""" Module that manage the behaviour of source allocated in a http """ import re import os from copy import deepcopy import urllib.request import urllib.parse import http.client import requests from tools.protocols.masterProtocol import MasterProtocol from tools import const class Http(MasterProtocol): ConnectionError = type('ConnectionError', (Exception,), {}) LoginError = type('LoginError', (Exception,), {}) NotEnoughPrivileges = type('NotEnoughPrivileges', (Exception, ), {}) NoDirectoryOrFile = type('NoDirectoryOrFile', (Exception,), {}) DirectoryOrFileExists = type('DirectoryOrFileExists', (Exception, ), {}) CONNECTION_REQUIRED = {'user': 'user', 'password': 'password', 'ip': 'ip', 'fileType': 'driver_type', 'port': 'port', 'filePath': 'source', 'domain': 'domain', 'layerName': 'layer_name', 'subtype': 'type'} NAME = const.HTTP_KEY VSI_CURL_STREAMING_STRING = '/vsicurl_streaming/' def __init__(self, ip, port, user="", password=""): if ip == 'www.dropbox.com': self.ip = 'www.dl.dropboxusercontent.com' else: self.ip = ip self.port = port self.user = user self.password = password self.conn = http.client.HTTPConnection(self.ip, self.port) self.http_scheme = 'http' def exists_layer(self, layer): if self.exists(layer['source']): return True raise self.NoDirectoryOrFile('{} not exists on {}:{}'.format( layer['source'], self.ip, self.port)) def connect(self): try: self.conn.connect() except ConnectionRefusedError: raise self.ConnectionError('{} on {}:{} off'.format( self.http_scheme, self.ip, self.port)) def exists(self, layer): try: path = layer.get_source() if self.port: port = ':{}'.format(self.port) else: port = '' url = """{}://{}{}{}""".format(self.http_scheme, self.ip, port, path) if '.zip/' in url: file = url.split("/")[-1] url = url.replace("/" + file, "") response = requests.get(url) if response.status_code in [200, 201]: return response.status_code == 200 except Exception: return False def list_files(self, directory, suffix=None): if self.ip == 'www.dl.dropboxusercontent.com': return True else: list_files = [] if not self.user: login_string = '' else: login_string = '{}:{}@'.format(urllib.parse.quote(self.user), urllib.parse.quote(self.password)) src_directory = "{}://{}{}:{}{}".format(self.http_scheme, login_string, self.ip, self.port, directory) page = urllib.request.urlopen(src_directory) try: page_decode = str(page.read().decode()) except UnicodeDecodeError: return False regex = re.compile(r'
  • .*?>(.*?)<.*') for word in page_decode.splitlines(): if regex.search(word): list_files.append(regex.search(word).groups()[0]) page.close() return list_files def is_file(self, path): if self.ip == 'www.dl.dropboxusercontent.com': return True else: files = self.list_files(path) if isinstance(files, bool): return True else: if len(files) <= 0: return True return False def is_directory(self, _): return False def gdal_layer(self, layer, with_vsi): if '.' in layer['source']: layer_name = layer['source'] else: if layer['driver_type'] == 'SHAPE': # Shapefiles must be in separate folder to read it, # at least for now list_n = self.list_files(layer['source']) list_copy = deepcopy(list_n) list_n = [] for i in list_copy: if ".shp" in i: list_n.append(i) layer_name = '{}/{}'.format(layer['source'], list_n[0]) else: if 'layer_name' in layer.cloud_parameters: layer_name = '{}/{}'.format(layer['source'], layer['layer_name']) return self.gdal_url(layer_name, with_vsi=with_vsi, driver_prefix=self.VSI_CURL_STREAMING_STRING) def gdal_url(self, source, with_vsi=False, driver_prefix=''): if source[0] == '/': source = source[1:] if not self.user: login_string = '' else: login_string = '{}:{}@'.format(urllib.parse.quote(self.user), urllib.parse.quote(self.password)) vsi_string = '' if with_vsi: vsi_string = '{}'.format(driver_prefix) if self.port: port = ':{}'.format(self.port) else: port = '' return "{}{}://{}{}{}/{}".format(vsi_string, self.http_scheme, login_string, self.ip, port, source) def join(self, *args): args = list(args) for arg_index, arg in enumerate(args[1:]): if arg[0] == '/': args[arg_index+1] = arg[1:] join_string = os.path.join(*args) return join_string def upload(self, file, path): pass def upload_dir(self, src_directory, dst_directory): pass def download(self, file, path): testfile = urllib.request.URLopener() testfile.retrieve("""{}://{}:{}{}""".format(self.http_scheme, self.ip, self.port, file), path) def format_parameters(self, _, parameters): # Format source parameters.update({'source': re.sub(' +', '_', parameters['source'])}) return parameters