from monitorT.task import Task from ..StaticBase.staticBase import StaticBase import datetime import tempfile import os from tools.layer import Layer, OutLayer # Fix gdal 2.40 and 3.3 integration problems try: import gdal except ModuleNotFoundError: from osgeo import gdal class TaskCustom(Task): CONTROL = True def set_up(self): self.FLAG_PAUSE = False def pause(self): self.FLAG_PAUSE = True def start(self): self.FLAG_PAUSE = False def post(task, user=None, output_layer: OutLayer = None, aoi: Layer = None, base_type=None, user_ms=None, password_ms=None, cloud_coverage_max=None, date='now', sort=False): """ Service to update a static database through files downloading. Parameters ---------- user: str User id. Used to load all user data. (Not implemented yet , TODO) output_layer: FTP Connection Connection to ftp where image will be downloaded base_type: str StaticBase to be updated. Options: sentinel aoi: FTP Connection Connection to ftp where AOI is hosted user_ms: str User ID of sentinel platform password_ms: str Password to sentinel platform cloud_coverage_max: float Max cloud coverage, hundred per cent. date: str Date to get the images. Options: "month/year" or "now" sort: bool Returns ------- response: list of str List containing the path of all images downloaded. Examples -------- >>> import requests >>> import json >>> array_bytes = open('AOI_TEST.kml', 'rb').read() >>> int_array = list(array_bytes) >>> params = {"user": 'test', "base": 'sentinel', "params": {"AOI": int_array, "date": '02/2019'}} >>> response = requests.post(url=':/MG/update/staticBase', json=params) [sentinel_repo/29THQ/2019/2, sentinel_repo/29THP/2019/2, ......., sentinel_repo/30TVK/2019/2] """ ################################################### # Allow different users repositories # ################################################### # user_data = get_user_data(user) # # user_repo = user_data['repo'] # # if 'repo' in user_data.keys(): # # user_repo = user_data['repo'] # # data_base = StaticBase.load(user_repo) # # else: # # data_base = settings.STATIC_DATABASE[base] # ################################################### # Creating tmp files tmp_directory = tempfile.mkdtemp(dir='tmp') sentinel_vrt_path = tempfile.NamedTemporaryFile(suffix='.vrt', prefix='orto_', dir=tmp_directory).name orto_result = tempfile.NamedTemporaryFile(suffix='.tif', prefix='ms_orto_aoi_', dir=tmp_directory).name # Sentinel ############################################# apihub_path_tmp = tempfile.mktemp(suffix='.txt', dir=tmp_directory) apihub_write = open(apihub_path_tmp, 'w') # ## Some characters falls the bash execution password_ms = password_ms.replace('!', '\!') password_ms = password_ms.replace('?', '\?') password_ms = password_ms.replace('$', '\$') # ########################################### apihub_write.write("{} {}\n".format(user_ms, password_ms)) apihub_write.close() apihub_path_tmp = os.path.join(os.path.abspath(''), apihub_path_tmp) # apihub_path_tmp = os.path.join(os.path.abspath(''), 'MG','update','StaticBase','apihub.txt') ######################################################## data_base = StaticBase.load({base_type: {'cloud_coverage_max': cloud_coverage_max, 'path': tmp_directory, 'apihub_path': apihub_path_tmp}}) data_base = data_base[base_type] if date == 'now': month = datetime.datetime.now().month year = datetime.datetime.now().year date = '{}/{}'.format(month, year) task.status_description = "Downloading tiles" task.set_progress(20) data_base_entries = data_base.update(task, aoi.gdal_layer(), date=date) # Remove file with password os.remove(apihub_path_tmp) task.status_description = "Extracting and merging bands" task.set_progress(50) # Create vrt of sentinel images vrt_string = 'gdalbuildvrt {result} {files}' # ORTO VRT ###################################################################################### vrt_files = [] results = [] # ## Getting files is_vrt = True if len(data_base_entries) == 1: is_vrt = False for entry in data_base_entries: # Extracting entry.extract() # Crop sentinel by AOI task.status_description = "Cropping by AOI" task.set_progress(80) gdal_warp_sentece = 'gdalwarp -q -multi -cutline "{cut_shape}" -dstnodata 0 -overwrite -crop_to_cutline ' \ '-r near "{source}" "{dst}" ' os.system(gdal_warp_sentece.format(cut_shape=aoi.gdal_layer(), source=os.path.join(entry.path, entry.name), dst=orto_result)) vrt_files.append(orto_result.split(os.sep)[-1]) results.append(orto_result) ds = gdal.Open(os.path.join(entry.path, entry.name)) # # Generate metadata string metadata = ds.GetMetadata() metadata_string = '' for metadata_key in metadata: metadata_string += '-mo \"{}={}\" '.format(metadata_key, metadata[metadata_key]) gdal_edit_command = "gdal_edit.py {} {}".format(metadata_string, orto_result) os.system(gdal_edit_command) if is_vrt: task.status_description = "Creating vrt's" task.set_progress(80) # Creatin vrt's command = vrt_string.format(result=sentinel_vrt_path, files=' '.join(vrt_files)) os.system(command) output_layer.attach_sources((sentinel_vrt_path)) output_layer.attach_sources(results) return output_layer