a ˆ#½eÙã@sNddlZddlZddlZddlZddlZddlZddlmZddl Z ddl m Z ddl Z ddl Z ddlZddlZddlZddlmZddlZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZm Z dd l!m"Z"dd l!m#Z#ddl!m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:m;Z;mm?Z?ddl@mAZAddlBmCZCddlDmEZFmGZGddlHmIZIddlJZJdd lKmLZLe Md!¡ZNGd"d#„d#eOƒZPd$d%„ZQd&d'„ZRd(d)„ZSd*d+„ZTd,d-„ZUd2d.d/„ZVGd0d1„d1e"jWƒZXdS)3éN)Úzipfly)Úquote)Úcopyfile)ÚImage)Ú GDALRaster)Ú OGRGeometry)Ú GEOSGeometry)Úfields)ÚInMemoryUploadedFile)ÚValidationErrorÚSuspiciousFileOperation)Úmodels)Ú transaction)Ú connection)Útimezone)ÚReadTimeoutError)Úpending_actions)Ú GeometryField)Ú assure_cogeo)Úis_pointcloud_georeferenced)Ú testWatch)Úpath_traversal_check)Ú status_codes)ÚProcessingNode)ÚNodeResponseErrorÚNodeConnectionErrorÚNodeServerErrorÚOdmError)Úsettings)ÚGCPFileé)ÚProject)Ú gettext_lazyÚgettext)Úpartial)ÚConsolez app.loggerc@s eZdZdS)ÚTaskInterruptedExceptionN)Ú__name__Ú __module__Ú __qualname__©r*r*ú/webodm/app/models/task.pyr&6sr&cCs d ||¡S)Nzproject/{0}/task/{1}/)Úformat)ÚtaskIdÚ projectIdr*r*r+Útask_directory_path9sr/cGstjjtjt||ƒg|¢RŽS©N)ÚosÚpathÚjoinrÚ MEDIA_ROOTr/)r-r.Úargsr*r*r+Úfull_task_directory_path=sr6cCsd t||ƒ|¡S)Nz{0}{1})r,r/)r-r.Úfilenamer*r*r+Úassets_directory_pathAsr8cCst|j|jj|ƒSr0)r8ÚidÚproject)Útaskr7r*r*r+Úgcp_directory_pathFsr<cCsXt|ƒdkrdSz.|D]$}|ds*tdƒ‚|dstdƒ‚qWntdƒ‚Yn0dS)zB Make sure that the format of this options field is valid rNÚnamezName key not found in optionÚvaluezValue key not found in optionzInvalid options)Úlenr )r>Úoptionr*r*r+Úvalidate_task_optionsJsrAc Cs~zd}t d|tj¡}|r"d}n†zdt |¡dtjj}t|tƒrJd}n:t|t ƒrrt |ƒdkrr|dt |ƒk}nt   d|¡d}Wn t y¦t   d|¡Yn0|sÆt   d |¡|dd œWSt |¡}tj |¡\}}tj |d |¡} |j\} } t| | ƒ} | |kr:t   d  || |¡¡| ¡|dd œWSt|ƒt| ƒ} t| | ƒ}t| | ƒ}| ||ftj¡}i}|r†d |d<d|jvrÀt |jd¡}|j| fdt |¡i|¤Žn|j| fi|¤Ž| ¡t |¡t | |¡t  d |||¡¡WnVt t!t"j#fy\}z4t   d |t$|ƒ¡¡|durF|ƒWYd}~dSd}~00|| d œ}|durz||ƒ|S)z½ :param image_path: path to the image :param resize_to: target size to resize this image to (largest side) :param done: optional callback :return: path and resize ratio Fz .*\.jpe?g$TZ0thr )ézACannot determine if image %s can be resized, hoping for the best!z$Cannot find BitsPerSample tag for %szCannot resize %s)r2Ú resize_ratioz.resizedzIYou asked to make {} bigger ({} --> {}), but we are not going to do that.édÚqualityÚexifzResized {} to {}x{}zCannot resize {}: {}.N)%ÚreÚmatchÚ IGNORECASEÚpiexifÚloadZImageIFDÚ BitsPerSampleÚ isinstanceÚintÚtupler?ÚloggerÚwarningÚKeyErrorrÚopenr1r2Úsplitextr3ÚsizeÚmaxr,ÚcloseÚfloatÚresizeÚLANCZOSÚinfoÚsaveÚdumpÚremoveÚrenameÚIOErrorÚ ValueErrorÚstructÚerrorÚstr)Z image_pathÚ resize_toÚdoneZ can_resizeZis_jpegZbpsÚimr2ÚextZresized_image_pathÚwidthÚheightZmax_sideÚratioZ resized_widthZresized_heightÚparamsZ exif_dictÚeÚretvalr*r*r+Ú resize_imageYsf                rocs¬eZdZdddœej dd¡ej dd¡ej dd¡ej dd¡ej d d ¡ej d d ¡ej d d ¡ej d d ¡ddddœej dd¡dej dd¡dœdej dd¡dœej dd¡ej dd¡dddœdddœdd dœd!ej d"d#¡ej d"d$¡ej d d%¡d&œZejd'fej d(fej d)fej d*fej d+ffZ ejd,fejd-fejd.fejd/fejd0ffZd1Zejd2ejd2d3d3ed4ƒd5Zejd6d2d7d2ed8ƒed9ƒd:Zejeej ed;ƒed<ƒd=Z!ejd6d2d2ed>ƒed?ƒd@Z"ej#dAedBƒedCƒdDZ$eje%ej&d2d2edEƒedFƒdGZ'ej(d2edHƒedIƒdDZ)ej#e d2d2d2edJƒedKƒdLZ*ej+d2d2edMƒedNƒdOZ,e-j.e/d2edPƒe0gedQƒdRZ1e-j2ejdSdTe3d2edUƒedVƒdWZ4e5d2d2dXedYƒedZƒd[Z6e5d2d2dXd\ed]ƒd[Z7e5d2d2dXd^ed_ƒd[Z8ej9e:j;ed`ƒedaƒdDZej#dAedfƒedgƒdDZ?ej@dhediƒedjƒd2dkZAej@dhedlƒedmƒd2dkZBej@dhednƒedoƒd2dkZCej+d3d7d2edpƒedqƒdrZDej#d3d2dsedtƒeduƒdvZEej(d3edwƒedxƒdDZFe-j.e/d2edyƒedzƒdWZGej#d2d{d2ed|ƒd}drZHej+d2d7d2ed~ƒedƒd€ZIe-j.e3d2edƒed‚ƒdWZJej@dhd2edƒƒed„ƒdWZKGd…d†„d†ƒZL‡fd‡dˆ„ZMd‰dŠ„ZNd‹dŒ„ZO‡fddŽ„ZPdd„ZQd‘d’„ZRd“d”„ZSd•d–„ZTd—d˜„ZUdËd™dš„ZVd›dœ„ZWddž„ZXdŸd „ZYd¡d¢„ZZd£d¤„Z[d¥d¦„Z\d§d¨„Z]d©dª„Z^d«d¬„Z_dÌd­d®„Z`dÍd¯d°„ZadÎd±d²„ZbdÏd³d´„ZcdЇfdµd¶„ Zdd·d¸„Zed¹dº„Zfd»d¼„Zgd½d¾„Zhd¿dÀ„ZidÁd„ZjdÃdÄ„ZkdÅdÆ„ZldÇdÈ„ZmdÑdÉdÊ„Zn‡ZoS)ÒÚTaskúall.zipÚ.)Ú deferred_pathÚdeferred_compress_dirÚodm_orthophotoúodm_orthophoto.tifzodm_orthophoto.pngzodm_orthophoto.mbtileszodm_orthophoto.kmzZodm_georeferencingzodm_georeferenced_model.laszodm_georeferenced_model.lazzodm_georeferenced_model.plyzodm_georeferenced_model.csvútextured_model.zipZ odm_texturing)úodm_textured_model_geo.glb)rsrtÚdeferred_exclude_filesrxú3d_tiles_model.zipZ3d_tilesÚmodelú3d_tiles_pointcloud.zipÚ pointcloudÚodm_demúdtm.tifúdsm.tifú dtm_tiles.zipZ dtm_tilesú dsm_tiles.zipZ dsm_tilesúorthophoto_tiles.zipZorthophoto_tilesú cameras.jsonÚ odm_reportú shots.geojsonú report.pdfúground_control_points.geojson)rqúorthophoto.tifzorthophoto.pngzorthophoto.mbtileszorthophoto.kmzzgeoreferenced_model.lasúgeoreferenced_model.lazzgeoreferenced_model.plyzgeoreferenced_model.csvrwztextured_model.glbrzr|rr€rr‚rƒr„r†r‡rˆÚQUEUEDÚRUNNINGÚFAILEDÚ COMPLETEDÚCANCELEDÚCANCELÚREMOVEÚRESTARTÚRESIZEÚIMPORTg333333ë?TFZId)Ú primary_keyÚdefaultÚuniqueÚ serializeÚeditableÚ verbose_nameéÿÚz3Identifier of the task (as returned by NodeODM API)ÚUUID)Ú max_lengthÚdb_indexr–ÚblankÚ help_textršz!Project that this task belongs tor!)Ú on_deleter¡ršzA label for the taskÚName)ržÚnullr r¡ršéÿÿÿÿztNumber of milliseconds that elapsed since the beginning of this task (-1 indicates that no information is available)zProcessing Time)r–r¡ršzXProcessing node assigned to this task (or null if this task has not been associated yet)zProcessing Node)r¢r¤r r¡ršzVA flag indicating whether this task should be automatically assigned a processing nodezAuto Processing NodezCurrent status of the taskZStatus)ÚchoicesrŸr¤r r¡ršz"The last processing error receivedz Last Error)r¤r r¡ršz0Options that are being used to process this taskÚOptions)r–r r¡Ú validatorsršéP)ržz$List of available assets to downloadzAvailable Assets)r–r r¡ršiæzExtent of the orthophotozOrthophoto Extent)r¤r Úsridr¡ršzExtent of the DSMz DSM ExtentzExtent of the DTMz DTM Extentz Creation datez Created atzzA requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.zPending Actionz>A flag indicating whether this task is available to the publicZPublicz˜When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing.z Resize Togz`Value between 0 and 1 indicating the upload progress of this task's files to the processing nodezUpload Progress)r–r¡ršr zJValue between 0 and 1 indicating the resize progress of this task's imageszResize ProgresszNValue between 0 and 1 indicating the running progress (estimated) of this taskzRunning Progressz8URL this task is imported from (only for imported tasks)z Import URL)r¤r–r r¡ršrz*Number of images associated with this taskz Images Count)r¤r r–r¡ršzˆA flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing.ZPartialzXSerialized potree scene information used to save/load measurements and camera view anglez Potree SceneNz+EPSG code of the dataset (if georeferenced)ZEPSGz Task tagsÚTags)rŸr–r r¡ršzList of orthophoto bandszOrthophoto Bandsz%Size of the task on disk in megabytesÚSizec@seZdZedƒZedƒZdS)z Task.MetarpZTasksN)r'r(r)Ú_ršÚverbose_name_pluralr*r*r*r+ÚMetasr¯cs4tt|ƒj|i|¤Ž|jj|_t| d¡ƒ|_dS)Nzconsole_output.txt) ÚsuperrpÚ__init__r:r9Ú_Task__original_project_idr%Ú data_pathÚconsole)Úselfr5Úkwargs©Ú __class__r*r+r±!s z Task.__init__cCs&|jdur|jntdƒ}d ||j¡S)NZunnamedzTask [{}] ({}))r=r#r,r9)rµr=r*r*r+Ú__str__)sz Task.__str__c CsÔt|j|ƒ}t|j|ƒ}tj tj |tj¡¡}zftj |¡r€tj |¡s€tj |¡s`t |¡t   ||¡t   d  ||¡¡nt  d  |||¡¡Wn8t jyÎ}zt  d  ||¡¡WYd}~n d}~00dS)zc Moves the task's folder, update ImageFields and orthophoto files to a new project zMoved task folder from {} to {}z‡Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.zlCould not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}N)r6r9r1r2Úabspathr3ÚpardirÚexistsÚmakedirsÚshutilÚmoverPr[r,rQÚError)rµZold_project_idZnew_project_idZold_task_folderZnew_task_folderZnew_task_folder_parentrmr*r*r+Ú move_assets.s      þ zTask.move_assetsc sæ|jj|jkr*| |j|jj¡|jj|_i}|jjD]x}|jdvr6t||jƒ}|jrb||j vrbq6zt ||j|  ||¡ƒWq6t y¬}z|j ||j<WYd}~q6d}~00q6|r¼t |ƒ‚|  ¡| ¡tt|ƒj|i|¤ŽdS)N)Úoptions)r:r9r²rÁÚ_metar ÚattnameÚgetattrr Ú empty_valuesÚsetattrÚcleanr Ú error_listr=Úvalidate_uniquer°rpr\)rµr5r¶ÚerrorsÚfÚ raw_valuermr·r*r+r\Fs$    $z Task.savecGs|jdg|¢RŽS)zJ Get a path relative to the place where assets are stored Zassets©Ú task_path©rµr5r*r*r+Ú assets_pathbszTask.assets_pathcGs|jdg|¢RŽS)z^ Path to task data that does not fit in database fields (e.g. console output) ÚdatarÎrÐr*r*r+r³hszTask.data_pathcGs&tjjtjt|j|jjdƒg|¢RŽS)z> Get path relative to the root task directory rœ)r1r2r3rr4r8r9r:rÐr*r*r+rÏns  ÿþzTask.task_pathcCs\||jvrX|j|}t|tƒr0tj | |¡¡St|tƒrXd|vrXtj | |d¡¡SdS)a Checks whether a particular asset is available in the file system Generally this should never be used directly, as it's slow. Use the available_assets field in the database instead. :param asset: one of ASSETS_MAP keys :return: boolean rtF)Ú ASSETS_MAPrMrdr1r2r¼rÑÚdict©rµÚassetr>r*r*r+Úis_asset_available_slowvs    zTask.is_asset_available_slowc Cs<| dd¡}tj |¡r4z:t|ƒ}t | ¡¡}Wdƒn1sH0YWn>ty’}z&t   d  |t |ƒ¡¡iWYd}~Sd}~00d}|  di¡  dd¡rÖ|  di¡  di¡  d ig¡d   d ¡}n|  d i¡  d ¡}d|i|  di¡  d¡|  di¡  d¡|  di¡  d¡|  di¡  d¡dœSiSdS)z5 Parse ODM's stats.json if available r…z stats.jsonNzMalformed JSON {}: {}Zpoint_cloud_statisticsZdenseFÚstatsZ statisticrÚcountZreconstruction_statisticsZreconstructed_points_countÚpointsZodm_processing_statisticsZ average_gsdZprocessing_statisticsÚareaÚ start_dateÚend_date)r}ZgsdrÛrÜrÝ)rÑr1r2r¼rSÚjsonÚloadsÚreadÚ ExceptionrPrQr,rdÚget)rµZ stats_jsonrÌÚjrmrÚr*r*r+Úget_statisticsˆs(  0*ÿù zTask.get_statisticsc Csjz(t ¡tjj|jd}d|_|r>tdƒd|ji|_t  ¡|_ |  ¡|  ¡t  d ||¡¡tj | ¡¡rèztj| ¡| ¡tjdWqøtyä}z4t  d t|ƒ¡¡t | ¡| ¡¡WYd}~qød}~00nt  d |¡¡|jjj ¡Wdƒn1s0Y|WStyd}z t  d t|ƒ¡¡WYd}~n d}~00d S) N©ÚpkzCopy of %(task)sr;zDuplicating {} to {})Ú copy_functionzHCannot duplicate task using hard links, will use normal copy instead: {}z.Task {} doesn't have folder, will skip copyingzCannot duplicate task: {}F)rÚatomicrpÚobjectsrârær#r=rÚnowÚ created_atr\Úrefresh_from_dbrPr[r,r1r2ÚisdirrÏr¾ÚcopytreeÚlinkrárQrdr:ÚownerÚprofileÚclear_used_quota_cache)rµÚ set_new_namer;rmr*r*r+Ú duplicate§s,  ,.*zTask.duplicatecsè||jvrÖ|j|‰tˆtƒr,| ˆ¡dfStˆtƒrÆdˆvr¶dˆvr¶| ˆd¡‰‡fdd„t ˆ¡Dƒ}dˆvr”tˆdtƒr”‡fdd„|Dƒ}t|ƒdkr¨t d ƒ‚t   |¡d fSt d   |¡ƒ‚qät d   |¡ƒ‚nt d   |¡ƒ‚dS)z‰ Get a stream to an asset :param asset: one of ASSETS_MAP keys :return: (path|stream, is_zipstream:bool) Frsrtc sDg|]<\}}}|D],}tj tj ||¡ˆ¡tj ||¡dœ‘qqS))ÚnÚfs)r1r2Úrelpathr3)Ú.0ÚdpÚdnÚ filenamesrÌ)Úzip_dirr*r+Ú Óóz4Task.get_asset_file_or_zipstream..rycs(g|] }tj |d¡ˆdvr|‘qS)röry)r1r2Úbasename)røÚp)r>r*r+rýÕrþrzNo files available for downloadTú-{} is not a valid asset (invalid dict values)ú%{} is not a valid asset (invalid map)ú{} is not a valid assetN) rÓrMrdrÑrÔr1ÚwalkrOr?ÚFileNotFoundErrorrZ ZipStreamr,)rµrÖÚpathsr*)r>rür+Úget_asset_file_or_zipstreamÅs      z Task.get_asset_file_or_zipstreamcCs|||jvrj|j|}t|tƒr(| |¡St|tƒrZd|vrJd|vrJ|dStd |¡ƒ‚qxtd |¡ƒ‚ntd |¡ƒ‚dS)zv Get the path to an asset download :param asset: one of ASSETS_MAP keys :return: path rsrtrrrN)rÓrMrdrÑrÔrr,rÕr*r*r+Úget_asset_download_pathàs     zTask.get_asset_download_pathc CsÞ|jtdƒd7_| ¡| d¡}|jrtj |¡s|j d¡ràtj  t j d¡}tj  t j d|j  dd¡¡}z$t ||ƒ}tj |¡r˜t||ƒWn@tyÚ}z(t d |j|¡¡t|ƒ‚WYd}~n d}~00n6zøt d |j|¡¡tj|jd d d }|j d ¡}|dur&t|ƒnd}d } d } t|dƒˆ} | d¡D]l} | t| ƒ7} t ¡| dkr¨|dur˜tjj |j!dj"t#| ƒ|dd| $¡t ¡} |  %| ¡qHWdƒn1sÌ0YWn<tj&j'tj&j(t)fy}zt|ƒ‚WYd}~n d}~00| *¡z | +¡Wn"t,j-yLttdƒƒ‚Yn0| d¡} tj | ¡rÆzBt| ƒ$}t. /|¡}t|ƒ|_0Wdƒn1sœ0YWnt 1d |¡¡Yn0d|_2d |_3| ¡dS)NzImporting assets...Ú rqzfile://ÚimportsrœzIError due importing assets from {} for {} in cause of path checking errorz$Importing task assets from {} for {}Té )ÚstreamÚtimeoutzcontent-lengthrÚwbiérågÍÌÌÌÌÌì?©Úrunning_progressúInvalid zip filez images.jsonz.Cannot read images count from imported task {})4r´r#r\rÑÚ import_urlr1r2r¼Ú startswithr3rr4ÚreplacerÚisfilerr rPrcr,rr[ÚrequestsrâÚheadersrNrSÚ iter_contentr?ÚtimerpréÚfilterr9ÚupdaterXÚcheck_if_canceledÚwriteÚ exceptionsÚTimeoutÚConnectionErrorrrìÚextract_assets_and_completeÚzipfileÚ BadZipFilerÞrKÚ images_countrQÚpending_actionÚprocessing_time)rµÚzip_pathZimports_folder_pathZunsafe_path_to_import_fileZchecked_path_to_filermÚdownload_streamÚcontent_lengthÚ total_lengthÚ downloadedÚ last_updateÚfdÚchunkZ images_jsonrÌÚimagesr*r*r+Ú handle_importõsb    "    $2    .zTask.handle_importc s^zªˆjtjkrˆ ¡ˆjtjkrLˆ ¡}ˆ ¡ˆ |¡dˆ_ˆ ¡ˆj r ˆj t j t j fvr ˆjdur¸t ¡ˆ_ˆjr¸ˆjjd7_ˆj ¡t d ˆjˆ¡¡ˆ ¡ˆjr ˆj ¡s ˆj t jkr t d ˆjˆ¡¡dˆ_dˆ_dˆ_ ˆ ¡nˆj t jkr tdƒ‚ˆjrþˆjsþˆjdurþˆj durþt d ˆ¡¡ˆ ¡‰‡fdd „ˆ ¡Dƒ}d ‰‡‡fd d „}zˆj |ˆjˆj|¡}Wn>tyà}z$tt d ƒdt!|ƒiƒ‚WYd}~n d}~00ˆ ¡dˆ_"|ˆ_ˆ ¡ˆjdurJˆjtj#kr¢t d ˆ¡¡ˆjrˆˆjrˆzˆj $ˆj¡Wn$t%ynt &d ˆ¡¡Yn0t j ˆ_ dˆ_ˆ ¡nt j ˆ_ dˆ_ˆ ¡n¨ˆjtj'kræt d ˆ¡¡ˆjrØd}ˆjrzˆj (ˆj¡}|jˆjk}Wnt%yYn0d}|rjzˆj )ˆjˆj¡Wn>tt*fyf}z t &d ˆ¡¡d}WYd}~n d}~00nd}|r¦t d ˆ¡¡dˆ_t+t,dd„ˆjƒƒˆ_d ˆ_"ˆj- .¡dˆ_/dˆ_ dˆ_0dˆ_d ˆ_1ˆ ¡n tt dƒƒ‚ndˆjtj2krJt d ˆ¡¡ˆjr<ˆjrj? @| ¡rvt d | ˆ¡¡tA B| ¡t> C| ¡d } d} d ‰‡‡fd d „}| s d ‰t d! ˆ¡¡ˆjjDˆj| |tEdtFd"d#| ƒƒd$} ˆ =d%¡}t> G| |¡t d& ˆ¡¡zˆ H¡d} WnPtIjJyb| d'krRt &d( |¡¡| d7} t> K|¡n tt d)ƒƒ‚Yn0qšn6ˆ ¡ˆj t j krªd d*lLmM}|jNjOˆjPˆjQd+nˆ ¡Wn¬tt*fyä}zˆ Rt!|ƒ¡WYd}~n~d}~0ty}z"t &d, ˆt!|ƒ¡¡WYd}~nDd}~0tSyX}z"t &d- ˆt!|ƒ¡¡WYd}~n d}~00dS).a' This method contains the logic for processing tasks asynchronously from a background thread or from a worker. Here tasks that are ready to be processed execute some logic. This could be communication with a processing node or executing a pending action. Nr z/Automatically assigned processing node {} to {}z2Processing node {} went offline, reassigning {}...rœzZProcessing node went offline. This could be due to insufficient memory or a network error.zProcessing... {}csg|]}tj ˆ|¡‘qSr*)r1r2r3)røÚi)Ú images_pathr*r+rýtrþz Task.process..rcsRt ¡ˆdk}|rNt d¡ˆ ¡tjjˆjdjt |ƒddt ¡‰dS)NrzTask.process.callbackråçY@)Úupload_progress) rrZmanual_log_callrrprérr9rrX©ÚprogressZtime_has_elapsed©r-rµr*r+Úcallbackys   zTask.process..callbackzConnection error: %(error)srcçð?z Canceling {}z?Could not cancel {} on processing node. We'll proceed anyway...z Restarting {}Fz*Could not restart {}, will start a new oneTz{} needs to be reprocessedcSs |ddkS)Nr=z rerun-fromr*)Údr*r*r+ÚÎrþzTask.process..r¥z1Cannot restart a task that has no processing nodez Removing {}r r4zProcessing status: {} for {}z(Removing old assets directory: {} for {}csVt ¡ˆdk}|s t|ƒdkrRtjjˆjdjˆjt|ƒdddt ¡‰dS)NrrDrår4gš™™™™™¹?r) rrNrprérr9rÚTASK_PROGRESS_LAST_VALUErXr6r8r*r+r9s ÿzDownloading all.zip for {}ér)Úprogress_callbackÚparallel_downloadsrqzExtracting all.zip for {}éz{} seems corrupted. Retrying...r©Úsignals©ÚsenderÚtask_idzI{} connection/timeout error: {}. We'll try reprocessing at the next tick.z{} interrupted)Tr&rr”r1r“Ú resize_imagesrìÚ resize_gcpr\Úauto_processing_nodeÚstatusrrrÚprocessing_noderZfind_best_available_nodeZ queue_countrPr[r,Z is_onliner‹ÚuuidrŒrrÏÚ scan_imagesZprocess_new_taskr=rÂrr#rdr5rZ cancel_taskrrQr’Z get_task_infoZ restart_taskrÚlistrr´Úresetr'Ú last_errorrr‘Z remove_taskÚdeleteÚoutputr?Úsplitr>r3r7r=rŽrÑr1r2r¼r¾Úrmtreer½Zdownload_task_assetsrVrNr_r"r#r$r^Ú app.pluginsrCZ task_failedÚ send_robustr¸r9Ú set_failurer&)rµÚresized_imagesr0r9rLrmZuuid_still_existsr[Zneed_to_reprocessZcurrent_lines_countÚ assets_dirZ retry_numZ extractedr(Z all_zip_pathÚplugin_signalsr*)r3r-rµr+Úprocess:s&         .            (      "*z Task.processc Cs"| d¡}| d¡}t |d¡}| |¡Wdƒn1s@0Yt d |¡¡t |¡tj   | dd¡¡dftj   | d d ¡¡d ftj   | d d ¡¡d fg}|D]ü\}}tj   |¡r®z t |ƒWn<t y }z"t d|t|ƒf¡WYd}~n d}~00t|ƒ}t |j¡} t ¡F} |  d|jg¡| jdkr`ttdƒdt|jƒiƒ‚Wdƒn1sv0Yt||t| j|jdƒt d ||¡¡q®| ¡| ¡|  ¡| !¡i|_"d|_#|j$tdƒd7_$t%j&|_'| (¡ddl)m*} | j+j,|j-|j.ddS)z Extracts assets/all.zip, populates task fields where required and assure COGs It will raise a zipfile.BadZipFile exception is the archive is corrupted. :return: rœrqÚrNzExtracted all.zip for {}rurvÚorthophoto_extentr~r€Ú dsm_extentrÚ dtm_extentzjCannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance.z0SELECT SRID FROM spatial_ref_sys WHERE SRID = %srzFUnsupported SRS %(code)s. Please make sure you picked a supported SRS.Úcode)rªz%Populated extent field with {} for {}r:zDone!r rBrD)/rÑr#ÚZipFileÚ extractallrPr[r,r1r^r2Úrealpathr¼rr`rQrdrrÚ from_bboxÚextentrÚcursorÚexecuterªÚrowcountrr#rÇrÚwktÚupdate_available_assets_fieldÚupdate_epsg_fieldÚupdate_orthophoto_bands_fieldÚ update_sizeÚ potree_scenerr´rrŽrJr\rUrCZtask_completedrVr¸r9) rµrYr(Úzip_hZ extent_fieldsZ raster_pathÚfieldrmÚrasterrerfrZr*r*r+r"LsR  ( ÿÿÿû   ,   : z Task.extract_assets_and_completecCs| d |¡||d |¡¡S)Nz{}_tilesz{}.png)rÑr,)rµÚ tile_typeÚzÚxÚyr*r*r+Ú get_tile_pathŽszTask.get_tile_pathcCs |dkr d}d |jj|j|¡S)NÚplantÚ orthophotoz/api/projects/{}/tasks/{}/{}/)r,r:r9)rµrrr*r*r+Úget_tile_base_url‘szTask.get_tile_base_urlc sÆg}dˆjvr"| d¡| d¡dˆjvr6| d¡dˆjvrJ| d¡d}d ˆjvrjd  ˆjjˆj¡}d}d ˆjvrŠd  ˆjjˆj¡}‡fd d„|Dƒdtˆjƒˆjjˆj||ˆjˆjdœidœS)Nr‰rxrwr€ZdsmrZdtmrœr†z0/api/projects/{}/tasks/{}/download/shots.geojsonrˆz@/api/projects/{}/tasks/{}/download/ground_control_points.geojsoncsg|]}ˆ |¡|dœ‘qS))ÚurlÚtype)ry)røÚt©rµr*r+rý§rþz&Task.get_map_items..r;)r9r:ÚpublicÚ camera_shotsÚground_control_pointsÚepsgÚorthophoto_bands)ZtilesÚmeta) Úavailable_assetsÚappendr,r:r9rdr~rr‚)rµÚtypesrr€r*r}r+Ú get_map_items˜s,   ùÿþzTask.get_map_itemscCs t|jƒ|jj|j|j|jdœS)zK Subset of a task fields used in the 3D model display view )r9r:r„r~r)rdr9r:r„r~rr}r*r*r+Úget_model_display_paramsµs ûzTask.get_model_display_paramscCsX| |¡}| |¡}tj |¡s.td |¡ƒ‚tj |¡sTt tj |¡dd|¡|S)aA :param archive: path of the destination .zip file (relative to /assets/ directory) :param directory: path of the source directory to compress (relative to /assets/ directory) :param stream: return a stream instead of a path to the file :return: full path of the generated archive z{} does not existrÚzip) rÑr1r2r¼rr,r¾Ú make_archiverT)rµÚarchiveÚ directoryr Ú archive_pathZdirectory_pathr*r*r+Úgenerate_deferred_assetÁs    zTask.generate_deferred_assetcs2tˆj ¡ƒ}‡fdd„|Dƒˆ_|r.ˆ ¡dS)zÄ Updates the available_assets field with the actual types of assets available :param commit: when True also saves the model, otherwise the user should manually call save() csg|]}ˆ |¡r|‘qSr*)r×)rørÖr}r*r+rýÙrþz6Task.update_available_assets_field..N)rNrÓÚkeysr„r\)rµÚcommitZ all_assetsr*r}r+rjÓsz"Task.update_available_assets_fieldc Csd}dD]¤}| |j|¡}tj |¡rzTt |¡6}|jdur\|j ¡}WdƒWq®Wdƒn1sp0YWqt yª}zt   |¡WYd}~qd}~00q| |jd¡}|durîtj |¡rît |ƒsît   d |¡¡d}||_|r| ¡dS)z¥ Updates the epsg field with the correct value :param commit: when True also saves the model, otherwise the user should manually call save() N)r‰r€rrŠz{} is not georeferenced)rÑrÓr1r2rÚrasteriorSÚcrsZto_epsgrárPrQrr[r,rr\)rµrrrÖZ asset_pathrÌrmZ point_cloudr*r*r+rkÝs$    4"zTask.update_epsg_fieldcCs˜g}| |jd¡}tj |¡r‚t |¡H}dd„|jDƒ}t|ƒD]\}}|  ||j |dœ¡qDWdƒn1sx0Y||_ |r”|  ¡dS)z± Updates the orthophoto bands field with the correct value :param commit: when True also saves the model, otherwise the user should manually call save() r‰cSsg|] }|j‘qSr*)r=)røÚcr*r*r+rýrþz6Task.update_orthophoto_bands_field..)r=Ú descriptionN) rÑrÓr1r2rr‘rSZ colorinterpÚ enumerater…Ú descriptionsr‚r\)rµrÚbandsZorthophoto_pathrÌÚnamesr2rõr*r*r+rlús  þ(z"Task.update_orthophoto_bands_fieldc s´|j}ddlm}|jj|j|dtj t j t |j|j jƒ¡}t t|ƒ ||¡zt |¡Wn.tyŽ}zt |¡WYd}~n d}~00|j jj ¡|jj|j|ddS)NrrBrD)r9rUrCZ task_removingrVr¸r1r2r3rr4r/r:r°rprQr¾rTrrPrQrðrñròZ task_removed)rµÚusingÚ keep_parentsrFrZZdirectory_to_deletermr·r*r+rQs  ÿ z Task.deletecCs2t d ||¡¡||_tj|_d|_| ¡dS)NzFAILURE FOR {}: {}) rPrcr,rPrrrJr&r\)rµÚ error_messager*r*r+rW#s zTask.set_failurecs*t|j|jjƒ‰‡‡fdd„t ˆ¡DƒS)Ncs*g|]"}t ˆ|tj¡rtj ˆ|¡‘qSr*)rGrHrIr1r2r3)rørÌ©rŒÚregexr*r+rý,sÿz0Task.find_all_files_matching..)r6r9r:r1Úlistdir)rµrr*rœr+Úfind_all_files_matching*szTask.find_all_files_matchingcCs.tj d¡j|jdjtjtjfvr*t ƒ‚dS)Nr&rå) rpréÚonlyrâr9r&rrr‘r&r}r*r*r+r/sÿzTask.check_if_canceledcs„ˆjdkr t d ˆj¡¡gSˆ d¡}t|ƒ‰d‰d‰d ‡‡‡‡fdd„ }tttt ˆj|d|ƒƒ}t j j ˆj djd d |S) a Destructively resize this task's JPG images while retaining EXIF tags. Resulting images are always converted to JPG. TODO: add support for tiff files :return list containing paths of resized images and resize ratios rz=We were asked to resize images to {}, this might be an error.z.*\.(jpe?g|tiff?)$NcsPˆd7‰t ¡ˆdkrLtjjˆjdjtˆƒtˆƒdˆ ¡t ¡‰dS)Nr rrå©Úresize_progress)rrprérr9rrXr)rn©r-Zresized_images_countrµZ total_imagesr*r+r9Es $z$Task.resize_images..callback)rerfrår:r¡)N)rerPrQr,rŸr?rNÚmapr$rorprérr9r)rµr3r9rXr*r£r+rG5s   zTask.resize_imagesc CsÈ| d¡}ttdd„|ƒƒ}t|ƒdkr,dS|d}i}|D] }|d|tj |d¡ ¡<q.rNrCr2zResized GCP file {}z Could not resize GCP file {}: {})rŸrNrr?r1r2rÿr¥rZcreate_resized_copyrPr[r,rárQrd)rµrXZgcp_pathZ image_ratiosÚriZgcpFilermr*r*r+rHWs  zTask.resize_gcpc Cs^| d¡}zt |¡Wn@tyX}z(|jtjkrBtj |¡rBn‚WYd}~n d}~00dS)zP Create directories for this task (if they don't exist already) rœN)rÑr1r½ÚOSErrorÚerrnoÚEEXISTr2rí)rµrYÚexcr*r*r+Úcreate_task_directoriesus zTask.create_task_directoriescCs4| ¡}zdd„t |¡DƒWSgYS0dS)NcSsg|]}| ¡r|j‘qSr*)Úis_filer=)rørmr*r*r+rý…rþz$Task.scan_images..)rÏr1Úscandir)rµÚtpr*r*r+rM‚s zTask.scan_imagescCs| |¡}t|| ¡ƒSr0)rÏr)rµr7rr*r*r+Úget_image_path‰s zTask.get_image_pathc CsÒ|D]È}|j}|durq| ¡}tj |¡s:tj|dd| |¡}t|dƒn}t|t ƒrt|  ¡D]}|  |¡qbn:t|  ¡dƒ}t  ||¡Wdƒn1s¤0YWdƒq1sÂ0YqdS)NT)Úexist_okzwb+Úrb)r=rÏr1r2r¼r½r¯rSrMr ÚchunksrÚtemporary_file_pathr¾Ú copyfileobj) rµÚfilesÚfiler=r®Zdst_pathr.r/rÌr*r*r+Úhandle_images_uploads     zTask.handle_images_uploadc Cs¾z~d}t | ¡¡D]>\}}}|D].}tj ||¡}tj |¡s"|tj |¡7}q"q|dd|_|rn| ¡|j j j   ¡Wn:t y¸}z"t d |t|ƒ¡¡WYd}~n d}~00dS)Nriz"Cannot update size for task {}: {})r1rrÏr2r3ÚislinkÚgetsizerUr\r:rðrñròrárPÚwarnr,rd) rµrÚ total_bytesÚdirpathr­rûrÌÚfprmr*r*r+rm¡s  zTask.update_size)T)F)F)F)F)NF)F)pr'r(r)r1r2r3rÓrr‹rŒrrŽrZ STATUS_CODESrrr‘r’r“r”ZPENDING_ACTIONSr=r Ú UUIDFieldÚ uuid_moduleÚuuid4r­r9Ú CharFieldrLÚ ForeignKeyr!ÚCASCADEr:r=Ú IntegerFieldr'rÚSET_NULLrKÚ BooleanFieldrIrJÚ TextFieldrPr Z JSONFieldrÔrArÂZ ArrayFieldrNr„rr]r^r_Ú DateTimeFieldrrêrër&r~reÚ FloatFieldr5r¢rrr%r$rnrÚtagsr‚rUr¯r±r¹rÁr\rÑr³rÏr×rärôrrr1r[r"rvryr‡rˆrŽrjrkrlrQrWrŸrrGrHr«rMr¯r·rmÚ __classcell__r*r*r·r+rp®sþ        ý  þ þ  þþþ   Ô0û û $ýýý   EB   " rp)N)YÚloggingr1r¾rrbrLr¿Z app.vendorrrÞÚshlexrr¨rJrGr#r‘rrÚPILrÚdjango.contrib.gis.gdalrrÚdjango.contrib.gis.geosrZdjango.contrib.postgresr Údjango.core.files.uploadedfiler Údjango.core.exceptionsr r Ú django.dbr rrÚ django.utilsrZurllib3.exceptionsrÚapprÚ#django.contrib.gis.db.models.fieldsrZ app.cogeorZapp.pointcloud_utilsrZ app.testwatchrZ app.securityrÚnodeodmrZnodeodm.modelsrÚpyodm.exceptionsrrrrZwebodmrZapp.classes.gcprr:r!Údjango.utils.translationr"r­r#Ú functoolsr$Ú subprocessZapp.classes.consoler%Ú getLoggerrPrár&r/r6r8r<rAroÚModelrpr*r*r*r+Úsh                             U