Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
3# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data
4#
5# Copyright (C) 2020 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
6#
7# This software was developed within the context of the GeoMultiSens project funded
8# by the German Federal Ministry of Education and Research
9# (project grant code: 01 IS 14 010 A-C).
10#
11# This program is free software: you can redistribute it and/or modify it under
12# the terms of the GNU General Public License as published by the Free Software
13# Foundation, either version 3 of the License, or (at your option) any later version.
14# Please note the following exception: `gms_preprocessing` depends on tqdm, which
15# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files
16# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore".
17# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE.
18#
19# This program is distributed in the hope that it will be useful, but WITHOUT
20# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
21# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
22# details.
23#
24# You should have received a copy of the GNU Lesser General Public License along
25# with this program. If not, see <http://www.gnu.org/licenses/>.
27import os
28import fnmatch
29import tempfile
30import shutil
31import tarfile
32import zipfile
33import itertools
34import warnings
35import re
36from glob import glob
38# custom
39try:
40 from osgeo import gdal
41except ImportError:
42 import gdal
43import fmask
45from ..misc.helper_functions import convert_absPathArchive_to_GDALvsiPath, subcall_with_output
46from ..misc.definition_dicts import get_mask_classdefinition
47from ..misc.database_tools import get_info_from_postgreSQLdb
48from ..misc.exceptions import FmaskError, FmaskWarning
49from geoarray import GeoArray
51__author__ = 'Daniel Scheffler'
54class _FMASK_Runner(object):
55 """The FMASK runner base class (not to be called directly)."""
57 def __init__(self, path_providerArchive, satellite, extract_archive, tempdir_root):
58 # TODO provide options of fmask_usgsLandsatStacked argparser
59 # private attributes
60 self._gdal_path_archive = None
61 self._files_in_archive = None
62 self._angles_stack = None
63 self._metaFile = None
64 self._TOARef = None
66 assert fmask
68 self.path_archive = path_providerArchive
69 self.satellite = satellite
70 self.extract_archive = extract_archive
71 self.validate_inputs()
73 self.project_dir = os.path.abspath(os.path.curdir)
74 self.is_extracted = None
75 self.cloud_mask = None
76 self.cloud_mask = None
77 self.saturationmask_legend = {}
79 # create temporary directory
80 if tempdir_root and not os.path.isdir(tempdir_root):
81 os.makedirs(tempdir_root)
82 self.tempdir = tempfile.mkdtemp(dir=tempdir_root,
83 prefix='FMASK__%s__' % os.path.basename(self.path_archive))
85 # create subdirectory for FMASK internal intermediate files
86 os.makedirs(os.path.join(self.tempdir, 'FMASK_intermediates'))
88 def validate_inputs(self):
89 if not os.path.exists(self.path_archive):
90 raise FileNotFoundError(self.path_archive)
91 if self.satellite not in ['Landsat-4', 'Landsat-5', 'Landsat-7', 'Landsat-8', 'Sentinel-2A', 'Sentinel-2B']:
92 raise ValueError('%s is not a supported satellite for cloud mask calculation via FMASK.' % self.satellite)
94 @property
95 def is_GMSConfig_available(self):
96 from ..options.config import GMS_config as CFG
97 try:
98 if CFG is not None:
99 return True
100 except (EnvironmentError, OSError):
101 return False
103 @property
104 def gdal_path_archive(self):
105 if not self._gdal_path_archive:
106 self._gdal_path_archive = convert_absPathArchive_to_GDALvsiPath(self.path_archive)
107 return self._gdal_path_archive
109 @property
110 def files_in_archive(self):
111 if not self._files_in_archive:
112 os.chdir(os.path.dirname(self.path_archive))
113 self._files_in_archive = gdal.ReadDirRecursive(self.gdal_path_archive)
114 return self._files_in_archive
116 @staticmethod
117 def run_cmd(cmd):
118 output, exitcode, err = subcall_with_output(cmd)
119 if exitcode:
120 raise FmaskError("Error during FMASK cloud masking: \n" + err.decode('latin-1'))
121 if output:
122 return output.decode('UTF-8')
124 def extract_tar_archive(self):
125 with tarfile.open(self.path_archive) as tarF:
126 tarF.extractall(self.tempdir)
127 self.is_extracted = True
129 def extract_zip_archive(self):
130 with zipfile.ZipFile(self.path_archive, "r") as z:
131 z.extractall(self.tempdir)
132 self.is_extracted = True
134 def to_saved_rasterFile(self, value, attrname):
135 pathFile = os.path.join(self.tempdir, "%s.bsq" % attrname)
136 if isinstance(value, str) and os.path.exists(value):
137 pathFile = value
138 elif isinstance(value, GeoArray):
139 value.q = True
140 value.save(pathFile)
141 else:
142 raise TypeError("The attribute '%s' can only be set by an existing path or an instance of GeoArray. "
143 "Received %s" % (attrname, type(value)))
145 assert isinstance(pathFile, str) and os.path.exists(pathFile)
147 return pathFile
149 def calc_cloudMask(self, path_out=None, fmt=None):
150 if path_out:
151 gdal.Translate(path_out, gdal.Open(self.cloud_mask), format=fmt)
152 self.cloud_mask = GeoArray(path_out)
153 elif self.cloud_mask is not None and isinstance(self.cloud_mask, str) and os.path.exists(self.cloud_mask):
154 self.cloud_mask = GeoArray(self.cloud_mask).to_mem()
155 else:
156 self.cloud_mask = None
158 if self.cloud_mask:
159 if self.is_GMSConfig_available:
160 self.cloud_mask.legend = \
161 get_mask_classdefinition('mask_clouds', self.satellite)
162 else: # use default FMASK legend
163 warnings.warn('GMS configuration not available. Using default cloud mask legend.', FmaskWarning)
164 self.cloud_mask.legend = \
165 {'No Data': 0, 'Clear': 1, 'Cloud': 2, 'Shadow': 3, 'Snow': 4, 'Water': 5}
167 return self.cloud_mask
169 def clean(self):
170 shutil.rmtree(self.tempdir)
171 self.is_extracted = False
172 os.chdir(self.project_dir)
173 self._metaFile = None
174 self._angles_stack = None
175 self._TOARef = None
178class FMASK_Runner_Landsat(_FMASK_Runner):
179 def __init__(self, path_providerArchive, satellite, TOARef=None, opticalDNs=None, thermalDNs=None,
180 tempdir_root=None):
181 """FMASK wrapper class for Landsat 4-8.
183 :param path_providerArchive: file path of the provider .tar.gz archive
184 :param satellite: name of the satellite: 'Landsat-4', 'Landsat-5', 'Landsat-7', 'Landsat-8'
185 :param TOARef: file path or GeoArray instance of top-of-atmosphere reflectance data
186 scaled from 0 to 10000 (optional -> generated from archive if not given)
187 :param opticalDNs: file path or GeoArray instance of optical data (digital numbers)
188 :param thermalDNs: file path or GeoArray instance of thermal data (digital numbers)
189 :param tempdir_root: directory to write intermediate data (auto-determined if not given)
190 """
192 # private attributes
193 self._optical_stack = None
194 self._thermal_stack = None
195 self._saturationmask = None
197 self.FileMatchExp = {
198 'Landsat-4': dict(optical='L*_B[1-5,7].TIF', thermal='L*_B6.TIF', meta='*_MTL.txt'),
199 'Landsat-5': dict(optical='L*_B[1-5,7].TIF', thermal='L*_B6.TIF', meta='*_MTL.txt'),
200 'Landsat-7': dict(optical='L*_B[1-5,7].TIF', thermal='L*_B6_VCID_?.TIF', meta='*_MTL.txt'),
201 'Landsat-8': dict(optical='L*_B[1-7,9].TIF', thermal='L*_B1[0,1].TIF', meta='*_MTL.txt')
202 }[satellite]
204 super(FMASK_Runner_Landsat, self).__init__(path_providerArchive, satellite, extract_archive=False,
205 tempdir_root=tempdir_root)
207 # populate optional attributes
208 if TOARef is not None:
209 self.TOARef = TOARef
210 if opticalDNs is not None:
211 self.optical_stack = opticalDNs
212 if thermalDNs is not None:
213 self.thermal_stack = thermalDNs
215 @property
216 def optical_stack(self):
217 if self._optical_stack is None:
218 if not self.is_extracted:
219 self.extract_tar_archive()
220 opt_fNames = list(sorted(fnmatch.filter(os.listdir(self.tempdir), self.FileMatchExp['optical'])))
221 fNames_str = ' '.join([os.path.join(self.tempdir, fName) for fName in opt_fNames])
223 # create stack of optical bands
224 self._optical_stack = os.path.join(self.tempdir, 'optical_stack.vrt')
225 self.run_cmd('gdalbuildvrt %s -separate %s' % (self._optical_stack, fNames_str))
227 return self._optical_stack
229 @optical_stack.setter
230 def optical_stack(self, value):
231 self._optical_stack = super(FMASK_Runner_Landsat, self).to_saved_rasterFile(value, 'optical_stack')
233 @property
234 def thermal_stack(self):
235 if self._thermal_stack is None:
236 if not self.is_extracted:
237 self.extract_tar_archive()
238 opt_fNames = list(sorted(fnmatch.filter(os.listdir(self.tempdir), self.FileMatchExp['thermal'])))
239 fNames_str = ' '.join([os.path.join(self.tempdir, fName) for fName in opt_fNames])
241 # create stack of thermal bands
242 self._thermal_stack = os.path.join(self.tempdir, 'thermal_stack.vrt')
243 self.run_cmd('gdalbuildvrt %s -separate %s' % (self._thermal_stack, fNames_str))
245 return self._thermal_stack
247 @thermal_stack.setter
248 def thermal_stack(self, value):
249 self._thermal_stack = super(FMASK_Runner_Landsat, self).to_saved_rasterFile(value, 'thermal_stack')
251 @property
252 def metaFile(self):
253 if not self._metaFile:
254 if not self.is_extracted:
255 self.extract_tar_archive()
256 self._metaFile = os.path.join(self.tempdir, fnmatch.filter(os.listdir(self.tempdir), '*_MTL.txt')[0])
258 return self._metaFile
260 @property
261 def angles_stack(self):
262 if self._angles_stack is None:
263 self._angles_stack = os.path.join(self.tempdir, 'angles.vrt')
264 self.run_cmd('fmask_usgsLandsatMakeAnglesImage.py -m %s -t %s -o %s'
265 % (self.metaFile, self.optical_stack, self._angles_stack))
267 return self._angles_stack
269 @property
270 def saturationmask(self):
271 if self._saturationmask is None:
272 self._saturationmask = os.path.join(self.tempdir, 'saturationmask.vrt')
273 self.run_cmd('fmask_usgsLandsatSaturationMask.py -m %s -i %s -o %s'
274 % (self.metaFile, self.optical_stack, self._saturationmask))
275 self.saturationmask_legend = {'blue': 0, 'green': 1, 'red': 2}
277 return self._saturationmask
279 @property
280 def TOARef(self):
281 if self._TOARef is None:
282 self._TOARef = os.path.join(self.tempdir, 'TOARef.vrt')
283 self.run_cmd('fmask_usgsLandsatTOA.py -m %s -i %s -z %s -o %s'
284 % (self.metaFile, self.optical_stack, self.angles_stack, self._TOARef))
286 return self._TOARef
288 @TOARef.setter
289 def TOARef(self, value):
290 self._TOARef = super(FMASK_Runner_Landsat, self).to_saved_rasterFile(value, 'TOARef')
292 def calc_cloudMask(self, path_out=None, fmt=None):
293 # type: (str, str) -> any
295 """Calculate the cloud mask!
297 :param path_out: output path (if not given, a GeoArray instance is returned)
298 :param fmt: output GDAL driver code, e.g. 'ENVI' (only needed if path_out is given)
299 :return: a GeoArray instance of the computed cloud mask
300 """
302 try:
303 self.cloud_mask = os.path.join(self.tempdir, 'fmask_cloudmask.vrt')
304 self.run_cmd('fmask_usgsLandsatStacked.py %s'
305 % ' '.join(['-m %s' % self.metaFile,
306 '-a %s' % self.TOARef,
307 '-t %s' % self.thermal_stack,
308 '-z %s' % self.angles_stack,
309 '-s %s' % self.saturationmask,
310 '-o %s' % self.cloud_mask,
311 '-e %s' % os.path.join(self.tempdir, 'FMASK_intermediates')
312 ]))
313 return super(FMASK_Runner_Landsat, self).calc_cloudMask(path_out=path_out, fmt=fmt)
315 finally:
316 self.clean()
318 def clean(self):
319 self._thermal_stack = None
320 self._optical_stack = None
321 self._saturationmask = None
323 super(FMASK_Runner_Landsat, self).clean()
324 assert not os.path.isdir(self.tempdir), 'Error deleting temporary FMASK directory.'
327class FMASK_Runner_Sentinel2(_FMASK_Runner):
328 def __init__(self, path_providerArchive, satellite, scene_ID=None, granule_ID='', target_res=20, TOARef=None,
329 extract_archive=False, tempdir_root=None):
330 """FMASK wrapper class for Sentinel-2.
332 :param path_providerArchive: file path of the provider .zip archive
333 :param satellite: name of the satellite: 'Sentinel-2A' or 'Sentinel-2B'
334 :param scene_ID: the GeoMultiSens scene ID of the given scene (needed if granule_ID is not given)
335 :param granule_ID: the Sentinel-2 granule ID
336 :param target_res: target spatial resolution of the cloud mask (default: 20m)
337 :param TOARef: file path or GeoArray instance of top-of-atmosphere reflectance data
338 scaled from 0 to 10000 (optional -> read from archive if not given)
339 :param extract_archive: whether to extract the archive to disk or to read from the archive directly
340 (default: False); NOTE: There is no remarkable speed difference.
341 :param tempdir_root: directory to write intermediate data (auto-determined if not given)
342 """
344 self._granule_ID = granule_ID
346 self.scene_ID = scene_ID
347 self.tgt_res = target_res
349 oldStPref = '*GRANULE/' + self.granule_ID + '*/'
350 self.FileMatchExp = {
351 'Sentinel-2A': dict(opticalOLDStyle='%s*_B0[1-8].jp2 %s*_B8A.jp2 %s*_B09.jp2 %s*_B1[0-2].jp2'
352 % (oldStPref, oldStPref, oldStPref, oldStPref),
353 opticalNEWStyle='*_B0[1-8].jp2 *_B8A.jp2 *_B09.jp2 *_B1[0-2].jp2',
354 metaOLDStyle='%sS2A*.xml' % oldStPref,
355 metaNEWStyle='*MTD_TL.xml'),
356 'Sentinel-2B': dict(opticalOLDStyle='%s*_B0[1-8].jp2 %s*_B8A.jp2 %s*_B09.jp2 %s*_B1[0-2].jp2'
357 % (oldStPref, oldStPref, oldStPref, oldStPref),
358 opticalNEWStyle='*_B0[1-8].jp2 *_B8A.jp2 *_B09.jp2 *_B1[0-2].jp2',
359 metaOLDStyle='%sS2B*.xml' % oldStPref,
360 metaNEWStyle='*MTD_TL.xml'),
361 }[satellite]
363 super(FMASK_Runner_Sentinel2, self).__init__(path_providerArchive, satellite, extract_archive,
364 tempdir_root=tempdir_root)
366 # populate optional attributes
367 if TOARef is not None:
368 self.TOARef = TOARef
370 @property
371 def granule_ID(self):
372 """Gets the Sentinel-2 granule ID from the database using the scene ID in case the granule ID has not been
373 given."""
375 if not self._granule_ID and self.scene_ID and self.scene_ID != -9999 and self.is_GMSConfig_available:
376 from ..options.config import GMS_config as CFG
377 res = get_info_from_postgreSQLdb(CFG.conn_database, 'scenes', ['entityid'], {'id': self.scene_ID})
378 assert len(res) != 0, \
379 "Invalid SceneID given - no corresponding scene with the ID=%s found in database.\n" % self.scene_ID
380 assert len(res) == 1, "Error in database. The sceneid %s exists more than once. \n" % self.scene_ID
381 self._granule_ID = res[0][0]
383 return self._granule_ID
385 @property
386 def metaFile(self):
387 if not self._metaFile:
388 fNs_meta = fnmatch.filter(self.files_in_archive, self.FileMatchExp['metaNEWStyle'])
389 if not fNs_meta:
390 fNs_meta = fnmatch.filter(self.files_in_archive, self.FileMatchExp['metaOLDStyle'])
391 if len(fNs_meta) > 1:
392 raise RuntimeError('Found multiple metadata files for the given %s dataset. Please provide the '
393 'granule ID where you want to use the metadata from.' % self.satellite)
394 elif not fNs_meta:
395 raise RuntimeError('Could not find a metadata file for the given %s dataset.' % self.satellite)
396 fN_meta = fNs_meta[0]
398 # only extract the metadata file
399 self._metaFile = os.path.join(self.tempdir, 'meta.xml')
400 with zipfile.ZipFile(self.path_archive) as z:
401 with z.open(fN_meta) as zf, open(self._metaFile, 'wb') as f:
402 shutil.copyfileobj(zf, f)
404 return self._metaFile
406 @property
407 def angles_stack(self):
408 if self._angles_stack is None:
409 self._angles_stack = os.path.join(self.tempdir, 'angles.vrt')
410 self.run_cmd('fmask_sentinel2makeAnglesImage.py -i %s -o %s' % (self.metaFile, self._angles_stack))
412 return self._angles_stack
414 @property
415 def TOARef(self):
416 if self._TOARef is None:
417 if not self.extract_archive:
418 fileList = self.files_in_archive
419 else:
420 if not self.is_extracted:
421 self.extract_zip_archive()
422 fileList = glob(self.tempdir + '/**', recursive=True)
424 matchExps = self.FileMatchExp['opticalOLDStyle'].split()
425 opt_fNames = list(itertools.chain.from_iterable(
426 [list(sorted(fnmatch.filter(fileList, mE))) for mE in matchExps]))
427 if not opt_fNames:
428 matchExps = self.FileMatchExp['opticalNEWStyle'].split()
429 opt_fNames = list(itertools.chain.from_iterable(
430 [list(sorted(fnmatch.filter(fileList, mE))) for mE in matchExps]))
431 fNames_str = ' '.join(opt_fNames) if self.is_extracted else \
432 ' '.join([os.path.join(self.gdal_path_archive, fName) for fName in opt_fNames])
434 # create stack of TOARef bands
435 self._TOARef = os.path.join(self.tempdir, 'TOARef.vrt')
436 self.run_cmd('gdalbuildvrt %s -resolution user -tr %s %s -separate %s'
437 % (self._TOARef, self.tgt_res, self.tgt_res, fNames_str))
439 return self._TOARef
441 @TOARef.setter
442 def TOARef(self, value):
443 self._TOARef = super(FMASK_Runner_Sentinel2, self).to_saved_rasterFile(value, 'TOARef')
445 def calc_cloudMask(self, path_out=None, fmt=None):
446 # type: (str, str) -> any
448 """Calculate the cloud mask!
450 :param path_out: output path (if not given, a GeoArray instance is returned)
451 :param fmt: output GDAL driver code, e.g. 'ENVI' (only needed if path_out is given)
452 :return: a GeoArray instance of the computed cloud mask
453 """
455 try:
456 self.cloud_mask = os.path.join(self.tempdir, 'fmask_cloudmask.vrt')
457 self.run_cmd('fmask_sentinel2Stacked.py %s'
458 % ' '.join(['-a %s' % self.TOARef,
459 '-z %s' % self.angles_stack,
460 '-o %s' % self.cloud_mask,
461 '-e %s' % os.path.join(self.tempdir, 'FMASK_intermediates')
462 ]))
463 return super(FMASK_Runner_Sentinel2, self).calc_cloudMask(path_out=path_out, fmt=fmt)
465 finally:
466 self.clean()
468 def clean(self):
469 super(FMASK_Runner_Sentinel2, self).clean()
470 assert not os.path.isdir(self.tempdir), 'Error deleting temporary FMASK directory.'
473class Cloud_Mask_Creator(object):
474 def __init__(self, GMS_object, algorithm, target_res=None, tempdir_root=None):
475 """A class for creating cloud masks.
477 :param GMS_object: instance or subclass instance of model.gms_object.GMS_object
478 :param algorithm: 'FMASK' or 'Classical Bayesian
479 :param target_res: target resolution of the computed cloud mask (if not given, the appropriate resolution
480 needed for atmospheric correction is chosen)
481 """
483 self.GMS_obj = GMS_object
484 self.algorithm = algorithm
485 self.tgt_res = target_res if target_res else self.GMS_obj.ac_options['cld_mask']['target_resolution']
486 self.tempdir_root = tempdir_root
488 self.cloud_mask_geoarray = None
489 self.cloud_mask_array = None
490 self.cloud_mask_legend = None
491 self.success = None
493 if self.GMS_obj.satellite not in ['Landsat-4', 'Landsat-5', 'Landsat-7', 'Landsat-8',
494 'Sentinel-2A', 'Sentinel-2B']:
495 self.GMS_obj.error(
496 'Cloud masking is not yet implemented for %s %s...' % (self.GMS_obj.satellite, self.GMS_obj.sensor))
497 self.success = False
498 if algorithm not in ['FMASK', 'Classical Bayesian']: # TODO move validation to config
499 raise ValueError(algorithm)
501 def calc_cloud_mask(self):
502 if self.success is False:
503 return None, None, None
505 self.GMS_obj.logger.info("Calculating cloud mask based on '%s' algorithm..." % self.algorithm)
507 if self.algorithm == 'FMASK':
508 if re.search(r'Landsat', self.GMS_obj.satellite, re.I):
509 FMR = FMASK_Runner_Landsat(self.GMS_obj.path_archive, self.GMS_obj.satellite)
511 else:
512 FMR = FMASK_Runner_Sentinel2(self.GMS_obj.path_archive, self.GMS_obj.satellite,
513 scene_ID=self.GMS_obj.scene_ID,
514 target_res=self.tgt_res, tempdir_root=self.tempdir_root)
516 self.cloud_mask_geoarray = FMR.calc_cloudMask()
517 self.cloud_mask_array = self.cloud_mask_geoarray[:]
518 self.cloud_mask_legend = self.cloud_mask_geoarray.legend
520 else:
521 raise NotImplementedError("The cloud masking algorithm '%s' is not yet implemented." % self.algorithm)
523 self.validate_cloud_mask()
525 return self.cloud_mask_geoarray, self.cloud_mask_array, self.cloud_mask_legend
527 def validate_cloud_mask(self):
528 assert self.cloud_mask_geoarray.xgsd == self.cloud_mask_geoarray.ygsd == self.tgt_res
529 self.success = True