Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
3# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data
4#
5# Copyright (C) 2020 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
6#
7# This software was developed within the context of the GeoMultiSens project funded
8# by the German Federal Ministry of Education and Research
9# (project grant code: 01 IS 14 010 A-C).
10#
11# This program is free software: you can redistribute it and/or modify it under
12# the terms of the GNU General Public License as published by the Free Software
13# Foundation, either version 3 of the License, or (at your option) any later version.
14# Please note the following exception: `gms_preprocessing` depends on tqdm, which
15# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files
16# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore".
17# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE.
18#
19# This program is distributed in the hope that it will be useful, but WITHOUT
20# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
21# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
22# details.
23#
24# You should have received a copy of the GNU Lesser General Public License along
25# with this program. If not, see <http://www.gnu.org/licenses/>.
27"""Definition of gms options schema (as used by cerberus library)."""
30gms_schema_input = dict(
31 global_opts=dict(
32 type='dict', required=False,
33 schema=dict(
34 inmem_serialization=dict(type='boolean', required=False),
35 parallelization_level=dict(type='string', required=False, allowed=['scenes', 'tiles']),
36 spatial_index_server_host=dict(type='string', required=False),
37 spatial_index_server_port=dict(type='integer', required=False),
38 CPUs=dict(type='integer', required=False, nullable=True),
39 CPUs_all_jobs=dict(type='integer', required=False, nullable=True),
40 max_mem_usage=dict(type='integer', required=False, min=0, max=100),
41 critical_mem_usage=dict(type='integer', required=False, min=0, max=100),
42 max_parallel_reads_writes=dict(type='integer', required=False, min=0),
43 allow_subMultiprocessing=dict(type='boolean', required=False),
44 delete_old_output=dict(type='boolean', required=False),
45 disable_exception_handler=dict(type='boolean', required=False),
46 disable_IO_locks=dict(type='boolean', required=False),
47 disable_CPU_locks=dict(type='boolean', required=False),
48 disable_DB_locks=dict(type='boolean', required=False),
49 disable_memory_locks=dict(type='boolean', required=False),
50 min_version_mem_usage_stats=dict(type='string', required=False),
51 log_level=dict(type='string', required=False, allowed=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']),
52 tiling_block_size_XY=dict(type='list', required=False, schema=dict(type="integer"), minlength=2,
53 maxlength=2),
54 is_test=dict(type='boolean', required=False),
55 profiling=dict(type='boolean', required=False),
56 benchmark_global=dict(type='boolean', required=False),
57 )),
58 paths=dict(
59 type='dict', required=False,
60 schema=dict(
61 path_fileserver=dict(type='string', required=False),
62 path_archive=dict(type='string', required=False),
63 path_procdata_scenes=dict(type='string', required=False),
64 path_procdata_MGRS=dict(type='string', required=False),
65 path_tempdir=dict(type='string', required=False),
66 path_benchmarks=dict(type='string', required=False),
67 path_job_logs=dict(type='string', required=False),
68 path_spatIdxSrv=dict(type='string', required=False),
69 path_SNR_models=dict(type='string', required=False),
70 path_dem_proc_srtm_90m=dict(type='string', required=False),
71 path_earthSunDist=dict(type='string', required=False),
72 path_solar_irr=dict(type='string', required=False),
73 path_cloud_classif=dict(type='string', required=False),
74 path_custom_sicor_options=dict(type='string', required=False, nullable=True),
75 path_ECMWF_db=dict(type='string', required=False),
76 path_spechomo_classif=dict(type='string', required=False, nullable=True),
77 )),
78 processors=dict(
79 type='dict', required=False,
80 schema=dict(
81 general_opts=dict(type='dict', required=False, schema=dict(
82 skip_thermal=dict(type='boolean', required=False),
83 skip_pan=dict(type='boolean', required=False),
84 sort_bands_by_cwl=dict(type='boolean', required=False),
85 target_radunit_optical=dict(type='string', required=False, allowed=['Rad', 'TOA_Ref', 'BOA_Ref']),
86 target_radunit_thermal=dict(type='string', required=False, allowed=['Rad', 'Temp']),
87 scale_factor_TOARef=dict(type='integer', required=False),
88 scale_factor_BOARef=dict(type='integer', required=False),
89 mgrs_pixel_buffer=dict(type='integer', required=False),
90 output_data_compression=dict(type='boolean', required=False),
91 write_ENVIclassif_cloudmask=dict(type='boolean', required=False),
92 )),
93 L1A=dict(type='dict', required=False, schema=dict(
94 run_processor=dict(type='boolean', required=False),
95 write_output=dict(type='boolean', required=False),
96 delete_output=dict(type='boolean', required=False),
97 SZA_SAA_calculation_accurracy=dict(type='string', required=False, allowed=['coarse', 'fine']),
98 export_VZA_SZA_SAA_RAA_stats=dict(type='boolean', required=False),
99 )),
100 L1B=dict(type='dict', required=False, schema=dict(
101 run_processor=dict(type='boolean', required=False),
102 write_output=dict(type='boolean', required=False),
103 delete_output=dict(type='boolean', required=False),
104 skip_coreg=dict(type='boolean', required=False),
105 spatial_ref_min_overlap=dict(type='float', required=False, min=0, max=100),
106 spatial_ref_min_cloudcov=dict(type='float', required=False, min=0, max=100),
107 spatial_ref_max_cloudcov=dict(type='float', required=False, min=0, max=100),
108 spatial_ref_plusminus_days=dict(type='integer', required=False),
109 spatial_ref_plusminus_years=dict(type='integer', required=False),
110 coreg_band_wavelength_for_matching=dict(type='integer', required=False, min=350, max=2500),
111 coreg_max_shift_allowed=dict(type='float', required=False, min=0),
112 coreg_window_size=dict(type='list', required=False, minlength=0, maxlength=2,
113 schema=dict(type='integer', required=False, min=8)),
114 )),
115 L1C=dict(type='dict', required=False, schema=dict(
116 run_processor=dict(type='boolean', required=False),
117 write_output=dict(type='boolean', required=False),
118 delete_output=dict(type='boolean', required=False),
119 cloud_masking_algorithm=dict(type='dict', required=False, schema={
120 'Landsat-4': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']),
121 'Landsat-5': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']),
122 'Landsat-7': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']),
123 'Landsat-8': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']),
124 'Sentinel-2A': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian',
125 'SICOR']),
126 'Sentinel-2B': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian',
127 'SICOR']),
128 }),
129 export_L1C_obj_dumps=dict(type='boolean', required=False),
130 auto_download_ecmwf=dict(type='boolean', required=False),
131 ac_fillnonclear_areas=dict(type='boolean', required=False),
132 ac_clear_area_labels=dict(type='list', required=False, schema=dict(type='string', allowed=[
133 "Clear", "Snow", "Water", "Shadow", "Cirrus", "Cloud"])),
134 ac_scale_factor_errors=dict(type='integer', required=False),
135 ac_max_ram_gb=dict(type='integer', required=False),
136 ac_estimate_accuracy=dict(type='boolean', required=False),
137 ac_bandwise_accuracy=dict(type='boolean', required=False),
138 )),
139 L2A=dict(type='dict', required=False, schema=dict(
140 run_processor=dict(type='boolean', required=False),
141 write_output=dict(type='boolean', required=False),
142 delete_output=dict(type='boolean', required=False),
143 align_coord_grids=dict(type='boolean', required=False),
144 match_gsd=dict(type='boolean', required=False),
145 spatial_resamp_alg=dict(type='string', required=False,
146 allowed=['nearest', 'bilinear', 'cubic', 'cubic_spline', 'lanczos', 'average',
147 'mode', 'max', 'min', 'med', 'q1', 'q3']),
148 clip_to_extent=dict(type='boolean', required=False),
149 spathomo_estimate_accuracy=dict(type='boolean', required=False),
150 )),
151 L2B=dict(type='dict', required=False, schema=dict(
152 run_processor=dict(type='boolean', required=False),
153 write_output=dict(type='boolean', required=False),
154 delete_output=dict(type='boolean', required=False),
155 spechomo_method=dict(type='string', required=False, allowed=['LI', 'LR', 'QR', 'RFR']),
156 spechomo_n_clusters=dict(type='integer', required=False, allowed=[1, 5, 10, 15, 20, 30, 40, 50]),
157 spechomo_classif_alg=dict(type='string', required=False, allowed=['MinDist', 'kNN', 'SAM', 'SID']),
158 spechomo_kNN_n_neighbors=dict(type='integer', required=False, min=0),
159 spechomo_estimate_accuracy=dict(type='boolean', required=False),
160 spechomo_bandwise_accuracy=dict(type='boolean', required=False),
161 )),
162 L2C=dict(type='dict', required=False, schema=dict(
163 run_processor=dict(type='boolean', required=False),
164 write_output=dict(type='boolean', required=False),
165 delete_output=dict(type='boolean', required=False),
166 )),
167 )),
168 usecase=dict(
169 type='dict', required=False, schema=dict(
170 virtual_sensor_id=dict(type='integer', required=False), # TODO add possible values
171 datasetid_spatial_ref=dict(type='integer', required=False, nullable=True),
172 datasetid_spectral_ref=dict(type='integer', required=False, nullable=True),
173 target_CWL=dict(type='list', required=False, schema=dict(type='float')),
174 target_FWHM=dict(type='list', required=False, schema=dict(type='float')),
175 target_gsd=dict(type='list', required=False, schema=dict(type='float'), maxlength=2),
176 target_epsg_code=dict(type='integer', required=False, nullable=True),
177 spatial_ref_gridx=dict(type='list', required=False, schema=dict(type='float'), maxlength=2),
178 spatial_ref_gridy=dict(type='list', required=False, schema=dict(type='float'), maxlength=2),
179 )),
180)
183def get_updated_schema(source_schema, key2update, new_value):
184 def deep_update(schema, key2upd, new_val):
185 """Return true if update, else false"""
187 for key in schema:
188 if key == key2upd:
189 schema[key] = new_val
190 elif isinstance(schema[key], dict):
191 deep_update(schema[key], key2upd, new_val)
193 return schema
195 from copy import deepcopy
196 tgt_schema = deepcopy(source_schema)
197 return deep_update(tgt_schema, key2update, new_value)
200gms_schema_config_output = get_updated_schema(gms_schema_input, key2update='required', new_value=True)
203parameter_mapping = dict(
204 # global opts
205 inmem_serialization=('global_opts', 'inmem_serialization'),
206 parallelization_level=('global_opts', 'parallelization_level'),
207 spatial_index_server_host=('global_opts', 'spatial_index_server_host'),
208 spatial_index_server_port=('global_opts', 'spatial_index_server_port'),
209 CPUs=('global_opts', 'CPUs'),
210 CPUs_all_jobs=('global_opts', 'CPUs_all_jobs'),
211 max_mem_usage=('global_opts', 'max_mem_usage'),
212 critical_mem_usage=('global_opts', 'critical_mem_usage'),
213 max_parallel_reads_writes=('global_opts', 'max_parallel_reads_writes'),
214 allow_subMultiprocessing=('global_opts', 'allow_subMultiprocessing'),
215 delete_old_output=('global_opts', 'delete_old_output'),
216 disable_exception_handler=('global_opts', 'disable_exception_handler'),
217 disable_IO_locks=('global_opts', 'disable_IO_locks'),
218 disable_CPU_locks=('global_opts', 'disable_CPU_locks'),
219 disable_DB_locks=('global_opts', 'disable_DB_locks'),
220 disable_memory_locks=('global_opts', 'disable_memory_locks'),
221 min_version_mem_usage_stats=('global_opts', 'min_version_mem_usage_stats'),
222 log_level=('global_opts', 'log_level'),
223 tiling_block_size_XY=('global_opts', 'tiling_block_size_XY'),
224 is_test=('global_opts', 'is_test'),
225 profiling=('global_opts', 'profiling'),
226 benchmark_global=('global_opts', 'benchmark_global'),
228 # paths
229 path_fileserver=('paths', 'path_fileserver'),
230 path_archive=('paths', 'path_archive'),
231 path_procdata_scenes=('paths', 'path_procdata_scenes'),
232 path_procdata_MGRS=('paths', 'path_procdata_MGRS'),
233 path_tempdir=('paths', 'path_tempdir'),
234 path_benchmarks=('paths', 'path_benchmarks'),
235 path_job_logs=('paths', 'path_job_logs'),
236 path_spatIdxSrv=('paths', 'path_spatIdxSrv'),
237 path_SNR_models=('paths', 'path_SNR_models'),
238 path_dem_proc_srtm_90m=('paths', 'path_dem_proc_srtm_90m'),
239 path_earthSunDist=('paths', 'path_earthSunDist'),
240 path_solar_irr=('paths', 'path_solar_irr'),
241 path_cloud_classif=('paths', 'path_cloud_classif'),
242 path_custom_sicor_options=('paths', 'path_custom_sicor_options'),
243 path_ECMWF_db=('paths', 'path_ECMWF_db'),
244 path_spechomo_classif=('paths', 'path_spechomo_classif'),
246 # processors > general opts
247 skip_thermal=('processors', 'general_opts', 'skip_thermal'),
248 skip_pan=('processors', 'general_opts', 'skip_pan'),
249 sort_bands_by_cwl=('processors', 'general_opts', 'sort_bands_by_cwl'),
250 target_radunit_optical=('processors', 'general_opts', 'target_radunit_optical'),
251 target_radunit_thermal=('processors', 'general_opts', 'target_radunit_thermal'),
252 scale_factor_TOARef=('processors', 'general_opts', 'scale_factor_TOARef'),
253 scale_factor_BOARef=('processors', 'general_opts', 'scale_factor_BOARef'),
254 mgrs_pixel_buffer=('processors', 'general_opts', 'mgrs_pixel_buffer'),
255 output_data_compression=('processors', 'general_opts', 'output_data_compression'),
256 write_ENVIclassif_cloudmask=('processors', 'general_opts', 'write_ENVIclassif_cloudmask'),
258 # processors > L1A
259 exec_L1AP=('processors', 'L1A', ['run_processor', 'write_output', 'delete_output']),
260 SZA_SAA_calculation_accurracy=('processors', 'L1A', 'SZA_SAA_calculation_accurracy'),
261 export_VZA_SZA_SAA_RAA_stats=('processors', 'L1A', 'export_VZA_SZA_SAA_RAA_stats'),
263 # processors > L1B
264 exec_L1BP=('processors', 'L1B', ['run_processor', 'write_output', 'delete_output']),
265 skip_coreg=('processors', 'L1B', 'skip_coreg'),
266 spatial_ref_min_overlap=('processors', 'L1B', 'spatial_ref_min_overlap'),
267 spatial_ref_min_cloudcov=('processors', 'L1B', 'spatial_ref_min_cloudcov'),
268 spatial_ref_max_cloudcov=('processors', 'L1B', 'spatial_ref_max_cloudcov'),
269 spatial_ref_plusminus_days=('processors', 'L1B', 'spatial_ref_plusminus_days'),
270 spatial_ref_plusminus_years=('processors', 'L1B', 'spatial_ref_plusminus_years'),
271 coreg_band_wavelength_for_matching=('processors', 'L1B', 'coreg_band_wavelength_for_matching'),
272 coreg_max_shift_allowed=('processors', 'L1B', 'coreg_max_shift_allowed'),
273 coreg_window_size=('processors', 'L1B', 'coreg_window_size'),
275 # processors > L1C
276 exec_L1CP=('processors', 'L1C', ['run_processor', 'write_output', 'delete_output']),
277 cloud_masking_algorithm=('processors', 'L1C', 'cloud_masking_algorithm'),
278 export_L1C_obj_dumps=('processors', 'L1C', 'export_L1C_obj_dumps'),
279 auto_download_ecmwf=('processors', 'L1C', 'auto_download_ecmwf'),
280 ac_fillnonclear_areas=('processors', 'L1C', 'ac_fillnonclear_areas'),
281 ac_clear_area_labels=('processors', 'L1C', 'ac_clear_area_labels'),
282 ac_scale_factor_errors=('processors', 'L1C', 'ac_scale_factor_errors'),
283 ac_max_ram_gb=('processors', 'L1C', 'ac_max_ram_gb'),
284 ac_estimate_accuracy=('processors', 'L1C', 'ac_estimate_accuracy'),
285 ac_bandwise_accuracy=('processors', 'L1C', 'ac_bandwise_accuracy'),
287 # processors > L2A
288 exec_L2AP=('processors', 'L2A', ['run_processor', 'write_output', 'delete_output']),
289 align_coord_grids=('processors', 'L2A', 'align_coord_grids'),
290 match_gsd=('processors', 'L2A', 'match_gsd'),
291 spatial_resamp_alg=('processors', 'L2A', 'spatial_resamp_alg'),
292 clip_to_extent=('processors', 'L2A', 'clip_to_extent'),
293 spathomo_estimate_accuracy=('processors', 'L2A', 'spathomo_estimate_accuracy'),
295 # processors > L2B
296 exec_L2BP=('processors', 'L2B', ['run_processor', 'write_output', 'delete_output']),
297 spechomo_method=('processors', 'L2B', 'spechomo_method'),
298 spechomo_n_clusters=('processors', 'L2B', 'spechomo_n_clusters'),
299 spechomo_classif_alg=('processors', 'L2B', 'spechomo_classif_alg'),
300 spechomo_kNN_n_neighbors=('processors', 'L2B', 'spechomo_kNN_n_neighbors'),
301 spechomo_estimate_accuracy=('processors', 'L2B', 'spechomo_estimate_accuracy'),
302 spechomo_bandwise_accuracy=('processors', 'L2B', 'spechomo_bandwise_accuracy'),
304 # processors > L2C
305 exec_L2CP=('processors', 'L2C', ['run_processor', 'write_output', 'delete_output']),
307 # usecase
308 virtual_sensor_id=('usecase', 'virtual_sensor_id'),
309 datasetid_spatial_ref=('usecase', 'datasetid_spatial_ref'),
310 virtual_sensor_name=('usecase', 'virtual_sensor_name'),
311 datasetid_spectral_ref=('usecase', 'datasetid_spectral_ref'),
312 target_CWL=('usecase', 'target_CWL'),
313 target_FWHM=('usecase', 'target_FWHM'),
314 target_gsd=('usecase', 'target_gsd'),
315 target_epsg_code=('usecase', 'target_epsg_code'),
316 spatial_ref_gridx=('usecase', 'spatial_ref_gridx'),
317 spatial_ref_gridy=('usecase', 'spatial_ref_gridy'),
318)
321def get_param_from_json_config(paramname, json_config):
322 keymap = parameter_mapping[paramname] # tuple
324 dict2search = json_config
325 for i, k in enumerate(keymap):
326 if i < len(keymap) - 1:
327 # not the last element of the tuple -> contains a sub-dictionary
328 dict2search = dict2search[k]
329 elif isinstance(k, list):
330 return [dict2search[sk] for sk in k]
331 else:
332 return dict2search[k]