Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2 

3# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data 

4# 

5# Copyright (C) 2020 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de) 

6# 

7# This software was developed within the context of the GeoMultiSens project funded 

8# by the German Federal Ministry of Education and Research 

9# (project grant code: 01 IS 14 010 A-C). 

10# 

11# This program is free software: you can redistribute it and/or modify it under 

12# the terms of the GNU General Public License as published by the Free Software 

13# Foundation, either version 3 of the License, or (at your option) any later version. 

14# Please note the following exception: `gms_preprocessing` depends on tqdm, which 

15# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files 

16# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore". 

17# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE. 

18# 

19# This program is distributed in the hope that it will be useful, but WITHOUT 

20# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 

21# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 

22# details. 

23# 

24# You should have received a copy of the GNU Lesser General Public License along 

25# with this program. If not, see <http://www.gnu.org/licenses/>. 

26 

27"""Definition of gms options schema (as used by cerberus library).""" 

28 

29 

30gms_schema_input = dict( 

31 global_opts=dict( 

32 type='dict', required=False, 

33 schema=dict( 

34 inmem_serialization=dict(type='boolean', required=False), 

35 parallelization_level=dict(type='string', required=False, allowed=['scenes', 'tiles']), 

36 spatial_index_server_host=dict(type='string', required=False), 

37 spatial_index_server_port=dict(type='integer', required=False), 

38 CPUs=dict(type='integer', required=False, nullable=True), 

39 CPUs_all_jobs=dict(type='integer', required=False, nullable=True), 

40 max_mem_usage=dict(type='integer', required=False, min=0, max=100), 

41 critical_mem_usage=dict(type='integer', required=False, min=0, max=100), 

42 max_parallel_reads_writes=dict(type='integer', required=False, min=0), 

43 allow_subMultiprocessing=dict(type='boolean', required=False), 

44 delete_old_output=dict(type='boolean', required=False), 

45 disable_exception_handler=dict(type='boolean', required=False), 

46 disable_IO_locks=dict(type='boolean', required=False), 

47 disable_CPU_locks=dict(type='boolean', required=False), 

48 disable_DB_locks=dict(type='boolean', required=False), 

49 disable_memory_locks=dict(type='boolean', required=False), 

50 min_version_mem_usage_stats=dict(type='string', required=False), 

51 log_level=dict(type='string', required=False, allowed=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']), 

52 tiling_block_size_XY=dict(type='list', required=False, schema=dict(type="integer"), minlength=2, 

53 maxlength=2), 

54 is_test=dict(type='boolean', required=False), 

55 profiling=dict(type='boolean', required=False), 

56 benchmark_global=dict(type='boolean', required=False), 

57 )), 

58 paths=dict( 

59 type='dict', required=False, 

60 schema=dict( 

61 path_fileserver=dict(type='string', required=False), 

62 path_archive=dict(type='string', required=False), 

63 path_procdata_scenes=dict(type='string', required=False), 

64 path_procdata_MGRS=dict(type='string', required=False), 

65 path_tempdir=dict(type='string', required=False), 

66 path_benchmarks=dict(type='string', required=False), 

67 path_job_logs=dict(type='string', required=False), 

68 path_spatIdxSrv=dict(type='string', required=False), 

69 path_SNR_models=dict(type='string', required=False), 

70 path_dem_proc_srtm_90m=dict(type='string', required=False), 

71 path_earthSunDist=dict(type='string', required=False), 

72 path_solar_irr=dict(type='string', required=False), 

73 path_cloud_classif=dict(type='string', required=False), 

74 path_custom_sicor_options=dict(type='string', required=False, nullable=True), 

75 path_ECMWF_db=dict(type='string', required=False), 

76 path_spechomo_classif=dict(type='string', required=False, nullable=True), 

77 )), 

78 processors=dict( 

79 type='dict', required=False, 

80 schema=dict( 

81 general_opts=dict(type='dict', required=False, schema=dict( 

82 skip_thermal=dict(type='boolean', required=False), 

83 skip_pan=dict(type='boolean', required=False), 

84 sort_bands_by_cwl=dict(type='boolean', required=False), 

85 target_radunit_optical=dict(type='string', required=False, allowed=['Rad', 'TOA_Ref', 'BOA_Ref']), 

86 target_radunit_thermal=dict(type='string', required=False, allowed=['Rad', 'Temp']), 

87 scale_factor_TOARef=dict(type='integer', required=False), 

88 scale_factor_BOARef=dict(type='integer', required=False), 

89 mgrs_pixel_buffer=dict(type='integer', required=False), 

90 output_data_compression=dict(type='boolean', required=False), 

91 write_ENVIclassif_cloudmask=dict(type='boolean', required=False), 

92 )), 

93 L1A=dict(type='dict', required=False, schema=dict( 

94 run_processor=dict(type='boolean', required=False), 

95 write_output=dict(type='boolean', required=False), 

96 delete_output=dict(type='boolean', required=False), 

97 SZA_SAA_calculation_accurracy=dict(type='string', required=False, allowed=['coarse', 'fine']), 

98 export_VZA_SZA_SAA_RAA_stats=dict(type='boolean', required=False), 

99 )), 

100 L1B=dict(type='dict', required=False, schema=dict( 

101 run_processor=dict(type='boolean', required=False), 

102 write_output=dict(type='boolean', required=False), 

103 delete_output=dict(type='boolean', required=False), 

104 skip_coreg=dict(type='boolean', required=False), 

105 spatial_ref_min_overlap=dict(type='float', required=False, min=0, max=100), 

106 spatial_ref_min_cloudcov=dict(type='float', required=False, min=0, max=100), 

107 spatial_ref_max_cloudcov=dict(type='float', required=False, min=0, max=100), 

108 spatial_ref_plusminus_days=dict(type='integer', required=False), 

109 spatial_ref_plusminus_years=dict(type='integer', required=False), 

110 coreg_band_wavelength_for_matching=dict(type='integer', required=False, min=350, max=2500), 

111 coreg_max_shift_allowed=dict(type='float', required=False, min=0), 

112 coreg_window_size=dict(type='list', required=False, minlength=0, maxlength=2, 

113 schema=dict(type='integer', required=False, min=8)), 

114 )), 

115 L1C=dict(type='dict', required=False, schema=dict( 

116 run_processor=dict(type='boolean', required=False), 

117 write_output=dict(type='boolean', required=False), 

118 delete_output=dict(type='boolean', required=False), 

119 cloud_masking_algorithm=dict(type='dict', required=False, schema={ 

120 'Landsat-4': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']), 

121 'Landsat-5': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']), 

122 'Landsat-7': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']), 

123 'Landsat-8': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 'SICOR']), 

124 'Sentinel-2A': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 

125 'SICOR']), 

126 'Sentinel-2B': dict(type='string', required=False, allowed=['FMASK', 'Classical Bayesian', 

127 'SICOR']), 

128 }), 

129 export_L1C_obj_dumps=dict(type='boolean', required=False), 

130 auto_download_ecmwf=dict(type='boolean', required=False), 

131 ac_fillnonclear_areas=dict(type='boolean', required=False), 

132 ac_clear_area_labels=dict(type='list', required=False, schema=dict(type='string', allowed=[ 

133 "Clear", "Snow", "Water", "Shadow", "Cirrus", "Cloud"])), 

134 ac_scale_factor_errors=dict(type='integer', required=False), 

135 ac_max_ram_gb=dict(type='integer', required=False), 

136 ac_estimate_accuracy=dict(type='boolean', required=False), 

137 ac_bandwise_accuracy=dict(type='boolean', required=False), 

138 )), 

139 L2A=dict(type='dict', required=False, schema=dict( 

140 run_processor=dict(type='boolean', required=False), 

141 write_output=dict(type='boolean', required=False), 

142 delete_output=dict(type='boolean', required=False), 

143 align_coord_grids=dict(type='boolean', required=False), 

144 match_gsd=dict(type='boolean', required=False), 

145 spatial_resamp_alg=dict(type='string', required=False, 

146 allowed=['nearest', 'bilinear', 'cubic', 'cubic_spline', 'lanczos', 'average', 

147 'mode', 'max', 'min', 'med', 'q1', 'q3']), 

148 clip_to_extent=dict(type='boolean', required=False), 

149 spathomo_estimate_accuracy=dict(type='boolean', required=False), 

150 )), 

151 L2B=dict(type='dict', required=False, schema=dict( 

152 run_processor=dict(type='boolean', required=False), 

153 write_output=dict(type='boolean', required=False), 

154 delete_output=dict(type='boolean', required=False), 

155 spechomo_method=dict(type='string', required=False, allowed=['LI', 'LR', 'QR', 'RFR']), 

156 spechomo_n_clusters=dict(type='integer', required=False, allowed=[1, 5, 10, 15, 20, 30, 40, 50]), 

157 spechomo_classif_alg=dict(type='string', required=False, allowed=['MinDist', 'kNN', 'SAM', 'SID']), 

158 spechomo_kNN_n_neighbors=dict(type='integer', required=False, min=0), 

159 spechomo_estimate_accuracy=dict(type='boolean', required=False), 

160 spechomo_bandwise_accuracy=dict(type='boolean', required=False), 

161 )), 

162 L2C=dict(type='dict', required=False, schema=dict( 

163 run_processor=dict(type='boolean', required=False), 

164 write_output=dict(type='boolean', required=False), 

165 delete_output=dict(type='boolean', required=False), 

166 )), 

167 )), 

168 usecase=dict( 

169 type='dict', required=False, schema=dict( 

170 virtual_sensor_id=dict(type='integer', required=False), # TODO add possible values 

171 datasetid_spatial_ref=dict(type='integer', required=False, nullable=True), 

172 datasetid_spectral_ref=dict(type='integer', required=False, nullable=True), 

173 target_CWL=dict(type='list', required=False, schema=dict(type='float')), 

174 target_FWHM=dict(type='list', required=False, schema=dict(type='float')), 

175 target_gsd=dict(type='list', required=False, schema=dict(type='float'), maxlength=2), 

176 target_epsg_code=dict(type='integer', required=False, nullable=True), 

177 spatial_ref_gridx=dict(type='list', required=False, schema=dict(type='float'), maxlength=2), 

178 spatial_ref_gridy=dict(type='list', required=False, schema=dict(type='float'), maxlength=2), 

179 )), 

180) 

181 

182 

183def get_updated_schema(source_schema, key2update, new_value): 

184 def deep_update(schema, key2upd, new_val): 

185 """Return true if update, else false""" 

186 

187 for key in schema: 

188 if key == key2upd: 

189 schema[key] = new_val 

190 elif isinstance(schema[key], dict): 

191 deep_update(schema[key], key2upd, new_val) 

192 

193 return schema 

194 

195 from copy import deepcopy 

196 tgt_schema = deepcopy(source_schema) 

197 return deep_update(tgt_schema, key2update, new_value) 

198 

199 

200gms_schema_config_output = get_updated_schema(gms_schema_input, key2update='required', new_value=True) 

201 

202 

203parameter_mapping = dict( 

204 # global opts 

205 inmem_serialization=('global_opts', 'inmem_serialization'), 

206 parallelization_level=('global_opts', 'parallelization_level'), 

207 spatial_index_server_host=('global_opts', 'spatial_index_server_host'), 

208 spatial_index_server_port=('global_opts', 'spatial_index_server_port'), 

209 CPUs=('global_opts', 'CPUs'), 

210 CPUs_all_jobs=('global_opts', 'CPUs_all_jobs'), 

211 max_mem_usage=('global_opts', 'max_mem_usage'), 

212 critical_mem_usage=('global_opts', 'critical_mem_usage'), 

213 max_parallel_reads_writes=('global_opts', 'max_parallel_reads_writes'), 

214 allow_subMultiprocessing=('global_opts', 'allow_subMultiprocessing'), 

215 delete_old_output=('global_opts', 'delete_old_output'), 

216 disable_exception_handler=('global_opts', 'disable_exception_handler'), 

217 disable_IO_locks=('global_opts', 'disable_IO_locks'), 

218 disable_CPU_locks=('global_opts', 'disable_CPU_locks'), 

219 disable_DB_locks=('global_opts', 'disable_DB_locks'), 

220 disable_memory_locks=('global_opts', 'disable_memory_locks'), 

221 min_version_mem_usage_stats=('global_opts', 'min_version_mem_usage_stats'), 

222 log_level=('global_opts', 'log_level'), 

223 tiling_block_size_XY=('global_opts', 'tiling_block_size_XY'), 

224 is_test=('global_opts', 'is_test'), 

225 profiling=('global_opts', 'profiling'), 

226 benchmark_global=('global_opts', 'benchmark_global'), 

227 

228 # paths 

229 path_fileserver=('paths', 'path_fileserver'), 

230 path_archive=('paths', 'path_archive'), 

231 path_procdata_scenes=('paths', 'path_procdata_scenes'), 

232 path_procdata_MGRS=('paths', 'path_procdata_MGRS'), 

233 path_tempdir=('paths', 'path_tempdir'), 

234 path_benchmarks=('paths', 'path_benchmarks'), 

235 path_job_logs=('paths', 'path_job_logs'), 

236 path_spatIdxSrv=('paths', 'path_spatIdxSrv'), 

237 path_SNR_models=('paths', 'path_SNR_models'), 

238 path_dem_proc_srtm_90m=('paths', 'path_dem_proc_srtm_90m'), 

239 path_earthSunDist=('paths', 'path_earthSunDist'), 

240 path_solar_irr=('paths', 'path_solar_irr'), 

241 path_cloud_classif=('paths', 'path_cloud_classif'), 

242 path_custom_sicor_options=('paths', 'path_custom_sicor_options'), 

243 path_ECMWF_db=('paths', 'path_ECMWF_db'), 

244 path_spechomo_classif=('paths', 'path_spechomo_classif'), 

245 

246 # processors > general opts 

247 skip_thermal=('processors', 'general_opts', 'skip_thermal'), 

248 skip_pan=('processors', 'general_opts', 'skip_pan'), 

249 sort_bands_by_cwl=('processors', 'general_opts', 'sort_bands_by_cwl'), 

250 target_radunit_optical=('processors', 'general_opts', 'target_radunit_optical'), 

251 target_radunit_thermal=('processors', 'general_opts', 'target_radunit_thermal'), 

252 scale_factor_TOARef=('processors', 'general_opts', 'scale_factor_TOARef'), 

253 scale_factor_BOARef=('processors', 'general_opts', 'scale_factor_BOARef'), 

254 mgrs_pixel_buffer=('processors', 'general_opts', 'mgrs_pixel_buffer'), 

255 output_data_compression=('processors', 'general_opts', 'output_data_compression'), 

256 write_ENVIclassif_cloudmask=('processors', 'general_opts', 'write_ENVIclassif_cloudmask'), 

257 

258 # processors > L1A 

259 exec_L1AP=('processors', 'L1A', ['run_processor', 'write_output', 'delete_output']), 

260 SZA_SAA_calculation_accurracy=('processors', 'L1A', 'SZA_SAA_calculation_accurracy'), 

261 export_VZA_SZA_SAA_RAA_stats=('processors', 'L1A', 'export_VZA_SZA_SAA_RAA_stats'), 

262 

263 # processors > L1B 

264 exec_L1BP=('processors', 'L1B', ['run_processor', 'write_output', 'delete_output']), 

265 skip_coreg=('processors', 'L1B', 'skip_coreg'), 

266 spatial_ref_min_overlap=('processors', 'L1B', 'spatial_ref_min_overlap'), 

267 spatial_ref_min_cloudcov=('processors', 'L1B', 'spatial_ref_min_cloudcov'), 

268 spatial_ref_max_cloudcov=('processors', 'L1B', 'spatial_ref_max_cloudcov'), 

269 spatial_ref_plusminus_days=('processors', 'L1B', 'spatial_ref_plusminus_days'), 

270 spatial_ref_plusminus_years=('processors', 'L1B', 'spatial_ref_plusminus_years'), 

271 coreg_band_wavelength_for_matching=('processors', 'L1B', 'coreg_band_wavelength_for_matching'), 

272 coreg_max_shift_allowed=('processors', 'L1B', 'coreg_max_shift_allowed'), 

273 coreg_window_size=('processors', 'L1B', 'coreg_window_size'), 

274 

275 # processors > L1C 

276 exec_L1CP=('processors', 'L1C', ['run_processor', 'write_output', 'delete_output']), 

277 cloud_masking_algorithm=('processors', 'L1C', 'cloud_masking_algorithm'), 

278 export_L1C_obj_dumps=('processors', 'L1C', 'export_L1C_obj_dumps'), 

279 auto_download_ecmwf=('processors', 'L1C', 'auto_download_ecmwf'), 

280 ac_fillnonclear_areas=('processors', 'L1C', 'ac_fillnonclear_areas'), 

281 ac_clear_area_labels=('processors', 'L1C', 'ac_clear_area_labels'), 

282 ac_scale_factor_errors=('processors', 'L1C', 'ac_scale_factor_errors'), 

283 ac_max_ram_gb=('processors', 'L1C', 'ac_max_ram_gb'), 

284 ac_estimate_accuracy=('processors', 'L1C', 'ac_estimate_accuracy'), 

285 ac_bandwise_accuracy=('processors', 'L1C', 'ac_bandwise_accuracy'), 

286 

287 # processors > L2A 

288 exec_L2AP=('processors', 'L2A', ['run_processor', 'write_output', 'delete_output']), 

289 align_coord_grids=('processors', 'L2A', 'align_coord_grids'), 

290 match_gsd=('processors', 'L2A', 'match_gsd'), 

291 spatial_resamp_alg=('processors', 'L2A', 'spatial_resamp_alg'), 

292 clip_to_extent=('processors', 'L2A', 'clip_to_extent'), 

293 spathomo_estimate_accuracy=('processors', 'L2A', 'spathomo_estimate_accuracy'), 

294 

295 # processors > L2B 

296 exec_L2BP=('processors', 'L2B', ['run_processor', 'write_output', 'delete_output']), 

297 spechomo_method=('processors', 'L2B', 'spechomo_method'), 

298 spechomo_n_clusters=('processors', 'L2B', 'spechomo_n_clusters'), 

299 spechomo_classif_alg=('processors', 'L2B', 'spechomo_classif_alg'), 

300 spechomo_kNN_n_neighbors=('processors', 'L2B', 'spechomo_kNN_n_neighbors'), 

301 spechomo_estimate_accuracy=('processors', 'L2B', 'spechomo_estimate_accuracy'), 

302 spechomo_bandwise_accuracy=('processors', 'L2B', 'spechomo_bandwise_accuracy'), 

303 

304 # processors > L2C 

305 exec_L2CP=('processors', 'L2C', ['run_processor', 'write_output', 'delete_output']), 

306 

307 # usecase 

308 virtual_sensor_id=('usecase', 'virtual_sensor_id'), 

309 datasetid_spatial_ref=('usecase', 'datasetid_spatial_ref'), 

310 virtual_sensor_name=('usecase', 'virtual_sensor_name'), 

311 datasetid_spectral_ref=('usecase', 'datasetid_spectral_ref'), 

312 target_CWL=('usecase', 'target_CWL'), 

313 target_FWHM=('usecase', 'target_FWHM'), 

314 target_gsd=('usecase', 'target_gsd'), 

315 target_epsg_code=('usecase', 'target_epsg_code'), 

316 spatial_ref_gridx=('usecase', 'spatial_ref_gridx'), 

317 spatial_ref_gridy=('usecase', 'spatial_ref_gridy'), 

318) 

319 

320 

321def get_param_from_json_config(paramname, json_config): 

322 keymap = parameter_mapping[paramname] # tuple 

323 

324 dict2search = json_config 

325 for i, k in enumerate(keymap): 

326 if i < len(keymap) - 1: 

327 # not the last element of the tuple -> contains a sub-dictionary 

328 dict2search = dict2search[k] 

329 elif isinstance(k, list): 

330 return [dict2search[sk] for sk in k] 

331 else: 

332 return dict2search[k]