Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2 

3# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data 

4# 

5# Copyright (C) 2020 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de) 

6# 

7# This software was developed within the context of the GeoMultiSens project funded 

8# by the German Federal Ministry of Education and Research 

9# (project grant code: 01 IS 14 010 A-C). 

10# 

11# This program is free software: you can redistribute it and/or modify it under 

12# the terms of the GNU General Public License as published by the Free Software 

13# Foundation, either version 3 of the License, or (at your option) any later version. 

14# Please note the following exception: `gms_preprocessing` depends on tqdm, which 

15# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files 

16# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore". 

17# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE. 

18# 

19# This program is distributed in the hope that it will be useful, but WITHOUT 

20# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 

21# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more 

22# details. 

23# 

24# You should have received a copy of the GNU Lesser General Public License along 

25# with this program. If not, see <http://www.gnu.org/licenses/>. 

26 

27import os 

28import sys 

29import warnings 

30 

31try: 

32 from osgeo import gdal 

33except ImportError: 

34 import gdal 

35from logging import getLogger 

36from ecmwfapi.api import APIKeyFetchError, get_apikey_values 

37 

38from ..options.config import GMS_config as CFG 

39from .spatial_index_mediator import SpatialIndexMediatorServer, Connection 

40from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning 

41from ..misc.locks import redis_conn 

42 

43__author__ = 'Daniel Scheffler' 

44 

45 

46class GMSEnvironment(object): 

47 """GeoMultiSens Environment class.""" 

48 

49 def __init__(self, logger=None): 

50 self.logger = logger or getLogger('GMSEnvironment') 

51 self.logger.info('Checking system environment...') 

52 self.spatIdxSrvRunning = None 

53 

54 def _check_spatial_index_mediator_server(self): 

55 try: 

56 # get instance of SpatialIndexMediatorServer (raises GMSEnvironmentError if bindings are missing) 

57 SpatIdxSrv = SpatialIndexMediatorServer(CFG.path_spatIdxSrv, logger=self.logger) 

58 

59 # check if spatial index mediator is running and start if needed 

60 if not SpatIdxSrv.is_running: 

61 SpatIdxSrv.start() 

62 

63 # test connection 

64 conn = Connection(host=CFG.spatial_index_server_host, port=CFG.spatial_index_server_port, timeout=5.0) 

65 conn.disconnect() 

66 

67 os.environ['GMS_SPAT_IDX_SRV_STATUS'] = 'available' 

68 

69 except Exception as e: 

70 self.logger.error(e, exc_info=False) 

71 self.logger.warning('Coregistration will be skipped!') 

72 os.environ['GMS_SPAT_IDX_SRV_STATUS'] = 'unavailable' 

73 

74 def _check_redis_server(self): 

75 if CFG.max_parallel_reads_writes > 0 and not redis_conn: 

76 self.logger.warning("Unable to connect to redis server. Is the server installed and running? For " 

77 "installation on Ubuntu, use 'sudo apt install redis-server'. \n" 

78 "NOTE: Without connection to redis server, any kind of locking (IO / RAM / database) " 

79 "is disabled!") 

80 

81 def _check_nonpip_packages(self): 

82 """Check for not pip-installable packages.""" 

83 

84 # fmask # conda install -c conda-forge python-fmask 

85 try: 

86 # noinspection PyUnresolvedReferences 

87 import fmask # noqa F401 unused 

88 except ImportError: 

89 if 'FMASK' in list(CFG.cloud_masking_algorithm.values()): 

90 msg = "FMASK library is not installed because it is not pip-installable and must be installed " \ 

91 "manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!" 

92 self.logger.warning(MissingNonPipLibraryWarning(msg)) 

93 

94 # 'pyhdf', # conda install --yes -c conda-forge pyhdf 

95 try: 

96 # noinspection PyUnresolvedReferences 

97 import pyhdf # noqa F401 unused 

98 except ImportError: 

99 if gdal.GetDriverByName('HDF4') is None: 

100 msg = "The library 'pyhdf' is missing and the HDF4 driver of GDAL is not available. ASTER data cannot "\ 

101 "be processed! For Anaconda, run 'conda install --yes -c conda-forge pyhdf' to fix that!" 

102 self.logger.warning(MissingNonPipLibraryWarning(msg)) 

103 

104 # # 'sicor', # pip install git+https://git.gfz-potsdam.de/hollstei/sicor.git 

105 # try: 

106 # # noinspection PyUnresolvedReferences 

107 # import sicor # noqa F401 unused 

108 # except ImportError: 

109 # msg = "The library 'sicor' has not been installed automatically because installation requires login " \ 

110 # "credentials. See installation instrucions here: https://git.gfz-potsdam.de/EnMAP/sicor" 

111 # self.logger.warning(MissingNonPipLibraryWarning(msg)) 

112 

113 def check_dependencies(self): 

114 # pyLibs = [] 

115 # javaLibs = [] 

116 

117 self._check_spatial_index_mediator_server() 

118 self._check_redis_server() 

119 self._check_nonpip_packages() 

120 

121 def check_ports(self): 

122 # portsDict = { 

123 # 8654: 'SpatialIndexMediator' 

124 # } 

125 

126 pass 

127 

128 @staticmethod 

129 def check_read_write_permissions(): 

130 if not os.path.isdir(CFG.path_tempdir): 

131 os.makedirs(CFG.path_tempdir, exist_ok=True) 

132 

133 if not os.access(CFG.path_tempdir, os.R_OK): 

134 raise PermissionError('No read-permissions at %s.' % CFG.path_tempdir) 

135 

136 if not os.access(CFG.path_tempdir, os.W_OK): 

137 raise PermissionError('No write-permissions at %s.' % CFG.path_tempdir) 

138 

139 if not os.access(CFG.path_tempdir, os.X_OK): 

140 raise PermissionError('No delete-permissions at %s.' % CFG.path_tempdir) 

141 

142 @staticmethod 

143 def check_paths(): # TODO to be completed 

144 # check existence of database paths, etc. 

145 # check existence of cloud classifier dill objects from PG.get_path_cloud_class_obj() 

146 

147 if not os.path.exists(CFG.path_archive): 

148 raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.path_archive) 

149 

150 @staticmethod 

151 def ensure_properly_activated_GDAL(): 

152 if 'conda' in sys.version: 

153 conda_rootdir = sys.prefix 

154 gdal_data_dir = os.path.join(conda_rootdir, 'share', 'gdal') 

155 

156 # GDAL_DATA 

157 if 'GDAL_DATA' not in os.environ: 

158 # Prevents "Unable to open EPSG support file gcs.csv". 

159 warnings.warn("GDAL_DATA variable not set. Setting it to '%s'." % gdal_data_dir) 

160 os.environ['GDAL_DATA'] = gdal_data_dir 

161 if os.path.abspath(os.environ['GDAL_DATA']) != gdal_data_dir: 

162 warnings.warn("GDAL_DATA variable seems to be incorrectly set since Anaconda´s Python is " 

163 "executed from %s and GDAL_DATA = %s. Setting it to '%s'." 

164 % (conda_rootdir, os.environ['GDAL_DATA'], gdal_data_dir)) 

165 os.environ['GDAL_DATA'] = gdal_data_dir 

166 

167 # CPL_ZIP_ENCODING 

168 if 'CPL_ZIP_ENCODING' not in os.environ: 

169 # Prevents "Warning 1: Recode from CP437 to UTF-8 failed with the error: "Invalid argument"." 

170 # during gdal.Open(). 

171 os.environ['CPL_ZIP_ENCODING'] = 'UTF-8' 

172 

173 def check_ecmwf_api_creds(self): 

174 try: 

175 get_apikey_values() 

176 except APIKeyFetchError as e: 

177 self.logger.error(e, exc_info=False) 

178 self.logger.warning("ECMWF API credentials could not be found! Atmospheric correction may fail or use " 

179 "default input parameters as fallback. Place a credentials file called .ecmwfapirc " 

180 "into the user root directory that runs GeoMultiSens or set the environment variables " 

181 "'ECMWF_API_KEY', 'ECMWF_API_URL' and 'ECMWF_API_EMAIL'!")