Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
3# gms_preprocessing, spatial and spectral homogenization of satellite remote sensing data
4#
5# Copyright (C) 2020 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
6#
7# This software was developed within the context of the GeoMultiSens project funded
8# by the German Federal Ministry of Education and Research
9# (project grant code: 01 IS 14 010 A-C).
10#
11# This program is free software: you can redistribute it and/or modify it under
12# the terms of the GNU General Public License as published by the Free Software
13# Foundation, either version 3 of the License, or (at your option) any later version.
14# Please note the following exception: `gms_preprocessing` depends on tqdm, which
15# is distributed under the Mozilla Public Licence (MPL) v2.0 except for the files
16# "tqdm/_tqdm.py", "setup.py", "README.rst", "MANIFEST.in" and ".gitignore".
17# Details can be found here: https://github.com/tqdm/tqdm/blob/master/LICENCE.
18#
19# This program is distributed in the hope that it will be useful, but WITHOUT
20# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
21# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
22# details.
23#
24# You should have received a copy of the GNU Lesser General Public License along
25# with this program. If not, see <http://www.gnu.org/licenses/>.
27import os
28import sys
29import warnings
31try:
32 from osgeo import gdal
33except ImportError:
34 import gdal
35from logging import getLogger
36from ecmwfapi.api import APIKeyFetchError, get_apikey_values
38from ..options.config import GMS_config as CFG
39from .spatial_index_mediator import SpatialIndexMediatorServer, Connection
40from .exceptions import GMSEnvironmentError, MissingNonPipLibraryWarning
41from ..misc.locks import redis_conn
43__author__ = 'Daniel Scheffler'
46class GMSEnvironment(object):
47 """GeoMultiSens Environment class."""
49 def __init__(self, logger=None):
50 self.logger = logger or getLogger('GMSEnvironment')
51 self.logger.info('Checking system environment...')
52 self.spatIdxSrvRunning = None
54 def _check_spatial_index_mediator_server(self):
55 try:
56 # get instance of SpatialIndexMediatorServer (raises GMSEnvironmentError if bindings are missing)
57 SpatIdxSrv = SpatialIndexMediatorServer(CFG.path_spatIdxSrv, logger=self.logger)
59 # check if spatial index mediator is running and start if needed
60 if not SpatIdxSrv.is_running:
61 SpatIdxSrv.start()
63 # test connection
64 conn = Connection(host=CFG.spatial_index_server_host, port=CFG.spatial_index_server_port, timeout=5.0)
65 conn.disconnect()
67 os.environ['GMS_SPAT_IDX_SRV_STATUS'] = 'available'
69 except Exception as e:
70 self.logger.error(e, exc_info=False)
71 self.logger.warning('Coregistration will be skipped!')
72 os.environ['GMS_SPAT_IDX_SRV_STATUS'] = 'unavailable'
74 def _check_redis_server(self):
75 if CFG.max_parallel_reads_writes > 0 and not redis_conn:
76 self.logger.warning("Unable to connect to redis server. Is the server installed and running? For "
77 "installation on Ubuntu, use 'sudo apt install redis-server'. \n"
78 "NOTE: Without connection to redis server, any kind of locking (IO / RAM / database) "
79 "is disabled!")
81 def _check_nonpip_packages(self):
82 """Check for not pip-installable packages."""
84 # fmask # conda install -c conda-forge python-fmask
85 try:
86 # noinspection PyUnresolvedReferences
87 import fmask # noqa F401 unused
88 except ImportError:
89 if 'FMASK' in list(CFG.cloud_masking_algorithm.values()):
90 msg = "FMASK library is not installed because it is not pip-installable and must be installed " \
91 "manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
92 self.logger.warning(MissingNonPipLibraryWarning(msg))
94 # 'pyhdf', # conda install --yes -c conda-forge pyhdf
95 try:
96 # noinspection PyUnresolvedReferences
97 import pyhdf # noqa F401 unused
98 except ImportError:
99 if gdal.GetDriverByName('HDF4') is None:
100 msg = "The library 'pyhdf' is missing and the HDF4 driver of GDAL is not available. ASTER data cannot "\
101 "be processed! For Anaconda, run 'conda install --yes -c conda-forge pyhdf' to fix that!"
102 self.logger.warning(MissingNonPipLibraryWarning(msg))
104 # # 'sicor', # pip install git+https://git.gfz-potsdam.de/hollstei/sicor.git
105 # try:
106 # # noinspection PyUnresolvedReferences
107 # import sicor # noqa F401 unused
108 # except ImportError:
109 # msg = "The library 'sicor' has not been installed automatically because installation requires login " \
110 # "credentials. See installation instrucions here: https://git.gfz-potsdam.de/EnMAP/sicor"
111 # self.logger.warning(MissingNonPipLibraryWarning(msg))
113 def check_dependencies(self):
114 # pyLibs = []
115 # javaLibs = []
117 self._check_spatial_index_mediator_server()
118 self._check_redis_server()
119 self._check_nonpip_packages()
121 def check_ports(self):
122 # portsDict = {
123 # 8654: 'SpatialIndexMediator'
124 # }
126 pass
128 @staticmethod
129 def check_read_write_permissions():
130 if not os.path.isdir(CFG.path_tempdir):
131 os.makedirs(CFG.path_tempdir, exist_ok=True)
133 if not os.access(CFG.path_tempdir, os.R_OK):
134 raise PermissionError('No read-permissions at %s.' % CFG.path_tempdir)
136 if not os.access(CFG.path_tempdir, os.W_OK):
137 raise PermissionError('No write-permissions at %s.' % CFG.path_tempdir)
139 if not os.access(CFG.path_tempdir, os.X_OK):
140 raise PermissionError('No delete-permissions at %s.' % CFG.path_tempdir)
142 @staticmethod
143 def check_paths(): # TODO to be completed
144 # check existence of database paths, etc.
145 # check existence of cloud classifier dill objects from PG.get_path_cloud_class_obj()
147 if not os.path.exists(CFG.path_archive):
148 raise GMSEnvironmentError("The given provider archive path '%s' does not exist!" % CFG.path_archive)
150 @staticmethod
151 def ensure_properly_activated_GDAL():
152 if 'conda' in sys.version:
153 conda_rootdir = sys.prefix
154 gdal_data_dir = os.path.join(conda_rootdir, 'share', 'gdal')
156 # GDAL_DATA
157 if 'GDAL_DATA' not in os.environ:
158 # Prevents "Unable to open EPSG support file gcs.csv".
159 warnings.warn("GDAL_DATA variable not set. Setting it to '%s'." % gdal_data_dir)
160 os.environ['GDAL_DATA'] = gdal_data_dir
161 if os.path.abspath(os.environ['GDAL_DATA']) != gdal_data_dir:
162 warnings.warn("GDAL_DATA variable seems to be incorrectly set since Anaconda´s Python is "
163 "executed from %s and GDAL_DATA = %s. Setting it to '%s'."
164 % (conda_rootdir, os.environ['GDAL_DATA'], gdal_data_dir))
165 os.environ['GDAL_DATA'] = gdal_data_dir
167 # CPL_ZIP_ENCODING
168 if 'CPL_ZIP_ENCODING' not in os.environ:
169 # Prevents "Warning 1: Recode from CP437 to UTF-8 failed with the error: "Invalid argument"."
170 # during gdal.Open().
171 os.environ['CPL_ZIP_ENCODING'] = 'UTF-8'
173 def check_ecmwf_api_creds(self):
174 try:
175 get_apikey_values()
176 except APIKeyFetchError as e:
177 self.logger.error(e, exc_info=False)
178 self.logger.warning("ECMWF API credentials could not be found! Atmospheric correction may fail or use "
179 "default input parameters as fallback. Place a credentials file called .ecmwfapirc "
180 "into the user root directory that runs GeoMultiSens or set the environment variables "
181 "'ECMWF_API_KEY', 'ECMWF_API_URL' and 'ECMWF_API_EMAIL'!")