diff -Nru eoxserver-1.1.2/.bumpversion.cfg eoxserver-1.2.2/.bumpversion.cfg --- eoxserver-1.1.2/.bumpversion.cfg 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/.bumpversion.cfg 2023-02-20 12:10:15.000000000 +0000 @@ -1,16 +1,15 @@ [bumpversion] -current_version = 1.1.2 +current_version = 1.2.2 commit = True tag = True -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+).(?P\d+))? serialize = - {major}.{minor}.{patch}-{release}{build} + {major}.{minor}.{patch}-{release}.{build} {major}.{minor}.{patch} tag_name = release-{new_version} [bumpversion:part:release] -optional_value = final -first_value = dev +optional_value = dev values = dev alpha diff -Nru eoxserver-1.1.2/debian/changelog eoxserver-1.2.2/debian/changelog --- eoxserver-1.1.2/debian/changelog 2022-07-22 10:54:15.000000000 +0000 +++ eoxserver-1.2.2/debian/changelog 2023-03-06 09:44:43.000000000 +0000 @@ -1,3 +1,21 @@ +eoxserver (1.2.2-0~jammy2) jammy; urgency=low + + * Applied upstream patch https://github.com/EOxServer/eoxserver/commit/20f33cf01a3928e08aa84b9c94771a4c03166f12. + + -- Angelos Tzotsos Mon, 06 Mar 2023 12:00:00 +0200 + +eoxserver (1.2.2-0~jammy1) jammy; urgency=low + + * Updated dependency list. + + -- Angelos Tzotsos Sun, 05 Mar 2023 14:00:00 +0200 + +eoxserver (1.2.2-0~jammy0) jammy; urgency=low + + * New upstream version. + + -- Angelos Tzotsos Sun, 05 Mar 2023 14:00:00 +0200 + eoxserver (1.1.2-1~jammy0) jammy; urgency=low * Patch for VSIFile return value. diff -Nru eoxserver-1.1.2/debian/control eoxserver-1.2.2/debian/control --- eoxserver-1.1.2/debian/control 2022-07-03 11:00:00.000000000 +0000 +++ eoxserver-1.2.2/debian/control 2023-03-05 18:19:39.000000000 +0000 @@ -9,6 +9,8 @@ autoconf, automake, autotools-dev, + libgdal-dev, + gdal-bin, python3-all, python3-all-dev Standards-Version: 3.9.3 @@ -24,10 +26,17 @@ python3-libxml2, python3, debconf, + gdal-bin, python3-ply, python3-django-model-utils, python3-zipstream, python3-dateutil, + python3-pytzdata, + python3-psycopg2, + python3-matplotlib, + python3-keystoneclient, + python3-swiftclient, + python3-mapscript, python3-gdal, python3-pycql, python3-pyows, diff -Nru eoxserver-1.1.2/debian/patches/charonpdp.patch eoxserver-1.2.2/debian/patches/charonpdp.patch --- eoxserver-1.1.2/debian/patches/charonpdp.patch 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/debian/patches/charonpdp.patch 2023-03-06 09:45:44.000000000 +0000 @@ -0,0 +1,32 @@ +Description: Syntax typo fix + * Applied upstream patch https://github.com/EOxServer/eoxserver/commit/20f33cf01a3928e08aa84b9c94771a4c03166f12. +Author: Angelos Tzotsos + +--- +The information above should follow the Patch Tagging Guidelines, please +checkout http://dep.debian.net/deps/dep3/ to learn about the format. Here +are templates for supplementary fields that you might want to add: + +Origin: , +Bug: +Bug-Debian: https://bugs.debian.org/ +Bug-Ubuntu: https://launchpad.net/bugs/ +Forwarded: +Reviewed-By: +Last-Update: 2023-03-06 + +--- eoxserver-1.2.2.orig/eoxserver/services/auth/charonpdp.py ++++ eoxserver-1.2.2/eoxserver/services/auth/charonpdp.py +@@ -293,7 +293,11 @@ class AuthorisationClient(object): + # Get XML snippet for the Environment part of the XACMLAuthzDecisionQuery + def _getPartEnvironment(self): + now = datetime.datetime.now() +- formattedNow = now.strftime("%Y-%m-%dT%H:%M:%S.%f%z")urlparse ++ formattedNow = now.strftime("%Y-%m-%dT%H:%M:%S.%f%z") ++ ++ return template_attribute.format(attrib_current_date, \ ++ dt_date, \ ++ formattedNow) + # Get the full XACMLAuthzDecisionQuery + def _getFullRequest(self, userAttributes, resourceAttributes, action): + return template_request.format(self._getPartSubject(userAttributes), \ diff -Nru eoxserver-1.1.2/debian/patches/series eoxserver-1.2.2/debian/patches/series --- eoxserver-1.1.2/debian/patches/series 2022-07-22 10:56:13.000000000 +0000 +++ eoxserver-1.2.2/debian/patches/series 2023-03-06 09:45:09.000000000 +0000 @@ -1 +1 @@ -vsi.patch +charonpdp.patch diff -Nru eoxserver-1.1.2/.dockerignore eoxserver-1.2.2/.dockerignore --- eoxserver-1.1.2/.dockerignore 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/.dockerignore 2023-02-20 12:10:15.000000000 +0000 @@ -3,4 +3,5 @@ build dist documentation -schemas \ No newline at end of file +schemas +debian/ diff -Nru eoxserver-1.1.2/eoxserver/backends/access.py eoxserver-1.2.2/eoxserver/backends/access.py --- eoxserver-1.1.2/eoxserver/backends/access.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/backends/access.py 2023-02-20 12:10:15.000000000 +0000 @@ -143,7 +143,16 @@ location = data_item.location storage = data_item.storage - return get_vsi_storage_path(storage, location) + vsi_path = get_vsi_storage_path(storage, location) + subdataset_type = getattr(data_item, 'subdataset_type', None) + subdataset_locator = getattr(data_item, 'subdataset_locator', None) + + if subdataset_type: + vsi_path = '%s:"%s"' % (subdataset_type, vsi_path) + if subdataset_locator: + vsi_path = '%s:%s' % (vsi_path, subdataset_locator) + + return vsi_path def get_vsi_storage_path(storage, location=None): diff -Nru eoxserver-1.1.2/eoxserver/backends/models.py eoxserver-1.2.2/eoxserver/backends/models.py --- eoxserver-1.1.2/eoxserver/backends/models.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/backends/models.py 2023-02-20 12:10:15.000000000 +0000 @@ -29,7 +29,6 @@ from django.db import models from django.core.exceptions import ValidationError -from django.utils.encoding import python_2_unicode_compatible from eoxserver.backends.storages import get_handler_class_by_name @@ -43,7 +42,6 @@ # ============================================================================== -@python_2_unicode_compatible class StorageAuth(models.Model): """ Model to symbolize authorization for storages. """ @@ -59,7 +57,6 @@ validate_storage_auth(self) -@python_2_unicode_compatible class Storage(models.Model): """ Model to symbolize storages that provide file or other types of access to data items. @@ -78,7 +75,6 @@ validate_storage(self) -@python_2_unicode_compatible class DataItem(models.Model): """ Abstract model for locateable data items contributing to a dataset. """ diff -Nru eoxserver-1.1.2/eoxserver/COMMITTERS eoxserver-1.2.2/eoxserver/COMMITTERS --- eoxserver-1.1.2/eoxserver/COMMITTERS 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/COMMITTERS 1970-01-01 00:00:00.000000000 +0000 @@ -1,15 +0,0 @@ -============== ===================== =================================== ================================== - Login Name Email / Contact Area(s) -============== ===================== =================================== ================================== -meissls Stephan Meissl stephan.meissl at eox.at Overall -krauses Stephan Krause stephan.krause at eox.at Overall -schindlerf Fabian Schindler fabian.schindler at eox.at OGC Services -novacek Milan Novacek milan.novacek at siemens.com S2P (WCS) Proxy -martin.paces Martin Paces martin.paces at eox.at WCS-T, WPS -abonitz Arndt Bonitz arndt.bonitz at ait.ac.at IDM -MiroslavHoudek Miroslav Houdek miroslav.houdek at iguassu.cz n/a -ungarj Joachim Ungar joachim.ungar at eox.at Usability & Testing -schillerc Christian Schiller christian.schiller at eox.at Documentation, Usability & Testing -locherm Marko Locher marko.locher at eox.at OSGeo Live Package -santilland Daniel Santillan daniel.santillan at eox.at Client & 3D extensions -============== ===================== =================================== ================================== diff -Nru eoxserver-1.1.2/eoxserver/contrib/vsi.py eoxserver-1.2.2/eoxserver/contrib/vsi.py --- eoxserver-1.1.2/eoxserver/contrib/vsi.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/contrib/vsi.py 2023-02-20 12:10:15.000000000 +0000 @@ -1,9 +1,9 @@ -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # # Project: EOxServer # Authors: Fabian Schindler # -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Copyright (C) 2013 EOX IT Services GmbH # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -13,8 +13,8 @@ # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # -# The above copyright notice and this permission notice shall be included in all -# copies of this Software or works derived from this Software. +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -23,7 +23,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ """ This module provides Python file-object like access to VSI files. @@ -33,6 +33,11 @@ import os from uuid import uuid4 from functools import wraps +import mimetypes +from urllib.parse import quote + +from django.http import StreamingHttpResponse + if os.environ.get('READTHEDOCS', None) != 'True': import numpy @@ -111,7 +116,14 @@ """ value = VSIFReadL(1, size, self._handle) - return value if value is not None else '' + if isinstance(value, bytes): + return value + elif isinstance(value, bytearray): + return bytes(value) + elif value is None: + return b'' + else: + raise ValueError(value) @_ensure_open def write(self, data): @@ -291,3 +303,49 @@ else: parts.extend(new) return '/'.join(parts) + + +class VSIFileResponse(StreamingHttpResponse): + """ Subclass of StreamingHttpResponse, a replacement for Django's + FileResponse which does not work for VSIFiles in Django v3.2.15 + """ + # inspired from https://github.com/django/django/blob/bd062445cffd3f6cc6dcd20d13e2abed818fa173/django/http/response.py#L500 + + block_size = 4096 + + def __init__(self, *args, as_attachment=False, filename='', **kwargs): + self.as_attachment = as_attachment + self.filename = filename + super().__init__(*args, **kwargs) + + def _set_streaming_content(self, value): + filelike = value + self._resource_closers.append(filelike.close) + value = iter(lambda: filelike.read(self.block_size), b'') + self.set_headers(filelike) + super()._set_streaming_content(value) + + def set_headers(self, filelike): + self.headers["Content-Length"] = filelike.size + content_type, encoding = mimetypes.guess_type(filelike.name) + content_type = { + "bzip2": "application/x-bzip", + "gzip": "application/gzip", + "xz": "application/x-xz", + }.get(encoding, content_type) + self.headers["Content-Type"] = ( + content_type or "application/octet-stream" + ) + + filename = self.filename or filelike.name + disposition = "attachment" if self.as_attachment else "inline" + try: + filename.encode("ascii") + file_expr = 'filename="{}"'.format( + filename.replace("\\", "\\\\").replace('"', r"\"") + ) + except UnicodeEncodeError: + file_expr = "filename*=utf-8''{}".format(quote(filename)) + self.headers["Content-Disposition"] = "{}; {}".format( + disposition, file_expr + ) diff -Nru eoxserver-1.1.2/eoxserver/COPYING eoxserver-1.2.2/eoxserver/COPYING --- eoxserver-1.1.2/eoxserver/COPYING 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/COPYING 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ - EOxServer Open License - Version 1, 8 June 2011 - -Copyright (C) 2011 EOX IT Services GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies of this Software or works derived from this Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff -Nru eoxserver-1.1.2/eoxserver/__init__.py eoxserver-1.2.2/eoxserver/__init__.py --- eoxserver-1.1.2/eoxserver/__init__.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/__init__.py 2023-02-20 12:10:15.000000000 +0000 @@ -28,7 +28,7 @@ # ------------------------------------------------------------------------------ -__version__ = '1.1.2' +__version__ = '1.2.2' def get_version(): diff -Nru eoxserver-1.1.2/eoxserver/INSTALL eoxserver-1.2.2/eoxserver/INSTALL --- eoxserver-1.1.2/eoxserver/INSTALL 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/INSTALL 1970-01-01 00:00:00.000000000 +0000 @@ -1,110 +0,0 @@ --------------------------------------------------------------------------------- - - Project: EOxServer - Purpose: - Authors: Stephan Krause - Stephan Meissl - --------------------------------------------------------------------------------- -Copyright (C) 2011 EOX IT Services GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies of this Software or works derived from this Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. --------------------------------------------------------------------------------- - - -############################################## -# Quick installation guide for the impatient # -############################################## - -sudo pip install eoxserver -eoxserver-admin.py create_instance YOUR_INSTANCE_ID --init_spatialite -cd YOUR_INSTANCE_ID -python manage.py syncdb - - -+--------------------------------------------+ -| Running from the command-line -+--------------------------------------------+ - -python manage.py runserver - -# Point your browser to: "http://localhost:8000/" - - -+--------------------------------------------+ -| Running via WSGI interface -+--------------------------------------------+ - -mkdir static -python manage.py collectstatic --noinput - -# Add the following to your Apache web server configuration -# (e.g. /etc/apache2/sites-enabled/eoxserver): ------------------------------------------------------------------ -Alias /static "/static" -Alias /eoxserver "/wsgi.py" -"> - AllowOverride None - Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch - AddHandler wsgi-script .py - Order allow,deny - allow from all - ------------------------------------------------------------------ -# Restart Apache web server and point your browser to: -# "http://. - - --------------------------------------------------------------------------------- -License (see also file named COPYING) --------------------------------------------------------------------------------- - -Copyright (C) 2011 EOX IT Services GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies of this Software or works derived from this Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Credits --------------------------------------------------------------------------------- - -Work on EOxServer has been partly funded by the European Space Agency (ESA) -in the frame of the HMA-FO and O3S projects. -Link: http://rssportal.esa.int/tiki-index.php?page=Open%20Software diff -Nru eoxserver-1.1.2/eoxserver/render/browse/functions.py eoxserver-1.2.2/eoxserver/render/browse/functions.py --- eoxserver-1.1.2/eoxserver/render/browse/functions.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/browse/functions.py 2023-02-20 12:10:15.000000000 +0000 @@ -28,12 +28,14 @@ import logging from uuid import uuid4 from functools import wraps +from typing import List, SupportsFloat as Numeric import numpy as np from eoxserver.contrib import gdal from eoxserver.contrib import ogr from eoxserver.contrib import gdal_array +from eoxserver.render.browse.util import convert_dtype logger = logging.getLogger(__name__) @@ -53,15 +55,10 @@ processing, **kwargs ) - - out_ds = gdal.Open(filename) - band = out_ds.GetRasterBand(1) - out_data = band.ReadAsArray() - del out_ds - finally: + except Exception: gdal.Unlink(filename) - - return gdal_array.OpenNumPyArray(out_data, False) + out_ds = gdal.Open(filename) + return out_ds def hillshade(data, zfactor=1, scale=1, azimuth=315, altitude=45, alg='Horn'): @@ -170,14 +167,11 @@ outputBounds=[xmin, ymin, xmax, ymax], ) - out_ds = gdal.Open(out_filename) - band = out_ds.GetRasterBand(1) - out_data = gdal_array.OpenNumPyArray(band.ReadAsArray(), False) - del out_ds - gdal.Unlink(out_filename) + out_data = gdal.Open(out_filename) elif format == 'vector': out_data = vector_ds - + except Exception: + gdal.Unlink(out_filename) finally: vector_driver.DeleteDataSource(vec_filename) @@ -203,6 +197,12 @@ ) out_ds = gdal_array.OpenNumPyArray(ds.ReadAsArray(), True) + # restore original nodata from pan band to output ds + nodata_value = pan_ds.GetRasterBand(1).GetNoDataValue() + if nodata_value is not None: + for i in range(out_ds.RasterCount): + out_ds.GetRasterBand(i + 1).SetNoDataValue(nodata_value) + return out_ds @@ -212,10 +212,6 @@ if histogram: min_, max_, _, buckets = histogram bucket_diff = (max_ - min_) / len(buckets) - nodata = band.GetNoDataValue() - if nodata is not None: - # Set bucket of nodata value to 0 - buckets[round((nodata - min_) / bucket_diff)] = 0 cumsum = np.cumsum(buckets) bucket_index = np.searchsorted(cumsum, cumsum[-1] * (perc / 100)) return min_ + (bucket_index * bucket_diff) @@ -225,6 +221,7 @@ def _has_stats(band): return 'STATISTICS_MINIMUM' in band.GetMetadata() + def statistics_min(ds, default=0): band = ds.GetRasterBand(1) if _has_stats(band): @@ -240,6 +237,7 @@ return max_ return default + def statistics_mean(ds, default=0): band = ds.GetRasterBand(1) if _has_stats(band): @@ -256,52 +254,101 @@ return default -def interpolate(ds, x1, x2, y1, y2): - """Perform linear interpolation for x between (x1,y1) and (x2,y2) """ +def interpolate( + ds:"gdal.Dataset", x1:Numeric, x2:Numeric, y1:Numeric, y2:Numeric, clip:bool=False, nodata_range:List[Numeric]=None + ): + """Perform linear interpolation for x between (x1,y1) and (x2,y2) with + optional clamp and additional masking out multiple no data value ranges + + Args: + ds (gdal.Dataset): input gdal dataset + x1 (Numeric): linear interpolate from min + x2 (Numeric): linear interpolate from max + y1 (Numeric): linear interpolate to min + y2 (Numeric): linear interpolate to max + clip (bool, optional): if set to True, performs clip (values below y1 set to y1, values above y2 set to y2). Defaults to False. + additional_no_data (List, optional): additionally masks out (sets to band no_data_value) a range of values. Defaults to []. Example [1,5] + + Returns: + gdal.Dataset: Interpolated dataset + """ band = ds.GetRasterBand(1) - x = band.ReadAsArray() - x = ((y2 - y1) * x + x2 * y1 - x1 * y2) / (x2 - x1) - return gdal_array.OpenNumPyArray(x, True) + nodata_value = band.GetNoDataValue() + orig_image = band.ReadAsArray() + # NOTE: the interpolate formula uses large numbers which lead to overflows on uint16 + if orig_image.dtype != convert_dtype(orig_image.dtype): + orig_image = orig_image.astype(convert_dtype(orig_image.dtype)) + interpolated_image = ((y2 - y1) * orig_image + x2 * y1 - x1 * y2) / (x2 - x1) + if clip: + # clamp values below min to min and above max to max + np.clip(interpolated_image, y1, y2, out=interpolated_image) + if nodata_value is not None: + # restore nodata pixels on interpolated array from original array + interpolated_image[orig_image == nodata_value] = nodata_value + if nodata_range: + # apply mask of additional nodata ranges from original array on interpolated array + interpolated_image[(orig_image >= nodata_range[0]) & (orig_image <= nodata_range[1])] = nodata_value + + ds = gdal_array.OpenNumPyArray(interpolated_image, True) + if nodata_value is not None: + ds.GetRasterBand(1).SetNoDataValue(nodata_value) + return ds def wrap_numpy_func(function): @wraps(function) - def inner(ds, *args, **kwargs): - band = ds.GetRasterBand(1) - data = band.ReadAsArray() - function(data, *args, **kwargs) - band.WriteArray(data) - return ds + def inner(*args, **kwargs): + converted_args = [] + for arg in args: + if isinstance(arg, gdal.Dataset): + band = arg.GetRasterBand(1) + data = band.ReadAsArray() + converted_args.append(data) + else: + converted_args.append(arg) + + result = function(*converted_args, **kwargs) + + if np.isscalar(result): + return result + + arg = args[0] + if isinstance(arg, gdal.Dataset): + band = arg.GetRasterBand(1) + band.WriteArray(result) + return arg + else: + return gdal_array.OpenNumPyArray(result, False) return inner function_map = { - 'sin': np.sin, - 'cos': np.cos, - 'tan': np.tan, - 'arcsin': np.arcsin, - 'arccos': np.arccos, - 'arctan': np.arctan, - 'hypot': np.hypot, - 'arctan2': np.arctan2, - 'degrees': np.degrees, - 'radians': np.radians, - 'unwrap': np.unwrap, - 'deg2rad': np.deg2rad, - 'rad2deg': np.rad2deg, - 'sinh': np.sinh, - 'cosh': np.cosh, - 'tanh': np.tanh, - 'arcsinh': np.arcsinh, - 'arccosh': np.arccosh, - 'arctanh': np.arctanh, - 'exp': np.exp, - 'expm1': np.expm1, - 'exp2': np.exp2, - 'log': np.log, - 'log10': np.log10, - 'log2': np.log2, - 'log1p': np.log1p, + 'sin': wrap_numpy_func(np.sin), + 'cos': wrap_numpy_func(np.cos), + 'tan': wrap_numpy_func(np.tan), + 'arcsin': wrap_numpy_func(np.arcsin), + 'arccos': wrap_numpy_func(np.arccos), + 'arctan': wrap_numpy_func(np.arctan), + 'hypot': wrap_numpy_func(np.hypot), + 'arctan2': wrap_numpy_func(np.arctan2), + 'degrees': wrap_numpy_func(np.degrees), + 'radians': wrap_numpy_func(np.radians), + 'unwrap': wrap_numpy_func(np.unwrap), + 'deg2rad': wrap_numpy_func(np.deg2rad), + 'rad2deg': wrap_numpy_func(np.rad2deg), + 'sinh': wrap_numpy_func(np.sinh), + 'cosh': wrap_numpy_func(np.cosh), + 'tanh': wrap_numpy_func(np.tanh), + 'arcsinh': wrap_numpy_func(np.arcsinh), + 'arccosh': wrap_numpy_func(np.arccosh), + 'arctanh': wrap_numpy_func(np.arctanh), + 'exp': wrap_numpy_func(np.exp), + 'expm1': wrap_numpy_func(np.expm1), + 'exp2': wrap_numpy_func(np.exp2), + 'log': wrap_numpy_func(np.log), + 'log10': wrap_numpy_func(np.log10), + 'log2': wrap_numpy_func(np.log2), + 'log1p': wrap_numpy_func(np.log1p), 'hillshade': hillshade, 'slopeshade': slopeshade, 'aspect': aspect, diff -Nru eoxserver-1.1.2/eoxserver/render/browse/generate.py eoxserver-1.2.2/eoxserver/render/browse/generate.py --- eoxserver-1.1.2/eoxserver/render/browse/generate.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/browse/generate.py 2023-02-20 12:10:15.000000000 +0000 @@ -116,6 +116,7 @@ _ast.Add, _ast.Sub, _ast.Num if hasattr(_ast, 'Num') else _ast.Constant, + _ast.List, _ast.BitAnd, _ast.BitOr, @@ -475,11 +476,18 @@ expr.value, fields_and_datasets, variables, cache ) # assume that we will only use a single index - slice_ = expr.slice.value.value + if isinstance(expr.slice.value, int): + slice_ = expr.slice.value # python 3.10 + else: + slice_ = expr.slice.value.value # python 3.8 # Get a copy of the selected band data = value.GetRasterBand(slice_ + 1).ReadAsArray() result = gdal_array.OpenNumPyArray(data, True) + # restore nodata on output + nodata_value = value.GetRasterBand(slice_ + 1).GetNoDataValue() + if nodata_value is not None: + result.GetRasterBand(1).SetNoDataValue(nodata_value) elif hasattr(_ast, 'Num') and isinstance(expr, _ast.Num): result = expr.n @@ -487,6 +495,12 @@ elif hasattr(_ast, 'Constant') and isinstance(expr, _ast.Constant): result = expr.value + elif hasattr(_ast, 'List') and isinstance(expr, _ast.List): + result = [ + _evaluate_expression( + item, fields_and_datasets, variables, cache, + ) for item in expr.elts + ] else: raise BandExpressionError('Invalid expression node %s' % expr) diff -Nru eoxserver-1.1.2/eoxserver/render/browse/objects.py eoxserver-1.2.2/eoxserver/render/browse/objects.py --- eoxserver-1.1.2/eoxserver/render/browse/objects.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/browse/objects.py 2023-02-20 12:10:15.000000000 +0000 @@ -25,6 +25,8 @@ # THE SOFTWARE. # ------------------------------------------------------------------------------ +from typing import List, Tuple, Optional, Union + from django.contrib.gis.geos import Polygon from django.contrib.gis.gdal import SpatialReference, CoordTransform, DataSource @@ -38,6 +40,9 @@ BROWSE_MODE_GRAYSCALE = "grayscale" +OptionalNumeric = Optional[Union[float, int]] + + class Browse(object): def __init__(self, name, filename, env, size, extent, crs, mode, footprint): self._name = name @@ -136,7 +141,9 @@ class GeneratedBrowse(Browse): def __init__(self, name, band_expressions, ranges, nodata_values, - fields_and_coverages, field_list, footprint, variables): + fields_and_coverages, field_list, footprint, variables, + show_out_of_bounds_data=False, + ): self._name = name self._band_expressions = band_expressions self._ranges = ranges @@ -145,6 +152,7 @@ self._field_list = field_list self._footprint = footprint self._variables = variables + self._show_out_of_bounds_data = show_out_of_bounds_data @property def name(self): @@ -185,11 +193,11 @@ return self._band_expressions @property - def ranges(self): + def ranges(self) -> List[Tuple[OptionalNumeric, OptionalNumeric]]: return self._ranges @property - def nodata_values(self): + def nodata_values(self) -> List[OptionalNumeric]: return self._nodata_values @property @@ -204,10 +212,14 @@ def variables(self): return self._variables + @property + def show_out_of_bounds_data(self) -> bool: + return self._show_out_of_bounds_data + @classmethod def from_coverage_models(cls, band_expressions, ranges, nodata_values, fields_and_coverage_models, - product_model, variables): + product_model, variables, show_out_of_bounds_data): fields_and_coverages = { field_name: [ @@ -230,6 +242,7 @@ ], product_model.footprint, variables, + show_out_of_bounds_data, ) diff -Nru eoxserver-1.1.2/eoxserver/render/browse/util.py eoxserver-1.2.2/eoxserver/render/browse/util.py --- eoxserver-1.1.2/eoxserver/render/browse/util.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/browse/util.py 2023-02-20 12:10:15.000000000 +0000 @@ -1,8 +1,12 @@ from uuid import uuid4 +import numpy as np +import logging from eoxserver.contrib import gdal, osr from eoxserver.resources.coverages import crss +logger = logging.getLogger(__name__) + def create_mem_ds(width, height, data_type): driver = gdal.GetDriverByName('MEM') @@ -78,3 +82,27 @@ gdal.Unlink(vrt_filename) return out_ds + + +def convert_dtype(dtype:np.dtype): + """Maps numpy dtype to a larger itemsize + to avoid value overflow during mathematical operations + + Args: + dtype (np.dtype): input dtype + + Returns: + dtype (np.dtype): either one size larger dtype or original dtype + """ + mapping = { + np.dtype(np.int8): np.dtype(np.int16), + np.dtype(np.int16): np.dtype(np.int32), + np.dtype(np.int32): np.dtype(np.int64), + np.dtype(np.uint8): np.dtype(np.int16), + np.dtype(np.uint16): np.dtype(np.int32), + np.dtype(np.uint32): np.dtype(np.int64), + np.dtype(np.float16): np.dtype(np.float32), + np.dtype(np.float32): np.dtype(np.float64), + } + output_dtype = mapping.get(dtype, dtype) + return output_dtype diff -Nru eoxserver-1.1.2/eoxserver/render/coverage/objects.py eoxserver-1.2.2/eoxserver/render/coverage/objects.py --- eoxserver-1.1.2/eoxserver/render/coverage/objects.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/coverage/objects.py 2023-02-20 12:10:15.000000000 +0000 @@ -31,6 +31,7 @@ from itertools import zip_longest as izip_longest from copy import deepcopy +from typing import List, Optional, Union from django.utils.six import string_types from eoxserver.core.util.timetools import parse_iso8601, parse_duration @@ -63,7 +64,7 @@ self._data_type_range = data_type_range @property - def index(self): + def index(self) -> int: return self._index @property @@ -476,11 +477,11 @@ self._band_statistics = band_statistics @property - def start_field(self): + def start_field(self) -> int: return self._start_field @property - def end_field(self): + def end_field(self) -> int: return self._end_field @property @@ -539,7 +540,7 @@ return self._origin @property - def grid(self): + def grid(self) -> Grid: return self._grid @property @@ -555,7 +556,7 @@ ) @property - def arraydata_locations(self): + def arraydata_locations(self) -> List[ArraydataLocation]: return self._arraydata_locations @property @@ -596,7 +597,9 @@ elif self.footprint: return self.footprint.extent - def lookup_field(self, field_or_identifier): + def lookup_field( + self, field_or_identifier: Union[Field, str] + ) -> Optional[Field]: if isinstance(field_or_identifier, Field): field = field_or_identifier if field not in self.range_type: @@ -612,7 +615,9 @@ except StopIteration: return None - def get_location_for_field(self, field_or_identifier): + def get_location_for_field( + self, field_or_identifier: Union[Field, str], + ) -> Optional[ArraydataLocation]: field = self.lookup_field(field_or_identifier) index = field.index @@ -738,7 +743,7 @@ return self._eo_metadata.end_time if self._eo_metadata else None @property - def range_type(self): + def range_type(self) -> RangeType: return self._range_type @property diff -Nru eoxserver-1.1.2/eoxserver/render/map/objects.py eoxserver-1.2.2/eoxserver/render/map/objects.py --- eoxserver-1.1.2/eoxserver/render/map/objects.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/map/objects.py 2023-02-20 12:10:15.000000000 +0000 @@ -26,9 +26,10 @@ # ------------------------------------------------------------------------------ from weakref import proxy +from typing import List from eoxserver.render.coverage.objects import ( - GRID_TYPE_TEMPORAL, GRID_TYPE_ELEVATION + GRID_TYPE_TEMPORAL, GRID_TYPE_ELEVATION, Coverage, Mosaic, ) @@ -71,7 +72,7 @@ self._ranges = ranges @property - def coverage(self): + def coverage(self) -> Coverage: return self._coverage @property @@ -109,7 +110,7 @@ self._ranges = ranges @property - def coverages(self): + def coverages(self) -> List[Coverage]: return self._coverages @property @@ -184,7 +185,7 @@ self._ranges = ranges @property - def mosaic(self): + def mosaic(self) -> Mosaic: return self._mosaic @property @@ -295,7 +296,7 @@ class Map(object): """ Abstract interpretation of a map to be drawn. """ - def __init__(self, layers, width, height, format, bbox, crs, bgcolor=None, + def __init__(self, layers: List[Layer], width, height, format, bbox, crs, bgcolor=None, transparent=True, time=None, elevation=None): self._layers = layers self._width = int(width) @@ -312,7 +313,7 @@ layer.map = self @property - def layers(self): + def layers(self) -> List[Layer]: return self._layers @property diff -Nru eoxserver-1.1.2/eoxserver/render/mapserver/factories.py eoxserver-1.2.2/eoxserver/render/mapserver/factories.py --- eoxserver-1.1.2/eoxserver/render/mapserver/factories.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/mapserver/factories.py 2023-02-20 12:10:15.000000000 +0000 @@ -26,6 +26,7 @@ # ------------------------------------------------------------------------------ from os.path import join +from typing import List, Type, Iterable, Tuple from uuid import uuid4 try: from itertools import izip_longest @@ -47,8 +48,10 @@ from eoxserver.render.map.objects import ( CoverageLayer, CoveragesLayer, MosaicLayer, OutlinedCoveragesLayer, BrowseLayer, OutlinedBrowseLayer, - MaskLayer, MaskedBrowseLayer, OutlinesLayer + MaskLayer, MaskedBrowseLayer, OutlinesLayer, + Layer, Map, ) +from eoxserver.render.coverage.objects import Coverage, Field from eoxserver.render.mapserver.config import ( DEFAULT_EOXS_MAPSERVER_LAYER_FACTORIES, ) @@ -64,23 +67,25 @@ class BaseMapServerLayerFactory(object): - handled_layer_types = [] + handled_layer_types: List[Type[Layer]] = [] @classmethod - def supports(self, layer_type): + def supports(self, layer_type: Type[Layer]): return layer_type in self.handled_layer_types - def create(self, map_obj, layer): + def create(self, map_obj: Map, layer: Layer): pass - def destroy(self, map_obj, layer, data): + def destroy(self, map_obj: Map, layer: Layer, data): pass class CoverageLayerFactoryMixIn(object): """ Base class for factories dealing with coverages. """ - def get_fields(self, fields, bands, wavelengths): + def get_fields( + self, fields: Iterable[Field], bands, wavelengths + ) -> List[Field]: """ Get the field subset for the given bands/wavelengths selection """ if bands: @@ -112,7 +117,7 @@ return fields - def create_coverage_layer(self, map_obj, coverage, fields, + def create_coverage_layer(self, map_obj: Map, coverage: Coverage, fields: List[Field], style=None, ranges=None): """ Creates a mapserver layer object for the given coverage """ @@ -132,8 +137,8 @@ # TODO: apply subsets in time/elevation dims num_locations = len(set(locations)) if num_locations == 1: + location = field_locations[0][1] if not coverage.grid.is_referenceable: - location = field_locations[0][1] data = location.path ms.set_env(map_obj, location.env, True) else: @@ -145,12 +150,13 @@ wkt = osr.SpatialReference(map_obj.getProjection()).wkt # TODO: env? - reftools.create_rectified_vrt( - field_locations[0][1].path, vrt_path, - order=1, max_error=10, - resolution=(resx, -resy), srid_or_wkt=wkt - ) - data = vrt_path + with gdal.config_env(location.env): + reftools.create_rectified_vrt( + location.path, vrt_path, + order=1, max_error=10, + resolution=(resx, -resy), srid_or_wkt=wkt + ) + data = vrt_path elif num_locations > 1: paths_set = set( @@ -270,7 +276,7 @@ BaseMapServerLayerFactory): handled_layer_types = [OutlinedCoveragesLayer] - def create(self, map_obj, layer): + def create(self, map_obj, layer: CoveragesLayer): coverages = layer.coverages style = layer.style @@ -311,7 +317,7 @@ class MosaicLayerFactory(CoverageLayerFactoryMixIn, BaseMapServerLayerFactory): handled_layer_types = [MosaicLayer] - def create(self, map_obj, layer): + def create(self, map_obj, layer: MosaicLayer): mosaic = layer.mosaic fields = self.get_fields( mosaic.range_type, layer.bands, layer.wavelengths @@ -395,9 +401,9 @@ else: browse_iter = enumerate( - zip(browse.field_list, browse.ranges), start=1 + zip(browse.field_list, browse.ranges, browse.nodata_values), start=1 ) - for i, (field, field_range) in browse_iter: + for i, (field, field_range, nodata_value) in browse_iter: if ranges: if len(ranges) == 1: range_ = ranges[0] @@ -409,10 +415,34 @@ range_ = _get_range(field) for layer_obj in layer_objs: - layer_obj.setProcessingKey( - "SCALE_%d" % i, - "%s,%s" % tuple(range_) - ) + # NOTE: Only works if browsetype nodata is lower than browse_type_min by at least 1 + if browse.show_out_of_bounds_data: + # final LUT for min,max 200,700 and nodata=0 should look like: + # 0:0,1:1,200:1,700:256 + lut_inputs = { + range_[0]: 1, + range_[1]: 256, + } + if nodata_value is not None: + # no_data_value_plus +1 to ensure that only no_data_value is + # rendered as black (transparent) + nodata_value_plus = nodata_value + 1 + lut_inputs[nodata_value] = 0 + lut_inputs[nodata_value_plus] = 1 + + # LUT inputs needs to be ascending + sorted_inputs = { + k: v for k, v in sorted(list(lut_inputs.items())) + } + lut = ",".join("%d:%d" % (k,v) for k,v in sorted_inputs.items()) + + layer_obj.setProcessingKey("LUT_%d" % i, lut) + else: + # due to offsite 0,0,0 will make all pixels below or equal to min transparent + layer_obj.setProcessingKey( + "SCALE_%d" % i, + "%s,%s" % tuple(range_) + ) elif isinstance(browse, Browse): layer_objs = _create_raster_layer_objs( @@ -632,7 +662,7 @@ def _create_raster_layer_objs(map_obj, extent, sr, data, filename_generator, - resample=None): + resample=None) -> List[ms.layerObj]: layer_obj = ms.layerObj(map_obj) layer_obj.type = ms.MS_LAYER_RASTER layer_obj.status = ms.MS_ON @@ -853,7 +883,7 @@ layer.insertClass(cls) -def _get_range(field, range_=None): +def _get_range(field: Field, range_=None) -> Tuple[int, int]: """ Gets the numeric range of a field """ if range_: @@ -885,7 +915,7 @@ ] -def get_layer_factories(): +def get_layer_factories() -> List[BaseMapServerLayerFactory]: if LAYER_FACTORIES is None: _setup_factories() return LAYER_FACTORIES diff -Nru eoxserver-1.1.2/eoxserver/render/mapserver/map_renderer.py eoxserver-1.2.2/eoxserver/render/mapserver/map_renderer.py --- eoxserver-1.1.2/eoxserver/render/mapserver/map_renderer.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/render/mapserver/map_renderer.py 2023-02-20 12:10:15.000000000 +0000 @@ -27,12 +27,14 @@ import logging import tempfile +from typing import List, Tuple, Type from uuid import uuid4 from eoxserver.contrib import mapserver as ms from eoxserver.contrib import vsi from eoxserver.render.colors import BASE_COLORS, COLOR_SCALES -from eoxserver.render.mapserver.factories import get_layer_factories +from eoxserver.render.mapserver.factories import BaseMapServerLayerFactory, get_layer_factories +from eoxserver.render.map.objects import Map, Layer from eoxserver.resources.coverages.formats import getFormatRegistry @@ -68,7 +70,7 @@ def get_supported_formats(self): return getFormatRegistry().getSupportedFormatsWMS() - def render_map(self, render_map): + def render_map(self, render_map: Map): # TODO: get layer creators for each layer type in the map map_obj = ms.mapObj() @@ -153,7 +155,10 @@ for layer, factory, data in layers_plus_factories_plus_data: factory.destroy(map_obj, layer, data) - def _get_layers_plus_factories(self, render_map): + def _get_layers_plus_factories( + self, + render_map: Map, + ) -> List[Tuple[Layer, BaseMapServerLayerFactory]]: layers_plus_factories = [] type_to_layer_factory = {} for layer in render_map.layers: @@ -168,7 +173,7 @@ return layers_plus_factories - def _get_layer_factory(self, layer_type): + def _get_layer_factory(self, layer_type: Type[Layer]): for factory in get_layer_factories(): if factory.supports(layer_type): return factory diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/admin.py eoxserver-1.2.2/eoxserver/resources/coverages/admin.py --- eoxserver-1.1.2/eoxserver/resources/coverages/admin.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/admin.py 2023-02-20 12:10:15.000000000 +0000 @@ -93,7 +93,11 @@ 'alpha_expression', 'alpha_nodata_value', ('alpha_range_min', 'alpha_range_max'), ) - }) + }), + ("Show out of bounds data", { + 'classes': ('collapse', 'collapsed'), + 'fields': ('show_out_of_bounds_data',), + }), ) diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/browsetype.py eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/browsetype.py --- eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/browsetype.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/browsetype.py 2023-02-20 12:10:15.000000000 +0000 @@ -104,6 +104,12 @@ '--alpha-nodata', type=float, dest='alpha_nodata', default=None, ) + create_parser.add_argument( + '--show-out-of-bounds-data', + action="store_true", + default=False, + ) + list_parser.add_argument( 'product_type_name', nargs=1, @@ -134,6 +140,7 @@ blue_range=(None, None), alpha_range=(None, None), red_or_grey_nodata=None, green_nodata=None, blue_nodata=None, alpha_nodata=None, + show_out_of_bounds_data=False, *args, **kwargs): """ Handle the creation of a new browse type. """ @@ -171,6 +178,7 @@ green_nodata_value=green_nodata, blue_nodata_value=blue_nodata, alpha_nodata_value=alpha_nodata, + show_out_of_bounds_data=show_out_of_bounds_data, ) if not browse_type_name: diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/product.py eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/product.py --- eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/product.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/product.py 2023-02-20 12:10:15.000000000 +0000 @@ -216,7 +216,7 @@ discover_parser.add_argument( 'identifier', default=None, - help='The identifier of the product to descover.' + help='The identifier of the product to discover.' ) discover_parser.add_argument( diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/stac.py eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/stac.py --- eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/stac.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/stac.py 2023-02-20 12:10:15.000000000 +0000 @@ -139,7 +139,7 @@ product, replaced = register_stac_product( values, type_name, replace=replace, - file_href=location if not stdin else None, + self_href=location if not stdin else None, ) self.print_msg( "Successfully %s product %s" % ( diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/timeseries.py eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/timeseries.py --- eoxserver-1.1.2/eoxserver/resources/coverages/management/commands/timeseries.py 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/management/commands/timeseries.py 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,172 @@ +# ------------------------------------------------------------------------------ +# +# Project: EOxServer +# Authors: Bernhard Mallinger +# +# ------------------------------------------------------------------------------ +# Copyright (C) 2022 EOX IT Services GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ------------------------------------------------------------------------------ + + +from django.core.management.base import BaseCommand +from django.db import transaction + +from eoxserver.resources.coverages.registration.timeseries import register_time_series +from eoxserver.resources.coverages import models +from eoxserver.resources.coverages.management.commands import ( + CommandOutputMixIn, + SubParserMixIn, +) + + +def parse_collection(identifier): + try: + return models.Collection.objects.get(identifier=identifier) + except models.Collection.DoesNotExist: + raise ValueError(f'No collection with identifier "{identifier}" found.') + + +def parse_coverage_type_mapping(mapping): + # raises value error if not exactly 1 ":" + dim, coverage_type_name = mapping.split(":") + return (dim, coverage_type_name) + + +class Command(CommandOutputMixIn, SubParserMixIn, BaseCommand): + """Command to manage time series. This command uses sub-commands for the + specific tasks: register, deregister + """ + + def add_arguments(self, parser): + register_parser = self.add_subparser(parser, "register") + deregister_parser = self.add_subparser(parser, "deregister") + + register_parser.add_argument( + "--collection", + "--collection-identifier", + "-c", + dest="collection", + required=True, + type=parse_collection, + help="Register timeseries for this collection", + ) + register_parser.add_argument( + "--storage", + help="The storage to use", + ) + register_parser.add_argument( + "--path", + required=True, + help="Path to timeseries file", + ) + register_parser.add_argument( + "--product-type-name", + required=True, + help="The product type name", + ) + register_parser.add_argument( + "--coverage-type-mapping", + action="append", + type=parse_coverage_type_mapping, + required=True, + help="Which dimension to map to which coverage type. " + 'Use : as separator, e.g. --coverage-type-mapping "/Band1:b1"', + ) + register_parser.add_argument( + "--x-dim-name", + required=True, + help="Name of the X dimension", + ) + register_parser.add_argument( + "--y-dim-name", + required=True, + help="Name of the Y dimension", + ) + register_parser.add_argument( + "--time-dim-name", + required=True, + help="Name of the time dimension", + ) + register_parser.add_argument( + "--product-template", + required=True, + help="Format string for product identifier. " + "Can use the following template variables: " + "collection_identifier, file_identifier, index, " + "product_type, begin_time, end_time", + ) + register_parser.add_argument( + "--replace", + "-r", + dest="replace", + action="store_true", + default=False, + help=( + "Optional. If the time series with the given identifier already " + "exists, replace it. Without this flag, this would result in " + "an error." + ), + ) + + @transaction.atomic + def handle(self, subcommand, *args, **kwargs): + """Dispatch sub-commands: register, deregister.""" + if subcommand == "register": + self.handle_register(*args, **kwargs) + elif subcommand == "deregister": + self.handle_deregister(*args, **kwargs) + + def handle_register( + self, + collection, + storage, + path, + product_type_name, + coverage_type_mapping, + x_dim_name, + y_dim_name, + time_dim_name, + product_template, + replace, + **kwargs, + ): + timeseries_path, replaced = register_time_series( + collection=collection, + storage=storage, + path=path, + product_type_name=product_type_name, + coverage_type_mapping=dict(coverage_type_mapping), + x_dim_name=x_dim_name, + y_dim_name=y_dim_name, + time_dim_name=time_dim_name, + product_template=product_template, + replace=replace, + ) + + self.print_msg( + ( + f"Successfully {'replaced' if replaced else 'registered'}" + f" timeseries {timeseries_path}" + ) + ) + + def handle_deregister(self, **kwargs): + raise NotImplementedError() diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/migrations/0012_out_of_bounds_data_flag.py eoxserver-1.2.2/eoxserver/resources/coverages/migrations/0012_out_of_bounds_data_flag.py --- eoxserver-1.1.2/eoxserver/resources/coverages/migrations/0012_out_of_bounds_data_flag.py 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/migrations/0012_out_of_bounds_data_flag.py 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-11-29 15:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('coverages', '0011_bandstatistics'), + ] + + operations = [ + migrations.AddField( + model_name='browsetype', + name='show_out_of_bounds_data', + field=models.BooleanField(default=False), + ), + ] diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/models.py eoxserver-1.2.2/eoxserver/resources/coverages/models.py --- eoxserver-1.1.2/eoxserver/resources/coverages/models.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/models.py 2023-02-20 12:10:15.000000000 +0000 @@ -42,7 +42,6 @@ from django.db.models import Min, Max, Q, F, ExpressionWrapper from django.db.models.functions import Cast from django.utils.timezone import now -from django.utils.encoding import python_2_unicode_compatible from model_utils.managers import InheritanceManager from jsonfield import JSONField @@ -202,6 +201,8 @@ blue_range_max = models.FloatField(**optional) alpha_range_max = models.FloatField(**optional) + show_out_of_bounds_data = models.BooleanField(default=False, **mandatory) + def __str__(self): if self.name: return self.name @@ -314,7 +315,6 @@ identifier_validators[0](value) -@python_2_unicode_compatible class EOObject(models.Model): """ Base class for Collections, Products and Coverages """ @@ -530,7 +530,6 @@ # ============================================================================== -@python_2_unicode_compatible class AbstractCommonValue(models.Model): value = models.CharField(max_length=256, db_index=True, unique=True) diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/registration/base.py eoxserver-1.2.2/eoxserver/resources/coverages/registration/base.py --- eoxserver-1.1.2/eoxserver/resources/coverages/registration/base.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/registration/base.py 2023-02-20 12:10:15.000000000 +0000 @@ -434,6 +434,8 @@ raise RegistrationError( 'Grid %r does not exist' % definition ) + elif isinstance(definition, models.Grid): + grid = definition elif definition: axis_names = definition.get('axis_names', []) axis_types = definition['axis_types'] diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/registration/stac.py eoxserver-1.2.2/eoxserver/resources/coverages/registration/stac.py --- eoxserver-1.1.2/eoxserver/resources/coverages/registration/stac.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/registration/stac.py 2023-02-20 12:10:15.000000000 +0000 @@ -27,8 +27,12 @@ from itertools import zip_longest import json -from urllib.parse import urljoin, urlparse, urlunparse +from urllib.parse import ( + urljoin, urlparse, urlunparse, uses_netloc, uses_relative +) import logging +from typing import Optional +import mimetypes from django.contrib.gis.geos import GEOSGeometry from django.db.models import Q @@ -51,7 +55,7 @@ ) -logger = logging.getLogger() +logger = logging.getLogger(__name__) def get_product_type_name(stac_item): @@ -98,7 +102,14 @@ return '_'.join(parts) -def get_path_from_href(href, file_href=None): +# allow to urljoin s3:// URLs +if 's3' not in uses_netloc: + uses_netloc.append('s3') +if 's3' not in uses_relative: + uses_relative.append('s3') + + +def get_path_from_href(href: str, self_href: Optional[str] = None): """ Extract the path from the given HREF. For S3 URLs this excludes the bucket name. Leading and trailing slashes will be stripped, so resulting paths are always relative. @@ -107,11 +118,11 @@ >>> get_path_from_href('s3://bucket/prefix/file.ext') 'prefix/file.ext' - >>> get_path_from_href('https://www.example.com/path/to#res.ext') + >>> get_path_from_href('https://www.example.com/path/to/res.ext') 'path/to/res.ext' """ - if file_href: - href = urljoin(file_href, href) + if self_href: + href = urljoin(self_href, href) parsed = urlparse(href) if parsed.scheme: @@ -120,11 +131,80 @@ return urlunparse(parsed).strip('/') +def resolve_storage( + asset: dict, base_storage: Optional[backends.Storage] +) -> Optional[backends.Storage]: + """Resolve the actual storage for a given asset. This is required when an + intermediate storage is introduced for a ZIP or TAR archive file. + + Args: + asset (dict): The asset to resolve te storage for + base_storage (Optional[backends.Storage]): The potential base storage + everything is relative to. + + Returns: + Optional[backends.Storage]: The resulting storage, either the base + storage or a fetched/newly created Storage or None. + """ + href = asset['href'] + if "archive:href" in asset: + storage_type = None + + file_type = asset.get("type") + archive_type = asset.get("archive:type") + storage_type = None + for type_ in [file_type, archive_type, mimetypes.guess_type(href)]: + if type_ == "application/zip": + storage_type = "ZIP" + break + elif type_ == "application/x-tar": + storage_type = "TAR" + break + + if storage_type is None: + raise TypeError(f"Unsupported archive type for file {href}") + + return backends.Storage.objects.get_or_create( + url=href, + storage_type=storage_type, + name="%s__%s" % (base_storage.name, href) if base_storage else href, + parent=base_storage, + )[0] + else: + return base_storage + + +def resolve_location(asset: dict, self_href: Optional[str]) -> str: + """Resolves the assets location, relative to its storage. This takes the + archive:href into account, returning the relative location of the asset + within the archive file. + + Args: + asset (dict): The asset to resolve the location for + self_href (Optional[str]): The STAC Items self href, which absolute + assets need to be made relative to + + Returns: + str: the resulting location relative to any storage + """ + archive_href = asset.get("archive:href") + if archive_href: + # the archive:href is the path of the file within the archive. So + # here we have to make sure, that it is not made relative to the + # STAC Items self_href + return archive_href + else: + # we have to make sure that the assets href is made relative to the + # STAC Items self_href + return get_path_from_href(asset['href'], self_href) + + @transaction.atomic def register_stac_product(stac_item, product_type=None, storage=None, replace=False, coverage_mapping={}, browse_mapping=None, metadata_asset_names=None, - file_href=None): + simplify_footprint_tolerance=None, + self_href=None): """ Registers a single parsed STAC item as a Product. The product type to be used can be specified via the product_type_name argument. @@ -205,8 +285,8 @@ metadata_items = [ models.MetaDataItem( - location=get_path_from_href(asset['href'], file_href), - storage=storage, + location=resolve_location(asset, self_href), + storage=resolve_storage(asset, storage), ) for asset in metadata_assets ] @@ -278,6 +358,11 @@ # read footprint from metadata if it was not already defined footprint = footprint or metadata.get('footprint') + if simplify_footprint_tolerance is not None and footprint: + footprint = footprint.simplify( + simplify_footprint_tolerance, preserve_topology=True + ) + # finally create the product and its metadata object product = models.Product.objects.create( identifier=identifier, @@ -297,8 +382,14 @@ registrator = GDALRegistrator() - # handling coverages + if len(data_assets) == 0: + logger.info( + 'No data assets found in STAC item for Product %s' % ( + identifier, + ) + ) + # handling coverages for asset_name, asset in data_assets.items(): overrides = {} coverage_type = None @@ -312,6 +403,18 @@ ) break else: + logger.info( + '''Data asset "%s" was not mapped to any coverage_mapping %s. + Asset will not be added as Coverage to Product %s''' % ( + asset_name, + { + coverage_type_name: mapping['assets'] + for coverage_type_name, mapping + in coverage_mapping.items() + }, + identifier, + ) + ) continue # if no mapping is defined, we try to figure out the coverage type via @@ -319,6 +422,10 @@ else: bands = asset.get('eo:bands') if bands is None: + logger.info( + 'No eo:bands information present in Item.' + 'Skipping data asset %s.' % asset_name + ) continue if not isinstance(bands, list): @@ -347,17 +454,12 @@ continue overrides['identifier'] = '%s_%s' % (identifier, asset_name) - # create the storage item - path = get_path_from_href(asset['href'], file_href) - coverage_footprint = None if 'proj:geometry' in asset: coverage_footprint = GEOSGeometry( json.dumps(asset['proj:geometry']) ) - if footprint: - coverage_footprint = footprint if coverage_footprint: overrides['footprint'] = coverage_footprint.wkt @@ -371,7 +473,7 @@ overrides['size'] = [shape[1], shape[0]] if transform: - overrides['origin'] = [transform[1], transform[5]] + overrides['origin'] = [transform[2], transform[5]] if epsg and transform: sr = osr.SpatialReference(epsg) @@ -390,13 +492,24 @@ ) ) + location = resolve_location(asset, self_href) + asset_storage = resolve_storage(asset, storage) + + if asset_storage is None: + data_locations = [[location]] + elif asset_storage == storage: + data_locations = [[storage.name, location]] + else: + data_locations = [[storage.name, asset_storage.name, location]] + report = registrator.register( - data_locations=[([storage.name] if storage else []) + [path]], + data_locations=data_locations, metadata_locations=[], coverage_type_name=coverage_type.name, footprint_from_extent=False, overrides=overrides, replace=replace, + simplify_footprint_tolerance=simplify_footprint_tolerance, statistics=[ [ dict( @@ -424,7 +537,7 @@ ) ) register_browse_for_asset( - asset, file_href, product, storage, browse_type + asset, self_href, product, storage, browse_type ) break else: @@ -442,7 +555,7 @@ # # name='' # # ).first() register_browse_for_asset( - asset, file_href, product, storage, None + asset, self_href, product, storage, None ) else: if browse_type is None: @@ -452,16 +565,33 @@ if browse_type: register_browse_for_asset( - asset, file_href, product, storage, browse_type + asset, self_href, product, storage, browse_type ) + # adding thumbnail image, which is the first one with role thumbnail + thumbnail_asset = next( + ( + asset + for asset in assets.values() + if 'thumbnail' in asset.get('roles', []) + ), + None + ) + if thumbnail_asset: + models.MetaDataItem.objects.create( + eo_object=product, + semantic=models.MetaDataItem.semantic_codes['thumbnail'], + storage=resolve_storage(thumbnail_asset, storage), + location=resolve_location(thumbnail_asset, self_href), + ) + return (product, replaced) -def register_browse_for_asset(asset, file_href, product, storage, browse_type): +def register_browse_for_asset(asset, self_href, product, storage, browse_type): browse = models.Browse( - storage=storage, - location=get_path_from_href(asset['href'], file_href), + storage=resolve_storage(asset, storage), + location=resolve_location(asset, self_href), product=product, browse_type=browse_type, ) @@ -476,7 +606,7 @@ browse.coordinate_reference_system = sr.wkt x_a = transform[0] - x_b = transform[0] + transform[1] * browse.width + x_b = transform[0] + transform[2] * browse.width y_a = transform[3] y_b = transform[3] + transform[5] * browse.height @@ -582,7 +712,7 @@ if not bands_list: raise RegistrationError( - 'Failed to extract band defintion from STAC Item' + 'Failed to extract band definition from STAC Item' ) # create product type itself diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/registration/timeseries.py eoxserver-1.2.2/eoxserver/resources/coverages/registration/timeseries.py --- eoxserver-1.1.2/eoxserver/resources/coverages/registration/timeseries.py 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/registration/timeseries.py 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,301 @@ +# ------------------------------------------------------------------------------ +# +# Project: EOxServer +# Authors: Mussab Abdalla +# Fabian Schindler +# +# ------------------------------------------------------------------------------ +# Copyright (C) 2022 EOX IT Services GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ------------------------------------------------------------------------------ + +import re +import datetime +import itertools +import logging + + +from osgeo.osr import SpatialReference, CoordinateTransformation +from django.db import transaction +from django.contrib.gis.geos import Polygon +from django.core.management.base import CommandError +import numpy as np + +from eoxserver.backends.access import get_vsi_env, get_vsi_storage_path +from eoxserver.contrib.gdal import config_env +from eoxserver.resources.coverages.registration.exceptions import ( + RegistrationError +) +from eoxserver.contrib.gdal import open_with_env +from eoxserver.contrib import gdal +from eoxserver.resources.coverages import models +from eoxserver.backends import models as backends +from eoxserver.resources.coverages.registration.registrators.gdal import ( + GDALRegistrator +) + +logger = logging.getLogger(__name__) + + +def create_product( + collection, + begin_time, + end_time, + footprint, + product_type, + coverage_type_mapping, + replace, + driver_name, + storage, + path, + index, + all_overrides, + file_identifier, + product_template, +): + template_values = { + "collection_identifier": collection.identifier, + "file_identifier": file_identifier, + "index": index, + "product_type": product_type, + "begin_time": begin_time.strftime('%Y%m%d'), + "end_time": end_time.strftime('%Y%m%d') + } + + product_identifier = product_template.format(**template_values) + + replaced = False + + # check if the product already exists + if models.Product.objects.filter( + identifier=product_identifier).exists(): + if replace: + logger.info('Deleting existing Product %s', product_identifier) + models.Product.objects.filter( + identifier=product_identifier).delete() + replaced = True + else: + raise RegistrationError( + 'Product %s already exists' % product_identifier + ) + + product = models.Product.objects.create( + identifier=product_identifier, + begin_time=begin_time, + end_time=end_time, + footprint=footprint, + product_type=product_type, + ) + models.collection_insert_eo_object(collection, product) + + logger.info('Successfully created product %s', product_identifier) + + registrator = GDALRegistrator() + + # adding coverages: + for dim_name, coverage_type_name in coverage_type_mapping.items(): + overrides = dict(all_overrides) + coverage_type = None + overrides['identifier'] = '%s_%s' % (product_identifier, coverage_type_name) + overrides['footprint'] = footprint + + file_path = '%s:"%s":%s:%s' % (driver_name, path, dim_name, index) + + # TODO: coverage types created ? or configured and + # all needed is the name? + try: + coverage_type = models.CoverageType.objects.get( + name=coverage_type_name + ) + except models.CoverageType.DoesNotExist: + raise CommandError( + "Coverage type %r does not exist." % coverage_type_name + ) + + report = registrator.register( + data_locations=[([ + storage.name] if storage else []) + [file_path]], + metadata_locations=[], + coverage_type_name=coverage_type.name, + footprint_from_extent=False, + overrides=overrides, + replace=replace, + use_subdatasets=True, + ) + models.product_add_coverage(product, report.coverage) + + # cache grid, size and origin, as it will be the same for all slices + if not all_overrides: + coverage = report.coverage + all_overrides['grid'] = coverage.grid + all_overrides['size'] = coverage.size + all_overrides['origin'] = coverage.origin + + logger.info('Successfully created coverage %s' % overrides['identifier']) + + return (product, replaced) + + +def extent_to_footprint(crs_wkt, extent): + dcrs = SpatialReference() + dcrs.ImportFromWkt(crs_wkt) + + wgs84 = SpatialReference() + wgs84.ImportFromEPSG(4326) + + dcrs2wgs84 = CoordinateTransformation(dcrs, wgs84) + ll = dcrs2wgs84.TransformPoint(extent[0], extent[1]) + ur = dcrs2wgs84.TransformPoint(extent[2], extent[3]) + + footprint = Polygon( + ( + (ll[1], ll[0]), + (ur[1], ll[0]), + (ur[1], ur[0]), + (ll[1], ur[0]), + (ll[1], ll[0]), + ) + ) + + return footprint + + +def compute_min_max(dim): + dimension = dim.ReadAsArray() + return [dimension[0, 0], dimension[0, dimension.size - 1]] + + +def compute_extent(x_path, y_path): + x_ds = open_with_env(x_path, {}) + y_ds = open_with_env(y_path, {}) + + minmax_x = compute_min_max(x_ds) + minmax_y = compute_min_max(y_ds) + bbox = [minmax_x[0], minmax_y[0], minmax_x[1], minmax_y[1]] + + return bbox + + +UNIT_RE = re.compile(r"(\w+) since (.*)") + + +def get_time_offset_and_step(unit): + match = UNIT_RE.match(unit) + if match: + step_unit, offset = match.groups() + offset = datetime.datetime.fromisoformat(offset) + offset = offset.replace(tzinfo=datetime.timezone.utc) + step = datetime.timedelta(**{step_unit: 1}) + return offset, step + raise ValueError("Failed to parse time unit") + + +def pairwise(iterable): + # pairwise('ABCDEFG') --> AB BC CD DE EF FG + a, b = itertools.tee(iterable) + next(b, None) + return zip(a, b) + + +def get_intervals(dates_array, unit): + datetimes = [] + offset, step = get_time_offset_and_step(unit) + + # create a list of datetime objects from the raw values + for value in np.nditer(dates_array): + datetimes.append(offset + (step * value)) + + # append the last end-datetime + datetimes.append(datetimes[-1] + (datetimes[-1] - datetimes[-2])) + + # create the list of start/end-tuples + return [ + (start, end - datetime.timedelta(seconds=1)) + for start, end in pairwise(datetimes) + ] + + +@transaction.atomic +def register_time_series( + collection, + storage, + path, + product_type_name, + coverage_type_mapping, + x_dim_name, + y_dim_name, + time_dim_name, + product_template, + replace=True +): + + file_identifier = path.split("/")[-1].split(".")[0] + + if isinstance(storage, str): + storage = backends.Storage.objects.get(name=storage) + + with config_env(get_vsi_env(storage)): + vsi_path = get_vsi_storage_path(storage, path) if storage else path + metadata = gdal.MultiDimInfo(vsi_path) + driver_name = metadata['driver'].upper() + + fixed_dimensions = [] + for dimension in metadata['dimensions']: + fixed_dimensions.append(dimension['name']) + + # footprint is the same for all ? + + extent = compute_extent( + '%s:"%s":%s' % (driver_name, vsi_path, x_dim_name), + '%s:"%s":%s' % (driver_name, vsi_path, y_dim_name), + ) + + for dim_name, dim in metadata['arrays'].items(): + if dim_name not in fixed_dimensions: + footprint = extent_to_footprint(dim['srs']['wkt'], extent) + break + + time_path = '%s:"%s":%s' % (driver_name, vsi_path, time_dim_name) + time_ds = open_with_env(time_path, {}) + date_array = get_intervals( + time_ds.ReadAsArray() , metadata['arrays']['time']['unit']) + + product_type = models.ProductType.objects.get(name=product_type_name) + + overrides = {} + for i, (begin_time, end_time) in enumerate(date_array): + create_product( + collection, + begin_time, + end_time, + footprint, + product_type, + coverage_type_mapping, + replace, + driver_name, + storage, + path, + i, + overrides, + file_identifier, + product_template + ) + + return path, replace diff -Nru eoxserver-1.1.2/eoxserver/resources/coverages/views.py eoxserver-1.2.2/eoxserver/resources/coverages/views.py --- eoxserver-1.1.2/eoxserver/resources/coverages/views.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/resources/coverages/views.py 2023-02-20 12:10:15.000000000 +0000 @@ -11,7 +11,8 @@ import shutil from django.http import ( - HttpResponse, HttpResponseBadRequest, HttpResponseNotAllowed, FileResponse + HttpResponse, HttpResponseBadRequest, HttpResponseNotAllowed, FileResponse, + Http404 ) from django.contrib.gis.geos import GEOSGeometry from django.db import transaction @@ -20,6 +21,7 @@ from eoxserver.core.config import get_eoxserver_config from eoxserver.core.decoders import config +from eoxserver.contrib.vsi import VSIFileResponse from eoxserver.backends.access import vsi_open from eoxserver.resources.coverages import models from eoxserver.resources.coverages.registration.product import ( @@ -73,10 +75,13 @@ frmt = request.GET.get('format') - semantic_code = { - name: code - for code, name in models.MetaDataItem.SEMANTIC_CHOICES - }[semantic] + try: + semantic_code = { + name: code + for code, name in models.MetaDataItem.SEMANTIC_CHOICES + }[semantic] + except KeyError as exc: + raise Http404(semantic) from exc qs = models.MetaDataItem.objects.filter( eo_object__identifier=identifier, semantic=semantic_code, @@ -86,8 +91,9 @@ metadata_item = get_object_or_404(qs) - return FileResponse( - vsi_open(metadata_item), content_type=metadata_item.format + return VSIFileResponse( + vsi_open(metadata_item), + content_type=metadata_item.format ) diff -Nru eoxserver-1.1.2/eoxserver/scripts/eoxserver-admin.py eoxserver-1.2.2/eoxserver/scripts/eoxserver-admin.py --- eoxserver-1.1.2/eoxserver/scripts/eoxserver-admin.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/scripts/eoxserver-admin.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,39 +0,0 @@ -#!/usr/bin/python -#------------------------------------------------------------------------------- -# -# Project: EOxServer -# Authors: Stephan Krause -# Stephan Meissl -# Fabian Schindler -# -#------------------------------------------------------------------------------- -# Copyright (C) 2011 EOX IT Services GmbH -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies of this Software or works derived from this Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -#------------------------------------------------------------------------------- - -from eoxserver.core.management import execute_from_commandline -import warnings - -if __name__ == "__main__": - warnings.warn( - "'eoxserver-admin.py' is deprecated. Use 'eoxserver-instance.py' to " - "create an instance", DeprecationWarning - ) - execute_from_commandline() diff -Nru eoxserver-1.1.2/eoxserver/services/auth/charonpdp.py eoxserver-1.2.2/eoxserver/services/auth/charonpdp.py --- eoxserver-1.1.2/eoxserver/services/auth/charonpdp.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/auth/charonpdp.py 2023-02-20 12:10:15.000000000 +0000 @@ -29,10 +29,10 @@ import logging import os import datetime -import httplib +import httplib # TODO Python2 relic swap out import xml.dom.minidom import eoxserver -from urlparse import urlparse +from urllib.parse import urlparse from django.utils.six import iteritems @@ -293,12 +293,7 @@ # Get XML snippet for the Environment part of the XACMLAuthzDecisionQuery def _getPartEnvironment(self): now = datetime.datetime.now() - formattedNow = now.strftime("%Y-%m-%dT%H:%M:%S.%f%z") - return template_attribute.format(attrib_current_date, \ - dt_date, \ - formattedNow) - - + formattedNow = now.strftime("%Y-%m-%dT%H:%M:%S.%f%z")urlparse # Get the full XACMLAuthzDecisionQuery def _getFullRequest(self, userAttributes, resourceAttributes, action): return template_request.format(self._getPartSubject(userAttributes), \ diff -Nru eoxserver-1.1.2/eoxserver/services/config.py eoxserver-1.2.2/eoxserver/services/config.py --- eoxserver-1.1.2/eoxserver/services/config.py 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/config.py 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,45 @@ +# ------------------------------------------------------------------------------ +# +# Project: EOxServer +# Authors: Bernhard Mallinger +# +# ------------------------------------------------------------------------------ +# Copyright (C) 2022 EOX IT Services GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ------------------------------------------------------------------------------ + +from typing import Optional + +from django.conf import settings +from django.views.decorators.cache import cache_control + + +def _get_cache_time() -> Optional[int]: + _cache_time_str = getattr(settings, "EOXS_RENDERER_CACHE_TIME", None) + return int(_cache_time_str) if _cache_time_str is not None else None + + +def apply_cache_header(view): + cache_time = _get_cache_time() + return ( + cache_control(max_age=cache_time)(view) + if cache_time is not None + else view + ) diff -Nru eoxserver-1.1.2/eoxserver/services/opensearch/config.py eoxserver-1.2.2/eoxserver/services/opensearch/config.py --- eoxserver-1.1.2/eoxserver/services/opensearch/config.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/opensearch/config.py 2023-02-20 12:10:15.000000000 +0000 @@ -60,6 +60,8 @@ # default ordering (field name) for opensearch queries DEFAULT_EOXS_OPENSEARCH_DEFAULT_ORDERING = None +# when True, adds exceptions=text/html to all GetCoverage links in opensearch response +DEFAULT_EOXS_OPENSEARCH_GETCOVERAGE_HTML_EXCEPTION = False def get_opensearch_record_model(): class_name = getattr( diff -Nru eoxserver-1.1.2/eoxserver/services/opensearch/formats/base.py eoxserver-1.2.2/eoxserver/services/opensearch/formats/base.py --- eoxserver-1.1.2/eoxserver/services/opensearch/formats/base.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/opensearch/formats/base.py 2023-02-20 12:10:15.000000000 +0000 @@ -31,9 +31,9 @@ from django.http import QueryDict from lxml.builder import ElementMaker try: - from django.core.urlresolvers import reverse + from django.core.urlresolvers import reverse, NoReverseMatch except ImportError: - from django.urls import reverse + from django.urls import reverse, NoReverseMatch from django.conf import settings from django.utils.module_loading import import_string @@ -45,7 +45,8 @@ from eoxserver.resources.coverages import models from eoxserver.services.gml.v32.encoders import GML32Encoder from eoxserver.services.opensearch.config import ( - DEFAULT_EOXS_RESULT_ITEM_FEED_LINK_GENERATORS + DEFAULT_EOXS_RESULT_ITEM_FEED_LINK_GENERATORS, + DEFAULT_EOXS_OPENSEARCH_GETCOVERAGE_HTML_EXCEPTION, ) @@ -300,7 +301,7 @@ ) ) - if wms_small: + if thumbnail_link or wms_small: # "Thumbnail" image links.append( MEDIA("content", @@ -510,14 +511,22 @@ return None def _create_coverage_link(self, request, coverage): + options = dict( + service="WCS", + version="2.0.1", + request="GetCoverage", + coverageId=coverage.identifier, + ) + if getattr( + settings, + 'EOXS_OPENSEARCH_GETCOVERAGE_HTML_EXCEPTION', + DEFAULT_EOXS_OPENSEARCH_GETCOVERAGE_HTML_EXCEPTION, + ): + options["exceptions"] = "text/html" + return request.build_absolute_uri( "%s?%s" % ( - reverse("ows"), urlencode(dict( - service="WCS", - version="2.0.1", - request="GetCoverage", - coverageId=coverage.identifier, - )) + reverse("ows"), urlencode(options) ) ) @@ -590,12 +599,15 @@ def _create_thumbail_link(self, request, item): semantic = models.MetaDataItem.semantic_codes['thumbnail'] if item.metadata_items.filter(semantic=semantic).exists(): - return request.build_absolute_uri( - reverse("metadata", kwargs={ - 'identifier': item.identifier, - 'semantic': 'thumbnail' - }) - ) + try: + return request.build_absolute_uri( + reverse("metadata", kwargs={ + 'identifier': item.identifier, + 'semantic': 'thumbnail' + }) + ) + except NoReverseMatch: + return None def _make_metadata_href(self, request, item, metadata_item): semantic_name = models.MetaDataItem.semantic_names[metadata_item.semantic] diff -Nru eoxserver-1.1.2/eoxserver/services/opensearch/urls.py eoxserver-1.2.2/eoxserver/services/opensearch/urls.py --- eoxserver-1.1.2/eoxserver/services/opensearch/urls.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/opensearch/urls.py 2023-02-20 12:10:15.000000000 +0000 @@ -31,16 +31,20 @@ except ImportError: from django.urls import include, re_path +from eoxserver.services.config import apply_cache_header from eoxserver.services.opensearch.views import description, search + +search_cached = apply_cache_header(search) + app_name = 'opensearch' urlpatterns = [ re_path(r'^$', description, name='description'), - re_path(r'^(?P[^/]+)/$', search, name='search'), + re_path(r'^(?P[^/]+)/$', search_cached, name='search'), re_path(r'^collections/(?P[^/]+)/', include(([ re_path(r'^$', description, name='description'), re_path( - r'^(?P[^/]+)/$', search, + r'^(?P[^/]+)/$', search_cached, name='search' ) ], 'collection'))) diff -Nru eoxserver-1.1.2/eoxserver/services/opensearch/views.py eoxserver-1.2.2/eoxserver/services/opensearch/views.py --- eoxserver-1.1.2/eoxserver/services/opensearch/views.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/opensearch/views.py 2023-02-20 12:10:15.000000000 +0000 @@ -26,6 +26,7 @@ #------------------------------------------------------------------------------- +from django.conf import settings from django.http import HttpResponse from django.views.decorators.csrf import csrf_exempt diff -Nru eoxserver-1.1.2/eoxserver/services/ows/common/v20/encoders.py eoxserver-1.2.2/eoxserver/services/ows/common/v20/encoders.py --- eoxserver-1.1.2/eoxserver/services/ows/common/v20/encoders.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/common/v20/encoders.py 2023-02-20 12:10:15.000000000 +0000 @@ -199,7 +199,7 @@ class OWS20ExceptionXMLEncoder(XMLEncoder): - def encode_exception(self, message, version, code, locator=None): + def encode_exception(self, message, version, code, locator=None, request=None, exception=None): exception_attributes = { "exceptionCode": str(code) } diff -Nru eoxserver-1.1.2/eoxserver/services/ows/config.py eoxserver-1.2.2/eoxserver/services/ows/config.py --- eoxserver-1.1.2/eoxserver/services/ows/config.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/config.py 2023-02-20 12:10:15.000000000 +0000 @@ -66,3 +66,5 @@ 'eoxserver.services.ows.wcs.v20.exceptionhandler.WCS20ExceptionHandler', 'eoxserver.services.ows.wms.v13.exceptionhandler.WMS13ExceptionHandler', ] + +DEFAULT_EOXS_WCS_ERROR_HTML_TEMPLATE = "wcs/error_template.html" diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wcs/v20/exceptionhandler.py eoxserver-1.2.2/eoxserver/services/ows/wcs/v20/exceptionhandler.py --- eoxserver-1.1.2/eoxserver/services/ows/wcs/v20/exceptionhandler.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wcs/v20/exceptionhandler.py 2023-02-20 12:10:15.000000000 +0000 @@ -25,13 +25,21 @@ # THE SOFTWARE. #------------------------------------------------------------------------------- +import traceback + +from django.conf import settings +from django.template.loader import render_to_string from eoxserver.core import Component, implements +from eoxserver.core.decoders import kvp, lower, xml from eoxserver.services.ows.interfaces import ExceptionHandlerInterface from eoxserver.services.ows.common.v20.encoders import OWS20ExceptionXMLEncoder +from eoxserver.services.ows.config import DEFAULT_EOXS_WCS_ERROR_HTML_TEMPLATE +from eoxserver.services.ows.wcs.v20.util import ns_wcs from eoxserver.core.decoders import ( DecodingException, MissingParameterException ) +from eoxserver.core.util.xmltools import NameSpace, NameSpaceMap CODES_404 = frozenset(( @@ -46,6 +54,49 @@ )) +class WCS20ExceptionHandlerKVPDecoder(kvp.Decoder): + exceptions = kvp.Parameter(num="?", type=lower, default="application/xml") + + +class WCS20ExceptionHandlerXMLDecoder(xml.Decoder): + namespaces = NameSpaceMap( + ns_wcs, NameSpace("http://eoxserver.org/eoxs/1.0", "eoxs") + ) + exceptions = xml.Parameter("wcs:Extension/eoxs:exceptions/text()", num="?", type=lower, default="application/xml") + + +class OWS20ExceptionHTMLEncoder(object): + @property + def content_type(self): + return "text/html" + + def serialize(self, message): + # content is already str + return message + + def encode_exception(self, message, version, code, locator=None, request=None, exception=None): + template_name = getattr( + settings, + 'EOXS_ERROR_HTML_TEMPLATE', + DEFAULT_EOXS_WCS_ERROR_HTML_TEMPLATE, + ) + # pass in original traceback and debug to allow usage in template + debug = getattr(settings, 'DEBUG', False) + stack_trace = traceback.format_exc() + + template_params = { + "message": message, + "exception": exception, + "debug": debug, + "stack_trace": stack_trace, + } + return render_to_string( + template_name, + context=template_params, + request=request + ) + + class WCS20ExceptionHandler(Component): implements(ExceptionHandlerInterface) @@ -53,6 +104,17 @@ versions = ("2.0.0", "2.0.1") request = None + def get_encoder(self, request): + if request.method == "GET": + decoder = WCS20ExceptionHandlerKVPDecoder(request.GET) + elif request.method == "POST": + decoder = WCS20ExceptionHandlerXMLDecoder(request.body) + + if decoder.exceptions == "text/html": + return OWS20ExceptionHTMLEncoder() + else: + return OWS20ExceptionXMLEncoder() + def handle_exception(self, request, exception): message = str(exception) code = getattr(exception, "code", None) @@ -72,9 +134,9 @@ elif code in ("OperationNotSupported", "OptionNotSupported"): status = 501 - encoder = OWS20ExceptionXMLEncoder() + encoder = self.get_encoder(request) xml = encoder.serialize( - encoder.encode_exception(message, "2.0.1", code, locator) + encoder.encode_exception(message, "2.0.1", code, locator, request=request, exception=exception) ) return (xml, encoder.content_type, status) diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wcs/v20/geteocoverageset.py eoxserver-1.2.2/eoxserver/services/ows/wcs/v20/geteocoverageset.py --- eoxserver-1.1.2/eoxserver/services/ows/wcs/v20/geteocoverageset.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wcs/v20/geteocoverageset.py 2023-02-20 12:10:15.000000000 +0000 @@ -48,7 +48,7 @@ from eoxserver.render.coverage import objects from eoxserver.resources.coverages import models from eoxserver.services.ows.wcs.v20.util import ( - nsmap, parse_subset_kvp, parse_subset_xml, parse_scaleaxis_kvp, + nsmapGetEoCoverageSet, parse_subset_kvp, parse_subset_xml, parse_scaleaxis_kvp, parse_scaleaxis_xml, parse_scaleextent_kvp, parse_scaleextent_xml, parse_scalesize_kvp, parse_scalesize_xml, parse_interpolation ) @@ -91,6 +91,26 @@ return PACKAGE_WRITERS +class TempfileIterator(object): + def __init__(self, filename, chunksize=2048, delete=True): + self.filename = filename + self.chunksize = chunksize + self.delete = delete + + def close(self): + if self.delete and os.path.exists(self.filename): + os.remove(self.filename) + + def __iter__(self): + with open(self.filename, 'rb') as file_obj: + while True: + data = file_obj.read(self.chunksize) + if not data: + break + yield data + self.close() + + class WCS20GetEOCoverageSetHandler(object): service = "WCS" versions = ("2.0.0", "2.0.1") @@ -323,9 +343,8 @@ package, package_format, format_params ) writer.cleanup(package) - response = StreamingHttpResponse( - tempfile_iterator(pkg_filename), mime_type + TempfileIterator(pkg_filename), mime_type ) response["Content-Disposition"] = 'inline; filename="ows%s"' % ext response["Content-Length"] = str(os.path.getsize(pkg_filename)) @@ -333,18 +352,6 @@ return response -def tempfile_iterator(filename, chunksize=2048, delete=True): - with open(filename, 'rb') as file_obj: - while True: - data = file_obj.read(chunksize) - if not data: - break - yield data - - if delete: - os.remove(filename) - - def pos_int(value): value = int(value) if value < 0: @@ -391,26 +398,24 @@ interpolation = kvp.Parameter("interpolation", type=parse_interpolation, num="?") subsettingcrs = kvp.Parameter("subsettingcrs", num="?") outputcrs = kvp.Parameter("outputcrs", num="?") - interpolation = kvp.Parameter("interpolation", type=parse_interpolation, num="?") class WCS20GetEOCoverageSetXMLDecoder(xml.Decoder): eo_ids = xml.Parameter("wcseo11:eoId/text()", num="+", locator="eoid") subsets = xml.Parameter("wcs:DimensionTrim", type=parse_subset_xml, num="*") - containment = xml.Parameter("wcseo11:containment/text()", type=containment_enum, locator="containment") + containment = xml.Parameter("wcseo11:containment/text()", num="?", type=containment_enum, locator="containment") count = xml.Parameter("@count", type=pos_int, num="?", default=MAXSIZE, locator="count") start_index = xml.Parameter("@startIndex", type=pos_int, num="?", default=0, locator="startIndex") package_format = xml.Parameter("wcseo11:packageFormat/text()", type=parse_package_format, num="?", locator="packageFormat") - mediatype = xml.Parameter("wcs:mediaType/text()", num="?", locator="mediatype") + mediatype = xml.Parameter("wcseo11:mediaType/text()", num="?", locator="mediatype") format = xml.Parameter("wcseo11:format/text()", num="?", locator="format") apply_subset = xml.Parameter("wcseo11:applySubset/text()", type=parse_apply_subset, num="?", locator="format") - scalefactor = xml.Parameter("scal:ScaleByFactor/scal:scaleFactor/text()", type=float, num="?", locator="scalefactor") - scaleaxes = xml.Parameter("scal:ScaleByAxesFactor/scal:ScaleAxis", type=parse_scaleaxis_xml, num="*", default=(), locator="scaleaxes") - scalesize = xml.Parameter("scal:ScaleToSize/scal:TargetAxisSize", type=parse_scalesize_xml, num="*", default=(), locator="scalesize") - scaleextent = xml.Parameter("scal:ScaleToExtent/scal:TargetAxisExtent", type=parse_scaleextent_xml, num="*", default=(), locator="scaleextent") - interpolation = xml.Parameter("int:Interpolation/int:globalInterpolation/text()", type=parse_interpolation, num="?", locator="interpolation") - subsettingcrs = xml.Parameter("crs:subsettingCrs/text()", num="?", locator="subsettingcrs") - outputcrs = xml.Parameter("crs:outputCrs/text()", num="?", locator="outputcrs") - interpolation = xml.Parameter("int:Interpolation/int:globalInterpolation/text()", type=parse_interpolation, num="?", locator="interpolation") + scalefactor = xml.Parameter("wcs:Extension/scal:ScaleByFactor/scal:scaleFactor/text()", type=float, num="?", locator="scalefactor") + scaleaxes = xml.Parameter("wcs:Extension/scal:ScaleByAxesFactor/scal:ScaleAxis", type=parse_scaleaxis_xml, num="*", default=(), locator="scaleaxes") + scalesize = xml.Parameter("wcs:Extension/scal:ScaleToSize/scal:TargetAxisSize", type=parse_scalesize_xml, num="*", default=(), locator="scalesize") + scaleextent = xml.Parameter("wcs:Extension/scal:ScaleToExtent/scal:TargetAxisExtent", type=parse_scaleextent_xml, num="*", default=(), locator="scaleextent") + interpolation = xml.Parameter("wcs:Extension/int:Interpolation/int:globalInterpolation/text()", type=parse_interpolation, num="?", locator="interpolation") + subsettingcrs = xml.Parameter("wcs:Extension/crs:subsettingCrs/text()", num="?", locator="subsettingcrs") + outputcrs = xml.Parameter("wcs:Extension/crs:outputCrs/text()", num="?", locator="outputcrs") - namespaces = nsmap + namespaces = nsmapGetEoCoverageSet diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wcs/v20/util.py eoxserver-1.2.2/eoxserver/services/ows/wcs/v20/util.py --- eoxserver-1.1.2/eoxserver/services/ows/wcs/v20/util.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wcs/v20/util.py 2023-02-20 12:10:15.000000000 +0000 @@ -63,7 +63,10 @@ # namespace map nsmap = NameSpaceMap( ns_xlink, ns_ogc, ns_ows, ns_gml, ns_gmlcov, ns_wcs, ns_crs, ns_rsub, - ns_eowcs, ns_om, ns_eop, ns_swe, ns_int, ns_scal + ns_eowcs, ns_om, ns_eop, ns_swe, ns_int, ns_scal, +) +nsmapGetEoCoverageSet = NameSpaceMap( + ns_wcs, ns_crs, ns_int, ns_scal, ns_wcseo11 ) # Element factories diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wms/basehandlers.py eoxserver-1.2.2/eoxserver/services/ows/wms/basehandlers.py --- eoxserver-1.1.2/eoxserver/services/ows/wms/basehandlers.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wms/basehandlers.py 2023-02-20 12:10:15.000000000 +0000 @@ -59,7 +59,6 @@ ) from eoxserver.services import filters from eoxserver.services.ows.wms.layermapper import LayerMapper -from eoxserver.services import views class WMSBaseGetCapabilitiesHandler(object): @@ -122,7 +121,7 @@ conf = CapabilitiesConfigReader(get_eoxserver_config()) return encoder.serialize( encoder.encode_capabilities( - conf, request.build_absolute_uri(reverse(views.ows)), + conf, request.build_absolute_uri(reverse('ows')), crss.getSupportedCRS_WMS(format_function=crss.asShortCode), map_renderer.get_supported_formats(), [], layer_descriptions diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wms/layermapper.py eoxserver-1.2.2/eoxserver/services/ows/wms/layermapper.py --- eoxserver-1.1.2/eoxserver/services/ows/wms/layermapper.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wms/layermapper.py 2023-02-20 12:10:15.000000000 +0000 @@ -608,7 +608,7 @@ color = config.Option(type=str, default='grey') -def _generate_browse_from_browse_type(product, browse_type, variables): +def _generate_browse_from_browse_type(product: models.Product, browse_type: models.BrowseType, variables): if not browse_type.red_or_grey_expression: return None @@ -666,6 +666,7 @@ fields_and_coverages, product, variables, + show_out_of_bounds_data=browse_type.show_out_of_bounds_data, ) return None diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/config.py eoxserver-1.2.2/eoxserver/services/ows/wps/config.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/config.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/config.py 2023-02-20 12:10:15.000000000 +0000 @@ -30,7 +30,8 @@ 'eoxserver.services.ows.wps.processes.get_time_data.GetTimeDataProcess', 'eoxserver.services.ows.wps.processes.get_height_profile.GetHeightProfileProcess', 'eoxserver.services.ows.wps.processes.get_statistics.GetStatisticsProcess', - 'eoxserver.services.ows.wps.processes.get_dem_processing.DemProcessingProcess' + 'eoxserver.services.ows.wps.processes.get_dem_processing.DemProcessingProcess', + 'eoxserver.services.ows.wps.processes.get_cloud_coverage.CloudCoverageProcess' ] DEFAULT_EOXS_ASYNC_BACKENDS = [ diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/complexdata.py eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/complexdata.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/complexdata.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/complexdata.py 2023-02-20 12:10:15.000000000 +0000 @@ -51,7 +51,7 @@ from lxml import etree from .base import Parameter from .formats import Format -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from django.utils.six import string_types, text_type, itervalues, binary_type #------------------------------------------------------------------------------- @@ -172,7 +172,7 @@ """ def __init__(self, data=u'', *args, **kwargs): # NOTE: StringIO is an old-style class and super cannot be used! - StringIO.__init__(self, smart_text(data)) + StringIO.__init__(self, smart_str(data)) CDBase.__init__(self, *args, **kwargs) self.text_encoding = kwargs.get('text_encoding', None) @@ -183,9 +183,9 @@ def write(self, data): if self.text_encoding is None: - return StringIO.write(self, smart_text(data)) + return StringIO.write(self, smart_str(data)) else: - return StringIO.write(self, smart_text(data, self.text_encoding)) + return StringIO.write(self, smart_str(data, self.text_encoding)) def read(self, size=None): if size is None: @@ -532,7 +532,7 @@ if isinstance(data, text_type): return data elif isinstance(data, bytes): - return smart_text(data, encoding) + return smart_str(data, encoding) raise TypeError( "Byte or Unicode string expected, %s received!" % type(data) ) diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/crs.py eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/crs.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/crs.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/crs.py 2023-02-20 12:10:15.000000000 +0000 @@ -31,7 +31,7 @@ asURL, fromURL, fromURN, fromShortCode, validateEPSGCode, parseEPSGCode, ) from .data_types import BaseType -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str class CRSType(BaseType): @@ -66,7 +66,7 @@ if value == 0: return u'ImageCRS' elif validateEPSGCode(value): - return smart_text(asURL(value)) + return smart_str(asURL(value)) raise ValueError("Invalid CRS %r!" % value) @classmethod diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/data_types.py eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/data_types.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/data_types.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/data_types.py 2023-02-20 12:10:15.000000000 +0000 @@ -30,7 +30,7 @@ from datetime import datetime, date, time, timedelta from django.utils.dateparse import parse_date, parse_datetime, parse_time, utc from django.utils.six import PY2, PY3, string_types -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from eoxserver.core.util.timetools import parse_duration try: @@ -68,7 +68,7 @@ @classmethod def encode(cls, value): """ Encode value to a Unicode string.""" - return smart_text(value) + return smart_str(value) @classmethod def get_diff_dtype(cls): # difference type - change if differs from the base @@ -97,7 +97,7 @@ def parse(cls, raw_value): if isinstance(raw_value, string_types): - raw_value = smart_text(raw_value.lower()) + raw_value = smart_str(raw_value.lower()) if raw_value in ('1', 'true'): return True elif raw_value in ('0', 'false'): @@ -130,7 +130,7 @@ @classmethod def encode(cls, value): """ Encode value to a Unicode string.""" - return smart_text(int(value)) + return smart_str(int(value)) @classmethod def as_number(cls, value): @@ -176,9 +176,9 @@ def encode(cls, value): """ Encode value to a Unicode string.""" try: - return smart_text(value) + return smart_str(value) except UnicodeDecodeError: - return smart_text(value, cls.encoding) + return smart_str(value, cls.encoding) @classmethod def parse(cls, raw_value): @@ -228,7 +228,7 @@ elif seconds != 0: items.append('%dS' % seconds) - return smart_text("".join(items)) + return smart_str("".join(items)) @classmethod def as_number(cls, value): @@ -261,7 +261,7 @@ @classmethod def encode(cls, value): if isinstance(value, cls.dtype): - return smart_text(value.isoformat()) + return smart_str(value.isoformat()) raise ValueError("Invalid value type '%s'!" % type(value)) @classmethod @@ -292,7 +292,7 @@ @classmethod def encode(cls, value): if isinstance(value, cls.dtype): - return smart_text(value.isoformat()) + return smart_str(value.isoformat()) raise ValueError("Invalid value type '%s'!" % type(value)) @classmethod @@ -329,7 +329,7 @@ @classmethod def encode(cls, value): if isinstance(value, cls.dtype): - return smart_text(cls._isoformat(value)) + return smart_str(cls._isoformat(value)) raise ValueError("Invalid value type '%s'!" % type(value)) @classmethod diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/literaldata.py eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/literaldata.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/parameters/literaldata.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/parameters/literaldata.py 2023-02-20 12:10:15.000000000 +0000 @@ -38,7 +38,7 @@ from .data_types import BaseType, String, DTYPES from .allowed_values import BaseAllowed, AllowedAny, AllowedEnum from .units import UnitOfMeasure, UnitLinear -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from django.utils.six import text_type @@ -195,9 +195,9 @@ if isinstance(raw_value, text_type): _value = raw_value elif isinstance(raw_value, str): - _value = smart_text(raw_value, encoding) + _value = smart_str(raw_value, encoding) else: - _value = smart_text(raw_value) + _value = smart_str(raw_value) _value = self._dtype.parse(raw_value) _value = self.strip_uom(_value, uom or self.default_uom) _value = self._allowed_values.verify(_value) diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/processes/get_cloud_coverage.py eoxserver-1.2.2/eoxserver/services/ows/wps/processes/get_cloud_coverage.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/processes/get_cloud_coverage.py 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/processes/get_cloud_coverage.py 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,212 @@ +# ----------------------------------------------------------------------------- +# +# Project: EOxServer +# Authors: Bernhard Mallinger +# +# ----------------------------------------------------------------------------- +# Copyright (C) 2022 EOX IT Services GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ----------------------------------------------------------------------------- + +import contextlib +import concurrent +import functools +from datetime import datetime +from uuid import uuid4 + +from osgeo import ogr, osr + +from eoxserver.core import Component +from eoxserver.contrib import gdal +from eoxserver.resources.coverages import models +from eoxserver.backends.access import gdal_open +from eoxserver.services.ows.wps.parameters import ( + LiteralData, + ComplexData, + FormatJSON, + FormatText, + CDObject, +) +import logging + +logger = logging.getLogger(__name__) + + +class CloudCoverageProcess(Component): + + identifier = "CloudCoverage" + title = "Cloud coverage information about images of an AOI/TOI" + description = "" + metadata = {} + profiles = ["EOxServer:CloudCoverage"] + + inputs = { + "begin_time": LiteralData( + "begin_time", + datetime, + title="Start of the time interval.", + ), + "end_time": LiteralData( + "end_time", + datetime, + title="End of the time interval.", + ), + "geometry": ComplexData( + "geometry", + title="Geometry", + formats=[FormatText()], + ), + } + + outputs = { + "result": ComplexData( + "result", + title="output data", + abstract="Information about cloud coverage", + formats=(FormatJSON(),), + ), + } + + SCL_LAYER_NO_DATA = 0 + SCL_LAYER_CLOUD_MEDIUM_PROBABILITY = 8 + SCL_LAYER_CLOUD_HIGH_PROBABILITY = 9 + SCL_LAYER_THIN_CIRRUS = 10 + SCL_LAYER_SATURATED_OR_DEFECTIVE = 1 + + @staticmethod + def execute( + begin_time, + end_time, + geometry, + result, + ): + wkt_geometry = geometry[0].text + + # TODO Use queue object for more complex query if parent_product__footprint is not enough + coverages = models.Coverage.objects.filter( + parent_product__begin_time__lte=end_time, + parent_product__end_time__gte=begin_time, + parent_product__footprint__intersects=wkt_geometry, + coverage_type__name="SCL", + ).order_by("parent_product__begin_time") + + logger.info("Matched %s coverages for cloud coverage", coverages.count()) + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as e: + cloud_coverage_ratios = e.map( + functools.partial( + cloud_coverage_ratio_in_geometry, + wkt_geometry=wkt_geometry, + ), + [coverage.arraydata_items.get() for coverage in coverages], + ) + + result = { + "result": { + coverage.parent_product.begin_time.isoformat(): cloud_cover_ratio + for coverage, cloud_cover_ratio in zip(coverages, cloud_coverage_ratios) + } + } + return CDObject( + result, + format=FormatJSON(), + filename=("cloud_coverage.json"), + ) + + +def cloud_coverage_ratio_in_geometry( + data_item: models.ArrayDataItem, + wkt_geometry: str, +) -> float: + # NOTE: this is executed in threads, but all gdal operations are contained + # in here, so each thread has separate gdal data + + tmp_ds = f"/vsimem/{uuid4()}.tif" + original_ds = gdal_open(data_item) + + with _create_geometry_feature_in_memory(wkt_geometry) as geometry_mem_path: + result_ds = gdal.Warp( + tmp_ds, + original_ds, + # TODO: ideally only cut relevant band. possibly retrieve + # single band and only bbox with with gdal_translate + options=gdal.WarpOptions( + cutlineDSName=geometry_mem_path, + cropToCutline=True, + warpOptions=["CUTLINE_ALL_TOUCHED=TRUE"], + ), + ) + + # NOTE: using histogram is safe because it defaults to a bin distribution + # which captures integers + histogram = result_ds.GetRasterBand(1).GetHistogram( + approx_ok=False, + include_out_of_range=True, + ) + + num_cloud = sum( + histogram[scl_value] + for scl_value in [ + CloudCoverageProcess.SCL_LAYER_CLOUD_MEDIUM_PROBABILITY, + CloudCoverageProcess.SCL_LAYER_CLOUD_HIGH_PROBABILITY, + CloudCoverageProcess.SCL_LAYER_THIN_CIRRUS, + CloudCoverageProcess.SCL_LAYER_SATURATED_OR_DEFECTIVE, + ] + ) + + num_no_data = histogram[CloudCoverageProcess.SCL_LAYER_NO_DATA] + + num_pixels = sum(histogram) - num_no_data + + cloud_coverage_ratio = num_cloud / num_pixels if num_pixels != 0 else 0 + + gdal.Unlink(tmp_ds) + + return cloud_coverage_ratio + + +@contextlib.contextmanager +def _create_geometry_feature_in_memory(wkt_geometry: str): + memory_path = f"/vsimem/{uuid4()}.shp" + + ogr_geometry = ogr.CreateGeometryFromWkt(wkt_geometry) + + drv = ogr.GetDriverByName("ESRI Shapefile") + + feature_ds = drv.CreateDataSource(memory_path) + + srs = osr.SpatialReference() + # TODO: always this value? + srs.ImportFromEPSG(4326) + feature_layer = feature_ds.CreateLayer("layer", srs, geom_type=ogr.wkbPolygon) + + featureDefnHeaders = feature_layer.GetLayerDefn() + + outFeature = ogr.Feature(featureDefnHeaders) + + outFeature.SetGeometry(ogr_geometry) + + feature_layer.CreateFeature(outFeature) + + feature_ds.FlushCache() + + yield memory_path + + gdal.Unlink(memory_path) diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/processes/get_height_profile.py eoxserver-1.2.2/eoxserver/services/ows/wps/processes/get_height_profile.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/processes/get_height_profile.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/processes/get_height_profile.py 2023-02-20 12:10:15.000000000 +0000 @@ -28,6 +28,7 @@ import csv from uuid import uuid4 +import json from math import radians, cos, sin, asin, sqrt @@ -123,7 +124,7 @@ """ if isinstance(line, str): - line = eval(line) + line = json.loads('[%s]' % line) line_distance = haversine(line[0], line[1], line[2], line[3]) diff -Nru eoxserver-1.1.2/eoxserver/services/ows/wps/v20/execute.py eoxserver-1.2.2/eoxserver/services/ows/wps/v20/execute.py --- eoxserver-1.1.2/eoxserver/services/ows/wps/v20/execute.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/ows/wps/v20/execute.py 2023-02-20 12:10:15.000000000 +0000 @@ -25,8 +25,8 @@ # THE SOFTWARE. # ------------------------------------------------------------------------------- -import typing from logging import getLogger +from typing import Any from eoxserver.services.ows.wps.util import get_process_by_identifier from eoxserver.services.ows.wps.interfaces import ProcessInterface @@ -110,7 +110,7 @@ raise OperationNotSupportedError("Document mode not implemented") -def _input_value(input_: pyows_types.Input) -> typing.Any: +def _input_value(input_: pyows_types.Input) -> Any: if isinstance(input_.data, pyows_types.Data): data_value = input_.data.value @@ -124,6 +124,7 @@ crs=data_value.crs, ) else: - raise OperationNotSupportedError("Unsupported input element") + # not a common type, process needs to handle it on its own + return data_value else: raise OperationNotSupportedError("References as input are not implemented") diff -Nru eoxserver-1.1.2/eoxserver/services/templates/wcs/error_template.html eoxserver-1.2.2/eoxserver/services/templates/wcs/error_template.html --- eoxserver-1.1.2/eoxserver/services/templates/wcs/error_template.html 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/templates/wcs/error_template.html 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,17 @@ + + + Error: {{ message }} + + +

Error: {{ message }}

+ {% if exception.code == 404 %} +

Not found.

+ {% endif %} + {% if debug %} +

DEBUG: Full stack trace

+

+ {{ stack_trace }} +

+ {% endif %} + + diff -Nru eoxserver-1.1.2/eoxserver/services/tests.py eoxserver-1.2.2/eoxserver/services/tests.py --- eoxserver-1.1.2/eoxserver/services/tests.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/tests.py 2023-02-20 12:10:15.000000000 +0000 @@ -25,11 +25,17 @@ # THE SOFTWARE. #------------------------------------------------------------------------------- +import http from textwrap import dedent +import importlib +import sys -from django.test import TestCase, TransactionTestCase + +from django.conf import settings +from django.test import TestCase, TransactionTestCase, Client, override_settings from django.contrib.gis.geos import Polygon, MultiPolygon from django.utils.six import assertCountEqual, b +from django.urls import clear_url_caches from eoxserver.core.util import multiparttools as mp from eoxserver.core.util.timetools import parse_iso8601 @@ -37,6 +43,9 @@ from eoxserver.services.subset import Subsets, Trim, Slice from eoxserver.services.result import result_set_from_raw_data from eoxserver.resources.coverages import models +import eoxserver.services.config +import eoxserver.services.views + class MultipartTest(TestCase): @@ -738,3 +747,28 @@ self.evaluate_subsets( self.make_subsets("2000-01-01T00:00:40Z"), "contains", ("H",) ) + +class CachingTest(TestCase): + def _reload_ows_views(self): + # NOTE: we have to do this dance because the setting + # is read at import time + importlib.reload(eoxserver.services.views) + importlib.reload(eoxserver.services.urls) + importlib.reload(sys.modules[settings.ROOT_URLCONF]) + clear_url_caches() + + + def test_ows_view_not_cached_by_default(self): + response = Client().get("/ows", {"service": "WMS", "request": "GetCapabilities"}) + self.assertEqual(response.status_code, http.HTTPStatus.OK) + self.assertNotIn('Cache-Control', response) + + def test_ows_view_cached_if_configured(self): + with override_settings(EOXS_RENDERER_CACHE_TIME="3"): + self._reload_ows_views() + response = Client().get("/ows", {"service": "WMS", "request": "GetCapabilities"}) + + self._reload_ows_views() + + self.assertEqual(response.status_code, http.HTTPStatus.OK) + self.assertEqual(response['Cache-Control'], "max-age=3") diff -Nru eoxserver-1.1.2/eoxserver/services/urls.py eoxserver-1.2.2/eoxserver/services/urls.py --- eoxserver-1.1.2/eoxserver/services/urls.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/urls.py 2023-02-20 12:10:15.000000000 +0000 @@ -37,7 +37,7 @@ from eoxserver.services import views urlpatterns = [ - re_path(r'^$', views.ows, name='ows') + re_path(r'^$', views.ows, name='ows',) ] diff -Nru eoxserver-1.1.2/eoxserver/services/views.py eoxserver-1.2.2/eoxserver/services/views.py --- eoxserver-1.1.2/eoxserver/services/views.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/eoxserver/services/views.py 2023-02-20 12:10:15.000000000 +0000 @@ -33,6 +33,7 @@ import logging import traceback +from django.conf import settings from django.http import HttpResponse try: from django.http import StreamingHttpResponse @@ -48,12 +49,14 @@ from eoxserver.services.ows.dispatch import ( query_service_handler, query_exception_handler ) +from eoxserver.services.config import apply_cache_header logger = logging.getLogger(__name__) @csrf_exempt +@apply_cache_header def ows(request): """ Main entry point for OWS requests against EOxServer. It uses the :class:`ServiceComponent @@ -111,3 +114,8 @@ ) except ValueError: pass + + +# NOTE: we need to apply caching here because the name `views.ows` +# is being url-reversed at some point, so this name needs to be registered +# with django diff -Nru eoxserver-1.1.2/.github/ISSUE_TEMPLATE/bug_report.md eoxserver-1.2.2/.github/ISSUE_TEMPLATE/bug_report.md --- eoxserver-1.1.2/.github/ISSUE_TEMPLATE/bug_report.md 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/.github/ISSUE_TEMPLATE/bug_report.md 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff -Nru eoxserver-1.1.2/.github/ISSUE_TEMPLATE/feature_request.md eoxserver-1.2.2/.github/ISSUE_TEMPLATE/feature_request.md --- eoxserver-1.1.2/.github/ISSUE_TEMPLATE/feature_request.md 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/.github/ISSUE_TEMPLATE/feature_request.md 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff -Nru eoxserver-1.1.2/.github/workflows/ci.yml eoxserver-1.2.2/.github/workflows/ci.yml --- eoxserver-1.1.2/.github/workflows/ci.yml 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/.github/workflows/ci.yml 2023-02-20 12:10:15.000000000 +0000 @@ -1,60 +1,77 @@ name: CI -on: push +run-name: CI pipeline triggered by @${{ github.actor }} +on: + push: + paths: + - '.github/workflows/**' + - 'setup.py' + - 'setup.cfg' + - 'MANIFEST.in' + - 'pyproject.toml' + - 'eoxserver/**' jobs: - run: + build-docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build the eoxserver docker image + run: | + docker build -t eoxserver . + docker save eoxserver | gzip > eoxserver.tar.gz + - uses: actions/upload-artifact@v3 + with: + name: eoxserver + path: eoxserver.tar.gz + retention-days: 2 + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() + + test: runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - # TODO: deactivated as django 1.11 does not seem to work with the GEOS version supplied by Ubuntu 20.04 - # - os: ubuntu - # python: py3 - # db: postgis - # django: "1.11.26" - # python_bin: python3 - # pip_bin: pip3 - - os: ubuntu - python: py3 - db: postgis - django: "2.2.17" + - db: postgis python_bin: python3 - pip_bin: pip3 - - os: ubuntu - python: py3 - db: spatialite - django: "2.2.17" + - db: spatialite python_bin: python3 - pip_bin: pip3 - latest: true + needs: build-docker steps: - - uses: actions/checkout@v2 - - name: Build the eoxserver docker image + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3.0.1 + with: + name: eoxserver + - name: Import docker image run: | - docker build -t eoxserver --build-arg DJANGO=${{ matrix.django }} -f docker/${{ matrix.os }}/${{ matrix.python }}/Dockerfile . + docker load --input eoxserver.tar.gz - name: Run the tests env: COMPOSE_INTERACTIVE_NO_CLI: 1 run: | - cd autotest - echo "DB=${{ matrix.db }}" >> eoxserver.env + echo "DB=${{ matrix.db }}" >> sample.env docker-compose config docker-compose up -d docker-compose ps - docker exec -i autotest_autotest_1 ${{ matrix.pip_bin }} install scipy - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_data_types - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_allowed_values - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.core -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.backends -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.services -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.resources.coverages -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs20 -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs11 -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs10 -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wms -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wps -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag opensearch -v2 - cd .. + docker exec -i eoxserver_autotest_1 pip3 install scipy + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_data_types + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_allowed_values + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.core -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.backends -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.services -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=./eoxserver/ eoxserver.resources.coverages -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs20 -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs11 -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs10 -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wms -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wps -v2 + docker exec -i eoxserver_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag opensearch -v2 - name: Upload logs and outputs of failed tests uses: 'actions/upload-artifact@v2' with: @@ -64,64 +81,113 @@ autotest/autotest/responses/* retention-days: 5 if: failure() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() - # get branch/tag name for later stages + publish-docker: + runs-on: ubuntu-latest + needs: test + if: contains(github.ref, 'refs/tags/') || github.ref == 'refs/heads/master' + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3.0.1 + with: + name: eoxserver - name: Branch name id: branch_name run: | - echo ::set-output name=SOURCE_BRANCH:: $([[ $GITHUB_REF == refs/heads/* ]] && echo ${GITHUB_REF#refs/heads/} || echo "") echo ::set-output name=SOURCE_TAG::$([[ $GITHUB_REF == refs/tags/* ]] && echo ${GITHUB_REF#refs/tags/} || echo "") - - # docker image tagging/publishing - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} if: success() - - # conditionally tag docker images and push them to dockerhub - - name: Tag docker latest master image + - name: Import docker image run: | - docker tag eoxserver eoxa/eoxserver:master - if: success() && steps.branch_name.outputs.SOURCE_BRANCH == 'master' && matrix.latest + docker load --input eoxserver.tar.gz - name: Tag docker latest image run: | docker tag eoxserver eoxa/eoxserver:latest - if: success() && matrix.latest - - name: Tag docker latest release image + if: github.ref == 'refs/heads/master' + - name: Tag docker release image run: | docker tag eoxserver eoxa/eoxserver:${{ steps.branch_name.outputs.SOURCE_TAG }} - if: success() && steps.branch_name.outputs.SOURCE_TAG && matrix.latest - - name: Tag docker release image with OS and Python/Django versions - run: | - docker tag eoxserver eoxa/eoxserver:${{ steps.branch_name.outputs.SOURCE_TAG }}-${{ matrix.os }}-${{ matrix.python }}-django${{ matrix.django }} - if: success() && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') + if: steps.branch_name.outputs.SOURCE_TAG - name: Push docker images run: | # TODO: --all-tags does not seem to work with the version on github-actions # docker push --all-tags eoxa/eoxserver for tag in $(docker image ls --format "{{.Tag}}" eoxa/eoxserver) ; do docker push "eoxa/eoxserver:$tag" ; done if: success() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() - # build a Python package and publish it on pypi + publish-pypi: + runs-on: ubuntu-latest + needs: test + if: contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v3 + - name: Branch name + id: branch_name + run: | + echo ::set-output name=SOURCE_TAG::$([[ $GITHUB_REF == refs/tags/* ]] && echo ${GITHUB_REF#refs/tags/} || echo "") - name: Build Python package id: build_python_release run: | python -m pip install --upgrade pip pip install setuptools wheel + sudo apt-get install -y gdal-bin python setup.py sdist bdist_wheel - echo ::set-output name=WHEEL_FILE::$(ls dist/*.whl) - echo ::set-output name=SRC_DIST_FILE::$(ls dist/*.tar.gz) + - uses: actions/upload-artifact@v3 + with: + name: eoxserver-dist + path: ./dist/ + retention-days: 2 - name: Push package to pypi uses: pypa/gh-action-pypi-publish@master with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') - - # draft a github release and add files - - name: Create Release + if: success() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() + + publish-github: + runs-on: ubuntu-latest + needs: publish-pypi + if: contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3.0.1 + with: + name: eoxserver-dist + - name: Branch name + id: branch_name + run: | + echo ::set-output name=SOURCE_TAG::$([[ $GITHUB_REF == refs/tags/* ]] && echo ${GITHUB_REF#refs/tags/} || echo "") + echo ::set-output name=WHEEL_FILE::$(ls *.whl) + echo ::set-output name=SRC_DIST_FILE::$(ls *.tar.gz) + - name: Draft Release id: create_release uses: actions/create-release@v1 env: @@ -130,38 +196,50 @@ tag_name: ${{ github.ref }} release_name: ${{ steps.branch_name.outputs.SOURCE_TAG }} draft: true - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') - name: Upload Release Asset Wheel uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.build_python_release.outputs.WHEEL_FILE }} - asset_name: ${{ steps.build_python_release.outputs.WHEEL_FILE }} + asset_path: ${{ steps.branch_name.outputs.WHEEL_FILE }} + asset_name: ${{ steps.branch_name.outputs.WHEEL_FILE }} asset_content_type: application/x-wheel+zip - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') + if: success() - name: Upload Release Asset Source Dist uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: ${{ steps.build_python_release.outputs.SRC_DIST_FILE }} - asset_name: ${{ steps.build_python_release.outputs.SRC_DIST_FILE }} + asset_path: ${{ steps.branch_name.outputs.SRC_DIST_FILE }} + asset_name: ${{ steps.branch_name.outputs.SRC_DIST_FILE }} asset_content_type: application/tar+gzip - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') - - note: + if: success() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() + + notify: runs-on: ubuntu-20.04 - needs: run + needs: [publish-github, publish-docker] steps: - # send Slack notifications to the eox organization - - name: action-slack + - name: Slack Notify uses: 8398a7/action-slack@v3.8.0 with: - status: ${{ needs.run.result }} + status: ${{ job.status }} fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + custom_payload: | + { + attachments: [{ + text: `Publish tag finished`, + }] + } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} if: always() diff -Nru eoxserver-1.1.2/.gitignore eoxserver-1.2.2/.gitignore --- eoxserver-1.1.2/.gitignore 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/.gitignore 2023-02-20 12:10:15.000000000 +0000 @@ -23,3 +23,4 @@ .devcontainer/ instances/ +.minio.sys/ diff -Nru eoxserver-1.1.2/LICENSE eoxserver-1.2.2/LICENSE --- eoxserver-1.1.2/LICENSE 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/LICENSE 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,20 @@ +EOxServer Open License +Version 1, 8 June 2011 + +Copyright (C) 2011 EOX IT Services GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the “Software”), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies +of this Software or works derived from this Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff -Nru eoxserver-1.1.2/MANIFEST.in eoxserver-1.2.2/MANIFEST.in --- eoxserver-1.1.2/MANIFEST.in 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/MANIFEST.in 2023-02-20 12:10:15.000000000 +0000 @@ -1,4 +1,4 @@ -include README.rst +include README.md include MANIFEST.in graft eoxserver recursive-exclude eoxserver *.pyc diff -Nru eoxserver-1.1.2/pyproject.toml eoxserver-1.2.2/pyproject.toml --- eoxserver-1.1.2/pyproject.toml 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/pyproject.toml 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.distutils.bdist_wheel] +universal = true diff -Nru eoxserver-1.1.2/README.md eoxserver-1.2.2/README.md --- eoxserver-1.1.2/README.md 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/README.md 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,68 @@ +# EOxServer + +EOxServer is a Python application and library for presenting Earth +Observation (EO) data and metadata. + +![build](https://github.com/EOxServer/eoxserver/actions/workflows/ci.yml/badge.svg) +[![PyPi](https://badge.fury.io/py/EOxServer.svg)](https://pypi.org/project/EOxServer/) +[![ReadTheDocs](https://readthedocs.org/projects/eoxserver/badge/?version=master)](http://docs.eoxserver.org/en/master) + +EOxServer implements the [OGC](http://www.opengeospatial.org) +Implementation Specifications EO-WCS and EO-WMS on top of +[MapServer's](http://mapserver.org) [WCS](http://www.opengeospatial.org/standards/wcs) and +[WMS](http://www.opengeospatial.org/standards/wms) implementations. +EOxServer is released under the +[EOxServer Open License](https://docs.eoxserver.org/en/stable/copyright.html) an MIT-style +license and written in python and entirely based on open source software including: + +- [MapServer](http://mapserver.org) +- [Django/GeoDjango](https://www.djangoproject.com) +- [GDAL](http://www.gdal.org>) +- [SpatiaLite](http://www.gaia-gis.it/spatialite) +- [PostGIS](http://postgis.refractions.net/>) +- [PROJ.4](http://trac.osgeo.org/proj/>) + +More information is available at [https://eoxserver.org](https://eoxserver.org). Documentation +is available at [readthedocs](https://docs.eoxserver.org/en/stable/) + +## Docker + +To run with SpatiaLite database simply run: + +```sh +docker run -it --rm -p 8080:8000 eoxa/eoxserver +``` + +EOxServer is now accessible at [http://localhost:8080/](http://localhost:8080/). +And you can login to the `Admin Client` using: + +- username: admin +- password: admin + +The following environment variables control configuration: + +- `DB`: Specify the used database type. either `spatialite` or `postgis` +- `DB_PW`, `DB_NAME`, `DB_HOST`, `DB_USER`: these credentials will be used to establish a + connection to the postgres database when DB is set to `postgis` in order to wait + for it to come online +- `INSTANCE_NAME`: the name of the instance passed to `eoxserver-instance.py` - defaults + to `instance` +- `INSTANCE_DIR`: the directory of the instance. Defaults to `/opt/instance` +- `DJANGO_USER`, `DJANGO_MAIL`, `DJANGO_PASSWORD`: when set, these credentials will be + used to create a superuser to be used for the Django Admin. By default, no user is + created +- `COLLECT_STATIC`: if set to "true" (the default), static files will be collected + upon initialization +- `PREINIT_SCRIPTS`: the list of commands that will be executed before + the instance is initialized +- `INIT_SCRIPTS`: the list of commands that will be executed once + when the instance is initialized +- `STARTUP_SCRIPTS`: the list of commands that will be executed before + the command is run +- `GUNICORN_CMD_ARGS`: gunicorn command arguments. Defaults to + `--config /opt/eoxserver/gunicorn.conf.py ${INSTANCE_NAME}.wsgi:application` + +## Development + +The autotest instance can be used for development and testing. +More information in `./autotest/README.md` diff -Nru eoxserver-1.1.2/README.rst eoxserver-1.2.2/README.rst --- eoxserver-1.1.2/README.rst 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/README.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -EOxServer -========= - -.. image:: https://travis-ci.org/EOxServer/eoxserver.svg?branch=master - :target: https://travis-ci.org/EOxServer/eoxserver - -.. image:: https://badge.fury.io/py/EOxServer.svg - :target: https://pypi.org/project/EOxServer/ - -.. image:: https://readthedocs.org/projects/eoxserver/badge/?version=master - :alt: Documentation Status - :scale: 100% - :target: http://docs.eoxserver.org/en/master - -EOxServer is a Python application and framework for presenting Earth -Observation (EO) data and metadata. - -EOxServer implements the `OGC `_ -Implementation Specifications EO-WCS and EO-WMS on top of -`MapServer's `_ -`WCS `_ and -`WMS `_ implementations. - -EOxServer is released under the `EOxServer Open License -`_ a MIT-style -license and written in `Python `_ and entirely based on -Open Source software including `MapServer `_, -`Django/GeoDjango `_, -`GDAL `_, -`SpatiaLite `_, or -`PostGIS `_, and -`PROJ.4 `_. - -More information is available at `http://eoxserver.org `_. diff -Nru eoxserver-1.1.2/requirements.txt eoxserver-1.2.2/requirements.txt --- eoxserver-1.1.2/requirements.txt 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/requirements.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -gunicorn -eventlet==0.30.2 -greenlet>=0.4.14 \ No newline at end of file diff -Nru eoxserver-1.1.2/sample.env eoxserver-1.2.2/sample.env --- eoxserver-1.1.2/sample.env 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/sample.env 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,12 @@ +INSTANCE_NAME=autotest +DB_USER=user +DB_PW=pw +DB_HOST=database +DB_PORT=5432 +DB_NAME=dbms +DB=postgis +DJANGO_USER=admin +DJANGO_PASSWORD=admin +DJANGO_MAIL=admin@sample.com +XML_CATALOG_FILES=/opt/schemas/catalog.xml +GUNICORN_CMD_ARGS="--config /opt/eoxserver/gunicorn.conf.py ${INSTANCE_NAME}.wsgi:application" diff -Nru eoxserver-1.1.2/schemas/HOWTO eoxserver-1.2.2/schemas/HOWTO --- eoxserver-1.1.2/schemas/HOWTO 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/schemas/HOWTO 1970-01-01 00:00:00.000000000 +0000 @@ -1,35 +0,0 @@ -#------------------------------------------------------------------------------- -# $Id$ -# -# Project: EOxServer -# Authors: Stephan Krause -# Stephan Meissl -# -#------------------------------------------------------------------------------- -# Copyright (C) 2011 EOX IT Services GmbH -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies of this Software or works derived from this Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -#------------------------------------------------------------------------------- - -####################### -# Schema HowTo # -####################### - -For validation in Eclipse comment attribute definition in -"schemas/SCHEMAS_OPENGIS_NET/gml/3.1.1/smil/xml-mod.xsd". diff -Nru eoxserver-1.1.2/schemas/README.md eoxserver-1.2.2/schemas/README.md --- eoxserver-1.1.2/schemas/README.md 1970-01-01 00:00:00.000000000 +0000 +++ eoxserver-1.2.2/schemas/README.md 2023-02-20 12:10:15.000000000 +0000 @@ -0,0 +1,17 @@ +# XML schemas used by EOxServer + +This repository hosts a copy of the OGC schemas available from +[http://schemas.opengis.net] as well as an excerpt of the W3 schemas from [http://www.w3.org]. + +Additionally there are proposed changes as well as extensions to those XML schemas available. + +These extensions include work in progress that are currently discussed in +the relevant working groups of the OGC mainly the WCS.SWG and might change +without notice. There is no guarantee whatsoever that these proposals will +get officially adopted by OGC. + +For validation in Eclipse comment attribute definition in +"schemas/SCHEMAS_OPENGIS_NET/gml/3.1.1/smil/xml-mod.xsd". + +More information about EOxServer is available at +`http://eoxserver.org `_. diff -Nru eoxserver-1.1.2/schemas/README.rst eoxserver-1.2.2/schemas/README.rst --- eoxserver-1.1.2/schemas/README.rst 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/schemas/README.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -XML schemas used by EOxServer -============================= - -This repository hosts a copy of the OGC schemas available from -http://schemas.opengis.net as well as an excerpt of the W3 schemas from -http://www.w3.org. - -Additionally there are proposed changes as well as extensions to those XML -schemas available. - -These extensions include work in progress that are currently discussed in -the relevant working groups of the OGC mainly the WCS.SWG and might change -without notice. There is no guarantee whatsoever that these proposals will -get officially adopted by OGC. - -More information about EOxServer is available at -`http://eoxserver.org `_. diff -Nru eoxserver-1.1.2/setup.cfg eoxserver-1.2.2/setup.cfg --- eoxserver-1.1.2/setup.cfg 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/setup.cfg 2023-02-20 12:10:15.000000000 +0000 @@ -5,3 +5,12 @@ libxml2-python python-lxml python ply + +[flake8] +max-line-length = 90 +exclude = .venv, build, tests, docs, autotest +ignore = W503,E203 + +[mypy] +exclude = (.venv|build|tests|docs|autotest) +ignore_missing_imports = True diff -Nru eoxserver-1.1.2/setup.py eoxserver-1.2.2/setup.py --- eoxserver-1.1.2/setup.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/setup.py 2023-02-20 12:10:15.000000000 +0000 @@ -28,6 +28,7 @@ # ----------------------------------------------------------------------------- import os +import subprocess from setuptools import setup # get version number @@ -79,6 +80,11 @@ if on_rtd: ext_modules = [] +try: + gdal_version = subprocess.check_output(['gdal-config','--version']).decode('utf-8').strip() +except FileNotFoundError: + gdal_version = subprocess.check_output(['gdalinfo','--version']).decode('utf-8').split(' ')[1].strip(',') + setup( name='EOxServer', version=version.replace(' ', '-'), @@ -86,27 +92,33 @@ data_files=data_files, include_package_data=True, scripts=[ - "eoxserver/scripts/eoxserver-admin.py", "eoxserver/scripts/eoxserver-instance.py", "tools/eoxserver-atpd.py", "tools/eoxserver-validate_xml.py", "tools/eoxserver-preprocess.py" ], install_requires=[ - 'django<3', - 'python-dateutil', - 'django-model-utils<4.0.0', - 'zipstream', - 'psycopg2', - 'lxml', - 'pycql==0.0.8', - 'matplotlib', - 'pyows>=0.2.6', - 'jsonfield', + "django<4", + "python-dateutil", + "django-model-utils<5.0.0", + "django-utils-six==2.0", + 'tzdata', + "zipstream", + "psycopg2", + "lxml", + "pycql==0.0.8", + "matplotlib", + "pyows>=0.2.6", + "python-keystoneclient<6.0.0", + "python-swiftclient<5.0.0", + "jsonfield", + "gunicorn", + f"gdal=={gdal_version}", + "mapscript" ], extras_require={ - 'dev': ['scipy'], - ':python_version == "2.7"': ['futures'] + 'dev': ['scipy', 'flake8', 'mypy', 'black'], + 'docs': ['sphinx', 'sphinx_rtd_theme', 'myst-parser'], }, zip_safe=False, @@ -117,7 +129,8 @@ maintainer_email="packages@eox.at", description="EOxServer is a server for Earth Observation (EO) data", - long_description=read("README.rst"), + long_description=read("README.md"), + long_description_content_type='text/markdown', classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -132,9 +145,9 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.5', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Database', 'Topic :: Internet', 'Topic :: Internet :: WWW/HTTP', diff -Nru eoxserver-1.1.2/tools/eoxserver-atpd.py eoxserver-1.2.2/tools/eoxserver-atpd.py --- eoxserver-1.1.2/tools/eoxserver-atpd.py 2022-05-16 06:36:56.000000000 +0000 +++ eoxserver-1.2.2/tools/eoxserver-atpd.py 2023-02-20 12:10:15.000000000 +0000 @@ -1,26 +1,26 @@ -#!/usr/bin/env python +#!/usr/bin/env python #----------------------------------------------------------------------- # -# Description: +# Description: # # asynchronous processing master daemon # -# This is the master server which keeps track of the aynchronous tasks in the -# queue and distributes task to the workers -# +# This is the master server which keeps track of the aynchronous tasks in the +# queue and distributes task to the workers +# #------------------------------------------------------------------------------- # # Project: EOxServer # Authors: Martin Paces # #------------------------------------------------------------------------------- -# Copyright (C) 2011 Iguassu Software Systems, a.s +# Copyright (C) 2011 Iguassu Software Systems, a.s # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all @@ -35,30 +35,30 @@ # THE SOFTWARE. #------------------------------------------------------------------------------- -# default django settings module +# default django settings module DJANGO_SETTINGS_DEFAULT = "settings" DJANGO_DB_DEFAULT = "default" #------------------------------------------------------------------------------- -import os -import sys -import signal +import os +import sys +import signal import logging import traceback -import os.path -import time +import os.path +import time import struct -import socket -from datetime import datetime, timedelta +import socket +from datetime import datetime, timedelta -from multiprocessing import Lock, Process, Queue , cpu_count +from multiprocessing import Lock, Process, Queue , cpu_count from multiprocessing.queues import Empty as MPQEmpty from multiprocessing.queues import Full as MPQFull -from django.utils.encoding import smart_text -try: import cPickle as pickle -except: import pickle +from django.utils.encoding import smart_str +try: import cPickle as pickle +except: import pickle try: # Python 2 xrange @@ -67,361 +67,361 @@ xrange = range #------------------------------------------------------------------------------- -QUEUE_EMPTY_QUERY_DELAY=1.5 # time in seconds of next query to empty queue -QUEUE_PUT_TIMEOUT=1.0 # time out used by internal task queue put operation -QUEUE_CLEAN_UP_COUNT=300 +QUEUE_EMPTY_QUERY_DELAY=1.5 # time in seconds of next query to empty queue +QUEUE_PUT_TIMEOUT=1.0 # time out used by internal task queue put operation +QUEUE_CLEAN_UP_COUNT=300 #------------------------------------------------------------------------------- -# generate unique server instance ID +# generate unique server instance ID -SERVER_ID=0 -while 0 == SERVER_ID : +SERVER_ID=0 +while 0 == SERVER_ID : tmp = os.urandom(8) SERVER_ID = struct.unpack( 'q' , tmp )[0] - SERVER_ID_STR = "0x%16.16X"%( struct.unpack( 'Q' , tmp )[0] ) + SERVER_ID_STR = "0x%16.16X"%( struct.unpack( 'Q' , tmp )[0] ) #------------------------------------------------------------------------------- dbLock = Lock() writeLock = Lock() -def write(msg) : +def write(msg) : writeLock.acquire() - sys.stdout.write("[%s] %s"%(SERVER_ID_STR,msg) ) + sys.stdout.write("[%s] %s"%(SERVER_ID_STR,msg) ) writeLock.release() -def debug( msg ) : write( ("DEBUG: %s\n"%(msg)).encode('UTF-8') ) -def info( msg ) : write( ("INFO: %s\n"%(msg)).encode('UTF-8') ) -def warn( msg ) : write( ("WARNINIG: %s\n"%(msg)).encode('UTF-8') ) -def error( msg ) : write( ("ERROR: %s\n"%(msg)).encode('UTF-8') ) +def debug( msg ) : write( ("DEBUG: %s\n"%(msg)).encode('UTF-8') ) +def info( msg ) : write( ("INFO: %s\n"%(msg)).encode('UTF-8') ) +def warn( msg ) : write( ("WARNINIG: %s\n"%(msg)).encode('UTF-8') ) +def error( msg ) : write( ("ERROR: %s\n"%(msg)).encode('UTF-8') ) #------------------------------------------------------------------------------- # global worker pool -global GWP +global GWP -GWP = None +GWP = None #------------------------------------------------------------------------------- # iterrupt and terminate signal handlers -class ExcTerminate( Exception ) : pass +class ExcTerminate( Exception ) : pass -SS = { signal.SIGINT:"SIGINT" , signal.SIGTERM:"SIGTERM" } +SS = { signal.SIGINT:"SIGINT" , signal.SIGTERM:"SIGTERM" } def signal_handler_dummy(sig, frm): pass -def signal_handler_sigint(sig, frm): - global GWP - if GWP is not None : - GWP.terminate = True - -def signal_handler_sigterm(sig, frm): - global GWP - if GWP is not None : - GWP.terminate = True - GWP.killChild = True # force immediate termination +def signal_handler_sigint(sig, frm): + global GWP + if GWP is not None : + GWP.terminate = True + +def signal_handler_sigterm(sig, frm): + global GWP + if GWP is not None : + GWP.terminate = True + GWP.killChild = True # force immediate termination #------------------------------------------------------------------------------- -class Importer( object ) : - """ smart importer of the handler subroutine - - E.g. function 'd()' in module 'a.b.c' +class Importer( object ) : + """ smart importer of the handler subroutine + + E.g. function 'd()' in module 'a.b.c' given by a path 'a.b.c.d' is imported """ - def __init__( self , path ) : - """Initialize class from the given handler + def __init__( self , path ) : + """Initialize class from the given handler subroutine dot-path""" - self.path = path + self.path = path self.module , _ , self.func = path.rpartition(".") - self.modules = [] - self.handler = None + self.modules = [] + self.handler = None + + def loadHandler( self ) : + """ load the handler subroutine """ - def loadHandler( self ) : - """ load the handler subroutine """ - - if not self.handler : + if not self.handler : - # initial list of modules - ml0 = set( sys.modules ) - - # new list of modules - ml1 = set( sys.modules ) + # initial list of modules + ml0 = set( sys.modules ) - self.handler = getattr( __import__( self.module , fromlist=[self.func] ) , self.func ) + # new list of modules + ml1 = set( sys.modules ) - # store list of loaded modules - self.modules = ml1 - ml0 + self.handler = getattr( __import__( self.module , fromlist=[self.func] ) , self.func ) - return self.handler + # store list of loaded modules + self.modules = ml1 - ml0 - def unloadHandler( self ) : - """ unload the handler subroutine """ + return self.handler - if self.handler : + def unloadHandler( self ) : + """ unload the handler subroutine """ - self.handler = None + if self.handler : - # unload the loaded modules - for m in self.modules : - del( sys.modules[m] ) + self.handler = None - # store list of loaded modules - self.modules = [] + # unload the loaded modules + for m in self.modules : + del( sys.modules[m] ) + # store list of loaded modules + self.modules = [] -def taskDispatch( taskID , threadID ) : - """ - task dispatcher - based on the request class the right request hadler is used - to process the asynchronous requets - """ - # status logger - pStatus = TaskStatus( taskID , dbLock ) +def taskDispatch( taskID , threadID ) : + """ + task dispatcher + + based on the request class the right request hadler is used + to process the asynchronous requets + """ + # status logger + pStatus = TaskStatus( taskID , dbLock ) - try: + try: - # get task parameters - requestType , requestID , requestHandler , inputs = dbLocker( dbLock , startTask , taskID ) + # get task parameters + requestType , requestID , requestHandler , inputs = dbLocker( dbLock , startTask , taskID ) - info( "[%3.3i] PROCESS: %s %s is running ... " % ( threadID , requestType , requestID ) ) + info( "[%3.3i] PROCESS: %s %s is running ... " % ( threadID , requestType , requestID ) ) - # create importer object - imp = Importer( requestHandler ) + # create importer object + imp = Importer( requestHandler ) - # try to load the right module and handler - imp.loadHandler() + # try to load the right module and handler + imp.loadHandler() - # execute handler - proper status logging is duty of the callback - imp.handler( pStatus , inputs ) + # execute handler - proper status logging is duty of the callback + imp.handler( pStatus , inputs ) - # try to unload the handler - imp.unloadHandler() + # try to unload the handler + imp.unloadHandler() - # if no terminating status has been set do it right now + # if no terminating status has been set do it right now dbLocker( dbLock , stopTaskSuccessIfNotFinished , taskID ) - info( "[%3.3i] PROCESS: %s %s is finished ... " % ( threadID , requestType , requestID ) ) + info( "[%3.3i] PROCESS: %s %s is finished ... " % ( threadID , requestType , requestID ) ) - except (KeyboardInterrupt,SystemExit): raise - except Exception as e : + except (KeyboardInterrupt,SystemExit): raise + except Exception as e : - pStatus.setFailure( smart_text((e) ) + pStatus.setFailure( smart_str((e) ) - # finish the task - error( "[%3.3i] %s " % ( threadID , smart_text((e) ) ) + # finish the task + error( "[%3.3i] %s " % ( threadID , smart_str((e) ) ) #------------------------------------------------------------------------------- def worker( queue , id ) : - """ worker function executed by worker subprocesses """ + """ worker function executed by worker subprocesses """ #def signal_handler(sig, frm): raise KeyboardInterrupt #signal.signal( signal.SIGINT, signal_handler ) - # use the dummy hadlers + # use the dummy hadlers signal.signal( signal.SIGINT, signal_handler_dummy ) #signal.signal( signal.SIGTERM, signal_handler_dummy ) - try : + try : - while True : + while True : - try : - item = queue.get() - except IOError as e : - warn( str(e) ) - continue + try : + item = queue.get() + except IOError as e : + warn( str(e) ) + continue - # gracefull termination - if ( item is None ) : break + # gracefull termination + if ( item is None ) : break - # run the task - taskDispatch( item , id ) + # run the task + taskDispatch( item , id ) - except (KeyboardInterrupt,SystemExit): pass + except (KeyboardInterrupt,SystemExit): pass info( "[%3.3i] PROCESS: termination " % id ) -def cleanup() : - """ cleanup function performing reenquing of zombie tasks """ +def cleanup() : + """ cleanup function performing reenquing of zombie tasks """ - tasks = dbLocker( dbLock , reenqueueZombieTasks , "Reenqueued by ATPD after timeout." ) + tasks = dbLocker( dbLock , reenqueueZombieTasks , "Reenqueued by ATPD after timeout." ) - for (id,task) in tasks : - warn( "[MASTER] Task %i:%s renqueued after timeout!"%(id,task) ) + for (id,task) in tasks : + warn( "[MASTER] Task %i:%s renqueued after timeout!"%(id,task) ) - tasks = dbLocker( dbLock , deleteRetiredTasks ) + tasks = dbLocker( dbLock , deleteRetiredTasks ) - for (id,task) in tasks : - info( "[MASTER] Task %i:%s deleted after expiration!"%(id,task) ) + for (id,task) in tasks : + info( "[MASTER] Task %i:%s deleted after expiration!"%(id,task) ) -class WorkerPool( object ) : +class WorkerPool( object ) : def __init__( self , nthread ) : self.queue = Queue( nthread ) - self.terminate = False - self.killChild = False - self.proces = [] - - # start subprocesses - for i in xrange( nthread ) : - p = Process( target=worker , args=( self.queue , i ) ) - p.start() + self.terminate = False + self.killChild = False + self.proces = [] + + # start subprocesses + for i in xrange( nthread ) : + p = Process( target=worker , args=( self.queue , i ) ) + p.start() self.proces.append(p) - - def __del__( self ) : - # if possible process gracefull termination - debug( "[MASTER]: enqueueing terminators ... " ) - for p in self.proces : - self.queue.put( None ) - - if not self.killChild : - debug( "[MASTER]: joining subprocesses ... " ) - for p in self.proces : - p.join() - else : - debug( "[MASTER]: terminating subprocesses ... " ) - for p in self.proces : - p.terminate() + def __del__( self ) : + # if possible process gracefull termination + debug( "[MASTER]: enqueueing terminators ... " ) + for p in self.proces : + self.queue.put( None ) + + if not self.killChild : + debug( "[MASTER]: joining subprocesses ... " ) + for p in self.proces : + p.join() + else : + debug( "[MASTER]: terminating subprocesses ... " ) + for p in self.proces : + p.terminate() - def startLoop( self ) : + + def startLoop( self ) : # reenqueue hanging tasks - # TODO: reenqueuePendingTasks() + # TODO: reenqueuePendingTasks() - cnt = 0 - taskIds = [] - self.terminate = False + cnt = 0 + taskIds = [] + self.terminate = False - while not self.terminate : + while not self.terminate : - try: + try: - # get a pending task from the queue - taskIds = dbLocker( dbLock , dequeueTask , SERVER_ID ) + # get a pending task from the queue + taskIds = dbLocker( dbLock , dequeueTask , SERVER_ID ) - except QueueEmpty : # no task to be processed + except QueueEmpty : # no task to be processed - # perform DB cleanup + # perform DB cleanup cleanup() - # wait some ammount of time - time.sleep( QUEUE_EMPTY_QUERY_DELAY ) + # wait some ammount of time + time.sleep( QUEUE_EMPTY_QUERY_DELAY ) - # clear counter - cnt = 0 + # clear counter + cnt = 0 - continue + continue - # send task to worker - for taskId in list(taskIds) : - while not self.terminate : - try : self.queue.put(taskId,True,QUEUE_PUT_TIMEOUT) - except MPQFull : continue - taskIds.remove(taskId) - break + # send task to worker + for taskId in list(taskIds) : + while not self.terminate : + try : self.queue.put(taskId,True,QUEUE_PUT_TIMEOUT) + except MPQFull : continue + taskIds.remove(taskId) + break - # increment counter - cnt += 1 + # increment counter + cnt += 1 - # perform DB cleanup - if ( cnt > QUEUE_CLEAN_UP_COUNT ) : + # perform DB cleanup + if ( cnt > QUEUE_CLEAN_UP_COUNT ) : cleanup() - cnt = 0 + cnt = 0 - info( "[MASTER]: termination in progress ... " ) + info( "[MASTER]: termination in progress ... " ) - # try to reenequeue processes taken from the DB task queue - for item in taskIds : - debug( "[MASTER]: reenquing task ID=%i ... " % item ) - dbLocker( dbLock , reenqueueTask , item , message = "Reenqued by ATPD." ) - try: - while True : + # try to reenequeue processes taken from the DB task queue + for item in taskIds : + debug( "[MASTER]: reenquing task ID=%i ... " % item ) + dbLocker( dbLock , reenqueueTask , item , message = "Reenqued by ATPD." ) + try: + while True : item = self.queue.get(False) - debug( "[MASTER]: reenquing task ID=%i ... " % item ) - dbLocker( dbLock , reenqueueTask , item , message = "Reenqued by ATPD." ) - except MPQEmpty : pass -#------------------------------------------------------------------------------- - -def usage() : - """ print usage info """ - - s = [] - s.append( "USAGE: %s [-h][-p ][-s ][-d ][-n ] " % ( os.path.basename( sys.argv[0] ) ) ) - s.append( "" ) - s.append( "PARAMETERS: " ) - s.append( " -h print this info" ) - s.append( " -p append an addition Python search path (can be repeated)" ) - s.append( " -n number of worker instance to be started ( N >= 1 , number of CPUs used by default )" ) - s.append( " -s django settings module (default '%s')"%DJANGO_SETTINGS_DEFAULT ) - #s.append( " -d django DB name (default '%s')"%DJANGO_DB_DEFAULT ) - s.append( "" ) - + debug( "[MASTER]: reenquing task ID=%i ... " % item ) + dbLocker( dbLock , reenqueueTask , item , message = "Reenqued by ATPD." ) + except MPQEmpty : pass +#------------------------------------------------------------------------------- + +def usage() : + """ print usage info """ + + s = [] + s.append( "USAGE: %s [-h][-p ][-s ][-d ][-n ] " % ( os.path.basename( sys.argv[0] ) ) ) + s.append( "" ) + s.append( "PARAMETERS: " ) + s.append( " -h print this info" ) + s.append( " -p append an addition Python search path (can be repeated)" ) + s.append( " -n number of worker instance to be started ( N >= 1 , number of CPUs used by default )" ) + s.append( " -s django settings module (default '%s')"%DJANGO_SETTINGS_DEFAULT ) + #s.append( " -d django DB name (default '%s')"%DJANGO_DB_DEFAULT ) + s.append( "" ) + return "\n".join(s) - + #------------------------------------------------------------------------------- -if __name__ == "__main__" : +if __name__ == "__main__" : - # django settings module + # django settings module - DJANGO_SETTINGS = os.environ.get("DJANGO_SETTINGS_MODULE",DJANGO_SETTINGS_DEFAULT) + DJANGO_SETTINGS = os.environ.get("DJANGO_SETTINGS_MODULE",DJANGO_SETTINGS_DEFAULT) DJANGO_DB = DJANGO_DB_DEFAULT # try to get number of CPUs - try : + try : NTHREAD = cpu_count() - except NotImplementedError : - NTHREAD = 1 - warn( "Failed to get number of CPUs! Setting to 1 asynchronous execution thread." ) - - info( "Default number of working threads: %i" % NTHREAD ) - - # parse commandline arguments - - idx = 1 - while idx < len(sys.argv) : - arg = sys.argv[idx] ; idx +=1 ; - if arg == '-p' : - sys.path.append( sys.argv[idx] ) + except NotImplementedError : + NTHREAD = 1 + warn( "Failed to get number of CPUs! Setting to 1 asynchronous execution thread." ) + + info( "Default number of working threads: %i" % NTHREAD ) + + # parse commandline arguments + + idx = 1 + while idx < len(sys.argv) : + arg = sys.argv[idx] ; idx +=1 ; + if arg == '-p' : + sys.path.append( sys.argv[idx] ) info("'%s' ... adding to Python search path." % sys.argv[idx] ) - idx += 1 - elif arg == '-s' : + idx += 1 + elif arg == '-s' : DJANGO_SETTINGS = sys.argv[idx] - idx += 1 - elif arg == '-d' : + idx += 1 + elif arg == '-d' : DJANGO_DB = sys.argv[idx] - idx += 1 - elif arg == '-n' : + idx += 1 + elif arg == '-n' : NTHREAD = max( 1 , int(sys.argv[idx]) ) - info("Setting number of working threads to: %i" % NTHREAD ) + info("Setting number of working threads to: %i" % NTHREAD ) idx += 1 - elif arg == '-h' : - sys.stderr.write(usage()) ; sys.exit(0) - else : - sys.stderr.write(usage()) - error( "Invalid commandline option '%s' !" % arg ) - sys.exit(1) + elif arg == '-h' : + sys.stderr.write(usage()) ; sys.exit(0) + else : + sys.stderr.write(usage()) + error( "Invalid commandline option '%s' !" % arg ) + sys.exit(1) #------------------------------------------------------------------- - # initialize the working enviroment + # initialize the working enviroment - # django settings module - os.environ["DJANGO_SETTINGS_MODULE"] = DJANGO_SETTINGS + # django settings module + os.environ["DJANGO_SETTINGS_MODULE"] = DJANGO_SETTINGS info("'%s' ... is set as the Django settings module " % DJANGO_SETTINGS ) info("'%s' ... is set as the Django database " % DJANGO_DB ) @@ -431,30 +431,30 @@ from eoxserver.core.system import System from eoxserver.resources.processes.tracker import TaskStatus, QueueEmpty, \ dequeueTask, startTask, reenqueueTask, stopTaskSuccessIfNotFinished, \ - reenqueueZombieTasks, deleteRetiredTasks, dbLocker + reenqueueZombieTasks, deleteRetiredTasks, dbLocker - # initialize the system + # initialize the system System.init() #------------------------------------------------------------------- - info( "ATPD Asynchronous Task Processing Daemon has just been started!") - info( "ATPD: id=%s (%i)" % ( SERVER_ID_STR , SERVER_ID ) ) - info( "ATPD: hostname=%s" % socket.getfqdn() ) + info( "ATPD Asynchronous Task Processing Daemon has just been started!") + info( "ATPD: id=%s (%i)" % ( SERVER_ID_STR , SERVER_ID ) ) + info( "ATPD: hostname=%s" % socket.getfqdn() ) info( "ATPD: pid=%i " % os.getpid() ) #------------------------------------------------------------------- - # start the worker pool - - GWP = WorkerPool( NTHREAD ) + # start the worker pool + + GWP = WorkerPool( NTHREAD ) - # use the GWP terminating hadlers + # use the GWP terminating hadlers signal.signal( signal.SIGINT, signal_handler_sigint ) signal.signal( signal.SIGTERM, signal_handler_sigterm ) - # start the main loop + # start the main loop GWP.startLoop() - # use the dummy hadlers + # use the dummy hadlers signal.signal( signal.SIGINT, signal_handler_dummy ) signal.signal( signal.SIGTERM, signal_handler_dummy )