diff -Nru mapproxy-1.12.0/CHANGES.txt mapproxy-1.15.1/CHANGES.txt --- mapproxy-1.12.0/CHANGES.txt 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/CHANGES.txt 2022-06-14 12:39:29.000000000 +0000 @@ -1,3 +1,76 @@ +Nightly +~~~~~~~~~~~~~~~~~ +1.15.1 2022-06-14 +~~~~~~~~~~~~~~~~~ + +Fixes: + +- Fixup release on PyPI + +1.15.0 2022-06-14 +~~~~~~~~~~~~~~~~~ + +Improvements: + +- WMS Dimension caching (#449) +- Add a mechanism to define plugins (#578) +- Support of non-EPSG SRS authorities (#572) +- Support for python 3.10 (#582) + +Fixes: + +- Several minor bugfixes + +1.14.0 2021-11-24 +~~~~~~~~~~~~~~~~~ + +Improvements: + +- Refresh while serving (#518). +- Enabled commandline option `skip uncached` (#515) +- Several dependencies updated +- Support for python 3.5 has been dropped because of its EOL, 3.9 has been added + +Fixes: + +- Several minor bugfixes +- Security fix to avoid potential web cache poisoning. + +1.13.2 2021-07-14 +~~~~~~~~~~~~~~~~~ + +Fixes: + +- Hotfix: Fixup demo service (#528). + +1.13.1 2021-07-13 +~~~~~~~~~~~~~~~~~ + +Improvements: + +- Support cookie management for HTTP sources. + +Fixes: + +- Security fix for local file disclosure (#526). + +1.13.0 2020-11-18 +~~~~~~~~~~~~~~~~~ + +Improvements: + +- Proj: Support for PROJ>=5 via pyproj. +- Services: New hide_exception_url option to hide source URLs. +- Tile sources: Support '@' in URL path (e.g. /0/0/0@2x.png) + +Fixes: + +- Various fixes for Python 3.8 compatibility. +- WMS: Always query WMS server in supported SRS. +- Fix warnings for tagged layer sources in layers. +- Demo: Fix capabilites "as HTML" when running behind a proxy + + 1.12.0 2019-08-30 ~~~~~~~~~~~~~~~~~ diff -Nru mapproxy-1.12.0/debian/changelog mapproxy-1.15.1/debian/changelog --- mapproxy-1.12.0/debian/changelog 2019-09-02 04:44:53.000000000 +0000 +++ mapproxy-1.15.1/debian/changelog 2022-09-03 11:00:00.000000000 +0000 @@ -1,3 +1,74 @@ +mapproxy (1.15.1-1~focal0) focal; urgency=medium + + * No change rebuild for Focal. + + -- Angelos Tzotsos Sat, 03 Sep 2022 14:00:00 +0300 + +mapproxy (1.15.1-1) unstable; urgency=medium + + * New upstream release. + + -- Bas Couwenberg Tue, 14 Jun 2022 15:18:31 +0200 + +mapproxy (1.15.0-1) unstable; urgency=medium + + * New upstream release. + * Update copyright file. + * Add python3-pyproj to build dependencies. + + -- Bas Couwenberg Tue, 14 Jun 2022 12:38:12 +0200 + +mapproxy (1.14.0-1) unstable; urgency=medium + + * New upstream release. + * Bump Standards-Version to 4.6.0, no changes. + * Bump debhelper compat to 12, changes: + - Drop --list-missing from dh_install + * Update lintian overrides. + + -- Bas Couwenberg Wed, 24 Nov 2021 15:06:43 +0100 + +mapproxy (1.13.2-1) unstable; urgency=medium + + * Move from experimental to unstable. + + -- Bas Couwenberg Sun, 15 Aug 2021 11:24:56 +0200 + +mapproxy (1.13.2-1~exp1) experimental; urgency=medium + + * New upstream release. + + -- Bas Couwenberg Wed, 14 Jul 2021 16:28:11 +0200 + +mapproxy (1.13.1-1~exp1) experimental; urgency=medium + + * New upstream release. + * Bump Standards-Version to 4.5.1, no changes. + * Update watch file for GitHub URL changes. + + -- Bas Couwenberg Tue, 13 Jul 2021 13:30:08 +0200 + +mapproxy (1.13.0-1) unstable; urgency=medium + + * New upstream release. + * Bump watch file version to 4. + * Drop patches applied/included upstream. + * Update lintian overrides. + + -- Bas Couwenberg Thu, 19 Nov 2020 17:15:50 +0100 + +mapproxy (1.12.0-2) unstable; urgency=medium + + * Update override for embedded-javascript-library. + * Bump Standards-Version to 4.5.0, no changes. + * Drop Name field from upstream metadata. + * Bump debhelper compat to 10. + * Add patches for compatibility with Python 3.8. + * Add patch to fix PIL version check. + * Add lintian override for manpage-without-executable. + + -- Bas Couwenberg Sun, 24 May 2020 15:00:06 +0200 + mapproxy (1.12.0-1) unstable; urgency=medium * Move from experimental to unstable. diff -Nru mapproxy-1.12.0/debian/compat mapproxy-1.15.1/debian/compat --- mapproxy-1.12.0/debian/compat 2018-07-20 17:53:56.000000000 +0000 +++ mapproxy-1.15.1/debian/compat 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -9 diff -Nru mapproxy-1.12.0/debian/control mapproxy-1.15.1/debian/control --- mapproxy-1.12.0/debian/control 2019-09-01 11:07:44.000000000 +0000 +++ mapproxy-1.15.1/debian/control 2022-06-14 10:49:56.000000000 +0000 @@ -3,13 +3,14 @@ Uploaders: Bas Couwenberg Section: python Priority: optional -Build-Depends: debhelper (>= 9), +Build-Depends: debhelper-compat (= 12), dh-python, libgdal-dev, python3-all, python3-lxml, python3-pil, python3-pkg-resources, + python3-pyproj, python3-pytest, python3-redis, python3-requests, @@ -23,7 +24,7 @@ docbook-xsl, docbook-xml, xsltproc -Standards-Version: 4.4.0 +Standards-Version: 4.6.0 Vcs-Browser: https://salsa.debian.org/debian-gis-team/mapproxy Vcs-Git: https://salsa.debian.org/debian-gis-team/mapproxy.git Homepage: http://mapproxy.org/ diff -Nru mapproxy-1.12.0/debian/copyright mapproxy-1.15.1/debian/copyright --- mapproxy-1.12.0/debian/copyright 2018-07-20 17:53:56.000000000 +0000 +++ mapproxy-1.15.1/debian/copyright 2022-06-14 10:37:29.000000000 +0000 @@ -5,6 +5,7 @@ Files: * Copyright: 2010-2017, Omniscale + 2022, Even Rouault License: Apache-2.0 Files: mapproxy/image/fonts/*.ttf diff -Nru mapproxy-1.12.0/debian/mapproxy-doc.lintian-overrides mapproxy-1.15.1/debian/mapproxy-doc.lintian-overrides --- mapproxy-1.12.0/debian/mapproxy-doc.lintian-overrides 2018-07-20 17:53:56.000000000 +0000 +++ mapproxy-1.15.1/debian/mapproxy-doc.lintian-overrides 2021-03-12 12:21:48.000000000 +0000 @@ -1,5 +1,5 @@ # libjs-twitter-bootstrap is not compatible -embedded-javascript-library usr/share/doc/mapproxy/html/_static/bootstrap-*/js/bootstrap.js please use libjs-twitter-bootstrap +embedded-javascript-library usr/share/doc/mapproxy/html/_static/bootstrap-*/js/bootstrap.js please use libjs-bootstrap font-in-non-font-package usr/share/doc/mapproxy/html/_static/boot*/fonts/* font-outside-font-dir usr/share/doc/mapproxy/html/_static/boots*/fonts/* diff -Nru mapproxy-1.12.0/debian/rules mapproxy-1.15.1/debian/rules --- mapproxy-1.12.0/debian/rules 2019-09-01 11:07:44.000000000 +0000 +++ mapproxy-1.15.1/debian/rules 2021-09-12 15:30:46.000000000 +0000 @@ -77,6 +77,6 @@ done override_dh_install: - dh_install --list-missing + dh_install $(RM) debian/*/usr/share/python*-mapproxy/test/schemas/*/*/ReadMe.txt diff -Nru mapproxy-1.12.0/debian/source/lintian-overrides mapproxy-1.15.1/debian/source/lintian-overrides --- mapproxy-1.12.0/debian/source/lintian-overrides 2018-07-31 19:42:19.000000000 +0000 +++ mapproxy-1.15.1/debian/source/lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# Not worth the effort -testsuite-autopkgtest-missing - diff -Nru mapproxy-1.12.0/debian/upstream/metadata mapproxy-1.15.1/debian/upstream/metadata --- mapproxy-1.12.0/debian/upstream/metadata 2018-07-20 17:53:56.000000000 +0000 +++ mapproxy-1.15.1/debian/upstream/metadata 2021-03-12 12:21:48.000000000 +0000 @@ -1,6 +1,5 @@ --- Bug-Database: https://github.com/mapproxy/mapproxy/issues Bug-Submit: https://github.com/mapproxy/mapproxy/issues/new -Name: MapProxy Repository: https://github.com/mapproxy/mapproxy.git Repository-Browse: https://github.com/mapproxy/mapproxy diff -Nru mapproxy-1.12.0/debian/watch mapproxy-1.15.1/debian/watch --- mapproxy-1.12.0/debian/watch 2019-09-01 11:07:44.000000000 +0000 +++ mapproxy-1.15.1/debian/watch 2021-10-23 17:12:10.000000000 +0000 @@ -1,7 +1,7 @@ -version=3 +version=4 opts=\ dversionmangle=s/\+(debian|dfsg|ds|deb)\d*$//,\ uversionmangle=s/(\d)[_\.\-\+]?((RC|rc|pre|dev|beta|alpha)\d*)$/$1~$2/;s/RC/rc/,\ filenamemangle=s/(?:.*?\/)?(?:rel|v|mapproxy)?[\-\_]?(\d\S+)\.(tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))/mapproxy-$1.$2/ \ -https://github.com/mapproxy/mapproxy/releases \ -(?:.*?/archive/)?(?:rel|v|mapproxy)?[\-\_]?(\d\S+)\.(?:tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) +https://github.com/mapproxy/mapproxy/tags \ +(?:.*?/archive/(?:.*?/)?)?(?:rel|v|mapproxy)?[\-\_]?(\d\S+)\.(?:tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) diff -Nru mapproxy-1.12.0/DEVELOPMENT.md mapproxy-1.15.1/DEVELOPMENT.md --- mapproxy-1.12.0/DEVELOPMENT.md 1970-01-01 00:00:00.000000000 +0000 +++ mapproxy-1.15.1/DEVELOPMENT.md 2022-06-14 12:39:29.000000000 +0000 @@ -0,0 +1,83 @@ +Dev Setup +========= + +* Create parent directory for source, applications and the virtual env +* Clone source into directory mapproxy: `git clone ` +* Install dependencies: https://mapproxy.org/docs/latest/install.html#install-dependencies +* Create virtualenv: `python3.6 -m venv ./venv` +* Activate virtualenv: `source venv/bin/activate` +* Install mapproxy: `pip install -e mapproxy/` +* Install dev dependencies: `pip install -r mapproxy/requirements-tests.txt` +* Run tests: + * `cd mapproxy` + * `pytest mapproxy` + * Run single test: `pytest mapproxy/test/unit/test_grid.py -v` +* Create an application: `mapproxy-util create -t base-config apps/base` + +* Start a dev server in debug mode: `mapproxy-util serve-develop apps/base/mapproxy.yaml --debug` + + +Coding Style +------------ + +PEP8: https://www.python.org/dev/peps/pep-0008/ + + +Debugging +--------- + + - With PyCharm: + * Attach to dev server with https://www.jetbrains.com/help/pycharm/attaching-to-local-process.html + + - With ipython: + * `pip install ipython ipdb` + + - With Visual Studio Code: + * After creating a virtual env and mapproxy configuration: + * Create a `launch.json` file in the project-root/.vscode directory with the following content: + ``` + { + "version": "0.2.0", + "configurations": [ + { + "name": "Debug local mapproxy", + "type": "python", + "request": "launch", + "program": ".venv/bin/mapproxy-util", + "args": ["serve-develop", "-b", ":1234", "config/mapproxy.yaml"], + "console": "integratedTerminal", + "autoReload": { + "enable": true + } + } + ] + } + ``` + + * Then start debugging by hitting `F5`. + + +Some more details in the documentation +-------------------------------------- + +See https://mapproxy.org/docs/latest/development.html + + +Some incomplete notes about the structure of the software +--------------------------------------------------------- + +A mapproxy app decides on the request-URL which handler it starts. There exist different handlers for WMS, WMTS. + +Incoming http requests are transformed into own request objects (for example `WMSRequest`). + +The class `TileManager` decides if tiles are served from cache or from a source. + +All caches need to implement the interface `TileCacheBase`. + +The code in `config/` builds mapproxy out of a configuration. `config/spec.py` validates the config. + +The sources live in `source/` which in turn use low-level functions from `client/` to request the data. + +The file `layer.py` merges/clips/transforms tiles. + +The whole of MapProxy is stateless apart from the chache which uses locks on file system level. diff -Nru mapproxy-1.12.0/doc/caching_layer_dimensions.rst mapproxy-1.15.1/doc/caching_layer_dimensions.rst --- mapproxy-1.12.0/doc/caching_layer_dimensions.rst 1970-01-01 00:00:00.000000000 +0000 +++ mapproxy-1.15.1/doc/caching_layer_dimensions.rst 2022-06-14 12:39:29.000000000 +0000 @@ -0,0 +1,193 @@ +Caching Dimensions +################## + +WMS servers have the capability to offer layers with 1..n dimensions. A WMS +layer may provide dimensions such as time, elevation or other axes to be able +to visualize maps of multidimensional data. The `OGC OGC Best Practice for +using Web Map Services (WMS) with Time-Dependent or Elevation-Dependent Data`_ +is an example typically implemented by WMS providers of multidimensional +data. + +MapProxy supports caching layers with dimensions and making them available +through standard WMS mechanisms. + +Configuration +============= + +To enable dimension caching, the underlying WMS layer must have dimensions +defined and available. The following properties are required in configuration +to successfully cache any layer with dimensions: + +- ``forward_req_params``: list of 1..n query parameter names to send when + caching the layer, if request by the MapProxy client. This property is + specified in a source +- ``dimensions``: an object of 1..n keys of dimension definitions. The + dimension names must match with those specified in ``forward_req_params`` + for a given layer/source combination. Each dimension object requires + a default dimension (in cases where not specified by the client) as well + as a list of dimension values. Dimension lists can be intervals, or a + a compound value of ``start-date-time/end-date-time/duration`` + +An example is shown below: + +.. code-block:: yaml + + sources: + test: + type: wms + req: + url: https://example.org/wms + layers: global-air-temperature-15km + forward_req_params: + - time + - dim_reference_time + +.. code-block:: yaml + + layers: + - name: global-air-temperature-15km + title: Global Air Temperature (°C) - 15km + sources: [test_cache] + dimensions: + time: + values: + - "2020-09-22T11:20:00Z/2020-09-22T14:20:00Z/PT2H" + default: "2020-09-22T14:20:00Z" + dim_reference_time: + values: + - "2020-09-22T11:20:00Z/2020-09-22T14:20:00Z/PT2H" + default: "2020-09-22T14:20:00Z" + + +ISO 8601 Interval +================= + +For example: + ``2020-03-25T12:00:00Z/2020-03-27T00:00:00Z/PT12H30M`` + +A time interval is the intervening time between two time points. The amount of intervening time is expressed by a duration (as described in the previous section). The two time points (start and end) are expressed by either a combined date and time representation or just a date representation. + +There are four ways to express a time interval: + 1. Start and end, such as "2007-03-01T13:00:00Z/2008-05-11T15:30:00Z" + 2. Start and duration, such as "2007-03-01T13:00:00Z/P1Y2M10DT2H30M" + 3. Duration and end, such as "P1Y2M10DT2H30M/2008-05-11T15:30:00Z" + 4. Duration only, such as "P1Y2M10DT2H30M", with additional context information + + P is the duration designator (for period) placed at the start of the duration representation. + - Y is the year designator that follows the value for the number of years. + - M is the month designator that follows the value for the number of months. + - W is the week designator that follows the value for the number of weeks. + - D is the day designator that follows the value for the number of days. + T is the time designator that precedes the time components of the representation. + - H is the hour designator that follows the value for the number of hours. + - M is the minute designator that follows the value for the number of minutes. + - S is the second designator that follows the value for the number of seconds. + +WMS Capabilities +================ + +The following is an example of the resulting WMS capabilities in MapProxy: + +.. code-block:: xml + + + + + WMS + Test Dimension + + + none + none + 4000 + 4000 + + + + + text/xml + + + + + + + + + + image/png + image/jpeg + image/gif + image/GeoTIFF + image/tiff + + + + + + + + + + text/plain + text/html + text/xml + + + + + + + + + + + XML + INIMAGE + BLANK + + + global-air-temperature-15km + Global Air Temperature (°C) - 15km + EPSG:4326 + EPSG:3857 + + -180 + 180 + -89.999999 + 89.999999 + + + + + 2020-09-22T11:20:00Z,2020-09-22T13:20:00Z,2020-09-22T15:20:00Z + 2020-09-22T11:20:00Z,2020-09-22T13:20:00Z,2020-09-22T15:20:00Z + + + + + +Known limitations +================= + +- some WMS time-enabled servers provide dimension support for real-time + data with ongoing updates to retention time. In this case, a given + WMS layer's temporal extent may be updated a few hours after, for + example. It is up to the MapProxy configuration to manage dimensions/ + extents accordingly. This can be done with custom scripts + to run WMS ``GetCapabilities`` requests and write the updated temporal + dimensions into the MapProxy configuration. An example of such a tool + is `geomet-mapproxy`_ +- caches of layers with dimensions need to be cleaned/deleted by the MapProxy + administrator. This can typically be done via cron/schedule accordingly +- dimemsion support is only implemented in the default file cache backend + at this time + + +Tests +===== + +All tests related to caching layer dimensions: ``mapproxy/test/system/test_dimensions.py`` + +.. _`OGC OGC Best Practice for using Web Map Services (WMS) with Time-Dependent or Elevation-Dependent Data`: https://portal.ogc.org/files/?artifact_id=56394 +.. _`geomet-mapproxy`: https://github.com/ECCC-MSC/geomet-mapproxy diff -Nru mapproxy-1.12.0/doc/configuration_examples.rst mapproxy-1.15.1/doc/configuration_examples.rst --- mapproxy-1.12.0/doc/configuration_examples.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/configuration_examples.rst 2022-06-14 12:39:29.000000000 +0000 @@ -667,7 +667,7 @@ - 3000 default: 0 -You can know access this layer with the elevation and time dimensions via the WMTS KVP service. +You can now access this layer with the elevation and time dimensions via the WMTS KVP service. The RESTful service requires a custom URL template that contains the dimensions. For example:: services: @@ -708,6 +708,8 @@ url: https://username:mypassword@example.org/service? layers: securelayer +.. note:: If the source requires session handling through cookies, have a look at the ``manage_cookies`` configuration option. + .. _http_proxy: Access sources through HTTP proxy @@ -734,11 +736,28 @@ Add a username and password to the URL if your HTTP proxy requires authentication. For example ``http://username:password@example.com:3128``. +.. note:: If the source requires session handling through cookies, have a look at the ``manage_cookies`` configuration option. + You can use the ``no_proxy`` environment variable if you need to bypass the proxy for some hosts:: $ export no_proxy="localhost,127.0.0.1,196.168.1.99" +Cookie Management +================= + +MapProxy can handle server cookies of HTTP sources, like browsers do. That is, MapProxy accepts cookies and passes them back +on subsequent calls. This is useful for sources that use cookie for session management or rate-limiting for example:: + + sources: + wms_with_session_management: + type: wms + http: + manage_cookies: True + req: + url: http://example.org/service? + layers: layer0 + .. _paster_urlmap: Serve multiple MapProxy instances diff -Nru mapproxy-1.12.0/doc/configuration.rst mapproxy-1.15.1/doc/configuration.rst --- mapproxy-1.12.0/doc/configuration.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/configuration.rst 2022-06-14 12:39:29.000000000 +0000 @@ -513,6 +513,43 @@ Tiles created by the ``upscale_tiles`` or ``downscale_tiles`` option are only stored in the cache if this option is set to true. +``refresh_before`` +""""""""""""""""""" + +Here you can force MapProxy to refresh tiles from the source while serving if they are found to be expired. +The validity conditions are the same as for seeding: + +Explanation:: + + # absolute as ISO time + refresh_before: + time: 2010-10-21T12:35:00 + + # relative from the time of the tile request + refresh_before: + weeks: 1 + days: 7 + hours: 4 + minutes: 15 + + # modification time of a given file + refresh_before: + mtime: path/to/file + +Example +~~~~~~~~ + +:: + + caches: + osm_cache: + grids: ['osm_grid'] + sources: [OSM] + disable_storage: false + refresh_before: + days: 1 + + ``disable_storage`` """""""""""""""""""" @@ -908,10 +945,12 @@ .. versionadded:: 1.12.0 ``proj_data_dir`` - MapProxy uses Proj4 for all coordinate transformations. If you need custom projections - or need to tweak existing definitions (e.g. add towgs parameter set) you can point - MapProxy to your own set of proj4 init files. The path should contain an ``epsg`` file - with the EPSG definitions. + + MapProxy uses PROJ for all coordinate transformations. If you need custom projections + or need to tweak existing definitions. You can point MapProxy to your own set of PROJ data files. + + This path should contain an ``epsg`` file with the EPSG definitions for installations with PROJ version 4. + PROJ>=5 uses a different configuration format. Please refer to the PROJ documentation. The configured path can be absolute or relative to the mapproxy.yaml. @@ -943,6 +982,10 @@ If you need to override one of the default values, then you need to define both axis order options, even if one is empty. + .. versionchanged:: 1.13.0 + MapProxy can now determine the correct axis order for all coordinate systems when using pyproj>=2. The axis_order_ne/axis_order_en are ignored in this case. + + .. _http_ssl: ``http`` @@ -1018,6 +1061,24 @@ Sets the ``Access-control-allow-origin`` header to HTTP responses for `Cross-origin resource sharing `_. This header is required for WebGL or Canvas web clients. Defaults to `*`. Leave empty to disable the header. This option is only available in `globals`. +``manage_cookies`` +^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 1.14.0 + +Enables MapProxy cookie management for HTTP sources. When enabled MapProxy will accept and store server cookies. Accepted cookies will be passed +back to the source on subsequent requests. Usefull for sources which require to maintain an HTTP session to work efficiently, maybe in combination +with basic authentication. Depending on your deployment MapProxy will still start multiple sessions (e.g. one per MapProxy process). +Cookie handling is based on Python `CookieJar `_. Disabled by default. + +``hide_error_details`` +^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 1.13.0 + +When enabled, MapProxy will only report generic error messages to the client in case of any errors while fetching source services. +The full error message might contain confidential information like internal URLs. You will find the full error message in the logs, regardless of this option. The option is enabled by default, i.e. the details are hidden. + ``tiles`` """""""""" diff -Nru mapproxy-1.12.0/doc/conf.py mapproxy-1.15.1/doc/conf.py --- mapproxy-1.12.0/doc/conf.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/conf.py 2022-06-14 12:39:29.000000000 +0000 @@ -49,9 +49,9 @@ # built documents. # # The short X.Y version. -version = '1.12' +version = '1.15' # The full version, including alpha/beta/rc tags. -release = '1.12.0' +release = '1.15.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff -Nru mapproxy-1.12.0/doc/deployment.rst mapproxy-1.15.1/doc/deployment.rst --- mapproxy-1.12.0/doc/deployment.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/deployment.rst 2022-06-14 12:39:29.000000000 +0000 @@ -95,9 +95,9 @@ -``mod_wsgi`` has a lot of options for more fine tuning. ``WSGIPythonHome`` or ``WSGIPythonPath`` lets you configure your ``virtualenv`` and ``WSGIDaemonProcess``/``WSGIProcessGroup`` allows you to start multiple processes. See the `mod_wsgi configuration directives documentation `_. Using Mapnik also requires the ``WSGIApplicationGroup`` option. +``mod_wsgi`` has a lot of options for more fine tuning. ``WSGIPythonHome`` or ``WSGIPythonPath`` lets you configure your ``virtualenv`` and ``WSGIDaemonProcess``/``WSGIProcessGroup`` allows you to start multiple processes. See the `mod_wsgi configuration directives documentation `_. Using Mapnik also requires the ``WSGIApplicationGroup`` option. -.. note:: On Windows only the ``WSGIPythonPath`` option is supported. Linux/Unix supports ``WSGIPythonPath`` and ``WSGIPythonHome``. See also the `mod_wsgi documentation for virtualenv `_ for detailed information when using multiple virtualenvs. +.. note:: On Windows only the ``WSGIPythonPath`` option is supported. Linux/Unix supports ``WSGIPythonPath`` and ``WSGIPythonHome``. See also the `mod_wsgi documentation for virtualenv `_ for detailed information when using multiple virtualenvs. A more complete configuration might look like:: @@ -121,7 +121,7 @@ .. _mod_wsgi: http://www.modwsgi.org/ -.. _mod_wsgi installation: http://code.google.com/p/modwsgi/wiki/InstallationInstructions +.. _mod_wsgi installation: https://modwsgi.readthedocs.io/en/latest/installation.html Behind HTTP server or proxy --------------------------- diff -Nru mapproxy-1.12.0/doc/index.rst mapproxy-1.15.1/doc/index.rst --- mapproxy-1.12.0/doc/index.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/index.rst 2022-06-14 12:39:29.000000000 +0000 @@ -23,6 +23,7 @@ auth decorate_img development + plugins mapproxy_2 .. todolist:: diff -Nru mapproxy-1.12.0/doc/install.rst mapproxy-1.15.1/doc/install.rst --- mapproxy-1.12.0/doc/install.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/install.rst 2022-06-14 12:39:29.000000000 +0000 @@ -33,7 +33,7 @@ source mapproxy/bin/activate -This will change the ``PATH`` for you `current` session. +This will change the ``PATH`` for your `current` session. Install Dependencies @@ -48,7 +48,7 @@ On a Debian or Ubuntu system, you need to install the following packages:: - sudo apt-get install python-pil python-yaml libproj12 + sudo apt-get install python-pil python-yaml python-proj To get all optional packages:: @@ -59,15 +59,22 @@ Dependency details ^^^^^^^^^^^^^^^^^^ -libproj -~~~~~~~ -MapProxy uses the Proj4 C Library for all coordinate transformation tasks. It is included in most distributions as ``libproj`` or ``libprojXX`` where ``XX`` is a number. +pyproj or libproj +~~~~~~~~~~~~~~~~~ + +MapProxy uses the PROJ C library for all coordinate transformation tasks. MapProxy can directly use the C library or via the pyproj Python package. +The internal API of PROJ was updated with PROJ >=5. The old PROJ 4 API is now deprecated and will be removed from future PROJ releases. MapProxy only supports the new API via pyproj and it is therefore recommended to use a recent pyproj version. + + +.. versionchanged:: 1.13 + Support for new PROJ API via pyproj. + .. _dependencies_pil: Pillow ~~~~~~ -Pillow, the successor of the Python Image Library (PIL), is used for the image processing and it is included in most distributions as ``python-pil`` or ``python-imaging``. Please make sure that you have Pillow installed as MapProxy is no longer compatible with the original PIL. The version of ``python-imaging`` should be >=2. +Pillow, the successor of the Python Image Library (PIL), is used for the image processing and it is included in most distributions as ``python-pil`` or ``python-imaging``. Please make sure that you have Pillow installed as MapProxy is no longer compatible with the original PIL. The version of ``python-imaging`` should be >=3.1. You can install a new version of Pillow from source with:: @@ -101,7 +108,7 @@ Install MapProxy ---------------- -Your virtual environment should contains `pip`_, a tool to install Python packages. +Your virtual environment should contain `pip`_, a tool to install Python packages. To install you need to call:: @@ -179,6 +186,6 @@ Changes ^^^^^^^ -New releases of MapProxy are backwards compatible with older configuration files. MapProxy will issue warnings on startup if a behavior will change in the next releases. You are advised to upgrade in single release steps (e.g. 1.9.0 to 1.10.0 to 1.11.0) and to check the output of ``mapproxy-util serve-develop`` for any warnings. You should also refer to the Changes Log of each release to see if there is anything to pay attention for. +New releases of MapProxy are backwards compatible with older configuration files. MapProxy will issue warnings on start-up if a behavior will change in the next releases. You are advised to upgrade in single release steps (e.g. 1.9.0 to 1.10.0 to 1.11.0) and to check the output of ``mapproxy-util serve-develop`` for any warnings. You should also refer to the Changes Log of each release to see if there is anything to pay attention for. -If you upgrade from 0.8, please read the `old mirgation documentation `_. +If you upgrade from 0.8, please read the `old migration documentation `_. diff -Nru mapproxy-1.12.0/doc/install_windows.rst mapproxy-1.15.1/doc/install_windows.rst --- mapproxy-1.12.0/doc/install_windows.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/install_windows.rst 2022-06-14 12:39:29.000000000 +0000 @@ -52,12 +52,13 @@ PyProj ~~~~~~ -Since libproj4 is generally not available on a Windows system, you will also need to install the Python package ``pyproj``. -You need to manually download the ``pyproj`` package for your system. See below for *Platform dependent packages*. +Since PROJ is generally not available on a Windows system, you will also need to install the Python package ``pyproj``. :: - pip install path\to\pyproj-xxx.whl + pip install pyproj + +See *Platform dependent packages* below if this installation fails as Windows packages might not be available for pyproj. Shapely and GEOS *(optional)* diff -Nru mapproxy-1.12.0/doc/mapproxy_util.rst mapproxy-1.15.1/doc/mapproxy_util.rst --- mapproxy-1.12.0/doc/mapproxy_util.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/mapproxy_util.rst 2022-06-14 12:39:29.000000000 +0000 @@ -73,7 +73,7 @@ log-ini: - Creates an example logging configuration. You need to pass the target filename to the command. + Creates an example logging configuration. You need to pass the target filename to the command (i.e. `my-app/log.ini`). wsgi-app: Creates an example server script for the given MapProxy configuration (:option:`--f/--mapproxy-conf`) . You need to pass the target filename to the command. @@ -105,6 +105,10 @@ The server address where the HTTP server should listen for incomming connections. Can be a port (``:8080``), a host (``localhost``) or both (``localhost:8081``). The default is ``localhost:8080``. You need to use ``0.0.0.0`` to be able to connect to the server from external clients. +.. cmdoption:: --debug + + The server outputs debug logging information to the console. + Example ------- diff -Nru mapproxy-1.12.0/doc/plugins.rst mapproxy-1.15.1/doc/plugins.rst --- mapproxy-1.12.0/doc/plugins.rst 1970-01-01 00:00:00.000000000 +0000 +++ mapproxy-1.15.1/doc/plugins.rst 2022-06-14 12:39:29.000000000 +0000 @@ -0,0 +1,303 @@ +Writing plugins +=============== + +Since MapProxy 1.15, it is possible to write plugins for MapProxy that can +add new sources, services or commands. This requires Python >= 3.7 + +Example +------- + +The mapproxy_hips plugin at https://github.com/rouault/mapproxy_hips is an +example of a plugin, which adds a new source, service and customizes the demo +service, demonstrating all the below points. + +How to add a plugin ? +--------------------- + +A plugin should be written as a Python package whose setuptools ``setup()`` +method has a ``entry_points`` keyword with a group ``mapproxy`` pointing to +a module with a ``plugin_entrypoint`` method. + +.. code-block:: python + + entry_points={"mapproxy": ["hips = mapproxy_hips.pluginmodule"]}, + + +In this example, the ``mapproxy_hips/pluginmodule.py`` file should have +a ``plugin_entrypoint`` method taking no argument and returning nothing. + +.. code-block:: python + + def plugin_entrypoint(): + # call different registration methods, like register_service_configuration(), + # register_source_configuration() + pass + +That method is in charge of registering the various registration methods +detailed hereafter. + +Plugins will often by dependent on MapProxy internal classes. It is their +responsibility to check the MapProxy version, in case the MapProxy internal +API or behavior would change and make them incompatible. + +Adding a new service +-------------------- + +The ``mapproxy.config.loader`` module has a ``register_service_configuration()`` +method to register a new service and specify the allowed keywords for it in +the YAML configuration file. + +.. code-block:: python + + def register_service_configuration(service_name, service_creator, + yaml_spec_service_name = None, yaml_spec_service_def = None): + """ Method used by plugins to register a new service. + + :param config_name: Name of the service + :type config_name: str + :param service_creator: Creator method of the service + :type service_creator: method of type (serviceConfiguration: ServiceConfiguration, conf: dict) -> Server + :param yaml_spec_service_name: Name of the service in the YAML configuration file + :type yaml_spec_service_name: str + :param yaml_spec_service_def: Definition of the service in the YAML configuration file + :type yaml_spec_service_def: dict + """ + + +This can for example by used like the following snippet: + +.. code-block:: python + + from mapproxy.config.loader import register_service_configuration + from mapproxy.service.base import Server + + class MyExtraServiceServer(Server): + # Look at classes at https://github.com/mapproxy/mapproxy/tree/master/mapproxy/service + # for a real implementation + names = ('my_extra_service',) + def __init__(self): + pass + + def my_extra_service_method(serviceConfiguration, conf): + return MyExtraServiceServer() + + register_service_configuration('my_extra_service', my_extra_service_method, + 'my_extra_service', {'foo': str()}) + + +This allows the following declaration in the YAML mapproxy configuration file: + +.. code-block:: yaml + + services: + my_extra_service: + foo: bar + +A real-world implementation can be found at https://github.com/rouault/mapproxy_hips/blob/master/mapproxy_hips/service/hips.py + + +Customizing layer metadata in YAML configuration file +----------------------------------------------------- + +When implementing a new service, it might be useful to add per-layer metadata +for it. The YAML validator needs to be updated to recognize the new keywords. +The ``add_subcategory_to_layer_md()`` method of the ``mapproxy.config.spec`` module +can be used to do that. + +.. code-block:: python + + + def add_subcategory_to_layer_md(category_name, category_def): + """ Add a new category to wms_130_layer_md. + Used by plugins + """ + +This can for example be used like in the following snippet: + +.. code-block:: python + + from mapproxy.config.spec import add_subcategory_to_layer_md + + # Add a 'hips' subcategory to layer spec to be able to define hips service + # specific layer metadata + add_subcategory_to_layer_md('hips', anything()) + + +Adding a new source +------------------- + +The ``mapproxy.config.loader`` module has a ``register_source_configuration()`` +method to register a new source and specify the allowed keywords for it in +the YAML configuration file. + +.. code-block:: python + + + def register_source_configuration(config_name, config_class, + yaml_spec_source_name = None, yaml_spec_source_def = None): + """ Method used by plugins to register a new source configuration. + + :param config_name: Name of the source configuration + :type config_name: str + :param config_class: Class of the source configuration + :type config_name: SourceConfiguration + :param yaml_spec_source_name: Name of the source in the YAML configuration file + :type yaml_spec_source_name: str + :param yaml_spec_source_def: Definition of the source in the YAML configuration file + :type yaml_spec_source_def: dict + """ + + +This can for example by used like the following snippet: + +.. code-block:: python + + from mapproxy.config.loader import register_source_configuration + from mapproxy.config.loader import SourceConfiguration + + class my_source_configuration(SourceConfiguration): + source_type = ('my_extra_source',) + + def source(self, params=None): + # Look at classes at https://github.com/mapproxy/mapproxy/tree/master/mapproxy/source + # for a real implementation + class MySource(object): + def __init__(self): + self.extent = None + return MySource() + + register_source_configuration('my_extra_source', my_source_configuration, + 'my_extra_source', {'foo': str()}) + + +This allows the following declaration in the YAML mapproxy configuration file: + +.. code-block:: yaml + + sources: + some_source_name: + type: my_extra_source + foo: bar + +A real-world implementation can be found at https://github.com/rouault/mapproxy_hips/blob/master/mapproxy_hips/source/hips.py + +Customizing the demo service +---------------------------- + +The :ref:`demo_service_label` can be customized in two ways: + +- Customizing the output of the ``/demo`` HTML output, typically by adding entries + for new services. This is done with the ``register_extra_demo_substitution_handler()`` + method of the ``mapproxy.service.demo`` module. + + .. code-block:: python + + def register_extra_demo_substitution_handler(handler): + """ Method used by plugins to register a new handler for doing substitutions + to the HTML template used by the demo service. + The handler passed to this method is invoked by the DemoServer._render_template() + method. The handler may modify the passed substitutions dictionary + argument. Keys of particular interest are 'extra_services_html_beginning' + and 'extra_services_html_end' to add HTML content before/after built-in + services. + + :param handler: New handler for incoming requests + :type handler: function that takes 3 arguments(DemoServer instance, req and a substitutions dictionary argument). + """ + +- Handling new request paths under the ``/demo/`` hierarchy, typically to implement a new + service. This is done with the ``register_extra_demo_server_handler()`` + method of the ``mapproxy.service.demo`` module. + + .. code-block:: python + + def register_extra_demo_server_handler(handler): + """ Method used by plugins to register a new handler for the demo service. + The handler passed to this method is invoked by the DemoServer.handle() + method when receiving an incoming request. This enables handlers to + process it, in case it is relevant to it. + + :param handler: New handler for incoming requests + :type handler: function that takes 2 arguments (DemoServer instance and req) and + returns a string with HTML content or None + """ + +This can for example be used like in the following snippet: + +.. code-block:: python + + from mapproxy.service.demo import register_extra_demo_server_handler, register_extra_demo_substitution_handler + + def demo_server_handler(demo_server, req): + if 'my_service' in req.args: + return 'my_return' + return None + + def demo_substitution_handler(demo_server, req, substitutions): + html = '

My extra service

' + html += 'My service' + substitutions['extra_services_html_beginning'] += html + + register_extra_demo_server_handler(demo_server_handler) + register_extra_demo_substitution_handler(demo_substitution_handler) + + +A real-world example can be found at https://github.com/rouault/mapproxy_hips/blob/master/mapproxy_hips/service/demo_extra.py + + +Adding new commands to mapproxy-util +------------------------------------ + +New commands can be added to :ref:`mapproxy-util` by using the +``register_command()`` method of the ``mapproxy.script.util`` module + +.. code-block:: python + + def register_command(command_name, command_spec): + """ Method used by plugins to register a command. + + :param command_name: Name of the command + :type command_name: str + :param command_spec: Definition of the command. Dictionary with a 'func' and 'help' member + :type command_spec: dict + """ + +This can for example be used like in the following snippet: + +.. code-block:: python + + import optparse + from mapproxy.script.util import register_command + + def my_command(args=None): + parser = optparse.OptionParser("%prog my_command [options] -f mapproxy_conf -l layer") + parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", + help="MapProxy configuration.") + parser.add_option("-l", "--layer", dest="layer", help="Layer") + + if args: + args = args[1:] # remove script name + + (options, args) = parser.parse_args(args) + if not options.mapproxy_conf or not options.layer: + parser.print_help() + sys.exit(1) + + # Do something + + + register_command('my_command', { + 'func': my_command, + 'help': 'Do something.' + }) + + +A real-world example can be found at +https://github.com/rouault/mapproxy_hips/blob/master/mapproxy_hips/script/hipsallsky.py +` + +Credits +------- + +The development of the plugin mechanism has been funded by +Centre National d'Etudes Spatiales (CNES): https://cnes.fr diff -Nru mapproxy-1.12.0/doc/seed.rst mapproxy-1.15.1/doc/seed.rst --- mapproxy-1.12.0/doc/seed.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/seed.rst 2022-06-14 12:39:29.000000000 +0000 @@ -69,6 +69,16 @@ This will simulate the seed/cleanup process without requesting, creating or removing any tiles. +.. option:: -l N, --skip-geoms-for-last-levels N + + This will skip checking the intersections between tiles and seed geometries on the last N levels. + +.. option:: --skip-uncached + + This will seed only tiles which are already in the cache. This option is interesting in combination with + the configuration entry `refresh_before`_ to refresh regularly the existing tiles and to avoid loading + all available tiles. + .. option:: --summary Print a summary of all seeding and cleanup tasks and exit. diff -Nru mapproxy-1.12.0/doc/sources.rst mapproxy-1.15.1/doc/sources.rst --- mapproxy-1.12.0/doc/sources.rst 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/doc/sources.rst 2022-06-14 12:39:29.000000000 +0000 @@ -184,6 +184,7 @@ - ``client_timeout`` - ``ssl_ca_certs`` - ``ssl_no_cert_checks`` +- ``manage_cookies`` See :ref:`HTTP Options ` for detailed documentation. @@ -241,6 +242,12 @@ You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency. +``authorize_stale`` + + Set this to ``True`` if MapProxy should serve in priority stale tiles present in cache. If the specified source error occurs, MapProxy will serve a stale tile which is still in cache instead of the error reponse, even if the tile in cache should be refreshed according to refresh_before date. Otherwise (``False``) MapProxy will serve the unicolor error response defined by the error handler if the source is faulty and the tile is not in cache, or is stale. + +You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency. + :: my_tile_source: @@ -249,6 +256,10 @@ url: http://localhost:8080/service? layers: base on_error: + 404: + response: 'transparent' + cache: False + authorize_stale: True 500: response: '#ede9e3' cache: False @@ -432,6 +443,7 @@ - ``client_timeout`` - ``ssl_ca_certs`` - ``ssl_no_cert_checks`` +- ``manage_cookies`` See :ref:`HTTP Options ` for detailed documentation. diff -Nru mapproxy-1.12.0/.github/pull_request_template.md mapproxy-1.15.1/.github/pull_request_template.md --- mapproxy-1.12.0/.github/pull_request_template.md 1970-01-01 00:00:00.000000000 +0000 +++ mapproxy-1.15.1/.github/pull_request_template.md 2022-06-14 12:39:29.000000000 +0000 @@ -0,0 +1,20 @@ + diff -Nru mapproxy-1.12.0/.github/workflows/ghpages.yml mapproxy-1.15.1/.github/workflows/ghpages.yml --- mapproxy-1.12.0/.github/workflows/ghpages.yml 1970-01-01 00:00:00.000000000 +0000 +++ mapproxy-1.15.1/.github/workflows/ghpages.yml 2022-06-14 12:39:29.000000000 +0000 @@ -0,0 +1,31 @@ +name: Build and deploy documentation on github pages + +on: + workflow_dispatch: + push: + tags: + - '*' + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Install dependencies ⏬ + run: pip install sphinx sphinx-bootstrap-theme + + - name: Run documentation build and publish 🏗️ + run: | + git config --global user.name 'ghpages' + git config --global user.email 'ghpages@users.noreply.github.com' + git fetch --all + git checkout gh-pages + git pull origin gh-pages + git rebase origin/master + sphinx-build doc/ docs + git add docs + git commit -m "Automated documentation" + git push -f origin gh-pages diff -Nru mapproxy-1.12.0/.github/workflows/test.yml mapproxy-1.15.1/.github/workflows/test.yml --- mapproxy-1.12.0/.github/workflows/test.yml 1970-01-01 00:00:00.000000000 +0000 +++ mapproxy-1.15.1/.github/workflows/test.yml 2022-06-14 12:39:29.000000000 +0000 @@ -0,0 +1,75 @@ +name: Running mapproxy tests + +on: + workflow_dispatch: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test: + runs-on: ubuntu-20.04 + services: + redis-server: + image: redis + ports: + - 6379:6379 + couchdb: + image: couchdb:2 + ports: + - 5984:5984 + riak: + image: basho/riak-kv + ports: + - 8087:8087 + - 8098:8098 + + strategy: + matrix: + python-version: [2.7, 3.6, 3.7, 3.8, 3.9, "3.10"] + + env: + MAPPROXY_TEST_COUCHDB: 'http://localhost:5984' + MAPPROXY_TEST_REDIS: '127.0.0.1:6379' + MAPPROXY_TEST_RIAK_HTTP: 'http://localhost:8098' + MAPPROXY_TEST_RIAK_PBC: 'pbc://localhost:8087' + # do not load /etc/boto.cfg with Python 3 incompatible plugin + # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882 + BOTO_CONFIG: '/doesnotexist' + + steps: + - name: Install packages + run: | + sudo apt update + sudo apt install proj-bin libgeos-dev libgdal-dev libxslt1-dev libxml2-dev build-essential python-dev libjpeg-dev zlib1g-dev libfreetype6-dev protobuf-compiler libprotoc-dev -y + + - name: Checkout sources + uses: actions/checkout@v2 + + - name: Use python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache python deps 💾 + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.OS }}-python-${{ hashFiles('**/requirements-tests.txt') }} + restore-keys: | + ${{ runner.OS }}-python- + ${{ runner.OS }}- + + - name: Install dependencies ⏬ + run: | + pip install -r requirements-tests.txt + if [[ ${{ matrix.python-version }} = 2.7 || ${{ matrix.python-version }} = 3.8 ]]; then pip install -U "Pillow!=8.3.0,!=8.3.1"; fi + pip freeze + + - name: Run tests 🏗️ + run: | + export LD_PRELOAD=/lib/x86_64-linux-gnu/libstdc++.so.6:$LD_PRELOAD + pytest mapproxy diff -Nru mapproxy-1.12.0/mapproxy/cache/base.py mapproxy-1.15.1/mapproxy/cache/base.py --- mapproxy-1.12.0/mapproxy/cache/base.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/base.py 2022-06-14 12:39:29.000000000 +0000 @@ -42,40 +42,40 @@ supports_timestamp = True - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): raise NotImplementedError() - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): all_succeed = True for tile in tiles: - if not self.load_tile(tile, with_metadata=with_metadata): + if not self.load_tile(tile, with_metadata=with_metadata, dimensions=dimensions): all_succeed = False return all_succeed - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): raise NotImplementedError() - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions=None): all_succeed = True for tile in tiles: - if not self.store_tile(tile): + if not self.store_tile(tile, dimensions=dimensions): all_succeed = False return all_succeed - def remove_tile(self, tile): + def remove_tile(self, tile, dimensions=None): raise NotImplementedError() - def remove_tiles(self, tiles): + def remove_tiles(self, tiles, dimensions=None): for tile in tiles: - self.remove_tile(tile) + self.remove_tile(tile, dimensions=dimensions) - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): """ Return ``True`` if the tile is cached. """ raise NotImplementedError() - def load_tile_metadata(self, tile): + def load_tile_metadata(self, tile, dimensions=None): """ Fill the metadata attributes of `tile`. Sets ``.timestamp`` and ``.size``. diff -Nru mapproxy-1.12.0/mapproxy/cache/compact.py mapproxy-1.15.1/mapproxy/cache/compact.py --- mapproxy-1.12.0/mapproxy/cache/compact.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/compact.py 2022-06-14 12:39:29.000000000 +0000 @@ -53,21 +53,21 @@ bundle_fname, offset = self._get_bundle_fname_and_offset(tile_coord) return self.bundle_class(bundle_fname, offset=offset) - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.coord is None: return True if tile.source: return True - return self._get_bundle(tile.coord).is_cached(tile) + return self._get_bundle(tile.coord).is_cached(tile, dimensions=dimensions) - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return True - return self._get_bundle(tile.coord).store_tile(tile) + return self._get_bundle(tile.coord).store_tile(tile, dimensions=dimensions) - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions=None): if len(tiles) > 1: # Check if all tiles are from a single bundle. bundle_files = set() @@ -78,23 +78,23 @@ bundle_files.add(self._get_bundle_fname_and_offset(t.coord)[0]) tile_coord = t.coord if len(bundle_files) == 1: - return self._get_bundle(tile_coord).store_tiles(tiles) + return self._get_bundle(tile_coord).store_tiles(tiles, dimensions=dimensions) # Tiles are across multiple bundles failed = False for tile in tiles: - if not self.store_tile(tile): + if not self.store_tile(tile, dimensions=dimensions): failed = True return not failed - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): if tile.source or tile.coord is None: return True - return self._get_bundle(tile.coord).load_tile(tile) + return self._get_bundle(tile.coord).load_tile(tile, dimensions=dimensions) - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): if len(tiles) > 1: # Check if all tiles are from a single bundle. bundle_files = set() @@ -105,23 +105,23 @@ bundle_files.add(self._get_bundle_fname_and_offset(t.coord)[0]) tile_coord = t.coord if len(bundle_files) == 1: - return self._get_bundle(tile_coord).load_tiles(tiles) + return self._get_bundle(tile_coord).load_tiles(tiles, dimensions=dimensions) # No support_bulk_load or tiles are across multiple bundles missing = False for tile in tiles: - if not self.load_tile(tile, with_metadata=with_metadata): + if not self.load_tile(tile, with_metadata=with_metadata, dimensions=dimensions): missing = True return not missing - def remove_tile(self, tile): + def remove_tile(self, tile, dimensions=None): if tile.coord is None: return True - return self._get_bundle(tile.coord).remove_tile(tile) + return self._get_bundle(tile.coord).remove_tile(tile, dimensions=dimensions) - def load_tile_metadata(self, tile): - if self.load_tile(tile): + def load_tile_metadata(self, tile, dimensions=None): + if self.load_tile(tile, dimensions=dimensions): tile.timestamp = -1 def remove_level_tiles_before(self, level, timestamp): @@ -152,7 +152,7 @@ def index(self): return BundleIndexV1(self.base_filename + BUNDLEX_V1_EXT) - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.source or tile.coord is None: return True @@ -168,12 +168,12 @@ size = bundle.read_size(offset) return size != 0 - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return True - return self.store_tiles([tile]) + return self.store_tiles([tile], dimensions=dimensions) - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions=None): tiles_data = [] for t in tiles: if t.stored: @@ -194,12 +194,12 @@ return True - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): if tile.source or tile.coord is None: return True - return self.load_tiles([tile], with_metadata) + return self.load_tiles([tile], with_metadata, dimensions=dimensions) - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): missing = False with self.index().readonly() as idx: @@ -223,7 +223,7 @@ return not missing - def remove_tile(self, tile): + def remove_tile(self, tile, dimensions=None): if tile.coord is None: return True @@ -501,7 +501,7 @@ offset = val - (size << 40) return offset, size - def _load_tile(self, fh, tile): + def _load_tile(self, fh, tile, dimensions=None): if tile.source or tile.coord is None: return True @@ -516,13 +516,13 @@ tile.source = ImageSource(BytesIO(data)) return True - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): if tile.source or tile.coord is None: return True - return self.load_tiles([tile], with_metadata) + return self.load_tiles([tile], with_metadata, dimensions=dimensions) - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): missing = False with self._readonly() as fh: @@ -537,7 +537,7 @@ return not missing - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): with self._readonly() as fh: if not fh: return False @@ -576,7 +576,7 @@ fh.seek(24) fh.write(struct.pack("0: + items = list(dimensions.keys()) + items.sort() + dimensions_str = ['{key}-{value}'.format(key=i, value=dimensions[i].replace('/', '_')) for i in items] + cache_dir = os.path.join(self.cache_dir, '_'.join(dimensions_str)) + return self._tile_location(tile, self.cache_dir, self.file_ext, create_dir=create_dir, dimensions=dimensions) - def level_location(self, level): + def level_location(self, level, dimensions=None): """ Return the path where all tiles for `level` will be stored. @@ -56,7 +61,7 @@ >>> c.level_location(2) '/tmp/cache/02' """ - return self._level_location(level, self.cache_dir) + return self._level_location(level, self.cache_dir, dimensions) def _single_color_tile_location(self, color, create_dir=False): """ @@ -74,8 +79,8 @@ ensure_directory(location) return location - def load_tile_metadata(self, tile): - location = self.tile_location(tile) + def load_tile_metadata(self, tile, dimensions=None): + location = self.tile_location(tile, dimensions=dimensions) try: stats = os.lstat(location) tile.timestamp = stats.st_mtime @@ -85,12 +90,12 @@ tile.timestamp = 0 tile.size = 0 - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): """ Returns ``True`` if the tile data is present. """ if tile.is_missing(): - location = self.tile_location(tile) + location = self.tile_location(tile, dimensions=dimensions) if os.path.exists(location): return True else: @@ -98,7 +103,7 @@ else: return True - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): """ Fills the `Tile.source` of the `tile` if it is cached. If it is not cached or if the ``.coord`` is ``None``, nothing happens. @@ -106,23 +111,23 @@ if not tile.is_missing(): return True - location = self.tile_location(tile) + location = self.tile_location(tile, dimensions=dimensions) if os.path.exists(location): if with_metadata: - self.load_tile_metadata(tile) + self.load_tile_metadata(tile, dimensions=dimensions) tile.source = ImageSource(location) return True return False - def remove_tile(self, tile): - location = self.tile_location(tile) + def remove_tile(self, tile, dimensions=None): + location = self.tile_location(tile, dimensions=dimensions) try: os.remove(location) except OSError as ex: if ex.errno != errno.ENOENT: raise - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): """ Add the given `tile` to the file cache. Stores the `Tile.source` to `FileCache.tile_location`. @@ -130,7 +135,7 @@ if tile.stored: return - tile_loc = self.tile_location(tile, create_dir=True) + tile_loc = self.tile_location(tile, create_dir=True, dimensions=dimensions) if self.link_single_color_images: color = is_single_color_image(tile.source.as_image()) diff -Nru mapproxy-1.12.0/mapproxy/cache/geopackage.py mapproxy-1.15.1/mapproxy/cache/geopackage.py --- mapproxy-1.12.0/mapproxy/cache/geopackage.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/geopackage.py 2022-06-14 12:39:29.000000000 +0000 @@ -355,21 +355,21 @@ db.commit() db.close() - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.coord is None: return True if tile.source: return True - return self.load_tile(tile) + return self.load_tile(tile, dimensions=dimensions) - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return True return self._store_bulk([tile]) - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions=None): tiles = [t for t in tiles if not t.stored] return self._store_bulk(tiles) @@ -395,11 +395,11 @@ cursor.executemany(stmt, records) self.db.commit() except sqlite3.OperationalError as ex: - log.warn('unable to store tile: %s', ex) + log.warning('unable to store tile: %s', ex) return False return True - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): if tile.source or tile.coord is None: return True @@ -416,7 +416,7 @@ else: return False - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): # associate the right tiles with the cursor tile_dict = {} coords = [] @@ -459,7 +459,7 @@ return loaded_tiles == len(tile_dict) - def remove_tile(self, tile): + def remove_tile(self, tile, dimensions=None): cursor = self.db.cursor() cursor.execute( "DELETE FROM [{0}] WHERE (tile_column = ? AND tile_row = ? AND zoom_level = ?)".format(self.table_name), @@ -480,13 +480,13 @@ return True return False - def load_tile_metadata(self, tile): + def load_tile_metadata(self, tile, dimensions=None): if not self.supports_timestamp: # GPKG specification does not include tile timestamps. # This sets the timestamp of the tile to epoch (1970s) tile.timestamp = -1 else: - self.load_tile(tile) + self.load_tile(tile, dimensions=dimensions) class GeopackageLevelCache(TileCacheBase): @@ -527,35 +527,35 @@ for gp in self._geopackage.values(): gp.cleanup() - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.coord is None: return True if tile.source: return True - return self._get_level(tile.coord[2]).is_cached(tile) + return self._get_level(tile.coord[2]).is_cached(tile, dimensions=dimensions) - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return True - return self._get_level(tile.coord[2]).store_tile(tile) + return self._get_level(tile.coord[2]).store_tile(tile, dimensions=dimensions) - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions=None): failed = False for level, tiles in itertools.groupby(tiles, key=lambda t: t.coord[2]): tiles = [t for t in tiles if not t.stored] - res = self._get_level(level).store_tiles(tiles) + res = self._get_level(level).store_tiles(tiles, dimensions=dimensions) if not res: failed = True return failed - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False,dimensions=None): if tile.source or tile.coord is None: return True - return self._get_level(tile.coord[2]).load_tile(tile, with_metadata=with_metadata) + return self._get_level(tile.coord[2]).load_tile(tile, with_metadata=with_metadata, dimensions=dimensions) - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): level = None for tile in tiles: if tile.source or tile.coord is None: @@ -566,13 +566,13 @@ if not level: return True - return self._get_level(level).load_tiles(tiles, with_metadata=with_metadata) + return self._get_level(level).load_tiles(tiles, with_metadata=with_metadata, dimensions=dimensions) - def remove_tile(self, tile): + def remove_tile(self, tile, dimensions=None): if tile.coord is None: return True - return self._get_level(tile.coord[2]).remove_tile(tile) + return self._get_level(tile.coord[2]).remove_tile(tile, dimensions=dimensions) def remove_level_tiles_before(self, level, timestamp): level_cache = self._get_level(level) diff -Nru mapproxy-1.12.0/mapproxy/cache/mbtiles.py mapproxy-1.15.1/mapproxy/cache/mbtiles.py --- mapproxy-1.12.0/mapproxy/cache/mbtiles.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/mbtiles.py 2022-06-14 12:39:29.000000000 +0000 @@ -134,20 +134,20 @@ db.commit() db.close() - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.coord is None: return True if tile.source: return True - return self.load_tile(tile) + return self.load_tile(tile, dimensions=dimensions) - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return True return self._store_bulk([tile]) - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions=None): tiles = [t for t in tiles if not t.stored] return self._store_bulk(tiles) @@ -178,11 +178,11 @@ cursor.executemany(stmt, records) self.db.commit() except sqlite3.OperationalError as ex: - log.warn('unable to store tile: %s', ex) + log.warning('unable to store tile: %s', ex) return False return True - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): if tile.source or tile.coord is None: return True @@ -208,7 +208,7 @@ else: return False - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): #associate the right tiles with the cursor tile_dict = {} coords = [] @@ -287,13 +287,13 @@ return True return False - def load_tile_metadata(self, tile): + def load_tile_metadata(self, tile, dimensions=None): if not self.supports_timestamp: # MBTiles specification does not include timestamps. # This sets the timestamp of the tile to epoch (1970s) tile.timestamp = -1 else: - self.load_tile(tile) + self.load_tile(tile, dimensions=dimensions) class MBTilesLevelCache(TileCacheBase): supports_timestamp = True @@ -330,25 +330,25 @@ for mbtile in self._mbtiles.values(): mbtile.cleanup() - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.coord is None: return True if tile.source: return True - return self._get_level(tile.coord[2]).is_cached(tile) + return self._get_level(tile.coord[2]).is_cached(tile, dimensions=dimensions) - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return True - return self._get_level(tile.coord[2]).store_tile(tile) + return self._get_level(tile.coord[2]).store_tile(tile, dimensions=dimensions) - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions): failed = False for level, tiles in itertools.groupby(tiles, key=lambda t: t.coord[2]): tiles = [t for t in tiles if not t.stored] - res = self._get_level(level).store_tiles(tiles) + res = self._get_level(level).store_tiles(tiles, dimensions=dimensions) if not res: failed = True return failed @@ -358,7 +358,7 @@ return self._get_level(tile.coord[2]).load_tile(tile, with_metadata=with_metadata) - def load_tiles(self, tiles, with_metadata=False): + def load_tiles(self, tiles, with_metadata=False, dimensions=None): level = None for tile in tiles: if tile.source or tile.coord is None: @@ -369,7 +369,7 @@ if not level: return True - return self._get_level(level).load_tiles(tiles, with_metadata=with_metadata) + return self._get_level(level).load_tiles(tiles, with_metadata=with_metadata, dimensions=dimensions) def remove_tile(self, tile): if tile.coord is None: @@ -377,8 +377,8 @@ return self._get_level(tile.coord[2]).remove_tile(tile) - def load_tile_metadata(self, tile): - self.load_tile(tile) + def load_tile_metadata(self, tile, dimensions): + self.load_tile(tile, dimensions=dimensions) def remove_level_tiles_before(self, level, timestamp): level_cache = self._get_level(level) diff -Nru mapproxy-1.12.0/mapproxy/cache/path.py mapproxy-1.15.1/mapproxy/cache/path.py --- mapproxy-1.12.0/mapproxy/cache/path.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/path.py 2022-06-14 12:39:29.000000000 +0000 @@ -16,7 +16,7 @@ import os from mapproxy.compat import string_type from mapproxy.util.fs import ensure_directory - +from mapproxy.request.base import NoCaseMultiDict def location_funcs(layout): if layout == 'tc': @@ -34,18 +34,39 @@ else: raise ValueError('unknown directory_layout "%s"' % layout) -def level_location(level, cache_dir): +def level_location(level, cache_dir, dimensions=None): """ Return the path where all tiles for `level` will be stored. >>> os.path.abspath(level_location(2, '/tmp/cache')) == os.path.abspath('/tmp/cache/02') True """ + dim_path = dimensions_part(dimensions) + if isinstance(level, string_type): - return os.path.join(cache_dir, level) + return os.path.join(cache_dir, dim_path, level) else: - return os.path.join(cache_dir, "%02d" % level) + return os.path.join(cache_dir, dim_path, "%02d" % level) +def dimensions_part(dimensions): + """ + Return the subpath where all tiles for `dimensions` will be stored. + Dimensions prefixed with "dim_" are sorted after the predefined elevation and time dimensions + >>> dimensions_part({'time': '2020-08-25T00:00:00Z'}) + 'time-2020-08-25T00:00:00Z' + >>> dimensions_part({'time': '2020-08-25T00:00:00Z', 'dim_reference_time': '2020-08-25T00:00:00Z', 'dim_level': '700'}) + 'time-2020-08-25T00:00:00Z/dim_level-700/dim_reference_time-2020-08-25T00:00:00Z' + + """ + if dimensions: + dims = NoCaseMultiDict(dimensions) + predefined_dims, custom_dims = [], [] + for dim in dims.keys(): + (custom_dims if dim.startswith('dim_') else predefined_dims).append(dim) + dim_keys = sorted(predefined_dims) + sorted(custom_dims) + return os.path.join(*(map(lambda k: k + "-" + str(dims.get(k, 'default')), dim_keys))) + else: + return "" def level_part(level): """ @@ -62,7 +83,7 @@ return "%02d" % level -def tile_location_tc(tile, cache_dir, file_ext, create_dir=False): +def tile_location_tc(tile, cache_dir, file_ext, create_dir=False, dimensions=None): """ Return the location of the `tile`. Caches the result as ``location`` property of the `tile`. @@ -77,7 +98,9 @@ """ if tile.location is None: x, y, z = tile.coord + parts = (cache_dir, + dimensions_part(dimensions), level_part(z), "%03d" % int(x / 1000000), "%03d" % (int(x / 1000) % 1000), @@ -90,7 +113,7 @@ ensure_directory(tile.location) return tile.location -def tile_location_mp(tile, cache_dir, file_ext, create_dir=False): +def tile_location_mp(tile, cache_dir, file_ext, create_dir=False, dimensions=None): """ Return the location of the `tile`. Caches the result as ``location`` property of the `tile`. @@ -108,6 +131,7 @@ if tile.location is None: x, y, z = tile.coord parts = (cache_dir, + dimensions_part(dimensions), level_part(z), "%04d" % int(x / 10000), "%04d" % (int(x) % 10000), @@ -118,7 +142,7 @@ ensure_directory(tile.location) return tile.location -def tile_location_tms(tile, cache_dir, file_ext, create_dir=False): +def tile_location_tms(tile, cache_dir, file_ext, create_dir=False, dimensions=None): """ Return the location of the `tile`. Caches the result as ``location`` property of the `tile`. @@ -134,14 +158,14 @@ if tile.location is None: x, y, z = tile.coord tile.location = os.path.join( - cache_dir, level_part(str(z)), + cache_dir,dimensions_part(dimensions) ,level_part(str(z)), str(x), str(y) + '.' + file_ext ) if create_dir: ensure_directory(tile.location) return tile.location -def tile_location_reverse_tms(tile, cache_dir, file_ext, create_dir=False): +def tile_location_reverse_tms(tile, cache_dir, file_ext, create_dir=False, dimensions=None): """ Return the location of the `tile`. Caches the result as ``location`` property of the `tile`. @@ -157,16 +181,16 @@ if tile.location is None: x, y, z = tile.coord tile.location = os.path.join( - cache_dir, str(y), str(x), str(z) + '.' + file_ext + cache_dir,dimensions_part(dimensions),str(y), str(x), str(z) + '.' + file_ext ) if create_dir: ensure_directory(tile.location) return tile.location -def level_location_tms(level, cache_dir): +def level_location_tms(level, cache_dir, dimensions=None): return level_location(str(level), cache_dir=cache_dir) -def tile_location_quadkey(tile, cache_dir, file_ext, create_dir=False): +def tile_location_quadkey(tile, cache_dir, file_ext, create_dir=False, dimensions=None): """ Return the location of the `tile`. Caches the result as ``location`` property of the `tile`. @@ -197,11 +221,11 @@ ensure_directory(tile.location) return tile.location -def no_level_location(level, cache_dir): +def no_level_location(level, cache_dir, dimensions=None): # dummy for quadkey cache which stores all tiles in one directory raise NotImplementedError('cache does not have any level location') -def tile_location_arcgiscache(tile, cache_dir, file_ext, create_dir=False): +def tile_location_arcgiscache(tile, cache_dir, file_ext, create_dir=False, dimensions=None): """ Return the location of the `tile`. Caches the result as ``location`` property of the `tile`. @@ -222,5 +246,5 @@ ensure_directory(tile.location) return tile.location -def level_location_arcgiscache(z, cache_dir): - return level_location('L%02d' % z, cache_dir=cache_dir) \ No newline at end of file +def level_location_arcgiscache(z, cache_dir, dimensions=None): + return level_location('L%02d' % z, cache_dir=cache_dir, dimensions=dimensions) diff -Nru mapproxy-1.12.0/mapproxy/cache/redis.py mapproxy-1.15.1/mapproxy/cache/redis.py --- mapproxy-1.12.0/mapproxy/cache/redis.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/redis.py 2022-06-14 12:39:29.000000000 +0000 @@ -48,13 +48,13 @@ x, y, z = tile.coord return self.prefix + '-%d-%d-%d' % (z, x, y) - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.coord is None or tile.source: return True return self.r.exists(self._key(tile)) - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return True @@ -69,7 +69,7 @@ self.r.pexpire(key, int(self.ttl * 1000)) return r - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): if tile.source or tile.coord is None: return True key = self._key(tile) @@ -79,7 +79,7 @@ return True return False - def remove_tile(self, tile): + def remove_tile(self, tile, dimensions=None): if tile.coord is None: return True diff -Nru mapproxy-1.12.0/mapproxy/cache/riak.py mapproxy-1.15.1/mapproxy/cache/riak.py --- mapproxy-1.12.0/mapproxy/cache/riak.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/riak.py 2022-06-14 12:39:29.000000000 +0000 @@ -72,7 +72,7 @@ try: obj = self.bucket.get(key, r=1, timeout=self.request_timeout) except Exception as e: - log.warn('error while requesting %s: %s', key, e) + log.warning('error while requesting %s: %s', key, e) if not obj: obj = self.bucket.new(key=key, data=None, content_type='application/octet-stream') @@ -87,7 +87,7 @@ obj.usermeta = {'timestamp': '0'} return 0.0 - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): return self.load_tile(tile, True) def _store_bulk(self, tiles): @@ -107,29 +107,29 @@ try: res.store(w=1, dw=1, pw=1, return_body=False, timeout=self.request_timeout) except riak.RiakError as ex: - log.warn('unable to store tile: %s', ex) + log.warning('unable to store tile: %s', ex) return False return True - def store_tile(self, tile): + def store_tile(self, tile,dimensions=None): if tile.stored: return True return self._store_bulk([tile]) - def store_tiles(self, tiles): + def store_tiles(self, tiles,dimensions=None): tiles = [t for t in tiles if not t.stored] return self._store_bulk(tiles) - def load_tile_metadata(self, tile): + def load_tile_metadata(self, tile,dimensions=None): if tile.timestamp: return # is_cached loads metadata self.load_tile(tile, True) - def load_tile(self, tile, with_metadata=False): + def load_tile(self, tile, with_metadata=False, dimensions=None): if tile.timestamp is None: tile.timestamp = 0 if tile.source or tile.coord is None: @@ -158,7 +158,7 @@ try: res.delete(w=1, r=1, dw=1, pw=1, timeout=self.request_timeout) except riak.RiakError as ex: - log.warn('unable to remove tile: %s', ex) + log.warning('unable to remove tile: %s', ex) return False return True diff -Nru mapproxy-1.12.0/mapproxy/cache/s3.py mapproxy-1.15.1/mapproxy/cache/s3.py --- mapproxy-1.12.0/mapproxy/cache/s3.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/s3.py 2022-06-14 12:39:29.000000000 +0000 @@ -55,6 +55,7 @@ super(S3Cache, self).__init__() self.lock_cache_id = hashlib.md5(base_path.encode('utf-8') + bucket_name.encode('utf-8')).hexdigest() self.bucket_name = bucket_name + self.profile_name = profile_name self.region_name = region_name self.endpoint_url = endpoint_url self.access_control_list = access_control_list @@ -86,14 +87,14 @@ raise ImportError("S3 Cache requires 'boto3' package.") try: - return s3_session().client("s3", region_name=self.region_name, endpoint_url=self.endpoint_url) + return s3_session(self.profile_name).client("s3", region_name=self.region_name, endpoint_url=self.endpoint_url) except Exception as e: raise S3ConnectionError('Error during connection %s' % e) - def load_tile_metadata(self, tile): + def load_tile_metadata(self, tile, dimensions=None): if tile.timestamp: return - self.is_cached(tile) + self.is_cached(tile, dimensions=dimensions) def _set_metadata(self, response, tile): if 'LastModified' in response: @@ -101,7 +102,7 @@ if 'ContentLength' in response: tile.size = response['ContentLength'] - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): if tile.is_missing(): key = self.tile_key(tile) try: @@ -114,11 +115,11 @@ return True - def load_tiles(self, tiles, with_metadata=True): + def load_tiles(self, tiles, with_metadata=True, dimensions=None): p = async_.Pool(min(4, len(tiles))) return all(p.map(self.load_tile, tiles)) - def load_tile(self, tile, with_metadata=True): + def load_tile(self, tile, with_metadata=True, dimensions=None): if not tile.is_missing(): return True @@ -142,11 +143,11 @@ log.debug('remove_tile, key: %s' % key) self.conn().delete_object(Bucket=self.bucket_name, Key=key) - def store_tiles(self, tiles): + def store_tiles(self, tiles, dimensions=None): p = async_.Pool(min(self._concurrent_writer, len(tiles))) p.map(self.store_tile, tiles) - def store_tile(self, tile): + def store_tile(self, tile, dimensions=None): if tile.stored: return diff -Nru mapproxy-1.12.0/mapproxy/cache/tile.py mapproxy-1.15.1/mapproxy/cache/tile.py --- mapproxy-1.12.0/mapproxy/cache/tile.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/cache/tile.py 2022-06-14 12:39:29.000000000 +0000 @@ -34,8 +34,6 @@ """ - - from functools import partial from contextlib import contextmanager from mapproxy.grid import MetaGrid @@ -44,8 +42,12 @@ from mapproxy.image.merge import merge_images from mapproxy.image.tile import TileSplitter, TiledImage from mapproxy.layer import MapQuery, BlankImage +from mapproxy.source import SourceError from mapproxy.util import async_ -from mapproxy.util.py import reraise +from mapproxy.util.py import reraise, reraise_exception +import sys +import logging +log = logging.getLogger('mapproxy.cache.tile') class TileManager(object): @@ -64,6 +66,7 @@ bulk_meta_tiles=False, rescale_tiles=0, cache_rescaled_tiles=False, + dimensions=None ): self.grid = grid self.cache = cache @@ -76,9 +79,11 @@ self.sources = sources self.minimize_meta_requests = minimize_meta_requests self._expire_timestamp = None + self._refresh_before = {} self.pre_store_filter = pre_store_filter or [] self.concurrent_tile_creators = concurrent_tile_creators self.tile_creator_class = tile_creator_class or TileCreator + self.dimensions = dimensions self.rescale_tiles = rescale_tiles self.cache_rescaled_tiles = cache_rescaled_tiles @@ -149,7 +154,7 @@ return tiles def _load_tile_coords(self, tiles, dimensions=None, with_metadata=False, - rescale_till_zoom=None, rescaled_tiles=None, + rescale_till_zoom=None, rescaled_tiles=None ): uncached_tiles = [] @@ -159,7 +164,7 @@ t.source = rescaled_tiles[t.coord].source # load all in batch - self.cache.load_tiles(tiles, with_metadata) + self.cache.load_tiles(tiles, with_metadata, dimensions=dimensions) for tile in tiles: if tile.coord is not None and not self.is_cached(tile, dimensions=dimensions): @@ -198,11 +203,13 @@ tile = Tile(tile) if tile.coord is None: return True - cached = self.cache.is_cached(tile) + cached = self.cache.is_cached(tile, dimensions=dimensions) max_mtime = self.expire_timestamp(tile) if cached and max_mtime is not None: self.cache.load_tile_metadata(tile) - stale = tile.timestamp < max_mtime + # file time stamp must be rounded to integer since time conversion functions + # mktime and timetuple strip decimals from seconds + stale = int(tile.timestamp) <= max_mtime if stale: cached = False return cached @@ -213,9 +220,9 @@ """ if isinstance(tile, tuple): tile = Tile(tile) - if self.cache.is_cached(tile): + if self.cache.is_cached(tile, dimensions=dimensions): # tile exists - if not self.is_cached(tile): + if not self.is_cached(tile, dimensions=dimensions): # expired return True return False @@ -228,6 +235,9 @@ :note: Returns _expire_timestamp by default. """ + if self._refresh_before: + from mapproxy.seed.config import before_timestamp_from_options + return before_timestamp_from_options(self._refresh_before) return self._expire_timestamp def apply_tile_filter(self, tile): @@ -312,11 +322,17 @@ self.dimensions = dimensions self.image_merger = image_merger - def is_cached(self, tile): + def is_cached(self, tile, dimensions=None): """ Return True if the tile is cached. """ - return self.tile_mgr.is_cached(tile) + return self.tile_mgr.is_cached(tile, dimensions=dimensions) + + def is_stale(self, tile): + """ + Return True if the tile exists in cache and is expired. + """ + return self.tile_mgr.is_stale(tile) def create_tiles(self, tiles): if not self.sources: @@ -356,14 +372,27 @@ result.extend(new_tiles) return result - def _create_single_tile(self, tile): + def _create_single_tile(self, tile, dimensions=None): tile_bbox = self.grid.tile_bbox(tile.coord) query = MapQuery(tile_bbox, self.grid.tile_size, self.grid.srs, self.tile_mgr.request_format, dimensions=self.dimensions) with self.tile_mgr.lock(tile): - if not self.is_cached(tile): - source = self._query_sources(query) + if not self.is_cached(tile, dimensions=dimensions): + source = None + try: + source = self._query_sources(query) + # if source is not available, try to serve tile in cache + except SourceError as e: + if self.is_stale(tile): + self.cache.load_tile(tile) + else: + reraise_exception(e, sys.exc_info()) if not source: return [] + if source.authorize_stale and self.is_stale(tile): + # The configuration authorises blank tiles generated by the error_handler + # to be replaced by stale tiles from cache. + self.cache.load_tile(tile) + return [tile] if self.tile_mgr.image_opts != source.image_opts: # call as_buffer to force conversion into cache format source.as_buffer(self.tile_mgr.image_opts) @@ -436,18 +465,18 @@ dimensions=self.dimensions) main_tile = Tile(meta_tile.main_tile_coord) with self.tile_mgr.lock(main_tile): - if not all(self.is_cached(t) for t in meta_tile.tiles if t is not None): + if not all(self.is_cached(t, dimensions=self.dimensions) for t in meta_tile.tiles if t is not None): meta_tile_image = self._query_sources(query) if not meta_tile_image: return [] splitted_tiles = split_meta_tiles(meta_tile_image, meta_tile.tile_patterns, tile_size, self.tile_mgr.image_opts) splitted_tiles = [self.tile_mgr.apply_tile_filter(t) for t in splitted_tiles] if meta_tile_image.cacheable: - self.cache.store_tiles(splitted_tiles) + self.cache.store_tiles(splitted_tiles,dimensions=self.dimensions) return splitted_tiles # else tiles = [Tile(coord) for coord in meta_tile.tiles] - self.cache.load_tiles(tiles) + self.cache.load_tiles(tiles, dimensions=self.dimensions) return tiles def _create_bulk_meta_tile(self, meta_tile): @@ -458,7 +487,7 @@ tile_size = self.grid.tile_size main_tile = Tile(meta_tile.main_tile_coord) with self.tile_mgr.lock(main_tile): - if not all(self.is_cached(t) for t in meta_tile.tiles if t is not None): + if not all(self.is_cached(t, dimensions=self.dimensions) for t in meta_tile.tiles if t is not None): async_pool = async_.Pool(self.tile_mgr.concurrent_tile_creators) def query_tile(coord): try: @@ -499,7 +528,7 @@ # else tiles = [Tile(coord) for coord in meta_tile.tiles] - self.cache.load_tiles(tiles) + self.cache.load_tiles(tiles, dimensions=self.dimensions) return tiles diff -Nru mapproxy-1.12.0/mapproxy/client/cgi.py mapproxy-1.15.1/mapproxy/client/cgi.py --- mapproxy-1.12.0/mapproxy/client/cgi.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/client/cgi.py 2022-06-14 12:39:29.000000000 +0000 @@ -118,7 +118,7 @@ else: headers, content = split_cgi_response(stdout) - status_match = re.match('(\d\d\d) ', headers.get('Status', '')) + status_match = re.match(r'(\d\d\d) ', headers.get('Status', '')) if status_match: status_code = status_match.group(1) else: diff -Nru mapproxy-1.12.0/mapproxy/client/http.py mapproxy-1.15.1/mapproxy/client/http.py --- mapproxy-1.12.0/mapproxy/client/http.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/client/http.py 2022-06-14 12:39:29.000000000 +0000 @@ -29,12 +29,15 @@ if PY2: import urllib2 - from urllib2 import URLError, HTTPError + from urllib2 import URLError, HTTPError, HTTPCookieProcessor import httplib + from cookielib import CookieJar else: from urllib import request as urllib2 from urllib.error import URLError, HTTPError + from urllib.request import HTTPCookieProcessor from http import client as httplib + from http.cookiejar import CookieJar import socket import ssl @@ -45,9 +48,10 @@ supports_ssl_default_context = True class HTTPClientError(Exception): - def __init__(self, arg, response_code=None): + def __init__(self, arg, response_code=None, full_msg=None): Exception.__init__(self, arg) self.response_code = response_code + self.full_msg = full_msg def build_https_handler(ssl_ca_certs, insecure): @@ -130,8 +134,8 @@ def __init__(self): self._opener = {} - def __call__(self, ssl_ca_certs, url, username, password, insecure=False): - cache_key = (ssl_ca_certs, insecure) + def __call__(self, ssl_ca_certs, url, username, password, insecure=False, manage_cookies=False): + cache_key = (ssl_ca_certs, insecure, manage_cookies) if cache_key not in self._opener: handlers = [] https_handler = build_https_handler(ssl_ca_certs, insecure) @@ -142,6 +146,9 @@ handlers.append(authhandler) authhandler = urllib2.HTTPDigestAuthHandler(passman) handlers.append(authhandler) + if manage_cookies: + cj = CookieJar() + handlers.append(HTTPCookieProcessor(cj)) opener = urllib2.build_opener(*handlers) @@ -160,7 +167,8 @@ class HTTPClient(object): def __init__(self, url=None, username=None, password=None, insecure=False, - ssl_ca_certs=None, timeout=None, headers=None): + ssl_ca_certs=None, timeout=None, headers=None, hide_error_details=False, + manage_cookies=False): self._timeout = timeout if url and url.startswith('https'): if insecure: @@ -169,19 +177,22 @@ raise HTTPClientError('No ca_certs file set (http.ssl_ca_certs). ' 'Set file or disable verification with http.ssl_no_cert_checks option.') - self.opener = create_url_opener(ssl_ca_certs, url, username, password, insecure=insecure) + self.opener = create_url_opener(ssl_ca_certs, url, username, password, insecure=insecure, manage_cookies=manage_cookies) self.header_list = headers.items() if headers else [] + self.hide_error_details = hide_error_details - def open(self, url, data=None): + def open(self, url, data=None, method=None): code = None result = None try: req = urllib2.Request(url, data=data) except ValueError as e: - reraise_exception(HTTPClientError('URL not correct "%s": %s' - % (url, e.args[0])), sys.exc_info()) + err = self.handle_url_exception(url, 'URL not correct', e.args[0]) + reraise_exception(err, sys.exc_info()) for key, value in self.header_list: req.add_header(key, value) + if method: + req.method=method try: start_time = time.time() if self._timeout is not None: @@ -190,25 +201,24 @@ result = self.opener.open(req) except HTTPError as e: code = e.code - reraise_exception(HTTPClientError('HTTP Error "%s": %d' - % (url, e.code), response_code=code), sys.exc_info()) + err = self.handle_url_exception(url, 'HTTP Error', str(code), response_code=code) + reraise_exception(err, sys.exc_info()) except URLError as e: if isinstance(e.reason, ssl.SSLError): - e = HTTPClientError('Could not verify connection to URL "%s": %s' - % (url, e.reason.args[1])) - reraise_exception(e, sys.exc_info()) + err = self.handle_url_exception(url, 'Could not verify connection to URL', e.reason.args[1]) + reraise_exception(err, sys.exc_info()) try: reason = e.reason.args[1] except (AttributeError, IndexError): reason = e.reason - reraise_exception(HTTPClientError('No response from URL "%s": %s' - % (url, reason)), sys.exc_info()) + err = self.handle_url_exception(url, 'No response from URL', reason) + reraise_exception(err, sys.exc_info()) except ValueError as e: - reraise_exception(HTTPClientError('URL not correct "%s": %s' - % (url, e.args[0])), sys.exc_info()) + err = self.handle_url_exception(url, 'URL not correct', e.args[0]) + reraise_exception(err, sys.exc_info()) except Exception as e: - reraise_exception(HTTPClientError('Internal HTTP error "%s": %r' - % (url, e)), sys.exc_info()) + err = self.handle_url_exception(url, 'Internal HTTP error', repr(e)) + reraise_exception(err, sys.exc_info()) else: code = getattr(result, 'code', 200) if code == 204: @@ -224,8 +234,24 @@ raise HTTPClientError('response is not an image: (%s)' % (resp.read())) return ImageSource(resp) + def handle_url_exception(self, url, message, reason, response_code=None): + full_msg = '%s "%s": %s' % (message, url, reason) + if self.hide_error_details: + return HTTPClientError( + '{} (see logs for URL and reason).'.format(message), + response_code=response_code, + full_msg=full_msg, + ) + else: + return HTTPClientError( + full_msg, + response_code=response_code, + ) + def auth_data_from_url(url): """ + >>> auth_data_from_url('invalid_url') + ('invalid_url', (None, None)) >>> auth_data_from_url('http://localhost/bar') ('http://localhost/bar', (None, None)) >>> auth_data_from_url('http://bar@localhost/bar') @@ -238,16 +264,35 @@ ('http://localhost/bar', ('bar foo; foo@bar', 'b:az@')) >>> auth_data_from_url('https://bar:foo#;%$@localhost/bar') ('https://localhost/bar', ('bar', 'foo#;%$')) + >>> auth_data_from_url('http://localhost/bar@2x') + ('http://localhost/bar@2x', (None, None)) + >>> auth_data_from_url('http://bar@localhost/bar@2x') + ('http://localhost/bar@2x', ('bar', None)) + >>> auth_data_from_url('http://bar:baz@localhost/bar@2x') + ('http://localhost/bar@2x', ('bar', 'baz')) + >>> auth_data_from_url('https://bar@localhost/bar/0/0/0@2x.png') + ('https://localhost/bar/0/0/0@2x.png', ('bar', None)) + >>> auth_data_from_url('http://bar:baz@localhost/bar@2x.png') + ('http://localhost/bar@2x.png', ('bar', 'baz')) """ + if not url or '://' not in url: + # be silent for invalid URLs + return url, (None, None) + + schema, url = url.split('://', 1) + if '/' in url: + host, request = url.split('/', 1) + else: + host, request = url, '' + username = password = None - if '@' in url: - head, url = url.rsplit('@', 1) - schema, auth_data = head.split('//', 1) - url = schema + '//' + url + if '@' in host: + auth_data, host = host.rsplit('@', 1) if ':' in auth_data: username, password = auth_data.split(':', 1) else: username = auth_data + url = schema + "://" + host + "/" + request return url, (username, password) diff -Nru mapproxy-1.12.0/mapproxy/client/wms.py mapproxy-1.15.1/mapproxy/client/wms.py --- mapproxy-1.12.0/mapproxy/client/wms.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/client/wms.py 2022-06-14 12:39:29.000000000 +0000 @@ -16,6 +16,8 @@ """ WMS clients for maps and information. """ +import sys + from mapproxy.compat import text_type from mapproxy.request.base import split_mime_type from mapproxy.layer import InfoQuery @@ -39,6 +41,7 @@ self.fwd_req_params = fwd_req_params or set() def retrieve(self, query, format): + log.debug(query) if self.http_method == 'POST': request_method = 'POST' elif self.http_method == 'GET': @@ -72,10 +75,19 @@ log_size = 8000 # larger xml exception else: log_size = 100 # image? - data = resp.read(log_size) - if len(data) == log_size: - data += '... truncated' - log.warn("no image returned from source WMS: %s, response was: %s" % (url, data)) + data = resp.read(log_size+1) + + truncated = '' + if len(data) == log_size+1: + data = data[:-1] + truncated = ' [output truncated]' + + if sys.version_info >= (3, 5, 0): + data = data.decode('utf-8', 'backslashreplace') + else: + data = data.decode('ascii', 'ignore') + + log.warning("no image returned from source WMS: {}, response was: '{}'{}".format(url, data, truncated)) raise SourceError('no image returned from source WMS: %s' % (url, )) def _query_url(self, query, format): @@ -91,7 +103,6 @@ req.params.size = query.size req.params.srs = query.srs.srs_code req.params.format = format - # also forward dimension request params if available in the query req.params.update(query.dimensions_for_params(self.fwd_req_params)) return req diff -Nru mapproxy-1.12.0/mapproxy/compat/image.py mapproxy-1.15.1/mapproxy/compat/image.py --- mapproxy-1.12.0/mapproxy/compat/image.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/compat/image.py 2022-06-14 12:39:29.000000000 +0000 @@ -25,6 +25,7 @@ # prevent pyflakes warnings Image, ImageColor, ImageDraw, ImageFont, ImagePalette, ImageChops, ImageMath ImageFileDirectory_v2, TiffTags + PIL_VERSION = getattr(PIL, '__version__') or getattr(PIL, 'PILLOW_VERSION') except ImportError: # allow MapProxy to start without PIL (for tilecache only). # issue warning and raise ImportError on first use of @@ -41,18 +42,21 @@ Image.NEAREST = Image.BILINEAR = Image.BICUBIC = 1 Image.Image = NoPIL ImageColor = NoPIL() + ImageFileDirectory_v2 = NoPIL() + TiffTags = NoPIL() + ImageMath = NoPIL() ImageColor.getrgb = lambda x: x + PIL_VERSION = None def has_alpha_composite_support(): return hasattr(Image, 'alpha_composite') def transform_uses_center(): - # transformation behavior changed with Pillow 3.4 + # transformation behavior changed with Pillow 3.4 to use pixel centers # https://github.com/python-pillow/Pillow/commit/5232361718bae0f0ccda76bfd5b390ebf9179b18 - if hasattr(PIL, 'PILLOW_VERSION'): - if not PIL.PILLOW_VERSION.startswith(('1.', '2.', '3.0', '3.1', '3.2', '3.3')): - return True - return False + if not PIL_VERSION or PIL_VERSION.startswith(('1.', '2.', '3.0', '3.1', '3.2', '3.3')): + return False + return True def quantize_pil(img, colors=256, alpha=False, defaults=None): if hasattr(Image, 'FASTOCTREE'): diff -Nru mapproxy-1.12.0/mapproxy/compat/modules.py mapproxy-1.15.1/mapproxy/compat/modules.py --- mapproxy-1.12.0/mapproxy/compat/modules.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/compat/modules.py 2022-06-14 12:39:29.000000000 +0000 @@ -5,5 +5,9 @@ if PY2: import urlparse; urlparse + from cgi import parse_qsl, escape + from urllib import quote else: - from urllib import parse as urlparse \ No newline at end of file + from html import escape + from urllib import parse as urlparse + from urllib.parse import parse_qsl, quote diff -Nru mapproxy-1.12.0/mapproxy/config/defaults.py mapproxy-1.15.1/mapproxy/config/defaults.py --- mapproxy-1.12.0/mapproxy/config/defaults.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/config/defaults.py 2022-06-14 12:39:29.000000000 +0000 @@ -93,4 +93,6 @@ concurrent_requests = 0, method = 'AUTO', access_control_allow_origin = '*', + hide_error_details = True, + manage_cookies = False, ) diff -Nru mapproxy-1.12.0/mapproxy/config/loader.py mapproxy-1.15.1/mapproxy/config/loader.py --- mapproxy-1.12.0/mapproxy/config/loader.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/config/loader.py 2022-06-14 12:39:29.000000000 +0000 @@ -30,7 +30,7 @@ from mapproxy.config import load_default_config, finish_base_config, defaults from mapproxy.config.validator import validate_references -from mapproxy.config.spec import validate_options +from mapproxy.config.spec import validate_options, add_source_to_mapproxy_yaml_spec, add_service_to_mapproxy_yaml_spec from mapproxy.util.py import memoize from mapproxy.util.ext.odict import odict from mapproxy.util.yaml import load_yaml_file, YAMLError @@ -84,6 +84,7 @@ def load_sources(self): self.sources = SourcesCollection() for source_name, source_conf in iteritems((self.configuration.get('sources') or {})): + source_conf['name'] = source_name self.sources[source_name] = SourceConfiguration.load(conf=source_conf, context=self) def load_tile_layers(self): @@ -293,7 +294,7 @@ global_key='image.max_shrink_factor') if conf.get('origin') is None: - log.warn('grid %s does not have an origin. default origin will change from sw (south/west) to nw (north-west) with MapProxy 2.0', + log.warning('grid %s does not have an origin. default origin will change from sw (south/west) to nw (north-west) with MapProxy 2.0', conf['name'], ) @@ -591,10 +592,13 @@ timeout = self.context.globals.get_value('http.client_timeout', self.conf) headers = self.context.globals.get_value('http.headers', self.conf) + hide_error_details = self.context.globals.get_value('http.hide_error_details', self.conf) + manage_cookies = self.context.globals.get_value('http.manage_cookies', self.conf) http_client = HTTPClient(url, username, password, insecure=insecure, ssl_ca_certs=ssl_ca_certs, timeout=timeout, - headers=headers) + headers=headers, hide_error_details=hide_error_details, + manage_cookies=manage_cookies) return http_client, url @memoize @@ -608,11 +612,12 @@ raise ConfigurationError("invalid error code %r in on_error", status_code) cacheable = response_conf.get('cache', False) color = response_conf.get('response', 'transparent') + authorize_stale = response_conf.get('authorize_stale', False) if color == 'transparent': color = (255, 255, 255, 0) else: color = parse_color(color) - error_handler.add_handler(status_code, color, cacheable) + error_handler.add_handler(status_code, color, cacheable, authorize_stale) return error_handler @@ -915,7 +920,8 @@ if concurrent_requests: from mapproxy.util.lock import SemLock lock_dir = self.context.globals.get_path('cache.lock_dir', self.conf) - md5 = hashlib.md5(self.conf['mapfile']) + mapfile = self.conf['mapfile'] + md5 = hashlib.md5(mapfile.encode('utf-8')) lock_file = os.path.join(lock_dir, md5.hexdigest() + '.lck') lock = lambda: SemLock(lock_file, concurrent_requests) @@ -974,7 +980,7 @@ grid_name = self.conf.get('grid') if grid_name is None: - log.warn("tile source for %s does not have a grid configured and defaults to GLOBAL_MERCATOR. default will change with MapProxy 2.0", url) + log.warning("tile source for %s does not have a grid configured and defaults to GLOBAL_MERCATOR. default will change with MapProxy 2.0", url) grid_name = "GLOBAL_MERCATOR" grid = self.context.grids[grid_name].tile_grid() @@ -1014,6 +1020,32 @@ } +def register_source_configuration(config_name, config_class, + yaml_spec_source_name = None, yaml_spec_source_def = None): + """ Method used by plugins to register a new source configuration. + + :param config_name: Name of the source configuration + :type config_name: str + :param config_class: Class of the source configuration + :type config_name: SourceConfiguration + :param yaml_spec_source_name: Name of the source in the YAML configuration file + :type yaml_spec_source_name: str + :param yaml_spec_source_def: Definition of the source in the YAML configuration file + :type yaml_spec_source_def: dict + + Example: + register_source_configuration('hips', HIPSSourceConfiguration, + 'hips', { required('url'): str(), + 'resampling_method': str(), + 'image': image_opts, + }) + """ + log.info('Registering configuration for plugin source %s' % config_name) + source_configuration_types[config_name] = config_class + if yaml_spec_source_name is not None and yaml_spec_source_def is not None: + add_source_to_mapproxy_yaml_spec(yaml_spec_source_name, yaml_spec_source_def) + + class CacheConfiguration(ConfigurationBase): defaults = {'format': 'image/png'} @@ -1022,7 +1054,7 @@ cache_dir = self.conf.get('cache', {}).get('directory') if cache_dir: if self.conf.get('cache_dir'): - log.warn('found cache.directory and cache_dir option for %s, ignoring cache_dir', + log.warning('found cache.directory and cache_dir option for %s, ignoring cache_dir', self.conf['name']) return self.context.globals.abspath(cache_dir) @@ -1060,9 +1092,10 @@ global_key='cache.link_single_color_images') if link_single_color_images and sys.platform == 'win32': - log.warn('link_single_color_images not supported on windows') + log.warning('link_single_color_images not supported on windows') link_single_color_images = False + return FileCache( cache_dir, file_ext=file_ext, @@ -1451,8 +1484,8 @@ from mapproxy.layer import map_extent_from_grid, merge_layer_extents base_image_opts = self.image_opts() - if self.conf.get('format') == 'mixed' and not self.conf.get('request_format') == 'image/png': - raise ConfigurationError('request_format must be set to image/png if mixed mode is enabled') + if self.conf.get('format') == 'mixed' and not self.conf.get('request_format') in [ 'image/png', 'image/vnd.jpeg-png' ]: + raise ConfigurationError('request_format must be set to image/png or image/vnd.jpeg-png if mixed mode is enabled') request_format = self.conf.get('request_format') or self.conf.get('format') if '/' in request_format: request_format_ext = request_format.split('/', 1)[1] @@ -1558,6 +1591,7 @@ lock_timeout=self.context.globals.get_value('http.client_timeout', {}), lock_cache_id=cache.lock_cache_id, ) + mgr = TileManager(tile_grid, cache, sources, image_opts.format.ext, locker=locker, image_opts=image_opts, identifier=identifier, @@ -1571,6 +1605,8 @@ cache_rescaled_tiles=cache_rescaled_tiles, rescale_tiles=rescale_tiles, ) + if self.conf['name'] in self.context.caches: + mgr._refresh_before = self.context.caches[self.conf['name']].conf.get('refresh_before', {}) extent = merge_layer_extents(sources) if extent.is_default: extent = map_extent_from_grid(tile_grid) @@ -1581,7 +1617,7 @@ def grid_confs(self): grid_names = self.conf.get('grids') if grid_names is None: - log.warn('cache %s does not have any grids. default will change from [GLOBAL_MERCATOR] to [GLOBAL_WEBMERCATOR] with MapProxy 2.0', + log.warning('cache %s does not have any grids. default will change from [GLOBAL_MERCATOR] to [GLOBAL_WEBMERCATOR] with MapProxy 2.0', self.conf['name']) grid_names = ['GLOBAL_MERCATOR'] return [(g, self.context.grids[g]) for g in grid_names] @@ -1711,18 +1747,32 @@ lg_sources.append(lg_source) res_range = resolution_range(self.conf) + dimensions = None + if 'dimensions' in self.conf.keys(): + dimensions = self.dimensions() layer = WMSLayer(self.conf.get('name'), self.conf.get('title'), - sources, fi_sources, lg_sources, res_range=res_range, md=self.conf.get('md')) + sources, fi_sources, lg_sources, res_range=res_range, md=self.conf.get('md'),dimensions=dimensions) return layer @memoize def dimensions(self): from mapproxy.layer import Dimension + from mapproxy.util.ext.wmsparse.util import parse_datetime_range dimensions = {} - for dimension, conf in iteritems(self.conf.get('dimensions', {})): - values = [str(val) for val in conf.get('values', ['default'])] + raw_values = conf.get('values') + if len(raw_values) == 1: + # look for time or dim_reference_time + if 'time' in dimension.lower(): + log.debug('Determining values as datetime strings') + values = parse_datetime_range(raw_values[0]) + else: + log.debug('Determining values as plain strings') + values = raw_values[0].strip().split('/') + else: + values = [str(val) for val in conf.get('values', ['default'])] + default = conf.get('default', values[-1]) dimensions[dimension.lower()] = Dimension(dimension, values, default=default) return dimensions @@ -1731,7 +1781,7 @@ def tile_layers(self, grid_name_as_path=False): from mapproxy.service.tile import TileLayer from mapproxy.cache.dummy import DummyCache - + from mapproxy.cache.file import FileCache sources = [] fi_only_sources = [] if 'tile_sources' in self.conf: @@ -1771,9 +1821,14 @@ fi_source = self.context.sources[fi_source_name].fi_source() if fi_source: fi_sources.append(fi_source) - + for grid, extent, cache_source in self.context.caches[cache_name].caches(): - if dimensions and not isinstance(cache_source.cache, DummyCache): + disable_storage = self.context.configuration['caches'][cache_name].get('disable_storage', False) + if disable_storage: + supported_cache_class = DummyCache + else: + supported_cache_class = FileCache + if dimensions and not isinstance(cache_source.cache, supported_cache_class): # caching of dimension layers is not supported yet raise ConfigurationError( "caching of dimension layer (%s) is not supported yet." @@ -1833,6 +1888,28 @@ return extents +plugin_services = {} + +def register_service_configuration(service_name, service_creator, + yaml_spec_service_name = None, yaml_spec_service_def = None): + """ Method used by plugins to register a new service. + + :param service_name: Name of the service + :type service_name: str + :param service_creator: Creator method of the service + :type service_creator: method of type (serviceConfiguration: ServiceConfiguration, conf: dict) -> Server + :param yaml_spec_service_name: Name of the service in the YAML configuration file + :type yaml_spec_service_name: str + :param yaml_spec_service_def: Definition of the service in the YAML configuration file + :type yaml_spec_service_def: dict + """ + + log.info('Registering configuration for plugin service %s' % service_name) + plugin_services[service_name] = service_creator + if yaml_spec_service_name is not None and yaml_spec_service_def is not None: + add_service_to_mapproxy_yaml_spec(yaml_spec_service_name, yaml_spec_service_def) + + class ServiceConfiguration(ConfigurationBase): def __init__(self, conf, context): if 'wms' in conf: @@ -1849,9 +1926,15 @@ for service_name, service_conf in iteritems(self.conf): creator = getattr(self, service_name + '_service', None) if not creator: - raise ValueError('unknown service: %s' % service_name) + # If not a known service, try to use the plugin mechanism + global plugin_services + creator = plugin_services.get(service_name, None) + if not creator: + raise ValueError('unknown service: %s' % service_name) + new_services = creator(self, service_conf or {}) + else: + new_services = creator(service_conf or {}) - new_services = creator(service_conf or {}) # a creator can return a list of services... if not isinstance(new_services, (list, tuple)): new_services = [new_services] @@ -1936,7 +2019,7 @@ fi_template = conf.get('restful_featureinfo_template') if template and '{{' in template: # TODO remove warning in 1.6 - log.warn("double braces in WMTS restful_template are deprecated {{x}} -> {x}") + log.warning("double braces in WMTS restful_template are deprecated {{x}} -> {x}") services.append( WMTSRestServer( layers, md, template=template, @@ -1974,7 +2057,7 @@ for format in image_formats_names: opts = self.context.globals.image_options.image_opts({}, format) if opts.format in image_formats: - log.warn('duplicate mime-type for WMS image_formats: "%s" already configured, will use last format', + log.warning('duplicate mime-type for WMS image_formats: "%s" already configured, will use last format', opts.format) image_formats[opts.format] = opts info_types = conf.get('featureinfo_types') @@ -2053,7 +2136,27 @@ image_formats=image_formats, srs=srs, services=services, restful_template=restful_template) +def load_plugins(): + """ Locate plugins that belong to the 'mapproxy' group and load them """ + if sys.version_info >= (3, 8): + from importlib import metadata as importlib_metadata + else: + try: + import importlib_metadata + except ImportError: + return + + for dist in importlib_metadata.distributions(): + for ep in dist.entry_points: + if ep.group == 'mapproxy': + log.info('Loading plugin from package %s' % dist.metadata['name']) + ep.load().plugin_entrypoint() + + def load_configuration(mapproxy_conf, seed=False, ignore_warnings=True, renderd=False): + + load_plugins() + conf_base_dir = os.path.abspath(os.path.dirname(mapproxy_conf)) # A configuration is checked/validated four times, each step has a different @@ -2068,16 +2171,14 @@ conf_dict = load_configuration_file([os.path.basename(mapproxy_conf)], conf_base_dir) except YAMLError as ex: raise ConfigurationError(ex) - errors, informal_only = validate_options(conf_dict) for error in errors: - log.warn(error) + log.warning(error) if not informal_only or (errors and not ignore_warnings): raise ConfigurationError('invalid configuration') - errors = validate_references(conf_dict) for error in errors: - log.warn(error) + log.warning(error) return ProxyConfiguration(conf_dict, conf_base_dir=conf_base_dir, seed=seed, renderd=renderd) @@ -2092,9 +2193,7 @@ conf_file = os.path.normpath(os.path.join(working_dir, conf_file)) log.info('reading: %s' % conf_file) current_dict = load_yaml_file(conf_file) - conf_dict['__config_files__'][os.path.abspath(conf_file)] = os.path.getmtime(conf_file) - if 'base' in current_dict: current_working_dir = os.path.dirname(conf_file) base_files = current_dict.pop('base') @@ -2102,9 +2201,8 @@ base_files = [base_files] imported_dict = load_configuration_file(base_files, current_working_dir) current_dict = merge_dict(current_dict, imported_dict) - conf_dict = merge_dict(conf_dict, current_dict) - + return conf_dict def merge_dict(conf, base): @@ -2149,5 +2247,3 @@ return r, g, b, a return r, g, b - - diff -Nru mapproxy-1.12.0/mapproxy/config/spec.py mapproxy-1.15.1/mapproxy/config/spec.py --- mapproxy-1.12.0/mapproxy/config/spec.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/config/spec.py 2022-06-14 12:39:29.000000000 +0000 @@ -36,6 +36,16 @@ else: return [], True +time_spec = { + 'seconds': number(), + 'minutes': number(), + 'hours': number(), + 'days': number(), + 'weeks': number(), + 'time': anything(), + 'mtime': str(), +} + coverage = recursive({ 'polygons': str(), 'polygons_srs': str(), @@ -71,9 +81,11 @@ 'client_timeout': number(), 'ssl_no_cert_checks': bool(), 'ssl_ca_certs': str(), + 'hide_error_details': bool(), 'headers': { anything(): str() }, + 'manage_cookies': bool(), } mapserver_opts = { @@ -177,6 +189,7 @@ anything(): { required('response'): one_of([int], str), 'cache': bool, + 'authorize_stale': bool } } @@ -417,6 +430,7 @@ 'cache_rescaled_tiles': bool(), 'upscale_tiles': int(), 'downscale_tiles': int(), + 'refresh_before': time_spec, 'watermark': { 'text': string_type, 'font_size': number(), @@ -586,8 +600,33 @@ } })]) ), - # `parts` can be used for partial configurations that are referenced - # from other sections (e.g. coverages, dimensions, etc.) + # `parts` can be used for partial configurations that are referenced + # from other sections (e.g. coverages, dimensions, etc.) 'parts': anything(), } + +def add_source_to_mapproxy_yaml_spec(source_name, source_spec): + """ Add a new source type to mapproxy_yaml_spec. + Used by plugins. + """ + + # sources has a single anything() : {} member + values = list(mapproxy_yaml_spec['sources'].values()) + assert len(values) == 1 + values[0].add_subspec(source_name, source_spec) + + +def add_service_to_mapproxy_yaml_spec(service_name, service_spec): + """ Add a new service type to mapproxy_yaml_spec. + Used by plugins. + """ + + mapproxy_yaml_spec['services'][service_name] = service_spec + + +def add_subcategory_to_layer_md(category_name, category_def): + """ Add a new category to wms_130_layer_md. + Used by plugins + """ + wms_130_layer_md[category_name] = category_def diff -Nru mapproxy-1.12.0/mapproxy/config/validator.py mapproxy-1.15.1/mapproxy/config/validator.py --- mapproxy-1.12.0/mapproxy/config/validator.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/config/validator.py 2022-06-14 12:39:29.000000000 +0000 @@ -80,8 +80,9 @@ if source in self.caches_conf: self._validate_cache(source, self.caches_conf[source]) continue + + source, layers = self._split_tagged_source(source) if source in self.sources_conf: - source, layers = self._split_tagged_source(source) self._validate_source(source, self.sources_conf[source], layers) continue @@ -104,7 +105,6 @@ ) ) - def _split_tagged_source(self, source_name): layers = None if ':' in str(source_name): diff -Nru mapproxy-1.12.0/mapproxy/config_template/base_config/full_example.yaml mapproxy-1.15.1/mapproxy/config_template/base_config/full_example.yaml --- mapproxy-1.12.0/mapproxy/config_template/base_config/full_example.yaml 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/config_template/base_config/full_example.yaml 2022-06-14 12:39:29.000000000 +0000 @@ -429,6 +429,21 @@ transparent: true layers: securelayer + # WMS source that requires authentication and session management + # through HTTP cookies + session_source: + type: wms + http: + # Accept session cookies and forward on subsequent requests + manage_cookies: true + # Use basic auth header directly + headers: + Authorization: Basic YWRtaW46Z2Vvc2VydmVy + req: + url: https://my-service.com/service? + transparent: true + layers: securelayer + feature_info_source: type: wms wms_opts: diff -Nru mapproxy-1.12.0/mapproxy/config_template/base_config/mapproxy.yaml mapproxy-1.15.1/mapproxy/config_template/base_config/mapproxy.yaml --- mapproxy-1.12.0/mapproxy/config_template/base_config/mapproxy.yaml 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/config_template/base_config/mapproxy.yaml 2022-06-14 12:39:29.000000000 +0000 @@ -18,7 +18,7 @@ # first tile: http://localhost:8080/tiles/osm/webmercator/0/0/0.png # TMS: # note: TMS is not compatible with OSM/Google Maps/etc. -# fist tile: http://localhost:8080/tms/1.0.0/osm/webmercator/0/0/0.png +# first tile: http://localhost:8080/tms/1.0.0/osm/webmercator/0/0/0.png # KML: # initial doc: http://localhost:8080/kml/osm/webmercator diff -Nru mapproxy-1.12.0/mapproxy/exception.py mapproxy-1.15.1/mapproxy/exception.py --- mapproxy-1.12.0/mapproxy/exception.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/exception.py 2022-06-14 12:39:29.000000000 +0000 @@ -16,8 +16,8 @@ """ Service exception handling (WMS exceptions, XML, in_image, etc.). """ -import cgi from mapproxy.response import Response +from mapproxy.compat.modules import escape class RequestError(Exception): """ @@ -44,11 +44,13 @@ """ if self.request is not None: handler = self.request.exception_handler - return handler.render(self) + resp = handler.render(self) elif self.status is not None: - return Response(self.msg, status=self.status) + resp = Response(self.msg, status=self.status) else: - return Response('internal error: %s' % self.msg, status=500) + resp = Response('internal error: %s' % self.msg, status=500) + resp.cache_headers(no_cache=True) + return resp def __str__(self): return 'RequestError("%s", code=%r, request=%r)' % (self.msg, self.code, @@ -116,7 +118,7 @@ """ status_code = self.status_codes.get(request_error.code, self.status_code) # escape &<> in error message (e.g. URL params) - msg = cgi.escape(request_error.msg) + msg = escape(request_error.msg) result = self.template.substitute(exception=msg, code=request_error.code) return Response(result, mimetype=self.mimetype, content_type=self.content_type, diff -Nru mapproxy-1.12.0/mapproxy/image/__init__.py mapproxy-1.15.1/mapproxy/image/__init__.py --- mapproxy-1.12.0/mapproxy/image/__init__.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/image/__init__.py 2022-06-14 12:39:29.000000000 +0000 @@ -89,7 +89,24 @@ return tags -class ImageSource(object): +class BaseImageSource(object): + """ + Virtual parent class for ImageSource and BlankImageSource + """ + def __init__(self): + raise Exception("Virtual class BaseImageSource, cannot be instanciated.") + + def as_image(self): + raise Exception("Virtual class BaseImageSource, method as_image cannot be called.") + + def as_buffer(self, image_opts=None, format=None, seekable=False): + raise Exception("Virtual class BaseImageSource, method as_buffer cannot be called.") + + def close_buffers(self): + pass + + +class ImageSource(BaseImageSource): """ This class wraps either a PIL image, a file-like object, or a file name. You can access the result as an image (`as_image` ) or a file-like buffer @@ -111,6 +128,7 @@ self._size = size self.cacheable = cacheable self.georef = georef + self.authorize_stale = False @property def source(self): @@ -238,7 +256,7 @@ img.paste(subimg, offset) return ImageSource(img, size=size, image_opts=new_image_opts, cacheable=cacheable) -class BlankImageSource(object): +class BlankImageSource(BaseImageSource): """ ImageSource for transparent or solid-color images. Implements optimized as_buffer() method. @@ -249,6 +267,7 @@ self._buf = None self._img = None self.cacheable = cacheable + self.authorize_stale = False def as_image(self): if not self._img: diff -Nru mapproxy-1.12.0/mapproxy/image/message.py mapproxy-1.15.1/mapproxy/image/message.py --- mapproxy-1.12.0/mapproxy/image/message.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/image/message.py 2022-06-14 12:39:29.000000000 +0000 @@ -99,11 +99,11 @@ self.font_size) except ImportError: _pil_ttf_support = False - log_system.warn("Couldn't load TrueType fonts, " + log_system.warning("Couldn't load TrueType fonts, " "PIL needs to be build with freetype support.") except IOError: _pil_ttf_support = False - log_system.warn("Couldn't load find TrueType font ", self.font_name) + log_system.warning("Couldn't load find TrueType font ", self.font_name) if self._font is None: self._font = ImageFont.load_default() return self._font diff -Nru mapproxy-1.12.0/mapproxy/image/tile.py mapproxy-1.15.1/mapproxy/image/tile.py --- mapproxy-1.12.0/mapproxy/image/tile.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/image/tile.py 2022-06-14 12:39:29.000000000 +0000 @@ -65,7 +65,7 @@ source.close_buffers() except IOError as e: if e.errno is None: # PIL error - log.warn('unable to load tile %s, removing it (reason was: %s)' + log.warning('unable to load tile %s, removing it (reason was: %s)' % (source, str(e))) if getattr(source, 'filename'): if os.path.exists(source.filename): diff -Nru mapproxy-1.12.0/mapproxy/layer.py mapproxy-1.15.1/mapproxy/layer.py --- mapproxy-1.12.0/mapproxy/layer.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/layer.py 2022-06-14 12:39:29.000000000 +0000 @@ -136,7 +136,11 @@ return dict((k, v) for k, v in iteritems(self.dimensions) if k.lower() in params) def __repr__(self): - return "MapQuery(bbox=%(bbox)s, size=%(size)s, srs=%(srs)r, format=%(format)s)" % self.__dict__ + info = self.__dict__ + serialized_dimensions = ", ".join(["'%s': '%s'" % (key, value) for (key, value) in self.dimensions.items()]) + info["serialized_dimensions"] = serialized_dimensions + return "MapQuery(bbox=%(bbox)s, size=%(size)s, srs=%(srs)r, format=%(format)s, dimensions={%(serialized_dimensions)s)}" % info + class InfoQuery(object): def __init__(self, bbox, size, srs, pos, info_format, format=None, @@ -402,7 +406,7 @@ size, offset, bbox = bbox_position_in_image(query.bbox, query.size, self.extent.bbox_for(query.srs)) if size[0] == 0 or size[1] == 0: raise BlankImage() - src_query = MapQuery(bbox, size, query.srs, query.format) + src_query = MapQuery(bbox, size, query.srs, query.format, dimensions=query.dimensions) resp = self._image(src_query) result = SubImageSource(resp, size=query.size, offset=offset, image_opts=self.image_opts, cacheable=resp.cacheable) @@ -429,7 +433,7 @@ num_tiles = tile_grid[0] * tile_grid[1] if self.max_tile_limit and num_tiles >= self.max_tile_limit: - raise MapBBOXError("too many tiles") + raise MapBBOXError("too many tiles, max_tile_limit: %s, num_tiles: %s" % (self.max_tile_limit, num_tiles)) if query.tiled_only: if num_tiles > 1: @@ -440,7 +444,7 @@ raise MapBBOXError("query does not align to tile boundaries") with self.tile_manager.session(): - tile_collection = self.tile_manager.load_tile_coords(affected_tile_coords, with_metadata=query.tiled_only) + tile_collection = self.tile_manager.load_tile_coords(affected_tile_coords, with_metadata=query.tiled_only, dimensions=query.dimensions) if tile_collection.empty: raise BlankImage() diff -Nru mapproxy-1.12.0/mapproxy/proj.py mapproxy-1.15.1/mapproxy/proj.py --- mapproxy-1.12.0/mapproxy/proj.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/proj.py 2022-06-14 12:39:29.000000000 +0000 @@ -14,7 +14,7 @@ # limitations under the License. """ -ctypes based replacement of pyroj (with pyproj fallback). +ctypes based replacement of pyroj (with pyproj fallback) of the old PROJ 4 API. This module implements the `Proj`, `transform` and `set_datapath` class/functions. This module is a drop-in replacement for pyproj. It does implement just enough to work for @@ -24,6 +24,8 @@ to pyroj. You can force the usage of either backend by setting the environment variables MAPPROXY_USE_LIBPROJ or MAPPROXY_USE_PYPROJ to any value. +The new PROJ >=5 API is only supported via pyproj. See USE_PROJ4_API. + """ from __future__ import print_function @@ -57,6 +59,10 @@ if libproj is None: return + if hasattr(libproj, 'proj_create'): + log_system.warning('Found libproj >=5. Using this library without pyproj is ' + 'deprecated and not fully supported. Please install pyproj >= 2.') + libproj.pj_init_plus.argtypes = [c_char_p] libproj.pj_init_plus.restype = c_void_p @@ -127,14 +133,24 @@ class ProjInitError(ProjError): pass -def try_pyproj_import(): +def try_pyproj4_import(): try: from pyproj import Proj, transform, set_datapath except ImportError: return False - log_system.info('using pyproj for coordinate transformation') + log_system.info('using pyproj with old Proj4 API for coordinate transformation') return Proj, transform, set_datapath +def try_pyproj_import(): + try: + from pyproj import CRS + from pyproj.transformer import Transformer + from pyproj.datadir import set_data_dir + except ImportError: + return False + log_system.info('using pyproj for coordinate transformation') + return CRS, Transformer, set_data_dir + def try_libproj_import(): libproj = init_libproj() @@ -244,22 +260,33 @@ proj_imports = [try_libproj_import] if 'MAPPROXY_USE_PYPROJ' in os.environ: - proj_imports = [try_pyproj_import] + proj_imports = [try_pyproj_import, try_pyproj4_import] if not proj_imports: if sys.platform == 'win32': # prefer pyproj on windows - proj_imports = [try_pyproj_import, try_libproj_import] + proj_imports = [try_pyproj_import, try_pyproj4_import, try_libproj_import] else: - proj_imports = [try_libproj_import, try_pyproj_import] + proj_imports = [try_pyproj_import, try_libproj_import, try_pyproj4_import] +# try different imports in previously defined order for try_import in proj_imports: - res = try_import() - if res: - Proj, transform, set_datapath = res - break + if try_import == try_pyproj_import: + res = try_import() + if res: + CRS, Transformer, set_datapath = res + Proj, transform = None, None + USE_PROJ4_API = False + break + else: + res = try_import() + if res: + Proj, transform, set_datapath = res + CRS, Transformer = None, None + USE_PROJ4_API = True + break else: - raise ImportError('could not find libproj or pyproj') + raise ImportError('could not find pyproj (Python library) or libproj (C library, deprecated)') if __name__ == '__main__': diff -Nru mapproxy-1.12.0/mapproxy/request/base.py mapproxy-1.15.1/mapproxy/request/base.py --- mapproxy-1.12.0/mapproxy/request/base.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/request/base.py 2022-06-14 12:39:29.000000000 +0000 @@ -16,15 +16,9 @@ """ Service requests (parsing, handling, etc). """ -import cgi - +from mapproxy.compat import PY2, iteritems, text_type +from mapproxy.compat.modules import parse_qsl, urlparse, quote from mapproxy.util.py import cached_property -from mapproxy.compat import iteritems, PY2, text_type - -if PY2: - from urllib import quote -else: - from urllib.parse import quote class NoCaseMultiDict(dict): """ @@ -178,7 +172,7 @@ Parse query string `qs` and return a `NoCaseMultiDict`. """ tmp = [] - for key, value in cgi.parse_qsl(qs, include_empty): + for key, value in parse_qsl(qs, include_empty): if PY2: if decode_keys: key = key.decode(charset, errors) @@ -265,12 +259,27 @@ def host_url(self): return '%s://%s/' % (self.url_scheme, self.host) + @cached_property + def server_url(self): + return 'http://%s:%s/' % ( + self.environ['SERVER_NAME'], + self.environ['SERVER_PORT'] + ) + @property def script_url(self): "Full script URL without trailing /" return (self.host_url.rstrip('/') + quote(self.environ.get('SCRIPT_NAME', '/').rstrip('/')) ) + + @property + def server_script_url(self): + "Internal script URL" + return self.script_url.replace( + self.host_url.rstrip('/'), + self.server_url.rstrip('/') + ) @property def base_url(self): @@ -393,7 +402,7 @@ """ request_params = RequestParams - def __init__(self, param=None, url='', validate=False, http=None): + def __init__(self, param=None, url='', validate=False, http=None, dimensions=None): self.delimiter = ',' self.http = http diff -Nru mapproxy-1.12.0/mapproxy/request/wms/__init__.py mapproxy-1.15.1/mapproxy/request/wms/__init__.py --- mapproxy-1.12.0/mapproxy/request/wms/__init__.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/request/wms/__init__.py 2022-06-14 12:39:29.000000000 +0000 @@ -17,6 +17,7 @@ Service requests (parsing, handling, etc). """ import codecs +import re from mapproxy.request.wms import exception from mapproxy.exception import RequestError from mapproxy.srs import SRS, make_lin_transf @@ -185,10 +186,29 @@ #pylint: disable-msg=E1102 xml_exception_handler = None prevent_image_exception = False + dimension_params = ['time', 'elevation'] + dimension_prefix = 'dim_' def __init__(self, param=None, url='', validate=False, non_strict=False, **kw): + self.dimensions = self._get_dimensions(param) WMSRequest.__init__(self, param=param, url=url, validate=validate, non_strict=non_strict, **kw) + + def _get_dimensions(self, param): + if param: + regex = "(?i)%s%s" % (("^%s|" % self.dimension_prefix if self.dimension_prefix else ""), + "^(%s)$" % "|".join(self.dimension_params)) + keys = [] + if isinstance(param, RequestParams): + keys = list(map (lambda k: k[0], param.iteritems())) + else: + keys = list(param.keys()) + if len(keys) > 0: + return dict(map(lambda k: (k, param.get(k)), filter (lambda k: re.search(regex, k), keys))) + else: + return None + else: + return None def validate(self): self.validate_param() @@ -280,6 +300,8 @@ fixed_params = {'request': 'map', 'wmtver': '1.0.0'} expected_param = ['wmtver', 'request', 'layers', 'styles', 'srs', 'bbox', 'width', 'height', 'format'] + dimension_params = [] + dimension_prefix = '' def adapt_to_111(self): del self.params['wmtver'] self.params['version'] = '1.0.0' @@ -331,7 +353,7 @@ if SRS(srs).is_axis_order_ne: return bbox[1], bbox[0], bbox[3], bbox[2] except RuntimeError: - log.warn('unknown SRS %s' % srs) + log.warning('unknown SRS %s' % srs) return bbox def _switch_bbox(self): diff -Nru mapproxy-1.12.0/mapproxy/request/wmts.py mapproxy-1.15.1/mapproxy/request/wmts.py --- mapproxy-1.12.0/mapproxy/request/wmts.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/request/wmts.py 2022-06-14 12:39:29.000000000 +0000 @@ -318,7 +318,7 @@ if self._regexp: return self._regexp converted_re = self.var_re.sub(self.substitute_var, re.escape(self.template)) - wmts_re = re.compile('/wmts' + converted_re) + wmts_re = re.compile(r'/wmts' + converted_re) if not self.found.issuperset(self.required): raise InvalidWMTSTemplate('missing required variables in WMTS restful template: %s' % self.required.difference(self.found)) diff -Nru mapproxy-1.12.0/mapproxy/response.py mapproxy-1.15.1/mapproxy/response.py --- mapproxy-1.12.0/mapproxy/response.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/response.py 2022-06-14 12:39:29.000000000 +0000 @@ -42,6 +42,12 @@ content_type = self.default_content_type self.headers['Content-type'] = content_type + if content_type.startswith(('text/', 'application/')): + # Capability documents can be dependent on the value of a few X-headers. + # Tell this caching proxies via the Vary HTTP header. This also prevents + # malicious cache poisoning. + self.headers['Vary'] = 'X-Script-Name, X-Forwarded-Host, X-Forwarded-Proto' + def _status_set(self, status): if isinstance(status, int): status = status_code(status) diff -Nru mapproxy-1.12.0/mapproxy/script/conf/app.py mapproxy-1.15.1/mapproxy/script/conf/app.py --- mapproxy-1.12.0/mapproxy/script/conf/app.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/script/conf/app.py 2022-06-14 12:39:29.000000000 +0000 @@ -71,7 +71,10 @@ @contextmanager def file_or_stdout(name): if name == '-': - yield codecs.getwriter('utf-8')(sys.stdout) + if hasattr(sys.stdout, 'buffer'): + yield codecs.getwriter('utf-8')(sys.stdout.buffer) + else: + yield codecs.getwriter('utf-8')(sys.stdout) else: with open(name, 'wb') as f: yield codecs.getwriter('utf-8')(f) diff -Nru mapproxy-1.12.0/mapproxy/script/conf/sources.py mapproxy-1.15.1/mapproxy/script/conf/sources.py --- mapproxy-1.12.0/mapproxy/script/conf/sources.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/script/conf/sources.py 2022-06-14 12:39:29.000000000 +0000 @@ -34,7 +34,7 @@ SRS(srs) _checked_srs[srs] = True except Exception as ex: - logging.getLogger(__name__).warn('unable to initialize srs for %s: %s', srs, ex) + logging.getLogger(__name__).warning('unable to initialize srs for %s: %s', srs, ex) _checked_srs[srs] = False return _checked_srs[srs] diff -Nru mapproxy-1.12.0/mapproxy/script/defrag.py mapproxy-1.15.1/mapproxy/script/defrag.py --- mapproxy-1.12.0/mapproxy/script/defrag.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/script/defrag.py 2022-06-14 12:39:29.000000000 +0000 @@ -108,7 +108,7 @@ >>> bundle_offset("path/to/R0380C1380.bundle") (4992, 896) """ - match = re.search('R([A-F0-9]{4,})C([A-F0-9]{4,}).bundle$', fname, re.IGNORECASE) + match = re.search(r'R([A-F0-9]{4,})C([A-F0-9]{4,}).bundle$', fname, re.IGNORECASE) if match: r = int(match.group(1), 16) c = int(match.group(2), 16) diff -Nru mapproxy-1.12.0/mapproxy/script/export.py mapproxy-1.15.1/mapproxy/script/export.py --- mapproxy-1.12.0/mapproxy/script/export.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/script/export.py 2022-06-14 12:39:29.000000000 +0000 @@ -48,7 +48,7 @@ levels = set() for part in level_str.split(','): part = part.strip() - if re.match('\d+..\d+', part): + if re.match(r'\d+..\d+', part): from_level, to_level = part.split('..') levels.update(list(range(int(from_level), int(to_level) + 1))) else: diff -Nru mapproxy-1.12.0/mapproxy/script/util.py mapproxy-1.15.1/mapproxy/script/util.py --- mapproxy-1.12.0/mapproxy/script/util.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/script/util.py 2022-06-14 12:39:29.000000000 +0000 @@ -25,6 +25,7 @@ import logging from mapproxy.compat import iteritems +from mapproxy.config.loader import load_plugins from mapproxy.script.conf.app import config_command from mapproxy.script.defrag import defrag_command from mapproxy.script.export import export_command @@ -155,7 +156,7 @@ if ':' in address: host, port = address.split(':', 1) port = int(port) - elif re.match('^\d+$', address): + elif re.match(r'^\d+$', address): host = default[0] port = int(address) else: @@ -321,6 +322,18 @@ } +def register_command(command_name, command_spec): + """ Method used by plugins to register a command. + + :param command_name: Name of the service + :type command_name: str + :param command_spec: Definition of the command. Dictionary with a 'func' and 'help' member + :type command_spec: dict + """ + + commands[command_name] = command_spec + + class NonStrictOptionParser(optparse.OptionParser): def _process_args(self, largs, rargs, values): while rargs: @@ -361,6 +374,9 @@ print(' %s%s' % (name, help), file=sys.stdout) def main(): + + load_plugins() + parser = NonStrictOptionParser("usage: %prog COMMAND [options]", add_help_option=False) options, args = parser.parse_args() @@ -387,4 +403,4 @@ commands[command]['func'](args) if __name__ == '__main__': - main() \ No newline at end of file + main() diff -Nru mapproxy-1.12.0/mapproxy/seed/config.py mapproxy-1.15.1/mapproxy/seed/config.py --- mapproxy-1.12.0/mapproxy/seed/config.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/seed/config.py 2022-06-14 12:39:29.000000000 +0000 @@ -58,7 +58,7 @@ else: errors, informal_only = validate_seed_conf(conf) for error in errors: - log.warn(error) + log.warning(error) if not informal_only: raise SeedConfigurationError('invalid configuration') seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf) diff -Nru mapproxy-1.12.0/mapproxy/seed/script.py mapproxy-1.15.1/mapproxy/seed/script.py --- mapproxy-1.12.0/mapproxy/seed/script.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/seed/script.py 2022-06-14 12:39:29.000000000 +0000 @@ -105,6 +105,11 @@ metavar="N", help="do not check for intersections between tiles" " and seed geometries on the last N levels") + parser.add_option("--skip-uncached", + action="store_true", dest="skip_uncached", default=False, + help="only treat tiles which are already present in the cache." + " This can be used with the configuration entry `refresh_before`" + " to refresh only the existing cache.") parser.add_option("--summary", action="store_true", dest="summary", default=False, help="print summary with all seeding tasks and exit." @@ -243,7 +248,8 @@ progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, - skip_geoms_for_last_levels=options.geom_levels) + skip_geoms_for_last_levels=options.geom_levels, + skip_uncached=options.skip_uncached) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % ( diff -Nru mapproxy-1.12.0/mapproxy/seed/seeder.py mapproxy-1.15.1/mapproxy/seed/seeder.py --- mapproxy-1.12.0/mapproxy/seed/seeder.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/seed/seeder.py 2022-06-14 12:39:29.000000000 +0000 @@ -43,9 +43,13 @@ CONTAINS = -1 INTERSECTS = 1 -# do not use multiprocessing on windows, it blows -# no lambdas, no anonymous functions/classes, no base_config(), etc. -if sys.platform == 'win32': +# Decide whether to use multiprocessing or threading. multiprocessing should be faster but +# it is not well supported on all platforms. Especially regarding lambdas and anonymous +# function/classes which are used in proj.py for example. +# +# Since Python 3.8, MacOS uses a non-forking start method for multiprocessing which +# inhibits similar restrictions to Windows. +if sys.platform == 'win32' or (sys.platform == 'darwin' and sys.version_info >= (3, 8)): import threading proc_class = threading.Thread queue_class = Queue.Queue @@ -83,7 +87,7 @@ alive = True break if not alive: - log.warn('no workers left, stopping') + log.warning('no workers left, stopping') raise SeedInterrupted continue else: @@ -473,7 +477,7 @@ return NONE def seed(tasks, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0, - progress_logger=None, cache_locker=None): + progress_logger=None, cache_locker=None, skip_uncached=False): if cache_locker is None: cache_locker = DummyCacheLocker() @@ -492,7 +496,7 @@ start_progress = None seed_progress = SeedProgress(old_progress_identifier=start_progress) seed_task(task, concurrency, dry_run, skip_geoms_for_last_levels, progress_logger, - seed_progress=seed_progress) + seed_progress=seed_progress, skip_uncached=skip_uncached) except CacheLockedError: print(' ...cache is locked, skipping') active_tasks = [task] + active_tasks[:-1] @@ -501,7 +505,7 @@ def seed_task(task, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0, - progress_logger=None, seed_progress=None): + progress_logger=None, seed_progress=None, skip_uncached=False): if task.coverage is False: return if task.refresh_timestamp is not None: @@ -514,7 +518,11 @@ tile_worker_pool = TileWorkerPool(task, TileSeedWorker, dry_run=dry_run, size=concurrency, progress_logger=progress_logger) - tile_walker = TileWalker(task, tile_worker_pool, handle_uncached=True, + # If the configuration requests to only refresh tiles which are already in cache, + # tile walker parameters shall be adapted + handle_stale = skip_uncached + handle_uncached = not skip_uncached + tile_walker = TileWalker(task, tile_worker_pool, handle_uncached=handle_uncached, handle_stale=handle_stale, skip_geoms_for_last_levels=skip_geoms_for_last_levels, progress_logger=progress_logger, seed_progress=seed_progress, work_on_metatiles=work_on_metatiles, diff -Nru mapproxy-1.12.0/mapproxy/seed/spec.py mapproxy-1.15.1/mapproxy/seed/spec.py --- mapproxy-1.12.0/mapproxy/seed/spec.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/seed/spec.py 2022-06-14 12:39:29.000000000 +0000 @@ -17,7 +17,7 @@ from mapproxy.util.ext.dictspec.spec import one_off, anything, number from mapproxy.util.ext.dictspec.spec import required -from mapproxy.config.spec import coverage +from mapproxy.config.spec import coverage, time_spec def validate_seed_conf(conf_dict): """ @@ -31,16 +31,6 @@ else: return [], True -time_spec = { - 'seconds': number(), - 'minutes': number(), - 'hours': number(), - 'days': number(), - 'weeks': number(), - 'time': anything(), - 'mtime': str(), -} - from_to_spec = { 'from': number(), 'to': number(), diff -Nru mapproxy-1.12.0/mapproxy/seed/util.py mapproxy-1.15.1/mapproxy/seed/util.py --- mapproxy-1.12.0/mapproxy/seed/util.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/seed/util.py 2022-06-14 12:39:29.000000000 +0000 @@ -191,7 +191,7 @@ time.sleep(0.01) except exceptions as ex: if n >= max_repeat: - print("An error occured. Giving up", file=sys.strerr) + print("An error occured. Giving up", file=sys.stderr) raise BackoffError wait_for = start_backoff_sec * 2**n if wait_for > max_backoff: diff -Nru mapproxy-1.12.0/mapproxy/service/demo.py mapproxy-1.15.1/mapproxy/service/demo.py --- mapproxy-1.12.0/mapproxy/service/demo.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/demo.py 2022-06-14 12:39:29.000000000 +0000 @@ -41,6 +41,40 @@ env = {'bunch': bunch} get_template = template_loader(__name__, 'templates', namespace=env) +# Used by plugins +extra_demo_server_handlers = set() + +def register_extra_demo_server_handler(handler): + """ Method used by plugins to register a new handler for the demo service. + The handler passed to this method is invoked by the DemoServer.handle() + method when receiving an incoming request, and let a chance to the + handler to process it, if it is relevant to it. + + :param handler: New handler for incoming requests + :type handler: function that takes 2 arguments (DemoServer instance and req) and + returns a string with HTML content or None + """ + + extra_demo_server_handlers.add(handler) + + +extra_substitution_handlers = set() + +def register_extra_demo_substitution_handler(handler): + """ Method used by plugins to register a new handler for doing substitutions + to the HTML template used by the demo service. + The handler passed to this method is invoked by the DemoServer._render_template() + method. The handler may modify the passed substitutions dictionary + argument. Keys of particular interest are 'extra_services_html_beginning' + and 'extra_services_html_end' to add HTML content before/after built-in + services. + + :param handler: New handler for incoming requests + :type handler: function that takes 3 arguments(DemoServer instance, req and a substitutions dictionary argument). + """ + + extra_substitution_handlers.add(handler) + def static_filename(name): if base_config().template_dir: @@ -68,6 +102,8 @@ def handle(self, req): if req.path.startswith('/demo/static/'): + if '..' in req.path: + return Response('file not found', content_type='text/plain', status=404) filename = req.path.lstrip('/') filename = static_filename(filename) if not os.path.isfile(filename): @@ -84,6 +120,11 @@ if not authorized: return Response('forbidden', content_type='text/plain', status=403) + for handler in extra_demo_server_handlers: + demo = handler(self, req) + if demo is not None: + return Response(demo, content_type='text/html') + if 'wms_layer' in req.args: demo = self._render_wms_template('demo/wms_demo.html', req) elif 'tms_layer' in req.args: @@ -91,33 +132,38 @@ elif 'wmts_layer' in req.args: demo = self._render_wmts_template('demo/wmts_demo.html', req) elif 'wms_capabilities' in req.args: - url = '%s/service?REQUEST=GetCapabilities'%(req.script_url) - capabilities = urllib2.urlopen(url) + internal_url = '%s/service?REQUEST=GetCapabilities'%(req.server_script_url) + url = internal_url.replace(req.server_script_url, req.script_url) + capabilities = urllib2.urlopen(internal_url) demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMS', url) elif 'wmsc_capabilities' in req.args: - url = '%s/service?REQUEST=GetCapabilities&tiled=true'%(req.script_url) - capabilities = urllib2.urlopen(url) + internal_url = '%s/service?REQUEST=GetCapabilities&tiled=true'%(req.server_script_url) + url = internal_url.replace(req.server_script_url, req.script_url) + capabilities = urllib2.urlopen(internal_url) demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMS-C', url) elif 'wmts_capabilities_kvp' in req.args: - url = '%s/service?REQUEST=GetCapabilities&SERVICE=WMTS' % (req.script_url) - capabilities = urllib2.urlopen(url) + internal_url = '%s/service?REQUEST=GetCapabilities&SERVICE=WMTS' % (req.server_script_url) + url = internal_url.replace(req.server_script_url, req.script_url) + capabilities = urllib2.urlopen(internal_url) demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMTS', url) elif 'wmts_capabilities' in req.args: - url = '%s/wmts/1.0.0/WMTSCapabilities.xml' % (req.script_url) - capabilities = urllib2.urlopen(url) + internal_url = '%s/wmts/1.0.0/WMTSCapabilities.xml' % (req.server_script_url) + url = internal_url.replace(req.server_script_url, req.script_url) + capabilities = urllib2.urlopen(internal_url) demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'WMTS', url) elif 'tms_capabilities' in req.args: if 'layer' in req.args and 'srs' in req.args: # prevent dir traversal (seems it's not possible with urllib2, but better safe then sorry) layer = req.args['layer'].replace('..', '') srs = req.args['srs'].replace('..', '') - url = '%s/tms/1.0.0/%s/%s'%(req.script_url, layer, srs) + internal_url = '%s/tms/1.0.0/%s/%s'%(req.server_script_url, layer, srs) else: - url = '%s/tms/1.0.0/'%(req.script_url) - capabilities = urllib2.urlopen(url) + internal_url = '%s/tms/1.0.0/'%(req.server_script_url) + capabilities = urllib2.urlopen(internal_url) + url = internal_url.replace(req.server_script_url, req.script_url) demo = self._render_capabilities_template('demo/capabilities_demo.html', capabilities, 'TMS', url) elif req.path == '/demo/': - demo = self._render_template('demo/demo.html') + demo = self._render_template(req, 'demo/demo.html') else: resp = Response('', status=301) resp.headers['Location'] = req.script_url.rstrip('/') + '/demo/' @@ -150,26 +196,40 @@ if srs_code not in cached_srs: uncached_srs.append(srs_code) - sorted_cached_srs = sorted(cached_srs, key=lambda srs: get_epsg_num(srs)) - sorted_uncached_srs = sorted(uncached_srs, key=lambda srs: get_epsg_num(srs)) + def get_srs_key(srs): + epsg_num = get_epsg_num(srs) + return str(epsg_num if epsg_num else srs) + + sorted_cached_srs = sorted(cached_srs, key=lambda srs: get_srs_key(srs)) + sorted_uncached_srs = sorted(uncached_srs, key=lambda srs: get_srs_key(srs)) sorted_cached_srs = [(s + '*', s) for s in sorted_cached_srs] sorted_uncached_srs = [(s, s) for s in sorted_uncached_srs] return sorted_cached_srs + sorted_uncached_srs - def _render_template(self, template): + def _render_template(self, req, template): template = get_template(template, default_inherit="demo/static.html") tms_tile_layers = defaultdict(list) for layer in self.tile_layers: name = self.tile_layers[layer].md.get('name') tms_tile_layers[name].append(self.tile_layers[layer]) wmts_layers = tms_tile_layers.copy() - return template.substitute(layers=self.layers, - formats=self.image_formats, - srs=self.srs, - layer_srs=self.layer_srs, - tms_layers=tms_tile_layers, - wmts_layers=wmts_layers, - services=self.services) + + substitutions = dict( + extra_services_html_beginning='', + extra_services_html_end='', + layers=self.layers, + formats=self.image_formats, + srs=self.srs, + layer_srs=self.layer_srs, + tms_layers=tms_tile_layers, + wmts_layers=wmts_layers, + services=self.services + ) + + for add_substitution in extra_substitution_handlers: + add_substitution(self, req, substitutions) + + return template.substitute(substitutions) def _render_wms_template(self, template, req): template = get_template(template, default_inherit="demo/static.html") diff -Nru mapproxy-1.12.0/mapproxy/service/template_helper.py mapproxy-1.15.1/mapproxy/service/template_helper.py --- mapproxy-1.12.0/mapproxy/service/template_helper.py 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/template_helper.py 2022-06-14 12:39:29.000000000 +0000 @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from cgi import escape from mapproxy.template import bunch +from mapproxy.compat.modules import escape __all__ = ['escape', 'indent', 'bunch', 'wms100format', 'wms100info_format', 'wms111metadatatype'] diff -Nru mapproxy-1.12.0/mapproxy/service/templates/demo/capabilities_demo.html mapproxy-1.15.1/mapproxy/service/templates/demo/capabilities_demo.html --- mapproxy-1.12.0/mapproxy/service/templates/demo/capabilities_demo.html 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/templates/demo/capabilities_demo.html 2022-06-14 12:39:29.000000000 +0000 @@ -1,5 +1,5 @@ {{py: -import cgi +from mapproxy.compat.modules import escape import textwrap wrapper = textwrap.TextWrapper(replace_whitespace=False, width=90, @@ -12,7 +12,7 @@ {{url}} 
 {{for line in capabilities}}
-{{cgi.escape(wrapper.fill(line.decode('utf8')))}}
+{{escape(wrapper.fill(line.decode('utf8')))}}
 {{endfor}}
diff -Nru mapproxy-1.12.0/mapproxy/service/templates/demo/demo.html mapproxy-1.15.1/mapproxy/service/templates/demo/demo.html --- mapproxy-1.12.0/mapproxy/service/templates/demo/demo.html 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/templates/demo/demo.html 2022-06-14 12:39:29.000000000 +0000 @@ -35,6 +35,7 @@ {{enddef}}

About

MapProxy Version {{version}}

+ {{ extra_services_html_beginning }}

WMS

{{if 'wms' in services}}
@@ -177,3 +178,4 @@ This service is not available with the current configuration.
{{endif}} + {{ extra_services_html_end }} diff -Nru mapproxy-1.12.0/mapproxy/service/templates/demo/tms_demo.html mapproxy-1.15.1/mapproxy/service/templates/demo/tms_demo.html --- mapproxy-1.12.0/mapproxy/service/templates/demo/tms_demo.html 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/templates/demo/tms_demo.html 2022-06-14 12:39:29.000000000 +0000 @@ -1,5 +1,5 @@ {{py: -import cgi +from mapproxy.compat.modules import escape import textwrap wrapper = textwrap.TextWrapper(replace_whitespace=False, width=90, @@ -76,6 +76,6 @@

JavaScript code

 {{for line in jscript_openlayers().split('\n')}}
-{{cgi.escape(wrapper.fill(line))}}
+{{escape(wrapper.fill(line))}}
 {{endfor}}
-
\ No newline at end of file + diff -Nru mapproxy-1.12.0/mapproxy/service/templates/demo/wms_demo.html mapproxy-1.15.1/mapproxy/service/templates/demo/wms_demo.html --- mapproxy-1.12.0/mapproxy/service/templates/demo/wms_demo.html 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/templates/demo/wms_demo.html 2022-06-14 12:39:29.000000000 +0000 @@ -1,5 +1,5 @@ {{py: -import cgi +from mapproxy.compat.modules import escape import textwrap wrapper = textwrap.TextWrapper(replace_whitespace=False, width=90, @@ -26,11 +26,19 @@ var layer = new OpenLayers.Layer.WMS( "WMS {{layer.name}}", "../service?", {layers: "{{layer.name}}", format: "{{format}}", srs:"{{srs}}", + {{if layer.dimensions}}{{for d in layer.dimensions.keys()}}"{{d}}": "{{layer.dimensions[d].default}}",{{endfor}}{{endif}} exceptions: "application/vnd.ogc.se_inimage"{{if format == 'image/png'}}, transparent: true{{endif}}}, {singleTile: true, ratio: 1, isBaseLayer: true} ); map.addLayer(layer); map.zoomToMaxExtent(); + + + } + + function setDimension (layer, dimension, value) { + layer.params[dimension.toUpperCase()] = value; + layer.redraw(); } {{enddef}} @@ -40,6 +48,9 @@ Coordinate System Image format + {{for dim in layer.dimensions}} + {{str(dim)}} + {{endfor}} @@ -64,6 +75,19 @@ {{endfor}} + {{for dim in layer.dimensions}} + + + + {{endfor}} @@ -72,6 +96,6 @@

JavaScript code

 {{for line in jscript_openlayers().split('\n')}}
-{{cgi.escape(wrapper.fill(line))}}
+{{escape(wrapper.fill(line))}}
 {{endfor}}
-
\ No newline at end of file + diff -Nru mapproxy-1.12.0/mapproxy/service/templates/demo/wmts_demo.html mapproxy-1.15.1/mapproxy/service/templates/demo/wmts_demo.html --- mapproxy-1.12.0/mapproxy/service/templates/demo/wmts_demo.html 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/templates/demo/wmts_demo.html 2022-06-14 12:39:29.000000000 +0000 @@ -1,5 +1,5 @@ {{py: -import cgi +from mapproxy.compat.modules import escape import textwrap wrapper = textwrap.TextWrapper(replace_whitespace=False, width=90, @@ -70,6 +70,6 @@

JavaScript code

 {{for line in jscript_openlayers().split('\n')}}
-{{cgi.escape(wrapper.fill(line))}}
+{{escape(wrapper.fill(line))}}
 {{endfor}}
-
\ No newline at end of file + diff -Nru mapproxy-1.12.0/mapproxy/service/templates/wms100capabilities.xml mapproxy-1.15.1/mapproxy/service/templates/wms100capabilities.xml --- mapproxy-1.12.0/mapproxy/service/templates/wms100capabilities.xml 2019-08-30 07:34:08.000000000 +0000 +++ mapproxy-1.15.1/mapproxy/service/templates/wms100capabilities.xml 2022-06-14 12:39:29.000000000 +0000 @@ -78,6 +78,17 @@ {{for srs_code, bbox in layer_srs_bbox(layer)}} {{endfor}} + {{if hasattr(layer, 'dimensions')}} + {{if layer.dimensions}} + {{for d in layer.dimensions}} + {{if "time" in d }} + {{','.join(layer.dimensions[d])}} + {{else}} + {{','.join(layer.dimensions[d])}} + {{endif}} + {{endfor}} + {{endif}} + {{endif}} {{if layer.is_active and layer.has_legend and layer.legend_url}}