diff -Nru fiona-1.8.4/appveyor.yml fiona-1.8.6/appveyor.yml --- fiona-1.8.4/appveyor.yml 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/appveyor.yml 2019-03-19 04:25:07.000000000 +0000 @@ -45,8 +45,8 @@ PYTHON_VERSION: "3.6.4" PYTHON_ARCH: "64" GDAL_VERSION: "2.4.0" - GIS_INTERNALS: "release-1911-x64-gdal-mapserver.zip" - GIS_INTERNALS_LIBS: "release-1911-x64-gdal-mapserver-libs.zip" + GIS_INTERNALS: "release-1911-x64-gdal-2-4-0-mapserver-7-2-2.zip" + GIS_INTERNALS_LIBS: "release-1911-x64-gdal-2-4-0-mapserver-7-2-2-libs.zip" install: @@ -125,7 +125,7 @@ # install the wheel - ps: python -m pip install --upgrade pip - - ps: pip install --force-reinstall --ignore-installed (gci dist\*.whl | % { "$_" }) + - ps: python -m pip install --force-reinstall --ignore-installed (gci dist\*.whl | % { "$_" }) - ps: move fiona fiona.build @@ -140,7 +140,7 @@ matrix: allow_failures: - - GDAL_VERSION: 2.4.0 + - PYTHON_VERSION: "2.7.14" artifacts: - path: dist\*.whl diff -Nru fiona-1.8.4/CHANGES.txt fiona-1.8.6/CHANGES.txt --- fiona-1.8.4/CHANGES.txt 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/CHANGES.txt 2019-03-19 04:25:07.000000000 +0000 @@ -3,6 +3,38 @@ All issue numbers are relative to https://github.com/Toblerity/Fiona/issues. +1.8.6 (2019-03-18) +------------------ + +- The advertisement for JSON driver enablement in 1.8.5 was false (#176), but + in this release they are ready for use. + +1.8.5 (2019-03-15) +------------------ + +- GDAL seems to work best if GDAL_DATA is set as early as possible. Ideally it + is set when building the library or in the environment before importing + Fiona, but for wheels we patch GDAL_DATA into os.environ when fiona.env + is imported. This resolves #731. +- A combination of bugs which allowed .cpg files to be overlooked has been + fixed (#726). +- On entering a collection context (Collection.__enter__) a new anonymous GDAL + environment is created if needed and entered. This makes `with + fiona.open(...) as collection:` roughly equivalent to `with fiona.open(...) + as collection, Env():`. This helps prevent bugs when Collections are created + and then used later or in different scopes. +- Missing GDAL support for TopoJSON, GeoJSONSeq, and ESRIJSON has been enabled + (#721). +- A regression in handling of polygons with M values (#724) has been fixed. +- Per-feature debug logging calls in OGRFeatureBuilder methods have been + eliminated to improve feature writing performance (#718). +- Native support for datasets in Google Cloud Storage identified by "gs" + resource names has been added (#709). +- Support has been added for triangle, polyhedral surface, and TIN geometry + types (#679). +- Notes about using the MemoryFile and ZipMemoryFile classes has been added to + the manual (#674). + 1.8.4 (2018-12-10) ------------------ diff -Nru fiona-1.8.4/debian/changelog fiona-1.8.6/debian/changelog --- fiona-1.8.4/debian/changelog 2019-02-25 19:00:00.000000000 +0000 +++ fiona-1.8.6/debian/changelog 2019-07-14 14:00:00.000000000 +0000 @@ -1,8 +1,32 @@ -fiona (1.8.4-1~bionic0) bionic; urgency=medium +fiona (1.8.6-1~bionic1) bionic; urgency=medium + * No change rebuild for GDAL 2.4.2 transition. + + -- Angelos Tzotsos Sun, 14 Jul 2019 16:00:00 +0200 + +fiona (1.8.6-1~bionic0) bionic; urgency=medium + * No change rebuild for Bionic. - -- Angelos Tzotsos Mon, 25 Feb 2019 21:00:00 +0200 + -- Angelos Tzotsos Thu, 02 May 2019 12:00:00 +0200 + +fiona (1.8.6-1~exp1) experimental; urgency=medium + + * Team upload. + * New upstream release. + * Ignore test_data_paths. + + -- Bas Couwenberg Tue, 19 Mar 2019 06:50:58 +0100 + +fiona (1.8.5-1~exp1) experimental; urgency=medium + + * Team upload. + * New upstream release. + * Bump Standards-Version to 4.3.0, no changes. + * Refresh patches. + * Add python{,3}-mock to build dependencies. + + -- Bas Couwenberg Sat, 16 Mar 2019 08:34:02 +0100 fiona (1.8.4-1) unstable; urgency=medium diff -Nru fiona-1.8.4/debian/control fiona-1.8.6/debian/control --- fiona-1.8.4/debian/control 2018-10-31 19:19:18.000000000 +0000 +++ fiona-1.8.6/debian/control 2019-03-16 07:46:46.000000000 +0000 @@ -22,6 +22,8 @@ python-cligj, python3-cligj, python-enum34, + python-mock, + python3-mock, python-munch, python3-munch, python-pytest, @@ -32,7 +34,7 @@ python3-six, python-sphinx, python3-sphinx -Standards-Version: 4.2.1 +Standards-Version: 4.3.0 Vcs-Browser: https://salsa.debian.org/debian-gis-team/fiona Vcs-Git: https://salsa.debian.org/debian-gis-team/fiona.git Homepage: https://github.com/Toblerity/Fiona diff -Nru fiona-1.8.4/debian/patches/0001-Rename-fio-command-to-fiona-to-avoid-name-clash.patch fiona-1.8.6/debian/patches/0001-Rename-fio-command-to-fiona-to-avoid-name-clash.patch --- fiona-1.8.4/debian/patches/0001-Rename-fio-command-to-fiona-to-avoid-name-clash.patch 2018-11-16 05:56:15.000000000 +0000 +++ fiona-1.8.6/debian/patches/0001-Rename-fio-command-to-fiona-to-avoid-name-clash.patch 2019-03-16 07:33:36.000000000 +0000 @@ -9,7 +9,7 @@ --- a/setup.py +++ b/setup.py -@@ -306,7 +306,7 @@ setup_args = dict( +@@ -309,7 +309,7 @@ setup_args = dict( packages=['fiona', 'fiona.fio'], entry_points=''' [console_scripts] diff -Nru fiona-1.8.4/debian/patches/0006-Remove-unknown-distribution-options.patch fiona-1.8.6/debian/patches/0006-Remove-unknown-distribution-options.patch --- fiona-1.8.4/debian/patches/0006-Remove-unknown-distribution-options.patch 2018-11-16 05:56:17.000000000 +0000 +++ fiona-1.8.6/debian/patches/0006-Remove-unknown-distribution-options.patch 2019-03-16 07:33:40.000000000 +0000 @@ -6,7 +6,7 @@ --- a/setup.py +++ b/setup.py -@@ -288,11 +288,8 @@ extras_require['all'] = list(set(it.chai +@@ -291,11 +291,8 @@ extras_require['all'] = list(set(it.chai setup_args = dict( cmdclass={'sdist': sdist_multi_gdal}, diff -Nru fiona-1.8.4/debian/rules fiona-1.8.6/debian/rules --- fiona-1.8.4/debian/rules 2018-10-31 19:19:42.000000000 +0000 +++ fiona-1.8.6/debian/rules 2019-03-19 05:50:58.000000000 +0000 @@ -17,6 +17,7 @@ export PYBUILD_AFTER_TEST=rm -rf {build_dir}/tests export PYBUILD_TEST_ARGS=--ignore tests/test_bytescollection.py \ --ignore tests/test_collection.py \ + --ignore tests/test_data_paths.py \ --ignore tests/test_feature.py \ --ignore tests/test_filter_vsi.py \ --ignore tests/test_fio_bounds.py \ diff -Nru fiona-1.8.4/fiona/collection.py fiona-1.8.6/fiona/collection.py --- fiona-1.8.4/fiona/collection.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/collection.py 2019-03-19 04:25:07.000000000 +0000 @@ -11,11 +11,11 @@ from fiona.ogrext import buffer_to_virtual_file, remove_virtual_file, GEOMETRY_TYPES from fiona.errors import (DriverError, SchemaError, CRSError, UnsupportedGeometryTypeError, DriverSupportError) from fiona.logutils import FieldSkipLogFilter -from fiona._env import driver_count, get_gdal_release_name, get_gdal_version_tuple -from fiona.env import Env +from fiona._env import get_gdal_release_name, get_gdal_version_tuple +from fiona.env import env_ctx_if_needed from fiona.errors import FionaDeprecationWarning from fiona.drvsupport import supported_drivers -from fiona.path import Path, UnparsedPath, vsi_path, parse_path +from fiona.path import Path, vsi_path, parse_path from six import string_types, binary_type @@ -150,7 +150,7 @@ raise CRSError("crs lacks init or proj parameter") self._driver = driver - kwargs.update(encoding=encoding or '') + kwargs.update(encoding=encoding) self.encoding = encoding try: @@ -166,8 +166,6 @@ if self.session is not None: self.guard_driver_mode() - if not self.encoding: - self.encoding = self.session.get_fileencoding().lower() if self.mode in ("a", "w"): self._valid_geom_types = _get_valid_geom_types(self.schema, self.driver) @@ -459,10 +457,13 @@ def __enter__(self): logging.getLogger('fiona.ogrext').addFilter(self.field_skip_log_filter) + self._env = env_ctx_if_needed() + self._env.__enter__() return self def __exit__(self, type, value, traceback): logging.getLogger('fiona.ogrext').removeFilter(self.field_skip_log_filter) + self._env.__exit__() self.close() def __del__(self): diff -Nru fiona-1.8.4/fiona/drvsupport.py fiona-1.8.6/fiona/drvsupport.py --- fiona-1.8.4/fiona/drvsupport.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/drvsupport.py 2019-03-19 04:25:07.000000000 +0000 @@ -43,11 +43,15 @@ ("OpenFileGDB", "r"), # ESRI Personal GeoDatabase PGeo No Yes No, needs ODBC library # ESRI ArcSDE SDE No Yes No, needs ESRI SDE + # ESRIJSON ESRIJSON No Yes Yes + ("ESRIJSON", "r"), # ESRI Shapefile ESRI Shapefile Yes Yes Yes ("ESRI Shapefile", "raw"), # FMEObjects Gateway FMEObjects Gateway No Yes No, needs FME # GeoJSON GeoJSON Yes Yes Yes ("GeoJSON", "rw"), + # GeoJSONSeq GeoJSON sequences Yes Yes Yes + ("GeoJSONSeq", "rw"), # Géoconcept Export Geoconcept Yes Yes Yes # multi-layers # ("Geoconcept", "raw"), @@ -114,6 +118,8 @@ # SUA SUA No Yes Yes ("SUA", "r"), # SVG SVG No Yes No, needs libexpat + # TopoJSON TopoJSON No Yes Yes + ("TopoJSON", "r"), # UK .NTF UK. NTF No Yes Yes # multi-layer # ("UK. NTF", "r"), diff -Nru fiona-1.8.4/fiona/env.py fiona-1.8.6/fiona/env.py --- fiona-1.8.4/fiona/env.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/env.py 2019-03-19 04:25:07.000000000 +0000 @@ -3,6 +3,7 @@ from contextlib import contextmanager from functools import wraps, total_ordering import logging +import os import re import threading @@ -11,7 +12,7 @@ from fiona._env import ( GDALEnv, calc_gdal_version_num, get_gdal_version_num, get_gdal_config, - set_gdal_config, get_gdal_release_name) + set_gdal_config, get_gdal_release_name, GDALDataFinder, PROJDataFinder) from fiona.compat import getargspec from fiona.errors import EnvError, GDALVersionError from fiona.session import Session, DummySession @@ -581,3 +582,32 @@ return wrapper return decorator + + +# Patch the environment if needed, such as in the installed wheel case. + +if 'GDAL_DATA' not in os.environ: + + # See https://github.com/mapbox/rasterio/issues/1631. + if GDALDataFinder().find_file("header.dxf"): + log.debug("GDAL data files are available at built-in paths") + + else: + path = GDALDataFinder().search() + + if path: + os.environ['GDAL_DATA'] = path + log.debug("GDAL_DATA not found in environment, set to %r.", path) + +if 'PROJ_LIB' not in os.environ: + + # See https://github.com/mapbox/rasterio/issues/1631. + if PROJDataFinder().has_data(): + log.debug("PROJ data files are available at built-in paths") + + else: + path = PROJDataFinder().search() + + if path: + os.environ['PROJ_LIB'] = path + log.debug("PROJ data not found in environment, set to %r.", path) diff -Nru fiona-1.8.4/fiona/_env.pyx fiona-1.8.6/fiona/_env.pyx --- fiona-1.8.4/fiona/_env.pyx 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_env.pyx 2019-03-19 04:25:07.000000000 +0000 @@ -17,6 +17,10 @@ import sys import threading +from fiona._err cimport exc_wrap_int, exc_wrap_ogrerr +from fiona._err import CPLE_BaseError +from fiona.errors import EnvError + level_map = { 0: 0, @@ -54,6 +58,13 @@ cdef bint is_64bit = sys.maxsize > 2 ** 32 +cdef _safe_osr_release(OGRSpatialReferenceH srs): + """Wrapper to handle OSR release when NULL.""" + if srs != NULL: + OSRRelease(srs) + srs = NULL + + def calc_gdal_version_num(maj, min, rev): """Calculates the internal gdal version number based on major, minor and revision @@ -237,12 +248,41 @@ class GDALDataFinder(object): """Finds GDAL data files - Note: this class is private in 1.8.x and not in the public API. + Note: this is not part of the 1.8.x public API. """ + def find_file(self, basename): + """Returns path of a GDAL data file or None + + Parameters + ---------- + basename : str + Basename of a data file such as "header.dxf" + + Returns + ------- + str (on success) or None (on failure) + + """ + cdef const char *path_c = NULL + basename_b = basename.encode('utf-8') + path_c = CPLFindFile("gdal", basename_b) + if path_c == NULL: + return None + else: + path = path_c + return path def search(self, prefix=None): - """Returns GDAL_DATA location""" + """Returns GDAL data directory + + Note well that os.environ is not consulted. + + Returns + ------- + str or None + + """ path = self.search_wheel(prefix or __file__) if not path: path = self.search_prefix(prefix or sys.prefix) @@ -264,20 +304,47 @@ def search_debian(self, prefix=sys.prefix): """Check Debian locations""" - gdal_version = get_gdal_version_tuple() - datadir = os.path.join(prefix, 'share', 'gdal', '{}.{}'.format(gdal_version.major, gdal_version.minor)) + gdal_release_name = GDALVersionInfo("RELEASE_NAME") + datadir = os.path.join(prefix, 'share', 'gdal', '{}.{}'.format(*gdal_release_name.split('.')[:2])) return datadir if os.path.exists(os.path.join(datadir, 'pcs.csv')) else None class PROJDataFinder(object): """Finds PROJ data files - Note: this class is private in 1.8.x and not in the public API. + Note: this is not part of the public 1.8.x API. """ + def has_data(self): + """Returns True if PROJ's data files can be found + + Returns + ------- + bool + + """ + cdef OGRSpatialReferenceH osr = OSRNewSpatialReference(NULL) + + try: + exc_wrap_ogrerr(exc_wrap_int(OSRImportFromProj4(osr, "+init=epsg:4326"))) + except CPLE_BaseError: + return False + else: + return True + finally: + _safe_osr_release(osr) + def search(self, prefix=None): - """Returns PROJ_LIB location""" + """Returns PROJ data directory + + Note well that os.environ is not consulted. + + Returns + ------- + str or None + + """ path = self.search_wheel(prefix or __file__) if not path: path = self.search_prefix(prefix or sys.prefix) @@ -322,6 +389,10 @@ self.update_config_options(GDAL_DATA=os.environ['GDAL_DATA']) log.debug("GDAL_DATA found in environment: %r.", os.environ['GDAL_DATA']) + # See https://github.com/mapbox/rasterio/issues/1631. + elif GDALDataFinder().find_file("header.dxf"): + log.debug("GDAL data files are available at built-in paths") + else: path = GDALDataFinder().search() @@ -329,8 +400,13 @@ self.update_config_options(GDAL_DATA=path) log.debug("GDAL_DATA not found in environment, set to %r.", path) - if 'PROJ_LIB' not in os.environ: + if 'PROJ_LIB' in os.environ: + log.debug("PROJ_LIB found in environment: %r.", os.environ['PROJ_LIB']) + elif PROJDataFinder().has_data(): + log.debug("PROJ data files are available at built-in paths") + + else: path = PROJDataFinder().search() if path: diff -Nru fiona-1.8.4/fiona/_err.pxd fiona-1.8.6/fiona/_err.pxd --- fiona-1.8.4/fiona/_err.pxd 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_err.pxd 2019-03-19 04:25:07.000000000 +0000 @@ -4,7 +4,12 @@ ctypedef FILE VSILFILE +cdef extern from "ogr_core.h": + + ctypedef int OGRErr + cdef int exc_wrap_int(int retval) except -1 +cdef OGRErr exc_wrap_ogrerr(OGRErr retval) except -1 cdef void *exc_wrap_pointer(void *ptr) except NULL cdef VSILFILE *exc_wrap_vsilfile(VSILFILE *f) except NULL diff -Nru fiona-1.8.4/fiona/_err.pyx fiona-1.8.6/fiona/_err.pyx --- fiona-1.8.4/fiona/_err.pyx 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_err.pyx 2019-03-19 04:25:07.000000000 +0000 @@ -249,6 +249,17 @@ return err +cdef OGRErr exc_wrap_ogrerr(OGRErr err) except -1: + """Wrap a function that returns OGRErr but does not use the + CPL error stack. + + """ + if err == 0: + return err + else: + raise CPLE_BaseError(3, err, "OGR Error code {}".format(err)) + + cdef void *exc_wrap_pointer(void *ptr) except NULL: """Wrap a GDAL/OGR function that returns GDALDatasetH etc (void *) Raises a Rasterio exception if a non-fatal error has be set. diff -Nru fiona-1.8.4/fiona/gdal.pxi fiona-1.8.6/fiona/gdal.pxi --- fiona-1.8.4/fiona/gdal.pxi 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/gdal.pxi 2019-03-19 04:25:07.000000000 +0000 @@ -10,6 +10,7 @@ void CPLSetThreadLocalConfigOption(const char* key, const char* val) void CPLSetConfigOption(const char* key, const char* val) const char* CPLGetConfigOption(const char* key, const char* default) + const char *CPLFindFile(const char *pszClass, const char *pszBasename) cdef extern from "cpl_error.h" nogil: diff -Nru fiona-1.8.4/fiona/_geometry.pxd fiona-1.8.6/fiona/_geometry.pxd --- fiona-1.8.4/fiona/_geometry.pxd 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_geometry.pxd 2019-03-19 04:25:07.000000000 +0000 @@ -142,3 +142,5 @@ cdef unsigned int geometry_type_code(object name) except? 9999 cdef object normalize_geometry_type_code(unsigned int code) +cdef unsigned int base_geometry_type_code(unsigned int code) + diff -Nru fiona-1.8.4/fiona/_geometry.pyx fiona-1.8.6/fiona/_geometry.pyx --- fiona-1.8.4/fiona/_geometry.pyx 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_geometry.pyx 2019-03-19 04:25:07.000000000 +0000 @@ -80,6 +80,17 @@ return code +cdef inline unsigned int base_geometry_type_code(unsigned int code): + """ Returns base geometry code without Z, M and ZM types """ + # Remove 2.5D flag. + code = code & (~0x80000000) + + # Normalize Z, M, and ZM types. Fiona 1.x does not support M + # and doesn't treat OGC 'Z' variants as special types of their + # own. + return code % 1000 + + # Geometry related functions and classes follow. cdef void * _createOgrGeomFromWKB(object wkb) except NULL: """Make an OGR geometry from a WKB string""" @@ -162,13 +173,7 @@ cdef unsigned int etype = OGR_G_GetGeometryType(geom) - # Remove 2.5D flag. - self.code = etype & (~0x80000000) - - # Normalize Z, M, and ZM types. Fiona 1.x does not support M - # and doesn't treat OGC 'Z' variants as special types of their - # own. - self.code = self.code % 1000 + self.code = base_geometry_type_code(etype) if self.code not in GEOMETRY_TYPES: raise UnsupportedGeometryTypeError(self.code) diff -Nru fiona-1.8.4/fiona/__init__.py fiona-1.8.6/fiona/__init__.py --- fiona-1.8.4/fiona/__init__.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/__init__.py 2019-03-19 04:25:07.000000000 +0000 @@ -101,7 +101,7 @@ __all__ = ['bounds', 'listlayers', 'open', 'prop_type', 'prop_width'] -__version__ = "1.8.4" +__version__ = "1.8.6" __gdal_version__ = get_gdal_release_name() gdal_version = get_gdal_version_tuple() diff -Nru fiona-1.8.4/fiona/ogrext1.pxd fiona-1.8.6/fiona/ogrext1.pxd --- fiona-1.8.4/fiona/ogrext1.pxd 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/ogrext1.pxd 2019-03-19 04:25:07.000000000 +0000 @@ -196,6 +196,7 @@ void * OGR_F_GetFieldDefnRef (void *feature, int n) int OGR_F_GetFieldIndex (void *feature, char *name) void * OGR_F_GetGeometryRef (void *feature) + void * OGR_F_StealGeometry (void *feature) void OGR_F_SetFieldDateTime (void *feature, int n, int y, int m, int d, int hh, int mm, int ss, int tz) void OGR_F_SetFieldDouble (void *feature, int n, double value) void OGR_F_SetFieldInteger (void *feature, int n, int value) @@ -235,6 +236,9 @@ double OGR_G_GetZ (void *geometry, int n) void OGR_G_ImportFromWkb (void *geometry, unsigned char *bytes, int nbytes) int OGR_G_WkbSize (void *geometry) + void * OGR_G_ForceToMultiPolygon (void *geometry) + void * OGR_G_ForceToPolygon (void *geometry) + void * OGR_G_Clone(void *geometry) OGRErr OGR_L_CreateFeature (void *layer, void *feature) OGRErr OGR_L_CreateField (void *layer, void *fielddefn, int flexible) OGRErr OGR_L_GetExtent (void *layer, void *extent, int force) diff -Nru fiona-1.8.4/fiona/ogrext2.pxd fiona-1.8.6/fiona/ogrext2.pxd --- fiona-1.8.4/fiona/ogrext2.pxd 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/ogrext2.pxd 2019-03-19 04:25:07.000000000 +0000 @@ -255,6 +255,7 @@ void * OGR_F_GetFieldDefnRef (void *feature, int n) int OGR_F_GetFieldIndex (void *feature, char *name) void * OGR_F_GetGeometryRef (void *feature) + void * OGR_F_StealGeometry (void *feature) void OGR_F_SetFieldDateTime (void *feature, int n, int y, int m, int d, int hh, int mm, int ss, int tz) void OGR_F_SetFieldDouble (void *feature, int n, double value) void OGR_F_SetFieldInteger (void *feature, int n, int value) @@ -297,6 +298,9 @@ double OGR_G_GetZ (void *geometry, int n) void OGR_G_ImportFromWkb (void *geometry, unsigned char *bytes, int nbytes) int OGR_G_WkbSize (void *geometry) + void * OGR_G_ForceToMultiPolygon (void *geometry) + void * OGR_G_ForceToPolygon (void *geometry) + void * OGR_G_Clone(void *geometry) OGRErr OGR_L_CreateFeature (void *layer, void *feature) OGRErr OGR_L_CreateField (void *layer, void *fielddefn, int flexible) OGRErr OGR_L_GetExtent (void *layer, void *extent, int force) diff -Nru fiona-1.8.4/fiona/ogrext.pyx fiona-1.8.6/fiona/ogrext.pyx --- fiona-1.8.4/fiona/ogrext.pyx 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/ogrext.pyx 2019-03-19 04:25:07.000000000 +0000 @@ -18,7 +18,7 @@ from fiona._geometry cimport ( GeomBuilder, OGRGeomBuilder, geometry_type_code, - normalize_geometry_type_code) + normalize_geometry_type_code, base_geometry_type_code) from fiona._err cimport exc_wrap_int, exc_wrap_pointer, exc_wrap_vsilfile import fiona @@ -42,6 +42,7 @@ from libc.string cimport strcmp from cpython cimport PyBytes_FromStringAndSize, PyBytes_AsString + cdef extern from "ogr_api.h" nogil: ctypedef void * OGRLayerH @@ -125,8 +126,7 @@ argument is not destroyed. """ - cdef build(self, void *feature, encoding='utf-8', bbox=False, driver=None, - ignore_fields=None, ignore_geometry=False): + cdef build(self, void *feature, encoding='utf-8', bbox=False, driver=None, ignore_fields=None, ignore_geometry=False): """Build a Fiona feature object from an OGR feature Parameters @@ -262,24 +262,40 @@ props[key] = None cdef void *cogr_geometry = NULL + cdef void *org_geometry = NULL if not ignore_geometry: cogr_geometry = OGR_F_GetGeometryRef(feature) if cogr_geometry is not NULL: - code = OGR_G_GetGeometryType(cogr_geometry) - if 7 < code < 100: # Curves. + code = base_geometry_type_code(OGR_G_GetGeometryType(cogr_geometry)) + + if 8 <= code <= 14: # Curves. cogr_geometry = get_linear_geometry(cogr_geometry) geom = GeomBuilder().build(cogr_geometry) OGR_G_DestroyGeometry(cogr_geometry) + elif 15 <= code <= 17: + # We steal the geometry: the geometry of the in-memory feature is now null + # and we are responsible for cogr_geometry. + org_geometry = OGR_F_StealGeometry(feature) + + if code in (15, 16): + cogr_geometry = OGR_G_ForceToMultiPolygon(org_geometry) + elif code == 17: + cogr_geometry = OGR_G_ForceToPolygon(org_geometry) + + geom = GeomBuilder().build(cogr_geometry) + OGR_G_DestroyGeometry(cogr_geometry) + else: geom = GeomBuilder().build(cogr_geometry) fiona_feature["geometry"] = geom else: + fiona_feature["geometry"] = None return fiona_feature @@ -313,9 +329,8 @@ feature['geometry']) OGR_F_SetGeometryDirectly(cogr_feature, cogr_geometry) - # OGR_F_SetFieldString takes UTF-8 encoded strings ('bytes' in - # Python 3). - encoding = session.get_internalencoding() + # OGR_F_SetFieldString takes encoded strings ('bytes' in Python 3). + encoding = session._get_internal_encoding() for key, value in feature['properties'].items(): log.debug( @@ -412,12 +427,10 @@ cdef void *cogr_geometry = OGR_F_GetGeometryRef(cogr_feature) if cogr_geometry == NULL: raise ValueError("Null geometry") - log.debug("Geometry: %s" % OGR_G_ExportToJson(cogr_geometry)) - encoding = collection.encoding or 'utf-8' result = FeatureBuilder().build( cogr_feature, + encoding='utf-8', bbox=False, - encoding=encoding, driver=collection.driver ) _deleteOgrFeature(cogr_feature) @@ -453,7 +466,7 @@ path_b = collection.path.encode('utf-8') path_c = path_b - userencoding = kwargs.get('encoding') + self._fileencoding = kwargs.get('encoding') or collection.encoding # We have two ways of specifying drivers to try. Resolve the # values into a single set of driver short names. @@ -464,16 +477,18 @@ else: drivers = None + encoding = kwargs.pop('encoding', None) + if encoding: + kwargs['encoding'] = encoding.upper() + self.cogr_ds = gdal_open_vector(path_c, 0, drivers, kwargs) if isinstance(collection.name, string_types): name_b = collection.name.encode('utf-8') name_c = name_b - self.cogr_layer = GDALDatasetGetLayerByName( - self.cogr_ds, name_c) + self.cogr_layer = GDALDatasetGetLayerByName(self.cogr_ds, name_c) elif isinstance(collection.name, int): - self.cogr_layer = GDALDatasetGetLayer( - self.cogr_ds, collection.name) + self.cogr_layer = GDALDatasetGetLayer(self.cogr_ds, collection.name) name_c = OGR_L_GetName(self.cogr_layer) name_b = name_c collection.name = name_b.decode('utf-8') @@ -481,21 +496,16 @@ if self.cogr_layer == NULL: raise ValueError("Null layer: " + repr(collection.name)) - self._fileencoding = userencoding or ( - OGR_L_TestCapability( - self.cogr_layer, OLC_STRINGSASUTF8) and - 'utf-8') or ( - "Shapefile" in self.get_driver() and - 'ISO-8859-1') or locale.getpreferredencoding().upper() + encoding = self._get_internal_encoding() if collection.ignore_fields: try: for name in collection.ignore_fields: try: - name = name.encode(self._fileencoding) + name_b = name.encode(encoding) except AttributeError: raise TypeError("Ignored field \"{}\" has type \"{}\", expected string".format(name, name.__class__.__name__)) - ignore_fields = CSLAddString(ignore_fields, name) + ignore_fields = CSLAddString(ignore_fields, name_b) OGR_L_SetIgnoredFields(self.cogr_layer, ignore_fields) finally: CSLDestroy(ignore_fields) @@ -509,16 +519,54 @@ self.cogr_ds = NULL def get_fileencoding(self): + """DEPRECATED""" + warnings.warn("get_fileencoding is deprecated and will be removed in a future version.", FionaDeprecationWarning) return self._fileencoding - def get_internalencoding(self): - if not self._encoding: - fileencoding = self.get_fileencoding() - self._encoding = ( - OGR_L_TestCapability( - self.cogr_layer, OLC_STRINGSASUTF8) and - 'utf-8') or fileencoding - return self._encoding + def _get_fallback_encoding(self): + """Determine a format-specific fallback encoding to use when using OGR_F functions + + Parameters + ---------- + None + + Returns + ------- + str + + """ + if "Shapefile" in self.get_driver(): + return 'iso-8859-1' + else: + return locale.getpreferredencoding() + + + def _get_internal_encoding(self): + """Determine the encoding to use when use OGR_F functions + + Parameters + ---------- + None + + Returns + ------- + str + + Notes + ----- + If the layer implements RFC 23 support for UTF-8, the return + value will be 'utf-8' and callers can be certain that this is + correct. If the layer does not have the OLC_STRINGSASUTF8 + capability marker, it is not possible to know exactly what the + internal encoding is and this method returns best guesses. That + means ISO-8859-1 for shapefiles and the locale's preferred + encoding for other formats such as CSV files. + + """ + if OGR_L_TestCapability(self.cogr_layer, OLC_STRINGSASUTF8): + return 'utf-8' + else: + return self._fileencoding or self._get_fallback_encoding() def get_length(self): if self.cogr_layer == NULL: @@ -553,6 +601,8 @@ if cogr_featuredefn == NULL: raise ValueError("Null feature definition") + encoding = self._get_internal_encoding() + n = OGR_FD_GetFieldCount(cogr_featuredefn) for i from 0 <= i < n: @@ -566,7 +616,7 @@ if not bool(key_b): raise ValueError("Invalid field name ref: %s" % key) - key = key_b.decode(self.get_internalencoding()) + key = key_b.decode(encoding) if key in ignore_fields: log.debug("By request, ignoring field %r", key) @@ -771,8 +821,8 @@ if cogr_feature != NULL: feature = FeatureBuilder().build( cogr_feature, + encoding=self._get_internal_encoding(), bbox=False, - encoding=self.get_internalencoding(), driver=self.collection.driver, ignore_fields=self.collection.ignore_fields, ignore_geometry=self.collection.ignore_geometry, @@ -806,8 +856,8 @@ return None feature = FeatureBuilder().build( cogr_feature, + encoding=self._get_internal_encoding(), bbox=False, - encoding=self.get_internalencoding(), driver=self.collection.driver, ignore_fields=self.collection.ignore_fields, ignore_geometry=self.collection.ignore_geometry, @@ -856,7 +906,7 @@ self.cogr_ds = gdal_open_vector(path_c, 1, None, kwargs) if isinstance(collection.name, string_types): - name_b = collection.name.encode() + name_b = collection.name.encode('utf-8') name_c = name_b self.cogr_layer = exc_wrap_pointer(GDALDatasetGetLayerByName(self.cogr_ds, name_c)) @@ -870,11 +920,7 @@ raise DriverError(u"{}".format(exc)) else: - self._fileencoding = (userencoding or ( - OGR_L_TestCapability(self.cogr_layer, OLC_STRINGSASUTF8) and - 'utf-8') or ( - self.get_driver() == "ESRI Shapefile" and - 'ISO-8859-1') or locale.getpreferredencoding()).upper() + self._fileencoding = userencoding or self._get_fallback_encoding() elif collection.mode == 'w': @@ -975,19 +1021,16 @@ self.cogr_layer = NULL raise CRSError(u"{}".format(exc)) - - # Figure out what encoding to use. The encoding parameter given - # to the collection constructor takes highest precedence, then - # 'iso-8859-1', then the system's default encoding as last resort. + # Determine which encoding to use. The encoding parameter given to + # the collection constructor takes highest precedence, then + # 'iso-8859-1' (for shapefiles), then the system's default encoding + # as last resort. sysencoding = locale.getpreferredencoding() - self._fileencoding = (userencoding or ( - "Shapefile" in collection.driver and - 'ISO-8859-1') or sysencoding).upper() + self._fileencoding = userencoding or ("Shapefile" in collection.driver and 'iso-8859-1') or sysencoding if "Shapefile" in collection.driver: - fileencoding = self.get_fileencoding() - if fileencoding: - fileencoding_b = fileencoding.encode('utf-8') + if self._fileencoding: + fileencoding_b = self._fileencoding.upper().encode('utf-8') fileencoding_c = fileencoding_b options = CSLSetNameValue(options, "ENCODING", fileencoding_c) @@ -1017,6 +1060,9 @@ for k, v in kwargs.items(): + if v is None: + continue + # We need to remove encoding from the layer creation # options if we're not creating a shapefile. if k == 'encoding' and "Shapefile" not in collection.driver: @@ -1068,6 +1114,9 @@ # Next, make a layer definition from the given schema properties, # which are an ordered dict since Fiona 1.0.1. + + encoding = self._get_internal_encoding() + for key, value in collection.schema['properties'].items(): log.debug("Begin creating field: %r value: %r", key, value) @@ -1104,7 +1153,7 @@ value = 'int32' field_type = FIELD_TYPES.index(value) - encoding = self.get_internalencoding() + try: key_bytes = key.encode(encoding) cogr_fielddefn = exc_wrap_pointer(OGR_Fld_Create(key_bytes, field_type)) @@ -1116,6 +1165,7 @@ # subtypes are new in GDAL 2.x, ignored in 1.x set_field_subtype(cogr_fielddefn, field_subtype) exc_wrap_int(OGR_L_CreateField(self.cogr_layer, cogr_fielddefn, 1)) + except (UnicodeEncodeError, CPLE_BaseError) as exc: OGRReleaseDataSource(self.cogr_ds) self.cogr_ds = NULL @@ -1250,9 +1300,9 @@ OGR_G_DestroyGeometry(cogr_geometry) else: - OGR_L_SetSpatialFilter( - cogr_layer, NULL) - self.encoding = session.get_internalencoding() + OGR_L_SetSpatialFilter(cogr_layer, NULL) + + self.encoding = session._get_internal_encoding() self.fastindex = OGR_L_TestCapability( session.cogr_layer, OLC_FASTSETNEXTBYINDEX) @@ -1352,8 +1402,8 @@ try: return FeatureBuilder().build( cogr_feature, + encoding=self.collection.session._get_internal_encoding(), bbox=False, - encoding=self.encoding, driver=self.collection.driver, ignore_fields=self.collection.ignore_fields, ignore_geometry=self.collection.ignore_geometry, @@ -1379,12 +1429,11 @@ if cogr_feature == NULL: raise StopIteration - fid = OGR_F_GetFID(cogr_feature) feature = FeatureBuilder().build( cogr_feature, + encoding=self.collection.session._get_internal_encoding(), bbox=False, - encoding=self.encoding, driver=self.collection.driver, ignore_fields=self.collection.ignore_fields, ignore_geometry=self.collection.ignore_geometry, @@ -1449,7 +1498,7 @@ def _remove_layer(path, layer, driver=None): cdef void *cogr_ds cdef int layer_index - + if isinstance(layer, integer_types): layer_index = layer layer_str = str(layer_index) @@ -1460,11 +1509,11 @@ except ValueError: raise ValueError("Layer \"{}\" does not exist in datasource: {}".format(layer, path)) layer_str = '"{}"'.format(layer) - + if layer_index < 0: layer_names = _listlayers(path) layer_index = len(layer_names) + layer_index - + try: cogr_ds = gdal_open_vector(path.encode("utf-8"), 1, None, {}) except (DriverError, FionaNullPointerError): diff -Nru fiona-1.8.4/fiona/path.py fiona-1.8.6/fiona/path.py --- fiona-1.8.4/fiona/path.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/path.py 2019-03-19 04:25:07.000000000 +0000 @@ -17,13 +17,14 @@ 's3': 's3', 'tar': 'tar', 'zip': 'zip', - 'file': 'file' + 'file': 'file', + 'gs': 'gs', } CURLSCHEMES = set([k for k, v in SCHEMES.items() if v == 'curl']) # TODO: extend for other cloud plaforms. -REMOTESCHEMES = set([k for k, v in SCHEMES.items() if v in ('curl', 's3')]) +REMOTESCHEMES = set([k for k, v in SCHEMES.items() if v in ('curl', 's3', 'gs')]) class Path(object): diff -Nru fiona-1.8.4/fiona/session.py fiona-1.8.6/fiona/session.py --- fiona-1.8.4/fiona/session.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/session.py 2019-03-19 04:25:07.000000000 +0000 @@ -186,3 +186,52 @@ return {'AWS_NO_SIGN_REQUEST': 'YES'} else: return {k.upper(): v for k, v in self.credentials.items()} + +class GSSession(Session): + """Configures access to secured resources stored in Google Cloud Storage + """ + def __init__(self, google_application_credentials=None): + """Create new Google Cloude Storage session + + Parameters + ---------- + google_application_credentials: string + Path to the google application credentials JSON file. + """ + + self._creds = {} + if google_application_credentials is not None: + self._creds['google_application_credentials'] = google_application_credentials + + @classmethod + def hascreds(cls, config): + """Determine if the given configuration has proper credentials + + Parameters + ---------- + cls : class + A Session class. + config : dict + GDAL configuration as a dict. + + Returns + ------- + bool + + """ + return 'GOOGLE_APPLICATION_CREDENTIALS' in config + + @property + def credentials(self): + """The session credentials as a dict""" + return self._creds + + def get_credential_options(self): + """Get credentials as GDAL configuration options + + Returns + ------- + dict + + """ + return {k.upper(): v for k, v in self.credentials.items()} diff -Nru fiona-1.8.4/fiona/_shim1.pyx fiona-1.8.6/fiona/_shim1.pyx --- fiona-1.8.4/fiona/_shim1.pyx 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_shim1.pyx 2019-03-19 04:25:07.000000000 +0000 @@ -1,3 +1,5 @@ +"""Shims on top of ogrext for GDAL versions < 2""" + import logging from fiona.ogrext1 cimport * @@ -44,14 +46,11 @@ for name in drivers: name_b = name.encode() name_c = name_b - #log.debug("Trying driver: %s", name) drv = OGRGetDriverByName(name_c) if drv != NULL: ds = OGR_Dr_Open(drv, path_c, mode) if ds != NULL: cogr_ds = ds - # TODO - #collection._driver = name break else: cogr_ds = OGROpen(path_c, mode, NULL) @@ -95,22 +94,32 @@ finally: CSLDestroy(opts) + # transactions are not supported in GDAL 1.x cdef OGRErr gdal_start_transaction(void* cogr_ds, int force): return OGRERR_NONE + + cdef OGRErr gdal_commit_transaction(void* cogr_ds): return OGRERR_NONE + + cdef OGRErr gdal_rollback_transaction(void* cogr_ds): return OGRERR_NONE + # field subtypes are not supported in GDAL 1.x cdef OGRFieldSubType get_field_subtype(void *fielddefn): return OFSTNone + + cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype): pass + cdef bint check_capability_create_layer(void *cogr_ds): return OGR_DS_TestCapability(cogr_ds, ODsCCreateLayer) + cdef void *get_linear_geometry(void *geom): return geom diff -Nru fiona-1.8.4/fiona/_shim22.pyx fiona-1.8.6/fiona/_shim22.pyx --- fiona-1.8.4/fiona/_shim22.pyx 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_shim22.pyx 2019-03-19 04:25:07.000000000 +0000 @@ -1,6 +1,10 @@ +"""Shims on top of ogrext for GDAL versions >= 2.2""" + cdef extern from "ogr_api.h": + int OGR_F_IsFieldNull(void *feature, int n) + from fiona.ogrext2 cimport * from fiona._err cimport exc_wrap_pointer from fiona._err import cpl_errs, CPLE_BaseError, FionaNullPointerError @@ -30,7 +34,7 @@ GDALFlushCache(cogr_ds) -cdef void* gdal_open_vector(const char* path_c, int mode, drivers, options) except NULL: +cdef void* gdal_open_vector(char* path_c, int mode, drivers, options) except NULL: cdef void* cogr_ds = NULL cdef char **drvs = NULL cdef void* drv = NULL @@ -46,12 +50,15 @@ for name in drivers: name_b = name.encode() name_c = name_b - #log.debug("Trying driver: %s", name) drv = GDALGetDriverByName(name_c) if drv != NULL: drvs = CSLAddString(drvs, name_c) for k, v in options.items(): + + if v is None: + continue + k = k.upper().encode('utf-8') if isinstance(v, bool): v = ('ON' if v else 'OFF').encode('utf-8') @@ -64,7 +71,7 @@ try: cogr_ds = exc_wrap_pointer( - GDALOpenEx(path_c, flags, drvs, open_opts, NULL) + GDALOpenEx(path_c, flags, drvs, open_opts, NULL) ) return cogr_ds except FionaNullPointerError: @@ -102,20 +109,26 @@ cdef OGRErr gdal_start_transaction(void* cogr_ds, int force): return GDALDatasetStartTransaction(cogr_ds, force) + cdef OGRErr gdal_commit_transaction(void* cogr_ds): return GDALDatasetCommitTransaction(cogr_ds) + cdef OGRErr gdal_rollback_transaction(void* cogr_ds): return GDALDatasetRollbackTransaction(cogr_ds) - + + cdef OGRFieldSubType get_field_subtype(void *fielddefn): return OGR_Fld_GetSubType(fielddefn) + cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype): OGR_Fld_SetSubType(fielddefn, subtype) + cdef bint check_capability_create_layer(void *cogr_ds): return GDALDatasetTestCapability(cogr_ds, ODsCCreateLayer) + cdef void *get_linear_geometry(void *geom): return OGR_G_GetLinearGeometry(geom, 0.0, NULL) diff -Nru fiona-1.8.4/fiona/_shim2.pyx fiona-1.8.6/fiona/_shim2.pyx --- fiona-1.8.4/fiona/_shim2.pyx 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/_shim2.pyx 2019-03-19 04:25:07.000000000 +0000 @@ -1,3 +1,5 @@ +"""Shims on top of ogrext for GDAL versions > 2""" + from fiona.ogrext2 cimport * from fiona._err cimport exc_wrap_pointer from fiona._err import cpl_errs, CPLE_BaseError, FionaNullPointerError @@ -46,6 +48,10 @@ drvs = CSLAddString(drvs, name_c) for k, v in options.items(): + + if v is None: + continue + k = k.upper().encode('utf-8') if isinstance(v, bool): v = ('ON' if v else 'OFF').encode('utf-8') @@ -95,20 +101,27 @@ cdef OGRErr gdal_start_transaction(void* cogr_ds, int force): return GDALDatasetStartTransaction(cogr_ds, force) + cdef OGRErr gdal_commit_transaction(void* cogr_ds): return GDALDatasetCommitTransaction(cogr_ds) + cdef OGRErr gdal_rollback_transaction(void* cogr_ds): return GDALDatasetRollbackTransaction(cogr_ds) + cdef OGRFieldSubType get_field_subtype(void *fielddefn): return OGR_Fld_GetSubType(fielddefn) + cdef void set_field_subtype(void *fielddefn, OGRFieldSubType subtype): OGR_Fld_SetSubType(fielddefn, subtype) + cdef bint check_capability_create_layer(void *cogr_ds): return GDALDatasetTestCapability(cogr_ds, ODsCCreateLayer) + cdef void *get_linear_geometry(void *geom): return OGR_G_GetLinearGeometry(geom, 0.0, NULL) + diff -Nru fiona-1.8.4/fiona/vfs.py fiona-1.8.6/fiona/vfs.py --- fiona-1.8.4/fiona/vfs.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/fiona/vfs.py 2019-03-19 04:25:07.000000000 +0000 @@ -16,12 +16,13 @@ 's3': 's3', 'tar': 'tar', 'zip': 'zip', + 'gs': 'gs', } CURLSCHEMES = set([k for k, v in SCHEMES.items() if v == 'curl']) # TODO: extend for other cloud plaforms. -REMOTESCHEMES = set([k for k, v in SCHEMES.items() if v in ('curl', 's3')]) +REMOTESCHEMES = set([k for k, v in SCHEMES.items() if v in ('curl', 's3', 'gs')]) def valid_vsi(vsi): diff -Nru fiona-1.8.4/README.rst fiona-1.8.6/README.rst --- fiona-1.8.4/README.rst 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/README.rst 2019-03-19 04:25:07.000000000 +0000 @@ -24,7 +24,7 @@ For more details, see: * Fiona `home page `__ -* Fiona `docs and manual `__ +* Fiona `docs and manual `__ * Fiona `examples `__ Usage @@ -54,7 +54,7 @@ # collection's ``meta`` property and then modify them as # desired. - meta = source.meta + meta = src.meta meta['schema']['geometry'] = 'Point' # Open an output file, using the same format driver and @@ -64,7 +64,7 @@ with fiona.open('test_write.shp', 'w', **meta) as dst: # Process only the records intersecting a box. - for f in source.filter(bbox=(-107.0, 37.0, -105.0, 39.0)): + for f in src.filter(bbox=(-107.0, 37.0, -105.0, 39.0)): # Get a point on the boundary of the record's # geometry. @@ -214,7 +214,7 @@ Installation ============ -Fiona requires Python 2.6, 2.7, 3.3, or 3.4 and GDAL/OGR 1.8+. To build from +Fiona requires Python 2.7 or 3.4+ and GDAL/OGR 1.8+. To build from a source distribution you will need a C compiler and GDAL and Python development headers and libraries (libgdal1-dev for Debian/Ubuntu, gdal-dev for CentOS/Fedora). diff -Nru fiona-1.8.4/setup.py fiona-1.8.6/setup.py --- fiona-1.8.4/setup.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/setup.py 2019-03-19 04:25:07.000000000 +0000 @@ -233,7 +233,10 @@ Extension('fiona._drivers', ['fiona/_drivers.pyx'], **ext_options), Extension('fiona._err', ['fiona/_err.pyx'], **ext_options), Extension('fiona._shim', ['fiona/_shim.pyx'], **ext_options), - Extension('fiona.ogrext', ['fiona/ogrext.pyx'], **ext_options)]) + Extension('fiona.ogrext', ['fiona/ogrext.pyx'], **ext_options) + ], + compiler_directives={"language_level": "3"} + ) # If there's no manifest template, as in an sdist, we just specify .c files. elif "clean" not in sys.argv: @@ -280,7 +283,7 @@ extras_require = { 'calc': ['shapely'], 's3': ['boto3>=1.2.4'], - 'test': ['pytest>=3', 'pytest-cov', 'boto3>=1.2.4'], + 'test': ['pytest>=3', 'pytest-cov', 'boto3>=1.2.4', 'mock; python_version<"3.4"'] } extras_require['all'] = list(set(it.chain(*extras_require.values()))) diff -Nru fiona-1.8.4/tests/conftest.py fiona-1.8.6/tests/conftest.py --- fiona-1.8.4/tests/conftest.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/conftest.py 2019-03-19 04:25:07.000000000 +0000 @@ -71,6 +71,16 @@ @pytest.fixture(scope='session') +def path_test_tin_shp(data_dir): + """Path to ```test_tin.shp``""" + return os.path.join(data_dir, 'test_tin.shp') + +@pytest.fixture(scope='session') +def path_test_tin_csv(data_dir): + """Path to ```test_tin.csv``""" + return os.path.join(data_dir, 'test_tin.csv') + +@pytest.fixture(scope='session') def path_coutwildrnp_shp(data_dir): """Path to ```coutwildrnp.shp``""" return os.path.join(data_dir, 'coutwildrnp.shp') @@ -250,6 +260,10 @@ not gdal_version.at_least('2.2'), reason="Requires GDAL 2.2.x") +requires_gdal24 = pytest.mark.skipif( + not gdal_version.at_least('2.4'), + reason="Requires GDAL 2.4.x") + @pytest.fixture(scope="class") def unittest_data_dir(data_dir, request): diff -Nru fiona-1.8.4/tests/data/coutwildrnp.cpg fiona-1.8.6/tests/data/coutwildrnp.cpg --- fiona-1.8.4/tests/data/coutwildrnp.cpg 1970-01-01 00:00:00.000000000 +0000 +++ fiona-1.8.6/tests/data/coutwildrnp.cpg 2019-03-19 04:25:07.000000000 +0000 @@ -0,0 +1 @@ +ISO-8859-1 diff -Nru fiona-1.8.4/tests/data/test_tin.csv fiona-1.8.6/tests/data/test_tin.csv --- fiona-1.8.4/tests/data/test_tin.csv 1970-01-01 00:00:00.000000000 +0000 +++ fiona-1.8.6/tests/data/test_tin.csv 2019-03-19 04:25:07.000000000 +0000 @@ -0,0 +1,3 @@ +WKT,id +"TIN (((0 0 0, 0 0 1, 0 1 0, 0 0 0)), ((0 0 0, 0 1 0, 1 1 0, 0 0 0)))",1 +"TRIANGLE((0 0 0,0 1 0,1 1 0,0 0 0))",2 Binary files /tmp/tmpSBdxIc/rmgd3LzYWt/fiona-1.8.4/tests/data/test_tin.dbf and /tmp/tmpSBdxIc/KDM03XdV9g/fiona-1.8.6/tests/data/test_tin.dbf differ Binary files /tmp/tmpSBdxIc/rmgd3LzYWt/fiona-1.8.4/tests/data/test_tin.shp and /tmp/tmpSBdxIc/KDM03XdV9g/fiona-1.8.6/tests/data/test_tin.shp differ Binary files /tmp/tmpSBdxIc/rmgd3LzYWt/fiona-1.8.4/tests/data/test_tin.shx and /tmp/tmpSBdxIc/KDM03XdV9g/fiona-1.8.6/tests/data/test_tin.shx differ diff -Nru fiona-1.8.4/tests/test_collection_legacy.py fiona-1.8.6/tests/test_collection_legacy.py --- fiona-1.8.4/tests/test_collection_legacy.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/test_collection_legacy.py 2019-03-19 04:25:07.000000000 +0000 @@ -41,8 +41,8 @@ def test_mode(self): assert self.c.mode == 'r' - def test_collection(self): - assert self.c.encoding == 'iso-8859-1' + def test_encoding(self): + assert self.c.encoding is None def test_iter(self): assert iter(self.c) diff -Nru fiona-1.8.4/tests/test_collection.py fiona-1.8.6/tests/test_collection.py --- fiona-1.8.4/tests/test_collection.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/test_collection.py 2019-03-19 04:25:07.000000000 +0000 @@ -1,7 +1,6 @@ # Testing collections and workspaces import datetime -import logging import sys import re @@ -9,6 +8,7 @@ import fiona from fiona.collection import Collection, supported_drivers +from fiona.env import getenv from fiona.errors import FionaValueError, DriverError from .conftest import WGS84PATTERN @@ -115,8 +115,8 @@ def test_mode(self): assert self.c.mode == 'r' - def test_collection(self): - assert self.c.encoding == 'iso-8859-1' + def test_encoding(self): + assert self.c.encoding is None def test_iter(self): assert iter(self.c) @@ -872,7 +872,7 @@ def test_encoding_option_warning(tmpdir, caplog): """There is no ENCODING creation option log warning for GeoJSON""" - ds = fiona.Collection(str(tmpdir.join("test.geojson")), "w", driver="GeoJSON", crs="epsg:4326", + fiona.Collection(str(tmpdir.join("test.geojson")), "w", driver="GeoJSON", crs="epsg:4326", schema={"geometry": "Point", "properties": {"foo": "int"}}) assert not caplog.text @@ -881,7 +881,21 @@ """Confirm fix for issue #687""" src = fiona.open(path_coutwildrnp_shp) itr = iter(src) - feats = list(itr) + list(itr) src.close() with pytest.raises(FionaValueError): next(itr) + + +def test_collection_no_env(path_coutwildrnp_shp): + """We have no GDAL env left over from open""" + collection = fiona.open(path_coutwildrnp_shp) + assert collection + with pytest.raises(Exception): + getenv() + + +def test_collection_env(path_coutwildrnp_shp): + """We have a GDAL env within collection context""" + with fiona.open(path_coutwildrnp_shp): + assert 'FIONA_ENV' in getenv() diff -Nru fiona-1.8.4/tests/test_data_paths.py fiona-1.8.6/tests/test_data_paths.py --- fiona-1.8.4/tests/test_data_paths.py 1970-01-01 00:00:00.000000000 +0000 +++ fiona-1.8.6/tests/test_data_paths.py 2019-03-19 04:25:07.000000000 +0000 @@ -0,0 +1,54 @@ +"""Tests of GDAL and PROJ data finding""" + +import os.path + +from click.testing import CliRunner +import pytest + +import fiona +from fiona._env import GDALDataFinder, PROJDataFinder +from fiona.fio.main import main_group + + +@pytest.mark.wheel +def test_gdal_data_wheel(): + """Get GDAL data path from a wheel""" + assert GDALDataFinder().search() == os.path.join(os.path.dirname(fiona.__file__), 'gdal_data') + + +@pytest.mark.wheel +def test_proj_data_wheel(): + """Get GDAL data path from a wheel""" + assert PROJDataFinder().search() == os.path.join(os.path.dirname(fiona.__file__), 'proj_data') + + +@pytest.mark.wheel +def test_env_gdal_data_wheel(): + runner = CliRunner() + result = runner.invoke(main_group, ['env', '--gdal-data']) + assert result.exit_code == 0 + assert result.output.strip() == os.path.join(os.path.dirname(fiona.__file__), 'gdal_data') + + +@pytest.mark.wheel +def test_env_proj_data_wheel(): + runner = CliRunner() + result = runner.invoke(main_group, ['env', '--proj-data']) + assert result.exit_code == 0 + assert result.output.strip() == os.path.join(os.path.dirname(fiona.__file__), 'proj_data') + + +def test_env_gdal_data_environ(monkeypatch): + monkeypatch.setenv('GDAL_DATA', '/foo/bar') + runner = CliRunner() + result = runner.invoke(main_group, ['env', '--gdal-data']) + assert result.exit_code == 0 + assert result.output.strip() == '/foo/bar' + + +def test_env_proj_data_environ(monkeypatch): + monkeypatch.setenv('PROJ_LIB', '/foo/bar') + runner = CliRunner() + result = runner.invoke(main_group, ['env', '--proj-data']) + assert result.exit_code == 0 + assert result.output.strip() == '/foo/bar' diff -Nru fiona-1.8.4/tests/test_drvsupport.py fiona-1.8.6/tests/test_drvsupport.py --- fiona-1.8.4/tests/test_drvsupport.py 1970-01-01 00:00:00.000000000 +0000 +++ fiona-1.8.6/tests/test_drvsupport.py 2019-03-19 04:25:07.000000000 +0000 @@ -0,0 +1,14 @@ +"""Tests of driver support""" + +import pytest + +from .conftest import requires_gdal24 + +import fiona.drvsupport + + +@requires_gdal24 +@pytest.mark.parametrize('format', ['GeoJSON', 'ESRIJSON', 'TopoJSON', 'GeoJSONSeq']) +def test_geojsonseq(format): + """Format is available""" + assert format in fiona.drvsupport.supported_drivers.keys() diff -Nru fiona-1.8.4/tests/test_encoding.py fiona-1.8.6/tests/test_encoding.py --- fiona-1.8.4/tests/test_encoding.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/test_encoding.py 2019-03-19 04:25:07.000000000 +0000 @@ -9,6 +9,8 @@ import fiona +from .conftest import requires_gdal2 + @pytest.fixture(scope='function') def gre_shp_cp1252(tmpdir): @@ -20,17 +22,32 @@ tmpdir = tmpdir.mkdir('data') for filename in test_files: shutil.copy(filename, str(tmpdir)) - tmpdir.join('gre.cpg').write('cp1252') + tmpdir.join('gre.cpg').write('CP1252') yield tmpdir.join('gre.shp') +@requires_gdal2 def test_broken_encoding(gre_shp_cp1252): """Reading as cp1252 mis-encodes a Russian name""" with fiona.open(str(gre_shp_cp1252)) as src: - assert next(iter(src))['properties']['name_ru'] != u'Гренада' + assert src.session._get_internal_encoding() == 'utf-8' + feat = next(iter(src)) + assert feat['properties']['name_ru'] != u'Гренада' + + +@requires_gdal2 +def test_cpg_encoding(gre_shp_cp1252): + """Reads a Russian name""" + gre_shp_cp1252.join('../gre.cpg').write('UTF-8') + with fiona.open(str(gre_shp_cp1252)) as src: + assert src.session._get_internal_encoding() == 'utf-8' + feat = next(iter(src)) + assert feat['properties']['name_ru'] == u'Гренада' +@requires_gdal2 def test_override_encoding(gre_shp_cp1252): """utf-8 override succeeds""" with fiona.open(str(gre_shp_cp1252), encoding='utf-8') as src: + assert src.session._get_internal_encoding() == 'utf-8' assert next(iter(src))['properties']['name_ru'] == u'Гренада' diff -Nru fiona-1.8.4/tests/test__env.py fiona-1.8.6/tests/test__env.py --- fiona-1.8.4/tests/test__env.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/test__env.py 2019-03-19 04:25:07.000000000 +0000 @@ -1,6 +1,10 @@ """Tests of _env util module""" import pytest +try: + from unittest import mock +except ImportError: + import mock from fiona._env import GDALDataFinder, PROJDataFinder diff -Nru fiona-1.8.4/tests/test_env.py fiona-1.8.6/tests/test_env.py --- fiona-1.8.4/tests/test_env.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/test_env.py 2019-03-19 04:25:07.000000000 +0000 @@ -2,13 +2,17 @@ import os import sys +try: + from unittest import mock +except ImportError: + import mock import pytest import fiona from fiona import _env from fiona.env import getenv, ensure_env, ensure_env_with_credentials -from fiona.session import AWSSession +from fiona.session import AWSSession, GSSession def test_nested_credentials(monkeypatch): @@ -46,12 +50,14 @@ assert f() == '/lol/wut' -def test_ensure_env_decorator_sets_gdal_data_prefix(gdalenv, monkeypatch, tmpdir): +@mock.patch("fiona._env.GDALDataFinder.find_file") +def test_ensure_env_decorator_sets_gdal_data_prefix(find_file, gdalenv, monkeypatch, tmpdir): """fiona.env.ensure_env finds GDAL data under a prefix""" @ensure_env def f(): return getenv()['GDAL_DATA'] + find_file.return_value = None tmpdir.ensure("share/gdal/pcs.csv") monkeypatch.delenv('GDAL_DATA', raising=False) monkeypatch.setattr(_env, '__file__', str(tmpdir.join("fake.py"))) @@ -60,12 +66,14 @@ assert f() == str(tmpdir.join("share").join("gdal")) -def test_ensure_env_decorator_sets_gdal_data_wheel(gdalenv, monkeypatch, tmpdir): +@mock.patch("fiona._env.GDALDataFinder.find_file") +def test_ensure_env_decorator_sets_gdal_data_wheel(find_file, gdalenv, monkeypatch, tmpdir): """fiona.env.ensure_env finds GDAL data in a wheel""" @ensure_env def f(): return getenv()['GDAL_DATA'] + find_file.return_value = None tmpdir.ensure("gdal_data/pcs.csv") monkeypatch.delenv('GDAL_DATA', raising=False) monkeypatch.setattr(_env, '__file__', str(tmpdir.join(os.path.basename(_env.__file__)))) @@ -73,12 +81,14 @@ assert f() == str(tmpdir.join("gdal_data")) -def test_ensure_env_with_decorator_sets_gdal_data_wheel(gdalenv, monkeypatch, tmpdir): +@mock.patch("fiona._env.GDALDataFinder.find_file") +def test_ensure_env_with_decorator_sets_gdal_data_wheel(find_file, gdalenv, monkeypatch, tmpdir): """fiona.env.ensure_env finds GDAL data in a wheel""" @ensure_env_with_credentials def f(*args): return getenv()['GDAL_DATA'] + find_file.return_value = None tmpdir.ensure("gdal_data/pcs.csv") monkeypatch.delenv('GDAL_DATA', raising=False) monkeypatch.setattr(_env, '__file__', str(tmpdir.join(os.path.basename(_env.__file__)))) @@ -89,3 +99,17 @@ def test_ensure_env_crs(path_coutwildrnp_shp): """Decoration of .crs works""" assert fiona.open(path_coutwildrnp_shp).crs + + +def test_nested_gs_credentials(monkeypatch): + """Check that rasterio.open() doesn't wipe out surrounding credentials""" + + @ensure_env_with_credentials + def fake_opener(path): + return fiona.env.getenv() + + with fiona.env.Env(session=GSSession(google_application_credentials='foo')): + assert fiona.env.getenv()['GOOGLE_APPLICATION_CREDENTIALS'] == 'foo' + + gdalenv = fake_opener('gs://foo/bar') + assert gdalenv['GOOGLE_APPLICATION_CREDENTIALS'] == 'foo' diff -Nru fiona-1.8.4/tests/test_fio_ls.py fiona-1.8.6/tests/test_fio_ls.py --- fiona-1.8.4/tests/test_fio_ls.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/test_fio_ls.py 2019-03-19 04:25:07.000000000 +0000 @@ -15,7 +15,7 @@ result = CliRunner().invoke(main_group, ['ls', data_dir]) assert result.exit_code == 0 assert len(result.output.splitlines()) == 1 - assert sorted(json.loads(result.output)) == ['coutwildrnp', 'gre'] + assert sorted(json.loads(result.output)) == ['coutwildrnp', 'gre', 'test_tin'] def test_fio_ls_indent(path_coutwildrnp_shp): diff -Nru fiona-1.8.4/tests/test_listing.py fiona-1.8.6/tests/test_listing.py --- fiona-1.8.4/tests/test_listing.py 2018-12-10 23:58:01.000000000 +0000 +++ fiona-1.8.6/tests/test_listing.py 2019-03-19 04:25:07.000000000 +0000 @@ -22,11 +22,11 @@ def test_directory(data_dir): - assert sorted(fiona.listlayers(data_dir)) == ['coutwildrnp', 'gre'] + assert sorted(fiona.listlayers(data_dir)) == ['coutwildrnp', 'gre', 'test_tin'] def test_directory_trailing_slash(data_dir): - assert sorted(fiona.listlayers(data_dir)) == ['coutwildrnp', 'gre'] + assert sorted(fiona.listlayers(data_dir)) == ['coutwildrnp', 'gre', 'test_tin'] def test_zip_path(path_coutwildrnp_zip): diff -Nru fiona-1.8.4/tests/test_rfc64_tin.py fiona-1.8.6/tests/test_rfc64_tin.py --- fiona-1.8.4/tests/test_rfc64_tin.py 1970-01-01 00:00:00.000000000 +0000 +++ fiona-1.8.6/tests/test_rfc64_tin.py 2019-03-19 04:25:07.000000000 +0000 @@ -0,0 +1,49 @@ +"""Tests of features related to GDAL RFC 64 + +See https://trac.osgeo.org/gdal/wiki/rfc64_triangle_polyhedralsurface_tin. +""" + +import fiona + +from .conftest import requires_gdal22 + + +def test_tin_shp(path_test_tin_shp): + """Convert TIN to MultiPolygon""" + with fiona.open(path_test_tin_shp) as col: + assert col.schema['geometry'] == 'Unknown' + features = list(col) + assert len(features) == 1 + assert features[0]['geometry']['type'] == 'MultiPolygon' + assert features[0]['geometry']['coordinates'] == [[[(0.0, 0.0, 0.0), + (0.0, 0.0, 1.0), + (0.0, 1.0, 0.0), + (0.0, 0.0, 0.0)]], + [[(0.0, 0.0, 0.0), + (0.0, 1.0, 0.0), + (1.0, 1.0, 0.0), + (0.0, 0.0, 0.0)]]] + + +@requires_gdal22 +def test_tin_csv(path_test_tin_csv): + """Convert TIN to MultiPolygon and Triangle to Polygon""" + with fiona.open(path_test_tin_csv) as col: + assert col.schema['geometry'] == 'Unknown' + features = list(col) + assert len(features) == 2 + assert features[0]['geometry']['type'] == 'MultiPolygon' + assert features[0]['geometry']['coordinates'] == [[[(0.0, 0.0, 0.0), + (0.0, 0.0, 1.0), + (0.0, 1.0, 0.0), + (0.0, 0.0, 0.0)]], + [[(0.0, 0.0, 0.0), + (0.0, 1.0, 0.0), + (1.0, 1.0, 0.0), + (0.0, 0.0, 0.0)]]] + + assert features[1]['geometry']['type'] == 'Polygon' + assert features[1]['geometry']['coordinates'] == [[(0.0, 0.0, 0.0), + (0.0, 1.0, 0.0), + (1.0, 1.0, 0.0), + (0.0, 0.0, 0.0)]]