diff -Nru trollimage-1.12.0/debian/changelog trollimage-1.13.0/debian/changelog --- trollimage-1.12.0/debian/changelog 2020-03-04 06:56:32.000000000 +0000 +++ trollimage-1.13.0/debian/changelog 2020-06-13 06:11:28.000000000 +0000 @@ -1,3 +1,11 @@ +trollimage (1.13.0-1) unstable; urgency=medium + + * New upstream release. + * debian/patches: + - refresh all patches + + -- Antonio Valentino Sat, 13 Jun 2020 06:11:28 +0000 + trollimage (1.12.0-1) unstable; urgency=medium * New upstream release. diff -Nru trollimage-1.12.0/debian/patches/0001-No-display.patch trollimage-1.13.0/debian/patches/0001-No-display.patch --- trollimage-1.12.0/debian/patches/0001-No-display.patch 2020-03-04 06:56:32.000000000 +0000 +++ trollimage-1.13.0/debian/patches/0001-No-display.patch 2020-06-13 06:11:28.000000000 +0000 @@ -8,10 +8,10 @@ 1 file changed, 1 insertion(+) diff --git a/trollimage/tests/test_image.py b/trollimage/tests/test_image.py -index 97328bb..5d6b775 100644 +index bb628a4..cedc688 100644 --- a/trollimage/tests/test_image.py +++ b/trollimage/tests/test_image.py -@@ -1927,6 +1927,7 @@ class TestXRImage(unittest.TestCase): +@@ -1966,6 +1966,7 @@ class TestXRImage(unittest.TestCase): """Test putalpha.""" pass diff -Nru trollimage-1.12.0/doc/conf.py trollimage-1.13.0/doc/conf.py --- trollimage-1.12.0/doc/conf.py 2020-03-02 19:11:54.000000000 +0000 +++ trollimage-1.13.0/doc/conf.py 2020-06-03 15:04:09.000000000 +0000 @@ -251,4 +251,5 @@ 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), 'xarray': ('https://xarray.pydata.org/en/stable', None), 'dask': ('https://dask.pydata.org/en/latest', None), + 'rasterio': ('https://rasterio.readthedocs.io/en/latest', None), } diff -Nru trollimage-1.12.0/trollimage/tests/test_image.py trollimage-1.13.0/trollimage/tests/test_image.py --- trollimage-1.12.0/trollimage/tests/test_image.py 2020-03-02 19:11:54.000000000 +0000 +++ trollimage-1.13.0/trollimage/tests/test_image.py 2020-06-03 15:04:09.000000000 +0000 @@ -808,6 +808,10 @@ img = xrimage.XRImage(data) with NamedTemporaryFile(suffix='.jpg') as tmp: img.save(tmp.name, fill_value=0) + # Jpeg fails without fill value (no alpha handling) + with NamedTemporaryFile(suffix='.jpg') as tmp: + # make sure fill_value is mentioned in the error message + self.assertRaisesRegex(OSError, "fill_value", img.save, tmp.name) # As PNG that support alpha channel img = xrimage.XRImage(data) with NamedTemporaryFile(suffix='.png') as tmp: @@ -1022,6 +1026,17 @@ np.testing.assert_allclose(file_data[1], exp[:, :, 1]) np.testing.assert_allclose(file_data[2], exp[:, :, 2]) np.testing.assert_allclose(file_data[3], 255) + # test .tiff too + with NamedTemporaryFile(suffix='.tiff') as tmp: + img.save(tmp.name) + with rio.open(tmp.name) as f: + file_data = f.read() + self.assertEqual(file_data.shape, (4, 5, 5)) # alpha band added + exp = np.arange(75).reshape(5, 5, 3) + np.testing.assert_allclose(file_data[0], exp[:, :, 0]) + np.testing.assert_allclose(file_data[1], exp[:, :, 1]) + np.testing.assert_allclose(file_data[2], exp[:, :, 2]) + np.testing.assert_allclose(file_data[3], 255) data = xr.DataArray(da.from_array(np.arange(75).reshape(5, 5, 3), chunks=5), dims=['y', 'x', 'bands'], @@ -1212,6 +1227,30 @@ with rio.open(tmp.name) as f: self.assertEqual(len(f.overviews(1)), 2) + # auto-levels + data = np.zeros(25*25*3, dtype=np.uint8).reshape(25, 25, 3) + data = xr.DataArray(data, dims=[ + 'y', 'x', 'bands'], coords={'bands': ['R', 'G', 'B']}) + img = xrimage.XRImage(data) + self.assertTrue(np.issubdtype(img.data.dtype, np.integer)) + with NamedTemporaryFile(suffix='.tif') as tmp: + img.save(tmp.name, overviews=[], overviews_minsize=2) + with rio.open(tmp.name) as f: + self.assertEqual(len(f.overviews(1)), 4) + + # auto-levels and resampling + data = np.zeros(25*25*3, dtype=np.uint8).reshape(25, 25, 3) + data = xr.DataArray(data, dims=[ + 'y', 'x', 'bands'], coords={'bands': ['R', 'G', 'B']}) + img = xrimage.XRImage(data) + self.assertTrue(np.issubdtype(img.data.dtype, np.integer)) + with NamedTemporaryFile(suffix='.tif') as tmp: + img.save(tmp.name, overviews=[], overviews_minsize=2, + overviews_resampling='average') + with rio.open(tmp.name) as f: + # no way to check resampling method from the file + self.assertEqual(len(f.overviews(1)), 4) + @unittest.skipIf(sys.platform.startswith('win'), "'NamedTemporaryFile' not supported on Windows") def test_save_tags(self): """Test saving geotiffs with tags.""" @@ -1956,21 +1995,42 @@ return pil_obj img = xrimage.XRImage(data) - pi = mock.MagicMock() - img.pil_image = pi - res = img.apply_pil(dummy_fun, 'RGB') - # check that the pil image generation is delayed - pi.assert_not_called() - # make it happen - res.data.data.compute() - pi.return_value.convert.assert_called_with('RGB') - - img = xrimage.XRImage(data) - pi = mock.MagicMock() - img.pil_image = pi - res = img.apply_pil(dummy_fun, 'RGB', - fun_args=('Hey', 'Jude'), - fun_kwargs={'chorus': "La lala lalalala"}) - self.assertEqual(dummy_args, [({}, ), {}]) - res.data.data.compute() - self.assertEqual(dummy_args, [(OrderedDict(), 'Hey', 'Jude'), {'chorus': "La lala lalalala"}]) + with mock.patch.object(xrimage, "PILImage") as pi: + pil_img = mock.MagicMock() + pi.fromarray = mock.Mock(wraps=lambda *args, **kwargs: pil_img) + res = img.apply_pil(dummy_fun, 'RGB') + # check that the pil image generation is delayed + pi.fromarray.assert_not_called() + # make it happen + res.data.data.compute() + pil_img.convert.assert_called_with('RGB') + + img = xrimage.XRImage(data) + with mock.patch.object(xrimage, "PILImage") as pi: + pil_img = mock.MagicMock() + pi.fromarray = mock.Mock(wraps=lambda *args, **kwargs: pil_img) + res = img.apply_pil(dummy_fun, 'RGB', + fun_args=('Hey', 'Jude'), + fun_kwargs={'chorus': "La lala lalalala"}) + self.assertEqual(dummy_args, [({}, ), {}]) + res.data.data.compute() + self.assertEqual(dummy_args, [(OrderedDict(), 'Hey', 'Jude'), {'chorus': "La lala lalalala"}]) + + # Test HACK for _burn_overlay + dummy_args = [(OrderedDict(), ), {}] + + def _burn_overlay(pil_obj, *args, **kwargs): + dummy_args[0] = args + dummy_args[1] = kwargs + return pil_obj + + img = xrimage.XRImage(data) + with mock.patch.object(xrimage, "PILImage") as pi: + pil_img = mock.MagicMock() + pi.fromarray = mock.Mock(wraps=lambda *args, **kwargs: pil_img) + res = img.apply_pil(_burn_overlay, 'RGB') + # check that the pil image generation is delayed + pi.fromarray.assert_not_called() + # make it happen + res.data.data.compute() + pil_img.convert.assert_called_with('RGB') diff -Nru trollimage-1.12.0/trollimage/version.py trollimage-1.13.0/trollimage/version.py --- trollimage-1.12.0/trollimage/version.py 2020-03-02 19:11:54.000000000 +0000 +++ trollimage-1.13.0/trollimage/version.py 2020-06-03 15:04:09.000000000 +0000 @@ -23,9 +23,9 @@ # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). - git_refnames = " (HEAD -> master, tag: v1.12.0)" - git_full = "19eb077dce183fafc5be452efd5022c1378ccb48" - git_date = "2020-03-02 13:11:54 -0600" + git_refnames = " (HEAD -> master, tag: v1.13.0)" + git_full = "1f63856486d83faccc1aafea2bea4585a5ab1530" + git_date = "2020-06-03 10:04:09 -0500" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords diff -Nru trollimage-1.12.0/trollimage/xrimage.py trollimage-1.13.0/trollimage/xrimage.py --- trollimage-1.12.0/trollimage/xrimage.py 2020-03-02 19:11:54.000000000 +0000 +++ trollimage-1.13.0/trollimage/xrimage.py 2020-06-03 15:04:09.000000000 +0000 @@ -47,6 +47,7 @@ try: import rasterio + from rasterio.enums import Resampling except ImportError: rasterio = None @@ -75,6 +76,16 @@ self.lock = threading.Lock() @property + def width(self): + """Width of the band images.""" + return self.kwargs['width'] + + @property + def height(self): + """Height of the band images.""" + return self.kwargs['height'] + + @property def closed(self): """Check if the file is closed.""" return self.rfile is None or self.rfile.closed @@ -160,10 +171,15 @@ class RIODataset: """A wrapper for a rasterio dataset.""" - def __init__(self, rfile, overviews=None): + def __init__(self, rfile, overviews=None, overviews_resampling=None, + overviews_minsize=256): """Init the rasterio dataset.""" self.rfile = rfile self.overviews = overviews + if overviews_resampling is None: + overviews_resampling = 'nearest' + self.overviews_resampling = Resampling[overviews_resampling] + self.overviews_minsize = overviews_minsize def __setitem__(self, key, item): """Put the data chunk in the image.""" @@ -190,9 +206,19 @@ def close(self): """Close the file.""" - if self.overviews: - logger.debug('Building overviews %s', str(self.overviews)) - self.rfile.build_overviews(self.overviews) + if self.overviews is not None: + overviews = self.overviews + # it's an empty list + if len(overviews) == 0: + from rasterio.rio.overview import get_maximum_overview_level + width = self.rfile.width + height = self.rfile.height + max_level = get_maximum_overview_level( + width, height, self.overviews_minsize) + overviews = [2 ** j for j in range(1, max_level + 1)] + logger.debug('Building overviews %s with %s resampling', + str(overviews), self.overviews_resampling.name) + self.rfile.build_overviews(overviews, resampling=self.overviews_resampling) return self.rfile.close() @@ -263,6 +289,25 @@ return 1 / scale, -offset / scale +@delayed(nout=1, pure=True) +def delayed_pil_save(img, *args, **kwargs): + """Dask delayed saving of PIL Image object. + + Special wrapper to handle `fill_value` try/except catch and provide a + more useful error message. + + """ + try: + img.save(*args, **kwargs) + except OSError as e: + # ex: cannot write mode LA as JPEG + if "A as JPEG" in str(e): + new_msg = ("Image mode not supported for this format. Specify " + "`fill_value=0` to set invalid values to black.") + raise OSError(new_msg) from e + raise + + class XRImage(object): """Image class using an :class:`xarray.DataArray` as internal storage. @@ -370,7 +415,7 @@ """ kwformat = format_kwargs.pop('format', None) fformat = fformat or kwformat or os.path.splitext(filename)[1][1:] - if fformat in ('tif', 'jp2') and rasterio: + if fformat in ('tif', 'tiff', 'jp2') and rasterio: return self.rio_save(filename, fformat=fformat, fill_value=fill_value, compute=compute, keep_palette=keep_palette, cmap=cmap, @@ -382,6 +427,7 @@ def rio_save(self, filename, fformat=None, fill_value=None, dtype=np.uint8, compute=True, tags=None, keep_palette=False, cmap=None, overviews=None, + overviews_minsize=256, overviews_resampling=None, include_scale_offset_tags=False, **format_kwargs): """Save the image using rasterio. @@ -405,6 +451,20 @@ img.rio_save('myfile.tif', overviews=[2, 4, 8, 16]) + If provided as an empty list, then levels will be + computed as powers of two until the last level has less + pixels than `overviews_minsize`. + Default is to not add overviews. + overviews_minsize (int): Minimum number of pixels for the smallest + overview size generated when `overviews` is auto-generated. + Defaults to 256. + overviews_resampling (str): Resampling method + to use when generating overviews. This must be the name of an + enum value from :class:`rasterio.enums.Resampling` and + only takes effect if the `overviews` keyword argument is + provided. Common values include `nearest` (default), + `bilinear`, `average`, and many others. See the rasterio + documentation for more information. include_scale_offset_tags (bool): Whether or not (default) to include a ``scale`` and an ``offset`` tag in the data that would help retrieving original data values from pixel values. @@ -417,6 +477,7 @@ drivers = {'jpg': 'JPEG', 'png': 'PNG', 'tif': 'GTiff', + 'tiff': 'GTiff', 'jp2': 'JP2OpenJPEG'} driver = drivers.get(fformat, fformat) @@ -515,7 +576,9 @@ continue r_file.rfile.update_tags(**tags) - r_dataset = RIODataset(r_file, overviews) + r_dataset = RIODataset(r_file, overviews, + overviews_resampling=overviews_resampling, + overviews_minsize=overviews_minsize) to_store = (data.data, r_dataset) if da_tags: @@ -551,7 +614,7 @@ format_kwargs['pnginfo'] = self._pngmeta() img = self.pil_image(fill_value, compute=False) - delay = img.save(filename, fformat, **format_kwargs) + delay = delayed_pil_save(img, filename, fformat, **format_kwargs) if compute: return delay.compute() return delay @@ -571,19 +634,15 @@ return combine_scales_offsets(*scaling) @delayed(nout=1, pure=True) - def _delayed_apply_pil(self, fun, pil_args, pil_kwargs, fun_args, fun_kwargs, + def _delayed_apply_pil(self, fun, pil_image, fun_args, fun_kwargs, image_metadata=None, output_mode=None): - if pil_args is None: - pil_args = tuple() - if pil_kwargs is None: - pil_kwargs = dict() if fun_args is None: fun_args = tuple() if fun_kwargs is None: fun_kwargs = dict() if image_metadata is None: image_metadata = dict() - new_img = fun(self.pil_image(*pil_args, **pil_kwargs), image_metadata, *fun_args, **fun_kwargs) + new_img = fun(pil_image, image_metadata, *fun_args, **fun_kwargs) if output_mode is not None: new_img = new_img.convert(output_mode) return np.array(new_img) / self.data.dtype.type(255.0) @@ -599,8 +658,39 @@ The pil_args and pil_kwargs are passed to the `pil_image` method of the XRImage instance. """ - new_array = self._delayed_apply_pil(fun, pil_args, pil_kwargs, fun_args, fun_kwargs, - self.data.attrs, output_mode) + if pil_args is None: + pil_args = tuple() + if pil_kwargs is None: + pil_kwargs = dict() + pil_image = self.pil_image(*pil_args, compute=False, **pil_kwargs) + + # HACK: aggdraw.Font objects cause segmentation fault in dask tokenize + # Remove this when aggdraw is either updated to allow type(font_obj) + # or pycoast is updated to not accept Font objects + # See https://github.com/pytroll/pycoast/issues/43 + # The last positional argument to the _burn_overlay function in Satpy + # is the 'overlay' dict. This could include aggdraw.Font objects so we + # completely remove it. + delayed_kwargs = {} + if fun.__name__ == "_burn_overlay": + from dask.base import tokenize + from dask.utils import funcname + func = self._delayed_apply_pil + if fun_args is None: + fun_args = tuple() + if fun_kwargs is None: + fun_kwargs = dict() + tokenize_args = (fun, pil_image, fun_args[:-1], fun_kwargs, + self.data.attrs, output_mode) + dask_key_name = "%s-%s" % ( + funcname(func), + tokenize(func.key, *tokenize_args, pure=True), + ) + delayed_kwargs['dask_key_name'] = dask_key_name + + new_array = self._delayed_apply_pil(fun, pil_image, fun_args, fun_kwargs, + self.data.attrs, output_mode, + **delayed_kwargs) bands = len(output_mode) arr = da.from_delayed(new_array, dtype=self.data.dtype, shape=(self.data.sizes['y'], self.data.sizes['x'], bands))