diff -Nru libcommons-compress-java-1.13/debian/changelog libcommons-compress-java-1.18/debian/changelog --- libcommons-compress-java-1.13/debian/changelog 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/changelog 2019-03-01 16:57:29.000000000 +0000 @@ -1,3 +1,39 @@ +libcommons-compress-java (1.18-1~18.04) bionic; urgency=medium + + * Backport for OpenJDK 11 (dependency of libapache-poi-java). LP: #1814133. + + -- Matthias Klose Fri, 01 Mar 2019 17:57:29 +0100 + +libcommons-compress-java (1.18-1) unstable; urgency=medium + + * Team upload. + * New upstream version 1.18. + - Fix CVE-2018-11771. + When reading a specially crafted ZIP archive, the read method of Apache + Commons Compress ZipArchiveInputStream can fail to return the correct EOF + indication after the end of the stream has been reached. When combined + with a java.io.InputStreamReader this can lead to an infinite stream, + which can be used to mount a denial of service attack against services + that use Compress' zip package. Thanks to Salvatore Bonaccorso for the + report. (Closes: #906301) + * Declare compliance with Debian Policy 4.2.0. + + -- Markus Koschany Wed, 22 Aug 2018 21:43:55 +0200 + +libcommons-compress-java (1.17-1) unstable; urgency=medium + + * Team upload. + * New upstream release + - Removed the CVE-2018-1324 patch (fixed upstream) + - New build dependency on libmaven-antrun-plugin-java + - Disabled Brotli support (dependency not in Debian) + - Disabled Zstandard support (dependency not in Debian) + * Rebuilt with Java 8 compatibility (Closes: #903774) + * Standards-Version updated to 4.1.5 + * Use salsa.debian.org Vcs-* URLs + + -- Emmanuel Bourg Mon, 16 Jul 2018 13:11:58 +0200 + libcommons-compress-java (1.13-2) unstable; urgency=medium * Team upload. diff -Nru libcommons-compress-java-1.13/debian/control libcommons-compress-java-1.18/debian/control --- libcommons-compress-java-1.13/debian/control 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/control 2018-08-22 19:43:55.000000000 +0000 @@ -12,14 +12,15 @@ javahelper, junit4, libcommons-parent-java, + libmaven-antrun-plugin-java, libmaven-bundle-plugin-java, libmaven-javadoc-plugin-java, libpowermock-java, libxz-java (>= 1.5), maven-debian-helper -Standards-Version: 4.1.4 -Vcs-Git: https://anonscm.debian.org/git/pkg-java/libcommons-compress-java.git -Vcs-Browser: https://anonscm.debian.org/cgit/pkg-java/libcommons-compress-java.git +Standards-Version: 4.2.0 +Vcs-Git: https://salsa.debian.org/java-team/libcommons-compress-java.git +Vcs-Browser: https://salsa.debian.org/java-team/libcommons-compress-java Homepage: https://commons.apache.org/proper/commons-compress/ Package: libcommons-compress-java diff -Nru libcommons-compress-java-1.13/debian/copyright libcommons-compress-java-1.18/debian/copyright --- libcommons-compress-java-1.13/debian/copyright 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/copyright 2018-08-22 19:43:55.000000000 +0000 @@ -1,15 +1,18 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: Apache Commons Compress Upstream-Contact: Apache Commons Developers -Source: https://commons.apache.org/proper/commons-compress/ +Source: https://commons.apache.org/compress/ Files: * -Copyright: 2001-2016, The Apache Software Foundation. +Copyright: 2001-2018, The Apache Software Foundation. License: Apache-2.0 Files: debian/* -Copyright: 2007-2009, Torsten Werner - 2018, tony mancill +Copyright: 2007-2011, Torsten Werner + 2012, Damien Raude-Morvan + 2012, Miguel Landaeta + 2013-2018, tony mancill + 2013-2018, Emmanuel Bourg License: Apache-2.0 License: Apache-2.0 diff -Nru libcommons-compress-java-1.13/debian/maven.ignoreRules libcommons-compress-java-1.18/debian/maven.ignoreRules --- libcommons-compress-java-1.13/debian/maven.ignoreRules 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/maven.ignoreRules 2018-08-22 19:43:55.000000000 +0000 @@ -1,8 +1,13 @@ +com.github.luben zstd-jni * * * * org.apache.maven.plugins maven-assembly-plugin * * * * org.apache.maven.plugins maven-javadoc-plugin * * * * org.apache.maven.plugins maven-pmd-plugin * * * * org.apache.maven.plugins maven-scm-publish-plugin * * * * org.apache.rat apache-rat-plugin * * * * +org.brotli dec * * * * org.codehaus.mojo cobertura-maven-plugin * * * * org.codehaus.mojo findbugs-maven-plugin * * * * +org.eluder.coveralls coveralls-maven-plugin * * * * +org.ops4j.pax.exam * * * * * +com.github.siom79.japicmp japicmp-maven-plugin * * * * diff -Nru libcommons-compress-java-1.13/debian/maven.properties libcommons-compress-java-1.18/debian/maven.properties --- libcommons-compress-java-1.13/debian/maven.properties 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/maven.properties 2018-08-22 19:43:55.000000000 +0000 @@ -5,3 +5,4 @@ project.build.sourceEncoding=UTF-8 maven.test.failure.ignore=true +maven.compiler.release=8 diff -Nru libcommons-compress-java-1.13/debian/patches/CVE-2018-1324.patch libcommons-compress-java-1.18/debian/patches/CVE-2018-1324.patch --- libcommons-compress-java-1.13/debian/patches/CVE-2018-1324.patch 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/patches/CVE-2018-1324.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -Description: CVE-2018-1324 -Author: Stefan Bodewig -Forwarded: not-needed -Source: https://git-wip-us.apache.org/repos/asf?p=commons-compress.git;a=blobdiff;f=src/main/java/org/apache/commons/compress/archivers/zip/X0017_StrongEncryptionHeader.java;h=acc3b22346b49845e85b5ef27a5814b69e834139;hp=0feb9c98cc622cde1defa3bbd268ef82b4ae5c18;hb=2a2f1dc48e22a34ddb72321a4db211da91aa933b;hpb=dcb0486fb4cb2b6592c04d6ec2edbd3f690df5f2 -Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=893174 - ---- a/src/main/java/org/apache/commons/compress/archivers/zip/X0017_StrongEncryptionHeader.java -+++ b/src/main/java/org/apache/commons/compress/archivers/zip/X0017_StrongEncryptionHeader.java -@@ -310,7 +310,7 @@ public class X0017_StrongEncryptionHeader extends PKWareExtraHeader { - this.hashAlg = HashAlgorithm.getAlgorithmByCode(ZipShort.getValue(data, offset + 12)); - this.hashSize = ZipShort.getValue(data, offset + 14); - // srlist... hashed public keys -- for (int i = 0; i < this.rcount; i++) { -+ for (long i = 0; i < this.rcount; i++) { - for (int j = 0; j < this.hashSize; j++) { - // ZipUtil.signedByteToUnsignedInt(data[offset + 16 + (i * this.hashSize) + j])); - } diff -Nru libcommons-compress-java-1.13/debian/patches/disable-brotli.patch libcommons-compress-java-1.18/debian/patches/disable-brotli.patch --- libcommons-compress-java-1.13/debian/patches/disable-brotli.patch 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/debian/patches/disable-brotli.patch 2018-08-22 19:43:55.000000000 +0000 @@ -0,0 +1,45 @@ +--- a/pom.xml ++++ b/pom.xml +@@ -314,6 +314,17 @@ + + + ++ maven-compiler-plugin ++ ++ ++ **/brotli/** ++ ++ ++ **/brotli/** ++ ++ ++ ++ + + maven-assembly-plugin + +--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java ++++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java +@@ -31,8 +31,6 @@ + import java.util.SortedMap; + import java.util.TreeMap; + +-import org.apache.commons.compress.compressors.brotli.BrotliCompressorInputStream; +-import org.apache.commons.compress.compressors.brotli.BrotliUtils; + import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; + import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; + import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream; +@@ -585,13 +583,6 @@ + return new BZip2CompressorInputStream(in, actualDecompressConcatenated); + } + +- if (BROTLI.equalsIgnoreCase(name)) { +- if (!BrotliUtils.isBrotliCompressionAvailable()) { +- throw new CompressorException("Brotli compression is not available." + YOU_NEED_BROTLI_DEC); +- } +- return new BrotliCompressorInputStream(in); +- } +- + if (XZ.equalsIgnoreCase(name)) { + if (!XZUtils.isXZCompressionAvailable()) { + throw new CompressorException("XZ compression is not available." + YOU_NEED_XZ_JAVA); diff -Nru libcommons-compress-java-1.13/debian/patches/disable-osgi-tests.patch libcommons-compress-java-1.18/debian/patches/disable-osgi-tests.patch --- libcommons-compress-java-1.13/debian/patches/disable-osgi-tests.patch 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/debian/patches/disable-osgi-tests.patch 2018-08-22 19:43:55.000000000 +0000 @@ -0,0 +1,10 @@ +--- a/pom.xml ++++ b/pom.xml +@@ -323,6 +323,7 @@ + + **/brotli/** + **/zstandard/** ++ **/OsgiITest.java + + + diff -Nru libcommons-compress-java-1.13/debian/patches/disable-zstd.patch libcommons-compress-java-1.18/debian/patches/disable-zstd.patch --- libcommons-compress-java-1.13/debian/patches/disable-zstd.patch 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/debian/patches/disable-zstd.patch 2018-08-22 19:43:55.000000000 +0000 @@ -0,0 +1,91 @@ +--- a/pom.xml ++++ b/pom.xml +@@ -318,9 +318,11 @@ + + + **/brotli/** ++ **/zstandard/** + + + **/brotli/** ++ **/zstandard/** + + + +--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java ++++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java +@@ -54,9 +54,6 @@ + import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; + import org.apache.commons.compress.compressors.xz.XZUtils; + import org.apache.commons.compress.compressors.z.ZCompressorInputStream; +-import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; +-import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; +-import org.apache.commons.compress.compressors.zstandard.ZstdUtils; + import org.apache.commons.compress.utils.IOUtils; + import org.apache.commons.compress.utils.Lists; + import org.apache.commons.compress.utils.ServiceLoaderIterator; +@@ -516,10 +513,6 @@ + return LZ4_FRAMED; + } + +- if (ZstdUtils.matches(signature, signatureLength)) { +- return ZSTANDARD; +- } +- + throw new CompressorException("No Compressor found for the stream signature."); + } + /** +@@ -590,13 +583,6 @@ + return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb); + } + +- if (ZSTANDARD.equalsIgnoreCase(name)) { +- if (!ZstdUtils.isZstdCompressionAvailable()) { +- throw new CompressorException("Zstandard compression is not available." + YOU_NEED_ZSTD_JNI); +- } +- return new ZstdCompressorInputStream(in); +- } +- + if (LZMA.equalsIgnoreCase(name)) { + if (!LZMAUtils.isLZMACompressionAvailable()) { + throw new CompressorException("LZMA compression is not available" + YOU_NEED_XZ_JAVA); +@@ -708,10 +694,6 @@ + if (LZ4_FRAMED.equalsIgnoreCase(name)) { + return new FramedLZ4CompressorOutputStream(out); + } +- +- if (ZSTANDARD.equalsIgnoreCase(name)) { +- return new ZstdCompressorOutputStream(out); +- } + } catch (final IOException e) { + throw new CompressorException("Could not create CompressorOutputStream", e); + } +--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java ++++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java +@@ -38,7 +38,6 @@ + import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream; + import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +-import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; + import org.junit.Test; + + @SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated +@@ -114,10 +113,6 @@ + assertNotNull(zlib); + assertTrue(zlib instanceof DeflateCompressorInputStream); + +- final CompressorInputStream zstd = getStreamFor("bla.tar.zst"); +- assertNotNull(zstd); +- assertTrue(zstd instanceof ZstdCompressorInputStream); +- + try { + factory.createCompressorInputStream(new ByteArrayInputStream(new byte[0])); + fail("No exception thrown for an empty input stream"); +@@ -138,7 +133,6 @@ + assertEquals(CompressorStreamFactory.LZMA, detect("bla.tar.lzma")); + assertEquals(CompressorStreamFactory.SNAPPY_FRAMED, detect("bla.tar.sz")); + assertEquals(CompressorStreamFactory.Z, detect("bla.tar.Z")); +- assertEquals(CompressorStreamFactory.ZSTANDARD, detect("bla.tar.zst")); + + //make sure we don't oom on detect + assertEquals(CompressorStreamFactory.Z, detect("COMPRESS-386")); diff -Nru libcommons-compress-java-1.13/debian/patches/series libcommons-compress-java-1.18/debian/patches/series --- libcommons-compress-java-1.13/debian/patches/series 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/patches/series 2018-08-22 19:43:55.000000000 +0000 @@ -1 +1,3 @@ -CVE-2018-1324.patch +disable-brotli.patch +disable-zstd.patch +disable-osgi-tests.patch diff -Nru libcommons-compress-java-1.13/debian/watch libcommons-compress-java-1.18/debian/watch --- libcommons-compress-java-1.13/debian/watch 2018-04-09 03:58:50.000000000 +0000 +++ libcommons-compress-java-1.18/debian/watch 2018-08-22 19:43:55.000000000 +0000 @@ -1,4 +1,4 @@ version=4 opts="compression=xz,\ repack" \ - http://archive.apache.org/dist/commons/compress/source/commons-compress-([\d.]+)-src\.tar\.gz + https://archive.apache.org/dist/commons/compress/source/commons-compress-([\d.]+)-src\.tar\.gz diff -Nru libcommons-compress-java-1.13/findbugs-exclude-filter.xml libcommons-compress-java-1.18/findbugs-exclude-filter.xml --- libcommons-compress-java-1.13/findbugs-exclude-filter.xml 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/findbugs-exclude-filter.xml 2018-05-02 20:17:13.000000000 +0000 @@ -49,6 +49,11 @@ + + + + + @@ -185,5 +190,18 @@ + + + + + + + + + + + + diff -Nru libcommons-compress-java-1.13/NOTICE.txt libcommons-compress-java-1.18/NOTICE.txt --- libcommons-compress-java-1.13/NOTICE.txt 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/NOTICE.txt 2018-05-23 12:50:54.000000000 +0000 @@ -1,8 +1,8 @@ Apache Commons Compress -Copyright 2002-2016 The Apache Software Foundation +Copyright 2002-2018 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +The Apache Software Foundation (https://www.apache.org/). The files in the package org.apache.commons.compress.archivers.sevenz were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), diff -Nru libcommons-compress-java-1.13/pom.xml libcommons-compress-java-1.18/pom.xml --- libcommons-compress-java-1.13/pom.xml 2016-12-25 12:09:13.000000000 +0000 +++ libcommons-compress-java-1.18/pom.xml 2018-08-13 07:13:46.000000000 +0000 @@ -20,42 +20,59 @@ org.apache.commons commons-parent - 41 + 47 - org.apache.commons commons-compress - 1.13 + 1.18 Apache Commons Compress - http://commons.apache.org/proper/commons-compress/ + https://commons.apache.org/proper/commons-compress/ Apache Commons Compress software defines an API for working with compression and archive formats. These include: bzip2, gzip, pack200, -lzma, xz, Snappy, traditional Unix Compress, DEFLATE and ar, cpio, -jar, tar, zip, dump, 7z, arj. +lzma, xz, Snappy, traditional Unix Compress, DEFLATE, DEFLATE64, LZ4, +Brotli, Zstandard and ar, cpio, jar, tar, zip, dump, 7z, arj. 1.7 1.7 compress + org.apache.commons.compress COMPRESS 12310904 ${project.version} RC1 - 1.6.4 - 3.7 - 0.9.3 + 1.7.3 + 3.8 + + ${project.build.outputDirectory}/META-INF + ${commons.manifestlocation}/MANIFEST.MF + + org.tukaani.xz;resolution:=optional, + org.brotli.dec;resolution:=optional, + com.github.luben.zstd;resolution:=optional, + javax.crypto.*;resolution:=optional, + * + true + + + false + + 0.11.1 + + 4.11.0 + 1.7.21 jira - http://issues.apache.org/jira/browse/COMPRESS + https://issues.apache.org/jira/browse/COMPRESS @@ -66,9 +83,21 @@ test + com.github.luben + zstd-jni + 1.3.3-3 + true + + + org.brotli + dec + 0.1.2 + true + + org.tukaani xz - 1.6 + 1.8 true @@ -83,6 +112,57 @@ ${powermock.version} test + + + + org.ops4j.pax.exam + pax-exam-container-native + ${pax.exam.version} + test + + + org.ops4j.pax.exam + pax-exam-junit4 + ${pax.exam.version} + test + + + org.ops4j.pax.exam + pax-exam-cm + ${pax.exam.version} + test + + + org.ops4j.pax.exam + pax-exam-link-mvn + ${pax.exam.version} + test + + + org.apache.felix + org.apache.felix.framework + 5.6.10 + test + + + javax.inject + javax.inject + 1 + test + + + org.slf4j + slf4j-api + ${slf4j.version} + test + + + + org.osgi + org.osgi.core + 6.0.0 + provided + @@ -126,6 +206,11 @@ ggregory ggregory at apache.org + + Rob Tompkins + chtompki + chtompki at apache.org + @@ -155,10 +240,18 @@ BELUGA BEHR + + Simon Spero + sesuncedu@gmail.com + + + Michael Hausegger + hausegger.michael@googlemail.com + - scm:git:http://git-wip-us.apache.org/repos/asf/commons-compress.git + scm:git:https://git-wip-us.apache.org/repos/asf/commons-compress.git scm:git:https://git-wip-us.apache.org/repos/asf/commons-compress.git https://git-wip-us.apache.org/repos/asf?p=commons-compress.git @@ -170,11 +263,12 @@ org.apache.maven.plugins maven-javadoc-plugin + ${commons.javadoc.version} true ${maven.compiler.source} ${commons.encoding} - ${commons.docEncoding} + ${commons.docEncoding} true ${commons.javadoc.java.link} @@ -209,18 +303,28 @@ src/test/resources/** .pmd .projectile + .mvn/** + org.eluder.coveralls + coveralls-maven-plugin + + false + + + + org.apache.felix + maven-bundle-plugin + ${commons.felix.version} + + + com.github.siom79.japicmp japicmp-maven-plugin - ${commons.japicmp.version} - - true - false - + false @@ -244,6 +348,7 @@ org.apache.commons.compress.archivers.Lister org.apache.commons.compress + ${commons.module.name} @@ -252,9 +357,7 @@ org.apache.felix maven-bundle-plugin - - org.tukaani.xz;resolution:=optional - + ${commons.manifestlocation} @@ -271,6 +374,34 @@ maven-pmd-plugin ${commons.pmd-plugin.version} + + org.apache.maven.plugins + maven-antrun-plugin + + + process-test-resources + + + + + + + run + + + + + + maven-surefire-plugin + + + ${karaf.version} + ${project.version} + + + @@ -297,7 +428,7 @@ true ${maven.compiler.source} ${commons.encoding} - ${commons.docEncoding} + ${commons.docEncoding} true ${commons.javadoc.java.link} @@ -325,7 +456,7 @@ org.codehaus.mojo findbugs-maven-plugin - 3.0.4 + 3.0.5 Normal Default @@ -386,34 +517,20 @@ - - travis + java9+ - - env.TRAVIS - true - + [9,) - - - - org.jacoco - jacoco-maven-plugin - ${commons.jacoco.version} - - - org.eluder.coveralls - coveralls-maven-plugin - 4.3.0 - - EpochMillis - - - - + + 9 + 0.7.9 + true + + true + + diff -Nru libcommons-compress-java-1.13/README.txt libcommons-compress-java-1.18/README.txt --- libcommons-compress-java-1.13/README.txt 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/README.txt 2018-05-23 12:50:54.000000000 +0000 @@ -4,7 +4,7 @@ Commons Compress is a Java library for working with various compression and archiving formats. -For full documentation see http://commons.apache.org/proper/commons-compress/ +For full documentation see https://commons.apache.org/proper/commons-compress/ ## Apache Commons Compress was derived from various sources, including: diff -Nru libcommons-compress-java-1.13/RELEASE-NOTES.txt libcommons-compress-java-1.18/RELEASE-NOTES.txt --- libcommons-compress-java-1.13/RELEASE-NOTES.txt 2016-12-25 12:16:34.000000000 +0000 +++ libcommons-compress-java-1.18/RELEASE-NOTES.txt 2018-08-13 06:56:10.000000000 +0000 @@ -2,8 +2,279 @@ Apache Commons Compress software defines an API for working with compression and archive formats. These include: bzip2, gzip, pack200, -lzma, xz, Snappy, traditional Unix Compress, DEFLATE and ar, cpio, -jar, tar, zip, dump, 7z, arj. +lzma, xz, Snappy, traditional Unix Compress, DEFLATE, DEFLATE64, LZ4, +Brotli, Zstandard and ar, cpio, jar, tar, zip, dump, 7z, arj. + +Release 1.18 +------------ + +New features: +o It is now possible to specify the arguments of zstd-jni's + ZstdOutputStream constructors via Commons Compress as well. + Issue: COMPRESS-460. + Thanks to Carmi Grushko. + +Fixed Bugs: +o The example Expander class has been vulnerable to a path + traversal in the edge case that happens when the target + directory has a sibling directory and the name of the target + directory is a prefix of the sibling directory's name. + Thanks to Didier Loiseau. +o Changed the OSGi Import-Package to also optionally import + javax.crypto so encrypted archives can be read. + Issue: COMPRESS-456. +o Changed various implementations of the close method to better + ensure all held resources get closed even if exceptions are + thrown during the closing the stream. + Issue: COMPRESS-457. +o ZipArchiveInputStream can now detect the APK Signing Block + used in signed Android APK files and treats it as an "end of + archive" marker. + Issue: COMPRESS-455. +o The cpio streams didn't handle archives using a multi-byte + encoding properly. + Issue: COMPRESS-459. + Thanks to Jens Reimann. +o ZipArchiveInputStream#read would silently return -1 on a + corrupted stored entry and even return > 0 after hitting the + end of the archive. + Issue: COMPRESS-463. +o ArArchiveInputStream#read would allow to read from the stream + without opening an entry at all. + Issue: COMPRESS-462. + +Release 1.17 +------------ + +New features: +o Added a unit test that is supposed to fail if we break the + OSGi manifest entries again. + Issue: COMPRESS-443. +o Add a new SkipShieldingInputStream class that can be used with + streams that throw an IOException when skip is invoked. + Issue: COMPRESS-449. +o New constructors have been added to SevenZFile that accept + char[]s rather than byte[]s in order to avoid a common error + of using the wrong encoding when creating the byte[]. This + change may break source compatibility for client code that + uses one of the constructors expecting a password and passes + in null as password. We recommend to change the code to use a + constructor without password argument. + Issue: COMPRESS-452. + +Fixed Bugs: +o Removed the objenesis dependency from the pom as it is not + needed at all. +o Fixed resource leak in ParallelScatterZipCreator#writeTo. + Issue: COMPRESS-446. +o Certain errors when parsing ZIP extra fields in corrupt + archives are now turned into ZipException, they used to + manifest as ArrayIndexOutOfBoundsException before. + Issue: COMPRESS-447. +o IOUtils.copy now verifies the buffer size is bigger than 0. + Issue: COMPRESS-451. +o ZipArchiveInputStream failed to read some files with stored + entries using a data descriptor. + Issue: COMPRESS-454. + +Changes: +o Fixed some code examples. + Github Pull Request #63. + Thanks to Marchenko Sergey. +o The streams returned by ZipFile and most other decompressing + streams now provide information about the number of compressed + and uncompressed bytes read so far. This may be used to detect + a ZipBomb if the compression ratio exceeds a certain + threshold, for example. + For SevenZFile a new method returns the statistics for the + current entry. + Issue: COMPRESS-445. + Thanks to Andreas Beeker. +o Added a workaround for a bug in AdoptOpenJDK for S/390 to + BZip2CompressorInputStream. + Issue: COMPRESS-453. + +Release 1.16.1 +-------------- + +Fixed Bug: +o Fixed the OSGi manifest entry for imports that has been broken + in 1.16. + Issue: COMPRESS-442. + +Release 1.16 +------------ + +New features: +o Add read-only support for Zstandard compression based on the + Zstd-jni project. + Issue: COMPRESS-423. Thanks to Andre F de Miranda. +o Added auto-detection for Zstandard compressed streams. + Issue: COMPRESS-425. +o Added write-support for Zstandard compression. + Issue: COMPRESS-426. +o Added read-only DEFLATE64 support to ZIP archives and as + stand-alone CompressorInputStream. + Issue: COMPRESS-380. Thanks to Christian Marquez Grabia. +o Added read-only DEFLATE64 support to 7z archives. + Issue: COMPRESS-437. + +Fixed Bugs: +o Synchronized iteration over a synchronizedList in + ParallelScatterZipCreator. + Issue: COMPRESS-430. Thanks to Bruno P. Kinoshita. +o ZipFile could get stuck in an infinite loop when parsing ZIP + archives with certain strong encryption headers. + Issue: COMPRESS-432. +o Added improved checks to detect corrupted bzip2 streams and + throw the expected IOException rather than obscure + RuntimeExceptions. + Issue: COMPRESS-424. + +Changes: +o Replaces instanceof checks with a type marker in LZ77 support code. + Issue: COMPRESS-435. Thanks to BELUGA BEHR. +o Updated XZ for Java dependency to 1.8 in order to pick up bug fix + to LZMA2InputStream's available method. +o ZipArchiveEntry now exposes how the name or comment have been + determined when the entry was read. + Issue: COMPRESS-429. Thanks to Damiano Albani. +o ZipFile.getInputStream will now always buffer the stream + internally in order to improve read performance. + Issue: COMPRESS-438. +o Speed improvement for DEFLATE64 decompression. + Issue: COMPRESS-440. Thanks to Dawid Weiss. +o Added a few extra sanity checks for the rarer compression + methods used in ZIP archives. + Issue: COMPRESS-436. +o Simplified the special handling for the dummy byte required by + zlib when using java.util.zip.Inflater. + Issue: COMPRESS-441. +o Various code cleanups. + Github Pull Request #61. Thanks to Shahab Kondri. +o TarArchiveEntry's preserveLeadingSlashes constructor argument + has been renamed and can now also be used to preserve the + drive letter on Windows. + +Release 1.15 +------------ + +New features: +o Added magic MANIFEST entry Automatic-Module-Name so the module + name will be org.apache.commons.compress when the jar is used + as an automatic module in Java9. + Issue: COMPRESS-397. +o Added a new utility class FixedLengthBlockOutputStream that + can be used to ensure writing always happens in blocks of a + given size. + Issue: COMPRESS-405. Thanks to Simon Spero. +o It is now possible to specify/read custom PAX headers when + writing/reading tar archives. + Issue: COMPRESS-400. Thanks to Simon Spero. + +Fixed Bugs: +o Make sure "version needed to extract" in local file header and + central directory of a ZIP archive agree with each other. + Also ensure the version is set to 2.0 if DEFLATE is used. + Issue: COMPRESS-394. +o Don't use a data descriptor in ZIP archives when copying a raw + entry that already knows its size and CRC information. + Issue: COMPRESS-395. +o Travis build redundantly repeats compilation and tests redundantly + GitHub Pull Request #43. Thanks to Simon Spero. + Issue: COMPRESS-413 +o The MANIFEST of 1.14 lacks an OSGi Import-Package for XZ for + Java. + Issue: COMPRESS-396. +o BUILDING.md now passes the RAT check. + Issue: COMPRESS-406. Thanks to Simon Spero. +o Made sure ChecksumCalculatingInputStream receives valid + checksum and input stream instances via the constructor. + Issue: COMPRESS-412. Thanks to Michael Hausegger. +o TarArchiveOutputStream now verifies the block and record sizes + specified at construction time are compatible with the tar + specification. In particular 512 is the only record size + accepted and the block size must be a multiple of 512. + Issue: COMPRESS-407. Thanks to Simon Spero. +o Fixed class names of CpioArchiveEntry and + CpioArchiveInputStream in various Javadocs. + Issue: COMPRESS-415. +o The code of the extended timestamp zip extra field incorrectly + assumed the time was stored as unsigned 32-bit int and thus + created incorrect results for years after 2037. + Issue: COMPRESS-416. Thanks to Simon Spero. +o Removed ZipEncoding code that became obsolete when we started + to require Java 5 as baseline long ago. + Issue: COMPRESS-410. Thanks to Simon Spero. +o The tar package will no longer try to parse the major and + minor device numbers unless the entry represents a character + or block special file. + Issue: COMPRESS-417. +o When reading tar headers with name fields containing embedded + NULs, the name will now be terminated at the first NUL byte. + Issue: COMPRESS-421. Thanks to Roel Spilker. +o Simplified TarArchiveOutputStream by replacing the internal + buffering with new class FixedLengthBlockOutputStream. + Issue: COMPRESS-409. + +Release 1.14 +------------ + +New features: +o Added write support for Snappy. + Issue: COMPRESS-246. +o Added support for LZ4 (block and frame format). + Issue: COMPRESS-271. +o Add static detect(InputStream in) to CompressorStreamFactory + and ArchiveStreamFactory + Issue: COMPRESS-385. +o Added a way to limit amount of memory ZCompressorStream may + use. + Issue: COMPRESS-382. Thanks to Tim Allison. +o Added a way to limit amount of memory ZCompressorStream may + use. + Issue: COMPRESS-386. Thanks to Tim Allison. +o Added a way to limit amount of memory LZMACompressorStream and + XZCompressorInputStream may use. + Issue: COMPRESS-382. Thanks to Tim Allison. +o Add Brotli decoder based on the Google Brotli library. + Issue: COMPRESS-392. Thanks to Philippe Mouawad. +o ZipEntry now exposes its data offset. + Issue: COMPRESS-390. Thanks to Zbynek Vyskovsky. +o Using ZipArchiveEntry's setAlignment it is now possible to + ensure the data offset of an entry starts at a file position + that at word or page boundaries. + A new extra field has been added for this purpose. + Issue: COMPRESS-391. Thanks to Zbynek Vyskovsky. + +Fixed Bugs: +o SnappyCompressorInputStream slides the window too early + leading to ArrayIndexOutOfBoundsExceptions for some streams. + Issue: COMPRESS-378. +o ZipArchiveEntry#isUnixSymlink now only returns true if the + corresponding link flag is the only file-type flag set. + Issue: COMPRESS-379. Thanks to Guillaume Boué. +o Fixed an integer overflow in CPIO's CRC calculation. + Pull Request #17. Thanks to Daniel Collin. +o Make unit tests work on Windows paths with spaces in their names. + Issue: COMPRESS-387. +o Internal location pointer in ZipFile could get incremented + even if nothing had been read. + Issue: COMPRESS-389. +o LZMACompressorOutputStream#flush would throw an exception + rather than be the NOP it promised to be. + Issue: COMPRESS-393. + +Changes: +o The blocksize for FramedSnappyCompressorInputStream can now be + configured as some IWA files seem to be using blocks larger + than the default 32k. + Issue: COMPRESS-358. +o BZip2CompressorInputstream now uses BitInputStream internally. + Pull Request #13. Thanks to Thomas Meyer. +o Improved performance for concurrent reads from ZipFile when + reading from a file. + Issue: COMPRESS-388. Thanks to Zbynek Vyskovsky. Release 1.13 ------------ @@ -90,9 +361,9 @@ Changes: o Update requirement from Java 5 to 6. - Issue: COMPRESS-349. + Issue: COMPRESS-349. o TarArchiveEntry wastefully allocates empty arrays. - Issue: COMPRESS-350. + Issue: COMPRESS-350. o Javadoc for BZip2CompressorInputStream(InputStream, boolean) should refer to IOEx, not NPE. Issue: COMPRESS-353. @@ -285,7 +556,7 @@ on how to submit bug reports, patches, or suggestions for improvement, see the Apache Commons Compress website: -http://commons.apache.org/compress/ +https://commons.apache.org/compress/ Old Release Notes ================= @@ -603,7 +874,7 @@ ------------- This is a security bugfix release, see -http://commons.apache.org/proper/commons-compress/security.html#Fixed_in_Apache_Commons_Compress_1.4.1 +https://commons.apache.org/proper/commons-compress/security.html#Fixed_in_Apache_Commons_Compress_1.4.1 Fixed Bugs: diff -Nru libcommons-compress-java-1.13/src/changes/changes.xml libcommons-compress-java-1.18/src/changes/changes.xml --- libcommons-compress-java-1.13/src/changes/changes.xml 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/changes/changes.xml 2018-08-10 14:03:28.000000000 +0000 @@ -42,7 +42,355 @@ commons-compress - + + The example Expander class has been vulnerable to a path + traversal in the edge case that happens when the target + directory has a sibling directory and the name of the target + directory is a prefix of the sibling directory's name. + + + Changed the OSGi Import-Package to also optionally import + javax.crypto so encrypted archives can be read. + + + Changed various implementations of the close method to better + ensure all held resources get closed even if exceptions are + thrown during the closing the stream. + + + ZipArchiveInputStream can now detect the APK Signing Block + used in signed Android APK files and treats it as an "end of + archive" marker. + + + The cpio streams didn't handle archives using a multi-byte + encoding properly. + + + It is now possible to specify the arguments of zstd-jni's + ZstdOutputStream constructors via Commons Compress as well. + + + ZipArchiveInputStream#read would silently return -1 on a + corrupted stored entry and even return > 0 after hitting the + end of the archive. + + + ArArchiveInputStream#read would allow to read from the stream + without opening an entry at all. + + + + + Removed the objenesis dependency from the pom as it is not + needed at all. + + + Fixed resource leak in ParallelScatterZipCreator#writeTo. + + + Fixed some code examples. + Github Pull Request #63. + + + Certain errors when parsing ZIP extra fields in corrupt + archives are now turned into ZipException, they used to + manifest as ArrayIndexOutOfBoundsException before. + + + The streams returned by ZipFile and most other decompressing + streams now provide information about the number of compressed + and uncompressed bytes read so far. This may be used to detect + a ZipBomb if the compression ratio exceeds a certain + threshold, for example. + For SevenZFile a new method returns the statistics for the + current entry. + + + Added a unit test that is supposed to fail if we break the + OSGi manifest entries again. + + + Add a new SkipShieldingInputStream class that can be used with + streams that throw an IOException when skip is invoked. + + + IOUtils.copy now verifies the buffer size is bigger than 0. + + + New constructors have been added to SevenZFile that accept + char[]s rather than byte[]s in order to avoid a common error + of using the wrong encoding when creating the byte[]. This + change may break source compatibility for client code that + uses one of the constructors expecting a password and passes + in null as password. We recommend to change the code to use a + constructor without password argument. + + + Added a workaround for a bug in AdoptOpenJDK for S/390 to + BZip2CompressorInputStream. + + + ZipArchiveInputStream failed to read some files with stored + entries using a data descriptor. + + + + + Fixed the OSGi manifest entry for imports that has been broken + in 1.16. + + + + + Add read-only support for Zstandard compression based on the + Zstd-jni project. + + + Added auto-detection for Zstandard compressed streams. + + + Synchronized iteration over a synchronizedList in ParallelScatterZipCreator. + + + ZipFile could get stuck in an infinite loop when parsing ZIP + archives with certain strong encryption headers. + + + Replaces instanceof checks with a type marker in LZ77 support code. + + + Added write-support for Zstandard compression. + + + Added improved checks to detect corrupted bzip2 streams and + throw the expected IOException rather than obscure + RuntimeExceptions. + + + Updated XZ for Java dependency to 1.8 in order to pick up bug + fix to LZMA2InputStream's available method. + + + ZipArchiveEntry now exposes how the name or comment have been + determined when the entry was read. + + + Added read-only DEFLATE64 support to ZIP archives and as + stand-alone CompressorInputStream. + + + ZipFile.getInputStream will now always buffer the stream + internally in order to improve read performance. + + + Speed improvement for DEFLATE64 decompression. + + + Added read-only DEFLATE64 support to 7z archives. + + + Added a few extra sanity checks for the rarer compression + methods used in ZIP archives. + + + Simplified the special handling for the dummy byte required by + zlib when using java.util.zip.Inflater. + + + Various code cleanups. + Github Pull Request #61. + + + TarArchiveEntry's preserveLeadingSlashes constructor argument + has been renamed and can now also be used to preserve the + drive letter on Windows. + + + + + Make sure "version needed to extract" in local file header and + central directory of a ZIP archive agree with each other. + Also ensure the version is set to 2.0 if DEFLATE is used. + + + Don't use a data descriptor in ZIP archives when copying a raw + entry that already knows its size and CRC information. + + + Travis build redundantly repeats compilation and tests redundantly #43. + + + Added magic MANIFEST entry Automatic-Module-Name so the module + name will be org.apache.commons.compress when the jar is used + as an automatic module in Java9. + + + The MANIFEST of 1.14 lacks an OSGi Import-Package for XZ for + Java. + + + BUILDING.md now passes the RAT check. + + + Added a new utility class FixedLengthBlockOutputStream that + can be used to ensure writing always happens in blocks of a + given size. + + + Made sure ChecksumCalculatingInputStream receives valid + checksum and input stream instances via the constructor. + + + TarArchiveOutputStream now verifies the block and record sizes + specified at construction time are compatible with the tar + specification. In particular 512 is the only record size + accepted and the block size must be a multiple of 512. + At the same time the default block size in + TarArchiveOutputStream has been changed from 10240 to 512 + bytes. + + + It is now possible to specify/read custom PAX headers when + writing/reading tar archives. + + + Fixed class names of CpioArchiveEntry and + CpioArchiveInputStream in various Javadocs. + + + The code of the extended timestamp zip extra field incorrectly + assumed the time was stored as unsigned 32-bit int and thus + created incorrect results for years after 2037. + + + Removed ZipEncoding code that became obsolete when we started + to require Java 5 as baseline long ago. + + + The tar package will no longer try to parse the major and + minor device numbers unless the entry represents a character + or block special file. + + + When reading tar headers with name fields containing embedded + NULs, the name will now be terminated at the first NUL byte. + + + Simplified TarArchiveOutputStream by replacing the internal + buffering with new class FixedLengthBlockOutputStream. + + + + + SnappyCompressorInputStream slides the window too early + leading to ArrayIndexOutOfBoundsExceptions for some streams. + + + Added write support for Snappy. + + + The blocksize for FramedSnappyCompressorInputStream can now be + configured as some IWA files seem to be using blocks larger + than the default 32k. + + + ZipArchiveEntry#isUnixSymlink now only returns true if the + corresponding link flag is the only file-type flag set. + + + Added support for LZ4 (block and frame format). + + + BZip2CompressorInputstream now uses BitInputStream internally. + Pull Request #13. + + + Fixed an integer overflow in CPIO's CRC calculation. + Pull Request #17. + + + Add static detect(InputStream in) to CompressorStreamFactory + and ArchiveStreamFactory + + + Make unit tests work on Windows paths with spaces in their names. + + + Improved performance for concurrent reads from ZipFile when + reading from a file. + + + Added a way to limit amount of memory ZCompressorStream may + use. + + + Added a way to limit amount of memory ZCompressorStream may + use. + + + Added a way to limit amount of memory LZMACompressorStream and + XZCompressorInputStream may use. + + + Internal location pointer in ZipFile could get incremented + even if nothing had been read. + + + Add Brotli decoder based on the Google Brotli library. + + + ZipEntry now exposes its data offset. + + + LZMACompressorOutputStream#flush would throw an exception + rather than be the NOP it promised to be. + + + Using ZipArchiveEntry's setAlignment it is now possible to + ensure the data offset of an entry starts at a file position + that at word or page boundaries. + A new extra field has been added for this purpose. + + + Update Java requirement from 6 to 7. diff -Nru libcommons-compress-java-1.13/src/changes/release-notes.vm libcommons-compress-java-1.18/src/changes/release-notes.vm --- libcommons-compress-java-1.13/src/changes/release-notes.vm 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/changes/release-notes.vm 2018-05-23 12:50:54.000000000 +0000 @@ -474,7 +474,7 @@ ------------- This is a security bugfix release, see -http://commons.apache.org/proper/commons-compress/security.html#Fixed_in_Apache_Commons_Compress_1.4.1 +https://commons.apache.org/proper/commons-compress/security.html#Fixed_in_Apache_Commons_Compress_1.4.1 Fixed Bugs: diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveEntry.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveEntry.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveEntry.java 2018-05-23 12:50:54.000000000 +0000 @@ -25,10 +25,10 @@ /** * Represents an archive entry in the "ar" format. - * + * * Each AR archive starts with "!<arch>" followed by a LF. After these 8 bytes * the archive entries are listed. The format of an entry header is as it follows: - * + * * 
  * START BYTE   END BYTE    NAME                    FORMAT      LENGTH
  * 0            15          File name               ASCII       16
@@ -39,15 +39,15 @@
  * 48           57          File size (bytes)       Decimal     10
  * 58           59          File magic              \140\012    2
  *
- * + * * This specifies that an ar archive entry header contains 60 bytes. - * + * * Due to the limitation of the file name length to 16 bytes GNU and * BSD has their own variants of this format. Currently Commons * Compress can read but not write the GNU variant. It fully supports * the BSD variant. - * - * @see ar man page + * + * @see ar man page * * @Immutable */ @@ -68,7 +68,7 @@ private final int userId; private final int groupId; private final int mode; - private static final int DEFAULT_MODE = 33188; // = (octal) 0100644 + private static final int DEFAULT_MODE = 33188; // = (octal) 0100644 private final long lastModified; private final long length; @@ -179,12 +179,9 @@ } final ArArchiveEntry other = (ArArchiveEntry) obj; if (name == null) { - if (other.name != null) { - return false; - } - } else if (!name.equals(other.name)) { - return false; + return other.name == null; + } else { + return name.equals(other.name); } - return true; } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStream.java 2018-08-10 04:45:14.000000000 +0000 @@ -29,9 +29,9 @@ /** * Implements the "ar" archive format as an input stream. - * + * * @NotThreadSafe - * + * */ public class ArArchiveInputStream extends ArchiveInputStream { @@ -54,16 +54,27 @@ */ private long entryOffset = -1; - // cached buffers - must only be used locally in the class (COMPRESS-172 - reduce garbage collection) - private final byte[] nameBuf = new byte[16]; - private final byte[] lastModifiedBuf = new byte[12]; - private final byte[] idBuf = new byte[6]; - private final byte[] fileModeBuf = new byte[8]; - private final byte[] lengthBuf = new byte[10]; + // offsets and length of meta data parts + private static final int NAME_OFFSET = 0; + private static final int NAME_LEN = 16; + private static final int LAST_MODIFIED_OFFSET = NAME_LEN; + private static final int LAST_MODIFIED_LEN = 12; + private static final int USER_ID_OFFSET = LAST_MODIFIED_OFFSET + LAST_MODIFIED_LEN; + private static final int USER_ID_LEN = 6; + private static final int GROUP_ID_OFFSET = USER_ID_OFFSET + USER_ID_LEN; + private static final int GROUP_ID_LEN = 6; + private static final int FILE_MODE_OFFSET = GROUP_ID_OFFSET + GROUP_ID_LEN; + private static final int FILE_MODE_LEN = 8; + private static final int LENGTH_OFFSET = FILE_MODE_OFFSET + FILE_MODE_LEN; + private static final int LENGTH_LEN = 10; + + // cached buffer for meta data - must only be used locally in the class (COMPRESS-172 - reduce garbage collection) + private final byte[] metaData = + new byte[NAME_LEN + LAST_MODIFIED_LEN + USER_ID_LEN + GROUP_ID_LEN + FILE_MODE_LEN + LENGTH_LEN]; /** * Constructs an Ar input stream with the referenced stream - * + * * @param pInput * the ar input stream */ @@ -74,7 +85,7 @@ /** * Returns the next AR entry in this stream. - * + * * @return the next AR entry. * @throws IOException * if the entry could not be read @@ -82,14 +93,16 @@ public ArArchiveEntry getNextArEntry() throws IOException { if (currentEntry != null) { final long entryEnd = entryOffset + currentEntry.getLength(); - IOUtils.skip(this, entryEnd - offset); + long skipped = IOUtils.skip(input, entryEnd - offset); + trackReadBytes(skipped); currentEntry = null; } if (offset == 0) { final byte[] expected = ArchiveUtils.toAsciiBytes(ArArchiveEntry.HEADER); final byte[] realized = new byte[expected.length]; - final int read = IOUtils.readFully(this, realized); + final int read = IOUtils.readFully(input, realized); + trackReadBytes(read); if (read != expected.length) { throw new IOException("failed to read header. Occured at byte: " + getBytesRead()); } @@ -100,27 +113,31 @@ } } - if (offset % 2 != 0 && read() < 0) { - // hit eof - return null; + if (offset % 2 != 0) { + if (input.read() < 0) { + // hit eof + return null; + } + trackReadBytes(1); } if (input.available() == 0) { return null; } - IOUtils.readFully(this, nameBuf); - IOUtils.readFully(this, lastModifiedBuf); - IOUtils.readFully(this, idBuf); - final int userId = asInt(idBuf, true); - IOUtils.readFully(this, idBuf); - IOUtils.readFully(this, fileModeBuf); - IOUtils.readFully(this, lengthBuf); + { + final int read = IOUtils.readFully(input, metaData); + trackReadBytes(read); + if (read < metaData.length) { + throw new IOException("truncated ar archive"); + } + } { final byte[] expected = ArchiveUtils.toAsciiBytes(ArArchiveEntry.TRAILER); final byte[] realized = new byte[expected.length]; - final int read = IOUtils.readFully(this, realized); + final int read = IOUtils.readFully(input, realized); + trackReadBytes(read); if (read != expected.length) { throw new IOException("failed to read entry trailer. Occured at byte: " + getBytesRead()); } @@ -136,13 +153,13 @@ // GNU ar uses a '/' to mark the end of the filename; this allows for the use of spaces without the use of an extended filename. // entry name is stored as ASCII string - String temp = ArchiveUtils.toAsciiString(nameBuf).trim(); + String temp = ArchiveUtils.toAsciiString(metaData, NAME_OFFSET, NAME_LEN).trim(); if (isGNUStringTable(temp)) { // GNU extended filenames entry - currentEntry = readGNUStringTable(lengthBuf); + currentEntry = readGNUStringTable(metaData, LENGTH_OFFSET, LENGTH_LEN); return getNextArEntry(); } - long len = asLong(lengthBuf); + long len = asLong(metaData, LENGTH_OFFSET, LENGTH_LEN); if (temp.endsWith("/")) { // GNU terminator temp = temp.substring(0, temp.length() - 1); } else if (isGNULongName(temp)) { @@ -158,16 +175,17 @@ entryOffset += nameLen; } - currentEntry = new ArArchiveEntry(temp, len, userId, - asInt(idBuf, true), - asInt(fileModeBuf, 8), - asLong(lastModifiedBuf)); + currentEntry = new ArArchiveEntry(temp, len, + asInt(metaData, USER_ID_OFFSET, USER_ID_LEN, true), + asInt(metaData, GROUP_ID_OFFSET, GROUP_ID_LEN, true), + asInt(metaData, FILE_MODE_OFFSET, FILE_MODE_LEN, 8), + asLong(metaData, LAST_MODIFIED_OFFSET, LAST_MODIFIED_LEN)); return currentEntry; } /** * Get an extended name from the GNU extended name buffer. - * + * * @param offset pointer to entry within the buffer * @return the extended file name; without trailing "/" if present. * @throws IOException if name not found or buffer not set up @@ -187,24 +205,24 @@ throw new IOException("Failed to read entry: " + offset); } - private long asLong(final byte[] byteArray) { - return Long.parseLong(ArchiveUtils.toAsciiString(byteArray).trim()); + private long asLong(final byte[] byteArray, int offset, int len) { + return Long.parseLong(ArchiveUtils.toAsciiString(byteArray, offset, len).trim()); } - private int asInt(final byte[] byteArray) { - return asInt(byteArray, 10, false); + private int asInt(final byte[] byteArray, int offset, int len) { + return asInt(byteArray, offset, len, 10, false); } - private int asInt(final byte[] byteArray, final boolean treatBlankAsZero) { - return asInt(byteArray, 10, treatBlankAsZero); + private int asInt(final byte[] byteArray, int offset, int len, final boolean treatBlankAsZero) { + return asInt(byteArray, offset, len, 10, treatBlankAsZero); } - private int asInt(final byte[] byteArray, final int base) { - return asInt(byteArray, base, false); + private int asInt(final byte[] byteArray, int offset, int len, final int base) { + return asInt(byteArray, offset, len, base, false); } - private int asInt(final byte[] byteArray, final int base, final boolean treatBlankAsZero) { - final String string = ArchiveUtils.toAsciiString(byteArray).trim(); + private int asInt(final byte[] byteArray, int offset, int len, final int base, final boolean treatBlankAsZero) { + final String string = ArchiveUtils.toAsciiString(byteArray, offset, len).trim(); if (string.length() == 0 && treatBlankAsZero) { return 0; } @@ -213,7 +231,7 @@ /* * (non-Javadoc) - * + * * @see * org.apache.commons.compress.archivers.ArchiveInputStream#getNextEntry() */ @@ -224,7 +242,7 @@ /* * (non-Javadoc) - * + * * @see java.io.InputStream#close() */ @Override @@ -238,30 +256,30 @@ /* * (non-Javadoc) - * + * * @see java.io.InputStream#read(byte[], int, int) */ @Override public int read(final byte[] b, final int off, final int len) throws IOException { + if (currentEntry == null) { + throw new IllegalStateException("No current ar entry"); + } int toRead = len; - if (currentEntry != null) { - final long entryEnd = entryOffset + currentEntry.getLength(); - if (len > 0 && entryEnd > offset) { - toRead = (int) Math.min(len, entryEnd - offset); - } else { - return -1; - } + final long entryEnd = entryOffset + currentEntry.getLength(); + if (len > 0 && entryEnd > offset) { + toRead = (int) Math.min(len, entryEnd - offset); + } else { + return -1; } final int ret = this.input.read(b, off, toRead); - count(ret); - offset += ret > 0 ? ret : 0; + trackReadBytes(ret); return ret; } /** * Checks if the signature matches ASCII "!<arch>" followed by a single LF * control character - * + * * @param signature * the bytes to check * @param length @@ -271,35 +289,11 @@ public static boolean matches(final byte[] signature, final int length) { // 3c21 7261 6863 0a3e - if (length < 8) { - return false; - } - if (signature[0] != 0x21) { - return false; - } - if (signature[1] != 0x3c) { - return false; - } - if (signature[2] != 0x61) { - return false; - } - if (signature[3] != 0x72) { - return false; - } - if (signature[4] != 0x63) { - return false; - } - if (signature[5] != 0x68) { - return false; - } - if (signature[6] != 0x3e) { - return false; - } - if (signature[7] != 0x0a) { - return false; - } - - return true; + return length >= 8 && signature[0] == 0x21 && + signature[1] == 0x3c && signature[2] == 0x61 && + signature[3] == 0x72 && signature[4] == 0x63 && + signature[5] == 0x68 && signature[6] == 0x3e && + signature[7] == 0x0a; } static final String BSD_LONGNAME_PREFIX = "#1/"; @@ -346,7 +340,8 @@ final int nameLen = Integer.parseInt(bsdLongName.substring(BSD_LONGNAME_PREFIX_LEN)); final byte[] name = new byte[nameLen]; - final int read = IOUtils.readFully(this, name); + final int read = IOUtils.readFully(input, name); + trackReadBytes(read); if (read != nameLen) { throw new EOFException(); } @@ -366,7 +361,7 @@ *

A header references an extended filename by storing a "/" * followed by a decimal offset to the start of the filename in * the extended filename data section.

- * + * *

The format of the "//" file itself is simply a list of the * long filenames, each separated by one or more LF * characters. Note that the decimal offsets are number of @@ -376,15 +371,23 @@ return GNU_STRING_TABLE_NAME.equals(name); } + private void trackReadBytes(final long read) { + count(read); + if (read > 0) { + offset += read; + } + } + /** * Reads the GNU archive String Table. * * @see #isGNUStringTable */ - private ArArchiveEntry readGNUStringTable(final byte[] length) throws IOException { - final int bufflen = asInt(length); // Assume length will fit in an int + private ArArchiveEntry readGNUStringTable(final byte[] length, final int offset, final int len) throws IOException { + final int bufflen = asInt(length, offset, len); // Assume length will fit in an int namebuffer = new byte[bufflen]; - final int read = IOUtils.readFully(this, namebuffer, 0, bufflen); + final int read = IOUtils.readFully(input, namebuffer, 0, bufflen); + trackReadBytes(read); if (read != bufflen){ throw new IOException("Failed to read complete // record: expected=" + bufflen + " read=" + read); diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveOutputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ar/ArArchiveOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -28,7 +28,7 @@ /** * Implements the "ar" archive format as an output stream. - * + * * @NotThreadSafe */ public class ArArchiveOutputStream extends ArchiveOutputStream { @@ -110,7 +110,7 @@ haveUnclosedEntry = true; } - private long fill( final long pOffset, final long pNewOffset, final char pFill ) throws IOException { + private long fill( final long pOffset, final long pNewOffset, final char pFill ) throws IOException { final long diff = pNewOffset - pOffset; if (diff > 0) { @@ -137,7 +137,7 @@ if (LONGFILE_ERROR == longFileMode && n.length() > 16) { throw new IOException("filename too long, > 16 chars: "+n); } - if (LONGFILE_BSD == longFileMode && + if (LONGFILE_BSD == longFileMode && (n.length() > 16 || n.contains(" "))) { mustAppendName = true; offset += write(ArArchiveInputStream.BSD_LONGNAME_PREFIX @@ -206,11 +206,14 @@ */ @Override public void close() throws IOException { - if(!finished) { - finish(); + try { + if (!finished) { + finish(); + } + } finally { + out.close(); + prevEntry = null; } - out.close(); - prevEntry = null; } @Override diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveEntry.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveEntry.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveEntry.java 2018-05-23 12:50:54.000000000 +0000 @@ -27,14 +27,16 @@ /** * Gets the name of the entry in this archive. May refer to a file or directory or other item. - * + * + *

This method returns the raw name as it is stored inside of the archive.

+ * * @return The name of this entry in the archive. */ String getName(); /** * Gets the uncompressed size of this entry. May be -1 (SIZE_UNKNOWN) if the size is unknown - * + * * @return the uncompressed size of this entry. */ long getSize(); @@ -44,14 +46,14 @@ /** * Returns true if this entry refers to a directory. - * + * * @return true if this entry refers to a directory. */ boolean isDirectory(); /** * Gets the last modified date of this entry. - * + * * @return the last modified date of this entry. * @since 1.1 */ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveException.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveException.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveException.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveException.java 2018-05-02 20:17:13.000000000 +0000 @@ -19,7 +19,7 @@ package org.apache.commons.compress.archivers; /** - * Archiver related Exception + * Archiver related Exception */ public class ArchiveException extends Exception { @@ -29,7 +29,7 @@ /** * Constructs a new exception with the specified detail message. The cause * is not initialized. - * + * * @param message * the detail message */ @@ -39,7 +39,7 @@ /** * Constructs a new exception with the specified detail message and cause. - * + * * @param message * the detail message * @param cause diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -22,7 +22,7 @@ import java.io.InputStream; /** - * Archive input streams MUST override the + * Archive input streams MUST override the * {@link #read(byte[], int, int)} - or {@link #read()} - * method so that reading from the stream generates EOF for the end of * data in each entry as well as at the end of the file proper. @@ -35,7 +35,7 @@ * public static boolean matches(byte[] signature, int length) * * which is used by the {@link ArchiveStreamFactory} to autodetect - * the archive type from the first few bytes of a stream. + * the archive type from the first few bytes of a stream. */ public abstract class ArchiveInputStream extends InputStream { @@ -66,12 +66,12 @@ /** * Reads a byte of data. This method will block until enough input is * available. - * + * * Simply calls the {@link #read(byte[], int, int)} method. - * + * * MUST be overridden if the {@link #read(byte[], int, int)} method * is not overridden; may be overridden otherwise. - * + * * @return the byte read, or -1 if end of input is reached * @throws IOException * if an I/O error has occurred @@ -85,7 +85,7 @@ /** * Increments the counter of already read bytes. * Doesn't increment if the EOF has been hit (read == -1) - * + * * @param read the number of bytes read */ protected void count(final int read) { @@ -95,7 +95,7 @@ /** * Increments the counter of already read bytes. * Doesn't increment if the EOF has been hit (read == -1) - * + * * @param read the number of bytes read * @since 1.1 */ @@ -107,7 +107,7 @@ /** * Decrements the counter of already read bytes. - * + * * @param pushedBack the number of bytes pushed back. * @since 1.1 */ @@ -137,15 +137,15 @@ /** * Whether this stream is able to read the given entry. - * + * *

* Some archive formats support variants or details that are not supported (yet). *

- * + * * @param archiveEntry * the entry to test * @return This implementation always returns true. - * + * * @since 1.1 */ public boolean canReadEntryData(final ArchiveEntry archiveEntry) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveOutputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveOutputStream.java 2018-05-11 18:53:30.000000000 +0000 @@ -27,7 +27,7 @@ * {@link #write(byte[], int, int)} method to improve performance. * They should also override {@link #close()} to ensure that any necessary * trailers are added. - * + * *

The normal sequence of calls when working with ArchiveOutputStreams is:

*
    *
  • Create ArchiveOutputStream object,
    • @@ -58,7 +58,7 @@ * Writes the headers for an archive entry to the output stream. * The caller must then write the content to the stream and call * {@link #closeArchiveEntry()} to complete the process. - * + * * @param entry describes the entry * @throws IOException if an I/O error occurs */ @@ -74,18 +74,18 @@ /** * Finishes the addition of entries to this stream, without closing it. * Additional data can be written, if the format supports it. - * + * * @throws IOException if the user forgets to close the entry. */ public abstract void finish() throws IOException; /** * Create an archive entry using the inputFile and entryName provided. - * + * * @param inputFile the file to create the entry from * @param entryName name to use for the entry * @return the ArchiveEntry set up with details from the file - * + * * @throws IOException if an I/O error occurs */ public abstract ArchiveEntry createArchiveEntry(File inputFile, String entryName) throws IOException; @@ -99,7 +99,7 @@ * *

    MUST be overridden if the {@link #write(byte[], int, int)} method * is not overridden; may be overridden otherwise. - * + * * @param b The byte to be written. * @throws IOException on error */ @@ -112,7 +112,7 @@ /** * Increments the counter of already written bytes. * Doesn't increment if EOF has been hit ({@code written == -1}). - * + * * @param written the number of bytes written */ protected void count(final int written) { @@ -122,7 +122,7 @@ /** * Increments the counter of already written bytes. * Doesn't increment if EOF has been hit ({@code written == -1}). - * + * * @param written the number of bytes written * @since 1.1 */ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamFactory.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamFactory.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamFactory.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamFactory.java 2018-05-02 20:17:13.000000000 +0000 @@ -55,30 +55,30 @@ * the InputStream. In order to add other implementations, you should extend * ArchiveStreamFactory and override the appropriate methods (and call their * implementation from super of course). - * + * * Compressing a ZIP-File: - * + * * 

    - * final OutputStream out = new FileOutputStream(output); 
+ * final OutputStream out = Files.newOutputStream(output.toPath());
  * ArchiveOutputStream os = new ArchiveStreamFactory().createArchiveOutputStream(ArchiveStreamFactory.ZIP, out);
- * 
+ *
  * os.putArchiveEntry(new ZipArchiveEntry("testdata/test1.xml"));
- * IOUtils.copy(new FileInputStream(file1), os);
+ * IOUtils.copy(Files.newInputStream(file1.toPath()), os);
  * os.closeArchiveEntry();
  *
  * os.putArchiveEntry(new ZipArchiveEntry("testdata/test2.xml"));
- * IOUtils.copy(new FileInputStream(file2), os);
+ * IOUtils.copy(Files.newInputStream(file2.toPath()), os);
  * os.closeArchiveEntry();
  * os.close();
  *
    - * + * * Decompressing a ZIP-File: - * + * * 
    - * final InputStream is = new FileInputStream(input); 
+ * final InputStream is = Files.newInputStream(input.toPath());
  * ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.ZIP, is);
  * ZipArchiveEntry entry = (ZipArchiveEntry)in.getNextEntry();
- * OutputStream out = new FileOutputStream(new File(dir, entry.getName()));
+ * OutputStream out = Files.newOutputStream(dir.toPath().resolve(entry.getName()));
  * IOUtils.copy(in, out);
  * out.close();
  * in.close();
@@ -95,51 +95,51 @@
     private static final int SIGNATURE_SIZE = 12;
 
     private static final ArchiveStreamFactory SINGLETON = new ArchiveStreamFactory();
-    
+
     /**
      * Constant (value {@value}) used to identify the AR archive format.
      * @since 1.1
      */
     public static final String AR = "ar";
-    
+
     /**
      * Constant (value {@value}) used to identify the ARJ archive format.
      * Not supported as an output stream type.
      * @since 1.6
      */
     public static final String ARJ = "arj";
-    
+
     /**
      * Constant (value {@value}) used to identify the CPIO archive format.
      * @since 1.1
      */
     public static final String CPIO = "cpio";
-    
+
     /**
      * Constant (value {@value}) used to identify the Unix DUMP archive format.
      * Not supported as an output stream type.
      * @since 1.3
      */
     public static final String DUMP = "dump";
-    
+
     /**
      * Constant (value {@value}) used to identify the JAR archive format.
      * @since 1.1
      */
     public static final String JAR = "jar";
-    
+
     /**
      * Constant used to identify the TAR archive format.
      * @since 1.1
      */
     public static final String TAR = "tar";
-    
+
     /**
      * Constant (value {@value}) used to identify the ZIP archive format.
      * @since 1.1
      */
     public static final String ZIP = "zip";
-    
+
     /**
      * Constant (value {@value}) used to identify the 7z archive format.
      * @since 1.8
@@ -170,7 +170,7 @@
             map.put(toKey(name), provider);
         }
     }
-    
+
     private static Iterator serviceLoaderIterator() {
         return new ServiceLoaderIterator<>(ArchiveStreamProvider.class);
     }
@@ -295,11 +295,11 @@
 
     /**
      * Sets the encoding to use for arj, jar, zip, dump, cpio and tar files. Use null for the archiver default.
-     * 
+     *
      * @param entryEncoding the entry encoding, null uses the archiver default.
      * @since 1.5
      * @deprecated 1.10 use {@link #ArchiveStreamFactory(String)} to specify the encoding
-     * @throws IllegalStateException if the constructor {@link #ArchiveStreamFactory(String)} 
+     * @throws IllegalStateException if the constructor {@link #ArchiveStreamFactory(String)}
      * was used to specify the factory encoding.
      */
     @Deprecated
@@ -313,7 +313,7 @@
 
     /**
      * Creates an archive input stream from an archiver name and an input stream.
-     * 
+     *
      * @param archiverName the archive name,
      * i.e. {@value #AR}, {@value #ARJ}, {@value #ZIP}, {@value #TAR}, {@value #JAR}, {@value #CPIO}, {@value #DUMP} or {@value #SEVEN_Z}
      * @param in the input stream
@@ -327,7 +327,7 @@
             throws ArchiveException {
         return createArchiveInputStream(archiverName, in, entryEncoding);
     }
-    
+
     @Override
     public ArchiveInputStream createArchiveInputStream(final String archiverName, final InputStream in,
             final String actualEncoding) throws ArchiveException {
@@ -393,9 +393,9 @@
 
     /**
      * Creates an archive output stream from an archiver name and an output stream.
-     * 
+     *
      * @param archiverName the archive name,
-     * i.e. {@value #AR}, {@value #ZIP}, {@value #TAR}, {@value #JAR} or {@value #CPIO} 
+     * i.e. {@value #AR}, {@value #ZIP}, {@value #TAR}, {@value #JAR} or {@value #CPIO}
      * @param out the output stream
      * @return the archive output stream
      * @throws ArchiveException if the archiver name is not known
@@ -407,7 +407,7 @@
             throws ArchiveException {
         return createArchiveOutputStream(archiverName, out, entryEncoding);
     }
-    
+
     @Override
     public ArchiveOutputStream createArchiveOutputStream(
             final String archiverName, final OutputStream out, final String actualEncoding)
@@ -463,7 +463,7 @@
      * Create an archive input stream from an input stream, autodetecting
      * the archive type from the first few bytes of the stream. The InputStream
      * must support marks, like BufferedInputStream.
-     * 
+     *
      * @param in the input stream
      * @return the archive input stream
      * @throws ArchiveException if the archiver name is not known
@@ -473,6 +473,17 @@
      */
     public ArchiveInputStream createArchiveInputStream(final InputStream in)
             throws ArchiveException {
+        return createArchiveInputStream(detect(in), in);
+    }
+
+    /**
+     * Try to determine the type of Archiver
+     * @param in input stream
+     * @return type of archiver if found
+     * @throws ArchiveException if an archiver cannot be detected in the stream
+     * @since 1.14
+     */
+    public static String detect(InputStream in) throws ArchiveException {
         if (in == null) {
             throw new IllegalArgumentException("Stream must not be null.");
         }
@@ -483,62 +494,72 @@
 
         final byte[] signature = new byte[SIGNATURE_SIZE];
         in.mark(signature.length);
+        int signatureLength = -1;
         try {
-            int signatureLength = IOUtils.readFully(in, signature);
+            signatureLength = IOUtils.readFully(in, signature);
             in.reset();
-            if (ZipArchiveInputStream.matches(signature, signatureLength)) {
-                return createArchiveInputStream(ZIP, in);
-            } else if (JarArchiveInputStream.matches(signature, signatureLength)) {
-                return createArchiveInputStream(JAR, in);
-            } else if (ArArchiveInputStream.matches(signature, signatureLength)) {
-                return createArchiveInputStream(AR, in);
-            } else if (CpioArchiveInputStream.matches(signature, signatureLength)) {
-                return createArchiveInputStream(CPIO, in);
-            } else if (ArjArchiveInputStream.matches(signature, signatureLength)) {
-                return createArchiveInputStream(ARJ, in);
-            } else if (SevenZFile.matches(signature, signatureLength)) {
-                throw new StreamingNotSupportedException(SEVEN_Z);
-            }
-
-            // Dump needs a bigger buffer to check the signature;
-            final byte[] dumpsig = new byte[DUMP_SIGNATURE_SIZE];
-            in.mark(dumpsig.length);
+        } catch (IOException e) {
+            throw new ArchiveException("IOException while reading signature.", e);
+        }
+
+        if (ZipArchiveInputStream.matches(signature, signatureLength)) {
+            return ZIP;
+        } else if (JarArchiveInputStream.matches(signature, signatureLength)) {
+            return JAR;
+        } else if (ArArchiveInputStream.matches(signature, signatureLength)) {
+            return AR;
+        } else if (CpioArchiveInputStream.matches(signature, signatureLength)) {
+            return CPIO;
+        } else if (ArjArchiveInputStream.matches(signature, signatureLength)) {
+            return ARJ;
+        } else if (SevenZFile.matches(signature, signatureLength)) {
+            return SEVEN_Z;
+        }
+
+        // Dump needs a bigger buffer to check the signature;
+        final byte[] dumpsig = new byte[DUMP_SIGNATURE_SIZE];
+        in.mark(dumpsig.length);
+        try {
             signatureLength = IOUtils.readFully(in, dumpsig);
             in.reset();
-            if (DumpArchiveInputStream.matches(dumpsig, signatureLength)) {
-                return createArchiveInputStream(DUMP, in);
-            }
+        } catch (IOException e) {
+            throw new ArchiveException("IOException while reading dump signature", e);
+        }
+        if (DumpArchiveInputStream.matches(dumpsig, signatureLength)) {
+            return DUMP;
+        }
 
-            // Tar needs an even bigger buffer to check the signature; read the first block
-            final byte[] tarHeader = new byte[TAR_HEADER_SIZE];
-            in.mark(tarHeader.length);
+        // Tar needs an even bigger buffer to check the signature; read the first block
+        final byte[] tarHeader = new byte[TAR_HEADER_SIZE];
+        in.mark(tarHeader.length);
+        try {
             signatureLength = IOUtils.readFully(in, tarHeader);
             in.reset();
-            if (TarArchiveInputStream.matches(tarHeader, signatureLength)) {
-                return createArchiveInputStream(TAR, in);
-            }
-            // COMPRESS-117 - improve auto-recognition
-            if (signatureLength >= TAR_HEADER_SIZE) {
-                TarArchiveInputStream tais = null;
-                try {
-                    tais = new TarArchiveInputStream(new ByteArrayInputStream(tarHeader));
-                    // COMPRESS-191 - verify the header checksum
-                    if (tais.getNextTarEntry().isCheckSumOK()) {
-                        return createArchiveInputStream(TAR, in);
-                    }
-                } catch (final Exception e) { // NOPMD
-                    // can generate IllegalArgumentException as well
-                    // as IOException
-                    // autodetection, simply not a TAR
-                    // ignored
-                } finally {
-                    IOUtils.closeQuietly(tais);
+        } catch (IOException e) {
+            throw new ArchiveException("IOException while reading tar signature", e);
+        }
+        if (TarArchiveInputStream.matches(tarHeader, signatureLength)) {
+            return TAR;
+        }
+
+        // COMPRESS-117 - improve auto-recognition
+        if (signatureLength >= TAR_HEADER_SIZE) {
+            TarArchiveInputStream tais = null;
+            try {
+                tais = new TarArchiveInputStream(new ByteArrayInputStream(tarHeader));
+                // COMPRESS-191 - verify the header checksum
+                if (tais.getNextTarEntry().isCheckSumOK()) {
+                    return TAR;
                 }
+            } catch (final Exception e) { // NOPMD // NOSONAR
+                // can generate IllegalArgumentException as well
+                // as IOException
+                // autodetection, simply not a TAR
+                // ignored
+            } finally {
+                IOUtils.closeQuietly(tais);
             }
-        } catch (final IOException e) {
-            throw new ArchiveException("Could not use reset and mark operations.", e);
         }
-
         throw new ArchiveException("No Archiver found for the stream signature");
     }
 
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamProvider.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamProvider.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamProvider.java	2016-12-25 11:57:03.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/ArchiveStreamProvider.java	2018-05-02 20:17:13.000000000 +0000
@@ -25,7 +25,7 @@
 
 /**
  * Creates Archive {@link ArchiveInputStream}s and {@link ArchiveOutputStream}s.
- * 
+ *
  * @since 1.13
  */
 public interface ArchiveStreamProvider {
@@ -33,7 +33,7 @@
     /**
      * Creates an archive input stream from an archiver name and an input
      * stream.
-     * 
+     *
      * @param name
      *            the archive name, i.e.
      *            {@value org.apache.commons.compress.archivers.ArchiveStreamFactory#AR},
@@ -63,7 +63,7 @@
     /**
      * Creates an archive output stream from an archiver name and an output
      * stream.
-     * 
+     *
      * @param name
      *            the archive name, i.e.
      *            {@value org.apache.commons.compress.archivers.ArchiveStreamFactory#AR},
@@ -89,14 +89,14 @@
 
     /**
      * Gets all the input stream archive names for this provider
-     * 
+     *
      * @return all the input archive names for this provider
      */
     Set getInputStreamArchiveNames();
 
     /**
      * Gets all the output stream archive names for this provider
-     * 
+     *
      * @return all the output archive names for this provider
      */
     Set getOutputStreamArchiveNames();
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveEntry.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveEntry.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveEntry.java	2018-05-23 12:50:54.000000000 +0000
@@ -26,17 +26,17 @@
 
 /**
  * An entry in an ARJ archive.
- * 
+ *
  * @NotThreadSafe
  * @since 1.6
  */
 public class ArjArchiveEntry implements ArchiveEntry {
     private final LocalFileHeader localFileHeader;
-    
+
     public ArjArchiveEntry() {
         localFileHeader = new LocalFileHeader();
     }
-    
+
     ArjArchiveEntry(final LocalFileHeader localFileHeader) {
         this.localFileHeader = localFileHeader;
     }
@@ -44,6 +44,8 @@
     /**
      * Get this entry's name.
      *
+     * 
    This method returns the raw name as it is stored inside of the archive.
    
+     *
      * @return This entry's name.
      */
     @Override
@@ -91,7 +93,7 @@
      */
     @Override
     public Date getLastModifiedDate() {
-        final long ts = isHostOsUnix() ? localFileHeader.dateTimeModified * 1000l
+        final long ts = isHostOsUnix() ? localFileHeader.dateTimeModified * 1000L
             : ZipUtil.dosToJavaTime(0xFFFFFFFFL & localFileHeader.dateTimeModified);
         return new Date(ts);
     }
@@ -158,5 +160,5 @@
         public static final int WIN95 = 10;
         public static final int WIN32 = 11;
     }
-    
+
 }
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStream.java	2016-12-25 11:57:03.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStream.java	2018-05-02 20:17:13.000000000 +0000
@@ -47,7 +47,7 @@
     private final MainHeader mainHeader;
     private LocalFileHeader currentLocalFileHeader = null;
     private InputStream currentInputStream = null;
-    
+
     /**
      * Constructs the ArjInputStream, taking ownership of the inputStream that is passed in.
      * @param inputStream the underlying stream, whose ownership is taken
@@ -82,7 +82,7 @@
             throws ArchiveException {
         this(inputStream, "CP437");
     }
-    
+
     @Override
     public void close() throws IOException {
         in.close();
@@ -105,7 +105,7 @@
         count(4);
         return Integer.reverseBytes(value);
     }
-    
+
     private String readString(final DataInputStream dataIn) throws IOException {
         final ByteArrayOutputStream buffer = new ByteArrayOutputStream();
         int nextByte;
@@ -118,13 +118,13 @@
         // intentionally using the default encoding as that's the contract for a null charsetName
         return new String(buffer.toByteArray());
     }
-    
+
     private void readFully(final DataInputStream dataIn, final byte[] b)
         throws IOException {
         dataIn.readFully(b);
         count(b.length);
     }
-    
+
     private byte[] readHeader() throws IOException {
         boolean found = false;
         byte[] basicHeaderBytes = null;
@@ -153,7 +153,7 @@
         } while (!found);
         return basicHeaderBytes;
     }
-    
+
     private MainHeader readMainHeader() throws IOException {
         final byte[] basicHeaderBytes = readHeader();
         if (basicHeaderBytes == null) {
@@ -161,13 +161,13 @@
         }
         final DataInputStream basicHeader = new DataInputStream(
                 new ByteArrayInputStream(basicHeaderBytes));
-        
+
         final int firstHeaderSize = basicHeader.readUnsignedByte();
         final byte[] firstHeaderBytes = new byte[firstHeaderSize - 1];
         basicHeader.readFully(firstHeaderBytes);
         final DataInputStream firstHeader = new DataInputStream(
                 new ByteArrayInputStream(firstHeaderBytes));
-        
+
         final MainHeader hdr = new MainHeader();
         hdr.archiverVersionNumber = firstHeader.readUnsignedByte();
         hdr.minVersionToExtract = firstHeader.readUnsignedByte();
@@ -185,7 +185,7 @@
         pushedBackBytes(20); // count has already counted them via readFully
         hdr.encryptionVersion = firstHeader.readUnsignedByte();
         hdr.lastChapter = firstHeader.readUnsignedByte();
-        
+
         if (firstHeaderSize >= 33) {
             hdr.arjProtectionFactor = firstHeader.readUnsignedByte();
             hdr.arjFlags2 = firstHeader.readUnsignedByte();
@@ -195,7 +195,7 @@
 
         hdr.name = readString(basicHeader);
         hdr.comment = readString(basicHeader);
-        
+
         final  int extendedHeaderSize = read16(in);
         if (extendedHeaderSize > 0) {
             hdr.extendedHeaderBytes = new byte[extendedHeaderSize];
@@ -207,10 +207,10 @@
                 throw new IOException("Extended header CRC32 verification failure");
             }
         }
-        
+
         return hdr;
     }
-    
+
     private LocalFileHeader readLocalFileHeader() throws IOException {
         final byte[] basicHeaderBytes = readHeader();
         if (basicHeaderBytes == null) {
@@ -265,7 +265,7 @@
             }
         }
     }
-    
+
     private void readExtraData(final int firstHeaderSize, final DataInputStream firstHeader,
                                final LocalFileHeader localFileHeader) throws IOException {
         if (firstHeaderSize >= 33) {
@@ -294,7 +294,7 @@
                 (0xff & signature[0]) == ARJ_MAGIC_1 &&
                 (0xff & signature[1]) == ARJ_MAGIC_2;
     }
-    
+
     /**
      * Gets the archive's recorded name.
      * @return the archive's name
@@ -302,7 +302,7 @@
     public String getArchiveName() {
         return mainHeader.name;
     }
-    
+
     /**
      * Gets the archive's comment.
      * @return the archive's comment
@@ -310,7 +310,7 @@
     public String getArchiveComment() {
         return mainHeader.comment;
     }
-    
+
     @Override
     public ArjArchiveEntry getNextEntry() throws IOException {
         if (currentInputStream != null) {
@@ -320,7 +320,7 @@
             currentLocalFileHeader = null;
             currentInputStream = null;
         }
-        
+
         currentLocalFileHeader = readLocalFileHeader();
         if (currentLocalFileHeader != null) {
             currentInputStream = new BoundedInputStream(in, currentLocalFileHeader.compressedSize);
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/LocalFileHeader.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/LocalFileHeader.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/LocalFileHeader.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/LocalFileHeader.java	2018-05-02 20:17:13.000000000 +0000
@@ -35,17 +35,17 @@
     int fileAccessMode;
     int firstChapter;
     int lastChapter;
-    
+
     int extendedFilePosition;
     int dateTimeAccessed;
     int dateTimeCreated;
     int originalSizeEvenForVolumes;
-    
+
     String name;
     String comment;
-    
+
     byte[][] extendedHeaders = null;
-    
+
     static class Flags {
         static final int GARBLED = 0x01;
         static final int VOLUME = 0x04;
@@ -53,7 +53,7 @@
         static final int PATHSYM = 0x10;
         static final int BACKUP = 0x20;
     }
-    
+
     static class FileTypes {
         static final int BINARY = 0;
         static final int SEVEN_BIT_TEXT = 1;
@@ -61,7 +61,7 @@
         static final int VOLUME_LABEL = 4;
         static final int CHAPTER_LABEL = 5;
     }
-    
+
     static class Methods {
         static final int STORED = 0;
         static final int COMPRESSED_MOST = 1;
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/MainHeader.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/MainHeader.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/arj/MainHeader.java	2016-01-13 17:48:31.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/arj/MainHeader.java	2018-05-02 20:17:13.000000000 +0000
@@ -40,7 +40,7 @@
     String name;
     String comment;
     byte[] extendedHeaderBytes = null;
-    
+
     static class Flags {
         static final int GARBLED = 0x01;
         static final int OLD_SECURED_NEW_ANSI_PAGE = 0x02;
@@ -52,7 +52,7 @@
         static final int ALTNAME = 0x80;
     }
 
-    
+
     @Override
     public String toString() {
         final StringBuilder builder = new StringBuilder();
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveEntry.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveEntry.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveEntry.java	2018-07-11 16:59:06.000000000 +0000
@@ -19,6 +19,7 @@
 package org.apache.commons.compress.archivers.cpio;
 
 import java.io.File;
+import java.nio.charset.Charset;
 import java.util.Date;
 
 import org.apache.commons.compress.archivers.ArchiveEntry;
@@ -27,7 +28,7 @@
  * A cpio archive consists of a sequence of files. There are several types of
  * headers defided in two categories of new and old format. The headers are
  * recognized by magic numbers:
- * 
+ *
  * 
      
  * 
    • "070701" ASCII for new portable format
      • 
  * 
    • "070702" ASCII for new portable format with CRC
      • 
@@ -39,38 +40,38 @@
  * 
      The old binary format is limited to 16 bits for user id, group
  * id, device, and inode numbers. It is limited to 4 gigabyte file
  * sizes.
- * 
+ *
  * The old ASCII format is limited to 18 bits for the user id, group
  * id, device, and inode numbers. It is limited to 8 gigabyte file
  * sizes.
- * 
+ *
  * The new ASCII format is limited to 4 gigabyte file sizes.
- * 
+ *
  * CPIO 2.5 knows also about tar, but it is not recognized here.
      
- * 
- * 
+ *
+ *
  * 
      OLD FORMAT
      
- * 
+ *
  * 
      Each file has a 76 (ascii) / 26 (binary) byte header, a variable
  * length, NUL terminated filename, and variable length file data. A
  * header for a filename "TRAILER!!!" indicates the end of the
  * archive.
      
- * 
+ *
  * 
      All the fields in the header are ISO 646 (approximately ASCII)
  * strings of octal numbers, left padded, not NUL terminated.
      
- * 
+ *
  * 
      - * FIELDNAME        NOTES 
+ * FIELDNAME        NOTES
  * c_magic          The integer value octal 070707.  This value can be used to deter-
  *                  mine whether this archive is written with little-endian or big-
  *                  endian integers.
- * c_dev            Device that contains a directory entry for this file 
- * c_ino            I-node number that identifies the input file to the file system 
+ * c_dev            Device that contains a directory entry for this file
+ * c_ino            I-node number that identifies the input file to the file system
  * c_mode           The mode specifies both the regular permissions and the file type.
- * c_uid            Numeric User ID of the owner of the input file 
- * c_gid            Numeric Group ID of the owner of the input file 
- * c_nlink          Number of links that are connected to the input file 
- * c_rdev           For block special and character special entries, this field 
+ * c_uid            Numeric User ID of the owner of the input file
+ * c_gid            Numeric Group ID of the owner of the input file
+ * c_nlink          Number of links that are connected to the input file
+ * c_rdev           For block special and character special entries, this field
  *                  contains the associated device number.  For all other entry types,
  *                  it should be set to zero by writers and ignored by readers.
  * c_mtime[2]       Modification time of the file, indicated as the number of seconds
@@ -78,33 +79,33 @@
  *                  four-byte integer is stored with the most-significant 16 bits
  *                  first followed by the least-significant 16 bits.  Each of the two
  *                  16 bit values are stored in machine-native byte order.
- * c_namesize       Length of the path name, including the terminating null byte 
- * c_filesize[2]    Length of the file in bytes. This is the length of the data 
- *                  section that follows the header structure. Must be 0 for 
+ * c_namesize       Length of the path name, including the terminating null byte
+ * c_filesize[2]    Length of the file in bytes. This is the length of the data
+ *                  section that follows the header structure. Must be 0 for
  *                  FIFOs and directories
  *
  * All fields are unsigned short fields with 16-bit integer values
  * apart from c_mtime and c_filesize which are 32-bit integer values
  * 
      
- * 
+ *
  * 
      If necessary, the filename and file data are padded with a NUL byte to an even length
      
- * 
+ *
  * 
      Special files, directories, and the trailer are recorded with
  * the h_filesize field equal to 0.
      
- * 
+ *
  * 
      In the ASCII version of this format, the 16-bit entries are represented as 6-byte octal numbers,
  * and the 32-bit entries are represented as 11-byte octal numbers. No padding is added.
      
- * 
+ *
  * 
      NEW FORMAT
      
- * 
+ *
  * 
      Each file has a 110 byte header, a variable length, NUL
  * terminated filename, and variable length file data. A header for a
  * filename "TRAILER!!!" indicates the end of the archive. All the
  * fields in the header are ISO 646 (approximately ASCII) strings of
  * hexadecimal numbers, left padded, not NUL terminated.
      
- * 
+ *
  * 
      - * FIELDNAME        NOTES 
+ * FIELDNAME        NOTES
  * c_magic[6]       The string 070701 for new ASCII, the string 070702 for new ASCII with CRC
  * c_ino[8]
  * c_mode[8]
@@ -112,36 +113,36 @@
  * c_gid[8]
  * c_nlink[8]
  * c_mtim[8]
- * c_filesize[8]    must be 0 for FIFOs and directories 
+ * c_filesize[8]    must be 0 for FIFOs and directories
  * c_maj[8]
- * c_min[8] 
- * c_rmaj[8]        only valid for chr and blk special files 
- * c_rmin[8]        only valid for chr and blk special files 
- * c_namesize[8]    count includes terminating NUL in pathname 
+ * c_min[8]
+ * c_rmaj[8]        only valid for chr and blk special files
+ * c_rmin[8]        only valid for chr and blk special files
+ * c_namesize[8]    count includes terminating NUL in pathname
  * c_check[8]       0 for "new" portable format; for CRC format
  *                  the sum of all the bytes in the file
  * 
      
- * 
+ *
  * 
      New ASCII Format The "new" ASCII format uses 8-byte hexadecimal
  * fields for all numbers and separates device numbers into separate
  * fields for major and minor numbers.
      
- * 
+ *
  * 
      The pathname is followed by NUL bytes so that the total size of
  * the fixed header plus pathname is a multiple of four. Likewise, the
  * file data is padded to a multiple of four bytes.
      
- * 
+ *
  * 
      This class uses mutable fields and is not considered to be
  * threadsafe.
      
- * 
+ *
  * 
      Based on code from the jRPM project (http://jrpm.sourceforge.net).
      
  *
  * 
      The MAGIC numbers and other constants are defined in {@link CpioConstants}
      
- * 
+ *
  * 
      
  * N.B. does not handle the cpio "tar" format
  * 
      
  * @NotThreadSafe
- * @see http://people.freebsd.org/~kientzle/libarchive/man/cpio.5.txt
+ * @see https://people.freebsd.org/~kientzle/libarchive/man/cpio.5.txt
  */
 public class CpioArchiveEntry implements CpioConstants, ArchiveEntry {
 
@@ -150,7 +151,7 @@
     /**
      * See constructor documenation for possible values.
      */
-    private final short fileFormat; 
+    private final short fileFormat;
 
     /** The number of bytes in each header record; depends on the file format */
     private final int headerSize;
@@ -188,8 +189,8 @@
     private long uid = 0;
 
     /**
-     * Creates a CPIOArchiveEntry with a specified format.
-     * 
+     * Creates a CpioArchiveEntry with a specified format.
+     *
      * @param format
      *            The cpio format for this entry.
      * 
      
@@ -226,9 +227,9 @@
     }
 
     /**
-     * Creates a CPIOArchiveEntry with a specified name. The format of
+     * Creates a CpioArchiveEntry with a specified name. The format of
      * this entry will be the new format.
-     * 
+     *
      * @param name
      *            The name of this entry.
      */
@@ -237,8 +238,8 @@
     }
 
     /**
-     * Creates a CPIOArchiveEntry with a specified name.
-     * 
+     * Creates a CpioArchiveEntry with a specified name.
+     *
      * @param format
      *            The cpio format for this entry.
      * @param name
@@ -251,7 +252,7 @@
      * CpioConstants.FORMAT_OLD_BINARY
      * CpioConstants.FORMAT_OLD_ASCII
      *
    - * + * * @since 1.1 */ public CpioArchiveEntry(final short format, final String name) { @@ -260,9 +261,9 @@ } /** - * Creates a CPIOArchiveEntry with a specified name. The format of + * Creates a CpioArchiveEntry with a specified name. The format of * this entry will be the new format. - * + * * @param name * The name of this entry. * @param size @@ -274,8 +275,8 @@ } /** - * Creates a CPIOArchiveEntry with a specified name. - * + * Creates a CpioArchiveEntry with a specified name. + * * @param format * The cpio format for this entry. * @param name @@ -290,7 +291,7 @@ * CpioConstants.FORMAT_OLD_BINARY * CpioConstants.FORMAT_OLD_ASCII * - * + * * @since 1.1 */ public CpioArchiveEntry(final short format, final String name, @@ -300,10 +301,10 @@ } /** - * Creates a CPIOArchiveEntry with a specified name for a + * Creates a CpioArchiveEntry with a specified name for a * specified file. The format of this entry will be the new * format. - * + * * @param inputFile * The file to gather information from. * @param entryName @@ -314,9 +315,9 @@ } /** - * Creates a CPIOArchiveEntry with a specified name for a + * Creates a CpioArchiveEntry with a specified name for a * specified file. - * + * * @param format * The cpio format for this entry. * @param inputFile @@ -331,7 +332,7 @@ * CpioConstants.FORMAT_OLD_BINARY * CpioConstants.FORMAT_OLD_ASCII * - * + * * @since 1.1 */ public CpioArchiveEntry(final short format, final File inputFile, @@ -370,21 +371,21 @@ /** * Get the checksum. * Only supported for the new formats. - * + * * @return Returns the checksum. * @throws UnsupportedOperationException if the format is not a new format */ public long getChksum() { checkNewFormat(); - return this.chksum; + return this.chksum & 0xFFFFFFFFL; } /** * Get the device id. - * + * * @return Returns the device id. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with a new + * if this method is called for a CpioArchiveEntry with a new * format. */ public long getDevice() { @@ -394,10 +395,10 @@ /** * Get the major device id. - * + * * @return Returns the major device id. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with an old + * if this method is called for a CpioArchiveEntry with an old * format. */ public long getDeviceMaj() { @@ -407,7 +408,7 @@ /** * Get the minor device id - * + * * @return Returns the minor device id. * @throws UnsupportedOperationException if format is not a new format */ @@ -418,7 +419,7 @@ /** * Get the filesize. - * + * * @return Returns the filesize. * @see org.apache.commons.compress.archivers.ArchiveEntry#getSize() */ @@ -429,7 +430,7 @@ /** * Get the format for this entry. - * + * * @return Returns the format. */ public short getFormat() { @@ -438,7 +439,7 @@ /** * Get the group id. - * + * * @return Returns the group id. */ public long getGID() { @@ -447,7 +448,7 @@ /** * Get the header size for this CPIO format - * + * * @return Returns the header size in bytes. */ public int getHeaderSize() { @@ -456,7 +457,7 @@ /** * Get the alignment boundary for this CPIO format - * + * * @return Returns the aligment boundary (0, 2, 4) in bytes */ public int getAlignmentBoundary() { @@ -465,17 +466,53 @@ /** * Get the number of bytes needed to pad the header to the alignment boundary. - * + * + * @deprecated This method doesn't properly work for multi-byte encodings. And + * creates corrupt archives. Use {@link #getHeaderPadCount(Charset)} + * or {@link #getHeaderPadCount(long)} in any case. * @return the number of bytes needed to pad the header (0,1,2,3) */ + @Deprecated public int getHeaderPadCount(){ + return getHeaderPadCount(null); + } + + /** + * Get the number of bytes needed to pad the header to the alignment boundary. + * + * @param charset + * The character set used to encode the entry name in the stream. + * @return the number of bytes needed to pad the header (0,1,2,3) + * @since 1.18 + */ + public int getHeaderPadCount(Charset charset) { + if (name == null) { + return 0; + } + if (charset == null) { + return getHeaderPadCount(name.length()); + } + return getHeaderPadCount(name.getBytes(charset).length); + } + + /** + * Get the number of bytes needed to pad the header to the alignment boundary. + * + * @param namesize + * The length of the name in bytes, as read in the stream. + * Without the trailing zero byte. + * @return the number of bytes needed to pad the header (0,1,2,3) + * + * @since 1.18 + */ + public int getHeaderPadCount(long namesize) { if (this.alignmentBoundary == 0) { return 0; } int size = this.headerSize + 1; // Name has terminating null if (name != null) { - size += name.length(); + size += namesize; } final int remain = size % this.alignmentBoundary; - if (remain > 0){ + if (remain > 0) { return this.alignmentBoundary - remain; } return 0; @@ -483,7 +520,7 @@ /** * Get the number of bytes needed to pad the data to the alignment boundary. - * + * * @return the number of bytes needed to pad the data (0,1,2,3) */ public int getDataPadCount(){ @@ -498,7 +535,7 @@ /** * Set the inode. - * + * * @return Returns the inode. */ public long getInode() { @@ -507,7 +544,7 @@ /** * Get the mode of this entry (e.g. directory, regular file). - * + * * @return Returns the mode. */ public long getMode() { @@ -516,7 +553,9 @@ /** * Get the name. - * + * + *

    This method returns the raw name as it is stored inside of the archive.

    + * * @return Returns the name. */ @Override @@ -526,7 +565,7 @@ /** * Get the number of links. - * + * * @return Returns the number of links. */ public long getNumberOfLinks() { @@ -537,10 +576,10 @@ /** * Get the remote device id. - * + * * @return Returns the remote device id. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with a new + * if this method is called for a CpioArchiveEntry with a new * format. */ public long getRemoteDevice() { @@ -550,10 +589,10 @@ /** * Get the remote major device id. - * + * * @return Returns the remote major device id. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with an old + * if this method is called for a CpioArchiveEntry with an old * format. */ public long getRemoteDeviceMaj() { @@ -563,10 +602,10 @@ /** * Get the remote minor device id. - * + * * @return Returns the remote minor device id. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with an old + * if this method is called for a CpioArchiveEntry with an old * format. */ public long getRemoteDeviceMin() { @@ -576,7 +615,7 @@ /** * Get the time in seconds. - * + * * @return Returns the time. */ public long getTime() { @@ -590,7 +629,7 @@ /** * Get the user id. - * + * * @return Returns the user id. */ public long getUID() { @@ -599,7 +638,7 @@ /** * Check if this entry represents a block device. - * + * * @return TRUE if this entry is a block device. */ public boolean isBlockDevice() { @@ -608,7 +647,7 @@ /** * Check if this entry represents a character device. - * + * * @return TRUE if this entry is a character device. */ public boolean isCharacterDevice() { @@ -617,7 +656,7 @@ /** * Check if this entry represents a directory. - * + * * @return TRUE if this entry is a directory. */ @Override @@ -627,7 +666,7 @@ /** * Check if this entry represents a network device. - * + * * @return TRUE if this entry is a network device. */ public boolean isNetwork() { @@ -636,7 +675,7 @@ /** * Check if this entry represents a pipe. - * + * * @return TRUE if this entry is a pipe. */ public boolean isPipe() { @@ -645,7 +684,7 @@ /** * Check if this entry represents a regular file. - * + * * @return TRUE if this entry is a regular file. */ public boolean isRegularFile() { @@ -654,7 +693,7 @@ /** * Check if this entry represents a socket. - * + * * @return TRUE if this entry is a socket. */ public boolean isSocket() { @@ -663,7 +702,7 @@ /** * Check if this entry represents a symbolic link. - * + * * @return TRUE if this entry is a symbolic link. */ public boolean isSymbolicLink() { @@ -673,22 +712,22 @@ /** * Set the checksum. The checksum is calculated by adding all bytes of a * file to transfer (crc += buf[pos] & 0xFF). - * + * * @param chksum * The checksum to set. */ public void setChksum(final long chksum) { checkNewFormat(); - this.chksum = chksum; + this.chksum = chksum & 0xFFFFFFFFL; } /** * Set the device id. - * + * * @param device * The device id to set. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with a new + * if this method is called for a CpioArchiveEntry with a new * format. */ public void setDevice(final long device) { @@ -698,7 +737,7 @@ /** * Set major device id. - * + * * @param maj * The major device id to set. */ @@ -709,7 +748,7 @@ /** * Set the minor device id - * + * * @param min * The minor device id to set. */ @@ -720,7 +759,7 @@ /** * Set the filesize. - * + * * @param size * The filesize to set. */ @@ -734,7 +773,7 @@ /** * Set the group id. - * + * * @param gid * The group id to set. */ @@ -744,7 +783,7 @@ /** * Set the inode. - * + * * @param inode * The inode to set. */ @@ -754,7 +793,7 @@ /** * Set the mode of this entry (e.g. directory, regular file). - * + * * @param mode * The mode to set. */ @@ -773,7 +812,7 @@ default: throw new IllegalArgumentException( "Unknown mode. " - + "Full: " + Long.toHexString(mode) + + "Full: " + Long.toHexString(mode) + " Masked: " + Long.toHexString(maskedMode)); } @@ -782,7 +821,7 @@ /** * Set the name. - * + * * @param name * The name to set. */ @@ -792,7 +831,7 @@ /** * Set the number of links. - * + * * @param nlink * The number of links to set. */ @@ -802,11 +841,11 @@ /** * Set the remote device id. - * + * * @param device * The remote device id to set. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with a new + * if this method is called for a CpioArchiveEntry with a new * format. */ public void setRemoteDevice(final long device) { @@ -816,11 +855,11 @@ /** * Set the remote major device id. - * + * * @param rmaj * The remote major device id to set. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with an old + * if this method is called for a CpioArchiveEntry with an old * format. */ public void setRemoteDeviceMaj(final long rmaj) { @@ -830,11 +869,11 @@ /** * Set the remote minor device id. - * + * * @param rmin * The remote minor device id to set. * @throws UnsupportedOperationException - * if this method is called for a CPIOArchiveEntry with an old + * if this method is called for a CpioArchiveEntry with an old * format. */ public void setRemoteDeviceMin(final long rmin) { @@ -844,7 +883,7 @@ /** * Set the time in seconds. - * + * * @param time * The time to set. */ @@ -854,7 +893,7 @@ /** * Set the user id. - * + * * @param uid * The user id to set. */ @@ -886,12 +925,9 @@ } final CpioArchiveEntry other = (CpioArchiveEntry) obj; if (name == null) { - if (other.name != null) { - return false; - } - } else if (!name.equals(other.name)) { - return false; + return other.name == null; + } else { + return name.equals(other.name); } - return true; } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStream.java 2018-08-09 17:44:19.000000000 +0000 @@ -31,7 +31,7 @@ import org.apache.commons.compress.utils.IOUtils; /** - * CPIOArchiveInputStream is a stream for reading cpio streams. All formats of + * CpioArchiveInputStream is a stream for reading cpio streams. All formats of * cpio are supported (old ascii, old binary, new portable format and the new * portable format with crc). * @@ -41,9 +41,9 @@ * specified by the entry. *

    * 
    - * CPIOArchiveInputStream cpioIn = new CPIOArchiveInputStream(
- *         new FileInputStream(new File("test.cpio")));
- * CPIOArchiveEntry cpioEntry;
+ * CpioArchiveInputStream cpioIn = new CpioArchiveInputStream(
+ *         Files.newInputStream(Paths.get("test.cpio")));
+ * CpioArchiveEntry cpioEntry;
  *
  * while ((cpioEntry = cpioIn.getNextEntry()) != null) {
  *     System.out.println(cpioEntry.getName());
@@ -58,9 +58,9 @@
  *
    *

    * Note: This implementation should be compatible to cpio 2.5 - * + * *

    This class uses mutable fields and is not considered to be threadsafe. - * + * *

    Based on code from the jRPM project (jrpm.sourceforge.net) */ @@ -100,7 +100,7 @@ * Construct the cpio input stream with a blocksize of {@link * CpioConstants#BLOCK_SIZE BLOCK_SIZE} and expecting ASCII file * names. - * + * * @param in * The cpio stream */ @@ -111,7 +111,7 @@ /** * Construct the cpio input stream with a blocksize of {@link * CpioConstants#BLOCK_SIZE BLOCK_SIZE}. - * + * * @param in * The cpio stream * @param encoding @@ -127,7 +127,7 @@ * Construct the cpio input stream with a blocksize of {@link * CpioConstants#BLOCK_SIZE BLOCK_SIZE} expecting ASCII file * names. - * + * * @param in * The cpio stream * @param blockSize @@ -140,7 +140,7 @@ /** * Construct the cpio input stream with a blocksize of {@link CpioConstants#BLOCK_SIZE BLOCK_SIZE}. - * + * * @param in * The cpio stream * @param blockSize @@ -163,7 +163,7 @@ *

    * Programs should not count on this method to return the actual number of * bytes that could be read without blocking. - * + * * @return 1 before EOF and 0 after EOF has reached for current entry. * @throws IOException * if an I/O error has occurred or if a CPIO file error has @@ -180,7 +180,7 @@ /** * Closes the CPIO input stream. - * + * * @throws IOException * if an I/O error has occurred */ @@ -195,7 +195,7 @@ /** * Closes the current CPIO entry and positions the stream for reading the * next entry. - * + * * @throws IOException * if an I/O error has occurred or if a CPIO file error has * occurred @@ -210,7 +210,7 @@ /** * Check to make sure that this stream has not been closed - * + * * @throws IOException * if the stream is already closed */ @@ -223,8 +223,8 @@ /** * Reads the next CPIO file entry and positions stream at the beginning of * the entry data. - * - * @return the CPIOArchiveEntry just read + * + * @return the CpioArchiveEntry just read * @throws IOException * if an I/O error has occurred or if a CPIO file error has * occurred @@ -283,7 +283,7 @@ /** * Reads from the current CPIO entry into an array of bytes. Blocks until * some input is available. - * + * * @param b * the buffer into which the data is read * @param off @@ -329,9 +329,12 @@ if (this.entry.getFormat() == FORMAT_NEW_CRC) { for (int pos = 0; pos < tmpread; pos++) { this.crc += b[pos] & 0xFF; + this.crc &= 0xFFFFFFFFL; } } - this.entryBytesRead += tmpread; + if (tmpread > 0) { + this.entryBytesRead += tmpread; + } return tmpread; } @@ -392,7 +395,7 @@ + ArchiveUtils.sanitize(name) + " Occured at byte: " + getBytesRead()); } - skip(ret.getHeaderPadCount()); + skip(ret.getHeaderPadCount(namesize - 1)); return ret; } @@ -448,7 +451,7 @@ + ArchiveUtils.sanitize(name) + "Occured at byte: " + getBytesRead()); } - skip(ret.getHeaderPadCount()); + skip(ret.getHeaderPadCount(namesize - 1)); return ret; } @@ -463,7 +466,7 @@ /** * Skips specified number of bytes in the current CPIO entry. - * + * * @param n * the number of bytes to skip * @return the actual number of bytes skipped @@ -519,15 +522,15 @@ /** * Checks if the signature matches one of the following magic values: - * + * * Strings: * * "070701" - MAGIC_NEW * "070702" - MAGIC_NEW_CRC * "070707" - MAGIC_OLD_ASCII - * + * * Octal Binary value: - * + * * 070707 - MAGIC_OLD_BINARY (held as a short) = 0x71C7 or 0xC771 * @param signature data to match * @param length length of data diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveOutputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioArchiveOutputStream.java 2018-08-10 14:06:09.000000000 +0000 @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.HashMap; import org.apache.commons.compress.archivers.ArchiveEntry; @@ -32,7 +33,7 @@ import org.apache.commons.compress.utils.CharsetNames; /** - * CPIOArchiveOutputStream is a stream for writing CPIO streams. All formats of + * CpioArchiveOutputStream is a stream for writing CPIO streams. All formats of * CPIO are supported (old ASCII, old binary, new portable format and the new * portable format with CRC). * @@ -56,9 +57,9 @@ * * *

    Note: This implementation should be compatible to cpio 2.5

    - * + * *

    This class uses mutable fields and is not considered threadsafe.

    - * + * *

    based on code from the jRPM project (jrpm.sourceforge.net)

    */ public class CpioArchiveOutputStream extends ArchiveOutputStream implements @@ -101,7 +102,7 @@ * Construct the cpio output stream with a specified format, a * blocksize of {@link CpioConstants#BLOCK_SIZE BLOCK_SIZE} and * using ASCII as the file name encoding. - * + * * @param out * The cpio stream * @param format @@ -114,25 +115,25 @@ /** * Construct the cpio output stream with a specified format using * ASCII as the file name encoding. - * + * * @param out * The cpio stream * @param format * The format of the stream * @param blockSize * The block size of the archive. - * + * * @since 1.1 */ public CpioArchiveOutputStream(final OutputStream out, final short format, final int blockSize) { this(out, format, blockSize, CharsetNames.US_ASCII); - } + } /** * Construct the cpio output stream with a specified format using * ASCII as the file name encoding. - * + * * @param out * The cpio stream * @param format @@ -142,7 +143,7 @@ * @param encoding * The encoding of file names to write - use null for * the platform's default. - * + * * @since 1.6 */ public CpioArchiveOutputStream(final OutputStream out, final short format, @@ -167,7 +168,7 @@ /** * Construct the cpio output stream. The format for this CPIO stream is the * "new" format using ASCII encoding for file names - * + * * @param out * The cpio stream */ @@ -178,7 +179,7 @@ /** * Construct the cpio output stream. The format for this CPIO stream is the * "new" format. - * + * * @param out * The cpio stream * @param encoding @@ -192,7 +193,7 @@ /** * Check to make sure that this stream has not been closed - * + * * @throws IOException * if the stream is already closed */ @@ -208,7 +209,7 @@ * current time will be used if the entry has no set modification time and * the default header format will be used if no other format is specified in * the entry. - * + * * @param entry * the CPIO cpioEntry to be written * @throws IOException @@ -299,10 +300,11 @@ writeAsciiLong(devMin, 8, 16); writeAsciiLong(entry.getRemoteDeviceMaj(), 8, 16); writeAsciiLong(entry.getRemoteDeviceMin(), 8, 16); - writeAsciiLong(entry.getName().length() + 1l, 8, 16); + byte[] name = encode(entry.getName()); + writeAsciiLong(name.length + 1L, 8, 16); writeAsciiLong(entry.getChksum(), 8, 16); - writeCString(entry.getName()); - pad(entry.getHeaderPadCount()); + writeCString(name); + pad(entry.getHeaderPadCount(name.length)); } private void writeOldAsciiEntry(final CpioArchiveEntry entry) @@ -330,9 +332,10 @@ writeAsciiLong(entry.getNumberOfLinks(), 6, 8); writeAsciiLong(entry.getRemoteDevice(), 6, 8); writeAsciiLong(entry.getTime(), 11, 8); - writeAsciiLong(entry.getName().length() + 1l, 6, 8); + byte[] name = encode(entry.getName()); + writeAsciiLong(name.length + 1L, 6, 8); writeAsciiLong(entry.getSize(), 11, 8); - writeCString(entry.getName()); + writeCString(name); } private void writeOldBinaryEntry(final CpioArchiveEntry entry, @@ -360,14 +363,15 @@ writeBinaryLong(entry.getNumberOfLinks(), 2, swapHalfWord); writeBinaryLong(entry.getRemoteDevice(), 2, swapHalfWord); writeBinaryLong(entry.getTime(), 4, swapHalfWord); - writeBinaryLong(entry.getName().length() + 1l, 2, swapHalfWord); + byte[] name = encode(entry.getName()); + writeBinaryLong(name.length + 1L, 2, swapHalfWord); writeBinaryLong(entry.getSize(), 4, swapHalfWord); - writeCString(entry.getName()); - pad(entry.getHeaderPadCount()); + writeCString(name); + pad(entry.getHeaderPadCount(name.length)); } /*(non-Javadoc) - * + * * @see * org.apache.commons.compress.archivers.ArchiveOutputStream#closeArchiveEntry * () @@ -402,7 +406,7 @@ /** * Writes an array of bytes to the current CPIO entry data. This method will * block until all the bytes are written. - * + * * @param b * the data to be written * @param off @@ -434,6 +438,7 @@ if (this.entry.getFormat() == FORMAT_NEW_CRC) { for (int pos = 0; pos < len; pos++) { this.crc += b[pos] & 0xFF; + this.crc &= 0xFFFFFFFFL; } } count(len); @@ -443,7 +448,7 @@ * Finishes writing the contents of the CPIO output stream without closing * the underlying stream. Use this method when applying multiple filters in * succession to the same output stream. - * + * * @throws IOException * if an I/O exception has occurred or if a CPIO file error has * occurred @@ -474,20 +479,22 @@ /** * Closes the CPIO output stream as well as the stream being filtered. - * + * * @throws IOException * if an I/O error has occurred or if a CPIO file error has * occurred */ @Override public void close() throws IOException { - if(!finished) { - finish(); - } - - if (!this.closed) { - out.close(); - this.closed = true; + try { + if (!finished) { + finish(); + } + } finally { + if (!this.closed) { + out.close(); + this.closed = true; + } } } @@ -533,21 +540,32 @@ } /** - * Writes an ASCII string to the stream followed by \0 + * Encodes the given string using the configured encoding. + * * @param str the String to write * @throws IOException if the string couldn't be written + * @return result of encoding the string */ - private void writeCString(final String str) throws IOException { + private byte[] encode(final String str) throws IOException { final ByteBuffer buf = zipEncoding.encode(str); final int len = buf.limit() - buf.position(); - out.write(buf.array(), buf.arrayOffset(), len); + return Arrays.copyOfRange(buf.array(), buf.arrayOffset(), buf.arrayOffset() + len); + } + + /** + * Writes an encoded string to the stream followed by \0 + * @param str the String to write + * @throws IOException if the string couldn't be written + */ + private void writeCString(byte[] str) throws IOException { + out.write(str); out.write('\0'); - count(len + 1); + count(str.length + 1); } /** * Creates a new ArchiveEntry. The entryName must be an ASCII encoded string. - * + * * @see org.apache.commons.compress.archivers.ArchiveOutputStream#createArchiveEntry(java.io.File, java.lang.String) */ @Override diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioConstants.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioConstants.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioConstants.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioConstants.java 2018-05-02 20:17:13.000000000 +0000 @@ -20,9 +20,9 @@ /** * All constants needed by CPIO. - * - * based on code from the jRPM project (jrpm.sourceforge.net) - * + * + * based on code from the jRPM project (jrpm.sourceforge.net) + * */ public interface CpioConstants { /** magic number of a cpio entry in the new format */ @@ -39,16 +39,16 @@ // These FORMAT_ constants are internal to the code - /** write/read a CPIOArchiveEntry in the new format */ + /** write/read a CpioArchiveEntry in the new format */ short FORMAT_NEW = 1; - /** write/read a CPIOArchiveEntry in the new format with crc */ + /** write/read a CpioArchiveEntry in the new format with crc */ short FORMAT_NEW_CRC = 2; - /** write/read a CPIOArchiveEntry in the old ascii format */ + /** write/read a CpioArchiveEntry in the old ascii format */ short FORMAT_OLD_ASCII = 4; - /** write/read a CPIOArchiveEntry in the old binary format */ + /** write/read a CpioArchiveEntry in the old binary format */ short FORMAT_OLD_BINARY = 8; /** Mask for both new formats */ @@ -136,7 +136,7 @@ /** * The default block size. - * + * * @since 1.1 */ int BLOCK_SIZE = 512; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioUtil.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioUtil.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/cpio/CpioUtil.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/cpio/CpioUtil.java 2018-05-02 20:17:13.000000000 +0000 @@ -20,7 +20,7 @@ /** * Package private utility class for Cpio - * + * * @Immutable */ class CpioUtil { @@ -35,7 +35,7 @@ /** * Converts a byte array to a long. Halfwords can be swapped by setting * swapHalfWord=true. - * + * * @param number * An array of bytes containing a number * @param swapHalfWord @@ -71,12 +71,12 @@ } /** - * Converts a long number to a byte array + * Converts a long number to a byte array * Halfwords can be swapped by setting swapHalfWord=true. - * - * @param number + * + * @param number * the input long number to be converted - * + * * @param length * The length of the returned array * @param swapHalfWord diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/Dirent.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/Dirent.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/Dirent.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/Dirent.java 2018-05-23 12:50:54.000000000 +0000 @@ -68,6 +68,9 @@ /** * Get name of directory entry. + * + *

    This method returns the raw name as it is stored inside of the archive.

    + * * @return the directory name */ String getName() { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveConstants.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveConstants.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveConstants.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveConstants.java 2018-05-02 20:17:13.000000000 +0000 @@ -49,7 +49,7 @@ int code; - private SEGMENT_TYPE(final int code) { + SEGMENT_TYPE(final int code) { this.code = code; } @@ -74,7 +74,7 @@ int code; - private COMPRESSION_TYPE(final int code) { + COMPRESSION_TYPE(final int code) { this.code = code; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveEntry.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveEntry.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveEntry.java 2018-05-23 12:50:54.000000000 +0000 @@ -409,7 +409,7 @@ final DumpArchiveEntry rhs = (DumpArchiveEntry) o; - if ((header == null) || (rhs.header == null)) { + if (rhs.header == null) { return false; } @@ -417,8 +417,9 @@ return false; } - if ((summary == null && rhs.summary != null) - || (summary != null && !summary.equals(rhs.summary))) { + // summary is always null right now, but this may change some day + if ((summary == null && rhs.summary != null) // NOSONAR + || (summary != null && !summary.equals(rhs.summary))) { // NOSONAR return false; } @@ -561,6 +562,9 @@ /** * Returns the name of the entry. + * + *

    This method returns the raw name as it is stored inside of the archive.

    + * * @return the name of the entry. */ @Override @@ -785,7 +789,7 @@ private int code; - private TYPE(final int code) { + TYPE(final int code) { this.code = code; } @@ -818,7 +822,7 @@ private int code; - private PERMISSION(final int code) { + PERMISSION(final int code) { this.code = code; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -413,7 +413,7 @@ * @return full path for specified archive entry, or null if there's a gap. */ private String getPath(final DumpArchiveEntry entry) { - // build the stack of elements. It's possible that we're + // build the stack of elements. It's possible that we're // still missing an intermediate value and if so we final Stack elements = new Stack<>(); Dirent dirent = null; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveUtil.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveUtil.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveUtil.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/DumpArchiveUtil.java 2018-05-02 20:17:13.000000000 +0000 @@ -19,7 +19,9 @@ package org.apache.commons.compress.archivers.dump; import java.io.IOException; +import java.util.Arrays; import org.apache.commons.compress.archivers.zip.ZipEncoding; +import org.apache.commons.compress.utils.ByteUtils; /** * Various utilities for dump archives. @@ -64,11 +66,7 @@ //verify checksum... final int checksum = convert32(buffer, 28); - if (checksum != calculateChecksum(buffer)) { - return false; - } - - return true; + return checksum == calculateChecksum(buffer); } /** @@ -88,17 +86,7 @@ * @return the 8-byte entry as a long */ public static final long convert64(final byte[] buffer, final int offset) { - long i = 0; - i += (((long) buffer[offset + 7]) << 56); - i += (((long) buffer[offset + 6] << 48) & 0x00FF000000000000L); - i += (((long) buffer[offset + 5] << 40) & 0x0000FF0000000000L); - i += (((long) buffer[offset + 4] << 32) & 0x000000FF00000000L); - i += (((long) buffer[offset + 3] << 24) & 0x00000000FF000000L); - i += (((long) buffer[offset + 2] << 16) & 0x0000000000FF0000L); - i += (((long) buffer[offset + 1] << 8) & 0x000000000000FF00L); - i += (buffer[offset] & 0x00000000000000FFL); - - return i; + return ByteUtils.fromLittleEndian(buffer, offset, 8); } /** @@ -109,13 +97,7 @@ * @return the 4-byte entry as an int */ public static final int convert32(final byte[] buffer, final int offset) { - int i = 0; - i = buffer[offset + 3] << 24; - i += (buffer[offset + 2] << 16) & 0x00FF0000; - i += (buffer[offset + 1] << 8) & 0x0000FF00; - i += buffer[offset] & 0x000000FF; - - return i; + return (int) ByteUtils.fromLittleEndian(buffer, offset, 4); } /** @@ -126,11 +108,7 @@ * @return the 2-byte entry as an int */ public static final int convert16(final byte[] buffer, final int offset) { - int i = 0; - i += (buffer[offset + 1] << 8) & 0x0000FF00; - i += buffer[offset] & 0x000000FF; - - return i; + return (int) ByteUtils.fromLittleEndian(buffer, offset, 2); } /** @@ -138,8 +116,6 @@ */ static String decode(final ZipEncoding encoding, final byte[] b, final int offset, final int len) throws IOException { - final byte[] copy = new byte[len]; - System.arraycopy(b, offset, copy, 0, len); - return encoding.decode(copy); + return encoding.decode(Arrays.copyOfRange(b, offset, offset + len)); } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/TapeInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/TapeInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/dump/TapeInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/dump/TapeInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -107,7 +107,7 @@ /** * {@inheritDoc} * - *

    reads the full given length unless EOF is reached.

    + *

    reads the full given length unless EOF is reached.

    * * @param len length to read, must be a multiple of the stream's * record size @@ -126,8 +126,12 @@ // we need to read from the underlying stream. // this will reset readOffset value. // return -1 if there's a problem. - if ((readOffset == blockSize) && !readBlock(true)) { - return -1; + if (readOffset == blockSize) { + try { + readBlock(true); + } catch (ShortFileException sfe) { // NOSONAR + return -1; + } } int n = 0; @@ -153,7 +157,7 @@ /** * Skip bytes. Same as read but without the arraycopy. * - *

    skips the full given length unless EOF is reached.

    + *

    skips the full given length unless EOF is reached.

    * * @param len length to read, must be a multiple of the stream's * record size @@ -173,16 +177,19 @@ // this will reset readOffset value. We do not perform // any decompression if we won't eventually read the data. // return -1 if there's a problem. - if ((readOffset == blockSize) && - !readBlock((len - bytes) < blockSize)) { - return -1; + if (readOffset == blockSize) { + try { + readBlock((len - bytes) < blockSize); + } catch (ShortFileException sfe) { // NOSONAR + return -1; + } } long n = 0; if ((readOffset + (len - bytes)) <= blockSize) { // we can read entirely from the buffer. - n = (long) len - bytes; + n = len - bytes; } else { // copy what we can from the buffer. n = (long) blockSize - readOffset; @@ -218,8 +225,12 @@ // we need to read from the underlying stream. This // isn't a problem since it would be the first step in // any subsequent read() anyway. - if ((readOffset == blockSize) && !readBlock(true)) { - return null; + if (readOffset == blockSize) { + try { + readBlock(true); + } catch (ShortFileException sfe) { // NOSONAR + return null; + } } // copy data, increment counters. @@ -252,23 +263,18 @@ * * @param decompress if false the buffer will not be decompressed. * This is an optimization for longer seeks. - * @return false if End-Of-File, else true */ - private boolean readBlock(final boolean decompress) throws IOException { - boolean success = true; - + private void readBlock(final boolean decompress) throws IOException { if (in == null) { throw new IOException("input buffer is closed"); } if (!isCompressed || (currBlkIdx == -1)) { // file is not compressed - success = readFully(blockBuffer, 0, blockSize); + readFully(blockBuffer, 0, blockSize); bytesRead += blockSize; } else { - if (!readFully(blockBuffer, 0, 4)) { - return false; - } + readFully(blockBuffer, 0, 4); bytesRead += 4; final int h = DumpArchiveUtil.convert32(blockBuffer, 0); @@ -276,14 +282,14 @@ if (!compressed) { // file is compressed but this block is not. - success = readFully(blockBuffer, 0, blockSize); + readFully(blockBuffer, 0, blockSize); bytesRead += blockSize; } else { // this block is compressed. final int flags = (h >> 1) & 0x07; int length = (h >> 4) & 0x0FFFFFFF; final byte[] compBuffer = new byte[length]; - success = readFully(compBuffer, 0, length); + readFully(compBuffer, 0, length); bytesRead += length; if (!decompress) { @@ -327,21 +333,17 @@ currBlkIdx++; readOffset = 0; - - return success; } /** * Read buffer */ - private boolean readFully(final byte[] b, final int off, final int len) + private void readFully(final byte[] b, final int off, final int len) throws IOException { final int count = IOUtils.readFully(in, b, off, len); if (count < len) { throw new ShortFileException(); } - - return true; } /** diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/EntryStreamOffsets.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/EntryStreamOffsets.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/EntryStreamOffsets.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/EntryStreamOffsets.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers; + + +/** + * Provides information about ArchiveEntry stream offsets. + */ +public interface EntryStreamOffsets { + + /** Special value indicating that the offset is unknown. */ + long OFFSET_UNKNOWN = -1; + + /** + * Gets the offset of data stream within the archive file, + * + * @return + * the offset of entry data stream, {@code OFFSET_UNKNOWN} if not known. + */ + long getDataOffset(); + + /** + * Indicates whether the stream is contiguous, i.e. not split among + * several archive parts, interspersed with control blocks, etc. + * + * @return + * true if stream is contiguous, false otherwise. + */ + boolean isStreamContiguous(); +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/examples/Archiver.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/examples/Archiver.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/examples/Archiver.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/examples/Archiver.java 2018-06-07 19:11:34.000000000 +0000 @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.examples; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.utils.IOUtils; + +/** + * Provides a high level API for creating archives. + * @since 1.17 + */ +public class Archiver { + + private interface ArchiveEntryCreator { + ArchiveEntry create(File f, String entryName) throws IOException; + } + + private interface ArchiveEntryConsumer { + void accept(File source, ArchiveEntry entry) throws IOException; + } + + private interface Finisher { + void finish() throws IOException; + } + + /** + * Creates an archive {@code target} using the format {@code + * format} by recursively including all files and directories in + * {@code directory}. + * + * @param format the archive format. This uses the same format as + * accepted by {@link ArchiveStreamFactory}. + * @param target the file to write the new archive to. + * @param directory the directory that contains the files to archive. + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be created for other reasons + */ + public void create(String format, File target, File directory) throws IOException, ArchiveException { + if (prefersSeekableByteChannel(format)) { + try (SeekableByteChannel c = FileChannel.open(target.toPath(), StandardOpenOption.WRITE, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + create(format, c, directory); + } + return; + } + try (OutputStream o = Files.newOutputStream(target.toPath())) { + create(format, o, directory); + } + } + + /** + * Creates an archive {@code target} using the format {@code + * format} by recursively including all files and directories in + * {@code directory}. + * + * @param format the archive format. This uses the same format as + * accepted by {@link ArchiveStreamFactory}. + * @param target the stream to write the new archive to. + * @param directory the directory that contains the files to archive. + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be created for other reasons + */ + public void create(String format, OutputStream target, File directory) throws IOException, ArchiveException { + create(new ArchiveStreamFactory().createArchiveOutputStream(format, target), directory); + } + + /** + * Creates an archive {@code target} using the format {@code + * format} by recursively including all files and directories in + * {@code directory}. + * + * @param format the archive format. This uses the same format as + * accepted by {@link ArchiveStreamFactory}. + * @param target the channel to write the new archive to. + * @param directory the directory that contains the files to archive. + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be created for other reasons + */ + public void create(String format, SeekableByteChannel target, File directory) + throws IOException, ArchiveException { + if (!prefersSeekableByteChannel(format)) { + create(format, Channels.newOutputStream(target), directory); + } else if (ArchiveStreamFactory.ZIP.equalsIgnoreCase(format)) { + create(new ZipArchiveOutputStream(target), directory); + } else if (ArchiveStreamFactory.SEVEN_Z.equalsIgnoreCase(format)) { + create(new SevenZOutputFile(target), directory); + } else { + // never reached as prefersSeekableByteChannel only returns true for ZIP and 7z + throw new ArchiveException("don't know how to handle format " + format); + } + } + + /** + * Creates an archive {@code target} by recursively including all + * files and directories in {@code directory}. + * + * @param target the stream to write the new archive to. + * @param directory the directory that contains the files to archive. + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be created for other reasons + */ + public void create(final ArchiveOutputStream target, File directory) + throws IOException, ArchiveException { + create(directory, new ArchiveEntryCreator() { + public ArchiveEntry create(File f, String entryName) throws IOException { + return target.createArchiveEntry(f, entryName); + } + }, new ArchiveEntryConsumer() { + public void accept(File source, ArchiveEntry e) throws IOException { + target.putArchiveEntry(e); + if (!e.isDirectory()) { + try (InputStream in = new BufferedInputStream(Files.newInputStream(source.toPath()))) { + IOUtils.copy(in, target); + } + } + target.closeArchiveEntry(); + } + }, new Finisher() { + public void finish() throws IOException { + target.finish(); + } + }); + } + + /** + * Creates an archive {@code target} by recursively including all + * files and directories in {@code directory}. + * + * @param target the file to write the new archive to. + * @param directory the directory that contains the files to archive. + * @throws IOException if an I/O error occurs + */ + public void create(final SevenZOutputFile target, File directory) throws IOException { + create(directory, new ArchiveEntryCreator() { + public ArchiveEntry create(File f, String entryName) throws IOException { + return target.createArchiveEntry(f, entryName); + } + }, new ArchiveEntryConsumer() { + public void accept(File source, ArchiveEntry e) throws IOException { + target.putArchiveEntry(e); + if (!e.isDirectory()) { + final byte[] buffer = new byte[8024]; + int n = 0; + long count = 0; + try (InputStream in = new BufferedInputStream(Files.newInputStream(source.toPath()))) { + while (-1 != (n = in.read(buffer))) { + target.write(buffer, 0, n); + count += n; + } + } + } + target.closeArchiveEntry(); + } + }, new Finisher() { + public void finish() throws IOException { + target.finish(); + } + }); + } + + private boolean prefersSeekableByteChannel(String format) { + return ArchiveStreamFactory.ZIP.equalsIgnoreCase(format) || ArchiveStreamFactory.SEVEN_Z.equalsIgnoreCase(format); + } + + private void create(File directory, ArchiveEntryCreator creator, ArchiveEntryConsumer consumer, + Finisher finisher) throws IOException { + create("", directory, creator, consumer); + finisher.finish(); + } + + private void create(String prefix, File directory, ArchiveEntryCreator creator, ArchiveEntryConsumer consumer) + throws IOException { + File[] children = directory.listFiles(); + if (children == null) { + return; + } + for (File f : children) { + String entryName = prefix + f.getName() + (f.isDirectory() ? "/" : ""); + consumer.accept(f, creator.create(f, entryName)); + if (f.isDirectory()) { + create(entryName, f, creator, consumer); + } + } + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/examples/Expander.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/examples/Expander.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/examples/Expander.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/examples/Expander.java 2018-06-28 15:53:45.000000000 +0000 @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.examples; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Enumeration; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.archivers.sevenz.SevenZFile; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.compress.utils.IOUtils; + +/** + * Provides a high level API for expanding archives. + * @since 1.17 + */ +public class Expander { + + private interface ArchiveEntrySupplier { + ArchiveEntry getNextReadableEntry() throws IOException; + } + + private interface EntryWriter { + void writeEntryDataTo(ArchiveEntry entry, OutputStream out) throws IOException; + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + *

    Tries to auto-detect the archive's format.

    + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(File archive, File targetDirectory) throws IOException, ArchiveException { + String format = null; + try (InputStream i = new BufferedInputStream(Files.newInputStream(archive.toPath()))) { + format = new ArchiveStreamFactory().detect(i); + } + expand(format, archive, targetDirectory); + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @param format the archive format. This uses the same format as + * accepted by {@link ArchiveStreamFactory}. + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(String format, File archive, File targetDirectory) throws IOException, ArchiveException { + if (prefersSeekableByteChannel(format)) { + try (SeekableByteChannel c = FileChannel.open(archive.toPath(), StandardOpenOption.READ)) { + expand(format, c, targetDirectory); + } + return; + } + try (InputStream i = new BufferedInputStream(Files.newInputStream(archive.toPath()))) { + expand(format, i, targetDirectory); + } + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + *

    Tries to auto-detect the archive's format.

    + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(InputStream archive, File targetDirectory) throws IOException, ArchiveException { + expand(new ArchiveStreamFactory().createArchiveInputStream(archive), targetDirectory); + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @param format the archive format. This uses the same format as + * accepted by {@link ArchiveStreamFactory}. + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(String format, InputStream archive, File targetDirectory) + throws IOException, ArchiveException { + expand(new ArchiveStreamFactory().createArchiveInputStream(format, archive), targetDirectory); + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @param format the archive format. This uses the same format as + * accepted by {@link ArchiveStreamFactory}. + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(String format, SeekableByteChannel archive, File targetDirectory) + throws IOException, ArchiveException { + if (!prefersSeekableByteChannel(format)) { + expand(format, Channels.newInputStream(archive), targetDirectory); + } else if (ArchiveStreamFactory.ZIP.equalsIgnoreCase(format)) { + expand(new ZipFile(archive), targetDirectory); + } else if (ArchiveStreamFactory.SEVEN_Z.equalsIgnoreCase(format)) { + expand(new SevenZFile(archive), targetDirectory); + } else { + // never reached as prefersSeekableByteChannel only returns true for ZIP and 7z + throw new ArchiveException("don't know how to handle format " + format); + } + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(final ArchiveInputStream archive, File targetDirectory) + throws IOException, ArchiveException { + expand(new ArchiveEntrySupplier() { + @Override + public ArchiveEntry getNextReadableEntry() throws IOException { + ArchiveEntry next = archive.getNextEntry(); + while (next != null && !archive.canReadEntryData(next)) { + next = archive.getNextEntry(); + } + return next; + } + }, new EntryWriter() { + @Override + public void writeEntryDataTo(ArchiveEntry entry, OutputStream out) throws IOException { + IOUtils.copy(archive, out); + } + }, targetDirectory); + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(final ZipFile archive, File targetDirectory) + throws IOException, ArchiveException { + final Enumeration entries = archive.getEntries(); + expand(new ArchiveEntrySupplier() { + @Override + public ArchiveEntry getNextReadableEntry() throws IOException { + ZipArchiveEntry next = entries.hasMoreElements() ? entries.nextElement() : null; + while (next != null && !archive.canReadEntryData(next)) { + next = entries.hasMoreElements() ? entries.nextElement() : null; + } + return next; + } + }, new EntryWriter() { + @Override + public void writeEntryDataTo(ArchiveEntry entry, OutputStream out) throws IOException { + try (InputStream in = archive.getInputStream((ZipArchiveEntry) entry)) { + IOUtils.copy(in, out); + } + } + }, targetDirectory); + } + + /** + * Expands {@code archive} into {@code targetDirectory}. + * + * @param archive the file to expand + * @param targetDirectory the directory to write to + * @throws IOException if an I/O error occurs + * @throws ArchiveException if the archive cannot be read for other reasons + */ + public void expand(final SevenZFile archive, File targetDirectory) + throws IOException, ArchiveException { + expand(new ArchiveEntrySupplier() { + @Override + public ArchiveEntry getNextReadableEntry() throws IOException { + return archive.getNextEntry(); + } + }, new EntryWriter() { + @Override + public void writeEntryDataTo(ArchiveEntry entry, OutputStream out) throws IOException { + final byte[] buffer = new byte[8024]; + int n = 0; + long count = 0; + while (-1 != (n = archive.read(buffer))) { + out.write(buffer, 0, n); + count += n; + } + } + }, targetDirectory); + } + + private boolean prefersSeekableByteChannel(String format) { + return ArchiveStreamFactory.ZIP.equalsIgnoreCase(format) || ArchiveStreamFactory.SEVEN_Z.equalsIgnoreCase(format); + } + + private void expand(ArchiveEntrySupplier supplier, EntryWriter writer, File targetDirectory) + throws IOException { + String targetDirPath = targetDirectory.getCanonicalPath(); + if (!targetDirPath.endsWith(File.separator)) { + targetDirPath += File.separatorChar; + } + ArchiveEntry nextEntry = supplier.getNextReadableEntry(); + while (nextEntry != null) { + File f = new File(targetDirectory, nextEntry.getName()); + if (!f.getCanonicalPath().startsWith(targetDirPath)) { + throw new IOException("expanding " + nextEntry.getName() + + " would create file outside of " + targetDirectory); + } + if (nextEntry.isDirectory()) { + if (!f.isDirectory() && !f.mkdirs()) { + throw new IOException("failed to create directory " + f); + } + } else { + File parent = f.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("failed to create directory " + parent); + } + try (OutputStream o = Files.newOutputStream(f.toPath())) { + writer.writeEntryDataTo(nextEntry, o); + } + } + nextEntry = supplier.getNextReadableEntry(); + } + } + +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/examples/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/examples/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/examples/package.html 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/examples/package.html 2018-06-07 19:11:34.000000000 +0000 @@ -0,0 +1,25 @@ + + + +

    Contains example code that is not guaranteed to provide a + stable API across releases of Commons Compress.

    + + + diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveEntry.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveEntry.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveEntry.java 2018-05-02 20:17:13.000000000 +0000 @@ -74,7 +74,7 @@ */ @Deprecated public Certificate[] getCertificates() { - if (certificates != null) { // never true currently + if (certificates != null) { // never true currently // NOSONAR final Certificate[] certs = new Certificate[certificates.length]; System.arraycopy(certificates, 0, certs, 0, certs.length); return certs; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -27,14 +27,14 @@ /** * Implements an input stream that can read entries from jar files. - * + * * @NotThreadSafe */ public class JarArchiveInputStream extends ZipArchiveInputStream { /** * Creates an instance from the input stream using the default encoding. - * + * * @param inputStream the input stream to wrap */ public JarArchiveInputStream( final InputStream inputStream ) { @@ -43,7 +43,7 @@ /** * Creates an instance from the input stream using the specified encoding. - * + * * @param inputStream the input stream to wrap * @param encoding the encoding to use * @since 1.10 @@ -65,7 +65,7 @@ /** * Checks if the signature matches what is expected for a jar file * (in this case it is the same as for a zip file). - * + * * @param signature * the bytes to check * @param length diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveOutputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/jar/JarArchiveOutputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -30,7 +30,7 @@ * Subclass that adds a special extra field to the very first entry * which allows the created archive to be used as an executable jar on * Solaris. - * + * * @NotThreadSafe */ public class JarArchiveOutputStream extends ZipArchiveOutputStream { @@ -43,7 +43,7 @@ /** * Create and instance that wraps the output stream using the provided encoding. - * + * * @param out the output stream to wrap * @param encoding the encoding to use. Use null for the platform default. * @since 1.10 diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/Lister.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/Lister.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/Lister.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/Lister.java 2018-05-23 12:50:54.000000000 +0000 @@ -20,8 +20,10 @@ import java.io.BufferedInputStream; import java.io.File; -import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import org.apache.commons.compress.archivers.sevenz.SevenZFile; /** * Simple command line application that lists the contents of an archive. @@ -44,7 +46,16 @@ if (!f.isFile()) { System.err.println(f + " doesn't exist or is a directory"); } - try (final InputStream fis = new BufferedInputStream(new FileInputStream(f)); + String format = args.length > 1 ? args[1] : detectFormat(f); + if (ArchiveStreamFactory.SEVEN_Z.equalsIgnoreCase(format)) { + list7z(f); + } else { + listStream(f, args); + } + } + + private static void listStream(File f, String[] args) throws ArchiveException, IOException { + try (final InputStream fis = new BufferedInputStream(Files.newInputStream(f.toPath())); final ArchiveInputStream ais = createArchiveInputStream(args, fis)) { System.out.println("Created " + ais.toString()); ArchiveEntry ae; @@ -62,8 +73,24 @@ return factory.createArchiveInputStream(fis); } + private static String detectFormat(File f) throws ArchiveException, IOException { + try (final InputStream fis = new BufferedInputStream(Files.newInputStream(f.toPath()))) { + return factory.detect(fis); + } + } + + private static void list7z(File f) throws ArchiveException, IOException { + try (SevenZFile z = new SevenZFile(f)) { + System.out.println("Created " + z.toString()); + ArchiveEntry ae; + while ((ae = z.getNextEntry()) != null) { + System.out.println(ae.getName()); + } + } + } + private static void usage() { System.out.println("Parameters: archive-name [archive-type]"); } -} \ No newline at end of file +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256Decoder.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256Decoder.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256Decoder.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256Decoder.java 2018-05-02 20:17:13.000000000 +0000 @@ -36,7 +36,7 @@ return new InputStream() { private boolean isInitialized = false; private CipherInputStream cipherInputStream = null; - + private CipherInputStream init() throws IOException { if (isInitialized) { return cipherInputStream; @@ -99,17 +99,17 @@ generalSecurityException); } } - + @Override public int read() throws IOException { return init().read(); } - + @Override public int read(final byte[] b, final int off, final int len) throws IOException { return init().read(b, off, len); } - + @Override public void close() { } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/Archive.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/Archive.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/Archive.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/Archive.java 2018-05-02 20:17:13.000000000 +0000 @@ -46,10 +46,10 @@ } private static String lengthOf(final long[] a) { - return a == null ? "(null)" : String.valueOf(a.length); + return a == null ? "(null)" : String.valueOf(a.length); } private static String lengthOf(final Object[] a) { - return a == null ? "(null)" : String.valueOf(a.length); + return a == null ? "(null)" : String.valueOf(a.length); } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/CLI.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/CLI.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/CLI.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/CLI.java 2018-05-02 20:17:13.000000000 +0000 @@ -23,9 +23,8 @@ public class CLI { - private static final byte[] BUF = new byte[8192]; - private static enum Mode { + private enum Mode { LIST("Analysing") { @Override public void takeAction(final SevenZFile archive, final SevenZArchiveEntry entry) { @@ -65,8 +64,9 @@ } }, EXTRACT("Extracting") { + private final byte[] buf = new byte[8192]; @Override - public void takeAction(final SevenZFile archive, final SevenZArchiveEntry entry) + public void takeAction(final SevenZFile archive, final SevenZArchiveEntry entry) throws IOException { final File outFile = new File(entry.getName()); if (entry.isDirectory()) { @@ -86,8 +86,8 @@ final long total = entry.getSize(); long off = 0; while (off < total) { - final int toRead = (int) Math.min(total - off, BUF.length); - final int bytesRead = archive.read(BUF, 0, toRead); + final int toRead = (int) Math.min(total - off, buf.length); + final int bytesRead = archive.read(buf, 0, toRead); if (bytesRead < 1) { throw new IOException("reached end of entry " + entry.getName() @@ -96,14 +96,14 @@ + total); } off += bytesRead; - fos.write(BUF, 0, bytesRead); + fos.write(buf, 0, bytesRead); } } } }; private final String message; - private Mode(final String message) { + Mode(final String message) { this.message = message; } public String getMessage() { @@ -111,7 +111,7 @@ } public abstract void takeAction(SevenZFile archive, SevenZArchiveEntry entry) throws IOException; - } + } public static void main(final String[] args) throws Exception { if (args.length == 0) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/Coders.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/Coders.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/Coders.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/Coders.java 2018-05-23 12:50:54.000000000 +0000 @@ -17,10 +17,11 @@ */ package org.apache.commons.compress.archivers.sevenz; -import java.io.FilterInputStream; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.SequenceInputStream; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -31,6 +32,7 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream; import org.apache.commons.compress.utils.FlushShieldFilterOutputStream; import org.tukaani.xz.ARMOptions; import org.tukaani.xz.ARMThumbOptions; @@ -50,6 +52,7 @@ put(SevenZMethod.LZMA, new LZMADecoder()); put(SevenZMethod.LZMA2, new LZMA2Decoder()); put(SevenZMethod.DEFLATE, new DeflateDecoder()); + put(SevenZMethod.DEFLATE64, new Deflate64Decoder()); put(SevenZMethod.BZIP2, new BZIP2Decoder()); put(SevenZMethod.AES256SHA256, new AES256SHA256Decoder()); put(SevenZMethod.BCJ_X86_FILTER, new BCJDecoder(new X86Options())); @@ -75,7 +78,7 @@ } return cb.decode(archiveName, is, uncompressedLength, coder, password); } - + static OutputStream addEncoder(final OutputStream out, final SevenZMethod method, final Object options) throws IOException { final CoderBase cb = findByMethod(method); @@ -89,7 +92,7 @@ @Override InputStream decode(final String archiveName, final InputStream in, final long uncompressedLength, final Coder coder, final byte[] password) throws IOException { - return in; + return in; } @Override OutputStream encode(final OutputStream out, final Object options) { @@ -111,86 +114,131 @@ } catch (final AssertionError e) { throw new IOException("BCJ filter used in " + archiveName + " needs XZ for Java > 1.4 - see " - + "http://commons.apache.org/proper/commons-compress/limitations.html#7Z", + + "https://commons.apache.org/proper/commons-compress/limitations.html#7Z", e); } } - + @SuppressWarnings("resource") @Override OutputStream encode(final OutputStream out, final Object options) { return new FlushShieldFilterOutputStream(opts.getOutputStream(new FinishableWrapperOutputStream(out))); } } - + static class DeflateDecoder extends CoderBase { + private static final byte[] ONE_ZERO_BYTE = new byte[1]; DeflateDecoder() { super(Number.class); } + @SuppressWarnings("resource") // caller must close the InputStream @Override InputStream decode(final String archiveName, final InputStream in, final long uncompressedLength, final Coder coder, final byte[] password) throws IOException { final Inflater inflater = new Inflater(true); - final InflaterInputStream inflaterInputStream = new InflaterInputStream(new DummyByteAddingInputStream(in), - inflater); - return new InputStream() { - @Override - public int read() throws IOException { - return inflaterInputStream.read(); - } - - @Override - public int read(final byte[] b, final int off, final int len) throws IOException { - return inflaterInputStream.read(b, off, len); - } - - @Override - public int read(final byte[] b) throws IOException { - return inflaterInputStream.read(b); - } - - @Override - public void close() throws IOException { - try { - inflaterInputStream.close(); - } finally { - inflater.end(); - } - } - }; + // Inflater with nowrap=true has this odd contract for a zero padding + // byte following the data stream; this used to be zlib's requirement + // and has been fixed a long time ago, but the contract persists so + // we comply. + // https://docs.oracle.com/javase/7/docs/api/java/util/zip/Inflater.html#Inflater(boolean) + final InflaterInputStream inflaterInputStream = new InflaterInputStream(new SequenceInputStream(in, + new ByteArrayInputStream(ONE_ZERO_BYTE)), inflater); + return new DeflateDecoderInputStream(inflaterInputStream, inflater); } @Override OutputStream encode(final OutputStream out, final Object options) { final int level = numberOptionOrDefault(options, 9); final Deflater deflater = new Deflater(level, true); final DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(out, deflater); - return new OutputStream() { - @Override - public void write(final int b) throws IOException { - deflaterOutputStream.write(b); - } + return new DeflateDecoderOutputStream(deflaterOutputStream, deflater); + } - @Override - public void write(final byte[] b) throws IOException { - deflaterOutputStream.write(b); - } + static class DeflateDecoderInputStream extends InputStream { + + InflaterInputStream inflaterInputStream; + Inflater inflater; - @Override - public void write(final byte[] b, final int off, final int len) throws IOException { - deflaterOutputStream.write(b, off, len); + public DeflateDecoderInputStream(InflaterInputStream inflaterInputStream, + Inflater inflater) { + this.inflaterInputStream = inflaterInputStream; + this.inflater = inflater; + } + + @Override + public int read() throws IOException { + return inflaterInputStream.read(); + } + + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + return inflaterInputStream.read(b, off, len); + } + + @Override + public int read(final byte[] b) throws IOException { + return inflaterInputStream.read(b); + } + + @Override + public void close() throws IOException { + try { + inflaterInputStream.close(); + } finally { + inflater.end(); } + } + } + + static class DeflateDecoderOutputStream extends OutputStream { + + DeflaterOutputStream deflaterOutputStream; + Deflater deflater; + + public DeflateDecoderOutputStream(DeflaterOutputStream deflaterOutputStream, + Deflater deflater) { + this.deflaterOutputStream = deflaterOutputStream; + this.deflater = deflater; + } - @Override - public void close() throws IOException { - try { - deflaterOutputStream.close(); - } finally { - deflater.end(); - } + @Override + public void write(final int b) throws IOException { + deflaterOutputStream.write(b); + } + + @Override + public void write(final byte[] b) throws IOException { + deflaterOutputStream.write(b); + } + + @Override + public void write(final byte[] b, final int off, final int len) throws IOException { + deflaterOutputStream.write(b, off, len); + } + + @Override + public void close() throws IOException { + try { + deflaterOutputStream.close(); + } finally { + deflater.end(); } - }; + } + } + } + + static class Deflate64Decoder extends CoderBase { + Deflate64Decoder() { + super(Number.class); + } + + @SuppressWarnings("resource") // caller must close the InputStream + @Override + InputStream decode(final String archiveName, final InputStream in, final long uncompressedLength, + final Coder coder, final byte[] password) + throws IOException { + return new Deflate64CompressorInputStream(in); } } @@ -213,38 +261,4 @@ } } - /** - * ZLIB requires an extra dummy byte. - * - * @see java.util.zip.Inflater#Inflater(boolean) - * @see org.apache.commons.compress.archivers.zip.ZipFile.BoundedInputStream - */ - private static class DummyByteAddingInputStream extends FilterInputStream { - private boolean addDummyByte = true; - - private DummyByteAddingInputStream(final InputStream in) { - super(in); - } - - @Override - public int read() throws IOException { - int result = super.read(); - if (result == -1 && addDummyByte) { - addDummyByte = false; - result = 0; - } - return result; - } - - @Override - public int read(final byte[] b, final int off, final int len) throws IOException { - final int result = super.read(b, off, len); - if (result == -1 && addDummyByte) { - addDummyByte = false; - b[off] = 0; - return 1; - } - return result; - } - } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/Folder.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/Folder.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/Folder.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/Folder.java 2018-05-02 20:17:13.000000000 +0000 @@ -70,7 +70,7 @@ } return -1; } - + int findBindPairForOutStream(final int index) { for (int i = 0; i < bindPairs.length; i++) { if (bindPairs[i].outIndex == index) { @@ -79,7 +79,7 @@ } return -1; } - + long getUnpackSize() { if (totalOutputStreams == 0) { return 0; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/LZMADecoder.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/LZMADecoder.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/LZMADecoder.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/LZMADecoder.java 2018-05-02 20:17:13.000000000 +0000 @@ -21,6 +21,7 @@ import java.io.InputStream; import java.io.OutputStream; +import org.apache.commons.compress.utils.ByteUtils; import org.apache.commons.compress.utils.FlushShieldFilterOutputStream; import org.tukaani.xz.LZMA2Options; import org.tukaani.xz.LZMAInputStream; @@ -55,13 +56,10 @@ final LZMA2Options options = getOptions(opts); final byte props = (byte) ((options.getPb() * 5 + options.getLp()) * 9 + options.getLc()); int dictSize = options.getDictSize(); - return new byte[] { - props, - (byte) (dictSize & 0xff), - (byte) ((dictSize >> 8) & 0xff), - (byte) ((dictSize >> 16) & 0xff), - (byte) ((dictSize >> 24) & 0xff), - }; + byte[] o = new byte[5]; + o[0] = props; + ByteUtils.toLittleEndian(o, dictSize, 1, 4); + return o; } @Override @@ -80,11 +78,7 @@ } private int getDictionarySize(final Coder coder) throws IllegalArgumentException { - long dictSize = coder.properties[1]; - for (int i = 1; i < 4; i++) { - dictSize |= (coder.properties[i + 1] & 0xffl) << (8 * i); - } - return (int) dictSize; + return (int) ByteUtils.fromLittleEndian(coder.properties, 1, 4); } private LZMA2Options getOptions(final Object opts) throws IOException { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZArchiveEntry.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZArchiveEntry.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZArchiveEntry.java 2018-05-23 12:50:54.000000000 +0000 @@ -27,7 +27,7 @@ /** * An entry in a 7z archive. - * + * * @NotThreadSafe * @since 1.6 */ @@ -48,20 +48,22 @@ private long crc, compressedCrc; private long size, compressedSize; private Iterable contentMethods; - + public SevenZArchiveEntry() { } /** * Get this entry's name. * + *

    This method returns the raw name as it is stored inside of the archive.

    + * * @return This entry's name. */ @Override public String getName() { return name; } - + /** * Set this entry's name. * @@ -96,7 +98,7 @@ public boolean isDirectory() { return isDirectory; } - + /** * Sets whether or not this entry represents a directory. * @@ -105,10 +107,10 @@ public void setDirectory(final boolean isDirectory) { this.isDirectory = isDirectory; } - + /** * Indicates whether this is an "anti-item" used in differential backups, - * meaning it should delete the same file from a previous backup. + * meaning it should delete the same file from a previous backup. * @return true if it is an anti-item, false otherwise */ public boolean isAntiItem() { @@ -118,7 +120,7 @@ /** * Sets whether this is an "anti-item" used in differential backups, * meaning it should delete the same file from a previous backup. - * @param isAntiItem true if it is an anti-item, false otherwise + * @param isAntiItem true if it is an anti-item, false otherwise */ public void setAntiItem(final boolean isAntiItem) { this.isAntiItem = isAntiItem; @@ -131,7 +133,7 @@ public boolean getHasCreationDate() { return hasCreationDate; } - + /** * Sets whether this entry has got a creation date at all. * @param hasCreationDate whether the entry has got a creation date @@ -139,7 +141,7 @@ public void setHasCreationDate(final boolean hasCreationDate) { this.hasCreationDate = hasCreationDate; } - + /** * Gets the creation date. * @throws UnsupportedOperationException if the entry hasn't got a @@ -153,7 +155,7 @@ throw new UnsupportedOperationException( "The entry doesn't have this timestamp"); } - + /** * Sets the creation date using NTFS time (100 nanosecond units * since 1 January 1601) @@ -162,7 +164,7 @@ public void setCreationDate(final long ntfsCreationDate) { this.creationDate = ntfsCreationDate; } - + /** * Sets the creation date, * @param creationDate the creation date @@ -205,7 +207,7 @@ throw new UnsupportedOperationException( "The entry doesn't have this timestamp"); } - + /** * Sets the last modified date using NTFS time (100 nanosecond * units since 1 January 1601) @@ -214,7 +216,7 @@ public void setLastModifiedDate(final long ntfsLastModifiedDate) { this.lastModifiedDate = ntfsLastModifiedDate; } - + /** * Sets the last modified date, * @param lastModifiedDate the last modified date @@ -225,7 +227,7 @@ this.lastModifiedDate = javaTimeToNtfsTime(lastModifiedDate); } } - + /** * Returns whether this entry has got an access date at all. * @return whether this entry has got an access date at all. @@ -255,7 +257,7 @@ throw new UnsupportedOperationException( "The entry doesn't have this timestamp"); } - + /** * Sets the access date using NTFS time (100 nanosecond units * since 1 January 1601) @@ -264,7 +266,7 @@ public void setAccessDate(final long ntfsAccessDate) { this.accessDate = ntfsAccessDate; } - + /** * Sets the access date, * @param accessDate the access date @@ -411,7 +413,7 @@ public long getSize() { return size; } - + /** * Set this entry's file size. * @@ -429,7 +431,7 @@ long getCompressedSize() { return compressedSize; } - + /** * Set this entry's compressed file size. * @@ -497,7 +499,7 @@ final long realTime = ntfsEpoch.getTimeInMillis() + (ntfsTime / (10*1000)); return new Date(realTime); } - + /** * Converts Java time to NTFS time. * @param date the Java time diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZFile.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZFile.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZFile.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZFile.java 2018-06-07 19:11:34.000000000 +0000 @@ -22,11 +22,15 @@ import java.io.Closeable; import java.io.DataInputStream; import java.io.File; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.CharBuffer; import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.charset.CharsetEncoder; import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.util.ArrayList; @@ -40,6 +44,7 @@ import org.apache.commons.compress.utils.CRC32VerifyingInputStream; import org.apache.commons.compress.utils.CharsetNames; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; /** * Reads a 7z file, using SeekableByteChannel under @@ -66,7 +71,7 @@ * encrypted, neither file names nor file * contents can be read, but the use of * encryption isn't plausibly deniable. - * + * * @NotThreadSafe * @since 1.6 */ @@ -81,6 +86,9 @@ private InputStream currentFolderInputStream = null; private byte[] password; + private long compressedBytesReadFromCurrentEntry; + private long uncompressedBytesReadFromCurrentEntry; + private final ArrayList deferredBlockStreams = new ArrayList<>(); // shared with SevenZOutputFile and tests, neither mutates it @@ -92,10 +100,24 @@ * Reads a file as 7z archive * * @param filename the file to read + * @param password optional password if the archive is encrypted + * @throws IOException if reading the archive fails + * @since 1.17 + */ + public SevenZFile(final File filename, final char[] password) throws IOException { + this(Files.newByteChannel(filename.toPath(), EnumSet.of(StandardOpenOption.READ)), + filename.getAbsolutePath(), utf16Decode(password), true); + } + + /** + * Reads a file as 7z archive + * + * @param filename the file to read * @param password optional password if the archive is encrypted - * the byte array is supposed to be the UTF16-LE encoded * representation of the password. * @throws IOException if reading the archive fails + * @deprecated use the char[]-arg version for the password instead */ public SevenZFile(final File filename, final byte[] password) throws IOException { this(Files.newByteChannel(filename.toPath(), EnumSet.of(StandardOpenOption.READ)), @@ -114,7 +136,59 @@ * @since 1.13 */ public SevenZFile(final SeekableByteChannel channel) throws IOException { - this(channel, "unknown archive", null); + this(channel, "unknown archive", (char[]) null); + } + + /** + * Reads a SeekableByteChannel as 7z archive + * + *

    {@link + * org.apache.commons.compress.utils.SeekableInMemoryByteChannel} + * allows you to read from an in-memory archive.

    + * + * @param channel the channel to read + * @param password optional password if the archive is encrypted + * @throws IOException if reading the archive fails + * @since 1.17 + */ + public SevenZFile(final SeekableByteChannel channel, + final char[] password) throws IOException { + this(channel, "unknown archive", utf16Decode(password)); + } + + /** + * Reads a SeekableByteChannel as 7z archive + * + *

    {@link + * org.apache.commons.compress.utils.SeekableInMemoryByteChannel} + * allows you to read from an in-memory archive.

    + * + * @param channel the channel to read + * @param filename name of the archive - only used for error reporting + * @param password optional password if the archive is encrypted + * @throws IOException if reading the archive fails + * @since 1.17 + */ + public SevenZFile(final SeekableByteChannel channel, String filename, + final char[] password) throws IOException { + this(channel, filename, utf16Decode(password), false); + } + + /** + * Reads a SeekableByteChannel as 7z archive + * + *

    {@link + * org.apache.commons.compress.utils.SeekableInMemoryByteChannel} + * allows you to read from an in-memory archive.

    + * + * @param channel the channel to read + * @param filename name of the archive - only used for error reporting + * @throws IOException if reading the archive fails + * @since 1.17 + */ + public SevenZFile(final SeekableByteChannel channel, String filename) + throws IOException { + this(channel, filename, null, false); } /** @@ -130,6 +204,7 @@ * representation of the password. * @throws IOException if reading the archive fails * @since 1.13 + * @deprecated use the char[]-arg version for the password instead */ public SevenZFile(final SeekableByteChannel channel, final byte[] password) throws IOException { @@ -150,6 +225,7 @@ * representation of the password. * @throws IOException if reading the archive fails * @since 1.13 + * @deprecated use the char[]-arg version for the password instead */ public SevenZFile(final SeekableByteChannel channel, String filename, final byte[] password) throws IOException { @@ -164,8 +240,7 @@ try { archive = readHeaders(password); if (password != null) { - this.password = new byte[password.length]; - System.arraycopy(password, 0, this.password, 0, password.length); + this.password = Arrays.copyOf(password, password.length); } else { this.password = null; } @@ -184,7 +259,7 @@ * @throws IOException if reading the archive fails */ public SevenZFile(final File filename) throws IOException { - this(filename, null); + this(filename, (char[]) null); } /** @@ -220,6 +295,7 @@ ++currentEntryIndex; final SevenZArchiveEntry entry = archive.files[currentEntryIndex]; buildDecodingStream(); + uncompressedBytesReadFromCurrentEntry = compressedBytesReadFromCurrentEntry = 0; return entry; } @@ -672,7 +748,7 @@ files[i] = new SevenZArchiveEntry(); } BitSet isEmptyStream = null; - BitSet isEmptyFile = null; + BitSet isEmptyFile = null; BitSet isAnti = null; while (true) { final int propertyType = getUnsignedByte(header); @@ -803,7 +879,7 @@ int nonEmptyFileCounter = 0; int emptyFileCounter = 0; for (int i = 0; i < files.length; i++) { - files[i].setHasStream(isEmptyStream == null ? true : !isEmptyStream.get(i)); + files[i].setHasStream(isEmptyStream == null || !isEmptyStream.get(i)); if (files[i].hasStream()) { files[i].setDirectory(false); files[i].setAntiItem(false); @@ -812,8 +888,8 @@ files[i].setSize(archive.subStreamsInfo.unpackSizes[nonEmptyFileCounter]); ++nonEmptyFileCounter; } else { - files[i].setDirectory(isEmptyFile == null ? true : !isEmptyFile.get(emptyFileCounter)); - files[i].setAntiItem(isAnti == null ? false : isAnti.get(emptyFileCounter)); + files[i].setDirectory(isEmptyFile == null || !isEmptyFile.get(emptyFileCounter)); + files[i].setAntiItem(isAnti != null && isAnti.get(emptyFileCounter)); files[i].setHasCrc(false); files[i].setSize(0); ++emptyFileCounter; @@ -839,7 +915,7 @@ streamMap.packStreamOffsets = new long[numPackSizes]; for (int i = 0; i < numPackSizes; i++) { streamMap.packStreamOffsets[i] = nextPackStreamOffset; - nextPackStreamOffset += archive.packSizes[i]; + nextPackStreamOffset += archive.packSizes[i]; } streamMap.folderFirstFileIndex = new int[numFolders]; @@ -920,10 +996,33 @@ private InputStream buildDecoderStack(final Folder folder, final long folderOffset, final int firstPackStreamIndex, final SevenZArchiveEntry entry) throws IOException { channel.position(folderOffset); - InputStream inputStreamStack = - new BufferedInputStream( + InputStream inputStreamStack = new FilterInputStream(new BufferedInputStream( new BoundedSeekableByteChannelInputStream(channel, - archive.packSizes[firstPackStreamIndex])); + archive.packSizes[firstPackStreamIndex]))) { + @Override + public int read() throws IOException { + final int r = in.read(); + if (r >= 0) { + count(1); + } + return r; + } + @Override + public int read(final byte[] b) throws IOException { + return read(b, 0, b.length); + } + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + final int r = in.read(b, off, len); + if (r >= 0) { + count(r); + } + return r; + } + private void count(int c) { + compressedBytesReadFromCurrentEntry += c; + } + }; final LinkedList methods = new LinkedList<>(); for (final Coder coder : folder.getOrderedCoders()) { if (coder.numInStreams != 1 || coder.numOutStreams != 1) { @@ -945,13 +1044,17 @@ /** * Reads a byte of data. - * + * * @return the byte read, or -1 if end of input is reached * @throws IOException * if an I/O error has occurred */ public int read() throws IOException { - return getCurrentStream().read(); + int b = getCurrentStream().read(); + if (b >= 0) { + uncompressedBytesReadFromCurrentEntry++; + } + return b; } private InputStream getCurrentStream() throws IOException { @@ -969,6 +1072,7 @@ try (final InputStream stream = deferredBlockStreams.remove(0)) { IOUtils.skip(stream, Long.MAX_VALUE); } + compressedBytesReadFromCurrentEntry = 0; } return deferredBlockStreams.get(0); @@ -976,7 +1080,7 @@ /** * Reads data into an array of bytes. - * + * * @param b the array to write data to * @return the number of bytes read, or -1 if end of input is reached * @throws IOException @@ -988,7 +1092,7 @@ /** * Reads data into an array of bytes. - * + * * @param b the array to write data to * @param off offset into the buffer to start filling at * @param len of bytes to read @@ -997,7 +1101,30 @@ * if an I/O error has occurred */ public int read(final byte[] b, final int off, final int len) throws IOException { - return getCurrentStream().read(b, off, len); + int cnt = getCurrentStream().read(b, off, len); + if (cnt > 0) { + uncompressedBytesReadFromCurrentEntry += cnt; + } + return cnt; + } + + /** + * Provides statistics for bytes read from the current entry. + * + * @return statistics for bytes read from the current entry + * @since 1.17 + */ + public InputStreamStatistics getStatisticsForCurrentEntry() { + return new InputStreamStatistics() { + @Override + public long getCompressedCount() { + return compressedBytesReadFromCurrentEntry; + } + @Override + public long getUncompressedCount() { + return uncompressedBytesReadFromCurrentEntry; + } + }; } private static long readUint64(final ByteBuffer in) throws IOException { @@ -1066,4 +1193,19 @@ public String toString() { return archive.toString(); } + + private static final CharsetEncoder PASSWORD_ENCODER = StandardCharsets.UTF_16LE.newEncoder(); + + private static byte[] utf16Decode(char[] chars) throws IOException { + if (chars == null) { + return null; + } + ByteBuffer encoded = PASSWORD_ENCODER.encode(CharBuffer.wrap(chars)); + if (encoded.hasArray()) { + return encoded.array(); + } + byte[] e = new byte[encoded.remaining()]; + encoded.get(e); + return e; + } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZMethod.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZMethod.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZMethod.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZMethod.java 2018-05-02 20:17:13.000000000 +0000 @@ -42,6 +42,11 @@ LZMA2(new byte[] { (byte)0x21 }), /** Deflate */ DEFLATE(new byte[] { (byte)0x04, (byte)0x01, (byte)0x08 }), + /** + * Deflate64 + * @since 1.16 + */ + DEFLATE64(new byte[] { (byte)0x04, (byte)0x01, (byte)0x09 }), /** BZIP2 */ BZIP2(new byte[] { (byte)0x04, (byte)0x02, (byte)0x02 }), /** @@ -87,7 +92,7 @@ private final byte[] id; - private SevenZMethod(final byte[] id) { + SevenZMethod(final byte[] id) { this.id = id; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFile.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFile.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFile.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFile.java 2018-07-01 09:53:29.000000000 +0000 @@ -125,24 +125,27 @@ /** * Closes the archive, calling {@link #finish} if necessary. - * + * * @throws IOException on error */ @Override public void close() throws IOException { - if (!finished) { - finish(); + try { + if (!finished) { + finish(); + } + } finally { + channel.close(); } - channel.close(); } /** * Create an archive entry using the inputFile and entryName provided. - * + * * @param inputFile file to create an entry from * @param entryName the name to use * @return the ArchiveEntry set up with details from the file - * + * * @throws IOException on error */ public SevenZArchiveEntry createArchiveEntry(final File inputFile, @@ -159,7 +162,7 @@ * * The caller must then write the content to the archive and call * {@link #closeArchiveEntry()} to complete the process. - * + * * @param archiveEntry describes the entry * @throws IOException on error */ @@ -240,7 +243,7 @@ /** * Finishes the addition of entries to this archive, without closing it. - * + * * @throws IOException if archive is already closed. */ public void finish() throws IOException { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/StreamingNotSupportedException.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/StreamingNotSupportedException.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/StreamingNotSupportedException.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/StreamingNotSupportedException.java 2018-05-02 20:17:13.000000000 +0000 @@ -20,18 +20,18 @@ /** * Exception thrown by ArchiveStreamFactory if a format is requested/detected that doesn't support streaming. - * + * * @since 1.8 */ public class StreamingNotSupportedException extends ArchiveException { - + private static final long serialVersionUID = 1L; - + private final String format; /** * Creates a new StreamingNotSupportedException. - * + * * @param format the format that has been requested/detected. */ public StreamingNotSupportedException(final String format) { @@ -41,11 +41,11 @@ /** * Returns the format that has been requested/detected. - * + * * @return the format that has been requested/detected. */ public String getFormat() { return format; } - + } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveEntry.java 2018-05-23 12:50:54.000000000 +0000 @@ -20,10 +20,11 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.Locale; import java.util.Map; - import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipEncoding; import org.apache.commons.compress.utils.ArchiveUtils; @@ -133,7 +134,7 @@ * char prefix[131]; // offset 345 * char atime[12]; // offset 476 * char ctime[12]; // offset 488 - * char mfill[8]; // offset 500 + * char mfill[8]; // offset 500 * char xmagic[4]; // offset 508 "tar" * }; * @@ -142,14 +143,14 @@ * @NotThreadSafe */ -public class TarArchiveEntry implements TarConstants, ArchiveEntry { +public class TarArchiveEntry implements ArchiveEntry, TarConstants { private static final TarArchiveEntry[] EMPTY_TAR_ARCHIVE_ENTRIES = new TarArchiveEntry[0]; /** The entry's name. */ private String name = ""; - /** Whether to enforce leading slashes on the name */ - private boolean preserveLeadingSlashes; + /** Whether to allow leading slashes or drive names inside the name */ + private final boolean preserveAbsolutePath; /** The entry's permission mode. */ private int mode; @@ -207,6 +208,9 @@ /** The entry's file reference */ private final File file; + /** Extra, user supplied pax headers */ + private final Map extraPaxHeaders = new HashMap<>(); + /** Maximum length of a user's name in the tar file */ public static final int MAX_NAMELEN = 31; @@ -219,10 +223,11 @@ /** Convert millis to seconds */ public static final int MILLIS_PER_SECOND = 1000; + /** * Construct an empty entry and prepares the header values. */ - private TarArchiveEntry() { + private TarArchiveEntry(boolean preserveAbsolutePath) { String user = System.getProperty("user.name", ""); if (user.length() > MAX_NAMELEN) { @@ -231,12 +236,17 @@ this.userName = user; this.file = null; + this.preserveAbsolutePath = preserveAbsolutePath; } /** * Construct an entry with only a name. This allows the programmer * to construct the entry's header "by hand". File is set to null. * + *

    The entry's name will be the value of the {@code name} + * argument with all file separators replaced by forward slashes + * and leading slashes as well as Windows drive letters stripped.

    + * * @param name the entry name */ public TarArchiveEntry(final String name) { @@ -247,18 +257,21 @@ * Construct an entry with only a name. This allows the programmer * to construct the entry's header "by hand". File is set to null. * + *

    The entry's name will be the value of the {@code name} + * argument with all file separators replaced by forward slashes. + * Leading slashes and Windows drive letters are stripped if + * {@code preserveAbsolutePath} is {@code false}.

    + * * @param name the entry name - * @param preserveLeadingSlashes whether to allow leading slashes - * in the name. + * @param preserveAbsolutePath whether to allow leading slashes + * or drive letters in the name. * * @since 1.1 */ - public TarArchiveEntry(String name, final boolean preserveLeadingSlashes) { - this(); - - this.preserveLeadingSlashes = preserveLeadingSlashes; + public TarArchiveEntry(String name, final boolean preserveAbsolutePath) { + this(preserveAbsolutePath); - name = normalizeFileName(name, preserveLeadingSlashes); + name = normalizeFileName(name, preserveAbsolutePath); final boolean isDir = name.endsWith("/"); this.name = name; @@ -271,6 +284,11 @@ /** * Construct an entry with a name and a link flag. * + *

    The entry's name will be the value of the {@code name} + * argument with all file separators replaced by forward slashes + * and leading slashes as well as Windows drive letters + * stripped.

    + * * @param name the entry name * @param linkFlag the entry link flag. */ @@ -281,15 +299,20 @@ /** * Construct an entry with a name and a link flag. * + *

    The entry's name will be the value of the {@code name} + * argument with all file separators replaced by forward slashes. + * Leading slashes and Windows drive letters are stripped if + * {@code preserveAbsolutePath} is {@code false}.

    + * * @param name the entry name * @param linkFlag the entry link flag. - * @param preserveLeadingSlashes whether to allow leading slashes - * in the name. + * @param preserveAbsolutePath whether to allow leading slashes + * or drive letters in the name. * * @since 1.5 */ - public TarArchiveEntry(final String name, final byte linkFlag, final boolean preserveLeadingSlashes) { - this(name, preserveLeadingSlashes); + public TarArchiveEntry(final String name, final byte linkFlag, final boolean preserveAbsolutePath) { + this(name, preserveAbsolutePath); this.linkFlag = linkFlag; if (linkFlag == LF_GNUTYPE_LONGNAME) { magic = MAGIC_GNU; @@ -302,6 +325,12 @@ * header is constructed from information from the file. * The name is set from the normalized file path. * + *

    The entry's name will be the value of the {@code file}'s + * path with all file separators replaced by forward slashes and + * leading slashes as well as Windows drive letters stripped. The + * name will end in a slash if the {@code file} represents a + * directory.

    + * * @param file The file that the entry represents. */ public TarArchiveEntry(final File file) { @@ -312,6 +341,12 @@ * Construct an entry for a file. File is set to file, and the * header is constructed from information from the file. * + *

    The entry's name will be the value of the {@code fileName} + * argument with all file separators replaced by forward slashes + * and leading slashes as well as Windows drive letters stripped. + * The name will end in a slash if the {@code file} represents a + * directory.

    + * * @param file The file that the entry represents. * @param fileName the name to be used for the entry. */ @@ -338,6 +373,7 @@ this.modTime = file.lastModified() / MILLIS_PER_SECOND; this.userName = ""; + preserveAbsolutePath = false; } /** @@ -348,7 +384,7 @@ * @throws IllegalArgumentException if any of the numeric fields have an invalid format */ public TarArchiveEntry(final byte[] headerBuf) { - this(); + this(false); parseTarHeader(headerBuf); } @@ -364,7 +400,7 @@ */ public TarArchiveEntry(final byte[] headerBuf, final ZipEncoding encoding) throws IOException { - this(); + this(false); parseTarHeader(headerBuf, encoding); } @@ -419,6 +455,8 @@ /** * Get this entry's name. * + *

    This method returns the raw name as it is stored inside of the archive.

    + * * @return This entry's name. */ @Override @@ -432,7 +470,7 @@ * @param name This entry's new name. */ public void setName(final String name) { - this.name = normalizeFileName(name, this.preserveLeadingSlashes); + this.name = normalizeFileName(name, this.preserveAbsolutePath); } /** @@ -859,11 +897,7 @@ return true; } - if (!isPaxHeader() && !isGlobalPaxHeader() && getName().endsWith("/")) { - return true; - } - - return false; + return !isPaxHeader() && !isGlobalPaxHeader() && getName().endsWith("/"); } /** @@ -943,6 +977,147 @@ } /** + * get extra PAX Headers + * @return read-only map containing any extra PAX Headers + * @since 1.15 + */ + public Map getExtraPaxHeaders() { + return Collections.unmodifiableMap(extraPaxHeaders); + } + + /** + * clear all extra PAX headers. + * @since 1.15 + */ + public void clearExtraPaxHeaders() { + extraPaxHeaders.clear(); + } + + /** + * add a PAX header to this entry. If the header corresponds to an existing field in the entry, + * that field will be set; otherwise the header will be added to the extraPaxHeaders Map + * @param name The full name of the header to set. + * @param value value of header. + * @since 1.15 + */ + public void addPaxHeader(String name,String value) { + processPaxHeader(name,value); + } + + /** + * get named extra PAX header + * @param name The full name of an extended PAX header to retrieve + * @return The value of the header, if any. + * @since 1.15 + */ + public String getExtraPaxHeader(String name) { + return extraPaxHeaders.get(name); + } + + /** + * Update the entry using a map of pax headers. + * @param headers + * @since 1.15 + */ + void updateEntryFromPaxHeaders(Map headers) { + for (final Map.Entry ent : headers.entrySet()) { + final String key = ent.getKey(); + final String val = ent.getValue(); + processPaxHeader(key, val, headers); + } + } + + /** + * process one pax header, using the entries extraPaxHeaders map as source for extra headers + * used when handling entries for sparse files. + * @param key + * @param val + * @since 1.15 + */ + private void processPaxHeader(String key, String val) { + processPaxHeader(key,val,extraPaxHeaders); + } + + /** + * Process one pax header, using the supplied map as source for extra headers to be used when handling + * entries for sparse files + * + * @param key the header name. + * @param val the header value. + * @param headers map of headers used for dealing with sparse file. + * @since 1.15 + */ + private void processPaxHeader(String key, String val, Map headers) { + /* + * The following headers are defined for Pax. + * atime, ctime, charset: cannot use these without changing TarArchiveEntry fields + * mtime + * comment + * gid, gname + * linkpath + * size + * uid,uname + * SCHILY.devminor, SCHILY.devmajor: don't have setters/getters for those + * + * GNU sparse files use additional members, we use + * GNU.sparse.size to detect the 0.0 and 0.1 versions and + * GNU.sparse.realsize for 1.0. + * + * star files use additional members of which we use + * SCHILY.filetype in order to detect star sparse files. + * + * If called from addExtraPaxHeader, these additional headers must be already present . + */ + switch (key) { + case "path": + setName(val); + break; + case "linkpath": + setLinkName(val); + break; + case "gid": + setGroupId(Long.parseLong(val)); + break; + case "gname": + setGroupName(val); + break; + case "uid": + setUserId(Long.parseLong(val)); + break; + case "uname": + setUserName(val); + break; + case "size": + setSize(Long.parseLong(val)); + break; + case "mtime": + setModTime((long) (Double.parseDouble(val) * 1000)); + break; + case "SCHILY.devminor": + setDevMinor(Integer.parseInt(val)); + break; + case "SCHILY.devmajor": + setDevMajor(Integer.parseInt(val)); + break; + case "GNU.sparse.size": + fillGNUSparse0xData(headers); + break; + case "GNU.sparse.realsize": + fillGNUSparse1xData(headers); + break; + case "SCHILY.filetype": + if ("sparse".equals(val)) { + fillStarSparseData(headers); + } + break; + default: + extraPaxHeaders.put(key,val); + } + } + + + + /** * If this entry represents a file, and the file is a directory, return * an array of TarEntries for this entry's children. * @@ -1047,7 +1222,7 @@ private int writeEntryHeaderField(final long value, final byte[] outbuf, final int offset, final int length, final boolean starMode) { if (!starMode && (value < 0 - || value >= 1l << 3 * (length - 1))) { + || value >= 1L << 3 * (length - 1))) { // value doesn't fit into field when written as octal // number, will be written to PAX header or causes an // error @@ -1125,10 +1300,14 @@ groupName = oldStyle ? TarUtils.parseName(header, offset, GNAMELEN) : TarUtils.parseName(header, offset, GNAMELEN, encoding); offset += GNAMELEN; - devMajor = (int) TarUtils.parseOctalOrBinary(header, offset, DEVLEN); - offset += DEVLEN; - devMinor = (int) TarUtils.parseOctalOrBinary(header, offset, DEVLEN); - offset += DEVLEN; + if (linkFlag == LF_CHR || linkFlag == LF_BLK) { + devMajor = (int) TarUtils.parseOctalOrBinary(header, offset, DEVLEN); + offset += DEVLEN; + devMinor = (int) TarUtils.parseOctalOrBinary(header, offset, DEVLEN); + offset += DEVLEN; + } else { + offset += 2 * DEVLEN; + } final int type = evaluateType(header); switch (type) { @@ -1142,7 +1321,7 @@ isExtended = TarUtils.parseBoolean(header, offset); offset += ISEXTENDEDLEN_GNU; realSize = TarUtils.parseOctal(header, offset, REALSIZELEN_GNU); - offset += REALSIZELEN_GNU; + offset += REALSIZELEN_GNU; // NOSONAR - assignment as documentation break; } case FORMAT_XSTAR: { @@ -1176,29 +1355,31 @@ * turns path separators into forward slahes. */ private static String normalizeFileName(String fileName, - final boolean preserveLeadingSlashes) { - final String osname = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); - - if (osname != null) { - - // Strip off drive letters! - // REVIEW Would a better check be "(File.separator == '\')"? - - if (osname.startsWith("windows")) { - if (fileName.length() > 2) { - final char ch1 = fileName.charAt(0); - final char ch2 = fileName.charAt(1); - - if (ch2 == ':' - && (ch1 >= 'a' && ch1 <= 'z' - || ch1 >= 'A' && ch1 <= 'Z')) { - fileName = fileName.substring(2); + final boolean preserveAbsolutePath) { + if (!preserveAbsolutePath) { + final String osname = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + + if (osname != null) { + + // Strip off drive letters! + // REVIEW Would a better check be "(File.separator == '\')"? + + if (osname.startsWith("windows")) { + if (fileName.length() > 2) { + final char ch1 = fileName.charAt(0); + final char ch2 = fileName.charAt(1); + + if (ch2 == ':' + && (ch1 >= 'a' && ch1 <= 'z' + || ch1 >= 'A' && ch1 <= 'Z')) { + fileName = fileName.substring(2); + } + } + } else if (osname.contains("netware")) { + final int colon = fileName.indexOf(':'); + if (colon != -1) { + fileName = fileName.substring(colon + 1); } - } - } else if (osname.contains("netware")) { - final int colon = fileName.indexOf(':'); - if (colon != -1) { - fileName = fileName.substring(colon + 1); } } } @@ -1208,7 +1389,7 @@ // No absolute pathnames // Windows (and Posix?) paths can start with "\\NetworkDrive\", // so we loop on starting /'s. - while (!preserveLeadingSlashes && fileName.startsWith("/")) { + while (!preserveAbsolutePath && fileName.startsWith("/")) { fileName = fileName.substring(1); } return fileName; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -28,7 +28,6 @@ import java.io.InputStream; import java.util.HashMap; import java.util.Map; -import java.util.Map.Entry; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; @@ -128,7 +127,7 @@ * @param recordSize the record size to use */ public TarArchiveInputStream(final InputStream is, final int blockSize, final int recordSize) { - this(is, blockSize, recordSize, null); + this(is, blockSize, recordSize, null); } /** @@ -190,7 +189,7 @@ return (int) (entrySize - entryOffset); } - + /** * Skips over and discards n bytes of data from this input * stream. The skip method may, for a variety of reasons, end @@ -199,8 +198,8 @@ * or end of entry before n bytes have been skipped; are only * two possibilities. The actual number of bytes skipped is returned. If * n is negative, no bytes are skipped. - * - * + * + * * @param n * the number of bytes to be skipped. * @return the actual number of bytes skipped. @@ -214,7 +213,7 @@ } final long available = entrySize - entryOffset; - final long skipped = is.skip(Math.min(n, available)); + final long skipped = IOUtils.skip(is, Math.min(n, available)); count(skipped); entryOffset += skipped; return skipped; @@ -260,7 +259,7 @@ * @throws IOException on error */ public TarArchiveEntry getNextTarEntry() throws IOException { - if (hasHitEOF) { + if (isAtEOF()) { return null; } @@ -333,7 +332,7 @@ return currEntry; } - + /** * The last record block should be written at the full size, so skip any * additional space used to fill a record after an entry @@ -396,8 +395,8 @@ */ private byte[] getRecord() throws IOException { byte[] headerBuf = readRecord(); - hasHitEOF = isEOFRecord(headerBuf); - if (hasHitEOF && headerBuf != null) { + setAtEOF(isEOFRecord(headerBuf)); + if (isAtEOF() && headerBuf != null) { tryToConsumeSecondEOFRecord(); consumeRemainderOfLastBlock(); headerBuf = null; @@ -415,7 +414,7 @@ protected boolean isEOFRecord(final byte[] record) { return record == null || ArchiveUtils.isArrayZero(record, recordSize); } - + /** * Read a record from the input stream and return the data. * @@ -504,55 +503,8 @@ } private void applyPaxHeadersToCurrentEntry(final Map headers) { - /* - * The following headers are defined for Pax. - * atime, ctime, charset: cannot use these without changing TarArchiveEntry fields - * mtime - * comment - * gid, gname - * linkpath - * size - * uid,uname - * SCHILY.devminor, SCHILY.devmajor: don't have setters/getters for those - * - * GNU sparse files use additional members, we use - * GNU.sparse.size to detect the 0.0 and 0.1 versions and - * GNU.sparse.realsize for 1.0. - * - * star files use additional members of which we use - * SCHILY.filetype in order to detect star sparse files. - */ - for (final Entry ent : headers.entrySet()){ - final String key = ent.getKey(); - final String val = ent.getValue(); - if ("path".equals(key)){ - currEntry.setName(val); - } else if ("linkpath".equals(key)){ - currEntry.setLinkName(val); - } else if ("gid".equals(key)){ - currEntry.setGroupId(Long.parseLong(val)); - } else if ("gname".equals(key)){ - currEntry.setGroupName(val); - } else if ("uid".equals(key)){ - currEntry.setUserId(Long.parseLong(val)); - } else if ("uname".equals(key)){ - currEntry.setUserName(val); - } else if ("size".equals(key)){ - currEntry.setSize(Long.parseLong(val)); - } else if ("mtime".equals(key)){ - currEntry.setModTime((long) (Double.parseDouble(val) * 1000)); - } else if ("SCHILY.devminor".equals(key)){ - currEntry.setDevMinor(Integer.parseInt(val)); - } else if ("SCHILY.devmajor".equals(key)){ - currEntry.setDevMajor(Integer.parseInt(val)); - } else if ("GNU.sparse.size".equals(key)) { - currEntry.fillGNUSparse0xData(headers); - } else if ("GNU.sparse.realsize".equals(key)) { - currEntry.fillGNUSparse1xData(headers); - } else if ("SCHILY.filetype".equals(key) && "sparse".equals(val)) { - currEntry.fillStarSparseData(headers); - } - } + currEntry.updateEntryFromPaxHeaders(headers); + } /** @@ -599,7 +551,7 @@ public ArchiveEntry getNextEntry() throws IOException { return getNextTarEntry(); } - + /** * Tries to read the next record rewinding the stream if it is not a EOF record. * @@ -643,7 +595,7 @@ public int read(final byte[] buf, final int offset, int numToRead) throws IOException { int totalRead = 0; - if (hasHitEOF || isDirectory() || entryOffset >= entrySize) { + if (isAtEOF() || isDirectory() || entryOffset >= entrySize) { return -1; } @@ -652,14 +604,14 @@ } numToRead = Math.min(numToRead, available()); - + totalRead = is.read(buf, offset, numToRead); - + if (totalRead == -1) { if (numToRead > 0) { throw new IOException("Truncated TAR archive"); } - hasHitEOF = true; + setAtEOF(true); } else { count(totalRead); entryOffset += totalRead; @@ -684,7 +636,7 @@ /** * Get the current TAR Archive Entry that this input stream is processing - * + * * @return The current Archive Entry */ public TarArchiveEntry getCurrentEntry() { @@ -752,15 +704,11 @@ return true; } // COMPRESS-107 - recognise Ant tar files - if (ArchiveUtils.matchAsciiBuffer(TarConstants.MAGIC_ANT, + return ArchiveUtils.matchAsciiBuffer(TarConstants.MAGIC_ANT, signature, TarConstants.MAGIC_OFFSET, TarConstants.MAGICLEN) - && - ArchiveUtils.matchAsciiBuffer(TarConstants.VERSION_ANT, - signature, TarConstants.VERSION_OFFSET, TarConstants.VERSIONLEN) - ){ - return true; - } - return false; + && + ArchiveUtils.matchAsciiBuffer(TarConstants.VERSION_ANT, + signature, TarConstants.VERSION_OFFSET, TarConstants.VERSIONLEN); } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.StringWriter; +import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Date; @@ -33,56 +34,84 @@ import org.apache.commons.compress.archivers.zip.ZipEncodingHelper; import org.apache.commons.compress.utils.CharsetNames; import org.apache.commons.compress.utils.CountingOutputStream; +import org.apache.commons.compress.utils.FixedLengthBlockOutputStream; /** - * The TarOutputStream writes a UNIX tar archive as an OutputStream. - * Methods are provided to put entries, and then write their contents - * by writing to this stream using write(). + * The TarOutputStream writes a UNIX tar archive as an OutputStream. Methods are provided to put + * entries, and then write their contents by writing to this stream using write(). + * + *

    tar archives consist of a sequence of records of 512 bytes each + * that are grouped into blocks. Prior to Apache Commons Compress 1.14 + * it has been possible to configure a record size different from 512 + * bytes and arbitrary block sizes. Starting with Compress 1.15 512 is + * the only valid option for the record size and the block size must + * be a multiple of 512. Also the default block size changed from + * 10240 bytes prior to Compress 1.15 to 512 bytes with Compress + * 1.15.

    + * * @NotThreadSafe */ public class TarArchiveOutputStream extends ArchiveOutputStream { - /** Fail if a long file name is required in the archive. */ + + /** + * Fail if a long file name is required in the archive. + */ public static final int LONGFILE_ERROR = 0; - /** Long paths will be truncated in the archive. */ + /** + * Long paths will be truncated in the archive. + */ public static final int LONGFILE_TRUNCATE = 1; - /** GNU tar extensions are used to store long file names in the archive. */ + /** + * GNU tar extensions are used to store long file names in the archive. + */ public static final int LONGFILE_GNU = 2; - /** POSIX/PAX extensions are used to store long file names in the archive. */ + /** + * POSIX/PAX extensions are used to store long file names in the archive. + */ public static final int LONGFILE_POSIX = 3; - /** Fail if a big number (e.g. size > 8GiB) is required in the archive. */ + /** + * Fail if a big number (e.g. size > 8GiB) is required in the archive. + */ public static final int BIGNUMBER_ERROR = 0; - /** star/GNU tar/BSD tar extensions are used to store big number in the archive. */ + /** + * star/GNU tar/BSD tar extensions are used to store big number in the archive. + */ public static final int BIGNUMBER_STAR = 1; - /** POSIX/PAX extensions are used to store big numbers in the archive. */ + /** + * POSIX/PAX extensions are used to store big numbers in the archive. + */ public static final int BIGNUMBER_POSIX = 2; + private static final int RECORD_SIZE = 512; - private long currSize; - private String currName; - private long currBytes; - private final byte[] recordBuf; - private int assemLen; - private final byte[] assemBuf; - private int longFileMode = LONGFILE_ERROR; - private int bigNumberMode = BIGNUMBER_ERROR; + private long currSize; + private String currName; + private long currBytes; + private final byte[] recordBuf; + private int longFileMode = LONGFILE_ERROR; + private int bigNumberMode = BIGNUMBER_ERROR; private int recordsWritten; private final int recordsPerBlock; - private final int recordSize; private boolean closed = false; - /** Indicates if putArchiveEntry has been called without closeArchiveEntry */ + /** + * Indicates if putArchiveEntry has been called without closeArchiveEntry + */ private boolean haveUnclosedEntry = false; - /** indicates if this archive is finished */ + /** + * indicates if this archive is finished + */ private boolean finished = false; - private final OutputStream out; + private final FixedLengthBlockOutputStream out; + private final CountingOutputStream countingOut; private final ZipEncoding zipEncoding; @@ -93,81 +122,114 @@ private static final ZipEncoding ASCII = ZipEncodingHelper.getZipEncoding("ASCII"); + private static final int BLOCK_SIZE_UNSPECIFIED = -511; + /** - * Constructor for TarInputStream. + * Constructor for TarArchiveOutputStream. + * + *

    Uses a block size of 512 bytes.

    + * * @param os the output stream to use */ public TarArchiveOutputStream(final OutputStream os) { - this(os, TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE); + this(os, BLOCK_SIZE_UNSPECIFIED); } /** - * Constructor for TarInputStream. + * Constructor for TarArchiveOutputStream. + * + *

    Uses a block size of 512 bytes.

    + * * @param os the output stream to use * @param encoding name of the encoding to use for file names * @since 1.4 */ public TarArchiveOutputStream(final OutputStream os, final String encoding) { - this(os, TarConstants.DEFAULT_BLKSIZE, TarConstants.DEFAULT_RCDSIZE, encoding); + this(os, BLOCK_SIZE_UNSPECIFIED, encoding); } /** - * Constructor for TarInputStream. + * Constructor for TarArchiveOutputStream. + * * @param os the output stream to use - * @param blockSize the block size to use + * @param blockSize the block size to use. Must be a multiple of 512 bytes. */ public TarArchiveOutputStream(final OutputStream os, final int blockSize) { - this(os, blockSize, TarConstants.DEFAULT_RCDSIZE); + this(os, blockSize, null); } + /** - * Constructor for TarInputStream. + * Constructor for TarArchiveOutputStream. + * * @param os the output stream to use * @param blockSize the block size to use - * @param encoding name of the encoding to use for file names - * @since 1.4 + * @param recordSize the record size to use. Must be 512 bytes. + * @deprecated recordSize must always be 512 bytes. An IllegalArgumentException will be thrown + * if any other value is used */ + @Deprecated public TarArchiveOutputStream(final OutputStream os, final int blockSize, - final String encoding) { - this(os, blockSize, TarConstants.DEFAULT_RCDSIZE, encoding); + final int recordSize) { + this(os, blockSize, recordSize, null); } /** - * Constructor for TarInputStream. + * Constructor for TarArchiveOutputStream. + * * @param os the output stream to use - * @param blockSize the block size to use - * @param recordSize the record size to use + * @param blockSize the block size to use . Must be a multiple of 512 bytes. + * @param recordSize the record size to use. Must be 512 bytes. + * @param encoding name of the encoding to use for file names + * @since 1.4 + * @deprecated recordSize must always be 512 bytes. An IllegalArgumentException will be thrown + * if any other value is used. */ - public TarArchiveOutputStream(final OutputStream os, final int blockSize, final int recordSize) { - this(os, blockSize, recordSize, null); + @Deprecated + public TarArchiveOutputStream(final OutputStream os, final int blockSize, + final int recordSize, final String encoding) { + this(os, blockSize, encoding); + if (recordSize != RECORD_SIZE) { + throw new IllegalArgumentException( + "Tar record size must always be 512 bytes. Attempt to set size of " + recordSize); + } + } /** - * Constructor for TarInputStream. + * Constructor for TarArchiveOutputStream. + * * @param os the output stream to use - * @param blockSize the block size to use - * @param recordSize the record size to use + * @param blockSize the block size to use. Must be a multiple of 512 bytes. * @param encoding name of the encoding to use for file names * @since 1.4 */ public TarArchiveOutputStream(final OutputStream os, final int blockSize, - final int recordSize, final String encoding) { - out = new CountingOutputStream(os); + final String encoding) { + int realBlockSize; + if (BLOCK_SIZE_UNSPECIFIED == blockSize) { + realBlockSize = RECORD_SIZE; + } else { + realBlockSize = blockSize; + } + + if (realBlockSize <=0 || realBlockSize % RECORD_SIZE != 0) { + throw new IllegalArgumentException("Block size must be a multiple of 512 bytes. Attempt to use set size of " + blockSize); + } + out = new FixedLengthBlockOutputStream(countingOut = new CountingOutputStream(os), + RECORD_SIZE); this.encoding = encoding; this.zipEncoding = ZipEncodingHelper.getZipEncoding(encoding); - this.assemLen = 0; - this.assemBuf = new byte[recordSize]; - this.recordBuf = new byte[recordSize]; - this.recordSize = recordSize; - this.recordsPerBlock = blockSize / recordSize; + this.recordBuf = new byte[RECORD_SIZE]; + this.recordsPerBlock = realBlockSize / RECORD_SIZE; } /** - * Set the long file mode. - * This can be LONGFILE_ERROR(0), LONGFILE_TRUNCATE(1) or LONGFILE_GNU(2). - * This specifies the treatment of long file names (names >= TarConstants.NAMELEN). - * Default is LONGFILE_ERROR. + * Set the long file mode. This can be LONGFILE_ERROR(0), LONGFILE_TRUNCATE(1) or + * LONGFILE_GNU(2). This specifies the treatment of long file names (names >= + * TarConstants.NAMELEN). Default is LONGFILE_ERROR. + * * @param longFileMode the mode to use */ public void setLongFileMode(final int longFileMode) { @@ -175,10 +237,11 @@ } /** - * Set the big number mode. - * This can be BIGNUMBER_ERROR(0), BIGNUMBER_POSIX(1) or BIGNUMBER_STAR(2). - * This specifies the treatment of big files (sizes > TarConstants.MAXSIZE) and other numeric values to big to fit into a traditional tar header. + * Set the big number mode. This can be BIGNUMBER_ERROR(0), BIGNUMBER_POSIX(1) or + * BIGNUMBER_STAR(2). This specifies the treatment of big files (sizes > + * TarConstants.MAXSIZE) and other numeric values to big to fit into a traditional tar header. * Default is BIGNUMBER_ERROR. + * * @param bigNumberMode the mode to use * @since 1.4 */ @@ -188,8 +251,9 @@ /** * Whether to add a PAX extension header for non-ASCII file names. - * @since 1.4 + * * @param b whether to add a PAX extension header for non-ASCII file names. + * @since 1.4 */ public void setAddPaxHeadersForNonAsciiNames(final boolean b) { addPaxHeadersForNonAsciiNames = b; @@ -203,16 +267,16 @@ @Override public long getBytesWritten() { - return ((CountingOutputStream) out).getBytesWritten(); + return countingOut.getBytesWritten(); } /** * Ends the TAR archive without closing the underlying OutputStream. - * + * * An archive consists of a series of file entries terminated by an - * end-of-archive entry, which consists of two 512 blocks of zero bytes. + * end-of-archive entry, which consists of two 512 blocks of zero bytes. * POSIX.1 requires two EOF records, like some other implementations. - * + * * @throws IOException on error */ @Override @@ -222,7 +286,7 @@ } if (haveUnclosedEntry) { - throw new IOException("This archives contains unclosed entries."); + throw new IOException("This archive contains unclosed entries."); } writeEOFRecord(); writeEOFRecord(); @@ -233,17 +297,20 @@ /** * Closes the underlying OutputStream. + * * @throws IOException on error */ @Override public void close() throws IOException { - if (!finished) { - finish(); - } - - if (!closed) { - out.close(); - closed = true; + try { + if (!finished) { + finish(); + } + } finally { + if (!closed) { + out.close(); + closed = true; + } } } @@ -251,19 +318,19 @@ * Get the record size being used by this stream's TarBuffer. * * @return The TarBuffer record size. + * @deprecated */ + @Deprecated public int getRecordSize() { - return this.recordSize; + return RECORD_SIZE; } /** - * Put an entry on the output stream. This writes the entry's - * header record and positions the output stream for writing - * the contents of the entry. Once this method is called, the - * stream is ready for calls to write() to write the entry's - * contents. Once the contents are written, closeArchiveEntry() - * MUST be called to ensure that all buffered data - * is completely written to the output stream. + * Put an entry on the output stream. This writes the entry's header record and positions the + * output stream for writing the contents of the entry. Once this method is called, the stream + * is ready for calls to write() to write the entry's contents. Once the contents are written, + * closeArchiveEntry() MUST be called to ensure that all buffered data is completely + * written to the output stream. * * @param archiveEntry The TarEntry to be written to the archive. * @throws IOException on error @@ -275,60 +342,70 @@ throw new IOException("Stream has already been finished"); } final TarArchiveEntry entry = (TarArchiveEntry) archiveEntry; - final Map paxHeaders = new HashMap<>(); - final String entryName = entry.getName(); - final boolean paxHeaderContainsPath = handleLongName(entry, entryName, paxHeaders, "path", - TarConstants.LF_GNUTYPE_LONGNAME, "file name"); - - final String linkName = entry.getLinkName(); - final boolean paxHeaderContainsLinkPath = linkName != null && linkName.length() > 0 - && handleLongName(entry, linkName, paxHeaders, "linkpath", - TarConstants.LF_GNUTYPE_LONGLINK, "link name"); - - if (bigNumberMode == BIGNUMBER_POSIX) { - addPaxHeadersForBigNumbers(paxHeaders, entry); - } else if (bigNumberMode != BIGNUMBER_STAR) { - failForBigNumbers(entry); - } + if (entry.isGlobalPaxHeader()) { + final byte[] data = encodeExtendedPaxHeadersContents(entry.getExtraPaxHeaders()); + entry.setSize(data.length); + entry.writeEntryHeader(recordBuf, zipEncoding, bigNumberMode == BIGNUMBER_STAR); + writeRecord(recordBuf); + currSize= entry.getSize(); + currBytes = 0; + this.haveUnclosedEntry = true; + write(data); + closeArchiveEntry(); + } else { + final Map paxHeaders = new HashMap<>(); + final String entryName = entry.getName(); + final boolean paxHeaderContainsPath = handleLongName(entry, entryName, paxHeaders, "path", + TarConstants.LF_GNUTYPE_LONGNAME, "file name"); + + final String linkName = entry.getLinkName(); + final boolean paxHeaderContainsLinkPath = linkName != null && linkName.length() > 0 + && handleLongName(entry, linkName, paxHeaders, "linkpath", + TarConstants.LF_GNUTYPE_LONGLINK, "link name"); + + if (bigNumberMode == BIGNUMBER_POSIX) { + addPaxHeadersForBigNumbers(paxHeaders, entry); + } else if (bigNumberMode != BIGNUMBER_STAR) { + failForBigNumbers(entry); + } - if (addPaxHeadersForNonAsciiNames && !paxHeaderContainsPath - && !ASCII.canEncode(entryName)) { - paxHeaders.put("path", entryName); - } + if (addPaxHeadersForNonAsciiNames && !paxHeaderContainsPath + && !ASCII.canEncode(entryName)) { + paxHeaders.put("path", entryName); + } - if (addPaxHeadersForNonAsciiNames && !paxHeaderContainsLinkPath - && (entry.isLink() || entry.isSymbolicLink()) - && !ASCII.canEncode(linkName)) { - paxHeaders.put("linkpath", linkName); - } + if (addPaxHeadersForNonAsciiNames && !paxHeaderContainsLinkPath + && (entry.isLink() || entry.isSymbolicLink()) + && !ASCII.canEncode(linkName)) { + paxHeaders.put("linkpath", linkName); + } + paxHeaders.putAll(entry.getExtraPaxHeaders()); - if (paxHeaders.size() > 0) { - writePaxHeaders(entry, entryName, paxHeaders); - } + if (paxHeaders.size() > 0) { + writePaxHeaders(entry, entryName, paxHeaders); + } - entry.writeEntryHeader(recordBuf, zipEncoding, - bigNumberMode == BIGNUMBER_STAR); - writeRecord(recordBuf); + entry.writeEntryHeader(recordBuf, zipEncoding, bigNumberMode == BIGNUMBER_STAR); + writeRecord(recordBuf); - currBytes = 0; + currBytes = 0; - if (entry.isDirectory()) { - currSize = 0; - } else { - currSize = entry.getSize(); + if (entry.isDirectory()) { + currSize = 0; + } else { + currSize = entry.getSize(); + } + currName = entryName; + haveUnclosedEntry = true; } - currName = entryName; - haveUnclosedEntry = true; } /** - * Close an entry. This method MUST be called for all file - * entries that contain data. The reason is that we must - * buffer data written to the stream in order to satisfy - * the buffer's record based writes. Thus, there may be - * data fragments still being assembled that must be written - * to the output stream before this entry is closed and the - * next entry written. + * Close an entry. This method MUST be called for all file entries that contain data. The reason + * is that we must buffer data written to the stream in order to satisfy the buffer's record + * based writes. Thus, there may be data fragments still being assembled that must be written to + * the output stream before this entry is closed and the next entry written. + * * @throws IOException on error */ @Override @@ -336,37 +413,27 @@ if (finished) { throw new IOException("Stream has already been finished"); } - if (!haveUnclosedEntry){ + if (!haveUnclosedEntry) { throw new IOException("No current entry to close"); } - if (assemLen > 0) { - for (int i = assemLen; i < assemBuf.length; ++i) { - assemBuf[i] = 0; - } - - writeRecord(assemBuf); - - currBytes += assemLen; - assemLen = 0; - } - + out.flushBlock(); if (currBytes < currSize) { throw new IOException("entry '" + currName + "' closed at '" - + currBytes - + "' before the '" + currSize - + "' bytes specified in the header were written"); + + currBytes + + "' before the '" + currSize + + "' bytes specified in the header were written"); + } + recordsWritten += (currSize / RECORD_SIZE); + if (0 != currSize % RECORD_SIZE) { + recordsWritten++; } haveUnclosedEntry = false; } /** - * Writes bytes to the current tar archive entry. This method - * is aware of the current entry and will throw an exception if - * you attempt to write bytes past the length specified for the - * current entry. The method is also (painfully) aware of the - * record buffering required by TarBuffer, and manages buffers - * that are not a multiple of recordsize in length, including - * assembling records from small buffers. + * Writes bytes to the current tar archive entry. This method is aware of the current entry and + * will throw an exception if you attempt to write bytes past the length specified for the + * current entry. * * @param wBuf The buffer to write to the archive. * @param wOffset The offset in the buffer from which to get bytes. @@ -380,83 +447,39 @@ } if (currBytes + numToWrite > currSize) { throw new IOException("request to write '" + numToWrite - + "' bytes exceeds size in header of '" - + currSize + "' bytes for entry '" - + currName + "'"); - - // - // We have to deal with assembly!!! - // The programmer can be writing little 32 byte chunks for all - // we know, and we must assemble complete records for writing. - // REVIEW Maybe this should be in TarBuffer? Could that help to - // eliminate some of the buffer copying. - // - } - - if (assemLen > 0) { - if (assemLen + numToWrite >= recordBuf.length) { - final int aLen = recordBuf.length - assemLen; - - System.arraycopy(assemBuf, 0, recordBuf, 0, - assemLen); - System.arraycopy(wBuf, wOffset, recordBuf, - assemLen, aLen); - writeRecord(recordBuf); - - currBytes += recordBuf.length; - wOffset += aLen; - numToWrite -= aLen; - assemLen = 0; - } else { - System.arraycopy(wBuf, wOffset, assemBuf, assemLen, - numToWrite); - - wOffset += numToWrite; - assemLen += numToWrite; - numToWrite = 0; - } - } - - // - // When we get here we have EITHER: - // o An empty "assemble" buffer. - // o No bytes to write (numToWrite == 0) - // - while (numToWrite > 0) { - if (numToWrite < recordBuf.length) { - System.arraycopy(wBuf, wOffset, assemBuf, assemLen, - numToWrite); - - assemLen += numToWrite; - - break; - } - - writeRecord(wBuf, wOffset); - - final int num = recordBuf.length; - - currBytes += num; - numToWrite -= num; - wOffset += num; + + "' bytes exceeds size in header of '" + + currSize + "' bytes for entry '" + + currName + "'"); } + out.write(wBuf, wOffset, numToWrite); + currBytes += numToWrite; } /** * Writes a PAX extended header with the given map as contents. + * * @since 1.4 */ void writePaxHeaders(final TarArchiveEntry entry, - final String entryName, - final Map headers) throws IOException { + final String entryName, + final Map headers) throws IOException { String name = "./PaxHeaders.X/" + stripTo7Bits(entryName); if (name.length() >= TarConstants.NAMELEN) { name = name.substring(0, TarConstants.NAMELEN - 1); } final TarArchiveEntry pex = new TarArchiveEntry(name, - TarConstants.LF_PAX_EXTENDED_HEADER_LC); + TarConstants.LF_PAX_EXTENDED_HEADER_LC); transferModTime(entry, pex); + final byte[] data = encodeExtendedPaxHeadersContents(headers); + pex.setSize(data.length); + putArchiveEntry(pex); + write(data); + closeArchiveEntry(); + } + + private byte[] encodeExtendedPaxHeadersContents(Map headers) + throws UnsupportedEncodingException { final StringWriter w = new StringWriter(); for (final Map.Entry h : headers.entrySet()) { final String key = h.getKey(); @@ -478,11 +501,7 @@ } w.write(line); } - final byte[] data = w.toString().getBytes(CharsetNames.UTF_8); - pex.setSize(data.length); - putArchiveEntry(pex); - write(data); - closeArchiveEntry(); + return w.toString().getBytes(CharsetNames.UTF_8); } private String stripTo7Bits(final String name) { @@ -500,8 +519,8 @@ } /** - * @return true if the character could lead to problems when used - * inside a TarArchiveEntry name for a PAX header. + * @return true if the character could lead to problems when used inside a TarArchiveEntry name + * for a PAX header. */ private boolean shouldBeReplaced(final char c) { return c == 0 // would be read as Trailing null @@ -510,8 +529,8 @@ } /** - * Write an EOF (end of archive) record to the tar archive. - * An EOF record consists of a record of all zeros. + * Write an EOF (end of archive) record to the tar archive. An EOF record consists of a record + * of all zeros. */ private void writeEOFRecord() throws IOException { Arrays.fill(recordBuf, (byte) 0); @@ -525,13 +544,13 @@ @Override public ArchiveEntry createArchiveEntry(final File inputFile, final String entryName) - throws IOException { - if(finished) { + throws IOException { + if (finished) { throw new IOException("Stream has already been finished"); } return new TarArchiveEntry(inputFile, entryName); } - + /** * Write an archive record to the archive. * @@ -539,38 +558,16 @@ * @throws IOException on error */ private void writeRecord(final byte[] record) throws IOException { - if (record.length != recordSize) { + if (record.length != RECORD_SIZE) { throw new IOException("record to write has length '" - + record.length - + "' which is not the record size of '" - + recordSize + "'"); + + record.length + + "' which is not the record size of '" + + RECORD_SIZE + "'"); } out.write(record); recordsWritten++; } - - /** - * Write an archive record to the archive, where the record may be - * inside of a larger array buffer. The buffer must be "offset plus - * record size" long. - * - * @param buf The buffer containing the record data to write. - * @param offset The offset of the record data within buf. - * @throws IOException on error - */ - private void writeRecord(final byte[] buf, final int offset) throws IOException { - - if (offset + recordSize > buf.length) { - throw new IOException("record has length '" + buf.length - + "' with offset '" + offset - + "' which is less than the record size of '" - + recordSize + "'"); - } - - out.write(buf, offset, recordSize); - recordsWritten++; - } private void padAsNeeded() throws IOException { final int start = recordsWritten % recordsPerBlock; @@ -582,28 +579,28 @@ } private void addPaxHeadersForBigNumbers(final Map paxHeaders, - final TarArchiveEntry entry) { + final TarArchiveEntry entry) { addPaxHeaderForBigNumber(paxHeaders, "size", entry.getSize(), - TarConstants.MAXSIZE); + TarConstants.MAXSIZE); addPaxHeaderForBigNumber(paxHeaders, "gid", entry.getLongGroupId(), - TarConstants.MAXID); + TarConstants.MAXID); addPaxHeaderForBigNumber(paxHeaders, "mtime", - entry.getModTime().getTime() / 1000, - TarConstants.MAXSIZE); + entry.getModTime().getTime() / 1000, + TarConstants.MAXSIZE); addPaxHeaderForBigNumber(paxHeaders, "uid", entry.getLongUserId(), - TarConstants.MAXID); + TarConstants.MAXID); // star extensions by J\u00f6rg Schilling addPaxHeaderForBigNumber(paxHeaders, "SCHILY.devmajor", - entry.getDevMajor(), TarConstants.MAXID); + entry.getDevMajor(), TarConstants.MAXID); addPaxHeaderForBigNumber(paxHeaders, "SCHILY.devminor", - entry.getDevMinor(), TarConstants.MAXID); + entry.getDevMinor(), TarConstants.MAXID); // there is no PAX header for file mode failForBigNumber("mode", entry.getMode(), TarConstants.MAXID); } private void addPaxHeaderForBigNumber(final Map paxHeaders, - final String header, final long value, - final long maxValue) { + final String header, final long value, + final long maxValue) { if (value < 0 || value > maxValue) { paxHeaders.put(header, String.valueOf(value)); } @@ -613,45 +610,43 @@ failForBigNumber("entry size", entry.getSize(), TarConstants.MAXSIZE); failForBigNumberWithPosixMessage("group id", entry.getLongGroupId(), TarConstants.MAXID); failForBigNumber("last modification time", - entry.getModTime().getTime() / 1000, - TarConstants.MAXSIZE); + entry.getModTime().getTime() / 1000, + TarConstants.MAXSIZE); failForBigNumber("user id", entry.getLongUserId(), TarConstants.MAXID); failForBigNumber("mode", entry.getMode(), TarConstants.MAXID); failForBigNumber("major device number", entry.getDevMajor(), - TarConstants.MAXID); + TarConstants.MAXID); failForBigNumber("minor device number", entry.getDevMinor(), - TarConstants.MAXID); + TarConstants.MAXID); } private void failForBigNumber(final String field, final long value, final long maxValue) { failForBigNumber(field, value, maxValue, ""); } - private void failForBigNumberWithPosixMessage(final String field, final long value, final long maxValue) { - failForBigNumber(field, value, maxValue, " Use STAR or POSIX extensions to overcome this limit"); + private void failForBigNumberWithPosixMessage(final String field, final long value, + final long maxValue) { + failForBigNumber(field, value, maxValue, + " Use STAR or POSIX extensions to overcome this limit"); } - private void failForBigNumber(final String field, final long value, final long maxValue, final String additionalMsg) { + private void failForBigNumber(final String field, final long value, final long maxValue, + final String additionalMsg) { if (value < 0 || value > maxValue) { throw new RuntimeException(field + " '" + value //NOSONAR - + "' is too big ( > " - + maxValue + " )." + additionalMsg); + + "' is too big ( > " + + maxValue + " )." + additionalMsg); } } /** * Handles long file or link names according to the longFileMode setting. * - *

    I.e. if the given name is too long to be written to a plain - * tar header then - *

      - *
    • it creates a pax header who's name is given by the - * paxHeaderName parameter if longFileMode is POSIX
      • - *
    • it creates a GNU longlink entry who's type is given by - * the linkType parameter if longFileMode is GNU
      • - *
    • it throws an exception if longFileMode is ERROR
      • - *
    • it truncates the name if longFileMode is TRUNCATE
      • - *

    + *

    I.e. if the given name is too long to be written to a plain tar header then

    • it + * creates a pax header who's name is given by the paxHeaderName parameter if longFileMode is + * POSIX
    • it creates a GNU longlink entry who's type is given by the linkType parameter + * if longFileMode is GNU
    • it throws an exception if longFileMode is ERROR
    • it + * truncates the name if longFileMode is TRUNCATE

    * * @param entry entry the name belongs to * @param name the name to write @@ -661,9 +656,9 @@ * @param fieldName the name of the field * @return whether a pax header has been written. */ - private boolean handleLongName(final TarArchiveEntry entry , final String name, - final Map paxHeaders, - final String paxHeaderName, final byte linkType, final String fieldName) + private boolean handleLongName(final TarArchiveEntry entry, final String name, + final Map paxHeaders, + final String paxHeaderName, final byte linkType, final String fieldName) throws IOException { final ByteBuffer encodedName = zipEncoding.encode(name); final int len = encodedName.limit() - encodedName.position(); @@ -675,9 +670,10 @@ } else if (longFileMode == LONGFILE_GNU) { // create a TarEntry for the LongLink, the contents // of which are the link's name - final TarArchiveEntry longLinkEntry = new TarArchiveEntry(TarConstants.GNU_LONGLINK, linkType); + final TarArchiveEntry longLinkEntry = new TarArchiveEntry(TarConstants.GNU_LONGLINK, + linkType); - longLinkEntry.setSize(len + 1l); // +1 for NUL + longLinkEntry.setSize(len + 1L); // +1 for NUL transferModTime(entry, longLinkEntry); putArchiveEntry(longLinkEntry); write(encodedName.array(), encodedName.arrayOffset(), len); @@ -685,8 +681,8 @@ closeArchiveEntry(); } else if (longFileMode != LONGFILE_TRUNCATE) { throw new RuntimeException(fieldName + " '" + name //NOSONAR - + "' is too long ( > " - + TarConstants.NAMELEN + " bytes)"); + + "' is too long ( > " + + TarConstants.NAMELEN + " bytes)"); } } return false; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java 2016-03-30 12:43:48.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarConstants.java 2018-05-02 20:17:13.000000000 +0000 @@ -73,7 +73,7 @@ * be expressed in octal char notation (that's 7 sevens, octal). */ long MAXID = 07777777L; - + /** * The length of the checksum field in a header buffer. */ @@ -92,7 +92,7 @@ int SIZELEN = 12; /** - * The maximum size of a file in a tar archive + * The maximum size of a file in a tar archive * which can be expressed in octal char notation (that's 11 sevens, octal). */ long MAXSIZE = 077777777777L; @@ -133,67 +133,67 @@ /** * Length of the prefix field. - * + * */ int PREFIXLEN = 155; /** * The length of the access time field in an old GNU header buffer. - * + * */ int ATIMELEN_GNU = 12; /** * The length of the created time field in an old GNU header buffer. - * + * */ int CTIMELEN_GNU = 12; /** - * The length of the multivolume start offset field in an old GNU header buffer. - * + * The length of the multivolume start offset field in an old GNU header buffer. + * */ int OFFSETLEN_GNU = 12; /** - * The length of the long names field in an old GNU header buffer. - * + * The length of the long names field in an old GNU header buffer. + * */ int LONGNAMESLEN_GNU = 4; /** - * The length of the padding field in an old GNU header buffer. - * + * The length of the padding field in an old GNU header buffer. + * */ int PAD2LEN_GNU = 1; /** - * The sum of the length of all sparse headers in an old GNU header buffer. - * + * The sum of the length of all sparse headers in an old GNU header buffer. + * */ int SPARSELEN_GNU = 96; /** - * The length of the is extension field in an old GNU header buffer. - * + * The length of the is extension field in an old GNU header buffer. + * */ int ISEXTENDEDLEN_GNU = 1; /** - * The length of the real size field in an old GNU header buffer. - * + * The length of the real size field in an old GNU header buffer. + * */ int REALSIZELEN_GNU = 12; /** - * The sum of the length of all sparse headers in a sparse header buffer. - * + * The sum of the length of all sparse headers in a sparse header buffer. + * */ int SPARSELEN_GNU_SPARSE = 504; /** - * The length of the is extension field in a sparse header buffer. - * + * The length of the is extension field in a sparse header buffer. + * */ int ISEXTENDEDLEN_GNU_SPARSE = 1; @@ -337,21 +337,21 @@ /** * Length of the prefix field in xstar archives. - * + * * @since 1.11 */ int PREFIXLEN_XSTAR = 131; /** * The length of the access time field in a xstar header buffer. - * + * * @since 1.11 */ int ATIMELEN_XSTAR = 12; /** * The length of the created time field in a xstar header buffer. - * + * * @since 1.11 */ int CTIMELEN_XSTAR = 12; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/tar/TarUtils.java 2018-05-23 12:50:54.000000000 +0000 @@ -147,7 +147,7 @@ return result; } - /** + /** * Compute the value contained in a byte buffer. If the most * significant bit of the first byte in the buffer is set, this * bit is ignored and the rest of the buffer is interpreted as a @@ -245,8 +245,7 @@ String string = new String(buffer, offset, length); string=string.replaceAll("\0", "{NUL}"); // Replace NULs to allow string to be printed - final String s = "Invalid byte "+currentByte+" at offset "+(current-offset)+" in '"+string+"' len="+length; - return s; + return "Invalid byte "+currentByte+" at offset "+(current-offset)+" in '"+string+"' len="+length; } /** @@ -290,11 +289,9 @@ final ZipEncoding encoding) throws IOException { - int len = length; - for (; len > 0; len--) { - if (buffer[offset + len - 1] != 0) { - break; - } + int len = 0; + for (int i = offset; len < length && buffer[i] != 0; i++) { + len++; } if (len > 0) { final byte[] b = new byte[len]; @@ -307,7 +304,7 @@ /** * Copy a name into a buffer. * Copies characters from the name into the buffer - * starting at the specified offset. + * starting at the specified offset. * If the buffer is longer than the name, the buffer * is filled with trailing NULs. * If the name is longer than the buffer, @@ -336,7 +333,7 @@ /** * Copy a name into a buffer. * Copies characters from the name into the buffer - * starting at the specified offset. + * starting at the specified offset. * If the buffer is longer than the name, the buffer * is filled with trailing NULs. * If the name is longer than the buffer, @@ -373,7 +370,7 @@ /** * Fill buffer with unsigned octal number, padded with leading zeroes. - * + * * @param value number to convert to octal - treated as unsigned * @param buffer destination buffer * @param offset starting offset in buffer @@ -411,7 +408,7 @@ * Uses {@link #formatUnsignedOctalString} to format * the value as an octal string with leading zeros. * The converted number is followed by space and NUL - * + * * @param value The value to write * @param buf The buffer to receive the output * @param offset The starting offset into the buffer @@ -432,11 +429,11 @@ /** * Write an octal long integer into a buffer. - * + * * Uses {@link #formatUnsignedOctalString} to format * the value as an octal string with leading zeros. * The converted number is followed by a space. - * + * * @param value The value to write as octal * @param buf The destinationbuffer. * @param offset The starting offset into the buffer. @@ -457,11 +454,11 @@ /** * Write an long integer into a buffer as an octal string if this * will fit, or as a binary number otherwise. - * + * * Uses {@link #formatUnsignedOctalString} to format * the value as an octal string with leading zeros. * The converted number is followed by a space. - * + * * @param value The value to write into the buffer. * @param buf The destination buffer. * @param offset The starting offset into the buffer. @@ -484,8 +481,9 @@ if (length < 9) { formatLongBinary(value, buf, offset, length, negative); + } else { + formatBigIntegerBinary(value, buf, offset, length, negative); } - formatBigIntegerBinary(value, buf, offset, length, negative); buf[offset] = (byte) (negative ? 0xff : 0x80); return offset + length; @@ -495,16 +493,16 @@ final int offset, final int length, final boolean negative) { final int bits = (length - 1) * 8; - final long max = 1l << bits; - long val = Math.abs(value); - if (val >= max) { + final long max = 1L << bits; + long val = Math.abs(value); // Long.MIN_VALUE stays Long.MIN_VALUE + if (val < 0 || val >= max) { throw new IllegalArgumentException("Value " + value + " is too large for " + length + " byte field."); } if (negative) { val ^= max - 1; - val |= 0xff << bits; val++; + val |= 0xffL << bits; } for (int i = offset + length - 1; i >= offset; i--) { buf[i] = (byte) val; @@ -519,6 +517,10 @@ final BigInteger val = BigInteger.valueOf(value); final byte[] b = val.toByteArray(); final int len = b.length; + if (len > length - 1) { + throw new IllegalArgumentException("Value " + value + + " is too large for " + length + " byte field."); + } final int off = offset + length - len; System.arraycopy(b, 0, buf, off, len); final byte fill = (byte) (negative ? 0xff : 0); @@ -529,7 +531,7 @@ /** * Writes an octal value into a buffer. - * + * * Uses {@link #formatUnsignedOctalString} to format * the value as an octal string with leading zeros. * The converted number is followed by NUL and then space. @@ -569,7 +571,7 @@ } /** - * Wikipedia says: + * Wikipedia says: *
    * The checksum is calculated by taking the sum of the unsigned byte values * of the header block with the eight checksum bytes taken to be ascii diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/AbstractUnicodeExtraField.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/AbstractUnicodeExtraField.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/AbstractUnicodeExtraField.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/AbstractUnicodeExtraField.java 2018-05-02 20:17:13.000000000 +0000 @@ -39,7 +39,7 @@ /** * Assemble as unicode extension from the name/comment and * encoding of the original zip entry. - * + * * @param text The file name or comment. * @param bytes The encoded of the filename or comment in the zip * file. @@ -63,7 +63,7 @@ /** * Assemble as unicode extension from the name/comment and * encoding of the original zip entry. - * + * * @param text The file name or comment. * @param bytes The encoded of the filename or comment in the zip * file. diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/BinaryTree.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/BinaryTree.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/BinaryTree.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/BinaryTree.java 2018-05-02 20:17:13.000000000 +0000 @@ -26,7 +26,7 @@ /** * Binary tree of positive values. - * + * * @author Emmanuel Bourg * @since 1.7 */ @@ -34,11 +34,11 @@ /** Value in the array indicating an undefined node */ private static final int UNDEFINED = -1; - + /** Value in the array indicating a non leaf node */ private static final int NODE = -2; - /** + /** * The array representing the binary tree. The root is at index 0, * the left children are at 2*i+1 and the right children at 2*i+2. */ @@ -51,7 +51,7 @@ /** * Adds a leaf to the tree. - * + * * @param node the index of the node where the path is appended * @param path the path to the leaf (bits are parsed from the right to the left) * @param depth the number of nodes in the path @@ -68,7 +68,7 @@ } else { // mark the current node as a non leaf node tree[node] = NODE; - + // move down the path recursively final int nextChild = 2 * node + 1 + (path & 1); addLeaf(nextChild, path >>> 1, depth - 1, value); @@ -77,7 +77,7 @@ /** * Reads a value from the specified bit stream. - * + * * @param stream * @return the value decoded, or -1 if the end of the stream is reached */ @@ -102,7 +102,7 @@ } } } - + /** * Decodes the packed binary tree from the specified stream. @@ -119,7 +119,7 @@ /** The maximum bit length for a value (16 or lower) */ int maxLength = 0; - + final int[] originalBitLengths = new int[totalNumberOfValues]; int pos = 0; for (final byte b : encodedTree) { @@ -130,7 +130,7 @@ for (int j = 0; j < numberOfValues; j++) { originalBitLengths[pos++] = bitLength; } - + maxLength = Math.max(maxLength, bitLength); } @@ -139,7 +139,7 @@ for (int k = 0; k < permutation.length; k++) { permutation[k] = k; } - + int c = 0; final int[] sortedBitLengths = new int[originalBitLengths.length]; for (int k = 0; k < originalBitLengths.length; k++) { @@ -149,10 +149,10 @@ if (originalBitLengths[l] == k) { // put the value at the current position in the sorted array... sortedBitLengths[c] = k; - + // ...and memorize the permutation - permutation[c] = l; - + permutation[c] = l; + c++; } } @@ -173,10 +173,10 @@ } codes[permutation[i]] = code; } - + // build the tree final BinaryTree tree = new BinaryTree(maxLength); - + for (int k = 0; k < codes.length; k++) { final int bitLength = originalBitLengths[k]; if (bitLength > 0) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/BitStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/BitStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/BitStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/BitStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -38,7 +38,7 @@ /** * Returns the next bit. - * + * * @return The next bit (0 or 1) or -1 if the end of the stream has been reached */ int nextBit() throws IOException { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/CharsetAccessor.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/CharsetAccessor.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/CharsetAccessor.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/CharsetAccessor.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.commons.compress.archivers.zip; + +import java.nio.charset.Charset; + +/** + * An interface added to allow access to the character set associated with an {@link NioZipEncoding}, + * without requiring a new method to be added to {@link ZipEncoding}. + *

    + * This avoids introducing a + * potentially breaking change, or making {@link NioZipEncoding} a public class. + *

    + * @since 1.15 + */ +public interface CharsetAccessor { + + /** + * Provides access to the character set associated with an object. + *

    + * This allows nio oriented code to use more natural character encoding/decoding methods, + * whilst allowing existing code to continue to rely on special-case error handling for UTF-8. + *

    + * @return the character set associated with this object + */ + Charset getCharset(); +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/CircularBuffer.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/CircularBuffer.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/CircularBuffer.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/CircularBuffer.java 2018-05-02 20:17:13.000000000 +0000 @@ -21,7 +21,7 @@ /** * Circular byte buffer. - * + * * @author Emmanuel Bourg * @since 1.7 */ @@ -29,7 +29,7 @@ /** Size of the buffer */ private final int size; - + /** The buffer */ private final byte[] buffer; @@ -73,7 +73,7 @@ /** * Copy a previous interval in the buffer to the current position. - * + * * @param distance the distance from the current write position * @param length the number of bytes to copy */ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ExplodingInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ExplodingInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ExplodingInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ExplodingInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -19,6 +19,9 @@ package org.apache.commons.compress.archivers.zip; +import org.apache.commons.compress.utils.CountingInputStream; +import org.apache.commons.compress.utils.InputStreamStatistics; + import java.io.IOException; import java.io.InputStream; @@ -28,17 +31,17 @@ * method. *

    * The algorithm is described in the ZIP File Format Specification. - * - * @see ZIP File Format Specification - * + * + * @see ZIP File Format Specification + * * @author Emmanuel Bourg * @since 1.7 */ -class ExplodingInputStream extends InputStream { +class ExplodingInputStream extends InputStream implements InputStreamStatistics { /** The underlying stream containing the compressed data */ private final InputStream in; - + /** The stream of bits read from the input stream */ private BitStream bits; @@ -62,6 +65,10 @@ /** Output buffer holding the decompressed data */ private final CircularBuffer buffer = new CircularBuffer(32 * 1024); + private long uncompressedCount = 0; + + private long treeSizes = 0; + /** * Create a new stream decompressing the content of the specified stream * using the explode algorithm. @@ -85,18 +92,26 @@ /** * Reads the encoded binary trees and prepares the bit stream. - * + * * @throws IOException */ private void init() throws IOException { if (bits == null) { - if (numberOfTrees == 3) { - literalTree = BinaryTree.decode(in, 256); + try (CountingInputStream i = new CountingInputStream(in) { + @Override + public void close() { + // we do not want to close in + } + }) { + if (numberOfTrees == 3) { + literalTree = BinaryTree.decode(i, 256); + } + + lengthTree = BinaryTree.decode(i, 64); + distanceTree = BinaryTree.decode(i, 64); + treeSizes += i.getBytesRead(); } - lengthTree = BinaryTree.decode(in, 64); - distanceTree = BinaryTree.decode(in, 64); - bits = new BitStream(in); } } @@ -107,7 +122,35 @@ fillBuffer(); } - return buffer.get(); + final int ret = buffer.get(); + if (ret > -1) { + uncompressedCount++; + } + return ret; + } + + /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return bits.getBytesRead() + treeSizes; + } + + /** + * @since 1.17 + */ + @Override + public long getUncompressedCount() { + return uncompressedCount; + } + + /** + * @since 1.17 + */ + @Override + public void close() throws IOException { + in.close(); } /** @@ -116,7 +159,7 @@ */ private void fillBuffer() throws IOException { init(); - + final int bit = bits.nextBit(); if (bit == 1) { // literal value @@ -131,7 +174,7 @@ // end of stream reached, nothing left to decode return; } - + buffer.put(literal); } else if (bit == 0) { @@ -144,7 +187,7 @@ return; } final int distance = distanceHigh << distanceLowSize | distanceLow; - + int length = lengthTree.read(bits); if (length == 63) { length += bits.nextBits(8); diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ExtraFieldUtils.java 2018-05-23 12:50:54.000000000 +0000 @@ -52,6 +52,7 @@ register(X0016_CertificateIdForCentralDirectory.class); register(X0017_StrongEncryptionHeader.class); register(X0019_EncryptionRecipientCertificateList.class); + register(ResourceAlignmentExtraField.class); } /** @@ -174,11 +175,15 @@ } try { final ZipExtraField ze = createExtraField(headerId); - if (local) { - ze.parseFromLocalFileData(data, start + WORD, length); - } else { - ze.parseFromCentralDirectoryData(data, start + WORD, - length); + try { + if (local) { + ze.parseFromLocalFileData(data, start + WORD, length); + } else { + ze.parseFromCentralDirectoryData(data, start + WORD, length); + } + } catch (ArrayIndexOutOfBoundsException aiobe) { + throw (ZipException) new ZipException("Failed to parse corrupt ZIP extra field of type " + + Integer.toHexString(headerId.getValue())).initCause(aiobe); } v.add(ze); } catch (final InstantiationException | IllegalAccessException ie) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/FallbackZipEncoding.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/FallbackZipEncoding.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/FallbackZipEncoding.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/FallbackZipEncoding.java 1970-01-01 00:00:00.000000000 +0000 @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.commons.compress.archivers.zip; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * A fallback ZipEncoding, which uses a java.io means to encode names. - * - *

    This implementation is not suitable for encodings other than - * UTF-8, because java.io encodes unmappable character as question - * marks leading to unreadable ZIP entries on some operating - * systems.

    - * - *

    Furthermore this implementation is unable to tell whether a - * given name can be safely encoded or not.

    - * - *

    This implementation acts as a last resort implementation, when - * neither {@link Simple8BitZipEnoding} nor {@link NioZipEncoding} is - * available.

    - * - *

    The methods of this class are reentrant.

    - * @Immutable - */ -class FallbackZipEncoding implements ZipEncoding { - private final String charsetName; - - /** - * Construct a fallback zip encoding, which uses the platform's - * default charset. - */ - public FallbackZipEncoding() { - this.charsetName = null; - } - - /** - * Construct a fallback zip encoding, which uses the given charset. - * - * @param charsetName The name of the charset or {@code null} for - * the platform's default character set. - */ - public FallbackZipEncoding(final String charsetName) { - this.charsetName = charsetName; - } - - /** - * @see - * org.apache.commons.compress.archivers.zip.ZipEncoding#canEncode(java.lang.String) - */ - @Override - public boolean canEncode(final String name) { - return true; - } - - /** - * @see - * org.apache.commons.compress.archivers.zip.ZipEncoding#encode(java.lang.String) - */ - @Override - public ByteBuffer encode(final String name) throws IOException { - if (this.charsetName == null) { // i.e. use default charset, see no-args constructor - return ByteBuffer.wrap(name.getBytes()); - } - return ByteBuffer.wrap(name.getBytes(this.charsetName)); - } - - /** - * @see - * org.apache.commons.compress.archivers.zip.ZipEncoding#decode(byte[]) - */ - @Override - public String decode(final byte[] data) throws IOException { - if (this.charsetName == null) { // i.e. use default charset, see no-args constructor - return new String(data); - } - return new String(data,this.charsetName); - } -} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/GeneralPurposeBit.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/GeneralPurposeBit.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/GeneralPurposeBit.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/GeneralPurposeBit.java 2018-05-02 20:17:13.000000000 +0000 @@ -20,7 +20,7 @@ /** * Parser/encoder for the "general purpose bit" field in ZIP's local * file and central directory headers. - * + * * @since 1.1 * @NotThreadSafe */ @@ -196,7 +196,7 @@ /** * Parses the supported flags from the given archive data. - * + * * @param data local file header or a central directory entry. * @param offset offset at which the general purpose bit starts * @return parsed flags diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/InflaterInputStreamWithStatistics.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/InflaterInputStreamWithStatistics.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/InflaterInputStreamWithStatistics.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/InflaterInputStreamWithStatistics.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.commons.compress.archivers.zip; + +import org.apache.commons.compress.utils.InputStreamStatistics; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +/** + * Helper class to provide statistics + * + * @since 1.17 + */ +/* package */ class InflaterInputStreamWithStatistics extends InflaterInputStream + implements InputStreamStatistics { + private long compressedCount = 0; + private long uncompressedCount = 0; + + public InflaterInputStreamWithStatistics(InputStream in) { + super(in); + } + + public InflaterInputStreamWithStatistics(InputStream in, Inflater inf) { + super(in, inf); + } + + public InflaterInputStreamWithStatistics(InputStream in, Inflater inf, int size) { + super(in, inf, size); + } + + @Override + protected void fill() throws IOException { + super.fill(); + compressedCount += inf.getRemaining(); + } + + @Override + public int read() throws IOException { + final int b = super.read(); + if (b > -1) { + uncompressedCount++; + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + final int bytes = super.read(b, off, len); + if (bytes > -1) { + uncompressedCount += bytes; + } + return bytes; + } + + @Override + public long getCompressedCount() { + return compressedCount; + } + + @Override + public long getUncompressedCount() { + return uncompressedCount; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/NioZipEncoding.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/NioZipEncoding.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/NioZipEncoding.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/NioZipEncoding.java 2018-05-02 20:17:13.000000000 +0000 @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; import java.nio.charset.CharsetEncoder; import java.nio.charset.CoderResult; import java.nio.charset.CodingErrorAction; @@ -30,82 +31,92 @@ /** * A ZipEncoding, which uses a java.nio {@link * java.nio.charset.Charset Charset} to encode names. - * - *

    This implementation works for all cases under java-1.5 or - * later. However, in java-1.4, some charsets don't have a java.nio - * implementation, most notably the default ZIP encoding Cp437.

    - * *

    The methods of this class are reentrant.

    * @Immutable */ -class NioZipEncoding implements ZipEncoding { +class NioZipEncoding implements ZipEncoding, CharsetAccessor { + private final Charset charset; + private final boolean useReplacement; + private static final char REPLACEMENT = '?'; + private static final byte[] REPLACEMENT_BYTES = { (byte) REPLACEMENT }; + private static final String REPLACEMENT_STRING = String.valueOf(REPLACEMENT); + private static final char[] HEX_CHARS = new char[] { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + /** - * Construct an NIO based zip encoding, which wraps the given - * charset. - * - * @param charset The NIO charset to wrap. + * Construct an NioZipEncoding using the given charset. + * @param charset The character set to use. + * @param useReplacement should invalid characters be replaced, or reported. */ - public NioZipEncoding(final Charset charset) { + NioZipEncoding(final Charset charset, boolean useReplacement) { this.charset = charset; + this.useReplacement = useReplacement; + } + + @Override + public Charset getCharset() { + return charset; } /** - * @see - * org.apache.commons.compress.archivers.zip.ZipEncoding#canEncode(java.lang.String) + * @see ZipEncoding#canEncode(java.lang.String) */ @Override public boolean canEncode(final String name) { - final CharsetEncoder enc = this.charset.newEncoder(); - enc.onMalformedInput(CodingErrorAction.REPORT); - enc.onUnmappableCharacter(CodingErrorAction.REPORT); + final CharsetEncoder enc = newEncoder(); return enc.canEncode(name); } /** - * @see - * org.apache.commons.compress.archivers.zip.ZipEncoding#encode(java.lang.String) + * @see ZipEncoding#encode(java.lang.String) */ @Override public ByteBuffer encode(final String name) { - final CharsetEncoder enc = this.charset.newEncoder(); - - enc.onMalformedInput(CodingErrorAction.REPORT); - enc.onUnmappableCharacter(CodingErrorAction.REPORT); + final CharsetEncoder enc = newEncoder(); final CharBuffer cb = CharBuffer.wrap(name); - ByteBuffer out = ByteBuffer.allocate(name.length() - + (name.length() + 1) / 2); + CharBuffer tmp = null; + ByteBuffer out = ByteBuffer.allocate(estimateInitialBufferSize(enc, cb.remaining())); while (cb.remaining() > 0) { - final CoderResult res = enc.encode(cb, out,true); + final CoderResult res = enc.encode(cb, out, false); if (res.isUnmappable() || res.isMalformed()) { // write the unmappable characters in utf-16 // pseudo-URL encoding style to ByteBuffer. - if (res.length() * 6 > out.remaining()) { - out = ZipEncodingHelper.growBuffer(out, out.position() - + res.length() * 6); - } - for (int i=0; i out.remaining()) { + // if the destination buffer isn't over sized, assume that the presence of one + // unmappable character makes it likely that there will be more. Find all the + // un-encoded characters and allocate space based on those estimates. + int charCount = 0; + for (int i = cb.position() ; i < cb.limit(); i++) { + charCount += !enc.canEncode(cb.get(i)) ? 6 : 1; + } + int totalExtraSpace = estimateIncrementalEncodingSize(enc, charCount); + out = ZipEncodingHelper.growBufferBy(out, totalExtraSpace - out.remaining()); + } + if (tmp == null) { + tmp = CharBuffer.allocate(6); + } + for (int i = 0; i < res.length(); ++i) { + out = encodeFully(enc, encodeSurrogate(tmp, cb.get()), out); } } else if (res.isOverflow()) { - - out = ZipEncodingHelper.growBuffer(out, 0); - - } else if (res.isUnderflow()) { - - enc.flush(out); - break; - + int increment = estimateIncrementalEncodingSize(enc, cb.remaining()); + out = ZipEncodingHelper.growBufferBy(out, increment); } } + // tell the encoder we are done + enc.encode(cb, out, true); + // may have caused underflow, but that's been ignored traditionally out.limit(out.position()); out.rewind(); @@ -114,13 +125,92 @@ /** * @see - * org.apache.commons.compress.archivers.zip.ZipEncoding#decode(byte[]) + * ZipEncoding#decode(byte[]) */ @Override public String decode(final byte[] data) throws IOException { - return this.charset.newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) + return newDecoder() .decode(ByteBuffer.wrap(data)).toString(); } + + private static ByteBuffer encodeFully(CharsetEncoder enc, CharBuffer cb, ByteBuffer out) { + ByteBuffer o = out; + while (cb.hasRemaining()) { + CoderResult result = enc.encode(cb, o, false); + if (result.isOverflow()) { + int increment = estimateIncrementalEncodingSize(enc, cb.remaining()); + o = ZipEncodingHelper.growBufferBy(o, increment); + } + } + return o; + } + + private static CharBuffer encodeSurrogate(CharBuffer cb, char c) { + cb.position(0).limit(6); + cb.put('%'); + cb.put('U'); + + cb.put(HEX_CHARS[(c >> 12) & 0x0f]); + cb.put(HEX_CHARS[(c >> 8) & 0x0f]); + cb.put(HEX_CHARS[(c >> 4) & 0x0f]); + cb.put(HEX_CHARS[c & 0x0f]); + cb.flip(); + return cb; + } + + private CharsetEncoder newEncoder() { + if (useReplacement) { + return charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE) + .replaceWith(REPLACEMENT_BYTES); + } else { + return charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + } + } + + private CharsetDecoder newDecoder() { + if (!useReplacement) { + return this.charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + } else { + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE) + .replaceWith(REPLACEMENT_STRING); + } + } + + /** + * Estimate the initial encoded size (in bytes) for a character buffer. + *

    + * The estimate assumes that one character consumes uses the maximum length encoding, + * whilst the rest use an average size encoding. This accounts for any BOM for UTF-16, at + * the expense of a couple of extra bytes for UTF-8 encoded ASCII. + *

    + * + * @param enc encoder to use for estimates + * @param charChount number of characters in string + * @return estimated size in bytes. + */ + private static int estimateInitialBufferSize(CharsetEncoder enc, int charChount) { + float first = enc.maxBytesPerChar(); + float rest = (charChount - 1) * enc.averageBytesPerChar(); + return (int) Math.ceil(first + rest); + } + + /** + * Estimate the size needed for remaining characters + * + * @param enc encoder to use for estimates + * @param charCount number of characters remaining + * @return estimated size in bytes. + */ + private static int estimateIncrementalEncodingSize(CharsetEncoder enc, int charCount) { + return (int) Math.ceil(charCount * enc.averageBytesPerChar()); + } + } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java 2018-05-23 12:50:54.000000000 +0000 @@ -227,6 +227,10 @@ * before calling this method. *

    * + *

    Calling this method will shut down the {@link ExecutorService} used by this class. If any of the {@link + * Callable}s {@link #submit}ted to this instance throws an exception, the archive can not be created properly and + * this method will throw an exception.

    + * * @param targetStream The {@link ZipArchiveOutputStream} to receive the contents of the scatter streams * @throws IOException If writing fails * @throws InterruptedException If we get interrupted @@ -236,19 +240,24 @@ throws IOException, InterruptedException, ExecutionException { // Make sure we catch any exceptions from parallel phase - for (final Future future : futures) { - future.get(); + try { + for (final Future future : futures) { + future.get(); + } + } finally { + es.shutdown(); } - es.shutdown(); - es.awaitTermination(1000 * 60l, TimeUnit.SECONDS); // == Infinity. We really *must* wait for this to complete + es.awaitTermination(1000 * 60L, TimeUnit.SECONDS); // == Infinity. We really *must* wait for this to complete // It is important that all threads terminate before we go on, ensure happens-before relationship compressionDoneAt = System.currentTimeMillis(); - for (final ScatterZipOutputStream scatterStream : streams) { - scatterStream.writeTo(targetStream); - scatterStream.close(); + synchronized (streams) { + for (final ScatterZipOutputStream scatterStream : streams) { + scatterStream.writeTo(targetStream); + scatterStream.close(); + } } scatterDoneAt = System.currentTimeMillis(); diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/PKWareExtraHeader.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/PKWareExtraHeader.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/PKWareExtraHeader.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/PKWareExtraHeader.java 2018-05-02 20:17:13.000000000 +0000 @@ -80,7 +80,7 @@ protected PKWareExtraHeader(final ZipShort headerId) { this.headerId = headerId; } - + /** * Get the header id. * @@ -193,7 +193,7 @@ setLocalFileDataData(tmp); } } - + /** * Encryption algorithm. * @@ -218,7 +218,7 @@ static { final Map cte = new HashMap<>(); for (final EncryptionAlgorithm method : values()) { - cte.put(Integer.valueOf(method.getCode()), method); + cte.put(method.getCode(), method); } codeToEnum = Collections.unmodifiableMap(cte); } @@ -247,7 +247,7 @@ * if the method is not known */ public static EncryptionAlgorithm getAlgorithmByCode(final int code) { - return codeToEnum.get(Integer.valueOf(code)); + return codeToEnum.get(code); } } @@ -273,7 +273,7 @@ static { final Map cte = new HashMap<>(); for (final HashAlgorithm method : values()) { - cte.put(Integer.valueOf(method.getCode()), method); + cte.put(method.getCode(), method); } codeToEnum = Collections.unmodifiableMap(cte); } @@ -302,7 +302,7 @@ * if the method is not known */ public static HashAlgorithm getAlgorithmByCode(final int code) { - return codeToEnum.get(Integer.valueOf(code)); + return codeToEnum.get(code); } } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ResourceAlignmentExtraField.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ResourceAlignmentExtraField.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ResourceAlignmentExtraField.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ResourceAlignmentExtraField.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.zip; + + +import java.util.zip.ZipException; + +/** + * An extra field who's sole purpose is to align and pad the local file header + * so that the entry's data starts at a certain position. + * + *

    The padding content of the padding is ignored and not retained + * when reading a padding field.

    + * + *

    This enables Commons Compress to create "aligned" archives + * similar to Android's zipalign command line tool.

    + * + * @since 1.14 + * @see "https://developer.android.com/studio/command-line/zipalign.html" + * @see ZipArchiveEntry#setAlignment + */ +public class ResourceAlignmentExtraField implements ZipExtraField { + + /** + * Extra field id used for storing alignment and padding. + */ + public static final ZipShort ID = new ZipShort(0xa11e); + + public static final int BASE_SIZE = 2; + + private static final int ALLOW_METHOD_MESSAGE_CHANGE_FLAG = 0x8000; + + private short alignment; + + private boolean allowMethodChange; + + private int padding = 0; + + public ResourceAlignmentExtraField() { + } + + public ResourceAlignmentExtraField(int alignment) { + this(alignment, false); + } + + public ResourceAlignmentExtraField(int alignment, boolean allowMethodChange) { + this(alignment, allowMethodChange, 0); + } + + public ResourceAlignmentExtraField(int alignment, boolean allowMethodChange, int padding) { + if (alignment < 0 || alignment > 0x7fff) { + throw new IllegalArgumentException("Alignment must be between 0 and 0x7fff, was: " + alignment); + } + this.alignment = (short) alignment; + this.allowMethodChange = allowMethodChange; + this.padding = padding; + } + + /** + * Gets requested alignment. + * + * @return + * requested alignment. + */ + public short getAlignment() { + return alignment; + } + + /** + * Indicates whether method change is allowed when re-compressing the zip file. + * + * @return + * true if method change is allowed, false otherwise. + */ + public boolean allowMethodChange() { + return allowMethodChange; + } + + @Override + public ZipShort getHeaderId() { + return ID; + } + + @Override + public ZipShort getLocalFileDataLength() { + return new ZipShort(BASE_SIZE + padding); + } + + @Override + public ZipShort getCentralDirectoryLength() { + return new ZipShort(BASE_SIZE); + } + + @Override + public byte[] getLocalFileDataData() { + byte[] content = new byte[BASE_SIZE + padding]; + ZipShort.putShort(alignment | (allowMethodChange ? ALLOW_METHOD_MESSAGE_CHANGE_FLAG : 0), + content, 0); + return content; + } + + @Override + public byte[] getCentralDirectoryData() { + return ZipShort.getBytes(alignment | (allowMethodChange ? ALLOW_METHOD_MESSAGE_CHANGE_FLAG : 0)); + } + + @Override + public void parseFromLocalFileData(byte[] buffer, int offset, int length) throws ZipException { + parseFromCentralDirectoryData(buffer, offset, length); + this.padding = length - BASE_SIZE; + } + + @Override + public void parseFromCentralDirectoryData(byte[] buffer, int offset, int length) throws ZipException { + if (length < BASE_SIZE) { + throw new ZipException("Too short content for ResourceAlignmentExtraField (0xa11e): " + length); + } + int alignmentValue = ZipShort.getValue(buffer, offset); + this.alignment = (short) (alignmentValue & (ALLOW_METHOD_MESSAGE_CHANGE_FLAG - 1)); + this.allowMethodChange = (alignmentValue & ALLOW_METHOD_MESSAGE_CHANGE_FLAG) != 0; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -124,8 +124,11 @@ */ @Override public void close() throws IOException { - backingStore.close(); - streamCompressor.close(); + try { + backingStore.close(); + } finally { + streamCompressor.close(); + } } /** diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/Simple8BitZipEncoding.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/Simple8BitZipEncoding.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/Simple8BitZipEncoding.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/Simple8BitZipEncoding.java 1970-01-01 00:00:00.000000000 +0000 @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.commons.compress.archivers.zip; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * This ZipEncoding implementation implements a simple 8bit character - * set, which mets the following restrictions: - * - *
      - *
    • Characters 0x0000 to 0x007f are encoded as the corresponding - * byte values 0x00 to 0x7f.
      • - *
    • All byte codes from 0x80 to 0xff are mapped to a unique unicode - * character in the range 0x0080 to 0x7fff. (No support for - * UTF-16 surrogates) - *
    - * - *

    These restrictions most notably apply to the most prominent - * omissions of java-1.4's {@link java.nio.charset.Charset Charset} - * implementation, Cp437 and Cp850.

    - * - *

    The methods of this class are reentrant.

    - * @Immutable - */ -class Simple8BitZipEncoding implements ZipEncoding { - - /** - * A character entity, which is put to the reverse mapping table - * of a simple encoding. - */ - private static final class Simple8BitChar implements Comparable { - public final char unicode; - public final byte code; - - Simple8BitChar(final byte code, final char unicode) { - this.code = code; - this.unicode = unicode; - } - - @Override - public int compareTo(final Simple8BitChar a) { - return this.unicode - a.unicode; - } - - @Override - public String toString() { - return "0x" + Integer.toHexString(0xffff & unicode) - + "->0x" + Integer.toHexString(0xff & code); - } - - @Override - public boolean equals(final Object o) { - if (o instanceof Simple8BitChar) { - final Simple8BitChar other = (Simple8BitChar) o; - return unicode == other.unicode && code == other.code; - } - return false; - } - - @Override - public int hashCode() { - return unicode; - } - } - - /** - * The characters for byte values of 128 to 255 stored as an array of - * 128 chars. - */ - private final char[] highChars; - - /** - * A list of {@link Simple8BitChar} objects sorted by the unicode - * field. This list is used to binary search reverse mapping of - * unicode characters with a character code greater than 127. - */ - private final List reverseMapping; - - /** - * @param highChars The characters for byte values of 128 to 255 - * stored as an array of 128 chars. - */ - public Simple8BitZipEncoding(final char[] highChars) { - this.highChars = highChars.clone(); - final List temp = - new ArrayList<>(this.highChars.length); - - byte code = 127; - - for (final char highChar : this.highChars) { - temp.add(new Simple8BitChar(++code, highChar)); - } - - Collections.sort(temp); - this.reverseMapping = Collections.unmodifiableList(temp); - } - - /** - * Return the character code for a given encoded byte. - * - * @param b The byte to decode. - * @return The associated character value. - */ - public char decodeByte(final byte b) { - // code 0-127 - if (b >= 0) { - return (char) b; - } - - // byte is signed, so 128 == -128 and 255 == -1 - return this.highChars[128 + b]; - } - - /** - * @param c The character to encode. - * @return Whether the given unicode character is covered by this encoding. - */ - public boolean canEncodeChar(final char c) { - - if (c >= 0 && c < 128) { - return true; - } - - final Simple8BitChar r = this.encodeHighChar(c); - return r != null; - } - - /** - * Pushes the encoded form of the given character to the given byte buffer. - * - * @param bb The byte buffer to write to. - * @param c The character to encode. - * @return Whether the given unicode character is covered by this encoding. - * If {@code false} is returned, nothing is pushed to the - * byte buffer. - */ - public boolean pushEncodedChar(final ByteBuffer bb, final char c) { - - if (c >= 0 && c < 128) { - bb.put((byte) c); - return true; - } - - final Simple8BitChar r = this.encodeHighChar(c); - if (r == null) { - return false; - } - bb.put(r.code); - return true; - } - - /** - * @param c A unicode character in the range from 0x0080 to 0x7f00 - * @return A Simple8BitChar, if this character is covered by this encoding. - * A {@code null} value is returned, if this character is not - * covered by this encoding. - */ - private Simple8BitChar encodeHighChar(final char c) { - // for performance an simplicity, yet another reincarnation of - // binary search... - int i0 = 0; - int i1 = this.reverseMapping.size(); - - while (i1 > i0) { - - final int i = i0 + (i1 - i0) / 2; - - final Simple8BitChar m = this.reverseMapping.get(i); - - if (m.unicode == c) { - return m; - } - - if (m.unicode < c) { - i0 = i + 1; - } else { - i1 = i; - } - } - - if (i0 >= this.reverseMapping.size()) { - return null; - } - - final Simple8BitChar r = this.reverseMapping.get(i0); - - if (r.unicode != c) { - return null; - } - - return r; - } - - /** - * @see - * org.apache.commons.compress.archivers.zip.ZipEncoding#canEncode(java.lang.String) - */ - @Override - public boolean canEncode(final String name) { - - for (int i=0;iStores the UTF-8 version of the file comment as stored in the * central directory header.

    * - * @see PKWARE + * @see PKWARE * APPNOTE.TXT, section 4.6.8 * * @NotThreadSafe super-class is not thread-safe @@ -33,13 +33,13 @@ public static final ZipShort UCOM_ID = new ZipShort(0x6375); - public UnicodeCommentExtraField () { + public UnicodeCommentExtraField () { } /** * Assemble as unicode comment extension from the name given as * text as well as the encoded bytes actually written to the archive. - * + * * @param text The file name * @param bytes the bytes actually written to the archive * @param off The offset of the encoded comment in bytes. @@ -54,7 +54,7 @@ /** * Assemble as unicode comment extension from the comment given as * text as well as the bytes actually written to the archive. - * + * * @param comment The file comment * @param bytes the bytes actually written to the archive */ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnicodePathExtraField.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnicodePathExtraField.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnicodePathExtraField.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnicodePathExtraField.java 2018-05-23 12:50:54.000000000 +0000 @@ -21,10 +21,10 @@ /** * Info-ZIP Unicode Path Extra Field (0x7075): * - *

    Stores the UTF-8 version of the file name field as stored in the + *

    Stores the UTF-8 version of the file name field as stored in the * local header and central directory header.

    * - * @see PKWARE + * @see PKWARE * APPNOTE.TXT, section 4.6.9 * * @NotThreadSafe super-class is not thread-safe @@ -33,13 +33,13 @@ public static final ZipShort UPATH_ID = new ZipShort(0x7075); - public UnicodePathExtraField () { + public UnicodePathExtraField () { } /** * Assemble as unicode path extension from the name given as * text as well as the encoded bytes actually written to the archive. - * + * * @param text The file name * @param bytes the bytes actually written to the archive * @param off The offset of the encoded filename in bytes. @@ -53,7 +53,7 @@ /** * Assemble as unicode path extension from the name given as * text as well as the encoded bytes actually written to the archive. - * + * * @param name The file name * @param bytes the bytes actually written to the archive */ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnixStat.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnixStat.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnixStat.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnixStat.java 2018-05-02 20:17:13.000000000 +0000 @@ -29,6 +29,11 @@ */ int PERM_MASK = 07777; /** + * Bits used to indicate the filesystem object type. + * @since 1.14 + */ + int FILE_TYPE_FLAG = 0170000; + /** * Indicates symbolic links. */ int LINK_FLAG = 0120000; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnparseableExtraFieldData.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnparseableExtraFieldData.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnparseableExtraFieldData.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnparseableExtraFieldData.java 2018-05-23 12:50:54.000000000 +0000 @@ -22,7 +22,7 @@ * Wrapper for extra field data that doesn't conform to the recommended format of header-tag + size + data. * *

    The header-id is artificial (and not listed as a known ID in APPNOTE.TXT). Since it isn't used anywhere + * href="https://www.pkware.com/documents/casestudies/APPNOTE.TXT">APPNOTE.TXT). Since it isn't used anywhere * except to satisfy the ZipExtraField contract it shouldn't matter anyway.

    * * @since 1.1 diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnshrinkingInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnshrinkingInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnshrinkingInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnshrinkingInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -33,10 +33,10 @@ private static final int MAX_CODE_SIZE = 13; private static final int MAX_TABLE_SIZE = 1 << MAX_CODE_SIZE; private final boolean[] isUsed; - + /** * IOException is not actually thrown! - * + * * @param inputStream * @throws IOException IOException is not actually thrown! */ @@ -64,7 +64,7 @@ } return idx; } - + private void partialClear() { final boolean[] isParent = new boolean[MAX_TABLE_SIZE]; for (int i = 0; i < isUsed.length; i++) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnsupportedZipFeatureException.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnsupportedZipFeatureException.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/UnsupportedZipFeatureException.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/UnsupportedZipFeatureException.java 2018-05-02 20:17:13.000000000 +0000 @@ -93,6 +93,8 @@ * @since 1.1 */ public static class Feature implements java.io.Serializable { + + private static final long serialVersionUID = 4112582948775420359L; /** * The entry is encrypted. */ @@ -110,6 +112,13 @@ * @since 1.5 */ public static final Feature SPLITTING = new Feature("splitting"); + /** + * The archive contains entries with unknown compressed size + * for a compression method that doesn't support detection of + * the end of the compressed stream. + * @since 1.16 + */ + public static final Feature UNKNOWN_COMPRESSED_SIZE = new Feature("unknown compressed size"); private final String name; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X000A_NTFS.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X000A_NTFS.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X000A_NTFS.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X000A_NTFS.java 2018-05-02 20:17:13.000000000 +0000 @@ -382,12 +382,12 @@ private static ZipEightByteInteger dateToZip(final Date d) { if (d == null) { return null; } - return new ZipEightByteInteger((d.getTime() * 10000l) - EPOCH_OFFSET); + return new ZipEightByteInteger((d.getTime() * 10000L) - EPOCH_OFFSET); } private static Date zipToDate(final ZipEightByteInteger z) { if (z == null || ZipEightByteInteger.ZERO.equals(z)) { return null; } - final long l = (z.getLongValue() + EPOCH_OFFSET) / 10000l; + final long l = (z.getLongValue() + EPOCH_OFFSET) / 10000L; return new Date(l); } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X0017_StrongEncryptionHeader.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X0017_StrongEncryptionHeader.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X0017_StrongEncryptionHeader.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X0017_StrongEncryptionHeader.java 2018-05-02 20:17:13.000000000 +0000 @@ -310,7 +310,7 @@ this.hashAlg = HashAlgorithm.getAlgorithmByCode(ZipShort.getValue(data, offset + 12)); this.hashSize = ZipShort.getValue(data, offset + 14); // srlist... hashed public keys - for (int i = 0; i < this.rcount; i++) { + for (long i = 0; i < this.rcount; i++) { for (int j = 0; j < this.hashSize; j++) { // ZipUtil.signedByteToUnsignedInt(data[offset + 16 + (i * this.hashSize) + j])); } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X5455_ExtendedTimestamp.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X5455_ExtendedTimestamp.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X5455_ExtendedTimestamp.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X5455_ExtendedTimestamp.java 2018-05-02 20:17:13.000000000 +0000 @@ -25,14 +25,14 @@ /** *

    An extra field that stores additional file and directory timestamp data * for zip entries. Each zip entry can include up to three timestamps - * (modify, access, create*). The timestamps are stored as 32 bit unsigned + * (modify, access, create*). The timestamps are stored as 32 bit signed * integers representing seconds since UNIX epoch (Jan 1st, 1970, UTC). * This field improves on zip's default timestamp granularity, since it * allows one to store additional timestamps, and, in addition, the timestamps * are stored using per-second granularity (zip's default behaviour can only store * timestamps to the nearest even second). *

    - * Unfortunately, 32 (unsigned) bits can only store dates up to the year 2106, + * Unfortunately, 32 (signed) bits can only store dates up to the year 2037, * and so this extra field will eventually be obsolete. Enjoy it while it lasts! *

    *
      @@ -187,7 +187,7 @@ if (bit2_createTimePresent && createTime != null) { data[0] |= CREATE_TIME_BIT; System.arraycopy(createTime.getBytes(), 0, data, pos, 4); - pos += 4; + pos += 4; // NOSONAR - assignment as documentation } return data; } @@ -237,7 +237,7 @@ } if (bit2_createTimePresent && offset + 4 <= len) { createTime = new ZipLong(data, offset); - offset += 4; + offset += 4; // NOSONAR - assignment as documentation } } @@ -370,7 +370,7 @@ * @return modify time as java.util.Date or null. */ public Date getModifyJavaTime() { - return modifyTime != null ? new Date(modifyTime.getValue() * 1000) : null; + return zipLongToDate(modifyTime); } /** @@ -382,7 +382,7 @@ * @return access time as java.util.Date or null. */ public Date getAccessJavaTime() { - return accessTime != null ? new Date(accessTime.getValue() * 1000) : null; + return zipLongToDate(accessTime); } /** @@ -400,7 +400,7 @@ * @return create time as java.util.Date or null. */ public Date getCreateJavaTime() { - return createTime != null ? new Date(createTime.getValue() * 1000) : null; + return zipLongToDate(createTime); } /** @@ -518,12 +518,7 @@ private static ZipLong dateToZipLong(final Date d) { if (d == null) { return null; } - final long TWO_TO_32 = 0x100000000L; - final long l = d.getTime() / 1000; - if (l >= TWO_TO_32) { - throw new IllegalArgumentException("Cannot set an X5455 timestamp larger than 2^32: " + l); - } - return new ZipLong(l); + return unixTimeToZipLong(d.getTime() / 1000); } /** @@ -590,4 +585,15 @@ return hc; } + private static Date zipLongToDate(ZipLong unixTime) { + return unixTime != null ? new Date(unixTime.getIntValue() * 1000L) : null; + } + + private static ZipLong unixTimeToZipLong(long l) { + if (l < Integer.MIN_VALUE || l > Integer.MAX_VALUE) { + throw new IllegalArgumentException("X5455 timestamps must fit in a signed 32 bit integer: " + l); + } + return new ZipLong(l); + } + } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X7875_NewUnix.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X7875_NewUnix.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/X7875_NewUnix.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/X7875_NewUnix.java 2018-05-02 20:17:13.000000000 +0000 @@ -132,8 +132,10 @@ */ @Override public ZipShort getLocalFileDataLength() { - final int uidSize = trimLeadingZeroesForceMinLength(uid.toByteArray()).length; - final int gidSize = trimLeadingZeroesForceMinLength(gid.toByteArray()).length; + byte[] b = trimLeadingZeroesForceMinLength(uid.toByteArray()); + final int uidSize = b == null ? 0 : b.length; + b = trimLeadingZeroesForceMinLength(gid.toByteArray()); + final int gidSize = b == null ? 0 : b.length; // The 3 comes from: version=1 + uidsize=1 + gidsize=1 return new ZipShort(3 + uidSize + gidSize); @@ -165,26 +167,36 @@ // (e.g., so that the sign-bit is set to zero). We need to remove that // before sending the number over the wire. uidBytes = trimLeadingZeroesForceMinLength(uidBytes); + int uidBytesLen = uidBytes != null ? uidBytes.length : 0; gidBytes = trimLeadingZeroesForceMinLength(gidBytes); + int gidBytesLen = gidBytes != null ? gidBytes.length : 0; // Couldn't bring myself to just call getLocalFileDataLength() when we've // already got the arrays right here. Yeah, yeah, I know, premature // optimization is the root of all... // // The 3 comes from: version=1 + uidsize=1 + gidsize=1 - final byte[] data = new byte[3 + uidBytes.length + gidBytes.length]; + final byte[] data = new byte[3 + uidBytesLen + gidBytesLen]; // reverse() switches byte array from big-endian to little-endian. - reverse(uidBytes); - reverse(gidBytes); + if (uidBytes != null) { + reverse(uidBytes); + } + if (gidBytes != null) { + reverse(gidBytes); + } int pos = 0; data[pos++] = unsignedIntToSignedByte(version); - data[pos++] = unsignedIntToSignedByte(uidBytes.length); - System.arraycopy(uidBytes, 0, data, pos, uidBytes.length); - pos += uidBytes.length; - data[pos++] = unsignedIntToSignedByte(gidBytes.length); - System.arraycopy(gidBytes, 0, data, pos, gidBytes.length); + data[pos++] = unsignedIntToSignedByte(uidBytesLen); + if (uidBytes != null) { + System.arraycopy(uidBytes, 0, data, pos, uidBytesLen); + } + pos += uidBytesLen; + data[pos++] = unsignedIntToSignedByte(gidBytesLen); + if (gidBytes != null) { + System.arraycopy(gidBytes, 0, data, pos, gidBytesLen); + } return data; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/Zip64ExtendedInformationExtraField.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/Zip64ExtendedInformationExtraField.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/Zip64ExtendedInformationExtraField.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/Zip64ExtendedInformationExtraField.java 2018-05-23 12:50:54.000000000 +0000 @@ -34,7 +34,7 @@ *

      The implementation relies on data being read from the local file * header and assumes that both size values are always present.

      * - * @see PKWARE + * @see PKWARE * APPNOTE.TXT, section 4.5.3 * * @since 1.2 @@ -144,7 +144,7 @@ } if (diskStart != null) { System.arraycopy(diskStart.getBytes(), 0, data, off, WORD); - off += WORD; + off += WORD; // NOSONAR - assignment as documentation } return data; } @@ -174,8 +174,8 @@ } if (remaining >= WORD) { diskStart = new ZipLong(buffer, offset); - offset += WORD; - remaining -= WORD; + offset += WORD; // NOSONAR - assignment as documentation + remaining -= WORD; // NOSONAR - assignment as documentation } } @@ -256,7 +256,7 @@ } if (hasDiskStart) { diskStart = new ZipLong(rawCentralDirectoryData, offset); - offset += WORD; + offset += WORD; // NOSONAR - assignment as documentation } } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntry.java 2018-05-23 12:50:54.000000000 +0000 @@ -18,6 +18,7 @@ package org.apache.commons.compress.archivers.zip; import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.EntryStreamOffsets; import java.io.File; import java.util.ArrayList; @@ -48,7 +49,8 @@ * @NotThreadSafe */ public class ZipArchiveEntry extends java.util.zip.ZipEntry - implements ArchiveEntry { + implements ArchiveEntry, EntryStreamOffsets +{ public static final int PLATFORM_UNIX = 3; public static final int PLATFORM_FAT = 0; @@ -58,6 +60,50 @@ private static final byte[] EMPTY = new byte[0]; /** + * Indicates how the name of this entry has been determined. + * @since 1.16 + */ + public enum NameSource { + /** + * The name has been read from the archive using the encoding + * of the archive specified when creating the {@link + * ZipArchiveInputStream} or {@link ZipFile} (defaults to the + * platform's default encoding). + */ + NAME, + /** + * The name has been read from the archive and the archive + * specified the EFS flag which indicates the name has been + * encoded as UTF-8. + */ + NAME_WITH_EFS_FLAG, + /** + * The name has been read from an {@link UnicodePathExtraField + * Unicode Extra Field}. + */ + UNICODE_EXTRA_FIELD + } + + /** + * Indicates how the comment of this entry has been determined. + * @since 1.16 + */ + public enum CommentSource { + /** + * The comment has been read from the archive using the encoding + * of the archive specified when creating the {@link + * ZipArchiveInputStream} or {@link ZipFile} (defaults to the + * platform's default encoding). + */ + COMMENT, + /** + * The comment has been read from an {@link UnicodeCommentExtraField + * Unicode Extra Field}. + */ + UNICODE_EXTRA_FIELD + } + + /** * The {@link java.util.zip.ZipEntry} base class only supports * the compression methods STORED and DEFLATED. We override the * field so that any compression methods can be used. @@ -72,8 +118,11 @@ /** * The {@link java.util.zip.ZipEntry#setSize} method in the base * class throws an IllegalArgumentException if the size is bigger - * than 2GB for Java versions < 7. Need to keep our own size - * information for Zip64 support. + * than 2GB for Java versions < 7 and even in Java 7+ if the + * implementation in java.util.zip doesn't support Zip64 itself + * (it is an optional feature). + * + *

      We need to keep our own size information for Zip64 support.

      */ private long size = SIZE_UNKNOWN; @@ -83,12 +132,19 @@ private int platform = PLATFORM_FAT; private int rawFlag; private long externalAttributes = 0; + private int alignment = 0; private ZipExtraField[] extraFields; private UnparseableExtraFieldData unparseableExtra = null; private String name = null; private byte[] rawName = null; private GeneralPurposeBit gpb = new GeneralPurposeBit(); private static final ZipExtraField[] noExtraFields = new ZipExtraField[0]; + private long localHeaderOffset = OFFSET_UNKNOWN; + private long dataOffset = OFFSET_UNKNOWN; + private boolean isStreamContiguous = false; + private NameSource nameSource = NameSource.NAME; + private CommentSource commentSource = CommentSource.COMMENT; + /** * Creates a new zip entry with the specified name. @@ -166,7 +222,7 @@ * @param entryName name of the entry */ public ZipArchiveEntry(final File inputFile, final String entryName) { - this(inputFile.isDirectory() && !entryName.endsWith("/") ? + this(inputFile.isDirectory() && !entryName.endsWith("/") ? entryName + "/" : entryName); if (inputFile.isFile()){ setSize(inputFile.length()); @@ -294,7 +350,7 @@ * @return true if the entry represents a unix symlink, false otherwise. */ public boolean isUnixSymlink() { - return (getUnixMode() & UnixStat.LINK_FLAG) == UnixStat.LINK_FLAG; + return (getUnixMode() & UnixStat.FILE_TYPE_FLAG) == UnixStat.LINK_FLAG; } /** @@ -317,6 +373,32 @@ } /** + * Gets currently configured alignment. + * + * @return + * alignment for this entry. + * @since 1.14 + */ + protected int getAlignment() { + return this.alignment; + } + + /** + * Sets alignment for this entry. + * + * @param alignment + * requested alignment, 0 for default. + * @since 1.14 + */ + public void setAlignment(int alignment) { + if ((alignment & (alignment - 1)) != 0 || alignment > 0xffff) { + throw new IllegalArgumentException("Invalid value for alignment, must be power of two and no bigger than " + + 0xffff + " but is " + alignment); + } + this.alignment = alignment; + } + + /** * Replaces all currently attached extra fields with the new array. * @param fields an array of extra fields */ @@ -589,6 +671,9 @@ /** * Get the name of the entry. + * + *

      This method returns the raw name as it is stored inside of the archive.

      + * * @return the entry name */ @Override @@ -678,6 +763,38 @@ return null; } + protected long getLocalHeaderOffset() { + return this.localHeaderOffset; + } + + protected void setLocalHeaderOffset(long localHeaderOffset) { + this.localHeaderOffset = localHeaderOffset; + } + + @Override + public long getDataOffset() { + return dataOffset; + } + + /** + * Sets the data offset. + * + * @param dataOffset + * new value of data offset. + */ + protected void setDataOffset(long dataOffset) { + this.dataOffset = dataOffset; + } + + @Override + public boolean isStreamContiguous() { + return isStreamContiguous; + } + + protected void setStreamContiguous(boolean isStreamContiguous) { + this.isStreamContiguous = isStreamContiguous; + } + /** * Get the hashCode of the entry. * This uses the name as the hashcode. @@ -801,6 +918,8 @@ other.getCentralDirectoryExtra()) && Arrays.equals(getLocalFileDataExtra(), other.getLocalFileDataExtra()) + && localHeaderOffset == other.localHeaderOffset + && dataOffset == other.dataOffset && gpb.equals(other.gpb); } @@ -857,4 +976,41 @@ public void setRawFlag(final int rawFlag) { this.rawFlag = rawFlag; } + + /** + * The source of the name field value. + * @return source of the name field value + * @since 1.16 + */ + public NameSource getNameSource() { + return nameSource; + } + + /** + * Sets the source of the name field value. + * @param nameSource source of the name field value + * @since 1.16 + */ + public void setNameSource(NameSource nameSource) { + this.nameSource = nameSource; + } + + /** + * The source of the comment field value. + * @return source of the comment field value + * @since 1.16 + */ + public CommentSource getCommentSource() { + return commentSource; + } + + /** + * Sets the source of the comment field value. + * @param commentSource source of the comment field value + * @since 1.16 + */ + public void setCommentSource(CommentSource commentSource) { + this.commentSource = commentSource; + } + } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java 2018-08-09 18:37:01.000000000 +0000 @@ -24,7 +24,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.PushbackInputStream; +import java.math.BigInteger; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.zip.CRC32; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -34,8 +36,10 @@ import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream; import org.apache.commons.compress.utils.ArchiveUtils; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; import static org.apache.commons.compress.archivers.zip.ZipConstants.DWORD; import static org.apache.commons.compress.archivers.zip.ZipConstants.SHORT; @@ -74,7 +78,7 @@ * @see ZipFile * @NotThreadSafe */ -public class ZipArchiveInputStream extends ArchiveInputStream { +public class ZipArchiveInputStream extends ArchiveInputStream implements InputStreamStatistics { /** The zip encoding to use for filenames and the file comment. */ private final ZipEncoding zipEncoding; @@ -113,6 +117,9 @@ /** Whether the stream will try to read STORED entries that use a data descriptor. */ private boolean allowStoredEntriesWithDataDescriptor = false; + /** Count decompressed bytes for current entry */ + private long uncompressedCount = 0; + private static final int LFH_LEN = 30; /* local file header signature WORD @@ -217,6 +224,8 @@ } public ZipArchiveEntry getNextZipEntry() throws IOException { + uncompressedCount = 0; + boolean firstEntry = true; if (closed || hitCentralDirectory) { return null; @@ -226,6 +235,7 @@ firstEntry = false; } + long currentHeaderOffset = getBytesRead(); try { if (firstEntry) { // split archives have a special signature before the @@ -241,12 +251,12 @@ } final ZipLong sig = new ZipLong(lfhBuf); - if (sig.equals(ZipLong.CFH_SIG) || sig.equals(ZipLong.AED_SIG)) { - hitCentralDirectory = true; - skipRemainderOfArchive(); - return null; - } if (!sig.equals(ZipLong.LFH_SIG)) { + if (sig.equals(ZipLong.CFH_SIG) || sig.equals(ZipLong.AED_SIG) || isApkSigningBlock(lfhBuf)) { + hitCentralDirectory = true; + skipRemainderOfArchive(); + return null; + } throw new ZipException(String.format("Unexpected record signature: 0X%X", sig.getValue())); } @@ -291,11 +301,14 @@ off += SHORT; final int extraLen = ZipShort.getValue(lfhBuf, off); - off += SHORT; + off += SHORT; // NOSONAR - assignment as documentation final byte[] fileName = new byte[fileNameLen]; readFully(fileName); current.entry.setName(entryEncoding.decode(fileName), fileName); + if (hasUTF8Flag) { + current.entry.setNameSource(ZipArchiveEntry.NameSource.NAME_WITH_EFS_FLAG); + } final byte[] extraData = new byte[extraLen]; readFully(extraData); @@ -307,19 +320,41 @@ processZip64Extra(size, cSize); + current.entry.setLocalHeaderOffset(currentHeaderOffset); + current.entry.setDataOffset(getBytesRead()); + current.entry.setStreamContiguous(true); + + ZipMethod m = ZipMethod.getMethodByCode(current.entry.getMethod()); if (current.entry.getCompressedSize() != ArchiveEntry.SIZE_UNKNOWN) { - if (current.entry.getMethod() == ZipMethod.UNSHRINKING.getCode()) { - current.in = new UnshrinkingInputStream(new BoundedInputStream(in, current.entry.getCompressedSize())); - } else if (current.entry.getMethod() == ZipMethod.IMPLODING.getCode()) { - current.in = new ExplodingInputStream( + if (ZipUtil.canHandleEntryData(current.entry) && m != ZipMethod.STORED && m != ZipMethod.DEFLATED) { + InputStream bis = new BoundedInputStream(in, current.entry.getCompressedSize()); + switch (m) { + case UNSHRINKING: + current.in = new UnshrinkingInputStream(bis); + break; + case IMPLODING: + current.in = new ExplodingInputStream( current.entry.getGeneralPurposeBit().getSlidingDictionarySize(), current.entry.getGeneralPurposeBit().getNumberOfShannonFanoTrees(), - new BoundedInputStream(in, current.entry.getCompressedSize())); - } else if (current.entry.getMethod() == ZipMethod.BZIP2.getCode()) { - current.in = new BZip2CompressorInputStream(new BoundedInputStream(in, current.entry.getCompressedSize())); + bis); + break; + case BZIP2: + current.in = new BZip2CompressorInputStream(bis); + break; + case ENHANCED_DEFLATED: + current.in = new Deflate64CompressorInputStream(bis); + break; + default: + // we should never get here as all supported methods have been covered + // will cause an error when read is invoked, don't throw an exception here so people can + // skip unsupported entries + break; + } } + } else if (m == ZipMethod.ENHANCED_DEFLATED) { + current.in = new Deflate64CompressorInputStream(in); } - + entriesRead++; return current.entry; } @@ -353,7 +388,7 @@ */ private void processZip64Extra(final ZipLong size, final ZipLong cSize) { final Zip64ExtendedInformationExtraField z64 = - (Zip64ExtendedInformationExtraField) + (Zip64ExtendedInformationExtraField) current.entry.getExtraField(Zip64ExtendedInformationExtraField.HEADER_ID); current.usesZip64 = z64 != null; if (!current.hasDataDescriptor) { @@ -385,8 +420,8 @@ if (ae instanceof ZipArchiveEntry) { final ZipArchiveEntry ze = (ZipArchiveEntry) ae; return ZipUtil.canHandleEntryData(ze) - && supportsDataDescriptorFor(ze); - + && supportsDataDescriptorFor(ze) + && supportsCompressedSizeFor(ze); } return false; } @@ -405,12 +440,16 @@ if (offset > buffer.length || length < 0 || offset < 0 || buffer.length - offset < length) { throw new ArrayIndexOutOfBoundsException(); } - + ZipUtil.checkRequestedFeatures(current.entry); if (!supportsDataDescriptorFor(current.entry)) { throw new UnsupportedZipFeatureException(UnsupportedZipFeatureException.Feature.DATA_DESCRIPTOR, current.entry); } + if (!supportsCompressedSizeFor(current.entry)) { + throw new UnsupportedZipFeatureException(UnsupportedZipFeatureException.Feature.UNKNOWN_COMPRESSED_SIZE, + current.entry); + } int read; if (current.entry.getMethod() == ZipArchiveOutputStream.STORED) { @@ -419,21 +458,53 @@ read = readDeflated(buffer, offset, length); } else if (current.entry.getMethod() == ZipMethod.UNSHRINKING.getCode() || current.entry.getMethod() == ZipMethod.IMPLODING.getCode() + || current.entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode() || current.entry.getMethod() == ZipMethod.BZIP2.getCode()) { read = current.in.read(buffer, offset, length); } else { throw new UnsupportedZipFeatureException(ZipMethod.getMethodByCode(current.entry.getMethod()), current.entry); } - + if (read >= 0) { current.crc.update(buffer, offset, read); + uncompressedCount += read; } - + return read; } /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + if (current.entry.getMethod() == ZipArchiveOutputStream.STORED) { + return current.bytesRead; + } else if (current.entry.getMethod() == ZipArchiveOutputStream.DEFLATED) { + return getBytesInflated(); + } else if (current.entry.getMethod() == ZipMethod.UNSHRINKING.getCode()) { + return ((UnshrinkingInputStream) current.in).getCompressedCount(); + } else if (current.entry.getMethod() == ZipMethod.IMPLODING.getCode()) { + return ((ExplodingInputStream) current.in).getCompressedCount(); + } else if (current.entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode()) { + return ((Deflate64CompressorInputStream) current.in).getCompressedCount(); + } else if (current.entry.getMethod() == ZipMethod.BZIP2.getCode()) { + return ((BZip2CompressorInputStream) current.in).getCompressedCount(); + } else { + return -1; + } + } + + /** + * @since 1.17 + */ + @Override + public long getUncompressedCount() { + return uncompressedCount; + } + + /** * Implementation of read for STORED entries. */ private int readStored(final byte[] buffer, final int offset, final int length) throws IOException { @@ -454,7 +525,8 @@ buf.position(0); final int l = in.read(buf.array()); if (l == -1) { - return -1; + buf.limit(0); + throw new IOException("Truncated ZIP file"); } buf.limit(l); @@ -720,9 +792,14 @@ } private void readFully(final byte[] b) throws IOException { - final int count = IOUtils.readFully(in, b); + readFully(b, 0); + } + + private void readFully(final byte[] b, final int off) throws IOException { + final int len = b.length - off; + final int count = IOUtils.readFully(in, b, off, len); count(count); - if (count < b.length) { + if (count < len) { throw new EOFException(); } } @@ -741,7 +818,7 @@ // each, otherwise four bytes each. Unfortunately some // implementations - namely Java7 - use eight bytes without // using a ZIP64 extra field - - // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588 + // https://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588 // just read 16 bytes and check whether bytes nine to twelve // look like one of the signatures of what could follow a data @@ -765,13 +842,27 @@ * * @return true if allowStoredEntriesWithDataDescriptor is true, * the entry doesn't require any data descriptor or the method is - * DEFLATED. + * DEFLATED or ENHANCED_DEFLATED. */ private boolean supportsDataDescriptorFor(final ZipArchiveEntry entry) { return !entry.getGeneralPurposeBit().usesDataDescriptor() || (allowStoredEntriesWithDataDescriptor && entry.getMethod() == ZipEntry.STORED) - || entry.getMethod() == ZipEntry.DEFLATED; + || entry.getMethod() == ZipEntry.DEFLATED + || entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode(); + } + + /** + * Whether the compressed size for the entry is either known or + * not required by the compression method being used. + */ + private boolean supportsCompressedSizeFor(final ZipArchiveEntry entry) { + return entry.getCompressedSize() != ArchiveEntry.SIZE_UNKNOWN + || entry.getMethod() == ZipEntry.DEFLATED + || entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode() + || (entry.getGeneralPurposeBit().usesDataDescriptor() + && allowStoredEntriesWithDataDescriptor + && entry.getMethod() == ZipEntry.STORED); } /** @@ -839,7 +930,7 @@ boolean done = false; int readTooMuch = 0; - for (int i = 0; !done && i < lastRead - 4; i++) { + for (int i = 0; !done && i < offset + lastRead - 4; i++) { if (buf.array()[i] == LFH[0] && buf.array()[i + 1] == LFH[1]) { if ((buf.array()[i + 2] == LFH[2] && buf.array()[i + 3] == LFH[3]) || (buf.array()[i] == CFH[2] && buf.array()[i + 3] == CFH[3])) { @@ -1004,6 +1095,62 @@ return b == ZipArchiveOutputStream.EOCD_SIG[0]; } + private static final byte[] APK_SIGNING_BLOCK_MAGIC = new byte[] { + 'A', 'P', 'K', ' ', 'S', 'i', 'g', ' ', 'B', 'l', 'o', 'c', 'k', ' ', '4', '2', + }; + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); + + /** + * Checks whether this might be an APK Signing Block. + * + *

      Unfortunately the APK signing block does not start with some kind of signature, it rather ends with one. It + * starts with a length, so what we do is parse the suspect length, skip ahead far enough, look for the signature + * and if we've found it, return true.

      + * + * @param suspectLocalFileHeader the bytes read from the underlying stream in the expectation that they would hold + * the local file header of the next entry. + * + * @return true if this looks like a APK signing block + * + * @see https://source.android.com/security/apksigning/v2 + */ + private boolean isApkSigningBlock(byte[] suspectLocalFileHeader) throws IOException { + // length of block excluding the size field itself + BigInteger len = ZipEightByteInteger.getValue(suspectLocalFileHeader); + // LFH has already been read and all but the first eight bytes contain (part of) the APK signing block, + // also subtract 16 bytes in order to position us at the magic string + BigInteger toSkip = len.add(BigInteger.valueOf(DWORD - suspectLocalFileHeader.length + - APK_SIGNING_BLOCK_MAGIC.length)); + byte[] magic = new byte[APK_SIGNING_BLOCK_MAGIC.length]; + + try { + if (toSkip.signum() < 0) { + // suspectLocalFileHeader contains the start of suspect magic string + int off = suspectLocalFileHeader.length + toSkip.intValue(); + // length was shorter than magic length + if (off < DWORD) { + return false; + } + int bytesInBuffer = Math.abs(toSkip.intValue()); + System.arraycopy(suspectLocalFileHeader, off, magic, 0, Math.min(bytesInBuffer, magic.length)); + if (bytesInBuffer < magic.length) { + readFully(magic, bytesInBuffer); + } + } else { + while (toSkip.compareTo(LONG_MAX) > 0) { + realSkip(Long.MAX_VALUE); + toSkip = toSkip.add(LONG_MAX.negate()); + } + realSkip(toSkip.longValue()); + readFully(magic); + } + } catch (EOFException ex) { + // length was invalid + return false; + } + return Arrays.equals(magic, APK_SIGNING_BLOCK_MAGIC); + } + /** * Structure collecting information for the entry that is * currently being read. @@ -1032,7 +1179,7 @@ private long bytesRead; /** - * Number of bytes of entry content read so from the stream. + * Number of bytes of entry content read from the stream. * *

      This may be more than the actual entry's length as some * stuff gets buffered up and needs to be pushed back when the @@ -1064,7 +1211,7 @@ /** the number of bytes already returned */ private long pos = 0; - + /** * Creates a new BoundedInputStream that wraps the given input * stream and limits it to a certain size. @@ -1115,11 +1262,11 @@ @Override public long skip(final long n) throws IOException { final long toSkip = max >= 0 ? Math.min(n, max - pos) : n; - final long skippedBytes = in.skip(toSkip); + final long skippedBytes = IOUtils.skip(in, toSkip); pos += skippedBytes; return skippedBytes; } - + @Override public int available() throws IOException { if (max >= 0 && pos >= max) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -41,6 +41,7 @@ import org.apache.commons.compress.utils.IOUtils; import static org.apache.commons.compress.archivers.zip.ZipConstants.DATA_DESCRIPTOR_MIN_VERSION; +import static org.apache.commons.compress.archivers.zip.ZipConstants.DEFLATE_MIN_VERSION; import static org.apache.commons.compress.archivers.zip.ZipConstants.DWORD; import static org.apache.commons.compress.archivers.zip.ZipConstants.INITIAL_VERSION; import static org.apache.commons.compress.archivers.zip.ZipConstants.SHORT; @@ -201,9 +202,9 @@ private static final byte[] ONE = ZipLong.getBytes(1L); /** - * Holds the offsets of the LFH starts for each entry. + * Holds some book-keeping data for each entry. */ - private final Map offsets = + private final Map metaData = new HashMap<>(); /** @@ -241,7 +242,7 @@ * whether to use the general purpose bit flag when writing UTF-8 * filenames or not. */ - private boolean useUTF8Flag = true; + private boolean useUTF8Flag = true; /** * Whether to encode non-encodable file names as UTF-8. @@ -427,7 +428,7 @@ * this mode is not valid when the output stream is not seekable * and the uncompressed size is unknown when {@link * #putArchiveEntry} is called.

      - * + * *

      If no entry inside the resulting archive requires Zip64 * extensions then {@link Zip64Mode#Never Never} will create the * smallest archive. {@link Zip64Mode#AsNeeded AsNeeded} will @@ -472,7 +473,7 @@ cdLength = streamCompressor.getTotalBytesWritten() - cdOffset; writeZip64CentralDirectory(); writeCentralDirectoryEnd(); - offsets.clear(); + metaData.clear(); entries.clear(); streamCompressor.close(); finished = true; @@ -497,7 +498,7 @@ * Writes all necessary data for this entry. * @throws IOException on error * @throws Zip64RequiredException if the entry's uncompressed or - * compressed size exceeds 4 GByte and {@link #setUseZip64} + * compressed size exceeds 4 GByte and {@link #setUseZip64} * is {@link Zip64Mode#Never}. */ @Override @@ -538,7 +539,9 @@ rewriteSizesAndCrc(actuallyNeedsZip64); } - writeDataDescriptor(entry.entry); + if (!phased) { + writeDataDescriptor(entry.entry); + } entry = null; } @@ -639,10 +642,9 @@ } /** - * Ensures the current entry's size and CRC information is set to - * the values just written, verifies it isn't too big in the - * Zip64Mode.Never case and returns whether the entry would - * require a Zip64 extra field. + * Verifies the sizes aren't too big in the Zip64Mode.Never case + * and returns whether the entry would require a Zip64 extra + * field. */ private boolean checkIfNeedsZip64(final Zip64Mode effectiveMode) throws ZipException { @@ -695,7 +697,7 @@ // do some cleanup: // * rewrite version needed to extract channel.position(entry.localDataStart - 5 * SHORT); - writeOut(ZipShort.getBytes(INITIAL_VERSION)); + writeOut(ZipShort.getBytes(versionNeededToExtract(entry.entry.getMethod(), false, false))); // * remove ZIP64 extra so it doesn't get written // to the central directory @@ -714,10 +716,10 @@ } /** - * {@inheritDoc} + * {@inheritDoc} * @throws ClassCastException if entry is not an instance of ZipArchiveEntry * @throws Zip64RequiredException if the entry's uncompressed or - * compressed size is known to exceed 4 GByte and {@link #setUseZip64} + * compressed size is known to exceed 4 GByte and {@link #setUseZip64} * is {@link Zip64Mode#Never}. */ @Override @@ -758,18 +760,20 @@ final Zip64ExtendedInformationExtraField z64 = getZip64Extra(entry.entry); - // just a placeholder, real data will be in data - // descriptor or inserted later via SeekableByteChannel - ZipEightByteInteger size = ZipEightByteInteger.ZERO; - ZipEightByteInteger compressedSize = ZipEightByteInteger.ZERO; - if (phased){ + ZipEightByteInteger size; + ZipEightByteInteger compressedSize; + if (phased) { + // sizes are already known size = new ZipEightByteInteger(entry.entry.getSize()); compressedSize = new ZipEightByteInteger(entry.entry.getCompressedSize()); } else if (entry.entry.getMethod() == STORED && entry.entry.getSize() != ArchiveEntry.SIZE_UNKNOWN) { // actually, we already know the sizes - size = new ZipEightByteInteger(entry.entry.getSize()); - compressedSize = size; + compressedSize = size = new ZipEightByteInteger(entry.entry.getSize()); + } else { + // just a placeholder, real data will be in data + // descriptor or inserted later via SeekableByteChannel + compressedSize = size = ZipEightByteInteger.ZERO; } z64.setSize(size); z64.setCompressedSize(compressedSize); @@ -954,10 +958,13 @@ */ @Override public void close() throws IOException { - if (!finished) { - finish(); + try { + if (!finished) { + finish(); + } + } finally { + destroy(); } - destroy(); } /** @@ -1026,9 +1033,9 @@ addUnicodeExtraFields(ze, encodable, name); } - final byte[] localHeader = createLocalFileHeader(ze, name, encodable, phased); final long localHeaderStart = streamCompressor.getTotalBytesWritten(); - offsets.put(ze, localHeaderStart); + final byte[] localHeader = createLocalFileHeader(ze, name, encodable, phased, localHeaderStart); + metaData.put(ze, new EntryMetaData(localHeaderStart, usesDataDescriptor(ze.getMethod(), phased))); entry.localDataStart = localHeaderStart + LFH_CRC_OFFSET; // At crc offset writeCounted(localHeader); entry.dataStart = streamCompressor.getTotalBytesWritten(); @@ -1036,24 +1043,44 @@ private byte[] createLocalFileHeader(final ZipArchiveEntry ze, final ByteBuffer name, final boolean encodable, - final boolean phased) { + final boolean phased, long archiveOffset) { + ResourceAlignmentExtraField oldAlignmentEx = + (ResourceAlignmentExtraField) ze.getExtraField(ResourceAlignmentExtraField.ID); + if (oldAlignmentEx != null) { + ze.removeExtraField(ResourceAlignmentExtraField.ID); + } + + int alignment = ze.getAlignment(); + if (alignment <= 0 && oldAlignmentEx != null) { + alignment = oldAlignmentEx.getAlignment(); + } + + if (alignment > 1 || (oldAlignmentEx != null && !oldAlignmentEx.allowMethodChange())) { + int oldLength = LFH_FILENAME_OFFSET + + name.limit() - name.position() + + ze.getLocalFileDataExtra().length; + + int padding = (int) ((-archiveOffset - oldLength - ZipExtraField.EXTRAFIELD_HEADER_SIZE + - ResourceAlignmentExtraField.BASE_SIZE) & + (alignment - 1)); + ze.addExtraField(new ResourceAlignmentExtraField(alignment, + oldAlignmentEx != null && oldAlignmentEx.allowMethodChange(), padding)); + } + final byte[] extra = ze.getLocalFileDataExtra(); final int nameLen = name.limit() - name.position(); - final int len= LFH_FILENAME_OFFSET + nameLen + extra.length; + final int len = LFH_FILENAME_OFFSET + nameLen + extra.length; final byte[] buf = new byte[len]; System.arraycopy(LFH_SIG, 0, buf, LFH_SIG_OFFSET, WORD); //store method in local variable to prevent multiple method calls final int zipMethod = ze.getMethod(); + final boolean dataDescriptor = usesDataDescriptor(zipMethod, phased); - if (phased && !isZip64Required(entry.entry, zip64Mode)){ - putShort(INITIAL_VERSION, buf, LFH_VERSION_NEEDED_OFFSET); - } else { - putShort(versionNeededToExtract(zipMethod, hasZip64Extra(ze)), buf, LFH_VERSION_NEEDED_OFFSET); - } + putShort(versionNeededToExtract(zipMethod, hasZip64Extra(ze), dataDescriptor), buf, LFH_VERSION_NEEDED_OFFSET); - final GeneralPurposeBit generalPurposeBit = getGeneralPurposeBits(zipMethod, !encodable && fallbackToUTF8); + final GeneralPurposeBit generalPurposeBit = getGeneralPurposeBits(!encodable && fallbackToUTF8, dataDescriptor); generalPurposeBit.encode(buf, LFH_GPB_OFFSET); // compression method @@ -1097,7 +1124,9 @@ // file name System.arraycopy( name.array(), name.arrayOffset(), buf, LFH_FILENAME_OFFSET, nameLen); + // extra fields System.arraycopy(extra, 0, buf, LFH_FILENAME_OFFSET + nameLen, extra.length); + return buf; } @@ -1143,7 +1172,7 @@ * @throws IOException on error */ protected void writeDataDescriptor(final ZipArchiveEntry ze) throws IOException { - if (ze.getMethod() != DEFLATED || channel != null) { + if (!usesDataDescriptor(ze.getMethod(), false)) { return; } writeCounted(DD_SIG); @@ -1172,11 +1201,11 @@ private byte[] createCentralFileHeader(final ZipArchiveEntry ze) throws IOException { - final long lfhOffset = offsets.get(ze); + final EntryMetaData entryMetaData = metaData.get(ze); final boolean needsZip64Extra = hasZip64Extra(ze) || ze.getCompressedSize() >= ZIP64_MAGIC || ze.getSize() >= ZIP64_MAGIC - || lfhOffset >= ZIP64_MAGIC + || entryMetaData.offset >= ZIP64_MAGIC || zip64Mode == Zip64Mode.Always; if (needsZip64Extra && zip64Mode == Zip64Mode.Never) { @@ -1188,19 +1217,20 @@ } - handleZip64Extra(ze, lfhOffset, needsZip64Extra); + handleZip64Extra(ze, entryMetaData.offset, needsZip64Extra); - return createCentralFileHeader(ze, getName(ze), lfhOffset, needsZip64Extra); + return createCentralFileHeader(ze, getName(ze), entryMetaData, needsZip64Extra); } /** * Writes the central file header entry. * @param ze the entry to write * @param name The encoded name - * @param lfhOffset Local file header offset for this file + * @param entryMetaData meta data for this file * @throws IOException on error */ - private byte[] createCentralFileHeader(final ZipArchiveEntry ze, final ByteBuffer name, final long lfhOffset, + private byte[] createCentralFileHeader(final ZipArchiveEntry ze, final ByteBuffer name, + final EntryMetaData entryMetaData, final boolean needsZip64Extra) throws IOException { final byte[] extra = ze.getCentralDirectoryExtra(); @@ -1225,8 +1255,9 @@ final int zipMethod = ze.getMethod(); final boolean encodable = zipEncoding.canEncode(ze.getName()); - putShort(versionNeededToExtract(zipMethod, needsZip64Extra), buf, CFH_VERSION_NEEDED_OFFSET); - getGeneralPurposeBits(zipMethod, !encodable && fallbackToUTF8).encode(buf, CFH_GPB_OFFSET); + putShort(versionNeededToExtract(zipMethod, needsZip64Extra, entryMetaData.usesDataDescriptor), + buf, CFH_VERSION_NEEDED_OFFSET); + getGeneralPurposeBits(!encodable && fallbackToUTF8, entryMetaData.usesDataDescriptor).encode(buf, CFH_GPB_OFFSET); // compression method putShort(zipMethod, buf, CFH_METHOD_OFFSET); @@ -1266,10 +1297,10 @@ putLong(ze.getExternalAttributes(), buf, CFH_EXTERNAL_ATTRIBUTES_OFFSET); // relative offset of LFH - if (lfhOffset >= ZIP64_MAGIC || zip64Mode == Zip64Mode.Always) { + if (entryMetaData.offset >= ZIP64_MAGIC || zip64Mode == Zip64Mode.Always) { putLong(ZIP64_MAGIC, buf, CFH_LFH_OFFSET); } else { - putLong(Math.min(lfhOffset, ZIP64_MAGIC), buf, CFH_LFH_OFFSET); + putLong(Math.min(entryMetaData.offset, ZIP64_MAGIC), buf, CFH_LFH_OFFSET); } // file name @@ -1443,30 +1474,32 @@ } - private GeneralPurposeBit getGeneralPurposeBits(final int zipMethod, final boolean utfFallback) { + private GeneralPurposeBit getGeneralPurposeBits(final boolean utfFallback, boolean usesDataDescriptor) { final GeneralPurposeBit b = new GeneralPurposeBit(); b.useUTF8ForNames(useUTF8Flag || utfFallback); - if (isDeflatedToOutputStream(zipMethod)) { + if (usesDataDescriptor) { b.useDataDescriptor(true); } return b; } - private int versionNeededToExtract(final int zipMethod, final boolean zip64) { + private int versionNeededToExtract(final int zipMethod, final boolean zip64, final boolean usedDataDescriptor) { if (zip64) { return ZIP64_MIN_VERSION; } - // requires version 2 as we are going to store length info - // in the data descriptor - return (isDeflatedToOutputStream(zipMethod)) ? - DATA_DESCRIPTOR_MIN_VERSION : - INITIAL_VERSION; + if (usedDataDescriptor) { + return DATA_DESCRIPTOR_MIN_VERSION; + } + return versionNeededToExtractMethod(zipMethod); } - private boolean isDeflatedToOutputStream(final int zipMethod) { - return zipMethod == DEFLATED && channel == null; + private boolean usesDataDescriptor(final int zipMethod, boolean phased) { + return !phased && zipMethod == DEFLATED && channel == null; } + private int versionNeededToExtractMethod(int zipMethod) { + return zipMethod == DEFLATED ? DEFLATE_MIN_VERSION : INITIAL_VERSION; + } /** * Creates a new zip entry taking some information from the given @@ -1534,7 +1567,7 @@ /** * If the mode is AsNeeded and the entry is a compressed entry of - * unknown size that gets written to a non-seekable stream the + * unknown size that gets written to a non-seekable stream then * change the default to Never. * * @since 1.3 @@ -1567,11 +1600,14 @@ * corrupt archives so they can clean up any temporary files.

      */ void destroy() throws IOException { - if (channel != null) { - channel.close(); - } - if (out != null) { - out.close(); + try { + if (channel != null) { + channel.close(); + } + } finally { + if (out != null) { + out.close(); + } } } @@ -1646,4 +1682,12 @@ private boolean hasWritten; } + private static final class EntryMetaData { + private final long offset; + private final boolean usesDataDescriptor; + private EntryMetaData(long offset, boolean usesDataDescriptor) { + this.offset = offset; + this.usesDataDescriptor = usesDataDescriptor; + } + } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipConstants.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipConstants.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipConstants.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipConstants.java 2018-05-02 20:17:13.000000000 +0000 @@ -23,8 +23,6 @@ * @since 1.3 */ final class ZipConstants { - private ZipConstants() { } - /** Masks last eight bits */ static final int BYTE_MASK = 0xFF; @@ -40,6 +38,12 @@ /** Initial ZIP specification version */ static final int INITIAL_VERSION = 10; + /** + * ZIP specification version that introduced DEFLATE compression method. + * @since 1.15 + */ + static final int DEFLATE_MIN_VERSION = 20; + /** ZIP specification version that introduced data descriptor method */ static final int DATA_DESCRIPTOR_MIN_VERSION = 20; @@ -58,4 +62,6 @@ */ static final long ZIP64_MAGIC = 0xFFFFFFFFL; + private ZipConstants() { } + } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java 2018-05-02 20:17:13.000000000 +0000 @@ -24,7 +24,7 @@ /** * Utility class that represents an eight byte integer with conversion - * rules for the big endian byte order of ZIP files. + * rules for the little endian byte order of ZIP files. * @Immutable * * @since 1.2 diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncodingHelper.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncodingHelper.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncodingHelper.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncodingHelper.java 2018-05-23 12:50:54.000000000 +0000 @@ -20,175 +20,14 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.charset.UnsupportedCharsetException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.compress.utils.Charsets; /** - * Static helper functions for robustly encoding filenames in zip files. + * Static helper functions for robustly encoding filenames in zip files. */ public abstract class ZipEncodingHelper { - /** - * A class, which holds the high characters of a simple encoding - * and lazily instantiates a Simple8BitZipEncoding instance in a - * thread-safe manner. - */ - private static class SimpleEncodingHolder { - - private final char [] highChars; - private Simple8BitZipEncoding encoding; - - /** - * Instantiate a simple encoding holder. - * - * @param highChars The characters for byte codes 128 to 255. - * - * @see Simple8BitZipEncoding#Simple8BitZipEncoding(char[]) - */ - SimpleEncodingHolder(final char [] highChars) { - this.highChars = highChars; - } - - /** - * @return The associated {@link Simple8BitZipEncoding}, which - * is instantiated if not done so far. - */ - public synchronized Simple8BitZipEncoding getEncoding() { - if (this.encoding == null) { - this.encoding = new Simple8BitZipEncoding(this.highChars); - } - return this.encoding; - } - } - - private static final Map simpleEncodings; - - static { - final Map se = - new HashMap<>(); - - final char[] cp437_high_chars = - new char[] { 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, - 0x00e5, 0x00e7, 0x00ea, 0x00eb, 0x00e8, 0x00ef, - 0x00ee, 0x00ec, 0x00c4, 0x00c5, 0x00c9, 0x00e6, - 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9, - 0x00ff, 0x00d6, 0x00dc, 0x00a2, 0x00a3, 0x00a5, - 0x20a7, 0x0192, 0x00e1, 0x00ed, 0x00f3, 0x00fa, - 0x00f1, 0x00d1, 0x00aa, 0x00ba, 0x00bf, 0x2310, - 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb, - 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, - 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, - 0x255d, 0x255c, 0x255b, 0x2510, 0x2514, 0x2534, - 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, - 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, - 0x256c, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, - 0x2558, 0x2552, 0x2553, 0x256b, 0x256a, 0x2518, - 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580, - 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, - 0x00b5, 0x03c4, 0x03a6, 0x0398, 0x03a9, 0x03b4, - 0x221e, 0x03c6, 0x03b5, 0x2229, 0x2261, 0x00b1, - 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248, - 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, - 0x25a0, 0x00a0 }; - - final SimpleEncodingHolder cp437 = new SimpleEncodingHolder(cp437_high_chars); - - se.put("CP437", cp437); - se.put("Cp437", cp437); - se.put("cp437", cp437); - se.put("IBM437", cp437); - se.put("ibm437", cp437); - - final char[] cp850_high_chars = - new char[] { 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, - 0x00e5, 0x00e7, 0x00ea, 0x00eb, 0x00e8, 0x00ef, - 0x00ee, 0x00ec, 0x00c4, 0x00c5, 0x00c9, 0x00e6, - 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9, - 0x00ff, 0x00d6, 0x00dc, 0x00f8, 0x00a3, 0x00d8, - 0x00d7, 0x0192, 0x00e1, 0x00ed, 0x00f3, 0x00fa, - 0x00f1, 0x00d1, 0x00aa, 0x00ba, 0x00bf, 0x00ae, - 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb, - 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00c1, - 0x00c2, 0x00c0, 0x00a9, 0x2563, 0x2551, 0x2557, - 0x255d, 0x00a2, 0x00a5, 0x2510, 0x2514, 0x2534, - 0x252c, 0x251c, 0x2500, 0x253c, 0x00e3, 0x00c3, - 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, - 0x256c, 0x00a4, 0x00f0, 0x00d0, 0x00ca, 0x00cb, - 0x00c8, 0x0131, 0x00cd, 0x00ce, 0x00cf, 0x2518, - 0x250c, 0x2588, 0x2584, 0x00a6, 0x00cc, 0x2580, - 0x00d3, 0x00df, 0x00d4, 0x00d2, 0x00f5, 0x00d5, - 0x00b5, 0x00fe, 0x00de, 0x00da, 0x00db, 0x00d9, - 0x00fd, 0x00dd, 0x00af, 0x00b4, 0x00ad, 0x00b1, - 0x2017, 0x00be, 0x00b6, 0x00a7, 0x00f7, 0x00b8, - 0x00b0, 0x00a8, 0x00b7, 0x00b9, 0x00b3, 0x00b2, - 0x25a0, 0x00a0 }; - - final SimpleEncodingHolder cp850 = new SimpleEncodingHolder(cp850_high_chars); - - se.put("CP850", cp850); - se.put("Cp850", cp850); - se.put("cp850", cp850); - se.put("IBM850", cp850); - se.put("ibm850", cp850); - simpleEncodings = Collections.unmodifiableMap(se); - } - - /** - * Grow a byte buffer, so it has a minimal capacity or at least - * the double capacity of the original buffer - * - * @param b The original buffer. - * @param newCapacity The minimal requested new capacity. - * @return A byte buffer r with - * r.capacity() = max(b.capacity()*2,newCapacity) and - * all the data contained in b copied to the beginning - * of r. - * - */ - static ByteBuffer growBuffer(final ByteBuffer b, final int newCapacity) { - b.limit(b.position()); - b.rewind(); - - final int c2 = b.capacity() * 2; - final ByteBuffer on = ByteBuffer.allocate(c2 < newCapacity ? newCapacity : c2); - - on.put(b); - return on; - } - - - /** - * The hexadecimal digits 0,...,9,A,...,F encoded as - * ASCII bytes. - */ - private static final byte[] HEX_DIGITS = - new byte [] { - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, - 0x42, 0x43, 0x44, 0x45, 0x46 - }; - - /** - * Append %Uxxxx to the given byte buffer. - * The caller must assure, that bb.remaining()>=6. - * - * @param bb The byte buffer to write to. - * @param c The character to write. - */ - static void appendSurrogate(final ByteBuffer bb, final char c) { - - bb.put((byte) '%'); - bb.put((byte) 'U'); - - bb.put(HEX_DIGITS[(c >> 12)&0x0f]); - bb.put(HEX_DIGITS[(c >> 8)&0x0f]); - bb.put(HEX_DIGITS[(c >> 4)&0x0f]); - bb.put(HEX_DIGITS[c & 0x0f]); - } - /** * name of the encoding UTF-8 @@ -196,63 +35,62 @@ static final String UTF8 = "UTF8"; /** - * name of the encoding UTF-8 + * the encoding UTF-8 */ - static final ZipEncoding UTF8_ZIP_ENCODING = new FallbackZipEncoding(UTF8); + static final ZipEncoding UTF8_ZIP_ENCODING = getZipEncoding(UTF8); /** - * Instantiates a zip encoding. - * + * Instantiates a zip encoding. An NIO based character set encoder/decoder will be returned. + * As a special case, if the character set is UTF-8, the nio encoder will be configured replace malformed and + * unmappable characters with '?'. This matches existing behavior from the older fallback encoder. + *

      + * If the requested characer set cannot be found, the platform default will + * be used instead. + *

      * @param name The name of the zip encoding. Specify {@code null} for * the platform's default encoding. * @return A zip encoding for the given encoding name. */ public static ZipEncoding getZipEncoding(final String name) { - - // fallback encoding is good enough for UTF-8. - if (isUTF8(name)) { - return UTF8_ZIP_ENCODING; - } - - if (name == null) { - return new FallbackZipEncoding(); - } - - final SimpleEncodingHolder h = simpleEncodings.get(name); - - if (h!=null) { - return h.getEncoding(); - } - - try { - - final Charset cs = Charset.forName(name); - return new NioZipEncoding(cs); - - } catch (final UnsupportedCharsetException e) { - return new FallbackZipEncoding(name); + Charset cs = Charset.defaultCharset(); + if (name != null) { + try { + cs = Charset.forName(name); + } catch (UnsupportedCharsetException e) { // NOSONAR we use the default encoding instead + } } + boolean useReplacement = isUTF8(cs.name()); + return new NioZipEncoding(cs, useReplacement); } /** * Returns whether a given encoding is UTF-8. If the given name is null, then check the platform's default encoding. - * - * @param charsetName - * If the given name is null, then check the platform's default encoding. + * + * @param charsetName If the given name is null, then check the platform's default encoding. */ static boolean isUTF8(String charsetName) { if (charsetName == null) { // check platform's default encoding charsetName = Charset.defaultCharset().name(); } - if (Charsets.UTF_8.name().equalsIgnoreCase(charsetName)) { + if (StandardCharsets.UTF_8.name().equalsIgnoreCase(charsetName)) { return true; } - for (final String alias : Charsets.UTF_8.aliases()) { + for (final String alias : StandardCharsets.UTF_8.aliases()) { if (alias.equalsIgnoreCase(charsetName)) { return true; } } return false; } + + static ByteBuffer growBufferBy(ByteBuffer buffer, int increment) { + buffer.limit(buffer.position()); + buffer.rewind(); + + final ByteBuffer on = ByteBuffer.allocate(buffer.capacity() + increment); + + on.put(buffer); + return on; + } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncoding.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncoding.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncoding.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipEncoding.java 2018-05-02 20:17:13.000000000 +0000 @@ -30,14 +30,14 @@ * {@link java.nio.charset.Charset Charset} and one implementation, * which copes with simple 8 bit charsets, because java-1.4 did not * support Cp437 in java.nio.

      - * + * *

      The main reason for defining an own encoding layer comes from * the problems with {@link java.lang.String#getBytes(String) * String.getBytes}, which encodes unknown characters as ASCII * quotation marks ('?'). Quotation marks are per definition an * invalid filename on some operating systems like Windows, which * leads to ignored ZIP entries.

      - * + * *

      All implementations should implement this interface in a * reentrant way.

      */ @@ -45,7 +45,7 @@ /** * Check, whether the given string may be losslessly encoded using this * encoding. - * + * * @param name A filename or ZIP comment. * @return Whether the given name may be encoded with out any losses. */ @@ -54,15 +54,15 @@ /** * Encode a filename or a comment to a byte array suitable for * storing it to a serialized zip entry. - * + * *

      Examples for CP 437 (in pseudo-notation, right hand side is * C-style notation):

      * 
            *  encode("\u20AC_for_Dollar.txt") = "%U20AC_for_Dollar.txt"
      *  encode("\u00D6lf\u00E4sser.txt") = "\231lf\204sser.txt"
      *
      - * - * @param name A filename or ZIP comment. + * + * @param name A filename or ZIP comment. * @return A byte buffer with a backing array containing the * encoded name. Unmappable characters or malformed * character sequences are mapped to a sequence of utf-16 diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipExtraField.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipExtraField.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipExtraField.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipExtraField.java 2018-05-02 20:17:13.000000000 +0000 @@ -32,6 +32,12 @@ */ public interface ZipExtraField { /** + * Size of an extra field field header (id + length). + * @since 1.14 + */ + int EXTRAFIELD_HEADER_SIZE = 4; + + /** * The Header-ID. * * @return The HeaderId value diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java 2018-05-23 12:50:54.000000000 +0000 @@ -18,12 +18,15 @@ package org.apache.commons.compress.archivers.zip; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.EOFException; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.SequenceInputStream; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.StandardOpenOption; @@ -37,11 +40,13 @@ import java.util.List; import java.util.Map; import java.util.zip.Inflater; -import java.util.zip.InflaterInputStream; import java.util.zip.ZipException; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream; +import org.apache.commons.compress.utils.CountingInputStream; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; import static org.apache.commons.compress.archivers.zip.ZipConstants.DWORD; import static org.apache.commons.compress.archivers.zip.ZipConstants.SHORT; @@ -88,6 +93,7 @@ private static final int POS_1 = 1; private static final int POS_2 = 2; private static final int POS_3 = 3; + private static final byte[] ONE_ZERO_BYTE = new byte[1]; /** * List of entries in the order they appear inside the central @@ -102,11 +108,6 @@ private final Map> nameMap = new HashMap<>(HASH_SIZE); - private static final class OffsetEntry { - private long headerOffset = -1; - private long dataOffset = -1; - } - /** * The encoding to use for filenames and the file comment. * @@ -439,9 +440,8 @@ if (!(ze instanceof Entry)) { return null; } - final OffsetEntry offsetEntry = ((Entry) ze).getOffsetEntry(); - final long start = offsetEntry.dataOffset; - return new BoundedInputStream(start, ze.getCompressedSize()); + final long start = ze.getDataOffset(); + return createBoundedInputStream(start, ze.getCompressedSize()); } @@ -469,34 +469,41 @@ * Returns an InputStream for reading the contents of the given entry. * * @param ze the entry to get the stream for. - * @return a stream to read the entry from. + * @return a stream to read the entry from. The returned stream + * implements {@link InputStreamStatistics}. * @throws IOException if unable to create an input stream from the zipentry - * @throws ZipException if the zipentry uses an unsupported feature */ public InputStream getInputStream(final ZipArchiveEntry ze) - throws IOException, ZipException { + throws IOException { if (!(ze instanceof Entry)) { return null; } - // cast valididty is checked just above - final OffsetEntry offsetEntry = ((Entry) ze).getOffsetEntry(); + // cast validity is checked just above ZipUtil.checkRequestedFeatures(ze); - final long start = offsetEntry.dataOffset; - // doesn't get closed if the method is not supported, but doesn't hold any resources either - final BoundedInputStream bis = - new BoundedInputStream(start, ze.getCompressedSize()); //NOSONAR + final long start = ze.getDataOffset(); + + // doesn't get closed if the method is not supported - which + // should never happen because of the checkRequestedFeatures + // call above + final InputStream is = + new BufferedInputStream(createBoundedInputStream(start, ze.getCompressedSize())); //NOSONAR switch (ZipMethod.getMethodByCode(ze.getMethod())) { case STORED: - return bis; + return new StoredStatisticsStream(is); case UNSHRINKING: - return new UnshrinkingInputStream(bis); + return new UnshrinkingInputStream(is); case IMPLODING: return new ExplodingInputStream(ze.getGeneralPurposeBit().getSlidingDictionarySize(), - ze.getGeneralPurposeBit().getNumberOfShannonFanoTrees(), new BufferedInputStream(bis)); + ze.getGeneralPurposeBit().getNumberOfShannonFanoTrees(), is); case DEFLATED: - bis.addDummy(); final Inflater inflater = new Inflater(true); - return new InflaterInputStream(bis, inflater) { + // Inflater with nowrap=true has this odd contract for a zero padding + // byte following the data stream; this used to be zlib's requirement + // and has been fixed a long time ago, but the contract persists so + // we comply. + // https://docs.oracle.com/javase/7/docs/api/java/util/zip/Inflater.html#Inflater(boolean) + return new InflaterInputStreamWithStatistics(new SequenceInputStream(is, new ByteArrayInputStream(ONE_ZERO_BYTE)), + inflater) { @Override public void close() throws IOException { try { @@ -507,9 +514,10 @@ } }; case BZIP2: - return new BZip2CompressorInputStream(bis); - case AES_ENCRYPTED: + return new BZip2CompressorInputStream(is); case ENHANCED_DEFLATED: + return new Deflate64CompressorInputStream(is); + case AES_ENCRYPTED: case EXPANDING_LEVEL_1: case EXPANDING_LEVEL_2: case EXPANDING_LEVEL_3: @@ -521,6 +529,7 @@ case TOKENIZATION: case UNKNOWN: case WAVPACK: + case XZ: default: throw new ZipException("Found unsupported compression method " + ze.getMethod()); @@ -644,8 +653,7 @@ cfhBbuf.rewind(); IOUtils.readFully(archive, cfhBbuf); int off = 0; - final OffsetEntry offset = new OffsetEntry(); - final Entry ze = new Entry(offset); + final Entry ze = new Entry(); final int versionMadeBy = ZipShort.getValue(cfhBuf, off); off += SHORT; @@ -659,6 +667,9 @@ final boolean hasUTF8Flag = gpFlag.usesUTF8ForNames(); final ZipEncoding entryEncoding = hasUTF8Flag ? ZipEncodingHelper.UTF8_ZIP_ENCODING : zipEncoding; + if (hasUTF8Flag) { + ze.setNameSource(ZipArchiveEntry.NameSource.NAME_WITH_EFS_FLAG); + } ze.setGeneralPurposeBit(gpFlag); ze.setRawFlag(ZipShort.getValue(cfhBuf, off)); @@ -704,7 +715,7 @@ ze.setName(entryEncoding.decode(fileName), fileName); // LFH offset, - offset.headerOffset = ZipLong.getValue(cfhBuf, off); + ze.setLocalHeaderOffset(ZipLong.getValue(cfhBuf, off)); // data offset will be filled later entries.add(ze); @@ -712,7 +723,7 @@ IOUtils.readFully(archive, ByteBuffer.wrap(cdExtraData)); ze.setCentralDirectoryExtra(cdExtraData); - setSizesAndOffsetFromZip64Extra(ze, offset, diskStart); + setSizesAndOffsetFromZip64Extra(ze, diskStart); final byte[] comment = new byte[commentLen]; IOUtils.readFully(archive, ByteBuffer.wrap(comment)); @@ -736,7 +747,6 @@ * size would be invalid.

      */ private void setSizesAndOffsetFromZip64Extra(final ZipArchiveEntry ze, - final OffsetEntry offset, final int diskStart) throws IOException { final Zip64ExtendedInformationExtraField z64 = @@ -746,7 +756,7 @@ final boolean hasUncompressedSize = ze.getSize() == ZIP64_MAGIC; final boolean hasCompressedSize = ze.getCompressedSize() == ZIP64_MAGIC; final boolean hasRelativeHeaderOffset = - offset.headerOffset == ZIP64_MAGIC; + ze.getLocalHeaderOffset() == ZIP64_MAGIC; z64.reparseCentralDirectoryData(hasUncompressedSize, hasCompressedSize, hasRelativeHeaderOffset, @@ -765,8 +775,7 @@ } if (hasRelativeHeaderOffset) { - offset.headerOffset = - z64.getRelativeHeaderOffset().getLongValue(); + ze.setLocalHeaderOffset(z64.getRelativeHeaderOffset().getLongValue()); } } } @@ -997,7 +1006,7 @@ /** * Skips the given number of bytes or throws an EOFException if * skipping failed. - */ + */ private void skipBytes(final int count) throws IOException { long currentPosition = archive.position(); long newPosition = currentPosition + count; @@ -1036,8 +1045,7 @@ // entries is filled in populateFromCentralDirectory and // never modified final Entry ze = (Entry) zipArchiveEntry; - final OffsetEntry offsetEntry = ze.getOffsetEntry(); - final long offset = offsetEntry.headerOffset; + final long offset = ze.getLocalHeaderOffset(); archive.position(offset + LFH_OFFSET_FOR_FILENAME_LENGTH); wordBbuf.rewind(); IOUtils.readFully(archive, wordBbuf); @@ -1050,8 +1058,9 @@ final byte[] localExtraData = new byte[extraFieldLen]; IOUtils.readFully(archive, ByteBuffer.wrap(localExtraData)); ze.setExtra(localExtraData); - offsetEntry.dataOffset = offset + LFH_OFFSET_FOR_FILENAME_LENGTH - + SHORT + SHORT + fileNameLen + extraFieldLen; + ze.setDataOffset(offset + LFH_OFFSET_FOR_FILENAME_LENGTH + + SHORT + SHORT + fileNameLen + extraFieldLen); + ze.setStreamContiguous(true); if (entriesWithoutUTF8Flag.containsKey(ze)) { final NameAndComment nc = entriesWithoutUTF8Flag.get(ze); @@ -1081,98 +1090,106 @@ } /** + * Creates new BoundedInputStream, according to implementation of + * underlying archive channel. + */ + private BoundedInputStream createBoundedInputStream(long start, long remaining) { + return archive instanceof FileChannel ? + new BoundedFileChannelInputStream(start, remaining) : + new BoundedInputStream(start, remaining); + } + + /** * InputStream that delegates requests to the underlying * SeekableByteChannel, making sure that only bytes from a certain * range can be read. */ private class BoundedInputStream extends InputStream { - private static final int MAX_BUF_LEN = 8192; - private final ByteBuffer buffer; - private long remaining; + private ByteBuffer singleByteBuffer; + private final long end; private long loc; - private boolean addDummyByte = false; BoundedInputStream(final long start, final long remaining) { - this.remaining = remaining; - loc = start; - if (remaining < MAX_BUF_LEN && remaining > 0) { - buffer = ByteBuffer.allocate((int) remaining); - } else { - buffer = ByteBuffer.allocate(MAX_BUF_LEN); + this.end = start+remaining; + if (this.end < start) { + // check for potential vulnerability due to overflow + throw new IllegalArgumentException("Invalid length of stream at offset="+start+", length="+remaining); } + loc = start; } @Override - public int read() throws IOException { - if (remaining-- <= 0) { - if (addDummyByte) { - addDummyByte = false; - return 0; - } + public synchronized int read() throws IOException { + if (loc >= end) { return -1; } - synchronized (archive) { - archive.position(loc++); - int read = read(1); - if (read < 0) { - return read; - } - return buffer.get() & 0xff; + if (singleByteBuffer == null) { + singleByteBuffer = ByteBuffer.allocate(1); + } + else { + singleByteBuffer.rewind(); } + int read = read(loc, singleByteBuffer); + if (read < 0) { + return read; + } + loc++; + return singleByteBuffer.get() & 0xff; } @Override - public int read(final byte[] b, final int off, int len) throws IOException { - if (remaining <= 0) { - if (addDummyByte) { - addDummyByte = false; - b[off] = 0; - return 1; - } - return -1; - } - + public synchronized int read(final byte[] b, final int off, int len) throws IOException { if (len <= 0) { return 0; } - if (len > remaining) { - len = (int) remaining; - } - ByteBuffer buf; - int ret = -1; - synchronized (archive) { - archive.position(loc); - if (len <= buffer.capacity()) { - buf = buffer; - ret = read(len); - } else { - buf = ByteBuffer.allocate(len); - ret = archive.read(buf); - buf.flip(); + if (len > end-loc) { + if (loc >= end) { + return -1; } + len = (int)(end-loc); } + + ByteBuffer buf; + buf = ByteBuffer.wrap(b, off, len); + int ret = read(loc, buf); if (ret > 0) { - buf.get(b, off, ret); loc += ret; - remaining -= ret; + return ret; } return ret; } - private int read(int len) throws IOException { - buffer.rewind().limit(len); - int read = archive.read(buffer); - buffer.flip(); + protected int read(long pos, ByteBuffer buf) throws IOException { + int read; + synchronized (archive) { + archive.position(pos); + read = archive.read(buf); + } + buf.flip(); return read; } + } + + /** + * Lock-free implementation of BoundedInputStream. The + * implementation uses positioned reads on the underlying archive + * file channel and therefore performs significantly faster in + * concurrent environment. + */ + private class BoundedFileChannelInputStream extends BoundedInputStream { + private final FileChannel archive; + + BoundedFileChannelInputStream(final long start, final long remaining) { + super(start, remaining); + archive = (FileChannel)ZipFile.this.archive; + } - /** - * Inflater needs an extra dummy byte for nowrap - see - * Inflater's javadocs. - */ - void addDummy() { - addDummyByte = true; + @Override + protected int read(long pos, ByteBuffer buf) throws IOException { + int read = archive.read(buf, pos); + buf.flip(); + return read; } } @@ -1209,8 +1226,8 @@ if (ent2 == null) { return -1; } - final long val = (ent1.getOffsetEntry().headerOffset - - ent2.getOffsetEntry().headerOffset); + final long val = (ent1.getLocalHeaderOffset() + - ent2.getLocalHeaderOffset()); return val == 0 ? 0 : val < 0 ? -1 : +1; } }; @@ -1220,20 +1237,13 @@ */ private static class Entry extends ZipArchiveEntry { - private final OffsetEntry offsetEntry; - - Entry(final OffsetEntry offset) { - this.offsetEntry = offset; - } - - OffsetEntry getOffsetEntry() { - return offsetEntry; + Entry() { } @Override public int hashCode() { return 3 * super.hashCode() - + (int) (offsetEntry.headerOffset % Integer.MAX_VALUE); + + (int) getLocalHeaderOffset()+(int)(getLocalHeaderOffset()>>32); } @Override @@ -1241,12 +1251,28 @@ if (super.equals(other)) { // super.equals would return false if other were not an Entry final Entry otherEntry = (Entry) other; - return offsetEntry.headerOffset - == otherEntry.offsetEntry.headerOffset - && offsetEntry.dataOffset - == otherEntry.offsetEntry.dataOffset; + return getLocalHeaderOffset() + == otherEntry.getLocalHeaderOffset() + && getDataOffset() + == otherEntry.getDataOffset(); } return false; } } + + private static class StoredStatisticsStream extends CountingInputStream implements InputStreamStatistics { + StoredStatisticsStream(InputStream in) { + super(in); + } + + @Override + public long getCompressedCount() { + return super.getBytesRead(); + } + + @Override + public long getUncompressedCount() { + return getCompressedCount(); + } + } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipLong.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipLong.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipLong.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipLong.java 2018-05-02 20:17:13.000000000 +0000 @@ -17,33 +17,20 @@ */ package org.apache.commons.compress.archivers.zip; +import org.apache.commons.compress.utils.ByteUtils; + import java.io.Serializable; -import static org.apache.commons.compress.archivers.zip.ZipConstants.BYTE_MASK; import static org.apache.commons.compress.archivers.zip.ZipConstants.WORD; /** * Utility class that represents a four byte integer with conversion - * rules for the big endian byte order of ZIP files. + * rules for the little endian byte order of ZIP files. * @Immutable */ public final class ZipLong implements Cloneable, Serializable { private static final long serialVersionUID = 1L; - //private static final int BYTE_BIT_SIZE = 8; - - private static final int BYTE_1 = 1; - private static final int BYTE_1_MASK = 0xFF00; - private static final int BYTE_1_SHIFT = 8; - - private static final int BYTE_2 = 2; - private static final int BYTE_2_MASK = 0xFF0000; - private static final int BYTE_2_SHIFT = 16; - - private static final int BYTE_3 = 3; - private static final long BYTE_3_MASK = 0xFF000000L; - private static final int BYTE_3_SHIFT = 24; - private final long value; /** Central File Header Signature */ @@ -95,6 +82,15 @@ } /** + * create instance from a java int. + * @param value the int to store as a ZipLong + * @since 1.15 + */ + public ZipLong(int value) { + this.value = value; + } + + /** * Create instance from bytes. * @param bytes the bytes to store as a ZipLong */ @@ -128,6 +124,13 @@ } /** + * Get value as a (signed) java int + * @return value as int + * @since 1.15 + */ + public int getIntValue() { return (int)value;} + + /** * Get value as four bytes in big endian byte order. * @param value the value to convert * @return value as four bytes in big endian byte order @@ -148,10 +151,7 @@ */ public static void putLong(final long value, final byte[] buf, int offset) { - buf[offset++] = (byte) ((value & BYTE_MASK)); - buf[offset++] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT); - buf[offset++] = (byte) ((value & BYTE_2_MASK) >> BYTE_2_SHIFT); - buf[offset] = (byte) ((value & BYTE_3_MASK) >> BYTE_3_SHIFT); + ByteUtils.toLittleEndian(buf, value, offset, 4); } public void putLong(final byte[] buf, final int offset) { @@ -165,11 +165,7 @@ * @return the corresponding Java long value */ public static long getValue(final byte[] bytes, final int offset) { - long value = (bytes[offset + BYTE_3] << BYTE_3_SHIFT) & BYTE_3_MASK; - value += (bytes[offset + BYTE_2] << BYTE_2_SHIFT) & BYTE_2_MASK; - value += (bytes[offset + BYTE_1] << BYTE_1_SHIFT) & BYTE_1_MASK; - value += (bytes[offset] & BYTE_MASK); - return value; + return ByteUtils.fromLittleEndian(bytes, offset, 4); } /** diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipMethod.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipMethod.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipMethod.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipMethod.java 2018-05-23 12:50:54.000000000 +0000 @@ -23,17 +23,17 @@ import java.util.zip.ZipEntry; /** - * List of known compression methods - * + * List of known compression methods + * * Many of these methods are currently not supported by commons compress - * + * * @since 1.5 */ public enum ZipMethod { /** * Compression method 0 for uncompressed entries. - * + * * @see ZipEntry#STORED */ STORED(ZipEntry.STORED), @@ -41,123 +41,130 @@ /** * UnShrinking. * dynamic Lempel-Ziv-Welch-Algorithm - * - * @see Explanation of fields: compression + * + * @see Explanation of fields: compression * method: (2 bytes) */ UNSHRINKING(1), /** * Reduced with compression factor 1. - * - * @see Explanation of fields: compression + * + * @see Explanation of fields: compression * method: (2 bytes) */ EXPANDING_LEVEL_1(2), /** * Reduced with compression factor 2. - * - * @see Explanation of fields: compression + * + * @see Explanation of fields: compression * method: (2 bytes) */ EXPANDING_LEVEL_2(3), /** * Reduced with compression factor 3. - * - * @see Explanation of fields: compression + * + * @see Explanation of fields: compression * method: (2 bytes) */ EXPANDING_LEVEL_3(4), /** * Reduced with compression factor 4. - * - * @see Explanation of fields: compression + * + * @see Explanation of fields: compression * method: (2 bytes) */ EXPANDING_LEVEL_4(5), /** * Imploding. - * - * @see Explanation of fields: compression + * + * @see Explanation of fields: compression * method: (2 bytes) */ IMPLODING(6), /** * Tokenization. - * - * @see Explanation of fields: compression + * + * @see Explanation of fields: compression * method: (2 bytes) */ TOKENIZATION(7), /** * Compression method 8 for compressed (deflated) entries. - * + * * @see ZipEntry#DEFLATED */ DEFLATED(ZipEntry.DEFLATED), /** * Compression Method 9 for enhanced deflate. - * - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.winzip.com/wz54.htm */ ENHANCED_DEFLATED(9), /** * PKWARE Data Compression Library Imploding. - * - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.winzip.com/wz54.htm */ PKWARE_IMPLODING(10), /** * Compression Method 12 for bzip2. - * - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.winzip.com/wz54.htm */ BZIP2(12), /** * Compression Method 14 for LZMA. - * - * @see http://www.7-zip.org/sdk.html - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.7-zip.org/sdk.html + * @see https://www.winzip.com/wz54.htm */ LZMA(14), /** + * Compression Method 95 for XZ. + * + * @see https://www.winzip.com/wz54.htm + */ + XZ(95), + + /** * Compression Method 96 for Jpeg compression. - * - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.winzip.com/wz54.htm */ JPEG(96), /** * Compression Method 97 for WavPack. - * - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.winzip.com/wz54.htm */ WAVPACK(97), /** * Compression Method 98 for PPMd. - * - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.winzip.com/wz54.htm */ PPMD(98), /** * Compression Method 99 for AES encryption. - * - * @see http://www.winzip.com/wz54.htm + * + * @see https://www.winzip.com/wz54.htm */ AES_ENCRYPTED(99), @@ -193,9 +200,9 @@ /** * the code of the compression method. - * + * * @see ZipArchiveEntry#getMethod() - * + * * @return an integer code for the method */ public int getCode() { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipShort.java 2018-05-02 20:17:13.000000000 +0000 @@ -19,18 +19,21 @@ import java.io.Serializable; -import static org.apache.commons.compress.archivers.zip.ZipConstants.BYTE_MASK; +import org.apache.commons.compress.utils.ByteUtils; /** * Utility class that represents a two byte integer with conversion - * rules for the big endian byte order of ZIP files. + * rules for the little endian byte order of ZIP files. * @Immutable */ public final class ZipShort implements Cloneable, Serializable { - private static final long serialVersionUID = 1L; + /** + * ZipShort with a value of 0. + * @since 1.14 + */ + public static final ZipShort ZERO = new ZipShort(0); - private static final int BYTE_1_MASK = 0xFF00; - private static final int BYTE_1_SHIFT = 8; + private static final long serialVersionUID = 1L; private final int value; @@ -65,8 +68,7 @@ */ public byte[] getBytes() { final byte[] result = new byte[2]; - result[0] = (byte) (value & BYTE_MASK); - result[1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT); + ByteUtils.toLittleEndian(result, value, 0, 2); return result; } @@ -98,8 +100,7 @@ * must be non-negative and no larger than buf.length-2 */ public static void putShort(final int value, final byte[] buf, final int offset) { - buf[offset] = (byte) (value & BYTE_MASK); - buf[offset+1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT); + ByteUtils.toLittleEndian(buf, value, offset, 2); } /** @@ -109,9 +110,7 @@ * @return the corresponding java int value */ public static int getValue(final byte[] bytes, final int offset) { - int value = (bytes[offset + 1] << BYTE_1_SHIFT) & BYTE_1_MASK; - value += (bytes[offset] & BYTE_MASK); - return value; + return (int) ByteUtils.fromLittleEndian(bytes, offset, 2); } /** diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java 2018-05-02 20:17:13.000000000 +0000 @@ -234,11 +234,11 @@ final byte[] commentBytes) { final UnicodePathExtraField name = (UnicodePathExtraField) ze.getExtraField(UnicodePathExtraField.UPATH_ID); - final String originalName = ze.getName(); final String newName = getUnicodeStringIfOriginalMatches(name, originalNameBytes); - if (newName != null && !originalName.equals(newName)) { + if (newName != null) { ze.setName(newName); + ze.setNameSource(ZipArchiveEntry.NameSource.UNICODE_EXTRA_FIELD); } if (commentBytes != null && commentBytes.length > 0) { @@ -248,6 +248,7 @@ getUnicodeStringIfOriginalMatches(cmt, commentBytes); if (newComment != null) { ze.setComment(newComment); + ze.setCommentSource(ZipArchiveEntry.CommentSource.UNICODE_EXTRA_FIELD); } } } @@ -259,7 +260,7 @@ *

      If the field is null or the CRCs don't match, return null * instead.

      */ - private static + private static String getUnicodeStringIfOriginalMatches(final AbstractUnicodeExtraField f, final byte[] orig) { if (f != null) { @@ -295,6 +296,7 @@ } return null; } + static void copy(final byte[] from, final byte[] to, final int offset) { if (from != null) { System.arraycopy(from, 0, to, offset, from.length); @@ -323,13 +325,14 @@ * Whether this library supports the compression method used by * the given entry. * - * @return true if the compression method is STORED or DEFLATED + * @return true if the compression method is supported */ private static boolean supportsMethodOf(final ZipArchiveEntry entry) { return entry.getMethod() == ZipEntry.STORED || entry.getMethod() == ZipMethod.UNSHRINKING.getCode() || entry.getMethod() == ZipMethod.IMPLODING.getCode() || entry.getMethod() == ZipEntry.DEFLATED + || entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode() || entry.getMethod() == ZipMethod.BZIP2.getCode(); } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/Change.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/Change.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/Change.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/Change.java 2018-05-02 20:17:13.000000000 +0000 @@ -24,7 +24,7 @@ /** * Change holds meta information about a change. - * + * * @Immutable */ class Change { @@ -59,7 +59,7 @@ /** * Construct a change which adds an entry. - * + * * @param pEntry the entry details * @param pInput the InputStream for the entry data */ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/ChangeSet.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/ChangeSet.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/ChangeSet.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/ChangeSet.java 2018-05-02 20:17:13.000000000 +0000 @@ -29,7 +29,7 @@ * ChangeSet collects and performs changes to an archive. * Putting delete changes in this ChangeSet from multiple threads can * cause conflicts. - * + * * @NotThreadSafe */ public final class ChangeSet { @@ -37,8 +37,8 @@ private final Set changes = new LinkedHashSet<>(); /** - * Deletes the file with the filename from the archive. - * + * Deletes the file with the filename from the archive. + * * @param filename * the filename of the file to delete */ @@ -47,8 +47,8 @@ } /** - * Deletes the directory tree from the archive. - * + * Deletes the directory tree from the archive. + * * @param dirName * the name of the directory tree to delete */ @@ -58,7 +58,7 @@ /** * Adds a new archive entry to the archive. - * + * * @param pEntry * the entry to add * @param pInput @@ -72,7 +72,7 @@ * Adds a new archive entry to the archive. * If replace is set to true, this change will replace all other additions * done in this ChangeSet and all existing entries in the original stream. - * + * * @param pEntry * the entry to add * @param pInput @@ -86,7 +86,7 @@ /** * Adds an addition change. - * + * * @param pChange * the change which should result in an addition */ @@ -120,7 +120,7 @@ /** * Adds an delete change. - * + * * @param pChange * the change which should result in a deletion */ @@ -143,10 +143,8 @@ continue; } - if (Change.TYPE_DELETE == pChange.type() && source.equals(target)) { - it.remove(); - } else if (Change.TYPE_DELETE_DIR == pChange.type() && - target.matches(source + "/.*")) { + if (Change.TYPE_DELETE == pChange.type() && source.equals(target) || + (Change.TYPE_DELETE_DIR == pChange.type() && target.matches(source + "/.*"))) { it.remove(); } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/ChangeSetPerformer.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/ChangeSetPerformer.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/ChangeSetPerformer.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/ChangeSetPerformer.java 2018-05-02 20:17:13.000000000 +0000 @@ -37,7 +37,7 @@ * This class is thread safe and can be used multiple times. * It operates on a copy of the ChangeSet. If the ChangeSet changes, * a new Performer must be created. - * + * * @ThreadSafe * @Immutable */ @@ -55,10 +55,10 @@ /** * Performs all changes collected in this ChangeSet on the input stream and * streams the result to the output stream. Perform may be called more than once. - * + * * This method finishes the stream, no other entries should be added * after that. - * + * * @param in * the InputStream to perform the changes on * @param out @@ -75,10 +75,10 @@ /** * Performs all changes collected in this ChangeSet on the ZipFile and * streams the result to the output stream. Perform may be called more than once. - * + * * This method finishes the stream, no other entries should be added * after that. - * + * * @param in * the ZipFile to perform the changes on * @param out @@ -96,10 +96,10 @@ /** * Performs all changes collected in this ChangeSet on the input entries and * streams the result to the output stream. - * + * * This method finishes the stream, no other entries should be added * after that. - * + * * @param entryIterator * the entries to perform the changes on * @param out @@ -163,8 +163,8 @@ for (final Iterator it = workingSet.iterator(); it.hasNext();) { final Change change = it.next(); - if (change.type() == Change.TYPE_ADD && - !change.isReplaceMode() && + if (change.type() == Change.TYPE_ADD && + !change.isReplaceMode() && !results.hasBeenAdded(change.getEntry().getName())) { copyStream(change.getInput(), out, change.getEntry()); it.remove(); @@ -179,7 +179,7 @@ * Checks if an ArchiveEntry is deleted later in the ChangeSet. This is * necessary if an file is added with this ChangeSet, but later became * deleted in the same set. - * + * * @param entry * the entry to check * @return true, if this entry has an deletion change later, false otherwise @@ -205,7 +205,7 @@ /** * Copies the ArchiveEntry to the Output stream - * + * * @param in * the stream to read the data from * @param out diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/ChangeSetResults.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/ChangeSetResults.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/changes/ChangeSetResults.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/changes/ChangeSetResults.java 2018-05-02 20:17:13.000000000 +0000 @@ -38,7 +38,7 @@ } /** - * Adds the name of a file to the result list which has been + * Adds the name of a file to the result list which has been * copied from the source stream to the target stream. * @param fileName the file name which has been added from the original stream */ @@ -85,9 +85,6 @@ * @return true, if this filename already has been added */ boolean hasBeenAdded(final String filename) { - if(addedFromChangeSet.contains(filename) || addedFromStream.contains(filename)) { - return true; - } - return false; + return addedFromChangeSet.contains(filename) || addedFromStream.contains(filename); } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.brotli; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.utils.CountingInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; +import org.brotli.dec.BrotliInputStream; + +/** + * {@link CompressorInputStream} implementation to decode Brotli encoded stream. + * Library relies on Google brotli + * + * @since 1.14 + */ +public class BrotliCompressorInputStream extends CompressorInputStream + implements InputStreamStatistics { + + private final CountingInputStream countingStream; + private final BrotliInputStream decIS; + + public BrotliCompressorInputStream(final InputStream in) throws IOException { + decIS = new BrotliInputStream(countingStream = new CountingInputStream(in)); + } + + @Override + public int available() throws IOException { + return decIS.available(); + } + + @Override + public void close() throws IOException { + decIS.close(); + } + + @Override + public int read(final byte[] b) throws IOException { + return decIS.read(b); + } + + @Override + public long skip(final long n) throws IOException { + return IOUtils.skip(decIS, n); + } + + @Override + public void mark(final int readlimit) { + decIS.mark(readlimit); + } + + @Override + public boolean markSupported() { + return decIS.markSupported(); + } + + @Override + public int read() throws IOException { + final int ret = decIS.read(); + count(ret == -1 ? 0 : 1); + return ret; + } + + @Override + public int read(final byte[] buf, final int off, final int len) throws IOException { + final int ret = decIS.read(buf, off, len); + count(ret); + return ret; + } + + @Override + public String toString() { + return decIS.toString(); + } + + @Override + public void reset() throws IOException { + decIS.reset(); + } + + /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return countingStream.getBytesRead(); + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliUtils.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/brotli/BrotliUtils.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.brotli; + +/** + * Utility code for the Brotli compression format. + * @ThreadSafe + * @since 1.14 + */ +public class BrotliUtils { + + enum CachedAvailability { + DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE + } + + private static volatile CachedAvailability cachedBrotliAvailability; + + static { + cachedBrotliAvailability = CachedAvailability.DONT_CACHE; + try { + Class.forName("org.osgi.framework.BundleEvent"); + } catch (final Exception ex) { // NOSONAR + setCacheBrotliAvailablity(true); + } + } + + /** Private constructor to prevent instantiation of this utility class. */ + private BrotliUtils() { + } + + + /** + * Are the classes required to support Brotli compression available? + * @return true if the classes required to support Brotli compression are available + */ + public static boolean isBrotliCompressionAvailable() { + final CachedAvailability cachedResult = cachedBrotliAvailability; + if (cachedResult != CachedAvailability.DONT_CACHE) { + return cachedResult == CachedAvailability.CACHED_AVAILABLE; + } + return internalIsBrotliCompressionAvailable(); + } + + private static boolean internalIsBrotliCompressionAvailable() { + try { + Class.forName("org.brotli.dec.BrotliInputStream"); + return true; + } catch (NoClassDefFoundError | Exception error) { // NOSONAR + return false; + } + } + + /** + * Whether to cache the result of the Brotli for Java check. + * + *

      This defaults to {@code false} in an OSGi environment and {@code true} otherwise.

      + * @param doCache whether to cache the result + */ + public static void setCacheBrotliAvailablity(final boolean doCache) { + if (!doCache) { + cachedBrotliAvailability = CachedAvailability.DONT_CACHE; + } else if (cachedBrotliAvailability == CachedAvailability.DONT_CACHE) { + final boolean hasBrotli = internalIsBrotliCompressionAvailable(); + cachedBrotliAvailability = hasBrotli ? CachedAvailability.CACHED_AVAILABLE + : CachedAvailability.CACHED_UNAVAILABLE; + } + } + + // only exists to support unit tests + static CachedAvailability getCachedBrotliAvailability() { + return cachedBrotliAvailability; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/brotli/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/brotli/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/brotli/package.html 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/brotli/package.html 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,26 @@ + + + +

      Provides stream class for decompressing streams using the + Brotli algorithm based + on Google's Brotli + decoder.

      + + diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/BlockSort.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/BlockSort.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/BlockSort.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/BlockSort.java 2018-05-02 20:17:13.000000000 +0000 @@ -28,7 +28,7 @@ * blocksort.c in his libbzip2

      * *

      The Burrows-Wheeler transform is a reversible transform of the - * original data that is supposed to group similiar bytes close to + * original data that is supposed to group similar bytes close to * each other. The idea is to sort all permutations of the input and * only keep the last byte of each permutation. E.g. for "Commons * Compress" you'd get:

      @@ -36,7 +36,7 @@ * 
        *  CompressCommons
  * Commons Compress
- * CompressCommons 
+ * CompressCommons
  * essCommons Compr
  * mmons CompressCo
  * mons CompressCom
@@ -251,7 +251,7 @@
      * bucket 'ra' with sort index 5.  The fully sorted order then becomes.
      *
      * fmap = { 5, 3, 0, 4, 1, 2 }
-     * 
+     *
      */
 
     /**
@@ -261,12 +261,12 @@
      * @param eclass points from the index of a character inside the
      *        block to the first index in fmap that contains the
      *        bucket of its suffix that is sorted in this step.
-     * @param lo lower boundary of the fmap-interval to be sorted 
-     * @param hi upper boundary of the fmap-interval to be sorted 
+     * @param lo lower boundary of the fmap-interval to be sorted
+     * @param hi upper boundary of the fmap-interval to be sorted
      */
-    private void fallbackSimpleSort(final int[] fmap, 
-                                    final int[] eclass, 
-                                    final int lo, 
+    private void fallbackSimpleSort(final int[] fmap,
+                                    final int[] eclass,
+                                    final int lo,
                                     final int hi) {
         if (lo == hi) {
             return;
@@ -336,12 +336,12 @@
      * @param eclass points from the index of a character inside the
      *        block to the first index in fmap that contains the
      *        bucket of its suffix that is sorted in this step.
-     * @param loSt lower boundary of the fmap-interval to be sorted 
-     * @param hiSt upper boundary of the fmap-interval to be sorted 
+     * @param loSt lower boundary of the fmap-interval to be sorted
+     * @param hiSt upper boundary of the fmap-interval to be sorted
      */
-    private void fallbackQSort3(final int[] fmap, 
-                                final int[] eclass, 
-                                final int loSt, 
+    private void fallbackQSort3(final int[] fmap,
+                                final int[] eclass,
+                                final int loSt,
                                 final int hiSt) {
         int lo, unLo, ltLo, hi, unHi, gtHi, n;
 
@@ -359,9 +359,9 @@
             }
 
             /* LBZ2: Random partitioning.  Median of 3 sometimes fails to
-               avoid bad cases.  Median of 9 seems to help but 
+               avoid bad cases.  Median of 9 seems to help but
                looks rather expensive.  This too seems to work but
-               is cheaper.  Guidance for the magic constants 
+               is cheaper.  Guidance for the magic constants
                7621 and 32768 is taken from Sedgewick's algorithms
                book, chapter 35.
             */
@@ -369,7 +369,7 @@
             final long r3 = r % 3;
             long med;
             if (r3 == 0) {
-                med = eclass[fmap[lo]]; 
+                med = eclass[fmap[lo]];
             } else if (r3 == 1) {
                 med = eclass[fmap[(lo + hi) >>> 1]];
             } else {
@@ -387,10 +387,10 @@
                         break;
                     }
                     n = eclass[fmap[unLo]] - (int) med;
-                    if (n == 0) { 
-                        fswap(fmap, unLo, ltLo); 
-                        ltLo++; unLo++; 
-                        continue; 
+                    if (n == 0) {
+                        fswap(fmap, unLo, ltLo);
+                        ltLo++; unLo++;
+                        continue;
                     }
                     if (n > 0) {
                         break;
@@ -403,9 +403,9 @@
                     }
                     n = eclass[fmap[unHi]] - (int) med;
                     if (n == 0) {
-                        fswap(fmap, unHi, gtHi); 
-                        gtHi--; unHi--; 
-                        continue; 
+                        fswap(fmap, unHi, gtHi);
+                        gtHi--; unHi--;
+                        continue;
                     }
                     if (n < 0) {
                         break;
@@ -511,7 +511,7 @@
           --*/
 
         /*-- LBZ2: set sentinel bits for block-end detection --*/
-        for (i = 0; i < 32; i++) { 
+        for (i = 0; i < 32; i++) {
             bhtab.set(nblock + 2 * i);
             bhtab.clear(nblock + 2 * i + 1);
         }
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java	2018-06-07 19:11:34.000000000 +0000
@@ -26,16 +26,21 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteOrder;
+import java.util.Arrays;
 
 import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.utils.BitInputStream;
+import org.apache.commons.compress.utils.CloseShieldFilterInputStream;
+import org.apache.commons.compress.utils.InputStreamStatistics;
 
 /**
  * An input stream that decompresses from the BZip2 format to be read as any other stream.
- * 
+ *
  * @NotThreadSafe
  */
-public class BZip2CompressorInputStream extends CompressorInputStream implements
-                                                                          BZip2Constants {
+public class BZip2CompressorInputStream extends CompressorInputStream
+    implements BZip2Constants, InputStreamStatistics {
 
     /**
      * Index of the last char in the block, so the block size == last + 1.
@@ -55,13 +60,11 @@
 
     private boolean blockRandomised;
 
-    private int bsBuff;
-    private int bsLive;
     private final CRC crc = new CRC();
 
     private int nInUse;
 
-    private InputStream in;
+    private BitInputStream bin;
     private final boolean decompressConcatenated;
 
     private static final int EOF = 0;
@@ -99,7 +102,7 @@
      * Constructs a new BZip2CompressorInputStream which decompresses bytes
      * read from the specified stream. This doesn't suppprt decompressing
      * concatenated .bz2 files.
-     * 
+     *
      * @param in the InputStream from which this object should be created
      * @throws IOException
      *             if the stream content is malformed or an I/O error occurs.
@@ -125,7 +128,8 @@
      *             if {@code in == null}, the stream content is malformed, or an I/O error occurs.
      */
     public BZip2CompressorInputStream(final InputStream in, final boolean decompressConcatenated) throws IOException {
-        this.in = in;
+        this.bin = new BitInputStream(in == System.in ? new CloseShieldFilterInputStream(in) : in,
+            ByteOrder.BIG_ENDIAN);
         this.decompressConcatenated = decompressConcatenated;
 
         init(true);
@@ -134,7 +138,7 @@
 
     @Override
     public int read() throws IOException {
-        if (this.in != null) {
+        if (this.bin != null) {
             final int r = read0();
             count(r < 0 ? -1 : 1);
             return r;
@@ -144,7 +148,7 @@
 
     /*
      * (non-Javadoc)
-     * 
+     *
      * @see java.io.InputStream#read(byte[], int, int)
      */
     @Override
@@ -160,7 +164,7 @@
             throw new IndexOutOfBoundsException("offs(" + offs + ") + len("
                                                 + len + ") > dest.length(" + dest.length + ").");
         }
-        if (this.in == null) {
+        if (this.bin == null) {
             throw new IOException("stream closed");
         }
         if (len == 0) {
@@ -175,8 +179,15 @@
             count(1);
         }
 
-        final int c = (destOffs == offs) ? -1 : (destOffs - offs);
-        return c;
+        return (destOffs == offs) ? -1 : (destOffs - offs);
+    }
+
+    /**
+     * @since 1.17
+     */
+    @Override
+    public long getCompressedCount() {
+        return bin.getBytesRead();
     }
 
     private void makeMaps() {
@@ -225,17 +236,26 @@
         }
     }
 
+    private int readNextByte(BitInputStream in) throws IOException {
+        long b = in.readBits(8);
+        return (int) b;
+    }
+
     private boolean init(final boolean isFirstStream) throws IOException {
-        if (null == in) {
+        if (null == bin) {
             throw new IOException("No InputStream");
         }
 
-        final int magic0 = this.in.read();
+        if (!isFirstStream) {
+            bin.clearBitCache();
+        }
+
+        final int magic0 = readNextByte(this.bin);
         if (magic0 == -1 && !isFirstStream) {
             return false;
         }
-        final int magic1 = this.in.read();
-        final int magic2 = this.in.read();
+        final int magic1 = readNextByte(this.bin);
+        final int magic2 = readNextByte(this.bin);
 
         if (magic0 != 'B' || magic1 != 'Z' || magic2 != 'h') {
             throw new IOException(isFirstStream
@@ -243,20 +263,20 @@
                     : "Garbage after a valid BZip2 stream");
         }
 
-        final int blockSize = this.in.read();
+        final int blockSize = readNextByte(this.bin);
         if ((blockSize < '1') || (blockSize > '9')) {
             throw new IOException("BZip2 block size is invalid");
         }
 
         this.blockSize100k = blockSize - '0';
 
-        this.bsLive = 0;
         this.computedCombinedCRC = 0;
 
         return true;
     }
 
     private void initBlock() throws IOException {
+        BitInputStream bin = this.bin;
         char magic0;
         char magic1;
         char magic2;
@@ -266,12 +286,12 @@
 
         while (true) {
             // Get the block magic bytes.
-            magic0 = bsGetUByte();
-            magic1 = bsGetUByte();
-            magic2 = bsGetUByte();
-            magic3 = bsGetUByte();
-            magic4 = bsGetUByte();
-            magic5 = bsGetUByte();
+            magic0 = bsGetUByte(bin);
+            magic1 = bsGetUByte(bin);
+            magic2 = bsGetUByte(bin);
+            magic3 = bsGetUByte(bin);
+            magic4 = bsGetUByte(bin);
+            magic5 = bsGetUByte(bin);
 
             // If isn't end of stream magic, break out of the loop.
             if (magic0 != 0x17 || magic1 != 0x72 || magic2 != 0x45
@@ -297,8 +317,8 @@
             this.currentState = EOF;
             throw new IOException("bad block header");
         }
-        this.storedBlockCRC = bsGetInt();
-        this.blockRandomised = bsR(1) == 1;
+        this.storedBlockCRC = bsGetInt(bin);
+        this.blockRandomised = bsR(bin, 1) == 1;
 
         /**
          * Allocate data here instead in constructor, so we do not allocate
@@ -335,7 +355,7 @@
     }
 
     private boolean complete() throws IOException {
-        this.storedCombinedCRC = bsGetInt();
+        this.storedCombinedCRC = bsGetInt(bin);
         this.currentState = EOF;
         this.data = null;
 
@@ -350,53 +370,51 @@
 
     @Override
     public void close() throws IOException {
-        final InputStream inShadow = this.in;
+        final BitInputStream inShadow = this.bin;
         if (inShadow != null) {
             try {
-                if (inShadow != System.in) {
-                    inShadow.close();
-                }
+                inShadow.close();
             } finally {
                 this.data = null;
-                this.in = null;
+                this.bin = null;
             }
         }
     }
 
-    private int bsR(final int n) throws IOException {
-        int bsLiveShadow = this.bsLive;
-        int bsBuffShadow = this.bsBuff;
-
-        if (bsLiveShadow < n) {
-            final InputStream inShadow = this.in;
-            do {
-                final int thech = inShadow.read();
-
-                if (thech < 0) {
-                    throw new IOException("unexpected end of stream");
-                }
-
-                bsBuffShadow = (bsBuffShadow << 8) | thech;
-                bsLiveShadow += 8;
-            } while (bsLiveShadow < n);
-
-            this.bsBuff = bsBuffShadow;
+    /**
+     * read bits from the input stream
+     * @param n the number of bits to read, must not exceed 32?
+     * @return the requested bits combined into an int
+     * @throws IOException
+     */
+    private static int bsR(BitInputStream bin, final int n) throws IOException {
+        long thech = bin.readBits(n);
+        if (thech < 0) {
+            throw new IOException("unexpected end of stream");
         }
+        return (int) thech;
+    }
 
-        this.bsLive = bsLiveShadow - n;
-        return (bsBuffShadow >> (bsLiveShadow - n)) & ((1 << n) - 1);
+    private static boolean bsGetBit(BitInputStream bin) throws IOException {
+        return bsR(bin, 1) != 0;
     }
 
-    private boolean bsGetBit() throws IOException {
-        return bsR(1) != 0;
+    private static char bsGetUByte(BitInputStream bin) throws IOException {
+        return (char) bsR(bin, 8);
     }
 
-    private char bsGetUByte() throws IOException {
-        return (char) bsR(8);
+    private static int bsGetInt(BitInputStream bin) throws IOException {
+        return bsR(bin, 32);
     }
 
-    private int bsGetInt() throws IOException {
-        return (((((bsR(8) << 8) | bsR(8)) << 8) | bsR(8)) << 8) | bsR(8);
+    private static void checkBounds(final int checkVal, final int limitExclusive, String name)
+        throws IOException {
+        if (checkVal < 0) {
+            throw new IOException("Corrupted input, " + name + " value negative");
+        }
+        if (checkVal >= limitExclusive) {
+            throw new IOException("Corrupted input, " + name + " value too big");
+        }
     }
 
     /**
@@ -404,7 +422,8 @@
      */
     private static void hbCreateDecodeTables(final int[] limit,
                                              final int[] base, final int[] perm, final char[] length,
-                                             final int minLen, final int maxLen, final int alphaSize) {
+                                             final int minLen, final int maxLen, final int alphaSize)
+        throws IOException {
         for (int i = minLen, pp = 0; i <= maxLen; i++) {
             for (int j = 0; j < alphaSize; j++) {
                 if (length[j] == i) {
@@ -419,7 +438,9 @@
         }
 
         for (int i = 0; i < alphaSize; i++) {
-            base[length[i] + 1]++;
+            final int l = length[i];
+            checkBounds(l, MAX_ALPHA_SIZE, "length");
+            base[l + 1]++;
         }
 
         for (int i = 1, b = base[0]; i < MAX_CODE_LEN; i++) {
@@ -441,6 +462,7 @@
     }
 
     private void recvDecodingTables() throws IOException {
+        final BitInputStream bin = this.bin;
         final Data dataShadow = this.data;
         final boolean[] inUse = dataShadow.inUse;
         final byte[] pos = dataShadow.recvDecodingTables_pos;
@@ -451,20 +473,17 @@
 
         /* Receive the mapping table */
         for (int i = 0; i < 16; i++) {
-            if (bsGetBit()) {
+            if (bsGetBit(bin)) {
                 inUse16 |= 1 << i;
             }
         }
 
-        for (int i = 256; --i >= 0;) {
-            inUse[i] = false;
-        }
-
+        Arrays.fill(inUse, false);
         for (int i = 0; i < 16; i++) {
             if ((inUse16 & (1 << i)) != 0) {
                 final int i16 = i << 4;
                 for (int j = 0; j < 16; j++) {
-                    if (bsGetBit()) {
+                    if (bsGetBit(bin)) {
                         inUse[i16 + j] = true;
                     }
                 }
@@ -473,14 +492,16 @@
 
         makeMaps();
         final int alphaSize = this.nInUse + 2;
-
         /* Now the selectors */
-        final int nGroups = bsR(3);
-        final int nSelectors = bsR(15);
+        final int nGroups = bsR(bin, 3);
+        final int nSelectors = bsR(bin, 15);
+        checkBounds(alphaSize, MAX_ALPHA_SIZE + 1, "alphaSize");
+        checkBounds(nGroups, N_GROUPS + 1, "nGroups");
+        checkBounds(nSelectors, MAX_SELECTORS + 1, "nSelectors");
 
         for (int i = 0; i < nSelectors; i++) {
             int j = 0;
-            while (bsGetBit()) {
+            while (bsGetBit(bin)) {
                 j++;
             }
             selectorMtf[i] = (byte) j;
@@ -493,6 +514,7 @@
 
         for (int i = 0; i < nSelectors; i++) {
             int v = selectorMtf[i] & 0xff;
+            checkBounds(v, N_GROUPS, "selectorMtf");
             final byte tmp = pos[v];
             while (v > 0) {
                 // nearly all times v is zero, 4 in most other cases
@@ -507,11 +529,11 @@
 
         /* Now the coding tables */
         for (int t = 0; t < nGroups; t++) {
-            int curr = bsR(5);
+            int curr = bsR(bin, 5);
             final char[] len_t = len[t];
             for (int i = 0; i < alphaSize; i++) {
-                while (bsGetBit()) {
-                    curr += bsGetBit() ? -1 : 1;
+                while (bsGetBit(bin)) {
+                    curr += bsGetBit(bin) ? -1 : 1;
                 }
                 len_t[i] = (char) curr;
             }
@@ -525,7 +547,7 @@
      * Called by recvDecodingTables() exclusively.
      */
     private void createHuffmanDecodingTables(final int alphaSize,
-                                             final int nGroups) {
+                                             final int nGroups) throws IOException {
         final Data dataShadow = this.data;
         final char[][] len = dataShadow.temp_charArray2d;
         final int[] minLens = dataShadow.minLens;
@@ -553,10 +575,10 @@
     }
 
     private void getAndMoveToFrontDecode() throws IOException {
-        this.origPtr = bsR(24);
+        final BitInputStream bin = this.bin;
+        this.origPtr = bsR(bin, 24);
         recvDecodingTables();
 
-        final InputStream inShadow = this.in;
         final Data dataShadow = this.data;
         final byte[] ll8 = dataShadow.ll8;
         final int[] unzftab = dataShadow.unzftab;
@@ -582,11 +604,10 @@
         int groupNo = 0;
         int groupPos = G_SIZE - 1;
         final int eob = this.nInUse + 1;
-        int nextSym = getAndMoveToFrontDecode0(0);
-        int bsBuffShadow = this.bsBuff;
-        int bsLiveShadow = this.bsLive;
+        int nextSym = getAndMoveToFrontDecode0();
         int lastShadow = -1;
         int zt = selector[groupNo] & 0xff;
+        checkBounds(zt, N_GROUPS, "zt");
         int[] base_zt = base[zt];
         int[] limit_zt = limit[zt];
         int[] perm_zt = perm[zt];
@@ -607,7 +628,9 @@
 
                     if (groupPos == 0) {
                         groupPos = G_SIZE - 1;
-                        zt = selector[++groupNo] & 0xff;
+                        checkBounds(++groupNo, MAX_SELECTORS, "groupNo");
+                        zt = selector[groupNo] & 0xff;
+                        checkBounds(zt, N_GROUPS, "zt");
                         base_zt = base[zt];
                         limit_zt = limit[zt];
                         perm_zt = perm[zt];
@@ -617,57 +640,39 @@
                     }
 
                     int zn = minLens_zt;
-
-                    // Inlined:
-                    // int zvec = bsR(zn);
-                    while (bsLiveShadow < zn) {
-                        final int thech = inShadow.read();
-                        if (thech >= 0) {
-                            bsBuffShadow = (bsBuffShadow << 8) | thech;
-                            bsLiveShadow += 8;
-                            continue;
-                        }
-                        throw new IOException("unexpected end of stream");
-                    }
-                    int zvec = (bsBuffShadow >> (bsLiveShadow - zn))
-                        & ((1 << zn) - 1);
-                    bsLiveShadow -= zn;
-
-                    while (zvec > limit_zt[zn]) {
-                        zn++;
-                        while (bsLiveShadow < 1) {
-                            final int thech = inShadow.read();
-                            if (thech >= 0) {
-                                bsBuffShadow = (bsBuffShadow << 8) | thech;
-                                bsLiveShadow += 8;
-                                continue;
-                            }
-                            throw new IOException(
-                                                  "unexpected end of stream");
-                        }
-                        bsLiveShadow--;
-                        zvec = (zvec << 1)
-                            | ((bsBuffShadow >> bsLiveShadow) & 1);
+                    checkBounds(zn, MAX_ALPHA_SIZE, "zn");
+                    int zvec = bsR(bin, zn);
+                    while(zvec > limit_zt[zn]) {
+                        checkBounds(++zn, MAX_ALPHA_SIZE, "zn");
+                        zvec = (zvec << 1) | bsR(bin, 1);
                     }
-                    nextSym = perm_zt[zvec - base_zt[zn]];
+                    final int tmp = zvec - base_zt[zn];
+                    checkBounds(tmp, MAX_ALPHA_SIZE, "zvec");
+                    nextSym = perm_zt[tmp];
                 }
 
-                final byte ch = seqToUnseq[yy[0]];
+                final int yy0 = yy[0];
+                checkBounds(yy0, 256, "yy");
+                final byte ch = seqToUnseq[yy0];
                 unzftab[ch & 0xff] += s + 1;
 
-                while (s-- >= 0) {
-                    ll8[++lastShadow] = ch;
-                }
+                final int from = ++lastShadow;
+                lastShadow += s;
+                Arrays.fill(ll8, from, lastShadow + 1, ch);
 
                 if (lastShadow >= limitLast) {
-                    throw new IOException("block overrun");
+                    throw new IOException("block overrun while expanding RLE in MTF, "
+                        + lastShadow + " exceeds " + limitLast);
                 }
             } else {
                 if (++lastShadow >= limitLast) {
-                    throw new IOException("block overrun");
+                    throw new IOException("block overrun in MTF, "
+                        + lastShadow + " exceeds " + limitLast);
                 }
+                checkBounds(nextSym, 256 + 1, "nextSym");
 
                 final char tmp = yy[nextSym - 1];
+                checkBounds(tmp, 256, "yy");
                 unzftab[seqToUnseq[tmp] & 0xff]++;
                 ll8[lastShadow] = seqToUnseq[tmp];
 
@@ -688,7 +693,9 @@
 
                 if (groupPos == 0) {
                     groupPos = G_SIZE - 1;
-                    zt = selector[++groupNo] & 0xff;
+                    checkBounds(++groupNo, MAX_SELECTORS, "groupNo");
+                    zt = selector[groupNo] & 0xff;
+                    checkBounds(zt, N_GROUPS, "zt");
                     base_zt = base[zt];
                     limit_zt = limit[zt];
                     perm_zt = perm[zt];
@@ -698,75 +705,37 @@
                 }
 
                 int zn = minLens_zt;
-
-                // Inlined:
-                // int zvec = bsR(zn);
-                while (bsLiveShadow < zn) {
-                    final int thech = inShadow.read();
-                    if (thech >= 0) {
-                        bsBuffShadow = (bsBuffShadow << 8) | thech;
-                        bsLiveShadow += 8;
-                        continue;
-                    }
-                    throw new IOException("unexpected end of stream");
-                }
-                int zvec = (bsBuffShadow >> (bsLiveShadow - zn))
-                    & ((1 << zn) - 1);
-                bsLiveShadow -= zn;
-
-                while (zvec > limit_zt[zn]) {
-                    zn++;
-                    while (bsLiveShadow < 1) {
-                        final int thech = inShadow.read();
-                        if (thech >= 0) {
-                            bsBuffShadow = (bsBuffShadow << 8) | thech;
-                            bsLiveShadow += 8;
-                            continue;
-                        }
-                        throw new IOException("unexpected end of stream");
-                    }
-                    bsLiveShadow--;
-                    zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1);
-                }
-                nextSym = perm_zt[zvec - base_zt[zn]];
+                checkBounds(zn, MAX_ALPHA_SIZE, "zn");
+                int zvec = bsR(bin, zn);
+                while(zvec > limit_zt[zn]) {
+                    checkBounds(++zn, MAX_ALPHA_SIZE, "zn");
+                    zvec = (zvec << 1) | bsR(bin, 1);
+                }
+                final int idx = zvec - base_zt[zn];
+                checkBounds(idx, MAX_ALPHA_SIZE, "zvec");
+                nextSym = perm_zt[idx];
             }
         }
 
         this.last = lastShadow;
-        this.bsLive = bsLiveShadow;
-        this.bsBuff = bsBuffShadow;
     }
 
-    private int getAndMoveToFrontDecode0(final int groupNo) throws IOException {
-        final InputStream inShadow = this.in;
+    private int getAndMoveToFrontDecode0() throws IOException {
         final Data dataShadow = this.data;
-        final int zt = dataShadow.selector[groupNo] & 0xff;
+        final int zt = dataShadow.selector[0] & 0xff;
+        checkBounds(zt, N_GROUPS, "zt");
         final int[] limit_zt = dataShadow.limit[zt];
         int zn = dataShadow.minLens[zt];
-        int zvec = bsR(zn);
-        int bsLiveShadow = this.bsLive;
-        int bsBuffShadow = this.bsBuff;
-
+        checkBounds(zn, MAX_ALPHA_SIZE, "zn");
+        int zvec = bsR(bin, zn);
         while (zvec > limit_zt[zn]) {
-            zn++;
-            while (bsLiveShadow < 1) {
-                final int thech = inShadow.read();
-
-                if (thech >= 0) {
-                    bsBuffShadow = (bsBuffShadow << 8) | thech;
-                    bsLiveShadow += 8;
-                    continue;
-                }
-                throw new IOException("unexpected end of stream");
-            }
-            bsLiveShadow--;
-            zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1);
+            checkBounds(++zn, MAX_ALPHA_SIZE, "zn");
+            zvec = (zvec << 1) | bsR(bin, 1);
         }
+        final int tmp = zvec - dataShadow.base[zt][zn];
+        checkBounds(tmp, MAX_ALPHA_SIZE, "zvec");
 
-        this.bsLive = bsLiveShadow;
-        this.bsBuff = bsBuffShadow;
-
-        return dataShadow.perm[zt][zvec - dataShadow.base[zt][zn]];
+        return dataShadow.perm[zt][tmp];
     }
 
     private int setupBlock() throws IOException {
@@ -775,7 +744,8 @@
         }
 
         final int[] cftab = this.data.cftab;
-        final int[] tt = this.data.initTT(this.last + 1);
+        final int ttLen = this.last + 1;
+        final int[] tt = this.data.initTT(ttLen);
         final byte[] ll8 = this.data.ll8;
         cftab[0] = 0;
         System.arraycopy(this.data.unzftab, 0, cftab, 1, 256);
@@ -786,7 +756,9 @@
         }
 
         for (int i = 0, lastShadow = this.last; i <= lastShadow; i++) {
-            tt[cftab[ll8[i] & 0xff]++] = i;
+            final int tmp = cftab[ll8[i] & 0xff]++;
+            checkBounds(tmp, ttLen, "tt index");
+            tt[tmp] = i;
         }
 
         if ((this.origPtr < 0) || (this.origPtr >= tt.length)) {
@@ -810,6 +782,7 @@
         if (this.su_i2 <= this.last) {
             this.su_chPrev = this.su_ch2;
             int su_ch2Shadow = this.data.ll8[this.su_tPos] & 0xff;
+            checkBounds(this.su_tPos, this.data.tt.length, "su_tPos");
             this.su_tPos = this.data.tt[this.su_tPos];
             if (this.su_rNToGo == 0) {
                 this.su_rNToGo = Rand.rNums(this.su_rTPos) - 1;
@@ -835,6 +808,7 @@
             this.su_chPrev = this.su_ch2;
             final int su_ch2Shadow = this.data.ll8[this.su_tPos] & 0xff;
             this.su_ch2 = su_ch2Shadow;
+            checkBounds(this.su_tPos, this.data.tt.length, "su_tPos");
             this.su_tPos = this.data.tt[this.su_tPos];
             this.su_i2++;
             this.currentState = NO_RAND_PART_B_STATE;
@@ -854,6 +828,7 @@
             return setupRandPartA();
         } else if (++this.su_count >= 4) {
             this.su_z = (char) (this.data.ll8[this.su_tPos] & 0xff);
+            checkBounds(this.su_tPos, this.data.tt.length, "su_tPos");
             this.su_tPos = this.data.tt[this.su_tPos];
             if (this.su_rNToGo == 0) {
                 this.su_rNToGo = Rand.rNums(this.su_rTPos) - 1;
@@ -892,6 +867,7 @@
             this.su_count = 1;
             return setupNoRandPartA();
         } else if (++this.su_count >= 4) {
+            checkBounds(this.su_tPos, this.data.ll8.length, "su_tPos");
             this.su_z = (char) (this.data.ll8[this.su_tPos] & 0xff);
             this.su_tPos = this.data.tt[this.su_tPos];
             this.su_j2 = 0;
@@ -955,7 +931,7 @@
 
         /**
          * Initializes the {@link #tt} array.
-         * 
+         *
          * This method is called when the required length of the array is known.
          * I don't initialize it at construction time to avoid unneccessary
          * memory allocation when compressing small files.
@@ -978,33 +954,17 @@
 
     /**
      * Checks if the signature matches what is expected for a bzip2 file.
-     * 
+     *
      * @param signature
      *            the bytes to check
      * @param length
      *            the number of bytes to check
      * @return true, if this stream is a bzip2 compressed stream, false otherwise
-     * 
+     *
      * @since 1.1
      */
     public static boolean matches(final byte[] signature, final int length) {
-
-        if (length < 3) {
-            return false;
-        }
-
-        if (signature[0] != 'B') {
-            return false;
-        }
-
-        if (signature[1] != 'Z') {
-            return false;
-        }
-
-        if (signature[2] != 'h') {
-            return false;
-        }
-
-        return true;
+        return length >= 3 && signature[0] == 'B' &&
+                signature[1] == 'Z' && signature[2] == 'h';
     }
 }
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorOutputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorOutputStream.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorOutputStream.java	2018-07-01 09:53:29.000000000 +0000
@@ -344,7 +344,7 @@
     /**
      * Constructs a new {@code BZip2CompressorOutputStream} with a blocksize of 900k.
      *
-     * @param out 
+     * @param out
      *            the destination stream.
      *
      * @throws IOException
@@ -501,8 +501,11 @@
     public void close() throws IOException {
         if (!closed) {
             final OutputStream outShadow = this.out;
-            finish();
-            outShadow.close();
+            try {
+                finish();
+            } finally {
+                outShadow.close();
+            }
         }
     }
 
@@ -516,7 +519,7 @@
 
     /**
      * Writes magic bytes like BZ on the first position of the stream
-     * and bytes indiciating the file-format, which is 
+     * and bytes indiciating the file-format, which is
      * huffmanised, followed by a digit indicating blockSize100k.
      * @throws IOException if the magic bytes could not been written
      */
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/Rand.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/Rand.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/bzip2/Rand.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/bzip2/Rand.java	2018-05-02 20:17:13.000000000 +0000
@@ -81,7 +81,7 @@
 
     /**
      * Return the random number at a specific index.
-     * 
+     *
      * @param i the index
      * @return the random number
      */
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorException.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorException.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorException.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorException.java	2018-05-02 20:17:13.000000000 +0000
@@ -29,7 +29,7 @@
     /**
      * Constructs a new exception with the specified detail message. The cause
      * is not initialized.
-     * 
+     *
      * @param message
      *            the detail message
      */
@@ -39,7 +39,7 @@
 
     /**
      * Constructs a new exception with the specified detail message and cause.
-     * 
+     *
      * @param message
      *            the detail message
      * @param cause
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java	2018-05-23 12:50:54.000000000 +0000
@@ -26,7 +26,7 @@
     /**
      * Increments the counter of already read bytes.
      * Doesn't increment if the EOF has been hit (read == -1)
-     * 
+     *
      * @param read the number of bytes read
      *
      * @since 1.1
@@ -38,7 +38,7 @@
     /**
      * Increments the counter of already read bytes.
      * Doesn't increment if the EOF has been hit (read == -1)
-     * 
+     *
      * @param read the number of bytes read
      */
     protected void count(final long read) {
@@ -49,7 +49,7 @@
 
     /**
      * Decrements the counter of already read bytes.
-     * 
+     *
      * @param pushedBack the number of bytes pushed back.
      * @since 1.7
      */
@@ -77,4 +77,20 @@
     public long getBytesRead() {
         return bytesRead;
     }
+
+    /**
+     * Returns the amount of raw or compressed bytes read by the stream.
+     *
+     * 
      This implementation invokes {@link #getBytesRead}.
      
+     *
+     * 
      Provides half of {@link
+     * org.apache.commons.compress.utils.InputStreamStatistics}
+     * without forcing subclasses to implement the other half.
      
+     *
+     * @return the amount of decompressed bytes returned by the stream
+     * @since 1.17
+     */
+    public long getUncompressedCount() {
+        return getBytesRead();
+    }
 }
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java	2016-12-25 11:57:03.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java	2018-05-02 20:17:13.000000000 +0000
@@ -31,23 +31,34 @@
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import org.apache.commons.compress.compressors.brotli.BrotliCompressorInputStream;
+import org.apache.commons.compress.compressors.brotli.BrotliUtils;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
 import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
+import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream;
+import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorOutputStream;
+import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
+import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorOutputStream;
 import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream;
 import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream;
 import org.apache.commons.compress.compressors.lzma.LZMAUtils;
 import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
 import org.apache.commons.compress.compressors.pack200.Pack200CompressorOutputStream;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
+import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
 import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
 import org.apache.commons.compress.compressors.xz.XZUtils;
 import org.apache.commons.compress.compressors.z.ZCompressorInputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdUtils;
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.commons.compress.utils.Lists;
 import org.apache.commons.compress.utils.ServiceLoaderIterator;
@@ -59,27 +70,27 @@
  * implementations you should extend CompressorStreamFactory and override the
  * appropriate methods (and call their implementation from super of course).
  * 
      
- * 
+ *
  * Example (Compressing a file):
- * 
+ *
  * 
      - * final OutputStream out = new FileOutputStream(output);
+ * final OutputStream out = Files.newOutputStream(output.toPath());
  * CompressorOutputStream cos = new CompressorStreamFactory()
  *         .createCompressorOutputStream(CompressorStreamFactory.BZIP2, out);
- * IOUtils.copy(new FileInputStream(input), cos);
+ * IOUtils.copy(Files.newInputStream(input.toPath()), cos);
  * cos.close();
  * 
      
- * 
+ *
  * Example (Decompressing a file):
- * 
+ *
  * 
      - * final InputStream is = new FileInputStream(input);
+ * final InputStream is = Files.newInputStream(input.toPath());
  * CompressorInputStream in = new CompressorStreamFactory().createCompressorInputStream(CompressorStreamFactory.BZIP2,
  *         is);
- * IOUtils.copy(in, new FileOutputStream(output));
+ * IOUtils.copy(in, Files.newOutputStream(output.toPath()));
  * in.close();
  * 
      
- * 
+ *
  * @Immutable provided that the deprecated method setDecompressConcatenated is
  *            not used.
  * @ThreadSafe even if the deprecated method setDecompressConcatenated is used
@@ -88,18 +99,28 @@
 
     private static final CompressorStreamFactory SINGLETON = new CompressorStreamFactory();
 
+
+
+    /**
+     * Constant (value {@value}) used to identify the BROTLI compression
+     * algorithm.
+     *
+     * @since 1.14
+     */
+    public static final String BROTLI = "br";
+
     /**
      * Constant (value {@value}) used to identify the BZIP2 compression
      * algorithm.
-     * 
+     *
      * @since 1.1
      */
     public static final String BZIP2 = "bzip2";
 
     /**
      * Constant (value {@value}) used to identify the GZIP compression
-     * algorithm. Not supported as an output stream type.
-     * 
+     * algorithm.
+     *
      * @since 1.1
      */
     public static final String GZIP = "gz";
@@ -107,30 +128,29 @@
     /**
      * Constant (value {@value}) used to identify the PACK200 compression
      * algorithm.
-     * 
+     *
      * @since 1.3
      */
     public static final String PACK200 = "pack200";
 
     /**
      * Constant (value {@value}) used to identify the XZ compression method.
-     * 
+     *
      * @since 1.4
      */
     public static final String XZ = "xz";
 
     /**
      * Constant (value {@value}) used to identify the LZMA compression method.
-     * Not supported as an output stream type.
-     * 
+     *
      * @since 1.6
      */
     public static final String LZMA = "lzma";
 
     /**
      * Constant (value {@value}) used to identify the "framed" Snappy
-     * compression method. Not supported as an output stream type.
-     * 
+     * compression method.
+     *
      * @since 1.7
      */
     public static final String SNAPPY_FRAMED = "snappy-framed";
@@ -138,7 +158,7 @@
     /**
      * Constant (value {@value}) used to identify the "raw" Snappy compression
      * method. Not supported as an output stream type.
-     * 
+     *
      * @since 1.7
      */
     public static final String SNAPPY_RAW = "snappy-raw";
@@ -146,19 +166,58 @@
     /**
      * Constant (value {@value}) used to identify the traditional Unix compress
      * method. Not supported as an output stream type.
-     * 
+     *
      * @since 1.7
      */
     public static final String Z = "z";
 
     /**
      * Constant (value {@value}) used to identify the Deflate compress method.
-     * 
+     *
      * @since 1.9
      */
     public static final String DEFLATE = "deflate";
 
     /**
+     * Constant (value {@value}) used to identify the Deflate64 compress method.
+     *
+     * @since 1.16
+     */
+    public static final String DEFLATE64 = "deflate64";
+
+    /**
+     * Constant (value {@value}) used to identify the block LZ4
+     * compression method.
+     *
+     * @since 1.14
+     */
+    public static final String LZ4_BLOCK = "lz4-block";
+
+    /**
+     * Constant (value {@value}) used to identify the frame LZ4
+     * compression method.
+     *
+     * @since 1.14
+     */
+    public static final String LZ4_FRAMED = "lz4-framed";
+
+    /**
+     * Constant (value {@value}) used to identify the Zstandard compression
+     * algorithm. Not supported as an output stream type.
+     *
+     * @since 1.16
+     */
+    public static final String ZSTANDARD = "zstd";
+
+    private static final String YOU_NEED_BROTLI_DEC = youNeed("Google Brotli Dec", "https://github.com/google/brotli/");
+    private static final String YOU_NEED_XZ_JAVA = youNeed("XZ for Java", "https://tukaani.org/xz/java.html");
+    private static final String YOU_NEED_ZSTD_JNI = youNeed("Zstd JNI", "https://github.com/luben/zstd-jni");
+
+    private static String youNeed(String name, String url) {
+        return " In addition to Apache Commons Compress you need the " + name + " library - see " + url;
+    }
+
+    /**
      * Constructs a new sorted map from input stream provider names to provider
      * objects.
      *
@@ -243,7 +302,11 @@
     private static ArrayList findCompressorStreamProviders() {
         return Lists.newArrayList(serviceLoaderIterator());
     }
-    
+
+    public static String getBrotli() {
+        return BROTLI;
+    }
+
     public static String getBzip2() {
         return BZIP2;
     }
@@ -252,6 +315,14 @@
         return DEFLATE;
     }
 
+    /**
+     * @since 1.16
+     * @return the constant {@link #DEFLATE64}
+     */
+    public static String getDeflate64() {
+        return DEFLATE64;
+    }
+
     public static String getGzip() {
         return GZIP;
     }
@@ -284,6 +355,18 @@
         return Z;
     }
 
+    public static String getLZ4Framed() {
+        return LZ4_FRAMED;
+    }
+
+    public static String getLZ4Block() {
+        return LZ4_BLOCK;
+    }
+
+    public static String getZstandard() {
+        return ZSTANDARD;
+    }
+
     static void putAll(final Set names, final CompressorStreamProvider provider,
             final TreeMap map) {
         for (final String name : names) {
@@ -313,7 +396,7 @@
     private SortedMap compressorInputStreamProviders;
 
     private SortedMap compressorOutputStreamProviders;
-    
+
     /**
      * If true, decompress until the end of the input. If false, stop after the
      * first stream and leave the input position to point to the next byte after
@@ -321,45 +404,66 @@
      */
     private volatile boolean decompressConcatenated = false;
 
+    private final int memoryLimitInKb;
     /**
      * Create an instance with the decompress Concatenated option set to false.
      */
     public CompressorStreamFactory() {
         this.decompressUntilEOF = null;
+        this.memoryLimitInKb = -1;
     }
 
     /**
      * Create an instance with the provided decompress Concatenated option.
-     * 
+     *
      * @param decompressUntilEOF
      *            if true, decompress until the end of the input; if false, stop
      *            after the first stream and leave the input position to point
      *            to the next byte after the stream. This setting applies to the
      *            gzip, bzip2 and xz formats only.
-     * @since 1.10
+     *
+     * @param memoryLimitInKb
+     *            Some streams require allocation of potentially significant
+     *            byte arrays/tables, and they can offer checks to prevent OOMs
+     *            on corrupt files.  Set the maximum allowed memory allocation in KBs.
+     *
+     * @since 1.14
      */
-    public CompressorStreamFactory(final boolean decompressUntilEOF) {
-        this.decompressUntilEOF = Boolean.valueOf(decompressUntilEOF);
+    public CompressorStreamFactory(final boolean decompressUntilEOF, final int memoryLimitInKb) {
+        this.decompressUntilEOF = decompressUntilEOF;
         // Also copy to existing variable so can continue to use that as the
         // current value
         this.decompressConcatenated = decompressUntilEOF;
+        this.memoryLimitInKb = memoryLimitInKb;
     }
 
+
     /**
-     * Create an compressor input stream from an input stream, autodetecting the
-     * compressor type from the first few bytes of the stream. The InputStream
-     * must support marks, like BufferedInputStream.
-     * 
-     * @param in
-     *            the input stream
-     * @return the compressor input stream
-     * @throws CompressorException
-     *             if the compressor name is not known
-     * @throws IllegalArgumentException
-     *             if the stream is null or does not support mark
-     * @since 1.1
+     * Create an instance with the provided decompress Concatenated option.
+     *
+     * @param decompressUntilEOF
+     *            if true, decompress until the end of the input; if false, stop
+     *            after the first stream and leave the input position to point
+     *            to the next byte after the stream. This setting applies to the
+     *            gzip, bzip2 and xz formats only.
+     * @since 1.10
      */
-    public CompressorInputStream createCompressorInputStream(final InputStream in) throws CompressorException {
+    public CompressorStreamFactory(final boolean decompressUntilEOF) {
+        this(decompressUntilEOF, -1);
+    }
+
+    /**
+     * Try to detect the type of compressor stream.
+     *
+     * @param in input stream
+     * @return type of compressor stream detected
+     * @throws CompressorException if no compressor stream type was detected
+     *                             or if something else went wrong
+     * @throws IllegalArgumentException if stream is null or does not support mark
+     *
+     * @since 1.14
+     */
+    public static String detect(final InputStream in) throws CompressorException {
         if (in == null) {
             throw new IllegalArgumentException("Stream must not be null.");
         }
@@ -370,63 +474,92 @@
 
         final byte[] signature = new byte[12];
         in.mark(signature.length);
+        int signatureLength = -1;
         try {
-            final int signatureLength = IOUtils.readFully(in, signature);
+            signatureLength = IOUtils.readFully(in, signature);
             in.reset();
+        } catch (IOException e) {
+            throw new CompressorException("IOException while reading signature.", e);
+        }
 
-            if (BZip2CompressorInputStream.matches(signature, signatureLength)) {
-                return new BZip2CompressorInputStream(in, decompressConcatenated);
-            }
+        if (BZip2CompressorInputStream.matches(signature, signatureLength)) {
+            return BZIP2;
+        }
 
-            if (GzipCompressorInputStream.matches(signature, signatureLength)) {
-                return new GzipCompressorInputStream(in, decompressConcatenated);
-            }
+        if (GzipCompressorInputStream.matches(signature, signatureLength)) {
+            return GZIP;
+        }
 
-            if (Pack200CompressorInputStream.matches(signature, signatureLength)) {
-                return new Pack200CompressorInputStream(in);
-            }
+        if (Pack200CompressorInputStream.matches(signature, signatureLength)) {
+            return PACK200;
+        }
 
-            if (FramedSnappyCompressorInputStream.matches(signature, signatureLength)) {
-                return new FramedSnappyCompressorInputStream(in);
-            }
+        if (FramedSnappyCompressorInputStream.matches(signature, signatureLength)) {
+            return SNAPPY_FRAMED;
+        }
 
-            if (ZCompressorInputStream.matches(signature, signatureLength)) {
-                return new ZCompressorInputStream(in);
-            }
+        if (ZCompressorInputStream.matches(signature, signatureLength)) {
+            return Z;
+        }
 
-            if (DeflateCompressorInputStream.matches(signature, signatureLength)) {
-                return new DeflateCompressorInputStream(in);
-            }
+        if (DeflateCompressorInputStream.matches(signature, signatureLength)) {
+            return DEFLATE;
+        }
 
-            if (XZUtils.matches(signature, signatureLength) && XZUtils.isXZCompressionAvailable()) {
-                return new XZCompressorInputStream(in, decompressConcatenated);
-            }
+        if (XZUtils.matches(signature, signatureLength)) {
+            return XZ;
+        }
 
-            if (LZMAUtils.matches(signature, signatureLength) && LZMAUtils.isLZMACompressionAvailable()) {
-                return new LZMACompressorInputStream(in);
-            }
+        if (LZMAUtils.matches(signature, signatureLength)) {
+            return LZMA;
+        }
 
-        } catch (final IOException e) {
-            throw new CompressorException("Failed to detect Compressor from InputStream.", e);
+        if (FramedLZ4CompressorInputStream.matches(signature, signatureLength)) {
+            return LZ4_FRAMED;
+        }
+
+        if (ZstdUtils.matches(signature, signatureLength)) {
+            return ZSTANDARD;
         }
 
         throw new CompressorException("No Compressor found for the stream signature.");
     }
+    /**
+     * Create an compressor input stream from an input stream, autodetecting the
+     * compressor type from the first few bytes of the stream. The InputStream
+     * must support marks, like BufferedInputStream.
+     *
+     * @param in
+     *            the input stream
+     * @return the compressor input stream
+     * @throws CompressorException
+     *             if the compressor name is not known
+     * @throws IllegalArgumentException
+     *             if the stream is null or does not support mark
+     * @since 1.1
+     */
+    public CompressorInputStream createCompressorInputStream(final InputStream in) throws CompressorException {
+        return createCompressorInputStream(detect(in), in);
+    }
 
     /**
      * Creates a compressor input stream from a compressor name and an input
      * stream.
-     * 
+     *
      * @param name
      *            of the compressor, i.e. {@value #GZIP}, {@value #BZIP2},
      *            {@value #XZ}, {@value #LZMA}, {@value #PACK200},
-     *            {@value #SNAPPY_RAW}, {@value #SNAPPY_FRAMED}, {@value #Z} or
-     *            {@value #DEFLATE}
+     *            {@value #SNAPPY_RAW}, {@value #SNAPPY_FRAMED}, {@value #Z},
+     *            {@value #LZ4_BLOCK}, {@value #LZ4_FRAMED}, {@value #ZSTANDARD},
+     *            {@value #DEFLATE64}
+     *            or {@value #DEFLATE}
      * @param in
      *            the input stream
      * @return compressor input stream
      * @throws CompressorException
-     *             if the compressor name is not known
+     *             if the compressor name is not known or not available,
+     *             or if there's an IOException or MemoryLimitException thrown
+     *             during initialization
      * @throws IllegalArgumentException
      *             if the name or input stream is null
      */
@@ -452,12 +585,32 @@
                 return new BZip2CompressorInputStream(in, actualDecompressConcatenated);
             }
 
+            if (BROTLI.equalsIgnoreCase(name)) {
+                if (!BrotliUtils.isBrotliCompressionAvailable()) {
+                    throw new CompressorException("Brotli compression is not available." + YOU_NEED_BROTLI_DEC);
+                }
+                return new BrotliCompressorInputStream(in);
+            }
+
             if (XZ.equalsIgnoreCase(name)) {
-                return new XZCompressorInputStream(in, actualDecompressConcatenated);
+                if (!XZUtils.isXZCompressionAvailable()) {
+                    throw new CompressorException("XZ compression is not available." + YOU_NEED_XZ_JAVA);
+                }
+                return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb);
+            }
+
+            if (ZSTANDARD.equalsIgnoreCase(name)) {
+                if (!ZstdUtils.isZstdCompressionAvailable()) {
+                    throw new CompressorException("Zstandard compression is not available." + YOU_NEED_ZSTD_JNI);
+                }
+                return new ZstdCompressorInputStream(in);
             }
 
             if (LZMA.equalsIgnoreCase(name)) {
-                return new LZMACompressorInputStream(in);
+                if (!LZMAUtils.isLZMACompressionAvailable()) {
+                    throw new CompressorException("LZMA compression is not available" + YOU_NEED_XZ_JAVA);
+                }
+                return new LZMACompressorInputStream(in, memoryLimitInKb);
             }
 
             if (PACK200.equalsIgnoreCase(name)) {
@@ -473,13 +626,25 @@
             }
 
             if (Z.equalsIgnoreCase(name)) {
-                return new ZCompressorInputStream(in);
+                return new ZCompressorInputStream(in, memoryLimitInKb);
             }
 
             if (DEFLATE.equalsIgnoreCase(name)) {
                 return new DeflateCompressorInputStream(in);
             }
 
+            if (DEFLATE64.equalsIgnoreCase(name)) {
+                return new Deflate64CompressorInputStream(in);
+            }
+
+            if (LZ4_BLOCK.equalsIgnoreCase(name)) {
+                return new BlockLZ4CompressorInputStream(in);
+            }
+
+            if (LZ4_FRAMED.equalsIgnoreCase(name)) {
+                return new FramedLZ4CompressorInputStream(in, actualDecompressConcatenated);
+            }
+
         } catch (final IOException e) {
             throw new CompressorException("Could not create CompressorInputStream.", e);
         }
@@ -487,17 +652,19 @@
         if (compressorStreamProvider != null) {
             return compressorStreamProvider.createCompressorInputStream(name, in, actualDecompressConcatenated);
         }
-        
+
         throw new CompressorException("Compressor: " + name + " not found.");
     }
 
     /**
      * Creates an compressor output stream from an compressor name and an output
      * stream.
-     * 
+     *
      * @param name
      *            the compressor name, i.e. {@value #GZIP}, {@value #BZIP2},
-     *            {@value #XZ}, {@value #PACK200} or {@value #DEFLATE}
+     *            {@value #XZ}, {@value #PACK200}, {@value #SNAPPY_FRAMED},
+     *            {@value #LZ4_BLOCK}, {@value #LZ4_FRAMED}, {@value #ZSTANDARD}
+     *            or {@value #DEFLATE}
      * @param out
      *            the output stream
      * @return the compressor output stream
@@ -539,6 +706,21 @@
                 return new DeflateCompressorOutputStream(out);
             }
 
+            if (SNAPPY_FRAMED.equalsIgnoreCase(name)) {
+                return new FramedSnappyCompressorOutputStream(out);
+            }
+
+            if (LZ4_BLOCK.equalsIgnoreCase(name)) {
+                return new BlockLZ4CompressorOutputStream(out);
+            }
+
+            if (LZ4_FRAMED.equalsIgnoreCase(name)) {
+                return new FramedLZ4CompressorOutputStream(out);
+            }
+
+            if (ZSTANDARD.equalsIgnoreCase(name)) {
+                return new ZstdCompressorOutputStream(out);
+            }
         } catch (final IOException e) {
             throw new CompressorException("Could not create CompressorOutputStream", e);
         }
@@ -576,12 +758,13 @@
 
     @Override
     public Set getInputStreamCompressorNames() {
-        return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, SNAPPY_RAW, SNAPPY_FRAMED, Z, DEFLATE);
+        return Sets.newHashSet(GZIP, BROTLI, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, Z, LZ4_BLOCK,
+            LZ4_FRAMED, ZSTANDARD, DEFLATE64);
     }
 
     @Override
     public Set getOutputStreamCompressorNames() {
-        return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, DEFLATE);
+        return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_FRAMED, LZ4_BLOCK, LZ4_FRAMED, ZSTANDARD);
     }
 
     /**
@@ -610,5 +793,5 @@
         }
         this.decompressConcatenated = decompressConcatenated;
     }
-    
+
 }
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorStreamProvider.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorStreamProvider.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/CompressorStreamProvider.java	2016-12-25 11:57:03.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/CompressorStreamProvider.java	2018-05-02 20:17:13.000000000 +0000
@@ -26,7 +26,7 @@
 /**
  * Creates Compressor {@link CompressorInputStream}s and
  * {@link CompressorOutputStream}s.
- * 
+ *
  * @since 1.13
  */
 public interface CompressorStreamProvider {
@@ -34,7 +34,7 @@
     /**
      * Creates a compressor input stream from a compressor name and an input
      * stream.
-     * 
+     *
      * @param name
      *            of the compressor, i.e.
      *            {@value org.apache.commons.compress.compressors.CompressorStreamFactory#GZIP},
@@ -66,7 +66,7 @@
     /**
      * Creates a compressor output stream from an compressor name and an output
      * stream.
-     * 
+     *
      * @param name
      *            the compressor name, i.e.
      *            {@value org.apache.commons.compress.compressors.CompressorStreamFactory#GZIP},
@@ -88,14 +88,14 @@
 
     /**
      * Gets all the input stream compressor names for this provider
-     * 
+     *
      * @return all the input compressor names for this provider
      */
     Set getInputStreamCompressorNames();
 
     /**
      * Gets all the output stream compressor names for this provider
-     * 
+     *
      * @return all the output compressor names for this provider
      */
     Set getOutputStreamCompressorNames();
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStream.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStream.java	2018-05-23 12:50:54.000000000 +0000
@@ -24,18 +24,24 @@
 import java.util.zip.InflaterInputStream;
 
 import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.utils.CountingInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.compress.utils.InputStreamStatistics;
 
 /**
  * Deflate decompressor.
  * @since 1.9
  */
-public class DeflateCompressorInputStream extends CompressorInputStream {
+public class DeflateCompressorInputStream extends CompressorInputStream
+    implements InputStreamStatistics {
+
     private static final int MAGIC_1 = 0x78;
     private static final int MAGIC_2a = 0x01;
     private static final int MAGIC_2b = 0x5e;
     private static final int MAGIC_2c = 0x9c;
     private static final int MAGIC_2d = 0xda;
-    
+
+    private final CountingInputStream countingStream;
     private final InputStream in;
     private final Inflater inflater;
 
@@ -60,9 +66,9 @@
     public DeflateCompressorInputStream(final InputStream inputStream,
                                         final DeflateParameters parameters) {
         inflater = new Inflater(!parameters.withZlibHeader());
-        in = new InflaterInputStream(inputStream, inflater);
+        in = new InflaterInputStream(countingStream = new CountingInputStream(inputStream), inflater);
     }
-    
+
     /** {@inheritDoc} */
     @Override
     public int read() throws IOException {
@@ -82,7 +88,7 @@
     /** {@inheritDoc} */
     @Override
     public long skip(final long n) throws IOException {
-        return in.skip(n);
+        return IOUtils.skip(in, n);
     }
 
     /** {@inheritDoc} */
@@ -100,18 +106,26 @@
             inflater.end();
         }
     }
-    
+
+    /**
+     * @since 1.17
+     */
+    @Override
+    public long getCompressedCount() {
+        return countingStream.getBytesRead();
+    }
+
     /**
      * Checks if the signature matches what is expected for a zlib / deflated file
      *  with the zlib header.
-     * 
+     *
      * @param signature
      *            the bytes to check
      * @param length
      *            the number of bytes to check
      * @return true, if this stream is zlib / deflate compressed with a header
      * stream, false otherwise
-     * 
+     *
      * @since 1.10
      */
     public static boolean matches(final byte[] signature, final int length) {
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java	1970-01-01 00:00:00.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java	2018-07-01 09:53:29.000000000 +0000
@@ -0,0 +1,120 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.utils.InputStreamStatistics;
+
+import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
+
+/**
+ * Deflate64 decompressor.
+ *
+ * @since 1.16
+ * @NotThreadSafe
+ */
+public class Deflate64CompressorInputStream extends CompressorInputStream implements InputStreamStatistics {
+    private InputStream originalStream;
+    private HuffmanDecoder decoder;
+    private long compressedBytesRead;
+    private final byte[] oneByte = new byte[1];
+
+    /**
+     * Constructs a Deflate64CompressorInputStream.
+     *
+     * @param in the stream to read from
+     */
+    public Deflate64CompressorInputStream(InputStream in) {
+        this(new HuffmanDecoder(in));
+        originalStream = in;
+    }
+
+    Deflate64CompressorInputStream(HuffmanDecoder decoder) {
+        this.decoder = decoder;
+    }
+
+    /**
+     * @throws java.io.EOFException if the underlying stream is exhausted before the end of defalted data was reached.
+     */
+    @Override
+    public int read() throws IOException {
+        while (true) {
+            int r = read(oneByte);
+            switch (r) {
+                case 1:
+                    return oneByte[0] & 0xFF;
+                case -1:
+                    return -1;
+                case 0:
+                    continue;
+                default:
+                    throw new IllegalStateException("Invalid return value from read: " + r);
+            }
+        }
+    }
+
+    /**
+     * @throws java.io.EOFException if the underlying stream is exhausted before the end of defalted data was reached.
+     */
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        int read = -1;
+        if (decoder != null) {
+            read = decoder.decode(b, off, len);
+            compressedBytesRead = decoder.getBytesRead();
+            count(read);
+            if (read == -1) {
+                closeDecoder();
+            }
+        }
+        return read;
+    }
+
+    @Override
+    public int available() throws IOException {
+        return decoder != null ? decoder.available() : 0;
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            closeDecoder();
+        } finally {
+            if (originalStream != null) {
+                originalStream.close();
+                originalStream = null;
+            }
+        }
+    }
+
+    /**
+     * @since 1.17
+     */
+    @Override
+    public long getCompressedCount() {
+        return compressedBytesRead;
+    }
+
+    private void closeDecoder() {
+        closeQuietly(decoder);
+        decoder = null;
+    }
+}
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java	1970-01-01 00:00:00.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java	2018-05-23 12:50:54.000000000 +0000
@@ -0,0 +1,541 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import org.apache.commons.compress.utils.BitInputStream;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+
+import static org.apache.commons.compress.compressors.deflate64.HuffmanState.*;
+
+class HuffmanDecoder implements Closeable {
+    
+    /**
+     * 
      +     * --------------------------------------------------------------------
+     * idx  xtra  base     idx  xtra  base     idx  xtra  base
+     * --------------------------------------------------------------------
+     * 257   0     3       267   1   15,16     277   4   67-82
+     * 258   0     4       268   1   17,18     278   4   83-98
+     * 259   0     5       269   2   19-22     279   4   99-114
+     * 260   0     6       270   2   23-26     280   4   115-130
+     * 261   0     7       271   2   27-30     281   5   131-162
+     * 262   0     8       272   2   31-34     282   5   163-194
+     * 263   0     9       273   3   35-42     283   5   195-226
+     * 264   0     10      274   3   43-50     284   5   227-257
+     * 265   1     11,12   275   3   51-58     285   16  3
+     * 266   1     13,14   276   3   59-66
+     * --------------------------------------------------------------------
+     * 
      
+     * value = (base of run length) << 5 | (number of extra bits to read)
+     */
+    private static final short[] RUN_LENGTH_TABLE = {
+            96, 128, 160, 192, 224, 256, 288, 320, 353, 417, 481, 545, 610, 738, 866,
+            994, 1123, 1379, 1635, 1891, 2148, 2660, 3172, 3684, 4197, 5221, 6245, 7269, 112
+    };
+
+    /**
+     * 
      +     * --------------------------------------------------------------------
+     * idx  xtra  dist     idx  xtra  dist       idx  xtra  dist
+     * --------------------------------------------------------------------
+     * 0    0     1        10   4     33-48      20    9   1025-1536
+     * 1    0     2        11   4     49-64      21    9   1537-2048
+     * 2    0     3        12   5     65-96      22   10   2049-3072
+     * 3    0     4        13   5     97-128     23   10   3073-4096
+     * 4    1     5,6      14   6     129-192    24   11   4097-6144
+     * 5    1     7,8      15   6     193-256    25   11   6145-8192
+     * 6    2     9-12     16   7     257-384    26   12   8193-12288
+     * 7    2     13-16    17   7     385-512    27   12   12289-16384
+     * 8    3     17-24    18   8     513-768    28   13   16385-24576
+     * 9    3     25-32    19   8     769-1024   29   13   24577-32768
+     * 30   14   32769-49152
+     * 31   14   49153-65536
+     * --------------------------------------------------------------------
+     * 
      
+     * value = (base of distance) << 4 | (number of extra bits to read)
+     */
+    private static final int[] DISTANCE_TABLE = {
+            16, 32, 48, 64, 81, 113, 146, 210, 275, 403,  // 0-9
+            532, 788, 1045, 1557, 2070, 3094, 4119, 6167, 8216, 12312, // 10-19
+            16409, 24601, 32794, 49178, 65563, 98331, 131100, 196636, 262173, 393245, // 20-29
+            524318, 786462 // 30-31
+    };
+
+    /**
+     * When using dynamic huffman codes the order in which the values are stored
+     * follows the positioning below
+     */
+    private static final int[] CODE_LENGTHS_ORDER =
+            {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
+
+    /**
+     * Huffman Fixed Literal / Distance tables for mode 1
+     */
+    private static final int[] FIXED_LITERALS;
+    private static final int[] FIXED_DISTANCE;
+
+    static {
+        FIXED_LITERALS = new int[288];
+        Arrays.fill(FIXED_LITERALS, 0, 144, 8);
+        Arrays.fill(FIXED_LITERALS, 144, 256, 9);
+        Arrays.fill(FIXED_LITERALS, 256, 280, 7);
+        Arrays.fill(FIXED_LITERALS, 280, 288, 8);
+
+        FIXED_DISTANCE = new int[32];
+        Arrays.fill(FIXED_DISTANCE, 5);
+    }
+
+    private boolean finalBlock = false;
+    private DecoderState state;
+    private BitInputStream reader;
+    private final InputStream in;
+
+    private final DecodingMemory memory = new DecodingMemory();
+
+    HuffmanDecoder(InputStream in) {
+        this.reader = new BitInputStream(in, ByteOrder.LITTLE_ENDIAN);
+        this.in = in;
+        state = new InitialState();
+    }
+
+    @Override
+    public void close() {
+        state = new InitialState();
+        reader = null;
+    }
+
+    public int decode(byte[] b) throws IOException {
+        return decode(b, 0, b.length);
+    }
+
+    public int decode(byte[] b, int off, int len) throws IOException {
+        while (!finalBlock || state.hasData()) {
+            if (state.state() == INITIAL) {
+                finalBlock = readBits(1) == 1;
+                int mode = (int) readBits(2);
+                switch (mode) {
+                case 0:
+                    switchToUncompressedState();
+                    break;
+                case 1:
+                    state = new HuffmanCodes(FIXED_CODES, FIXED_LITERALS, FIXED_DISTANCE);
+                    break;
+                case 2:
+                    int[][] tables = readDynamicTables();
+                    state = new HuffmanCodes(DYNAMIC_CODES, tables[0], tables[1]);
+                    break;
+                default:
+                    throw new IllegalStateException("Unsupported compression: " + mode);
+                }
+            } else {
+                return state.read(b, off, len);
+            }
+        }
+        return -1;
+    }
+
+    /**
+     * @since 1.17
+     */
+    long getBytesRead() {
+        return reader.getBytesRead();
+    }
+
+    private void switchToUncompressedState() throws IOException {
+        reader.alignWithByteBoundary();
+        long bLen = readBits(16);
+        long bNLen = readBits(16);
+        if (((bLen ^ 0xFFFF) & 0xFFFF) != bNLen) {
+            //noinspection DuplicateStringLiteralInspection
+            throw new IllegalStateException("Illegal LEN / NLEN values");
+        }
+        state = new UncompressedState(bLen);
+    }
+
+    private int[][] readDynamicTables() throws IOException {
+        int[][] result = new int[2][];
+        int literals = (int) (readBits(5) + 257);
+        result[0] = new int[literals];
+
+        int distances = (int) (readBits(5) + 1);
+        result[1] = new int[distances];
+
+        populateDynamicTables(reader, result[0], result[1]);
+        return result;
+    }
+
+    int available() throws IOException {
+        return state.available();
+    }
+
+    private abstract static class DecoderState {
+        abstract HuffmanState state();
+
+        abstract int read(byte[] b, int off, int len) throws IOException;
+
+        abstract boolean hasData();
+
+        abstract int available() throws IOException ;
+    }
+
+    private class UncompressedState extends DecoderState {
+        private final long blockLength;
+        private long read;
+
+        private UncompressedState(long blockLength) {
+            this.blockLength = blockLength;
+        }
+
+        @Override
+        HuffmanState state() {
+            return read < blockLength ? STORED : INITIAL;
+        }
+
+        @Override
+        int read(byte[] b, int off, int len) throws IOException {
+            // as len is an int and (blockLength - read) is >= 0 the min must fit into an int as well
+            int max = (int) Math.min(blockLength - read, len);
+            int readSoFar = 0;
+            while (readSoFar < max) {
+                int readNow;
+                if (reader.bitsCached() > 0) {
+                    byte next = (byte) readBits(Byte.SIZE);
+                    b[off + readSoFar] = memory.add(next);
+                    readNow = 1;
+                } else {
+                    readNow = in.read(b, off + readSoFar, max - readSoFar);
+                    if (readNow == -1) {
+                        throw new EOFException("Truncated Deflate64 Stream");
+                    }
+                    memory.add(b, off + readSoFar, readNow);
+                }
+                read += readNow;
+                readSoFar += readNow;
+            }
+            return max;
+        }
+
+        @Override
+        boolean hasData() {
+            return read < blockLength;
+        }
+
+        @Override
+        int available() throws IOException {
+            return (int) Math.min(blockLength - read, reader.bitsAvailable() / Byte.SIZE);
+        }
+    }
+
+    private class InitialState extends DecoderState {
+        @Override
+        HuffmanState state() {
+            return INITIAL;
+        }
+
+        @Override
+        int read(byte[] b, int off, int len) throws IOException {
+            throw new IllegalStateException("Cannot read in this state");
+        }
+
+        @Override
+        boolean hasData() {
+            return false;
+        }
+
+        @Override
+        int available() {
+            return 0;
+        }
+    }
+
+    private class HuffmanCodes extends DecoderState {
+        private boolean endOfBlock = false;
+        private final HuffmanState state;
+        private final BinaryTreeNode lengthTree;
+        private final BinaryTreeNode distanceTree;
+
+        private int runBufferPos = 0;
+        private byte[] runBuffer = new byte[0];
+        private int runBufferLength = 0;
+
+        HuffmanCodes(HuffmanState state, int[] lengths, int[] distance) {
+            this.state = state;
+            lengthTree = buildTree(lengths);
+            distanceTree = buildTree(distance);
+        }
+
+        @Override
+        HuffmanState state() {
+            return endOfBlock ? INITIAL : state;
+        }
+
+        @Override
+        int read(byte[] b, int off, int len) throws IOException {
+            return decodeNext(b, off, len);
+        }
+
+        private int decodeNext(byte[] b, int off, int len) throws IOException {
+            if (endOfBlock) {
+                return -1;
+            }
+            int result = copyFromRunBuffer(b, off, len);
+
+            while (result < len) {
+                int symbol = nextSymbol(reader, lengthTree);
+                if (symbol < 256) {
+                    b[off + result++] = memory.add((byte) symbol);
+                } else if (symbol > 256) {
+                    int runMask = RUN_LENGTH_TABLE[symbol - 257];
+                    int run = runMask >>> 5;
+                    int runXtra = runMask & 0x1F;
+                    run += readBits(runXtra);
+
+                    int distSym = nextSymbol(reader, distanceTree);
+
+                    int distMask = DISTANCE_TABLE[distSym];
+                    int dist = distMask >>> 4;
+                    int distXtra = distMask & 0xF;
+                    dist += readBits(distXtra);
+
+                    if (runBuffer.length < run) {
+                        runBuffer = new byte[run];
+                    }
+                    runBufferLength = run;
+                    runBufferPos = 0;
+                    memory.recordToBuffer(dist, run, runBuffer);
+
+                    result += copyFromRunBuffer(b, off + result, len - result);
+                } else {
+                    endOfBlock = true;
+                    return result;
+                }
+            }
+
+            return result;
+        }
+
+        private int copyFromRunBuffer(byte[] b, int off, int len) {
+            int bytesInBuffer = runBufferLength - runBufferPos;
+            int copiedBytes = 0;
+            if (bytesInBuffer > 0) {
+                copiedBytes = Math.min(len, bytesInBuffer);
+                System.arraycopy(runBuffer, runBufferPos, b, off, copiedBytes);
+                runBufferPos += copiedBytes;
+            }
+            return copiedBytes;
+        }
+
+        @Override
+        boolean hasData() {
+            return !endOfBlock;
+        }
+
+        @Override
+        int available() {
+            return runBufferLength - runBufferPos;
+        }
+    }
+
+    private static int nextSymbol(BitInputStream reader, BinaryTreeNode tree) throws IOException {
+        BinaryTreeNode node = tree;
+        while (node != null && node.literal == -1) {
+            long bit = readBits(reader, 1);
+            node = bit == 0 ? node.leftNode : node.rightNode;
+        }
+        return node != null ? node.literal : -1;
+    }
+
+    private static void populateDynamicTables(BitInputStream reader, int[] literals, int[] distances) throws IOException {
+        int codeLengths = (int) (readBits(reader, 4) + 4);
+
+        int[] codeLengthValues = new int[19];
+        for (int cLen = 0; cLen < codeLengths; cLen++) {
+            codeLengthValues[CODE_LENGTHS_ORDER[cLen]] = (int) readBits(reader, 3);
+        }
+
+        BinaryTreeNode codeLengthTree = buildTree(codeLengthValues);
+
+        final int[] auxBuffer = new int[literals.length + distances.length];
+
+        int value = -1;
+        int length = 0;
+        int off = 0;
+        while (off < auxBuffer.length) {
+            if (length > 0) {
+                auxBuffer[off++] = value;
+                length--;
+            } else {
+                int symbol = nextSymbol(reader, codeLengthTree);
+                if (symbol < 16) {
+                    value = symbol;
+                    auxBuffer[off++] = value;
+                } else if (symbol == 16) {
+                    length = (int) (readBits(reader, 2) + 3);
+                } else if (symbol == 17) {
+                    value = 0;
+                    length = (int) (readBits(reader, 3) + 3);
+                } else if (symbol == 18) {
+                    value = 0;
+                    length = (int) (readBits(reader, 7) + 11);
+                }
+            }
+        }
+
+        System.arraycopy(auxBuffer, 0, literals, 0, literals.length);
+        System.arraycopy(auxBuffer, literals.length, distances, 0, distances.length);
+    }
+
+    private static class BinaryTreeNode {
+        private final int bits;
+        int literal = -1;
+        BinaryTreeNode leftNode;
+        BinaryTreeNode rightNode;
+
+        private BinaryTreeNode(int bits) {
+            this.bits = bits;
+        }
+
+        void leaf(int symbol) {
+            literal = symbol;
+            leftNode = null;
+            rightNode = null;
+        }
+
+        BinaryTreeNode left() {
+            if (leftNode == null && literal == -1) {
+                leftNode = new BinaryTreeNode(bits + 1);
+            }
+            return leftNode;
+        }
+
+        BinaryTreeNode right() {
+            if (rightNode == null && literal == -1) {
+                rightNode = new BinaryTreeNode(bits + 1);
+            }
+            return rightNode;
+        }
+    }
+
+    private static BinaryTreeNode buildTree(int[] litTable) {
+        int[] literalCodes = getCodes(litTable);
+
+        BinaryTreeNode root = new BinaryTreeNode(0);
+
+        for (int i = 0; i < litTable.length; i++) {
+            int len = litTable[i];
+            if (len != 0) {
+                BinaryTreeNode node = root;
+                int lit = literalCodes[len - 1];
+                for (int p = len - 1; p >= 0; p--) {
+                    int bit = lit & (1 << p);
+                    node = bit == 0 ? node.left() : node.right();
+                }
+                node.leaf(i);
+                literalCodes[len - 1]++;
+            }
+        }
+        return root;
+    }
+
+    private static int[] getCodes(int[] litTable) {
+        int max = 0;
+        int[] blCount = new int[65];
+
+        for (int aLitTable : litTable) {
+            max = Math.max(max, aLitTable);
+            blCount[aLitTable]++;
+        }
+        blCount = Arrays.copyOf(blCount, max + 1);
+
+        int code = 0;
+        int[] nextCode = new int[max + 1];
+        for (int i = 0; i <= max; i++) {
+            code = (code + blCount[i]) << 1;
+            nextCode[i] = code;
+        }
+
+        return nextCode;
+    }
+
+    private static class DecodingMemory {
+        private final byte[] memory;
+        private final int mask;
+        private int wHead;
+        private boolean wrappedAround;
+
+        private DecodingMemory() {
+            this(16);
+        }
+
+        private DecodingMemory(int bits) {
+            memory = new byte[1 << bits];
+            mask = memory.length - 1;
+        }
+
+        byte add(byte b) {
+            memory[wHead] = b;
+            wHead = incCounter(wHead);
+            return b;
+        }
+
+        void add(byte[] b, int off, int len) {
+            for (int i = off; i < off + len; i++) {
+                add(b[i]);
+            }
+        }
+
+        void recordToBuffer(int distance, int length, byte[] buff) {
+            if (distance > memory.length) {
+                throw new IllegalStateException("Illegal distance parameter: " + distance);
+            }
+            int start = (wHead - distance) & mask;
+            if (!wrappedAround && start >= wHead) {
+                throw new IllegalStateException("Attempt to read beyond memory: dist=" + distance);
+            }
+            for (int i = 0, pos = start; i < length; i++, pos = incCounter(pos)) {
+                buff[i] = add(memory[pos]);
+            }
+        }
+
+        private int incCounter(int counter) {
+            final int newCounter = (counter + 1) & mask;
+            if (!wrappedAround && newCounter < counter) {
+                wrappedAround = true;
+            }
+            return newCounter;
+        }
+    }
+
+    private long readBits(int numBits) throws IOException {
+        return readBits(reader, numBits);
+    }
+
+    private static long readBits(BitInputStream reader, int numBits) throws IOException {
+        long r = reader.readBits(numBits);
+        if (r == -1) {
+            throw new EOFException("Truncated Deflate64 Stream");
+        }
+        return r;
+    }
+}
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java	1970-01-01 00:00:00.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java	2018-05-02 20:17:13.000000000 +0000
@@ -0,0 +1,25 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+enum HuffmanState {
+    INITIAL,
+    STORED,
+    DYNAMIC_CODES,
+    FIXED_CODES
+}
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html	1970-01-01 00:00:00.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html	2018-05-02 20:17:13.000000000 +0000
@@ -0,0 +1,25 @@
+
+
+  
+    
      Provides a stream that allows decompressing streams using the
+      DEFLATE64(tm) algorithm. DEFLATE64 is a trademark of PKWARE,
+      Inc.
      
+  
+
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java	2018-05-23 12:50:54.000000000 +0000
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.EOFException;
 import java.io.InputStream;
+import java.io.DataInput;
 import java.io.DataInputStream;
 import java.io.BufferedInputStream;
 import java.util.zip.DataFormatException;
@@ -30,24 +31,51 @@
 import java.util.zip.CRC32;
 
 import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.utils.ByteUtils;
 import org.apache.commons.compress.utils.CharsetNames;
+import org.apache.commons.compress.utils.CountingInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.compress.utils.InputStreamStatistics;
 
 /**
  * Input stream that decompresses .gz files.
- * This supports decompressing concatenated .gz files which is important
- * when decompressing standalone .gz files.
+ *
+ * 
      This supports decompressing concatenated .gz files which is important
+ * when decompressing standalone .gz files.
      
+ *
  * 
      
  * {@link java.util.zip.GZIPInputStream} doesn't decompress concatenated .gz
  * files: it stops after the first member and silently ignores the rest.
  * It doesn't leave the read position to point to the beginning of the next
  * member, which makes it difficult workaround the lack of concatenation
  * support.
+ * 
      
+ *
  * 
      
  * Instead of using GZIPInputStream, this class has its own .gz
  * container format decoder. The actual decompression is done with
  * {@link java.util.zip.Inflater}.
+ * 
      
+ *
+ * 
      If you use the constructor {@code GzipCompressorInputStream(in)}
+ * or {@code GzipCompressorInputStream(in, false)} with some {@code
+ * InputStream} {@code in} then {@link #read} will return -1 as soon
+ * as the first internal member has been read completely. The stream
+ * {@code in} will be positioned at the start of the second gzip
+ * member if there is one.
      
+ *
+ * 
      If you use the constructor {@code GzipCompressorInputStream(in,
+ * true)} with some {@code InputStream} {@code in} then {@link #read}
+ * will return -1 once the stream {@code in} has been exhausted. The
+ * data read from a stream constructed this way will consist of the
+ * concatenated data of all gzip members contained inside {@code
+ * in}.
      
+ *
+ * @see "https://tools.ietf.org/html/rfc1952"
  */
-public class GzipCompressorInputStream extends CompressorInputStream {
+public class GzipCompressorInputStream extends CompressorInputStream
+    implements InputStreamStatistics {
+
     // Header flags
     // private static final int FTEXT = 0x01; // Uninteresting for us
     private static final int FHCRC = 0x02;
@@ -56,17 +84,20 @@
     private static final int FCOMMENT = 0x10;
     private static final int FRESERVED = 0xE0;
 
-    // Compressed input stream, possibly wrapped in a BufferedInputStream
+    private final CountingInputStream countingStream;
+
+    // Compressed input stream, possibly wrapped in a
+    // BufferedInputStream, always wrapped in countingStream above
     private final InputStream in;
 
-    // True if decompressing multimember streams.
+    // True if decompressing multi member streams.
     private final boolean decompressConcatenated;
 
     // Buffer to hold the input data
     private final byte[] buf = new byte[8192];
 
     // Amount of data in buf.
-    private int bufUsed = 0;
+    private int bufUsed;
 
     // Decompressor
     private Inflater inf = new Inflater(true);
@@ -123,12 +154,13 @@
     public GzipCompressorInputStream(final InputStream inputStream,
                                      final boolean decompressConcatenated)
             throws IOException {
+        countingStream = new CountingInputStream(inputStream);
         // Mark support is strictly needed for concatenated files only,
         // but it's simpler if it is always available.
-        if (inputStream.markSupported()) {
-            in = inputStream;
+        if (countingStream.markSupported()) {
+            in = countingStream;
         } else {
-            in = new BufferedInputStream(inputStream);
+            in = new BufferedInputStream(countingStream);
         }
 
         this.decompressConcatenated = decompressConcatenated;
@@ -165,7 +197,7 @@
         }
 
         // Parsing the rest of the header may throw EOFException.
-        final DataInputStream inData = new DataInputStream(in);
+        final DataInput inData = new DataInputStream(in);
         final int method = inData.readUnsignedByte();
         if (method != Deflater.DEFLATED) {
             throw new IOException("Unsupported compression method "
@@ -178,7 +210,7 @@
                     "Reserved flags are set in the .gz header");
         }
 
-        parameters.setModificationTime(readLittleEndianInt(inData) * 1000);
+        parameters.setModificationTime(ByteUtils.fromLittleEndian(inData, 4) * 1000);
         switch (inData.readUnsignedByte()) { // extra flags
         case 2:
             parameters.setCompressionLevel(Deflater.BEST_COMPRESSION);
@@ -233,7 +265,7 @@
         return true;
     }
 
-    private byte[] readToNull(final DataInputStream inData) throws IOException {
+    private static byte[] readToNull(final DataInput inData) throws IOException {
         final ByteArrayOutputStream bos = new ByteArrayOutputStream();
         int b = 0;
         while ((b = inData.readUnsignedByte()) != 0x00) { // NOPMD
@@ -242,13 +274,6 @@
         return bos.toByteArray();
     }
 
-    private long readLittleEndianInt(final DataInputStream inData) throws IOException {
-        return inData.readUnsignedByte()
-            | (inData.readUnsignedByte() << 8)
-            | (inData.readUnsignedByte() << 16)
-            | (((long) inData.readUnsignedByte()) << 24);
-    }
-
     @Override
     public int read() throws IOException {
         return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF;
@@ -299,20 +324,20 @@
                 // position to match the actual amount used.
                 //
                 // NOTE: The "if" is there just in case. Since we used
-                // in.mark earler, it should always skip enough.
+                // in.mark earlier, it should always skip enough.
                 in.reset();
 
                 final int skipAmount = bufUsed - inf.getRemaining();
-                if (in.skip(skipAmount) != skipAmount) {
+                if (IOUtils.skip(in, skipAmount) != skipAmount) {
                     throw new IOException();
                 }
 
                 bufUsed = 0;
 
-                final DataInputStream inData = new DataInputStream(in);
+                final DataInput inData = new DataInputStream(in);
 
                 // CRC32
-                final long crcStored = readLittleEndianInt(inData);
+                final long crcStored = ByteUtils.fromLittleEndian(inData, 4);
 
                 if (crcStored != crc.getValue()) {
                     throw new IOException("Gzip-compressed data is corrupt "
@@ -320,9 +345,9 @@
                 }
 
                 // Uncompressed size modulo 2^32 (ISIZE in the spec)
-                final long isize = readLittleEndianInt(inData);
+                final long isize = ByteUtils.fromLittleEndian(inData, 4);
 
-                if (isize != (inf.getBytesWritten() & 0xffffffffl)) {
+                if (isize != (inf.getBytesWritten() & 0xffffffffL)) {
                     throw new IOException("Gzip-compressed data is corrupt"
                                           + "(uncompressed size mismatch)");
                 }
@@ -350,20 +375,7 @@
      * @since 1.1
      */
     public static boolean matches(final byte[] signature, final int length) {
-
-        if (length < 2) {
-            return false;
-        }
-
-        if (signature[0] != 31) {
-            return false;
-        }
-
-        if (signature[1] != -117) {
-            return false;
-        }
-
-        return true;
+        return length >= 2 && signature[0] == 31 && signature[1] == -117;
     }
 
     /**
@@ -382,4 +394,12 @@
             this.in.close();
         }
     }
+
+    /**
+     * @since 1.17
+     */
+    @Override
+    public long getCompressedCount() {
+        return countingStream.getBytesRead();
+    }
 }
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java	2018-07-01 09:53:29.000000000 +0000
@@ -35,8 +35,8 @@
  * over the standard {@link GZIPOutputStream} class by allowing
  * the configuration of the compression level and the header metadata (filename,
  * comment, modification time, operating system and extra flags).
- * 
- * @see GZIP File Format Specification
+ *
+ * @see GZIP File Format Specification
  */
 public class GzipCompressorOutputStream extends CompressorOutputStream {
 
@@ -75,27 +75,27 @@
      * @param out the stream to compress to
      * @param parameters the parameters to use
      * @throws IOException if writing fails
-     * 
+     *
      * @since 1.7
      */
     public GzipCompressorOutputStream(final OutputStream out, final GzipParameters parameters) throws IOException {
         this.out = out;
         this.deflater = new Deflater(parameters.getCompressionLevel(), true);
-        
+
         writeHeader(parameters);
     }
 
     private void writeHeader(final GzipParameters parameters) throws IOException {
         final String filename = parameters.getFilename();
         final String comment = parameters.getComment();
-        
+
         final ByteBuffer buffer = ByteBuffer.allocate(10);
         buffer.order(ByteOrder.LITTLE_ENDIAN);
         buffer.putShort((short) GZIPInputStream.GZIP_MAGIC);
         buffer.put((byte) Deflater.DEFLATED); // compression method (8: deflate)
         buffer.put((byte) ((filename != null ? FNAME : 0) | (comment != null ? FCOMMENT : 0))); // flags
         buffer.putInt((int) (parameters.getModificationTime() / 1000));
-        
+
         // extra flags
         final int compressionLevel = parameters.getCompressionLevel();
         if (compressionLevel == Deflater.BEST_COMPRESSION) {
@@ -105,16 +105,16 @@
         } else {
             buffer.put((byte) 0);
         }
-        
+
         buffer.put((byte) parameters.getOperatingSystem());
-        
+
         out.write(buffer.array());
-        
+
         if (filename != null) {
             out.write(filename.getBytes(CharsetNames.ISO_8859_1));
             out.write(0);
         }
-        
+
         if (comment != null) {
             out.write(comment.getBytes(CharsetNames.ISO_8859_1));
             out.write(0);
@@ -137,7 +137,7 @@
 
     /**
      * {@inheritDoc}
-     * 
+     *
      * @since 1.1
      */
     @Override
@@ -147,7 +147,7 @@
 
     /**
      * {@inheritDoc}
-     * 
+     *
      * @since 1.1
      */
     @Override
@@ -157,11 +157,11 @@
 
         } else if (length > 0) {
             deflater.setInput(buffer, offset, length);
-            
+
             while (!deflater.needsInput()) {
                 deflate();
             }
-            
+
             crc.update(buffer, offset, length);
         }
     }
@@ -175,7 +175,7 @@
 
     /**
      * Finishes writing compressed data to the underlying stream without closing it.
-     * 
+     *
      * @since 1.7
      * @throws IOException on error
      */
@@ -186,14 +186,14 @@
             while (!deflater.finished()) {
                 deflate();
             }
-            
+
             writeTrailer();
         }
     }
 
     /**
      * {@inheritDoc}
-     * 
+     *
      * @since 1.7
      */
     @Override
@@ -204,10 +204,13 @@
     @Override
     public void close() throws IOException {
         if (!closed) {
-            finish();
-            deflater.end();
-            out.close();
-            closed = true;
+            try {
+                finish();
+            } finally {
+                deflater.end();
+                out.close();
+                closed = true;
+            }
         }
     }
 
diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java
--- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java	2016-06-18 15:07:49.000000000 +0000
+++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java	2018-05-02 20:17:13.000000000 +0000
@@ -23,7 +23,7 @@
 
 /**
  * Parameters for the GZIP compressor.
- * 
+ *
  * @since 1.7
  */
 public class GzipParameters {
@@ -40,8 +40,8 @@
 
     /**
      * Sets the compression level.
-     * 
-     * @param compressionLevel the compression level (between 0 and 9) 
+     *
+     * @param compressionLevel the compression level (between 0 and 9)
      * @see Deflater#NO_COMPRESSION
      * @see Deflater#BEST_SPEED
      * @see Deflater#DEFAULT_COMPRESSION
@@ -60,7 +60,7 @@
 
     /**
      * Sets the modification time of the compressed file.
-     * 
+     *
      * @param modificationTime the modification time, in milliseconds
      */
     public void setModificationTime(final long modificationTime) {
@@ -73,7 +73,7 @@
 
     /**
      * Sets the name of the compressed file.
-     * 
+     *
      * @param filename the name of the file without the directory path
      */
     public void setFilename(final String filename) {
@@ -112,7 +112,7 @@
      *   
    • 13: Acorn RISCOS
      • 
      *   
    • 255: Unknown
      • 
      *
    - * + * * @param operatingSystem the code of the operating system */ public void setOperatingSystem(final int operatingSystem) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.compress.compressors.lz77support.AbstractLZ77CompressorInputStream; +import org.apache.commons.compress.utils.ByteUtils; + +/** + * CompressorInputStream for the LZ4 block format. + * + * @see LZ4 Block Format Description + * @since 1.14 + * @NotThreadSafe + */ +public class BlockLZ4CompressorInputStream extends AbstractLZ77CompressorInputStream { + + static final int WINDOW_SIZE = 1 << 16; + static final int SIZE_BITS = 4; + static final int BACK_REFERENCE_SIZE_MASK = (1 << SIZE_BITS) - 1; + static final int LITERAL_SIZE_MASK = BACK_REFERENCE_SIZE_MASK << SIZE_BITS; + + /** Back-Reference-size part of the block starting byte. */ + private int nextBackReferenceSize; + + /** Current state of the stream */ + private State state = State.NO_BLOCK; + + /** + * Creates a new LZ4 input stream. + * + * @param is + * An InputStream to read compressed data from + * + * @throws IOException if reading fails + */ + public BlockLZ4CompressorInputStream(final InputStream is) throws IOException { + super(is, WINDOW_SIZE); + } + + /** + * {@inheritDoc} + */ + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + switch (state) { + case EOF: + return -1; + case NO_BLOCK: // NOSONAR - fallthrough intended + readSizes(); + /*FALLTHROUGH*/ + case IN_LITERAL: + int litLen = readLiteral(b, off, len); + if (!hasMoreDataInBlock()) { + state = State.LOOKING_FOR_BACK_REFERENCE; + } + return litLen > 0 ? litLen : read(b, off, len); + case LOOKING_FOR_BACK_REFERENCE: // NOSONAR - fallthrough intended + if (!initializeBackReference()) { + state = State.EOF; + return -1; + } + /*FALLTHROUGH*/ + case IN_BACK_REFERENCE: + int backReferenceLen = readBackReference(b, off, len); + if (!hasMoreDataInBlock()) { + state = State.NO_BLOCK; + } + return backReferenceLen > 0 ? backReferenceLen : read(b, off, len); + default: + throw new IOException("Unknown stream state " + state); + } + } + + private void readSizes() throws IOException { + int nextBlock = readOneByte(); + if (nextBlock == -1) { + throw new IOException("Premature end of stream while looking for next block"); + } + nextBackReferenceSize = nextBlock & BACK_REFERENCE_SIZE_MASK; + long literalSizePart = (nextBlock & LITERAL_SIZE_MASK) >> SIZE_BITS; + if (literalSizePart == BACK_REFERENCE_SIZE_MASK) { + literalSizePart += readSizeBytes(); + } + startLiteral(literalSizePart); + state = State.IN_LITERAL; + } + + private long readSizeBytes() throws IOException { + long accum = 0; + int nextByte; + do { + nextByte = readOneByte(); + if (nextByte == -1) { + throw new IOException("Premature end of stream while parsing length"); + } + accum += nextByte; + } while (nextByte == 255); + return accum; + } + + /** + * @return false if there is no more back-reference - this means this is the + * last block of the stream. + */ + private boolean initializeBackReference() throws IOException { + int backReferenceOffset = 0; + try { + backReferenceOffset = (int) ByteUtils.fromLittleEndian(supplier, 2); + } catch (IOException ex) { + if (nextBackReferenceSize == 0) { // the last block has no back-reference + return false; + } + throw ex; + } + long backReferenceSize = nextBackReferenceSize; + if (nextBackReferenceSize == BACK_REFERENCE_SIZE_MASK) { + backReferenceSize += readSizeBytes(); + } + // minimal match length 4 is encoded as 0 + startBackReference(backReferenceOffset, backReferenceSize + 4); + state = State.IN_BACK_REFERENCE; + return true; + } + + private enum State { + NO_BLOCK, IN_LITERAL, LOOKING_FOR_BACK_REFERENCE, IN_BACK_REFERENCE, EOF + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -0,0 +1,508 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Deque; +import java.util.Iterator; +import java.util.LinkedList; + +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.lz77support.LZ77Compressor; +import org.apache.commons.compress.compressors.lz77support.Parameters; +import org.apache.commons.compress.utils.ByteUtils; + +/** + * CompressorOutputStream for the LZ4 block format. + * + * @see LZ4 Block Format Description + * @since 1.14 + * @NotThreadSafe + */ +public class BlockLZ4CompressorOutputStream extends CompressorOutputStream { + + private static final int MIN_BACK_REFERENCE_LENGTH = 4; + private static final int MIN_OFFSET_OF_LAST_BACK_REFERENCE = 12; + + /* + + The LZ4 block format has a few properties that make it less + straight-forward than one would hope: + + * literal blocks and back-references must come in pairs (except + for the very last literal block), so consecutive literal + blocks created by the compressor must be merged into a single + block. + + * the start of a literal/back-reference pair contains the length + of the back-reference (at least some part of it) so we can't + start writing the literal before we know how long the next + back-reference is going to be. + + * there are special rules for the final blocks + + > There are specific parsing rules to respect in order to remain + > compatible with assumptions made by the decoder : + > + > 1. The last 5 bytes are always literals + > + > 2. The last match must start at least 12 bytes before end of + > block. Consequently, a block with less than 13 bytes cannot be + > compressed. + + which means any back-reference may need to get rewritten as a + literal block unless we know the next block is at least of + length 5 and the sum of this block's length and offset and the + next block's length is at least twelve. + + */ + + private final LZ77Compressor compressor; + private final OutputStream os; + + // used in one-arg write method + private final byte[] oneByte = new byte[1]; + + private boolean finished = false; + + private Deque pairs = new LinkedList<>(); + // keeps track of the last window-size bytes (64k) in order to be + // able to expand back-references when needed + private Deque expandedBlocks = new LinkedList<>(); + + /** + * Creates a new LZ4 output stream. + * + * @param os + * An OutputStream to read compressed data from + * + * @throws IOException if reading fails + */ + public BlockLZ4CompressorOutputStream(final OutputStream os) throws IOException { + this(os, createParameterBuilder().build()); + } + + /** + * Creates a new LZ4 output stream. + * + * @param os + * An OutputStream to read compressed data from + * @param params + * The parameters to use for LZ77 compression. + * + * @throws IOException if reading fails + */ + public BlockLZ4CompressorOutputStream(final OutputStream os, Parameters params) throws IOException { + this.os = os; + compressor = new LZ77Compressor(params, + new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) throws IOException { + switch (block.getType()) { + case LITERAL: + addLiteralBlock((LZ77Compressor.LiteralBlock) block); + break; + case BACK_REFERENCE: + addBackReference((LZ77Compressor.BackReference) block); + break; + case EOD: + writeFinalLiteralBlock(); + break; + } + } + }); + } + + @Override + public void write(int b) throws IOException { + oneByte[0] = (byte) (b & 0xff); + write(oneByte); + } + + @Override + public void write(byte[] data, int off, int len) throws IOException { + compressor.compress(data, off, len); + } + + @Override + public void close() throws IOException { + try { + finish(); + } finally { + os.close(); + } + } + + /** + * Compresses all remaining data and writes it to the stream, + * doesn't close the underlying stream. + * @throws IOException if an error occurs + */ + public void finish() throws IOException { + if (!finished) { + compressor.finish(); + finished = true; + } + } + + /** + * Adds some initial data to fill the window with. + * + * @param data the data to fill the window with. + * @param off offset of real data into the array + * @param len amount of data + * @throws IllegalStateException if the stream has already started to write data + * @see LZ77Compressor#prefill + */ + public void prefill(byte[] data, int off, int len) { + if (len > 0) { + byte[] b = Arrays.copyOfRange(data, off, off + len); + compressor.prefill(b); + recordLiteral(b); + } + } + + private void addLiteralBlock(LZ77Compressor.LiteralBlock block) throws IOException { + Pair last = writeBlocksAndReturnUnfinishedPair(block.getLength()); + recordLiteral(last.addLiteral(block)); + clearUnusedBlocksAndPairs(); + } + + private void addBackReference(LZ77Compressor.BackReference block) throws IOException { + Pair last = writeBlocksAndReturnUnfinishedPair(block.getLength()); + last.setBackReference(block); + recordBackReference(block); + clearUnusedBlocksAndPairs(); + } + + private Pair writeBlocksAndReturnUnfinishedPair(int length) throws IOException { + writeWritablePairs(length); + Pair last = pairs.peekLast(); + if (last == null || last.hasBackReference()) { + last = new Pair(); + pairs.addLast(last); + } + return last; + } + + private void recordLiteral(byte[] b) { + expandedBlocks.addFirst(b); + } + + private void clearUnusedBlocksAndPairs() { + clearUnusedBlocks(); + clearUnusedPairs(); + } + + private void clearUnusedBlocks() { + int blockLengths = 0; + int blocksToKeep = 0; + for (byte[] b : expandedBlocks) { + blocksToKeep++; + blockLengths += b.length; + if (blockLengths >= BlockLZ4CompressorInputStream.WINDOW_SIZE) { + break; + } + } + final int size = expandedBlocks.size(); + for (int i = blocksToKeep; i < size; i++) { + expandedBlocks.removeLast(); + } + } + + private void recordBackReference(LZ77Compressor.BackReference block) { + expandedBlocks.addFirst(expand(block.getOffset(), block.getLength())); + } + + private byte[] expand(final int offset, final int length) { + byte[] expanded = new byte[length]; + if (offset == 1) { // surprisingly common special case + byte[] block = expandedBlocks.peekFirst(); + byte b = block[block.length - 1]; + if (b != 0) { // the fresh array contains 0s anyway + Arrays.fill(expanded, b); + } + } else { + expandFromList(expanded, offset, length); + } + return expanded; + } + + private void expandFromList(final byte[] expanded, int offset, int length) { + int offsetRemaining = offset; + int lengthRemaining = length; + int writeOffset = 0; + while (lengthRemaining > 0) { + // find block that contains offsetRemaining + byte[] block = null; + int copyLen, copyOffset; + if (offsetRemaining > 0) { + int blockOffset = 0; + for (byte[] b : expandedBlocks) { + if (b.length + blockOffset >= offsetRemaining) { + block = b; + break; + } + blockOffset += b.length; + } + if (block == null) { + // should not be possible + throw new IllegalStateException("failed to find a block containing offset " + offset); + } + copyOffset = blockOffset + block.length - offsetRemaining; + copyLen = Math.min(lengthRemaining, block.length - copyOffset); + } else { + // offsetRemaining is negative or 0 and points into the expanded bytes + block = expanded; + copyOffset = -offsetRemaining; + copyLen = Math.min(lengthRemaining, writeOffset + offsetRemaining); + } + System.arraycopy(block, copyOffset, expanded, writeOffset, copyLen); + offsetRemaining -= copyLen; + lengthRemaining -= copyLen; + writeOffset += copyLen; + } + } + + private void clearUnusedPairs() { + int pairLengths = 0; + int pairsToKeep = 0; + for (Iterator it = pairs.descendingIterator(); it.hasNext(); ) { + Pair p = it.next(); + pairsToKeep++; + pairLengths += p.length(); + if (pairLengths >= BlockLZ4CompressorInputStream.WINDOW_SIZE) { + break; + } + } + final int size = pairs.size(); + for (int i = pairsToKeep; i < size; i++) { + Pair p = pairs.peekFirst(); + if (p.hasBeenWritten()) { + pairs.removeFirst(); + } else { + break; + } + } + } + + private void writeFinalLiteralBlock() throws IOException { + rewriteLastPairs(); + for (Pair p : pairs) { + if (!p.hasBeenWritten()) { + p.writeTo(os); + } + } + pairs.clear(); + } + + private void writeWritablePairs(int lengthOfBlocksAfterLastPair) throws IOException { + int unwrittenLength = lengthOfBlocksAfterLastPair; + for (Iterator it = pairs.descendingIterator(); it.hasNext(); ) { + Pair p = it.next(); + if (p.hasBeenWritten()) { + break; + } + unwrittenLength += p.length(); + } + for (Pair p : pairs) { + if (p.hasBeenWritten()) { + continue; + } + unwrittenLength -= p.length(); + if (p.canBeWritten(unwrittenLength)) { + p.writeTo(os); + } else { + break; + } + } + } + + private void rewriteLastPairs() { + LinkedList lastPairs = new LinkedList<>(); + LinkedList pairLength = new LinkedList<>(); + int offset = 0; + for (Iterator it = pairs.descendingIterator(); it.hasNext(); ) { + Pair p = it.next(); + if (p.hasBeenWritten()) { + break; + } + int len = p.length(); + pairLength.addFirst(len); + lastPairs.addFirst(p); + offset += len; + if (offset >= MIN_OFFSET_OF_LAST_BACK_REFERENCE) { + break; + } + } + for (Pair p : lastPairs) { + pairs.remove(p); + } + // lastPairs may contain between one and four Pairs: + // * the last pair may be a one byte literal + // * all other Pairs contain a back-reference which must be four bytes long at minimum + // we could merge them all into a single literal block but + // this may harm compression. For example compressing + // "bla.tar" from our tests yields a last block containing a + // back-reference of length > 2k and we'd end up with a last + // literal of that size rather than a 2k back-reference and a + // 12 byte literal at the end. + + // Instead we merge all but the first of lastPairs into a new + // literal-only Pair "replacement" and look at the + // back-reference in the first of lastPairs and see if we can + // split it. We can split it if it is longer than 16 - + // replacement.length (i.e. the minimal length of four is kept + // while making sure the last literal is at least twelve bytes + // long). If we can't split it, we expand the first of the pairs + // as well. + + // this is not optimal, we could get better compression + // results with more complex approaches as the last literal + // only needs to be five bytes long if the previous + // back-reference has an offset big enough + + final int lastPairsSize = lastPairs.size(); + int toExpand = 0; + for (int i = 1; i < lastPairsSize; i++) { + toExpand += pairLength.get(i); + } + Pair replacement = new Pair(); + if (toExpand > 0) { + replacement.prependLiteral(expand(toExpand, toExpand)); + } + Pair splitCandidate = lastPairs.get(0); + int stillNeeded = MIN_OFFSET_OF_LAST_BACK_REFERENCE - toExpand; + int brLen = splitCandidate.hasBackReference() ? splitCandidate.backReferenceLength() : 0; + if (splitCandidate.hasBackReference() && brLen >= MIN_BACK_REFERENCE_LENGTH + stillNeeded) { + replacement.prependLiteral(expand(toExpand + stillNeeded, stillNeeded)); + pairs.add(splitCandidate.splitWithNewBackReferenceLengthOf(brLen - stillNeeded)); + } else { + if (splitCandidate.hasBackReference()) { + replacement.prependLiteral(expand(toExpand + brLen, brLen)); + } + splitCandidate.prependTo(replacement); + } + pairs.add(replacement); + } + + /** + * Returns a builder correctly configured for the LZ4 algorithm. + * @return a builder correctly configured for the LZ4 algorithm + */ + public static Parameters.Builder createParameterBuilder() { + int maxLen = BlockLZ4CompressorInputStream.WINDOW_SIZE - 1; + return Parameters.builder(BlockLZ4CompressorInputStream.WINDOW_SIZE) + .withMinBackReferenceLength(MIN_BACK_REFERENCE_LENGTH) + .withMaxBackReferenceLength(maxLen) + .withMaxOffset(maxLen) + .withMaxLiteralLength(maxLen); + } + + final static class Pair { + private final Deque literals = new LinkedList<>(); + private int brOffset, brLength; + private boolean written; + + private void prependLiteral(byte[] data) { + literals.addFirst(data); + } + byte[] addLiteral(LZ77Compressor.LiteralBlock block) { + byte[] copy = Arrays.copyOfRange(block.getData(), block.getOffset(), + block.getOffset() + block.getLength()); + literals.add(copy); + return copy; + } + void setBackReference(LZ77Compressor.BackReference block) { + if (hasBackReference()) { + throw new IllegalStateException(); + } + brOffset = block.getOffset(); + brLength = block.getLength(); + } + boolean hasBackReference() { + return brOffset > 0; + } + boolean canBeWritten(int lengthOfBlocksAfterThisPair) { + return hasBackReference() + && lengthOfBlocksAfterThisPair >= MIN_OFFSET_OF_LAST_BACK_REFERENCE + MIN_BACK_REFERENCE_LENGTH; + } + int length() { + return literalLength() + brLength; + } + private boolean hasBeenWritten() { + return written; + } + void writeTo(OutputStream out) throws IOException { + int litLength = literalLength(); + out.write(lengths(litLength, brLength)); + if (litLength >= BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK) { + writeLength(litLength - BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK, out); + } + for (byte[] b : literals) { + out.write(b); + } + if (hasBackReference()) { + ByteUtils.toLittleEndian(out, brOffset, 2); + if (brLength - MIN_BACK_REFERENCE_LENGTH >= BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK) { + writeLength(brLength - MIN_BACK_REFERENCE_LENGTH + - BlockLZ4CompressorInputStream.BACK_REFERENCE_SIZE_MASK, out); + } + } + written = true; + } + private int literalLength() { + int length = 0; + for (byte[] b : literals) { + length += b.length; + } + return length; + } + private static int lengths(int litLength, int brLength) { + int l = litLength < 15 ? litLength : 15; + int br = brLength < 4 ? 0 : (brLength < 19 ? brLength - 4 : 15); + return (l << BlockLZ4CompressorInputStream.SIZE_BITS) | br; + } + private static void writeLength(int length, OutputStream out) throws IOException { + while (length >= 255) { + out.write(255); + length -= 255; + } + out.write(length); + } + private int backReferenceLength() { + return brLength; + } + private void prependTo(Pair other) { + Iterator listBackwards = literals.descendingIterator(); + while (listBackwards.hasNext()) { + other.prependLiteral(listBackwards.next()); + } + } + private Pair splitWithNewBackReferenceLengthOf(int newBackReferenceLength) { + Pair p = new Pair(); + p.literals.addAll(literals); + p.brOffset = brOffset; + p.brLength = newBackReferenceLength; + return p; + } + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.utils.BoundedInputStream; +import org.apache.commons.compress.utils.ByteUtils; +import org.apache.commons.compress.utils.ChecksumCalculatingInputStream; +import org.apache.commons.compress.utils.CountingInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; + +/** + * CompressorInputStream for the LZ4 frame format. + * + *

    Based on the "spec" in the version "1.5.1 (31/03/2015)"

    + * + * @see LZ4 Frame Format Description + * @since 1.14 + * @NotThreadSafe + */ +public class FramedLZ4CompressorInputStream extends CompressorInputStream + implements InputStreamStatistics { + + // used by FramedLZ4CompressorOutputStream as well + static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR + 4, 0x22, 0x4d, 0x18 + }; + private static final byte[] SKIPPABLE_FRAME_TRAILER = new byte[] { + 0x2a, 0x4d, 0x18 + }; + private static final byte SKIPPABLE_FRAME_PREFIX_BYTE_MASK = 0x50; + + static final int VERSION_MASK = 0xC0; + static final int SUPPORTED_VERSION = 0x40; + static final int BLOCK_INDEPENDENCE_MASK = 0x20; + static final int BLOCK_CHECKSUM_MASK = 0x10; + static final int CONTENT_SIZE_MASK = 0x08; + static final int CONTENT_CHECKSUM_MASK = 0x04; + static final int BLOCK_MAX_SIZE_MASK = 0x70; + static final int UNCOMPRESSED_FLAG_MASK = 0x80000000; + + // used in no-arg read method + private final byte[] oneByte = new byte[1]; + + private final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() { + @Override + public int getAsByte() throws IOException { + return readOneByte(); + } + }; + + private final CountingInputStream in; + private final boolean decompressConcatenated; + + private boolean expectBlockChecksum; + private boolean expectBlockDependency; + private boolean expectContentSize; + private boolean expectContentChecksum; + + private InputStream currentBlock; + private boolean endReached, inUncompressed; + + // used for frame header checksum and content checksum, if present + private final XXHash32 contentHash = new XXHash32(); + + // used for block checksum, if present + private final XXHash32 blockHash = new XXHash32(); + + // only created if the frame doesn't set the block independence flag + private byte[] blockDependencyBuffer; + + /** + * Creates a new input stream that decompresses streams compressed + * using the LZ4 frame format and stops after decompressing the + * first frame. + * @param in the InputStream from which to read the compressed data + * @throws IOException if reading fails + */ + public FramedLZ4CompressorInputStream(InputStream in) throws IOException { + this(in, false); + } + + /** + * Creates a new input stream that decompresses streams compressed + * using the LZ4 frame format. + * @param in the InputStream from which to read the compressed data + * @param decompressConcatenated if true, decompress until the end + * of the input; if false, stop after the first LZ4 frame + * and leave the input position to point to the next byte + * after the frame stream + * @throws IOException if reading fails + */ + public FramedLZ4CompressorInputStream(InputStream in, boolean decompressConcatenated) throws IOException { + this.in = new CountingInputStream(in); + this.decompressConcatenated = decompressConcatenated; + init(true); + } + + /** {@inheritDoc} */ + @Override + public int read() throws IOException { + return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF; + } + + /** {@inheritDoc} */ + @Override + public void close() throws IOException { + try { + if (currentBlock != null) { + currentBlock.close(); + currentBlock = null; + } + } finally { + in.close(); + } + } + + /** {@inheritDoc} */ + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + if (endReached) { + return -1; + } + int r = readOnce(b, off, len); + if (r == -1) { + nextBlock(); + if (!endReached) { + r = readOnce(b, off, len); + } + } + if (r != -1) { + if (expectBlockDependency) { + appendToBlockDependencyBuffer(b, off, r); + } + if (expectContentChecksum) { + contentHash.update(b, off, r); + } + } + return r; + } + + /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return in.getBytesRead(); + } + + private void init(boolean firstFrame) throws IOException { + if (readSignature(firstFrame)) { + readFrameDescriptor(); + nextBlock(); + } + } + + private boolean readSignature(boolean firstFrame) throws IOException { + String garbageMessage = firstFrame ? "Not a LZ4 frame stream" : "LZ4 frame stream followed by garbage"; + final byte[] b = new byte[4]; + int read = IOUtils.readFully(in, b); + count(read); + if (0 == read && !firstFrame) { + // good LZ4 frame and nothing after it + endReached = true; + return false; + } + if (4 != read) { + throw new IOException(garbageMessage); + } + + read = skipSkippableFrame(b); + if (0 == read && !firstFrame) { + // good LZ4 frame with only some skippable frames after it + endReached = true; + return false; + } + if (4 != read || !matches(b, 4)) { + throw new IOException(garbageMessage); + } + return true; + } + + private void readFrameDescriptor() throws IOException { + int flags = readOneByte(); + if (flags == -1) { + throw new IOException("Premature end of stream while reading frame flags"); + } + contentHash.update(flags); + if ((flags & VERSION_MASK) != SUPPORTED_VERSION) { + throw new IOException("Unsupported version " + (flags >> 6)); + } + expectBlockDependency = (flags & BLOCK_INDEPENDENCE_MASK) == 0; + if (expectBlockDependency) { + if (blockDependencyBuffer == null) { + blockDependencyBuffer = new byte[BlockLZ4CompressorInputStream.WINDOW_SIZE]; + } + } else { + blockDependencyBuffer = null; + } + expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0; + expectContentSize = (flags & CONTENT_SIZE_MASK) != 0; + expectContentChecksum = (flags & CONTENT_CHECKSUM_MASK) != 0; + int bdByte = readOneByte(); + if (bdByte == -1) { // max size is irrelevant for this implementation + throw new IOException("Premature end of stream while reading frame BD byte"); + } + contentHash.update(bdByte); + if (expectContentSize) { // for now we don't care, contains the uncompressed size + byte[] contentSize = new byte[8]; + int skipped = IOUtils.readFully(in, contentSize); + count(skipped); + if (8 != skipped) { + throw new IOException("Premature end of stream while reading content size"); + } + contentHash.update(contentSize, 0, contentSize.length); + } + int headerHash = readOneByte(); + if (headerHash == -1) { // partial hash of header. + throw new IOException("Premature end of stream while reading frame header checksum"); + } + int expectedHash = (int) ((contentHash.getValue() >> 8) & 0xff); + contentHash.reset(); + if (headerHash != expectedHash) { + throw new IOException("frame header checksum mismatch."); + } + } + + private void nextBlock() throws IOException { + maybeFinishCurrentBlock(); + long len = ByteUtils.fromLittleEndian(supplier, 4); + boolean uncompressed = (len & UNCOMPRESSED_FLAG_MASK) != 0; + int realLen = (int) (len & (~UNCOMPRESSED_FLAG_MASK)); + if (realLen == 0) { + verifyContentChecksum(); + if (!decompressConcatenated) { + endReached = true; + } else { + init(false); + } + return; + } + InputStream capped = new BoundedInputStream(in, realLen); + if (expectBlockChecksum) { + capped = new ChecksumCalculatingInputStream(blockHash, capped); + } + if (uncompressed) { + inUncompressed = true; + currentBlock = capped; + } else { + inUncompressed = false; + BlockLZ4CompressorInputStream s = new BlockLZ4CompressorInputStream(capped); + if (expectBlockDependency) { + s.prefill(blockDependencyBuffer); + } + currentBlock = s; + } + } + + private void maybeFinishCurrentBlock() throws IOException { + if (currentBlock != null) { + currentBlock.close(); + currentBlock = null; + if (expectBlockChecksum) { + verifyChecksum(blockHash, "block"); + blockHash.reset(); + } + } + } + + private void verifyContentChecksum() throws IOException { + if (expectContentChecksum) { + verifyChecksum(contentHash, "content"); + } + contentHash.reset(); + } + + private void verifyChecksum(XXHash32 hash, String kind) throws IOException { + byte[] checksum = new byte[4]; + int read = IOUtils.readFully(in, checksum); + count(read); + if (4 != read) { + throw new IOException("Premature end of stream while reading " + kind + " checksum"); + } + long expectedHash = hash.getValue(); + if (expectedHash != ByteUtils.fromLittleEndian(checksum)) { + throw new IOException(kind + " checksum mismatch."); + } + } + + private int readOneByte() throws IOException { + final int b = in.read(); + if (b != -1) { + count(1); + return b & 0xFF; + } + return -1; + } + + private int readOnce(byte[] b, int off, int len) throws IOException { + if (inUncompressed) { + int cnt = currentBlock.read(b, off, len); + count(cnt); + return cnt; + } + BlockLZ4CompressorInputStream l = (BlockLZ4CompressorInputStream) currentBlock; + long before = l.getBytesRead(); + int cnt = currentBlock.read(b, off, len); + count(l.getBytesRead() - before); + return cnt; + } + + private static boolean isSkippableFrameSignature(byte[] b) { + if ((b[0] & SKIPPABLE_FRAME_PREFIX_BYTE_MASK) != SKIPPABLE_FRAME_PREFIX_BYTE_MASK) { + return false; + } + for (int i = 1; i < 4; i++) { + if (b[i] != SKIPPABLE_FRAME_TRAILER[i - 1]) { + return false; + } + } + return true; + } + + /** + * Skips over the contents of a skippable frame as well as + * skippable frames following it. + * + *

    It then tries to read four more bytes which are supposed to + * hold an LZ4 signature and returns the number of bytes read + * while storing the bytes in the given array.

    + */ + private int skipSkippableFrame(byte[] b) throws IOException { + int read = 4; + while (read == 4 && isSkippableFrameSignature(b)) { + long len = ByteUtils.fromLittleEndian(supplier, 4); + long skipped = IOUtils.skip(in, len); + count(skipped); + if (len != skipped) { + throw new IOException("Premature end of stream while skipping frame"); + } + read = IOUtils.readFully(in, b); + count(read); + } + return read; + } + + private void appendToBlockDependencyBuffer(final byte[] b, final int off, int len) { + len = Math.min(len, blockDependencyBuffer.length); + if (len > 0) { + int keep = blockDependencyBuffer.length - len; + if (keep > 0) { + // move last keep bytes towards the start of the buffer + System.arraycopy(blockDependencyBuffer, len, blockDependencyBuffer, 0, keep); + } + // append new data + System.arraycopy(b, off, blockDependencyBuffer, keep, len); + } + } + + /** + * Checks if the signature matches what is expected for a .lz4 file. + * + *

    .lz4 files start with a four byte signature.

    + * + * @param signature the bytes to check + * @param length the number of bytes to check + * @return true if this is a .sz stream, false otherwise + */ + public static boolean matches(final byte[] signature, final int length) { + + if (length < LZ4_SIGNATURE.length) { + return false; + } + + byte[] shortenedSig = signature; + if (signature.length > LZ4_SIGNATURE.length) { + shortenedSig = new byte[LZ4_SIGNATURE.length]; + System.arraycopy(signature, 0, shortenedSig, 0, LZ4_SIGNATURE.length); + } + + return Arrays.equals(shortenedSig, LZ4_SIGNATURE); + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.utils.ByteUtils; + +/** + * CompressorOutputStream for the LZ4 frame format. + * + *

    Based on the "spec" in the version "1.5.1 (31/03/2015)"

    + * + * @see LZ4 Frame Format Description + * @since 1.14 + * @NotThreadSafe + */ +public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { + + private static final byte[] END_MARK = new byte[4]; + + // used in one-arg write method + private final byte[] oneByte = new byte[1]; + + private final byte[] blockData; + private final OutputStream out; + private final Parameters params; + private boolean finished = false; + private int currentIndex = 0; + + // used for frame header checksum and content checksum, if requested + private final XXHash32 contentHash = new XXHash32(); + // used for block checksum, if requested + private final XXHash32 blockHash; + + // only created if the config requires block dependency + private byte[] blockDependencyBuffer; + private int collectedBlockDependencyBytes; + + /** + * The block sizes supported by the format. + */ + public enum BlockSize { + /** Block size of 64K */ + K64(64 * 1024, 4), + /** Block size of 256K */ + K256(256 * 1024, 5), + /** Block size of 1M */ + M1(1024 * 1024, 6), + /** Block size of 4M */ + M4(4096 * 1024, 7); + + private final int size, index; + BlockSize(int size, int index) { + this.size = size; + this.index = index; + } + int getSize() { + return size; + } + int getIndex() { + return index; + } + } + + /** + * Parameters of the LZ4 frame format. + */ + public static class Parameters { + private final BlockSize blockSize; + private final boolean withContentChecksum, withBlockChecksum, withBlockDependency; + private final org.apache.commons.compress.compressors.lz77support.Parameters lz77params; + + /** + * The default parameters of 4M block size, enabled content + * checksum, disabled block checksums and independent blocks. + * + *

    This matches the defaults of the lz4 command line utility.

    + */ + public static final Parameters DEFAULT = new Parameters(BlockSize.M4, true, false, false); + + /** + * Sets up custom a custom block size for the LZ4 stream but + * otherwise uses the defaults of enabled content checksum, + * disabled block checksums and independent blocks. + * @param blockSize the size of a single block. + */ + public Parameters(BlockSize blockSize) { + this(blockSize, true, false, false); + } + /** + * Sets up custom a custom block size for the LZ4 stream but + * otherwise uses the defaults of enabled content checksum, + * disabled block checksums and independent blocks. + * @param blockSize the size of a single block. + * @param lz77params parameters used to fine-tune compression, + * in particular to balance compression ratio vs compression + * speed. + */ + public Parameters(BlockSize blockSize, + org.apache.commons.compress.compressors.lz77support.Parameters lz77params) { + this(blockSize, true, false, false, lz77params); + } + /** + * Sets up custom parameters for the LZ4 stream. + * @param blockSize the size of a single block. + * @param withContentChecksum whether to write a content checksum + * @param withBlockChecksum whether to write a block checksum. + * Note that block checksums are not supported by the lz4 + * command line utility + * @param withBlockDependency whether a block may depend on + * the content of a previous block. Enabling this may improve + * compression ratio but makes it impossible to decompress the + * output in parallel. + */ + public Parameters(BlockSize blockSize, boolean withContentChecksum, boolean withBlockChecksum, + boolean withBlockDependency) { + this(blockSize, withContentChecksum, withBlockChecksum, withBlockDependency, + BlockLZ4CompressorOutputStream.createParameterBuilder().build()); + } + + /** + * Sets up custom parameters for the LZ4 stream. + * @param blockSize the size of a single block. + * @param withContentChecksum whether to write a content checksum + * @param withBlockChecksum whether to write a block checksum. + * Note that block checksums are not supported by the lz4 + * command line utility + * @param withBlockDependency whether a block may depend on + * the content of a previous block. Enabling this may improve + * compression ratio but makes it impossible to decompress the + * output in parallel. + * @param lz77params parameters used to fine-tune compression, + * in particular to balance compression ratio vs compression + * speed. + */ + public Parameters(BlockSize blockSize, boolean withContentChecksum, boolean withBlockChecksum, + boolean withBlockDependency, + org.apache.commons.compress.compressors.lz77support.Parameters lz77params) { + this.blockSize = blockSize; + this.withContentChecksum = withContentChecksum; + this.withBlockChecksum = withBlockChecksum; + this.withBlockDependency = withBlockDependency; + this.lz77params = lz77params; + } + + @Override + public String toString() { + return "LZ4 Parameters with BlockSize " + blockSize + ", withContentChecksum " + withContentChecksum + + ", withBlockChecksum " + withBlockChecksum + ", withBlockDependency " + withBlockDependency; + } + } + + /** + * Constructs a new output stream that compresses data using the + * LZ4 frame format using the default block size of 4MB. + * @param out the OutputStream to which to write the compressed data + * @throws IOException if writing the signature fails + */ + public FramedLZ4CompressorOutputStream(OutputStream out) throws IOException { + this(out, Parameters.DEFAULT); + } + + /** + * Constructs a new output stream that compresses data using the + * LZ4 frame format using the given block size. + * @param out the OutputStream to which to write the compressed data + * @param params the parameters to use + * @throws IOException if writing the signature fails + */ + public FramedLZ4CompressorOutputStream(OutputStream out, Parameters params) throws IOException { + this.params = params; + blockData = new byte[params.blockSize.getSize()]; + this.out = out; + blockHash = params.withBlockChecksum ? new XXHash32() : null; + out.write(FramedLZ4CompressorInputStream.LZ4_SIGNATURE); + writeFrameDescriptor(); + blockDependencyBuffer = params.withBlockDependency + ? new byte[BlockLZ4CompressorInputStream.WINDOW_SIZE] + : null; + } + + @Override + public void write(int b) throws IOException { + oneByte[0] = (byte) (b & 0xff); + write(oneByte); + } + + @Override + public void write(byte[] data, int off, int len) throws IOException { + if (params.withContentChecksum) { + contentHash.update(data, off, len); + } + if (currentIndex + len > blockData.length) { + flushBlock(); + while (len > blockData.length) { + System.arraycopy(data, off, blockData, 0, blockData.length); + off += blockData.length; + len -= blockData.length; + currentIndex = blockData.length; + flushBlock(); + } + } + System.arraycopy(data, off, blockData, currentIndex, len); + currentIndex += len; + } + + @Override + public void close() throws IOException { + try { + finish(); + } finally { + out.close(); + } + } + + /** + * Compresses all remaining data and writes it to the stream, + * doesn't close the underlying stream. + * @throws IOException if an error occurs + */ + public void finish() throws IOException { + if (!finished) { + if (currentIndex > 0) { + flushBlock(); + } + writeTrailer(); + finished = true; + } + } + + private void writeFrameDescriptor() throws IOException { + int flags = FramedLZ4CompressorInputStream.SUPPORTED_VERSION; + if (!params.withBlockDependency) { + flags |= FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK; + } + if (params.withContentChecksum) { + flags |= FramedLZ4CompressorInputStream.CONTENT_CHECKSUM_MASK; + } + if (params.withBlockChecksum) { + flags |= FramedLZ4CompressorInputStream.BLOCK_CHECKSUM_MASK; + } + out.write(flags); + contentHash.update(flags); + int bd = (params.blockSize.getIndex() << 4) & FramedLZ4CompressorInputStream.BLOCK_MAX_SIZE_MASK; + out.write(bd); + contentHash.update(bd); + out.write((int) ((contentHash.getValue() >> 8) & 0xff)); + contentHash.reset(); + } + + private void flushBlock() throws IOException { + final boolean withBlockDependency = params.withBlockDependency; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (BlockLZ4CompressorOutputStream o = new BlockLZ4CompressorOutputStream(baos, params.lz77params)) { + if (withBlockDependency) { + o.prefill(blockDependencyBuffer, blockDependencyBuffer.length - collectedBlockDependencyBytes, + collectedBlockDependencyBytes); + } + o.write(blockData, 0, currentIndex); + } + if (withBlockDependency) { + appendToBlockDependencyBuffer(blockData, 0, currentIndex); + } + byte[] b = baos.toByteArray(); + if (b.length > currentIndex) { // compression increased size, maybe beyond blocksize + ByteUtils.toLittleEndian(out, currentIndex | FramedLZ4CompressorInputStream.UNCOMPRESSED_FLAG_MASK, + 4); + out.write(blockData, 0, currentIndex); + if (params.withBlockChecksum) { + blockHash.update(blockData, 0, currentIndex); + } + } else { + ByteUtils.toLittleEndian(out, b.length, 4); + out.write(b); + if (params.withBlockChecksum) { + blockHash.update(b, 0, b.length); + } + } + if (params.withBlockChecksum) { + ByteUtils.toLittleEndian(out, blockHash.getValue(), 4); + blockHash.reset(); + } + currentIndex = 0; + } + + private void writeTrailer() throws IOException { + out.write(END_MARK); + if (params.withContentChecksum) { + ByteUtils.toLittleEndian(out, contentHash.getValue(), 4); + } + } + + private void appendToBlockDependencyBuffer(final byte[] b, final int off, int len) { + len = Math.min(len, blockDependencyBuffer.length); + if (len > 0) { + int keep = blockDependencyBuffer.length - len; + if (keep > 0) { + // move last keep bytes towards the start of the buffer + System.arraycopy(blockDependencyBuffer, len, blockDependencyBuffer, 0, keep); + } + // append new data + System.arraycopy(b, off, blockDependencyBuffer, keep, len); + collectedBlockDependencyBytes = Math.min(collectedBlockDependencyBytes + len, + blockDependencyBuffer.length); + } + } + +} + diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/package.html 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/package.html 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,37 @@ + + + +

    Provides stream classes for the + LZ4 + algorithm.

    + +

    The block LZ4 format which only contains the compressed data is + supported by the BlockLZ4Compressor*putStream + classes while the frame format is implemented + by FramedLZ4Compressor*putStream. The + implementation in Commons Compress is based on the + specifications "Last revised: 2015-03-26" for the block format + and version "1.5.1 (31/03/2015)" for the frame format.

    + +

    Only the frame format can be auto-detected this means you have + to speficy the format explicitly if you want to read a block LZ4 + stream via CompressorStreamFactory.

    + + diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import static java.lang.Integer.rotateLeft; + +import java.util.zip.Checksum; + +import static org.apache.commons.compress.utils.ByteUtils.fromLittleEndian; + +/** + * Implementation of the xxhash32 hash algorithm. + * + * @see xxHash + * @NotThreadSafe + * @since 1.14 + */ +public class XXHash32 implements Checksum { + + private static final int BUF_SIZE = 16; + private static final int ROTATE_BITS = 13; + + private static final int PRIME1 = (int) 2654435761L; + private static final int PRIME2 = (int) 2246822519L; + private static final int PRIME3 = (int) 3266489917L; + private static final int PRIME4 = 668265263; + private static final int PRIME5 = 374761393; + + private final byte[] oneByte = new byte[1]; + private final int[] state = new int[4]; + // Note: the code used to use ByteBuffer but the manual method is 50% faster + // See: https://git-wip-us.apache.org/repos/asf/commons-compress/diff/2f56fb5c + private final byte[] buffer = new byte[BUF_SIZE]; + private final int seed; + + private int totalLen; + private int pos; + + /** + * Creates an XXHash32 instance with a seed of 0. + */ + public XXHash32() { + this(0); + } + + /** + * Creates an XXHash32 instance. + * @param seed the seed to use + */ + public XXHash32(int seed) { + this.seed = seed; + initializeState(); + } + + @Override + public void reset() { + initializeState(); + totalLen = 0; + pos = 0; + } + + @Override + public void update(int b) { + oneByte[0] = (byte) (b & 0xff); + update(oneByte, 0, 1); + } + + @Override + public void update(byte[] b, int off, final int len) { + if (len <= 0) { + return; + } + totalLen += len; + + final int end = off + len; + + if (pos + len < BUF_SIZE) { + System.arraycopy(b, off, buffer, pos, len); + pos += len; + return; + } + + if (pos > 0) { + final int size = BUF_SIZE - pos; + System.arraycopy(b, off, buffer, pos, size); + process(buffer, 0); + off += size; + } + + final int limit = end - BUF_SIZE; + while (off <= limit) { + process(b, off); + off += BUF_SIZE; + } + + if (off < end) { + pos = end - off; + System.arraycopy(b, off, buffer, 0, pos); + } + } + + @Override + public long getValue() { + int hash; + if (totalLen > BUF_SIZE) { + hash = + rotateLeft(state[0], 1) + + rotateLeft(state[1], 7) + + rotateLeft(state[2], 12) + + rotateLeft(state[3], 18); + } else { + hash = state[2] + PRIME5; + } + hash += totalLen; + + int idx = 0; + final int limit = pos - 4; + for (; idx <= limit; idx += 4) { + hash = rotateLeft(hash + getInt(buffer, idx) * PRIME3, 17) * PRIME4; + } + while (idx < pos) { + hash = rotateLeft(hash + (buffer[idx++] & 0xff) * PRIME5, 11) * PRIME1; + } + + hash ^= hash >>> 15; + hash *= PRIME2; + hash ^= hash >>> 13; + hash *= PRIME3; + hash ^= hash >>> 16; + return hash & 0xffffffffL; + } + + private static int getInt(byte[] buffer, int idx) { + return (int) (fromLittleEndian(buffer, idx, 4) & 0xffffffffL); + } + + private void initializeState() { + state[0] = seed + PRIME1 + PRIME2; + state[1] = seed + PRIME2; + state[2] = seed; + state[3] = seed - PRIME1; + } + + private void process(byte[] b, int offset) { + // local shadows for performance + int s0 = state[0]; + int s1 = state[1]; + int s2 = state[2]; + int s3 = state[3]; + + s0 = rotateLeft(s0 + getInt(b, offset) * PRIME2, ROTATE_BITS) * PRIME1; + s1 = rotateLeft(s1 + getInt(b, offset + 4) * PRIME2, ROTATE_BITS) * PRIME1; + s2 = rotateLeft(s2 + getInt(b, offset + 8) * PRIME2, ROTATE_BITS) * PRIME1; + s3 = rotateLeft(s3 + getInt(b, offset + 12) * PRIME2, ROTATE_BITS) * PRIME1; + + state[0] = s0; + state[1] = s1; + state[2] = s2; + state[3] = s3; + + pos = 0; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz77support; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.utils.ByteUtils; +import org.apache.commons.compress.utils.CountingInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; + +/** + * Encapsulates code common to LZ77 decompressors. + * + *

    Assumes the stream consists of blocks of literal data and + * back-references (called copies) in any order. Of course the first + * block must be a literal block for the scheme to work - unless the + * {@link #prefill prefill} method has been used to provide initial + * data that is never returned by {@link #read read} but only used for + * back-references.

    + * + *

    Subclasses must override the three-arg {@link #read read} method + * as the no-arg version delegates to it and the default + * implementation delegates to the no-arg version, leading to infinite + * mutual recursion and a {@code StackOverflowError} otherwise.

    + * + *

    The contract for subclasses' {@code read} implementation is:

    + *
      + * + *
    • keep track of the current state of the stream. Is it inside a + * literal block or a back-reference or in-between blocks?
      • + * + *
    • Use {@link #readOneByte} to access the underlying stream + * directly.
      • + * + *
    • If a new literal block starts, use {@link #startLiteral} to + * tell this class about it and read the literal data using {@link + * #readLiteral} until it returns {@code 0}. {@link + * #hasMoreDataInBlock} will return {@code false} before the next + * call to {@link #readLiteral} would return {@code 0}.
      • + * + *
    • If a new back-reference starts, use {@link #startBackReference} to + * tell this class about it and read the literal data using {@link + * #readBackReference} until it returns {@code 0}. {@link + * #hasMoreDataInBlock} will return {@code false} before the next + * call to {@link #readBackReference} would return {@code 0}.
      • + * + *
    • If the end of the stream has been reached, return {@code -1} + * as this class' methods will never do so themselves.
      • + * + *
    + * + *

    {@link #readOneByte} and {@link #readLiteral} update the counter + * for bytes read.

    + * + * @since 1.14 + */ +public abstract class AbstractLZ77CompressorInputStream extends CompressorInputStream + implements InputStreamStatistics { + + /** Size of the window - must be bigger than the biggest offset expected. */ + private final int windowSize; + + /** + * Buffer to write decompressed bytes to for back-references, will + * be three times windowSize big. + * + *

    Three times so we can slide the whole buffer a windowSize to + * the left once we've read twice windowSize and still have enough + * data inside of it to satisfy back-references.

    + */ + private final byte[] buf; + + /** One behind the index of the last byte in the buffer that was written, i.e. the next position to write to */ + private int writeIndex; + + /** Index of the next byte to be read. */ + private int readIndex; + + /** The underlying stream to read compressed data from */ + private final CountingInputStream in; + + /** Number of bytes still to be read from the current literal or back-reference. */ + private long bytesRemaining; + + /** Offset of the current back-reference. */ + private int backReferenceOffset; + + /** uncompressed size */ + private int size = 0; + + // used in no-arg read method + private final byte[] oneByte = new byte[1]; + + /** + * Supplier that delegates to {@link #readOneByte}. + */ + protected final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() { + @Override + public int getAsByte() throws IOException { + return readOneByte(); + } + }; + + /** + * Creates a new LZ77 input stream. + * + * @param is + * An InputStream to read compressed data from + * @param windowSize + * Size of the window kept for back-references, must be bigger than the biggest offset expected. + * + * @throws IOException if reading fails + */ + public AbstractLZ77CompressorInputStream(final InputStream is, int windowSize) throws IOException { + this.in = new CountingInputStream(is); + this.windowSize = windowSize; + buf = new byte[3 * windowSize]; + writeIndex = readIndex = 0; + bytesRemaining = 0; + } + + /** {@inheritDoc} */ + @Override + public int read() throws IOException { + return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF; + } + + /** {@inheritDoc} */ + @Override + public void close() throws IOException { + in.close(); + } + + /** {@inheritDoc} */ + @Override + public int available() { + return writeIndex - readIndex; + } + + /** + * Get the uncompressed size of the stream + * + * @return the uncompressed size + */ + public int getSize() { + return size; + } + + /** + * Adds some initial data to fill the window with. + * + *

    This is used if the stream has been cut into blocks and + * back-references of one block may refer to data of the previous + * block(s). One such example is the LZ4 frame format using block + * dependency.

    + * + * @param data the data to fill the window with. + * @throws IllegalStateException if the stream has already started to read data + */ + public void prefill(byte[] data) { + if (writeIndex != 0) { + throw new IllegalStateException("the stream has already been read from, can't prefill anymore"); + } + // we don't need more data than the big offset could refer to, so cap it + int len = Math.min(windowSize, data.length); + // we need the last data as we are dealing with *back*-references + System.arraycopy(data, data.length - len, buf, 0, len); + writeIndex += len; + readIndex += len; + } + + /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return in.getBytesRead(); + } + + /** + * Used by subclasses to signal the next block contains the given + * amount of literal data. + * @param length the length of the block + */ + protected final void startLiteral(long length) { + bytesRemaining = length; + } + + /** + * Is there still data remaining inside the current block? + * @return true if there is still data remaining inside the current block. + */ + protected final boolean hasMoreDataInBlock() { + return bytesRemaining > 0; + } + + /** + * Reads data from the current literal block. + * @param b buffer to write data to + * @param off offset to start writing to + * @param len maximum amount of data to read + * @return number of bytes read, may be 0. Will never return -1 as + * EOF-detection is the responsibility of the subclass + * @throws IOException if the underlying stream throws or signals + * an EOF before the amount of data promised for the block have + * been read + */ + protected final int readLiteral(final byte[] b, final int off, final int len) throws IOException { + final int avail = available(); + if (len > avail) { + tryToReadLiteral(len - avail); + } + return readFromBuffer(b, off, len); + } + + private void tryToReadLiteral(int bytesToRead) throws IOException { + // min of "what is still inside the literal", "what does the user want" and "how muc can fit into the buffer" + final int reallyTryToRead = Math.min((int) Math.min(bytesToRead, bytesRemaining), + buf.length - writeIndex); + final int bytesRead = reallyTryToRead > 0 + ? IOUtils.readFully(in, buf, writeIndex, reallyTryToRead) + : 0 /* happens for bytesRemaining == 0 */; + count(bytesRead); + if (reallyTryToRead != bytesRead) { + throw new IOException("Premature end of stream reading literal"); + } + writeIndex += reallyTryToRead; + bytesRemaining -= reallyTryToRead; + } + + private int readFromBuffer(final byte[] b, final int off, final int len) { + final int readable = Math.min(len, available()); + if (readable > 0) { + System.arraycopy(buf, readIndex, b, off, readable); + readIndex += readable; + if (readIndex > 2 * windowSize) { + slideBuffer(); + } + } + size += readable; + return readable; + } + + private void slideBuffer() { + System.arraycopy(buf, windowSize, buf, 0, windowSize * 2); + writeIndex -= windowSize; + readIndex -= windowSize; + } + + /** + * Used by subclasses to signal the next block contains a back-reference with the given coordinates. + * @param offset the offset of the back-reference + * @param length the length of the back-reference + */ + protected final void startBackReference(int offset, long length) { + backReferenceOffset = offset; + bytesRemaining = length; + } + + /** + * Reads data from the current back-reference. + * @param b buffer to write data to + * @param off offset to start writing to + * @param len maximum amount of data to read + * @return number of bytes read, may be 0. Will never return -1 as + * EOF-detection is the responsibility of the subclass + */ + protected final int readBackReference(final byte[] b, final int off, final int len) { + final int avail = available(); + if (len > avail) { + tryToCopy(len - avail); + } + return readFromBuffer(b, off, len); + } + + private void tryToCopy(int bytesToCopy) { + // this will fit into the buffer without sliding and not + // require more than is available inside the back-reference + int copy = Math.min((int) Math.min(bytesToCopy, bytesRemaining), + buf.length - writeIndex); + if (copy == 0) { + // NOP + } else if (backReferenceOffset == 1) { // pretty common special case + final byte last = buf[writeIndex - 1]; + Arrays.fill(buf, writeIndex, writeIndex + copy, last); + writeIndex += copy; + } else if (copy < backReferenceOffset) { + System.arraycopy(buf, writeIndex - backReferenceOffset, buf, writeIndex, copy); + writeIndex += copy; + } else { + // back-reference overlaps with the bytes created from it + // like go back two bytes and then copy six (by copying + // the last two bytes three time). + final int fullRots = copy / backReferenceOffset; + for (int i = 0; i < fullRots; i++) { + System.arraycopy(buf, writeIndex - backReferenceOffset, buf, writeIndex, backReferenceOffset); + writeIndex += backReferenceOffset; + } + + final int pad = copy - (backReferenceOffset * fullRots); + if (pad > 0) { + System.arraycopy(buf, writeIndex - backReferenceOffset, buf, writeIndex, pad); + writeIndex += pad; + } + } + bytesRemaining -= copy; + } + + /** + * Reads a single byte from the real input stream and ensures the data is accounted for. + * + * @return the byte read as value between 0 and 255 or -1 if EOF has been reached. + * @throws IOException if the underlying stream throws + */ + protected final int readOneByte() throws IOException { + final int b = in.read(); + if (b != -1) { + count(1); + return b & 0xFF; + } + return -1; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/LZ77Compressor.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/LZ77Compressor.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/LZ77Compressor.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/LZ77Compressor.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,559 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz77support; + +import java.io.IOException; +import java.util.Arrays; + +/** + * Helper class for compression algorithms that use the ideas of LZ77. + * + *

    Most LZ77 derived algorithms split input data into blocks of + * uncompressed data (called literal blocks) and back-references + * (pairs of offsets and lengths) that state "add length + * bytes that are the same as those already written starting + * offset bytes before the current position. The details + * of how those blocks and back-references are encoded are quite + * different between the algorithms and some algorithms perform + * additional steps (Huffman encoding in the case of DEFLATE for + * example).

    + * + *

    This class attempts to extract the core logic - finding + * back-references - so it can be re-used. It follows the algorithm + * explained in section 4 of RFC 1951 (DEFLATE) and currently doesn't + * implement the "lazy match" optimization. The three-byte hash + * function used in this class is the same as the one used by zlib and + * InfoZIP's ZIP implementation of DEFLATE. The whole class is + * strongly inspired by InfoZIP's implementation.

    + * + *

    LZ77 is used vaguely here (as well as many other places that + * talk about it :-), LZSS would likely be closer to the truth but + * LZ77 has become the synonym for a whole family of algorithms.

    + * + *

    The API consists of a compressor that is fed bytes + * and emits {@link Block}s to a registered callback where the blocks + * represent either {@link LiteralBlock literal blocks}, {@link + * BackReference back-references} or {@link EOD end of data + * markers}. In order to ensure the callback receives all information, + * the {@code #finish} method must be used once all data has been fed + * into the compressor.

    + * + *

    Several parameters influence the outcome of the "compression":

    + *
    + * + *
    windowSize
    the size of the sliding + * window, must be a power of two - this determines the maximum + * offset a back-reference can take. The compressor maintains a + * buffer of twice of windowSize - real world values are + * in the area of 32k.
    + * + *
    minBackReferenceLength
    + *
    Minimal length of a back-reference found. A true minimum of 3 is + * hard-coded inside of this implemention but bigger lengths can be + * configured.
    + * + *
    maxBackReferenceLength
    + *
    Maximal length of a back-reference found.
    + * + *
    maxOffset
    + *
    Maximal offset of a back-reference.
    + * + *
    maxLiteralLength
    + *
    Maximal length of a literal block.
    + *
    + * + * @see "https://tools.ietf.org/html/rfc1951#section-4" + * @since 1.14 + * @NotThreadSafe + */ +public class LZ77Compressor { + + /** + * Base class representing blocks the compressor may emit. + * + *

    This class is not supposed to be subclassed by classes + * outside of Commons Compress so it is considered internal and + * changed that would break subclasses may get introduced with + * future releases.

    + */ + public static abstract class Block { + /** Enumeration of the block types the compressor may emit. */ + public enum BlockType { + LITERAL, BACK_REFERENCE, EOD + } + public abstract BlockType getType(); + } + + /** + * Represents a literal block of data. + * + *

    For performance reasons this encapsulates the real data, not + * a copy of it. Don't modify the data and process it inside of + * {@link Callback#accept} immediately as it will get overwritten + * sooner or later.

    + */ + public static final class LiteralBlock extends Block { + private final byte[] data; + private final int offset, length; + public LiteralBlock(byte[] data, int offset, int length) { + this.data = data; + this.offset = offset; + this.length = length; + } + /** + * The literal data. + * + *

    This returns a life view of the actual data in order to + * avoid copying, modify the array at your own risk.

    + * @return the data + */ + public byte[] getData() { + return data; + } + /** + * Offset into data where the literal block starts. + * @return the offset + */ + public int getOffset() { + return offset; + } + /** + * Length of literal block. + * @return the length + */ + public int getLength() { + return length; + } + @Override + public BlockType getType() { + return BlockType.LITERAL; + } + @Override + public String toString() { + return "LiteralBlock starting at " + offset + " with length " + length; + } + } + + /** + * Represents a back-reference. + */ + public static final class BackReference extends Block { + private final int offset, length; + public BackReference(int offset, int length) { + this.offset = offset; + this.length = length; + } + /** + * Provides the offset of the back-reference. + * @return the offset + */ + public int getOffset() { + return offset; + } + /** + * Provides the length of the back-reference. + * @return the length + */ + public int getLength() { + return length; + } + @Override + public BlockType getType() { + return BlockType.BACK_REFERENCE; + } + @Override + public String toString() { + return "BackReference with offset " + offset + " and length " + length; + } + } + + /** A simple "we are done" marker. */ + public static final class EOD extends Block { + @Override + public BlockType getType() { + return BlockType.EOD; + } + } + + private static final Block THE_EOD = new EOD(); + + /** + * Callback invoked while the compressor processes data. + * + *

    The callback is invoked on the same thread that receives the + * bytes to compress and may be invoked multiple times during the + * execution of {@link #compress} or {@link #finish}.

    + */ + public interface Callback { + /** + * Consumes a block. + * @param b the block to consume + * @throws IOException in case of an error + */ + void accept(Block b) throws IOException; + } + + static final int NUMBER_OF_BYTES_IN_HASH = 3; + private static final int NO_MATCH = -1; + + private final Parameters params; + private final Callback callback; + + // the sliding window, twice as big as "windowSize" parameter + private final byte[] window; + // the head of hash-chain - indexed by hash-code, points to the + // location inside of window of the latest sequence of bytes with + // the given hash. + private final int[] head; + // for each window-location points to the latest earlier location + // with the same hash. Only stores values for the latest + // "windowSize" elements, the index is "window location modulo + // windowSize". + private final int[] prev; + + // bit mask used when indexing into prev + private final int wMask; + + private boolean initialized = false; + // the position inside of window that shall be encoded right now + private int currentPosition; + // the number of bytes available to compress including the one at + // currentPosition + private int lookahead = 0; + // the hash of the three bytes stating at the current position + private int insertHash = 0; + // the position inside of the window where the current literal + // block starts (in case we are inside of a literal block). + private int blockStart = 0; + // position of the current match + private int matchStart = NO_MATCH; + // number of missed insertString calls for the up to three last + // bytes of the last match that can only be performed once more + // data has been read + private int missedInserts = 0; + + /** + * Initializes a compressor with parameters and a callback. + * @param params the parameters + * @param callback the callback + * @throws NullPointerException if either parameter is null + */ + public LZ77Compressor(Parameters params, Callback callback) { + if (params == null) { + throw new NullPointerException("params must not be null"); + } + if (callback == null) { + throw new NullPointerException("callback must not be null"); + } + this.params = params; + this.callback = callback; + + final int wSize = params.getWindowSize(); + window = new byte[wSize * 2]; + wMask = wSize - 1; + head = new int[HASH_SIZE]; + Arrays.fill(head, NO_MATCH); + prev = new int[wSize]; + } + + /** + * Feeds bytes into the compressor which in turn may emit zero or + * more blocks to the callback during the execution of this + * method. + * @param data the data to compress - must not be null + * @throws IOException if the callback throws an exception + */ + public void compress(byte[] data) throws IOException { + compress(data, 0, data.length); + } + + /** + * Feeds bytes into the compressor which in turn may emit zero or + * more blocks to the callback during the execution of this + * method. + * @param data the data to compress - must not be null + * @param off the start offset of the data + * @param len the number of bytes to compress + * @throws IOException if the callback throws an exception + */ + public void compress(byte[] data, int off, int len) throws IOException { + final int wSize = params.getWindowSize(); + while (len > wSize) { // chop into windowSize sized chunks + doCompress(data, off, wSize); + off += wSize; + len -= wSize; + } + if (len > 0) { + doCompress(data, off, len); + } + } + + /** + * Tells the compressor to process all remaining data and signal + * end of data to the callback. + * + *

    The compressor will in turn emit at least one block ({@link + * EOD}) but potentially multiple blocks to the callback during + * the execution of this method.

    + * @throws IOException if the callback throws an exception + */ + public void finish() throws IOException { + if (blockStart != currentPosition || lookahead > 0) { + currentPosition += lookahead; + flushLiteralBlock(); + } + callback.accept(THE_EOD); + } + + /** + * Adds some initial data to fill the window with. + * + *

    This is used if the stream has been cut into blocks and + * back-references of one block may refer to data of the previous + * block(s). One such example is the LZ4 frame format using block + * dependency.

    + * + * @param data the data to fill the window with. + * @throws IllegalStateException if the compressor has already started to accept data + */ + public void prefill(byte[] data) { + if (currentPosition != 0 || lookahead != 0) { + throw new IllegalStateException("the compressor has already started to accept data, can't prefill anymore"); + } + + // don't need more than windowSize for back-references + final int len = Math.min(params.getWindowSize(), data.length); + System.arraycopy(data, data.length - len, window, 0, len); + + if (len >= NUMBER_OF_BYTES_IN_HASH) { + initialize(); + final int stop = len - NUMBER_OF_BYTES_IN_HASH + 1; + for (int i = 0; i < stop; i++) { + insertString(i); + } + missedInserts = NUMBER_OF_BYTES_IN_HASH - 1; + } else { // not enough data to hash anything + missedInserts = len; + } + blockStart = currentPosition = len; + } + + // we use a 15 bit hashcode as calculated in updateHash + private static final int HASH_SIZE = 1 << 15; + private static final int HASH_MASK = HASH_SIZE - 1; + private static final int H_SHIFT = 5; + + /** + * Assumes we are calculating the hash for three consecutive bytes + * as a rolling hash, i.e. for bytes ABCD if H is the hash of ABC + * the new hash for BCD is nextHash(H, D). + * + *

    The hash is shifted by five bits on each update so all + * effects of A have been swapped after the third update.

    + */ + private int nextHash(int oldHash, byte nextByte) { + final int nextVal = nextByte & 0xFF; + return ((oldHash << H_SHIFT) ^ nextVal) & HASH_MASK; + } + + // performs the actual algorithm with the pre-condition len <= windowSize + private void doCompress(byte[] data, int off, int len) throws IOException { + int spaceLeft = window.length - currentPosition - lookahead; + if (len > spaceLeft) { + slide(); + } + System.arraycopy(data, off, window, currentPosition + lookahead, len); + lookahead += len; + if (!initialized && lookahead >= params.getMinBackReferenceLength()) { + initialize(); + } + if (initialized) { + compress(); + } + } + + private void slide() throws IOException { + final int wSize = params.getWindowSize(); + if (blockStart != currentPosition && blockStart < wSize) { + flushLiteralBlock(); + blockStart = currentPosition; + } + System.arraycopy(window, wSize, window, 0, wSize); + currentPosition -= wSize; + matchStart -= wSize; + blockStart -= wSize; + for (int i = 0; i < HASH_SIZE; i++) { + int h = head[i]; + head[i] = h >= wSize ? h - wSize : NO_MATCH; + } + for (int i = 0; i < wSize; i++) { + int p = prev[i]; + prev[i] = p >= wSize ? p - wSize : NO_MATCH; + } + } + + private void initialize() { + for (int i = 0; i < NUMBER_OF_BYTES_IN_HASH - 1; i++) { + insertHash = nextHash(insertHash, window[i]); + } + initialized = true; + } + + private void compress() throws IOException { + final int minMatch = params.getMinBackReferenceLength(); + final boolean lazy = params.getLazyMatching(); + final int lazyThreshold = params.getLazyMatchingThreshold(); + + while (lookahead >= minMatch) { + catchUpMissedInserts(); + int matchLength = 0; + int hashHead = insertString(currentPosition); + if (hashHead != NO_MATCH && hashHead - currentPosition <= params.getMaxOffset()) { + // sets matchStart as a side effect + matchLength = longestMatch(hashHead); + + if (lazy && matchLength <= lazyThreshold && lookahead > minMatch) { + // try to find a longer match using the next position + matchLength = longestMatchForNextPosition(matchLength); + } + } + if (matchLength >= minMatch) { + if (blockStart != currentPosition) { + // emit preceeding literal block + flushLiteralBlock(); + blockStart = NO_MATCH; + } + flushBackReference(matchLength); + insertStringsInMatch(matchLength); + lookahead -= matchLength; + currentPosition += matchLength; + blockStart = currentPosition; + } else { + // no match, append to current or start a new literal + lookahead--; + currentPosition++; + if (currentPosition - blockStart >= params.getMaxLiteralLength()) { + flushLiteralBlock(); + blockStart = currentPosition; + } + } + } + } + + /** + * Inserts the current three byte sequence into the dictionary and + * returns the previous head of the hash-chain. + * + *

    Updates insertHash and prev as a + * side effect.

    + */ + private int insertString(int pos) { + insertHash = nextHash(insertHash, window[pos - 1 + NUMBER_OF_BYTES_IN_HASH]); + int hashHead = head[insertHash]; + prev[pos & wMask] = hashHead; + head[insertHash] = pos; + return hashHead; + } + + private int longestMatchForNextPosition(final int prevMatchLength) { + // save a bunch of values to restore them if the next match isn't better than the current one + final int prevMatchStart = matchStart; + final int prevInsertHash = insertHash; + + lookahead--; + currentPosition++; + int hashHead = insertString(currentPosition); + final int prevHashHead = prev[currentPosition & wMask]; + int matchLength = longestMatch(hashHead); + + if (matchLength <= prevMatchLength) { + // use the first match, as the next one isn't any better + matchLength = prevMatchLength; + matchStart = prevMatchStart; + + // restore modified values + head[insertHash] = prevHashHead; + insertHash = prevInsertHash; + currentPosition--; + lookahead++; + } + return matchLength; + } + + private void insertStringsInMatch(int matchLength) { + // inserts strings contained in current match + // insertString inserts the byte 2 bytes after position, which may not yet be available -> missedInserts + final int stop = Math.min(matchLength - 1, lookahead - NUMBER_OF_BYTES_IN_HASH); + // currentPosition has been inserted already + for (int i = 1; i <= stop; i++) { + insertString(currentPosition + i); + } + missedInserts = matchLength - stop - 1; + } + + private void catchUpMissedInserts() { + while (missedInserts > 0) { + insertString(currentPosition - missedInserts--); + } + } + + private void flushBackReference(int matchLength) throws IOException { + callback.accept(new BackReference(currentPosition - matchStart, matchLength)); + } + + private void flushLiteralBlock() throws IOException { + callback.accept(new LiteralBlock(window, blockStart, currentPosition - blockStart)); + } + + /** + * Searches the hash chain for real matches and returns the length + * of the longest match (0 if none were found) that isn't too far + * away (WRT maxOffset). + * + *

    Sets matchStart to the index of the start position of the + * longest match as a side effect.

    + */ + private int longestMatch(int matchHead) { + final int minLength = params.getMinBackReferenceLength(); + int longestMatchLength = minLength - 1; + final int maxPossibleLength = Math.min(params.getMaxBackReferenceLength(), lookahead); + final int minIndex = Math.max(0, currentPosition - params.getMaxOffset()); + final int niceBackReferenceLength = Math.min(maxPossibleLength, params.getNiceBackReferenceLength()); + final int maxCandidates = params.getMaxCandidates(); + for (int candidates = 0; candidates < maxCandidates && matchHead >= minIndex; candidates++) { + int currentLength = 0; + for (int i = 0; i < maxPossibleLength; i++) { + if (window[matchHead + i] != window[currentPosition + i]) { + break; + } + currentLength++; + } + if (currentLength > longestMatchLength) { + longestMatchLength = currentLength; + matchStart = matchHead; + if (currentLength >= niceBackReferenceLength) { + // no need to search any further + break; + } + } + matchHead = prev[matchHead & wMask]; + } + return longestMatchLength; // < minLength if no matches have been found, will be ignored in compress() + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/package.html 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/package.html 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,28 @@ + + + +

    Provides utility classes for LZ77 based algorithms.

    + +

    The classes in this package are currently used by the LZ4 and + Snappy implementations but might also help implementing other + algorithms that derive from LZ77 and LZSS.

    + + + diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/Parameters.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/Parameters.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lz77support/Parameters.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lz77support/Parameters.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz77support; + +/** + * Parameters of the {@link LZ77Compressor compressor}. + */ +public final class Parameters { + /** + * The hard-coded absolute minimal length of a back-reference. + */ + public static final int TRUE_MIN_BACK_REFERENCE_LENGTH = LZ77Compressor.NUMBER_OF_BYTES_IN_HASH; + + /** + * Initializes the builder for the compressor's parameters with a + * minBackReferenceLength of 3 and max*Length + * equal to windowSize - 1. + * + *

    It is recommended to not use this method directly but rather + * tune a pre-configured builder created by a format specific + * factory like {@link + * org.apache.commons.compress.compressors.snappy.SnappyCompressorOutputStream#createParameterBuilder}.

    + * + * @param windowSize the size of the sliding window - this + * determines the maximum offset a back-reference can take. Must + * be a power of two. + * @throws IllegalArgumentException if windowSize is not a power of two. + * @return a builder configured for the given window size + */ + public static Builder builder(int windowSize) { + return new Builder(windowSize); + } + + /** + * Builder for {@link Parameters} instances. + */ + public static class Builder { + private final int windowSize; + private int minBackReferenceLength, maxBackReferenceLength, maxOffset, maxLiteralLength; + private Integer niceBackReferenceLength, maxCandidates, lazyThreshold; + private Boolean lazyMatches; + + private Builder(int windowSize) { + if (windowSize < 2 || !isPowerOfTwo(windowSize)) { + throw new IllegalArgumentException("windowSize must be a power of two"); + } + this.windowSize = windowSize; + minBackReferenceLength = TRUE_MIN_BACK_REFERENCE_LENGTH; + maxBackReferenceLength = windowSize - 1; + maxOffset = windowSize - 1; + maxLiteralLength = windowSize; + } + + /** + * Sets the mininal length of a back-reference. + * + *

    Ensures maxBackReferenceLength is not + * smaller than minBackReferenceLength. + * + *

    It is recommended to not use this method directly but + * rather tune a pre-configured builder created by a format + * specific factory like {@link + * org.apache.commons.compress.compressors.snappy.SnappyCompressorOutputStream#createParameterBuilder}.

    + * + * @param minBackReferenceLength the minimal length of a back-reference found. A + * true minimum of 3 is hard-coded inside of this implemention + * but bigger lengths can be configured. + * @throws IllegalArgumentException if windowSize + * is smaller than minBackReferenceLength. + * @return the builder + */ + public Builder withMinBackReferenceLength(int minBackReferenceLength) { + this.minBackReferenceLength = Math.max(TRUE_MIN_BACK_REFERENCE_LENGTH, minBackReferenceLength); + if (windowSize < this.minBackReferenceLength) { + throw new IllegalArgumentException("minBackReferenceLength can't be bigger than windowSize"); + } + if (maxBackReferenceLength < this.minBackReferenceLength) { + maxBackReferenceLength = this.minBackReferenceLength; + } + return this; + } + + /** + * Sets the maximal length of a back-reference. + * + *

    It is recommended to not use this method directly but + * rather tune a pre-configured builder created by a format + * specific factory like {@link + * org.apache.commons.compress.compressors.snappy.SnappyCompressorOutputStream#createParameterBuilder}.

    + * + * @param maxBackReferenceLength maximal length of a + * back-reference found. A value smaller than + * minBackReferenceLength is interpreted as + * minBackReferenceLength. maxBackReferenceLength + * is capped at windowSize - 1. + * @return the builder + */ + public Builder withMaxBackReferenceLength(int maxBackReferenceLength) { + this.maxBackReferenceLength = maxBackReferenceLength < minBackReferenceLength ? minBackReferenceLength + : Math.min(maxBackReferenceLength, windowSize - 1); + return this; + } + + /** + * Sets the maximal offset of a back-reference. + * + *

    It is recommended to not use this method directly but + * rather tune a pre-configured builder created by a format + * specific factory like {@link + * org.apache.commons.compress.compressors.snappy.SnappyCompressorOutputStream#createParameterBuilder}.

    + * + * @param maxOffset maximal offset of a back-reference. A + * non-positive value as well as values bigger than + * windowSize - 1 are interpreted as windowSize + * - 1. + * @return the builder + */ + public Builder withMaxOffset(int maxOffset) { + this.maxOffset = maxOffset < 1 ? windowSize - 1 : Math.min(maxOffset, windowSize - 1); + return this; + } + + /** + * Sets the maximal length of a literal block. + * + *

    It is recommended to not use this method directly but + * rather tune a pre-configured builder created by a format + * specific factory like {@link + * org.apache.commons.compress.compressors.snappy.SnappyCompressorOutputStream#createParameterBuilder}.

    + * + * @param maxLiteralLength maximal length of a literal + * block. Negative numbers and 0 as well as values bigger than + * windowSize are interpreted as + * windowSize. + * @return the builder + */ + public Builder withMaxLiteralLength(int maxLiteralLength) { + this.maxLiteralLength = maxLiteralLength < 1 ? windowSize + : Math.min(maxLiteralLength, windowSize); + return this; + } + + /** + * Sets the "nice length" of a back-reference. + * + *

    When a back-references if this size has been found, stop searching for longer back-references.

    + * + *

    This settings can be used to tune the tradeoff between compression speed and compression ratio.

    + * @param niceLen the "nice length" of a back-reference + * @return the builder + */ + public Builder withNiceBackReferenceLength(int niceLen) { + niceBackReferenceLength = niceLen; + return this; + } + + /** + * Sets the maximum number of back-reference candidates that should be consulted. + * + *

    This settings can be used to tune the tradeoff between compression speed and compression ratio.

    + * @param maxCandidates maximum number of back-reference candidates + * @return the builder + */ + public Builder withMaxNumberOfCandidates(int maxCandidates) { + this.maxCandidates = maxCandidates; + return this; + } + + /** + * Sets whether lazy matching should be performed. + * + *

    Lazy matching means that after a back-reference for a certain position has been found the compressor will + * try to find a longer match for the next position.

    + * + *

    Lazy matching is enabled by default and disabled when tuning for speed.

    + * @param lazy whether lazy matching should be performed + * @return the builder + */ + public Builder withLazyMatching(boolean lazy) { + lazyMatches = lazy; + return this; + } + + /** + * Sets the threshold for lazy matching. + * + *

    Even if lazy matching is enabled it will not be performed if the length of the back-reference found for + * the current position is longer than this value.

    + * @param threshold the threshold for lazy matching + * @return the builder + */ + public Builder withLazyThreshold(int threshold) { + lazyThreshold = threshold; + return this; + } + + /** + * Changes the default setting for "nice back-reference length" and "maximum number of candidates" for improved + * compression speed at the cost of compression ratio. + * + *

    Use this method after configuring "maximum back-reference length".

    + * @return the builder + */ + public Builder tunedForSpeed() { + niceBackReferenceLength = Math.max(minBackReferenceLength, maxBackReferenceLength / 8); + maxCandidates = Math.max(32, windowSize / 1024); + lazyMatches = false; + lazyThreshold = minBackReferenceLength; + return this; + } + + /** + * Changes the default setting for "nice back-reference length" and "maximum number of candidates" for improved + * compression ratio at the cost of compression speed. + * + *

    Use this method after configuring "maximum back-reference length".

    + * @return the builder + */ + public Builder tunedForCompressionRatio() { + niceBackReferenceLength = lazyThreshold = maxBackReferenceLength; + maxCandidates = Math.max(32, windowSize / 16); + lazyMatches = true; + return this; + } + + /** + * Creates the {@link Parameters} instance. + * @return the configured {@link Parameters} instance. + */ + public Parameters build() { + // default settings tuned for a compromise of good compression and acceptable speed + int niceLen = niceBackReferenceLength != null ? niceBackReferenceLength + : Math.max(minBackReferenceLength, maxBackReferenceLength / 2); + int candidates = maxCandidates != null ? maxCandidates : Math.max(256, windowSize / 128); + boolean lazy = lazyMatches == null || lazyMatches; + int threshold = lazy ? (lazyThreshold != null ? lazyThreshold : niceLen) : minBackReferenceLength; + + return new Parameters(windowSize, minBackReferenceLength, maxBackReferenceLength, + maxOffset, maxLiteralLength, niceLen, candidates, lazy, threshold); + } + } + + private final int windowSize, minBackReferenceLength, maxBackReferenceLength, maxOffset, maxLiteralLength, + niceBackReferenceLength, maxCandidates, lazyThreshold; + private final boolean lazyMatching; + + private Parameters(int windowSize, int minBackReferenceLength, int maxBackReferenceLength, int maxOffset, + int maxLiteralLength, int niceBackReferenceLength, int maxCandidates, boolean lazyMatching, + int lazyThreshold) { + this.windowSize = windowSize; + this.minBackReferenceLength = minBackReferenceLength; + this.maxBackReferenceLength = maxBackReferenceLength; + this.maxOffset = maxOffset; + this.maxLiteralLength = maxLiteralLength; + this.niceBackReferenceLength = niceBackReferenceLength; + this.maxCandidates = maxCandidates; + this.lazyMatching = lazyMatching; + this.lazyThreshold = lazyThreshold; + } + + /** + * Gets the size of the sliding window - this determines the + * maximum offset a back-reference can take. + * @return the size of the sliding window + */ + public int getWindowSize() { + return windowSize; + } + /** + * Gets the minimal length of a back-reference found. + * @return the minimal length of a back-reference found + */ + public int getMinBackReferenceLength() { + return minBackReferenceLength; + } + /** + * Gets the maximal length of a back-reference found. + * @return the maximal length of a back-reference found + */ + public int getMaxBackReferenceLength() { + return maxBackReferenceLength; + } + /** + * Gets the maximal offset of a back-reference found. + * @return the maximal offset of a back-reference found + */ + public int getMaxOffset() { + return maxOffset; + } + /** + * Gets the maximal length of a literal block. + * @return the maximal length of a literal block + */ + public int getMaxLiteralLength() { + return maxLiteralLength; + } + + /** + * Gets the length of a back-reference that is considered nice enough to stop searching for longer ones. + * @return the length of a back-reference that is considered nice enough to stop searching + */ + public int getNiceBackReferenceLength() { + return niceBackReferenceLength; + } + + /** + * Gets the maximum number of back-reference candidates to consider. + * @return the maximum number of back-reference candidates to consider + */ + public int getMaxCandidates() { + return maxCandidates; + } + + /** + * Gets whether to perform lazy matching. + * @return whether to perform lazy matching + */ + public boolean getLazyMatching() { + return lazyMatching; + } + + /** + * Gets the threshold for lazy matching. + * @return the threshold for lazy matching + */ + public int getLazyMatchingThreshold() { + return lazyThreshold; + } + + private static final boolean isPowerOfTwo(int x) { + // pre-condition: x > 0 + return (x & (x - 1)) == 0; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -20,15 +20,23 @@ import java.io.IOException; import java.io.InputStream; + +import org.apache.commons.compress.MemoryLimitException; import org.tukaani.xz.LZMAInputStream; import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.utils.CountingInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; /** * LZMA decompressor. * @since 1.6 */ -public class LZMACompressorInputStream extends CompressorInputStream { +public class LZMACompressorInputStream extends CompressorInputStream + implements InputStreamStatistics { + + private final CountingInputStream countingStream; private final InputStream in; /** @@ -45,7 +53,34 @@ */ public LZMACompressorInputStream(final InputStream inputStream) throws IOException { - in = new LZMAInputStream(inputStream); + in = new LZMAInputStream(countingStream = new CountingInputStream(inputStream), -1); + } + + /** + * Creates a new input stream that decompresses LZMA-compressed data + * from the specified input stream. + * + * @param inputStream where to read the compressed data + * + * @param memoryLimitInKb calculated memory use threshold. Throws MemoryLimitException + * if calculate memory use is above this threshold + * + * @throws IOException if the input is not in the .lzma format, + * the input is corrupt or truncated, the .lzma + * headers specify sizes that are not supported + * by this implementation, or the underlying + * inputStream throws an exception + * + * @since 1.14 + */ + public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitInKb) + throws IOException { + try { + in = new LZMAInputStream(countingStream = new CountingInputStream(inputStream), memoryLimitInKb); + } catch (org.tukaani.xz.MemoryLimitException e) { + //convert to commons-compress exception + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); + } } /** {@inheritDoc} */ @@ -67,7 +102,7 @@ /** {@inheritDoc} */ @Override public long skip(final long n) throws IOException { - return in.skip(n); + return IOUtils.skip(in, n); } /** {@inheritDoc} */ @@ -83,34 +118,27 @@ } /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return countingStream.getBytesRead(); + } + + /** * Checks if the signature matches what is expected for an lzma file. - * + * * @param signature * the bytes to check * @param length * the number of bytes to check * @return true, if this stream is an lzma compressed stream, false otherwise - * + * * @since 1.10 */ public static boolean matches(final byte[] signature, final int length) { - - if (signature == null || length < 3) { - return false; - } - - if (signature[0] != 0x5d) { - return false; - } - - if (signature[1] != 0) { - return false; - } - - if (signature[2] != 0) { - return false; - } - - return true; + return signature != null && length >= 3 && + signature[0] == 0x5d && signature[1] == 0 && + signature[2] == 0; } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorOutputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorOutputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -60,7 +60,6 @@ */ @Override public void flush() throws IOException { - out.flush(); } /** diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/LZMAUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/LZMAUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/LZMAUtils.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/LZMAUtils.java 2018-05-02 20:17:13.000000000 +0000 @@ -38,7 +38,7 @@ (byte) 0x5D, 0, 0 }; - static enum CachedAvailability { + enum CachedAvailability { DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE } @@ -151,7 +151,7 @@ cachedLZMAAvailability = CachedAvailability.DONT_CACHE; } else if (cachedLZMAAvailability == CachedAvailability.DONT_CACHE) { final boolean hasLzma = internalIsLZMACompressionAvailable(); - cachedLZMAAvailability = hasLzma ? CachedAvailability.CACHED_AVAILABLE + cachedLZMAAvailability = hasLzma ? CachedAvailability.CACHED_AVAILABLE // NOSONAR : CachedAvailability.CACHED_UNAVAILABLE; } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzma/package.html 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzma/package.html 2018-05-23 12:50:54.000000000 +0000 @@ -18,13 +18,12 @@ --> -

    Provides a stream class decompressing streams using the - "stand-alone" LZMA algorithm.

    +

    Provides stream classes using the "stand-alone" LZMA + algorithm.

    -

    The class in this package is a wrapper around {@link - org.tukaani.xz.LZMAInputStream org.tukaani.xz.LZMAInputStream} - and provided by the public - domain XZ for Java +

    The classes in this package are wrappers around stream classes + provided by the public + domain XZ for Java library.

    In general you should prefer the more modern and robust XZ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -22,8 +22,10 @@ import java.io.InputStream; import java.nio.ByteOrder; +import org.apache.commons.compress.MemoryLimitException; import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.utils.BitInputStream; +import org.apache.commons.compress.utils.InputStreamStatistics; /** *

    Generic LZW implementation. It is used internally for @@ -33,7 +35,7 @@ * @NotThreadSafe * @since 1.10 */ -public abstract class LZWInputStream extends CompressorInputStream { +public abstract class LZWInputStream extends CompressorInputStream implements InputStreamStatistics { protected static final int DEFAULT_CODE_SIZE = 9; protected static final int UNUSED_PREFIX = -1; @@ -58,7 +60,7 @@ public void close() throws IOException { in.close(); } - + @Override public int read() throws IOException { final int ret = read(oneByte); @@ -67,7 +69,7 @@ } return 0xff & oneByte[0]; } - + @Override public int read(final byte[] b, final int off, final int len) throws IOException { int bytesRead = readFromStack(b, off, len); @@ -87,6 +89,14 @@ } /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return in.getBytesRead(); + } + + /** * Read the next code and expand it. * @return the expanded next code * @throws IOException on error @@ -113,6 +123,30 @@ /** * Initializes the arrays based on the maximum code size. + * First checks that the estimated memory usage is below memoryLimitInKb + * + * @param maxCodeSize maximum code size + * @param memoryLimitInKb maximum allowed estimated memory usage in Kb + * @throws MemoryLimitException if estimated memory usage is greater than memoryLimitInKb + */ + protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb) + throws MemoryLimitException { + + if (memoryLimitInKb > -1) { + final int maxTableSize = 1 << maxCodeSize; + //account for potential overflow + long memoryUsageInBytes = (long) maxTableSize * 6;//(4 (prefixes) + 1 (characters) +1 (outputStack)) + long memoryUsageInKb = memoryUsageInBytes >> 10; + + if (memoryUsageInKb > memoryLimitInKb) { + throw new MemoryLimitException(memoryUsageInKb, memoryLimitInKb); + } + } + initializeTables(maxCodeSize); + } + + /** + * Initializes the arrays based on the maximum code size. * @param maxCodeSize maximum code size */ protected void initializeTables(final int maxCodeSize) { @@ -139,7 +173,7 @@ } return (int) in.readBits(codeSize); } - + /** * Adds a new entry if the maximum table size hasn't been exceeded * and returns the new index. diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -28,11 +28,12 @@ import java.util.jar.Pack200; import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.utils.IOUtils; /** * An input stream that decompresses from the Pack200 format to be read * as any other stream. - * + * *

    The {@link CompressorInputStream#getCount getCount} and {@link * CompressorInputStream#getBytesRead getBytesRead} methods always * return 0.

    @@ -235,7 +236,7 @@ @Override public long skip(final long count) throws IOException { - return streamBridge.getInput().skip(count); + return IOUtils.skip(streamBridge.getInput(), count); } @Override @@ -257,7 +258,7 @@ /** * Checks if the signature matches what is expected for a pack200 * file (0xCAFED00D). - * + * * @param signature * the bytes to check * @param length diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorOutputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200CompressorOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -26,11 +26,10 @@ import java.util.jar.Pack200; import org.apache.commons.compress.compressors.CompressorOutputStream; -import org.apache.commons.compress.utils.IOUtils; /** * An output stream that compresses using the Pack200 format. - * + * * @NotThreadSafe * @since 1.3 */ @@ -115,11 +114,14 @@ @Override public void close() throws IOException { - finish(); try { - streamBridge.stop(); + finish(); } finally { - originalOutput.close(); + try { + streamBridge.stop(); + } finally { + originalOutput.close(); + } } } @@ -130,16 +132,8 @@ if (properties != null) { p.properties().putAll(properties); } - JarInputStream ji = null; - boolean success = false; - try { - ji = new JarInputStream(streamBridge.getInput()); + try (JarInputStream ji = new JarInputStream(streamBridge.getInput())) { p.pack(ji, originalOutput); - success = true; - } finally { - if (!success) { - IOUtils.closeQuietly(ji); - } } } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200Utils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200Utils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200Utils.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/Pack200Utils.java 2018-05-23 12:50:54.000000000 +0000 @@ -42,7 +42,7 @@ * and packed. * *

    As stated in Pack200.Packer's + * href="https://download.oracle.com/javase/1.5.0/docs/api/java/util/jar/Pack200.Packer.html">Pack200.Packer's * javadocs applying a Pack200 compression to a JAR archive will * in general make its sigantures invalid. In order to prepare a * JAR for signing it should be "normalized" by packing and @@ -64,7 +64,7 @@ * and packed. * *

    As stated in Pack200.Packer's + * href="https://download.oracle.com/javase/1.5.0/docs/api/java/util/jar/Pack200.Packer.html">Pack200.Packer's * javadocs applying a Pack200 compression to a JAR archive will * in general make its sigantures invalid. In order to prepare a * JAR for signing it should be "normalized" by packing and @@ -84,7 +84,7 @@ * Normalizes a JAR archive so it can be safely signed and packed. * *

    As stated in Pack200.Packer's + * href="https://download.oracle.com/javase/1.5.0/docs/api/java/util/jar/Pack200.Packer.html">Pack200.Packer's * javadocs applying a Pack200 compression to a JAR archive will * in general make its sigantures invalid. In order to prepare a * JAR for signing it should be "normalized" by packing and @@ -109,7 +109,7 @@ * Normalizes a JAR archive so it can be safely signed and packed. * *

    As stated in Pack200.Packer's + * href="https://download.oracle.com/javase/1.5.0/docs/api/java/util/jar/Pack200.Packer.html">Pack200.Packer's * javadocs applying a Pack200 compression to a JAR archive will * in general make its sigantures invalid. In order to prepare a * JAR for signing it should be "normalized" by packing and diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/package.html 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/package.html 2018-05-23 12:50:54.000000000 +0000 @@ -34,7 +34,7 @@

    JAR archives compressed with Pack200 will in general be different from the original archive when decompressed again. For details see - the API + the API documentation of Pack200.

    The streams of this package work on non-deflated streams, diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/TempFileCachingStreamBridge.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/TempFileCachingStreamBridge.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/pack200/TempFileCachingStreamBridge.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/pack200/TempFileCachingStreamBridge.java 2018-07-01 09:53:29.000000000 +0000 @@ -20,10 +20,10 @@ package org.apache.commons.compress.compressors.pack200; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; /** * StreamSwitcher that caches all data written to the output side in @@ -36,17 +36,20 @@ TempFileCachingStreamBridge() throws IOException { f = File.createTempFile("commons-compress", "packtemp"); f.deleteOnExit(); - out = new FileOutputStream(f); + out = Files.newOutputStream(f.toPath()); } @Override InputStream getInputView() throws IOException { out.close(); - return new FileInputStream(f) { + return new FilterInputStream(Files.newInputStream(f.toPath())) { @Override public void close() throws IOException { - super.close(); - f.delete(); + try { + super.close(); + } finally { + f.delete(); + } } }; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -25,7 +25,10 @@ import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.utils.BoundedInputStream; +import org.apache.commons.compress.utils.ByteUtils; +import org.apache.commons.compress.utils.CountingInputStream; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; /** * CompressorInputStream for the framing Snappy format. @@ -35,7 +38,8 @@ * @see Snappy framing format description * @since 1.7 */ -public class FramedSnappyCompressorInputStream extends CompressorInputStream { +public class FramedSnappyCompressorInputStream extends CompressorInputStream + implements InputStreamStatistics { /** * package private for tests only. @@ -43,22 +47,26 @@ static final long MASK_OFFSET = 0xa282ead8L; private static final int STREAM_IDENTIFIER_TYPE = 0xff; - private static final int COMPRESSED_CHUNK_TYPE = 0; + static final int COMPRESSED_CHUNK_TYPE = 0; private static final int UNCOMPRESSED_CHUNK_TYPE = 1; private static final int PADDING_CHUNK_TYPE = 0xfe; private static final int MIN_UNSKIPPABLE_TYPE = 2; private static final int MAX_UNSKIPPABLE_TYPE = 0x7f; private static final int MAX_SKIPPABLE_TYPE = 0xfd; - private static final byte[] SZ_SIGNATURE = new byte[] { + // used by FramedSnappyCompressorOutputStream as well + static final byte[] SZ_SIGNATURE = new byte[] { //NOSONAR (byte) STREAM_IDENTIFIER_TYPE, // tag 6, 0, 0, // length 's', 'N', 'a', 'P', 'p', 'Y' }; + private long unreadBytes; + private final CountingInputStream countingStream; + /** The underlying stream to read compressed data from */ private final PushbackInputStream in; - + /** The dialect to expect */ private final FramedSnappyDialect dialect; @@ -71,8 +79,16 @@ private int uncompressedBytesRemaining; private long expectedChecksum = -1; + private final int blockSize; private final PureJavaCrc32C checksum = new PureJavaCrc32C(); + private final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() { + @Override + public int getAsByte() throws IOException { + return readOneByte(); + } + }; + /** * Constructs a new input stream that decompresses * snappy-framed-compressed data from the specified input stream @@ -94,7 +110,25 @@ public FramedSnappyCompressorInputStream(final InputStream in, final FramedSnappyDialect dialect) throws IOException { - this.in = new PushbackInputStream(in, 1); + this(in, SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE, dialect); + } + + /** + * Constructs a new input stream that decompresses snappy-framed-compressed data + * from the specified input stream. + * @param in the InputStream from which to read the compressed data + * @param blockSize the block size to use for the compressed stream + * @param dialect the dialect used by the compressed stream + * @throws IOException if reading fails + * @since 1.14 + */ + public FramedSnappyCompressorInputStream(final InputStream in, + final int blockSize, + final FramedSnappyDialect dialect) + throws IOException { + countingStream = new CountingInputStream(in); + this.in = new PushbackInputStream(countingStream, 1); + this.blockSize = blockSize; this.dialect = dialect; if (dialect.hasStreamIdentifier()) { readStreamIdentifier(); @@ -110,11 +144,14 @@ /** {@inheritDoc} */ @Override public void close() throws IOException { - if (currentCompressedChunk != null) { - currentCompressedChunk.close(); - currentCompressedChunk = null; + try { + if (currentCompressedChunk != null) { + currentCompressedChunk.close(); + currentCompressedChunk = null; + } + } finally { + in.close(); } - in.close(); } /** {@inheritDoc} */ @@ -144,6 +181,14 @@ } /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return countingStream.getBytesRead() - unreadBytes; + } + + /** * Read from the current chunk into the given array. * * @return -1 if there is no current chunk or the number of bytes @@ -186,6 +231,7 @@ endReached = true; } else if (type == STREAM_IDENTIFIER_TYPE) { in.unread(type); + unreadBytes++; pushedBackBytes(1); readStreamIdentifier(); readNextBlock(); @@ -203,14 +249,14 @@ expectedChecksum = unmask(readCrc()); } else if (type == COMPRESSED_CHUNK_TYPE) { final boolean expectChecksum = dialect.usesChecksumWithCompressedChunks(); - final long size = readSize() - (expectChecksum ? 4l : 0l); + final long size = readSize() - (expectChecksum ? 4L : 0L); if (expectChecksum) { expectedChecksum = unmask(readCrc()); } else { expectedChecksum = -1; } currentCompressedChunk = - new SnappyCompressorInputStream(new BoundedInputStream(in, size)); + new SnappyCompressorInputStream(new BoundedInputStream(in, size), blockSize); // constructor reads uncompressed size count(currentCompressedChunk.getBytesRead()); } else { @@ -227,11 +273,7 @@ if (read != 4) { throw new IOException("premature end of stream"); } - long crc = 0; - for (int i = 0; i < 4; i++) { - crc |= (b[i] & 0xFFL) << (8 * i); - } - return crc; + return ByteUtils.fromLittleEndian(b); } static long unmask(long x) { @@ -243,16 +285,7 @@ } private int readSize() throws IOException { - int b = 0; - int sz = 0; - for (int i = 0; i < 3; i++) { - b = readOneByte(); - if (b == -1) { - throw new IOException("premature end of stream"); - } - sz |= (b << (i * 8)); - } - return sz; + return (int) ByteUtils.fromLittleEndian(supplier, 3); } private void skipBlock() throws IOException { @@ -294,7 +327,7 @@ * Checks if the signature matches what is expected for a .sz file. * *

    .sz files start with a chunk with tag 0xff and content sNaPpY.

    - * + * * @param signature the bytes to check * @param length the number of bytes to check * @return true if this is a .sz stream, false otherwise diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.snappy; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.lz77support.Parameters; +import org.apache.commons.compress.utils.ByteUtils; + +/** + * CompressorOutputStream for the framing Snappy format. + * + *

    Based on the "spec" in the version "Last revised: 2013-10-25"

    + * + * @see Snappy framing format description + * @since 1.14 + * @NotThreadSafe + */ +public class FramedSnappyCompressorOutputStream extends CompressorOutputStream { + // see spec: + // > However, we place an additional restriction that the uncompressed data + // > in a chunk must be no longer than 65536 bytes. This allows consumers to + // > easily use small fixed-size buffers. + private static final int MAX_COMPRESSED_BUFFER_SIZE = 1 << 16; + + private final OutputStream out; + private final Parameters params; + private final PureJavaCrc32C checksum = new PureJavaCrc32C(); + // used in one-arg write method + private final byte[] oneByte = new byte[1]; + private final byte[] buffer = new byte[MAX_COMPRESSED_BUFFER_SIZE]; + private int currentIndex = 0; + + private final ByteUtils.ByteConsumer consumer; + + /** + * Constructs a new output stream that compresses + * snappy-framed-compressed data to the specified output stream. + * @param out the OutputStream to which to write the compressed data + * @throws IOException if writing the signature fails + */ + public FramedSnappyCompressorOutputStream(final OutputStream out) throws IOException { + this(out, SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE) + .build()); + } + + /** + * Constructs a new output stream that compresses + * snappy-framed-compressed data to the specified output stream. + * @param out the OutputStream to which to write the compressed data + * @param params parameters used to fine-tune compression, in + * particular to balance compression ratio vs compression speed. + * @throws IOException if writing the signature fails + */ + public FramedSnappyCompressorOutputStream(final OutputStream out, Parameters params) throws IOException { + this.out = out; + this.params = params; + consumer = new ByteUtils.OutputStreamByteConsumer(out); + out.write(FramedSnappyCompressorInputStream.SZ_SIGNATURE); + } + + @Override + public void write(int b) throws IOException { + oneByte[0] = (byte) (b & 0xff); + write(oneByte); + } + + @Override + public void write(byte[] data, int off, int len) throws IOException { + if (currentIndex + len > MAX_COMPRESSED_BUFFER_SIZE) { + flushBuffer(); + while (len > MAX_COMPRESSED_BUFFER_SIZE) { + System.arraycopy(data, off, buffer, 0, MAX_COMPRESSED_BUFFER_SIZE); + off += MAX_COMPRESSED_BUFFER_SIZE; + len -= MAX_COMPRESSED_BUFFER_SIZE; + currentIndex = MAX_COMPRESSED_BUFFER_SIZE; + flushBuffer(); + } + } + System.arraycopy(data, off, buffer, currentIndex, len); + currentIndex += len; + } + + @Override + public void close() throws IOException { + try { + finish(); + } finally { + out.close(); + } + } + + /** + * Compresses all remaining data and writes it to the stream, + * doesn't close the underlying stream. + * @throws IOException if an error occurs + */ + public void finish() throws IOException { + if (currentIndex > 0) { + flushBuffer(); + } + } + + private void flushBuffer() throws IOException { + out.write(FramedSnappyCompressorInputStream.COMPRESSED_CHUNK_TYPE); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (OutputStream o = new SnappyCompressorOutputStream(baos, currentIndex, params)) { + o.write(buffer, 0, currentIndex); + } + byte[] b = baos.toByteArray(); + writeLittleEndian(3, b.length + 4L /* CRC */); + writeCrc(); + out.write(b); + currentIndex = 0; + } + + private void writeLittleEndian(final int numBytes, long num) throws IOException { + ByteUtils.toLittleEndian(consumer, num, numBytes); + } + + private void writeCrc() throws IOException { + checksum.update(buffer, 0, currentIndex); + writeLittleEndian(4, mask(checksum.getValue())); + checksum.reset(); + } + + static long mask(long x) { + // ugly, maybe we should just have used ints and deal with the + // overflow + x = ((x >> 15) | (x << 17)); + x += FramedSnappyCompressorInputStream.MASK_OFFSET; + x &= 0xffffFFFFL; + return x; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyDialect.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyDialect.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyDialect.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyDialect.java 2018-05-02 20:17:13.000000000 +0000 @@ -36,8 +36,8 @@ private final boolean streamIdentifier, checksumWithCompressedChunks; - private FramedSnappyDialect(final boolean hasStreamIdentifier, - final boolean usesChecksumWithCompressedChunks) { + FramedSnappyDialect(final boolean hasStreamIdentifier, + final boolean usesChecksumWithCompressedChunks) { this.streamIdentifier = hasStreamIdentifier; this.checksumWithCompressedChunks = usesChecksumWithCompressedChunks; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/package.html 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/package.html 2018-05-02 20:17:13.000000000 +0000 @@ -18,15 +18,15 @@ --> -

    Provides stream classes for decompressing streams using the - Snappy +

    Provides stream classes for the + Snappy algorithm.

    The raw Snappy format which only contains the compressed data - is supported by the SnappyCompressorInputStream - class while the so called "framing format" is implemented - by FramedSnappyCompressorInputStream. Note there - have been different versions of the fraing format specification, + is supported by the SnappyCompressor*putStream + classes while the so called "framing format" is implemented + by FramedSnappyCompressor*putStream. Note there + have been different versions of the framing format specification, the implementation in Commons Compress is based on the specification "Last revised: 2013-10-25".

    diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/PureJavaCrc32C.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/PureJavaCrc32C.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/PureJavaCrc32C.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/PureJavaCrc32C.java 2018-05-23 12:50:54.000000000 +0000 @@ -14,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * * Some portions of this file Copyright (c) 2004-2006 Intel Corportation * and licensed under the BSD license. */ @@ -28,7 +28,7 @@ * and implemented on many Intel chipsets supporting SSE4.2. * *

    This file is a copy of the implementation at the Apache Hadoop project.

    - * @see "http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java" + * @see "https://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java" * @NotThreadSafe * @since 1.7 */ @@ -90,7 +90,7 @@ default: /* nothing */ } - + // Publish crc out to object crc = localCrc; } @@ -99,7 +99,7 @@ final public void update(final int b) { crc = (crc >>> 8) ^ T[T8_0_START + ((crc ^ b) & 0xff)]; } - + // CRC polynomial tables generated by: // java -cp build/test/classes/:build/classes/ \ // org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78 @@ -115,524 +115,524 @@ private static final int[] T = new int[] { /* T8_0 */ - 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, - 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, - 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, - 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, - 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, - 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, - 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, - 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, - 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, - 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, - 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, - 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, - 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, - 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, - 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, - 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, - 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, - 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, - 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, - 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, - 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, - 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, - 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, - 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, - 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, - 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, - 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, - 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, - 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, - 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, - 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, - 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, - 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, - 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, - 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, - 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, - 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, - 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, - 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, - 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, - 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, - 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, - 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, - 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, - 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, - 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, - 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, - 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, - 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, - 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, - 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, - 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, - 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, - 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, - 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, - 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, - 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, - 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, - 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, - 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, - 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, - 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, - 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, - 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, + 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, + 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, + 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, + 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, + 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, + 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, + 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, + 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, + 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, + 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, + 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, + 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, + 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, + 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, + 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, + 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, + 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, + 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, + 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, + 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, + 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, + 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, + 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, + 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, + 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, + 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, + 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, + 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, /* T8_1 */ - 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, - 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, - 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, - 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, - 0x3FC5F181, 0x2C6769F6, 0x1880C16F, 0x0B225918, - 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, - 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, - 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, - 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, - 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, - 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, - 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, - 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, - 0x0EC4735F, 0x1D66EB28, 0x298143B1, 0x3A23DBC6, - 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, - 0x93D0B0E7, 0x80722890, 0xB4958009, 0xA737187E, - 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, - 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, - 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, - 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, - 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, - 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, - 0x5DC6F43D, 0x4E646C4A, 0x7A83C4D3, 0x69215CA4, - 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, - 0x809C2506, 0x933EBD71, 0xA7D915E8, 0xB47B8D9F, - 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, - 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, - 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, - 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, - 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, 0xC5341DC2, - 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, - 0x6CC776E3, 0x7F65EE94, 0x4B82460D, 0x5820DE7A, - 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, - 0xB5499B25, 0xA6EB0352, 0x920CABCB, 0x81AE33BC, - 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, - 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, - 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, - 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, - 0x5912C8C0, 0x4AB050B7, 0x7E57F82E, 0x6DF56059, - 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, - 0x844819FB, 0x97EA818C, 0xA30D2915, 0xB0AFB162, - 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, - 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, - 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, - 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, - 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, - 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, - 0x68134A1E, 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, - 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, - 0x4A5E5D21, 0x59FCC556, 0x6D1B6DCF, 0x7EB9F5B8, - 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, - 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, - 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, - 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, - 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, - 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, - 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, - 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, - 0xE64B1C47, 0xF5E98430, 0xC10E2CA9, 0xD2ACB4DE, - 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, - 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, - 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, - 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, - 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483, + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, + 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, + 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, + 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, + 0x3FC5F181, 0x2C6769F6, 0x1880C16F, 0x0B225918, + 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, + 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, + 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, + 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, + 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, + 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, + 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, + 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, + 0x0EC4735F, 0x1D66EB28, 0x298143B1, 0x3A23DBC6, + 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, + 0x93D0B0E7, 0x80722890, 0xB4958009, 0xA737187E, + 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, + 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, + 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, + 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, + 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, + 0x5DC6F43D, 0x4E646C4A, 0x7A83C4D3, 0x69215CA4, + 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, + 0x809C2506, 0x933EBD71, 0xA7D915E8, 0xB47B8D9F, + 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, + 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, + 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, + 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, + 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, 0xC5341DC2, + 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, + 0x6CC776E3, 0x7F65EE94, 0x4B82460D, 0x5820DE7A, + 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, + 0xB5499B25, 0xA6EB0352, 0x920CABCB, 0x81AE33BC, + 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, + 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, + 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, + 0x5912C8C0, 0x4AB050B7, 0x7E57F82E, 0x6DF56059, + 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, + 0x844819FB, 0x97EA818C, 0xA30D2915, 0xB0AFB162, + 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, + 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, + 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, + 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, + 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, + 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, + 0x68134A1E, 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, + 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, + 0x4A5E5D21, 0x59FCC556, 0x6D1B6DCF, 0x7EB9F5B8, + 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, + 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, + 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, + 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, + 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, + 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, + 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, + 0xE64B1C47, 0xF5E98430, 0xC10E2CA9, 0xD2ACB4DE, + 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, + 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, + 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, + 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, + 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483, /* T8_2 */ - 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, - 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, - 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, - 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, - 0x70A27D8A, 0xD5E3EFF4, 0x3FCD2F87, 0x9A8CBDF9, - 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, - 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, - 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, - 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, - 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, - 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, - 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, - 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, - 0x0F382284, 0xAA79B0FA, 0x40577089, 0xE516E2F7, - 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, - 0x37691C41, 0x92288E3F, 0x78064E4C, 0xDD47DC32, - 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, - 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, - 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, - 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, - 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, - 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, - 0x8F96C396, 0x2AD751E8, 0xC0F9919B, 0x65B803E5, - 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, - 0x26217BCD, 0x8360E9B3, 0x694E29C0, 0xCC0FBBBE, - 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, - 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, - 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, - 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, - 0xC85DA25D, 0x6D1C3023, 0x8732F050, 0x2273622E, - 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, - 0xF00C9C98, 0x554D0EE6, 0xBF63CE95, 0x1A225CEB, - 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, - 0x15F9D359, 0xB0B84127, 0x5A968154, 0xFFD7132A, - 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, - 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, - 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, - 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, - 0xC3D4340C, 0x6695A672, 0x8CBB6601, 0x29FAF47F, - 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, - 0x6A638C57, 0xCF221E29, 0x250CDE5A, 0x804D4C24, - 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, - 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, - 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, - 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, - 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, - 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, - 0xBC4E6B02, 0x190FF97C, 0xF321390F, 0x5660AB71, - 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, - 0xD29C5380, 0x77DDC1FE, 0x9DF3018D, 0x38B293F3, - 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, - 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, - 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, - 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, - 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, - 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, - 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, - 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, - 0x9557324B, 0x3016A035, 0xDA386046, 0x7F79F238, - 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, - 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, - 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, - 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, - 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8, + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, + 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, + 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, + 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, + 0x70A27D8A, 0xD5E3EFF4, 0x3FCD2F87, 0x9A8CBDF9, + 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, + 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, + 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, + 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, + 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, + 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, + 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, + 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, + 0x0F382284, 0xAA79B0FA, 0x40577089, 0xE516E2F7, + 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, + 0x37691C41, 0x92288E3F, 0x78064E4C, 0xDD47DC32, + 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, + 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, + 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, + 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, + 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, + 0x8F96C396, 0x2AD751E8, 0xC0F9919B, 0x65B803E5, + 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, + 0x26217BCD, 0x8360E9B3, 0x694E29C0, 0xCC0FBBBE, + 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, + 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, + 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, + 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, + 0xC85DA25D, 0x6D1C3023, 0x8732F050, 0x2273622E, + 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, + 0xF00C9C98, 0x554D0EE6, 0xBF63CE95, 0x1A225CEB, + 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, + 0x15F9D359, 0xB0B84127, 0x5A968154, 0xFFD7132A, + 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, + 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, + 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, + 0xC3D4340C, 0x6695A672, 0x8CBB6601, 0x29FAF47F, + 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, + 0x6A638C57, 0xCF221E29, 0x250CDE5A, 0x804D4C24, + 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, + 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, + 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, + 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, + 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, + 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, + 0xBC4E6B02, 0x190FF97C, 0xF321390F, 0x5660AB71, + 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, + 0xD29C5380, 0x77DDC1FE, 0x9DF3018D, 0x38B293F3, + 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, + 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, + 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, + 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, + 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, + 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, + 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, + 0x9557324B, 0x3016A035, 0xDA386046, 0x7F79F238, + 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, + 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, + 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, + 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, + 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8, /* T8_3 */ - 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, - 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, - 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, - 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, - 0xE964B13D, 0x34211B85, 0x560392BC, 0x8B463804, - 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, - 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, - 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, - 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, - 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, - 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, - 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, - 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, - 0x45639445, 0x98263EFD, 0xFA04B7C4, 0x27411D7C, - 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, - 0xB327F7A3, 0x6E625D1B, 0x0C40D422, 0xD1057E9A, - 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, - 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, - 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, - 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, - 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, - 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, - 0xB4868D3C, 0x69C32784, 0x0BE1AEBD, 0xD6A40405, - 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, - 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, 0x1EA1C255, - 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, - 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, - 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, - 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, - 0xEEC5CBA2, 0x3380611A, 0x51A2E823, 0x8CE7429B, - 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, - 0x1881A844, 0xC5C402FC, 0xA7E68BC5, 0x7AA3217D, - 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, - 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, - 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, - 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, - 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, - 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, - 0x4D801BE4, 0x90C5B15C, 0xF2E73865, 0x2FA292DD, - 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, - 0x8585DDB4, 0x58C0770C, 0x3AE2FE35, 0xE7A7548D, - 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, - 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, - 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, - 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, - 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, - 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, - 0xE1873E9C, 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, - 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, - 0x8224A72B, 0x5F610D93, 0x3D4384AA, 0xE0062E12, - 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, - 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, - 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, - 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, - 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, - 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, - 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, - 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, - 0xD867E1B5, 0x05224B0D, 0x6700C234, 0xBA45688C, - 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, - 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, - 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, - 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, - 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842, + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, + 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, + 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, + 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, + 0xE964B13D, 0x34211B85, 0x560392BC, 0x8B463804, + 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, + 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, + 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, + 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, + 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, + 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, + 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, + 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, + 0x45639445, 0x98263EFD, 0xFA04B7C4, 0x27411D7C, + 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, + 0xB327F7A3, 0x6E625D1B, 0x0C40D422, 0xD1057E9A, + 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, + 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, + 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, + 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, + 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, + 0xB4868D3C, 0x69C32784, 0x0BE1AEBD, 0xD6A40405, + 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, + 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, 0x1EA1C255, + 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, + 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, + 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, + 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, + 0xEEC5CBA2, 0x3380611A, 0x51A2E823, 0x8CE7429B, + 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, + 0x1881A844, 0xC5C402FC, 0xA7E68BC5, 0x7AA3217D, + 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, + 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, + 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, + 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, + 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, + 0x4D801BE4, 0x90C5B15C, 0xF2E73865, 0x2FA292DD, + 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, + 0x8585DDB4, 0x58C0770C, 0x3AE2FE35, 0xE7A7548D, + 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, + 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, + 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, + 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, + 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, + 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, + 0xE1873E9C, 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, + 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, + 0x8224A72B, 0x5F610D93, 0x3D4384AA, 0xE0062E12, + 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, + 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, + 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, + 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, + 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, + 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, + 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, + 0xD867E1B5, 0x05224B0D, 0x6700C234, 0xBA45688C, + 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, + 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, + 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, + 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, + 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842, /* T8_4 */ - 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, - 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, - 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, - 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, - 0x8F2261D3, 0xB7330E7F, 0xFF00BE8B, 0xC711D127, - 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, - 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, - 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, - 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, - 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, - 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, - 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, - 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, - 0x74CF6A34, 0x4CDE0598, 0x04EDB56C, 0x3CFCDAC0, - 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, - 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, 0xF99BD151, - 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, - 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, - 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, - 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, - 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, - 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, - 0x7D1400EC, 0x45056F40, 0x0D36DFB4, 0x3527B018, - 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, - 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, 0x64CA6F0D, - 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, - 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, - 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, - 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, - 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, - 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, - 0x86F90B0B, 0xBEE864A7, 0xF6DBD453, 0xCECABBFF, - 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, - 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, - 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, - 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, - 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, - 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, - 0x24E7BF1E, 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, - 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, - 0x750A600B, 0x4D1B0FA7, 0x0528BF53, 0x3D39D0FF, - 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, - 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, - 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, - 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, - 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, - 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, - 0xDF0AB4F9, 0xE71BDB55, 0xAF286BA1, 0x9739040D, - 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, - 0xB9B60142, 0x81A76EEE, 0xC994DE1A, 0xF185B1B6, - 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, - 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, - 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, - 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, - 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, - 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, - 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, - 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, - 0x873C0134, 0xBF2D6E98, 0xF71EDE6C, 0xCF0FB1C0, - 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, - 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, - 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, - 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, - 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3, + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, + 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, + 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, + 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, + 0x8F2261D3, 0xB7330E7F, 0xFF00BE8B, 0xC711D127, + 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, + 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, + 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, + 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, + 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, + 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, + 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, + 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, + 0x74CF6A34, 0x4CDE0598, 0x04EDB56C, 0x3CFCDAC0, + 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, + 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, 0xF99BD151, + 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, + 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, + 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, + 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, + 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, + 0x7D1400EC, 0x45056F40, 0x0D36DFB4, 0x3527B018, + 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, + 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, 0x64CA6F0D, + 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, + 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, + 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, + 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, + 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, + 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, + 0x86F90B0B, 0xBEE864A7, 0xF6DBD453, 0xCECABBFF, + 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, + 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, + 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, + 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, + 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, + 0x24E7BF1E, 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, + 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, + 0x750A600B, 0x4D1B0FA7, 0x0528BF53, 0x3D39D0FF, + 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, + 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, + 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, + 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, + 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, + 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, + 0xDF0AB4F9, 0xE71BDB55, 0xAF286BA1, 0x9739040D, + 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, + 0xB9B60142, 0x81A76EEE, 0xC994DE1A, 0xF185B1B6, + 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, + 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, + 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, + 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, + 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, + 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, + 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, + 0x873C0134, 0xBF2D6E98, 0xF71EDE6C, 0xCF0FB1C0, + 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, + 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, + 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, + 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, + 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3, /* T8_5 */ - 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, - 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, - 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, - 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, - 0xC00C303E, 0x2F3C5B27, 0x1B8090FD, 0xF4B0FBE4, - 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, - 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, - 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, - 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, - 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, - 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, - 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, - 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, - 0xF70D11C4, 0x183D7ADD, 0x2C81B107, 0xC3B1DA1E, - 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, - 0x970B09DB, 0x783B62C2, 0x4C87A918, 0xA3B7C201, - 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, - 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, - 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, - 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, - 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, - 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, - 0xAE0E73CA, 0x413E18D3, 0x7582D309, 0x9AB2B810, - 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, - 0x8BF04D66, 0x64C0267F, 0x507CEDA5, 0xBF4C86BC, - 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, - 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, - 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, - 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, - 0xF9094A2F, 0x16392136, 0x2285EAEC, 0xCDB581F5, - 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, - 0x990F5230, 0x763F3929, 0x4283F2F3, 0xADB399EA, - 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, - 0xAEFD80A1, 0x41CDEBB8, 0x75712062, 0x9A414B7B, - 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, - 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, - 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, - 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, - 0xBC029FF7, 0x5332F4EE, 0x678E3F34, 0x88BE542D, - 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, - 0x99FCA15B, 0x76CCCA42, 0x42700198, 0xAD406A81, - 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, - 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, - 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, - 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, - 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, - 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, - 0x8B03BE0D, 0x6433D514, 0x508F1ECE, 0xBFBF75D7, - 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, - 0xA0F9DB4A, 0x4FC9B053, 0x7B757B89, 0x94451090, - 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, - 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, - 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, - 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, - 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, - 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, - 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, - 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, - 0xF7FEE2AF, 0x18CE89B6, 0x2C72426C, 0xC3422975, - 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, - 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, - 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, - 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, - 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C, + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, + 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, + 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, + 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, + 0xC00C303E, 0x2F3C5B27, 0x1B8090FD, 0xF4B0FBE4, + 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, + 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, + 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, + 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, + 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, + 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, + 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, + 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, + 0xF70D11C4, 0x183D7ADD, 0x2C81B107, 0xC3B1DA1E, + 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, + 0x970B09DB, 0x783B62C2, 0x4C87A918, 0xA3B7C201, + 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, + 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, + 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, + 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, + 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, + 0xAE0E73CA, 0x413E18D3, 0x7582D309, 0x9AB2B810, + 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, + 0x8BF04D66, 0x64C0267F, 0x507CEDA5, 0xBF4C86BC, + 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, + 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, + 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, + 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, + 0xF9094A2F, 0x16392136, 0x2285EAEC, 0xCDB581F5, + 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, + 0x990F5230, 0x763F3929, 0x4283F2F3, 0xADB399EA, + 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, + 0xAEFD80A1, 0x41CDEBB8, 0x75712062, 0x9A414B7B, + 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, + 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, + 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, + 0xBC029FF7, 0x5332F4EE, 0x678E3F34, 0x88BE542D, + 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, + 0x99FCA15B, 0x76CCCA42, 0x42700198, 0xAD406A81, + 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, + 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, + 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, + 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, + 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, + 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, + 0x8B03BE0D, 0x6433D514, 0x508F1ECE, 0xBFBF75D7, + 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, + 0xA0F9DB4A, 0x4FC9B053, 0x7B757B89, 0x94451090, + 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, + 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, + 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, + 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, + 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, + 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, + 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, + 0xF7FEE2AF, 0x18CE89B6, 0x2C72426C, 0xC3422975, + 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, + 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, + 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, + 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, + 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C, /* T8_6 */ - 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, - 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, - 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, - 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, - 0x9C5BFAA6, 0xF458D66E, 0x4C5DA336, 0x245E8FFE, - 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, - 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, - 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, - 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, - 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, - 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, - 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, - 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, - 0x04E0BCCA, 0x6CE39002, 0xD4E6E55A, 0xBCE5C992, - 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, - 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, 0xF2C834C1, - 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, - 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, - 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, - 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, - 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, - 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, - 0xA8C1008F, 0xC0C22C47, 0x78C7591F, 0x10C475D7, - 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, - 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, 0xFFE9F19F, - 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, - 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, - 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, - 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, - 0x7E57BBB0, 0x16549778, 0xAE51E220, 0xC652CEE8, - 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, - 0x307A46E3, 0x58796A2B, 0xE07C1F73, 0x887F33BB, - 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, - 0x508ECB25, 0x388DE7ED, 0x808892B5, 0xE88BBE7D, - 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, - 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, - 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, - 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, - 0x27180901, 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, - 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, - 0xC8358D49, 0xA036A181, 0x1833D4D9, 0x7030F811, - 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, - 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, - 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, - 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, - 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, - 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, - 0xBFA34F6D, 0xD7A063A5, 0x6FA516FD, 0x07A63A35, - 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, - 0x2A39CC5F, 0x423AE097, 0xFA3F95CF, 0x923CB907, - 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, - 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, - 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, - 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, - 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, - 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, - 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, - 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, - 0xFCAF7760, 0x94AC5BA8, 0x2CA92EF0, 0x44AA0238, - 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, - 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, - 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, - 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, - 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F, + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, + 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, + 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, + 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, + 0x9C5BFAA6, 0xF458D66E, 0x4C5DA336, 0x245E8FFE, + 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, + 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, + 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, + 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, + 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, + 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, + 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, + 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, + 0x04E0BCCA, 0x6CE39002, 0xD4E6E55A, 0xBCE5C992, + 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, + 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, 0xF2C834C1, + 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, + 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, + 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, + 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, + 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, + 0xA8C1008F, 0xC0C22C47, 0x78C7591F, 0x10C475D7, + 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, + 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, 0xFFE9F19F, + 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, + 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, + 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, + 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, + 0x7E57BBB0, 0x16549778, 0xAE51E220, 0xC652CEE8, + 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, + 0x307A46E3, 0x58796A2B, 0xE07C1F73, 0x887F33BB, + 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, + 0x508ECB25, 0x388DE7ED, 0x808892B5, 0xE88BBE7D, + 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, + 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, + 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, + 0x27180901, 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, + 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, + 0xC8358D49, 0xA036A181, 0x1833D4D9, 0x7030F811, + 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, + 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, + 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, + 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, + 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, + 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, + 0xBFA34F6D, 0xD7A063A5, 0x6FA516FD, 0x07A63A35, + 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, + 0x2A39CC5F, 0x423AE097, 0xFA3F95CF, 0x923CB907, + 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, + 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, + 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, + 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, + 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, + 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, + 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, + 0xFCAF7760, 0x94AC5BA8, 0x2CA92EF0, 0x44AA0238, + 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, + 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, + 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, + 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, + 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F, /* T8_7 */ - 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, - 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, - 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, - 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, - 0x847609B4, 0xCD4A7493, 0x160EF3FA, 0x5F328EDD, - 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, - 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, - 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, - 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, - 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, - 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, - 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, - 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, - 0xA86BEE40, 0xE1579367, 0x3A13140E, 0x732F6929, - 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, - 0xEA50EA9A, 0xA36C97BD, 0x782810D4, 0x31146DF3, - 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, - 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, - 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, - 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, - 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, - 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, - 0xDC4DC65C, 0x9571BB7B, 0x4E353C12, 0x07094135, - 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, - 0x1700AEAB, 0x5E3CD38C, 0x857854E5, 0xCC4429C2, - 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, - 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, - 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, - 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, - 0xB26B2572, 0xFB575855, 0x2013DF3C, 0x692FA21B, - 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, - 0xF05021A8, 0xB96C5C8F, 0x6228DBE6, 0x2B14A6C1, - 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, - 0x151C1409, 0x5C20692E, 0x8764EE47, 0xCE589360, - 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, - 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, - 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, - 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, - 0xF24C9B0A, 0xBB70E62D, 0x60346144, 0x29081C63, - 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, - 0x3901F3FD, 0x703D8EDA, 0xAB7909B3, 0xE2457494, - 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, - 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, - 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, - 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, - 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, - 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, - 0xDE517CFE, 0x976D01D9, 0x4C2986B0, 0x0515FB97, - 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, - 0x0F1CDF3B, 0x4620A21C, 0x9D642575, 0xD4585852, - 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, - 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, - 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, - 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, - 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, - 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, - 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, - 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, - 0x613A3C15, 0x28064132, 0xF342C65B, 0xBA7EBB7C, - 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, - 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, - 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, - 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, - 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, + 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, + 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, + 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, + 0x847609B4, 0xCD4A7493, 0x160EF3FA, 0x5F328EDD, + 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, + 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, + 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, + 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, + 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, + 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, + 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, + 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, + 0xA86BEE40, 0xE1579367, 0x3A13140E, 0x732F6929, + 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, + 0xEA50EA9A, 0xA36C97BD, 0x782810D4, 0x31146DF3, + 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, + 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, + 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, + 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, + 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, + 0xDC4DC65C, 0x9571BB7B, 0x4E353C12, 0x07094135, + 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, + 0x1700AEAB, 0x5E3CD38C, 0x857854E5, 0xCC4429C2, + 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, + 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, + 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, + 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, + 0xB26B2572, 0xFB575855, 0x2013DF3C, 0x692FA21B, + 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, + 0xF05021A8, 0xB96C5C8F, 0x6228DBE6, 0x2B14A6C1, + 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, + 0x151C1409, 0x5C20692E, 0x8764EE47, 0xCE589360, + 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, + 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, + 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, + 0xF24C9B0A, 0xBB70E62D, 0x60346144, 0x29081C63, + 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, + 0x3901F3FD, 0x703D8EDA, 0xAB7909B3, 0xE2457494, + 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, + 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, + 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, + 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, + 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, + 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, + 0xDE517CFE, 0x976D01D9, 0x4C2986B0, 0x0515FB97, + 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, + 0x0F1CDF3B, 0x4620A21C, 0x9D642575, 0xD4585852, + 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, + 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, + 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, + 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, + 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, + 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, + 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, + 0x613A3C15, 0x28064132, 0xF342C65B, 0xBA7EBB7C, + 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, + 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, + 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, + 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, + 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 }; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -21,8 +21,8 @@ import java.io.IOException; import java.io.InputStream; -import org.apache.commons.compress.compressors.CompressorInputStream; -import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.compressors.lz77support.AbstractLZ77CompressorInputStream; +import org.apache.commons.compress.utils.ByteUtils; /** * CompressorInputStream for the raw Snappy format. @@ -35,10 +35,10 @@ * doesn't contain offsets bigger than 32k which is the default block * size used by this class.

    * - * @see Snappy compressed format description + * @see Snappy compressed format description * @since 1.7 */ -public class SnappyCompressorInputStream extends CompressorInputStream { +public class SnappyCompressorInputStream extends AbstractLZ77CompressorInputStream { /** Mask used to determine the type of "tag" is being processed */ private static final int TAG_MASK = 0x03; @@ -46,38 +46,23 @@ /** Default block size */ public static final int DEFAULT_BLOCK_SIZE = 32768; - /** Buffer to write decompressed bytes to for back-references */ - private final byte[] decompressBuf; - - /** One behind the index of the last byte in the buffer that was written */ - private int writeIndex; - - /** Index of the next byte to be read. */ - private int readIndex; - - /** The actual block size specified */ - private final int blockSize; - - /** The underlying stream to read compressed data from */ - private final InputStream in; - /** The size of the uncompressed data */ private final int size; /** Number of uncompressed bytes still to be read. */ private int uncompressedBytesRemaining; - // used in no-arg read method - private final byte[] oneByte = new byte[1]; + /** Current state of the stream */ + private State state = State.NO_BLOCK; private boolean endReached = false; /** * Constructor using the default buffer size of 32k. - * + * * @param is * An InputStream to read compressed data from - * + * * @throws IOException if reading fails */ public SnappyCompressorInputStream(final InputStream is) throws IOException { @@ -86,41 +71,20 @@ /** * Constructor using a configurable buffer size. - * + * * @param is * An InputStream to read compressed data from * @param blockSize * The block size used in compression - * + * * @throws IOException if reading fails */ public SnappyCompressorInputStream(final InputStream is, final int blockSize) throws IOException { - this.in = is; - this.blockSize = blockSize; - this.decompressBuf = new byte[blockSize * 3]; - this.writeIndex = readIndex = 0; + super(is, blockSize); uncompressedBytesRemaining = size = (int) readSize(); } - /** {@inheritDoc} */ - @Override - public int read() throws IOException { - return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF; - } - - /** {@inheritDoc} */ - @Override - public void close() throws IOException { - in.close(); - } - - /** {@inheritDoc} */ - @Override - public int available() { - return writeIndex - readIndex; - } - /** * {@inheritDoc} */ @@ -129,131 +93,118 @@ if (endReached) { return -1; } - final int avail = available(); - if (len > avail) { - fill(len - avail); - } - - final int readable = Math.min(len, available()); - if (readable == 0 && len > 0) { - return -1; - } - System.arraycopy(decompressBuf, readIndex, b, off, readable); - readIndex += readable; - if (readIndex > blockSize) { - slideBuffer(); + switch (state) { + case NO_BLOCK: + fill(); + return read(b, off, len); + case IN_LITERAL: + int litLen = readLiteral(b, off, len); + if (!hasMoreDataInBlock()) { + state = State.NO_BLOCK; + } + return litLen > 0 ? litLen : read(b, off, len); + case IN_BACK_REFERENCE: + int backReferenceLen = readBackReference(b, off, len); + if (!hasMoreDataInBlock()) { + state = State.NO_BLOCK; + } + return backReferenceLen > 0 ? backReferenceLen : read(b, off, len); + default: + throw new IOException("Unknown stream state " + state); } - return readable; } /** - * Try to fill the buffer with enough bytes to satisfy the current - * read request. - * - * @param len the number of uncompressed bytes to read + * Try to fill the buffer with the next block of data. */ - private void fill(final int len) throws IOException { + private void fill() throws IOException { if (uncompressedBytesRemaining == 0) { endReached = true; + return; + } + + int b = readOneByte(); + if (b == -1) { + throw new IOException("Premature end of stream reading block start"); } - int readNow = Math.min(len, uncompressedBytesRemaining); + int length = 0; + int offset = 0; + + switch (b & TAG_MASK) { + + case 0x00: - while (readNow > 0) { - final int b = readOneByte(); - int length = 0; - long offset = 0; - - switch (b & TAG_MASK) { - - case 0x00: - - length = readLiteralLength(b); - - if (expandLiteral(length)) { - return; - } - break; - - case 0x01: - - /* - * These elements can encode lengths between [4..11] bytes and - * offsets between [0..2047] bytes. (len-4) occupies three bits - * and is stored in bits [2..4] of the tag byte. The offset - * occupies 11 bits, of which the upper three are stored in the - * upper three bits ([5..7]) of the tag byte, and the lower - * eight are stored in a byte following the tag byte. - */ - - length = 4 + ((b >> 2) & 0x07); - offset = (b & 0xE0) << 3; - offset |= readOneByte(); - - if (expandCopy(offset, length)) { - return; - } - break; - - case 0x02: - - /* - * These elements can encode lengths between [1..64] and offsets - * from [0..65535]. (len-1) occupies six bits and is stored in - * the upper six bits ([2..7]) of the tag byte. The offset is - * stored as a little-endian 16-bit integer in the two bytes - * following the tag byte. - */ - - length = (b >> 2) + 1; - - offset = readOneByte(); - offset |= readOneByte() << 8; - - if (expandCopy(offset, length)) { - return; - } - break; - - case 0x03: - - /* - * These are like the copies with 2-byte offsets (see previous - * subsection), except that the offset is stored as a 32-bit - * integer instead of a 16-bit integer (and thus will occupy - * four bytes). - */ - - length = (b >> 2) + 1; - - offset = readOneByte(); - offset |= readOneByte() << 8; - offset |= readOneByte() << 16; - offset |= ((long) readOneByte()) << 24; - - if (expandCopy(offset, length)) { - return; - } - break; + length = readLiteralLength(b); + uncompressedBytesRemaining -= length; + startLiteral(length); + state = State.IN_LITERAL; + break; + + case 0x01: + + /* + * These elements can encode lengths between [4..11] bytes and + * offsets between [0..2047] bytes. (len-4) occupies three bits + * and is stored in bits [2..4] of the tag byte. The offset + * occupies 11 bits, of which the upper three are stored in the + * upper three bits ([5..7]) of the tag byte, and the lower + * eight are stored in a byte following the tag byte. + */ + + length = 4 + ((b >> 2) & 0x07); + uncompressedBytesRemaining -= length; + offset = (b & 0xE0) << 3; + b = readOneByte(); + if (b == -1) { + throw new IOException("Premature end of stream reading back-reference length"); } + offset |= b; + + startBackReference(offset, length); + state = State.IN_BACK_REFERENCE; + break; - readNow -= length; + case 0x02: + + /* + * These elements can encode lengths between [1..64] and offsets + * from [0..65535]. (len-1) occupies six bits and is stored in + * the upper six bits ([2..7]) of the tag byte. The offset is + * stored as a little-endian 16-bit integer in the two bytes + * following the tag byte. + */ + + length = (b >> 2) + 1; uncompressedBytesRemaining -= length; - } - } - /** - * Slide buffer. - * - *

    Move all bytes of the buffer after the first block down to - * the beginning of the buffer.

    - */ - private void slideBuffer() { - System.arraycopy(decompressBuf, blockSize, decompressBuf, 0, - blockSize * 2); - writeIndex -= blockSize; - readIndex -= blockSize; - } + offset = (int) ByteUtils.fromLittleEndian(supplier, 2); + startBackReference(offset, length); + state = State.IN_BACK_REFERENCE; + break; + + case 0x03: + + /* + * These are like the copies with 2-byte offsets (see previous + * subsection), except that the offset is stored as a 32-bit + * integer instead of a 16-bit integer (and thus will occupy + * four bytes). + */ + + length = (b >> 2) + 1; + uncompressedBytesRemaining -= length; + + offset = (int) ByteUtils.fromLittleEndian(supplier, 4) & 0x7fffffff; + + startBackReference(offset, length); + state = State.IN_BACK_REFERENCE; + break; + default: + // impossible as TAG_MASK is two bits and all four possible cases have been covered + break; + } + } /* * For literals up to and including 60 bytes in length, the @@ -270,21 +221,18 @@ switch (b >> 2) { case 60: length = readOneByte(); + if (length == -1) { + throw new IOException("Premature end of stream reading literal length"); + } break; case 61: - length = readOneByte(); - length |= readOneByte() << 8; + length = (int) ByteUtils.fromLittleEndian(supplier, 2); break; case 62: - length = readOneByte(); - length |= readOneByte() << 8; - length |= readOneByte() << 16; + length = (int) ByteUtils.fromLittleEndian(supplier, 3); break; case 63: - length = readOneByte(); - length |= readOneByte() << 8; - length |= readOneByte() << 16; - length |= (((long) readOneByte()) << 24); + length = (int) ByteUtils.fromLittleEndian(supplier, 4); break; default: length = b >> 2; @@ -295,111 +243,15 @@ } /** - * Literals are uncompressed data stored directly in the byte stream. - * - * @param length - * The number of bytes to read from the underlying stream - * - * @throws IOException - * If the first byte cannot be read for any reason other than - * end of file, or if the input stream has been closed, or if - * some other I/O error occurs. - * @return True if the decompressed data should be flushed - */ - private boolean expandLiteral(final int length) throws IOException { - final int bytesRead = IOUtils.readFully(in, decompressBuf, writeIndex, length); - count(bytesRead); - if (length != bytesRead) { - throw new IOException("Premature end of stream"); - } - - writeIndex += length; - return writeIndex >= 2 * this.blockSize; - } - - /** - * Copies are references back into previous decompressed data, telling the - * decompressor to reuse data it has previously decoded. They encode two - * values: The offset, saying how many bytes back from the current position - * to read, and the length, how many bytes to copy. Offsets of zero can be - * encoded, but are not legal; similarly, it is possible to encode - * backreferences that would go past the end of the block (offset > current - * decompressed position), which is also nonsensical and thus not allowed. - * - * @param off - * The offset from the backward from the end of expanded stream - * @param length - * The number of bytes to copy - * - * @throws IOException - * An the offset expands past the front of the decompression - * buffer - * @return True if the decompressed data should be flushed - */ - private boolean expandCopy(final long off, final int length) throws IOException { - if (off > blockSize) { - throw new IOException("Offset is larger than block size"); - } - final int offset = (int) off; - - if (offset == 1) { - final byte lastChar = decompressBuf[writeIndex - 1]; - for (int i = 0; i < length; i++) { - decompressBuf[writeIndex++] = lastChar; - } - } else if (length < offset) { - System.arraycopy(decompressBuf, writeIndex - offset, - decompressBuf, writeIndex, length); - writeIndex += length; - } else { - int fullRotations = length / offset; - final int pad = length - (offset * fullRotations); - - while (fullRotations-- != 0) { - System.arraycopy(decompressBuf, writeIndex - offset, - decompressBuf, writeIndex, offset); - writeIndex += offset; - } - - if (pad > 0) { - System.arraycopy(decompressBuf, writeIndex - offset, - decompressBuf, writeIndex, pad); - - writeIndex += pad; - } - } - return writeIndex >= 2 * this.blockSize; - } - - /** - * This helper method reads the next byte of data from the input stream. The - * value byte is returned as an int in the range 0 - * to 255. If no byte is available because the end of the - * stream has been reached, an Exception is thrown. - * - * @return The next byte of data - * @throws IOException - * EOF is reached or error reading the stream - */ - private int readOneByte() throws IOException { - final int b = in.read(); - if (b == -1) { - throw new IOException("Premature end of stream"); - } - count(1); - return b & 0xFF; - } - - /** * The stream starts with the uncompressed length (up to a maximum of 2^32 - * 1), stored as a little-endian varint. Varints consist of a series of * bytes, where the lower 7 bits are data and the upper bit is set iff there * are more bytes to be read. In other words, an uncompressed length of 64 * would be stored as 0x40, and an uncompressed length of 2097150 (0x1FFFFE) * would be stored as 0xFE 0xFF 0x7F. - * + * * @return The size of the uncompressed data - * + * * @throws IOException * Could not read a byte */ @@ -410,6 +262,9 @@ do { b = readOneByte(); + if (b == -1) { + throw new IOException("Premature end of stream reading size"); + } sz |= (b & 0x7f) << (index++ * 7); } while (0 != (b & 0x80)); return sz; @@ -417,11 +272,15 @@ /** * Get the uncompressed size of the stream - * + * * @return the uncompressed size */ + @Override public int getSize() { return size; } + private enum State { + NO_BLOCK, IN_LITERAL, IN_BACK_REFERENCE + } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/snappy/SnappyCompressorOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.snappy; + +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.lz77support.LZ77Compressor; +import org.apache.commons.compress.compressors.lz77support.Parameters; +import org.apache.commons.compress.utils.ByteUtils; + +/** + * CompressorOutputStream for the raw Snappy format. + * + *

    This implementation uses an internal buffer in order to handle + * the back-references that are at the heart of the LZ77 algorithm. + * The size of the buffer must be at least as big as the biggest + * offset used in the compressed stream. The current version of the + * Snappy algorithm as defined by Google works on 32k blocks and + * doesn't contain offsets bigger than 32k which is the default block + * size used by this class.

    + * + *

    The raw Snappy format requires the uncompressed size to be + * written at the beginning of the stream using a varint + * representation, i.e. the number of bytes needed to write the + * information is not known before the uncompressed size is + * known. We've chosen to make the uncompressedSize a parameter of the + * constructor in favor of buffering the whole output until the size + * is known. When using the {@link FramedSnappyCompressorOutputStream} + * this limitation is taken care of by the warpping framing + * format.

    + * + * @see Snappy compressed format description + * @since 1.14 + * @NotThreadSafe + */ +public class SnappyCompressorOutputStream extends CompressorOutputStream { + private final LZ77Compressor compressor; + private final OutputStream os; + private final ByteUtils.ByteConsumer consumer; + + // used in one-arg write method + private final byte[] oneByte = new byte[1]; + + private boolean finished = false; + + /** + * Constructor using the default block size of 32k. + * + * @param os the outputstream to write compressed data to + * @param uncompressedSize the uncompressed size of data + * @throws IOException if writing of the size fails + */ + public SnappyCompressorOutputStream(final OutputStream os, final long uncompressedSize) throws IOException { + this(os, uncompressedSize, SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE); + } + + /** + * Constructor using a configurable block size. + * + * @param os the outputstream to write compressed data to + * @param uncompressedSize the uncompressed size of data + * @param blockSize the block size used - must be a power of two + * @throws IOException if writing of the size fails + */ + public SnappyCompressorOutputStream(final OutputStream os, final long uncompressedSize, final int blockSize) + throws IOException { + this(os, uncompressedSize, createParameterBuilder(blockSize).build()); + } + + /** + * Constructor providing full control over the underlying LZ77 compressor. + * + * @param os the outputstream to write compressed data to + * @param uncompressedSize the uncompressed size of data + * @param params the parameters to use by the compressor - note + * that the format itself imposes some limits like a maximum match + * length of 64 bytes + * @throws IOException if writing of the size fails + */ + public SnappyCompressorOutputStream(final OutputStream os, final long uncompressedSize, Parameters params) + throws IOException { + this.os = os; + consumer = new ByteUtils.OutputStreamByteConsumer(os); + compressor = new LZ77Compressor(params, new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) throws IOException { + switch (block.getType()) { + case LITERAL: + writeLiteralBlock((LZ77Compressor.LiteralBlock) block); + break; + case BACK_REFERENCE: + writeBackReference((LZ77Compressor.BackReference) block); + break; + case EOD: + break; + } + } + }); + writeUncompressedSize(uncompressedSize); + } + + @Override + public void write(int b) throws IOException { + oneByte[0] = (byte) (b & 0xff); + write(oneByte); + } + + @Override + public void write(byte[] data, int off, int len) throws IOException { + compressor.compress(data, off, len); + } + + @Override + public void close() throws IOException { + try { + finish(); + } finally { + os.close(); + } + } + + /** + * Compresses all remaining data and writes it to the stream, + * doesn't close the underlying stream. + * @throws IOException if an error occurs + */ + public void finish() throws IOException { + if (!finished) { + compressor.finish(); + finished = true; + } + } + + private void writeUncompressedSize(long uncompressedSize) throws IOException { + boolean more = false; + do { + int currentByte = (int) (uncompressedSize & 0x7F); + more = uncompressedSize > currentByte; + if (more) { + currentByte |= 0x80; + } + os.write(currentByte); + uncompressedSize >>= 7; + } while (more); + } + + // literal length is stored as (len - 1) either inside the tag + // (six bits minus four flags) or in 1 to 4 bytes after the tag + private static final int MAX_LITERAL_SIZE_WITHOUT_SIZE_BYTES = 60; + private static final int MAX_LITERAL_SIZE_WITH_ONE_SIZE_BYTE = 1 << 8; + private static final int MAX_LITERAL_SIZE_WITH_TWO_SIZE_BYTES = 1 << 16; + private static final int MAX_LITERAL_SIZE_WITH_THREE_SIZE_BYTES = 1 << 24; + + private static final int ONE_SIZE_BYTE_MARKER = 60 << 2; + private static final int TWO_SIZE_BYTE_MARKER = 61 << 2; + private static final int THREE_SIZE_BYTE_MARKER = 62 << 2; + private static final int FOUR_SIZE_BYTE_MARKER = 63 << 2; + + private void writeLiteralBlock(LZ77Compressor.LiteralBlock block) throws IOException { + int len = block.getLength(); + if (len <= MAX_LITERAL_SIZE_WITHOUT_SIZE_BYTES) { + writeLiteralBlockNoSizeBytes(block, len); + } else if (len <= MAX_LITERAL_SIZE_WITH_ONE_SIZE_BYTE) { + writeLiteralBlockOneSizeByte(block, len); + } else if (len <= MAX_LITERAL_SIZE_WITH_TWO_SIZE_BYTES) { + writeLiteralBlockTwoSizeBytes(block, len); + } else if (len <= MAX_LITERAL_SIZE_WITH_THREE_SIZE_BYTES) { + writeLiteralBlockThreeSizeBytes(block, len); + } else { + writeLiteralBlockFourSizeBytes(block, len); + } + } + + private void writeLiteralBlockNoSizeBytes(LZ77Compressor.LiteralBlock block, int len) throws IOException { + writeLiteralBlockWithSize(len - 1 << 2, 0, len, block); + } + + private void writeLiteralBlockOneSizeByte(LZ77Compressor.LiteralBlock block, int len) throws IOException { + writeLiteralBlockWithSize(ONE_SIZE_BYTE_MARKER, 1, len, block); + } + + private void writeLiteralBlockTwoSizeBytes(LZ77Compressor.LiteralBlock block, int len) throws IOException { + writeLiteralBlockWithSize(TWO_SIZE_BYTE_MARKER, 2, len, block); + } + + private void writeLiteralBlockThreeSizeBytes(LZ77Compressor.LiteralBlock block, int len) throws IOException { + writeLiteralBlockWithSize(THREE_SIZE_BYTE_MARKER, 3, len, block); + } + + private void writeLiteralBlockFourSizeBytes(LZ77Compressor.LiteralBlock block, int len) throws IOException { + writeLiteralBlockWithSize(FOUR_SIZE_BYTE_MARKER, 4, len, block); + } + + private void writeLiteralBlockWithSize(int tagByte, int sizeBytes, int len, LZ77Compressor.LiteralBlock block) + throws IOException { + os.write(tagByte); + writeLittleEndian(sizeBytes, len - 1); + os.write(block.getData(), block.getOffset(), len); + } + + private void writeLittleEndian(final int numBytes, int num) throws IOException { + ByteUtils.toLittleEndian(consumer, num, numBytes); + } + + // Back-references ("copies") have their offset/size information + // in two, three or five bytes. + private static final int MIN_MATCH_LENGTH_WITH_ONE_OFFSET_BYTE = 4; + private static final int MAX_MATCH_LENGTH_WITH_ONE_OFFSET_BYTE = 11; + private static final int MAX_OFFSET_WITH_ONE_OFFSET_BYTE = 1 << 11 - 1; + private static final int MAX_OFFSET_WITH_TWO_OFFSET_BYTES = 1 << 16 - 1; + + private static final int ONE_BYTE_COPY_TAG = 1; + private static final int TWO_BYTE_COPY_TAG = 2; + private static final int FOUR_BYTE_COPY_TAG = 3; + + private void writeBackReference(LZ77Compressor.BackReference block) throws IOException { + final int len = block.getLength(); + final int offset = block.getOffset(); + if (len >= MIN_MATCH_LENGTH_WITH_ONE_OFFSET_BYTE && len <= MAX_MATCH_LENGTH_WITH_ONE_OFFSET_BYTE + && offset <= MAX_OFFSET_WITH_ONE_OFFSET_BYTE) { + writeBackReferenceWithOneOffsetByte(len, offset); + } else if (offset < MAX_OFFSET_WITH_TWO_OFFSET_BYTES) { + writeBackReferenceWithTwoOffsetBytes(len, offset); + } else { + writeBackReferenceWithFourOffsetBytes(len, offset); + } + } + + private void writeBackReferenceWithOneOffsetByte(int len, int offset) throws IOException { + os.write(ONE_BYTE_COPY_TAG | ((len - 4) << 2) | ((offset & 0x700) >> 3)); + os.write(offset & 0xff); + } + + private void writeBackReferenceWithTwoOffsetBytes(int len, int offset) throws IOException { + writeBackReferenceWithLittleEndianOffset(TWO_BYTE_COPY_TAG, 2, len, offset); + } + + private void writeBackReferenceWithFourOffsetBytes(int len, int offset) throws IOException { + writeBackReferenceWithLittleEndianOffset(FOUR_BYTE_COPY_TAG, 4, len, offset); + } + + private void writeBackReferenceWithLittleEndianOffset(int tag, int offsetBytes, int len, int offset) + throws IOException { + os.write(tag | ((len - 1) << 2)); + writeLittleEndian(offsetBytes, offset); + } + + // technically the format could use shorter matches but with a + // length of three the offset would be encoded as at least two + // bytes in addition to the tag, so yield no compression at all + private static final int MIN_MATCH_LENGTH = 4; + // Snappy stores the match length in six bits of the tag + private static final int MAX_MATCH_LENGTH = 64; + + /** + * Returns a builder correctly configured for the Snappy algorithm using the gven block size. + * @param blockSize the block size. + * @return a builder correctly configured for the Snappy algorithm using the gven block size + */ + public static Parameters.Builder createParameterBuilder(int blockSize) { + // the max offset and max literal length defined by the format + // are 2^32 - 1 and 2^32 respectively - with blockSize being + // an integer we will never exceed that + return Parameters.builder(blockSize) + .withMinBackReferenceLength(MIN_MATCH_LENGTH) + .withMaxBackReferenceLength(MAX_MATCH_LENGTH) + .withMaxOffset(blockSize) + .withMaxLiteralLength(blockSize); + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/xz/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/xz/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/xz/package.html 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/xz/package.html 2018-05-23 12:50:54.000000000 +0000 @@ -25,7 +25,7 @@ org.tukaani.xz.XZInputStream org.tukaani.xz.XZInputStream} and {@link org.tukaani.xz.XZOutputStream org.tukaani.xz.XZOutputStream} provided by the public - domain XZ for Java + domain XZ for Java library.

    diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -20,17 +20,25 @@ import java.io.IOException; import java.io.InputStream; + import org.tukaani.xz.XZ; import org.tukaani.xz.SingleXZInputStream; import org.tukaani.xz.XZInputStream; +import org.apache.commons.compress.MemoryLimitException; import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.utils.CountingInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; /** * XZ decompressor. * @since 1.4 */ -public class XZCompressorInputStream extends CompressorInputStream { +public class XZCompressorInputStream extends CompressorInputStream + implements InputStreamStatistics { + + private final CountingInputStream countingStream; private final InputStream in; /** @@ -92,30 +100,73 @@ public XZCompressorInputStream(final InputStream inputStream, final boolean decompressConcatenated) throws IOException { + this(inputStream, decompressConcatenated, -1); + } + + /** + * Creates a new input stream that decompresses XZ-compressed data + * from the specified input stream. + * + * @param inputStream where to read the compressed data + * @param decompressConcatenated + * if true, decompress until the end of the + * input; if false, stop after the first .xz + * stream and leave the input position to point + * to the next byte after the .xz stream + * @param memoryLimitInKb memory limit used when reading blocks. If + * the estimated memory limit is exceeded on {@link #read()}, + * a {@link MemoryLimitException} is thrown. + * + * @throws IOException if the input is not in the .xz format, + * the input is corrupt or truncated, the .xz + * headers specify options that are not supported + * by this implementation, + * or the underlying inputStream throws an exception + * + * @since 1.14 + */ + public XZCompressorInputStream(InputStream inputStream, + boolean decompressConcatenated, final int memoryLimitInKb) + throws IOException { + countingStream = new CountingInputStream(inputStream); if (decompressConcatenated) { - in = new XZInputStream(inputStream); + in = new XZInputStream(countingStream, memoryLimitInKb); } else { - in = new SingleXZInputStream(inputStream); + in = new SingleXZInputStream(countingStream, memoryLimitInKb); } } @Override public int read() throws IOException { - final int ret = in.read(); - count(ret == -1 ? -1 : 1); - return ret; + try { + final int ret = in.read(); + count(ret == -1 ? -1 : 1); + return ret; + } catch (org.tukaani.xz.MemoryLimitException e) { + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); + } } @Override public int read(final byte[] buf, final int off, final int len) throws IOException { - final int ret = in.read(buf, off, len); - count(ret); - return ret; + try { + final int ret = in.read(buf, off, len); + count(ret); + return ret; + } catch (org.tukaani.xz.MemoryLimitException e) { + //convert to commons-compress MemoryLimtException + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); + } } @Override public long skip(final long n) throws IOException { - return in.skip(n); + try { + return IOUtils.skip(in, n); + } catch (org.tukaani.xz.MemoryLimitException e) { + //convert to commons-compress MemoryLimtException + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); + } } @Override @@ -127,4 +178,12 @@ public void close() throws IOException { in.close(); } + + /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return countingStream.getBytesRead(); + } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/xz/XZUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/xz/XZUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/xz/XZUtils.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/xz/XZUtils.java 2018-05-02 20:17:13.000000000 +0000 @@ -41,7 +41,7 @@ (byte) 0xFD, '7', 'z', 'X', 'Z', '\0' }; - static enum CachedAvailability { + enum CachedAvailability { DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE } @@ -168,7 +168,7 @@ cachedXZAvailability = CachedAvailability.DONT_CACHE; } else if (cachedXZAvailability == CachedAvailability.DONT_CACHE) { final boolean hasXz = internalIsXZCompressionAvailable(); - cachedXZAvailability = hasXz ? CachedAvailability.CACHED_AVAILABLE + cachedXZAvailability = hasXz ? CachedAvailability.CACHED_AVAILABLE // NOSONAR : CachedAvailability.CACHED_UNAVAILABLE; } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -37,8 +37,9 @@ private final boolean blockMode; private final int maxCodeSize; private long totalCodesRead = 0; - - public ZCompressorInputStream(final InputStream inputStream) throws IOException { + + public ZCompressorInputStream(final InputStream inputStream, final int memoryLimitInKb) + throws IOException { super(inputStream, ByteOrder.LITTLE_ENDIAN); final int firstByte = (int) in.readBits(8); final int secondByte = (int) in.readBits(8); @@ -51,10 +52,14 @@ if (blockMode) { setClearCode(DEFAULT_CODE_SIZE); } - initializeTables(maxCodeSize); + initializeTables(maxCodeSize, memoryLimitInKb); clearEntries(); } - + + public ZCompressorInputStream(final InputStream inputStream) throws IOException { + this(inputStream, -1); + } + private void clearEntries() { setTableSize((1 << 8) + (blockMode ? 1 : 0)); } @@ -73,7 +78,7 @@ } return code; } - + private void reAlignReading() throws IOException { // "compress" works in multiples of 8 symbols, each codeBits bits long. // When codeBits changes, the remaining unused symbols in the current @@ -88,7 +93,7 @@ } in.clearBitCache(); } - + /** * {@inheritDoc} *

    This method is only protected for technical reasons @@ -146,17 +151,17 @@ return expandCodeToOutputStack(code, addedUnfinishedEntry); } } - + /** * Checks if the signature matches what is expected for a Unix compress file. - * + * * @param signature * the bytes to check * @param length * the number of bytes to check * @return true, if this stream is a Unix compress compressed * stream, false otherwise - * + * * @since 1.9 */ public static boolean matches(final byte[] signature, final int length) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/package.html 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,26 @@ + + + +

    Provides stream class for (de)compressing streams using the + Zstandard algorithm based + on Zstandard + JNI.

    + + diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.zstandard; + + +import java.io.IOException; +import java.io.InputStream; + +import com.github.luben.zstd.ZstdInputStream; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.utils.CountingInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; + +/** + * {@link CompressorInputStream} implementation to decode Zstandard encoded stream. + * Library relies on Zstandard JNI + * + * @since 1.16 + */ +public class ZstdCompressorInputStream extends CompressorInputStream + implements InputStreamStatistics { + + private final CountingInputStream countingStream; + private final ZstdInputStream decIS; + + public ZstdCompressorInputStream(final InputStream in) throws IOException { + this.decIS = new ZstdInputStream(countingStream = new CountingInputStream(in)); + } + + @Override + public int available() throws IOException { + return decIS.available(); + } + + @Override + public void close() throws IOException { + decIS.close(); + } + + @Override + public int read(final byte[] b) throws IOException { + return decIS.read(b); + } + + @Override + public long skip(final long n) throws IOException { + return IOUtils.skip(decIS, n); + } + + @Override + public void mark(final int readlimit) { + decIS.mark(readlimit); + } + + @Override + public boolean markSupported() { + return decIS.markSupported(); + } + + @Override + public int read() throws IOException { + final int ret = decIS.read(); + count(ret == -1 ? 0 : 1); + return ret; + } + + @Override + public int read(final byte[] buf, final int off, final int len) throws IOException { + final int ret = decIS.read(buf, off, len); + count(ret); + return ret; + } + + @Override + public String toString() { + return decIS.toString(); + } + + @Override + public void reset() throws IOException { + decIS.reset(); + } + + /** + * @since 1.17 + */ + @Override + public long getCompressedCount() { + return countingStream.getBytesRead(); + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorOutputStream.java 2018-08-10 13:50:47.000000000 +0000 @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.zstandard; + + +import java.io.IOException; +import java.io.OutputStream; + +import com.github.luben.zstd.ZstdOutputStream; +import org.apache.commons.compress.compressors.CompressorOutputStream; + +/** + * {@link CompressorOutputStream} implementation to create Zstandard encoded stream. + * Library relies on Zstandard JNI + * + * @since 1.16 + */ +public class ZstdCompressorOutputStream extends CompressorOutputStream { + + private final ZstdOutputStream encOS; + + /** + * Wraps the given stream into a zstd-jni ZstdOutputStream. + * @param outStream the stream to write to + * @param level value for zstd-jni's level argument + * @param closeFrameOnFlush value for zstd-jni's closeFrameOnFlush argument + * @param useChecksum value for zstd-jni's useChecksum argument + * @throws IOException if zstd-jni does + * @since 1.18 + */ + public ZstdCompressorOutputStream(final OutputStream outStream, int level, boolean closeFrameOnFlush, + boolean useChecksum) throws IOException { + this.encOS = new ZstdOutputStream(outStream, level, closeFrameOnFlush, useChecksum); + } + + /** + * Wraps the given stream into a zstd-jni ZstdOutputStream using the default value for {@code useChecksum}. + * @param outStream the stream to write to + * @param level value for zstd-jni's level argument + * @param closeFrameOnFlush value for zstd-jni's closeFrameOnFlush argument + * @throws IOException if zstd-jni does + * @since 1.18 + */ + public ZstdCompressorOutputStream(final OutputStream outStream, int level, boolean closeFrameOnFlush) + throws IOException { + this.encOS = new ZstdOutputStream(outStream, level, closeFrameOnFlush); + } + + /** + * Wraps the given stream into a zstd-jni ZstdOutputStream using the default values for {@code closeFrameOnFlush} + * and {@code useChecksum}. + * @param outStream the stream to write to + * @param level value for zstd-jni's level argument + * @throws IOException if zstd-jni does + * @since 1.18 + */ + public ZstdCompressorOutputStream(final OutputStream outStream, int level) throws IOException { + this.encOS = new ZstdOutputStream(outStream, level); + } + + /** + * Wraps the given stream into a zstd-jni ZstdOutputStream using the default values for {@code level}, {@code + * closeFrameOnFlush} and {@code useChecksum}. + * @param outStream the stream to write to + * @throws IOException if zstd-jni does + */ + public ZstdCompressorOutputStream(final OutputStream outStream) throws IOException { + this.encOS = new ZstdOutputStream(outStream); + } + + @Override + public void close() throws IOException { + encOS.close(); + } + + @Override + public void write(final int b) throws IOException { + encOS.write(b); + } + + @Override + public void write(final byte[] buf, final int off, final int len) throws IOException { + encOS.write(buf, off, len); + } + + @Override + public String toString() { + return encOS.toString(); + } + + @Override + public void flush() throws IOException { + encOS.flush(); + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.zstandard; + +/** + * Utility code for the Zstandard compression format. + * @ThreadSafe + * @since 1.16 + */ +public class ZstdUtils { + + enum CachedAvailability { + DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE + } + + /** + * Zstandard Frame Magic Bytes. + */ + private static final byte[] ZSTANDARD_FRAME_MAGIC = { + (byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD + }; + + /** + * Skippable Frame Magic Bytes - the three common bytes. + */ + private static final byte[] SKIPPABLE_FRAME_MAGIC = { + (byte) 0x2A, (byte) 0x4D, (byte) 0x18 + }; + + private static volatile CachedAvailability cachedZstdAvailability; + + static { + cachedZstdAvailability = CachedAvailability.DONT_CACHE; + try { + Class.forName("org.osgi.framework.BundleEvent"); + } catch (final Exception ex) { // NOSONAR + setCacheZstdAvailablity(true); + } + } + + /** Private constructor to prevent instantiation of this utility class. */ + private ZstdUtils() { + } + + /** + * Are the classes required to support Zstandard compression available? + * @return true if the classes required to support Zstandard compression are available + */ + public static boolean isZstdCompressionAvailable() { + final CachedAvailability cachedResult = cachedZstdAvailability; + if (cachedResult != CachedAvailability.DONT_CACHE) { + return cachedResult == CachedAvailability.CACHED_AVAILABLE; + } + return internalIsZstdCompressionAvailable(); + } + + private static boolean internalIsZstdCompressionAvailable() { + try { + Class.forName("com.github.luben.zstd.ZstdInputStream"); + return true; + } catch (NoClassDefFoundError | Exception error) { // NOSONAR + return false; + } + } + + /** + * Whether to cache the result of the Zstandard for Java check. + * + *

    This defaults to {@code false} in an OSGi environment and {@code true} otherwise.

    + * @param doCache whether to cache the result + */ + public static void setCacheZstdAvailablity(final boolean doCache) { + if (!doCache) { + cachedZstdAvailability = CachedAvailability.DONT_CACHE; + } else if (cachedZstdAvailability == CachedAvailability.DONT_CACHE) { + final boolean hasZstd = internalIsZstdCompressionAvailable(); + cachedZstdAvailability = hasZstd ? CachedAvailability.CACHED_AVAILABLE + : CachedAvailability.CACHED_UNAVAILABLE; + } + } + + /** + * Checks if the signature matches what is expected for a Zstandard file. + * + * @param signature the bytes to check + * @param length the number of bytes to check + * @return true if signature matches the Ztstandard or skippable + * frame magic bytes, false otherwise + */ + public static boolean matches(final byte[] signature, final int length) { + if (length < ZSTANDARD_FRAME_MAGIC.length) { + return false; + } + + boolean isZstandard = true; + for (int i = 0; i < ZSTANDARD_FRAME_MAGIC.length; ++i) { + if (signature[i] != ZSTANDARD_FRAME_MAGIC[i]) { + isZstandard = false; + break; + } + } + if (isZstandard) { + return true; + } + + if (0x50 == (signature[0] & 0xF0)) { + // skippable frame + for (int i = 0; i < SKIPPABLE_FRAME_MAGIC.length; ++i) { + if (signature[i + 1] != SKIPPABLE_FRAME_MAGIC[i]) { + return false; + } + } + + return true; + } + + return false; + } + + // only exists to support unit tests + static CachedAvailability getCachedZstdAvailability() { + return cachedZstdAvailability; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/MemoryLimitException.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/MemoryLimitException.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/MemoryLimitException.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/MemoryLimitException.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress; + +import java.io.IOException; + +/** + * If a stream checks for estimated memory allocation, and the estimate + * goes above the memory limit, this is thrown. This can also be thrown + * if a stream tries to allocate a byte array that is larger than + * the allowable limit. + * + * @since 1.14 + */ +public class MemoryLimitException extends IOException { + + private static final long serialVersionUID = 1L; + + //long instead of int to account for overflow for corrupt files + private final long memoryNeededInKb; + private final int memoryLimitInKb; + + public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb) { + super(buildMessage(memoryNeededInKb, memoryLimitInKb)); + this.memoryNeededInKb = memoryNeededInKb; + this.memoryLimitInKb = memoryLimitInKb; + } + + public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb, Exception e) { + super(buildMessage(memoryNeededInKb, memoryLimitInKb), e); + this.memoryNeededInKb = memoryNeededInKb; + this.memoryLimitInKb = memoryLimitInKb; + } + + public long getMemoryNeededInKb() { + return memoryNeededInKb; + } + + public int getMemoryLimitInKb() { + return memoryLimitInKb; + } + + private static String buildMessage(long memoryNeededInKb, int memoryLimitInKb) { + return memoryNeededInKb + " kb of memory would be needed; limit was " + + memoryLimitInKb + " kb. " + + "If the file is not corrupt, consider increasing the memory limit."; + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/parallel/FileBasedScatterGatherBackingStore.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/parallel/FileBasedScatterGatherBackingStore.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/parallel/FileBasedScatterGatherBackingStore.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/parallel/FileBasedScatterGatherBackingStore.java 2018-07-01 09:53:29.000000000 +0000 @@ -19,10 +19,10 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; /** * ScatterGatherBackingStore that is backed by a file. @@ -31,17 +31,24 @@ */ public class FileBasedScatterGatherBackingStore implements ScatterGatherBackingStore { private final File target; - private final FileOutputStream os; + private final OutputStream os; private boolean closed; public FileBasedScatterGatherBackingStore(final File target) throws FileNotFoundException { this.target = target; - os = new FileOutputStream(target); + try { + os = Files.newOutputStream(target.toPath()); + } catch (FileNotFoundException ex) { + throw ex; + } catch (IOException ex) { + // must convert exception to stay backwards compatible with Compress 1.10 to 1.13 + throw new RuntimeException(ex); // NOSONAR + } } @Override public InputStream getInputStream() throws IOException { - return new FileInputStream(target); + return Files.newInputStream(target.toPath()); } @Override @@ -60,7 +67,10 @@ @Override public void close() throws IOException { - closeForWriting(); - target.delete(); + try { + closeForWriting(); + } finally { + target.delete(); + } } } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ArchiveUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ArchiveUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ArchiveUtils.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ArchiveUtils.java 2018-05-02 20:17:13.000000000 +0000 @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress.utils; @@ -42,7 +42,7 @@ * - 2000 main.c * d 100 testfiles * - * + * * @param entry the entry * @return the representation of the entry */ @@ -62,7 +62,7 @@ /** * Check if buffer contents matches Ascii String. - * + * * @param expected expected string * @param buffer the buffer * @param offset offset to read from @@ -83,7 +83,7 @@ /** * Check if buffer contents matches Ascii String. - * + * * @param expected the expected strin * @param buffer the buffer * @return {@code true} if buffer is the same as the expected string @@ -95,7 +95,7 @@ /** * Convert a string to Ascii bytes. * Used for comparing "magic" strings which need to be independent of the default Locale. - * + * * @param inputString string to convert * @return the bytes */ @@ -110,7 +110,7 @@ /** * Convert an input byte array to a String using the ASCII character set. - * + * * @param inputBytes bytes to convert * @return the bytes, interpreted as an Ascii string */ @@ -125,7 +125,7 @@ /** * Convert an input byte array to a String using the ASCII character set. - * + * * @param inputBytes input byte array * @param offset offset within array * @param length length of array @@ -142,7 +142,7 @@ /** * Compare byte buffers, optionally ignoring trailing nulls - * + * * @param buffer1 first buffer * @param offset1 first offset * @param length1 first length @@ -186,7 +186,7 @@ /** * Compare byte buffers - * + * * @param buffer1 the first buffer * @param offset1 the first offset * @param length1 the first length @@ -203,7 +203,7 @@ /** * Compare byte buffers - * + * * @param buffer1 the first buffer * @param buffer2 the second buffer * @return {@code true} if buffer1 and buffer2 have same contents @@ -214,9 +214,9 @@ /** * Compare byte buffers, optionally ignoring trailing nulls - * + * * @param buffer1 the first buffer - * @param buffer2 the second buffer + * @param buffer2 the second buffer * @param ignoreTrailingNulls whether to ignore tariling nulls * @return {@code true} if buffer1 and buffer2 have same contents */ @@ -226,7 +226,7 @@ /** * Compare byte buffers, ignoring trailing nulls - * + * * @param buffer1 the first buffer * @param offset1 the first offset * @param length1 the first length @@ -240,10 +240,10 @@ final byte[] buffer2, final int offset2, final int length2){ return isEqual(buffer1, offset1, length1, buffer2, offset2, length2, true); } - + /** * Returns true if the first N bytes of an array are all zero - * + * * @param a * The array to check * @param size diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/BitInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/BitInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/BitInputStream.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/BitInputStream.java 2018-05-23 12:50:54.000000000 +0000 @@ -38,27 +38,27 @@ } } - private final InputStream in; + private final CountingInputStream in; private final ByteOrder byteOrder; private long bitsCached = 0; private int bitsCachedSize = 0; /** - * Constructor taking an InputStream and its bit arrangement. + * Constructor taking an InputStream and its bit arrangement. * @param in the InputStream * @param byteOrder the bit arrangement across byte boundaries, * either BIG_ENDIAN (aaaaabbb bb000000) or LITTLE_ENDIAN (bbbaaaaa 000000bb) */ public BitInputStream(final InputStream in, final ByteOrder byteOrder) { - this.in = in; + this.in = new CountingInputStream(in); this.byteOrder = byteOrder; } - + @Override public void close() throws IOException { in.close(); } - + /** * Clears the cache of bits that have been read from the * underlying stream but not yet provided via {@link #readBits}. @@ -67,7 +67,7 @@ bitsCached = 0; bitsCachedSize = 0; } - + /** * Returns at most 63 bits read from the underlying stream. * @@ -82,56 +82,122 @@ if (count < 0 || count > MAXIMUM_CACHE_SIZE) { throw new IllegalArgumentException("count must not be negative or greater than " + MAXIMUM_CACHE_SIZE); } + if (ensureCache(count)) { + return -1; + } + + if (bitsCachedSize < count) { + return processBitsGreater57(count); + } + return readCachedBits(count); + } + + /** + * Returns the number of bits that can be read from this input + * stream without reading from the underlying input stream at all. + * @return estimate of the number of bits that can be read without reading from the underlying stream + * @since 1.16 + */ + public int bitsCached() { + return bitsCachedSize; + } + + /** + * Returns an estimate of the number of bits that can be read from + * this input stream without blocking by the next invocation of a + * method for this input stream. + * @throws IOException if the underlying stream throws one when calling available + * @return estimate of the number of bits that can be read without blocking + * @since 1.16 + */ + public long bitsAvailable() throws IOException { + return bitsCachedSize + ((long) Byte.SIZE) * in.available(); + } + + /** + * Drops bits until the next bits will be read from a byte boundary. + * @since 1.16 + */ + public void alignWithByteBoundary() { + int toSkip = bitsCachedSize % Byte.SIZE; + if (toSkip > 0) { + readCachedBits(toSkip); + } + } + + /** + * Returns the number of bytes read from the underlying stream. + * + *

    This includes the bytes read to fill the current cache and + * not read as bits so far.

    + * @return the number of bytes read from the underlying stream + * @since 1.17 + */ + public long getBytesRead() { + return in.getBytesRead(); + } + + private long processBitsGreater57(final int count) throws IOException { + final long bitsOut; + int overflowBits = 0; + long overflow = 0L; + + // bitsCachedSize >= 57 and left-shifting it 8 bits would cause an overflow + int bitsToAddCount = count - bitsCachedSize; + overflowBits = Byte.SIZE - bitsToAddCount; + final long nextByte = in.read(); + if (nextByte < 0) { + return nextByte; + } + if (byteOrder == ByteOrder.LITTLE_ENDIAN) { + long bitsToAdd = nextByte & MASKS[bitsToAddCount]; + bitsCached |= (bitsToAdd << bitsCachedSize); + overflow = (nextByte >>> bitsToAddCount) & MASKS[overflowBits]; + } else { + bitsCached <<= bitsToAddCount; + long bitsToAdd = (nextByte >>> (overflowBits)) & MASKS[bitsToAddCount]; + bitsCached |= bitsToAdd; + overflow = nextByte & MASKS[overflowBits]; + } + bitsOut = bitsCached & MASKS[count]; + bitsCached = overflow; + bitsCachedSize = overflowBits; + return bitsOut; + } + + private long readCachedBits(int count) { + final long bitsOut; + if (byteOrder == ByteOrder.LITTLE_ENDIAN) { + bitsOut = (bitsCached & MASKS[count]); + bitsCached >>>= count; + } else { + bitsOut = (bitsCached >> (bitsCachedSize - count)) & MASKS[count]; + } + bitsCachedSize -= count; + return bitsOut; + } + + /** + * Fills the cache up to 56 bits + * @param count + * @return return true, when EOF + * @throws IOException + */ + private boolean ensureCache(final int count) throws IOException { while (bitsCachedSize < count && bitsCachedSize < 57) { final long nextByte = in.read(); if (nextByte < 0) { - return nextByte; + return true; } if (byteOrder == ByteOrder.LITTLE_ENDIAN) { bitsCached |= (nextByte << bitsCachedSize); } else { - bitsCached <<= 8; + bitsCached <<= Byte.SIZE; bitsCached |= nextByte; } - bitsCachedSize += 8; - } - int overflowBits = 0; - long overflow = 0l; - if (bitsCachedSize < count) { - // bitsCachedSize >= 57 and left-shifting it 8 bits would cause an overflow - int bitsToAddCount = count - bitsCachedSize; - overflowBits = 8 - bitsToAddCount; - final long nextByte = in.read(); - if (nextByte < 0) { - return nextByte; - } - if (byteOrder == ByteOrder.LITTLE_ENDIAN) { - long bitsToAdd = nextByte & MASKS[bitsToAddCount]; - bitsCached |= (bitsToAdd << bitsCachedSize); - overflow = (nextByte >>> bitsToAddCount) & MASKS[overflowBits]; - } else { - bitsCached <<= bitsToAddCount; - long bitsToAdd = (nextByte >>> (overflowBits)) & MASKS[bitsToAddCount]; - bitsCached |= bitsToAdd; - overflow = nextByte & MASKS[overflowBits]; - } - bitsCachedSize = count; + bitsCachedSize += Byte.SIZE; } - - final long bitsOut; - if (overflowBits == 0) { - if (byteOrder == ByteOrder.LITTLE_ENDIAN) { - bitsOut = (bitsCached & MASKS[count]); - bitsCached >>>= count; - } else { - bitsOut = (bitsCached >> (bitsCachedSize - count)) & MASKS[count]; - } - bitsCachedSize -= count; - } else { - bitsOut = bitsCached & MASKS[count]; - bitsCached = overflow; - bitsCachedSize = overflowBits; - } - return bitsOut; + return false; } + } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/BoundedInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -28,7 +28,7 @@ public class BoundedInputStream extends InputStream { private final InputStream in; private long bytesRemaining; - + /** * Creates the stream that will at most read the given amount of * bytes from the given stream. @@ -39,7 +39,7 @@ this.in = in; bytesRemaining = size; } - + @Override public int read() throws IOException { if (bytesRemaining > 0) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ByteUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ByteUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ByteUtils.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ByteUtils.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.commons.compress.utils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Utility methods for reading and writing bytes. + * @since 1.14 + */ +public final class ByteUtils { + private ByteUtils() { /* no instances */ } + + /** + * Used to supply bytes. + * @since 1.14 + */ + public interface ByteSupplier { + /** + * The contract is similar to {@link InputStream#read()}, return + * the byte as an unsigned int, -1 if there are no more bytes. + * @return the supplied byte or -1 if there are no more bytes + * @throws IOException if supplying fails + */ + int getAsByte() throws IOException; + } + + /** + * Used to consume bytes. + * @since 1.14 + */ + public interface ByteConsumer { + /** + * The contract is similar to {@link OutputStream#write(int)}, + * consume the lower eight bytes of the int as a byte. + * @param b the byte to consume + * @throws IOException if consuming fails + */ + void accept(int b) throws IOException; + } + + /** + * Reads the given byte array as a little endian long. + * @param bytes the byte array to convert + * @return the number read + */ + public static long fromLittleEndian(byte[] bytes) { + return fromLittleEndian(bytes, 0, bytes.length); + } + + /** + * Reads the given byte array as a little endian long. + * @param bytes the byte array to convert + * @param off the offset into the array that starts the value + * @param length the number of bytes representing the value + * @return the number read + * @throws IllegalArgumentException if len is bigger than eight + */ + public static long fromLittleEndian(byte[] bytes, final int off, final int length) { + checkReadLength(length); + long l = 0; + for (int i = 0; i < length; i++) { + l |= (bytes[off + i] & 0xffL) << (8 * i); + } + return l; + } + + /** + * Reads the given number of bytes from the given stream as a little endian long. + * @param in the stream to read from + * @param length the number of bytes representing the value + * @return the number read + * @throws IllegalArgumentException if len is bigger than eight + * @throws IOException if reading fails or the stream doesn't + * contain the given number of bytes anymore + */ + public static long fromLittleEndian(InputStream in, int length) throws IOException { + // somewhat duplicates the ByteSupplier version in order to save the creation of a wrapper object + checkReadLength(length); + long l = 0; + for (int i = 0; i < length; i++) { + long b = in.read(); + if (b == -1) { + throw new IOException("premature end of data"); + } + l |= (b << (i * 8)); + } + return l; + } + + /** + * Reads the given number of bytes from the given supplier as a little endian long. + * + *

    Typically used by our InputStreams that need to count the + * bytes read as well.

    + * + * @param supplier the supplier for bytes + * @param length the number of bytes representing the value + * @return the number read + * @throws IllegalArgumentException if len is bigger than eight + * @throws IOException if the supplier fails or doesn't supply the + * given number of bytes anymore + */ + public static long fromLittleEndian(ByteSupplier supplier, final int length) throws IOException { + checkReadLength(length); + long l = 0; + for (int i = 0; i < length; i++) { + long b = supplier.getAsByte(); + if (b == -1) { + throw new IOException("premature end of data"); + } + l |= (b << (i * 8)); + } + return l; + } + + /** + * Reads the given number of bytes from the given input as little endian long. + * @param in the input to read from + * @param length the number of bytes representing the value + * @return the number read + * @throws IllegalArgumentException if len is bigger than eight + * @throws IOException if reading fails or the stream doesn't + * contain the given number of bytes anymore + */ + public static long fromLittleEndian(DataInput in, int length) throws IOException { + // somewhat duplicates the ByteSupplier version in order to save the creation of a wrapper object + checkReadLength(length); + long l = 0; + for (int i = 0; i < length; i++) { + long b = in.readUnsignedByte(); + l |= (b << (i * 8)); + } + return l; + } + + /** + * Inserts the given value into the array as a little endian + * sequence of the given length starting at the given offset. + * @param b the array to write into + * @param value the value to insert + * @param off the offset into the array that receives the first byte + * @param length the number of bytes to use to represent the value + */ + public static void toLittleEndian(final byte[] b, final long value, final int off, final int length) { + long num = value; + for (int i = 0; i < length; i++) { + b[off + i] = (byte) (num & 0xff); + num >>= 8; + } + } + + /** + * Writes the given value to the given stream as a little endian + * array of the given length. + * @param out the stream to write to + * @param value the value to write + * @param length the number of bytes to use to represent the value + * @throws IOException if writing fails + */ + public static void toLittleEndian(OutputStream out, final long value, final int length) + throws IOException { + // somewhat duplicates the ByteConsumer version in order to save the creation of a wrapper object + long num = value; + for (int i = 0; i < length; i++) { + out.write((int) (num & 0xff)); + num >>= 8; + } + } + + /** + * Provides the given value to the given consumer as a little endian + * sequence of the given length. + * @param consumer the consumer to provide the bytes to + * @param value the value to provide + * @param length the number of bytes to use to represent the value + * @throws IOException if writing fails + */ + public static void toLittleEndian(ByteConsumer consumer, final long value, final int length) + throws IOException { + long num = value; + for (int i = 0; i < length; i++) { + consumer.accept((int) (num & 0xff)); + num >>= 8; + } + } + + /** + * Writes the given value to the given stream as a little endian + * array of the given length. + * @param out the output to write to + * @param value the value to write + * @param length the number of bytes to use to represent the value + * @throws IOException if writing fails + */ + public static void toLittleEndian(DataOutput out, final long value, final int length) + throws IOException { + // somewhat duplicates the ByteConsumer version in order to save the creation of a wrapper object + long num = value; + for (int i = 0; i < length; i++) { + out.write((int) (num & 0xff)); + num >>= 8; + } + } + + /** + * {@link ByteSupplier} based on {@link InputStream}. + * @since 1.14 + */ + public static class InputStreamByteSupplier implements ByteSupplier { + private final InputStream is; + public InputStreamByteSupplier(InputStream is) { + this.is = is; + } + @Override + public int getAsByte() throws IOException { + return is.read(); + } + } + + /** + * {@link ByteConsumer} based on {@link OutputStream}. + * @since 1.14 + */ + public static class OutputStreamByteConsumer implements ByteConsumer { + private final OutputStream os; + public OutputStreamByteConsumer(OutputStream os) { + this.os = os; + } + @Override + public void accept(int b) throws IOException { + os.write(b); + } + } + + private static final void checkReadLength(int length) { + if (length > 8) { + throw new IllegalArgumentException("can't read more than eight bytes into a long value"); + } + } +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CharsetNames.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CharsetNames.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CharsetNames.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CharsetNames.java 2018-06-15 17:46:02.000000000 +0000 @@ -13,22 +13,22 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress.utils; /** * Character encoding names required of every implementation of the Java platform. - * - * From the Java documentation Standard + * + * From the Java documentation Standard * charsets: *

    * Every implementation of the Java platform is required to support the following character encodings. Consult the * release documentation for your implementation to see if any other encodings are supported. Consult the release * documentation for your implementation to see if any other encodings are supported. *

    - * + * *
    *
    US-ASCII
    *
    Seven-bit ASCII, a.k.a. ISO646-US, a.k.a. the Basic Latin block of the Unicode character set.
    @@ -44,13 +44,12 @@ *
    Sixteen-bit Unicode Transformation Format, byte order specified by a mandatory initial byte-order mark (either order * accepted on input, big-endian used on output.)
    *
    - * + * *

    This perhaps would best belong in the [lang] project. Even if a similar interface is defined in [lang], it is not * foreseen that [compress] would be made to depend on [lang].

    - * - * @see Standard charsets + * + * @see Standard charsets * @since 1.4 - * @version $Id$ */ public class CharsetNames { /** @@ -58,8 +57,8 @@ *

    * Every implementation of the Java platform is required to support this character encoding. *

    - * - * @see Standard charsets + * + * @see Standard charsets */ public static final String ISO_8859_1 = "ISO-8859-1"; @@ -70,8 +69,8 @@ *

    * Every implementation of the Java platform is required to support this character encoding. *

    - * - * @see Standard charsets + * + * @see Standard charsets */ public static final String US_ASCII = "US-ASCII"; @@ -83,8 +82,8 @@ *

    * Every implementation of the Java platform is required to support this character encoding. *

    - * - * @see Standard charsets + * + * @see Standard charsets */ public static final String UTF_16 = "UTF-16"; @@ -95,8 +94,8 @@ *

    * Every implementation of the Java platform is required to support this character encoding. *

    - * - * @see Standard charsets + * + * @see Standard charsets */ public static final String UTF_16BE = "UTF-16BE"; @@ -107,8 +106,8 @@ *

    * Every implementation of the Java platform is required to support this character encoding. *

    - * - * @see Standard charsets + * + * @see Standard charsets */ public static final String UTF_16LE = "UTF-16LE"; @@ -119,8 +118,8 @@ *

    * Every implementation of the Java platform is required to support this character encoding. *

    - * - * @see Standard charsets + * + * @see Standard charsets */ public static final String UTF_8 = "UTF-8"; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Charsets.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Charsets.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Charsets.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Charsets.java 2018-05-23 12:50:54.000000000 +0000 @@ -19,11 +19,12 @@ package org.apache.commons.compress.utils; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; /** * Charsets required of every implementation of the Java platform. * - * From the Java documentation Standard + * From the Java documentation Standard * charsets: *

    * Every implementation of the Java platform is required to support the following character encodings. Consult the @@ -50,9 +51,9 @@ *

    This class best belongs in the Commons Lang or IO project. Even if a similar class is defined in another Commons * component, it is not foreseen that Commons Compress would be made to depend on another Commons component.

    * - * @see Standard charsets + * @see Standard charsets + * @see StandardCharsets * @since 1.4 - * @version $Id$ */ public class Charsets { @@ -93,9 +94,10 @@ * Every implementation of the Java platform is required to support this character encoding. *

    * - * @see Standard charsets + * @see Standard charsets + * @deprecated replaced by {@link StandardCharsets} in Java 7 */ - public static final Charset ISO_8859_1 = Charset.forName(CharsetNames.ISO_8859_1); + public static final Charset ISO_8859_1 = StandardCharsets.ISO_8859_1; /** *

    @@ -105,9 +107,10 @@ * Every implementation of the Java platform is required to support this character encoding. *

    * - * @see Standard charsets + * @see Standard charsets + * @deprecated replaced by {@link StandardCharsets} in Java 7 */ - public static final Charset US_ASCII = Charset.forName(CharsetNames.US_ASCII); + public static final Charset US_ASCII = StandardCharsets.US_ASCII; /** *

    @@ -118,9 +121,10 @@ * Every implementation of the Java platform is required to support this character encoding. *

    * - * @see Standard charsets + * @see Standard charsets + * @deprecated replaced by {@link StandardCharsets} in Java 7 */ - public static final Charset UTF_16 = Charset.forName(CharsetNames.UTF_16); + public static final Charset UTF_16 = StandardCharsets.UTF_16; /** *

    @@ -130,9 +134,10 @@ * Every implementation of the Java platform is required to support this character encoding. *

    * - * @see Standard charsets + * @see Standard charsets + * @deprecated replaced by {@link StandardCharsets} in Java 7 */ - public static final Charset UTF_16BE = Charset.forName(CharsetNames.UTF_16BE); + public static final Charset UTF_16BE = StandardCharsets.UTF_16BE; /** *

    @@ -142,9 +147,10 @@ * Every implementation of the Java platform is required to support this character encoding. *

    * - * @see Standard charsets + * @see Standard charsets + * @deprecated replaced by {@link StandardCharsets} in Java 7 */ - public static final Charset UTF_16LE = Charset.forName(CharsetNames.UTF_16LE); + public static final Charset UTF_16LE = StandardCharsets.UTF_16LE; /** *

    @@ -154,7 +160,8 @@ * Every implementation of the Java platform is required to support this character encoding. *

    * - * @see Standard charsets + * @see Standard charsets + * @deprecated replaced by {@link StandardCharsets} in Java 7 */ - public static final Charset UTF_8 = Charset.forName(CharsetNames.UTF_8); + public static final Charset UTF_8 = StandardCharsets.UTF_8; } diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.Checksum; + +/** + * A stream that calculates the checksum of the data read. + * @NotThreadSafe + * @since 1.14 + */ +public class ChecksumCalculatingInputStream extends InputStream { + private final InputStream in; + private final Checksum checksum; + + public ChecksumCalculatingInputStream(final Checksum checksum, final InputStream in) { + + if ( checksum == null ){ + throw new NullPointerException("Parameter checksum must not be null"); + } + + if ( in == null ){ + throw new NullPointerException("Parameter in must not be null"); + } + + this.checksum = checksum; + this.in = in; + } + + /** + * Reads a single byte from the stream + * @throws IOException if the underlying stream throws or the + * stream is exhausted and the Checksum doesn't match the expected + * value + */ + @Override + public int read() throws IOException { + final int ret = in.read(); + if (ret >= 0) { + checksum.update(ret); + } + return ret; + } + + /** + * Reads a byte array from the stream + * @throws IOException if the underlying stream throws or the + * stream is exhausted and the Checksum doesn't match the expected + * value + */ + @Override + public int read(final byte[] b) throws IOException { + return read(b, 0, b.length); + } + + /** + * Reads from the stream into a byte array. + * @throws IOException if the underlying stream throws or the + * stream is exhausted and the Checksum doesn't match the expected + * value + */ + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + final int ret = in.read(b, off, len); + if (ret >= 0) { + checksum.update(b, off, ret); + } + return ret; + } + + @Override + public long skip(final long n) throws IOException { + // Can't really skip, we have to hash everything to verify the checksum + if (read() >= 0) { + return 1; + } + return 0; + } + + /** + * Returns the calculated checksum. + * @return the calculated checksum. + */ + public long getValue() { + return checksum.getValue(); + } + +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -32,7 +32,7 @@ private long bytesRemaining; private final long expectedChecksum; private final Checksum checksum; - + public ChecksumVerifyingInputStream(final Checksum checksum, final InputStream in, final long size, final long expectedChecksum) { this.checksum = checksum; diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CloseShieldFilterInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CloseShieldFilterInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CloseShieldFilterInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CloseShieldFilterInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.commons.compress.utils; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Re-implements {@link FilterInputStream#close()} to do nothing. + * @since 1.14 + */ +public class CloseShieldFilterInputStream extends FilterInputStream { + + public CloseShieldFilterInputStream(InputStream in) { + super(in); + } + + @Override + public void close() throws IOException { + // NO IMPLEMENTATION. + } + +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CountingInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CountingInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CountingInputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CountingInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -57,7 +57,7 @@ /** * Increments the counter of already read bytes. * Doesn't increment if the EOF has been hit (read == -1) - * + * * @param read the number of bytes read */ protected final void count(final long read) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CountingOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CountingOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CountingOutputStream.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CountingOutputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -52,7 +52,7 @@ /** * Increments the counter of already written bytes. * Doesn't increment if the EOF has been hit (written == -1) - * + * * @param written the number of bytes written */ protected void count(final long written) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -27,14 +27,14 @@ * @since 1.6 */ public class CRC32VerifyingInputStream extends ChecksumVerifyingInputStream { - + /** * @param in the stream to wrap * @param size the of the stream's content * @param expectedCrc32 the expected checksum */ public CRC32VerifyingInputStream(final InputStream in, final long size, final int expectedCrc32) { - this(in, size, expectedCrc32 & 0xFFFFffffl); + this(in, size, expectedCrc32 & 0xFFFFffffL); } /** diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStream.java 2018-07-01 09:53:29.000000000 +0000 @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.utils; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.WritableByteChannel; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * This class supports writing to an Outputstream or WritableByteChannel in fixed length blocks. + *

    It can be be used to support output to devices such as tape drives that require output in this + * format. If the final block does not have enough content to fill an entire block, the output will + * be padded to a full block size.

    + * + *

    This class can be used to support TAR,PAX, and CPIO blocked output to character special devices. + * It is not recommended that this class be used unless writing to such devices, as the padding + * serves no useful purpose in such cases.

    + * + *

    This class should normally wrap a FileOutputStream or associated WritableByteChannel directly. + * If there is an intervening filter that modified the output, such as a CompressorOutputStream, or + * performs its own buffering, such as BufferedOutputStream, output to the device may + * no longer be of the specified size.

    + * + *

    Any content written to this stream should be self-delimiting and should tolerate any padding + * added to fill the last block.

    + * + * @since 1.15 + */ +public class FixedLengthBlockOutputStream extends OutputStream implements WritableByteChannel { + + private final WritableByteChannel out; + private final int blockSize; + private final ByteBuffer buffer; + private final AtomicBoolean closed = new AtomicBoolean(false); + + /** + * Create a fixed length block output stream with given destination stream and block size + * @param os The stream to wrap. + * @param blockSize The block size to use. + */ + public FixedLengthBlockOutputStream(OutputStream os, int blockSize) { + if (os instanceof FileOutputStream) { + FileOutputStream fileOutputStream = (FileOutputStream) os; + out = fileOutputStream.getChannel(); + buffer = ByteBuffer.allocateDirect(blockSize); + } else { + out = new BufferAtATimeOutputChannel(os); + buffer = ByteBuffer.allocate(blockSize); + } + this.blockSize = blockSize; + } + /** + * Create a fixed length block output stream with given destination writable byte channel and block size + * @param out The writable byte channel to wrap. + * @param blockSize The block size to use. + */ + public FixedLengthBlockOutputStream(WritableByteChannel out, int blockSize) { + this.out = out; + this.blockSize = blockSize; + this.buffer = ByteBuffer.allocateDirect(blockSize); + } + + private void maybeFlush() throws IOException { + if (!buffer.hasRemaining()) { + writeBlock(); + } + } + + private void writeBlock() throws IOException { + buffer.flip(); + int i = out.write(buffer); + boolean hasRemaining = buffer.hasRemaining(); + if (i != blockSize || hasRemaining) { + String msg = String + .format("Failed to write %,d bytes atomically. Only wrote %,d", + blockSize, i); + throw new IOException(msg); + } + buffer.clear(); + } + + @Override + public void write(int b) throws IOException { + if (!isOpen()) { + throw new ClosedChannelException(); + } + buffer.put((byte) b); + maybeFlush(); + } + + @Override + public void write(byte[] b, final int offset, final int length) throws IOException { + if (!isOpen()) { + throw new ClosedChannelException(); + } + int off = offset; + int len = length; + while (len > 0) { + int n = Math.min(len, buffer.remaining()); + buffer.put(b, off, n); + maybeFlush(); + len -= n; + off += n; + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + if (!isOpen()) { + throw new ClosedChannelException(); + } + int srcRemaining = src.remaining(); + + if (srcRemaining < buffer.remaining()) { + // if don't have enough bytes in src to fill up a block we must buffer + buffer.put(src); + } else { + int srcLeft = srcRemaining; + int savedLimit = src.limit(); + // If we're not at the start of buffer, we have some bytes already buffered + // fill up the reset of buffer and write the block. + if (buffer.position() != 0) { + int n = buffer.remaining(); + src.limit(src.position() + n); + buffer.put(src); + writeBlock(); + srcLeft -= n; + } + // whilst we have enough bytes in src for complete blocks, + // write them directly from src without copying them to buffer + while (srcLeft >= blockSize) { + src.limit(src.position() + blockSize); + out.write(src); + srcLeft -= blockSize; + } + // copy any remaining bytes into buffer + src.limit(savedLimit); + buffer.put(src); + } + return srcRemaining; + } + + @Override + public boolean isOpen() { + if (!out.isOpen()) { + closed.set(true); + } + return !closed.get(); + } + + /** + * Potentially pads and then writes the current block to the underlying stream. + * @throws IOException if writing fails + */ + public void flushBlock() throws IOException { + if (buffer.position() != 0) { + padBlock(); + writeBlock(); + } + } + + @Override + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + try { + flushBlock(); + } finally { + out.close(); + } + } + } + + private void padBlock() { + buffer.order(ByteOrder.nativeOrder()); + int bytesToWrite = buffer.remaining(); + if (bytesToWrite > 8) { + int align = buffer.position() & 7; + if (align != 0) { + int limit = 8 - align; + for (int i = 0; i < limit; i++) { + buffer.put((byte) 0); + } + bytesToWrite -= limit; + } + + while (bytesToWrite >= 8) { + buffer.putLong(0L); + bytesToWrite -= 8; + } + } + while (buffer.hasRemaining()) { + buffer.put((byte) 0); + } + } + + /** + * Helper class to provide channel wrapper for arbitrary output stream that doesn't alter the + * size of writes. We can't use Channels.newChannel, because for non FileOutputStreams, it + * breaks up writes into 8KB max chunks. Since the purpose of this class is to always write + * complete blocks, we need to write a simple class to take care of it. + */ + private static class BufferAtATimeOutputChannel implements WritableByteChannel { + + private final OutputStream out; + private final AtomicBoolean closed = new AtomicBoolean(false); + + private BufferAtATimeOutputChannel(OutputStream out) { + this.out = out; + } + + @Override + public int write(ByteBuffer buffer) throws IOException { + if (!isOpen()) { + throw new ClosedChannelException(); + } + if (!buffer.hasArray()) { + throw new IllegalArgumentException("direct buffer somehow written to BufferAtATimeOutputChannel"); + } + + try { + int pos = buffer.position(); + int len = buffer.limit() - pos; + out.write(buffer.array(), buffer.arrayOffset() + pos, len); + buffer.position(buffer.limit()); + return len; + } catch (IOException e) { + try { + close(); + } catch (IOException ignored) { //NOSONAR + } + throw e; + } + } + + @Override + public boolean isOpen() { + return !closed.get(); + } + + @Override + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + out.close(); + } + } + + } + + +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/InputStreamStatistics.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/InputStreamStatistics.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/InputStreamStatistics.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/InputStreamStatistics.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.commons.compress.utils; + +/** + * This interface provides statistics on the current decompression stream. + * The stream consumer can use that statistics to handle abnormal + * compression ratios, i.e. to prevent zip bombs. + * + * @since 1.17 + */ +public interface InputStreamStatistics { + /** + * @return the amount of raw or compressed bytes read by the stream + */ + long getCompressedCount(); + + /** + * @return the amount of decompressed bytes returned by the stream + */ + long getUncompressedCount(); +} diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/IOUtils.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/IOUtils.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/IOUtils.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/IOUtils.java 2018-05-23 12:50:54.000000000 +0000 @@ -35,7 +35,7 @@ private static final int COPY_BUF_SIZE = 8024; private static final int SKIP_BUF_SIZE = 4096; - + // This buffer does not need to be synchronised because it is write only; the contents are ignored // Does not affect Immutability private static final byte[] SKIP_BUF = new byte[SKIP_BUF_SIZE]; @@ -68,12 +68,17 @@ * @param output * the target Stream * @param buffersize - * the buffer size to use + * the buffer size to use, must be bigger than 0 * @return the number of bytes copied * @throws IOException * if an error occurs + * @throws IllegalArgumentException + * if buffersize is smaller than or equal to 0 */ public static long copy(final InputStream input, final OutputStream output, final int buffersize) throws IOException { + if (buffersize < 1) { + throw new IllegalArgumentException("buffersize must be bigger than 0"); + } final byte[] buffer = new byte[buffersize]; int n = 0; long count=0; @@ -83,7 +88,7 @@ } return count; } - + /** * Skips the given number of bytes by repeatedly invoking skip on * the given input stream if necessary. @@ -109,7 +114,7 @@ } numToSkip -= skipped; } - + while (numToSkip > 0) { final int read = readFully(input, SKIP_BUF, 0, (int) Math.min(numToSkip, SKIP_BUF_SIZE)); diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Iterators.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Iterators.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Iterators.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Iterators.java 2018-05-02 20:17:13.000000000 +0000 @@ -25,7 +25,7 @@ /** * Iterator utilities. - * + * * @since 1.13. */ public class Iterators { @@ -33,12 +33,15 @@ /** * Adds all the elements in the source {@code iterator} to the target * {@code collection}. - * + * *

    * When this method returns, the {@code iterator} will be "empty": its * {@code hasNext()} method returns {@code false}. *

    * + * @param type of the elements contained inside the collection + * @param collection target collection + * @param iterator source * @return {@code true} if the target {@code collection} was modified as a * result of this operation */ diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Lists.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Lists.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Lists.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Lists.java 2018-05-02 20:17:13.000000000 +0000 @@ -31,7 +31,8 @@ /** * Creates a new {@link ArrayList}. - * + * + * @param type of elements contained in new list * @return a new {@link ArrayList} */ public static ArrayList newArrayList() { @@ -41,9 +42,10 @@ /** * Creates a new {@link ArrayList} filled with the contents of the given * {@code iterator}. - * + * * @param iterator * the source iterator + * @param type of elements contained in new list * @return a new {@link ArrayList} */ public static ArrayList newArrayList(final Iterator iterator) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ServiceLoaderIterator.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ServiceLoaderIterator.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/ServiceLoaderIterator.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/ServiceLoaderIterator.java 2018-05-02 20:17:13.000000000 +0000 @@ -27,7 +27,7 @@ /** * Iterates all services for a given class through the standard * {@link ServiceLoader} mechanism. - * + * * @param * The service to load * @since 1.13 diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Sets.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Sets.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/Sets.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/Sets.java 2018-05-02 20:17:13.000000000 +0000 @@ -35,9 +35,10 @@ /** * Creates a new HashSet filled with the given elements - * + * * @param elements * the elements to fill the new set + * @param type of elements contained in new set * @return A new HasSet */ public static HashSet newHashSet(@SuppressWarnings("unchecked") E... elements) { diff -Nru libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/SkipShieldingInputStream.java libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/SkipShieldingInputStream.java --- libcommons-compress-java-1.13/src/main/java/org/apache/commons/compress/utils/SkipShieldingInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/main/java/org/apache/commons/compress/utils/SkipShieldingInputStream.java 2018-07-08 08:35:13.000000000 +0000 @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.utils; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * A wrapper that overwrites {@link #skip} and delegates to {@link #read} instead. + * + *

    Some implementations of {@link InputStream} implement {@link + * InputStream#skip} in a way that throws an exception if the stream + * is not seekable - {@link System#in System.in} is known to behave + * that way. For such a stream it is impossible to invoke skip at all + * and you have to read from the stream (and discard the data read) + * instead. Skipping is potentially much faster than reading so we do + * want to invoke {@code skip} when possible. We provide this class so + * you can wrap your own {@link InputStream} in it if you encounter + * problems with {@code skip} throwing an excpetion.

    + * + * @since 1.17 + */ +public class SkipShieldingInputStream extends FilterInputStream { + private static final int SKIP_BUFFER_SIZE = 8192; + // we can use a shared buffer as the content is discarded anyway + private static final byte[] SKIP_BUFFER = new byte[SKIP_BUFFER_SIZE]; + public SkipShieldingInputStream(InputStream in) { + super(in); + } + + @Override + public long skip(long n) throws IOException { + return n < 0 ? 0 : read(SKIP_BUFFER, 0, (int) Math.min(n, SKIP_BUFFER_SIZE)); + } +} diff -Nru libcommons-compress-java-1.13/src/site/site.xml libcommons-compress-java-1.18/src/site/site.xml --- libcommons-compress-java-1.13/src/site/site.xml 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/site.xml 2018-08-13 07:13:46.000000000 +0000 @@ -28,24 +28,32 @@ - + - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + diff -Nru libcommons-compress-java-1.13/src/site/xdoc/conventions.xml libcommons-compress-java-1.18/src/site/xdoc/conventions.xml --- libcommons-compress-java-1.13/src/site/xdoc/conventions.xml 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/conventions.xml 2018-06-07 19:11:34.000000000 +0000 @@ -35,7 +35,7 @@

    We use some of the annotations from JCIP - as JavaDoc tags. The used tags are: + as Javadoc tags. The used tags are:

    • @GuardedBy (field or method)
      • diff -Nru libcommons-compress-java-1.13/src/site/xdoc/download_compress.xml libcommons-compress-java-1.18/src/site/xdoc/download_compress.xml --- libcommons-compress-java-1.13/src/site/xdoc/download_compress.xml 2016-12-25 12:04:20.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/download_compress.xml 2018-08-13 07:13:46.000000000 +0000 @@ -64,7 +64,7 @@

      We recommend you use a mirror to download our release - builds, but you must verify the integrity of + builds, but you must verify the integrity of the downloaded files using signatures downloaded from our main distribution directories. Recent releases (48 hours) may not yet be available from all the mirrors. @@ -102,7 +102,7 @@ It is essential that you verify the integrity of downloaded files, preferably using the PGP signature (*.asc files); - failing that using the MD5 hash (*.md5 checksum files). + failing that using the SHA256 hash (*.sha256 checksum files).

      The KEYS @@ -111,32 +111,32 @@

      -
      +
      - - - + + + - - - + + +
      commons-compress-1.13-bin.tar.gzmd5pgpcommons-compress-1.18-bin.tar.gzsha256pgp
      commons-compress-1.13-bin.zipmd5pgpcommons-compress-1.18-bin.zipsha256pgp
      - - - + + + - - - + + +
      commons-compress-1.13-src.tar.gzmd5pgpcommons-compress-1.18-src.tar.gzsha256pgp
      commons-compress-1.13-src.zipmd5pgpcommons-compress-1.18-src.zipsha256pgp
      diff -Nru libcommons-compress-java-1.13/src/site/xdoc/examples.xml libcommons-compress-java-1.18/src/site/xdoc/examples.xml --- libcommons-compress-java-1.13/src/site/xdoc/examples.xml 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/examples.xml 2018-07-01 17:45:34.000000000 +0000 @@ -19,11 +19,11 @@ --> - Commons Compress Examples + Commons Compress User Guide Commons Documentation Team -
      +

      Commons Compress calls all formats that compress a single @@ -32,18 +32,18 @@ compressed) archive are archiver formats.

      The compressor formats supported are gzip, bzip2, xz, lzma, - Pack200, DEFLATE and Z, the archiver formats are 7z, ar, arj, + Pack200, DEFLATE, Brotli, DEFLATE64, ZStandard and Z, the archiver formats are 7z, ar, arj, cpio, dump, tar and zip. Pack200 is a special case as it can only compress JAR files.

      We currently only provide read support for arj, - dump and Z. arj can only read uncompressed archives, 7z can read + dump, Brotli, DEFLATE64 and Z. arj can only read uncompressed archives, 7z can read archives with many compression and encryption algorithms supported by 7z but doesn't support encryption when writing archives.

       - +

      The stream classes all wrap around streams provided by the calling code and they work on them directly without any additional buffering. On the other hand most of them will @@ -51,6 +51,7 @@ users wrap their stream in Buffered(In|Out)putStreams before using the Commons Compress API.

      +       @@ -81,7 +82,7 @@ .createCompressorInputStream(originalInput); ]]> -

      Note that there is no way to detect the lzma format so only +

      Note that there is no way to detect the lzma or Brotli formats so only the two-arg version of createCompressorInputStream can be used. Prior to Compress 1.9 the .Z format hasn't been auto-detected @@ -89,6 +90,36 @@ + +

      Starting with Compress 1.14 + CompressorStreamFactory has an optional + constructor argument that can be used to set an upper limit of + memory that may be used while decompressing or compressing a + stream. As of 1.14 this setting only affects decompressing Z, + XZ and LZMA compressed streams.

      +

      For the Snappy and LZ4 formats the amount of memory used + during compression is directly proportional to the window + size.

      +       + + +

      Starting with Compress 1.17 most of the + CompressorInputStream implementations as well as + ZipArchiveInputStream and all streams returned by + ZipFile.getInputStream implement the + InputStreamStatistics + interface. SevenZFile provides statistics for the + current entry via the + getStatisticsForCurrentEntry method. This + interface can be used to track progress while extracting a + stream or to detect potential zip bombs + when the compression ration becomes suspiciously large.

      +       + +
      +
      +

      Many of the supported formats have developed different dialects and extensions and some formats allow for features @@ -103,15 +134,278 @@ - -

      For the bzip2, gzip and xz formats a single compressed file - may actually consist of several streams that will be - concatenated by the command line utilities when decompressing - them. Starting with Commons Compress 1.4 the - *CompressorInputStreams for these formats support - concatenating streams as well, but they won't do so by - default. You must use the two-arg constructor and explicitly - enable the support.

      + +

      All archive formats provide meta data about the individual + archive entries via instances of ArchiveEntry (or + rather subclasses of it). When reading from an archive the + information provided the getName method is the + raw name as stored inside of the archive. There is no + guarantee the name represents a relative file name or even a + valid file name on your target operating system at all. You + should double check the outcome when you try to create file + names from entry names.

      +       + + +

      Apart from 7z all formats provide a subclass of + ArchiveInputStream that can be used to create an + archive. For 7z SevenZFile provides a similar API + that does not represent a stream as our implementation + requires random access to the input and cannot be used for + general streams. The ZIP implementation can benefit a lot from + random access as well, see the zip + page for details.

      + +

      Assuming you want to extract an archive to a target + directory you'd call getNextEntry, verify the + entry can be read, construct a sane file name from the entry's + name, create a File and write all contents to + it - here IOUtils.copy may come handy. You do so + for every entry until getNextEntry returns + null.

      + +

      A skeleton might look like:

      + + + +

      where the hypothetical fileName method is + written by you and provides the absolute name for the file + that is going to be written on disk. Here you should perform + checks that ensure the resulting file name actually is a valid + file name on your operating system or belongs to a file inside + of targetDir when using the entry's name as + input.

      + +

      If you want to combine an archive format with a compression + format - like when reading a "tar.gz" file - you wrap the + ArchiveInputStream around + CompressorInputStream for example:

      + + + +       + + +

      Apart from 7z all formats that support writing provide a + subclass of ArchiveOutputStream that can be used + to create an archive. For 7z SevenZOutputFile + provides a similar API that does not represent a stream as our + implementation requires random access to the output and cannot + be used for general streams. The + ZipArchiveOutputStream class will benefit from + random access as well but can be used for non-seekable streams + - but not all features will be available and the archive size + might be slightly bigger, see the zip page for + details.

      + +

      Assuming you want to add a collection of files to an + archive, you can first use createArchiveEntry for + each file. In general this will set a few flags (usually the + last modified time, the size and the information whether this + is a file or directory) based on the File + instance. Alternatively you can create the + ArchiveEntry subclass corresponding to your + format directly. Often you may want to set additional flags + like file permissions or owner information before adding the + entry to the archive.

      + +

      Next you use putArchiveEntry in order to add + the entry and then start using write to add the + content of the entry - here IOUtils.copy may + come handy. Finally you invoke + closeArchiveEntry once you've written all content + and before you add the next entry.

      + +

      Once all entries have been added you'd invoke + finish and finally close the + stream.

      + +

      A skeleton might look like:

      + + filesToArchive = ... +try (ArchiveOutputStream o = ... create the stream for your format ...) { + for (File f : filesToArchive) { + // maybe skip directories for formats like AR that don't store directories + ArchiveEntry entry = o.createArchiveEntry(f, entryName(f)); + // potentially add more flags to entry + o.putArchiveEntry(entry); + if (f.isFile()) { + try (InputStream i = Files.newInputStream(f.toPath())) { + IOUtils.copy(i, o); + } + } + o.closeArchiveEntry(); + } + out.finish(); +} +]]> + +

      where the hypothetical entryName method is + written by you and provides the name for the entry as it is + going to be written to the archive.

      + +

      If you want to combine an archive format with a compression + format - like when creating a "tar.gz" file - you wrap the + ArchiveOutputStream around a + CompressorOutputStream for example:

      + + + +       + + + +

      Note that Commons Compress currently only supports a subset + of compression and encryption algorithms used for 7z archives. + For writing only uncompressed entries, LZMA, LZMA2, BZIP2 and + Deflate are supported - in addition to those reading supports + AES-256/SHA-256 and DEFLATE64.

      + +

      Multipart archives are not supported at all.

      + +

      7z archives can use multiple compression and encryption + methods as well as filters combined as a pipeline of methods + for its entries. Prior to Compress 1.8 you could only specify + a single method when creating archives - reading archives + using more than one method has been possible before. Starting + with Compress 1.8 it is possible to configure the full + pipeline using the setContentMethods method of + SevenZOutputFile. Methods are specified in the + order they appear inside the pipeline when creating the + archive, you can also specify certain parameters for some of + the methods - see the Javadocs of + SevenZMethodConfiguration for details.

      + +

      When reading entries from an archive the + getContentMethods method of + SevenZArchiveEntry will properly represent the + compression/encryption/filter methods but may fail to + determine the configuration options used. As of Compress 1.8 + only the dictionary size used for LZMA2 can be read.

      + +

      Currently solid compression - compressing multiple files + as a single block to benefit from patterns repeating accross + files - is only supported when reading archives. This also + means compression ratio will likely be worse when using + Commons Compress compared to the native 7z executable.

      + +

      Reading or writing requires a + SeekableByteChannel that will be obtained + transparently when reading from or writing to a file. The + class + org.apache.commons.compress.utils.SeekableInMemoryByteChannel + allows you to read from or write to an in-memory archive.

      + +

      Adding an entry to a 7z archive:

      + + +

      Uncompressing a given 7z archive (you would + certainly add exception handling and make sure all streams + get closed properly):

      + + +

      Uncompressing a given in-memory 7z archive:

      + + +

      Encrypted 7z Archives

      + +

      Currently Compress supports reading but not writing of + encrypted archives. When reading an encrypted archive a + password has to be provided to one of + SevenZFile's constructors. If you try to read + an encrypted archive without specifying a password a + PasswordRequiredException (a subclass of + IOException) will be thrown.

      + +

      When specifying the password as a byte[] one + common mistake is to use the wrong encoding when creating + the byte[] from a String. The + SevenZFile class expects the bytes to + correspond to the UTF16-LE encoding of the password. An + example of reading an encrypted archive is

      + + + +

      Starting with Compress 1.17 new constructors have been + added that accept the password as char[] rather + than a byte[]. We recommend you use these in + order to avoid the problem above.

      + + +       @@ -148,6 +442,31 @@ longer than 16 characters using the BSD dialect, writing the SVR4/GNU dialect is not supported.

      + + + + + + + + + + + + + + + + + + + + + + + +
      Version of Apache Commons CompressSupport for Traditional AR FormatSupport for GNU/SRV4 DialectSupport for BSD Dialect
      1.0 to 1.2read/writeread-
      1.3 and laterread/writereadread/write
      +

      It is not possible to detect the end of an AR archive in a reliable way so ArArchiveInputStream will read until it reaches the end of the stream or fails to parse the @@ -155,6 +474,23 @@ + + +

      Note that Commons Compress doesn't support compressed, + encrypted or multi-volume ARJ archives, yet.

      + +

      Uncompressing a given arj archive (you would + certainly add exception handling and make sure all streams + get closed properly):

      + +       +

      In addition to the information stored @@ -195,6 +531,49 @@ + +

      In general, JAR archives are ZIP files, so the JAR package + supports all options provided by the ZIP package.

      + +

      To be interoperable JAR archives should always be created + using the UTF-8 encoding for file names (which is the + default).

      + +

      Archives created using JarArchiveOutputStream + will implicitly add a JarMarker extra field to + the very first archive entry of the archive which will make + Solaris recognize them as Java archives and allows them to + be used as executables.

      + +

      Note that ArchiveStreamFactory doesn't + distinguish ZIP archives from JAR archives, so if you use + the one-argument createArchiveInputStream + method on a JAR archive, it will still return the more + generic ZipArchiveInputStream.

      + +

      The JarArchiveEntry class contains fields for + certificates and attributes that are planned to be supported + in the future but are not supported as of Compress 1.0.

      + +

      Adding an entry to a jar archive:

      + + +

      Reading entries from an jar archive:

      + +       +

      In addition to the information stored @@ -332,46 +711,41 @@ - -

      In general, JAR archives are ZIP files, so the JAR package - supports all options provided by the ZIP package.

      - -

      To be interoperable JAR archives should always be created - using the UTF-8 encoding for file names (which is the - default).

      - -

      Archives created using JarArchiveOutputStream - will implicitly add a JarMarker extra field to - the very first archive entry of the archive which will make - Solaris recognize them as Java archives and allows them to - be used as executables.

      +
      +
      -

      Note that ArchiveStreamFactory doesn't - distinguish ZIP archives from JAR archives, so if you use - the one-argument createArchiveInputStream - method on a JAR archive, it will still return the more - generic ZipArchiveInputStream.

      + +

      For the bzip2, gzip and xz formats as well as the framed + lz4 format a single compressed file + may actually consist of several streams that will be + concatenated by the command line utilities when decompressing + them. Starting with Commons Compress 1.4 the + *CompressorInputStreams for these formats support + concatenating streams as well, but they won't do so by + default. You must use the two-arg constructor and explicitly + enable the support.

      +       -

      The JarArchiveEntry class contains fields for - certificates and attributes that are planned to be supported - in the future but are not supported as of Compress 1.0.

      + -

      Adding an entry to a jar archive:

      - +

      The implementation of this package is provided by the + Google Brotli dec library.

      -

      Reading entries from an jar archive:

      +

      Uncompressing a given Brotli compressed file (you would + certainly add exception handling and make sure all streams + get closed properly):

       @@ -379,17 +753,17 @@

      Note that BZipCompressorOutputStream keeps hold of some big data structures in memory. While it is - true recommended for any stream that you close it as soon as - you no longer needed, this is even more important + recommended for any stream that you close it as soon as + you no longer need it, this is even more important for BZipCompressorOutputStream.

      Uncompressing a given bzip2 compressed file (you would certainly add exception handling and make sure all streams get closed properly):

      +

      Compressing a given file using bzip2 (you would + certainly add exception handling and make sure all streams + get closed properly):

      + + - +

      The implementation of the DEFLATE/INFLATE code used by this package is provided by the java.util.zip package of the Java class library.

      -

      Uncompressing a given gzip compressed file (you would +

      Uncompressing a given DEFLATE compressed file (you would certainly add exception handling and make sure all streams get closed properly):

      -       - +

      Compressing a given file using DEFLATE (you would + certainly add exception handling and make sure all streams + get closed properly):

      + -

      The Pack200 package has a dedicated - documentation page.

      +       -

      The implementation of this package is provided by - the java.util.zip package of the Java class - library.

      + -

      Uncompressing a given pack200 compressed file (you would +

      Uncompressing a given DEFLATE64 compressed file (you would certainly add exception handling and make sure all streams get closed properly):

      -       - + -

      The implementation of this package is provided by the - public domain XZ - for Java library.

      + -

      When you try to open an XZ stream for reading using - CompressorStreamFactory, Commons Compress will - check whether the XZ for Java library is available. Starting - with Compress 1.9 the result of this check will be cached - unless Compress finds OSGi classes in its classpath. You can - use XZUtils#setCacheXZAvailability to overrride - this default behavior.

      +

      The implementation of the DEFLATE/INFLATE code used by this + package is provided by the java.util.zip package + of the Java class library.

      -

      Uncompressing a given XZ compressed file (you would +

      Uncompressing a given gzip compressed file (you would certainly add exception handling and make sure all streams get closed properly):

      + +

      Compressing a given file using gzip (you would + certainly add exception handling and make sure all streams + get closed properly):

      + +       - - -

      Uncompressing a given Z compressed file (you would + + +

      There are two different "formats" used for lz4. The format called + "block format" only contains the raw compressed data while the + other provides a higher level "frame format" - Commons + Compress offers two different stream classes for reading or + writing either format.

      + +

      Uncompressing a given frame LZ4 file (you would certainly add exception handling and make sure all streams get closed properly):

      +

      Compressing a given file using the LZ4 frame format (you would + certainly add exception handling and make sure all streams + get closed properly):

      + +

      The implementation of this package is provided by the - public domain XZ + public domain XZ for Java library.

      Uncompressing a given lzma compressed file (you would certainly add exception handling and make sure all streams get closed properly):

      -       - - -

      The implementation of the DEFLATE/INFLATE code used by this - package is provided by the java.util.zip package - of the Java class library.

      - -

      Uncompressing a given DEFLATE compressed file (you would +

      Compressing a given file using lzma (you would certainly add exception handling and make sure all streams get closed properly):

      -       - - - -

      Note that Commons Compress currently only supports a subset - of compression and encryption algorithms used for 7z archives. - For writing only uncompressed entries, LZMA, LZMA2, BZIP2 and - Deflate are supported - reading also supports - AES-256/SHA-256.

      -

      Multipart archives are not supported at all.

      - -

      7z archives can use multiple compression and encryption - methods as well as filters combined as a pipeline of methods - for its entries. Prior to Compress 1.8 you could only specify - a single method when creating archives - reading archives - using more than one method has been possible before. Starting - with Compress 1.8 it is possible to configure the full - pipeline using the setContentMethods method of - SevenZOutputFile. Methods are specified in the - order they appear inside the pipeline when creating the - archive, you can also specify certain parameters for some of - the methods - see the Javadocs of - SevenZMethodConfiguration for details.

      - -

      When reading entries from an archive the - getContentMethods method of - SevenZArchiveEntry will properly represent the - compression/encryption/filter methods but may fail to - determine the configuration options used. As of Compress 1.8 - only the dictionary size used for LZMA2 can be read.

      +       -

      Currently solid compression - compressing multiple files - as a single block to benefit from patterns repeating accross - files - is only supported when reading archives. This also - means compression ratio will likely be worse when using - Commons Compress compared to the native 7z executable.

      + -

      Reading or writing requires a - SeekableByteChannel that will be obtained - transparently when reading from or writing to a file. The - class - org.apache.commons.compress.utils.SeekableInMemoryByteChannel - allows you to read from or write to an in-memory archive.

      +

      The Pack200 package has a dedicated + documentation page.

      -

      Adding an entry to a 7z archive:

      - +

      The implementation of this package is provided by + the java.util.zip package of the Java class + library.

      -

      Uncompressing a given 7z archive (you would +

      Uncompressing a given pack200 compressed file (you would certainly add exception handling and make sure all streams get closed properly):

      -

      Uncompressing a given in-memory 7z archive:

      - -       - - - -

      Note that Commons Compress doesn't support compressed, - encrypted or multi-volume ARJ archives, yet.

      - -

      Uncompressing a given arj archive (you would +

      Compressing a given jar using pack200 (you would certainly add exception handling and make sure all streams get closed properly):

      +

      There are two different "formats" used for Snappy, one only + href="https://github.com/google/snappy/">Snappy, one only contains the raw compressed data while the other provides a higher level "framing format" - Commons Compress offers two different stream classes for reading either format.

      @@ -667,9 +1052,9 @@ certainly add exception handling and make sure all streams get closed properly):

      +

      Compressing a given file using framed Snappy (you would + certainly add exception handling and make sure all streams + get closed properly):

      + + +       + + + +

      The implementation of this package is provided by the + public domain XZ + for Java library.

      + +

      When you try to open an XZ stream for reading using + CompressorStreamFactory, Commons Compress will + check whether the XZ for Java library is available. Starting + with Compress 1.9 the result of this check will be cached + unless Compress finds OSGi classes in its classpath. You can + use XZUtils#setCacheXZAvailability to overrride + this default behavior.

      + +

      Uncompressing a given XZ compressed file (you would + certainly add exception handling and make sure all streams + get closed properly):

      + + +

      Compressing a given file using XZ (you would + certainly add exception handling and make sure all streams + get closed properly):

      + + +       + + + +

      Uncompressing a given Z compressed file (you would + certainly add exception handling and make sure all streams + get closed properly):

      + +       - - + + +

      The implementation of this package is provided by the + Zstandard JNI library.

      + +

      Uncompressing a given Zstandard compressed file (you would + certainly add exception handling and make sure all streams + get closed properly):

      + + +

      Compressing a given file using the Zstandard format (you + would certainly add exception handling and make sure all + streams get closed properly):

      + + +       +
      + +
      +

      Starting in release 1.13, it is now possible to add Compressor- and ArchiverStream implementations using the Java's ServiceLoader mechanism.

      - - + - +

      To provide your own compressor, you must make available on the classpath a file called META-INF/services/org.apache.commons.compress.compressors.CompressorStreamProvider. @@ -709,7 +1223,7 @@ org.apache.commons.compress.compressors.CompressorStreamProvider.

      - +

      @@ -728,7 +1242,7 @@ org.apache.commons.compress.archivers.ArchiveStreamProvider.

      - +
      diff -Nru libcommons-compress-java-1.13/src/site/xdoc/index.xml libcommons-compress-java-1.18/src/site/xdoc/index.xml --- libcommons-compress-java-1.13/src/site/xdoc/index.xml 2016-12-25 12:11:24.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/index.xml 2018-08-13 07:13:46.000000000 +0000 @@ -26,8 +26,9 @@

      The Apache Commons Compress library defines an API for - working with ar, cpio, Unix dump, tar, zip, gzip, XZ, Pack200, - bzip2, 7z, arj, lzma, snappy, DEFLATE and Z files. + working with ar, cpio, Unix dump, tar, zip, gzip, XZ, + Pack200, bzip2, 7z, arj, lzma, snappy, DEFLATE, lz4, + Brotli, Zstandard, DEFLATE64 and Z files.

      The code in this component has many origins: @@ -51,31 +52,22 @@

      -

      The current release is 1.13 and requires Java 7.

      +

      The current release is 1.18 and requires Java 7.

      Below we highlight some new features, for a full list of changes see the Changes Report.

      - +
        -
      • The 7z package as well as - ZipArchiveOutputStream and - ZipFile can now use - SeekableByteChannel when random acces is - needed. This allows archives to be read from inputs and - written to outputs that are seekable but are not - represented by Files.
        • -
      • It is now possible to add Compressor- and - ArchiverStream implementations using the JDK's - ServiceLoader mechanism. Please see - Extending Commons Compress.
        • -
      • Added support for writing the legacy LZMA format as - compressor stream and inside 7z archives - this requires - XZ for Java 1.6.
        • +
      • The CPIO package now properly handles file names + using a mult-byte encoding.
        • +
      • ZipArchiveInputStream can now deal with APK files + containing an APK signing block.
        • +
      • It is no possible to specifiy various parameters + for Zstd output.
       -
      @@ -87,23 +79,28 @@ by ArchiveEntry instances which in turn usually correspond to single files or directories.

      -

      Currently the bzip2, Pack200, XZ, gzip, lzma and Z formats are +

      Currently the bzip2, Pack200, XZ, gzip, lzma, brotli, + Zstandard and Z formats are supported as compressors where gzip support is mostly provided by the java.util.zip package and Pack200 support by the java.util.jar package of the Java class library. XZ and lzma support is provided by the public - domain XZ for - Java library. As of Commons Compress 1.13 support for - the Z and Snappy formats is read-only.

      + domain XZ for + Java library. Brotli support is provided by the MIT + licensed Google + Brotli decoder. Zstandard support is provided by the BSD + licensed Zstd-jni. + As of Commons Compress 1.18 support for the DEFLATE64, Z and Brotli + formats is read-only.

      The ar, arj, cpio, dump, tar, 7z and zip formats are supported as archivers where the zip implementation provides capabilities that go beyond the features found in java.util.zip. As of Commons Compress - 1.13 support for the dump and arj formats is + 1.18 support for the dump and arj formats is read-only - 7z can read most compressed and encrypted archives but only write unencrypted ones. LZMA(2) support - in 7z requires XZ for + in 7z requires XZ for Java as well.

      The compress component provides abstract base classes for @@ -114,7 +111,7 @@ implementation.

      - Download now! + Download now!

      diff -Nru libcommons-compress-java-1.13/src/site/xdoc/issue-tracking.xml libcommons-compress-java-1.18/src/site/xdoc/issue-tracking.xml --- libcommons-compress-java-1.13/src/site/xdoc/issue-tracking.xml 2016-12-25 12:06:52.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/issue-tracking.xml 2018-08-13 07:05:13.000000000 +0000 @@ -86,7 +86,7 @@

      For more information on subversion and creating patches see the - Apache Contributors Guide. + Apache Contributors Guide.

      diff -Nru libcommons-compress-java-1.13/src/site/xdoc/limitations.xml libcommons-compress-java-1.18/src/site/xdoc/limitations.xml --- libcommons-compress-java-1.13/src/site/xdoc/limitations.xml 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/limitations.xml 2018-05-23 12:50:54.000000000 +0000 @@ -25,10 +25,25 @@ Commons Compress™ grouped by the archiving/compression format they apply to.

      +
      +
        +
      • Several implementations of decompressors and unarchivers will + invoke skip + on the underlying InputStream which may throw an + IOException in some stream implementations. One + known case where this happens is when using + System.in as input. If you encounter an + exception with a message like "Illegal seek" we recommend you + wrap your stream in a SkipShieldingInputStream + from our utils package before passing it to Compress.
        • +
      +
      +
      • the format requires the otherwise optional XZ for Java + href="https://tukaani.org/xz/java.html">XZ for Java library.
      • only Files are supported as input/output, not streams. Starting with Compress 1.13 @@ -48,7 +63,7 @@
      • Support for some BCJ filters and the DELTA filter has been added with Compress 1.8. Because of a known bug in version 1.4 of the XZ for Java + href="https://tukaani.org/xz/java.html">XZ for Java library, archives using BCJ filters will cause an AssertionError when read. If you need support for BCJ filters you must use XZ for Java 1.5 or later.
        • @@ -71,6 +86,16 @@ archives
      +
      +
        +
      • the format requires the otherwise optional Google Brotli dec + library.
        • +
      • read-only support
        • +
      • CompressorStreamFactory is not able to auto-detect + streams using Brotli compression.
        • +
      +

      Versions of Compress prior to 1.4.1 are vulnerable to a possible denial of service attack, see the

    +
    +
      +
    • CompressorStreamFactory is not able to auto-detect + streams using DEFLATE64 compression.
      • +
    • read-only support
      • +
    +
    • read-only support
      • @@ -106,10 +138,18 @@ MANIFEST
    +
    +
      +
    • In theory LZ4 compressed streams can contain literals and + copies of arbitrary length while Commons Compress only + supports sizes up to 263 - 1 (i.e. ≈ 9.2 + EB).
      • +
    +
      -
    • read-only support
      • +
    • Commons Compress 1.13 and earlier only support reading + the format
    @@ -130,13 +171,13 @@ supported
  • In Compress 1.6 TarArchiveInputStream could fail to read the full contents of an entry unless the stream - was wrapped in a bufferng stream.
    • + was wrapped in a buffering stream.
  • the format requires the otherwise optional XZ for Java + href="https://tukaani.org/xz/java.html">XZ for Java library.
@@ -159,7 +200,7 @@ limitation of Compress' specific implementation.
  • only a subset of compression methods are supported, including the most common STORED and DEFLATEd. IMPLODE, - SHRINK and BZIP2 support is read-only.
    • + SHRINK, DEFLATE64 and BZIP2 support is read-only.
  • no support for encryption or multi-volume archives
  • In versions prior to Compress 1.6 ZipArchiveEntries read from an archive will @@ -173,6 +214,21 @@ ZipEntry#getTime under the covers which may return different times for the same archive when using different versions of Java.
    • +
  • In versions of Compress prior to 1.16 a specially crafted + ZIP archive can be used to cause an infinite loop inside of + Compress' extra field parser used by the ZipFile + and ZipArchiveInputStream classes. This can be + used to mount a denial of service attack against services + that use Compress' zip package. See the Security Reports page for + details.
    • + + +
    +
      +
    • the format requires the otherwise optional Zstandard JNI + library.
    diff -Nru libcommons-compress-java-1.13/src/site/xdoc/mail-lists.xml libcommons-compress-java-1.18/src/site/xdoc/mail-lists.xml --- libcommons-compress-java-1.13/src/site/xdoc/mail-lists.xml 2016-12-25 12:07:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/mail-lists.xml 2018-08-13 07:05:41.000000000 +0000 @@ -49,7 +49,7 @@

    Apache Commons Compress shares mailing lists with all the other - Commons Components. + Commons Components. To make it easier for people to only read messages related to components they are interested in, the convention in Commons is to prefix the subject line of messages with the component's name, for example: @@ -59,9 +59,9 @@

    Questions related to the usage of Apache Commons Compress should be posted to the - User List. + User List.
    - The Developer List + The Developer List is for questions and discussion related to the development of Apache Commons Compress.
    Please do not cross-post; developers are also subscribed to the user list. @@ -105,9 +105,9 @@ Subscribe Unsubscribe Post - mail-archives.apache.org - markmail.org
    - www.mail-archive.com
    + mail-archives.apache.org + markmail.org
    + www.mail-archive.com
    news.gmane.org @@ -123,9 +123,9 @@ Subscribe Unsubscribe Post - mail-archives.apache.org - markmail.org
    - www.mail-archive.com
    + mail-archives.apache.org + markmail.org
    + www.mail-archive.com
    news.gmane.org @@ -141,9 +141,9 @@ Subscribe Unsubscribe read only - mail-archives.apache.org - markmail.org
    - www.mail-archive.com + mail-archives.apache.org + markmail.org
    + www.mail-archive.com @@ -158,9 +158,9 @@ Subscribe Unsubscribe read only - mail-archives.apache.org - markmail.org
    - www.mail-archive.com + mail-archives.apache.org + markmail.org
    + www.mail-archive.com @@ -191,10 +191,10 @@ Subscribe Unsubscribe read only - mail-archives.apache.org - markmail.org
    + mail-archives.apache.org + markmail.org
    old.nabble.com
    - www.mail-archive.com
    + www.mail-archive.com
    news.gmane.org diff -Nru libcommons-compress-java-1.13/src/site/xdoc/pack200.xml libcommons-compress-java-1.18/src/site/xdoc/pack200.xml --- libcommons-compress-java-1.13/src/site/xdoc/pack200.xml 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/pack200.xml 2018-05-23 12:50:54.000000000 +0000 @@ -30,14 +30,14 @@ archives compressed with Pack200 will in general be different from the original archive when decompressed again. More information can be found in the Javadocs of the Pack200 + href="https://docs.oracle.com/javase/7/docs/api/java/util/jar/Pack200.Packer.html">Pack200.Packer class.

    While the pack200 command line utility of the JDK creates GZip compressed archives (.pack.gz) by default, the streams provided by the Pack200 package only perform the actual Pack200 operation. Wrap them in an - additional GZip(In|Out)putStream in order to deal + additional GzipCompressor(In|Out)putStream in order to deal with deflated streams.

    @@ -74,7 +74,7 @@ to "normalize" JAR archives prior to packing them that ensures signatures applied to the "normalized" JAR will still be valid aftre a pack/unpack cycle - see Pack200.Packer's + href="https://download.oracle.com/javase/7/docs/api/java/util/jar/Pack200.Packer.html">Pack200.Packer's javadocs.

    The Pack200Utils class in the diff -Nru libcommons-compress-java-1.13/src/site/xdoc/security-reports.xml libcommons-compress-java-1.18/src/site/xdoc/security-reports.xml --- libcommons-compress-java-1.13/src/site/xdoc/security-reports.xml 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/security-reports.xml 2018-05-23 12:50:54.000000000 +0000 @@ -25,7 +25,7 @@

    For information about reporting or asking questions about security problems, please see the security page + href="https://commons.apache.org/security.html">security page of the Commons project.

    @@ -54,9 +54,31 @@ the descriptions here are incomplete, please report them privately to the Apache Security Team. Thank you.

    + +

    Low: Denial of Service CVE-2018-1324

    + +

    A specially crafted ZIP archive can be used to cause an + infinite loop inside of Compress' extra field parser used by + the ZipFile and + ZipArchiveInputStream classes. This can be + used to mount a denial of service attack against services + that use Compress' zip package.

    + +

    This was fixed in revision 2a2f1dc4.

    + +

    This was first reported to the project's JIRA on 19 + December 2017.

    + +

    Affects: 1.11 - 1.15

    + +     +

    Low: Denial of Service CVE-2012-2098

    + href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2012-2098">CVE-2012-2098

    The bzip2 compressing streams in Apache Commons Compress internally use sorting algorithms with unacceptable @@ -68,18 +90,18 @@ service.

    This was fixed in revisions - 1332540, - 1332552, - 1333522, - 1337444, - 1340715, - 1340723, - 1340757, - 1340786, - 1340787, - 1340790, - 1340795 and - 1340799.

    + 1332540, + 1332552, + 1333522, + 1337444, + 1340715, + 1340723, + 1340757, + 1340786, + 1340787, + 1340790, + 1340795 and + 1340799.

    This was first reported to the Security Team on 12 April 2012 and made public on 23 May 2012.

    diff -Nru libcommons-compress-java-1.13/src/site/xdoc/tar.xml libcommons-compress-java-1.18/src/site/xdoc/tar.xml --- libcommons-compress-java-1.13/src/site/xdoc/tar.xml 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/tar.xml 2018-05-02 20:17:13.000000000 +0000 @@ -30,14 +30,16 @@ stores various attributes including information about the original owner and permissions.

    -

    There are several different tar formats and the TAR package - of Compress 1.4 mostly only provides the common functionality - of the existing variants.

    - -

    The original format (often called "ustar") didn't support - file names longer than 100 characters or bigger than 8 GiB and - the tar package will by default fail if you try to write an - entry that goes beyond those limits.

    +

    There are several different dialects of the TAR format, maybe + even different TAR formats. The tar package contains special + cases in order to read many of the existing dialects and will by + default try to create archives in the original format (often + called "ustar"). This original format didn't support file names + longer than 100 characters or bigger than 8 GiB and the tar + package will by default fail if you try to write an entry that + goes beyond those limits. "ustar" is the common denominator of + all the existing tar dialects and is understood by most of the + existing tools.

    The tar package does not support the full POSIX tar standard nor more modern GNU extension of said standard.

    @@ -151,6 +153,76 @@     + +

    The tar package has supported reading PAX extended headers + since 1.3 for local headers and 1.11 for global headers. The + following entries of PAX headers are applied when reading:

    + +
    +
    path
    +
    set the entry's name
    + +
    linkpath
    +
    set the entry's link name
    + +
    gid
    +
    set the entry's group id
    + +
    gname
    +
    set the entry's group name
    + +
    uid
    +
    set the entry's user id
    + +
    uname
    +
    set the entry's user name
    + +
    size
    +
    set the entry's size
    + +
    mtime
    +
    set the entry's modification time
    + +
    SCHILY.devminor
    +
    set the entry's minor device number
    + +
    SCHILY.devmajor
    +
    set the entry's major device number
    +
    + +

    in addition some fields used by GNU tar and star used to + signal sparse entries are supported and are used for the + is*GNUSparse and isStarSparse + methods.

    + +

    Some PAX extra headers may be set when writing archives, + for example for non-ASCII names or big numeric values. This + depends on various setting of the output stream - see the + previous sections.

    + +

    Since 1.15 you can directly access all PAX extension + headers that have been found when reading an entry or specify + extra headers to be written to a (local) PAX extended header + entry.

    + +

    Some hints if you try to set extended headers:

    + +
      +
    • pax header keywords should be ascii. star/gnutar + (SCHILY.xattr.* ) do not check for this. libarchive/bsdtar + (LIBARCHIVE.xattr.*) uses URL-Encoding.
      • +
    • pax header values should be encoded as UTF-8 characters + (including trailing \0). star/gnutar + (SCHILY.xattr.*) do not check for this. libarchive/bsdtar + (LIBARCHIVE.xattr.*) encode values using Base64.
      • +
    • libarchive/bsdtar will read SCHILY.xattr headers, but + will not generate them.
      • +
    • gnutar will complain about LIBARCHIVE.xattr (and any + other unknown) headers and will neither encode nor decode + them.
      • +
    +     +
    diff -Nru libcommons-compress-java-1.13/src/site/xdoc/zip.xml libcommons-compress-java-1.18/src/site/xdoc/zip.xml --- libcommons-compress-java-1.13/src/site/xdoc/zip.xml 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/site/xdoc/zip.xml 2018-06-15 18:00:32.000000000 +0000 @@ -105,7 +105,7 @@ over ZipArchiveInputStream.

    ZipFile requires a - SeekableByteChannel that will be obtain + SeekableByteChannel that will be obtained transparently when reading from a file. The class org.apache.commons.compress.utils.SeekableInMemoryByteChannel allows you to read from an in-memory archive.

    @@ -297,7 +297,8 @@ just fine. As of version 1.7, Commons Compress can also decompress entries compressed with the legacy SHRINK and IMPLODE algorithms of PKZIP 1.x. Version 1.11 of Commons - Compress adds read-only support for BZIP2.

    + Compress adds read-only support for BZIP2. Version 1.16 adds + read-only support for DEFLATE64 - also known as "enhanced DEFLATE".

    The ZIP specification allows for various other compression algorithms and also supports several different ways of @@ -312,6 +313,39 @@ mechanism. Using this method it is possible to at least detect and skip the entries that can not be extracted.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Version of Apache Commons CompressSupported Compression MethodsSupported Encryption Methods
    1.0 to 1.6STORED, DEFLATE-
    1.7 to 1.10STORED, DEFLATE, SHRINK, IMPLODE-
    1.11 to 1.15STORED, DEFLATE, SHRINK, IMPLODE, BZIP2-
    1.16 and laterSTORED, DEFLATE, SHRINK, IMPLODE, BZIP2, DEFLATE64 + (enhanced deflate)-
    + diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/ArchiveReadTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/ArchiveReadTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/ArchiveReadTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/ArchiveReadTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress; @@ -25,6 +25,7 @@ import java.io.File; import java.io.FileReader; import java.io.FilenameFilter; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; @@ -38,9 +39,9 @@ /** * Test that can read various archive file examples. - * + * * This is a very simple implementation. - * + * * Files must be in resources/archives, and there must be a file.txt containing * the list of files in the archives. */ @@ -48,9 +49,17 @@ public class ArchiveReadTest extends AbstractTestCase { private static final ClassLoader CLASSLOADER = ArchiveReadTest.class.getClassLoader(); - private static final File ARCDIR = new File(CLASSLOADER.getResource("archives").getFile()); + private static final File ARCDIR; private static final ArrayList FILELIST = new ArrayList<>(); + static { + try { + ARCDIR = new File(CLASSLOADER.getResource("archives").toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + private final File file; public ArchiveReadTest(final String file){ @@ -81,7 +90,7 @@ public boolean accept(final File dir, final String name) { return !name.endsWith(".txt"); } - })) + })) { params.add(new Object[] { f }); } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ar/ArArchiveInputStreamTest.java 2018-08-10 04:46:56.000000000 +0000 @@ -26,6 +26,7 @@ import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.utils.ArchiveUtils; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; public class ArArchiveInputStreamTest extends AbstractTestCase { @@ -58,4 +59,44 @@ assertNull(s.getNextEntry()); } } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + try (FileInputStream in = new FileInputStream(getFile("bla.ar")); + ArArchiveInputStream archive = new ArArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read()); + assertEquals(-1, archive.read()); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + byte[] buf = new byte[2]; + try (FileInputStream in = new FileInputStream(getFile("bla.ar")); + ArArchiveInputStream archive = new ArArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read(buf)); + assertEquals(-1, archive.read(buf)); + } + } + + @Test(expected=IllegalStateException.class) + public void cantReadWithoutOpeningAnEntry() throws Exception { + try (FileInputStream in = new FileInputStream(getFile("bla.ar")); + ArArchiveInputStream archive = new ArArchiveInputStream(in)) { + archive.read(); + } + } + + @Test(expected=IllegalStateException.class) + public void cantReadAfterClose() throws Exception { + try (FileInputStream in = new FileInputStream(getFile("bla.ar")); + ArArchiveInputStream archive = new ArArchiveInputStream(in)) { + archive.close(); + archive.read(); + } + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ArchiveOutputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ArchiveOutputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ArchiveOutputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ArchiveOutputStreamTest.java 2018-07-01 09:48:25.000000000 +0000 @@ -109,6 +109,7 @@ } catch (final IOException io) { // Exception expected } + finishTest.close(); } @Test @@ -175,12 +176,17 @@ fail("Should have raised IOException - close() called before closeArchiveEntry()"); } catch (final IOException expected) { } + + aos1 = createArchiveWithDummyEntry(archiveType, out1, dummy); aos1.closeArchiveEntry(); try { aos1.closeArchiveEntry(); fail("Should have raised IOException - closeArchiveEntry() called with no open entry"); } catch (final IOException expected) { } + + aos1 = createArchiveWithDummyEntry(archiveType, out1, dummy); + aos1.closeArchiveEntry(); aos1.finish(); aos1.close(); try { @@ -189,4 +195,14 @@ } catch (final IOException expected) { } } + + private ArchiveOutputStream createArchiveWithDummyEntry(String archiveType, OutputStream out1, File dummy) + throws Exception { + ArchiveOutputStream aos1 = factory.createArchiveOutputStream(archiveType, out1); + aos1.putArchiveEntry(aos1.createArchiveEntry(dummy, "dummy")); + try (InputStream is = new FileInputStream(dummy)) { + IOUtils.copy(is, aos1); + } + return aos1; + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ArchiveStreamFactoryTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ArchiveStreamFactoryTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ArchiveStreamFactoryTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ArchiveStreamFactoryTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -32,6 +32,7 @@ import java.io.InputStream; import java.lang.reflect.Field; +import org.apache.commons.compress.MockEvilInputStream; import org.apache.commons.compress.archivers.arj.ArjArchiveInputStream; import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream; import org.apache.commons.compress.archivers.dump.DumpArchiveInputStream; @@ -100,7 +101,7 @@ } /** - * Test case for + * Test case for * COMPRESS-267. */ @@ -117,7 +118,7 @@ } /** - * Test case for + * Test case for * COMPRESS-208. */ @@ -131,7 +132,7 @@ } } } - + @Test public void testEncodingCtor() { ArchiveStreamFactory fac = new ArchiveStreamFactory(); @@ -168,7 +169,7 @@ final String fieldName; final String type; final boolean hasOutputStream; - + TestData(final String testFile, final String type, final boolean hasOut, final String expectedEncoding, final ArchiveStreamFactory fac, final String fieldName) { this.testFile = testFile; this.expectedEncoding = expectedEncoding; @@ -177,7 +178,7 @@ this.type = type; this.hasOutputStream = hasOut; } - + @Override public String toString() { return "TestData [testFile=" + testFile + ", expectedEncoding=" + expectedEncoding + ", fac=" + fac @@ -226,6 +227,51 @@ DUMP_DEFAULT = dflt; } + @Test + public void testDetect() throws Exception { + for (String extension : new String[]{ + ArchiveStreamFactory.AR, + ArchiveStreamFactory.ARJ, + ArchiveStreamFactory.CPIO, + ArchiveStreamFactory.DUMP, + // Compress doesn't know how to detect JARs, see COMPRESS-91 + // ArchiveStreamFactory.JAR, + ArchiveStreamFactory.SEVEN_Z, + ArchiveStreamFactory.TAR, + ArchiveStreamFactory.ZIP + }) { + assertEquals(extension, detect("bla."+extension)); + } + + try { + ArchiveStreamFactory.detect(new BufferedInputStream(new ByteArrayInputStream(new byte[0]))); + fail("shouldn't be able to detect empty stream"); + } catch (ArchiveException e) { + assertEquals("No Archiver found for the stream signature", e.getMessage()); + } + + try { + ArchiveStreamFactory.detect(null); + fail("shouldn't be able to detect null stream"); + } catch (IllegalArgumentException e) { + assertEquals("Stream must not be null.", e.getMessage()); + } + + try { + ArchiveStreamFactory.detect(new BufferedInputStream(new MockEvilInputStream())); + fail("Expected ArchiveException"); + } catch (ArchiveException e) { + assertEquals("IOException while reading signature.", e.getMessage()); + } + } + + private String detect(String resource) throws IOException, ArchiveException { + try(InputStream in = new BufferedInputStream(new FileInputStream( + getFile(resource)))) { + return ArchiveStreamFactory.detect(in); + } + } + static final TestData[] TESTS = { new TestData("bla.arj", ArchiveStreamFactory.ARJ, false, ARJ_DEFAULT, FACTORY, "charsetName"), new TestData("bla.arj", ArchiveStreamFactory.ARJ, false, "UTF-8", FACTORY_UTF8, "charsetName"), @@ -342,7 +388,7 @@ } catch (final NoSuchFieldException e) { System.out.println("Cannot find " + name + " in class " + instance.getClass().getSimpleName()); return UNKNOWN; - } + } } final boolean isAccessible = fld.isAccessible(); try { diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStreamTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/arj/ArjArchiveInputStreamTest.java 2018-08-09 17:01:39.000000000 +0000 @@ -26,6 +26,8 @@ import java.util.TimeZone; import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; public class ArjArchiveInputStreamTest extends AbstractTestCase { @@ -81,4 +83,28 @@ assertEquals(cal.getTime(), entry.getLastModifiedDate()); in.close(); } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + try (FileInputStream in = new FileInputStream(getFile("bla.arj")); + ArjArchiveInputStream archive = new ArjArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read()); + assertEquals(-1, archive.read()); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + byte[] buf = new byte[2]; + try (FileInputStream in = new FileInputStream(getFile("bla.arj")); + ArjArchiveInputStream archive = new ArjArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read(buf)); + assertEquals(-1, archive.read(buf)); + } + } + } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/arj/CoverageTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/arj/CoverageTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/arj/CoverageTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/arj/CoverageTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.arj; + +import static org.junit.Assert.assertNotNull; + +import org.apache.commons.compress.archivers.arj.ArjArchiveEntry.HostOs; +import org.junit.Test; + +public class CoverageTest { + + @Test + public void testHostOsInstance() { + HostOs hostOs = new HostOs(); + assertNotNull(hostOs); + } + @Test + public void testHeaderInstances() { + assertNotNull(new LocalFileHeader.FileTypes()); + assertNotNull(new LocalFileHeader.Methods()); + assertNotNull(new LocalFileHeader.Flags()); + assertNotNull(new MainHeader.Flags()); + } + @Test + public void testCallLFHToString() { + LocalFileHeader lfh = new LocalFileHeader(); + assertNotNull(lfh.toString()); + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStreamTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveInputStreamTest.java 2018-08-09 17:40:44.000000000 +0000 @@ -23,6 +23,8 @@ import java.io.FileInputStream; import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; public class CpioArchiveInputStreamTest extends AbstractTestCase { @@ -65,4 +67,44 @@ assertEquals(count, 1); } + + @Test + public void testCpioUnarchiveMultibyteCharName() throws Exception { + final CpioArchiveInputStream in = + new CpioArchiveInputStream(new FileInputStream(getFile("COMPRESS-459.cpio")), "UTF-8"); + CpioArchiveEntry entry= null; + + int count = 0; + while ((entry = (CpioArchiveEntry) in.getNextEntry()) != null) { + count++; + assertNotNull(entry); + } + in.close(); + + assertEquals(2, count); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + try (FileInputStream in = new FileInputStream(getFile("bla.cpio")); + CpioArchiveInputStream archive = new CpioArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read()); + assertEquals(-1, archive.read()); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + byte[] buf = new byte[2]; + try (FileInputStream in = new FileInputStream(getFile("bla.cpio")); + CpioArchiveInputStream archive = new CpioArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read(buf)); + assertEquals(-1, archive.read(buf)); + } + } + } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/cpio/CpioArchiveTest.java 2018-07-11 16:41:35.000000000 +0000 @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.cpio; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.Arrays; +import java.util.Collection; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class CpioArchiveTest { + + @Parameters(name = "using {0}") + public static Collection factory() { + return Arrays.asList(new Object[][] { + new Object[] { CpioConstants.FORMAT_NEW }, + new Object[] { CpioConstants.FORMAT_NEW_CRC }, + new Object[] { CpioConstants.FORMAT_OLD_ASCII }, + new Object[] { CpioConstants.FORMAT_OLD_BINARY }, + }); + } + + private final short format; + + public CpioArchiveTest(short format) { + this.format = format; + } + + @Test + public void utf18RoundtripTest() throws Exception { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + try (CpioArchiveOutputStream os = new CpioArchiveOutputStream(baos, format, CpioConstants.BLOCK_SIZE, + "UTF-16LE")) { + CpioArchiveEntry entry = new CpioArchiveEntry(format, "T\u00e4st.txt", 4); + if (format == CpioConstants.FORMAT_NEW_CRC) { + entry.setChksum(10); + } + os.putArchiveEntry(entry); + os.write(new byte[] { 1, 2, 3, 4 }); + os.closeArchiveEntry(); + } + baos.close(); + try (ByteArrayInputStream bin = new ByteArrayInputStream(baos.toByteArray()); + CpioArchiveInputStream in = new CpioArchiveInputStream(bin, "UTF-16LE")) { + CpioArchiveEntry entry = (CpioArchiveEntry) in.getNextEntry(); + Assert.assertNotNull(entry); + Assert.assertEquals("T\u00e4st.txt", entry.getName()); + Assert.assertArrayEquals(new byte[] { 1, 2, 3, 4 }, IOUtils.toByteArray(in)); + } + } + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/cpio/CpioUtilTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/cpio/CpioUtilTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/cpio/CpioUtilTest.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/cpio/CpioUtilTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -18,11 +18,11 @@ */ package org.apache.commons.compress.archivers.cpio; +import org.junit.Test; + import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import org.junit.Test; - public class CpioUtilTest { @Test @@ -53,4 +53,31 @@ true)); } + + @Test(expected = UnsupportedOperationException.class) + public void testLong2byteArrayWithZeroThrowsUnsupportedOperationException() { + + CpioUtil.long2byteArray(0L, 0, false); + + } + + + @Test(expected = UnsupportedOperationException.class) + public void testLong2byteArrayWithPositiveThrowsUnsupportedOperationException() { + + CpioUtil.long2byteArray(0L, 1021, false); + + } + + + @Test(expected = UnsupportedOperationException.class) + public void testByteArray2longThrowsUnsupportedOperationException() { + + byte[] byteArray = new byte[1]; + + CpioUtil.byteArray2long(byteArray, true); + + } + + } \ No newline at end of file diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/dump/DumpArchiveInputStreamTest.java 2018-08-09 17:44:16.000000000 +0000 @@ -24,7 +24,9 @@ import java.io.InputStream; import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; public class DumpArchiveInputStreamTest extends AbstractTestCase { @@ -32,7 +34,7 @@ @Test public void testNotADumpArchive() throws Exception { try (FileInputStream is = new FileInputStream(getFile("bla.zip"))) { - new DumpArchiveInputStream(is); + new DumpArchiveInputStream(is).close(); fail("expected an exception"); } catch (final ArchiveException ex) { // expected @@ -43,7 +45,7 @@ @Test public void testNotADumpArchiveButBigEnough() throws Exception { try (FileInputStream is = new FileInputStream(getFile("zip64support.tar.bz2"))) { - new DumpArchiveInputStream(is); + new DumpArchiveInputStream(is).close(); fail("expected an exception"); } catch (final ArchiveException ex) { // expected @@ -68,4 +70,27 @@ dump.close(); } + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + try (FileInputStream in = new FileInputStream(getFile("bla.dump")); + DumpArchiveInputStream archive = new DumpArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read()); + assertEquals(-1, archive.read()); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + byte[] buf = new byte[2]; + try (FileInputStream in = new FileInputStream(getFile("bla.dump")); + DumpArchiveInputStream archive = new DumpArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read(buf)); + assertEquals(-1, archive.read(buf)); + } + } + } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/ExpanderTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/ExpanderTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/ExpanderTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/ExpanderTest.java 2018-06-15 17:46:02.000000000 +0000 @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.examples; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.archivers.StreamingNotSupportedException; +import org.apache.commons.compress.archivers.sevenz.SevenZFile; +import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class ExpanderTest extends AbstractTestCase { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private File archive; + + @Test + public void sevenZTwoFileVersion() throws IOException, ArchiveException { + setup7z(); + new Expander().expand("7z", archive, resultDir); + verifyTargetDir(); + } + + @Test + public void sevenZTwoFileVersionWithAutoDetection() throws IOException, ArchiveException { + setup7z(); + new Expander().expand(archive, resultDir); + verifyTargetDir(); + } + + @Test(expected = StreamingNotSupportedException.class) + public void sevenZInputStreamVersion() throws IOException, ArchiveException { + setup7z(); + try (InputStream i = new BufferedInputStream(Files.newInputStream(archive.toPath()))) { + new Expander().expand("7z", i, resultDir); + } + } + + @Test(expected = StreamingNotSupportedException.class) + public void sevenZInputStreamVersionWithAutoDetection() throws IOException, ArchiveException { + setup7z(); + try (InputStream i = new BufferedInputStream(Files.newInputStream(archive.toPath()))) { + new Expander().expand(i, resultDir); + } + } + + @Test + public void sevenZChannelVersion() throws IOException, ArchiveException { + setup7z(); + try (SeekableByteChannel c = FileChannel.open(archive.toPath(), StandardOpenOption.READ)) { + new Expander().expand("7z", c, resultDir); + } + verifyTargetDir(); + } + + @Test + public void sevenZFileVersion() throws IOException, ArchiveException { + setup7z(); + try (SevenZFile f = new SevenZFile(archive)) { + new Expander().expand(f, resultDir); + } + verifyTargetDir(); + } + + @Test + public void zipFileVersion() throws IOException, ArchiveException { + setupZip(); + try (ZipFile f = new ZipFile(archive)) { + new Expander().expand(f, resultDir); + } + verifyTargetDir(); + } + + @Test + public void fileCantEscapeViaAbsolutePath() throws IOException, ArchiveException { + setupZip("/tmp/foo"); + try (ZipFile f = new ZipFile(archive)) { + new Expander().expand(f, resultDir); + } + assertHelloWorld("tmp/foo", "1"); + } + + @Test + public void fileCantEscapeDoubleDotPath() throws IOException, ArchiveException { + thrown.expect(IOException.class); + thrown.expectMessage("expanding ../foo would create file outside of"); + setupZip("../foo"); + try (ZipFile f = new ZipFile(archive)) { + new Expander().expand(f, resultDir); + } + } + + @Test + public void fileCantEscapeDoubleDotPathWithSimilarSibling() throws IOException, ArchiveException { + String sibling = resultDir.getName() + "x"; + File s = new File(resultDir.getParentFile(), sibling); + Assume.assumeFalse(s.exists()); + s.mkdirs(); + Assume.assumeTrue(s.exists()); + s.deleteOnExit(); + try { + thrown.expect(IOException.class); + thrown.expectMessage("expanding ../" + sibling + "/a would create file outside of"); + setupZip("../" + sibling + "/a"); + try (ZipFile f = new ZipFile(archive)) { + new Expander().expand(f, resultDir); + } + } finally { + tryHardToDelete(s); + } + } + + private void setup7z() throws IOException, ArchiveException { + archive = new File(dir, "test.7z"); + File dummy = new File(dir, "x"); + try (OutputStream o = Files.newOutputStream(dummy.toPath())) { + o.write(new byte[14]); + } + try (SevenZOutputFile aos = new SevenZOutputFile(archive)) { + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a/b")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a/b/c")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dummy, "a/b/d.txt")); + aos.write("Hello, world 1".getBytes(StandardCharsets.UTF_8)); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dummy, "a/b/c/e.txt")); + aos.write("Hello, world 2".getBytes(StandardCharsets.UTF_8)); + aos.closeArchiveEntry(); + aos.finish(); + } + } + + private void setupZip() throws IOException, ArchiveException { + archive = new File(dir, "test.zip"); + File dummy = new File(dir, "x"); + try (OutputStream o = Files.newOutputStream(dummy.toPath())) { + o.write(new byte[14]); + } + try (ArchiveOutputStream aos = new ArchiveStreamFactory() + .createArchiveOutputStream("zip", Files.newOutputStream(archive.toPath()))) { + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a/b")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a/b/c")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dummy, "a/b/d.txt")); + aos.write("Hello, world 1".getBytes(StandardCharsets.UTF_8)); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dummy, "a/b/c/e.txt")); + aos.write("Hello, world 2".getBytes(StandardCharsets.UTF_8)); + aos.closeArchiveEntry(); + aos.finish(); + } + } + + private void setupZip(String entry) throws IOException, ArchiveException { + archive = new File(dir, "test.zip"); + File dummy = new File(dir, "x"); + try (OutputStream o = Files.newOutputStream(dummy.toPath())) { + o.write(new byte[14]); + } + try (ArchiveOutputStream aos = new ArchiveStreamFactory() + .createArchiveOutputStream("zip", Files.newOutputStream(archive.toPath()))) { + aos.putArchiveEntry(aos.createArchiveEntry(dummy, entry)); + aos.write("Hello, world 1".getBytes(StandardCharsets.UTF_8)); + aos.closeArchiveEntry(); + aos.finish(); + } + } + + private void verifyTargetDir() throws IOException { + Assert.assertTrue("a has not been created", new File(resultDir, "a").isDirectory()); + Assert.assertTrue("a/b has not been created", new File(resultDir, "a/b").isDirectory()); + Assert.assertTrue("a/b/c has not been created", new File(resultDir, "a/b/c").isDirectory()); + assertHelloWorld("a/b/d.txt", "1"); + assertHelloWorld("a/b/c/e.txt", "2"); + } + + private void assertHelloWorld(String fileName, String suffix) throws IOException { + Assert.assertTrue(fileName + " does not exist", new File(resultDir, fileName).isFile()); + byte[] expected = ("Hello, world " + suffix).getBytes(StandardCharsets.UTF_8); + try (InputStream is = Files.newInputStream(new File(resultDir, fileName).toPath())) { + byte[] actual = IOUtils.toByteArray(is); + Assert.assertArrayEquals(expected, actual); + } + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedArchiverTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedArchiverTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedArchiverTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedArchiverTest.java 2018-06-07 19:11:34.000000000 +0000 @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.examples; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class ParameterizedArchiverTest extends AbstractTestCase { + + // can't test 7z here as 7z cannot write to non-seekable streams + // and reading logic would be different as well - see + // SevenZArchiverTest class + @Parameters(name = "format={0}") + public static Collection data() { + return Arrays.asList( + new Object[] { "tar" }, + new Object[] { "cpio" }, + new Object[] { "zip" } + ); + } + + private final String format; + private File target; + + public ParameterizedArchiverTest(String format) { + this.format = format; + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + File c = new File(dir, "a/b/c"); + c.mkdirs(); + try (OutputStream os = Files.newOutputStream(new File(dir, "a/b/d.txt").toPath())) { + os.write("Hello, world 1".getBytes(StandardCharsets.UTF_8)); + } + try (OutputStream os = Files.newOutputStream(new File(dir, "a/b/c/e.txt").toPath())) { + os.write("Hello, world 2".getBytes(StandardCharsets.UTF_8)); + } + target = new File(resultDir, "test." + format); + } + + @Test + public void fileVersion() throws IOException, ArchiveException { + new Archiver().create(format, target, dir); + verifyContent(); + } + + @Test + public void outputStreamVersion() throws IOException, ArchiveException { + try (OutputStream os = Files.newOutputStream(target.toPath())) { + new Archiver().create(format, os, dir); + } + verifyContent(); + } + + @Test + public void channelVersion() throws IOException, ArchiveException { + try (SeekableByteChannel c = FileChannel.open(target.toPath(), StandardOpenOption.WRITE, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + new Archiver().create(format, c, dir); + } + verifyContent(); + } + + @Test + public void archiveStreamVersion() throws IOException, ArchiveException { + try (OutputStream os = Files.newOutputStream(target.toPath()); + ArchiveOutputStream aos = new ArchiveStreamFactory().createArchiveOutputStream(format, os)) { + new Archiver().create(aos, dir); + } + verifyContent(); + } + + private void verifyContent() throws IOException, ArchiveException { + try (InputStream is = Files.newInputStream(target.toPath()); + BufferedInputStream bis = new BufferedInputStream(is); + ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(format, bis)) { + assertDir("a", ais.getNextEntry()); + assertDir("a/b", ais.getNextEntry()); + ArchiveEntry n = ais.getNextEntry(); + Assert.assertNotNull(n); + // File.list may return a/b/c or a/b/d.txt first + if (n.getName().endsWith("/")) { + assertDir("a/b/c", n); + assertHelloWorld("a/b/c/e.txt", "2", ais.getNextEntry(), ais); + assertHelloWorld("a/b/d.txt", "1", ais.getNextEntry(), ais); + } else { + assertHelloWorld("a/b/d.txt", "1", n, ais); + assertDir("a/b/c", ais.getNextEntry()); + assertHelloWorld("a/b/c/e.txt", "2", ais.getNextEntry(), ais); + } + } + } + + private void assertDir(String expectedName, ArchiveEntry entry) { + Assert.assertNotNull(expectedName + " does not exists", entry); + Assert.assertEquals(expectedName + "/", entry.getName()); + Assert.assertTrue(expectedName + " is not a directory", entry.isDirectory()); + } + + private void assertHelloWorld(String expectedName, String suffix, ArchiveEntry entry, InputStream is) + throws IOException { + Assert.assertNotNull(expectedName + " does not exists", entry); + Assert.assertEquals(expectedName, entry.getName()); + Assert.assertFalse(expectedName + " is a directory", entry.isDirectory()); + byte[] expected = ("Hello, world " + suffix).getBytes(StandardCharsets.UTF_8); + byte[] actual = IOUtils.toByteArray(is); + Assert.assertArrayEquals(expected, actual); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedExpanderTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedExpanderTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedExpanderTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/ParameterizedExpanderTest.java 2018-06-07 19:11:34.000000000 +0000 @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.examples; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class ParameterizedExpanderTest extends AbstractTestCase { + + // 7z and ZIP using ZipFile is in ExpanderTest + @Parameters(name = "format={0}") + public static Collection data() { + return Arrays.asList( + new Object[] { "tar" }, + new Object[] { "cpio" }, + new Object[] { "zip" } + ); + } + + private final String format; + private File archive; + + public ParameterizedExpanderTest(String format) { + this.format = format; + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + archive = new File(dir, "test." + format); + File dummy = new File(dir, "x"); + try (OutputStream o = Files.newOutputStream(dummy.toPath())) { + o.write(new byte[14]); + } + try (ArchiveOutputStream aos = new ArchiveStreamFactory() + .createArchiveOutputStream(format, Files.newOutputStream(archive.toPath()))) { + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a/b")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dir, "a/b/c")); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dummy, "a/b/d.txt")); + aos.write("Hello, world 1".getBytes(StandardCharsets.UTF_8)); + aos.closeArchiveEntry(); + aos.putArchiveEntry(aos.createArchiveEntry(dummy, "a/b/c/e.txt")); + aos.write("Hello, world 2".getBytes(StandardCharsets.UTF_8)); + aos.closeArchiveEntry(); + aos.finish(); + } + } + + @Test + public void fileVersion() throws IOException, ArchiveException { + new Expander().expand(format, archive, resultDir); + verifyTargetDir(); + } + + @Test + public void fileVersionWithAutoDetection() throws IOException, ArchiveException { + new Expander().expand(archive, resultDir); + verifyTargetDir(); + } + + @Test + public void inputStreamVersion() throws IOException, ArchiveException { + try (InputStream i = new BufferedInputStream(Files.newInputStream(archive.toPath()))) { + new Expander().expand(format, i, resultDir); + } + verifyTargetDir(); + } + + @Test + public void inputStreamVersionWithAutoDetection() throws IOException, ArchiveException { + try (InputStream i = new BufferedInputStream(Files.newInputStream(archive.toPath()))) { + new Expander().expand(i, resultDir); + } + verifyTargetDir(); + } + + @Test + public void channelVersion() throws IOException, ArchiveException { + try (SeekableByteChannel c = FileChannel.open(archive.toPath(), StandardOpenOption.READ)) { + new Expander().expand(format, c, resultDir); + } + verifyTargetDir(); + } + + @Test + public void archiveInputStreamVersion() throws IOException, ArchiveException { + try (InputStream i = new BufferedInputStream(Files.newInputStream(archive.toPath())); + ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(format, i)) { + new Expander().expand(ais, resultDir); + } + verifyTargetDir(); + } + + private void verifyTargetDir() throws IOException { + Assert.assertTrue("a has not been created", new File(resultDir, "a").isDirectory()); + Assert.assertTrue("a/b has not been created", new File(resultDir, "a/b").isDirectory()); + Assert.assertTrue("a/b/c has not been created", new File(resultDir, "a/b/c").isDirectory()); + assertHelloWorld("a/b/d.txt", "1"); + assertHelloWorld("a/b/c/e.txt", "2"); + } + + private void assertHelloWorld(String fileName, String suffix) throws IOException { + Assert.assertTrue(fileName + " does not exist", new File(resultDir, fileName).isFile()); + byte[] expected = ("Hello, world " + suffix).getBytes(StandardCharsets.UTF_8); + try (InputStream is = Files.newInputStream(new File(resultDir, fileName).toPath())) { + byte[] actual = IOUtils.toByteArray(is); + Assert.assertArrayEquals(expected, actual); + } + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/SevenZArchiverTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/SevenZArchiverTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/examples/SevenZArchiverTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/examples/SevenZArchiverTest.java 2018-06-07 19:11:34.000000000 +0000 @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.examples; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.StreamingNotSupportedException; +import org.apache.commons.compress.archivers.sevenz.SevenZFile; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class SevenZArchiverTest extends AbstractTestCase { + private File target; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + File c = new File(dir, "a/b/c"); + c.mkdirs(); + try (OutputStream os = Files.newOutputStream(new File(dir, "a/b/d.txt").toPath())) { + os.write("Hello, world 1".getBytes(StandardCharsets.UTF_8)); + } + try (OutputStream os = Files.newOutputStream(new File(dir, "a/b/c/e.txt").toPath())) { + os.write("Hello, world 2".getBytes(StandardCharsets.UTF_8)); + } + target = new File(resultDir, "test.7z"); + } + + @Test + public void fileVersion() throws IOException, ArchiveException { + new Archiver().create("7z", target, dir); + verifyContent(); + } + + @Test(expected = StreamingNotSupportedException.class) + public void outputStreamVersion() throws IOException, ArchiveException { + try (OutputStream os = Files.newOutputStream(target.toPath())) { + new Archiver().create("7z", os, dir); + } + } + + @Test + public void channelVersion() throws IOException, ArchiveException { + try (SeekableByteChannel c = FileChannel.open(target.toPath(), StandardOpenOption.WRITE, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + new Archiver().create("7z", c, dir); + } + verifyContent(); + } + + // not really a 7z test but I didn't feel like adding a new test just for this + @Test(expected = ArchiveException.class) + public void unknownFormat() throws IOException, ArchiveException { + try (SeekableByteChannel c = FileChannel.open(target.toPath(), StandardOpenOption.WRITE, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + new Archiver().create("unknown format", c, dir); + } + } + + private void verifyContent() throws IOException, ArchiveException { + try (SevenZFile z = new SevenZFile(target)) { + assertDir("a", z.getNextEntry()); + assertDir("a/b", z.getNextEntry()); + ArchiveEntry n = z.getNextEntry(); + Assert.assertNotNull(n); + // File.list may return a/b/c or a/b/d.txt first + if (n.getName().endsWith("/")) { + assertDir("a/b/c", n); + assertHelloWorld("a/b/c/e.txt", "2", z.getNextEntry(), z); + assertHelloWorld("a/b/d.txt", "1", z.getNextEntry(), z); + } else { + assertHelloWorld("a/b/d.txt", "1", n, z); + assertDir("a/b/c", z.getNextEntry()); + assertHelloWorld("a/b/c/e.txt", "2", z.getNextEntry(), z); + } + } + } + + private void assertDir(String expectedName, ArchiveEntry entry) { + Assert.assertNotNull(expectedName + " does not exists", entry); + Assert.assertEquals(expectedName + "/", entry.getName()); + Assert.assertTrue(expectedName + " is not a directory", entry.isDirectory()); + } + + private void assertHelloWorld(String expectedName, String suffix, ArchiveEntry entry, SevenZFile z) + throws IOException { + Assert.assertNotNull(expectedName + " does not exists", entry); + Assert.assertEquals(expectedName, entry.getName()); + Assert.assertFalse(expectedName + " is a directory", entry.isDirectory()); + byte[] expected = ("Hello, world " + suffix).getBytes(StandardCharsets.UTF_8); + byte[] actual = new byte[expected.length]; + Assert.assertEquals(actual.length, z.read(actual)); + Assert.assertEquals(-1, z.read()); + Assert.assertArrayEquals(expected, actual); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/jar/JarMarkerTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/jar/JarMarkerTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/jar/JarMarkerTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/jar/JarMarkerTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.jar; + +import static org.junit.Assert.fail; + +import java.util.zip.ZipException; +import org.apache.commons.compress.archivers.zip.JarMarker; +import org.junit.Test; + +public class JarMarkerTest { + + @Test public void testJarMarkerLengthCheck() { + JarMarker jarMarker = JarMarker.getInstance(); + try { + jarMarker.parseFromLocalFileData(null,0,1); + fail("should have thrown exception due to length of 1"); + } catch (ZipException e) { + + } + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/LongPathTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/LongPathTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/LongPathTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/LongPathTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress.archivers; @@ -27,13 +27,12 @@ import java.io.FileInputStream; import java.io.FileReader; import java.io.FilenameFilter; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import junit.framework.AssertionFailedError; - import org.apache.commons.compress.AbstractTestCase; -import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ar.ArArchiveInputStream; import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -47,7 +46,7 @@ /** * Test that can read various tar file examples. - * + * * Files must be in resources/longpath, and there must be a file.txt containing * the list of files in the archives. */ @@ -55,11 +54,21 @@ public class LongPathTest extends AbstractTestCase { private static final ClassLoader CLASSLOADER = LongPathTest.class.getClassLoader(); - private static final File ARCDIR = new File(CLASSLOADER.getResource("longpath").getFile()); + private static final File ARCDIR; private static final ArrayList FILELIST = new ArrayList<>(); + static { + try { + ARCDIR = new File(CLASSLOADER.getResource("longpath").toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + private final File file; + + public LongPathTest(final String file){ this.file = new File(ARCDIR, file); } @@ -87,7 +96,7 @@ public boolean accept(final File dir, final String name) { return !name.endsWith(".txt"); } - })) + })) { params.add(new Object[] { f }); } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/LongSymLinkTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/LongSymLinkTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/LongSymLinkTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/LongSymLinkTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress.archivers; @@ -27,13 +27,12 @@ import java.io.FileInputStream; import java.io.FileReader; import java.io.FilenameFilter; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import junit.framework.AssertionFailedError; - import org.apache.commons.compress.AbstractTestCase; -import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ar.ArArchiveInputStream; import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -47,7 +46,7 @@ /** * Test that can read various tar file examples. - * + * * Files must be in resources/longsymlink, and there must be a file.txt containing * the list of files in the archives. */ @@ -55,9 +54,17 @@ public class LongSymLinkTest extends AbstractTestCase { private static final ClassLoader CLASSLOADER = LongSymLinkTest.class.getClassLoader(); - private static final File ARCDIR = new File(CLASSLOADER.getResource("longsymlink").getFile()); + private static final File ARCDIR; private static final ArrayList FILELIST = new ArrayList<>(); + static { + try { + ARCDIR = new File(CLASSLOADER.getResource("longsymlink").toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + private final File file; public LongSymLinkTest(final String file){ @@ -87,7 +94,7 @@ public boolean accept(final File dir, final String name) { return !name.endsWith(".txt"); } - })) + })) { params.add(new Object[] { f }); } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256DecoderTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256DecoderTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256DecoderTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/AES256SHA256DecoderTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.sevenz; + +import org.junit.Test; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; + +import static org.junit.Assert.*; + + +/** + * Unit tests for class {@link AES256SHA256Decoder}. + * + * @date 26.06.2017 + * @see AES256SHA256Decoder + **/ +public class AES256SHA256DecoderTest { + + + @Test + public void testDecodeWithNonEmptyString() throws IOException { + + AES256SHA256Decoder aES256SHA256Decoder = new AES256SHA256Decoder(); + BufferedInputStream bufferedInputStream = new BufferedInputStream(null, 3138); + Coder coder = new Coder(); + byte[] byteArray = new byte[8]; + byteArray[1] = (byte) (-72); + coder.properties = byteArray; + InputStream inputStream = aES256SHA256Decoder.decode("x", bufferedInputStream, 3138, coder, coder.properties); + + ObjectInputStream objectInputStream = null; + + try { + objectInputStream = new ObjectInputStream(inputStream); + fail("Expecting exception: IOException"); + } catch(Throwable e) { + assertEquals("Salt size + IV size too long in x",e.getMessage()); + assertEquals("org.apache.commons.compress.archivers.sevenz.AES256SHA256Decoder$1", e.getStackTrace()[0].getClassName()); + } + + } + + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/CoverageTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/CoverageTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/CoverageTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/CoverageTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.sevenz; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import org.junit.Test; + +public class CoverageTest { + + @Test public void testNidInstance() { + assertNotNull(new NID()); + } + + @Test public void testCLIInstance() { + CLI foo = new CLI(); + assertNotNull(foo); + try { + CLI.main(new String[]{"/dev/null/not-there"}); + fail("shouldn't be able to list contents of a file that isn't there"); + } catch (Exception ignored) { + + } + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/FolderTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/FolderTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/FolderTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/FolderTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.sevenz; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + + +/** + * Unit tests for class {@link Folder}. + * + * @date 26.06.2017 + * @see Folder + **/ +public class FolderTest { + + + @Test + public void testGetUnpackSizeForCoderOne() { + + Folder folder = new Folder(); + Coder[] coderArray = new Coder[5]; + Coder coder = new Coder(); + folder.coders = coderArray; + + assertEquals(0L, folder.getUnpackSizeForCoder(coder)); + + } + + + @Test + public void testGetUnpackSizeOne() { + + Folder folder = new Folder(); + folder.totalOutputStreams = 266L; + BindPair[] bindPairArray = new BindPair[1]; + BindPair bindPair = new BindPair(); + bindPairArray[0] = bindPair; + folder.bindPairs = bindPairArray; + folder.totalOutputStreams = 1L; + + assertEquals(0L, folder.getUnpackSize()); + + } + + + @Test + public void testGetUnpackSizeTwo() { + + Folder folder = new Folder(); + + assertEquals(0L, folder.getUnpackSize()); + + } + + + @Test + public void testFindBindPairForInStream() { + + Folder folder = new Folder(); + BindPair[] bindPairArray = new BindPair[1]; + BindPair bindPair = new BindPair(); + bindPairArray[0] = bindPair; + folder.bindPairs = bindPairArray; + + assertEquals(0, folder.findBindPairForInStream(0)); + + } + + +} \ No newline at end of file diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZFileTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZFileTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZFileTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZFileTest.java 2018-06-07 19:11:34.000000000 +0000 @@ -132,6 +132,11 @@ } @Test + public void test7zDeflate64Unarchive() throws Exception { + test7zUnarchive(getFile("bla.deflate64.7z"), SevenZMethod.DEFLATE64); + } + + @Test public void test7zDecryptUnarchive() throws Exception { if (isStrongCryptoAvailable()) { test7zUnarchive(getFile("bla.encrypted.7z"), SevenZMethod.LZMA, // stack LZMA + AES @@ -139,14 +144,22 @@ } } + @Test + public void test7zDecryptUnarchiveUsingCharArrayPassword() throws Exception { + if (isStrongCryptoAvailable()) { + test7zUnarchive(getFile("bla.encrypted.7z"), SevenZMethod.LZMA, // stack LZMA + AES + "foo".toCharArray()); + } + } + private void test7zUnarchive(final File f, final SevenZMethod m) throws Exception { - test7zUnarchive(f, m, null); + test7zUnarchive(f, m, (char[]) null); } @Test public void testEncryptedArchiveRequiresPassword() throws Exception { try { - new SevenZFile(getFile("bla.encrypted.7z")); + new SevenZFile(getFile("bla.encrypted.7z")).close(); fail("shouldn't decrypt without a password"); } catch (final PasswordRequiredException ex) { final String msg = ex.getMessage(); @@ -280,25 +293,35 @@ assertEquals(5, entries); } } - + private void test7zUnarchive(final File f, final SevenZMethod m, final byte[] password) throws Exception { try (SevenZFile sevenZFile = new SevenZFile(f, password)) { - SevenZArchiveEntry entry = sevenZFile.getNextEntry(); - assertEquals("test1.xml", entry.getName()); - assertEquals(m, entry.getContentMethods().iterator().next().getMethod()); - entry = sevenZFile.getNextEntry(); - assertEquals("test2.xml", entry.getName()); - assertEquals(m, entry.getContentMethods().iterator().next().getMethod()); - final byte[] contents = new byte[(int) entry.getSize()]; - int off = 0; - while ((off < contents.length)) { - final int bytesRead = sevenZFile.read(contents, off, contents.length - off); - assert (bytesRead >= 0); - off += bytesRead; - } - assertEquals(TEST2_CONTENT, new String(contents, "UTF-8")); - assertNull(sevenZFile.getNextEntry()); + test7zUnarchive(sevenZFile, m); + } + } + + private void test7zUnarchive(final File f, final SevenZMethod m, final char[] password) throws Exception { + try (SevenZFile sevenZFile = new SevenZFile(f, password)) { + test7zUnarchive(sevenZFile, m); + } + } + + private void test7zUnarchive(SevenZFile sevenZFile, final SevenZMethod m) throws Exception { + SevenZArchiveEntry entry = sevenZFile.getNextEntry(); + assertEquals("test1.xml", entry.getName()); + assertEquals(m, entry.getContentMethods().iterator().next().getMethod()); + entry = sevenZFile.getNextEntry(); + assertEquals("test2.xml", entry.getName()); + assertEquals(m, entry.getContentMethods().iterator().next().getMethod()); + final byte[] contents = new byte[(int) entry.getSize()]; + int off = 0; + while ((off < contents.length)) { + final int bytesRead = sevenZFile.read(contents, off, contents.length - off); + assert (bytesRead >= 0); + off += bytesRead; } + assertEquals(TEST2_CONTENT, new String(contents, "UTF-8")); + assertNull(sevenZFile.getNextEntry()); } private void checkHelloWorld(final String filename) throws Exception { diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZNativeHeapTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZNativeHeapTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZNativeHeapTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZNativeHeapTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -17,48 +17,251 @@ */ package org.apache.commons.compress.archivers.sevenz; -import org.apache.commons.compress.AbstractTestCase; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.io.OutputStream; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.DataFormatException; import java.util.zip.Deflater; import java.util.zip.Inflater; +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.archivers.sevenz.Coders.DeflateDecoder; +import org.apache.commons.compress.archivers.sevenz.Coders.DeflateDecoder.DeflateDecoderInputStream; +import org.apache.commons.compress.archivers.sevenz.Coders.DeflateDecoder.DeflateDecoderOutputStream; +import org.junit.Test; -@RunWith(PowerMockRunner.class) -@PrepareForTest(Coders.DeflateDecoder.class) public class SevenZNativeHeapTest extends AbstractTestCase { - @InjectMocks - Coders.DeflateDecoder deflateDecoder; @Test public void testEndDeflaterOnCloseStream() throws Exception { - final Deflater deflater = PowerMockito.spy(new Deflater()); - PowerMockito.whenNew(Deflater.class).withAnyArguments().thenReturn(deflater); + Coders.DeflateDecoder deflateDecoder = new DeflateDecoder(); - final OutputStream outputStream = deflateDecoder.encode(new ByteArrayOutputStream(), 9); + final DeflateDecoderOutputStream outputStream = + (DeflateDecoderOutputStream) deflateDecoder.encode(new ByteArrayOutputStream(), 9); + DelegatingDeflater delegatingDeflater = new DelegatingDeflater(outputStream.deflater); + outputStream.deflater = delegatingDeflater; outputStream.close(); + assertTrue(delegatingDeflater.isEnded.get()); - Mockito.verify(deflater).end(); } @Test public void testEndInflaterOnCloseStream() throws Exception { - final Inflater inflater = PowerMockito.spy(new Inflater()); - PowerMockito.whenNew(Inflater.class).withAnyArguments().thenReturn(inflater); - - final InputStream inputStream = deflateDecoder.decode("dummy",new ByteArrayInputStream(new byte[0]),0,null,null); + Coders.DeflateDecoder deflateDecoder = new DeflateDecoder(); + final DeflateDecoderInputStream inputStream = + (DeflateDecoderInputStream) deflateDecoder.decode("dummy",new ByteArrayInputStream(new byte[0]),0,null,null); + DelegatingInflater delegatingInflater = new DelegatingInflater(inputStream.inflater); + inputStream.inflater = delegatingInflater; inputStream.close(); - Mockito.verify(inflater).end(); + assertTrue(delegatingInflater.isEnded.get()); + } + + private class DelegatingInflater extends Inflater { + + private final Inflater inflater; + + public DelegatingInflater(Inflater inflater) { + this.inflater = inflater; + } + AtomicBoolean isEnded = new AtomicBoolean(); + + @Override + public void end() { + isEnded.set(true); + inflater.end(); + } + + @Override + public void setInput(byte[] b, int off, int len) { + inflater.setInput(b, off, len); + } + + @Override + public void setInput(byte[] b) { + inflater.setInput(b); + } + + @Override + public void setDictionary(byte[] b, int off, int len) { + inflater.setDictionary(b, off, len); + } + + @Override + public void setDictionary(byte[] b) { + inflater.setDictionary(b); + } + + @Override + public int getRemaining() { + return inflater.getRemaining(); + } + + @Override + public boolean needsInput() { + return inflater.needsInput(); + } + + @Override + public boolean needsDictionary() { + return inflater.needsDictionary(); + } + + @Override + public boolean finished() { + return inflater.finished(); + } + + @Override + public int inflate(byte[] b, int off, int len) throws DataFormatException { + return inflater.inflate(b, off, len); + } + + @Override + public int inflate(byte[] b) throws DataFormatException { + return inflater.inflate(b); + } + + @Override + public int getAdler() { + return inflater.getAdler(); + } + + @Override + public int getTotalIn() { + return inflater.getTotalIn(); + } + + @Override + public long getBytesRead() { + return inflater.getBytesRead(); + } + + @Override + public int getTotalOut() { + return inflater.getTotalOut(); + } + + @Override + public long getBytesWritten() { + return inflater.getBytesWritten(); + } + + @Override + public void reset() { + inflater.reset(); + } + + } + + private class DelegatingDeflater extends Deflater { + + private final Deflater deflater; + + public DelegatingDeflater(Deflater deflater) { + this.deflater = deflater; + } + + AtomicBoolean isEnded = new AtomicBoolean(); + + @Override + public void end() { + isEnded.set(true); + deflater.end(); + } + + @Override + public void setInput(byte[] b, int off, int len) { + deflater.setInput(b, off, len); + } + + @Override + public void setInput(byte[] b) { + deflater.setInput(b); + } + + @Override + public void setDictionary(byte[] b, int off, int len) { + deflater.setDictionary(b, off, len); + } + + @Override + public void setDictionary(byte[] b) { + deflater.setDictionary(b); + } + + @Override + public void setStrategy(int strategy) { + deflater.setStrategy(strategy); + } + + @Override + public void setLevel(int level) { + deflater.setLevel(level); + } + + @Override + public boolean needsInput() { + return deflater.needsInput(); + } + + @Override + public void finish() { + deflater.finish(); + } + + @Override + public boolean finished() { + return deflater.finished(); + } + + @Override + public int deflate(byte[] b, int off, int len) { + return deflater.deflate(b, off, len); + } + + @Override + public int deflate(byte[] b) { + return deflater.deflate(b); + } + + @Override + public int deflate(byte[] b, int off, int len, int flush) { + return deflater.deflate(b, off, len, flush); + } + + @Override + public int getAdler() { + return deflater.getAdler(); + } + + @Override + public int getTotalIn() { + return deflater.getTotalIn(); + } + + @Override + public long getBytesRead() { + return deflater.getBytesRead(); + } + + @Override + public int getTotalOut() { + return deflater.getTotalOut(); + } + + @Override + public long getBytesWritten() { + return deflater.getBytesWritten(); + } + + @Override + public void reset() { + deflater.reset(); + } + + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFileTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFileTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFileTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/sevenz/SevenZOutputFileTest.java 2018-06-07 19:11:34.000000000 +0000 @@ -38,7 +38,8 @@ static { final String version = org.tukaani.xz.XZ.class.getPackage().getImplementationVersion(); - XZ_BCJ_IS_BUGGY=version.equals("1.4"); + + XZ_BCJ_IS_BUGGY= version != null && version.equals("1.4"); if (XZ_BCJ_IS_BUGGY) { System.out.println("XZ version is " + version + " - skipping BCJ tests"); } @@ -500,8 +501,7 @@ outArchive.close(); } try (SevenZFile archive = - new SevenZFile(new SeekableInMemoryByteChannel(output.array()), "in memory", - null)) { + new SevenZFile(new SeekableInMemoryByteChannel(output.array()), "in memory")) { assertEquals(Boolean.TRUE, verifyFile(archive, 0, methods)); } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/SevenZTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/SevenZTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/SevenZTestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/SevenZTestCase.java 2018-08-09 15:25:21.000000000 +0000 @@ -22,21 +22,39 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import javax.crypto.Cipher; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry; import org.apache.commons.compress.archivers.sevenz.SevenZFile; import org.apache.commons.compress.archivers.sevenz.SevenZMethod; import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile; +import org.junit.Assume; +import org.junit.Before; import org.junit.Test; public class SevenZTestCase extends AbstractTestCase { + private File output; + private final File file1, file2; + + public SevenZTestCase() throws IOException { + file1 = getFile("test1.xml"); + file2 = getFile("test2.xml"); + } + + @Before + public void setUp() throws Exception { + super.setUp(); + output = new File(dir, "bla.7z"); + } + @Test public void testSevenZArchiveCreationUsingCopy() throws Exception { testSevenZArchiveCreation(SevenZMethod.COPY); } - + @Test public void testSevenZArchiveCreationUsingLZMA() throws Exception { testSevenZArchiveCreation(SevenZMethod.LZMA); @@ -46,32 +64,45 @@ public void testSevenZArchiveCreationUsingLZMA2() throws Exception { testSevenZArchiveCreation(SevenZMethod.LZMA2); } - + @Test public void testSevenZArchiveCreationUsingBZIP2() throws Exception { testSevenZArchiveCreation(SevenZMethod.BZIP2); } - + @Test public void testSevenZArchiveCreationUsingDeflate() throws Exception { testSevenZArchiveCreation(SevenZMethod.DEFLATE); } - + private void testSevenZArchiveCreation(final SevenZMethod method) throws Exception { - final File output = new File(dir, "bla.7z"); - final File file1 = getFile("test1.xml"); - final File file2 = getFile("test2.xml"); + createArchive(method); + try (SevenZFile archive = new SevenZFile(output)) { + SevenZArchiveEntry entry; + + entry = archive.getNextEntry(); + assert (entry != null); + assertEquals(entry.getName(), file1.getName()); + + entry = archive.getNextEntry(); + assert (entry != null); + assertEquals(entry.getName(), file2.getName()); + + assert (archive.getNextEntry() == null); + } + } + private void createArchive(final SevenZMethod method) throws Exception { final SevenZOutputFile outArchive = new SevenZOutputFile(output); outArchive.setContentCompression(method); try { SevenZArchiveEntry entry; - + entry = outArchive.createArchiveEntry(file1, file1.getName()); outArchive.putArchiveEntry(entry); copy(file1, outArchive); outArchive.closeArchiveEntry(); - + entry = outArchive.createArchiveEntry(file2, file2.getName()); outArchive.putArchiveEntry(entry); copy(file2, outArchive); @@ -79,23 +110,101 @@ } finally { outArchive.close(); } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingCopy() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.COPY); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingLZMA() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.LZMA); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingLZMA2() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.LZMA2); + } + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingBZIP2() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.BZIP2); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.DEFLATE); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingAES() throws Exception { + assumeStrongCryptoIsAvailable(); + try (SevenZFile archive = new SevenZFile(getFile("bla.encrypted.7z"), "foo".toCharArray())) { + singleByteReadConsistentlyReturnsMinusOneAtEof(archive); + } + } + + private void singleByteReadConsistentlyReturnsMinusOneAtEof(final SevenZMethod method) throws Exception { + createArchive(method); try (SevenZFile archive = new SevenZFile(output)) { - SevenZArchiveEntry entry; + singleByteReadConsistentlyReturnsMinusOneAtEof(archive); + } + } - entry = archive.getNextEntry(); - assert (entry != null); - assertEquals(entry.getName(), file1.getName()); + private void singleByteReadConsistentlyReturnsMinusOneAtEof(SevenZFile archive) throws Exception { + SevenZArchiveEntry entry = archive.getNextEntry(); + entry = archive.getNextEntry(); + readFully(archive); + assertEquals(-1, archive.read()); + assertEquals(-1, archive.read()); + } - entry = archive.getNextEntry(); - assert (entry != null); - assertEquals(entry.getName(), file2.getName()); + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingLZMA() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.LZMA); + } - assert (archive.getNextEntry() == null); + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingLZMA2() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.LZMA2); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingBZIP2() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.BZIP2); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(SevenZMethod.DEFLATE); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingAES() throws Exception { + assumeStrongCryptoIsAvailable(); + try (SevenZFile archive = new SevenZFile(getFile("bla.encrypted.7z"), "foo".toCharArray())) { + multiByteReadConsistentlyReturnsMinusOneAtEof(archive); } } - private void copy(final File src, final SevenZOutputFile dst) throws IOException { + private void multiByteReadConsistentlyReturnsMinusOneAtEof(final SevenZMethod method) throws Exception { + createArchive(method); + try (SevenZFile archive = new SevenZFile(output)) { + multiByteReadConsistentlyReturnsMinusOneAtEof(archive); + } + } + + private void multiByteReadConsistentlyReturnsMinusOneAtEof(SevenZFile archive) throws Exception { + final byte[] buf = new byte[2]; + SevenZArchiveEntry entry = archive.getNextEntry(); + entry = archive.getNextEntry(); + readFully(archive); + assertEquals(-1, archive.read(buf)); + assertEquals(-1, archive.read(buf)); + } + + private void copy(final File src, final SevenZOutputFile dst) throws IOException { FileInputStream fis = null; try { fis = new FileInputStream(src); @@ -110,4 +219,16 @@ } } } + + private void readFully(final SevenZFile archive) throws IOException { + final byte[] buf = new byte[1024]; + int x = 0; + while (0 <= (x = archive.read(buf))) { + ; + } + } + + private static void assumeStrongCryptoIsAvailable() throws NoSuchAlgorithmException { + Assume.assumeTrue("test requires strong crypto", Cipher.getMaxAllowedKeyLength("AES/ECB/PKCS5Padding") >= 256); + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveEntryTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveEntryTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveEntryTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveEntryTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -18,16 +18,25 @@ package org.apache.commons.compress.archivers.tar; -import static org.junit.Assert.*; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; import java.util.Locale; - import org.apache.commons.compress.AbstractTestCase; +import org.junit.Test; public class TarArchiveEntryTest implements TarConstants { @@ -121,6 +130,45 @@ t.setSize(0100000000000L); } + @Test public void testExtraPaxHeaders() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + TarArchiveOutputStream tos = new TarArchiveOutputStream(bos); + + TarArchiveEntry entry = new TarArchiveEntry("./weasels"); + entry.addPaxHeader("APACHE.mustelida","true"); + entry.addPaxHeader("SCHILY.xattr.user.org.apache.weasels","maximum weasels"); + entry.addPaxHeader("size","1"); + assertEquals("extra header count",2,entry.getExtraPaxHeaders().size()); + assertEquals("APACHE.mustelida","true", + entry.getExtraPaxHeader("APACHE.mustelida")); + assertEquals("SCHILY.xattr.user.org.apache.weasels","maximum weasels", + entry.getExtraPaxHeader("SCHILY.xattr.user.org.apache.weasels")); + assertEquals("size",entry.getSize(),1); + + tos.putArchiveEntry(entry); + tos.write('W'); + tos.closeArchiveEntry(); + tos.close(); + assertNotEquals("should have extra headers before clear",0,entry.getExtraPaxHeaders().size()); + entry.clearExtraPaxHeaders(); + assertEquals("extra headers should be empty after clear",0,entry.getExtraPaxHeaders().size()); + TarArchiveInputStream tis = new TarArchiveInputStream(new ByteArrayInputStream(bos.toByteArray())); + entry = tis.getNextTarEntry(); + assertNotNull("couldn't get entry",entry); + + assertEquals("extra header count",2,entry.getExtraPaxHeaders().size()); + assertEquals("APACHE.mustelida","true", + entry.getExtraPaxHeader("APACHE.mustelida")); + assertEquals("user.org.apache.weasels","maximum weasels", + entry.getExtraPaxHeader("SCHILY.xattr.user.org.apache.weasels")); + + assertEquals('W',tis.read()); + assertTrue("should be at end of entry",tis.read() <0); + + assertNull("should be at end of file",tis.getNextTarEntry()); + tis.close(); + } + @Test public void testLinkFlagConstructor() { final TarArchiveEntry t = new TarArchiveEntry("/foo", LF_GNUTYPE_LONGNAME); @@ -143,6 +191,15 @@ assertEquals("/foo", t.getName()); } + @Test + public void preservesDriveSpecOnWindowsAndNetwareIfAskedTo() { + assumeTrue("C:\\".equals(ROOT)); + TarArchiveEntry t = new TarArchiveEntry(ROOT + "foo.txt", true); + assertEquals("C:/foo.txt", t.getName()); + t = new TarArchiveEntry(ROOT + "foo.txt", LF_GNUTYPE_LONGNAME, true); + assertEquals("C:/foo.txt", t.getName()); + } + private void assertGnuMagic(final TarArchiveEntry t) { assertEquals(MAGIC_GNU + VERSION_GNU_SPACE, readMagic(t)); } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveInputStreamTest.java 2018-08-09 17:47:14.000000000 +0000 @@ -23,6 +23,7 @@ import static org.apache.commons.compress.AbstractTestCase.rmdir; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -40,6 +41,7 @@ import java.util.TimeZone; import java.util.zip.GZIPInputStream; +import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.utils.CharsetNames; import org.apache.commons.compress.utils.IOUtils; import org.junit.Test; @@ -312,6 +314,64 @@ assertNull(is.getNextTarEntry()); } } + @Test + public void testGetAndSetOfPaxEntry() throws Exception { + try (TarArchiveInputStream is = getTestStream("/COMPRESS-356.tar")) { + final TarArchiveEntry entry = is.getNextTarEntry(); + assertEquals("package/package.json", entry.getName()); + assertEquals(is.getCurrentEntry(),entry); + TarArchiveEntry weaselEntry = new TarArchiveEntry(entry.getName()); + weaselEntry.setSize(entry.getSize()); + is.setCurrentEntry(weaselEntry); + assertEquals(entry,is.getCurrentEntry()); + assertFalse(entry == is.getCurrentEntry()); + assertTrue(weaselEntry == is.getCurrentEntry()); + try { + is.setCurrentEntry(null); + is.read(); + fail("should abort because current entry is nulled"); + } catch(IllegalStateException e) { + // expected + } + is.setCurrentEntry(entry); + is.read(); + } + } + + /** + * @link "https://issues.apache.org/jira/browse/COMPRESS-417" + */ + @Test + public void skipsDevNumbersWhenEntryIsNoDevice() throws Exception { + try (TarArchiveInputStream is = getTestStream("/COMPRESS-417.tar")) { + assertEquals("test1.xml", is.getNextTarEntry().getName()); + assertEquals("test2.xml", is.getNextTarEntry().getName()); + assertNull(is.getNextTarEntry()); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + try (FileInputStream in = new FileInputStream(getFile("bla.tar")); + TarArchiveInputStream archive = new TarArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read()); + assertEquals(-1, archive.read()); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + byte[] buf = new byte[2]; + try (FileInputStream in = new FileInputStream(getFile("bla.tar")); + TarArchiveInputStream archive = new TarArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read(buf)); + assertEquals(-1, archive.read(buf)); + } + } private TarArchiveInputStream getTestStream(final String name) { return new TarArchiveInputStream( diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -18,7 +18,12 @@ package org.apache.commons.compress.archivers.tar; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -26,13 +31,15 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.security.MessageDigest; import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.TimeZone; - import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveOutputStream; @@ -100,10 +107,10 @@ tos.write(new byte[10 * 1024]); final byte[] data = bos.toByteArray(); assertEquals(0x80, - data[TarConstants.NAMELEN - + TarConstants.MODELEN - + TarConstants.UIDLEN - + TarConstants.GIDLEN] & 0x80); + data[TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN] & 0x80); final TarArchiveInputStream tin = new TarArchiveInputStream(new ByteArrayInputStream(data)); final TarArchiveEntry e = tin.getNextTarEntry(); @@ -126,12 +133,12 @@ tos.write(new byte[10 * 1024]); final byte[] data = bos.toByteArray(); assertEquals("00000000000 ", - new String(data, - 1024 + TarConstants.NAMELEN - + TarConstants.MODELEN - + TarConstants.UIDLEN - + TarConstants.GIDLEN, 12, - CharsetNames.UTF_8)); + new String(data, + 1024 + TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + CharsetNames.UTF_8)); final TarArchiveInputStream tin = new TarArchiveInputStream(new ByteArrayInputStream(data)); final TarArchiveEntry e = tin.getNextTarEntry(); @@ -148,11 +155,11 @@ m.put("a", "b"); final byte[] data = writePaxHeader(m); assertEquals("00000000006 ", - new String(data, TarConstants.NAMELEN - + TarConstants.MODELEN - + TarConstants.UIDLEN - + TarConstants.GIDLEN, 12, - CharsetNames.UTF_8)); + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + CharsetNames.UTF_8)); assertEquals("6 a=b\n", new String(data, 512, 6, CharsetNames.UTF_8)); } @@ -160,38 +167,38 @@ public void testPaxHeadersWithLength99() throws Exception { final Map m = new HashMap<>(); m.put("a", - "0123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789" - + "012"); + "0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "012"); final byte[] data = writePaxHeader(m); assertEquals("00000000143 ", - new String(data, TarConstants.NAMELEN - + TarConstants.MODELEN - + TarConstants.UIDLEN - + TarConstants.GIDLEN, 12, - CharsetNames.UTF_8)); + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + CharsetNames.UTF_8)); assertEquals("99 a=0123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789" - + "012\n", new String(data, 512, 99, CharsetNames.UTF_8)); + + "01234567890123456789012345678901234567890123456789" + + "012\n", new String(data, 512, 99, CharsetNames.UTF_8)); } @Test public void testPaxHeadersWithLength101() throws Exception { final Map m = new HashMap<>(); m.put("a", - "0123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789" - + "0123"); + "0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "0123"); final byte[] data = writePaxHeader(m); assertEquals("00000000145 ", - new String(data, TarConstants.NAMELEN - + TarConstants.MODELEN - + TarConstants.UIDLEN - + TarConstants.GIDLEN, 12, - CharsetNames.UTF_8)); + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + CharsetNames.UTF_8)); assertEquals("101 a=0123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789" - + "0123\n", new String(data, 512, 101, CharsetNames.UTF_8)); + + "01234567890123456789012345678901234567890123456789" + + "0123\n", new String(data, 512, 101, CharsetNames.UTF_8)); } private byte[] writePaxHeader(final Map m) throws Exception { @@ -226,7 +233,7 @@ tos.closeArchiveEntry(); final byte[] data = bos.toByteArray(); assertEquals("160 path=" + n + "\n", - new String(data, 512, 160, CharsetNames.UTF_8)); + new String(data, 512, 160, CharsetNames.UTF_8)); final TarArchiveInputStream tin = new TarArchiveInputStream(new ByteArrayInputStream(data)); final TarArchiveEntry e = tin.getNextTarEntry(); @@ -248,11 +255,11 @@ tos.write(new byte[10 * 1024]); final byte[] data = bos.toByteArray(); assertEquals((byte) 0xff, - data[TarConstants.NAMELEN - + TarConstants.MODELEN - + TarConstants.UIDLEN - + TarConstants.GIDLEN - + TarConstants.SIZELEN]); + data[TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN + + TarConstants.SIZELEN]); final TarArchiveInputStream tin = new TarArchiveInputStream(new ByteArrayInputStream(data)); final TarArchiveEntry e = tin.getNextTarEntry(); @@ -279,13 +286,13 @@ tos.write(new byte[10 * 1024]); final byte[] data = bos.toByteArray(); assertEquals("00000000000 ", - new String(data, - 1024 + TarConstants.NAMELEN - + TarConstants.MODELEN - + TarConstants.UIDLEN - + TarConstants.GIDLEN - + TarConstants.SIZELEN, 12, - CharsetNames.UTF_8)); + new String(data, + 1024 + TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN + + TarConstants.SIZELEN, 12, + CharsetNames.UTF_8)); final TarArchiveInputStream tin = new TarArchiveInputStream(new ByteArrayInputStream(data)); final TarArchiveEntry e = tin.getNextTarEntry(); @@ -328,7 +335,7 @@ tos.close(); final byte[] data = bos.toByteArray(); assertEquals("11 path=" + n + "\n", - new String(data, 512, 11, CharsetNames.UTF_8)); + new String(data, 512, 11, CharsetNames.UTF_8)); final TarArchiveInputStream tin = new TarArchiveInputStream(new ByteArrayInputStream(data)); final TarArchiveEntry e = tin.getNextTarEntry(); @@ -351,7 +358,7 @@ tos.close(); final byte[] data = bos.toByteArray(); assertEquals("15 linkpath=" + n + "\n", - new String(data, 512, 15, CharsetNames.UTF_8)); + new String(data, 512, 15, CharsetNames.UTF_8)); final TarArchiveInputStream tin = new TarArchiveInputStream(new ByteArrayInputStream(data)); final TarArchiveEntry e = tin.getNextTarEntry(); @@ -399,8 +406,8 @@ @Test public void testWriteLongDirectoryNameErrorMode() throws Exception { final String n = "01234567890123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789/"; + + "01234567890123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789/"; try { final TarArchiveEntry t = new TarArchiveEntry(n); @@ -524,8 +531,8 @@ @Test public void testWriteLongLinkNameErrorMode() throws Exception { final String linkname = "01234567890123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789" - + "01234567890123456789012345678901234567890123456789/test"; + + "01234567890123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789/test"; final TarArchiveEntry entry = new TarArchiveEntry("test", TarConstants.LF_SYMLINK); entry.setLinkName(linkname); @@ -548,7 +555,7 @@ final String linkname = "01234567890123456789012345678901234567890123456789" + "01234567890123456789012345678901234567890123456789" + "01234567890123456789012345678901234567890123456789/"; - final TarArchiveEntry entry = new TarArchiveEntry("test" , TarConstants.LF_SYMLINK); + final TarArchiveEntry entry = new TarArchiveEntry("test", TarConstants.LF_SYMLINK); entry.setLinkName(linkname); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); @@ -607,31 +614,136 @@ tin.close(); } + @SuppressWarnings("deprecation") + @Test public void testRecordSize() throws IOException { + try { + TarArchiveOutputStream tos = + new TarArchiveOutputStream(new ByteArrayOutputStream(),512,511); + fail("should have rejected recordSize of 511"); + } catch(IllegalArgumentException e) { + // expected; + } + try { + TarArchiveOutputStream tos = + new TarArchiveOutputStream(new ByteArrayOutputStream(),512,511,null); + fail("should have rejected recordSize of 511"); + } catch(IllegalArgumentException e) { + // expected; + } + try (TarArchiveOutputStream tos = new TarArchiveOutputStream(new ByteArrayOutputStream(), + 512, 512)) { + assertEquals("recordSize",512,tos.getRecordSize()); + } + try (TarArchiveOutputStream tos = new TarArchiveOutputStream(new ByteArrayOutputStream(), + 512, 512, null)) { + assertEquals("recordSize",512,tos.getRecordSize()); + } + } @Test - public void testPadsOutputToFullBlockLength() throws Exception { + public void testBlockSizes() throws Exception { + String fileName = "/test1.xml"; + byte[] contents = getResourceContents(fileName); + testPadding(TarConstants.DEFAULT_BLKSIZE, fileName, contents); // USTAR / pre-pax + testPadding(5120, fileName, contents); // PAX default + testPadding(1<<15, fileName, contents); //PAX max + testPadding(-2, fileName, contents); // don't specify a block size -> use minimum length + try { + testPadding(511, fileName, contents); // don't specify a block size -> use minimum length + fail("should have thrown an illegal argument exception"); + } catch (IllegalArgumentException e) { + //expected + } + try { + testPadding(0, fileName, contents); // don't specify a block size -> use minimum length + fail("should have thrown an illegal argument exception"); + } catch (IllegalArgumentException e) { + //expected + } + // test with "content" that is an exact multiple of record length + contents = new byte[2048]; + java.util.Arrays.fill(contents, (byte) 42); + testPadding(TarConstants.DEFAULT_BLKSIZE, fileName, contents); + } + + private void testPadding(int blockSize, String fileName, byte[] contents) throws IOException { final File f = File.createTempFile("commons-compress-padding", ".tar"); f.deleteOnExit(); final FileOutputStream fos = new FileOutputStream(f); - final TarArchiveOutputStream tos = new TarArchiveOutputStream(fos); - final File file1 = getFile("test1.xml"); - final TarArchiveEntry sEntry = new TarArchiveEntry(file1, file1.getName()); + final TarArchiveOutputStream tos; + if (blockSize != -2) { + tos = new TarArchiveOutputStream(fos, blockSize); + } else { + blockSize = 512; + tos = new TarArchiveOutputStream(fos); + } + TarArchiveEntry sEntry; + sEntry = new TarArchiveEntry(fileName); + sEntry.setSize(contents.length); tos.putArchiveEntry(sEntry); - final FileInputStream in = new FileInputStream(file1); - IOUtils.copy(in, tos); - in.close(); + tos.write(contents); tos.closeArchiveEntry(); tos.close(); - // test1.xml is small enough to fit into the default block size - assertEquals(TarConstants.DEFAULT_BLKSIZE, f.length()); + int fileRecordsSize = (int) Math.ceil((double) contents.length / 512) * 512; + final int headerSize = 512; + final int endOfArchiveSize = 1024; + int unpaddedSize = headerSize + fileRecordsSize + endOfArchiveSize; + int paddedSize = (int) Math.ceil((double)unpaddedSize/blockSize)*blockSize; + assertEquals(paddedSize, f.length()); + } + + private byte[] getResourceContents(String name) throws IOException { + ByteArrayOutputStream bos; + try (InputStream resourceAsStream = getClass().getResourceAsStream(name)) { + bos = new ByteArrayOutputStream(); + IOUtils.copy(resourceAsStream, bos); + } + return bos.toByteArray(); + } + @Test public void testPutGlobalPaxHeaderEntry() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + TarArchiveOutputStream tos = new TarArchiveOutputStream(bos); + int pid = 73; + int globCount = 1; + byte lfPaxGlobalExtendedHeader = TarConstants.LF_PAX_GLOBAL_EXTENDED_HEADER; + TarArchiveEntry globalHeader = new TarArchiveEntry("/tmp/GlobalHead." + pid + "." + globCount, + lfPaxGlobalExtendedHeader); + globalHeader.addPaxHeader("SCHILLY.xattr.user.org.apache.weasels","global-weasels"); + tos.putArchiveEntry(globalHeader); + TarArchiveEntry entry = new TarArchiveEntry("message"); + String x = "If at first you don't succeed, give up"; + entry.setSize(x.length()); + tos.putArchiveEntry(entry); + tos.write(x.getBytes()); + tos.closeArchiveEntry(); + entry = new TarArchiveEntry("counter-message"); + String y = "Nothing succeeds like excess"; + entry.setSize(y.length()); + entry.addPaxHeader("SCHILLY.xattr.user.org.apache.weasels.species","unknown"); + tos.putArchiveEntry(entry); + tos.write(y.getBytes()); + tos.closeArchiveEntry(); + tos.close(); + TarArchiveInputStream in = new TarArchiveInputStream(new ByteArrayInputStream(bos.toByteArray())); + TarArchiveEntry entryIn = in.getNextTarEntry(); + assertNotNull(entryIn); + assertEquals("message",entryIn.getName()); + assertEquals("global-weasels",entryIn.getExtraPaxHeader("SCHILLY.xattr.user.org.apache.weasels")); + Reader reader = new InputStreamReader(in); + for(int i=0;i= 2038 (Java8) + - Java hasn't read the extended time field at all (Java7- or early Java8) + */ + + final boolean zipTimeUsesExtendedTimestampCorrectly = rawZ.equals(m); + final boolean zipTimeUsesExtendedTimestampButUnsigned = year > 2037 && rawZ.getSeconds() == 1; + final boolean zipTimeUsesExtendedTimestamp = zipTimeUsesExtendedTimestampCorrectly + || zipTimeUsesExtendedTimestampButUnsigned; + final Date z = zipTimeUsesExtendedTimestamp ? rawZ : adjustFromGMTToExpectedOffset(rawZ); final Date a = xf.getAccessJavaTime(); @@ -131,65 +163,29 @@ final String modTime = DATE_FORMAT.format(m); final String accTime = DATE_FORMAT.format(a); - if (!zae.isDirectory()) { - final int x = name.lastIndexOf('/'); - final String yearString = name.substring(x + 1); - int year; - try { - year = Integer.parseInt(yearString); - } catch (final NumberFormatException nfe) { - year = -1; + switch (year) { + case 2109: + // All three timestamps have overflowed by 2109. + if (!zipTimeUsesExtendedTimestamp) { + assertEquals("1981-01-01/00:00:02 +0000", zipTime); } - if (year >= 0) { - switch (year) { - case 2107: - if (!zipTimeUsesExtendedTimestamp) { - // Zip time is okay up to 2107. - assertEquals(year + "-01-01/00:00:02 +0000", zipTime); - } - // But the X5455 data has overflowed: - assertEquals("1970-11-24/17:31:45 +0000", modTime); - assertEquals("1970-11-24/17:31:47 +0000", accTime); - break; - case 2108: - if (!zipTimeUsesExtendedTimestamp) { - // Zip time is still okay at Jan 1st midnight (UTC) in 2108 - // because we created the zip file in pacific time zone, so it's - // actually still 2107 in the zip file! - assertEquals(year + "-01-01/00:00:02 +0000", zipTime); - } - // The X5455 data is still overflowed, of course: - assertEquals("1971-11-24/17:31:45 +0000", modTime); - assertEquals("1971-11-24/17:31:47 +0000", accTime); - break; - case 2109: - // All three timestamps have overflowed by 2109. - if (!zipTimeUsesExtendedTimestamp) { - assertEquals("1981-01-01/00:00:02 +0000", zipTime); - } - assertEquals("1972-11-24/17:31:45 +0000", modTime); - assertEquals("1972-11-24/17:31:47 +0000", accTime); - - // Hmmm.... looks like one could examine both DOS time - // and the Unix time together to hack a nice workaround to - // get timestamps past 2106 in a reverse-compatible way. - - break; - default: - if (!zipTimeUsesExtendedTimestamp) { - // X5455 time is good from epoch (1970) to 2106. - // Zip time is good from 1980 to 2107. - if (year < 1980) { - assertEquals("1980-01-01/08:00:00 +0000", zipTime); - } else { - assertEquals(year + "-01-01/00:00:02 +0000", zipTime); - } - } - assertEquals(year + "-01-01/00:00:01 +0000", modTime); - assertEquals(year + "-01-01/00:00:03 +0000", accTime); - break; + break; + default: + if (!zipTimeUsesExtendedTimestamp) { + // X5455 time is good from epoch (1970) to 2037. + // Zip time is good from 1980 to 2107. + if (year < 1980) { + assertEquals("1980-01-01/08:00:00 +0000", zipTime); + } else { + assertEquals(year + "-01-01/00:00:02 +0000", zipTime); } } + + if (year < 2038) { + assertEquals(year + "-01-01/00:00:01 +0000", modTime); + assertEquals(year + "-01-01/00:00:03 +0000", accTime); + } + break; } } } finally { @@ -235,9 +231,10 @@ cal.set(Calendar.DATE, 1); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); - final Date timeMillis = cal.getTime(); - final ZipLong time = new ZipLong(timeMillis.getTime() / 1000); + final long timeMillis = cal.getTimeInMillis(); + final ZipLong time = new ZipLong(timeMillis / 1000); // set too big try { @@ -251,14 +248,15 @@ // get/set modify time xf.setModifyTime(time); assertEquals(time, xf.getModifyTime()); - assertEquals(timeMillis, xf.getModifyJavaTime()); - xf.setModifyJavaTime(timeMillis); + Date xfModifyJavaTime = xf.getModifyJavaTime(); + assertEquals(timeMillis, xfModifyJavaTime.getTime()); + xf.setModifyJavaTime(new Date(timeMillis)); assertEquals(time, xf.getModifyTime()); - assertEquals(timeMillis, xf.getModifyJavaTime()); + assertEquals(timeMillis, xf.getModifyJavaTime().getTime()); // Make sure milliseconds get zeroed out: - xf.setModifyJavaTime(new Date(timeMillis.getTime() + 123)); + xf.setModifyJavaTime(new Date(timeMillis + 123)); assertEquals(time, xf.getModifyTime()); - assertEquals(timeMillis, xf.getModifyJavaTime()); + assertEquals(timeMillis, xf.getModifyJavaTime().getTime()); // Null xf.setModifyTime(null); assertNull(xf.getModifyJavaTime()); @@ -268,14 +266,14 @@ // get/set access time xf.setAccessTime(time); assertEquals(time, xf.getAccessTime()); - assertEquals(timeMillis, xf.getAccessJavaTime()); - xf.setAccessJavaTime(timeMillis); + assertEquals(timeMillis, xf.getAccessJavaTime().getTime()); + xf.setAccessJavaTime(new Date(timeMillis)); assertEquals(time, xf.getAccessTime()); - assertEquals(timeMillis, xf.getAccessJavaTime()); + assertEquals(timeMillis, xf.getAccessJavaTime().getTime()); // Make sure milliseconds get zeroed out: - xf.setAccessJavaTime(new Date(timeMillis.getTime() + 123)); + xf.setAccessJavaTime(new Date(timeMillis + 123)); assertEquals(time, xf.getAccessTime()); - assertEquals(timeMillis, xf.getAccessJavaTime()); + assertEquals(timeMillis, xf.getAccessJavaTime().getTime()); // Null xf.setAccessTime(null); assertNull(xf.getAccessJavaTime()); @@ -285,14 +283,14 @@ // get/set create time xf.setCreateTime(time); assertEquals(time, xf.getCreateTime()); - assertEquals(timeMillis, xf.getCreateJavaTime()); - xf.setCreateJavaTime(timeMillis); + assertEquals(timeMillis, xf.getCreateJavaTime().getTime()); + xf.setCreateJavaTime(new Date(timeMillis)); assertEquals(time, xf.getCreateTime()); - assertEquals(timeMillis, xf.getCreateJavaTime()); + assertEquals(timeMillis, xf.getCreateJavaTime().getTime()); // Make sure milliseconds get zeroed out: - xf.setCreateJavaTime(new Date(timeMillis.getTime() + 123)); + xf.setCreateJavaTime(new Date(timeMillis + 123)); assertEquals(time, xf.getCreateTime()); - assertEquals(timeMillis, xf.getCreateJavaTime()); + assertEquals(timeMillis, xf.getCreateJavaTime().getTime()); // Null xf.setCreateTime(null); assertNull(xf.getCreateJavaTime()); @@ -388,15 +386,15 @@ final byte[] CR_CENTRAL = {4}; // central data only dontains the CR flag and no actual data final byte[] MOD_ZERO = {1, 0, 0, 0, 0}; - final byte[] MOD_MAX = {1, -1, -1, -1, -1}; + final byte[] MOD_MAX = {1, -1, -1, -1, 0x7f}; final byte[] AC_ZERO = {2, 0, 0, 0, 0}; - final byte[] AC_MAX = {2, -1, -1, -1, -1}; + final byte[] AC_MAX = {2, -1, -1, -1, 0x7f}; final byte[] CR_ZERO = {4, 0, 0, 0, 0}; - final byte[] CR_MAX = {4, -1, -1, -1, -1}; + final byte[] CR_MAX = {4, -1, -1, -1, 0x7f}; final byte[] MOD_AC_ZERO = {3, 0, 0, 0, 0, 0, 0, 0, 0}; - final byte[] MOD_AC_MAX = {3, -1, -1, -1, -1, -1, -1, -1, -1}; + final byte[] MOD_AC_MAX = {3, -1, -1, -1, 0x7f, -1, -1, -1, 0x7f}; final byte[] MOD_AC_CR_ZERO = {7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - final byte[] MOD_AC_CR_MAX = {7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + final byte[] MOD_AC_CR_MAX = {7, -1, -1, -1, 0x7f, -1, -1, -1, 0x7f, -1, -1, -1, 0x7f}; parseReparse(null, NULL_FLAGS, NULL_FLAGS); parseReparse(ZERO_TIME, MOD_ZERO, MOD_ZERO); @@ -441,7 +439,7 @@ } } out.close(); - + final ZipFile zf = new ZipFile(output); final ZipArchiveEntry ze = zf.getEntry("foo"); final X5455_ExtendedTimestamp ext = diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportIT.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportIT.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportIT.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportIT.java 2018-05-02 20:17:13.000000000 +0000 @@ -386,8 +386,8 @@ a.skipBytes(2 * 47 /* CD entry of file with file name length 1 and no extra data */ - + 2 * (mode == Zip64Mode.Always ? 4 : 0) - /* empty ZIP64 extra fields if mode is Always */ + + 2 * (mode == Zip64Mode.Always ? 28 : 0) + /* ZIP64 extra fields if mode is Always */ ); // grab third entry, verify offset is @@ -395,7 +395,7 @@ // information extra field final byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -409,15 +409,25 @@ }, header); // ignore timestamp, CRC, compressed size a.skipBytes(12); - final byte[] rest = new byte[23]; + // Original Size + final byte[] originalSize = new byte[4]; + a.readFully(originalSize); + if (mode == Zip64Mode.Always) { + assertArrayEquals("CDH original size", new byte[] { + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + }, originalSize); + } else { + assertArrayEquals("CDH original size", new byte[] { + 1, 0, 0, 0 + }, originalSize); + } + final byte[] rest = new byte[19]; a.readFully(rest); - assertArrayEquals(new byte[] { - // Original Size - 1, 0, 0, 0, + assertArrayEquals("CDH rest", new byte[] { // file name length 1, 0, // extra field length - 12, 0, + (byte) (mode == Zip64Mode.Always? 28 : 12), 0, // comment length 0, 0, // disk number @@ -431,14 +441,29 @@ // file name (byte) '2' }, rest); - final byte[] extra = new byte[4]; - a.readFully(extra); - assertArrayEquals(new byte[] { - // Header-ID - 1, 0, - // size - 8, 0 - }, extra); + if (mode == Zip64Mode.Always) { + final byte[] extra = new byte[12]; + a.readFully(extra); + assertArrayEquals("CDH extra", new byte[] { + // Header-ID + 1, 0, + // size + 24, 0, + // Original Size + 1, 0, 0, 0, 0, 0, 0, 0, + }, extra); + // skip compressed size + a.skipBytes(8); + } else { + final byte[] extra = new byte[4]; + a.readFully(extra); + assertArrayEquals("CDH extra", new byte[] { + // Header-ID + 1, 0, + // size + 8, 0, + }, extra); + } // read offset of LFH final byte[] offset = new byte[8]; @@ -447,7 +472,7 @@ a.seek(ZipEightByteInteger.getLongValue(offset)); final byte[] sig = new byte[4]; a.readFully(sig); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH signature", new byte[] { (byte) 0x50, (byte) 0x4b, 3, 4, }, sig); } @@ -594,7 +619,7 @@ // field byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -608,9 +633,9 @@ }, header); // ignore timestamp a.skipBytes(4); - byte[] rest = new byte[31]; + byte[] rest = new byte[26]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH rest", new byte[] { // CRC (byte) 0x50, (byte) 0x6F, (byte) 0x31, (byte) 0x5c, // Compressed Size @@ -620,7 +645,7 @@ // file name length 1, 0, // extra field length - 20, 0, + (byte) (mode == Zip64Mode.Always? 28 : 20), 0, // comment length 0, 0, // disk number @@ -628,19 +653,27 @@ // attributes 0, 0, 0, 0, 0, 0, - // offset - 0, 0, 0, 0, - // file name - (byte) '0' }, rest); + byte[] offset = new byte[4]; + a.readFully(offset); + if (mode == Zip64Mode.Always) { + assertArrayEquals("offset", new byte[] { + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + }, offset); + } else { + assertArrayEquals("offset", new byte[] { + 0, 0, 0, 0, + }, offset); + } + assertEquals('0', a.read()); final byte[] extra = new byte[20]; a.readFully(extra); // 5e9 == 0x12A05F200 - assertArrayEquals(new byte[] { + assertArrayEquals("CDH extra", new byte[] { // Header-ID 1, 0, // size of extra - 16, 0, + (byte) (mode == Zip64Mode.Always? 24 : 16), 0, // original size 0, (byte) 0xF2, 5, (byte) 0x2A, 1, 0, 0, 0, @@ -648,12 +681,19 @@ 0, (byte) 0xF2, 5, (byte) 0x2A, 1, 0, 0, 0, }, extra); + if (mode == Zip64Mode.Always) { + offset = new byte[8]; + a.readFully(offset); + assertArrayEquals("extra offset", new byte[] { + 0, 0, 0, 0, 0, 0, 0, 0, + }, offset); + } // and now validate local file header a.seek(0); header = new byte[10]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract @@ -667,7 +707,7 @@ a.skipBytes(4); rest = new byte[17]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH rest", new byte[] { // CRC (byte) 0x50, (byte) 0x6F, (byte) 0x31, (byte) 0x5c, // Compressed Size @@ -683,7 +723,7 @@ }, rest); a.readFully(extra); // 5e9 == 0x12A05F200 - assertArrayEquals(new byte[] { + assertArrayEquals("LFH extra", new byte[] { // Header-ID 1, 0, // size of extra @@ -833,7 +873,7 @@ // information extra field byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -847,9 +887,9 @@ }, header); // ignore timestamp a.skipBytes(4); - byte[] rest = new byte[31]; + byte[] rest = new byte[26]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH rest", new byte[] { // CRC (byte) 0x50, (byte) 0x6F, (byte) 0x31, (byte) 0x5c, // Compressed Size @@ -859,7 +899,7 @@ // file name length 1, 0, // extra field length - 20, 0, + (byte) (mode == Zip64Mode.Always? 28 : 20), 0, // comment length 0, 0, // disk number @@ -867,43 +907,56 @@ // attributes 0, 0, 0, 0, 0, 0, - // offset - 0, 0, 0, 0, - // file name - (byte) '0' }, rest); - final byte[] extra = new byte[20]; + byte[] offset = new byte[4]; + a.readFully(offset); + if (mode == Zip64Mode.Always) { + assertArrayEquals("offset", new byte[] { + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + }, offset); + } else { + assertArrayEquals("offset", new byte[] { + 0, 0, 0, 0, + }, offset); + } + assertEquals('0', a.read()); + byte[] extra = new byte[12]; a.readFully(extra); // 5e9 == 0x12A05F200 - assertArrayEquals(new byte[] { + assertArrayEquals("CDH extra", new byte[] { // Header-ID 1, 0, // size of extra - 16, 0, + (byte) (mode == Zip64Mode.Always? 24 : 16), 0, // original size 0, (byte) 0xF2, 5, (byte) 0x2A, 1, 0, 0, 0, - // compressed size - (byte) 0x68, (byte) 0x27, (byte) 0x4A, 0, - 0, 0, 0, 0, }, extra); + if (mode == Zip64Mode.Always) { + // skip compressed size + a.skipBytes(8); + offset = new byte[8]; + a.readFully(offset); + assertArrayEquals("extra offset", new byte[] { + 0, 0, 0, 0, 0, 0, 0, 0, + }, offset); + } // validate data descriptor a.seek(cfhPos - 24); byte[] dd = new byte[8]; a.readFully(dd); - assertArrayEquals(new byte[] { + assertArrayEquals("DD", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 7, 8, // CRC (byte) 0x50, (byte) 0x6F, (byte) 0x31, (byte) 0x5c, }, dd); - dd = new byte[16]; + // skip compressed size + a.skipBytes(8); + dd = new byte[8]; a.readFully(dd); - assertArrayEquals(new byte[] { - // compressed size - (byte) 0x68, (byte) 0x27, (byte) 0x4A, 0, - 0, 0, 0, 0, + assertArrayEquals("DD sizes", new byte[] { // original size 0, (byte) 0xF2, 5, (byte) 0x2A, 1, 0, 0, 0, @@ -913,7 +966,7 @@ a.seek(0); header = new byte[10]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract @@ -927,7 +980,7 @@ a.skipBytes(4); rest = new byte[17]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH rest", new byte[] { // CRC 0, 0, 0, 0, // Compressed Size @@ -941,8 +994,9 @@ // file name (byte) '0' }, rest); + extra = new byte[20]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH extra", new byte[] { // Header-ID 1, 0, // size of extra @@ -1073,7 +1127,7 @@ // information extra field byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -1087,9 +1141,9 @@ }, header); // ignore timestamp a.skipBytes(4); - byte[] rest = new byte[31]; + byte[] rest = new byte[26]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH rest", new byte[] { // CRC (byte) 0x50, (byte) 0x6F, (byte) 0x31, (byte) 0x5c, // Compressed Size @@ -1099,7 +1153,7 @@ // file name length 1, 0, // extra field length - 20, 0, + (byte) (mode == Zip64Mode.Always? 28 : 20), 0, // comment length 0, 0, // disk number @@ -1107,32 +1161,46 @@ // attributes 0, 0, 0, 0, 0, 0, - // offset - 0, 0, 0, 0, - // file name - (byte) '0' }, rest); - byte[] extra = new byte[20]; + byte[] offset = new byte[4]; + a.readFully(offset); + if (mode == Zip64Mode.Always) { + assertArrayEquals("offset", new byte[] { + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + }, offset); + } else { + assertArrayEquals("offset", new byte[] { + 0, 0, 0, 0, + }, offset); + } + assertEquals('0', a.read()); + byte[] extra = new byte[12]; a.readFully(extra); // 5e9 == 0x12A05F200 - assertArrayEquals(new byte[] { + assertArrayEquals("CDH extra", new byte[] { // Header-ID 1, 0, // size of extra - 16, 0, + (byte) (mode == Zip64Mode.Always? 24 : 16), 0, // original size 0, (byte) 0xF2, 5, (byte) 0x2A, 1, 0, 0, 0, - // compressed size - (byte) 0x68, (byte) 0x27, (byte) 0x4A, 0, - 0, 0, 0, 0, }, extra); + if (mode == Zip64Mode.Always) { + // skip compressed size + a.skipBytes(8); + offset = new byte[8]; + a.readFully(offset); + assertArrayEquals("extra offset", new byte[] { + 0, 0, 0, 0, 0, 0, 0, 0, + }, offset); + } // and now validate local file header a.seek(0); header = new byte[10]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract @@ -1160,7 +1228,7 @@ // file name (byte) '0' }, rest); - extra = new byte[20]; + extra = new byte[12]; a.readFully(extra); assertArrayEquals(new byte[] { // Header-ID @@ -1170,9 +1238,7 @@ // original size 0, (byte) 0xF2, 5, (byte) 0x2A, 1, 0, 0, 0, - // compressed size - (byte) 0x68, (byte) 0x27, (byte) 0x4A, 0, - 0, 0, 0, 0, + // skip compressed size }, extra); } } @@ -1328,7 +1394,7 @@ // at all byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -1345,7 +1411,7 @@ byte[] rest = new byte[31]; a.readFully(rest); // 1e6 == 0xF4240 - assertArrayEquals(new byte[] { + assertArrayEquals("CDH rest", new byte[] { // CRC (byte) 0x9E, (byte) 0xCB, (byte) 0x79, (byte) 0x12, // Compressed Size @@ -1378,7 +1444,7 @@ a.seek(0); header = new byte[10]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract @@ -1393,7 +1459,7 @@ rest = new byte[17]; a.readFully(rest); // 1e6 == 0xF4240 - assertArrayEquals(new byte[] { + assertArrayEquals("LFH rest", new byte[] { // CRC (byte) 0x9E, (byte) 0xCB, (byte) 0x79, (byte) 0x12, // Compressed Size @@ -1410,7 +1476,7 @@ if (hasExtra) { final byte[] extra = new byte[20]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("ZIP64 extra field", new byte[] { // Header-ID 1, 0, // size of extra @@ -1501,7 +1567,7 @@ // has an empty ZIP64 extended information extra field byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -1518,17 +1584,17 @@ byte[] rest = new byte[31]; a.readFully(rest); // 1e6 == 0xF4240 - assertArrayEquals(new byte[] { + assertArrayEquals("CDH rest", new byte[] { // CRC (byte) 0x9E, (byte) 0xCB, (byte) 0x79, (byte) 0x12, // Compressed Size - (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, // Original Size - (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, // file name length 1, 0, // extra field length - 4, 0, + 28, 0, // comment length 0, 0, // disk number @@ -1537,18 +1603,25 @@ 0, 0, 0, 0, 0, 0, // offset - 0, 0, 0, 0, + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, // file name (byte) '0' }, rest); - byte[] extra = new byte[4]; + byte[] extra = new byte[28]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH extra", new byte[] { // Header-ID 1, 0, // size of extra - 0, 0, + 24, 0, + // original size + (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, + 0, 0, 0, 0, + // compressed size + (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, }, extra); // and now validate local file header: this one @@ -1557,7 +1630,7 @@ a.seek(0); header = new byte[10]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract @@ -1572,7 +1645,7 @@ rest = new byte[17]; a.readFully(rest); // 1e6 == 0xF4240 - assertArrayEquals(new byte[] { + assertArrayEquals("LFH rest", new byte[] { // CRC (byte) 0x9E, (byte) 0xCB, (byte) 0x79, (byte) 0x12, // Compressed Size @@ -1589,7 +1662,7 @@ extra = new byte[20]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH extra", new byte[] { // Header-ID 1, 0, // size of extra @@ -1832,7 +1905,7 @@ // information extra field byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -1856,14 +1929,13 @@ a.skipBytes(4); byte[] rest = new byte[23]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH rest", new byte[] { // Original Size - (byte) 0x40, (byte) 0x42, - (byte) 0x0F, 0, + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, // file name length 1, 0, // extra field length - 4, 0, + 28, 0, // comment length 0, 0, // disk number @@ -1872,24 +1944,34 @@ 0, 0, 0, 0, 0, 0, // offset - 0, 0, 0, 0, + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, // file name (byte) '0' }, rest); - byte[] extra = new byte[4]; + byte[] extra = new byte[12]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH extra", new byte[] { // Header-ID 1, 0, // size of extra - 0, 0, + 24, 0, + // original size + (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, + 0, 0, 0, 0, }, extra); + // skip compressed size + a.skipBytes(8); + byte[] offset = new byte[8]; + a.readFully(offset); + assertArrayEquals("extra offset", new byte[] { + 0, 0, 0, 0, 0, 0, 0, 0, + }, offset); // validate data descriptor a.seek(cfhPos - 24); byte[] dd = new byte[8]; a.readFully(dd); - assertArrayEquals(new byte[] { + assertArrayEquals("DD", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 7, 8, // CRC @@ -1899,7 +1981,7 @@ a.skipBytes(8); dd = new byte[8]; a.readFully(dd); - assertArrayEquals(new byte[] { + assertArrayEquals("DD size", new byte[] { // original size (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, 0, 0, 0, 0 @@ -1909,7 +1991,7 @@ a.seek(0); header = new byte[10]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract @@ -1923,7 +2005,7 @@ a.skipBytes(4); rest = new byte[17]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH rest", new byte[] { // CRC 0, 0, 0, 0, // Compressed Size @@ -1940,7 +2022,7 @@ extra = new byte[20]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH extra", new byte[] { // Header-ID 1, 0, // size of extra @@ -2019,7 +2101,7 @@ // version made by 20, 0, // version needed to extract - 10, 0, + 20, 0, // GPB (EFS + *no* Data Descriptor) 0, 8, // method @@ -2065,7 +2147,7 @@ // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract - 10, 0, + 20, 0, // GPB (EFS bit, no DD) 0, 8, // method @@ -2183,7 +2265,7 @@ // information extra field byte[] header = new byte[12]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 1, 2, // version made by @@ -2199,20 +2281,20 @@ a.skipBytes(4); byte[] crc = new byte[4]; a.readFully(crc); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH CRC", new byte[] { (byte) 0x9E, (byte) 0xCB, (byte) 0x79, (byte) 0x12, }, crc); // skip compressed size a.skipBytes(4); byte[] rest = new byte[23]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH rest", new byte[] { // Original Size - (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, // file name length 1, 0, // extra field length - 4, 0, + 28, 0, // comment length 0, 0, // disk number @@ -2221,24 +2303,34 @@ 0, 0, 0, 0, 0, 0, // offset - 0, 0, 0, 0, + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, // file name (byte) '0' }, rest); - byte[] extra = new byte[4]; + byte[] extra = new byte[12]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("CDH extra", new byte[] { // Header-ID 1, 0, // size of extra - 0, 0, + 24, 0, + // original size + (byte) 0x40, (byte) 0x42, (byte) 0x0F, 0, + 0, 0, 0, 0, }, extra); + // skip compressed size + a.skipBytes(8); + byte[] offset = new byte[8]; + a.readFully(offset); + assertArrayEquals("extra offset", new byte[] { + 0, 0, 0, 0, 0, 0, 0, 0, + }, offset); // and now validate local file header a.seek(0); header = new byte[10]; a.readFully(header); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH start", new byte[] { // sig (byte) 0x50, (byte) 0x4b, 3, 4, // version needed to extract @@ -2252,14 +2344,14 @@ a.skipBytes(4); crc = new byte[4]; a.readFully(crc); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH CRC", new byte[] { (byte) 0x9E, (byte) 0xCB, (byte) 0x79, (byte) 0x12, }, crc); rest = new byte[13]; a.readFully(rest); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH rest", new byte[] { // Compressed Size (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, // Original Size @@ -2274,7 +2366,7 @@ extra = new byte[12]; a.readFully(extra); - assertArrayEquals(new byte[] { + assertArrayEquals("LFH extra", new byte[] { // Header-ID 1, 0, // size of extra diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -18,6 +18,7 @@ package org.apache.commons.compress.archivers.zip; +import static org.apache.commons.compress.AbstractTestCase.getFile; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; @@ -268,4 +269,25 @@ final ZipArchiveEntry copy = new ZipArchiveEntry(archiveEntry); assertEquals(archiveEntry, copy); } + + /** + * @see "https://issues.apache.org/jira/browse/COMPRESS-379" + */ + @Test + public void isUnixSymlinkIsFalseIfMoreThanOneFlagIsSet() throws Exception { + try (ZipFile zf = new ZipFile(getFile("COMPRESS-379.jar"))) { + ZipArchiveEntry ze = zf.getEntry("META-INF/maven/"); + assertFalse(ze.isUnixSymlink()); + } + } + + @Test + public void testIsUnixSymlink() { + ZipArchiveEntry ze = new ZipArchiveEntry("foo"); + ze.setUnixMode(UnixStat.LINK_FLAG); + assertTrue(ze.isUnixSymlink()); + ze.setUnixMode(UnixStat.LINK_FLAG | UnixStat.DIR_FLAG); + assertFalse(ze.isUnixSymlink()); + } + } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java 2018-08-09 18:31:11.000000000 +0000 @@ -21,18 +21,23 @@ import static org.apache.commons.compress.AbstractTestCase.getFile; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.EOFException; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; import java.util.zip.ZipException; +import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; import org.junit.Test; public class ZipArchiveInputStreamTest { @@ -130,22 +135,22 @@ @Test public void testUnshrinkEntry() throws Exception { final ZipArchiveInputStream in = new ZipArchiveInputStream(new FileInputStream(getFile("SHRUNK.ZIP"))); - + ZipArchiveEntry entry = in.getNextZipEntry(); assertEquals("method", ZipMethod.UNSHRINKING.getCode(), entry.getMethod()); assertTrue(in.canReadEntryData(entry)); - + FileInputStream original = new FileInputStream(getFile("test1.xml")); try { assertArrayEquals(IOUtils.toByteArray(original), IOUtils.toByteArray(in)); } finally { original.close(); } - + entry = in.getNextZipEntry(); assertEquals("method", ZipMethod.UNSHRINKING.getCode(), entry.getMethod()); assertTrue(in.canReadEntryData(entry)); - + original = new FileInputStream(getFile("test2.xml")); try { assertArrayEquals(IOUtils.toByteArray(original), IOUtils.toByteArray(in)); @@ -156,7 +161,7 @@ /** - * Test case for + * Test case for * COMPRESS-264. */ @@ -203,6 +208,41 @@ } /** + * @see "https://issues.apache.org/jira/browse/COMPRESS-380" + */ + @Test + public void readDeflate64CompressedStream() throws Exception { + final File input = getFile("COMPRESS-380/COMPRESS-380-input"); + final File archive = getFile("COMPRESS-380/COMPRESS-380.zip"); + try (FileInputStream in = new FileInputStream(input); + ZipArchiveInputStream zin = new ZipArchiveInputStream(new FileInputStream(archive))) { + byte[] orig = IOUtils.toByteArray(in); + ZipArchiveEntry e = zin.getNextZipEntry(); + byte[] fromZip = IOUtils.toByteArray(zin); + assertArrayEquals(orig, fromZip); + } + } + + @Test + public void readDeflate64CompressedStreamWithDataDescriptor() throws Exception { + // this is a copy of bla.jar with META-INF/MANIFEST.MF's method manually changed to ENHANCED_DEFLATED + final File archive = getFile("COMPRESS-380/COMPRESS-380-dd.zip"); + try (ZipArchiveInputStream zin = new ZipArchiveInputStream(new FileInputStream(archive))) { + ZipArchiveEntry e = zin.getNextZipEntry(); + assertEquals(-1, e.getSize()); + assertEquals(ZipMethod.ENHANCED_DEFLATED.getCode(), e.getMethod()); + byte[] fromZip = IOUtils.toByteArray(zin); + byte[] expected = new byte[] { + 'M', 'a', 'n', 'i', 'f', 'e', 's', 't', '-', 'V', 'e', 'r', 's', 'i', 'o', 'n', ':', ' ', '1', '.', '0', + '\r', '\n', '\r', '\n' + }; + assertArrayEquals(expected, fromZip); + zin.getNextZipEntry(); + assertEquals(25, e.getSize()); + } + } + + /** * Test case for * COMPRESS-364. @@ -248,6 +288,208 @@ } } + /** + * Test correct population of header and data offsets. + */ + @Test + public void testOffsets() throws Exception { + // mixed.zip contains both inflated and stored files + try (InputStream archiveStream = ZipArchiveInputStream.class.getResourceAsStream("/mixed.zip"); + ZipArchiveInputStream zipStream = new ZipArchiveInputStream((archiveStream)) + ) { + ZipArchiveEntry inflatedEntry = zipStream.getNextZipEntry(); + Assert.assertEquals("inflated.txt", inflatedEntry.getName()); + Assert.assertEquals(0x0000, inflatedEntry.getLocalHeaderOffset()); + Assert.assertEquals(0x0046, inflatedEntry.getDataOffset()); + ZipArchiveEntry storedEntry = zipStream.getNextZipEntry(); + Assert.assertEquals("stored.txt", storedEntry.getName()); + Assert.assertEquals(0x5892, storedEntry.getLocalHeaderOffset()); + Assert.assertEquals(0x58d6, storedEntry.getDataOffset()); + Assert.assertNull(zipStream.getNextZipEntry()); + } + } + + @Test + public void nameSourceDefaultsToName() throws Exception { + nameSource("bla.zip", "test1.xml", ZipArchiveEntry.NameSource.NAME); + } + + @Test + public void nameSourceIsSetToUnicodeExtraField() throws Exception { + nameSource("utf8-winzip-test.zip", "\u20AC_for_Dollar.txt", + ZipArchiveEntry.NameSource.UNICODE_EXTRA_FIELD); + } + + @Test + public void nameSourceIsSetToEFS() throws Exception { + nameSource("utf8-7zip-test.zip", "\u20AC_for_Dollar.txt", 3, + ZipArchiveEntry.NameSource.NAME_WITH_EFS_FLAG); + } + + @Test + public void properlyMarksEntriesAsUnreadableIfUncompressedSizeIsUnknown() throws Exception { + // we never read any data + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteArrayInputStream(new byte[0]))) { + ZipArchiveEntry e = new ZipArchiveEntry("test"); + e.setMethod(ZipMethod.DEFLATED.getCode()); + assertTrue(zis.canReadEntryData(e)); + e.setMethod(ZipMethod.ENHANCED_DEFLATED.getCode()); + assertTrue(zis.canReadEntryData(e)); + e.setMethod(ZipMethod.BZIP2.getCode()); + assertFalse(zis.canReadEntryData(e)); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("bla.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingStore() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-264.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingUnshrink() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("SHRUNK.ZIP")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingExplode() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("imploding-8Kdict-3trees.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate64() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-380/COMPRESS-380.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingBzip2() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("bzip2-zip.zip")); + } + + private void singleByteReadConsistentlyReturnsMinusOneAtEof(File file) throws Exception { + try (FileInputStream in = new FileInputStream(file); + ZipArchiveInputStream archive = new ZipArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read()); + assertEquals(-1, archive.read()); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("bla.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingStore() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-264.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingUnshrink() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("SHRUNK.ZIP")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingExplode() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("imploding-8Kdict-3trees.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate64() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-380/COMPRESS-380.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingBzip2() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("bzip2-zip.zip")); + } + + private void multiByteReadConsistentlyReturnsMinusOneAtEof(File file) throws Exception { + byte[] buf = new byte[2]; + try (FileInputStream in = new FileInputStream(getFile("bla.zip")); + ZipArchiveInputStream archive = new ZipArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + IOUtils.toByteArray(archive); + assertEquals(-1, archive.read(buf)); + assertEquals(-1, archive.read(buf)); + } + } + + @Test + public void singleByteReadThrowsAtEofForCorruptedStoredEntry() throws Exception { + byte[] content; + try (FileInputStream fs = new FileInputStream(getFile("COMPRESS-264.zip"))) { + content = IOUtils.toByteArray(fs); + } + // make size much bigger than entry's real size + for (int i = 17; i < 26; i++) { + content[i] = (byte) 0xff; + } + try (ByteArrayInputStream in = new ByteArrayInputStream(content); + ZipArchiveInputStream archive = new ZipArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + try { + IOUtils.toByteArray(archive); + fail("expected exception"); + } catch (IOException ex) { + assertEquals("Truncated ZIP file", ex.getMessage()); + } + try { + archive.read(); + fail("expected exception"); + } catch (IOException ex) { + assertEquals("Truncated ZIP file", ex.getMessage()); + } + try { + archive.read(); + fail("expected exception"); + } catch (IOException ex) { + assertEquals("Truncated ZIP file", ex.getMessage()); + } + } + } + + @Test + public void multiByteReadThrowsAtEofForCorruptedStoredEntry() throws Exception { + byte[] content; + try (FileInputStream fs = new FileInputStream(getFile("COMPRESS-264.zip"))) { + content = IOUtils.toByteArray(fs); + } + // make size much bigger than entry's real size + for (int i = 17; i < 26; i++) { + content[i] = (byte) 0xff; + } + byte[] buf = new byte[2]; + try (ByteArrayInputStream in = new ByteArrayInputStream(content); + ZipArchiveInputStream archive = new ZipArchiveInputStream(in)) { + ArchiveEntry e = archive.getNextEntry(); + try { + IOUtils.toByteArray(archive); + fail("expected exception"); + } catch (IOException ex) { + assertEquals("Truncated ZIP file", ex.getMessage()); + } + try { + archive.read(buf); + fail("expected exception"); + } catch (IOException ex) { + assertEquals("Truncated ZIP file", ex.getMessage()); + } + try { + archive.read(buf); + fail("expected exception"); + } catch (IOException ex) { + assertEquals("Truncated ZIP file", ex.getMessage()); + } + } + } + private static byte[] readEntry(ZipArchiveInputStream zip, ZipArchiveEntry zae) throws IOException { final int len = (int)zae.getSize(); final byte[] buff = new byte[len]; @@ -255,4 +497,20 @@ return buff; } + + private static void nameSource(String archive, String entry, ZipArchiveEntry.NameSource expected) throws Exception { + nameSource(archive, entry, 1, expected); + } + + private static void nameSource(String archive, String entry, int entryNo, ZipArchiveEntry.NameSource expected) + throws Exception { + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new FileInputStream(getFile(archive)))) { + ZipArchiveEntry ze; + do { + ze = zis.getNextZipEntry(); + } while (--entryNo > 0); + assertEquals(entry, ze.getName()); + assertEquals(expected, ze.getNameSource()); + } + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipClassCoverageTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipClassCoverageTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipClassCoverageTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipClassCoverageTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.archivers.zip; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.hamcrest.core.IsInstanceOf; +import org.junit.Test; + +public class ZipClassCoverageTest { + + @Test + public void testConstructZip64RequiredException() { + Zip64RequiredException e = new Zip64RequiredException("critique of pure"); + assertNotNull(e); + } + @Test + public void testMessageException() { + ZipArchiveEntry ze = new ZipArchiveEntry("hello"); + String entryTooBigMessage = Zip64RequiredException.getEntryTooBigMessage(ze); + assertEquals("hello's size exceeds the limit of 4GByte.", + entryTooBigMessage); + } + + @Test + public void testConstantConstructor() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + Class clazz = ZipConstants.class; + Constructor constructor = clazz.getDeclaredConstructor(); + assertFalse(constructor.isAccessible()); + constructor.setAccessible(true); + Object o = constructor.newInstance(); + assertThat(o, IsInstanceOf.instanceOf(clazz)); + constructor.setAccessible(false); + + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipEncodingTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipEncodingTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipEncodingTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipEncodingTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -19,13 +19,15 @@ package org.apache.commons.compress.archivers.zip; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.nio.ByteBuffer; - -import org.apache.commons.compress.utils.CharsetNames; - -import static org.junit.Assert.*; - +import java.nio.charset.Charset; +import org.hamcrest.core.IsInstanceOf; import org.junit.Assert; import org.junit.Test; @@ -33,6 +35,7 @@ * Test zip encodings. */ public class ZipEncodingTest { + private static final String UNENC_STRING = "\u2016"; // stress test for internal grow method. @@ -43,15 +46,53 @@ "%U2016%U2015%U2016%U2015%U2016%U2015%U2016%U2015%U2016%U2015%U2016"; @Test - public void testSimpleCp437Encoding() throws IOException { + public void testNothingToMakeCoverallsHappier() { + Object o = new ZipEncodingHelper() { + }; + assertNotNull(o); + } + + @Test + public void testGetNonexistentEncodng() throws IOException { + ZipEncoding ze = ZipEncodingHelper.getZipEncoding("I-am-a-banana"); + assertNotNull(ze); + if (ze instanceof CharsetAccessor) { + CharsetAccessor hasCharset = (CharsetAccessor) ze; + Assert.assertEquals(Charset.defaultCharset(), hasCharset.getCharset()); + } + } + + @Test + public void testIsUTF8() throws IOException { + assertTrue(ZipEncodingHelper.isUTF8("UTF-8")); + assertTrue(ZipEncodingHelper.isUTF8("UTF8")); + Assert.assertEquals(Charset.defaultCharset().name().equals("UTF-8"), ZipEncodingHelper.isUTF8(null)); + } - doSimpleEncodingTest("Cp437", null); + @Test + public void testSimpleCp437Encoding() throws IOException { + doSimpleEncodingsTest(437); } @Test public void testSimpleCp850Encoding() throws IOException { + doSimpleEncodingsTest(850); + } + + @Test + public void testEbcidic() throws IOException { + + doSimpleEncodingTest("IBM1047", null); + } - doSimpleEncodingTest("Cp850", null); + + private void doSimpleEncodingsTest(int n) throws IOException { + + doSimpleEncodingTest("Cp" + n, null); + doSimpleEncodingTest("cp" + n, null); + doSimpleEncodingTest("CP" + n, null); + doSimpleEncodingTest("IBM" + n, null); + doSimpleEncodingTest("ibm" + n, null); } @Test @@ -127,7 +168,7 @@ throws IOException { final ZipEncoding enc = ZipEncodingHelper.getZipEncoding(name); - + assertThat(enc, IsInstanceOf.instanceOf(NioZipEncoding.class)); if (testBytes == null) { testBytes = new byte[256]; @@ -145,10 +186,9 @@ assertEquals(testBytes, encoded); assertFalse(enc.canEncode(UNENC_STRING)); - assertEquals("%U2016".getBytes(CharsetNames.US_ASCII), enc.encode(UNENC_STRING)); + assertEquals("%U2016".getBytes(name), enc.encode(UNENC_STRING)); assertFalse(enc.canEncode(BAD_STRING)); - assertEquals(BAD_STRING_ENC.getBytes(CharsetNames.US_ASCII), - enc.encode(BAD_STRING)); + assertEquals(BAD_STRING_ENC.getBytes(name), enc.encode(BAD_STRING)); } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java 2018-08-09 18:03:55.000000000 +0000 @@ -21,21 +21,29 @@ import static org.apache.commons.compress.AbstractTestCase.getFile; import static org.junit.Assert.*; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.CRC32; import java.util.zip.ZipEntry; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; import org.junit.After; +import org.junit.Assert; import org.junit.Test; public class ZipFileTest { @@ -199,7 +207,7 @@ } /** - * Test case for + * Test case for * COMPRESS-208. */ @@ -306,7 +314,7 @@ } /** - * Test case for + * Test case for * COMPRESS-264. */ @@ -331,6 +339,435 @@ assertArrayEquals(expected, IOUtils.toByteArray(zf.getInputStream(ze))); } + @Test + public void testConcurrentReadSeekable() throws Exception { + // mixed.zip contains both inflated and stored files + byte[] data = null; + try (FileInputStream fis = new FileInputStream(getFile("mixed.zip"))) { + data = IOUtils.toByteArray(fis); + } + zf = new ZipFile(new SeekableInMemoryByteChannel(data), ZipEncodingHelper.UTF8); + + final Map content = new HashMap(); + for (ZipArchiveEntry entry: Collections.list(zf.getEntries())) { + content.put(entry.getName(), IOUtils.toByteArray(zf.getInputStream(entry))); + } + + final AtomicInteger passedCount = new AtomicInteger(); + Runnable run = new Runnable() { + @Override + public void run() { + for (ZipArchiveEntry entry: Collections.list(zf.getEntries())) { + assertAllReadMethods(content.get(entry.getName()), zf, entry); + } + passedCount.incrementAndGet(); + } + }; + Thread t0 = new Thread(run); + Thread t1 = new Thread(run); + t0.start(); + t1.start(); + t0.join(); + t1.join(); + assertEquals(2, passedCount.get()); + } + + @Test + public void testConcurrentReadFile() throws Exception { + // mixed.zip contains both inflated and stored files + final File archive = getFile("mixed.zip"); + zf = new ZipFile(archive); + + final Map content = new HashMap(); + for (ZipArchiveEntry entry: Collections.list(zf.getEntries())) { + content.put(entry.getName(), IOUtils.toByteArray(zf.getInputStream(entry))); + } + + final AtomicInteger passedCount = new AtomicInteger(); + Runnable run = new Runnable() { + @Override + public void run() { + for (ZipArchiveEntry entry: Collections.list(zf.getEntries())) { + assertAllReadMethods(content.get(entry.getName()), zf, entry); + } + passedCount.incrementAndGet(); + } + }; + Thread t0 = new Thread(run); + Thread t1 = new Thread(run); + t0.start(); + t1.start(); + t0.join(); + t1.join(); + assertEquals(2, passedCount.get()); + } + + /** + * Test correct population of header and data offsets. + */ + @Test + public void testOffsets() throws Exception { + // mixed.zip contains both inflated and stored files + final File archive = getFile("mixed.zip"); + try (ZipFile zf = new ZipFile(archive)) { + ZipArchiveEntry inflatedEntry = zf.getEntry("inflated.txt"); + Assert.assertEquals(0x0000, inflatedEntry.getLocalHeaderOffset()); + Assert.assertEquals(0x0046, inflatedEntry.getDataOffset()); + Assert.assertTrue(inflatedEntry.isStreamContiguous()); + ZipArchiveEntry storedEntry = zf.getEntry("stored.txt"); + Assert.assertEquals(0x5892, storedEntry.getLocalHeaderOffset()); + Assert.assertEquals(0x58d6, storedEntry.getDataOffset()); + Assert.assertTrue(inflatedEntry.isStreamContiguous()); + } + } + + /** + * Test correct population of header and data offsets when they are written after stream. + */ + @Test + public void testDelayedOffsetsAndSizes() throws Exception { + ByteArrayOutputStream zipContent = new ByteArrayOutputStream(); + try (ZipArchiveOutputStream zipOutput = new ZipArchiveOutputStream(zipContent)) { + ZipArchiveEntry inflatedEntry = new ZipArchiveEntry("inflated.txt"); + inflatedEntry.setMethod(ZipEntry.DEFLATED); + zipOutput.putArchiveEntry(inflatedEntry); + zipOutput.write("Hello Deflated\n".getBytes()); + zipOutput.closeArchiveEntry(); + + byte[] storedContent = "Hello Stored\n".getBytes(); + ZipArchiveEntry storedEntry = new ZipArchiveEntry("stored.txt"); + storedEntry.setMethod(ZipEntry.STORED); + storedEntry.setSize(storedContent.length); + storedEntry.setCrc(calculateCrc32(storedContent)); + zipOutput.putArchiveEntry(storedEntry); + zipOutput.write("Hello Stored\n".getBytes()); + zipOutput.closeArchiveEntry(); + + } + + try (ZipFile zf = new ZipFile(new SeekableInMemoryByteChannel(zipContent.toByteArray()))) { + ZipArchiveEntry inflatedEntry = zf.getEntry("inflated.txt"); + Assert.assertNotEquals(-1L, inflatedEntry.getLocalHeaderOffset()); + Assert.assertNotEquals(-1L, inflatedEntry.getDataOffset()); + Assert.assertTrue(inflatedEntry.isStreamContiguous()); + Assert.assertNotEquals(-1L, inflatedEntry.getCompressedSize()); + Assert.assertNotEquals(-1L, inflatedEntry.getSize()); + ZipArchiveEntry storedEntry = zf.getEntry("stored.txt"); + Assert.assertNotEquals(-1L, storedEntry.getLocalHeaderOffset()); + Assert.assertNotEquals(-1L, storedEntry.getDataOffset()); + Assert.assertTrue(inflatedEntry.isStreamContiguous()); + Assert.assertNotEquals(-1L, storedEntry.getCompressedSize()); + Assert.assertNotEquals(-1L, storedEntry.getSize()); + } + } + + /** + * Test entries alignment. + */ + @Test + public void testEntryAlignment() throws Exception { + SeekableInMemoryByteChannel zipContent = new SeekableInMemoryByteChannel(); + try (ZipArchiveOutputStream zipOutput = new ZipArchiveOutputStream(zipContent)) { + ZipArchiveEntry inflatedEntry = new ZipArchiveEntry("inflated.txt"); + inflatedEntry.setMethod(ZipEntry.DEFLATED); + inflatedEntry.setAlignment(1024); + zipOutput.putArchiveEntry(inflatedEntry); + zipOutput.write("Hello Deflated\n".getBytes(Charset.forName("UTF-8"))); + zipOutput.closeArchiveEntry(); + + ZipArchiveEntry storedEntry = new ZipArchiveEntry("stored.txt"); + storedEntry.setMethod(ZipEntry.STORED); + storedEntry.setAlignment(1024); + zipOutput.putArchiveEntry(storedEntry); + zipOutput.write("Hello Stored\n".getBytes(Charset.forName("UTF-8"))); + zipOutput.closeArchiveEntry(); + + ZipArchiveEntry storedEntry2 = new ZipArchiveEntry("stored2.txt"); + storedEntry2.setMethod(ZipEntry.STORED); + storedEntry2.setAlignment(1024); + storedEntry2.addExtraField(new ResourceAlignmentExtraField(1)); + zipOutput.putArchiveEntry(storedEntry2); + zipOutput.write("Hello overload-alignment Stored\n".getBytes(Charset.forName("UTF-8"))); + zipOutput.closeArchiveEntry(); + + ZipArchiveEntry storedEntry3 = new ZipArchiveEntry("stored3.txt"); + storedEntry3.setMethod(ZipEntry.STORED); + storedEntry3.addExtraField(new ResourceAlignmentExtraField(1024)); + zipOutput.putArchiveEntry(storedEntry3); + zipOutput.write("Hello copy-alignment Stored\n".getBytes(Charset.forName("UTF-8"))); + zipOutput.closeArchiveEntry(); + + } + + try (ZipFile zf = new ZipFile(new SeekableInMemoryByteChannel( + Arrays.copyOfRange(zipContent.array(), 0, (int)zipContent.size()) + ))) { + ZipArchiveEntry inflatedEntry = zf.getEntry("inflated.txt"); + ResourceAlignmentExtraField inflatedAlignmentEx = + (ResourceAlignmentExtraField)inflatedEntry.getExtraField(ResourceAlignmentExtraField.ID); + assertNotEquals(-1L, inflatedEntry.getCompressedSize()); + assertNotEquals(-1L, inflatedEntry.getSize()); + assertEquals(0L, inflatedEntry.getDataOffset()%1024); + assertNotNull(inflatedAlignmentEx); + assertEquals(1024, inflatedAlignmentEx.getAlignment()); + assertFalse(inflatedAlignmentEx.allowMethodChange()); + try (InputStream stream = zf.getInputStream(inflatedEntry)) { + Assert.assertEquals("Hello Deflated\n", + new String(IOUtils.toByteArray(stream), Charset.forName("UTF-8"))); + } + ZipArchiveEntry storedEntry = zf.getEntry("stored.txt"); + ResourceAlignmentExtraField storedAlignmentEx = + (ResourceAlignmentExtraField)storedEntry.getExtraField(ResourceAlignmentExtraField.ID); + assertNotEquals(-1L, storedEntry.getCompressedSize()); + assertNotEquals(-1L, storedEntry.getSize()); + assertEquals(0L, storedEntry.getDataOffset()%1024); + assertNotNull(storedAlignmentEx); + assertEquals(1024, storedAlignmentEx.getAlignment()); + assertFalse(storedAlignmentEx.allowMethodChange()); + try (InputStream stream = zf.getInputStream(storedEntry)) { + Assert.assertEquals("Hello Stored\n", + new String(IOUtils.toByteArray(stream), Charset.forName("UTF-8"))); + } + + ZipArchiveEntry storedEntry2 = zf.getEntry("stored2.txt"); + ResourceAlignmentExtraField stored2AlignmentEx = + (ResourceAlignmentExtraField)storedEntry2.getExtraField(ResourceAlignmentExtraField.ID); + assertNotEquals(-1L, storedEntry2.getCompressedSize()); + assertNotEquals(-1L, storedEntry2.getSize()); + assertEquals(0L, storedEntry2.getDataOffset()%1024); + assertNotNull(stored2AlignmentEx); + assertEquals(1024, stored2AlignmentEx.getAlignment()); + assertFalse(stored2AlignmentEx.allowMethodChange()); + try (InputStream stream = zf.getInputStream(storedEntry2)) { + Assert.assertEquals("Hello overload-alignment Stored\n", + new String(IOUtils.toByteArray(stream), Charset.forName("UTF-8"))); + } + + ZipArchiveEntry storedEntry3 = zf.getEntry("stored3.txt"); + ResourceAlignmentExtraField stored3AlignmentEx = + (ResourceAlignmentExtraField)storedEntry3.getExtraField(ResourceAlignmentExtraField.ID); + assertNotEquals(-1L, storedEntry3.getCompressedSize()); + assertNotEquals(-1L, storedEntry3.getSize()); + assertEquals(0L, storedEntry3.getDataOffset()%1024); + assertNotNull(stored3AlignmentEx); + assertEquals(1024, stored3AlignmentEx.getAlignment()); + assertFalse(stored3AlignmentEx.allowMethodChange()); + try (InputStream stream = zf.getInputStream(storedEntry3)) { + Assert.assertEquals("Hello copy-alignment Stored\n", + new String(IOUtils.toByteArray(stream), Charset.forName("UTF-8"))); + } + } + } + + /** + * Test too big alignment, resulting into exceeding extra field limit. + */ + @Test(expected = IllegalArgumentException.class) + public void testEntryAlignmentExceed() throws Exception { + SeekableInMemoryByteChannel zipContent = new SeekableInMemoryByteChannel(); + try (ZipArchiveOutputStream zipOutput = new ZipArchiveOutputStream(zipContent)) { + ZipArchiveEntry inflatedEntry = new ZipArchiveEntry("inflated.txt"); + inflatedEntry.setMethod(ZipEntry.STORED); + inflatedEntry.setAlignment(0x20000); + } + } + + /** + * Test non power of 2 alignment. + */ + @Test(expected = IllegalArgumentException.class) + public void testInvalidAlignment() throws Exception { + ZipArchiveEntry entry = new ZipArchiveEntry("dummy"); + entry.setAlignment(3); + } + + @Test + public void nameSourceDefaultsToName() throws Exception { + nameSource("bla.zip", "test1.xml", ZipArchiveEntry.NameSource.NAME); + } + + @Test + public void nameSourceIsSetToUnicodeExtraField() throws Exception { + nameSource("utf8-winzip-test.zip", "\u20AC_for_Dollar.txt", + ZipArchiveEntry.NameSource.UNICODE_EXTRA_FIELD); + } + + @Test + public void nameSourceIsSetToEFS() throws Exception { + nameSource("utf8-7zip-test.zip", "\u20AC_for_Dollar.txt", + ZipArchiveEntry.NameSource.NAME_WITH_EFS_FLAG); + } + + /** + * @see "https://issues.apache.org/jira/browse/COMPRESS-380" + */ + @Test + public void readDeflate64CompressedStream() throws Exception { + final File input = getFile("COMPRESS-380/COMPRESS-380-input"); + final File archive = getFile("COMPRESS-380/COMPRESS-380.zip"); + try (FileInputStream in = new FileInputStream(input); + ZipFile zf = new ZipFile(archive)) { + byte[] orig = IOUtils.toByteArray(in); + ZipArchiveEntry e = zf.getEntry("input2"); + try (InputStream s = zf.getInputStream(e)) { + byte[] fromZip = IOUtils.toByteArray(s); + assertArrayEquals(orig, fromZip); + } + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("bla.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingStore() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-264.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingUnshrink() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("SHRUNK.ZIP")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingExplode() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("imploding-8Kdict-3trees.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate64() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-380/COMPRESS-380.zip")); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofUsingBzip2() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(getFile("bzip2-zip.zip")); + } + + private void singleByteReadConsistentlyReturnsMinusOneAtEof(File file) throws Exception { + try (ZipFile archive = new ZipFile(file)) { + ZipArchiveEntry e = archive.getEntries().nextElement(); + try (InputStream is = archive.getInputStream(e)) { + IOUtils.toByteArray(is); + assertEquals(-1, is.read()); + assertEquals(-1, is.read()); + } + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("bla.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingStore() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-264.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingUnshrink() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("SHRUNK.ZIP")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingExplode() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("imploding-8Kdict-3trees.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingDeflate64() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("COMPRESS-380/COMPRESS-380.zip")); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofUsingBzip2() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(getFile("bzip2-zip.zip")); + } + + private void multiByteReadConsistentlyReturnsMinusOneAtEof(File file) throws Exception { + byte[] buf = new byte[2]; + try (ZipFile archive = new ZipFile(file)) { + ZipArchiveEntry e = archive.getEntries().nextElement(); + try (InputStream is = archive.getInputStream(e)) { + IOUtils.toByteArray(is); + assertEquals(-1, is.read(buf)); + assertEquals(-1, is.read(buf)); + } + } + } + + private void assertAllReadMethods(byte[] expected, ZipFile zipFile, ZipArchiveEntry entry) { + // simple IOUtil read + try (InputStream stream = zf.getInputStream(entry)) { + byte[] full = IOUtils.toByteArray(stream); + assertArrayEquals(expected, full); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + + // big buffer at the beginning and then chunks by IOUtils read + try (InputStream stream = zf.getInputStream(entry)) { + byte[] full; + byte[] bytes = new byte[0x40000]; + int read = stream.read(bytes); + if (read < 0) { + full = new byte[0]; + } + else { + full = readStreamRest(bytes, read, stream); + } + assertArrayEquals(expected, full); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + + // small chunk / single byte and big buffer then + try (InputStream stream = zf.getInputStream(entry)) { + byte[] full; + int single = stream.read(); + if (single < 0) { + full = new byte[0]; + } + else { + byte[] big = new byte[0x40000]; + big[0] = (byte)single; + int read = stream.read(big, 1, big.length-1); + if (read < 0) { + full = new byte[]{ (byte)single }; + } + else { + full = readStreamRest(big, read+1, stream); + } + } + assertArrayEquals(expected, full); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + /** + * Utility to append the rest of the stream to already read data. + */ + private byte[] readStreamRest(byte[] beginning, int length, InputStream stream) throws IOException { + byte[] rest = IOUtils.toByteArray(stream); + byte[] full = new byte[length+rest.length]; + System.arraycopy(beginning, 0, full, 0, length); + System.arraycopy(rest, 0, full, length, rest.length); + return full; + } + + private long calculateCrc32(byte[] content) { + CRC32 crc = new CRC32(); + crc.update(content); + return crc.getValue(); + } + /* * ordertest.zip has been handcrafted. * @@ -354,4 +791,12 @@ + expectedName + ".java", ze.getName()); } + + private static void nameSource(String archive, String entry, ZipArchiveEntry.NameSource expected) throws Exception { + try (ZipFile zf = new ZipFile(getFile(archive))) { + ZipArchiveEntry ze = zf.getEntry(entry); + assertEquals(entry, ze.getName()); + assertEquals(expected, ze.getNameSource()); + } + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipLongTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipLongTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/zip/ZipLongTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/zip/ZipLongTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -18,10 +18,10 @@ package org.apache.commons.compress.archivers.zip; -import static org.junit.Assert.*; - import org.junit.Test; +import static org.junit.Assert.*; + /** * JUnit testcases for org.apache.commons.compress.archivers.zip.ZipLong. * @@ -90,8 +90,15 @@ */ @Test public void testSign() { - final ZipLong zl = new ZipLong(new byte[] {(byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF}); + ZipLong zl = new ZipLong(new byte[] {(byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF}); assertEquals(0x00000000FFFFFFFFl, zl.getValue()); + assertEquals(-1,zl.getIntValue()); + + zl = new ZipLong(0xFFFF_FFFFL); + assertEquals(0x00000000FFFFFFFFl, zl.getValue()); + zl = new ZipLong(0xFFFF_FFFF); + assertEquals(0xFFFF_FFFF_FFFF_FFFFL, zl.getValue()); + } @Test diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java 2018-05-23 12:50:54.000000000 +0000 @@ -22,14 +22,17 @@ import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.Enumeration; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipException; @@ -37,11 +40,12 @@ import org.apache.commons.compress.archivers.zip.Zip64Mode; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveEntryPredicate; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.compress.archivers.zip.ZipMethod; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.InputStreamStatistics; import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; import org.junit.Assert; import org.junit.Test; @@ -124,8 +128,8 @@ IOUtils.readFully(new FileInputStream(file1), file1Contents); IOUtils.readFully(new FileInputStream(file2), file2Contents); - SeekableInMemoryByteChannel c = new SeekableInMemoryByteChannel(); - try (ZipArchiveOutputStream os = new ZipArchiveOutputStream(c)) { + SeekableInMemoryByteChannel channel = new SeekableInMemoryByteChannel(); + try (ZipArchiveOutputStream os = new ZipArchiveOutputStream(channel)) { os.putArchiveEntry(new ZipArchiveEntry("testdata/test1.xml")); os.write(file1Contents); os.closeArchiveEntry(); @@ -139,7 +143,7 @@ final List results = new ArrayList<>(); try (ArchiveInputStream in = new ArchiveStreamFactory() - .createArchiveInputStream("zip", new ByteArrayInputStream(c.array()))) { + .createArchiveInputStream("zip", new ByteArrayInputStream(channel.array()))) { ZipArchiveEntry entry; while((entry = (ZipArchiveEntry)in.getNextEntry()) != null) { @@ -160,17 +164,17 @@ @Test public void testZipUnarchive() throws Exception { final File input = getFile("bla.zip"); - final InputStream is = new FileInputStream(input); - final ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream("zip", is); - final ZipArchiveEntry entry = (ZipArchiveEntry)in.getNextEntry(); - final OutputStream out = new FileOutputStream(new File(dir, entry.getName())); - IOUtils.copy(in, out); - out.close(); - in.close(); + try (final InputStream is = new FileInputStream(input); + final ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream("zip", is)) { + final ZipArchiveEntry entry = (ZipArchiveEntry) in.getNextEntry(); + try (final OutputStream out = new FileOutputStream(new File(dir, entry.getName()))) { + IOUtils.copy(in, out); + } + } } /** - * Test case for + * Test case for * COMPRESS-208. */ @@ -197,7 +201,7 @@ assertTrue(bla.canReadEntryData(bla.getEntry("test1.xml"))); bla.close(); */ - + final ZipFile moby = new ZipFile(getFile("moby.zip")); final ZipArchiveEntry entry = moby.getEntry("README"); assertEquals("method", ZipMethod.TOKENIZATION.getCode(), entry.getMethod()); @@ -206,7 +210,7 @@ } /** - * Test case for being able to skip an entry in an + * Test case for being able to skip an entry in an * {@link ZipArchiveInputStream} even if the compression method of that * entry is unsupported. * @@ -234,9 +238,9 @@ * Checks if all entries from a nested archive can be read. * The archive: OSX_ArchiveWithNestedArchive.zip contains: * NestedArchiv.zip and test.xml3. - * + * * The nested archive: NestedArchive.zip contains test1.xml and test2.xml - * + * * @throws Exception */ @Test @@ -369,28 +373,28 @@ final File reference = createReferenceFile(tmp[0], Zip64Mode.Never, "expected."); final File a1 = File.createTempFile("src1.", ".zip", tmp[0]); - final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(a1); - zos.setUseZip64(Zip64Mode.Never); - createFirstEntry(zos).close(); + try (final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(a1)) { + zos.setUseZip64(Zip64Mode.Never); + createFirstEntry(zos).close(); + } final File a2 = File.createTempFile("src2.", ".zip", tmp[0]); - final ZipArchiveOutputStream zos1 = new ZipArchiveOutputStream(a2); - zos1.setUseZip64(Zip64Mode.Never); - createSecondEntry(zos1).close(); + try (final ZipArchiveOutputStream zos1 = new ZipArchiveOutputStream(a2)) { + zos1.setUseZip64(Zip64Mode.Never); + createSecondEntry(zos1).close(); + } - final ZipFile zf1 = new ZipFile(a1); - final ZipFile zf2 = new ZipFile(a2); - final File fileResult = File.createTempFile("file-actual.", ".zip", tmp[0]); - final ZipArchiveOutputStream zos2 = new ZipArchiveOutputStream(fileResult); - zf1.copyRawEntries(zos2, allFilesPredicate); - zf2.copyRawEntries(zos2, allFilesPredicate); - zos2.close(); - // copyRawEntries does not add superfluous zip64 header like regular zip output stream - // does when using Zip64Mode.AsNeeded so all the source material has to be Zip64Mode.Never, - // if exact binary equality is to be achieved - assertSameFileContents(reference, fileResult); - zf1.close(); - zf2.close(); + try (final ZipFile zf1 = new ZipFile(a1); final ZipFile zf2 = new ZipFile(a2)) { + final File fileResult = File.createTempFile("file-actual.", ".zip", tmp[0]); + try (final ZipArchiveOutputStream zos2 = new ZipArchiveOutputStream(fileResult)) { + zf1.copyRawEntries(zos2, allFilesPredicate); + zf2.copyRawEntries(zos2, allFilesPredicate); + } + // copyRawEntries does not add superfluous zip64 header like regular zip output stream + // does when using Zip64Mode.AsNeeded so all the source material has to be Zip64Mode.Never, + // if exact binary equality is to be achieved + assertSameFileContents(reference, fileResult); + } } @Test @@ -399,24 +403,25 @@ final File[] tmp = createTempDirAndFile(); final File reference = File.createTempFile("z64reference.", ".zip", tmp[0]); - final ZipArchiveOutputStream zos1 = new ZipArchiveOutputStream(reference); - zos1.setUseZip64(Zip64Mode.Always); - createFirstEntry(zos1); - zos1.close(); + try (final ZipArchiveOutputStream zos1 = new ZipArchiveOutputStream(reference)) { + zos1.setUseZip64(Zip64Mode.Always); + createFirstEntry(zos1); + } final File a1 = File.createTempFile("zip64src.", ".zip", tmp[0]); - final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(a1); - zos.setUseZip64(Zip64Mode.Always); - createFirstEntry(zos).close(); + try (final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(a1)) { + zos.setUseZip64(Zip64Mode.Always); + createFirstEntry(zos).close(); + } - final ZipFile zf1 = new ZipFile(a1); final File fileResult = File.createTempFile("file-actual.", ".zip", tmp[0]); - final ZipArchiveOutputStream zos2 = new ZipArchiveOutputStream(fileResult); - zos2.setUseZip64(Zip64Mode.Always); - zf1.copyRawEntries(zos2, allFilesPredicate); - zos2.close(); - assertSameFileContents(reference, fileResult); - zf1.close(); + try (final ZipFile zf1 = new ZipFile(a1)) { + try (final ZipArchiveOutputStream zos2 = new ZipArchiveOutputStream(fileResult)) { + zos2.setUseZip64(Zip64Mode.Always); + zf1.copyRawEntries(zos2, allFilesPredicate); + } + assertSameFileContents(reference, fileResult); + } } @Test @@ -425,27 +430,28 @@ final File[] tmp = createTempDirAndFile(); final File a1 = File.createTempFile("unixModeBits.", ".zip", tmp[0]); - final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(a1); + try (final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(a1)) { - final ZipArchiveEntry archiveEntry = new ZipArchiveEntry("fred"); - archiveEntry.setUnixMode(0664); - archiveEntry.setMethod(ZipEntry.DEFLATED); - zos.addRawArchiveEntry(archiveEntry, new ByteArrayInputStream("fud".getBytes())); - zos.close(); + final ZipArchiveEntry archiveEntry = new ZipArchiveEntry("fred"); + archiveEntry.setUnixMode(0664); + archiveEntry.setMethod(ZipEntry.DEFLATED); + zos.addRawArchiveEntry(archiveEntry, new ByteArrayInputStream("fud".getBytes())); + } - final ZipFile zf1 = new ZipFile(a1); - final ZipArchiveEntry fred = zf1.getEntry("fred"); - assertEquals(0664, fred.getUnixMode()); - zf1.close(); + try (final ZipFile zf1 = new ZipFile(a1)) { + final ZipArchiveEntry fred = zf1.getEntry("fred"); + assertEquals(0664, fred.getUnixMode()); + } } - private File createReferenceFile(final File directory, final Zip64Mode zipMode, final String prefix) throws IOException { + private File createReferenceFile(final File directory, final Zip64Mode zipMode, final String prefix) + throws IOException { final File reference = File.createTempFile(prefix, ".zip", directory); - final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(reference); - zos.setUseZip64(zipMode); - createFirstEntry(zos); - createSecondEntry(zos); - zos.close(); + try (final ZipArchiveOutputStream zos = new ZipArchiveOutputStream(reference)) { + zos.setUseZip64(zipMode); + createFirstEntry(zos); + createSecondEntry(zos); + } return reference; } @@ -462,39 +468,37 @@ private void assertSameFileContents(final File expectedFile, final File actualFile) throws IOException { final int size = (int) Math.max(expectedFile.length(), actualFile.length()); - final ZipFile expected = new ZipFile(expectedFile); - final ZipFile actual = new ZipFile(actualFile); - final byte[] expectedBuf = new byte[size]; - final byte[] actualBuf = new byte[size]; - - final Enumeration actualInOrder = actual.getEntriesInPhysicalOrder(); - final Enumeration expectedInOrder = expected.getEntriesInPhysicalOrder(); - - while (actualInOrder.hasMoreElements()){ - final ZipArchiveEntry actualElement = actualInOrder.nextElement(); - final ZipArchiveEntry expectedElement = expectedInOrder.nextElement(); - assertEquals( expectedElement.getName(), actualElement.getName()); - // Don't compare timestamps since they may vary; - // there's no support for stubbed out clock (TimeSource) in ZipArchiveOutputStream - assertEquals( expectedElement.getMethod(), actualElement.getMethod()); - assertEquals( expectedElement.getGeneralPurposeBit(), actualElement.getGeneralPurposeBit()); - assertEquals( expectedElement.getCrc(), actualElement.getCrc()); - assertEquals( expectedElement.getCompressedSize(), actualElement.getCompressedSize()); - assertEquals( expectedElement.getSize(), actualElement.getSize()); - assertEquals( expectedElement.getExternalAttributes(), actualElement.getExternalAttributes()); - assertEquals( expectedElement.getInternalAttributes(), actualElement.getInternalAttributes()); - - final InputStream actualIs = actual.getInputStream(actualElement); - final InputStream expectedIs = expected.getInputStream(expectedElement); - IOUtils.readFully(expectedIs, expectedBuf); - IOUtils.readFully(actualIs, actualBuf); - expectedIs.close(); - actualIs.close(); - Assert.assertArrayEquals(expectedBuf, actualBuf); // Buffers are larger than payload. dont care - } + try (final ZipFile expected = new ZipFile(expectedFile); final ZipFile actual = new ZipFile(actualFile)) { + final byte[] expectedBuf = new byte[size]; + final byte[] actualBuf = new byte[size]; + + final Enumeration actualInOrder = actual.getEntriesInPhysicalOrder(); + final Enumeration expectedInOrder = expected.getEntriesInPhysicalOrder(); + + while (actualInOrder.hasMoreElements()) { + final ZipArchiveEntry actualElement = actualInOrder.nextElement(); + final ZipArchiveEntry expectedElement = expectedInOrder.nextElement(); + assertEquals(expectedElement.getName(), actualElement.getName()); + // Don't compare timestamps since they may vary; + // there's no support for stubbed out clock (TimeSource) in ZipArchiveOutputStream + assertEquals(expectedElement.getMethod(), actualElement.getMethod()); + assertEquals(expectedElement.getGeneralPurposeBit(), actualElement.getGeneralPurposeBit()); + assertEquals(expectedElement.getCrc(), actualElement.getCrc()); + assertEquals(expectedElement.getCompressedSize(), actualElement.getCompressedSize()); + assertEquals(expectedElement.getSize(), actualElement.getSize()); + assertEquals(expectedElement.getExternalAttributes(), actualElement.getExternalAttributes()); + assertEquals(expectedElement.getInternalAttributes(), actualElement.getInternalAttributes()); + + final InputStream actualIs = actual.getInputStream(actualElement); + final InputStream expectedIs = expected.getInputStream(expectedElement); + IOUtils.readFully(expectedIs, expectedBuf); + IOUtils.readFully(actualIs, actualBuf); + expectedIs.close(); + actualIs.close(); + Assert.assertArrayEquals(expectedBuf, actualBuf); // Buffers are larger than payload. dont care + } - expected.close(); - actual.close(); + } } @@ -598,4 +602,109 @@ rmdir(tmp[0]); } } + + @Test + public void inputStreamStatisticsOfZipBombExcel() throws IOException, ArchiveException { + Map> expected = new HashMap>() {{ + put("[Content_Types].xml", Arrays.asList(8390036L, 8600L)); + put("xl/worksheets/sheet1.xml", Arrays.asList(1348L, 508L)); + }}; + testInputStreamStatistics("zipbomb.xlsx", expected); + } + + @Test + public void inputStreamStatisticsForImplodedEntry() throws IOException, ArchiveException { + Map> expected = new HashMap>() {{ + put("LICENSE.TXT", Arrays.asList(11560L, 4131L)); + }}; + testInputStreamStatistics("imploding-8Kdict-3trees.zip", expected); + } + + @Test + public void inputStreamStatisticsForShrunkEntry() throws IOException, ArchiveException { + Map> expected = new HashMap>() {{ + put("TEST1.XML", Arrays.asList(76L, 66L)); + put("TEST2.XML", Arrays.asList(81L, 76L)); + }}; + testInputStreamStatistics("SHRUNK.ZIP", expected); + } + + @Test + public void inputStreamStatisticsForStoredEntry() throws IOException, ArchiveException { + Map> expected = new HashMap>() {{ + put("test.txt", Arrays.asList(5L, 5L)); + }}; + testInputStreamStatistics("COMPRESS-264.zip", expected); + } + + @Test + public void inputStreamStatisticsForBzip2Entry() throws IOException, ArchiveException { + Map> expected = new HashMap>() {{ + put("lots-of-as", Arrays.asList(42L, 39L)); + }}; + testInputStreamStatistics("bzip2-zip.zip", expected); + } + + @Test + public void inputStreamStatisticsForDeflate64Entry() throws IOException, ArchiveException { + Map> expected = new HashMap>() {{ + put("input2", Arrays.asList(3072L, 2111L)); + }}; + testInputStreamStatistics("COMPRESS-380/COMPRESS-380.zip", expected); + } + + private void testInputStreamStatistics(String fileName, Map> expectedStatistics) + throws IOException, ArchiveException { + final File input = getFile(fileName); + + final Map>> actualStatistics = new HashMap<>(); + + // stream access + try (final FileInputStream fis = new FileInputStream(input); + final ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream("zip", fis)) { + for (ArchiveEntry entry; (entry = in.getNextEntry()) != null; ) { + readStream(in, entry, actualStatistics); + } + } + + // file access + try (final ZipFile zf = new ZipFile(input)) { + final Enumeration entries = zf.getEntries(); + while (entries.hasMoreElements()) { + final ZipArchiveEntry zae = entries.nextElement(); + try (InputStream in = zf.getInputStream(zae)) { + readStream(in, zae, actualStatistics); + } + } + } + + // compare statistics of stream / file access + for (Map.Entry>> me : actualStatistics.entrySet()) { + assertEquals("Mismatch of stats for: " + me.getKey(), + me.getValue().get(0), me.getValue().get(1)); + } + + for (Map.Entry> me : expectedStatistics.entrySet()) { + assertEquals("Mismatch of stats with expected value for: " + me.getKey(), + me.getValue(), actualStatistics.get(me.getKey()).get(0)); + } + } + + private void readStream(final InputStream in, final ArchiveEntry entry, final Map>> map) throws IOException { + final byte[] buf = new byte[4096]; + final InputStreamStatistics stats = (InputStreamStatistics) in; + while (in.read(buf) != -1); + + final String name = entry.getName(); + final List> l; + if (map.containsKey(name)) { + l = map.get(name); + } else { + map.put(name, l = new ArrayList<>()); + } + + final long t = stats.getUncompressedCount(); + final long b = stats.getCompressedCount(); + l.add(Arrays.asList(t, b)); + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/ArchiveUtilsTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/ArchiveUtilsTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/ArchiveUtilsTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/ArchiveUtilsTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -13,16 +13,17 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress; -import static org.junit.Assert.*; - +import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry; import org.apache.commons.compress.utils.ArchiveUtils; import org.junit.Test; +import static org.junit.Assert.*; + public class ArchiveUtilsTest extends AbstractTestCase { private static final int bytesToTest = 50; @@ -95,6 +96,53 @@ assertEquals(expected, ArchiveUtils.sanitize(input)); } + @Test + public void testIsEqualWithNullWithPositive() { + + byte[] byteArray = new byte[8]; + byteArray[1] = (byte) (-77); + + assertFalse(ArchiveUtils.isEqualWithNull(byteArray, 0, (byte)0, byteArray, (byte)0, (byte)80)); + + } + + @Test + public void testToAsciiBytes() { + + byte[] byteArray = ArchiveUtils.toAsciiBytes("SOCKET"); + + assertArrayEquals(new byte[] {(byte)83, (byte)79, (byte)67, (byte)75, (byte)69, (byte)84}, byteArray); + + assertFalse(ArchiveUtils.isEqualWithNull(byteArray, 0, 46, byteArray, 63, 0)); + + } + + @Test + public void testToStringWithNonNull() { + + SevenZArchiveEntry sevenZArchiveEntry = new SevenZArchiveEntry(); + String string = ArchiveUtils.toString(sevenZArchiveEntry); + + assertEquals("- 0 null", string); + + } + + @Test + public void testIsEqual() { + + assertTrue(ArchiveUtils.isEqual((byte[]) null, 0, 0, (byte[]) null, 0, 0)); + + } + + @Test(expected = StringIndexOutOfBoundsException.class) + public void testToAsciiStringThrowsStringIndexOutOfBoundsException() { + + byte[] byteArray = new byte[3]; + + ArchiveUtils.toAsciiString(byteArray, 940, 2730); + + } + private void asciiToByteAndBackOK(final String inputString) { assertEquals(inputString, ArchiveUtils.toAsciiString(ArchiveUtils.toAsciiBytes(inputString))); } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/ChainingTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/ChainingTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/ChainingTestCase.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/ChainingTestCase.java 2018-05-02 20:17:13.000000000 +0000 @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress; diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/changes/ChangeSetTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/changes/ChangeSetTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/changes/ChangeSetTestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/changes/ChangeSetTestCase.java 2018-05-02 20:17:13.000000000 +0000 @@ -74,7 +74,7 @@ * Adds an ArchiveEntry with the same name two times. * Only the latest addition should be found in the ChangeSet, * the first add should be replaced. - * + * * @throws Exception */ @Test @@ -109,7 +109,7 @@ * Adds an ArchiveEntry with the same name two times. * Only the first addition should be found in the ChangeSet, * the second add should never be added since replace = false - * + * * @throws Exception */ @Test @@ -145,7 +145,7 @@ * the deletion of bla/*, which actually means bla/test4.xml should be * removed from the archive. The file something/bla (without ending, named * like the folder) should not be deleted. - * + * * @throws Exception */ @Test @@ -187,7 +187,7 @@ /** * Tries to delete the folder "la" from an archive file. This should result in * the deletion of la/*, which should not match any files/folders. - * + * * @throws Exception */ @Test @@ -229,7 +229,7 @@ /** * Tries to delete the folder "test.txt" from an archive file. * This should not match any files/folders. - * + * * @throws Exception */ @Test @@ -271,7 +271,7 @@ /** * Tries to delete the file "bla/test5.xml" from an archive. This should * result in the deletion of "bla/test5.xml". - * + * * @throws Exception */ @Test @@ -314,7 +314,7 @@ /** * Tries to delete the file "bla" from an archive. This should * result in the deletion of nothing. - * + * * @throws Exception */ @Test @@ -358,7 +358,7 @@ * Tries to delete and then add a file with the same name. * Should delete test/test3.xml and adds test.txt with the name * test/test3.xml - * + * * @throws Exception */ @Test @@ -425,12 +425,12 @@ in.close(); } rmdir(check); - } + } } /** * Checks for the correct ChangeSetResults - * + * * @throws Exception */ @Test @@ -494,7 +494,7 @@ * Tries to delete a directory with a file and adds a new directory with a * new file and with the same name. Should delete dir1/* and add * dir1/test.txt at the end - * + * * @throws Exception */ @Test @@ -541,7 +541,7 @@ /** * Adds a file to a zip archive. Deletes an other file. - * + * * @throws Exception */ @Test @@ -588,7 +588,7 @@ /** * Adds a file to a zip archive. Deletes an other file. - * + * * @throws Exception */ @Test @@ -634,7 +634,7 @@ /** * add blub/test.txt + delete blub Should add blub/test.txt and delete it * afterwards. In this example, the archive should stay untouched. - * + * * @throws Exception */ @Test @@ -683,7 +683,7 @@ /** * delete bla + add bla/test.txt + delete bla Deletes dir1/* first, then * suppresses the add of bla.txt because there is a delete operation later. - * + * * @throws Exception */ @Test @@ -733,7 +733,7 @@ /** * Simple Delete from a zip file. - * + * * @throws Exception */ @Test @@ -774,7 +774,7 @@ /** * Simple delete from a tar file - * + * * @throws Exception */ @Test @@ -812,7 +812,7 @@ /** * Simple delete from a jar file - * + * * @throws Exception */ @Test @@ -902,7 +902,7 @@ /** * Delete from a jar file and add another file - * + * * @throws Exception */ @Test @@ -948,7 +948,7 @@ /** * Simple delete from an ar file - * + * * @throws Exception */ @Test @@ -987,7 +987,7 @@ /** * Deletes a file from an AR-archive and adds another - * + * * @throws Exception */ @Test @@ -1033,10 +1033,10 @@ /** * TODO: Move operations are not supported currently - * + * * mv dir1/test.text dir2/test.txt + delete dir1 Moves the file to dir2 and * deletes everything in dir1 - * + * * @throws Exception */ @Test @@ -1045,12 +1045,12 @@ /** * TODO: Move operations are not supported currently - * + * * add dir1/bla.txt + mv dir1/test.text dir2/test.txt + delete dir1 - * + * * Add dir1/bla.txt should be surpressed. All other dir1 files will be * deleted, except dir1/test.text will be moved - * + * * @throws Exception */ @Test @@ -1059,7 +1059,7 @@ /** * Check can add a file to an empty archive. - * + * * @throws Exception */ @Test @@ -1094,7 +1094,7 @@ out.close(); } if (ais != null) { - ais.close(); // will close is + ais.close(); // will close is } else if (is != null){ is.close(); } @@ -1105,7 +1105,7 @@ /** * Check can delete and add a file to an archive with a single file - * + * * @throws Exception */ @Test @@ -1143,7 +1143,7 @@ out.close(); } if (ais != null) { - ais.close(); // will close is + ais.close(); // will close is } else if (is != null){ is.close(); } @@ -1154,7 +1154,7 @@ /** * Check can add and delete a file to an archive with a single file - * + * * @throws Exception */ @Test @@ -1176,7 +1176,7 @@ out = factory.createArchiveOutputStream(archivename, new FileOutputStream(result)); final File file = getFile("test.txt"); - final ArchiveEntry entry = out.createArchiveEntry(file,"bla/test.txt"); + final ArchiveEntry entry = out.createArchiveEntry(file,"bla/test.txt"); changes.add(entry, new FileInputStream(file)); archiveList.add("bla/test.txt"); @@ -1192,7 +1192,7 @@ out.close(); } if (ais != null) { - ais.close(); // will close is + ais.close(); // will close is } else if (is != null){ is.close(); } @@ -1204,7 +1204,7 @@ /** * Adds a file with the same filename as an existing file from the stream. * Should lead to a replacement. - * + * * @throws Exception */ @Test @@ -1249,7 +1249,7 @@ /** * Adds a file with the same filename as an existing file from the stream. * Should lead to a replacement. - * + * * @throws Exception */ @Test diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/changes/ChangeTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/changes/ChangeTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/changes/ChangeTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/changes/ChangeTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.changes; + +import org.apache.commons.compress.archivers.memory.MemoryArchiveEntry; +import org.junit.Test; + +import java.io.PipedInputStream; + + +/** + * Unit tests for class {@link Change}. + * + * @date 16.06.2017 + * @see Change + **/ +public class ChangeTest { + + + @Test(expected = NullPointerException.class) + public void testFailsToCreateChangeTakingFourArgumentsThrowsNullPointerExceptionOne() { + + MemoryArchiveEntry memoryArchiveEntry = new MemoryArchiveEntry("x"); + + Change change = new Change(memoryArchiveEntry, null, false); + + } + + + @Test(expected = NullPointerException.class) + public void testFailsToCreateChangeTakingFourArgumentsThrowsNullPointerExceptionTwo() { + + PipedInputStream pipedInputStream = new PipedInputStream(1); + + Change change = new Change(null, pipedInputStream, false); + + } + + + @Test(expected = NullPointerException.class) + public void testFailsToCreateChangeTakingThreeArgumentsThrowsNullPointerException() { + + Change change = new Change(null, (-407)); + + } + + +} \ No newline at end of file diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/brotli/BrotliCompressorInputStreamTest.java 2018-08-09 05:50:20.000000000 +0000 @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.brotli; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public class BrotliCompressorInputStreamTest extends AbstractTestCase { + + /** + * Test bridge works fine + * @throws {@link IOException} + */ + @Test + public void testBrotliDecode() throws IOException { + final File input = getFile("brotli.testdata.compressed"); + final File expected = getFile("brotli.testdata.uncompressed"); + try (InputStream inputStream = new FileInputStream(input); + InputStream expectedStream = new FileInputStream(expected); + BrotliCompressorInputStream brotliInputStream = new BrotliCompressorInputStream(inputStream)) { + final byte[] b = new byte[20]; + IOUtils.readFully(expectedStream, b); + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + int readByte = -1; + while((readByte = brotliInputStream.read()) != -1) { + bos.write(readByte); + } + Assert.assertArrayEquals(b, bos.toByteArray()); + } + } + + @Test + public void testCachingIsEnabledByDefaultAndBrotliIsPresent() { + assertEquals(BrotliUtils.CachedAvailability.CACHED_AVAILABLE, BrotliUtils.getCachedBrotliAvailability()); + assertTrue(BrotliUtils.isBrotliCompressionAvailable()); + } + + @Test + public void testCanTurnOffCaching() { + try { + BrotliUtils.setCacheBrotliAvailablity(false); + assertEquals(BrotliUtils.CachedAvailability.DONT_CACHE, BrotliUtils.getCachedBrotliAvailability()); + assertTrue(BrotliUtils.isBrotliCompressionAvailable()); + } finally { + BrotliUtils.setCacheBrotliAvailablity(true); + } + } + + @Test + public void testTurningOnCachingReEvaluatesAvailability() { + try { + BrotliUtils.setCacheBrotliAvailablity(false); + assertEquals(BrotliUtils.CachedAvailability.DONT_CACHE, BrotliUtils.getCachedBrotliAvailability()); + BrotliUtils.setCacheBrotliAvailablity(true); + assertEquals(BrotliUtils.CachedAvailability.CACHED_AVAILABLE, BrotliUtils.getCachedBrotliAvailability()); + } finally { + BrotliUtils.setCacheBrotliAvailablity(true); + } + } + + + @Test + public void availableShouldReturnZero() throws IOException { + final File input = getFile("brotli.testdata.compressed"); + try (InputStream is = new FileInputStream(input)) { + final BrotliCompressorInputStream in = + new BrotliCompressorInputStream(is); + Assert.assertTrue(in.available() == 0); + in.close(); + } + } + + @Test + public void shouldBeAbleToSkipAByte() throws IOException { + final File input = getFile("brotli.testdata.compressed"); + try (InputStream is = new FileInputStream(input)) { + final BrotliCompressorInputStream in = + new BrotliCompressorInputStream(is); + Assert.assertEquals(1, in.skip(1)); + in.close(); + } + } + + @Test + public void singleByteReadWorksAsExpected() throws IOException { + final File input = getFile("brotli.testdata.compressed"); + try (InputStream is = new FileInputStream(input)) { + final BrotliCompressorInputStream in = + new BrotliCompressorInputStream(is); + // starts with filename "XXX" + Assert.assertEquals('X', in.read()); + in.close(); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("brotli.testdata.compressed"); + try (InputStream is = new FileInputStream(input)) { + final BrotliCompressorInputStream in = + new BrotliCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("brotli.testdata.compressed"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final BrotliCompressorInputStream in = + new BrotliCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } + + @Test + public void testBrotliUnarchive() throws Exception { + final File input = getFile("bla.tar.br"); + final File output = new File(dir, "bla.tar"); + try (InputStream is = new FileInputStream(input)) { + final CompressorInputStream in = new CompressorStreamFactory() + .createCompressorInputStream("br", is); + FileOutputStream out = null; + try { + out = new FileOutputStream(output); + IOUtils.copy(in, out); + } finally { + if (out != null) { + out.close(); + } + in.close(); + } + } + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/bzip2/BlockSortTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/bzip2/BlockSortTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/bzip2/BlockSortTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/bzip2/BlockSortTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -62,19 +62,19 @@ and the original line has been 0 */ - private static final byte[] FIXTURE_BWT = { (byte) 128, 0, 3, (byte) 254, 2, 1, + private static final byte[] FIXTURE_BWT = { (byte) 128, 0, 3, (byte) 254, 2, 1, (byte) 252, (byte) 255, (byte) 253 }; private static final int[] FIXTURE_SORTED = { 0, 1, 7, 6, 8, 2, 3, 5, 4 }; - private static final byte[] FIXTURE2 = { - 'C', 'o', 'm', 'm', 'o', 'n', 's', ' ', 'C', 'o', 'm', 'p', 'r', 'e', 's', 's', + private static final byte[] FIXTURE2 = { + 'C', 'o', 'm', 'm', 'o', 'n', 's', ' ', 'C', 'o', 'm', 'p', 'r', 'e', 's', 's', }; private static final byte[] FIXTURE2_BWT = { - 's', 's', ' ', 'r', 'o', 'm', 'o', 'o', 'C', 'C', 'm', 'm', 'p', 'n', 's', 'e', + 's', 's', ' ', 'r', 'o', 'm', 'o', 'o', 'C', 'C', 'm', 'm', 'p', 'n', 's', 'e', }; @Test diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStreamTest.java 2018-08-09 05:54:31.000000000 +0000 @@ -22,8 +22,11 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Assert; import org.junit.Test; @@ -32,7 +35,8 @@ @Test(expected = IOException.class) public void shouldThrowAnIOExceptionWhenAppliedToAZipFile() throws Exception { try (FileInputStream in = new FileInputStream(getFile("bla.zip"))) { - new BZip2CompressorInputStream(in); + BZip2CompressorInputStream bis = new BZip2CompressorInputStream(in); + bis.close(); } } @@ -66,4 +70,31 @@ bzipIn.close(); } + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.txt.bz2"); + try (InputStream is = new FileInputStream(input)) { + final BZip2CompressorInputStream in = + new BZip2CompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.txt.bz2"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final BZip2CompressorInputStream in = + new BZip2CompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } + } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate/DeflateCompressorInputStreamTest.java 2018-08-09 06:48:49.000000000 +0000 @@ -65,13 +65,28 @@ } @Test - public void singleByteReadReturnsMinusOneAtEof() throws IOException { + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { final File input = AbstractTestCase.getFile("bla.tar.deflatez"); try (InputStream is = new FileInputStream(input)) { final DeflateCompressorInputStream in = new DeflateCompressorInputStream(is); IOUtils.toByteArray(in); Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = AbstractTestCase.getFile("bla.tar.deflatez"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final DeflateCompressorInputStream in = + new DeflateCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); in.close(); } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.compressors.deflate64; + +import static org.apache.commons.compress.AbstractTestCase.getFile; + +import java.io.InputStream; +import java.util.Enumeration; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.junit.Test; + +public class Deflate64BugsTest { + + @Test + public void readBeyondMemoryException() throws Exception { + try (ZipFile zfile = new ZipFile(getFile("COMPRESS-380/COMPRESS-380-readbeyondmemory.zip"))) { + Enumeration entries = zfile.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry e = entries.nextElement(); + byte [] buf = new byte [1024 * 8]; + try (InputStream is = zfile.getInputStream(e)) { + while (true) { + int read = is.read(buf); + if (read == -1) { + break; + } + } + } + } + } + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java 2018-08-09 06:55:34.000000000 +0000 @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.compressors.deflate64; + +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.times; + +@RunWith(MockitoJUnitRunner.class) +public class Deflate64CompressorInputStreamTest { + private final HuffmanDecoder nullDecoder = null; + + @Mock + private HuffmanDecoder decoder; + + @Test + public void readWhenClosed() throws Exception { + Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder); + assertEquals(-1, input.read()); + assertEquals(-1, input.read(new byte[1])); + assertEquals(-1, input.read(new byte[1], 0, 1)); + } + + @Test + public void properSizeWhenClosed() throws Exception { + Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder); + assertEquals(0, input.available()); + } + + @Test + public void delegatesAvailable() throws Exception { + Mockito.when(decoder.available()).thenReturn(1024); + + Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder); + assertEquals(1024, input.available()); + } + + @Test + public void closeCallsDecoder() throws Exception { + + Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder); + input.close(); + + Mockito.verify(decoder, times(1)).close(); + } + + @Test + public void closeIsDelegatedJustOnce() throws Exception { + + Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder); + + input.close(); + input.close(); + + Mockito.verify(decoder, times(1)).close(); + } + + @Test + public void uncompressedBlock() throws Exception { + byte[] data = { + 1, 11, 0, -12, -1, + 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd' + }; + + try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data)); + BufferedReader br = new BufferedReader(new InputStreamReader(input))) { + assertEquals("Hello World", br.readLine()); + assertEquals(null, br.readLine()); + } + } + + @Test + public void uncompressedBlockViaFactory() throws Exception { + byte[] data = { + 1, 11, 0, -12, -1, + 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd' + }; + + try (InputStream input = new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.DEFLATE64, new ByteArrayInputStream(data)); + BufferedReader br = new BufferedReader(new InputStreamReader(input))) { + assertEquals("Hello World", br.readLine()); + assertEquals(null, br.readLine()); + } + } + + @Test + public void uncompressedBlockAvailable() throws Exception { + byte[] data = { + 1, 11, 0, -12, -1, + 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd' + }; + + try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data))) { + assertEquals('H', input.read()); + assertEquals(10, input.available()); + } + } + + @Test + public void streamIgnoresExtraBytesAfterDeflatedInput() throws Exception + { + byte[] data = { + 1, 11, 0, -12, -1, + 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', 'X' + }; + + try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data)); + BufferedReader br = new BufferedReader(new InputStreamReader(input))) { + assertEquals("Hello World", br.readLine()); + assertEquals(null, br.readLine()); + } + } + + @Test(expected = java.io.EOFException.class) + public void throwsEOFExceptionOnTruncatedStreams() throws Exception + { + byte[] data = { + 1, 11, 0, -12, -1, + 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', + }; + + try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data)); + BufferedReader br = new BufferedReader(new InputStreamReader(input))) { + assertEquals("Hello World", br.readLine()); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + try (final Deflate64CompressorInputStream in = + new Deflate64CompressorInputStream(nullDecoder)) { + IOUtils.toByteArray(in); + assertEquals(-1, in.read()); + assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws Exception { + byte[] buf = new byte[2]; + try (final Deflate64CompressorInputStream in = + new Deflate64CompressorInputStream(nullDecoder)) { + IOUtils.toByteArray(in); + assertEquals(-1, in.read(buf)); + assertEquals(-1, in.read(buf)); + in.close(); + } + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.compressors.deflate64; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class HuffmanDecoderTest { + @Test + public void decodeUncompressedBlock() throws Exception { + byte[] data = { + 0b1, // end of block + no compression mode + 11, 0, -12, -1, // len & ~len + 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd' + }; + + HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data)); + byte[] result = new byte[100]; + int len = decoder.decode(result); + + assertEquals(11, len); + assertEquals("Hello World", new String(result, 0, len)); + } + + @Test + public void decodeUncompressedBlockWithInvalidLenNLenValue() throws Exception { + byte[] data = { + 0b1, // end of block + no compression mode + 11, 0, -12, -2, // len & ~len + 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd' + }; + + HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data)); + byte[] result = new byte[100]; + try { + int len = decoder.decode(result); + fail("Should have failed but returned " + len + " entries: " + Arrays.toString(Arrays.copyOf(result, len))); + } catch (IllegalStateException e) { + assertEquals("Illegal LEN / NLEN values", e.getMessage()); + } + } + + @Test + public void decodeSimpleFixedHuffmanBlock() throws Exception { + byte[] data = { + //|--- binary filling ---|76543210 + 0b11111111111111111111111111110011, // final block + fixed huffman + H + 0b00000000000000000000000001001000, // H + e + 0b11111111111111111111111111001101, // e + l + 0b11111111111111111111111111001001, // l + l + 0b11111111111111111111111111001001, // l + o + 0b00000000000000000000000001010111, // o + ' ' + 0b00000000000000000000000000001000, // ' ' + W + 0b11111111111111111111111111001111, // W + o + 0b00000000000000000000000000101111, // o + r + 0b11111111111111111111111111001010, // r + l + 0b00000000000000000000000001001001, // l + d + 0b00000000000000000000000000000001, // d + end of block + 0b11111111111111111111111111111100 // end of block (00) + garbage + }; + + HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data)); + byte[] result = new byte[100]; + int len = decoder.decode(result); + + assertEquals(11, len); + assertEquals("Hello World", new String(result, 0, len)); + } + + @Test + public void decodeSimpleFixedHuffmanBlockToSmallBuffer() throws Exception { + byte[] data = { + //|--- binary filling ---|76543210 + 0b11111111111111111111111111110011, // final block + fixed huffman + H + 0b00000000000000000000000001001000, // H + e + 0b11111111111111111111111111001101, // e + l + 0b11111111111111111111111111001001, // l + l + 0b11111111111111111111111111001001, // l + o + 0b00000000000000000000000001010111, // o + ' ' + 0b00000000000000000000000000001000, // ' ' + W + 0b11111111111111111111111111001111, // W + o + 0b00000000000000000000000000101111, // o + r + 0b11111111111111111111111111001010, // r + l + 0b00000000000000000000000001001001, // l + d + 0b00000000000000000000000000000001, // d + end of block + 0b11111111111111111111111111111100 // end of block (00) + garbage + }; + + HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data)); + byte[] result = new byte[10]; + int len; + len = decoder.decode(result); + assertEquals(10, len); + assertEquals("Hello Worl", new String(result, 0, len)); + len = decoder.decode(result); + assertEquals(1, len); + assertEquals("d", new String(result, 0, len)); + } + + + @Test + public void decodeFixedHuffmanBlockWithMemoryLookup() throws Exception { + byte[] data = { + //|--- binary filling ---|76543210 + 0b11111111111111111111111111110011, // final block + fixed huffman + H + 0b00000000000000000000000001001000, // H + e + 0b11111111111111111111111111001101, // e + l + 0b11111111111111111111111111001001, // l + l + 0b11111111111111111111111111001001, // l + o + 0b00000000000000000000000001010111, // o + ' ' + 0b00000000000000000000000000001000, // ' ' + W + 0b11111111111111111111111111001111, // W + o + 0b00000000000000000000000000101111, // o + r + 0b11111111111111111111111111001010, // r + l + 0b00000000000000000000000001001001, // l + d + 0b11111111111111111111111111100001, // d + '\n' + 0b00000000000000000000000000100010, // '\n' + + 0b11111111111111111111111110000110, // + offset <001> + dist6 + 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000) + 0b11111111111111111111111111111000 // end of block (0000) + garbage + }; + + HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data)); + byte[] result = new byte[100]; + int len = decoder.decode(result); + + assertEquals(48, len); + assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len)); + } + + @Test + public void decodeFixedHuffmanBlockWithMemoryLookupInSmallBuffer() throws Exception { + byte[] data = { + //|--- binary filling ---|76543210 + 0b11111111111111111111111111110011, // final block + fixed huffman + H + 0b00000000000000000000000001001000, // H + e + 0b11111111111111111111111111001101, // e + l + 0b11111111111111111111111111001001, // l + l + 0b11111111111111111111111111001001, // l + o + 0b00000000000000000000000001010111, // o + ' ' + 0b00000000000000000000000000001000, // ' ' + W + 0b11111111111111111111111111001111, // W + o + 0b00000000000000000000000000101111, // o + r + 0b11111111111111111111111111001010, // r + l + 0b00000000000000000000000001001001, // l + d + 0b11111111111111111111111111100001, // d + '\n' + 0b00000000000000000000000000100010, // '\n' + + 0b11111111111111111111111110000110, // + offset <001> + dist6 + 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000) + 0b11111111111111111111111111111000 // end of block (0000) + garbage + }; + + HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data)); + byte[] result = new byte[30]; + int len; + + len = decoder.decode(result); + assertEquals(30, len); + assertEquals("Hello World\nHello World\nHello ", new String(result, 0, len)); + + len = decoder.decode(result); + assertEquals(18, len); + assertEquals("World\nHello World\n", new String(result, 0, len)); + } + + @Test + public void decodeFixedHuffmanBlockWithMemoryLookupInExactBuffer() throws Exception { + byte[] data = { + //|--- binary filling ---|76543210 + 0b11111111111111111111111111110011, // final block + fixed huffman + H + 0b00000000000000000000000001001000, // H + e + 0b11111111111111111111111111001101, // e + l + 0b11111111111111111111111111001001, // l + l + 0b11111111111111111111111111001001, // l + o + 0b00000000000000000000000001010111, // o + ' ' + 0b00000000000000000000000000001000, // ' ' + W + 0b11111111111111111111111111001111, // W + o + 0b00000000000000000000000000101111, // o + r + 0b11111111111111111111111111001010, // r + l + 0b00000000000000000000000001001001, // l + d + 0b11111111111111111111111111100001, // d + '\n' + 0b00000000000000000000000000100010, // '\n' + + 0b11111111111111111111111110000110, // + offset <001> + dist6 + 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000) + 0b11111111111111111111111111111000 // end of block (0000) + garbage + }; + + HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data)); + byte[] result = new byte[48]; + int len; + + len = decoder.decode(result); + assertEquals(48, len); + assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len)); + + len = decoder.decode(result); + assertEquals(0, len); + + len = decoder.decode(result); + assertEquals(-1, len); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/DeflateTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/DeflateTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/DeflateTestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/DeflateTestCase.java 2018-05-02 20:17:13.000000000 +0000 @@ -35,7 +35,7 @@ /** * Tests the creation of a DEFLATE archive with zlib header - * + * * @throws Exception */ @Test @@ -52,7 +52,7 @@ /** * Tests the creation of a "raw" DEFLATE archive (without zlib header) - * + * * @throws Exception */ @Test @@ -70,7 +70,7 @@ /** * Tests the extraction of a DEFLATE archive with zlib header - * + * * @throws Exception */ @Test @@ -95,7 +95,7 @@ /** * Tests the extraction of a "raw" DEFLATE archive (without zlib header) - * + * * @throws Exception */ @Test diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java 2018-05-02 20:17:13.000000000 +0000 @@ -19,21 +19,26 @@ package org.apache.commons.compress.compressors; import static org.apache.commons.compress.AbstractTestCase.getFile; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.compress.compressors.CompressorInputStream; -import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.MemoryLimitException; +import org.apache.commons.compress.MockEvilInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; import org.junit.Test; @SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated @@ -51,7 +56,7 @@ factorySetTrue = new CompressorStreamFactory(); factorySetTrue.setDecompressConcatenated(true); factorySetFalse = new CompressorStreamFactory(); - factorySetFalse.setDecompressConcatenated(false); + factorySetFalse.setDecompressConcatenated(false); } static class TestData { @@ -86,17 +91,17 @@ new TestData("multiple.xz", new char[]{'a'}, factorySetFalse, false), new TestData("multiple.xz", new char[]{'a'}, factory, false), }; - + @Test public void testDetection() throws Exception { - final CompressorInputStream bzip2 = getStreamFor("bla.txt.bz2"); + final CompressorInputStream bzip2 = getStreamFor("bla.txt.bz2"); assertNotNull(bzip2); assertTrue(bzip2 instanceof BZip2CompressorInputStream); final CompressorInputStream gzip = getStreamFor("bla.tgz"); assertNotNull(gzip); assertTrue(gzip instanceof GzipCompressorInputStream); - + final CompressorInputStream pack200 = getStreamFor("bla.pack"); assertNotNull(pack200); assertTrue(pack200 instanceof Pack200CompressorInputStream); @@ -109,6 +114,10 @@ assertNotNull(zlib); assertTrue(zlib instanceof DeflateCompressorInputStream); + final CompressorInputStream zstd = getStreamFor("bla.tar.zst"); + assertNotNull(zstd); + assertTrue(zstd instanceof ZstdCompressorInputStream); + try { factory.createCompressorInputStream(new ByteArrayInputStream(new byte[0])); fail("No exception thrown for an empty input stream"); @@ -118,6 +127,106 @@ } @Test + public void testDetect() throws Exception { + + assertEquals(CompressorStreamFactory.BZIP2, detect("bla.txt.bz2")); + assertEquals(CompressorStreamFactory.GZIP, detect("bla.tgz")); + assertEquals(CompressorStreamFactory.PACK200, detect("bla.pack")); + assertEquals(CompressorStreamFactory.XZ, detect("bla.tar.xz")); + assertEquals(CompressorStreamFactory.DEFLATE, detect("bla.tar.deflatez")); + assertEquals(CompressorStreamFactory.LZ4_FRAMED, detect("bla.tar.lz4")); + assertEquals(CompressorStreamFactory.LZMA, detect("bla.tar.lzma")); + assertEquals(CompressorStreamFactory.SNAPPY_FRAMED, detect("bla.tar.sz")); + assertEquals(CompressorStreamFactory.Z, detect("bla.tar.Z")); + assertEquals(CompressorStreamFactory.ZSTANDARD, detect("bla.tar.zst")); + + //make sure we don't oom on detect + assertEquals(CompressorStreamFactory.Z, detect("COMPRESS-386")); + assertEquals(CompressorStreamFactory.LZMA, detect("COMPRESS-382")); + + try { + CompressorStreamFactory.detect(new BufferedInputStream(new ByteArrayInputStream(new byte[0]))); + fail("shouldn't be able to detect empty stream"); + } catch (CompressorException e) { + assertEquals("No Compressor found for the stream signature.", e.getMessage()); + } + + try { + CompressorStreamFactory.detect(null); + fail("shouldn't be able to detect null stream"); + } catch (IllegalArgumentException e) { + assertEquals("Stream must not be null.", e.getMessage()); + } + + try { + CompressorStreamFactory.detect(new BufferedInputStream(new MockEvilInputStream())); + fail("Expected IOException"); + } catch (CompressorException e) { + assertEquals("IOException while reading signature.", e.getMessage()); + } + + + } + + private String detect(String testFileName) throws IOException, CompressorException { + String name = null; + try (InputStream is = new BufferedInputStream( + new FileInputStream(getFile(testFileName)))) { + name = CompressorStreamFactory.detect(is); + } + return name; + } + + @Test(expected = MemoryLimitException.class) + public void testLZMAMemoryLimit() throws Exception { + getStreamFor("COMPRESS-382", 100); + } + + @Test(expected = MemoryLimitException.class) + public void testZMemoryLimit() throws Exception { + getStreamFor("COMPRESS-386", 100); + } + + @Test(expected = MemoryLimitException.class) + public void testXZMemoryLimitOnRead() throws Exception { + //Even though the file is very small, the memory limit + //has to be quite large (8296 KiB) because of the dictionary size + + //This is triggered on read(); not during initialization. + //This test is here instead of the xz unit test to make sure + //that the parameter is properly passed via the CompressorStreamFactory + try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) { + compressorIs.read(); + } + } + + @Test(expected = MemoryLimitException.class) + public void testXZMemoryLimitOnSkip() throws Exception { + try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) { + compressorIs.skip(10); + } + } + + private InputStream getStreamFor(final String fileName, final int memoryLimitInKb) throws Exception { + CompressorStreamFactory fac = new CompressorStreamFactory(true, + memoryLimitInKb); + InputStream is = new BufferedInputStream( + new FileInputStream(getFile(fileName))); + try { + return fac.createCompressorInputStream(is); + } catch (CompressorException e) { + if (e.getCause() != null && e.getCause() instanceof Exception) { + //unwrap cause to reveal MemoryLimitException + throw (Exception)e.getCause(); + } else { + throw e; + } + } + + } + + + @Test public void testOverride() { CompressorStreamFactory fac = new CompressorStreamFactory(); assertFalse(fac.getDecompressConcatenated()); @@ -153,7 +262,7 @@ final CompressorInputStream in = getStreamFor(test.fileName, fac); assertNotNull("Test entry "+i,in); for (final char entry : test.entryNames) { - assertEquals("Test entry" + i, entry, in.read()); + assertEquals("Test entry" + i, entry, in.read()); } assertEquals(0, in.available()); assertEquals(-1, in.read()); @@ -173,5 +282,4 @@ new BufferedInputStream(new FileInputStream( getFile(resource)))); } - } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java 2018-05-02 20:17:13.000000000 +0000 @@ -26,6 +26,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Random; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream; @@ -95,4 +96,60 @@ } } + @Test + public void testRoundtrip() throws Exception { + testRoundtrip(getFile("test.txt")); + testRoundtrip(getFile("bla.tar")); + testRoundtrip(getFile("COMPRESS-256.7z")); + } + + @Test + public void testRoundtripWithOneBigWrite() throws Exception { + Random r = new Random(); + File input = new File(dir, "bigChunkTest"); + try (FileOutputStream fs = new FileOutputStream(input)) { + for (int i = 0 ; i < 1 << 17; i++) { + fs.write(r.nextInt(256)); + } + } + long start = System.currentTimeMillis(); + final File outputSz = new File(dir, input.getName() + ".sz"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(outputSz); + CompressorOutputStream sos = new CompressorStreamFactory() + .createCompressorOutputStream("snappy-framed", os)) { + byte[] b = IOUtils.toByteArray(is); + sos.write(b[0]); + sos.write(b, 1, b.length - 1); // must be split into multiple compressed chunks + } + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + try (FileInputStream is = new FileInputStream(input); + CompressorInputStream sis = new CompressorStreamFactory() + .createCompressorInputStream("snappy-framed", new FileInputStream(outputSz))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(sis); + assertArrayEquals(expected, actual); + } + } + + private void testRoundtrip(File input) throws Exception { + long start = System.currentTimeMillis(); + final File outputSz = new File(dir, input.getName() + ".sz"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(outputSz); + CompressorOutputStream sos = new CompressorStreamFactory() + .createCompressorOutputStream("snappy-framed", os)) { + IOUtils.copy(is, sos); + } + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + try (FileInputStream is = new FileInputStream(input); + CompressorInputStream sis = new CompressorStreamFactory() + .createCompressorInputStream("snappy-framed", new FileInputStream(outputSz))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(sis); + assertArrayEquals(expected, actual); + } + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java 2018-08-09 13:48:20.000000000 +0000 @@ -145,7 +145,7 @@ try (FileInputStream fis = new FileInputStream(getFile("test3.xml"))) { content = IOUtils.toByteArray(fis); } - + final ByteArrayOutputStream bout = new ByteArrayOutputStream(); final GzipParameters parameters = new GzipParameters(); @@ -171,7 +171,7 @@ try (FileInputStream fis = new FileInputStream(getFile("test3.xml"))) { content = IOUtils.toByteArray(fis); } - + final ByteArrayOutputStream bout = new ByteArrayOutputStream(); final GzipParameters parameters = new GzipParameters(); @@ -200,7 +200,7 @@ } catch (final IllegalArgumentException e) { // expected } - + try { parameters.setCompressionLevel(-5); fail("IllegalArgumentException not thrown"); @@ -214,16 +214,16 @@ try (FileInputStream fis = new FileInputStream(getFile("test3.xml"))) { content = IOUtils.toByteArray(fis); } - + final ByteArrayOutputStream bout = new ByteArrayOutputStream(); - + final GzipParameters parameters = new GzipParameters(); parameters.setCompressionLevel(compressionLevel); final GzipCompressorOutputStream out = new GzipCompressorOutputStream(bout, parameters); IOUtils.copy(new ByteArrayInputStream(content), out); out.flush(); out.close(); - + assertEquals("extra flags (XFL)", flag, bout.toByteArray()[8]); } @@ -241,7 +241,7 @@ public void testExtraFlagsDefaultCompression() throws Exception { testExtraFlags(Deflater.DEFAULT_COMPRESSION, 0); } - + @Test public void testOverWrite() throws Exception { final GzipCompressorOutputStream out = new GzipCompressorOutputStream(new ByteArrayOutputStream()); @@ -257,7 +257,7 @@ @Test public void testMetadataRoundTrip() throws Exception { final ByteArrayOutputStream bout = new ByteArrayOutputStream(); - + final GzipParameters parameters = new GzipParameters(); parameters.setCompressionLevel(Deflater.BEST_COMPRESSION); parameters.setModificationTime(123456000); @@ -268,7 +268,7 @@ ".xml"))) { IOUtils.copy(fis, out); } - + final GzipCompressorInputStream input = new GzipCompressorInputStream(new ByteArrayInputStream(bout.toByteArray())); input.close(); @@ -279,4 +279,31 @@ assertEquals("test3.xml", readParams.getFilename()); assertEquals("Umlaute möglich?", readParams.getComment()); } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tgz"); + try (InputStream is = new FileInputStream(input)) { + final GzipCompressorInputStream in = + new GzipCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tgz"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final GzipCompressorInputStream in = + new GzipCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStreamTest.java 2018-08-09 13:51:02.000000000 +0000 @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public class BlockLZ4CompressorInputStreamTest extends AbstractTestCase { + + @Test + public void readBlaLz4() throws IOException { + try (InputStream a = new BlockLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar.block_lz4"))); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + Assert.assertArrayEquals(expected, actual); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.block_lz4"); + try (InputStream is = new FileInputStream(input)) { + final BlockLZ4CompressorInputStream in = + new BlockLZ4CompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.block_lz4"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final BlockLZ4CompressorInputStream in = + new BlockLZ4CompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.compress.compressors.lz77support.LZ77Compressor; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +public class BlockLZ4CompressorOutputStreamTest { + + @Test + public void pairSeesBackReferenceWhenSet() { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + Assert.assertFalse(p.hasBackReference()); + p.setBackReference(new LZ77Compressor.BackReference(1, 4)); + Assert.assertTrue(p.hasBackReference()); + } + + @Test + public void canWriteBackReferenceFollowedByLongLiteral() { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 4)); + // a length of 11 would be enough according to the spec, but + // the algorithm we use for rewriting the last block requires + // 16 bytes + Assert.assertTrue(p.canBeWritten(16)); + } + + @Test + @Ignore("would pass if the algorithm used for rewriting the final pairs was smarter") + public void canWriteBackReferenceFollowedByShortLiteralIfOffsetIsBigEnough() { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(10, 4)); + Assert.assertTrue(p.canBeWritten(5)); + } + + @Test + @Ignore("would pass if the algorithm used for rewriting the final pairs was smarter") + public void canWriteBackReferenceFollowedByShortLiteralIfLengthIsBigEnough() { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 10)); + Assert.assertTrue(p.canBeWritten(5)); + } + + @Test + public void cantWriteBackReferenceFollowedByLiteralThatIsTooShort() { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(10, 14)); + Assert.assertFalse(p.canBeWritten(4)); + } + + @Test + public void cantWriteBackReferenceIfAccumulatedOffsetIsTooShort() { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 4)); + Assert.assertFalse(p.canBeWritten(5)); + } + + @Test + public void pairAccumulatesLengths() { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 4)); + byte[] b = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 1, 4)); + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 2, 5)); + Assert.assertEquals(13, p.length()); + } + + @Test + public void canWritePairWithoutLiterals() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 4)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { 0, 1, 0 }, bos.toByteArray()); + } + + @Test + public void writesCorrectSizeFor19ByteLengthBackReference() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 19)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { 15, 1, 0, 0 }, bos.toByteArray()); + } + + @Test + public void writesCorrectSizeFor273ByteLengthBackReference() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 273)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { 15, 1, 0, (byte) 254 }, bos.toByteArray()); + } + + @Test + public void writesCorrectSizeFor274ByteLengthBackReference() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + p.setBackReference(new LZ77Compressor.BackReference(1, 274)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { 15, 1, 0, (byte) 255, 0 }, bos.toByteArray()); + } + + @Test + public void canWritePairWithoutBackReference() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + byte[] b = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 1, 4)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { 4<<4, 2, 3, 4, 5 }, bos.toByteArray()); + } + + @Test + public void writesCorrectSizeFor15ByteLengthLiteral() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + byte[] b = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 0, 9)); + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 0, 6)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { (byte) (15<<4), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6 }, + bos.toByteArray()); + } + + @Test + public void writesCorrectSizeFor269ByteLengthLiteral() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + byte[] b = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + for (int i = 0; i < 26; i++) { + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 0, 10)); + } + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 0, 9)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { (byte) (15<<4), (byte) 254, 1 }, + Arrays.copyOfRange(bos.toByteArray(), 0, 3)); + } + + @Test + public void writesCorrectSizeFor270ByteLengthLiteral() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + byte[] b = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + for (int i = 0; i < 27; i++) { + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 0, 10)); + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { (byte) (15<<4), (byte) 255, 0, 1 }, + Arrays.copyOfRange(bos.toByteArray(), 0, 4)); + } + + @Test + public void writesCompletePair() throws IOException { + BlockLZ4CompressorOutputStream.Pair p = new BlockLZ4CompressorOutputStream.Pair(); + byte[] b = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + p.addLiteral(new LZ77Compressor.LiteralBlock(b, 1, 4)); + b[2] = 19; + p.setBackReference(new LZ77Compressor.BackReference(1, 5)); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + p.writeTo(bos); + Assert.assertArrayEquals(new byte[] { (4<<4) + 1, 2, 3, 4, 5, 1, 0 }, + bos.toByteArray()); + } + + @Test + public void rewritingOfFinalBlockWithoutTrailingLZ77Literals() throws IOException { + for (int i = 1; i < 13; i++) { + // according to the spec these are all too short be compressed + // LZ77Compressor will create a single byte literal + // followed by a back-reference starting with i = 5, + // though. (4 is the minimum length for a back-reference + // in LZ4 + byte[] compressed = compress(i); + byte[] expected = prepareExpected(i + 1); + expected[0] = (byte) (i<<4); + Assert.assertArrayEquals("input length is " + i, expected, compressed); + } + + for (int i = 13; i < 17; i++) { + // LZ77Compressor will still create a single byte literal + // followed by a back-reference + // according to the spec the back-reference could be split + // as we can cut out a five byte literal and the offset + // would be big enough, but our algorithm insists on a + // twelve byte literal trailer and the back-reference + // would fall below the minimal size + byte[] compressed = compress(i); + byte[] expected = prepareExpected(i < 15 ? i + 1 : i + 2); + if (i < 15) { + expected[0] = (byte) (i<<4); + } else { + expected[0] = (byte) (15<<4); + expected[1] = (byte) (i - 15); + } + Assert.assertArrayEquals("input length is " + i, expected, compressed); + } + + for (int i = 17; i < 20; i++) { + // LZ77Compressor will still create a single byte literal + // followed by a back-reference + // this time even our algorithm is willing to break up the + // back-reference + byte[] compressed = compress(i); + byte[] expected = prepareExpected(17); + expected[0] = (byte) ((1<<4) | i - 17); + // two-byte offset + expected[2] = 1; + expected[3] = 0; + expected[4] = (byte) (12<<4); + Assert.assertArrayEquals("input length is " + i, expected, compressed); + } + } + + @Test + public void rewritingOfFinalBlockWithTrailingLZ77Literals() throws IOException { + for (int i = 1; i < 5; i++) { + // LZ77Compressor will create a single byte literal + // followed by a back-reference of length 15 followed by a + // literal of length i + // we can split the back-reference and merge it with the literal + byte[] compressed = compress(16, i); + byte[] expected = prepareExpected(17); + expected[0] = (byte) ((1<<4) | i - 1); + // two-byte offset + expected[2] = 1; + expected[3] = 0; + expected[4] = (byte) (12<<4); + for (int j = 0; j < i; j++) { + expected[expected.length - 1 - j] = 1; + } + Assert.assertArrayEquals("trailer length is " + i, expected, compressed); + } + for (int i = 5; i < 12; i++) { + // LZ77Compressor will create a single byte literal + // followed by a back-reference of length 15 followed by + // another single byte literal and another back-reference + // of length i-1 + // according to the spec we could completely satisfy the + // requirements by just rewriting the last Pair, but our + // algorithm will chip off a few bytes from the first Pair + byte[] compressed = compress(16, i); + byte[] expected = prepareExpected(17); + expected[0] = (byte) ((1<<4) | i - 1); + // two-byte offset + expected[2] = 1; + expected[3] = 0; + expected[4] = (byte) (12<<4); + for (int j = 0; j < i; j++) { + expected[expected.length - 1 - j] = 1; + } + Assert.assertArrayEquals("trailer length is " + i, expected, compressed); + } + for (int i = 12; i < 15; i++) { + // LZ77Compressor will create a single byte literal + // followed by a back-reference of length 15 followed by + // another single byte literal and another back-reference + // of length i-1 + // this shouldn't affect the first pair at all as + // rewriting the second one is sufficient + byte[] compressed = compress(16, i); + byte[] expected = prepareExpected(i + 5); + expected[0] = (byte) ((1<<4) | 11); + // two-byte offset + expected[2] = 1; + expected[3] = 0; + expected[4] = (byte) (i<<4); + for (int j = 0; j < i; j++) { + expected[expected.length - 1 - j] = 1; + } + Assert.assertArrayEquals("trailer length is " + i, expected, compressed); + } + } + + @Test + public void rewritingOfFourPairs() throws IOException { + // LZ77Compressor creates three times a literal block followed + // by a back-reference (once 5 bytes long and twice four bytes + // long and a final literal block of length 1 + // in the result the three last pairs are merged into a single + // literal and one byte is chopped off of the first pair's + // back-reference + byte[] compressed = compress(6, 5, 5, 1); + byte[] expected = prepareExpected(17); + expected[0] = (byte) (1<<4); + // two-byte offset + expected[2] = 1; + expected[3] = 0; + expected[4] = (byte) (12<<4); + for (int i = 6; i < 11; i++) { + expected[i] = 1; + } + for (int i = 11; i < 16; i++) { + expected[i] = 2; + } + expected[16] = 3; + Assert.assertArrayEquals(expected, compressed); + } + + @Test + public void rewritingWithFinalBackreferenceAndOffsetBiggerThan1() throws IOException { + // this caused trouble when expandFromList() fell into the "offsetRemaining is negative" self-copy case as the + // calculation of copyOffset was wrong + byte[] toCompress = prepareExpected(25); + for (int i = 0; i < toCompress.length; i += 4) { + toCompress[i] = 1; + } + // LZ77Compressor creates a four byte literal and a back-reference with offset 4 and length 21 + // we'll need to split the back-reference and chop off the last 12 bytes + byte[] compressed = compress(toCompress); + byte[] expected = prepareExpected(1 + 4 + 2 + 1 + 12); + expected[0] = (byte) ((4<<4) | 5); + expected[1] = 1; + expected[5] = 4; + expected[6] = 0; + expected[7] = (byte) (12<<4); + for (int i = 11; i < expected.length; i += 4) { + expected[i] = 1; + } + Assert.assertArrayEquals(expected, compressed); + } + + private byte[] compress(int length) throws IOException { + return compress(length, 0); + } + + private byte[] compress(int lengthBeforeTrailer, int... lengthOfTrailers) throws IOException { + byte[] b = prepareExpected(lengthBeforeTrailer); + return compress(b, lengthOfTrailers); + } + + private byte[] compress(byte[] input, int... lengthOfTrailers) throws IOException { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BlockLZ4CompressorOutputStream lo = new BlockLZ4CompressorOutputStream(baos)) { + lo.write(input); + for (int i = 0; i < lengthOfTrailers.length; i++) { + int lengthOfTrailer = lengthOfTrailers[i]; + for (int j = 0; j < lengthOfTrailer; j++) { + lo.write(i + 1); + } + } + lo.close(); + return baos.toByteArray(); + } + } + + private byte[] prepareExpected(int length) { + byte[] b = new byte[length]; + Arrays.fill(b, (byte) -1); + return b; + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorRoundtripTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorRoundtripTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorRoundtripTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorRoundtripTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.lz77support.Parameters; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.junit.runner.RunWith; + +@RunWith(Parameterized.class) +public final class BlockLZ4CompressorRoundtripTest extends AbstractTestCase { + + @org.junit.runners.Parameterized.Parameters(name = "using {0}") + public static Collection factory() { + return Arrays.asList(new Object[][] { + new Object[] { "default", BlockLZ4CompressorOutputStream.createParameterBuilder().build() }, + new Object[] { "tuned for speed", + BlockLZ4CompressorOutputStream.createParameterBuilder().tunedForSpeed().build() }, + new Object[] { "tuned for compression ratio", + BlockLZ4CompressorOutputStream.createParameterBuilder().tunedForCompressionRatio().build() } + }); + } + + private final String config; + private final Parameters params; + + public BlockLZ4CompressorRoundtripTest(String config, Parameters params) { + this.config = config; + this.params = params; + } + + private void roundTripTest(String testFile) throws IOException { + File input = getFile(testFile); + long start = System.currentTimeMillis(); + final File outputSz = new File(dir, input.getName() + ".block.lz4"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(outputSz); + BlockLZ4CompressorOutputStream los = new BlockLZ4CompressorOutputStream(os, params)) { + IOUtils.copy(is, los); + } + System.err.println("Configuration: " + config); + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + start = System.currentTimeMillis(); + try (FileInputStream is = new FileInputStream(input); + BlockLZ4CompressorInputStream sis = new BlockLZ4CompressorInputStream(new FileInputStream(outputSz))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(sis); + Assert.assertArrayEquals(expected, actual); + } + System.err.println(outputSz.getName() + " read after " + (System.currentTimeMillis() - start) + "ms"); + } + + // should yield decent compression + @Test + public void blaTarRoundtrip() throws IOException { + roundTripTest("bla.tar"); + } + + // yields no compression at all + @Test + public void gzippedLoremIpsumRoundtrip() throws IOException { + roundTripTest("lorem-ipsum.txt.gz"); + } + + // yields no compression at all + @Test + public void biggerFileRoundtrip() throws IOException { + roundTripTest("COMPRESS-256.7z"); + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/FactoryTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/FactoryTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/FactoryTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/FactoryTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public class FactoryTest extends AbstractTestCase { + + @Test + public void frameRoundtripViaFactory() throws Exception { + roundtripViaFactory(CompressorStreamFactory.getLZ4Framed()); + } + + @Test + public void blockRoundtripViaFactory() throws Exception { + roundtripViaFactory(CompressorStreamFactory.getLZ4Block()); + } + + private void roundtripViaFactory(String format) throws Exception { + File input = getFile("bla.tar"); + long start = System.currentTimeMillis(); + final File outputSz = new File(dir, input.getName() + "." + format + ".lz4"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(outputSz); + OutputStream los = new CompressorStreamFactory().createCompressorOutputStream(format, os)) { + IOUtils.copy(is, los); + } + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + start = System.currentTimeMillis(); + try (FileInputStream is = new FileInputStream(input); + InputStream sis = new CompressorStreamFactory() + .createCompressorInputStream(format, new FileInputStream(outputSz))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(sis); + Assert.assertArrayEquals(expected, actual); + } + System.err.println(outputSz.getName() + " read after " + (System.currentTimeMillis() - start) + "ms"); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java 2018-08-09 13:57:47.000000000 +0000 @@ -0,0 +1,645 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Test; + +public final class FramedLZ4CompressorInputStreamTest + extends AbstractTestCase { + + @Test + public void testMatches() throws IOException { + assertFalse(FramedLZ4CompressorInputStream.matches(new byte[10], 4)); + final byte[] b = new byte[12]; + final File input = getFile("bla.tar.lz4"); + try (FileInputStream in = new FileInputStream(input)) { + IOUtils.readFully(in, b); + } + assertFalse(FramedLZ4CompressorInputStream.matches(b, 3)); + assertTrue(FramedLZ4CompressorInputStream.matches(b, 4)); + assertTrue(FramedLZ4CompressorInputStream.matches(b, 5)); + } + + @Test + public void readBlaLz4() throws IOException { + try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar.lz4"))); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void readBlaLz4ViaFactory() throws Exception { + try (InputStream a = new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), + new FileInputStream(getFile("bla.tar.lz4"))); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void readBlaLz4ViaFactoryAutoDetection() throws Exception { + try (InputStream a = new CompressorStreamFactory() + .createCompressorInputStream(new BufferedInputStream(new FileInputStream(getFile("bla.tar.lz4")))); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void readBlaLz4WithDecompressConcatenated() throws IOException { + try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar.lz4")), true); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void readDoubledBlaLz4WithDecompressConcatenatedTrue() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + @Override + public InputStream wrap(InputStream in) throws Exception { + return new FramedLZ4CompressorInputStream(in, true); + } + }, true); + } + + @Test + public void readDoubledBlaLz4WithDecompressConcatenatedFalse() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + @Override + public InputStream wrap(InputStream in) throws Exception { + return new FramedLZ4CompressorInputStream(in, false); + } + }, false); + } + + @Test + public void readDoubledBlaLz4WithoutExplicitDecompressConcatenated() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + @Override + public InputStream wrap(InputStream in) throws Exception { + return new FramedLZ4CompressorInputStream(in); + } + }, false); + } + + @Test + public void readBlaLz4ViaFactoryWithDecompressConcatenated() throws Exception { + try (InputStream a = new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), + new FileInputStream(getFile("bla.tar.lz4")), + true); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedTrue() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + @Override + public InputStream wrap(InputStream in) throws Exception { + return new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, true); + } + }, true); + } + + @Test + public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedFalse() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + @Override + public InputStream wrap(InputStream in) throws Exception { + return new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, false); + } + }, false); + } + + @Test + public void readDoubledBlaLz4ViaFactoryWithoutExplicitDecompressConcatenated() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + @Override + public InputStream wrap(InputStream in) throws Exception { + return new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in); + } + }, false); + } + + @Test + public void readBlaDumpLz4() throws IOException { + try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.dump.lz4"))); + FileInputStream e = new FileInputStream(getFile("bla.dump"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test(expected = IOException.class) + public void rejectsNonLZ4Stream() throws IOException { + try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar")))) { + fail("expected exception"); + } + } + + @Test + public void rejectsFileWithoutFrameDescriptor() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18 // signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("frame flags")); + } + } + + @Test + public void rejectsFileWithoutBlockSizeByte() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("BD byte")); + } + } + + @Test + public void rejectsFileWithWrongVersion() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x24, // flag - Version 00, block independent, no block checksum, no content size, with content checksum + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("version")); + } + } + + @Test + public void rejectsFileWithInsufficientContentSize() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x6C, // flag - Version 01, block independent, no block checksum, with content size, with content checksum + 0x70, // block size 4MB + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("content size")); + } + } + + @Test + public void rejectsFileWithoutHeaderChecksum() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum + 0x70, // block size 4MB + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("header checksum")); + } + } + + @Test + public void rejectsFileWithBadHeaderChecksum() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum + 0x70, // block size 4MB + 0, + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("header checksum mismatch")); + } + } + + @Test + public void readsUncompressedBlocks() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + }; + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(new byte[] { + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!' + }, actual); + } + } + + @Test + public void readsUncompressedBlocksUsingSingleByteRead() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + }; + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + int h = a.read(); + assertEquals('H', h); + } + } + + @Test + public void rejectsBlocksWithoutChecksum() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x70, // flag - Version 01, block independent, with block checksum, no content size, no content checksum + 0x70, // block size 4MB + 114, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("block checksum")); + } + } + + @Test + public void rejectsStreamsWithoutContentChecksum() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum + 0x70, // block size 4MB + (byte) 185, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("content checksum")); + } + } + + @Test + public void rejectsStreamsWithBadContentChecksum() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum + 0x70, // block size 4MB + (byte) 185, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 1, 2, 3, 4, + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("content checksum mismatch")); + } + } + + @Test + public void skipsOverSkippableFrames() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x5f, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 1, 0, 0, (byte) 0x80, // 1 bytes length and uncompressed bit set + '!', // content + 0, 0, 0, 0, // empty block marker + }; + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(new byte[] { + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', '!' + }, actual); + } + } + + @Test + public void skipsOverTrailingSkippableFrames() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x51, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + }; + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(new byte[] { + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!' + }, actual); + } + } + + @Test + public void rejectsSkippableFrameFollowedByJunk() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x50, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + 1, 0x22, 0x4d, 0x18, // bad signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsSkippableFrameFollowedByTooFewBytes() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x52, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + 4, // too short for signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsSkippableFrameWithPrematureEnd() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x50, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, // content of skippable frame (should be two bytes) + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("Premature end of stream while skipping frame")); + } + } + + @Test + public void rejectsSkippableFrameWithPrematureEndInLengthBytes() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x55, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, // should be four byte length + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("premature end of data")); + } + } + + @Test + public void rejectsSkippableFrameWithBadSignatureTrailer() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x51, 0x2a, 0x4d, 0x17, // broken skippable frame signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsSkippableFrameWithBadSignaturePrefix() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x60, 0x2a, 0x4d, 0x18, // broken skippable frame signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsTrailingBytesAfterValidFrame() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x56, 0x2a, 0x4d, // too short for any signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.lz4"); + try (InputStream is = new FileInputStream(input)) { + final FramedLZ4CompressorInputStream in = + new FramedLZ4CompressorInputStream(is); + IOUtils.toByteArray(in); + assertEquals(-1, in.read()); + assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.lz4"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final FramedLZ4CompressorInputStream in = + new FramedLZ4CompressorInputStream(is); + IOUtils.toByteArray(in); + assertEquals(-1, in.read(buf)); + assertEquals(-1, in.read(buf)); + in.close(); + } + } + + interface StreamWrapper { + InputStream wrap(InputStream in) throws Exception; + } + + private void readDoubledBlaLz4(StreamWrapper wrapper, boolean expectDuplicateOutput) throws Exception { + byte[] singleInput; + try (InputStream i = new FileInputStream(getFile("bla.tar.lz4"))) { + singleInput = IOUtils.toByteArray(i); + } + byte[] input = duplicate(singleInput); + try (InputStream a = wrapper.wrap(new ByteArrayInputStream(input)); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expectDuplicateOutput ? duplicate(expected) : expected, actual); + } + } + + private static byte[] duplicate(byte[] from) { + byte[] to = Arrays.copyOf(from, 2 * from.length); + System.arraycopy(from, 0, to, from.length, from.length); + return to; + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public final class FramedLZ4CompressorRoundtripTest extends AbstractTestCase { + + @Parameters(name = "using {0}") + public static Collection factory() { + return Arrays.asList(new Object[][] { + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64) }, + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K256) }, + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M1) }, + new Object[] { FramedLZ4CompressorOutputStream.Parameters.DEFAULT }, + // default without content checksum + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4, + false, false, false) }, + // default with block checksum + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4, + true, true, false) }, + // small blocksize (so we get enough blocks) and enabled block dependency, otherwise defaults + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64, + true, false, true) }, + // default, tuned for speed + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4, + true, false, false, BlockLZ4CompressorOutputStream.createParameterBuilder() + .tunedForSpeed().build()) }, + }); + } + + private final FramedLZ4CompressorOutputStream.Parameters params; + + public FramedLZ4CompressorRoundtripTest(FramedLZ4CompressorOutputStream.Parameters params) { + this.params = params; + } + + private void roundTripTest(String testFile) throws IOException { + File input = getFile(testFile); + long start = System.currentTimeMillis(); + final File outputSz = new File(dir, input.getName() + ".framed.lz4"); + byte[] expected; + try (FileInputStream is = new FileInputStream(input)) { + expected = IOUtils.toByteArray(is); + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (FramedLZ4CompressorOutputStream los = new FramedLZ4CompressorOutputStream(bos, + params)) { + IOUtils.copy(new ByteArrayInputStream(expected), los); + } + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + start = System.currentTimeMillis(); + try (FramedLZ4CompressorInputStream sis = new FramedLZ4CompressorInputStream( + new ByteArrayInputStream(bos.toByteArray()))) { + byte[] actual = IOUtils.toByteArray(sis); + Assert.assertArrayEquals(expected, actual); + } + + System.err.println(outputSz.getName() + " read after " + (System.currentTimeMillis() - start) + "ms"); + } + + // should yield decent compression + @Test + public void blaTarRoundtrip() throws IOException { + roundTripTest("bla.tar"); + } + + // yields no compression at all + @Test + public void gzippedLoremIpsumRoundtrip() throws IOException { + roundTripTest("lorem-ipsum.txt.gz"); + } + + @Test + public void biggerFileRoundtrip() throws IOException { + roundTripTest("COMPRESS-256.7z"); + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/XXHash32Test.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/XXHash32Test.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz4/XXHash32Test.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz4/XXHash32Test.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz4; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.utils.IOUtils; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runner.RunWith; + +@RunWith(Parameterized.class) +public class XXHash32Test { + + private final File file; + private final String expectedChecksum; + + public XXHash32Test(String fileName, String c) throws IOException { + file = AbstractTestCase.getFile(fileName); + expectedChecksum = c; + } + + @Parameters + public static Collection factory() { + return Arrays.asList(new Object[][] { + // reference checksums created with xxh32sum + { "bla.tar", "fbb5c8d1" }, + { "bla.tar.xz", "4106a208" }, + { "8.posix.tar.gz", "9fce116a" }, + }); + } + + @Test + public void verifyChecksum() throws IOException { + XXHash32 h = new XXHash32(); + try (FileInputStream s = new FileInputStream(file)) { + byte[] b = IOUtils.toByteArray(s); + h.update(b, 0, b.length); + } + Assert.assertEquals("checksum for " + file.getName(), expectedChecksum, Long.toHexString(h.getValue())); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz77support; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class AbstractLZ77CompressorInputStreamTest { + + private static class TestStream extends AbstractLZ77CompressorInputStream { + private boolean literal; + TestStream(InputStream in) throws IOException { + super(in, 1024); + } + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + if (literal) { + return readLiteral(b, off, len); + } + return readBackReference(b, off, len); + } + void literal(int len) { + startLiteral(len); + literal = true; + } + } + + @Test(expected = IllegalStateException.class) + public void cantPrefillAfterDataHasBeenRead() throws IOException { + byte[] data = new byte[] { 1, 2, 3, 4 }; + try (TestStream s = new TestStream(new ByteArrayInputStream(data))) { + s.literal(3); + assertEquals(1, s.read()); + s.prefill(new byte[] { 1, 2, 3 }); + } + } + + @Test + public void prefillCanBeUsedForBackReferences() throws IOException { + byte[] data = new byte[] { 1, 2, 3, 4 }; + try (TestStream s = new TestStream(new ByteArrayInputStream(new byte[0]))) { + s.prefill(data); + s.startBackReference(2, 4); + byte[] r = new byte[4]; + assertEquals(4, s.read(r)); + assertArrayEquals(new byte[] { 3, 4, 3, 4 }, r); + } + } + + @Test + public void ifPrefillExceedsWindowSizeTheLastBytesAreUsed() throws IOException { + byte[] data = new byte[2048]; + data[2046] = 3; + data[2047] = 4; + try (TestStream s = new TestStream(new ByteArrayInputStream(new byte[0]))) { + s.prefill(data); + s.startBackReference(2, 4); + byte[] r = new byte[4]; + assertEquals(4, s.read(r)); + assertArrayEquals(new byte[] { 3, 4, 3, 4 }, r); + } + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz77support/LZ77CompressorTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz77support/LZ77CompressorTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz77support/LZ77CompressorTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz77support/LZ77CompressorTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz77support; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class LZ77CompressorTest { + + private static final byte[] BLA, SAM, ONE_TO_TEN; + + static { + try { + /* + * Example from "An Explanation of the Deflate Algorithm" by "Antaeus Feldspar". + * @see "http://zlib.net/feldspar.html" + */ + BLA = "Blah blah blah blah blah!".getBytes("ASCII"); + + /* + * Example from Wikipedia article about LZSS. + * Note the example uses indices instead of offsets. + * @see "https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski" + */ + SAM = ("I am Sam\n" + + "\n" + + "Sam I am\n" + + "\n" + + "That Sam-I-am!\n" + + "That Sam-I-am!\n" + + "I do not like\n" + + "that Sam-I-am!\n" + + "\n" + + "Do you like green eggs and ham?\n" + + "\n" + + "I do not like them, Sam-I-am.\n" + + "I do not like green eggs and ham.").getBytes("ASCII"); + } catch (IOException ex) { + throw new RuntimeException("ASCII not supported"); + } + ONE_TO_TEN = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + } + + private List compress(Parameters params, byte[]... chunks) throws IOException { + final List blocks = new ArrayList<>(); + LZ77Compressor c = new LZ77Compressor(params, new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) { + //System.err.println(block); + if (block instanceof LZ77Compressor.LiteralBlock) { + // replace with a real copy of data so tests + // can see the results as they've been when + // the callback has been called + LZ77Compressor.LiteralBlock b = (LZ77Compressor.LiteralBlock) block; + int len = b.getLength(); + block = new LZ77Compressor.LiteralBlock( + Arrays.copyOfRange(b.getData(), b.getOffset(), b.getOffset() + len), + 0, len); + } + blocks.add(block); + } + }); + for (byte[] chunk : chunks) { + c.compress(chunk); + } + c.finish(); + return blocks; + } + + @Test + public void nonCompressableWithLengthSmallerThanLiteralMax() throws IOException { + List blocks = compress(newParameters(128), ONE_TO_TEN); + assertSize(2, blocks); + assertLiteralBlock(ONE_TO_TEN, blocks.get(0)); + } + + @Test + public void nonCompressableWithLengthGreaterThanLiteralMaxButLessThanTwiceWindowSize() throws IOException { + List blocks = compress(newParameters(8), ONE_TO_TEN); + assertSize(3, blocks); + assertLiteralBlock(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }, blocks.get(0)); + assertLiteralBlock(new byte[] { 9, 10 }, blocks.get(1)); + } + + @Test + public void nonCompressableWithLengthThatForcesWindowSlide() throws IOException { + List blocks = compress(newParameters(4), ONE_TO_TEN); + assertSize(4, blocks); + assertLiteralBlock(new byte[] { 1, 2, 3, 4, }, blocks.get(0)); + assertLiteralBlock(new byte[] { 5, 6, 7, 8 }, blocks.get(1)); + assertLiteralBlock(new byte[] { 9, 10 }, blocks.get(2)); + } + + @Test + public void nonCompressableSentAsSingleBytes() throws IOException { + List blocks = compress(newParameters(8), stagger(ONE_TO_TEN)); + assertSize(3, blocks); + assertLiteralBlock(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }, blocks.get(0)); + assertLiteralBlock(new byte[] { 9, 10 }, blocks.get(1)); + } + + @Test + public void blaExampleWithFullArrayAvailableForCompression() + throws IOException { + List blocks = compress(newParameters(128), BLA); + assertSize(4, blocks); + assertLiteralBlock("Blah b", blocks.get(0)); + assertBackReference(5, 18, blocks.get(1)); + assertLiteralBlock("!", blocks.get(2)); + } + + @Test + public void blaExampleWithShorterBackReferenceLength() throws IOException { + List blocks = compress(newParameters(128, 3, 5, 0, 0), BLA); + assertSize(7, blocks); + assertLiteralBlock("Blah b", blocks.get(0)); + assertBackReference(5, 5, blocks.get(1)); + assertBackReference(5, 5, blocks.get(2)); + assertBackReference(5, 5, blocks.get(3)); + assertBackReference(5, 3, blocks.get(4)); + assertLiteralBlock("!", blocks.get(5)); + } + + @Test + public void blaExampleSmallerWindowSize() throws IOException { + List blocks = compress(newParameters(8), BLA); + assertSize(6, blocks); + assertLiteralBlock("Blah b", blocks.get(0)); + assertBackReference(5, 7, blocks.get(1)); + assertBackReference(5, 3, blocks.get(2)); + assertBackReference(5, 7, blocks.get(3)); + assertLiteralBlock("h!", blocks.get(4)); + } + + @Test + public void blaExampleWithSingleByteWrites() throws IOException { + List blocks = compress(newParameters(128), stagger(BLA)); + assertEquals(9, blocks.size()); + assertLiteralBlock("Blah b", blocks.get(0)); + assertBackReference(5, 3, blocks.get(1)); + assertBackReference(5, 3, blocks.get(2)); + assertBackReference(5, 3, blocks.get(3)); + assertBackReference(5, 3, blocks.get(4)); + assertBackReference(5, 3, blocks.get(5)); + assertBackReference(5, 3, blocks.get(6)); + assertLiteralBlock("!", blocks.get(7)); + } + + @Test + public void samIAmExampleWithFullArrayAvailableForCompression() throws IOException { + List blocks = compress(newParameters(1024), SAM); + assertEquals(21, blocks.size()); + assertLiteralBlock("I am Sam\n\n", blocks.get(0)); + assertBackReference(5, 3, blocks.get(1)); + assertLiteralBlock(" ", blocks.get(2)); + assertBackReference(14, 4, blocks.get(3)); + assertLiteralBlock("\n\nThat", blocks.get(4)); + assertBackReference(20, 4, blocks.get(5)); + assertLiteralBlock("-I-am!", blocks.get(6)); + assertBackReference(15, 16, blocks.get(7)); + assertLiteralBlock("I do not like\nt", blocks.get(8)); + assertBackReference(29, 14, blocks.get(9)); + assertLiteralBlock("\nDo you", blocks.get(10)); + assertBackReference(28, 5, blocks.get(11)); + assertLiteralBlock(" green eggs and ham?\n", blocks.get(12)); + assertBackReference(63, 14, blocks.get(13)); + assertLiteralBlock(" them,", blocks.get(14)); + assertBackReference(64, 9, blocks.get(15)); + assertLiteralBlock(".", blocks.get(16)); + assertBackReference(30, 15, blocks.get(17)); + assertBackReference(65, 18, blocks.get(18)); + assertLiteralBlock(".", blocks.get(19)); + } + + @Test + public void blaExampleWithPrefill() throws IOException { + final List blocks = new ArrayList<>(); + LZ77Compressor c = new LZ77Compressor(newParameters(128), new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) { + //System.err.println(block); + if (block instanceof LZ77Compressor.LiteralBlock) { + // replace with a real copy of data so tests + // can see the results as they've been when + // the callback has been called + LZ77Compressor.LiteralBlock b = (LZ77Compressor.LiteralBlock) block; + int len = b.getLength(); + block = new LZ77Compressor.LiteralBlock( + Arrays.copyOfRange(b.getData(), b.getOffset(), b.getOffset() + len), + 0, len); + } + blocks.add(block); + } + }); + c.prefill(Arrays.copyOfRange(BLA, 0, 6)); + c.compress(Arrays.copyOfRange(BLA, 6, BLA.length)); + c.finish(); + assertSize(3, blocks); + assertBackReference(5, 18, blocks.get(0)); + assertLiteralBlock("!", blocks.get(1)); + } + + @Test + public void blaExampleWithShortPrefill() throws IOException { + final List blocks = new ArrayList<>(); + LZ77Compressor c = new LZ77Compressor(newParameters(128), new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) { + //System.err.println(block); + if (block instanceof LZ77Compressor.LiteralBlock) { + // replace with a real copy of data so tests + // can see the results as they've been when + // the callback has been called + LZ77Compressor.LiteralBlock b = (LZ77Compressor.LiteralBlock) block; + int len = b.getLength(); + block = new LZ77Compressor.LiteralBlock( + Arrays.copyOfRange(b.getData(), b.getOffset(), b.getOffset() + len), + 0, len); + } + blocks.add(block); + } + }); + c.prefill(Arrays.copyOfRange(BLA, 0, 2)); + c.compress(Arrays.copyOfRange(BLA, 2, BLA.length)); + c.finish(); + assertSize(4, blocks); + assertLiteralBlock("ah b", blocks.get(0)); + assertBackReference(5, 18, blocks.get(1)); + assertLiteralBlock("!", blocks.get(2)); + } + + @Test + public void blaExampleWithPrefillBiggerThanWindowSize() throws IOException { + final List blocks = new ArrayList<>(); + LZ77Compressor c = new LZ77Compressor(newParameters(4), new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) { + //System.err.println(block); + if (block instanceof LZ77Compressor.LiteralBlock) { + // replace with a real copy of data so tests + // can see the results as they've been when + // the callback has been called + LZ77Compressor.LiteralBlock b = (LZ77Compressor.LiteralBlock) block; + int len = b.getLength(); + block = new LZ77Compressor.LiteralBlock( + Arrays.copyOfRange(b.getData(), b.getOffset(), b.getOffset() + len), + 0, len); + } + blocks.add(block); + } + }); + c.prefill(Arrays.copyOfRange(BLA, 0, 6)); + c.compress(Arrays.copyOfRange(BLA, 6, BLA.length)); + c.finish(); + assertSize(6, blocks); + assertLiteralBlock("lah ", blocks.get(0)); + assertLiteralBlock("blah", blocks.get(1)); + assertLiteralBlock(" bla", blocks.get(2)); + assertLiteralBlock("h bl", blocks.get(3)); + assertLiteralBlock("ah!", blocks.get(4)); + } + + @Test(expected = IllegalStateException.class) + public void cantPrefillTwice() { + LZ77Compressor c = new LZ77Compressor(newParameters(128), new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) { + } + }); + c.prefill(Arrays.copyOfRange(BLA, 0, 2)); + c.prefill(Arrays.copyOfRange(BLA, 2, 4)); + } + + @Test(expected = IllegalStateException.class) + public void cantPrefillAfterCompress() throws IOException { + LZ77Compressor c = new LZ77Compressor(newParameters(128), new LZ77Compressor.Callback() { + @Override + public void accept(LZ77Compressor.Block block) { + } + }); + c.compress(Arrays.copyOfRange(BLA, 0, 2)); + c.prefill(Arrays.copyOfRange(BLA, 2, 4)); + } + + private static final void assertSize(int expectedSize, List blocks) { + assertEquals(expectedSize, blocks.size()); + assertEquals(LZ77Compressor.Block.BlockType.EOD, blocks.get(expectedSize - 1).getType()); + } + + private static final void assertLiteralBlock(String expectedContent, LZ77Compressor.Block block) + throws IOException { + assertLiteralBlock(expectedContent.getBytes("ASCII"), block); + } + + private static final void assertLiteralBlock(byte[] expectedContent, LZ77Compressor.Block block) { + assertEquals(LZ77Compressor.LiteralBlock.class, block.getClass()); + assertArrayEquals(expectedContent, ((LZ77Compressor.LiteralBlock) block).getData()); + } + + private static final void assertBackReference(int expectedOffset, int expectedLength, LZ77Compressor.Block block) { + assertEquals(LZ77Compressor.BackReference.class, block.getClass()); + LZ77Compressor.BackReference b = (LZ77Compressor.BackReference) block; + assertEquals(expectedOffset, b.getOffset()); + assertEquals(expectedLength, b.getLength()); + } + + private static final byte[][] stagger(byte[] data) { + byte[][] r = new byte[data.length][1]; + for (int i = 0; i < data.length; i++) { + r[i][0] = data[i]; + } + return r; + } + + private static Parameters newParameters(int windowSize) { + return Parameters.builder(windowSize).build(); + } + + private static Parameters newParameters(int windowSize, int minBackReferenceLength, int maxBackReferenceLength, + int maxOffset, int maxLiteralLength) { + return Parameters.builder(windowSize) + .withMinBackReferenceLength(minBackReferenceLength) + .withMaxBackReferenceLength(maxBackReferenceLength) + .withMaxOffset(maxOffset) + .withMaxLiteralLength(maxLiteralLength) + .tunedForCompressionRatio() + .build(); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz77support/ParametersTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz77support/ParametersTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/lz77support/ParametersTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/lz77support/ParametersTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.lz77support; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ParametersTest { + + @Test + public void defaultConstructor() { + Parameters p = newParameters(128); + assertEquals(128, p.getWindowSize()); + assertEquals(3, p.getMinBackReferenceLength()); + assertEquals(127, p.getMaxBackReferenceLength()); + assertEquals(127, p.getMaxOffset()); + assertEquals(128, p.getMaxLiteralLength()); + } + + @Test + public void minBackReferenceLengthIsAtLeastThree() { + Parameters p = newParameters(128, 2, 3, 4, 5); + assertEquals(3, p.getMinBackReferenceLength()); + } + + @Test + public void maxBackReferenceLengthIsMinBackReferenceLengthWhenSmallerThanMinBackReferenceLength() { + Parameters p = newParameters(128, 2, 2, 4, 5); + assertEquals(3, p.getMaxBackReferenceLength()); + } + + @Test + public void maxBackReferenceLengthIsMinBackReferenceLengthWhenSmallerThanMinBackReferenceLengthReversedInvocationOrder() { + Parameters p = Parameters.builder(128) + .withMaxBackReferenceLength(2) + .withMinBackReferenceLength(2) + .withMaxOffset(4) + .withMaxLiteralLength(5) + .build(); + assertEquals(3, p.getMaxBackReferenceLength()); + } + + @Test + public void maxBackReferenceLengthIsMinBackReferenceLengthIfBothAreEqual() { + Parameters p = newParameters(128, 2, 3, 4, 5); + assertEquals(3, p.getMaxBackReferenceLength()); + } + + @Test + public void maxOffsetIsWindowSizeMinus1IfSetTo0() { + Parameters p = newParameters(128, 2, 3, 0, 5); + assertEquals(127, p.getMaxOffset()); + } + + @Test + public void maxOffsetIsWindowSizeMinus1IfSetToANegativeValue() { + Parameters p = newParameters(128, 2, 3, -1, 5); + assertEquals(127, p.getMaxOffset()); + } + + @Test + public void maxOffsetIsWindowSizeMinus1IfBiggerThanWindowSize() { + Parameters p = newParameters(128, 2, 3, 129, 5); + assertEquals(127, p.getMaxOffset()); + } + + @Test + public void maxLiteralLengthIsWindowSizeIfSetTo0() { + Parameters p = newParameters(128, 2, 3, 4, 0); + assertEquals(128, p.getMaxLiteralLength()); + } + + @Test + public void maxLiteralLengthIsWindowSizeIfSetToANegativeValue() { + Parameters p = newParameters(128, 2, 3, 0, -1); + assertEquals(128, p.getMaxLiteralLength()); + } + + @Test + public void maxLiteralLengthIsWindowSizeIfSetToAValueTooBigToHoldInSlidingWindow() { + Parameters p = newParameters(128, 2, 3, 0, 259); + assertEquals(128, p.getMaxLiteralLength()); + } + + @Test + public void allParametersUsuallyTakeTheirSpecifiedValues() { + Parameters p = newParameters(256, 4, 5, 6, 7); + assertEquals(256, p.getWindowSize()); + assertEquals(4, p.getMinBackReferenceLength()); + assertEquals(5, p.getMaxBackReferenceLength()); + assertEquals(6, p.getMaxOffset()); + assertEquals(7, p.getMaxLiteralLength()); + } + + @Test(expected = IllegalArgumentException.class) + public void windowSizeMustNotBeSmallerThanMinBackReferenceLength() { + newParameters(128, 200, 300, 400, 500); + } + + @Test(expected = IllegalArgumentException.class) + public void windowSizeMustBeAPowerOfTwo() { + newParameters(100, 200, 300, 400, 500); + } + + private static Parameters newParameters(int windowSize) { + return Parameters.builder(windowSize).build(); + } + + private static Parameters newParameters(int windowSize, int minBackReferenceLength, int maxBackReferenceLength, + int maxOffset, int maxLiteralLength) { + return Parameters.builder(windowSize) + .withMinBackReferenceLength(minBackReferenceLength) + .withMaxBackReferenceLength(maxBackReferenceLength) + .withMaxOffset(maxOffset) + .withMaxLiteralLength(maxLiteralLength) + .build(); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/LZMATestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/LZMATestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/LZMATestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/LZMATestCase.java 2018-08-09 14:00:28.000000000 +0000 @@ -77,6 +77,33 @@ } } + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.lzma"); + try (InputStream is = new FileInputStream(input)) { + final LZMACompressorInputStream in = + new LZMACompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.lzma"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final LZMACompressorInputStream in = + new LZMACompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } + private void copy(final InputStream in, final File output) throws IOException { FileOutputStream out = null; try { diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/Pack200TestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/Pack200TestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/Pack200TestCase.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/Pack200TestCase.java 2018-08-09 14:07:55.000000000 +0000 @@ -198,4 +198,46 @@ os.close(); } } + + @Test + public void singleByteReadFromMemoryConsistentlyReturnsMinusOneAtEof() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(Pack200Strategy.IN_MEMORY); + } + + @Test + public void singleByteReadFromTempFileConsistentlyReturnsMinusOneAtEof() throws Exception { + singleByteReadConsistentlyReturnsMinusOneAtEof(Pack200Strategy.TEMP_FILE); + } + + private void singleByteReadConsistentlyReturnsMinusOneAtEof(Pack200Strategy s) throws Exception { + final File input = getFile("bla.pack"); + try (final Pack200CompressorInputStream in = new Pack200CompressorInputStream(input, s)) { + IOUtils.toByteArray(in); + assertEquals(-1, in.read()); + assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadFromMemoryConsistentlyReturnsMinusOneAtEof() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(Pack200Strategy.IN_MEMORY); + } + + @Test + public void multiByteReadFromTempFileConsistentlyReturnsMinusOneAtEof() throws Exception { + multiByteReadConsistentlyReturnsMinusOneAtEof(Pack200Strategy.TEMP_FILE); + } + + private void multiByteReadConsistentlyReturnsMinusOneAtEof(Pack200Strategy s) throws Exception { + final File input = getFile("bla.pack"); + byte[] buf = new byte[2]; + try (final Pack200CompressorInputStream in = new Pack200CompressorInputStream(input, s)) { + IOUtils.toByteArray(in); + assertEquals(-1, in.read(buf)); + assertEquals(-1, in.read(buf)); + in.close(); + } + } + } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java 2018-08-09 14:10:51.000000000 +0000 @@ -119,10 +119,9 @@ assertEquals(3, in.available()); // remainder of first uncompressed block assertEquals(3, in.read(new byte[5], 0, 3)); assertEquals('5', in.read()); - assertEquals(4, in.available()); // remainder of literal + assertEquals(0, in.available()); // end of chunk, must read next one assertEquals(4, in.read(new byte[5], 0, 4)); assertEquals('5', in.read()); - assertEquals(19, in.available()); // remainder of copy in.close(); } } @@ -170,6 +169,53 @@ } } + /** + * @see "https://issues.apache.org/jira/browse/COMPRESS-358" + */ + @Test + public void readIWAFileWithBiggerOffset() throws Exception { + File o = new File(dir, "COMPRESS-358.raw"); + try (InputStream is = new FileInputStream(getFile("COMPRESS-358.iwa")); + FramedSnappyCompressorInputStream in = + new FramedSnappyCompressorInputStream(is, 1<<16, FramedSnappyDialect.IWORK_ARCHIVE); + FileOutputStream out = new FileOutputStream(o)) { + IOUtils.copy(in, out); + } + try (FileInputStream a = new FileInputStream(o); + FileInputStream e = new FileInputStream(getFile("COMPRESS-358.uncompressed"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.sz"); + try (InputStream is = new FileInputStream(input)) { + final FramedSnappyCompressorInputStream in = + new FramedSnappyCompressorInputStream(is); + IOUtils.toByteArray(in); + assertEquals(-1, in.read()); + assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.sz"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final FramedSnappyCompressorInputStream in = + new FramedSnappyCompressorInputStream(is); + IOUtils.toByteArray(in); + assertEquals(-1, in.read(buf)); + assertEquals(-1, in.read(buf)); + in.close(); + } + } + private void testChecksumUnmasking(final long x) { assertEquals(Long.toHexString(x), Long.toHexString(FramedSnappyCompressorInputStream diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/snappy/SnappyRoundtripTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/snappy/SnappyRoundtripTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/snappy/SnappyRoundtripTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/snappy/SnappyRoundtripTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.snappy; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Random; +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.lz77support.Parameters; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public final class SnappyRoundtripTest extends AbstractTestCase { + + private void roundTripTest(String testFile) throws IOException { + roundTripTest(getFile(testFile), + SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE) + .build()); + } + + private void roundTripTest(final File input, Parameters params) throws IOException { + long start = System.currentTimeMillis(); + final File outputSz = new File(dir, input.getName() + ".raw.sz"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(outputSz); + SnappyCompressorOutputStream sos = new SnappyCompressorOutputStream(os, input.length(), params)) { + IOUtils.copy(is, sos); + } + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + start = System.currentTimeMillis(); + try (FileInputStream is = new FileInputStream(input); + SnappyCompressorInputStream sis = new SnappyCompressorInputStream(new FileInputStream(outputSz), + params.getWindowSize())) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(sis); + Assert.assertArrayEquals(expected, actual); + } + System.err.println(outputSz.getName() + " read after " + (System.currentTimeMillis() - start) + "ms"); + } + private void roundTripTest(final byte[] input, Parameters params) throws IOException { + long start = System.currentTimeMillis(); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try ( + SnappyCompressorOutputStream sos = new SnappyCompressorOutputStream(os, input.length, params)) { + sos.write(input); + } + System.err.println("byte array" + " written, uncompressed bytes: " + input.length + + ", compressed bytes: " + os.size() + " after " + (System.currentTimeMillis() - start) + "ms"); + start = System.currentTimeMillis(); + try ( + SnappyCompressorInputStream sis = new SnappyCompressorInputStream(new ByteArrayInputStream(os.toByteArray()), + params.getWindowSize())) { + byte[] expected = input; + byte[] actual = IOUtils.toByteArray(sis); + Assert.assertArrayEquals(expected, actual); + } + System.err.println("byte array" + " read after " + (System.currentTimeMillis() - start) + "ms"); + } + + // should yield decent compression + @Test + public void blaTarRoundtrip() throws IOException { + System.err.println("Configuration: default"); + roundTripTest("bla.tar"); + } + + @Test + public void blaTarRoundtripTunedForSpeed() throws IOException { + System.err.println("Configuration: tuned for speed"); + roundTripTest(getFile("bla.tar"), + SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE) + .tunedForSpeed() + .build()); + } + + @Test + public void blaTarRoundtripTunedForCompressionRatio() throws IOException { + System.err.println("Configuration: tuned for compression ratio"); + roundTripTest(getFile("bla.tar"), + SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE) + .tunedForCompressionRatio() + .build()); + } + + // yields no compression at all + @Test + public void gzippedLoremIpsumRoundtrip() throws IOException { + roundTripTest("lorem-ipsum.txt.gz"); + } + + // yields no compression at all + @Test + public void biggerFileRoundtrip() throws IOException { + roundTripTest("COMPRESS-256.7z"); + } + + @Test + public void tryReallyBigOffset() throws IOException { + // "normal" Snappy files will never reach offsets beyond + // 16bits (i.e. those using four bytes to encode the length) + // as the block size is only 32k. This means we never execute + // the code for four-byte length copies in either stream class + // using real-world Snappy files. + // This is an artifical stream using a bigger block size that + // may not even be expandable by other Snappy implementations. + // Start with the four byte sequence 0000 after that add > 64k + // of random noise that doesn't contain any 0000 at all, then + // add 0000. + File f = new File(dir, "reallyBigOffsetTest"); + ByteArrayOutputStream fs = new ByteArrayOutputStream((1<<16) + 1024); + fs.write(0); + fs.write(0); + fs.write(0); + fs.write(0); + int cnt = 1 << 16 + 5; + Random r = new Random(); + for (int i = 0 ; i < cnt; i++) { + fs.write(r.nextInt(255) + 1); + } + fs.write(0); + fs.write(0); + fs.write(0); + fs.write(0); + + roundTripTest(fs.toByteArray(), newParameters(1 << 17, 4, 64, 1 << 17 - 1, 1 << 17 - 1)); + } + + @Test + public void tryReallyLongLiterals() throws IOException { + // "normal" Snappy files will never reach literal blocks with + // length beyond 16bits (i.e. those using three or four bytes + // to encode the length) as the block size is only 32k. This + // means we never execute the code for the three/four byte + // length literals in either stream class using real-world + // Snappy files. + // What we'd need would be a sequence of bytes with no four + // byte subsequence repeated that is longer than 64k, we try + // our best with random, but will probably only hit the three byte + // methods in a few lucky cases. + // The four byte methods would require even more luck and a + // buffer (and a file written to disk) that was 2^5 bigger + // than the buffer used here. + File f = new File(dir, "reallyBigLiteralTest"); + try (FileOutputStream fs = new FileOutputStream(f)) { + int cnt = 1 << 19; + Random r = new Random(); + for (int i = 0 ; i < cnt; i++) { + fs.write(r.nextInt(256)); + } + } + roundTripTest(f, newParameters(1 << 18, 4, 64, 1 << 16 - 1, 1 << 18 - 1)); + } + + private static Parameters newParameters(int windowSize, int minBackReferenceLength, int maxBackReferenceLength, + int maxOffset, int maxLiteralLength) { + return Parameters.builder(windowSize) + .withMinBackReferenceLength(minBackReferenceLength) + .withMaxBackReferenceLength(maxBackReferenceLength) + .withMaxOffset(maxOffset) + .withMaxLiteralLength(maxLiteralLength) + .build(); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStreamTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStreamTest.java 2018-08-09 14:18:20.000000000 +0000 @@ -18,6 +18,13 @@ */ package org.apache.commons.compress.compressors.xz; +import static org.apache.commons.compress.AbstractTestCase.getFile; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.compress.utils.IOUtils; import org.junit.Assert; import org.junit.Test; @@ -33,4 +40,50 @@ data[5] = '0'; Assert.assertFalse(XZCompressorInputStream.matches(data, 6)); } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofNoDecompressConcatenated() throws IOException { + singleByteReadConsistentlyReturnsMinusOneAtEof(false); + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEofDecompressConcatenated() throws IOException { + singleByteReadConsistentlyReturnsMinusOneAtEof(true); + } + + private void singleByteReadConsistentlyReturnsMinusOneAtEof(boolean decompressConcatenated) throws IOException { + final File input = getFile("bla.tar.xz"); + try (InputStream is = new FileInputStream(input)) { + final XZCompressorInputStream in = + new XZCompressorInputStream(is, decompressConcatenated); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofNoDecompressConcatenated() throws IOException { + multiByteReadConsistentlyReturnsMinusOneAtEof(false); + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEofDecompressConcatenated() throws IOException { + multiByteReadConsistentlyReturnsMinusOneAtEof(true); + } + + private void multiByteReadConsistentlyReturnsMinusOneAtEof(boolean decompressConcatenated) throws IOException { + final File input = getFile("bla.tar.xz"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final XZCompressorInputStream in = + new XZCompressorInputStream(is, decompressConcatenated); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } + } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorOutputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorOutputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorOutputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/xz/XZCompressorOutputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.xz; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + + +/** + * Unit tests for class {@link XZCompressorOutputStream}. + * + * @date 16.06.2017 + * @see XZCompressorOutputStream + **/ +public class XZCompressorOutputStreamTest { + + + @Test + public void testWrite() throws IOException { + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4590); + try (XZCompressorOutputStream xZCompressorOutputStream = new XZCompressorOutputStream(byteArrayOutputStream)) { + xZCompressorOutputStream.write(4590); + } + + try (XZCompressorInputStream xZCompressorInputStream = + new XZCompressorInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))) { + assertEquals(4590 % 256, xZCompressorInputStream.read()); + assertEquals(-1, xZCompressorInputStream.read()); + } + } + + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/z/ZCompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/z/ZCompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/z/ZCompressorInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/z/ZCompressorInputStreamTest.java 2018-08-09 14:34:04.000000000 +0000 @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.compressors.z; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Enumeration; +import org.apache.commons.compress.utils.IOUtils; + +import static org.apache.commons.compress.AbstractTestCase.getFile; +import static org.mockito.Mockito.mock; +import static org.powermock.api.mockito.PowerMockito.doReturn; + + +/** + * Unit tests for class {@link ZCompressorInputStream}. + * + * @date 16.06.2017 + * @see ZCompressorInputStream + **/ +public class ZCompressorInputStreamTest { + + + @Test(expected = IOException.class) + public void testFailsToCreateZCompressorInputStreamAndThrowsIOException() throws IOException { + boolean java9 = false; + try { + Class.forName("java.lang.module.ModuleDescriptor"); + java9 = true; + } catch (Exception ex) { + // not Java9 + } + org.junit.Assume.assumeFalse("can't use PowerMock with Java9", java9); + + Enumeration enumeration = (Enumeration) mock(Enumeration.class); + SequenceInputStream sequenceInputStream = new SequenceInputStream(enumeration); + ZCompressorInputStream zCompressorInputStream = null; + + doReturn(false).when(enumeration).hasMoreElements(); + + zCompressorInputStream = new ZCompressorInputStream(sequenceInputStream); + + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.Z"); + try (InputStream is = new FileInputStream(input)) { + final ZCompressorInputStream in = + new ZCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("bla.tar.Z"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final ZCompressorInputStream in = + new ZCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java 2018-08-09 14:38:48.000000000 +0000 @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.zstandard; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public class ZstdCompressorInputStreamTest extends AbstractTestCase { + + /** + * Test bridge works fine. + * + * @throws IOException + */ + @Test + public void testZstdDecode() throws IOException { + final File input = getFile("zstandard.testdata.zst"); + final File expected = getFile("zstandard.testdata"); + try (InputStream inputStream = new FileInputStream(input); + InputStream expectedStream = new FileInputStream(expected); + ZstdCompressorInputStream zstdInputStream = new ZstdCompressorInputStream(inputStream)) { + final byte[] b = new byte[97]; + IOUtils.readFully(expectedStream, b); + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + int readByte = -1; + while((readByte = zstdInputStream.read()) != -1) { + bos.write(readByte); + } + Assert.assertArrayEquals(b, bos.toByteArray()); + } + } + + @Test + public void testCachingIsEnabledByDefaultAndZstdUtilsPresent() { + assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, ZstdUtils.getCachedZstdAvailability()); + assertTrue(ZstdUtils.isZstdCompressionAvailable()); + } + + @Test + public void testCanTurnOffCaching() { + try { + ZstdUtils.setCacheZstdAvailablity(false); + assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE, ZstdUtils.getCachedZstdAvailability()); + assertTrue(ZstdUtils.isZstdCompressionAvailable()); + } finally { + ZstdUtils.setCacheZstdAvailablity(true); + } + } + + @Test + public void testTurningOnCachingReEvaluatesAvailability() { + try { + ZstdUtils.setCacheZstdAvailablity(false); + assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE, ZstdUtils.getCachedZstdAvailability()); + ZstdUtils.setCacheZstdAvailablity(true); + assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, ZstdUtils.getCachedZstdAvailability()); + } finally { + ZstdUtils.setCacheZstdAvailablity(true); + } + } + + @Test + public void shouldBeAbleToSkipAByte() throws IOException { + final File input = getFile("zstandard.testdata.zst"); + try (InputStream is = new FileInputStream(input)) { + final ZstdCompressorInputStream in = + new ZstdCompressorInputStream(is); + Assert.assertEquals(1, in.skip(1)); + in.close(); + } + } + + @Test + public void singleByteReadWorksAsExpected() throws IOException { + + final File input = getFile("zstandard.testdata.zst"); + + final File original = getFile("zstandard.testdata"); + final long originalFileLength = original.length(); + + byte[] originalFileContent = new byte[((int) originalFileLength)]; + + try (InputStream ois = new FileInputStream(original)) { + ois.read(originalFileContent); + } + + try (InputStream is = new FileInputStream(input)) { + final ZstdCompressorInputStream in = + new ZstdCompressorInputStream(is); + + Assert.assertEquals(originalFileContent[0], in.read()); + in.close(); + } + } + + @Test + public void singleByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("zstandard.testdata.zst"); + try (InputStream is = new FileInputStream(input)) { + final ZstdCompressorInputStream in = + new ZstdCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(-1, in.read()); + in.close(); + } + } + + @Test + public void multiByteReadConsistentlyReturnsMinusOneAtEof() throws IOException { + final File input = getFile("zstandard.testdata.zst"); + byte[] buf = new byte[2]; + try (InputStream is = new FileInputStream(input)) { + final ZstdCompressorInputStream in = + new ZstdCompressorInputStream(is); + IOUtils.toByteArray(in); + Assert.assertEquals(-1, in.read(buf)); + Assert.assertEquals(-1, in.read(buf)); + in.close(); + } + } + + @Test + public void testZstandardUnarchive() throws Exception { + final File input = getFile("bla.tar.zst"); + final File output = new File(dir, "bla.tar"); + try (InputStream is = new FileInputStream(input)) { + final CompressorInputStream in = new CompressorStreamFactory() + .createCompressorInputStream("zstd", is); + FileOutputStream out = null; + try { + out = new FileOutputStream(output); + IOUtils.copy(in, out); + } finally { + if (out != null) { + out.close(); + } + in.close(); + } + } + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdRoundtripTest.java 2018-07-28 14:20:37.000000000 +0000 @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.zstandard; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import org.apache.commons.compress.AbstractTestCase; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorOutputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.Test; + +public class ZstdRoundtripTest extends AbstractTestCase { + + private interface OutputStreamCreator { + ZstdCompressorOutputStream wrap(FileOutputStream os) throws IOException; + } + + @Test + public void directRoundtrip() throws Exception { + roundtrip(new OutputStreamCreator() { + @Override + public ZstdCompressorOutputStream wrap(FileOutputStream os) throws IOException { + return new ZstdCompressorOutputStream(os); + } + }); + } + + private void roundtrip(OutputStreamCreator oc) throws IOException { + File input = getFile("bla.tar"); + long start = System.currentTimeMillis(); + final File output = new File(dir, input.getName() + ".zstd"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(output); + ZstdCompressorOutputStream zos = oc.wrap(os)) { + IOUtils.copy(is, zos); + } + System.err.println(input.getName() + " written, uncompressed bytes: " + input.length() + + ", compressed bytes: " + output.length() + " after " + (System.currentTimeMillis() - start) + "ms"); + start = System.currentTimeMillis(); + try (FileInputStream is = new FileInputStream(input); + ZstdCompressorInputStream zis = new ZstdCompressorInputStream(new FileInputStream(output))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(zis); + Assert.assertArrayEquals(expected, actual); + } + System.err.println(output.getName() + " read after " + (System.currentTimeMillis() - start) + "ms"); + } + + @Test + public void factoryRoundtrip() throws Exception { + File input = getFile("bla.tar"); + long start = System.currentTimeMillis(); + final File output = new File(dir, input.getName() + ".zstd"); + try (FileInputStream is = new FileInputStream(input); + FileOutputStream os = new FileOutputStream(output); + CompressorOutputStream zos = new CompressorStreamFactory().createCompressorOutputStream("zstd", os)) { + IOUtils.copy(is, zos); + } + start = System.currentTimeMillis(); + try (FileInputStream is = new FileInputStream(input); + CompressorInputStream zis = new CompressorStreamFactory() + .createCompressorInputStream("zstd", new FileInputStream(output))) { + byte[] expected = IOUtils.toByteArray(is); + byte[] actual = IOUtils.toByteArray(zis); + Assert.assertArrayEquals(expected, actual); + } + } + + @Test + public void roundtripWithCustomLevel() throws Exception { + roundtrip(new OutputStreamCreator() { + @Override + public ZstdCompressorOutputStream wrap(FileOutputStream os) throws IOException { + return new ZstdCompressorOutputStream(os, 1); + } + }); + } + + @Test + public void roundtripWithCloseFrameOnFlush() throws Exception { + roundtrip(new OutputStreamCreator() { + @Override + public ZstdCompressorOutputStream wrap(FileOutputStream os) throws IOException { + return new ZstdCompressorOutputStream(os, 3, true); + } + }); + } + + @Test + public void roundtripWithChecksum() throws Exception { + roundtrip(new OutputStreamCreator() { + @Override + public ZstdCompressorOutputStream wrap(FileOutputStream os) throws IOException { + return new ZstdCompressorOutputStream(os, 3, false, true); + } + }); + } + +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdUtilsTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdUtilsTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdUtilsTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/compressors/zstandard/ZstdUtilsTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.compress.compressors.zstandard; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class ZstdUtilsTest { + + @Test + public void testMatchesZstandardFrame() { + final byte[] data = { + (byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD, + }; + assertFalse(ZstdUtils.matches(data, 3)); + assertTrue(ZstdUtils.matches(data, 4)); + assertTrue(ZstdUtils.matches(data, 5)); + data[3] = '0'; + assertFalse(ZstdUtils.matches(data, 4)); + } + + @Test + public void testMatchesSkippableFrame() { + final byte[] data = { + 0, (byte) 0x2A, (byte) 0x4D, (byte) 0x18, + }; + assertFalse(ZstdUtils.matches(data, 4)); + for (byte b = (byte) 0x50; b < 0x60; b++) { + data[0] = b; + assertTrue(ZstdUtils.matches(data, 4)); + } + assertFalse(ZstdUtils.matches(data, 3)); + assertTrue(ZstdUtils.matches(data, 5)); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/DetectArchiverTestCase.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/DetectArchiverTestCase.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/DetectArchiverTestCase.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/DetectArchiverTestCase.java 2018-05-02 20:17:13.000000000 +0000 @@ -65,7 +65,7 @@ @Test public void testDetection() throws Exception { - final ArchiveInputStream ar = getStreamFor("bla.ar"); + final ArchiveInputStream ar = getStreamFor("bla.ar"); assertNotNull(ar); assertTrue(ar instanceof ArArchiveInputStream); @@ -84,7 +84,7 @@ final ArchiveInputStream cpio = getStreamFor("bla.cpio"); assertNotNull(cpio); assertTrue(cpio instanceof CpioArchiveInputStream); - + final ArchiveInputStream arj = getStreamFor("bla.arj"); assertNotNull(arj); assertTrue(arj instanceof ArjArchiveInputStream); diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/IOMethodsTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/IOMethodsTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/IOMethodsTest.java 2016-06-18 15:07:49.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/IOMethodsTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + * */ package org.apache.commons.compress; diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/MockEvilInputStream.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/MockEvilInputStream.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/MockEvilInputStream.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/MockEvilInputStream.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Simple mock InputStream that always throws an IOException + * when {@link #read()} or {@link #read(byte[], int, int)} + * is called. + */ +public class MockEvilInputStream extends InputStream { + + @Override + public int read() throws IOException { + throw new IOException("Evil"); + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + throw new IOException("Evil"); + } +} + diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/OsgiITest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/OsgiITest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/OsgiITest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/OsgiITest.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress; + +import static org.ops4j.pax.exam.CoreOptions.bundle; +import static org.ops4j.pax.exam.CoreOptions.composite; +import static org.ops4j.pax.exam.CoreOptions.mavenBundle; +import static org.ops4j.pax.exam.CoreOptions.systemProperty; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.ops4j.pax.exam.Configuration; +import org.ops4j.pax.exam.Option; +import org.ops4j.pax.exam.junit.PaxExam; + +@RunWith(PaxExam.class) +public class OsgiITest { + + @Configuration + public Option[] config() { + return new Option[] { + systemProperty("pax.exam.osgi.unresolved.fail").value("true"), + mavenBundle().groupId("org.apache.felix").artifactId("org.apache.felix.scr") + .version("2.0.14"), + mavenBundle().groupId("org.apache.felix").artifactId("org.apache.felix.configadmin") + .version("1.8.16"), + composite(systemProperty("pax.exam.invoker").value("junit"), + bundle("link:classpath:META-INF/links/org.ops4j.pax.tipi.junit.link"), + bundle("link:classpath:META-INF/links/org.ops4j.pax.exam.invoker.junit.link"), + mavenBundle().groupId("org.apache.servicemix.bundles") + .artifactId("org.apache.servicemix.bundles.hamcrest").version("1.3_1")), + bundle("reference:file:target/classes/").start() + }; + } + + @Test + public void loadBundle() { + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/BitInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/BitInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/BitInputStreamTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/BitInputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -171,6 +171,41 @@ } } + @Test + public void alignWithByteBoundaryWhenAtBoundary() throws Exception { + try (final BitInputStream bis = new BitInputStream(getStream(), ByteOrder.LITTLE_ENDIAN)) { + assertEquals(0xF8, bis.readBits(8)); + bis.alignWithByteBoundary(); + assertEquals(0, bis.readBits(4)); + } + } + + @Test + public void alignWithByteBoundaryWhenNotAtBoundary() throws Exception { + try (final BitInputStream bis = new BitInputStream(getStream(), ByteOrder.LITTLE_ENDIAN)) { + assertEquals(0x08, bis.readBits(4)); + assertEquals(4, bis.bitsCached()); + bis.alignWithByteBoundary(); + assertEquals(0, bis.bitsCached()); + assertEquals(0, bis.readBits(4)); + } + } + + @Test + public void availableWithoutCache() throws Exception { + try (final BitInputStream bis = new BitInputStream(getStream(), ByteOrder.LITTLE_ENDIAN)) { + assertEquals(32, bis.bitsAvailable()); + } + } + + @Test + public void availableWithCache() throws Exception { + try (final BitInputStream bis = new BitInputStream(getStream(), ByteOrder.LITTLE_ENDIAN)) { + assertEquals(0x08, bis.readBits(4)); + assertEquals(28, bis.bitsAvailable()); + } + } + private ByteArrayInputStream getStream() { return new ByteArrayInputStream(new byte[] { (byte) 0xF8, // 11111000 diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ByteUtilsTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ByteUtilsTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ByteUtilsTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ByteUtilsTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.commons.compress.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.junit.Test; + +import static org.apache.commons.compress.utils.ByteUtils.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class ByteUtilsTest { + + @Test + public void fromLittleEndianFromArrayOneArg() { + byte[] b = new byte[] { 2, 3, 4 }; + assertEquals(2 + 3 * 256 + 4 * 256 * 256, fromLittleEndian(b)); + } + + @Test + public void fromLittleEndianFromArrayOneArgUnsignedInt32() { + byte[] b = new byte[] { 2, 3, 4, (byte) 128 }; + assertEquals(2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, fromLittleEndian(b)); + } + + @Test(expected = IllegalArgumentException.class) + public void fromLittleEndianFromArrayOneArgThrowsForLengthTooBig() { + fromLittleEndian(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + } + + @Test + public void fromLittleEndianFromArray() { + byte[] b = new byte[] { 1, 2, 3, 4, 5 }; + assertEquals(2 + 3 * 256 + 4 * 256 * 256, fromLittleEndian(b, 1, 3)); + } + + @Test + public void fromLittleEndianFromArrayUnsignedInt32() { + byte[] b = new byte[] { 1, 2, 3, 4, (byte) 128 }; + assertEquals(2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, fromLittleEndian(b, 1, 4)); + } + + @Test(expected = IllegalArgumentException.class) + public void fromLittleEndianFromArrayThrowsForLengthTooBig() { + fromLittleEndian(new byte[0], 0, 9); + } + + @Test + public void fromLittleEndianFromStream() throws IOException { + ByteArrayInputStream bin = new ByteArrayInputStream(new byte[] { 2, 3, 4, 5 }); + assertEquals(2 + 3 * 256 + 4 * 256 * 256, fromLittleEndian(bin, 3)); + } + + @Test + public void fromLittleEndianFromStreamUnsignedInt32() throws IOException { + ByteArrayInputStream bin = new ByteArrayInputStream(new byte[] { 2, 3, 4, (byte) 128 }); + assertEquals(2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, fromLittleEndian(bin, 4)); + } + + @Test(expected = IllegalArgumentException.class) + public void fromLittleEndianFromStreamThrowsForLengthTooBig() throws IOException { + fromLittleEndian(new ByteArrayInputStream(new byte[0]), 9); + } + + @Test(expected = IOException.class) + public void fromLittleEndianFromStreamThrowsForPrematureEnd() throws IOException { + ByteArrayInputStream bin = new ByteArrayInputStream(new byte[] { 2, 3 }); + fromLittleEndian(bin, 3); + } + + @Test + public void fromLittleEndianFromSupplier() throws IOException { + ByteArrayInputStream bin = new ByteArrayInputStream(new byte[] { 2, 3, 4, 5 }); + assertEquals(2 + 3 * 256 + 4 * 256 * 256, fromLittleEndian(new InputStreamByteSupplier(bin), 3)); + } + + @Test + public void fromLittleEndianFromSupplierUnsignedInt32() throws IOException { + ByteArrayInputStream bin = new ByteArrayInputStream(new byte[] { 2, 3, 4, (byte) 128 }); + assertEquals(2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, + fromLittleEndian(new InputStreamByteSupplier(bin), 4)); + } + + @Test(expected = IllegalArgumentException.class) + public void fromLittleEndianFromSupplierThrowsForLengthTooBig() throws IOException { + fromLittleEndian(new InputStreamByteSupplier(new ByteArrayInputStream(new byte[0])), 9); + } + + @Test(expected = IOException.class) + public void fromLittleEndianFromSupplierThrowsForPrematureEnd() throws IOException { + ByteArrayInputStream bin = new ByteArrayInputStream(new byte[] { 2, 3 }); + fromLittleEndian(new InputStreamByteSupplier(bin), 3); + } + + @Test + public void fromLittleEndianFromDataInput() throws IOException { + DataInput din = new DataInputStream(new ByteArrayInputStream(new byte[] { 2, 3, 4, 5 })); + assertEquals(2 + 3 * 256 + 4 * 256 * 256, fromLittleEndian(din, 3)); + } + + @Test + public void fromLittleEndianFromDataInputUnsignedInt32() throws IOException { + DataInput din = new DataInputStream(new ByteArrayInputStream(new byte[] { 2, 3, 4, (byte) 128 })); + assertEquals(2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, fromLittleEndian(din, 4)); + } + + @Test(expected = IllegalArgumentException.class) + public void fromLittleEndianFromDataInputThrowsForLengthTooBig() throws IOException { + DataInput din = new DataInputStream(new ByteArrayInputStream(new byte[0])); + fromLittleEndian(din, 9); + } + + @Test(expected = java.io.EOFException.class) + public void fromLittleEndianFromDataInputThrowsForPrematureEnd() throws IOException { + DataInput din = new DataInputStream(new ByteArrayInputStream(new byte[] { 2, 3 })); + fromLittleEndian(din, 3); + } + + @Test + public void toLittleEndianToStream() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + toLittleEndian(bos, 2 + 3 * 256 + 4 * 256 * 256, 3); + bos.close(); + assertArrayEquals(new byte[] { 2, 3, 4 }, bos.toByteArray()); + } + + @Test + public void toLittleEndianToStreamUnsignedInt32() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + toLittleEndian(bos, 2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, 4); + bos.close(); + assertArrayEquals(new byte[] { 2, 3, 4, (byte) 128 }, bos.toByteArray()); + } + + @Test + public void toLittleEndianToConsumer() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + toLittleEndian(new OutputStreamByteConsumer(bos), 2 + 3 * 256 + 4 * 256 * 256, 3); + bos.close(); + assertArrayEquals(new byte[] { 2, 3, 4 }, bos.toByteArray()); + } + + @Test + public void toLittleEndianToConsumerUnsignedInt32() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + toLittleEndian(new OutputStreamByteConsumer(bos), 2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, 4); + bos.close(); + assertArrayEquals(new byte[] { 2, 3, 4, (byte) 128 }, bos.toByteArray()); + } + + @Test + public void toLittleEndianToDataOutput() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutput dos = new DataOutputStream(bos); + toLittleEndian(dos, 2 + 3 * 256 + 4 * 256 * 256, 3); + bos.close(); + assertArrayEquals(new byte[] { 2, 3, 4 }, bos.toByteArray()); + } + + @Test + public void toLittleEndianToDataOutputUnsignedInt32() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutput dos = new DataOutputStream(bos); + toLittleEndian(dos, 2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, 4); + bos.close(); + assertArrayEquals(new byte[] { 2, 3, 4, (byte) 128 }, bos.toByteArray()); + } + + + @Test + public void toLittleEndianToByteArray() throws IOException { + byte[] b = new byte[4]; + toLittleEndian(b, 2 + 3 * 256 + 4 * 256 * 256, 1, 3); + assertArrayEquals(new byte[] { 2, 3, 4 }, Arrays.copyOfRange(b, 1, 4)); + } + + @Test + public void toLittleEndianToByteArrayUnsignedInt32() throws IOException { + byte[] b = new byte[4]; + toLittleEndian(b, 2 + 3 * 256 + 4 * 256 * 256 + 128l * 256 * 256 * 256, 0, 4); + assertArrayEquals(new byte[] { 2, 3, 4, (byte) 128 }, b); + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/CharsetsTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/CharsetsTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/CharsetsTest.java 2016-01-13 17:48:31.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/CharsetsTest.java 2018-06-15 17:46:02.000000000 +0000 @@ -5,9 +5,9 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,8 +25,7 @@ /** * Sanity checks for {@link Charsets}. - * - * @version $Id$ + * */ public class CharsetsTest { @@ -38,34 +37,4 @@ Assert.assertEquals(Charset.forName("UTF-8"), Charsets.toCharset(Charset.forName("UTF-8"))); } - @Test - public void testIso8859_1() { - Assert.assertEquals("ISO-8859-1", Charsets.ISO_8859_1.name()); - } - - @Test - public void testUsAscii() { - Assert.assertEquals("US-ASCII", Charsets.US_ASCII.name()); - } - - @Test - public void testUtf16() { - Assert.assertEquals("UTF-16", Charsets.UTF_16.name()); - } - - @Test - public void testUtf16Be() { - Assert.assertEquals("UTF-16BE", Charsets.UTF_16BE.name()); - } - - @Test - public void testUtf16Le() { - Assert.assertEquals("UTF-16LE", Charsets.UTF_16LE.name()); - } - - @Test - public void testUtf8() { - Assert.assertEquals("UTF-8", Charsets.UTF_8.name()); - } - } diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.compress.utils; + +import org.junit.Test; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.zip.Adler32; +import java.util.zip.CRC32; + +import static org.junit.Assert.*; + +/** + * Unit tests for class {@link ChecksumCalculatingInputStream org.apache.commons.compress.utils.ChecksumCalculatingInputStream}. + * + * @date 13.06.2017 + * @see ChecksumCalculatingInputStream + **/ +public class ChecksumCalculatingInputStreamTest { + + + + @Test + public void testSkipReturningZero() throws IOException { + + Adler32 adler32 = new Adler32(); + byte[] byteArray = new byte[0]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + ChecksumCalculatingInputStream checksumCalculatingInputStream = new ChecksumCalculatingInputStream(adler32, byteArrayInputStream); + long skipResult = checksumCalculatingInputStream.skip(60L); + + assertEquals(0L, skipResult); + + assertEquals(1L, checksumCalculatingInputStream.getValue()); + + + } + + + @Test + public void testSkipReturningPositive() throws IOException { + + Adler32 adler32 = new Adler32(); + byte[] byteArray = new byte[6]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + ChecksumCalculatingInputStream checksumCalculatingInputStream = new ChecksumCalculatingInputStream(adler32, byteArrayInputStream); + long skipResult = checksumCalculatingInputStream.skip((byte)0); + + assertEquals(1L, skipResult); + + assertEquals(65537L, checksumCalculatingInputStream.getValue()); + + } + + + @Test + public void testReadTakingNoArguments() throws IOException { + + Adler32 adler32 = new Adler32(); + byte[] byteArray = new byte[6]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + ChecksumCalculatingInputStream checksumCalculatingInputStream = new ChecksumCalculatingInputStream(adler32, byteArrayInputStream); + BufferedInputStream bufferedInputStream = new BufferedInputStream(checksumCalculatingInputStream); + int inputStreamReadResult = bufferedInputStream.read(byteArray, 0, 1); + int checkSumCalculationReadResult = checksumCalculatingInputStream.read(); + + assertFalse(checkSumCalculationReadResult == inputStreamReadResult); + assertEquals((-1), checkSumCalculationReadResult); + + assertEquals(0, byteArrayInputStream.available()); + + assertEquals(393217L, checksumCalculatingInputStream.getValue()); + + } + + + @Test + public void testReadTakingByteArray() throws IOException { + + Adler32 adler32 = new Adler32(); + byte[] byteArray = new byte[6]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + ChecksumCalculatingInputStream checksumCalculatingInputStream = new ChecksumCalculatingInputStream(adler32, byteArrayInputStream); + int readResult = checksumCalculatingInputStream.read(byteArray); + + assertEquals(6, readResult); + + assertEquals(0, byteArrayInputStream.available()); + assertEquals(393217L, checksumCalculatingInputStream.getValue()); + + } + + + @Test(expected = NullPointerException.class) + public void testClassInstantiationWithParameterBeingNullThrowsNullPointerExceptionOne() { + + ChecksumCalculatingInputStream checksumCalculatingInputStream = new ChecksumCalculatingInputStream(null,null); + + + } + + + @Test(expected = NullPointerException.class) + public void testClassInstantiationWithParameterBeingNullThrowsNullPointerExceptionTwo() { + + ChecksumCalculatingInputStream checksumCalculatingInputStream = new ChecksumCalculatingInputStream(null,new ByteArrayInputStream(new byte[1])); + + + } + + + @Test(expected = NullPointerException.class) + public void testClassInstantiationWithParameterBeingNullThrowsNullPointerExceptionThree() { + + ChecksumCalculatingInputStream checksumCalculatingInputStream = new ChecksumCalculatingInputStream(new CRC32(),null); + + } + + +} \ No newline at end of file diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.compress.utils; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.zip.Adler32; +import java.util.zip.CRC32; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +/** + * Unit tests for class {@link ChecksumVerifyingInputStream org.apache.commons.compress.utils.ChecksumVerifyingInputStream}. + * + * @date 13.06.2017 + * @see ChecksumVerifyingInputStream + **/ +public class ChecksumVerifyingInputStreamTest { + + + + @Test(expected = IOException.class) + public void testReadTakingByteArrayThrowsIOException() throws IOException { + + Adler32 adler32 = new Adler32(); + byte[] byteArray = new byte[3]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + ChecksumVerifyingInputStream checksumVerifyingInputStream = new ChecksumVerifyingInputStream(adler32, byteArrayInputStream, (-1859L), (byte) (-68)); + + checksumVerifyingInputStream.read(byteArray); + + } + + + @Test(expected = IOException.class) + public void testReadTakingNoArgumentsThrowsIOException() throws IOException { + + CRC32 cRC32_ = new CRC32(); + byte[] byteArray = new byte[9]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + ChecksumVerifyingInputStream checksumVerifyingInputStream = new ChecksumVerifyingInputStream(cRC32_, byteArrayInputStream, (byte)1, (byte)1); + + checksumVerifyingInputStream.read(); + + } + + + @Test + public void testSkip() throws IOException { + + CRC32 cRC32_ = new CRC32(); + byte[] byteArray = new byte[4]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArray); + ChecksumVerifyingInputStream checksumVerifyingInputStream = new ChecksumVerifyingInputStream(cRC32_, byteArrayInputStream, (byte)33, 2303L); + int intOne = checksumVerifyingInputStream.read(byteArray); + + long skipReturnValue = checksumVerifyingInputStream.skip((byte)1); + + assertEquals(558161692L, cRC32_.getValue()); + assertEquals(0, byteArrayInputStream.available()); + + assertArrayEquals(new byte[] {(byte)0, (byte)0, (byte)0, (byte)0}, byteArray); + assertEquals(0L, skipReturnValue); + + } + + + +} \ No newline at end of file diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/FixedLengthBlockOutputStreamTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; +import org.hamcrest.core.IsInstanceOf; +import org.junit.Test; +import org.mockito.internal.matchers.GreaterOrEqual; + +public class FixedLengthBlockOutputStreamTest { + + @Test + public void testSmallWrite() throws IOException { + testWriteAndPad(10240, "hello world!\n", false); + testWriteAndPad(512, "hello world!\n", false); + testWriteAndPad(11, "hello world!\n", false); + testWriteAndPad(3, "hello world!\n", false); + } + + @Test + public void testSmallWriteToStream() throws IOException { + testWriteAndPadToStream(10240, "hello world!\n", false); + testWriteAndPadToStream(512, "hello world!\n", false); + testWriteAndPadToStream(11, "hello world!\n", false); + testWriteAndPadToStream(3, "hello world!\n", false); + } + + @Test + public void testWriteSingleBytes() throws IOException { + int blockSize = 4; + MockWritableByteChannel mock = new MockWritableByteChannel(blockSize, false); + ByteArrayOutputStream bos = mock.bos; + String text = "hello world avengers"; + byte msg[] = text.getBytes(); + int len = msg.length; + try (FixedLengthBlockOutputStream out = new FixedLengthBlockOutputStream(mock, blockSize)) { + for (int i = 0; i < len; i++) { + out.write(msg[i]); + } + } + byte[] output = bos.toByteArray(); + + validate(blockSize, msg, output); + } + + + @Test + public void testWriteBuf() throws IOException { + String hwa = "hello world avengers"; + testBuf(4, hwa); + testBuf(512, hwa); + testBuf(10240, hwa); + testBuf(11, hwa + hwa + hwa); + } + + @Test + public void testMultiWriteBuf() throws IOException { + int blockSize = 13; + MockWritableByteChannel mock = new MockWritableByteChannel(blockSize, false); + String testString = "hello world"; + byte msg[] = testString.getBytes(); + int reps = 17; + + try (FixedLengthBlockOutputStream out = new FixedLengthBlockOutputStream(mock, blockSize)) { + for (int i = 0; i < reps; i++) { + ByteBuffer buf = getByteBuffer(msg); + out.write(buf); + } + } + ByteArrayOutputStream bos = mock.bos; + double v = Math.ceil((reps * msg.length) / (double) blockSize) * blockSize; + assertEquals("wrong size", (long) v, bos.size()); + int strLen = msg.length * reps; + byte[] output = bos.toByteArray(); + String l = new String(output, 0, strLen); + StringBuilder buf = new StringBuilder(strLen); + for (int i = 0; i < reps; i++) { + buf.append(testString); + } + assertEquals(buf.toString(), l); + for (int i = strLen; i < output.length; i++) { + assertEquals(0, output[i]); + } + } + + @Test + public void testPartialWritingThrowsException() { + try { + testWriteAndPad(512, "hello world!\n", true); + fail("Exception for partial write not thrown"); + } catch (IOException e) { + String msg = e.getMessage(); + assertEquals("exception message", + "Failed to write 512 bytes atomically. Only wrote 511", msg); + } + + } + + @Test + public void testWriteFailsAfterFLClosedThrowsException() { + try { + FixedLengthBlockOutputStream out = getClosedFLBOS(); + out.write(1); + fail("expected Closed Channel Exception"); + } catch (IOException e) { + assertThat(e, IsInstanceOf.instanceOf(ClosedChannelException.class)); + // expected + } + try { + FixedLengthBlockOutputStream out = getClosedFLBOS(); + out.write(new byte[] {0,1,2,3}); + fail("expected Closed Channel Exception"); + } catch (IOException e) { + assertThat(e, IsInstanceOf.instanceOf(ClosedChannelException.class)); + // expected + } + + try { + FixedLengthBlockOutputStream out = getClosedFLBOS(); + out.write(ByteBuffer.wrap(new byte[] {0,1,2,3})); + fail("expected Closed Channel Exception"); + } catch (IOException e) { + assertThat(e, IsInstanceOf.instanceOf(ClosedChannelException.class)); + // expected + } + + } + + private FixedLengthBlockOutputStream getClosedFLBOS() throws IOException { + int blockSize = 512; + FixedLengthBlockOutputStream out = new FixedLengthBlockOutputStream( + new MockOutputStream(blockSize, false), blockSize); + out.write(1); + assertTrue(out.isOpen()); + out.close(); + assertFalse(out.isOpen()); + return out; + } + + @Test + public void testWriteFailsAfterDestClosedThrowsException() { + int blockSize = 2; + MockOutputStream mock = new MockOutputStream(blockSize, false); + FixedLengthBlockOutputStream out = + new FixedLengthBlockOutputStream(mock, blockSize); + try { + out.write(1); + assertTrue(out.isOpen()); + mock.close(); + out.write(1); + fail("expected IO Exception"); + } catch (IOException e) { + // expected + } + assertFalse(out.isOpen()); + } + + @Test + public void testWithFileOutputStream() throws IOException { + final Path tempFile = Files.createTempFile("xxx", "yyy"); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + try { + Files.deleteIfExists(tempFile); + } catch (IOException e) { + } + } + }); + int blockSize = 512; + int reps = 1000; + OutputStream os = new FileOutputStream(tempFile.toFile()); + try (FixedLengthBlockOutputStream out = new FixedLengthBlockOutputStream( + os, blockSize)) { + DataOutputStream dos = new DataOutputStream(out); + for (int i = 0; i < reps; i++) { + dos.writeInt(i); + } + } + long expectedDataSize = reps * 4L; + long expectedFileSize = (long)Math.ceil(expectedDataSize/(double)blockSize)*blockSize; + assertEquals("file size",expectedFileSize, Files.size(tempFile)); + DataInputStream din = new DataInputStream(Files.newInputStream(tempFile)); + for(int i=0;i(offset + expected.length)); + for (int i = 0; i < expected.length; i++) { + assertEquals(String.format("%s ([%d])", msg, i), expected[i], actual[i + offset]); + } + } + + private static class MockOutputStream extends OutputStream { + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + private final int requiredWriteSize; + private final boolean doPartialWrite; + private AtomicBoolean closed = new AtomicBoolean(); + + private MockOutputStream(int requiredWriteSize, boolean doPartialWrite) { + this.requiredWriteSize = requiredWriteSize; + this.doPartialWrite = doPartialWrite; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + checkIsOpen(); + assertEquals("write size", requiredWriteSize, len); + if (doPartialWrite) { + len--; + } + bos.write(b, off, len); + } + + private void checkIsOpen() throws IOException { + if (closed.get()) { + IOException e = new IOException("Closed"); + throw e; + } + } + + @Override + public void write(int b) throws IOException { + checkIsOpen(); + assertEquals("write size", requiredWriteSize, 1); + bos.write(b); + } + + @Override + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + bos.close(); + } + } + } + + private static class MockWritableByteChannel implements WritableByteChannel { + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + private final int requiredWriteSize; + private final boolean doPartialWrite; + + private MockWritableByteChannel(int requiredWriteSize, boolean doPartialWrite) { + this.requiredWriteSize = requiredWriteSize; + this.doPartialWrite = doPartialWrite; + } + + @Override + public int write(ByteBuffer src) throws IOException { + assertEquals("write size", requiredWriteSize, src.remaining()); + if (doPartialWrite) { + src.limit(src.limit() - 1); + } + int bytesOut = src.remaining(); + while (src.hasRemaining()) { + bos.write(src.get()); + } + return bytesOut; + } + + AtomicBoolean closed = new AtomicBoolean(); + + @Override + public boolean isOpen() { + return !closed.get(); + } + + @Override + public void close() throws IOException { + closed.compareAndSet(false, true); + } + } +} diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/IOUtilsTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/IOUtilsTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/IOUtilsTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/IOUtilsTest.java 2018-05-23 12:50:54.000000000 +0000 @@ -5,9 +5,9 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.apache.commons.compress.utils; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.FilterInputStream; import java.io.InputStream; @@ -101,6 +102,11 @@ readFully(source, b); } + @Test(expected = IllegalArgumentException.class) + public void copyThrowsOnZeroBufferSize() throws IOException { + IOUtils.copy(new ByteArrayInputStream(new byte[0]), new ByteArrayOutputStream(), 0); + } + private static void readFully(final byte[] source, ByteBuffer b) throws IOException { IOUtils.readFully(new ReadableByteChannel() { private int idx; diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/SeekableInMemoryByteChannelTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/SeekableInMemoryByteChannelTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/SeekableInMemoryByteChannelTest.java 2016-12-25 11:57:03.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/SeekableInMemoryByteChannelTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -45,6 +45,7 @@ assertEquals(testData.length, readCount); assertArrayEquals(testData, readBuffer.array()); assertEquals(testData.length, c.position()); + c.close(); } @Test @@ -58,6 +59,7 @@ assertEquals(testData.length, readCount); assertArrayEquals(testData, Arrays.copyOf(readBuffer.array(), testData.length)); assertEquals(testData.length, c.position()); + c.close(); } @Test @@ -72,6 +74,7 @@ assertEquals(4L, readCount); assertEquals("data", new String(readBuffer.array(), Charset.forName(UTF_8))); assertEquals(testData.length, c.position()); + c.close(); } @Test @@ -85,6 +88,7 @@ //then assertEquals(0L, readBuffer.position()); assertEquals(-1, readCount); + c.close(); } @Test(expected = ClosedChannelException.class) @@ -107,6 +111,7 @@ assertEquals(testData.length, writeCount); assertArrayEquals(testData, Arrays.copyOf(c.array(), (int) c.size())); assertEquals(testData.length, c.position()); + c.close(); } @Test @@ -123,6 +128,7 @@ assertEquals(testData.length, writeCount); assertArrayEquals(expectedData.array(), Arrays.copyOf(c.array(), (int) c.size())); assertEquals(testData.length + 5, c.position()); + c.close(); } @@ -144,6 +150,7 @@ //then byte[] bytes = Arrays.copyOf(c.array(), (int) c.size()); assertEquals("Some", new String(bytes, Charset.forName(UTF_8))); + c.close(); } @Test @@ -156,6 +163,7 @@ //then assertEquals(4L, c.position()); assertEquals(4L, c.size()); + c.close(); } @Test @@ -170,6 +178,7 @@ assertEquals(4L, posAtFour); assertEquals(c.size(), posAtTheEnd); assertEquals(posPastTheEnd, posPastTheEnd); + c.close(); } @Test(expected = IllegalArgumentException.class) @@ -178,6 +187,7 @@ SeekableInMemoryByteChannel c = new SeekableInMemoryByteChannel(); //when c.position(Integer.MAX_VALUE + 1L); + c.close(); } @Test(expected = ClosedChannelException.class) diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ServiceLoaderIteratorTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ServiceLoaderIteratorTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/ServiceLoaderIteratorTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/ServiceLoaderIteratorTest.java 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress.utils; + +import org.junit.Test; + +import java.util.NoSuchElementException; + +import static org.junit.Assert.assertFalse; + +/** + * Unit tests for class {@link ServiceLoaderIterator org.apache.commons.compress.utils.ServiceLoaderIterator}. + * + * @date 13.06.2017 + * @see ServiceLoaderIterator + **/ +public class ServiceLoaderIteratorTest { + + + + @Test(expected = NoSuchElementException.class) + public void testNextThrowsNoSuchElementException() { + + Class clasz = String.class; + ServiceLoaderIterator serviceLoaderIterator = new ServiceLoaderIterator(clasz); + + serviceLoaderIterator.next(); + + } + + + @Test + public void testHasNextReturnsFalse() { + + Class clasz = Object.class; + ServiceLoaderIterator serviceLoaderIterator = new ServiceLoaderIterator(clasz); + boolean result = serviceLoaderIterator.hasNext(); + + assertFalse(result); + + } + + + @Test(expected = UnsupportedOperationException.class) + public void testRemoveThrowsUnsupportedOperationException() { + + Class clasz = Integer.class; + ServiceLoaderIterator serviceLoaderIterator = new ServiceLoaderIterator(clasz); + + serviceLoaderIterator.remove(); + + + } + + + +} \ No newline at end of file diff -Nru libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/SkipShieldingInputStreamTest.java libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/SkipShieldingInputStreamTest.java --- libcommons-compress-java-1.13/src/test/java/org/apache/commons/compress/utils/SkipShieldingInputStreamTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/java/org/apache/commons/compress/utils/SkipShieldingInputStreamTest.java 2018-05-23 12:50:54.000000000 +0000 @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.utils; + +import java.io.IOException; +import java.io.InputStream; +import org.junit.Assert; +import org.junit.Test; + +public class SkipShieldingInputStreamTest { + + @Test + public void skipDelegatesToRead() throws IOException { + try (InputStream i = new SkipShieldingInputStream(new InputStream() { + @Override + public long skip(long n) { + Assert.fail("skip invoked"); + return -1; + } + @Override + public int read() { + return -1; + } + @Override + public int read(byte[] b, int off, int len) { + return len; + } + })) { + Assert.assertEquals(100, i.skip(100)); + } + } + + @Test + public void skipHasAnUpperBoundOnRead() throws IOException { + try (InputStream i = new SkipShieldingInputStream(new InputStream() { + @Override + public long skip(long n) { + Assert.fail("skip invoked"); + return -1; + } + @Override + public int read() { + return -1; + } + @Override + public int read(byte[] b, int off, int len) { + return len; + } + })) { + Assert.assertTrue(Integer.MAX_VALUE > i.skip(Long.MAX_VALUE)); + } + } + + @Test + public void skipSwallowsNegativeArguments() throws IOException { + try (InputStream i = new SkipShieldingInputStream(new InputStream() { + @Override + public long skip(long n) { + Assert.fail("skip invoked"); + return -1; + } + @Override + public int read() { + return -1; + } + @Override + public int read(byte[] b, int off, int len) { + Assert.fail("read invoked"); + return len; + } + })) { + Assert.assertEquals(0, i.skip(Long.MIN_VALUE)); + } + } + +} Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/bla.deflate64.7z and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/bla.deflate64.7z differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/bla.dump.lz4 and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/bla.dump.lz4 differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/bla.tar.block_lz4 and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/bla.tar.block_lz4 differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/bla.tar.br and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/bla.tar.br differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/bla.tar.lz4 and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/bla.tar.lz4 differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/brotli.testdata.compressed and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/brotli.testdata.compressed differ diff -Nru libcommons-compress-java-1.13/src/test/resources/brotli.testdata.uncompressed libcommons-compress-java-1.18/src/test/resources/brotli.testdata.uncompressed --- libcommons-compress-java-1.13/src/test/resources/brotli.testdata.uncompressed 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/resources/brotli.testdata.uncompressed 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1 @@ +XXXXXXXXXXYYYYYYYYYY \ No newline at end of file Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-358.iwa and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-358.iwa differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-358.uncompressed and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-358.uncompressed differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-379.jar and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-379.jar differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-380/COMPRESS-380-dd.zip and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-380/COMPRESS-380-dd.zip differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-380/COMPRESS-380-input and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-380/COMPRESS-380-input differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-380/COMPRESS-380-readbeyondmemory.zip and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-380/COMPRESS-380-readbeyondmemory.zip differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-380/COMPRESS-380.zip and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-380/COMPRESS-380.zip differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-382 and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-382 differ diff -Nru libcommons-compress-java-1.13/src/test/resources/COMPRESS-386 libcommons-compress-java-1.18/src/test/resources/COMPRESS-386 --- libcommons-compress-java-1.13/src/test/resources/COMPRESS-386 1970-01-01 00:00:00.000000000 +0000 +++ libcommons-compress-java-1.18/src/test/resources/COMPRESS-386 2018-05-02 20:17:13.000000000 +0000 @@ -0,0 +1 @@ +B \ No newline at end of file Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-417.tar and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-417.tar differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/COMPRESS-459.cpio and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/COMPRESS-459.cpio differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/mixed.zip and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/mixed.zip differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/zipbomb.xlsx and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/zipbomb.xlsx differ Binary files /tmp/tmp5KqhVT/ODnpqstMcN/libcommons-compress-java-1.13/src/test/resources/zstd-tests.tar and /tmp/tmp5KqhVT/IZRFD1THG0/libcommons-compress-java-1.18/src/test/resources/zstd-tests.tar differ