diff -Nru apt-cacher-ng-0.6.11/ChangeLog apt-cacher-ng-0.6.12/ChangeLog --- apt-cacher-ng-0.6.11/ChangeLog 2011-12-26 15:38:30.000000000 +0000 +++ apt-cacher-ng-0.6.12/ChangeLog 2012-01-15 15:10:33.000000000 +0000 @@ -1,3 +1,16 @@ +apt-cacher-ng (0.6.12) BETA; urgency=low + + * NEW: internal following of HTTP redirection responses with configurable + redirection limit + * FIX: unescaping RequestAppendix option value with multiple newlines + correctly + * FIX: more generic name pattern for volatile files, also treating *.db + files as compressed index data in expiration (automatically detected by + magic strings now) + * FIX: now really supporting legacy .lzma file format + + -- Eduard Bloch Sun, 15 Jan 2012 16:09:54 +0100 + apt-cacher-ng (0.6.11) BETA; urgency=low * FIX: when original source used chunked transfer encoding, the local diff -Nru apt-cacher-ng-0.6.11/conf/acng.conf apt-cacher-ng-0.6.12/conf/acng.conf --- apt-cacher-ng-0.6.11/conf/acng.conf 2011-12-26 15:30:31.000000000 +0000 +++ apt-cacher-ng-0.6.12/conf/acng.conf 2012-01-07 00:13:37.000000000 +0000 @@ -90,7 +90,7 @@ # Stop expiration when a critical problem appeared. Currently only failed # refresh of an index file is considered as critical. # -# WARNING: don't touch this option or set to a non-zero number. +# WARNING: don't touch this option or set to zero. # Anything else is DANGEROUS and may cause data loss. # # ExAbortOnProblems: 1 @@ -262,3 +262,15 @@ # any port. # # AllowUserPorts: 80 + +# Normally the HTTP redirection responses are forwarded to the original caller +# (i.e. APT) which starts a new download attempt from the new URL. This +# solution is ok for client configurations with proxy mode but doesn't work +# well with configurations using URL prefixes. To work around this the server +# can restart its own download with another URL. However, this might be used to +# circumvent download source policies by malicious users. +# The RedirMax option specifies how many such redirects the server should +# follow per request, 0 disables the internal redirection. If not set, +# default value is 0 if ForceManaged is used and 5 otherwise. +# +# RedirMax: 5 diff -Nru apt-cacher-ng-0.6.11/debian/changelog apt-cacher-ng-0.6.12/debian/changelog --- apt-cacher-ng-0.6.11/debian/changelog 2011-12-26 15:38:12.000000000 +0000 +++ apt-cacher-ng-0.6.12/debian/changelog 2012-01-15 15:10:10.000000000 +0000 @@ -1,3 +1,10 @@ +apt-cacher-ng (0.6.12-1) unstable; urgency=low + + * New upstream version + * Added use of dpkg-buildflags (for hardening) + + -- Eduard Bloch Sun, 15 Jan 2012 16:10:09 +0100 + apt-cacher-ng (0.6.11-1) unstable; urgency=low * Start init script at S18, not before bind (closes: 650726). Also adds the diff -Nru apt-cacher-ng-0.6.11/debian/rules apt-cacher-ng-0.6.12/debian/rules --- apt-cacher-ng-0.6.11/debian/rules 2011-12-26 11:26:52.000000000 +0000 +++ apt-cacher-ng-0.6.12/debian/rules 2012-01-15 13:35:57.000000000 +0000 @@ -5,13 +5,13 @@ SHELL=/bin/bash CDIR=$(CURDIR)/debian/apt-cacher-ng/etc/apt-cacher-ng/ -CFLAGS = -Wall -g +CFLAGS = $(shell dpkg-buildflags --get CFLAGS) $(shell dpkg-buildflags --get CPPFLAGS) -Wall +export CFLAGS +CXXFLAGS=$(CFLAGS) +export CXXFLAGS +LDFLAGS = $(shell dpkg-buildflags --get LDFLAGS) +export LDFLAGS -ifneq (,$(findstring noopt,$(DEB_BUILD_OPTIONS))) -CFLAGS += -O0 -else -CFLAGS += -O3 -endif PARJOBS=-j2 ifneq (,$(filter parallel=%,$(DEB_BUILD_OPTIONS))) @@ -19,9 +19,6 @@ endif -CXXFLAGS=$(CFLAGS) -export CXXFLAGS - configure: configure-stamp configure-stamp: dh_testdir diff -Nru apt-cacher-ng-0.6.11/include/acfg.h apt-cacher-ng-0.6.12/include/acfg.h --- apt-cacher-ng-0.6.11/include/acfg.h 2011-12-08 19:30:31.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/acfg.h 2012-01-14 13:19:57.000000000 +0000 @@ -31,7 +31,7 @@ extern int debug, numcores, offlinemode, foreground, verbose, stupidfs, forcemanaged, keepnver, verboselog, extreshhold, exfailabort, tpstandbymax, tpthreadmax, dnscachetime, dlbufsize, usewrap, exporigin, logxff, oldupdate, recompbz2, nettimeout, updinterval, forwardsoap, dirperms, fileperms, -maxtempdelay; +maxtempdelay, redirmax; // processed config settings extern tHttpUrl proxy_info; @@ -96,4 +96,5 @@ bool CompileExpressions(); } + #endif diff -Nru apt-cacher-ng-0.6.11/include/cacheman.h apt-cacher-ng-0.6.12/include/cacheman.h --- apt-cacher-ng-0.6.11/include/cacheman.h 2011-11-22 21:05:03.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/cacheman.h 2012-01-06 23:32:40.000000000 +0000 @@ -23,7 +23,7 @@ #define SZABSPATH(x) (CACHE_BASE+(x)).c_str() #define SABSPATH(x) (CACHE_BASE+(x)) -#define REDIRMAX 5 +#define REDIRMAX acfg::redirmax #define FAKEDATEMARK "Sat, 26 Apr 1986 01:23:39 GMT+3" diff -Nru apt-cacher-ng-0.6.11/include/config.h apt-cacher-ng-0.6.12/include/config.h --- apt-cacher-ng-0.6.11/include/config.h 2011-12-08 19:30:31.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/config.h 2012-01-14 13:24:00.000000000 +0000 @@ -4,7 +4,7 @@ #include "acsyscap.h" -#define ACVERSION "0.6.11" +#define ACVERSION "0.6.12" #define __STDC_FORMAT_MACROS #include diff -Nru apt-cacher-ng-0.6.11/include/conn.h apt-cacher-ng-0.6.12/include/conn.h --- apt-cacher-ng-0.6.11/include/conn.h 2011-08-12 15:04:16.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/conn.h 2012-01-06 23:32:40.000000000 +0000 @@ -72,6 +72,7 @@ UINT m_nProcessedJobs; #endif + job * AddNewJob(header *tempHeader); }; #endif diff -Nru apt-cacher-ng-0.6.11/include/csmapping.h apt-cacher-ng-0.6.12/include/csmapping.h --- apt-cacher-ng-0.6.11/include/csmapping.h 2011-10-16 01:02:43.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/csmapping.h 2012-01-07 00:13:37.000000000 +0000 @@ -71,7 +71,7 @@ } mstring GetCsAsString() const { - return CsBinToString(csum, CSTYPE_MD5==csType?16:20); + return BytesToHexString(csum, CSTYPE_MD5==csType?16:20); } operator mstring() const { diff -Nru apt-cacher-ng-0.6.11/include/fileio.h apt-cacher-ng-0.6.12/include/fileio.h --- apt-cacher-ng-0.6.11/include/fileio.h 2011-11-01 20:26:55.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/fileio.h 2012-01-14 21:32:53.000000000 +0000 @@ -57,7 +57,7 @@ #define forceclose(fd) { while(0 != ::close(fd)) { if(errno != EINTR) break; }; fd=-1; } #define checkforceclose(fd) if(fd>=0){ while(0 != ::close(fd)) { if(errno != EINTR) break; }; fd=-1; } -#define checkForceFclose(fh) if(fh) { int fd=fileno(fh); if(0!=::fclose(fh) && errno!=EBADF) forceclose(fd); fh=NULL;} +#define checkForceFclose(fh) if(fh) { int fd=::fileno(fh); if(0!=::fclose(fh) && errno!=EBADF) forceclose(fd); fh=NULL;} diff -Nru apt-cacher-ng-0.6.11/include/filereader.h apt-cacher-ng-0.6.12/include/filereader.h --- apt-cacher-ng-0.6.11/include/filereader.h 2011-08-17 21:53:59.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/filereader.h 2012-01-14 21:32:53.000000000 +0000 @@ -6,15 +6,8 @@ #include "acbuf.h" #include "fileio.h" -#ifdef HAVE_LIBBZ2 -#include -#endif -#ifdef HAVE_ZLIB -#include -#endif -#ifdef HAVE_LZMA -#include -#endif +class IDecompressor; + /*! * Helper class used to read files. @@ -35,15 +28,15 @@ */ bool OpenFile(const mstring & sFilename, bool bNoMagic=false); - mstring GetPureFilename(); + //mstring GetPureFilename(); //////! Filename with all prepended path and compressed suffix stripped //////void GetBaseFileName(mstring & sOut); //! Returns lines when beginning with non-space, otherwise empty string. //! @return False on errors. bool GetOneLine(mstring & sOut); - mstring GetPositionDescription() { return (tSS() << m_sOrigName << ":" << m_nCurLine); } - bool CheckGoodState(bool bTerminateOnErrors) const; + UINT GetCurrentLine() const { return m_nCurLine;} ; + bool CheckGoodState(bool bTerminateOnErrors, cmstring *reportFilePath=NULL) const; bool GetChecksum(int csType, uint8_t out[], off_t &scannedSize, FILE *pDumpFile=NULL); static bool GetChecksum(const mstring & sFileName, int csType, uint8_t out[], @@ -57,7 +50,15 @@ inline void AddEofLines(UINT nCount=1) { m_nEofLines=nCount; } private: - + /* + enum + { + GZIP, + BZIP2, + LZMA, + PLAIN + } m_ztype; +*/ UINT flags; char *m_szFileBuf; size_t m_nBufSize, m_nBufPos; @@ -65,7 +66,6 @@ acbuf m_UncompBuf; // uncompressed window // visible position reporting - mstring m_sOrigName; UINT m_nCurLine; int m_fd; @@ -73,18 +73,10 @@ int m_nEofLines; //! Returns: currently usable remaining text buffer size - inline void _UncompressMoreData(); - -#ifdef HAVE_LIBBZ2 - bz_stream m_bzStream; -#endif -#ifdef HAVE_ZLIB - z_stream m_zStream; -#endif -#ifdef HAVE_LZMA - lzma_stream m_xzStream; -#endif + inline void UncompressMoreData(); + MYSTD::auto_ptr m_Dec; + // not to be copied filereader& operator=(const filereader&); filereader(const filereader&); diff -Nru apt-cacher-ng-0.6.11/include/job.h apt-cacher-ng-0.6.12/include/job.h --- apt-cacher-ng-0.6.11/include/job.h 2011-11-05 16:05:40.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/job.h 2012-01-06 23:32:40.000000000 +0000 @@ -85,6 +85,9 @@ bool ParseRange(); off_t m_nRangeFrom, m_nRangeTo; + + int m_nRedirRemaining; + bool CreateRedirect(header &respHead); }; #endif diff -Nru apt-cacher-ng-0.6.11/include/meta.h apt-cacher-ng-0.6.12/include/meta.h --- apt-cacher-ng-0.6.11/include/meta.h 2011-11-05 11:52:35.000000000 +0000 +++ apt-cacher-ng-0.6.12/include/meta.h 2012-01-15 15:10:33.000000000 +0000 @@ -197,7 +197,7 @@ void appendLong(mstring &s, long val); -mstring CsBinToString(const uint8_t b[], unsigned short binLength); +mstring BytesToHexString(const uint8_t b[], unsigned short binLength); bool CsAsciiToBin(const char *a, uint8_t b[], unsigned short binLength); typedef const unsigned char CUCHAR; @@ -385,9 +385,23 @@ { mstring ret; for(cmstring::const_iterator it=s.begin();it!=s.end();++it) - ret+=char(*it=='\\' ? unEscape(*it) : *it); + { + if(*it=='\\') + { + ++it; + ret+=unEscape(*it); + } + else + ret+=*it; + } return ret; } +MYSTD::string BytesToHexString(const uint8_t sum[], unsigned short lengthBin); + +// STFU helpers, (void) casts are not effective for certain functions +static inline void ignore_value (int i) { (void) i; } +static inline void ignore_ptr (void* p) { (void) p; } + #endif // _META_H diff -Nru apt-cacher-ng-0.6.11/INSTALL apt-cacher-ng-0.6.12/INSTALL --- apt-cacher-ng-0.6.11/INSTALL 2011-02-18 22:05:16.000000000 +0000 +++ apt-cacher-ng-0.6.12/INSTALL 2012-01-07 00:13:37.000000000 +0000 @@ -13,7 +13,7 @@ Build and installation instructions: - run "make" - - if an error occurs, read the message, solve the problem, run "make clean" + - if an error occurs, read the message, solve the problem, run "make distclean" and retry with "make" - when done, copy build/apt-cacher-ng and build/in.acng to /usr/local/sbin - make apt-cacher-ng be executed at system startup time. diff -Nru apt-cacher-ng-0.6.11/source/acfg.cc apt-cacher-ng-0.6.12/source/acfg.cc --- apt-cacher-ng-0.6.11/source/acfg.cc 2011-11-22 21:05:03.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/acfg.cc 2012-01-15 15:10:33.000000000 +0000 @@ -107,6 +107,7 @@ ,{ "KeepExtraVersions", &keepnver, NULL, 10} ,{ "UseWrap", &usewrap, NULL, 10} ,{ "FreshIndexMaxAge", &maxtempdelay, NULL, 10} + ,{ "RedirMax", &redirmax, NULL, 10} ,{ "DirPerms", &dirperms, NULL, 8} ,{ "FilePerms", &fileperms, NULL, 8} @@ -395,7 +396,7 @@ if (0 >= --m_nRefCnt) { //system(cmdRel.c_str()); - downTimeNext= time(0) + downDuration; + downTimeNext= ::time(0) + downDuration; } } virtual void JobConnect() @@ -405,8 +406,8 @@ { if(downTimeNext) // huh, already ticking? reset downTimeNext=0; - else - system(cmdCon.c_str()); + else if(system(cmdCon.c_str())) + aclog::err(tSS() << "Warning: " << cmdCon << " returned with error code."); } } }; @@ -838,8 +839,8 @@ } else { - cerr << "Bad backend description, around line " - << itor.reader.GetPositionDescription() << endl; + cerr << "Bad backend description, around line " << sFile << ":" + << itor.reader.GetCurrentLine() << endl; exit(2); } } @@ -865,7 +866,7 @@ filereader reader; if(debug>4) cout << "Reading rewrite file: " << sFile <= LOG_MORE) { for (UINT i = 0; i < _countof(n2sTbl); i++) @@ -1200,7 +1207,8 @@ if(acfg::debug & LOG_FLUSH) aclog::flush(); - system(hooks.cmdRel.c_str()); + if(system(hooks.cmdRel.c_str())) + aclog::err(tSS() << "Warning: " << hooks.cmdRel << " returned with error code."); hooks.downTimeNext = 0; } else // in future, use the soonest time diff -Nru apt-cacher-ng-0.6.11/source/acfg_defaults.cc apt-cacher-ng-0.6.12/source/acfg_defaults.cc --- apt-cacher-ng-0.6.11/source/acfg_defaults.cc 2011-12-26 15:30:31.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/acfg_defaults.cc 2012-01-14 13:19:57.000000000 +0000 @@ -21,9 +21,12 @@ string cachedir("/var/tmp"), logdir("/var/tmp"), fifopath, pidfile, reportpage, confdir, adminauth, bindaddr, mirrorsrcs, suppdir("/usr/lib/apt-cacher-ng"); + #define COMPRLIST "(\\.gz|\\.bz2|\\.lzma|\\.xz)" -#define COMPONENT_OPTIONAL "(-[a-z0-9-])" -#define PARANOIASOURCE "(\\.orig|\\.debian)" +#define ALXPATTERN ".*\\.(db|files|abs)(\\.tar"COMPRLIST")?" +#define COMPOPT COMPRLIST"?" +//#define COMPONENT_OPTIONAL "(-[a-z0-9-])" +//#define PARANOIASOURCE "(\\.orig|\\.debian)" string pfilepat(".*(\\.d?deb|\\.rpm|\\.dsc|\\.tar"COMPRLIST"(\\.gpg)?" "|\\.diff"COMPRLIST"|\\.jigdo|\\.template|changelog|copyright" @@ -34,16 +37,16 @@ ")$"); string vfilepat( INFOLDER - "(Index|Packages"COMPRLIST"?|InRelease|Release|Release\\.gpg|" - "Sources"COMPRLIST"?|release|index\\.db-.*\\.gz|Contents-[^/]*"COMPRLIST"?" - "|pkglist[^/]*\\.bz2|rclist[^/]*\\.bz2|/meta-release[^/]*|Translation[^/]*"COMPRLIST"?" + "(Index|Packages"COMPOPT"|InRelease|Release|Release\\.gpg|" + "Sources"COMPOPT"|release|index\\.db-.*\\.gz|Contents-[^/]*"COMPOPT + "|pkglist[^/]*\\.bz2|rclist[^/]*\\.bz2|/meta-release[^/]*|Translation[^/]*"COMPOPT "|MD5SUMS|SHA1SUMS" // d-i stuff "|((setup|setup-legacy)(\\.ini|\\.bz2|\\.hint)(\\.sig)?)|mirrors\\.lst" // cygwin "|repo(index|md)\\.xml(\\.asc|\\.key)?|directory\\.yast" // opensuse "|products|content(\\.asc|\\.key)?|media" // opensuse 2, are they important? "|filelists\\.xml\\.gz|filelists\\.sqlite\\.bz2|repomd\\.xml" // SL, http://ra.khe.sh/computers/linux/apt-cacher-ng-with-yum.html "|packages\\.[a-zA-Z][a-zA-Z]\\.gz|info\\.txt|license\\.tar\\.gz|license\\.zip" //opensuse - "|.*\\.db(\\.tar\\.gz)?|.*\\.files\\.tar\\.gz|.*\\.abs\\.tar\\.gz" // Arch Linux + "|"ALXPATTERN // Arch Linux "|metalink\\?repo|.*prestodelta\\.xml\\.gz" // fedora core, maybe... never worked reliably ")$" // end of filename paterns. Any other some specific URLs? "|/dists/.*/installer-[^/]+/[^0-9][^/]+/images/.*"); // d-i stuff but not containing a number (year) in the revision directory (like "current", "beta", ...) @@ -56,7 +59,7 @@ "|Translation[^/]*"COMPRLIST"?" // to be checked, but they should never really go anywhere "|MD5SUMS|SHA1SUMS" "|.*\\.xml" // SUSE - "|.*\\.db\\.tar\\.gz|.*\\.files\\.tar\\.gz|.*\\.abs\\.tar\\.gz" // Arch Linux + "|"ALXPATTERN // Arch Linux "|[a-z]+32.exe" ")$" "|/dists/.*/installer-.*/images/.*"); @@ -68,7 +71,7 @@ int dnscachetime(1800), dlbufsize(70000), exfailabort(1), exporigin(false), numcores(1), logxff(false), oldupdate(false), recompbz2(false), nettimeout(60), updinterval(0), -forwardsoap(RESERVED_DEFVAL), usewrap(RESERVED_DEFVAL); +forwardsoap(RESERVED_DEFVAL), usewrap(RESERVED_DEFVAL), redirmax(RESERVED_DEFVAL); string agentname("Debian Apt-Cacher-NG/" ACVERSION); string remoteport("80"), port("3142"); diff -Nru apt-cacher-ng-0.6.11/source/cacheman.cc apt-cacher-ng-0.6.12/source/cacheman.cc --- apt-cacher-ng-0.6.11/source/cacheman.cc 2011-11-22 21:05:03.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/cacheman.cc 2012-01-14 21:32:53.000000000 +0000 @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -1412,9 +1413,11 @@ stripSuffix(sPureIfileName, ".xz"); stripSuffix(sPureIfileName, ".lzma"); if (sPureIfileName=="Packages") // Debian's Packages file - return EIDX_PACKAGES; - if (endsWithSzAr(sPureIfileName, ".db.tar")) + return EIDX_PACKAGES; + + if(endsWithSzAr(sPureIfileName, ".db") || endsWithSzAr(sPureIfileName, ".db.tar")) return EIDX_ARCHLXDB; + if (sPureIfileName == "setup") return EIDX_CYGSETUP; @@ -1434,7 +1437,6 @@ if (sPureIfileName == "Index") return endsWithSzAr(sPath, "i18n/Index") ? EIDX_TRANSIDX : EIDX_DIFFIDX; - return EIDX_UNSUPPORTED; } diff -Nru apt-cacher-ng-0.6.11/source/conn.cc apt-cacher-ng-0.6.12/source/conn.cc --- apt-cacher-ng-0.6.11/source/conn.cc 2011-10-04 13:15:53.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/conn.cc 2012-01-06 23:32:40.000000000 +0000 @@ -278,13 +278,8 @@ ldbg("Parsed REQUEST:" << m_pTmpHead->frontLine); ldbg("Rest: " << inBuf.size()); - job * j = new job(m_pTmpHead, this); - m_pTmpHead=NULL; // owned by job - j->PrepareDownload(); - m_jobs2send.push_back(j); -#ifdef DEBUG - m_nProcessedJobs++; -#endif + AddNewJob(m_pTmpHead); + m_pTmpHead=NULL; // owned by job now } MYCATCH(bad_alloc&) { @@ -411,3 +406,15 @@ sumOut=0; } +job* con::AddNewJob(header *pHead) + { + job * j = new job(pHead, this); + + j->PrepareDownload(); + m_jobs2send.push_back(j); +#ifdef DEBUG + m_nProcessedJobs++; +#endif + return j; + +} diff -Nru apt-cacher-ng-0.6.11/source/conserver.cc apt-cacher-ng-0.6.12/source/conserver.cc --- apt-cacher-ng-0.6.11/source/conserver.cc 2011-10-08 20:40:32.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/conserver.cc 2012-01-14 13:19:57.000000000 +0000 @@ -413,10 +413,10 @@ continue; } -#ifdef HAVE_LIBWRAP if (acfg::usewrap) { - // libwrap code is evil non-reentrant stuff, call it from here only +#ifdef HAVE_LIBWRAP + // libwrap is non-reentrant stuff, call it from here only request_info req; request_init(&req, RQ_DAEMON, "apt-cacher-ng", RQ_FILE, fd, 0); fromhost(&req); @@ -426,8 +426,10 @@ forceShutdownClose(fd); continue; } - } +#else + aclog::err("WARNING: attempted to use libwrap which was not enabled at build time"); #endif + } SetupConAndGo(fd, hbuf); } diff -Nru apt-cacher-ng-0.6.11/source/dlcon.cc apt-cacher-ng-0.6.12/source/dlcon.cc --- apt-cacher-ng-0.6.11/source/dlcon.cc 2011-12-08 19:30:31.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/dlcon.cc 2012-01-14 13:19:57.000000000 +0000 @@ -607,16 +607,9 @@ else if (r == 0) // looks like a timeout { dbgline; - - if(qActive.empty()) // there was nothing to do either - { - sErrorMsg = "500 Transfer timeout"; - return EFLAG_HINT_SWITCH; - } - - sErrorMsg = "500 Disconnected by peer"; - - return EFLAG_DISCON|EFLAG_JOB_BROKEN; + sErrorMsg = "500 Connection timeout"; + // was there anything to do at all? + return qActive.empty() ? EFLAG_HINT_SWITCH : (EFLAG_DISCON|EFLAG_JOB_BROKEN); } if (FD_ISSET(m_wakepipe[0], &rfds)) diff -Nru apt-cacher-ng-0.6.11/source/expiration.cc apt-cacher-ng-0.6.12/source/expiration.cc --- apt-cacher-ng-0.6.11/source/expiration.cc 2011-11-22 21:05:03.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/expiration.cc 2012-01-15 15:10:33.000000000 +0000 @@ -174,7 +174,7 @@ { SendFmt() << "BAD: " << sPathRel << " (truncating)
\n"; - ::truncate(sPathAbs.c_str(), 0); + ignore_value(::truncate(sPathAbs.c_str(), 0)); } else { diff -Nru apt-cacher-ng-0.6.11/source/fileitem.cc apt-cacher-ng-0.6.12/source/fileitem.cc --- apt-cacher-ng-0.6.11/source/fileitem.cc 2011-11-22 21:05:03.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/fileitem.cc 2012-01-15 15:10:33.000000000 +0000 @@ -293,8 +293,9 @@ if(m_status>FIST_FRESH) return false; m_status=FIST_INITED; - ::truncate(m_sPathAbs.c_str(), 0); - ::truncate((m_sPathAbs+".head").c_str(), 0); + if(::truncate(m_sPathAbs.c_str(), 0) || ::truncate((m_sPathAbs+".head").c_str(), 0)) + return false; + m_head.clear(); m_nSizeSeen=m_nSizeChecked=0; diff -Nru apt-cacher-ng-0.6.11/source/filereader.cc apt-cacher-ng-0.6.12/source/filereader.cc --- apt-cacher-ng-0.6.11/source/filereader.cc 2011-09-22 19:50:18.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/filereader.cc 2012-01-15 15:10:33.000000000 +0000 @@ -12,15 +12,10 @@ #include "sha1.h" #include "csmapping.h" -#define FLAG_PLAIN 1 -#define FLAG_GZ 2 -#define FLAG_BZ2 4 #define FLAG_EOF 8 #define FLAG_ERROR 16 #define FLAG_COMPEOF 32 -#define FLAG_LZMA 64 -//#include #include #define BUFSIZE 20000 @@ -33,9 +28,151 @@ using namespace MYSTD; + +class IDecompressor +{ +public: + bool inited; + IDecompressor() : inited(false) {}; + virtual ~IDecompressor() {}; + virtual UINT UncompMore(char *szInBuf, size_t nBufSize, size_t &nBufPos, acbuf &UncompBuf) + { return FLAG_ERROR; } + +}; + +#ifdef HAVE_LIBBZ2 +#include +class tBzDec : public IDecompressor +{ + bz_stream strm; +public: + tBzDec() + { + ::memset(&strm, 0, sizeof(strm)); + inited = (BZ_OK == BZ2_bzDecompressInit(&strm, 1, EXTREME_MEMORY_SAVING)); + } + ~tBzDec() + { + BZ2_bzDecompressEnd(&strm); + inited = false; + } + virtual UINT UncompMore(char *szInBuf, size_t nBufSize, size_t &nBufPos, acbuf &UncompBuf) + { + strm.next_in=szInBuf+nBufPos; + strm.avail_in=nBufSize-nBufPos; + strm.next_out=UncompBuf.wptr(); + strm.avail_out=UncompBuf.freecapa(); + + int ret=BZ2_bzDecompress(&strm); + if(ret==BZ_STREAM_END || ret==BZ_OK) + { + nBufPos += ( (nBufSize-nBufPos) - strm.avail_in); + unsigned int nGotBytes= UncompBuf.freecapa() - strm.avail_out; + UncompBuf.got(nGotBytes); + return ret == BZ_STREAM_END ? FLAG_COMPEOF : 0; + } + // or corrupted data? + return (FLAG_COMPEOF|FLAG_ERROR); + } +}; +#else +#define tBzDec IDecompressor +#endif + +#ifdef HAVE_ZLIB +#include +class tGzDec : public IDecompressor +{ + z_stream strm; +public: + tGzDec() + { + ::memset(&strm, 0, sizeof(strm)); + inited = (Z_OK == inflateInit2(&strm, 47)); + } + ~tGzDec() + { + deflateEnd(&strm); + inited = false; + } + virtual UINT UncompMore(char *szInBuf, size_t nBufSize, size_t &nBufPos, acbuf &UncompBuf) + { + strm.next_in = (uint8_t*) szInBuf + nBufPos; + strm.avail_in = nBufSize - nBufPos; + strm.next_out = (uint8_t*) UncompBuf.wptr(); + strm.avail_out = UncompBuf.freecapa(); + + int ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_END || ret == Z_OK) + { + nBufPos += ((nBufSize - nBufPos) - strm.avail_in); + unsigned int nGotBytes = UncompBuf.freecapa() - strm.avail_out; + UncompBuf.got(nGotBytes); + return ret == Z_STREAM_END ? FLAG_COMPEOF : 0; + } + // or corrupted data? + return (FLAG_COMPEOF | FLAG_ERROR); + } +}; + +#else +#define tGzDec IDecompressor +#endif + +#ifdef HAVE_LZMA +#include + +class tXzDec : public IDecompressor +{ + lzma_stream strm; +public: + tXzDec(bool lzmaFormat=false) + { + ::memset(&strm, 0, sizeof(strm)); + if(lzmaFormat) + inited = (LZMA_OK == lzma_alone_decoder(&strm, + EXTREME_MEMORY_SAVING ? 32000000 : MAX_VAL(uint64_t))); + else + inited = (LZMA_OK == lzma_stream_decoder(&strm, + EXTREME_MEMORY_SAVING ? 32000000 : MAX_VAL(uint64_t), + LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED)); + } + ~tXzDec() + { + lzma_end(&strm); + inited = false; + } + virtual UINT UncompMore(char *szInBuf, size_t nBufSize, size_t &nBufPos, acbuf &UncompBuf) + { + strm.next_in = (uint8_t*) szInBuf + nBufPos; + strm.avail_in = nBufSize - nBufPos; + strm.next_out = (uint8_t*) UncompBuf.wptr(); + strm.avail_out = UncompBuf.freecapa(); + + lzma_ret ret=lzma_code(&strm, LZMA_RUN); + if (ret == LZMA_STREAM_END || ret == LZMA_OK) + { + nBufPos += ((nBufSize - nBufPos) - strm.avail_in); + unsigned int nGotBytes = UncompBuf.freecapa() - strm.avail_out; + UncompBuf.got(nGotBytes); + return ret == LZMA_STREAM_END ? FLAG_COMPEOF : 0; + } + // or corrupted data? + return (FLAG_COMPEOF | FLAG_ERROR); + } +}; +#else +class tXzDec : public IDecompressor +{ +public: + tXzDec() {} + tXzDec(bool) {} +}; +#endif + filereader::filereader() : - flags(0), + flags(FLAG_ERROR|FLAG_EOF), m_szFileBuf((char*)MAP_FAILED), m_nBufSize(0), m_nBufPos(0), @@ -44,98 +181,77 @@ m_nEofLines(0) { }; - -string filereader::GetPureFilename() -{ - string x=m_sOrigName; - tStrPos p=x.rfind('/'); - if(stmiss!=p) - x.erase(0, p+1); - if(flags&FLAG_GZ) - x.erase(x.length()-3); - else if(flags&FLAG_BZ2) - x.erase(x.length()-4); - return x; -} +static const uint8_t gzMagic[] = +{ 0x1f, 0x8b, 0x8 }, bz2Magic[] = +{ 'B', 'Z', 'h' }, xzMagic[] = +{ 0xfd, '7', 'z', 'X', 'Z', 0x0 }, +lzmaMagic[] = {0x5d, 0, 0, 0x80}; bool filereader::OpenFile(const string & sFilename, bool bNoMagic) { Close(); // reset to clean state - m_sOrigName=sFilename; - m_fd=open(sFilename.c_str(), O_RDONLY); - - if(m_fd<0) - goto ofail; + m_fd = open(sFilename.c_str(), O_RDONLY); - if(bNoMagic) - flags|=FLAG_PLAIN; - else if(endsWithSzAr(sFilename, ".bz2")) - { -#ifdef HAVE_LIBBZ2 - m_bzStream.bzalloc = NULL; - m_bzStream.bzfree = NULL; - m_bzStream.opaque = NULL; - if(BZ_OK!=BZ2_bzDecompressInit ( & m_bzStream, 1, EXTREME_MEMORY_SAVING)) - goto ofail; - - // ok, now can mark for processing and cleanup - flags |= FLAG_BZ2; - m_UncompBuf.init(BUFSIZE); -#else - goto ofail; -#endif - } - else if(endsWithSzAr(sFilename, ".gz")) + if (m_fd < 0) + return false; + + if (bNoMagic) + m_Dec.reset(); + else if (endsWithSzAr(sFilename, ".bz2")) + m_Dec.reset(new tBzDec); + else if (endsWithSzAr(sFilename, ".gz")) + m_Dec.reset(new tGzDec); + else if(endsWithSzAr(sFilename, ".xz")) + m_Dec.reset(new tXzDec); + else if (endsWithSzAr(sFilename, ".lzma")) + m_Dec.reset(new tXzDec(true)); + else // unknown... ok, probe it { -#ifdef HAVE_ZLIB - memset(&m_zStream, 0, sizeof(m_zStream)); - - if(Z_OK != inflateInit2(&m_zStream, 47)) - goto ofail; - - flags |= FLAG_GZ; - m_UncompBuf.init(BUFSIZE); -#else - goto ofail; -#endif + filereader fh; + if (fh.OpenFile(sFilename, true)) + { + if (fh.GetSize() < 10) + return false; + if (0 == memcmp(gzMagic, fh.GetBuffer(), _countof(gzMagic))) + m_Dec.reset(new tGzDec); + else if (0 == memcmp(bz2Magic, fh.GetBuffer(), _countof(bz2Magic))) + m_Dec.reset(new tBzDec); + else if (0 == memcmp(xzMagic, fh.GetBuffer(), _countof(xzMagic))) + m_Dec.reset(new tXzDec); + else if (0 == memcmp(lzmaMagic, fh.GetBuffer(), _countof(lzmaMagic))) + m_Dec.reset(new tXzDec(true)); + + } } - else if(endsWithSzAr(sFilename, ".lzma") || endsWithSzAr(sFilename, ".xz")) - { -#ifdef HAVE_LZMA - memset(&m_xzStream, 0, sizeof(m_xzStream)); - if(LZMA_OK != lzma_stream_decoder (&m_xzStream, - EXTREME_MEMORY_SAVING ? 32000000 : MAX_VAL(uint64_t), - LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED)) - goto ofail; - flags |= FLAG_LZMA; + if (m_Dec.get()) + { + if(!m_Dec->inited) + { + //aclog::err("Unable to uncompress file, algo not available"); + return false; + } m_UncompBuf.init(BUFSIZE); -#else - goto ofail; -#endif } - else - flags |= FLAG_PLAIN; - - + struct stat statbuf; if(0!=fstat(m_fd, &statbuf)) - goto ofail; + return false; // LFS on 32bit? That's not good for mmap. Don't risk incorrect behaviour. if(uint64_t(statbuf.st_size) > MYSTD::numeric_limits::max()) { errno=EFBIG; - goto ofail; + return false; } if(statbuf.st_size>0) { m_szFileBuf = (char*) mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, m_fd, 0); if(m_szFileBuf==MAP_FAILED) - goto ofail; + return false; m_nBufSize = statbuf.st_size; } else @@ -151,21 +267,20 @@ m_nBufPos=0; m_nCurLine=0; + flags = 0; return true; - - ofail: - - flags |= (FLAG_ERROR|FLAG_EOF); - return false; } -bool filereader::CheckGoodState(bool bErrorsConsiderFatal) const +bool filereader::CheckGoodState(bool bErrorsConsiderFatal, cmstring *reportFilePath) const { if (flags&FLAG_ERROR) { if(bErrorsConsiderFatal) { - cerr << "Error opening file: "<< m_sOrigName << ", terminating."<=0) - { - fsync(m_fd); - forceclose(m_fd); - } - */ - checkforceclose(m_fd); - -#ifdef HAVE_LIBBZ2 - if(flags&FLAG_BZ2) - BZ2_bzDecompressEnd (&m_bzStream); -#endif -#ifdef HAVE_ZLIB - if(flags&FLAG_GZ) - deflateEnd(&m_zStream); -#endif + checkforceclose(m_fd); + m_Dec.reset(); -#ifdef HAVE_LZMA - if(flags&FLAG_LZMA) - lzma_end(&m_xzStream); -#endif - - flags=0; m_nBufSize=0; + + flags = (FLAG_ERROR|FLAG_EOF); // will be cleared in open method } filereader::~filereader() { @@ -232,7 +327,7 @@ maybe_got_more: - if(flags&FLAG_PLAIN) + if(!m_Dec.get()) { bCanRetry=false; @@ -292,7 +387,7 @@ if(bCanRetry) { bCanRetry=false; - _UncompressMoreData(); + UncompressMoreData(); goto maybe_got_more; } @@ -302,7 +397,7 @@ sOut.assign(rbuf, nLineLen); - if(flags&FLAG_PLAIN) + if(!m_Dec.get()) m_nBufPos+=nDropLen; else m_UncompBuf.drop(nDropLen); @@ -312,7 +407,7 @@ } //! @return: new text buffer size -inline void filereader::_UncompressMoreData() { +inline void filereader::UncompressMoreData() { // work with uncompressed buffer/window... m_UncompBuf.move(); // get unused space if possible @@ -328,83 +423,9 @@ return; } -#ifndef MINIBUILD - unsigned int nFeedLen=m_nBufSize-m_nBufPos; -#endif - -#ifdef HAVE_LIBBZ2 - if(flags&FLAG_BZ2) - { - m_bzStream.next_in=m_szFileBuf+m_nBufPos; - m_bzStream.avail_in=nFeedLen; - m_bzStream.next_out=m_UncompBuf.wptr(); - m_bzStream.avail_out=m_UncompBuf.freecapa(); + UINT wtf=m_Dec->UncompMore(m_szFileBuf, m_nBufSize, m_nBufPos, m_UncompBuf); + flags |= wtf; - int ret=BZ2_bzDecompress(&m_bzStream); - if(ret==BZ_STREAM_END) - { - // remember this later - flags|=FLAG_COMPEOF; - ret=BZ_OK; - } - if(ret==BZ_OK) - { - m_nBufPos += (nFeedLen-m_bzStream.avail_in); - unsigned int nGotBytes= m_UncompBuf.freecapa() - m_bzStream.avail_out; - m_UncompBuf.got(nGotBytes); - } - // or corrupted data? - else flags|=(FLAG_COMPEOF|FLAG_ERROR); - } -#endif -#ifdef HAVE_ZLIB - if(flags&FLAG_GZ) - { - m_zStream.next_in=(Bytef*) m_szFileBuf+m_nBufPos; - m_zStream.avail_in=nFeedLen; - m_zStream.next_out=(Bytef*) m_UncompBuf.wptr(); - m_zStream.avail_out=m_UncompBuf.freecapa(); - - int ret=inflate(&m_zStream, Z_NO_FLUSH); - if(ret==Z_STREAM_END) - { - flags|=FLAG_COMPEOF; - ret=Z_OK; - } - if(ret==Z_OK) - { //ok, accept the data - m_nBufPos += (nFeedLen-m_zStream.avail_in); - unsigned int nGotBytes= m_UncompBuf.freecapa() - m_zStream.avail_out; - m_UncompBuf.got(nGotBytes); - } - // or corrupted data? - else flags|=(FLAG_COMPEOF|FLAG_ERROR); - } -#endif -#ifdef HAVE_LZMA - if(flags&FLAG_LZMA) - { - m_xzStream.next_in=(const uint8_t*)m_szFileBuf+m_nBufPos; - m_xzStream.avail_in=nFeedLen; - m_xzStream.next_out=(uint8_t*)m_UncompBuf.wptr(); - m_xzStream.avail_out=m_UncompBuf.freecapa(); - - lzma_ret ret=lzma_code(&m_xzStream, LZMA_RUN); - if(ret==LZMA_STREAM_END) - { - flags|=FLAG_COMPEOF; - ret=LZMA_OK; - } - if(ret==LZMA_OK) - { //ok, accept the data - m_nBufPos += (nFeedLen-m_xzStream.avail_in); - unsigned int nGotBytes= m_UncompBuf.freecapa() - m_xzStream.avail_out; - m_UncompBuf.got(nGotBytes); - } - // or corrupted data? - else flags|=(FLAG_COMPEOF|FLAG_ERROR); - } -#endif } /* @@ -508,7 +529,7 @@ auto_ptr summer = csumBase::GetChecker(CSTYPES(csType)); scannedSize=0; - if (flags&FLAG_PLAIN) + if(!m_Dec.get()) { summer->add(m_szFileBuf, m_nBufSize); //sha_update(&ctx, (SHA_BYTE*) m_szFileBuf, m_nBufSize); @@ -520,7 +541,7 @@ { while (true) { - _UncompressMoreData(); + UncompressMoreData(); if (flags&FLAG_ERROR) return false; UINT nRest=m_UncompBuf.size(); @@ -558,7 +579,7 @@ ap = csumBase::GetChecker(CSTYPE_MD5); ap->add(testvec, sizeof(testvec) - 1); ap->finish(out); - if (CsBinToString(out, 16) != "900150983cd24fb0d6963f7d28e17f72") + if (BytesToHexString(out, 16) != "900150983cd24fb0d6963f7d28e17f72") { cerr << "Incorrect MD5 implementation detected, check compilation settings!\n"; exit(EXIT_FAILURE); @@ -611,7 +632,7 @@ return true; } -string CsBinToString(const uint8_t sum[], unsigned short lengthBin) +string BytesToHexString(const uint8_t sum[], unsigned short lengthBin) { string sRet; char hexmap[] = @@ -683,3 +704,17 @@ } #endif + +/* + * Playing (z|bz|xz)cat +int main(int argc, char **argv) +{ + filereader fh; + fh.OpenFile(argv[1]); + mstring sLine; + while(fh.GetOneLine(sLine)) + { + cout << sLine <frontLine << " and this: " << uintptr_t(this)); } @@ -485,6 +486,43 @@ return false; } +bool job::CreateRedirect(header & respHead) +{ + int st = respHead.getStatus(); + if (m_nRedirRemaining <= 0 || (st != 301 && st != 302) // redir limit reached or not redirect + || !respHead.h[header::LOCATION] || !respHead.h[header::LOCATION][0]) // or nonsense + return false; + + MYTRY + { + header *xheader = new header(*m_pReqHead); + // it's previous server's job to not send any crap. Just make sure that the format is right... + xheader->frontLine = string("GET "); + for (const char *p = respHead.h[header::LOCATION]; *p; ++p) + { + if (isspace((unsigned char) *p)) + { + xheader->frontLine += "%"; + xheader->frontLine += BytesToHexString((uint8_t*) p, 1); + } + else + xheader->frontLine += *p; + } + xheader->frontLine += " HTTP/1.1"; + job *pReplacement = m_pParentCon->AddNewJob(xheader); + if (!pReplacement) + return false; + pReplacement->m_nRedirRemaining = m_nRedirRemaining - 1; + } + MYCATCH(...) + { + return false; + } + return true; +} + + + void job::PrepareDownload() { LOGSTART("job::PrepareDownload"); @@ -833,6 +871,10 @@ return R_AGAIN; m_state=STATE_SEND_BUFFER; m_backstate=STATE_HEADER_SENT; // could be changed while creating header + + if(CreateRedirect(respHead)) + return R_DONE; + const char *szErr = BuildAndEnqueHeader(fistate, nGoodDataSize, respHead); if(szErr) THROW_ERROR(szErr); USRDBG(REPLEVEL, "Prepared response header for user: \n" << m_sendbuf ); @@ -1171,3 +1213,6 @@ //aclog::err(tSS() << "fileitem is now " << uintptr_t(m_pItem.get())); m_state=STATE_SEND_MAIN_HEAD; } + + + diff -Nru apt-cacher-ng-0.6.11/source/maintenance.cc apt-cacher-ng-0.6.12/source/maintenance.cc --- apt-cacher-ng-0.6.11/source/maintenance.cc 2011-10-16 19:41:51.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/maintenance.cc 2012-01-15 15:10:33.000000000 +0000 @@ -67,7 +67,7 @@ if(m_reportFD<3) { - write(m_reportFD, data, len); + ignore_value(::write(m_reportFD, data, len)); return; } diff -Nru apt-cacher-ng-0.6.11/source/meta.cc apt-cacher-ng-0.6.12/source/meta.cc --- apt-cacher-ng-0.6.11/source/meta.cc 2011-10-16 01:02:43.000000000 +0000 +++ apt-cacher-ng-0.6.12/source/meta.cc 2012-01-14 13:19:57.000000000 +0000 @@ -58,7 +58,7 @@ return ""; tStrPos end = in.find_last_not_of(CPATHSEP); // must be the last char of basename - if(end == stmiss) // just a slash? + if(end == stmiss) // empty, or just a slash? return "/"; tStrPos start = in.rfind(CPATHSEP, end); diff -Nru apt-cacher-ng-0.6.11/VERSION apt-cacher-ng-0.6.12/VERSION --- apt-cacher-ng-0.6.11/VERSION 2011-12-08 19:30:31.000000000 +0000 +++ apt-cacher-ng-0.6.12/VERSION 2012-01-07 00:13:37.000000000 +0000 @@ -1 +1 @@ -0.6.11 +0.6.12