diff -Nru zbackup-1.2/adler32.hh zbackup-1.4.1+20150403/adler32.hh --- zbackup-1.2/adler32.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/adler32.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef ADLER32_HH_INCLUDED__ -#define ADLER32_HH_INCLUDED__ +#ifndef ADLER32_HH_INCLUDED +#define ADLER32_HH_INCLUDED #include #include diff -Nru zbackup-1.2/appendallocator.cc zbackup-1.4.1+20150403/appendallocator.cc --- zbackup-1.2/appendallocator.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/appendallocator.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include diff -Nru zbackup-1.2/appendallocator.hh zbackup-1.4.1+20150403/appendallocator.hh --- zbackup-1.2/appendallocator.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/appendallocator.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef APPENDALLOCATOR_HH_INCLUDED__ -#define APPENDALLOCATOR_HH_INCLUDED__ +#ifndef APPENDALLOCATOR_HH_INCLUDED +#define APPENDALLOCATOR_HH_INCLUDED #include #include diff -Nru zbackup-1.2/backup_collector.cc zbackup-1.4.1+20150403/backup_collector.cc --- zbackup-1.2/backup_collector.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_collector.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,103 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include "backup_collector.hh" + +#include +#include + +#include "bundle.hh" +#include "chunk_index.hh" +#include "backup_restorer.hh" +#include "backup_file.hh" +#include "backup_exchanger.hh" + +#include "debug.hh" + +using std::string; + +void BundleCollector::startIndex( string const & indexFn ) +{ + indexModified = false; + indexTotalChunks = indexUsedChunks = 0; + indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0; +} + +void BundleCollector::finishIndex( string const & indexFn ) +{ + if ( indexModified ) + { + verbosePrintf( "Chunks: %d used / %d total, bundles: %d kept / %d modified / %d removed\n", + indexUsedChunks, indexTotalChunks, indexKeptBundles, indexModifiedBundles, indexRemovedBundles); + filesToUnlink.push_back( indexFn ); + } +} + +void BundleCollector::startBundle( Bundle::Id const & bundleId ) +{ + savedId = bundleId; + totalChunks = 0; + usedChunks = 0; +} + +void BundleCollector::processChunk( ChunkId const & chunkId ) +{ + totalChunks++; + if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() ) + { + usedChunks++; + } +} + +void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo const & info ) +{ + string i = Bundle::generateFileName( savedId, "", false ); + indexTotalChunks += totalChunks; + indexUsedChunks += usedChunks; + if ( usedChunks == 0 ) + { + if ( verbose ) + printf( "delete %s\n", i.c_str() ); + filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); + indexModified = true; + indexRemovedBundles++; + } + else if ( usedChunks < totalChunks ) + { + if ( verbose ) + printf( "%s: used %d/%d\n", i.c_str(), usedChunks, totalChunks ); + filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); + indexModified = true; + // Copy used chunks to the new index + string chunk; + size_t chunkSize; + for ( int x = info.chunk_record_size(); x--; ) + { + BundleInfo_ChunkRecord const & record = info.chunk_record( x ); + ChunkId id( record.id() ); + if ( usedChunkSet.find( id ) != usedChunkSet.end() ) + { + chunkStorageReader->get( id, chunk, chunkSize ); + chunkStorageWriter->add( id, chunk.data(), chunkSize ); + } + } + indexModifiedBundles++; + } + else + { + chunkStorageWriter->addBundle( info, savedId ); + if ( verbose ) + printf( "keep %s\n", i.c_str() ); + indexKeptBundles++; + } +} + +void BundleCollector::commit() +{ + for ( int i = filesToUnlink.size(); i--; ) + { + unlink( filesToUnlink[i].c_str() ); + } + filesToUnlink.clear(); + chunkStorageWriter->commit(); +} diff -Nru zbackup-1.2/backup_collector.hh zbackup-1.4.1+20150403/backup_collector.hh --- zbackup-1.2/backup_collector.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_collector.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,53 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef BACKUP_COLLECTOR_HH_INCLUDED +#define BACKUP_COLLECTOR_HH_INCLUDED + +#include "zbackup_base.hh" +#include "chunk_storage.hh" + +#include +#include +#include + +#include "bundle.hh" +#include "chunk_index.hh" +#include "backup_restorer.hh" +#include "backup_file.hh" +#include "backup_exchanger.hh" + +#include "debug.hh" + +using std::string; + +class BundleCollector: public IndexProcessor +{ +private: + Bundle::Id savedId; + int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks; + int indexModifiedBundles, indexKeptBundles, indexRemovedBundles; + bool indexModified; + vector< string > filesToUnlink; + +public: + string bundlesPath; + bool verbose; + ChunkStorage::Reader *chunkStorageReader; + ChunkStorage::Writer *chunkStorageWriter; + BackupRestorer::ChunkSet usedChunkSet; + + void startIndex( string const & indexFn ); + + void finishIndex( string const & indexFn ); + + void startBundle( Bundle::Id const & bundleId ); + + void processChunk( ChunkId const & chunkId ); + + void finishBundle( Bundle::Id const & bundleId, BundleInfo const & info ); + + void commit(); +}; + +#endif diff -Nru zbackup-1.2/backup_creator.cc zbackup-1.4.1+20150403/backup_creator.cc --- zbackup-1.2/backup_creator.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_creator.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include @@ -15,10 +15,10 @@ unsigned const MinChunkSize = 256; } -BackupCreator::BackupCreator( StorageInfo const & info, +BackupCreator::BackupCreator( Config const & config, ChunkIndex & chunkIndex, ChunkStorage::Writer & chunkStorageWriter ): - chunkMaxSize( info.chunk_max_size() ), + chunkMaxSize( config.GET_STORABLE( chunk, max_size ) ), chunkIndex( chunkIndex ), chunkStorageWriter( chunkStorageWriter ), ringBufferFill( 0 ), chunkToSaveFill( 0 ), diff -Nru zbackup-1.2/backup_creator.hh zbackup-1.4.1+20150403/backup_creator.hh --- zbackup-1.2/backup_creator.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_creator.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef BACKUP_CREATOR_HH_INCLUDED__ -#define BACKUP_CREATOR_HH_INCLUDED__ +#ifndef BACKUP_CREATOR_HH_INCLUDED +#define BACKUP_CREATOR_HH_INCLUDED #include #include @@ -17,6 +17,7 @@ #include "rolling_hash.hh" #include "sptr.hh" #include "zbackup.pb.h" +#include "config.hh" using std::vector; using std::string; @@ -69,7 +70,7 @@ virtual ChunkId const & getChunkId(); public: - BackupCreator( StorageInfo const &, ChunkIndex &, ChunkStorage::Writer & ); + BackupCreator( Config const &, ChunkIndex &, ChunkStorage::Writer & ); /// The data is fed the following way: the user fills getInputBuffer() with /// up to getInputBufferSize() bytes, then calls handleMoreData() with the diff -Nru zbackup-1.2/backup_exchanger.cc zbackup-1.4.1+20150403/backup_exchanger.cc --- zbackup-1.2/backup_exchanger.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_exchanger.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,49 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include "backup_exchanger.hh" +#include "dir.hh" +#include "debug.hh" + +namespace BackupExchanger { + +vector< string > findOrRebuild( string const & src, string const & dst, string const & relativePath ) +{ + vector< string > files; + + Dir::Listing lst ( Dir::addPath( src, relativePath ) ); + + Dir::Entry entry; + + while ( lst.getNext( entry ) ) + { + string currentRelativePath ( relativePath ); + if ( currentRelativePath.empty() ) + currentRelativePath.assign( entry.getFileName() ); + else + currentRelativePath.assign( Dir::addPath( relativePath, entry.getFileName() ) ); + + if ( entry.isDir() ) + { + verbosePrintf( "Found directory %s...\n", currentRelativePath.c_str() ); + string srcFullPath( Dir::addPath( src, currentRelativePath ) ); + string dstFullPath( Dir::addPath( dst, currentRelativePath ) ); + if ( !dst.empty() && !Dir::exists( dstFullPath.c_str() ) ) + { + verbosePrintf( "Directory %s not found in destination, creating...\n", + currentRelativePath.c_str() ); + Dir::create( dstFullPath.c_str() ); + } + vector< string > subFiles( findOrRebuild( src, dst, currentRelativePath ) ); + files.insert( files.end(), subFiles.begin(), subFiles.end() ); + } + else + { + verbosePrintf( "Found file %s...\n", currentRelativePath.c_str() ); + files.push_back( currentRelativePath ); + } + } + + return files; +} +} diff -Nru zbackup-1.2/backup_exchanger.hh zbackup-1.4.1+20150403/backup_exchanger.hh --- zbackup-1.2/backup_exchanger.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_exchanger.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,31 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef BACKUP_EXCHANGER_HH_INCLUDED +#define BACKUP_EXCHANGER_HH_INCLUDED + +#include +#include +#include "tmp_mgr.hh" + +namespace BackupExchanger { + +using std::string; +using std::vector; +using std::pair; + +enum { + backups, + bundles, + index, + Flags +}; + +/// Recreate source directory structure in destination +vector< string > findOrRebuild( string const & src, + string const & dst = std::string(), + string const & relativePath = std::string() ); +typedef pair< sptr< TemporaryFile >, string > PendingExchangeRename; +} + +#endif diff -Nru zbackup-1.2/backup_file.cc zbackup-1.4.1+20150403/backup_file.cc --- zbackup-1.2/backup_file.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_file.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "backup_file.hh" diff -Nru zbackup-1.2/backup_file.hh zbackup-1.4.1+20150403/backup_file.hh --- zbackup-1.2/backup_file.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_file.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef BACKUP_FILE_HH_INCLUDED__ -#define BACKUP_FILE_HH_INCLUDED__ +#ifndef BACKUP_FILE_HH_INCLUDED +#define BACKUP_FILE_HH_INCLUDED #include #include diff -Nru zbackup-1.2/backup_restorer.cc zbackup-1.4.1+20150403/backup_restorer.cc --- zbackup-1.2/backup_restorer.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_restorer.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include @@ -17,7 +17,7 @@ void restore( ChunkStorage::Reader & chunkStorageReader, std::string const & backupData, - DataSink & output ) + DataSink * output, ChunkSet * chunkSet ) { google::protobuf::io::ArrayInputStream is( backupData.data(), backupData.size() ); @@ -39,21 +39,58 @@ if ( instr.has_chunk_to_emit() ) { - // Need to emit a chunk, reading it from the store - size_t chunkSize; - chunkStorageReader.get( ChunkId( instr.chunk_to_emit() ), chunk, - chunkSize ); - output.saveData( chunk.data(), chunkSize ); + ChunkId id( instr.chunk_to_emit() ); + if ( output ) + { + // Need to emit a chunk, reading it from the store + size_t chunkSize; + chunkStorageReader.get( id, chunk, chunkSize ); + output->saveData( chunk.data(), chunkSize ); + } + if ( chunkSet ) + { + chunkSet->insert( id ); + } } - if ( instr.has_bytes_to_emit() ) + if ( output && instr.has_bytes_to_emit() ) { // Need to emit the bytes directly string const & bytes = instr.bytes_to_emit(); - output.saveData( bytes.data(), bytes.size() ); + output->saveData( bytes.data(), bytes.size() ); } } cis.PopLimit( limit ); } + +void restoreIterations( ChunkStorage::Reader & chunkStorageReader, + BackupInfo & backupInfo, std::string & backupData, ChunkSet * chunkSet ) +{ + // Perform the iterations needed to get to the actual user backup data + for ( ; ; ) + { + backupData.swap( *backupInfo.mutable_backup_data() ); + + if ( backupInfo.iterations() ) + { + struct StringWriter: public DataSink + { + string result; + + virtual void saveData( void const * data, size_t size ) + { + result.append( ( char const * ) data, size ); + } + } stringWriter; + + restore( chunkStorageReader, backupData, &stringWriter, chunkSet ); + backupInfo.mutable_backup_data()->swap( stringWriter.result ); + backupInfo.set_iterations( backupInfo.iterations() - 1 ); + } + else + break; + } +} + } diff -Nru zbackup-1.2/backup_restorer.hh zbackup-1.4.1+20150403/backup_restorer.hh --- zbackup-1.2/backup_restorer.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/backup_restorer.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,12 +1,13 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef BACKUP_RESTORER_HH_INCLUDED__ -#define BACKUP_RESTORER_HH_INCLUDED__ +#ifndef BACKUP_RESTORER_HH_INCLUDED +#define BACKUP_RESTORER_HH_INCLUDED #include #include #include +#include #include "chunk_storage.hh" #include "ex.hh" @@ -25,9 +26,14 @@ DEF_EX( Ex, "Backup restorer exception", std::exception ) DEF_EX( exTooManyBytesToEmit, "A backup record asks to emit too many bytes", Ex ) +typedef std::set< ChunkId > ChunkSet; + /// Restores the given backup void restore( ChunkStorage::Reader &, std::string const & backupData, - DataSink & ); + DataSink *, ChunkSet * ); + +/// Performs restore iterations on backupData +void restoreIterations( ChunkStorage::Reader &, BackupInfo &, std::string &, ChunkSet * ); } #endif diff -Nru zbackup-1.2/bundle.cc zbackup-1.4.1+20150403/bundle.cc --- zbackup-1.2/bundle.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/bundle.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,22 +1,30 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#include #include #include "bundle.hh" #include "check.hh" #include "dir.hh" -#include "encrypted_file.hh" #include "encryption.hh" #include "hex.hh" #include "message.hh" +#include "adler32.hh" +#include "compression.hh" namespace Bundle { enum { - FileFormatVersion = 1 + FileFormatVersion = 1, + + // This means, we don't use LZMA in this file. + FileFormatVersionNotLZMA, + + // <- add more versions here + + // This is the first version, we do not support. + FileFormatVersionFirstUnsupported }; void Creator::addChunk( string const & id, void const * data, size_t size ) @@ -27,14 +35,81 @@ payload.append( ( char const * ) data, size ); } -void Creator::write( std::string const & fileName, EncryptionKey const & key ) +void Creator::write( std::string const & fileName, EncryptionKey const & key, + Reader & reader ) +{ + EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv ); + + os.writeRandomIv(); + + Message::serialize( reader.getBundleHeader(), os ); + + Message::serialize( reader.getBundleInfo(), os ); + os.writeAdler32(); + + void * bufPrev = NULL; + const void * bufCurr = NULL; + int sizePrev = 0, sizeCurr = 0; + bool readPrev = false, readCurr = false; + + for ( ; ; ) + { + bool readCurr = reader.is.Next( &bufCurr, &sizeCurr ); + + if ( readCurr ) + { + if ( readPrev ) + { + os.write( bufPrev, sizePrev ); + + readPrev = readCurr; + free( bufPrev ); + bufPrev = malloc( sizeCurr ); + memcpy( bufPrev, bufCurr, sizeCurr ); + sizePrev = sizeCurr; + } + else + { + readPrev = readCurr; + bufPrev = malloc( sizeCurr ); + memcpy( bufPrev, bufCurr, sizeCurr ); + sizePrev = sizeCurr; + } + } + else + { + if ( readPrev ) + { + sizePrev -= sizeof( Adler32::Value ); + os.write( bufPrev, sizePrev ); + os.writeAdler32(); + free ( bufPrev ); + break; + } + } + } +} + +void Creator::write( Config const & config, std::string const & fileName, + EncryptionKey const & key ) { EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv ); os.writeRandomIv(); - FileHeader header; - header.set_version( FileFormatVersion ); + BundleFileHeader header; + + const_sptr compression = + Compression::CompressionMethod::selectedCompression; + header.set_compression_method( compression->getName() ); + + // The old code only support lzma, so we will bump up the version, if we're + // using lzma. This will make it fail cleanly. + if ( compression->getName() == "lzma" ) + header.set_version( FileFormatVersion ); + else + header.set_version( FileFormatVersionNotLZMA ); + Message::serialize( header, os ); Message::serialize( info, os ); @@ -42,16 +117,10 @@ // Compress - uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be - // the best option - lzma_stream strm = LZMA_STREAM_INIT; - lzma_ret ret; - - ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 ); - CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret ); + sptr encoder = compression->createEncoder( + config ); - strm.next_in = ( uint8_t const * ) payload.data(); - strm.avail_in = payload.size(); + encoder->setInput( payload.data(), payload.size() ); for ( ; ; ) { @@ -60,46 +129,38 @@ int size; if ( !os.Next( &data, &size ) ) { - lzma_end( &strm ); + encoder.reset(); throw exBundleWriteFailed(); } if ( !size ) continue; - strm.next_out = ( uint8_t * ) data; - strm.avail_out = size; + encoder->setOutput( data, size ); } // Perform the compression - ret = lzma_code( &strm, LZMA_FINISH ); - - if ( ret == LZMA_STREAM_END ) + if ( encoder->process( true ) ) { - if ( strm.avail_out ) - os.BackUp( strm.avail_out ); + if ( encoder->getAvailableOutput() ) + os.BackUp( encoder->getAvailableOutput() ); break; } - - CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret ); } - lzma_end( &strm ); + encoder.reset(); os.writeAdler32(); } -Reader::Reader( string const & fileName, EncryptionKey const & key ) +Reader::Reader( string const & fileName, EncryptionKey const & key, bool prohibitProcessing ): + is( fileName.c_str(), key, Encryption::ZeroIv ) { - EncryptedFile::InputStream is( fileName.c_str(), key, Encryption::ZeroIv ); - is.consumeRandomIv(); - FileHeader header; Message::parse( header, is ); - if ( header.version() != FileFormatVersion ) + if ( header.version() >= FileFormatVersionFirstUnsupported ) throw exUnsupportedVersion(); - BundleInfo info; Message::parse( info, is ); is.checkAdler32(); @@ -109,15 +170,13 @@ payload.resize( payloadSize ); - lzma_stream strm = LZMA_STREAM_INIT; - - lzma_ret ret; + if ( prohibitProcessing ) + return; - ret = lzma_stream_decoder( &strm, UINT64_MAX, 0 ); - CHECK( ret == LZMA_OK,"lzma_stream_decoder error: %d", (int) ret ); + sptr decoder = Compression::CompressionMethod::findCompression( + header.compression_method() )->createDecoder(); - strm.next_out = ( uint8_t * ) &payload[ 0 ]; - strm.avail_out = payload.size(); + decoder->setOutput( &payload[ 0 ], payload.size() ); for ( ; ; ) { @@ -126,35 +185,30 @@ int size; if ( !is.Next( &data, &size ) ) { - lzma_end( &strm ); + decoder.reset(); throw exBundleReadFailed(); } if ( !size ) continue; - strm.next_in = ( uint8_t const * ) data; - strm.avail_in = size; + decoder->setInput( data, size ); } - ret = lzma_code( &strm, LZMA_RUN ); - - if ( ret == LZMA_STREAM_END ) + if ( decoder->process( false ) ) { - if ( strm.avail_in ) - is.BackUp( strm.avail_in ); + if ( decoder->getAvailableInput() ) + is.BackUp( decoder->getAvailableInput() ); break; } - CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret ); - - if ( !strm.avail_out && strm.avail_in ) + if ( !decoder->getAvailableOutput() && decoder->getAvailableInput() ) { // Apparently we have more data than we were expecting - lzma_end( &strm ); + decoder.reset(); throw exTooMuchData(); } } - lzma_end( &strm ); + decoder.reset(); is.checkAdler32(); diff -Nru zbackup-1.2/bundle.hh zbackup-1.4.1+20150403/bundle.hh --- zbackup-1.2/bundle.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/bundle.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef BUNDLE_HH_INCLUDED__ -#define BUNDLE_HH_INCLUDED__ +#ifndef BUNDLE_HH_INCLUDED +#define BUNDLE_HH_INCLUDED #include #include @@ -16,6 +16,8 @@ #include "nocopy.hh" #include "static_assert.hh" #include "zbackup.pb.h" +#include "encrypted_file.hh" +#include "config.hh" namespace Bundle { @@ -44,6 +46,41 @@ STATIC_ASSERT( sizeof( Id ) == IdSize ); +/// Reads the bundle and allows accessing chunks +class Reader: NoCopy +{ + BundleInfo info; + BundleFileHeader header; + /// Unpacked payload + string payload; + /// Maps chunk id blob to its contents and size + typedef map< string, pair< char const *, size_t > > Chunks; + Chunks chunks; + +public: + DEF_EX( Ex, "Bundle reader exception", std::exception ) + DEF_EX( exBundleReadFailed, "Bundle read failed", Ex ) + DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex ) + DEF_EX( exTooMuchData, "More data than expected in a bundle", Ex ) + DEF_EX( exDuplicateChunks, "Chunks with the same id found in a bundle", Ex ) + + Reader( string const & fileName, EncryptionKey const & key, + bool prohibitProcessing = false ); + + /// Reads the chunk into chunkData and returns true, or returns false if there + /// was no such chunk in the bundle. chunkData may be enlarged but won't + /// be shrunk. The size of the actual chunk would be stored in chunkDataSize + bool get( string const & chunkId, string & chunkData, size_t & chunkDataSize ); + BundleInfo getBundleInfo() + { return info; } + BundleFileHeader getBundleHeader() + { return header; } + string getPayload() + { return payload; } + + EncryptedFile::InputStream is; +}; + /// Creates a bundle by adding chunks to it until it's full, then compressing /// it and writing out to disk class Creator @@ -65,37 +102,15 @@ /// Compresses and writes the bundle to the given file. The operation is /// time-consuming - calling this function from a worker thread could be /// warranted - void write( string const & fileName, EncryptionKey const & ); + void write( Config const &, string const & fileName, EncryptionKey const & ); + void write( string const & fileName, EncryptionKey const &, + Bundle::Reader & reader ); /// Returns the current BundleInfo record - this is used for index files BundleInfo const & getCurrentBundleInfo() const { return info; } }; -/// Reads the bundle and allows accessing chunks -class Reader: NoCopy -{ - /// Unpacked payload - string payload; - /// Maps chunk id blob to its contents and size - typedef map< string, pair< char const *, size_t > > Chunks; - Chunks chunks; - -public: - DEF_EX( Ex, "Bundle reader exception", std::exception ) - DEF_EX( exBundleReadFailed, "Bundle read failed", Ex ) - DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex ) - DEF_EX( exTooMuchData, "More data than expected in a bundle", Ex ) - DEF_EX( exDuplicateChunks, "Chunks with the same id found in a bundle", Ex ) - - Reader( string const & fileName, EncryptionKey const & ); - - /// Reads the chunk into chunkData and returns true, or returns false if there - /// was no such chunk in the bundle. chunkData may be enlarged but won't - /// be shrunk. The size of the actual chunk would be stored in chunkDataSize - bool get( string const & chunkId, string & chunkData, size_t & chunkDataSize ); -}; - /// Generates a full file name for a bundle with the given id. If createDirs /// is true, any intermediate directories will be created if they don't exist /// already diff -Nru zbackup-1.2/check.hh zbackup-1.4.1+20150403/check.hh --- zbackup-1.2/check.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/check.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef CHECK_HH_INCLUDED__ -#define CHECK_HH_INCLUDED__ +#ifndef CHECK_HH_INCLUDED +#define CHECK_HH_INCLUDED #include #include diff -Nru zbackup-1.2/chunk_id.cc zbackup-1.4.1+20150403/chunk_id.cc --- zbackup-1.2/chunk_id.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/chunk_id.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "chunk_id.hh" @@ -38,9 +38,17 @@ rollingHash = fromLittleEndian( v ); } +bool operator <( const ChunkId &lhs, const ChunkId &rhs ) +{ + int r = memcmp( &lhs.cryptoHash, &rhs.cryptoHash, sizeof( lhs.cryptoHash ) ); + if ( r != 0 ) + return r < 0; + return memcmp( &lhs.rollingHash, &rhs.rollingHash, sizeof( lhs.rollingHash ) ) < 0; +} + ChunkId::ChunkId( string const & blob ) { - CHECK( blob.size() == BlobSize, "incorrect blob sise: %zu", blob.size() ); + CHECK( blob.size() == BlobSize, "incorrect blob size: %zu", blob.size() ); setFromBlob( blob.data() ); } diff -Nru zbackup-1.2/chunk_id.hh zbackup-1.4.1+20150403/chunk_id.hh --- zbackup-1.2/chunk_id.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/chunk_id.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef CHUNK_ID_HH_INCLUDED__ -#define CHUNK_ID_HH_INCLUDED__ +#ifndef CHUNK_ID_HH_INCLUDED +#define CHUNK_ID_HH_INCLUDED #include #include "rolling_hash.hh" @@ -35,4 +35,6 @@ ChunkId( string const & blob ); }; +bool operator <( const ChunkId &lhs, const ChunkId &rhs ); + #endif diff -Nru zbackup-1.2/chunk_index.cc zbackup-1.4.1+20150403/chunk_index.cc --- zbackup-1.2/chunk_index.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/chunk_index.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include @@ -20,10 +20,10 @@ bool ChunkIndex::Chain::equalsTo( ChunkId const & id ) { - return memcmp( cryptoHash, id.cryptoHash, sizeof ( cryptoHash ) ) == 0; + return memcmp( cryptoHash, id.cryptoHash, sizeof( cryptoHash ) ) == 0; } -void ChunkIndex::loadIndex() +void ChunkIndex::loadIndex( IndexProcessor & ip ) { Dir::Listing lst( indexPath ); @@ -34,43 +34,81 @@ while( lst.getNext( entry ) ) { verbosePrintf( "Loading index file %s...\n", entry.getFileName().c_str() ); + try + { + string indexFn = Dir::addPath( indexPath, entry.getFileName() ); + IndexFile::Reader reader( key, indexFn ); - IndexFile::Reader reader( key, - Dir::addPath( indexPath, entry.getFileName() ) ); + ip.startIndex( indexFn ); - BundleInfo info; - Bundle::Id bundleId; - while( reader.readNextRecord( info, bundleId ) ) - { - Bundle::Id * savedId = storage.allocateObjects< Bundle::Id >( 1 ); - memcpy( savedId, &bundleId, sizeof( bundleId ) ); + BundleInfo info; + Bundle::Id bundleId; + while( reader.readNextRecord( info, bundleId ) ) + { + Bundle::Id * savedId = storage.allocateObjects< Bundle::Id >( 1 ); + memcpy( savedId, &bundleId, sizeof( bundleId ) ); - lastBundleId = savedId; + ChunkId id; - ChunkId id; + ip.startBundle( *savedId ); - for ( int x = info.chunk_record_size(); x--; ) - { - BundleInfo_ChunkRecord const & record = info.chunk_record( x ); + for ( int x = info.chunk_record_size(); x--; ) + { + BundleInfo_ChunkRecord const & record = info.chunk_record( x ); + + if ( record.id().size() != ChunkId::BlobSize ) + throw exIncorrectChunkIdSize(); - if ( record.id().size() != ChunkId::BlobSize ) - throw exIncorrectChunkIdSize(); + id.setFromBlob( record.id().data() ); + ip.processChunk( id ); + } - id.setFromBlob( record.id().data() ); - registerNewChunkId( id, savedId ); + ip.finishBundle( *savedId, info ); } + + ip.finishIndex( indexFn ); + } + catch( std::exception & e ) + { + verbosePrintf( "error: %s\n", e.what() ); + continue; } } verbosePrintf( "Index loaded.\n" ); } +void ChunkIndex::startIndex( string const & ) +{ +} + +void ChunkIndex::startBundle( Bundle::Id const & bundleId ) +{ + lastBundleId = &bundleId; +} + +void ChunkIndex::processChunk( ChunkId const & chunkId ) +{ + registerNewChunkId( chunkId, lastBundleId ); +} + +void ChunkIndex::finishBundle( Bundle::Id const &, BundleInfo const & ) +{ +} + +void ChunkIndex::finishIndex( string const & ) +{ +} + ChunkIndex::ChunkIndex( EncryptionKey const & key, TmpMgr & tmpMgr, - string const & indexPath ): + string const & indexPath, bool prohibitChunkIndexLoading ): key( key ), tmpMgr( tmpMgr ), indexPath( indexPath ), storage( 65536, 1 ), lastBundleId( NULL ) { - loadIndex(); + if ( !prohibitChunkIndexLoading ) + loadIndex( *this ); + dPrintf( "%s for %s is instantiated and initialized, hasKey: %s\n", + __CLASS, indexPath.c_str(), key.hasKey() ? "true" : "false" ); } Bundle::Id const * ChunkIndex::findChunk( ChunkId::RollingHashPart rollingHash, diff -Nru zbackup-1.2/chunk_index.hh zbackup-1.4.1+20150403/chunk_index.hh --- zbackup-1.2/chunk_index.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/chunk_index.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef CHUNK_INDEX_HH_INCLUDED__ -#define CHUNK_INDEX_HH_INCLUDED__ +#ifndef CHUNK_INDEX_HH_INCLUDED +#define CHUNK_INDEX_HH_INCLUDED // is obsolete, but requires C++11. Make up your // mind, GNU people! @@ -44,9 +44,19 @@ }; } +class IndexProcessor +{ +public: + virtual void startIndex( string const & ) = 0; + virtual void startBundle( Bundle::Id const & ) = 0; + virtual void processChunk( ChunkId const & ) = 0; + virtual void finishBundle( Bundle::Id const &, BundleInfo const & ) = 0; + virtual void finishIndex( string const & ) = 0; +}; + /// Maintains an in-memory hash table allowing to check whether we have a /// specific chunk or not, and if we do, get the bundle id it's in -class ChunkIndex: NoCopy +class ChunkIndex: NoCopy, IndexProcessor { struct Chain { @@ -77,7 +87,7 @@ DEF_EX( Ex, "Chunk index exception", std::exception ) DEF_EX( exIncorrectChunkIdSize, "Incorrect chunk id size encountered", Ex ) - ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath ); + ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool prohibitChunkIndexLoading ); struct ChunkInfoInterface { @@ -99,9 +109,15 @@ /// if added, false if existed already bool addChunk( ChunkId const &, Bundle::Id const & ); -private: - void loadIndex(); + void startIndex( string const & ); + void startBundle( Bundle::Id const & ); + void processChunk( ChunkId const & ); + void finishBundle( Bundle::Id const &, BundleInfo const & ); + void finishIndex( string const & ); + + void loadIndex( IndexProcessor & ); +private: /// Inserts new chunk id into the in-memory hash table. Returns the created /// Chain if it was inserted, NULL if it existed before Chain * registerNewChunkId( ChunkId const & id, Bundle::Id const * ); diff -Nru zbackup-1.2/chunk_storage.cc zbackup-1.4.1+20150403/chunk_storage.cc --- zbackup-1.2/chunk_storage.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/chunk_storage.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "check.hh" #include "chunk_storage.hh" @@ -10,11 +10,11 @@ namespace ChunkStorage { -Writer::Writer( StorageInfo const & storageInfo, +Writer::Writer( Config const & configIn, EncryptionKey const & encryptionKey, TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir, string const & indexDir, size_t maxCompressorsToRun ): - storageInfo( storageInfo ), encryptionKey( encryptionKey ), + config( configIn ), encryptionKey( encryptionKey ), tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ), indexDir( indexDir ), hasCurrentBundleId( false ), maxCompressorsToRun( maxCompressorsToRun ), runningCompressors( 0 ) @@ -34,7 +34,7 @@ { // Added to the index? Emit to the bundle then if ( getCurrentBundle().getPayloadSize() + size > - storageInfo.bundle_max_payload_size() ) + config.GET_STORABLE( bundle, max_payload_size ) ) finishCurrentBundle(); getCurrentBundle().addChunk( id.toBlob(), data, size ); @@ -45,6 +45,19 @@ return false; } +void Writer::addBundle( BundleInfo const & bundleInfo, Bundle::Id const & bundleId ) +{ + if ( !indexFile.get() ) + { + // Create a new index file + indexTempFile = tmpMgr.makeTemporaryFile(); + indexFile = new IndexFile::Writer( encryptionKey, + indexTempFile->getFileName() ); + } + + indexFile->add( bundleInfo, bundleId ); +} + void Writer::commit() { finishCurrentBundle(); @@ -68,7 +81,7 @@ // Generate a random filename unsigned char buf[ 24 ]; // Same comments as for Bundle::IdSize - Random::genaratePseudo( buf, sizeof( buf ) ); + Random::generatePseudo( buf, sizeof( buf ) ); indexTempFile->moveOverTo( Dir::addPath( indexDir, toHex( buf, sizeof( buf ) ) ) ); @@ -76,6 +89,20 @@ } } +void Writer::reset() +{ + finishCurrentBundle(); + + waitForAllCompressorsToFinish(); + + pendingBundleRenames.clear(); + + if ( indexFile.get() ) + { + indexFile.reset(); + } +} + Bundle::Creator & Writer::getCurrentBundle() { if ( !currentBundle.get() ) @@ -90,15 +117,7 @@ Bundle::Id const & bundleId = getCurrentBundleId(); - if ( !indexFile.get() ) - { - // Create a new index file - indexTempFile = tmpMgr.makeTemporaryFile(); - indexFile = new IndexFile::Writer( encryptionKey, - indexTempFile->getFileName() ); - } - - indexFile->add( currentBundle->getCurrentBundleInfo(), bundleId ); + addBundle( currentBundle->getCurrentBundleInfo(), bundleId ); sptr< TemporaryFile > file = tmpMgr.makeTemporaryFile(); @@ -111,7 +130,8 @@ while ( runningCompressors >= maxCompressorsToRun ) runningCompressorsCondition.wait( runningCompressorsMutex ); - Compressor * compressor = new Compressor( *this, currentBundle, + Compressor * compressor = new Compressor( config, + *this, currentBundle, file->getFileName() ); currentBundle.reset(); @@ -133,17 +153,18 @@ if ( !hasCurrentBundleId ) { // Generate a new one - Random::genaratePseudo( ¤tBundleId, sizeof( currentBundleId ) ); + Random::generatePseudo( ¤tBundleId, sizeof( currentBundleId ) ); hasCurrentBundleId = true; } return currentBundleId; } -Writer::Compressor::Compressor( Writer & writer, +Writer::Compressor::Compressor( Config const & configIn, Writer & writer, sptr< Bundle::Creator > const & bundleCreator, string const & fileName ): - writer( writer ), bundleCreator( bundleCreator ), fileName( fileName ) + writer( writer ), bundleCreator( bundleCreator ), fileName( fileName ), + config( configIn ) { } @@ -151,11 +172,11 @@ { try { - bundleCreator->write( fileName, writer.encryptionKey ); + bundleCreator->write( config, fileName, writer.encryptionKey ); } catch( std::exception & e ) { - FAIL( "Bunding writing failed: %s", e.what() ); + FAIL( "Bundle writing failed: %s", e.what() ); } { @@ -173,17 +194,18 @@ return NULL; } -Reader::Reader( StorageInfo const & storageInfo, +Reader::Reader( Config const & configIn, EncryptionKey const & encryptionKey, ChunkIndex & index, string const & bundlesDir, size_t maxCacheSizeBytes ): - storageInfo( storageInfo ), encryptionKey( encryptionKey ), + config( configIn ), encryptionKey( encryptionKey ), index( index ), bundlesDir( bundlesDir ), // We need to have at least one cached reader, otherwise we would have to // unpack a bundle each time a chunk is read, even for consecutive chunks // in the same bundle - cachedReaders( maxCacheSizeBytes < storageInfo.bundle_max_payload_size() ? - 1 : maxCacheSizeBytes / storageInfo.bundle_max_payload_size() ) + cachedReaders( + maxCacheSizeBytes < config.GET_STORABLE( bundle, max_payload_size ) ? + 1 : maxCacheSizeBytes / config.GET_STORABLE( bundle, max_payload_size ) ) { verbosePrintf( "Using up to %zu MB of RAM as cache\n", maxCacheSizeBytes / 1048576 ); diff -Nru zbackup-1.2/chunk_storage.hh zbackup-1.4.1+20150403/chunk_storage.hh --- zbackup-1.2/chunk_storage.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/chunk_storage.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef CHUNK_STORAGE_HH_INCLUDED__ -#define CHUNK_STORAGE_HH_INCLUDED__ +#ifndef CHUNK_STORAGE_HH_INCLUDED +#define CHUNK_STORAGE_HH_INCLUDED #include #include @@ -23,6 +23,7 @@ #include "sptr.hh" #include "tmp_mgr.hh" #include "zbackup.pb.h" +#include "config.hh" namespace ChunkStorage { @@ -40,7 +41,7 @@ /// All new bundles and index files are created as temp files. Call commit() /// to move them to their permanent locations. commit() is never called /// automatically! - Writer( StorageInfo const &, EncryptionKey const &, + Writer( Config const &, EncryptionKey const &, TmpMgr &, ChunkIndex & index, string const & bundlesDir, string const & indexDir, size_t maxCompressorsToRun ); @@ -48,10 +49,16 @@ /// in the index, does nothing and returns false bool add( ChunkId const &, void const * data, size_t size ); + /// Adds an existing bundle to the index + void addBundle( BundleInfo const &, Bundle::Id const & bundleId ); + /// Commits all newly created bundles. Must be called before destroying the /// object -- otherwise all work will be removed from the temp dir and lost void commit(); + /// Throw away all current changes. + void reset(); + ~Writer(); private: @@ -61,8 +68,9 @@ Writer & writer; sptr< Bundle::Creator > bundleCreator; string fileName; + Config const & config; public: - Compressor( Writer &, sptr< Bundle::Creator > const &, + Compressor( Config const &, Writer &, sptr< Bundle::Creator > const &, string const & fileName ); protected: virtual void * threadFunction() throw(); @@ -84,7 +92,7 @@ /// Wait for all compressors to finish void waitForAllCompressorsToFinish(); - StorageInfo const & storageInfo; + Config const & config; EncryptionKey const & encryptionKey; TmpMgr & tmpMgr; ChunkIndex & index; @@ -113,7 +121,7 @@ public: DEF_EX_STR( exNoSuchChunk, "no such chunk found:", Ex ) - Reader( StorageInfo const &, EncryptionKey const &, ChunkIndex & index, + Reader( Config const &, EncryptionKey const &, ChunkIndex & index, string const & bundlesDir, size_t maxCacheSizeBytes ); /// Loads the given chunk from the store into the given buffer. May throw file @@ -125,7 +133,7 @@ Bundle::Reader & getReaderFor( Bundle::Id const & ); private: - StorageInfo const & storageInfo; + Config const & config; EncryptionKey const & encryptionKey; ChunkIndex & index; string bundlesDir; diff -Nru zbackup-1.2/cmake/FindLibLZMA.cmake zbackup-1.4.1+20150403/cmake/FindLibLZMA.cmake --- zbackup-1.2/cmake/FindLibLZMA.cmake 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/cmake/FindLibLZMA.cmake 2015-04-03 09:33:05.000000000 +0000 @@ -1,16 +1,23 @@ -# - Find LibLZMA +#.rst: +# FindLibLZMA +# ----------- +# +# Find LibLZMA +# # Find LibLZMA headers and library # -# LIBLZMA_FOUND - True if liblzma is found. -# LIBLZMA_INCLUDE_DIRS - Directory where liblzma headers are located. -# LIBLZMA_LIBRARIES - Lzma libraries to link against. -# LIBLZMA_HAS_AUTO_DECODER - True if lzma_auto_decoder() is found (required). -# LIBLZMA_HAS_EASY_ENCODER - True if lzma_easy_encoder() is found (required). -# LIBLZMA_HAS_LZMA_PRESET - True if lzma_lzma_preset() is found (required). -# LIBLZMA_VERSION_MAJOR - The major version of lzma -# LIBLZMA_VERSION_MINOR - The minor version of lzma -# LIBLZMA_VERSION_PATCH - The patch version of lzma -# LIBLZMA_VERSION_STRING - version number as a string (ex: "5.0.3") +# :: +# +# LIBLZMA_FOUND - True if liblzma is found. +# LIBLZMA_INCLUDE_DIRS - Directory where liblzma headers are located. +# LIBLZMA_LIBRARIES - Lzma libraries to link against. +# LIBLZMA_HAS_AUTO_DECODER - True if lzma_auto_decoder() is found (required). +# LIBLZMA_HAS_EASY_ENCODER - True if lzma_easy_encoder() is found (required). +# LIBLZMA_HAS_LZMA_PRESET - True if lzma_lzma_preset() is found (required). +# LIBLZMA_VERSION_MAJOR - The major version of lzma +# LIBLZMA_VERSION_MINOR - The minor version of lzma +# LIBLZMA_VERSION_PATCH - The patch version of lzma +# LIBLZMA_VERSION_STRING - version number as a string (ex: "5.0.3") #============================================================================= # Copyright 2008 Per Øyvind Karlsen @@ -18,65 +25,15 @@ # Copyright 2009 Helio Chissini de Castro # Copyright 2012 Mario Bensi # -# Distributed under the OSI-approved BSD License (the "License"): -# -# CMake - Cross Platform Makefile Generator -# Copyright 2000-2011 Kitware, Inc., Insight Software Consortium -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the names of Kitware, Inc., the Insight Software Consortium, -# nor the names of their contributors may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# ------------------------------------------------------------------------------ -# -# The above copyright and license notice applies to distributions of -# CMake in source and binary form. Some source files contain additional -# notices of original copyright by their contributors; see each source -# for details. Third-party software packages supplied with CMake under -# compatible licenses provide their own copyright notices documented in -# corresponding subdirectories. -# -# ------------------------------------------------------------------------------ -# -# CMake was initially developed by Kitware with the following sponsorship: -# -# * National Library of Medicine at the National Institutes of Health -# as part of the Insight Segmentation and Registration Toolkit (ITK). -# -# * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel -# Visualization Initiative. -# -# * National Alliance for Medical Image Computing (NAMIC) is funded by the -# National Institutes of Health through the NIH Roadmap for Medical Research, -# Grant U54 EB005149. +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. # -# * Kitware, Inc. +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. #============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) find_path(LIBLZMA_INCLUDE_DIR lzma.h ) @@ -98,17 +55,21 @@ # Avoid using old codebase if (LIBLZMA_LIBRARY) include(CheckLibraryExists) + set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET}) + set(CMAKE_REQUIRED_QUIET ${LibLZMA_FIND_QUIETLY}) CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_auto_decoder "" LIBLZMA_HAS_AUTO_DECODER) CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_easy_encoder "" LIBLZMA_HAS_EASY_ENCODER) CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY} lzma_lzma_preset "" LIBLZMA_HAS_LZMA_PRESET) + set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE}) endif () include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZMA DEFAULT_MSG LIBLZMA_INCLUDE_DIR - LIBLZMA_LIBRARY - LIBLZMA_HAS_AUTO_DECODER - LIBLZMA_HAS_EASY_ENCODER - LIBLZMA_HAS_LZMA_PRESET +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZMA REQUIRED_VARS LIBLZMA_INCLUDE_DIR + LIBLZMA_LIBRARY + LIBLZMA_HAS_AUTO_DECODER + LIBLZMA_HAS_EASY_ENCODER + LIBLZMA_HAS_LZMA_PRESET + VERSION_VAR LIBLZMA_VERSION_STRING ) if (LIBLZMA_FOUND) diff -Nru zbackup-1.2/cmake/FindLibLZO.cmake zbackup-1.4.1+20150403/cmake/FindLibLZO.cmake --- zbackup-1.2/cmake/FindLibLZO.cmake 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/cmake/FindLibLZO.cmake 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,70 @@ +#.rst: +# FindLibLZO +# ----------- +# +# Find LibLZO +# +# Find LibLZO headers and library +# +# :: +# +# LIBLZO_FOUND - True if liblzo is found. +# LIBLZO_INCLUDE_DIRS - Directory where liblzo headers are located. +# LIBLZO_LIBRARIES - Lzo libraries to link against. +# LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE - True if lzo1x_decompress_safe() is found (required). +# LIBLZO_HAS_LZO1X_1_COMPRESS - True if lzo1x_1_compress() is found (required). +# LIBLZO_VERSION_STRING - version number as a string (ex: "5.0.3") + +#============================================================================= +# Copyright 2008 Per Øyvind Karlsen +# Copyright 2009 Alexander Neundorf +# Copyright 2009 Helio Chissini de Castro +# Copyright 2012 Mario Bensi +# Copyright 2012-2014 Konstantin Isakov +# Copyright 2013 Benjamin Koch (from lzma to lzo) +# Copyright 2014 Vladimir Stackov +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + + +find_path(LIBLZO_INCLUDE_DIR lzo/lzo1x.h ) +find_library(LIBLZO_LIBRARY lzo2) + +if(LIBLZO_INCLUDE_DIR AND EXISTS "${LIBLZO_INCLUDE_DIR}/lzo/lzoconf.h") + file(STRINGS "${LIBLZO_INCLUDE_DIR}/lzo/lzoconf.h" LIBLZO_HEADER_CONTENTS REGEX "#define LZO_VERSION_STRING.+\"[^\"]+\"") + string(REGEX REPLACE ".*#define LZO_VERSION_STRING.+\"([^\"]+)\".*" "\\1" LIBLZO_VERSION_STRING "${LIBLZO_HEADER_CONTENTS}") + unset(LIBLZO_HEADER_CONTENTS) +endif() + +# We're just using two functions. +if (LIBLZO_LIBRARY) + include(CheckLibraryExists) + set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET}) + set(CMAKE_REQUIRED_QUIET ${LibLZMA_FIND_QUIETLY}) + CHECK_LIBRARY_EXISTS(${LIBLZO_LIBRARY} lzo1x_decompress_safe "" LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE) + CHECK_LIBRARY_EXISTS(${LIBLZO_LIBRARY} lzo1x_1_compress "" LIBLZO_HAS_LZO1X_1_COMPRESS) + set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE}) +endif () + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibLZO REQUIRED_VARS LIBLZO_INCLUDE_DIR + LIBLZO_LIBRARY + LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE + LIBLZO_HAS_LZO1X_1_COMPRESS + VERSION_VAR LIBLZO_VERSION_STRING + ) + +if (LIBLZO_FOUND) + set(LIBLZO_LIBRARIES ${LIBLZO_LIBRARY}) + set(LIBLZO_INCLUDE_DIRS ${LIBLZO_INCLUDE_DIR}) +endif () + +mark_as_advanced( LIBLZO_INCLUDE_DIR LIBLZO_LIBRARY ) diff -Nru zbackup-1.2/cmake/FindLibUnwind.cmake zbackup-1.4.1+20150403/cmake/FindLibUnwind.cmake --- zbackup-1.2/cmake/FindLibUnwind.cmake 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/cmake/FindLibUnwind.cmake 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,77 @@ +#.rst: +# FindLibUnwind +# ----------- +# +# Find LibUnwind +# +# Find LibUnwind headers and library +# +# :: +# +# LIBUNWIND_FOUND - True if libunwind is found. +# LIBUNWIND_INCLUDE_DIRS - Directory where libunwind headers are located. +# LIBUNWIND_LIBRARIES - Unwind libraries to link against. +# LIBUNWIND_HAS_UNW_GETCONTEXT - True if unw_getcontext() is found (required). +# LIBUNWIND_HAS_UNW_INIT_LOCAL - True if unw_init_local() is found (required). +# LIBUNWIND_VERSION_STRING - version number as a string (ex: "5.0.3") + +#============================================================================= +# Copyright 2014 ZBackup contributors +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + + +find_path(LIBUNWIND_INCLUDE_DIR libunwind.h ) +if(NOT EXISTS "${LIBUNWIND_INCLUDE_DIR}/unwind.h") + MESSAGE("Found libunwind.h but corresponding unwind.h is absent!") + SET(LIBUNWIND_INCLUDE_DIR "") +endif() + +find_library(LIBUNWIND_LIBRARY unwind) + +if(LIBUNWIND_INCLUDE_DIR AND EXISTS "${LIBUNWIND_INCLUDE_DIR}/libunwind-common.h") + file(STRINGS "${LIBUNWIND_INCLUDE_DIR}/libunwind-common.h" LIBUNWIND_HEADER_CONTENTS REGEX "#define UNW_VERSION_[A-Z]+\t[0-9]*") + + string(REGEX REPLACE ".*#define UNW_VERSION_MAJOR\t([0-9]*).*" "\\1" LIBUNWIND_VERSION_MAJOR "${LIBUNWIND_HEADER_CONTENTS}") + string(REGEX REPLACE ".*#define UNW_VERSION_MINOR\t([0-9]*).*" "\\1" LIBUNWIND_VERSION_MINOR "${LIBUNWIND_HEADER_CONTENTS}") + string(REGEX REPLACE ".*#define UNW_VERSION_EXTRA\t([0-9]*).*" "\\1" LIBUNWIND_VERSION_EXTRA "${LIBUNWIND_HEADER_CONTENTS}") + + if(LIBUNWIND_VERSION_EXTRA) + set(LIBUNWIND_VERSION_STRING "${LIBUNWIND_VERSION_MAJOR}.${LIBUNWIND_VERSION_MINOR}.${LIBUNWIND_VERSION_EXTRA}") + else(not LIBUNWIND_VERSION_EXTRA) + set(LIBUNWIND_VERSION_STRING "${LIBUNWIND_VERSION_MAJOR}.${LIBUNWIND_VERSION_MINOR}") + endif() + unset(LIBUNWIND_HEADER_CONTENTS) +endif() + +if (LIBUNWIND_LIBRARY) + include(CheckSymbolExists) + set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET}) + set(CMAKE_REQUIRED_QUIET ${LibUnwind_FIND_QUIETLY}) + CHECK_SYMBOL_EXISTS(unw_getcontext "${LIBUNWIND_INCLUDE_DIR}/libunwind.h" LIBUNWIND_HAS_UNW_GETCONTEXT) + CHECK_SYMBOL_EXISTS(unw_init_local "${LIBUNWIND_INCLUDE_DIR}/libunwind.h" LIBUNWIND_HAS_UNW_INIT_LOCAL) + set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE}) +endif () + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibUnwind REQUIRED_VARS LIBUNWIND_INCLUDE_DIR + LIBUNWIND_LIBRARY + LIBUNWIND_HAS_UNW_GETCONTEXT + LIBUNWIND_HAS_UNW_INIT_LOCAL + VERSION_VAR LIBUNWIND_VERSION_STRING + ) + +if (LIBUNWIND_FOUND) + set(LIBUNWIND_LIBRARIES ${LIBUNWIND_LIBRARY}) + set(LIBUNWIND_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_DIR}) +endif () + +mark_as_advanced( LIBUNWIND_INCLUDE_DIR LIBUNWIND_LIBRARY ) diff -Nru zbackup-1.2/CMakeLists.txt zbackup-1.4.1+20150403/CMakeLists.txt --- zbackup-1.2/CMakeLists.txt 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/CMakeLists.txt 2015-04-03 09:33:05.000000000 +0000 @@ -1,15 +1,14 @@ -# Copyright (c) 2012-2013 Konstantin Isakov -# Part of ZBackup. Licensed under GNU GPLv2 or later +# Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +# Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE cmake_minimum_required( VERSION 2.8.2 ) project( zbackup ) -if( ${CMAKE_VERSION} VERSION_LESS "2.8.9" ) - # Use the included FindLibLZMA then - set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ) -endif() +list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ) -set( CMAKE_BUILD_TYPE Release ) +if( NOT CMAKE_BUILD_TYPE ) + set( CMAKE_BUILD_TYPE Release ) +endif( NOT CMAKE_BUILD_TYPE ) find_package( ZLIB REQUIRED ) include_directories( ${ZLIB_INCLUDE_DIRS} ) @@ -21,11 +20,11 @@ include_directories( ${PROTOBUF_INCLUDE_DIRS} ) include_directories( ${CMAKE_CURRENT_BINARY_DIR} ) -find_program(PROTOBUF_PROTOC_CHECK NAMES protoc DOC "Protobuf compiler binary") +find_program( PROTOBUF_PROTOC_CHECK NAMES protoc DOC "Protobuf compiler binary" ) -IF(${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND") - MESSAGE(FATAL_ERROR "Could not find protobuf compiler. Make sure protobuf-compiler package is installed.") -ENDIF(${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND") +IF( ${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND" ) + MESSAGE( FATAL_ERROR "Could not find protobuf compiler. Make sure protobuf-compiler package is installed." ) +ENDIF( ${PROTOBUF_PROTOC_CHECK} STREQUAL "PROTOBUF_PROTOC_CHECK-NOTFOUND" ) PROTOBUF_GENERATE_CPP( protoSrcs protoHdrs zbackup.proto ) @@ -34,6 +33,31 @@ find_package( LibLZMA REQUIRED ) include_directories( ${LIBLZMA_INCLUDE_DIRS} ) +find_package( LibLZO COMPONENTS LIBLZO_HAS_LZO1X_DECOMPRESS_SAFE LIBLZO_HAS_LZO1X_1_COMPRESS ) +if ( LIBLZO_FOUND ) + ADD_DEFINITIONS( -DHAVE_LIBLZO ) + include_directories( ${LIBLZO_INCLUDE_DIRS} ) +else ( LIBLZO_FOUND ) + set( LIBLZO_LIBRARIES ) +endif( LIBLZO_FOUND ) + +find_package( LibUnwind COMPONENTS LIBUNWIND_HAS_UNW_GETCONTEXT LIBUNWIND_HAS_INIT_LOCAL ) +if ( LIBUNWIND_FOUND ) + ADD_DEFINITIONS( -DHAVE_LIBUNWIND ) + include_directories( ${LIBUNWIND_INCLUDE_DIRS} ) +else ( LIBUNWIND_FOUND ) + set( LIBUNWIND_LIBRARIES ) +endif( LIBUNWIND_FOUND ) + +add_custom_target( invalidate_files ALL + COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt" ) +execute_process( OUTPUT_VARIABLE ZBACKUP_VERSION + COMMAND git describe --tags --always + OUTPUT_STRIP_TRAILING_WHITESPACE ) +if ( ZBACKUP_VERSION ) + ADD_DEFINITIONS( -DZBACKUP_VERSION="${ZBACKUP_VERSION}" ) +endif( ZBACKUP_VERSION ) + file( GLOB sourceFiles "*.cc" ) add_executable( zbackup ${sourceFiles} ${protoSrcs} ${protoHdrs} ) @@ -43,6 +67,8 @@ ${CMAKE_THREAD_LIBS_INIT} ${ZLIB_LIBRARIES} ${LIBLZMA_LIBRARIES} + ${LIBLZO_LIBRARIES} + ${LIBUNWIND_LIBRARIES} ) install( TARGETS zbackup DESTINATION bin ) diff -Nru zbackup-1.2/compression.cc zbackup-1.4.1+20150403/compression.cc --- zbackup-1.2/compression.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/compression.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,704 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include + +#include "compression.hh" +#include "check.hh" +#include "endian.hh" + +namespace Compression { + +EnDecoder::EnDecoder() +{ +} + +EnDecoder::~EnDecoder() +{ +} + +CompressionMethod::~CompressionMethod() +{ +} + +// LZMA + +#include + +class LZMAEnDecoder : public EnDecoder +{ +protected: + static lzma_stream initValue; + lzma_stream strm; +public: + LZMAEnDecoder() + { + strm = initValue; + } + + void setInput( const void* data, size_t size ) + { + strm.next_in = (const uint8_t *) data; + strm.avail_in = size; + } + + void setOutput( void* data, size_t size ) + { + strm.next_out = (uint8_t *) data; + strm.avail_out = size; + } + + size_t getAvailableInput() + { + return strm.avail_in; + } + + size_t getAvailableOutput() + { + return strm.avail_out; + } + + bool process( bool finish ) + { + lzma_ret ret = lzma_code( &strm, ( finish ? LZMA_FINISH : LZMA_RUN ) ); + + CHECK( ret == LZMA_OK || ret == LZMA_STREAM_END, "lzma_code error: %d", (int) ret ); + + return ( ret == LZMA_STREAM_END ); + } + + ~LZMAEnDecoder() + { + lzma_end( &strm ); + } +}; +lzma_stream LZMAEnDecoder::initValue = LZMA_STREAM_INIT; + +class LZMAEncoder : public LZMAEnDecoder +{ +public: + LZMAEncoder() + { + uint32_t preset = 6; + lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 ); + CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret ); + } + + LZMAEncoder( Config const & config ) + { + uint32_t compressionLevel = config.GET_STORABLE( lzma, compression_level ); + uint32_t preset = ( compressionLevel > 9 ) ? + ( compressionLevel - 10 ) | LZMA_PRESET_EXTREME : + compressionLevel; + lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 ); + CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret ); + } +}; + +class LZMADecoder : public LZMAEnDecoder +{ +public: + LZMADecoder() + { + lzma_ret ret = lzma_stream_decoder( &strm, UINT64_MAX, 0 ); + CHECK( ret == LZMA_OK,"lzma_stream_decoder error: %d", (int) ret ); + } +}; + +class LZMACompression : public CompressionMethod +{ +public: + sptr createEncoder( Config const & config ) const + { + return new LZMAEncoder( config ); + } + + sptr createEncoder() const + { + return new LZMAEncoder(); + } + + sptr createDecoder() const + { + return new LZMADecoder(); + } + + std::string getName() const { return "lzma"; } +}; + +// LZO + +// liblzo implements a lot of algorithms "for unlimited backward compatibility" + +// The web site says: +// "My experiments have shown that LZO1B is good with a large blocksize +// or with very redundant data, LZO1F is good with a small blocksize or +// with binary data and that LZO1X is often the best choice of all. +// LZO1Y and LZO1Z are almost identical to LZO1X - they can achieve a +// better compression ratio on some files. +// Beware, your mileage may vary." +// => I'm using LZO1X, as suggested + +#include + +// Unfortunately, liblzo always works with the whole data, so it doesn't support +// the streaming approach that most other libraries use. This means that we have +// to use a big buffer for the data. The class NoStreamEnDecoder implements this +// so we can use it, if there is another library like liblzo. + +// Collect all data and process it in one pass +class NoStreamEnDecoder : public EnDecoder +{ + std::string accDataIn, accDataOut; + const char* dataIn; + char* dataOut; + size_t availIn, availOut; + bool processed; + size_t posInAccDataOut; +protected: + // you must implement these: + + // Should we try with the existing output buffer which has availOut + // bytes of free space? If you know that this will fail, return false. + // You may peek into dataIn which contains the complete compressed data. + virtual bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) =0; + + // We will allocate a buffer for the output data. How big should it be? + // You may peek into dataIn which contains the complete compressed data. + virtual size_t suggestOutputSize( const char* dataIn, size_t availIn ) =0; + + // Is this input complete? + // An encoder should return false. + virtual bool isCompleteInput( const char* dataIn, size_t availIn ) =0; + + // Process the data in dataIn and put the result into dataOut. You musn't + // write more than availOut bytes! If the output buffer is big enough, + // process the data and store the output size in outputSize. If the output + // buffer is too small, return false and we will give you a bigger one. If + // any other error occurrs, abort the program. We don't have any better + // error handling. Sorry. Do NOT return false for errors that won't be + // remedied by a bigger buffer! + virtual bool doProcess( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ) =0; + + void setUnusedInput( size_t unused ) + { + this->dataIn += availIn - unused; + this->availIn = unused; + } +public: + NoStreamEnDecoder() + { + dataIn = dataOut = NULL; + availIn = availOut = posInAccDataOut = 0; + processed = false; + } + + void setInput( const void* data, size_t size ) + { + dataIn = (const char *) data; + availIn = size; + } + + void setOutput( void* data, size_t size ) + { + dataOut = (char *) data; + availOut = size; + } + + size_t getAvailableInput() + { + return availIn; + } + + size_t getAvailableOutput() + { + return availOut; + } + + bool process( bool finish ) + { + // try to process the input, if we haven't done it, yet + if ( !processed ) + { + // data has not been encoded + if ( accDataIn.empty() ) + { + // this is the first piece of data + if ( finish || isCompleteInput( dataIn, availIn ) ) + { + // special case: all the data has been passed at once + // -> process it without using accDataIn + processFinish( dataIn, availIn ); + } + } + + // if we didn't process the data, put it into accumulator + if ( !processed ) + { + // accumulate data in accDataIn + accDataIn.append( dataIn, availIn ); + + // If this was the last bit of data, we process it, now. + if ( finish || isCompleteInput( accDataIn.data(), accDataIn.size() ) ) + { + processFinish( accDataIn.data(), accDataIn.size() ); + } + } + } + + // If the input has been processed, try to copy some of it to the output buffer. + if ( processed ) + { + // data has been encoded or decoded, remaining output is in accDataOut + // -> copy to output + if ( availOut > 0 && accDataOut.size() - posInAccDataOut > 0 ) + { + size_t sz = availOut; + if ( sz > accDataOut.size() - posInAccDataOut ) + sz = accDataOut.size() - posInAccDataOut; + + memcpy( dataOut, accDataOut.data() + posInAccDataOut, sz ); + dataOut += sz; + availOut -= sz; + posInAccDataOut += sz; + } + + // no more data left? -> return true + return ( accDataOut.size() - posInAccDataOut == 0 ); + } + else + { + // not yet processed, so we cannot be done + return false; + } + } + +private: + void processFinish( const char* dataIn, size_t availIn ) + { + // should we try with the existing output buffer? + if ( shouldTryWith( dataIn, availIn, availOut ) ) + { + size_t outputSize; + if ( doProcess( dataIn, availIn, dataOut, availOut, outputSize ) ) + { + // it worked :-) + processed = true; + availOut -= outputSize; + return ; + } + } + + // we use our own buffer + size_t bufferSize = suggestOutputSize( dataIn, availIn ); + do { + accDataOut.resize( bufferSize ); + + size_t outputSize; + //TODO doc says we mustn't modify the pointer returned by data()... + if ( doProcess( dataIn, availIn, + (char*) accDataOut.data(), bufferSize, outputSize ) ) + { + // buffer is big enough + accDataOut.resize( outputSize ); + processed = true; + return ; + } + + // try a bigger one + bufferSize *= 2; + } while (true); + } +}; + +#ifdef __APPLE__ +#include +#elif __FreeBSD__ +#include +#else +#include +#endif + +// like NoStreamEnDecoder, but also adds the uncompressed size before the stream +//NOTE You should make sure that the compression function doesn't overwrite any +// memory, if this information is corrupted! This could be exploited by a +// malicious person and there is nothing I can do about it. I could check for +// an overflow, but when control gets back to this class, it is already too +// late, as one 'ret' instruction is enough to do harm. +class NoStreamAndUnknownSizeDecoder : public NoStreamEnDecoder +{ +protected: + // You implement this one: + // If you don't know the real decoded size, don't change outputSize. + virtual bool doProcessNoSize( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ) =0; + + bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) + { + return suggestOutputSize( dataIn, availIn ) <= availOut; + } + + // Is this input complete? + bool isCompleteInput( const char* dataIn, size_t availIn ) + { + if ( availIn < 2*sizeof(uint64_t) ) + return false; + + dataIn += sizeof(uint64_t); + + size_t inputSize = le32toh( *(uint32_t*) dataIn ); + + return ( availIn >= inputSize + 2*sizeof(uint64_t) ); + } + + size_t suggestOutputSize( const char* dataIn, size_t availIn ) + { + CHECK( availIn >= sizeof(uint64_t), "not enough input data" ); + // We're not using size_t because we need a type that has the same size on all + // architectures. A 32-bit host won't be able to open files with more than + // 4GB (actually much less), so 4 byte are enough. Even a 64-bit host would + // have some trouble with allocating 8GB of RAM just for our buffers ;-) + //NOTE If your compiler doesn't accept this cast, your size_t is smaller than + // uint32_t. In that case, you are in trouble... + size_t outputSize = le32toh( *(uint32_t*) dataIn ); + return outputSize; + } + + bool doProcess( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ) + { + if ( availIn < 2*sizeof( uint64_t ) ) + return false; + + //NOTE We skip 8 bytes. If we later decide to drop compatibility with 32-bit + // hosts, we can save a 64-bit size. Well, that will be much later, when + // we can easily hold two copies of a 4GB file in main memory :-D + + size_t neededOutputSize = le32toh( *(uint32_t*) dataIn ); + dataIn += sizeof(uint64_t); + size_t inputSize = le32toh( *(uint32_t*) dataIn ); + dataIn += sizeof(uint64_t); + /*if ( outputSize < neededOutputSize ) + return false;*/ + + outputSize = neededOutputSize; + + availIn -= 2*sizeof( uint64_t ); + + // We might not need all of our input data. + setUnusedInput( availIn - inputSize ); + availIn = inputSize; + + size_t reportedOutputSize = neededOutputSize; + if ( !doProcessNoSize( dataIn, availIn, dataOut, availOut, reportedOutputSize ) ) + return false; + + CHECK( reportedOutputSize == neededOutputSize, + "Size of decoded data is different than expected" ); + + return true; + } +}; + +// encoder for NoStreamAndUnknownSizeDecoder +class NoStreamAndUnknownSizeEncoder : public NoStreamEnDecoder +{ +protected: + // You implement this one: + virtual bool doProcessNoSize( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ) =0; + + bool shouldTryWith( const char*, size_t, size_t availOut ) + { + // If the compression doesn't use any spaces... + return availOut > sizeof( uint64_t ); + } + + bool isCompleteInput( const char* dataIn, size_t availIn ) + { + // We cannot know whether the user wants to send more data. + // -> return false; user must use finish=true to signal end of data + return false; + } + + size_t getOverhead() + { + return 2*sizeof( uint64_t ); + } + + size_t suggestOutputSize( const char*, size_t availIn ) + { + // We assume that the compression won't make the data any bigger. + return availIn + getOverhead(); + } + + bool doProcess( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ) + { + CHECK( availIn <= UINT32_MAX, + "You want to compress more than 4GB of data?! Sorry, we don't support that, yet." ); + + memcpy(dataOut, "ABCDEFGHIJKLMNOP", 16); + + // store size + *(uint32_t*)dataOut = htole32( availIn ); + uint32_t* compressedSize = (uint32_t*) ( dataOut + sizeof( uint64_t ) ); + + // compressed data goes after the size + // We skip more than we actually use; see NoStreamAndUnknownSizeDecoder::doProcess(...). + dataOut += getOverhead(); + availOut -= getOverhead(); + + if ( !doProcessNoSize( dataIn, availIn, dataOut, availOut, outputSize ) ) + return false; + + CHECK( outputSize <= UINT32_MAX, + "The compressed data is more than 4GB?! Sorry, we don't support that, yet." ); + *compressedSize = htole32( (uint32_t) outputSize ); + + outputSize += getOverhead(); + + return true; + } +}; + +#ifdef HAVE_LIBLZO + +#include + +// finally, we can implement lzo +class LZO1X_1_Decoder : public NoStreamAndUnknownSizeDecoder +{ +protected: + bool doProcessNoSize( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ) + { + // same argument is used for available output size and size of decompressed data + outputSize = availOut; + int ret = lzo1x_decompress_safe( (const lzo_bytep) dataIn, availIn, + (lzo_bytep) dataOut, (lzo_uintp) &outputSize, NULL ); + + if ( ret == LZO_E_OUTPUT_OVERRUN ) + return false; + + CHECK( ret >= LZO_E_OK, "lzo1x_decompress_safe failed (code %d)", ret ); + + return true; + } +}; +class LZO1X_1_Compression; +class LZO1X_1_Encoder : public NoStreamAndUnknownSizeEncoder +{ + const LZO1X_1_Compression* compression; + static size_t calcMaxCompressedSize( size_t availIn ); +public: + LZO1X_1_Encoder( const LZO1X_1_Compression* compression ) + { + this->compression = compression; + } + +protected: + bool doProcessNoSize( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ); + bool shouldTryWith( const char*, size_t, size_t availOut ); + size_t suggestOutputSize( const char*, size_t availIn ); +}; +class LZO1X_1_Compression : public CompressionMethod +{ + static bool initialized; + static void init() + { + //TODO This is not thread-safe. Does it have to be? + if (!initialized) + { + int ret = lzo_init(); + CHECK( ret == LZO_E_OK, "lzo_init failed (%d)", ret ); + initialized = true; + } + } +public: + sptr< EnDecoder > createEncoder( Config const & config ) const + { + init(); + return new LZO1X_1_Encoder(this); + } + + sptr< EnDecoder > createEncoder() const + { + init(); + return new LZO1X_1_Encoder(this); + } + + sptr< EnDecoder > createDecoder() const + { + init(); + return new LZO1X_1_Decoder(); + } + + std::string getName() const { return "lzo1x_1"; } + + lzo_voidp getWorkmem( size_t size ) const + { + return new char[size]; + } + + void giveBackWorkmem( lzo_voidp wrkmem ) const + { + //TODO I think we should keep the memory around and reuse it. After all + // it is only a few kilobytes and we will need it a lot. However, I + // won't risk anything here because I don't know whether this will be + // called by more than one thread. + delete[] (char*)wrkmem; + } +}; + +bool LZO1X_1_Compression::initialized = false; + +size_t LZO1X_1_Encoder::calcMaxCompressedSize( size_t availIn ) +{ + // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough. + // The documentation refers to example/simple.c which says: + // "Because the input block may be incompressible, we must provide a little more + // output space in case that compression is not possible." + // -> We use the same formula. + return (availIn + availIn / 16 + 64 + 3); +} + +bool LZO1X_1_Encoder::shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) +{ + return availOut >= suggestOutputSize( dataIn, availIn ); +} + +size_t LZO1X_1_Encoder::suggestOutputSize( const char*, size_t availIn ) +{ + // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough. + // The documentation refers to example/simple.c which says: + // "Because the input block may be incompressible, we must provide a little more + // output space in case that compression is not possible." + // -> We use the same formula. + return calcMaxCompressedSize( availIn ) + getOverhead(); +} + +bool LZO1X_1_Encoder::doProcessNoSize( const char* dataIn, size_t availIn, + char* dataOut, size_t availOut, size_t& outputSize ) +{ + // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough. + // Therefore, we won't try it unless we are sure that the buffer is big enough. + if ( availOut < calcMaxCompressedSize( availIn ) ) + return false; + + // same argument is used for available output size (haha, see above) + // and size of decompressed data + outputSize = availOut; + + lzo_voidp wrkmem = compression->getWorkmem(LZO1X_1_MEM_COMPRESS); + int ret = lzo1x_1_compress( (const lzo_bytep) dataIn, availIn, + (lzo_bytep) dataOut, (lzo_uintp) &outputSize, wrkmem ); + compression->giveBackWorkmem(wrkmem); + + if ( ret == LZO_E_OUTPUT_OVERRUN ) + return false; + + CHECK( ret >= LZO_E_OK, "lzo1x_1_compress failed (code %d)", ret ); + + return true; +} + +#endif // HAVE_LIBLZO + +// register them + +const_sptr< CompressionMethod > const CompressionMethod::compressions[] = { + new LZMACompression(), +# ifdef HAVE_LIBLZO + new LZO1X_1_Compression(), +# endif + // NULL entry marks end of list. Don't remove it! + NULL +}; + +const_sptr< CompressionMethod > CompressionMethod::selectedCompression = + compressions[ 0 ]; + +const_sptr< CompressionMethod > CompressionMethod::findCompression( + const std::string& name, bool optional ) +{ + for ( const const_sptr* c = compressions + 0; *c; ++c ) + { + if ( (*c)->getName() == name ) + { + return (*c); + } + } + if ( !optional ) + { + throw exUnsupportedCompressionMethod( name ); + } + return NULL; +} + +// iterator over compressions +CompressionMethod::iterator::iterator( const const_sptr< CompressionMethod > * ptr ): + ptr( ptr ) +{ +} + +CompressionMethod::iterator::iterator( const iterator & it ): + ptr( it.ptr ) +{ +} + +CompressionMethod::iterator& CompressionMethod::iterator::operator =( const iterator& it ) +{ + this->ptr = it.ptr; + return *this; +} + +bool CompressionMethod::iterator::operator ==( const iterator& other ) const +{ + // special case: one has ptr==NULL (end iterator returned by end()) and the + // other has *ptr==NULL (end iterator obtained by calling ++) + if ( !ptr && ( !other.ptr || !*other.ptr ) ) + return true; + else if ( !other.ptr && ( !ptr || !*ptr ) ) + return true; + else + return (ptr == other.ptr); +} +bool CompressionMethod::iterator::operator !=( const iterator& other ) const +{ + return !( *this == other ); +} + +bool CompressionMethod::iterator::atEnd() const +{ + return !ptr || !*ptr; +} + +CompressionMethod::iterator& CompressionMethod::iterator::operator ++() +{ + CHECK( ptr && *ptr, "Cannot increment the end iterator" ); + + ++ptr; + + return *this; +} + +const_sptr CompressionMethod::iterator::operator *() +{ + CHECK( ptr && *ptr, "Cannot dereference the end iterator" ); + + return *ptr; +} + +CompressionMethod::iterator CompressionMethod::begin() +{ + return iterator(compressions); +} +CompressionMethod::iterator CompressionMethod::end() +{ + return iterator(NULL); +} + +} diff -Nru zbackup-1.2/compression.hh zbackup-1.4.1+20150403/compression.hh --- zbackup-1.2/compression.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/compression.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,92 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef COMPRESSION_HH_INCLUDED +#define COMPRESSION_HH_INCLUDED + +#include "sptr.hh" +#include "ex.hh" +#include "nocopy.hh" +#include "config.hh" + +namespace Compression { + +DEF_EX( Ex, "Compression exception", std::exception ) +DEF_EX_STR( exUnsupportedCompressionMethod, "Unsupported compression method:", Ex ) + +// used for encoding or decoding +class EnDecoder: NoCopy +{ +protected: + EnDecoder(); +public: + virtual ~EnDecoder(); + + // encoder can read up to size bytes from data + virtual void setInput ( const void * data, size_t size ) = 0; + // how many bytes of the last input haven't been used, yet? + virtual size_t getAvailableInput() = 0; + + // encoder can write up to size bytes to output + virtual void setOutput( void * data, size_t size ) = 0; + // how many bytes of free space are remaining in the output buffer + virtual size_t getAvailableOutput() = 0; + + // process some bytes + // finish: will you pass more data to the encoder via setOutput? + // NOTE You must eventually set finish to true. + // returns, whether all output bytes have been written + virtual bool process( bool finish ) = 0; +}; + +// compression method +class CompressionMethod +{ +public: + virtual ~CompressionMethod(); + + // returns name of compression method + // This name is saved in the file header of the compressed file. + virtual std::string getName() const = 0; + + virtual sptr< EnDecoder > createEncoder( Config const & ) const = 0; + virtual sptr< EnDecoder > createEncoder() const = 0; + virtual sptr< EnDecoder > createDecoder() const = 0; + + // find a compression by name + // If optional is false, it will either return a valid CompressionMethod + // object or abort the program. If optional is true, it will return + // NULL, if it cannot find the a compression with that name. + static const_sptr< CompressionMethod > findCompression( + const std::string & name, bool optional = false ); + + static const_sptr< CompressionMethod > selectedCompression; + + static const_sptr< CompressionMethod > const compressions[]; + + class iterator + { + friend class CompressionMethod; + + const const_sptr< CompressionMethod > * ptr; + iterator( const const_sptr< CompressionMethod > * ptr ); + public: + iterator( const iterator & it ); + iterator & operator =( const iterator & it ); + + bool operator == ( const iterator & other ) const; + bool operator != ( const iterator & other ) const; + + bool atEnd() const; + + iterator & operator ++(); + + const_sptr< CompressionMethod > operator * (); + }; + static iterator begin(); + static iterator end(); +}; + +} + +#endif diff -Nru zbackup-1.2/config.cc zbackup-1.4.1+20150403/config.cc --- zbackup-1.2/config.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/config.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,564 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include +#include "config.hh" +#include "ex.hh" +#include "debug.hh" +#include "utils.hh" +#include "compression.hh" + +#define VALID_SUFFIXES "Valid suffixes:\n" \ + "B - multiply by 1 (bytes)\n" \ + "KiB - multiply by 1024 (kibibytes)\n" \ + "MiB - multiply by 1024*1024 (mebibytes)\n" \ + "GiB - multiply by 1024*1024*1024 (gibibytes)\n" \ + "KB - multiply by 1000 (kilobytes)\n" \ + "MB - multiply by 1000*1000 (megabytes)\n" \ + "GB - multiply by 1000*1000*1000 (gigabytes)\n" \ + +#define SKIP_ON_VALIDATION \ +{ \ + if ( validate ) \ + return true; \ +} + +// Some configurables could be just a switch +// So we introducing a macros that would indicate +// that this configurable is not a switch +#define REQUIRE_VALUE \ +{ \ + if ( !hasValue && !validate ) \ + return false; \ +} + +#define PARSE_OR_VALIDATE( parse_src, validate_src ) \ + ( !validate && ( parse_src ) ) || ( validate && ( validate_src ) ) + +DEF_EX_STR( exInvalidThreadsValue, "Invalid threads value specified:", std::exception ) + +void Config::prefillKeywords() +{ + /* Textual representations of the tokens. */ + + Keyword defaultKeywords[] = { + // Storable options + { + "chunk.max_size", + Config::oChunk_max_size, + Config::Storable, + "Maximum chunk size used when storing chunks\n" + "Affects deduplication ratio directly\n" + "Default is %s", + Utils::numberToString( GET_STORABLE( chunk, max_size ) ) + }, + { + "bundle.max_payload_size", + Config::oBundle_max_payload_size, + Config::Storable, + "Maximum number of bytes a bundle can hold. Only real chunk bytes are\n" + "counted, not metadata. Any bundle should be able to contain at least\n" + "one arbitrary single chunk, so this should not be smaller than\n" + "chunk.max_size\n" + "Default is %s", + Utils::numberToString( GET_STORABLE( bundle, max_payload_size ) ) + }, + { + "bundle.compression_method", + Config::oBundle_compression_method, + Config::Storable, + "Compression method for new bundles\n" + "Default is %s", + GET_STORABLE( bundle, compression_method ) + }, + { + "lzma.compression_level", + Config::oLZMA_compression_level, + Config::Storable, + "Compression level for new LZMA-compressed files\n" + "Valid values: 0-19 (values over 9 enables extreme mode)\n" + "Default is %s", + Utils::numberToString( GET_STORABLE( lzma, compression_level ) ) + }, + + // Shortcuts for storable options + { + "compression", + Config::oBundle_compression_method, + Config::Storable, + "Shortcut for bundle.compression_method\n" + "Default is %s", + GET_STORABLE( bundle, compression_method ) + }, + + // Runtime options + { + "threads", + Config::oRuntime_threads, + Config::Runtime, + "Maximum number of compressor threads to use in backup process\n" + "Default is %s on your system", + Utils::numberToString( runtime.threads ) + }, + { + "cache-size", + Config::oRuntime_cacheSize, + Config::Runtime, + "Cache size to use in restore process\n" + "Affects restore process speed directly\n" + VALID_SUFFIXES + "Default is %sMiB", + Utils::numberToString( runtime.cacheSize / 1024 / 1024 ) + }, + { + "exchange", + Config::oRuntime_exchange, + Config::Runtime, + "Data to exchange between repositories in import/export process\n" + "Can be specified multiple times\n" + "Valid values:\n" + "backups - exchange backup instructions (files in backups/ directory)\n" + "bundles - exchange bundles with data (files in bunles/ directory)\n" + "index - exchange indicies of chunks (files in index/ directory)\n" + "No default value, you should specify it explicitly" + }, + + { "", Config::oBadOption, Config::None } + }; + + keywords = new Keyword[ sizeof( defaultKeywords) / sizeof( Keyword ) ]; + std::copy( defaultKeywords, defaultKeywords + + sizeof( defaultKeywords) / sizeof( Keyword ), keywords ); + cleanup_keywords = true; +} + +Config::~Config() +{ + // prevent memleaks + // TODO: use sptr + if ( cleanup_storable ) + delete storable; + if ( cleanup_keywords ) + delete[] keywords; +} + +Config::Config(): + cleanup_storable( true ) +{ + storable = new ConfigInfo; + prefillKeywords(); + dPrintf( "%s is instantiated and initialized with default values\n", + __CLASS ); +} + +Config::Config( ConfigInfo * configInfo ): + cleanup_storable( false ) +{ + storable = configInfo; + prefillKeywords(); + dPrintf( "%s is instantiated and initialized with supplied ConfigInfo\n", + __CLASS ); +} + +Config::Config( const Config & configIn, ConfigInfo * configInfo ) +{ + configInfo->MergeFrom( *configIn.storable ); + *this = configIn; + storable = configInfo; + cleanup_storable = false; + cleanup_keywords = false; + dPrintf( "%s is instantiated and initialized with supplied values\n", + __CLASS ); +} + +Config::OpCodes Config::parseToken( const char * option, const OptionType type ) +{ + for ( u_int i = 0; !keywords[ i ].name.empty(); i++ ) + { + if ( strcasecmp( option, keywords[ i ].name.c_str() ) == 0 ) + { + if ( keywords[ i ].type != type ) + { + fprintf( stderr, "Invalid option type specified for %s\n", option ); + break; + } + + return keywords[ i ].opcode; + } + } + + return Config::oBadOption; +} + +bool Config::parseOrValidate( const string & option, const OptionType type, + bool validate ) +{ + string prefix; + if ( type == Runtime ) + prefix.assign( "runtime" ); + else + if ( type == Storable ) + prefix.assign( "storable" ); + + dPrintf( "%s %s option \"%s\"...\n", ( validate ? "Validating" : "Parsing" ), + prefix.c_str(), option.c_str() ); + + bool hasValue = false; + size_t optionLength = option.length() + 1; + char optionName[ optionLength ], optionValue[ optionLength ]; + + if ( sscanf( option.c_str(), "%[^=]=%s", optionName, optionValue ) == 2 ) + { + dPrintf( "%s option %s: %s\n", prefix.c_str(), optionName, optionValue ); + hasValue = true; + } + else + dPrintf( "%s option %s\n", prefix.c_str(), option.c_str() ); + + int opcode = parseToken( hasValue ? optionName : option.c_str(), type ); + + size_t sizeValue; + char suffix[ 16 ]; + int n; + unsigned int scale, scaleBase = 1; + uint32_t uint32Value; + + switch ( opcode ) + { + case oChunk_max_size: + SKIP_ON_VALIDATION; + REQUIRE_VALUE; + + if ( sscanf( optionValue, "%u %n", &uint32Value, &n ) == 1 + && !optionValue[ n ] ) + { + SET_STORABLE( chunk, max_size, uint32Value ); + dPrintf( "storable[chunk][max_size] = %u\n", + GET_STORABLE( chunk, max_size ) ); + + return true; + } + + return false; + /* NOTREACHED */ + break; + + case oBundle_max_payload_size: + SKIP_ON_VALIDATION; + REQUIRE_VALUE; + + if ( sscanf( optionValue, "%u %n", &uint32Value, &n ) == 1 + && !optionValue[ n ] ) + { + SET_STORABLE( bundle, max_payload_size, uint32Value ); + dPrintf( "storable[bundle][max_payload_size] = %u\n", + GET_STORABLE( bundle, max_payload_size ) ); + + return true; + } + + return false; + /* NOTREACHED */ + break; + + case oLZMA_compression_level: + REQUIRE_VALUE; + + if ( PARSE_OR_VALIDATE( + sscanf( optionValue, "%u %n", &uint32Value, &n ) != 1 || + optionValue[ n ] || uint32Value > 19, + GET_STORABLE( lzma, compression_level ) > 19 ) + ) + return false; + + SKIP_ON_VALIDATION; + SET_STORABLE( lzma, compression_level, uint32Value ); + dPrintf( "storable[lzma][compression_level] = %u\n", + GET_STORABLE( lzma, compression_level ) ); + + return true; + /* NOTREACHED */ + break; + + case oBundle_compression_method: + REQUIRE_VALUE; + + if ( PARSE_OR_VALIDATE( strcmp( optionValue, "lzma" ) == 0, + GET_STORABLE( bundle, compression_method ) == "lzma" ) ) + { + const_sptr< Compression::CompressionMethod > lzma = + Compression::CompressionMethod::findCompression( "lzma" ); + if ( !lzma ) + { + fprintf( stderr, "zbackup is compiled without LZMA support, but the code " + "would support it. If you install liblzma (including development files) " + "and recompile zbackup, you can use LZMA.\n" ); + return false; + } + Compression::CompressionMethod::selectedCompression = lzma; + } + else + if ( PARSE_OR_VALIDATE( + strcmp( optionValue, "lzo1x_1" ) == 0 || strcmp( optionValue, "lzo" ) == 0, + GET_STORABLE( bundle, compression_method ) == "lzo1x_1" ) ) + { + const_sptr< Compression::CompressionMethod > lzo = + Compression::CompressionMethod::findCompression( "lzo1x_1" ); + if ( !lzo ) + { + fprintf( stderr, "zbackup is compiled without LZO support, but the code " + "would support it. If you install liblzo2 (including development files) " + "and recompile zbackup, you can use LZO.\n" ); + return false; + } + Compression::CompressionMethod::selectedCompression = lzo; + } + else + { + fprintf( stderr, + "ZBackup doesn't support %s compression.\n" + "You probably need a newer version.\n", validate ? + GET_STORABLE( bundle, compression_method ).c_str() : optionValue ); + fprintf( stderr, "Supported compression methods:\n" ); + for ( const const_sptr< Compression::CompressionMethod > * c = + Compression::CompressionMethod::compressions; *c; ++c ) + { + fprintf( stderr, "%s\n", (*c)->getName().c_str() ); + } + fprintf( stderr, "\n" ); + + return false; + } + + SKIP_ON_VALIDATION; + SET_STORABLE( bundle, compression_method, + Compression::CompressionMethod::selectedCompression->getName() ); + dPrintf( "storable[bundle][compression_method] = %s\n", + GET_STORABLE( bundle, compression_method ).c_str() ); + + return true; + /* NOTREACHED */ + break; + + case oRuntime_threads: + REQUIRE_VALUE; + + sizeValue = runtime.threads; + if ( sscanf( optionValue, "%zu %n", &sizeValue, &n ) != 1 || + optionValue[ n ] || sizeValue < 1 ) + throw exInvalidThreadsValue( optionValue ); + runtime.threads = sizeValue; + + dPrintf( "runtime[threads] = %zu\n", runtime.threads ); + + return true; + /* NOTREACHED */ + break; + + case oRuntime_cacheSize: + REQUIRE_VALUE; + + sizeValue = runtime.cacheSize; + if ( sscanf( optionValue, "%zu %15s %n", + &sizeValue, suffix, &n ) == 2 && !optionValue[ n ] ) + { + // Check the suffix + for ( char * c = suffix; *c; ++c ) + *c = tolower( *c ); + + if ( strcmp( suffix, "b" ) == 0 ) + { + scale = 1; + } + else + if ( strcmp( suffix, "kib" ) == 0 ) + { + scaleBase = 1024; + scale = scaleBase; + } + else + if ( strcmp( suffix, "mib" ) == 0 ) + { + scaleBase = 1024; + scale = scaleBase * scaleBase; + } + else + if ( strcmp( suffix, "gib" ) == 0 ) + { + scaleBase = 1024; + scale = scaleBase * scaleBase * scaleBase; + } + else + if ( strcmp( suffix, "kb" ) == 0 ) + { + scaleBase = 1000; + scale = scaleBase; + } + else + if ( strcmp( suffix, "mb" ) == 0 ) + { + scaleBase = 1000; + scale = scaleBase * scaleBase; + } + else + if ( strcmp( suffix, "gb" ) == 0 ) + { + scaleBase = 1000; + scale = scaleBase * scaleBase * scaleBase; + } + else + { + // SI or IEC + fprintf( stderr, "Invalid suffix specified in cache size (%s): %s.\n" + VALID_SUFFIXES, optionValue, suffix ); + return false; + } + runtime.cacheSize = sizeValue * scale; + + dPrintf( "runtime[cacheSize] = %zu\n", runtime.cacheSize ); + + return true; + } + return false; + /* NOTREACHED */ + break; + + case oRuntime_exchange: + REQUIRE_VALUE; + + if ( strcmp( optionValue, "backups" ) == 0 ) + runtime.exchange.set( BackupExchanger::backups ); + else + if ( strcmp( optionValue, "bundles" ) == 0 ) + runtime.exchange.set( BackupExchanger::bundles ); + else + if ( strcmp( optionValue, "index" ) == 0 ) + runtime.exchange.set( BackupExchanger::index ); + else + { + fprintf( stderr, "Invalid exchange value specified: %s\n" + "Must be one of the following: backups, bundles, index.\n", + optionValue ); + return false; + } + + dPrintf( "runtime[exchange] = %s\n", runtime.exchange.to_string().c_str() ); + + return true; + /* NOTREACHED */ + break; + + case oBadOption: + default: + return false; + /* NOTREACHED */ + break; + } + + /* NOTREACHED */ + return false; +} + +void Config::showHelp( const OptionType type ) +{ + fprintf( stderr, +"Available %s options overview:\n\n" +"== help ==\n" +"show this message\n" +"", ( type == Runtime ? "runtime" : ( type == Storable ? "storable" : "" ) ) ); + + for ( u_int i = 0; !keywords[ i ].name.empty(); i++ ) + { + if ( keywords[ i ].type != type ) + continue; + + fprintf( stderr, "\n== %s ==\n", keywords[ i ].name.c_str() ); + fprintf( stderr, keywords[ i ].description.c_str(), + keywords[ i ].defaultValue.c_str() ); + fprintf( stderr, "\n" ); + } +} + +bool Config::parseProto( const string & str, google::protobuf::Message * mutable_message ) +{ + return google::protobuf::TextFormat::ParseFromString( str, mutable_message ); +} + +string Config::toString( google::protobuf::Message const & message ) +{ + std::string str; + google::protobuf::TextFormat::PrintToString( message, &str ); + + return str; +} + +bool Config::validateProto( const string & oldConfigData, const string & configData ) +{ + Config config; + dPrintf( "Validating proto...\n" ); + if ( !parseProto( configData, config.storable ) ) + return false; + + const ::google::protobuf::Descriptor * configDescriptor = + config.storable->descriptor(); + for ( int i = 0; i < configDescriptor->field_count(); i++ ) + { + const ::google::protobuf::FieldDescriptor * storage = + configDescriptor->field( i ); + dPrintf( "Storage: %s - %d - %d\n", storage->name().c_str(), + storage->label(), storage->type()); + + // TODO: support for top-level fields + if ( storage->type() == ::google::protobuf::FieldDescriptor::TYPE_MESSAGE ) + { + const ::google::protobuf::Descriptor * storageDescriptor = + storage->message_type(); + + for ( int j = 0; j < storageDescriptor->field_count(); j++ ) + { + const ::google::protobuf::FieldDescriptor * field = + storageDescriptor->field( j ); + + dPrintf( "Field: %s - %d - %d\n", field->name().c_str(), + field->label(), field->type()); + + string option = storage->name() + "." + field->name(); + + if ( !config.parseOrValidate( option.c_str(), Storable, true ) ) + { + fprintf( stderr, "Invalid option specified: %s\n", + option.c_str() ); + return false; + } + } + } + } + + return true; +} + +void Config::reset_storable() +{ + // TODO: Use protobuf introspection + // to fill messages in loop with default values + // without explicit declaration + Config defaultConfig; + + SET_STORABLE( chunk, max_size, defaultConfig.GET_STORABLE( chunk, max_size ) ); + SET_STORABLE( bundle, max_payload_size, defaultConfig.GET_STORABLE( + bundle, max_payload_size ) ); + SET_STORABLE( bundle, compression_method, defaultConfig.GET_STORABLE( + bundle, compression_method ) ); + SET_STORABLE( lzma, compression_level, defaultConfig.GET_STORABLE( + lzma, compression_level ) ); +} + +void Config::show() +{ + printf( "%s", toString( *storable ).c_str() ); +} + +void Config::show( const ConfigInfo & config ) +{ + printf( "%s", toString( config ).c_str() ); +} diff -Nru zbackup-1.2/config.hh zbackup-1.4.1+20150403/config.hh --- zbackup-1.2/config.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/config.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,108 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef CONFIG_HH_INCLUDED +#define CONFIG_HH_INCLUDED + +#include +#include +#include +#include "zbackup.pb.h" +#include "mt.hh" +#include "backup_exchanger.hh" + +// TODO: make *_storable to be variadic +#define SET_STORABLE( storage, property, value ) \ + storable->mutable_##storage()->set_##property( value ) + +#define GET_STORABLE( storage, property ) \ + storable->storage().property() + +using std::string; +using std::bitset; + +class Config +{ +public: + struct RuntimeConfig + { + size_t threads; + size_t cacheSize; + bitset< BackupExchanger::Flags > exchange; + + // Default runtime config + RuntimeConfig(): + threads( getNumberOfCpus() ), + cacheSize( 40 * 1024 * 1024 ) // 40 MB + { + } + }; + + enum OptionType + { + Runtime, + Storable, + None + }; + + /* Keyword tokens. */ + typedef enum + { + oBadOption, + + oChunk_max_size, + oBundle_max_payload_size, + oBundle_compression_method, + oLZMA_compression_level, + + oRuntime_threads, + oRuntime_cacheSize, + oRuntime_exchange, + + oDeprecated, oUnsupported + } OpCodes; + + // Validator for user-supplied storable configuration + static bool validateProto( const string &, const string & ); + + static bool parseProto( const string &, google::protobuf::Message * ); + + static string toString( google::protobuf::Message const & ); + + // Print configuration to screen + static void show( const ConfigInfo & ); + void show(); + + void showHelp( const OptionType ); + + OpCodes parseToken( const char *, const OptionType ); + bool parseOrValidate( const string &, const OptionType, bool validate = false ); + + Config( const Config &, ConfigInfo * ); + Config( ConfigInfo * ); + Config(); + ~Config(); + + void reset_storable(); + + RuntimeConfig runtime; + ConfigInfo * storable; +private: + struct Keyword + { + string name; + Config::OpCodes opcode; + Config::OptionType type; + string description; + string defaultValue; + }; + + Keyword * keywords; + + bool cleanup_storable; + bool cleanup_keywords; + + void prefillKeywords(); +}; + +#endif diff -Nru zbackup-1.2/CONTRIBUTORS zbackup-1.4.1+20150403/CONTRIBUTORS --- zbackup-1.2/CONTRIBUTORS 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/CONTRIBUTORS 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,20 @@ +This file contains a list of people who have made contributions +to the ZBackup. + +Original design and implementation: + Konstantin Isakov + +Core maintainers: + Konstantin Isakov + Vladimir Stackov + +Code contributions: + Benjamin Koch + Gleb Golubitsky + Igor Katson + Eugene Agafonov + Antonia Stevens + Frank Groeneveld + +Feel free to add yourself to this list in your pull-request. +Please modify this file instead of source headers. diff -Nru zbackup-1.2/debian/changelog zbackup-1.4.1+20150403/debian/changelog --- zbackup-1.2/debian/changelog 2014-04-27 05:56:22.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/changelog 2015-06-26 23:37:17.000000000 +0000 @@ -1,13 +1,24 @@ -zbackup (1.2-1.2~eugenesan~precise1) precise; urgency=medium +zbackup (1.4.1+20150403-1~eugenesan~precise2) precise; urgency=medium - * Import pre-release patches from upstream - - 7c2205a Add support for big-endian architectures - - eaadb44 customizable storage and runtime parameters and some more - - 2c5a3cc Switch to bytes for all sizes and minor cosmeitcs - - 1e8e941 revert disabled bundle naming scaling code - - 9e13d6b distinguish compression level per algorithm and related cosmetics + * New upstream snapshot + * Import upstream fix for temp files creation - -- Eugene San (eugenesan) Sun, 27 Apr 2014 08:51:08 +0300 + -- Eugene San (eugenesan) Tue, 19 May 2015 09:55:20 -0400 + +zbackup (1.4.1-1) unstable; urgency=medium + + * New upstream release. + + -- Laszlo Boszormenyi (GCS) Sun, 28 Dec 2014 07:33:17 +0000 + +zbackup (1.3-1) unstable; urgency=low + + * Upstream added OpenSSL linking exception (closes: #761381). + * Remove big-endian-support.patch , this release contains it. + * Update Standards-Version to 3.9.6 . + * New maintainer (closes: #755773). + + -- Laszlo Boszormenyi (GCS) Sun, 12 Oct 2014 10:20:33 +0000 zbackup (1.2-1.1) unstable; urgency=medium diff -Nru zbackup-1.2/debian/control zbackup-1.4.1+20150403/debian/control --- zbackup-1.2/debian/control 2013-08-28 02:08:35.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/control 2014-10-12 10:22:47.000000000 +0000 @@ -1,10 +1,10 @@ Source: zbackup Section: admin Priority: optional -Maintainer: Howard Chan +Maintainer: Laszlo Boszormenyi (GCS) Build-Depends: debhelper (>= 9), cmake (>= 2.8.2), libssl-dev, - libprotobuf-dev, libprotoc-dev, protobuf-compiler, liblzma-dev, zlib1g-dev -Standards-Version: 3.9.4 + libprotobuf-dev, libprotoc-dev, protobuf-compiler, liblzma-dev, zlib1g-dev +Standards-Version: 3.9.6 Homepage: http://zbackup.org #Vcs-Git: git://git.debian.org/collab-maint/zbackup.git #Vcs-Browser: http://git.debian.org/?p=collab-maint/zbackup.git;a=summary diff -Nru zbackup-1.2/debian/copyright zbackup-1.4.1+20150403/debian/copyright --- zbackup-1.2/debian/copyright 2013-08-27 07:41:35.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/copyright 2014-10-12 10:27:27.000000000 +0000 @@ -3,7 +3,7 @@ Source: http://zbackup.org Files: * -Copyright: 2012-2013 Konstantin Isakov +Copyright: 2012-2014 Konstantin Isakov License: GPL-2+-openssl This package is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -36,7 +36,8 @@ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". Files: debian/* -Copyright: 2013 Howard Chan +Copyright: 2014 Laszlo Boszormenyi (GCS) , + 2013-2014 Howard Chan License: GPL-2+ This package is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff -Nru zbackup-1.2/debian/patches/0001-Add-support-for-big-endian-architectures.patch zbackup-1.4.1+20150403/debian/patches/0001-Add-support-for-big-endian-architectures.patch --- zbackup-1.2/debian/patches/0001-Add-support-for-big-endian-architectures.patch 2014-04-27 05:49:48.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/patches/0001-Add-support-for-big-endian-architectures.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,52 +0,0 @@ -From 7c2205a3e0034a3ce6df0917cc6263262f489e80 Mon Sep 17 00:00:00 2001 -From: Konstantin Isakov -Date: Wed, 15 Jan 2014 21:48:05 +0400 -Subject: [PATCH 1/5] Add support for big-endian architectures. - ---- - endian.hh | 24 +++++++++++++++++++++--- - 1 file changed, 21 insertions(+), 3 deletions(-) - -diff --git a/endian.hh b/endian.hh -index e96c2e5..95167b5 100644 ---- a/endian.hh -+++ b/endian.hh -@@ -12,9 +12,7 @@ - #include - #endif - --#if __BYTE_ORDER != __LITTLE_ENDIAN --#error Please add support for architectures different from little-endian. --#endif -+#if __BYTE_ORDER == __LITTLE_ENDIAN - - /// Converts the given host-order value to big-endian value - inline uint32_t toBigEndian( uint32_t v ) { return htonl( v ); } -@@ -25,4 +23,24 @@ inline uint64_t toLittleEndian( uint64_t v ) { return v; } - inline uint32_t fromLittleEndian( uint32_t v ) { return v; } - inline uint64_t fromLittleEndian( uint64_t v ) { return v; } - -+#elif __BYTE_ORDER == __BIG_ENDIAN -+ -+// Note: the functions used are non-standard. Add more ifdefs if needed -+ -+/// Converts the given host-order value to big-endian value -+inline uint32_t toBigEndian( uint32_t v ) { return v; } -+/// Converts the given host-order value to little-endian value -+inline uint32_t toLittleEndian( uint32_t v ) { return htole32( v ); } -+inline uint64_t toLittleEndian( uint64_t v ) { return htole64( v ); } -+/// Converts the given little-endian value to host-order value -+inline uint32_t fromLittleEndian( uint32_t v ) { return le32toh( v ); } -+inline uint64_t fromLittleEndian( uint64_t v ) { return le64toh( v ); } -+ -+#else -+ -+#error Please add support for architectures different from little-endian and\ -+ big-endian. -+ -+#endif -+ - #endif --- -1.9.2 - diff -Nru zbackup-1.2/debian/patches/0002-customizable-storage-and-runtime-parameters-and-some.patch zbackup-1.4.1+20150403/debian/patches/0002-customizable-storage-and-runtime-parameters-and-some.patch --- zbackup-1.2/debian/patches/0002-customizable-storage-and-runtime-parameters-and-some.patch 2014-04-27 05:49:48.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/patches/0002-customizable-storage-and-runtime-parameters-and-some.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,468 +0,0 @@ -From eaadb445c62b88e7396c4597c54c306838c17398 Mon Sep 17 00:00:00 2001 -From: "Eugene San (eugenesan)" -Date: Sat, 8 Feb 2014 11:03:46 +0200 -Subject: [PATCH 2/5] customizable storage and runtime parameters and some more - - * compression level - * chunk size - * bundle size - * debugging bits - * minor cosmetics and comments ---- - bundle.cc | 45 +++++++++++++++------ - bundle.hh | 4 +- - chunk_storage.cc | 15 +++---- - chunk_storage.hh | 3 +- - file.cc | 5 ++- - zbackup.cc | 116 ++++++++++++++++++++++++++++++++++++++++++++++++------- - zbackup.hh | 4 +- - 7 files changed, 154 insertions(+), 38 deletions(-) - -diff --git a/bundle.cc b/bundle.cc -index 14ccee3..98b0238 100644 ---- a/bundle.cc -+++ b/bundle.cc -@@ -6,6 +6,7 @@ - - #include "bundle.hh" - #include "check.hh" -+#include "debug.hh" - #include "dir.hh" - #include "encrypted_file.hh" - #include "encryption.hh" -@@ -27,7 +28,7 @@ void Creator::addChunk( string const & id, void const * data, size_t size ) - payload.append( ( char const * ) data, size ); - } - --void Creator::write( std::string const & fileName, EncryptionKey const & key ) -+void Creator::write( std::string const & fileName, EncryptionKey const & key, size_t compression ) - { - EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv ); - -@@ -42,10 +43,11 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key ) - - // Compress - -- uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be -- // the best option -- lzma_stream strm = LZMA_STREAM_INIT; -- lzma_ret ret; -+ uint32_t preset = ( compression > 9 ) ? ( compression - 10 ) | LZMA_PRESET_EXTREME : compression; -+ dPrintf( "Compression level: %lu, lzma preset %u\n", compression, preset); -+ -+ lzma_stream strm = LZMA_STREAM_INIT; -+ lzma_ret ret; - - ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 ); - CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret ); -@@ -82,7 +84,9 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key ) - CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret ); - } - -- lzma_end( &strm ); -+ dPrintf( "Waiting for compression to finish\n"); -+ -+ lzma_end( &strm ); - - os.writeAdler32(); - } -@@ -154,6 +158,8 @@ Reader::Reader( string const & fileName, EncryptionKey const & key ) - } - } - -+ dPrintf( "Waiting for de-compression to finish\n"); -+ - lzma_end( &strm ); - - is.checkAdler32(); -@@ -192,13 +198,30 @@ bool Reader::get( string const & chunkId, string & chunkData, - } - - string generateFileName( Id const & id, string const & bundlesDir, -- bool createDirs ) -+ bool createDirs, size_t bundleMaxPayloadSize ) - { - string hex( toHex( ( unsigned char * ) &id, sizeof( id ) ) ); -- -- // TODO: make this scheme more flexible and allow it to scale, or at least -- // be configurable -- string level1( Dir::addPath( bundlesDir, hex.substr( 0, 2 ) ) ); -+ // TODO: We might still need a better scaling. -+ // Currently we assume ~18GB repository with 2MB bundles as reference. -+ // 2MB buldes results in ~25k files across 256 folders with ~100 bundles in each -+ // 32MB bundles results in ~1.5K bundles across 256 folders with ~5 files in each. -+ // Assuming above seems rational to have: -+ // 2 character folders for bundles 2MB and lower -+ // 1 character folders for up to 16M bundles -+ // no folders for 32MB bundles 32MB and above -+ // In mean time we will be using 2 characters to be able to restore with older versions -+ size_t BundleScale = 2; -+ -+ /* -+ if ( bundleMaxPayloadSize > ( 1024 * 1024 * 16 ) ) -+ BundleScale = 1; -+ if ( bundleMaxPayloadSize > ( 1024 * 1024 * 32 ) ) -+ BundleScale = 0; -+ -+ dPrintf( "Bundle size/scale %lu/%lu\n", bundleMaxPayloadSize, BundleScale); -+ */ -+ -+ string level1( Dir::addPath( bundlesDir, hex.substr( 0, BundleScale ) ) ); - - if ( createDirs && !Dir::exists( level1 ) ) - Dir::create( level1 ); -diff --git a/bundle.hh b/bundle.hh -index f6cb631..c1113cd 100644 ---- a/bundle.hh -+++ b/bundle.hh -@@ -65,7 +65,7 @@ public: - /// Compresses and writes the bundle to the given file. The operation is - /// time-consuming - calling this function from a worker thread could be - /// warranted -- void write( string const & fileName, EncryptionKey const & ); -+ void write( string const & fileName, EncryptionKey const & , size_t compression); - - /// Returns the current BundleInfo record - this is used for index files - BundleInfo const & getCurrentBundleInfo() const -@@ -100,7 +100,7 @@ public: - /// is true, any intermediate directories will be created if they don't exist - /// already - string generateFileName( Id const &, string const & bundlesDir, -- bool createDirs ); -+ bool createDirs, size_t bundleMaxPayloadSize ); - } - - #endif -diff --git a/chunk_storage.cc b/chunk_storage.cc -index 56c4add..764ddfd 100644 ---- a/chunk_storage.cc -+++ b/chunk_storage.cc -@@ -13,14 +13,14 @@ namespace ChunkStorage { - Writer::Writer( StorageInfo const & storageInfo, - EncryptionKey const & encryptionKey, - TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir, -- string const & indexDir, size_t maxCompressorsToRun ): -+ string const & indexDir, size_t maxCompressorsToRun, size_t compression ): - storageInfo( storageInfo ), encryptionKey( encryptionKey ), - tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ), - indexDir( indexDir ), hasCurrentBundleId( false ), -- maxCompressorsToRun( maxCompressorsToRun ), runningCompressors( 0 ) -+ maxCompressorsToRun( maxCompressorsToRun ), compression( compression ), runningCompressors( 0 ) - { -- verbosePrintf( "Using up to %zu thread(s) for compression\n", -- maxCompressorsToRun ); -+ verbosePrintf( "Using up to %zu thread(s) with compression level %zu\n", -+ maxCompressorsToRun, compression ); - } - - Writer::~Writer() -@@ -56,7 +56,7 @@ void Writer::commit() - { - PendingBundleRename & r = pendingBundleRenames[ x ]; - r.first->moveOverTo( Bundle::generateFileName( r.second, bundlesDir, -- true ) ); -+ true, storageInfo.bundle_max_payload_size() ) ); - } - - pendingBundleRenames.clear(); -@@ -119,6 +119,7 @@ void Writer::finishCurrentBundle() - - compressor->start(); - ++runningCompressors; -+ dPrintf( "Running compressors: %lu\n", runningCompressors ); - } - - void Writer::waitForAllCompressorsToFinish() -@@ -151,7 +152,7 @@ void * Writer::Compressor::Compressor::threadFunction() throw() - { - try - { -- bundleCreator->write( fileName, writer.encryptionKey ); -+ bundleCreator->write( fileName, writer.encryptionKey , writer.compression); - } - catch( std::exception & e ) - { -@@ -213,7 +214,7 @@ Bundle::Reader & Reader::getReaderFor( Bundle::Id const & id ) - { - // Load the bundle - reader = -- new Bundle::Reader( Bundle::generateFileName( id, bundlesDir, false ), -+ new Bundle::Reader( Bundle::generateFileName( id, bundlesDir, false, storageInfo.bundle_max_payload_size() ), - encryptionKey ); - } - -diff --git a/chunk_storage.hh b/chunk_storage.hh -index d94400e..fa68f16 100644 ---- a/chunk_storage.hh -+++ b/chunk_storage.hh -@@ -42,7 +42,7 @@ public: - /// automatically! - Writer( StorageInfo const &, EncryptionKey const &, - TmpMgr &, ChunkIndex & index, string const & bundlesDir, -- string const & indexDir, size_t maxCompressorsToRun ); -+ string const & indexDir, size_t maxCompressorsToRun, size_t compression ); - - /// Adds the given chunk to the store. If such a chunk has already existed - /// in the index, does nothing and returns false -@@ -97,6 +97,7 @@ private: - bool hasCurrentBundleId; - - size_t maxCompressorsToRun; -+ size_t compression; - Mutex runningCompressorsMutex; - Condition runningCompressorsCondition; - size_t runningCompressors; -diff --git a/file.cc b/file.cc -index b91b05e..ae12114 100644 ---- a/file.cc -+++ b/file.cc -@@ -12,8 +12,9 @@ - enum - { - // We employ a writing buffer to considerably speed up file operations when -- // they consists of many small writes. The default size for the buffer is 64k -- WriteBufferSize = 65536 -+ // they consists of many small writes. The default size for the buffer is 64KB -+ // TODO: Make write buffer customizable -+ WriteBufferSize = 64 * 1024 - }; - - bool File::exists( char const * filename ) throw() -diff --git a/zbackup.cc b/zbackup.cc -index 7729fe4..06d5abc 100644 ---- a/zbackup.cc -+++ b/zbackup.cc -@@ -75,12 +75,15 @@ StorageInfo ZBackupBase::loadStorageInfo() - - void ZBackupBase::initStorage( string const & storageDir, - string const & password, -- bool isEncrypted ) -+ bool isEncrypted, -+ size_t chunkMaxSizeKB, -+ size_t bundleMaxSizeMB ) - { - StorageInfo storageInfo; -- // TODO: make the following configurable -- storageInfo.set_chunk_max_size( 65536 ); -- storageInfo.set_bundle_max_payload_size( 0x200000 ); -+ storageInfo.set_chunk_max_size( chunkMaxSizeKB * 1024 ); -+ storageInfo.set_bundle_max_payload_size( bundleMaxSizeMB * 1024 * 1024 ); -+ -+ verbosePrintf( "Backup repository parameters: chunk_max[%u], bundle_max[%u]\n", storageInfo.chunk_max_size(), storageInfo.bundle_max_payload_size() ); - - if ( isEncrypted ) - EncryptionKey::generate( password, -@@ -127,10 +130,10 @@ string ZBackupBase::deriveStorageDirFromBackupsFile( string const & - } - - ZBackup::ZBackup( string const & storageDir, string const & password, -- size_t threads ): -+ size_t threads, size_t compression ): - ZBackupBase( storageDir, password ), - chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkIndex, -- getBundlesPath(), getIndexPath(), threads ) -+ getBundlesPath(), getIndexPath(), threads, compression ) - { - } - -@@ -145,6 +148,8 @@ void ZBackup::backupFromStdin( string const & outputFileName ) - Sha256 sha256; - BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter ); - -+ verbosePrintf( "Backup repository parameters: chunk_max[%u], bundle_max[%u]\n", storageInfo.chunk_max_size(), storageInfo.bundle_max_payload_size() ); -+ - time_t startTime = time( 0 ); - uint64_t totalDataSize = 0; - -@@ -297,6 +302,7 @@ void ZRestore::restoreToStdin( string const & inputFileName ) - DEF_EX( exNonEncryptedWithKey, "--non-encrypted and --password-file are incompatible", std::exception ) - DEF_EX( exSpecifyEncryptionOptions, "Specify either --password-file or --non-encrypted", std::exception ) - DEF_EX_STR( exInvalidThreadsValue, "Invalid threads value specified:", std::exception ) -+DEF_EX_STR( exInvalidCompressionValue, "Invalid compression level value specified:", std::exception ) - - int main( int argc, char *argv[] ) - { -@@ -304,10 +310,22 @@ int main( int argc, char *argv[] ) - { - char const * passwordFile = 0; - bool nonEncrypted = false; -+ - size_t const defaultThreads = getNumberOfCpus(); - size_t threads = defaultThreads; -- size_t const defaultCacheSizeMb = 40; -- size_t cacheSizeMb = defaultCacheSizeMb; -+ -+ size_t const defaultCompression = 6; -+ size_t compression = defaultCompression; -+ -+ size_t const defaultCacheSizeMB = 40; -+ size_t cacheSizeMB = defaultCacheSizeMB; -+ -+ size_t const defaultChunkMaxSizeKB = 64; -+ size_t chunkMaxSizeKB = defaultChunkMaxSizeKB; -+ -+ size_t const defaultBundleMaxSizeMB = 2; -+ size_t bundleMaxSizeMB = defaultBundleMaxSizeMB; -+ - vector< char const * > args; - - for( int x = 1; x < argc; ++x ) -@@ -333,12 +351,21 @@ int main( int argc, char *argv[] ) - ++x; - } - else -+ if ( strcmp( argv[ x ], "--compression" ) == 0 && x + 1 < argc ) -+ { -+ int n; -+ if ( sscanf( argv[ x + 1 ], "%zu %n", &compression, &n ) != 1 || -+ argv[ x + 1 ][ n ] || ( compression < 0 ) || ( compression > 19 ) ) -+ throw exInvalidCompressionValue( argv[ x + 1 ] ); -+ ++x; -+ } -+ else - if ( strcmp( argv[ x ], "--cache-size" ) == 0 && x + 1 < argc ) - { - char suffix[ 16 ]; - int n; - if ( sscanf( argv[ x + 1 ], "%zu %15s %n", -- &cacheSizeMb, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) -+ &cacheSizeMB, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) - { - // Check the suffix - for ( char * c = suffix; *c; ++c ) -@@ -363,6 +390,66 @@ int main( int argc, char *argv[] ) - } - } - else -+ if ( strcmp( argv[ x ], "--chunk-max-size" ) == 0 && x + 1 < argc ) -+ { -+ char suffix[ 16 ]; -+ int n; -+ if ( sscanf( argv[ x + 1 ], "%zu %15s %n", -+ &chunkMaxSizeKB, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) -+ { -+ // Check the suffix -+ for ( char * c = suffix; *c; ++c ) -+ *c = tolower( *c ); -+ -+ if ( strcmp( suffix, "kb" ) != 0 ) -+ { -+ fprintf( stderr, "Invalid suffix specified in chunk max size: %s. " -+ "The only supported suffix is 'kb' for kilobytes\n", -+ argv[ x + 1 ] ); -+ return EXIT_FAILURE; -+ } -+ -+ ++x; -+ } -+ else -+ { -+ fprintf( stderr, "Invalid chunk max size value specified: %s. " -+ "Must be a number with the 'kb' suffix, e.g. '16kb'\n", -+ argv[ x + 1 ] ); -+ return EXIT_FAILURE; -+ } -+ } -+ else -+ if ( strcmp( argv[ x ], "--bundle-max-size" ) == 0 && x + 1 < argc ) -+ { -+ char suffix[ 16 ]; -+ int n; -+ if ( sscanf( argv[ x + 1 ], "%zu %15s %n", -+ &bundleMaxSizeMB, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) -+ { -+ // Check the suffix -+ for ( char * c = suffix; *c; ++c ) -+ *c = tolower( *c ); -+ -+ if ( strcmp( suffix, "mb" ) != 0 ) -+ { -+ fprintf( stderr, "Invalid suffix specified in bundle max size: %s. " -+ "The only supported suffix is 'mb' for megabytes\n", -+ argv[ x + 1 ] ); -+ return EXIT_FAILURE; -+ } -+ -+ ++x; -+ } -+ else -+ { -+ fprintf( stderr, "Invalid bundle max size value specified: %s. " -+ "Must be a number with the 'mb' suffix, e.g. '128mb'\n", -+ argv[ x + 1 ] ); -+ return EXIT_FAILURE; -+ } -+ } -+ else - args.push_back( argv[ x ] ); - } - -@@ -381,12 +468,15 @@ int main( int argc, char *argv[] ) - " Flags: --non-encrypted|--password-file \n" - " --silent (default is verbose)\n" - " --threads (default is %zu on your system)\n" -+" --compression (default is %zu, 0-9 normal and 10-19 are extra preset)\n" - " --cache-size MB (default is %zu)\n" -+" --chunk-max-size KB (default is %zu)\n" -+" --bundle-max-size MB (default is %zu)\n" - " Commands:\n" - " init - initializes new storage;\n" - " backup - performs a backup from stdin;\n" - " restore - restores a backup to stdout.\n", *argv, -- defaultThreads, defaultCacheSizeMb ); -+ defaultThreads, defaultCompression, defaultCacheSizeMB, defaultChunkMaxSizeKB, defaultBundleMaxSizeMB ); - return EXIT_FAILURE; - } - -@@ -416,7 +506,7 @@ int main( int argc, char *argv[] ) - if ( !nonEncrypted && !passwordFile ) - throw exSpecifyEncryptionOptions(); - -- ZBackup::initStorage( args[ 1 ], passwordData, !nonEncrypted ); -+ ZBackup::initStorage( args[ 1 ], passwordData, !nonEncrypted, chunkMaxSizeKB, bundleMaxSizeMB ); - } - else - if ( strcmp( args[ 0 ], "backup" ) == 0 ) -@@ -429,7 +519,7 @@ int main( int argc, char *argv[] ) - return EXIT_FAILURE; - } - ZBackup zb( ZBackup::deriveStorageDirFromBackupsFile( args[ 1 ] ), -- passwordData, threads ); -+ passwordData, threads, compression ); - zb.backupFromStdin( args[ 1 ] ); - } - else -@@ -443,7 +533,7 @@ int main( int argc, char *argv[] ) - return EXIT_FAILURE; - } - ZRestore zr( ZRestore::deriveStorageDirFromBackupsFile( args[ 1 ] ), -- passwordData, cacheSizeMb * 1048576 ); -+ passwordData, cacheSizeMB * 1024 * 1024 ); - zr.restoreToStdin( args[ 1 ] ); - } - else -diff --git a/zbackup.hh b/zbackup.hh -index 1981ddf..71ed64a 100644 ---- a/zbackup.hh -+++ b/zbackup.hh -@@ -54,7 +54,7 @@ public: - - /// Creates new storage - static void initStorage( string const & storageDir, string const & password, -- bool isEncrypted ); -+ bool isEncrypted, size_t chunkMaxSizeKB, size_t bundleMaxSizeMB ); - - /// For a given file within the backups/ dir in the storage, returns its - /// storage dir or throws an exception -@@ -76,7 +76,7 @@ class ZBackup: public ZBackupBase - - public: - ZBackup( string const & storageDir, string const & password, -- size_t threads ); -+ size_t threads, size_t compression ); - - /// Backs up the data from stdin - void backupFromStdin( string const & outputFileName ); --- -1.9.2 - diff -Nru zbackup-1.2/debian/patches/0003-Switch-to-bytes-for-all-sizes-and-minor-cosmeitcs.patch zbackup-1.4.1+20150403/debian/patches/0003-Switch-to-bytes-for-all-sizes-and-minor-cosmeitcs.patch --- zbackup-1.2/debian/patches/0003-Switch-to-bytes-for-all-sizes-and-minor-cosmeitcs.patch 2014-04-27 05:49:48.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/patches/0003-Switch-to-bytes-for-all-sizes-and-minor-cosmeitcs.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,151 +0,0 @@ -From 2c5a3cc9c2b1253a1b4c840fd74c4b57dac19774 Mon Sep 17 00:00:00 2001 -From: "Eugene San (eugenesan)" -Date: Mon, 10 Feb 2014 16:02:15 +0200 -Subject: [PATCH 3/5] Switch to bytes for all sizes and minor cosmeitcs - ---- - chunk_storage.cc | 2 +- - zbackup.cc | 44 ++++++++++++++++++++++++++------------------ - 2 files changed, 27 insertions(+), 19 deletions(-) - -diff --git a/chunk_storage.cc b/chunk_storage.cc -index 764ddfd..4d1dfc4 100644 ---- a/chunk_storage.cc -+++ b/chunk_storage.cc -@@ -152,7 +152,7 @@ void * Writer::Compressor::Compressor::threadFunction() throw() - { - try - { -- bundleCreator->write( fileName, writer.encryptionKey , writer.compression); -+ bundleCreator->write( fileName, writer.encryptionKey, writer.compression ); - } - catch( std::exception & e ) - { -diff --git a/zbackup.cc b/zbackup.cc -index 06d5abc..1a9dfca 100644 ---- a/zbackup.cc -+++ b/zbackup.cc -@@ -76,12 +76,12 @@ StorageInfo ZBackupBase::loadStorageInfo() - void ZBackupBase::initStorage( string const & storageDir, - string const & password, - bool isEncrypted, -- size_t chunkMaxSizeKB, -- size_t bundleMaxSizeMB ) -+ size_t chunkMaxSize, -+ size_t bundleMaxSize ) - { - StorageInfo storageInfo; -- storageInfo.set_chunk_max_size( chunkMaxSizeKB * 1024 ); -- storageInfo.set_bundle_max_payload_size( bundleMaxSizeMB * 1024 * 1024 ); -+ storageInfo.set_chunk_max_size( chunkMaxSize ); -+ storageInfo.set_bundle_max_payload_size( bundleMaxSize ); - - verbosePrintf( "Backup repository parameters: chunk_max[%u], bundle_max[%u]\n", storageInfo.chunk_max_size(), storageInfo.bundle_max_payload_size() ); - -@@ -317,14 +317,14 @@ int main( int argc, char *argv[] ) - size_t const defaultCompression = 6; - size_t compression = defaultCompression; - -- size_t const defaultCacheSizeMB = 40; -- size_t cacheSizeMB = defaultCacheSizeMB; -+ size_t const defaultCacheSize = 40 * 1024 * 1024; -+ size_t cacheSize = defaultCacheSize; - -- size_t const defaultChunkMaxSizeKB = 64; -- size_t chunkMaxSizeKB = defaultChunkMaxSizeKB; -+ size_t const defaultChunkMaxSize = 64 * 1024; -+ size_t chunkMaxSize = defaultChunkMaxSize; - -- size_t const defaultBundleMaxSizeMB = 2; -- size_t bundleMaxSizeMB = defaultBundleMaxSizeMB; -+ size_t const defaultBundleMaxSize = 2 * 1024 * 1024; -+ size_t bundleMaxSize = defaultBundleMaxSize; - - vector< char const * > args; - -@@ -351,7 +351,7 @@ int main( int argc, char *argv[] ) - ++x; - } - else -- if ( strcmp( argv[ x ], "--compression" ) == 0 && x + 1 < argc ) -+ if ( strcmp( argv[ x ], "--compression-level" ) == 0 && x + 1 < argc ) - { - int n; - if ( sscanf( argv[ x + 1 ], "%zu %n", &compression, &n ) != 1 || -@@ -365,8 +365,10 @@ int main( int argc, char *argv[] ) - char suffix[ 16 ]; - int n; - if ( sscanf( argv[ x + 1 ], "%zu %15s %n", -- &cacheSizeMB, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) -+ &cacheSize, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) - { -+ // Convert to bytes -+ cacheSize *= 1024 * 1024; - // Check the suffix - for ( char * c = suffix; *c; ++c ) - *c = tolower( *c ); -@@ -395,8 +397,11 @@ int main( int argc, char *argv[] ) - char suffix[ 16 ]; - int n; - if ( sscanf( argv[ x + 1 ], "%zu %15s %n", -- &chunkMaxSizeKB, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) -+ &chunkMaxSize, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) - { -+ // Convert to bytes -+ chunkMaxSize *= 1024; -+ - // Check the suffix - for ( char * c = suffix; *c; ++c ) - *c = tolower( *c ); -@@ -425,8 +430,11 @@ int main( int argc, char *argv[] ) - char suffix[ 16 ]; - int n; - if ( sscanf( argv[ x + 1 ], "%zu %15s %n", -- &bundleMaxSizeMB, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) -+ &bundleMaxSize, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) - { -+ // Convert to bytes -+ bundleMaxSize *= 1024 * 1024; -+ - // Check the suffix - for ( char * c = suffix; *c; ++c ) - *c = tolower( *c ); -@@ -468,7 +476,7 @@ int main( int argc, char *argv[] ) - " Flags: --non-encrypted|--password-file \n" - " --silent (default is verbose)\n" - " --threads (default is %zu on your system)\n" --" --compression (default is %zu, 0-9 normal and 10-19 are extra preset)\n" -+" --compression-level (default is %zu, 0-9 normal and 10-19 are extra preset)\n" - " --cache-size MB (default is %zu)\n" - " --chunk-max-size KB (default is %zu)\n" - " --bundle-max-size MB (default is %zu)\n" -@@ -476,7 +484,7 @@ int main( int argc, char *argv[] ) - " init - initializes new storage;\n" - " backup - performs a backup from stdin;\n" - " restore - restores a backup to stdout.\n", *argv, -- defaultThreads, defaultCompression, defaultCacheSizeMB, defaultChunkMaxSizeKB, defaultBundleMaxSizeMB ); -+ defaultThreads, defaultCompression, defaultCacheSize, defaultChunkMaxSize, defaultBundleMaxSize ); - return EXIT_FAILURE; - } - -@@ -506,7 +514,7 @@ int main( int argc, char *argv[] ) - if ( !nonEncrypted && !passwordFile ) - throw exSpecifyEncryptionOptions(); - -- ZBackup::initStorage( args[ 1 ], passwordData, !nonEncrypted, chunkMaxSizeKB, bundleMaxSizeMB ); -+ ZBackup::initStorage( args[ 1 ], passwordData, !nonEncrypted, chunkMaxSize, bundleMaxSize ); - } - else - if ( strcmp( args[ 0 ], "backup" ) == 0 ) -@@ -533,7 +541,7 @@ int main( int argc, char *argv[] ) - return EXIT_FAILURE; - } - ZRestore zr( ZRestore::deriveStorageDirFromBackupsFile( args[ 1 ] ), -- passwordData, cacheSizeMB * 1024 * 1024 ); -+ passwordData, cacheSize ); - zr.restoreToStdin( args[ 1 ] ); - } - else --- -1.9.2 - diff -Nru zbackup-1.2/debian/patches/0004-revert-disabled-bundle-naming-scaling-code.patch zbackup-1.4.1+20150403/debian/patches/0004-revert-disabled-bundle-naming-scaling-code.patch --- zbackup-1.2/debian/patches/0004-revert-disabled-bundle-naming-scaling-code.patch 2014-04-27 05:49:48.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/patches/0004-revert-disabled-bundle-naming-scaling-code.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,48 +0,0 @@ -From 1e8e941c6c0bb9377405dd78cd08f4d169564a81 Mon Sep 17 00:00:00 2001 -From: "Eugene San (eugenesan)" -Date: Mon, 10 Feb 2014 17:24:59 +0200 -Subject: [PATCH 4/5] revert disabled bundle naming scaling code - ---- - bundle.cc | 25 ++++--------------------- - 1 file changed, 4 insertions(+), 21 deletions(-) - -diff --git a/bundle.cc b/bundle.cc -index 98b0238..dbe3eaf 100644 ---- a/bundle.cc -+++ b/bundle.cc -@@ -201,27 +201,10 @@ string generateFileName( Id const & id, string const & bundlesDir, - bool createDirs, size_t bundleMaxPayloadSize ) - { - string hex( toHex( ( unsigned char * ) &id, sizeof( id ) ) ); -- // TODO: We might still need a better scaling. -- // Currently we assume ~18GB repository with 2MB bundles as reference. -- // 2MB buldes results in ~25k files across 256 folders with ~100 bundles in each -- // 32MB bundles results in ~1.5K bundles across 256 folders with ~5 files in each. -- // Assuming above seems rational to have: -- // 2 character folders for bundles 2MB and lower -- // 1 character folders for up to 16M bundles -- // no folders for 32MB bundles 32MB and above -- // In mean time we will be using 2 characters to be able to restore with older versions -- size_t BundleScale = 2; -- -- /* -- if ( bundleMaxPayloadSize > ( 1024 * 1024 * 16 ) ) -- BundleScale = 1; -- if ( bundleMaxPayloadSize > ( 1024 * 1024 * 32 ) ) -- BundleScale = 0; -- -- dPrintf( "Bundle size/scale %lu/%lu\n", bundleMaxPayloadSize, BundleScale); -- */ -- -- string level1( Dir::addPath( bundlesDir, hex.substr( 0, BundleScale ) ) ); -+ -+ // TODO: make this scheme more flexible and allow it to scale, or at least -+ // be configurable -+ string level1( Dir::addPath( bundlesDir, hex.substr( 0, 2 ) ) ); - - if ( createDirs && !Dir::exists( level1 ) ) - Dir::create( level1 ); --- -1.9.2 - diff -Nru zbackup-1.2/debian/patches/0005-distinguish-compression-level-per-algorithm-and-rela.patch zbackup-1.4.1+20150403/debian/patches/0005-distinguish-compression-level-per-algorithm-and-rela.patch --- zbackup-1.2/debian/patches/0005-distinguish-compression-level-per-algorithm-and-rela.patch 2014-04-27 05:49:48.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/patches/0005-distinguish-compression-level-per-algorithm-and-rela.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,229 +0,0 @@ -From 9e13d6b696cb68ba522b3e0841d106f2d8b5d047 Mon Sep 17 00:00:00 2001 -From: "Eugene San (eugenesan)" -Date: Thu, 27 Mar 2014 21:48:46 +0200 -Subject: [PATCH 5/5] distinguish compression level per algorithm and related - cosmetics - ---- - bundle.cc | 12 ++++++------ - bundle.hh | 4 ++-- - chunk_storage.cc | 12 ++++++------ - chunk_storage.hh | 4 ++-- - zbackup.cc | 18 +++++++++--------- - 5 files changed, 25 insertions(+), 25 deletions(-) - -diff --git a/bundle.cc b/bundle.cc -index dbe3eaf..71bcea0 100644 ---- a/bundle.cc -+++ b/bundle.cc -@@ -28,7 +28,7 @@ void Creator::addChunk( string const & id, void const * data, size_t size ) - payload.append( ( char const * ) data, size ); - } - --void Creator::write( std::string const & fileName, EncryptionKey const & key, size_t compression ) -+void Creator::write( std::string const & fileName, EncryptionKey const & key, size_t compressionLevel ) - { - EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv ); - -@@ -43,8 +43,8 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key, si - - // Compress - -- uint32_t preset = ( compression > 9 ) ? ( compression - 10 ) | LZMA_PRESET_EXTREME : compression; -- dPrintf( "Compression level: %lu, lzma preset %u\n", compression, preset); -+ uint32_t preset = ( compressionLevel > 9 ) ? ( compressionLevel - 10 ) | LZMA_PRESET_EXTREME : compressionLevel; -+ dPrintf( "LZMA Compression level: %lu, preset %u\n", compressionLevel, preset ); - - lzma_stream strm = LZMA_STREAM_INIT; - lzma_ret ret; -@@ -84,7 +84,7 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key, si - CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret ); - } - -- dPrintf( "Waiting for compression to finish\n"); -+ dPrintf( "Waiting for compression to finish\n" ); - - lzma_end( &strm ); - -@@ -158,7 +158,7 @@ Reader::Reader( string const & fileName, EncryptionKey const & key ) - } - } - -- dPrintf( "Waiting for de-compression to finish\n"); -+ dPrintf( "Waiting for de-compression to finish\n" ); - - lzma_end( &strm ); - -@@ -198,7 +198,7 @@ bool Reader::get( string const & chunkId, string & chunkData, - } - - string generateFileName( Id const & id, string const & bundlesDir, -- bool createDirs, size_t bundleMaxPayloadSize ) -+ bool createDirs ) - { - string hex( toHex( ( unsigned char * ) &id, sizeof( id ) ) ); - -diff --git a/bundle.hh b/bundle.hh -index c1113cd..2afb636 100644 ---- a/bundle.hh -+++ b/bundle.hh -@@ -65,7 +65,7 @@ public: - /// Compresses and writes the bundle to the given file. The operation is - /// time-consuming - calling this function from a worker thread could be - /// warranted -- void write( string const & fileName, EncryptionKey const & , size_t compression); -+ void write( string const & fileName, EncryptionKey const &, size_t compressionLevel ); - - /// Returns the current BundleInfo record - this is used for index files - BundleInfo const & getCurrentBundleInfo() const -@@ -100,7 +100,7 @@ public: - /// is true, any intermediate directories will be created if they don't exist - /// already - string generateFileName( Id const &, string const & bundlesDir, -- bool createDirs, size_t bundleMaxPayloadSize ); -+ bool createDirs ); - } - - #endif -diff --git a/chunk_storage.cc b/chunk_storage.cc -index 4d1dfc4..c8370f1 100644 ---- a/chunk_storage.cc -+++ b/chunk_storage.cc -@@ -13,14 +13,14 @@ namespace ChunkStorage { - Writer::Writer( StorageInfo const & storageInfo, - EncryptionKey const & encryptionKey, - TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir, -- string const & indexDir, size_t maxCompressorsToRun, size_t compression ): -+ string const & indexDir, size_t maxCompressorsToRun, size_t compressionLevel ): - storageInfo( storageInfo ), encryptionKey( encryptionKey ), - tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ), - indexDir( indexDir ), hasCurrentBundleId( false ), -- maxCompressorsToRun( maxCompressorsToRun ), compression( compression ), runningCompressors( 0 ) -+ maxCompressorsToRun( maxCompressorsToRun ), compressionLevel( compressionLevel ), runningCompressors( 0 ) - { - verbosePrintf( "Using up to %zu thread(s) with compression level %zu\n", -- maxCompressorsToRun, compression ); -+ maxCompressorsToRun, compressionLevel ); - } - - Writer::~Writer() -@@ -56,7 +56,7 @@ void Writer::commit() - { - PendingBundleRename & r = pendingBundleRenames[ x ]; - r.first->moveOverTo( Bundle::generateFileName( r.second, bundlesDir, -- true, storageInfo.bundle_max_payload_size() ) ); -+ true ) ); - } - - pendingBundleRenames.clear(); -@@ -152,7 +152,7 @@ void * Writer::Compressor::Compressor::threadFunction() throw() - { - try - { -- bundleCreator->write( fileName, writer.encryptionKey, writer.compression ); -+ bundleCreator->write( fileName, writer.encryptionKey, writer.compressionLevel ); - } - catch( std::exception & e ) - { -@@ -214,7 +214,7 @@ Bundle::Reader & Reader::getReaderFor( Bundle::Id const & id ) - { - // Load the bundle - reader = -- new Bundle::Reader( Bundle::generateFileName( id, bundlesDir, false, storageInfo.bundle_max_payload_size() ), -+ new Bundle::Reader( Bundle::generateFileName( id, bundlesDir, false ), - encryptionKey ); - } - -diff --git a/chunk_storage.hh b/chunk_storage.hh -index fa68f16..469593a 100644 ---- a/chunk_storage.hh -+++ b/chunk_storage.hh -@@ -42,7 +42,7 @@ public: - /// automatically! - Writer( StorageInfo const &, EncryptionKey const &, - TmpMgr &, ChunkIndex & index, string const & bundlesDir, -- string const & indexDir, size_t maxCompressorsToRun, size_t compression ); -+ string const & indexDir, size_t maxCompressorsToRun, size_t compressionLevel ); - - /// Adds the given chunk to the store. If such a chunk has already existed - /// in the index, does nothing and returns false -@@ -97,7 +97,7 @@ private: - bool hasCurrentBundleId; - - size_t maxCompressorsToRun; -- size_t compression; -+ size_t compressionLevel; - Mutex runningCompressorsMutex; - Condition runningCompressorsCondition; - size_t runningCompressors; -diff --git a/zbackup.cc b/zbackup.cc -index 1a9dfca..877ff46 100644 ---- a/zbackup.cc -+++ b/zbackup.cc -@@ -130,10 +130,10 @@ string ZBackupBase::deriveStorageDirFromBackupsFile( string const & - } - - ZBackup::ZBackup( string const & storageDir, string const & password, -- size_t threads, size_t compression ): -+ size_t threads, size_t compressionLevel ): - ZBackupBase( storageDir, password ), - chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkIndex, -- getBundlesPath(), getIndexPath(), threads, compression ) -+ getBundlesPath(), getIndexPath(), threads, compressionLevel ) - { - } - -@@ -156,7 +156,7 @@ void ZBackup::backupFromStdin( string const & outputFileName ) - for ( ; ; ) - { - size_t toRead = backupCreator.getInputBufferSize(); --// dPrintf( "Reading up to %u bytes from stdin\n", toRead ); -+ //dPrintf( "Reading up to %u bytes from stdin\n", toRead ); - - void * inputBuffer = backupCreator.getInputBuffer(); - size_t rd = fread( inputBuffer, 1, toRead, stdin ); -@@ -315,7 +315,7 @@ int main( int argc, char *argv[] ) - size_t threads = defaultThreads; - - size_t const defaultCompression = 6; -- size_t compression = defaultCompression; -+ size_t compressionLevel = defaultCompression; - - size_t const defaultCacheSize = 40 * 1024 * 1024; - size_t cacheSize = defaultCacheSize; -@@ -351,11 +351,11 @@ int main( int argc, char *argv[] ) - ++x; - } - else -- if ( strcmp( argv[ x ], "--compression-level" ) == 0 && x + 1 < argc ) -+ if ( strcmp( argv[ x ], "--lzma-compression-level" ) == 0 && x + 1 < argc ) - { - int n; -- if ( sscanf( argv[ x + 1 ], "%zu %n", &compression, &n ) != 1 || -- argv[ x + 1 ][ n ] || ( compression < 0 ) || ( compression > 19 ) ) -+ if ( sscanf( argv[ x + 1 ], "%zu %n", &compressionLevel, &n ) != 1 || -+ argv[ x + 1 ][ n ] || ( compressionLevel > 19 ) ) - throw exInvalidCompressionValue( argv[ x + 1 ] ); - ++x; - } -@@ -476,7 +476,7 @@ int main( int argc, char *argv[] ) - " Flags: --non-encrypted|--password-file \n" - " --silent (default is verbose)\n" - " --threads (default is %zu on your system)\n" --" --compression-level (default is %zu, 0-9 normal and 10-19 are extra preset)\n" -+" --lzma-compression-level (default is %zu, 0-9 normal and 10-19 are extra preset)\n" - " --cache-size MB (default is %zu)\n" - " --chunk-max-size KB (default is %zu)\n" - " --bundle-max-size MB (default is %zu)\n" -@@ -527,7 +527,7 @@ int main( int argc, char *argv[] ) - return EXIT_FAILURE; - } - ZBackup zb( ZBackup::deriveStorageDirFromBackupsFile( args[ 1 ] ), -- passwordData, threads, compression ); -+ passwordData, threads, compressionLevel ); - zb.backupFromStdin( args[ 1 ] ); - } - else --- -1.9.2 - diff -Nru zbackup-1.2/debian/patches/Corrected-the-umask-setting-for-temporary-file-creation.patch zbackup-1.4.1+20150403/debian/patches/Corrected-the-umask-setting-for-temporary-file-creation.patch --- zbackup-1.2/debian/patches/Corrected-the-umask-setting-for-temporary-file-creation.patch 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/patches/Corrected-the-umask-setting-for-temporary-file-creation.patch 2015-06-26 13:59:43.000000000 +0000 @@ -0,0 +1,37 @@ +From 4c73159ad5897b87478dcc1eccf3b67f95ef542d Mon Sep 17 00:00:00 2001 +From: Brian Klahn +Date: Tue, 14 Apr 2015 13:20:39 -0400 +Subject: [PATCH] Corrected the umask setting for temporary file creation. + Added reset to old umask, afterwards. + +Perhaps explicitly setting the file mode, after the file is created +(before potentially sensitive data is written to the file.), could be +done, instead of using umask. +e.g. +fchmod(fd, S_IRUSR | S_IWUSR ); +--- + tmp_mgr.cc | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/tmp_mgr.cc b/tmp_mgr.cc +index a022f99..3d9532c 100644 +--- a/tmp_mgr.cc ++++ b/tmp_mgr.cc +@@ -44,8 +44,13 @@ sptr< TemporaryFile > TmpMgr::makeTemporaryFile() + { + string name( Dir::addPath( path, "XXXXXX") ); + +- umask( S_IRUSR | S_IWUSR | S_IRGRP ); ++ // Set the umask to remove permisions of grp and other ++ // in case an insecure (older) mkstemp is used. ++ mode_t old_mask = umask( S_IRWXG | S_IRWXO ); ++ // Create the temporary file. + int fd = mkstemp( &name[ 0 ] ); ++ // Reset the umask to the previous setting ++ umask(old_mask); + + if ( fd == -1 || close( fd ) != 0 ) + throw exCantCreate( path ); +-- +2.4.5 + diff -Nru zbackup-1.2/debian/patches/series zbackup-1.4.1+20150403/debian/patches/series --- zbackup-1.2/debian/patches/series 2014-04-27 05:50:54.000000000 +0000 +++ zbackup-1.4.1+20150403/debian/patches/series 2015-06-26 14:00:30.000000000 +0000 @@ -1,5 +1 @@ -0001-Add-support-for-big-endian-architectures.patch -0002-customizable-storage-and-runtime-parameters-and-some.patch -0003-Switch-to-bytes-for-all-sizes-and-minor-cosmeitcs.patch -0004-revert-disabled-bundle-naming-scaling-code.patch -0005-distinguish-compression-level-per-algorithm-and-rela.patch +Corrected-the-umask-setting-for-temporary-file-creation.patch diff -Nru zbackup-1.2/debug.cc zbackup-1.4.1+20150403/debug.cc --- zbackup-1.2/debug.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/debug.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,4 +1,11 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE bool verboseMode = true; + +#ifndef NDEBUG +#ifdef HAVE_LIBUNWIND +#include "debug.hh" + +#endif +#endif diff -Nru zbackup-1.2/debug.hh zbackup-1.4.1+20150403/debug.hh --- zbackup-1.2/debug.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/debug.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,20 +1,38 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef DEBUG_HH_INCLUDED__ -#define DEBUG_HH_INCLUDED__ +#ifndef DEBUG_HH_INCLUDED +#define DEBUG_HH_INCLUDED #include +#include // Macros we use to output debugging information +#define __CLASS typeid( *this ).name() + #ifndef NDEBUG -#define dPrintf( ... ) (fprintf( stderr, __VA_ARGS__ )) +#define __FILE_BASE (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) + +#define dPrintf( ... ) ({ fprintf( stderr, "[DEBUG] at %s( %s:%d ): ", __func__,\ + __FILE_BASE, __LINE__ );\ + fprintf( stderr, __VA_ARGS__ ); }) + +#ifdef HAVE_LIBUNWIND +#define UNW_LOCAL_ONLY +#include + +// TODO: pretty backtraces +#define dPrintBacktrace( ... ) () +#else +#define dPrintBacktrace( ... ) () +#endif #else #define dPrintf( ... ) +#define dPrintBacktrace( ... ) () #endif diff -Nru zbackup-1.2/dir.cc zbackup-1.4.1+20150403/dir.cc --- zbackup-1.2/dir.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/dir.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include @@ -66,6 +66,13 @@ return dirname( copy.data() ); } +bool isDirEmpty( string const & path ) +{ + Listing lst(path); + Entry tmp; + return !lst.getNext(tmp); +} + Listing::Listing( string const & dirName ): dirName( dirName ) { dir = opendir( dirName.c_str() ); diff -Nru zbackup-1.2/dir.hh zbackup-1.4.1+20150403/dir.hh --- zbackup-1.2/dir.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/dir.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef DIR_HH_INCLUDED__ -#define DIR_HH_INCLUDED__ +#ifndef DIR_HH_INCLUDED +#define DIR_HH_INCLUDED #include #include @@ -42,6 +42,9 @@ /// Returns the directory part of the given path string getDirName( string const & ); +/// Checkes whether directory is empty +bool isDirEmpty( string const & ); + /// A separator used to separate names in the path. inline char separator() { return '/'; } diff -Nru zbackup-1.2/encrypted_file.cc zbackup-1.4.1+20150403/encrypted_file.cc --- zbackup-1.2/encrypted_file.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/encrypted_file.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include @@ -9,6 +9,7 @@ #include "endian.hh" #include "page_size.hh" #include "random.hh" +#include "debug.hh" namespace EncryptedFile { @@ -22,6 +23,7 @@ buffer( std::max( getPageSize(), ( unsigned ) BlockSize * 2 ) ), fill( 0 ), remainder( 0 ), backedUp( false ) { + dPrintf( "Loading %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" ); if ( key.hasKey() ) { memcpy( iv, iv_, sizeof( iv ) ); @@ -239,6 +241,7 @@ file( fileName, UnbufferedFile::WriteOnly ), filePos( 0 ), key( key ), buffer( getPageSize() ), start( buffer.data() ), avail( 0 ), backedUp( false ) { + dPrintf( "Saving %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" ); if ( key.hasKey() ) memcpy( iv, iv_, sizeof( iv ) ); } @@ -341,7 +344,7 @@ if ( key.hasKey() ) { char iv[ Encryption::IvSize ]; - Random::genaratePseudo( iv, sizeof( iv ) ); + Random::generatePseudo( iv, sizeof( iv ) ); write( iv, sizeof( iv ) ); } } diff -Nru zbackup-1.2/encrypted_file.hh zbackup-1.4.1+20150403/encrypted_file.hh --- zbackup-1.2/encrypted_file.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/encrypted_file.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef ENCRYPTED_FILE_HH_INCLUDED__ -#define ENCRYPTED_FILE_HH_INCLUDED__ +#ifndef ENCRYPTED_FILE_HH_INCLUDED +#define ENCRYPTED_FILE_HH_INCLUDED #include #include diff -Nru zbackup-1.2/encryption.cc zbackup-1.4.1+20150403/encryption.cc --- zbackup-1.2/encryption.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/encryption.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include diff -Nru zbackup-1.2/encryption.hh zbackup-1.4.1+20150403/encryption.hh --- zbackup-1.2/encryption.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/encryption.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef ENCRYPTION_HH_INCLUDED__ -#define ENCRYPTION_HH_INCLUDED__ +#ifndef ENCRYPTION_HH_INCLUDED +#define ENCRYPTION_HH_INCLUDED #include #include diff -Nru zbackup-1.2/encryption_key.cc zbackup-1.4.1+20150403/encryption_key.cc --- zbackup-1.2/encryption_key.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/encryption_key.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include @@ -66,12 +66,13 @@ } void EncryptionKey::generate( string const & password, - EncryptionKeyInfo & info ) + EncryptionKeyInfo & info, + EncryptionKey & encryptionkey ) { // Use this buf for salts - char buf[ 16 ]; + char buf[ KeySize ]; - Random::genaratePseudo( buf, sizeof( buf ) ); + Random::generatePseudo( buf, sizeof( buf ) ); info.set_salt( buf, sizeof( buf ) ); info.set_rounds( 10000 ); // TODO: make this configurable @@ -79,11 +80,13 @@ deriveKey( password, info, derivedKey, sizeof( derivedKey ) ); char key[ KeySize ]; - - Random::genarateTrue( key, sizeof( key ) ); + if ( encryptionkey.hasKey() ) + memcpy( key, encryptionkey.getKey(), KeySize ); + else + Random::generateTrue( key, sizeof( key ) ); // Fill in the HMAC verification part - Random::genaratePseudo( buf, sizeof( buf ) ); + Random::generatePseudo( buf, sizeof( buf ) ); info.set_key_check_input( buf, sizeof( buf ) ); info.set_key_check_hmac( calculateKeyHmac( key, sizeof( key ), info.key_check_input() ) ); diff -Nru zbackup-1.2/encryption_key.hh zbackup-1.4.1+20150403/encryption_key.hh --- zbackup-1.2/encryption_key.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/encryption_key.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef ENCRYPTION_KEY_HH_INCLUDED__ -#define ENCRYPTION_KEY_HH_INCLUDED__ +#ifndef ENCRYPTION_KEY_HH_INCLUDED +#define ENCRYPTION_KEY_HH_INCLUDED #include #include @@ -41,7 +41,8 @@ { return sizeof( key ); } /// Generates new key info using the given password - static void generate( string const & password, EncryptionKeyInfo & ); + static void generate( string const & password, EncryptionKeyInfo &, + EncryptionKey & encryptionkey ); /// Returns a static instance without any key set static EncryptionKey const & noKey(); diff -Nru zbackup-1.2/endian.hh zbackup-1.4.1+20150403/endian.hh --- zbackup-1.2/endian.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/endian.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,20 +1,40 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef ENDIAN_HH_INCLUDED__ -#define ENDIAN_HH_INCLUDED__ +#ifndef ENDIAN_HH_INCLUDED +#define ENDIAN_HH_INCLUDED #include #include + #ifdef __APPLE__ #include +#include + +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) + +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) + +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) +//__APPLE__ + +#elif __FreeBSD__ +#include + #else #include #endif -#if __BYTE_ORDER != __LITTLE_ENDIAN -#error Please add support for architectures different from little-endian. -#endif +#if __BYTE_ORDER == __LITTLE_ENDIAN /// Converts the given host-order value to big-endian value inline uint32_t toBigEndian( uint32_t v ) { return htonl( v ); } @@ -25,4 +45,24 @@ inline uint32_t fromLittleEndian( uint32_t v ) { return v; } inline uint64_t fromLittleEndian( uint64_t v ) { return v; } +#elif __BYTE_ORDER == __BIG_ENDIAN + +// Note: the functions used are non-standard. Add more ifdefs if needed + +/// Converts the given host-order value to big-endian value +inline uint32_t toBigEndian( uint32_t v ) { return v; } +/// Converts the given host-order value to little-endian value +inline uint32_t toLittleEndian( uint32_t v ) { return htole32( v ); } +inline uint64_t toLittleEndian( uint64_t v ) { return htole64( v ); } +/// Converts the given little-endian value to host-order value +inline uint32_t fromLittleEndian( uint32_t v ) { return le32toh( v ); } +inline uint64_t fromLittleEndian( uint64_t v ) { return le64toh( v ); } + +#else + +#error Please add support for architectures different from little-endian and\ + big-endian. + +#endif + #endif diff -Nru zbackup-1.2/ex.hh zbackup-1.4.1+20150403/ex.hh --- zbackup-1.2/ex.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/ex.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef EX_HH_INCLUDED__ -#define EX_HH_INCLUDED__ +#ifndef EX_HH_INCLUDED +#define EX_HH_INCLUDED #include #include diff -Nru zbackup-1.2/file.cc zbackup-1.4.1+20150403/file.cc --- zbackup-1.2/file.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/file.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,13 +1,21 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include #include #include #include +#if defined( __APPLE__ ) || defined( __OpenBSD__ ) || defined(__FreeBSD__) + #include +#else + #include +#endif +#include +#include #include "file.hh" +#include "utils.hh" enum { @@ -36,10 +44,56 @@ } void File::rename( std::string const & from, - std::string const & to ) throw( exCantRename ) + std::string const & to ) throw( exCantRename, + exCantErase ) { - if ( ::rename( from.c_str(), to.c_str() ) != 0 ) - throw exCantRename( from + " to " + to ); + int res = 0; + res = ::rename( from.c_str(), to.c_str() ); + if ( 0 != res ) + { + if ( EXDEV == errno ) + { + int read_fd; + int write_fd; + struct stat stat_buf; + off_t offset = 0; + + /* Open the input file. */ + read_fd = ::open( from.c_str(), O_RDONLY ); + /* Stat the input file to obtain its size. */ + if ( fstat( read_fd, &stat_buf ) != 0 ) + throw exCantRename( from + " to " + to ); + /* Open the output file for writing, with the same permissions as the + source file. */ + write_fd = ::open( to.c_str(), O_WRONLY | O_CREAT, stat_buf.st_mode ); + /* Blast the bytes from one file to the other. */ + #if defined( __APPLE__ ) + if ( -1 == sendfile( write_fd, read_fd, offset, &stat_buf.st_size, NULL, 0 ) ) + throw exCantRename( from + " to " + to ); + #elif defined( __OpenBSD__ ) || defined(__FreeBSD__) + + size_t BUFSIZE = 4096, size; + char buf[BUFSIZE]; + + while ( ( size = ::read( read_fd, buf, BUFSIZE ) ) != -1 && size != 0 ) + ::write( write_fd, buf, size ); + + if ( size == -1 ) + throw exCantRename( from + " to " + to ); + + #else + if ( -1 == sendfile( write_fd, read_fd, &offset, stat_buf.st_size ) ) + throw exCantRename( from + " to " + to ); + #endif + + /* Close up. */ + ::close( read_fd ); + ::close( write_fd ); + File::erase ( from ); + } + else + throw exCantRename( from + " to " + to ); + } } void File::open( char const * filename, OpenMode mode ) throw( exCantOpen ) @@ -64,6 +118,28 @@ throw exCantOpen( std::string( filename ) + ": " + strerror( errno ) ); } +void File::open( int fd, OpenMode mode ) throw( exCantOpen ) +{ + char const * m; + + switch( mode ) + { + case Update: + m = "r+b"; + break; + case WriteOnly: + m = "wb"; + break; + default: + m = "rb"; + } + + f = fdopen( fd, m ); + + if ( !f ) + throw exCantOpen( "fd#" + Utils::numberToString( fd ) + ": " + strerror( errno ) ); +} + File::File( char const * filename, OpenMode mode ) throw( exCantOpen ): writeBuffer( 0 ) { @@ -76,6 +152,12 @@ open( filename.c_str(), mode ); } +File::File( int fd, OpenMode mode ) + throw( exCantOpen ): writeBuffer( 0 ) +{ + open( fd, mode ); +} + void File::read( void * buf, size_t size ) throw( exReadError, exWriteError ) { if ( !size ) @@ -262,6 +344,13 @@ return feof( f ); } +int File::error() throw( exReadError ) +{ + int result = ferror( f ); + + return result; +} + FILE * File::file() throw( exWriteError ) { flushWriteBuffer(); diff -Nru zbackup-1.2/file.hh zbackup-1.4.1+20150403/file.hh --- zbackup-1.2/file.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/file.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef FILE_HH_INCLUDED__ -#define FILE_HH_INCLUDED__ +#ifndef FILE_HH_INCLUDED +#define FILE_HH_INCLUDED #include #include @@ -45,6 +45,9 @@ File( std::string const & filename, OpenMode ) throw( exCantOpen ); + File( int fd, OpenMode ) + throw( exCantOpen ); + /// Reads the number of bytes to the buffer, throws an error if it /// failed to fill the whole buffer (short read, i/o error etc) void read( void * buf, size_t size ) throw( exReadError, exWriteError ); @@ -108,6 +111,9 @@ /// Returns true if end-of-file condition is set bool eof() throw( exWriteError ); + /// Returns ferror + int error() throw( exReadError ); + /// Returns the underlying FILE * record, so other operations can be /// performed on it FILE * file() throw( exWriteError ); @@ -132,7 +138,8 @@ /// Renames the given file static void rename( std::string const & from, - std::string const & to ) throw( exCantRename ); + std::string const & to ) throw( exCantRename, + exCantErase ); /// Throwing this class instead of exReadError will make the description /// include the file name @@ -153,6 +160,7 @@ private: void open( char const * filename, OpenMode ) throw( exCantOpen ); + void open( int fd, OpenMode ) throw( exCantOpen ); void flushWriteBuffer() throw( exWriteError ); void releaseWriteBuffer() throw( exWriteError ); }; diff -Nru zbackup-1.2/.gitignore zbackup-1.4.1+20150403/.gitignore --- zbackup-1.2/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/.gitignore 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,8 @@ +*.o +CMakeFiles/ +CMakeCache.txt +Makefile +cmake_install.cmake +/zbackup.pb.cc +/zbackup.pb.h +/zbackup diff -Nru zbackup-1.2/hex.cc zbackup-1.4.1+20150403/hex.cc --- zbackup-1.2/hex.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/hex.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "hex.hh" diff -Nru zbackup-1.2/hex.hh zbackup-1.4.1+20150403/hex.hh --- zbackup-1.2/hex.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/hex.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef HEX_HH_INCLUDED__ -#define HEX_HH_INCLUDED__ +#ifndef HEX_HH_INCLUDED +#define HEX_HH_INCLUDED #include diff -Nru zbackup-1.2/index_file.cc zbackup-1.4.1+20150403/index_file.cc --- zbackup-1.2/index_file.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/index_file.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include diff -Nru zbackup-1.2/index_file.hh zbackup-1.4.1+20150403/index_file.hh --- zbackup-1.2/index_file.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/index_file.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef INDEX_FILE_HH_INCLUDED__ -#define INDEX_FILE_HH_INCLUDED__ +#ifndef INDEX_FILE_HH_INCLUDED +#define INDEX_FILE_HH_INCLUDED #include #include diff -Nru zbackup-1.2/LICENSE zbackup-1.4.1+20150403/LICENSE --- zbackup-1.2/LICENSE 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/LICENSE 1970-01-01 00:00:00.000000000 +0000 @@ -1,339 +0,0 @@ -GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - ZBackup, a versatile deduplicating backup tool - Copyright (C) 2013 zbackup - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - {signature of Ty Coon}, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff -Nru zbackup-1.2/licenses/LICENSE zbackup-1.4.1+20150403/licenses/LICENSE --- zbackup-1.2/licenses/LICENSE 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/licenses/LICENSE 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,58 @@ +ZBackup, a versatile deduplicating backup tool +Copyright (c) 2012-2014 Konstantin Isakov and ZBackup +contributors, see CONTRIBUTORS + +This program is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License (GPL) as published by +the Free Software Foundation; either version 2 of the License, or (at +your option) any later version. The full text of versions 2 and 3 of +the GPL can be found respectively in the files LICENSE-GPLV2 and +LICENSE-GPLV3. + +EXCEPTION: This distribution of ZBackup may be linked against OpenSSL +according to the terms of the section below entitled "OpenSSL Exception." + +ADDITION: This distribution of ZBackup uses sources from CMake project +which is distributed under the OSI-approved BSD 3-clause License. +See LICENSE-CMAKE for details. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + + _OpenSSL Exception_ + +0. Definitions + +"ZBackup" means ZBackup software licensed under version 2 or any later +version of the GNU General Public License (collectively, "GPL"), or a +work based on such software and licensed under the GPL. + +"OpenSSL" means OpenSSL toolkit software distributed by the OpenSSL +Project and licensed under the OpenSSL Licenses, or a work based on such +software and licensed under the OpenSSL Licenses. + +"OpenSSL Licenses" means the OpenSSL License and Original SSLeay License +under which the OpenSSL Project distributes the OpenSSL toolkit software, +as those licenses appear in the file LICENSE-OPENSSL. + +1. Exception + +You have permission to copy, modify, propagate, and distribute a work +formed by combining OpenSSL with ZBackup, or a work derivative of such a +combination, even if such copying, modification, propagation, or +distribution would otherwise violate the terms of the GPL. You must +comply with the GPL in all respects for all of the code used other than +OpenSSL. + +You may include this OpenSSL Exception and its grant of permissions when +you distribute ZBackup. Inclusion of this notice with such a distribution +constitutes a grant of such permission. If you do not wish to grant these +permissions, remove this section entitled "OpenSSL Exception" from your +distribution. diff -Nru zbackup-1.2/licenses/LICENSE-CMAKE zbackup-1.4.1+20150403/licenses/LICENSE-CMAKE --- zbackup-1.2/licenses/LICENSE-CMAKE 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/licenses/LICENSE-CMAKE 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,57 @@ +CMake - Cross Platform Makefile Generator +Copyright 2000-2014 Kitware, Inc. +Copyright 2000-2011 Insight Software Consortium +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +* Neither the names of Kitware, Inc., the Insight Software Consortium, + nor the names of their contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------ + +The above copyright and license notice applies to distributions of +CMake in source and binary form. Some source files contain additional +notices of original copyright by their contributors; see each source +for details. Third-party software packages supplied with CMake under +compatible licenses provide their own copyright notices documented in +corresponding subdirectories. + +------------------------------------------------------------------------------ + +CMake was initially developed by Kitware with the following sponsorship: + + * National Library of Medicine at the National Institutes of Health + as part of the Insight Segmentation and Registration Toolkit (ITK). + + * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel + Visualization Initiative. + + * National Alliance for Medical Image Computing (NAMIC) is funded by the + National Institutes of Health through the NIH Roadmap for Medical Research, + Grant U54 EB005149. + + * Kitware, Inc. diff -Nru zbackup-1.2/licenses/LICENSE-GPLV2 zbackup-1.4.1+20150403/licenses/LICENSE-GPLV2 --- zbackup-1.2/licenses/LICENSE-GPLV2 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/licenses/LICENSE-GPLV2 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff -Nru zbackup-1.2/licenses/LICENSE-GPLV3 zbackup-1.4.1+20150403/licenses/LICENSE-GPLV3 --- zbackup-1.2/licenses/LICENSE-GPLV3 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/licenses/LICENSE-GPLV3 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff -Nru zbackup-1.2/licenses/LICENSE-OPENSSL zbackup-1.4.1+20150403/licenses/LICENSE-OPENSSL --- zbackup-1.2/licenses/LICENSE-OPENSSL 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/licenses/LICENSE-OPENSSL 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,127 @@ + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a dual license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. Actually both licenses are BSD-style + Open Source licenses. In case of any license issues related to OpenSSL + please contact openssl-core@openssl.org. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + diff -Nru zbackup-1.2/message.cc zbackup-1.4.1+20150403/message.cc --- zbackup-1.2/message.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/message.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "message.hh" diff -Nru zbackup-1.2/message.hh zbackup-1.4.1+20150403/message.hh --- zbackup-1.2/message.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/message.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef MESSAGE_HH_INCLUDED__ -#define MESSAGE_HH_INCLUDED__ +#ifndef MESSAGE_HH_INCLUDED +#define MESSAGE_HH_INCLUDED #include #include diff -Nru zbackup-1.2/mt.cc zbackup-1.4.1+20150403/mt.cc --- zbackup-1.2/mt.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/mt.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "mt.hh" diff -Nru zbackup-1.2/mt.hh zbackup-1.4.1+20150403/mt.hh --- zbackup-1.2/mt.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/mt.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef MT_HH_INCLUDED__ -#define MT_HH_INCLUDED__ +#ifndef MT_HH_INCLUDED +#define MT_HH_INCLUDED #include #include diff -Nru zbackup-1.2/nocopy.hh zbackup-1.4.1+20150403/nocopy.hh --- zbackup-1.2/nocopy.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/nocopy.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef NOCOPY_HH_INCLUDED__ -#define NOCOPY_HH_INCLUDED__ +#ifndef NOCOPY_HH_INCLUDED +#define NOCOPY_HH_INCLUDED /// A simple class to disallow copying of the class objects. Inherit from it to /// use it diff -Nru zbackup-1.2/objectcache.cc zbackup-1.4.1+20150403/objectcache.cc --- zbackup-1.2/objectcache.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/objectcache.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "objectcache.hh" diff -Nru zbackup-1.2/objectcache.hh zbackup-1.4.1+20150403/objectcache.hh --- zbackup-1.2/objectcache.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/objectcache.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef OBJECTCACHE_HH_INCLUDED__ -#define OBJECTCACHE_HH_INCLUDED__ +#ifndef OBJECTCACHE_HH_INCLUDED +#define OBJECTCACHE_HH_INCLUDED #include #include diff -Nru zbackup-1.2/page_size.cc zbackup-1.4.1+20150403/page_size.cc --- zbackup-1.2/page_size.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/page_size.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "page_size.hh" diff -Nru zbackup-1.2/page_size.hh zbackup-1.4.1+20150403/page_size.hh --- zbackup-1.2/page_size.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/page_size.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef PAGE_SIZE_HH_INCLUDED__ -#define PAGE_SIZE_HH_INCLUDED__ +#ifndef PAGE_SIZE_HH_INCLUDED +#define PAGE_SIZE_HH_INCLUDED /// Returns the page size used by this system unsigned getPageSize(); diff -Nru zbackup-1.2/random.cc zbackup-1.4.1+20150403/random.cc --- zbackup-1.2/random.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/random.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,17 +1,17 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "random.hh" #include namespace Random { -void genarateTrue( void * buf, unsigned size ) +void generateTrue( void * buf, unsigned size ) { if ( RAND_bytes( (unsigned char *) buf, size ) != 1 ) throw exCantGenerate(); } -void genaratePseudo( void * buf, unsigned size ) +void generatePseudo( void * buf, unsigned size ) { if ( RAND_pseudo_bytes( (unsigned char *) buf, size ) < 0 ) throw exCantGenerate(); diff -Nru zbackup-1.2/random.hh zbackup-1.4.1+20150403/random.hh --- zbackup-1.2/random.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/random.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef RANDOM_HH_INCLUDED__ -#define RANDOM_HH_INCLUDED__ +#ifndef RANDOM_HH_INCLUDED +#define RANDOM_HH_INCLUDED #include @@ -12,10 +12,10 @@ DEF_EX( exCantGenerate, "Error generating random sequence, try later", std::exception ) /// This one fills the buffer with true randomness, suitable for a key -void genarateTrue( void * buf, unsigned size ); +void generateTrue( void * buf, unsigned size ); /// This one fills the buffer with pseudo randomness, suitable for salts but not /// keys -void genaratePseudo( void * buf, unsigned size ); +void generatePseudo( void * buf, unsigned size ); } #endif diff -Nru zbackup-1.2/README.md zbackup-1.4.1+20150403/README.md --- zbackup-1.2/README.md 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/README.md 2015-04-03 09:33:05.000000000 +0000 @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.org/zbackup/zbackup.svg)](https://travis-ci.org/zbackup/zbackup) [![Coverity Scan Build Status](https://scan.coverity.com/projects/4229/badge.svg)](https://scan.coverity.com/projects/4229) + # Introduction **zbackup** is a globally-deduplicating backup tool, based on the ideas found in [rsync](http://rsync.samba.org/). Feed a large `.tar` into it, and it will store duplicate regions of it only once, then compress and optionally encrypt the result. Feed another `.tar` file, and it will also re-use any data found in any previous backups. This way only new changes are stored, and as long as the files are not very different, the amount of storage required is very low. Any of the backup files stored previously can be read back in full at any time. The program is format-agnostic, so you can feed virtually any files to it (any types of archives, proprietary formats, even raw disk images -- but see [Caveats](#caveats)). @@ -8,13 +10,14 @@ The program has the following features: - * Parallel LZMA compression of the stored data + * Parallel LZMA or LZO compression of the stored data * Built-in AES encryption of the stored data - * Possibility to delete old backup data in the future + * Possibility to delete old backup data * Use of a 64-bit rolling hash, keeping the amount of soft collisions to zero * Repository consists of immutable files. No existing files are ever modified * Written in C++ only with only modest library dependencies * Safe to use in production (see [below](#safety)) + * Possibility to exchange data between repos without recompression # Build dependencies @@ -22,11 +25,12 @@ * `libssl-dev` for all encryption, hashing and random numbers * `libprotobuf-dev` and `protobuf-compiler` for data serialization * `liblzma-dev` for compression + * `liblzo2-dev` for compression (optional) * `zlib1g-dev` for adler32 calculation # Quickstart -To build: +To build and install: ```bash cd zbackup @@ -36,6 +40,8 @@ # or just run as ./zbackup ``` +`zbackup` is also part of the [Fedora/EPEL](https://apps.fedoraproject.org/packages/zbackup), [Debian](https://packages.debian.org/search?keywords=zbackup), [Ubuntu](http://packages.ubuntu.com/search?keywords=zbackup), [Arch Linux](https://aur.archlinux.org/packages/zbackup/) and [FreeBSD](http://www.freshports.org/sysutils/zbackup/). + To use: ```bash @@ -73,17 +79,14 @@ * While you can pipe any data into the program, the data should be uncompressed and unencrypted -- otherwise no deduplication could be performed on it. `zbackup` would compress and encrypt the data itself, so there's no need to do that yourself. So just run `tar c` and pipe it into `zbackup` directly. If backing up disk images employing encryption, pipe the unencrypted version (the one you normally mount). If you create `.zip` or `.rar` files, use no compression (`-0` or `-m0`) and no encryption. * Parallel LZMA compression uses a lot of RAM (several hundreds of megabytes, depending on the number of threads used), and ten times more virtual address space. The latter is only relevant on 32-bit architectures where it's limited to 2 or 3 GB. If you hit the ceiling, lower the number of threads with `--threads`. * Since the data is deduplicated, there's naturally no redundancy in it. A loss of a single file can lead to a loss of virtually all data. Make sure you store it on a redundant storage (RAID1, a cloud provider etc). - * The encryption key, if used, is stored in the `info` file in the root of the repo. It is encrypted with your password. Technically thus you can change your password without re-encrypting any data, and as long as no one possesses the old `info` file and knows your old password, you would be safe (even though the actual option to change password is not implemented yet -- someone who needs this is welcome to create a pull request -- the possibility is all there). Also note that it is crucial you don't lose your `info` file, as otherwise the whole backup would be lost. + * The encryption key, if used, is stored in the `info` file in the root of the repo. It is encrypted with your password. Technically thus you can change your password without re-encrypting any data, and as long as no one possesses the old `info` file and knows your old password, you would be safe (note that ability to change repo type between encrypted and non-encrypted is not implemented yet -- someone who needs this is welcome to create a pull request -- the possibility is all there). Also note that it is crucial you don't lose your `info` file, as otherwise the whole backup would be lost. # Limitations * Right now the only modes supported are reading from standard input and writing to standard output. FUSE mounts and NBD servers may be added later if someone contributes the code. * The program keeps all known blocks in an in-RAM hash table, which may create scalability problems for very large repos (see [below](#scalability)). * The only encryption mode currently implemented is `AES-128` in `CBC` mode with `PKCS#7` padding. If you believe that this is not secure enough, patches are welcome. Before you jump to conclusions however, read [this article](http://www.schneier.com/blog/archives/2009/07/another_new_aes.html). - * The only compression mode supported is LZMA, which suits backups very nicely. * It's only possible to fully restore the backup in order to get to a required file, without any option to quickly pick it out. `tar` would not allow to do it anyway, but e.g. for `zip` files it could have been possible. This is possible to implement though, e.g. by exposing the data over a FUSE filesystem. - * There's no option to delete old backup data yet. The possibility is all there, though. Someone needs to implement it (see [below](#improvements)). - * There's no option to specify block and bundle sizes other than the default ones (currently `64k` and `2MB` respectively), though it's trivial to add command-line switches for those. Most of those limitations can be lifted by implementing the respective features. @@ -136,13 +139,29 @@ * `AES-128` in `CBC` mode with `PKCS#7` padding is used for encryption. This seems to be a reasonbly safe classic solution. Each encrypted file has a random IV as its first 16 bytes. * We use Google's [protocol buffers](https://developers.google.com/protocol-buffers/) to represent data structures in binary form. They are very efficient and relatively simple to use. +# Compression + +`zbackup` uses LZMA to compress stored data. It compresses very well, but it will slow down your backup +(unless you have a very fast CPU). + +LZO is much faster, but the files will be bigger. If you don't +want your backup process to be cpu-bound, you should consider using LZO. However, there are some caveats: + +* LZO is so fast that other parts of `zbackup` consume significant portions of the CPU. In fact, it is only using one core on my machine because compression is the only thing that can run in parallel. +* I've hacked the LZO support in a day. You shouldn't trust it. Please make sure that restore works before you assume that your data is safe. That may still be faster than a backup with LZMA ;-) +* LZMA is still the default, so make sure that you use the `-o bundle.compression_method=lzo` argument when you init the repo or whenever you do a backup. + +You can mix LZMA and LZO in a repository. Each bundle file has a field that says how it was compressed, so +`zbackup` will use the right method to decompress it. You could use an old `zbackup` respository with only LZMA +bundles and start using LZO. However, please think twice before you do that because old versions of `zbackup` +won't be able to read those bundles. + # Improvements There's a lot to be improved in the program. It was released with the minimum amount of functionality to be useful. It is also stable. This should hopefully stimulate people to join the development and add all those other fancy features. Here's a list of ideas: - * Additional options, such as configurable chunk and bundle sizes etc. - * A command to change password. - * A command to perform garbage collection. The program should skim through all backups and note which chunks are used by all of them. Then it should skim through all bundles and see which chunks among the ones stored were never used by the backups. If a bundle has more than *X%* of unused chunks, the remaining chunks should be transferred into brand new bundles. The old bundles should be deleted then. Once the process finishes, a new single index file with all existing chunk ids should be written, replacing all previous index files. With this command, it would become possible to remove old backups. + * Ability to change bundle type (between encrypted and non-encrypted). + * Improved garbage collection. The program should support ability to specify maximum index file size / maximum index file count (for better compatibility with cloud storages as well) or something like retention policy. * A command to fsck the repo by doing something close to what garbage collection does, but also checking all hashes and so on. * Parallel decompression. Right now decompression is single-threaded, but it is possible to look ahead in the stream and perform prefetching. * Support for mounting the repo over FUSE. Random access to data would then be possible. @@ -164,14 +183,15 @@ `zbackup` is certainly not the first project to embrace the idea of using a rolling hash for deduplication. Here's a list of other projects the author found on the web: * [bup](https://github.com/bup/bup), based on storing data in `git` packs. No possibility of removing old data. This program was the initial inspiration for `zbackup`. - * [ddar](http://www.synctus.com/ddar/), seems to be a little bit outdated. Contains a nice list of alternatives with comparisons. + * [ddar](https://github.com/basak/ddar), seems to be a little bit outdated. Contains a nice list of alternatives with comparisons. * [rdiff-backup](http://www.nongnu.org/rdiff-backup/), based on the original `rsync` algorithm. Does not do global deduplication, only working over the files with the same file name. * [duplicity](http://duplicity.nongnu.org/), which looks similar to `rdiff-backup` with regards to mode of operation. * Some filesystems (most notably [ZFS](http://en.wikipedia.org/wiki/ZFS) and [Btrfs](http://en.wikipedia.org/wiki/Btrfs)) provide deduplication features. They do so only at block level though, without a sliding window, so they can not accomodate to arbitrary byte insertion/deletion in the middle of data. + * [Attic](https://attic-backup.org/), which looks very similar to `zbackup`. # Credits -Copyright (c) 2013-2013 Konstantin Isakov (). Licensed under GNU GPLv2 or later. +Copyright (c) 2012-2014 Konstantin Isakov () and ZBackup contributors, see CONTRIBUTORS. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. diff -Nru zbackup-1.2/rolling_hash.cc zbackup-1.4.1+20150403/rolling_hash.cc --- zbackup-1.2/rolling_hash.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/rolling_hash.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "rolling_hash.hh" diff -Nru zbackup-1.2/rolling_hash.hh zbackup-1.4.1+20150403/rolling_hash.hh --- zbackup-1.2/rolling_hash.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/rolling_hash.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef ROLLING_HASH_HH_INCLUDED__ -#define ROLLING_HASH_HH_INCLUDED__ +#ifndef ROLLING_HASH_HH_INCLUDED +#define ROLLING_HASH_HH_INCLUDED #include #include diff -Nru zbackup-1.2/sha256.cc zbackup-1.4.1+20150403/sha256.cc --- zbackup-1.2/sha256.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/sha256.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "sha256.hh" diff -Nru zbackup-1.2/sha256.hh zbackup-1.4.1+20150403/sha256.hh --- zbackup-1.2/sha256.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/sha256.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef SHA256_HH_INCLUDED__ -#define SHA256_HH_INCLUDED__ +#ifndef SHA256_HH_INCLUDED +#define SHA256_HH_INCLUDED #include #include diff -Nru zbackup-1.2/sptr.hh zbackup-1.4.1+20150403/sptr.hh --- zbackup-1.2/sptr.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/sptr.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef SPTR_HH_INCLUDED__ -#define SPTR_HH_INCLUDED__ +#ifndef SPTR_HH_INCLUDED +#define SPTR_HH_INCLUDED /// A generic non-intrusive smart-pointer template. We could use boost::, tr1:: /// or whatever, but since there's no standard solution yet, it isn't worth @@ -74,6 +74,9 @@ { if ( &other != this ) { reset(); p = other.p; count = other.count; increment(); } return * this; } + operator bool( void ) const + { return !!p; } + bool operator ! ( void ) const { return !p; } diff -Nru zbackup-1.2/static_assert.hh zbackup-1.4.1+20150403/static_assert.hh --- zbackup-1.2/static_assert.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/static_assert.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef STATIC_ASSERT_HH_INCLUDED__ -#define STATIC_ASSERT_HH_INCLUDED__ +#ifndef STATIC_ASSERT_HH_INCLUDED +#define STATIC_ASSERT_HH_INCLUDED // Based on the one from the Boost library. It wouldn't make sense to depend on // boost just for that diff -Nru zbackup-1.2/storage_info_file.cc zbackup-1.4.1+20150403/storage_info_file.cc --- zbackup-1.2/storage_info_file.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/storage_info_file.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,11 +1,12 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "encrypted_file.hh" #include "message.hh" #include "storage_info_file.hh" +#include "debug.hh" namespace StorageInfoFile { @@ -16,6 +17,7 @@ void save( string const & fileName, StorageInfo const & storageInfo ) { + dPrintf( "Saving storage info...\n" ); EncryptedFile::OutputStream os( fileName.c_str(), EncryptionKey::noKey(), NULL ); FileHeader header; @@ -28,6 +30,7 @@ void load( string const & fileName, StorageInfo & storageInfo ) { + dPrintf( "Loading storage info...\n" ); EncryptedFile::InputStream is( fileName.c_str(), EncryptionKey::noKey(), NULL ); FileHeader header; @@ -39,4 +42,48 @@ is.checkAdler32(); } +} + +namespace ExtendedStorageInfoFile { + +enum +{ + FileFormatVersion = 1 +}; + +void save( string const & fileName, EncryptionKey const & encryptionKey, + ExtendedStorageInfo const & extendedStorageInfo ) +{ + dPrintf( "Saving extended storage info, hasKey: %s\n", + encryptionKey.hasKey() ? "true" : "false" ); + EncryptedFile::OutputStream os( fileName.c_str(), encryptionKey, + Encryption::ZeroIv ); + os.writeRandomIv(); + + FileHeader header; + header.set_version( FileFormatVersion ); + Message::serialize( header, os ); + + Message::serialize( extendedStorageInfo, os ); + os.writeAdler32(); +} + +void load( string const & fileName, EncryptionKey const & encryptionKey, + ExtendedStorageInfo & extendedStorageInfo ) +{ + dPrintf( "Loading extended storage info, hasKey: %s\n", + encryptionKey.hasKey() ? "true" : "false" ); + EncryptedFile::InputStream is( fileName.c_str(), encryptionKey, + Encryption::ZeroIv ); + is.consumeRandomIv(); + + FileHeader header; + Message::parse( header, is ); + if ( header.version() != FileFormatVersion ) + throw exUnsupportedVersion(); + + Message::parse( extendedStorageInfo, is ); + is.checkAdler32(); +} + } diff -Nru zbackup-1.2/storage_info_file.hh zbackup-1.4.1+20150403/storage_info_file.hh --- zbackup-1.2/storage_info_file.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/storage_info_file.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef STORAGE_INFO_FILE_HH_INCLUDED__ -#define STORAGE_INFO_FILE_HH_INCLUDED__ +#ifndef STORAGE_INFO_FILE_HH_INCLUDED +#define STORAGE_INFO_FILE_HH_INCLUDED #include #include @@ -25,4 +25,18 @@ void load( string const & fileName, StorageInfo & ); } +namespace ExtendedStorageInfoFile { + +using std::string; + +DEF_EX( Ex, "Extended storage info file exception", std::exception ) +DEF_EX( exUnsupportedVersion, "Unsupported version of the extended storage info file format", Ex ) + +/// Saves the given ExtendedStorageInfo data into the given file +void save( string const & fileName, EncryptionKey const &, ExtendedStorageInfo const & ); + +/// Loads the given ExtendedStorageInfo data from the given file +void load( string const & fileName, EncryptionKey const &, ExtendedStorageInfo & ); +} + #endif diff -Nru zbackup-1.2/tartool/CMakeLists.txt zbackup-1.4.1+20150403/tartool/CMakeLists.txt --- zbackup-1.2/tartool/CMakeLists.txt 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/tartool/CMakeLists.txt 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -# Copyright (c) 2012-2013 Konstantin Isakov -# Part of ZBackup. Licensed under GNU GPLv2 or later +# Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +# Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE cmake_minimum_required( VERSION 2.6.0 ) project( tartool ) diff -Nru zbackup-1.2/tartool/tartool.cc zbackup-1.4.1+20150403/tartool/tartool.cc --- zbackup-1.2/tartool/tartool.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/tartool/tartool.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include diff -Nru zbackup-1.2/tests/bundle/bundle.pro zbackup-1.4.1+20150403/tests/bundle/bundle.pro --- zbackup-1.2/tests/bundle/bundle.pro 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/tests/bundle/bundle.pro 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,50 @@ +###################################################################### +# Automatically generated by qmake (2.01a) Sun Jul 14 20:54:52 2013 +###################################################################### + +TEMPLATE = app +TARGET = +DEPENDPATH += . +INCLUDEPATH += . + +CONFIG = debug + +LIBS += -lcrypto -lprotobuf -lz -lprotobuf -llzma -llzo2 +DEFINES += __STDC_FORMAT_MACROS +DEFINES += HAVE_LIBLZO + +# Input +SOURCES += test_bundle.cc \ + ../../unbuffered_file.cc \ + ../../tmp_mgr.cc \ + ../../page_size.cc \ + ../../random.cc \ + ../../encryption_key.cc \ + ../../encryption.cc \ + ../../encrypted_file.cc \ + ../../file.cc \ + ../../dir.cc \ + ../../bundle.cc \ + ../../message.cc \ + ../../hex.cc \ + ../../compression.cc \ + ../../zbackup.pb.cc + +HEADERS += \ + ../../unbuffered_file.hh \ + ../../tmp_mgr.hh \ + ../../adler32.hh \ + ../../page_size.hh \ + ../../random.hh \ + ../../encryption_key.hh \ + ../../encrypted_file.hh \ + ../../encryption.hh \ + ../../ex.hh \ + ../../file.hh \ + ../../dir.hh \ + ../../bundle.hh \ + ../../message.hh \ + ../../hex.hh \ + ../../compression.hh \ + ../../message.hh \ + ../../zbackup.pb.h diff -Nru zbackup-1.2/tests/bundle/test_bundle.cc zbackup-1.4.1+20150403/tests/bundle/test_bundle.cc --- zbackup-1.2/tests/bundle/test_bundle.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/tests/bundle/test_bundle.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,171 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include +#include +#include +#include "../../encrypted_file.hh" +#include "../../encryption_key.hh" +#include "../../random.hh" +#include "../../tmp_mgr.hh" +#include "../../check.hh" +#include "../../adler32.hh" +#include "../../bundle.hh" +#include "../../compression.hh" +#include "../../message.hh" + +using namespace Compression; + + +char tmpbuf[100]; + +void testCompatibility() +{ + // The LZO code uses a different file header than the previous code + // because it adds the compression_method field. Nevertheless, it + // must be compatible with previous code. + + TmpMgr tmpMgr( "/dev/shm" ); + sptr< TemporaryFile > tempFile = tmpMgr.makeTemporaryFile(); + std::string fileName = tempFile->getFileName(); + + EncryptionKey noKey( std::string(), NULL ); + + + // Write old header, read as new header + { + { + EncryptedFile::OutputStream os( fileName.c_str(), noKey, Encryption::ZeroIv ); + + FileHeader header; + header.set_version( 42 ); + Message::serialize( header, os ); + } + + { + EncryptedFile::InputStream is( fileName.c_str(), noKey, Encryption::ZeroIv ); + + BundleFileHeader header; + Message::parse( header, is ); + + CHECK( header.version() == 42, "version is wrong when reading old header with new program" ); + CHECK( header.compression_method() == "lzma", "compression_method is wrong when reading old header with new program" ); + } + } + + // Write new header, read as old header + //NOTE In the real code, this will only work, if the file uses LZMA. If it doesn't, the version + // field is increased and the old code will refuse to read the file. + { + { + EncryptedFile::OutputStream os( fileName.c_str(), noKey, Encryption::ZeroIv ); + + BundleFileHeader header; + header.set_version( 42 ); + Message::serialize( header, os ); + } + + { + EncryptedFile::InputStream is( fileName.c_str(), noKey, Encryption::ZeroIv ); + + FileHeader header; + Message::parse( header, is ); + + CHECK( header.version() == 42, "version is wrong when reading new header with old program" ); + // cannot check compression_method because the field doesn't exist + } + } + + printf("compatibility test successful.\n"); +} + +void readAndWrite( EncryptionKey const & key, + const_sptr compression1, const_sptr compression2 ) +{ + // temporary file for the bundle + TmpMgr tmpMgr( "/dev/shm" ); + sptr< TemporaryFile > tempFile = tmpMgr.makeTemporaryFile(); + + // some chunk data + int chunkCount = rand() % 30; + size_t chunkSize = rand() % 20 ? 64*1024 : 10; + char** chunks = new char*[chunkCount]; + string* chunkIds = new string[chunkCount]; + + CompressionMethod::defaultCompression = compression1; + + // write bundle + { + Bundle::Creator bundle; + + for (int i=0;igetFileName().c_str(), key ); + } + + CompressionMethod::defaultCompression = compression2; + + // read it and compare + { + Bundle::Reader bundle( tempFile->getFileName().c_str(), key ); + + for (int i=0;i > compressions; + for ( CompressionMethod::iterator it = CompressionMethod::begin(); it!=CompressionMethod::end(); ++it ) { + printf( "supported compression: %s\n", (*it)->getName().c_str() ); + compressions.push_back( *it ); + } + + for ( size_t iteration = 100; iteration--; ) { + // default compression while writing the file + const_sptr compression1 = compressions[ rand() % compressions.size() ]; + // default compression while reading the file + // The reader should ignore it and always use the compression that was used for the file. + const_sptr compression2 = compressions[ rand() % compressions.size() ]; + + readAndWrite( ( rand() & 1 ) ? key : noKey, compression1, compression2 ); + } + + printf("\n"); + + return 0; +} diff -Nru zbackup-1.2/tests/encrypted_file/test_encrypted_file.cc zbackup-1.4.1+20150403/tests/encrypted_file/test_encrypted_file.cc --- zbackup-1.2/tests/encrypted_file/test_encrypted_file.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/tests/encrypted_file/test_encrypted_file.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include "../../encrypted_file.hh" diff -Nru zbackup-1.2/tests/rolling_hash/test_rolling_hash.cc zbackup-1.4.1+20150403/tests/rolling_hash/test_rolling_hash.cc --- zbackup-1.2/tests/rolling_hash/test_rolling_hash.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/tests/rolling_hash/test_rolling_hash.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include #include diff -Nru zbackup-1.2/tmp_mgr.cc zbackup-1.4.1+20150403/tmp_mgr.cc --- zbackup-1.2/tmp_mgr.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/tmp_mgr.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,9 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #include "tmp_mgr.hh" +#include #include #include #include "dir.hh" @@ -43,6 +44,7 @@ { string name( Dir::addPath( path, "XXXXXX") ); + umask( S_IRUSR | S_IWUSR | S_IRGRP ); int fd = mkstemp( &name[ 0 ] ); if ( fd == -1 || close( fd ) != 0 ) diff -Nru zbackup-1.2/tmp_mgr.hh zbackup-1.4.1+20150403/tmp_mgr.hh --- zbackup-1.2/tmp_mgr.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/tmp_mgr.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef TMP_MGR_HH_INCLUDED__ -#define TMP_MGR_HH_INCLUDED__ +#ifndef TMP_MGR_HH_INCLUDED +#define TMP_MGR_HH_INCLUDED #include #include diff -Nru zbackup-1.2/.travis.yml zbackup-1.4.1+20150403/.travis.yml --- zbackup-1.2/.travis.yml 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/.travis.yml 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,6 @@ +language: cpp +before_install: + - "sudo apt-get update -qq" + - "sudo apt-get install -qq cmake libssl-dev libprotobuf-dev protobuf-compiler liblzma-dev zlib1g-dev liblzo2-dev" +install: true +script: mkdir objdir && cd objdir && cmake ../ && make diff -Nru zbackup-1.2/unbuffered_file.cc zbackup-1.4.1+20150403/unbuffered_file.cc --- zbackup-1.2/unbuffered_file.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/unbuffered_file.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE #define _LARGEFILE64_SOURCE @@ -13,7 +13,7 @@ #include "unbuffered_file.hh" -#ifdef __APPLE__ +#if defined( __APPLE__ ) || defined( __OpenBSD__ ) || defined(__FreeBSD__) #define lseek64 lseek #endif @@ -24,7 +24,7 @@ int flags = ( mode == WriteOnly ? ( O_WRONLY | O_CREAT | O_TRUNC ) : O_RDONLY ); -#ifndef __APPLE__ +#if !defined( __APPLE__ ) && !defined( __OpenBSD__ ) && !defined(__FreeBSD__) flags |= O_LARGEFILE; #endif fd = open( fileName, flags, 0666 ); diff -Nru zbackup-1.2/unbuffered_file.hh zbackup-1.4.1+20150403/unbuffered_file.hh --- zbackup-1.2/unbuffered_file.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/unbuffered_file.hh 2015-04-03 09:33:05.000000000 +0000 @@ -1,8 +1,8 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef UNBUFFERED_FILE_HH_INCLUDED__ -#define UNBUFFERED_FILE_HH_INCLUDED__ +#ifndef UNBUFFERED_FILE_HH_INCLUDED +#define UNBUFFERED_FILE_HH_INCLUDED #include #include diff -Nru zbackup-1.2/utils.hh zbackup-1.4.1+20150403/utils.hh --- zbackup-1.2/utils.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/utils.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,21 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef UTILS_HH_INCLUDED +#define UTILS_HH_INCLUDED + +#include + +namespace Utils { + +template +std::string numberToString( T pNumber ) +{ + std::ostringstream oOStrStream; + oOStrStream << pNumber; + return oOStrStream.str(); +} + +} + +#endif diff -Nru zbackup-1.2/version.cc zbackup-1.4.1+20150403/version.cc --- zbackup-1.2/version.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/version.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,9 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include +#ifndef ZBACKUP_VERSION +std::string zbackup_version( "1.4" ); +#else +std::string zbackup_version( ZBACKUP_VERSION ); +#endif diff -Nru zbackup-1.2/version.hh zbackup-1.4.1+20150403/version.hh --- zbackup-1.2/version.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/version.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,10 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef VERSION_HH_INCLUDED +#define VERSION_HH_INCLUDED +#include + +extern std::string zbackup_version; + +#endif diff -Nru zbackup-1.2/zbackup_base.cc zbackup-1.4.1+20150403/zbackup_base.cc --- zbackup-1.2/zbackup_base.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/zbackup_base.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,451 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include +#include +#include +#include + +#include "zbackup_base.hh" + +#include "storage_info_file.hh" +#include "compression.hh" +#include "debug.hh" + +// TODO: make configurable by cmake +#if defined(PATH_VI) +# define EDITOR PATH_VI +#else +# define EDITOR "/bin/vi" +#endif + +#ifndef PATH_BSHELL +# define PATH_BSHELL "/bin/sh" +#endif + +using std::string; + +Paths::Paths( string const & storageDir ): storageDir( storageDir ) +{ +} + +string Paths::getTmpPath() +{ + return string( Dir::addPath( storageDir, "tmp" ) ); +} + +string Paths::getBundlesPath() +{ + return string( Dir::addPath( storageDir, "bundles" ) ); +} + +string Paths::getStorageInfoPath() +{ + return string( Dir::addPath( storageDir, "info" ) ); +} + +string Paths::getExtendedStorageInfoPath() +{ + return string( Dir::addPath( storageDir, "info_extended" ) ); +} + +string Paths::getIndexPath() +{ + return string( Dir::addPath( storageDir, "index" ) ); +} + +string Paths::getBackupsPath() +{ + return string( Dir::addPath( storageDir, "backups" ) ); +} + +ZBackupBase::ZBackupBase( string const & storageDir, string const & password ): + Paths( storageDir ), storageInfo( loadStorageInfo() ), + encryptionkey( password, storageInfo.has_encryption_key() ? + &storageInfo.encryption_key() : 0 ), + extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ), + tmpMgr( getTmpPath() ), + chunkIndex( encryptionkey, tmpMgr, getIndexPath(), false ), + config( extendedStorageInfo.mutable_config() ) +{ + propagateUpdate(); + dPrintf("%s for %s is instantiated and initialized\n", __CLASS, + storageDir.c_str() ); +} + +ZBackupBase::ZBackupBase( string const & storageDir, string const & password, + Config & configIn ): + Paths( storageDir ), storageInfo( loadStorageInfo() ), + encryptionkey( password, storageInfo.has_encryption_key() ? + &storageInfo.encryption_key() : 0 ), + extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ), + tmpMgr( getTmpPath() ), + chunkIndex( encryptionkey, tmpMgr, getIndexPath(), false ), + config( configIn, extendedStorageInfo.mutable_config() ) +{ + propagateUpdate(); + dPrintf("%s for %s is instantiated and initialized\n", __CLASS, + storageDir.c_str() ); +} + +ZBackupBase::ZBackupBase( string const & storageDir, string const & password, + bool prohibitChunkIndexLoading ): + Paths( storageDir ), storageInfo( loadStorageInfo() ), + encryptionkey( password, storageInfo.has_encryption_key() ? + &storageInfo.encryption_key() : 0 ), + extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ), + tmpMgr( getTmpPath() ), + chunkIndex( encryptionkey, tmpMgr, getIndexPath(), prohibitChunkIndexLoading ), + config( extendedStorageInfo.mutable_config() ) +{ + propagateUpdate(); + dPrintf("%s for %s is instantiated and initialized\n", __CLASS, + storageDir.c_str() ); +} + +ZBackupBase::ZBackupBase( string const & storageDir, string const & password, + Config & configIn, bool prohibitChunkIndexLoading ): + Paths( storageDir ), storageInfo( loadStorageInfo() ), + encryptionkey( password, storageInfo.has_encryption_key() ? + &storageInfo.encryption_key() : 0 ), + extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ), + tmpMgr( getTmpPath() ), + chunkIndex( encryptionkey, tmpMgr, getIndexPath(), prohibitChunkIndexLoading ), + config( configIn, extendedStorageInfo.mutable_config() ) +{ + propagateUpdate(); + dPrintf("%s for %s is instantiated and initialized\n", __CLASS, + storageDir.c_str() ); +} + +// Update all internal variables according to real configuration +// Dunno why someone need to store duplicate information +// in deduplication utility +void ZBackupBase::propagateUpdate() +{ + const_sptr< Compression::CompressionMethod > compression = + Compression::CompressionMethod::findCompression( + config.GET_STORABLE( bundle, compression_method ) ); + Compression::CompressionMethod::selectedCompression = compression; +} + +StorageInfo ZBackupBase::loadStorageInfo() +{ + StorageInfo storageInfo; + + StorageInfoFile::load( getStorageInfoPath(), storageInfo ); + + return storageInfo; +} + +ExtendedStorageInfo ZBackupBase::loadExtendedStorageInfo( + EncryptionKey const & encryptionkey ) +{ + try + { + ExtendedStorageInfo extendedStorageInfo; + + ExtendedStorageInfoFile::load( getExtendedStorageInfoPath(), encryptionkey, + extendedStorageInfo ); + + return extendedStorageInfo; + } + catch ( UnbufferedFile::exCantOpen & ex ) + { + verbosePrintf( "Can't open extended storage info (info_extended)!\n" + "Attempting to start repo migration.\n" ); + + if ( !File::exists( getExtendedStorageInfoPath() ) ) + { + ExtendedStorageInfo extendedStorageInfo; + Config config( extendedStorageInfo.mutable_config() ); + config.reset_storable(); + config.SET_STORABLE( chunk, max_size, storageInfo.chunk_max_size() ); + config.SET_STORABLE( bundle, max_payload_size, + storageInfo.bundle_max_payload_size() ); + config.SET_STORABLE( bundle, compression_method, + storageInfo.default_compression_method() ); + + ExtendedStorageInfoFile::save( getExtendedStorageInfoPath(), encryptionkey, + extendedStorageInfo ); + + verbosePrintf( "Done.\n" ); + + return loadExtendedStorageInfo( encryptionkey ); + } + else + { + fprintf( stderr, "info_extended exists but can't be opened!\n" + "Please check file permissions.\n" ); + } + } +} + +void ZBackupBase::initStorage( string const & storageDir, + string const & password, + bool isEncrypted, + Config const & configIn ) +{ + StorageInfo storageInfo; + ExtendedStorageInfo extendedStorageInfo; + Config config( extendedStorageInfo.mutable_config() ); + config.reset_storable(); + config.storable->MergeFrom( *configIn.storable ); + + EncryptionKey encryptionkey = EncryptionKey::noKey(); + + if ( isEncrypted ) + EncryptionKey::generate( password, + *storageInfo.mutable_encryption_key(), + encryptionkey ); + + Paths paths( storageDir ); + + if ( !Dir::exists( storageDir ) ) + Dir::create( storageDir ); + + if ( !Dir::exists( paths.getBundlesPath() ) ) + Dir::create( paths.getBundlesPath() ); + + if ( !Dir::exists( paths.getBackupsPath() ) ) + Dir::create( paths.getBackupsPath() ); + + if ( !Dir::exists( paths.getIndexPath() ) ) + Dir::create( paths.getIndexPath() ); + + string storageInfoPath( paths.getStorageInfoPath() ); + string extendedStorageInfoPath( paths.getExtendedStorageInfoPath() ); + + if ( File::exists( storageInfoPath ) ) + throw exWontOverwrite( storageInfoPath ); + + encryptionkey = EncryptionKey( password, storageInfo.has_encryption_key() ? + &storageInfo.encryption_key() : 0 ); + + StorageInfoFile::save( storageInfoPath, storageInfo ); + ExtendedStorageInfoFile::save( extendedStorageInfoPath, encryptionkey, extendedStorageInfo ); +} + +string ZBackupBase::deriveStorageDirFromBackupsFile( string const & + backupsFile, bool allowOutside ) +{ + // TODO: handle cases when there's a backup/ folder within the backup/ folder + // correctly + if ( allowOutside ) + return Dir::getRealPath( backupsFile ); + + string realPath = Dir::getRealPath( Dir::getDirName( backupsFile ) ); + size_t pos; + if ( realPath.size() >= 8 && strcmp( realPath.c_str() + realPath.size() - 8, + "/backups") == 0 ) + pos = realPath.size() - 8; + else + pos = realPath.rfind( "/backups/" ); + if ( pos == string::npos ) + throw exCantDeriveStorageDir( backupsFile ); + else + return realPath.substr( 0, pos ); +} + +void ZBackupBase::setPassword( string const & password ) +{ + EncryptionKey::generate( password, + *storageInfo.mutable_encryption_key(), encryptionkey ); + + StorageInfoFile::save( getStorageInfoPath(), storageInfo ); + + EncryptionKey encryptionkey( password, storageInfo.has_encryption_key() ? + &storageInfo.encryption_key() : 0 ); +} + +void ZBackupBase::saveExtendedStorageInfo() +{ + ExtendedStorageInfoFile::save( getExtendedStorageInfoPath(), encryptionkey, + extendedStorageInfo ); +} + +bool ZBackupBase::spawnEditor( string & data, bool( * validator ) + ( string const &, string const & ) ) +{ + // Based on ideas found in cronie-1.4.4-12.el6 + // Initially it was just a copy-paste from edit_cmd (crontab.c) + + /* Turn off signals. */ + (void) signal( SIGHUP, SIG_IGN ); + (void) signal( SIGINT, SIG_IGN ); + (void) signal( SIGQUIT, SIG_IGN ); + + sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile(); + const char * tmpFileName = tmpFile->getFileName().c_str(); + + sptr< File> tmpDataFile = new File( tmpFileName, File::WriteOnly ); + tmpDataFile->writeRecords( data.c_str(), data.size(), 1 ); + +again: + tmpDataFile->rewind(); + if ( tmpDataFile->error() ) + { + verbosePrintf( "Error while writing data to %s\n", tmpFileName ); +fatal: + tmpFile.reset(); + exit( EXIT_FAILURE ); + } + + char * editorEnv; + string editor; + if ( ( ( editorEnv = getenv( "VISUAL" ) ) == NULL || *editorEnv == '\0' ) && + ( ( editorEnv = getenv( "EDITOR" ) ) == NULL || *editorEnv == '\0' ) ) + editor.assign( EDITOR ); + else + editor.assign( editorEnv ); + + /* we still have the file open. editors will generally rewrite the + * original file rather than renaming/unlinking it and starting a + * new one; even backup files are supposed to be made by copying + * rather than by renaming. if some editor does not support this, + * then don't use it. the security problems are more severe if we + * close and reopen the file around the edit. + */ + + string shellArgs; + shellArgs += editor; + shellArgs += " "; + shellArgs += tmpFileName; + + pid_t pid, xpid; + + switch ( pid = fork() ) + { + case -1: + perror( "fork" ); + goto fatal; + case 0: + /* child */ + dPrintf( "Spawning editor: %s %s %s %s\n", PATH_BSHELL, PATH_BSHELL, + "-c", shellArgs.c_str() ); + execlp( PATH_BSHELL, PATH_BSHELL, "-c", shellArgs.c_str(), (char *) 0 ); + perror( editor.c_str() ); + exit( EXIT_FAILURE ); + /*NOTREACHED*/ + default: + /* parent */ + break; + } + + /* parent */ + int waiter; + for ( ; ; ) + { + xpid = waitpid( pid, &waiter, 0 ); + if ( xpid == -1 ) + { + if ( errno != EINTR ) + verbosePrintf( "waitpid() failed waiting for PID %ld from \"%s\": %s\n", + (long) pid, editor.c_str(), strerror( errno ) ); + } + else + if (xpid != pid) + { + verbosePrintf( "wrong PID (%ld != %ld) from \"%s\"\n", + (long) xpid, (long) pid, editor.c_str() ); + goto fatal; + } + else + if ( WIFEXITED( waiter ) && WEXITSTATUS( waiter ) ) + { + verbosePrintf( "\"%s\" exited with status %d\n", + editor.c_str(), WEXITSTATUS( waiter ) ); + goto fatal; + } + else + if ( WIFSIGNALED( waiter ) ) + { + verbosePrintf( "\"%s\" killed; signal %d (%score dumped)\n", + editor.c_str(), WTERMSIG( waiter ), + WCOREDUMP( waiter ) ? "" : "no "); + goto fatal; + } + else + break; + } + (void) signal( SIGHUP, SIG_DFL ); + (void) signal( SIGINT, SIG_DFL ); + (void) signal( SIGQUIT, SIG_DFL ); + + tmpDataFile->close(); + tmpDataFile = new File( tmpFileName, File::ReadOnly ); + + string newData; + newData.resize( tmpDataFile->size() ); + tmpDataFile->read( &newData[ 0 ], newData.size() ); + bool isChanged = false; + + bool valid = validator( data, newData ); + + switch ( valid ) + { + case true: + goto success; + case false: + for ( ; ; ) + { + fprintf( stderr, "Supplied data is not valid\n" ); + fflush( stderr ); + printf( "Do you want to retry the same edit? " ); + fflush( stdout ); + + string input; + input.resize( 131072 ); // Should I choose another magic value? + if ( fgets( &input[ 0 ], input.size(), stdin ) == 0L ) + continue; + + switch ( input[ 0 ] ) + { + case 'y': + case 'Y': + goto again; + case 'n': + case 'N': + verbosePrintf( "Data is kept intact\n" ); + goto end; + default: + fprintf( stderr, "Enter Y or N\n" ); + } + } + } + +success: + isChanged = true; + data.assign( newData ); + +end: + tmpDataFile.reset(); + tmpFile.reset(); + + return isChanged; +} + +bool ZBackupBase::editConfigInteractively() +{ + string configData( Config::toString( *config.storable ) ); + + if ( !spawnEditor( configData, &Config::validateProto ) ) + return false; + + ConfigInfo newConfig; + Config::parseProto( configData, &newConfig ); + if ( Config::toString( *config.storable ) == + Config::toString( newConfig ) ) + { + verbosePrintf( "No changes made to config\n" ); + return false; + } + + verbosePrintf( "Updating configuration...\n" ); + config.storable->MergeFrom( newConfig ); + verbosePrintf( +"Configuration successfully updated!\n" +"Updated configuration:\n%s", Config::toString( *config.storable ).c_str() ); + + return true; +} diff -Nru zbackup-1.2/zbackup_base.hh zbackup-1.4.1+20150403/zbackup_base.hh --- zbackup-1.2/zbackup_base.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/zbackup_base.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,86 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef ZBACKUP_BASE_HH_INCLUDED +#define ZBACKUP_BASE_HH_INCLUDED + +#include +#include + +#include "ex.hh" +#include "chunk_index.hh" +#include "config.hh" + +struct Paths +{ + std::string storageDir; + + Paths( std::string const & storageDir ); + + std::string getTmpPath(); + std::string getRestorePath(); + std::string getCreatePath(); + std::string getBundlesPath(); + std::string getStorageInfoPath(); + std::string getExtendedStorageInfoPath(); + std::string getIndexPath(); + std::string getBackupsPath(); +}; + +class ZBackupBase: public Paths +{ +public: + DEF_EX( Ex, "ZBackup exception", std::exception ) + DEF_EX_STR( exWontOverwrite, "Won't overwrite existing file", Ex ) + DEF_EX( exStdinError, "Error reading from standard input", Ex ) + DEF_EX( exWontReadFromTerminal, "Won't read data from a terminal", exStdinError ) + DEF_EX( exStdoutError, "Error writing to standard output", Ex ) + DEF_EX( exWontWriteToTerminal, "Won't write data to a terminal", exStdoutError ) + DEF_EX( exSerializeError, "Failed to serialize data", Ex ) + DEF_EX( exParseError, "Failed to parse data", Ex ) + DEF_EX( exChecksumError, "Checksum error", Ex ) + DEF_EX_STR( exCantDeriveStorageDir, "The path must be within the backups/ dir:", Ex ) + + /// Opens the storage + ZBackupBase( std::string const & storageDir, std::string const & password ); + ZBackupBase( std::string const & storageDir, std::string const & password, Config & configIn ); + ZBackupBase( std::string const & storageDir, std::string const & password, + bool prohibitChunkIndexLoading ); + ZBackupBase( std::string const & storageDir, std::string const & password, Config & configIn, + bool prohibitChunkIndexLoading ); + + /// Creates new storage + static void initStorage( std::string const & storageDir, std::string const & password, + bool isEncrypted, Config const & ); + + /// For a given file within the backups/ dir in the storage, returns its + /// storage dir or throws an exception + static std::string deriveStorageDirFromBackupsFile( std::string const & backupsFile, bool allowOutside = false ); + + void propagateUpdate(); + + void saveExtendedStorageInfo(); + + void setPassword( std::string const & password ); + + // returns true if data is changed + bool spawnEditor( std::string & data, bool( * validator ) + ( string const &, string const & ) ); + + // Edit current configuration + // returns true if configuration is changed + bool editConfigInteractively(); + + StorageInfo storageInfo; + EncryptionKey encryptionkey; + ExtendedStorageInfo extendedStorageInfo; + TmpMgr tmpMgr; + ChunkIndex chunkIndex; + Config config; + +private: + StorageInfo loadStorageInfo(); + ExtendedStorageInfo loadExtendedStorageInfo( EncryptionKey const & ); +}; + +#endif diff -Nru zbackup-1.2/zbackup.cc zbackup-1.4.1+20150403/zbackup.cc --- zbackup-1.2/zbackup.cc 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/zbackup.cc 2015-04-03 09:33:05.000000000 +0000 @@ -1,422 +1,214 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#include -#include -#include -#include -#include -#include -#include -#include - -#include "backup_creator.hh" -#include "backup_file.hh" -#include "backup_restorer.hh" +#include "zutils.hh" #include "debug.hh" -#include "dir.hh" -#include "encryption_key.hh" -#include "ex.hh" -#include "file.hh" -#include "mt.hh" -#include "sha256.hh" -#include "sptr.hh" -#include "storage_info_file.hh" -#include "zbackup.hh" - -using std::vector; - -Paths::Paths( string const & storageDir ): storageDir( storageDir ) -{ -} - -string Paths::getTmpPath() -{ - return string( Dir::addPath( storageDir, "tmp" ) ); -} - -string Paths::getBundlesPath() -{ - return string( Dir::addPath( storageDir, "bundles" ) ); -} - -string Paths::getStorageInfoPath() -{ - return string( Dir::addPath( storageDir, "info" ) ); -} - -string Paths::getIndexPath() -{ - return string( Dir::addPath( storageDir, "index" ) ); -} - -string Paths::getBackupsPath() -{ - return string( Dir::addPath( storageDir, "backups" ) ); -} - -ZBackupBase::ZBackupBase( string const & storageDir, string const & password ): - Paths( storageDir ), storageInfo( loadStorageInfo() ), - encryptionkey( password, storageInfo.has_encryption_key() ? - &storageInfo.encryption_key() : 0 ), - tmpMgr( getTmpPath() ), - chunkIndex( encryptionkey, tmpMgr, getIndexPath() ) -{ -} - -StorageInfo ZBackupBase::loadStorageInfo() -{ - StorageInfo storageInfo; - - StorageInfoFile::load( getStorageInfoPath(), storageInfo ); - - return storageInfo; -} - -void ZBackupBase::initStorage( string const & storageDir, - string const & password, - bool isEncrypted ) -{ - StorageInfo storageInfo; - // TODO: make the following configurable - storageInfo.set_chunk_max_size( 65536 ); - storageInfo.set_bundle_max_payload_size( 0x200000 ); - - if ( isEncrypted ) - EncryptionKey::generate( password, - *storageInfo.mutable_encryption_key() ); - - Paths paths( storageDir ); - - if ( !Dir::exists( storageDir ) ) - Dir::create( storageDir ); - - if ( !Dir::exists( paths.getBundlesPath() ) ) - Dir::create( paths.getBundlesPath() ); - - if ( !Dir::exists( paths.getBackupsPath() ) ) - Dir::create( paths.getBackupsPath() ); - - if ( !Dir::exists( paths.getIndexPath() ) ) - Dir::create( paths.getIndexPath() ); - - string storageInfoPath( paths.getStorageInfoPath() ); - - if ( File::exists( storageInfoPath ) ) - throw exWontOverwrite( storageInfoPath ); - - StorageInfoFile::save( storageInfoPath, storageInfo ); -} - -string ZBackupBase::deriveStorageDirFromBackupsFile( string const & - backupsFile ) -{ - // TODO: handle cases when there's a backup/ folder within the backup/ folder - // correctly - string realPath = Dir::getRealPath( Dir::getDirName( backupsFile ) ); - size_t pos; - if ( realPath.size() >= 8 && strcmp( realPath.c_str() + realPath.size() - 8, - "/backups") == 0 ) - pos = realPath.size() - 8; - else - pos = realPath.rfind( "/backups/" ); - if ( pos == string::npos ) - throw exCantDeriveStorageDir( backupsFile ); - else - return realPath.substr( 0, pos ); -} - -ZBackup::ZBackup( string const & storageDir, string const & password, - size_t threads ): - ZBackupBase( storageDir, password ), - chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkIndex, - getBundlesPath(), getIndexPath(), threads ) -{ -} - -void ZBackup::backupFromStdin( string const & outputFileName ) -{ - if ( isatty( fileno( stdin ) ) ) - throw exWontReadFromTerminal(); - - if ( File::exists( outputFileName ) ) - throw exWontOverwrite( outputFileName ); - - Sha256 sha256; - BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter ); - - time_t startTime = time( 0 ); - uint64_t totalDataSize = 0; - - for ( ; ; ) - { - size_t toRead = backupCreator.getInputBufferSize(); -// dPrintf( "Reading up to %u bytes from stdin\n", toRead ); - - void * inputBuffer = backupCreator.getInputBuffer(); - size_t rd = fread( inputBuffer, 1, toRead, stdin ); - - if ( !rd ) - { - if ( feof( stdin ) ) - { - dPrintf( "No more input on stdin\n" ); - break; - } - else - throw exStdinError(); - } - - sha256.add( inputBuffer, rd ); - - backupCreator.handleMoreData( rd ); - - totalDataSize += rd; - } - - // Finish up with the creator - backupCreator.finish(); - - string serialized; - backupCreator.getBackupData( serialized ); - - BackupInfo info; - - info.set_sha256( sha256.finish() ); - info.set_size( totalDataSize ); - - // Shrink the serialized data iteratively until it wouldn't shrink anymore - for ( ; ; ) - { - BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter ); - char const * ptr = serialized.data(); - size_t left = serialized.size(); - while( left ) - { - size_t bufferSize = backupCreator.getInputBufferSize(); - size_t toCopy = bufferSize > left ? left : bufferSize; - - memcpy( backupCreator.getInputBuffer(), ptr, toCopy ); - backupCreator.handleMoreData( toCopy ); - ptr += toCopy; - left -= toCopy; - } - - backupCreator.finish(); - - string newGen; - backupCreator.getBackupData( newGen ); - - if ( newGen.size() < serialized.size() ) - { - serialized.swap( newGen ); - info.set_iterations( info.iterations() + 1 ); - } - else - break; - } - - dPrintf( "Iterations: %u\n", info.iterations() ); - - info.mutable_backup_data()->swap( serialized ); - - info.set_time( time( 0 ) - startTime ); - - // Commit the bundles to the disk before creating the final output file - chunkStorageWriter.commit(); - - // Now save the resulting BackupInfo - - sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile(); - BackupFile::save( tmpFile->getFileName(), encryptionkey, info ); - tmpFile->moveOverTo( outputFileName ); -} - -ZRestore::ZRestore( string const & storageDir, string const & password, - size_t cacheSize ): - ZBackupBase( storageDir, password ), - chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(), - cacheSize ) -{ -} - -void ZRestore::restoreToStdin( string const & inputFileName ) -{ - if ( isatty( fileno( stdout ) ) ) - throw exWontWriteToTerminal(); - - BackupInfo backupInfo; - - BackupFile::load( inputFileName, encryptionkey, backupInfo ); - - string backupData; - - // Perform the iterations needed to get to the actual user backup data - for ( ; ; ) - { - backupData.swap( *backupInfo.mutable_backup_data() ); - - if ( backupInfo.iterations() ) - { - struct StringWriter: public DataSink - { - string result; - - virtual void saveData( void const * data, size_t size ) - { - result.append( ( char const * ) data, size ); - } - } stringWriter; - - BackupRestorer::restore( chunkStorageReader, backupData, stringWriter ); - backupInfo.mutable_backup_data()->swap( stringWriter.result ); - backupInfo.set_iterations( backupInfo.iterations() - 1 ); - } - else - break; - } - - struct StdoutWriter: public DataSink - { - Sha256 sha256; - - virtual void saveData( void const * data, size_t size ) - { - sha256.add( data, size ); - if ( fwrite( data, size, 1, stdout ) != 1 ) - throw exStdoutError(); - } - } stdoutWriter; - - BackupRestorer::restore( chunkStorageReader, backupData, stdoutWriter ); - - if ( stdoutWriter.sha256.finish() != backupInfo.sha256() ) - throw exChecksumError(); -} +#include "version.hh" +#include "utils.hh" +DEF_EX( exSpecifyTwoKeys, "Specify password flag (--non-encrypted or --password-file)" + " for import/export/passwd operation twice (first for source and second for destination)", std::exception ) DEF_EX( exNonEncryptedWithKey, "--non-encrypted and --password-file are incompatible", std::exception ) DEF_EX( exSpecifyEncryptionOptions, "Specify either --password-file or --non-encrypted", std::exception ) -DEF_EX_STR( exInvalidThreadsValue, "Invalid threads value specified:", std::exception ) int main( int argc, char *argv[] ) { try { - char const * passwordFile = 0; - bool nonEncrypted = false; - size_t const defaultThreads = getNumberOfCpus(); - size_t threads = defaultThreads; - size_t const defaultCacheSizeMb = 40; - size_t cacheSizeMb = defaultCacheSizeMb; + dPrintf( "ZBackup version %s\n", zbackup_version.c_str() ); + + bool printHelp = false; vector< char const * > args; + vector< string > passwords; + Config config; for( int x = 1; x < argc; ++x ) { + string option; + Config::OptionType optionType = Config::Runtime; + if ( strcmp( argv[ x ], "--password-file" ) == 0 && x + 1 < argc ) { - passwordFile = argv[ x + 1 ]; + // Read the password + char const * passwordFile = argv[ x + 1 ]; + string passwordData; + if ( passwordFile ) + { + File f( passwordFile, File::ReadOnly ); + passwordData.resize( f.size() ); + f.read( &passwordData[ 0 ], passwordData.size() ); + + // If the password ends with \n, remove that last \n. Many editors will + // add \n there even if a user doesn't want them to + if ( !passwordData.empty() && + passwordData[ passwordData.size() - 1 ] == '\n' ) + passwordData.resize( passwordData.size() - 1 ); + passwords.push_back( passwordData ); + } ++x; } else if ( strcmp( argv[ x ], "--non-encrypted" ) == 0 ) - nonEncrypted = true; + { + passwords.push_back( "" ); + } else if ( strcmp( argv[ x ], "--silent" ) == 0 ) verboseMode = false; else + if ( strcmp( argv[ x ], "--exchange" ) == 0 && x + 1 < argc ) + { + fprintf( stderr, "%s is deprecated, use -O exchange instead\n", argv[ x ] ); + option = argv[ x ] + 2;//; + "=" + argv[ x + 1 ]; + option += "="; + option += argv[ x + 1 ]; + goto parse_option; + } + else if ( strcmp( argv[ x ], "--threads" ) == 0 && x + 1 < argc ) { - int n; - if ( sscanf( argv[ x + 1 ], "%zu %n", &threads, &n ) != 1 || - argv[ x + 1 ][ n ] || threads < 1 ) - throw exInvalidThreadsValue( argv[ x + 1 ] ); - ++x; + fprintf( stderr, "%s is deprecated, use -O threads instead\n", argv[ x ] ); + option = argv[ x ] + 2; + option += "="; + option += argv[ x + 1 ]; + goto parse_option; } else if ( strcmp( argv[ x ], "--cache-size" ) == 0 && x + 1 < argc ) { + fprintf( stderr, "%s is deprecated, use -O cache-size instead\n", argv[ x ] ); + size_t cacheSizeMb; char suffix[ 16 ]; int n; if ( sscanf( argv[ x + 1 ], "%zu %15s %n", - &cacheSizeMb, suffix, &n ) == 2 && !argv[ x + 1 ][ n ] ) + &cacheSizeMb, suffix, &n ) == 2 && !argv[ x + 1][ n ] ) { - // Check the suffix - for ( char * c = suffix; *c; ++c ) - *c = tolower( *c ); + option = argv[ x ] + 2; + option += "=" + Utils::numberToString( cacheSizeMb ) + "MiB"; + goto parse_option; + } + } + else + if ( strcmp( argv[ x ], "--compression" ) == 0 && x + 1 < argc ) + { + fprintf( stderr, "%s is deprecated, use -o bundle.compression_method instead\n", argv[ x ] ); + option = argv[ x ] + 2; + option += "="; + option += argv[ x + 1 ]; + optionType = Config::Storable; + goto parse_option; + } + else + if ( strcmp( argv[ x ], "--help" ) == 0 || strcmp( argv[ x ], "-h" ) == 0 ) + { + printHelp = true; + } + else + if ( ( strcmp( argv[ x ], "-o" ) == 0 || strcmp( argv[ x ], "-O" ) == 0 ) + && x + 1 < argc ) + { + option = argv[ x + 1 ]; + if ( !option.empty() ) + { + if ( strcmp( argv[ x ], "-O" ) == 0 ) + optionType = Config::Runtime; + else + if ( strcmp( argv[ x ], "-o" ) == 0 ) + optionType = Config::Storable; - if ( strcmp( suffix, "mb" ) != 0 ) + if ( strcmp( option.c_str(), "help" ) == 0 ) { - fprintf( stderr, "Invalid suffix specified in cache size: %s. " - "The only supported suffix is 'mb' for megabytes\n", - argv[ x + 1 ] ); - return EXIT_FAILURE; + config.showHelp( optionType ); + return EXIT_SUCCESS; + } + else + { +parse_option: + if ( !config.parseOrValidate( option, optionType ) ) + goto invalid_option; } - - ++x; } else { - fprintf( stderr, "Invalid cache size value specified: %s. " - "Must be a number with the 'mb' suffix, e.g. '100mb'\n", - argv[ x + 1 ] ); +invalid_option: + fprintf( stderr, "Invalid option specified: %s\n", + option.c_str() ); return EXIT_FAILURE; } + ++x; } else args.push_back( argv[ x ] ); } - if ( nonEncrypted && passwordFile ) - throw exNonEncryptedWithKey(); - - if ( args.size() < 1 ) + if ( args.size() < 1 || printHelp ) { fprintf( stderr, -"ZBackup, a versatile deduplicating backup tool, version 1.2\n" -"Copyright (c) 2012-2013 Konstantin Isakov \n" -"Comes with no warranty. Licensed under GNU GPLv2 or later.\n" +"ZBackup, a versatile deduplicating backup tool, version %s\n" +"Copyright (c) 2012-2014 Konstantin Isakov and\n" +"ZBackup contributors\n" +"Comes with no warranty. Licensed under GNU GPLv2 or later + OpenSSL.\n" "Visit the project's home page at http://zbackup.org/\n\n" -"Usage: %s [flags] [command args]\n" +"Usage: %s [flags] [command args]\n" " Flags: --non-encrypted|--password-file \n" +" password flag should be specified twice if\n" +" import/export/passwd command specified\n" " --silent (default is verbose)\n" -" --threads (default is %zu on your system)\n" -" --cache-size MB (default is %zu)\n" +" --help|-h show this message\n" +" -O (overrides runtime configuration,\n" +" can be specified multiple times,\n" +" for detailed runtime options overview run with -O help)\n" +" -o (overrides storable repository\n" +" configuration, can be specified multiple times,\n" +" for detailed storable options overview run with -o help)\n" " Commands:\n" -" init - initializes new storage;\n" -" backup - performs a backup from stdin;\n" -" restore - restores a backup to stdout.\n", *argv, - defaultThreads, defaultCacheSizeMb ); +" init - initializes new storage\n" +" backup - performs a backup from stdin\n" +" restore - restores a backup to stdout\n" +" export -\n" +" performs export from source to destination storage\n" +" import -\n" +" performs import from source to destination storage,\n" +" for export/import storage path must be\n" +" a valid (initialized) storage\n" +" gc - performs chunk garbage collection\n" +" passwd - changes repo info file passphrase\n" +//" info - shows repo information\n" +" config [show|edit|set|reset] - performs\n" +" configuration manipulations (default is show)\n" +"", zbackup_version.c_str(), *argv ); return EXIT_FAILURE; } - // Read the password - string passwordData; - if ( passwordFile ) - { - File f( passwordFile, File::ReadOnly ); - passwordData.resize( f.size() ); - f.read( &passwordData[ 0 ], passwordData.size() ); - - // If the password ends with \n, remove that last \n. Many editors will - // add \n there even if a user doesn't want them to - if ( !passwordData.empty() && - passwordData[ passwordData.size() - 1 ] == '\n' ) - passwordData.resize( passwordData.size() - 1 ); - } + if ( passwords.size() > 1 && + ( ( passwords[ 0 ].empty() && !passwords[ 1 ].empty() ) || + ( !passwords[ 0 ].empty() && passwords[ 1 ].empty() ) ) && + ( strcmp( args[ 0 ], "export" ) != 0 && + strcmp( args[ 0 ], "import" ) != 0 && + strcmp( args[ 0 ], "passwd" ) ) ) + throw exNonEncryptedWithKey(); + else + if ( passwords.size() != 2 && + ( strcmp( args[ 0 ], "export" ) == 0 || + strcmp( args[ 0 ], "import" ) == 0 || + strcmp( args[ 0 ], "passwd" ) == 0 ) ) + throw exSpecifyTwoKeys(); + else + if ( passwords.size() < 1 ) + throw exSpecifyEncryptionOptions(); if ( strcmp( args[ 0 ], "init" ) == 0 ) { // Perform the init if ( args.size() != 2 ) { - fprintf( stderr, "Usage: %s init \n", *argv ); + fprintf( stderr, "Usage: %s %s \n", *argv, args[ 0 ] ); return EXIT_FAILURE; } - if ( !nonEncrypted && !passwordFile ) - throw exSpecifyEncryptionOptions(); - ZBackup::initStorage( args[ 1 ], passwordData, !nonEncrypted ); + ZBackup::initStorage( args[ 1 ], + passwords[ 0 ], !passwords[ 0 ].empty(), config ); } else if ( strcmp( args[ 0 ], "backup" ) == 0 ) @@ -424,12 +216,12 @@ // Perform the backup if ( args.size() != 2 ) { - fprintf( stderr, "Usage: %s backup \n", - *argv ); + fprintf( stderr, "Usage: %s %s \n", + *argv, args[ 0 ] ); return EXIT_FAILURE; } ZBackup zb( ZBackup::deriveStorageDirFromBackupsFile( args[ 1 ] ), - passwordData, threads ); + passwords[ 0 ], config ); zb.backupFromStdin( args[ 1 ] ); } else @@ -438,15 +230,156 @@ // Perform the restore if ( args.size() != 2 ) { - fprintf( stderr, "Usage: %s restore \n", - *argv ); + fprintf( stderr, "Usage: %s %s \n", + *argv , args[ 0 ] ); return EXIT_FAILURE; } ZRestore zr( ZRestore::deriveStorageDirFromBackupsFile( args[ 1 ] ), - passwordData, cacheSizeMb * 1048576 ); + passwords[ 0 ], config ); zr.restoreToStdin( args[ 1 ] ); } else + if ( strcmp( args[ 0 ], "export" ) == 0 || strcmp( args[ 0 ], "import" ) == 0 ) + { + if ( args.size() != 3 ) + { + fprintf( stderr, "Usage: %s %s \n", + *argv, args[ 0 ] ); + return EXIT_FAILURE; + } + if ( config.runtime.exchange.none() ) + { + fprintf( stderr, "Specify any --exchange flag\n" ); + return EXIT_FAILURE; + } + + int src, dst; + if ( strcmp( args[ 0 ], "export" ) == 0 ) + { + src = 1; + dst = 2; + } + else + if ( strcmp( args[ 0 ], "import" ) == 0 ) + { + src = 2; + dst = 1; + } + dPrintf( "%s src: %s\n", args[ 0 ], args[ src ] ); + dPrintf( "%s dst: %s\n", args[ 0 ], args[ dst ] ); + + ZExchange ze( ZBackupBase::deriveStorageDirFromBackupsFile( args[ src ], true ), + passwords[ src - 1 ], + ZBackupBase::deriveStorageDirFromBackupsFile( args[ dst ], true ), + passwords[ dst - 1 ], + config ); + ze.exchange(); + } + else + if ( strcmp( args[ 0 ], "gc" ) == 0 ) + { + // Perform the restore + if ( args.size() != 2 ) + { + fprintf( stderr, "Usage: %s %s \n", + *argv, args[ 0 ] ); + return EXIT_FAILURE; + } + ZCollector zc( args[ 1 ], passwords[ 0 ], config ); + zc.gc(); + } + else + if ( strcmp( args[ 0 ], "passwd" ) == 0 ) + { + // Perform the password change + if ( args.size() != 2 ) + { + fprintf( stderr, "Usage: %s %s \n", + *argv, args[ 0 ] ); + return EXIT_FAILURE; + } + + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ), + passwords[ 0 ], true ); + + if ( passwords[ 0 ].empty() != passwords[ 1 ].empty() ) + { + fprintf( stderr, +"Changing repo encryption type (non-encrypted to encrypted and vice versa) is possible " +"only via import/export operations.\n" +"Current repo type: %s.\n", zbb.encryptionkey.hasKey() ? "encrypted" : "non-encrypted" ); + return EXIT_FAILURE; + } + zbb.setPassword( passwords[ 1 ] ); + } + else + if ( strcmp( args[ 0 ], "info" ) == 0 ) + { + // Show repo info + if ( args.size() != 2 ) + { + fprintf( stderr, "Usage: %s %s \n", + *argv, args[ 0 ] ); + return EXIT_FAILURE; + } + + // TODO: implementation in ZBackupBase + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ), + passwords[ 0 ], true ); + fprintf( stderr, "NOT IMPLEMENTED YET!\n" ); + return EXIT_FAILURE; + } + else + if ( strcmp( args[ 0 ], "config" ) == 0 ) + { + if ( args.size() < 2 || args.size() > 3 ) + { + fprintf( stderr, "Usage: %s %s [show|edit|set|reset] \n", + *argv, args[ 0 ] ); + return EXIT_FAILURE; + } + + int fieldStorage = 1; + int fieldAction = 2; + + if ( args.size() == 3 ) + { + fieldStorage = 2; + fieldAction = 1; + } + + if ( args.size() > 2 && strcmp( args[ fieldAction ], "edit" ) == 0 ) + { + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), + passwords[ 0 ], true ); + if ( zbb.editConfigInteractively() ) + zbb.saveExtendedStorageInfo(); + } + else + if ( args.size() > 2 && strcmp( args[ fieldAction ], "set" ) == 0 ) + { + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), + passwords[ 0 ], config, true ); + zbb.config.show(); + zbb.saveExtendedStorageInfo(); + } + else + if ( args.size() > 2 && strcmp( args[ fieldAction ], "reset" ) == 0 ) + { + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), + passwords[ 0 ], true ); + zbb.config.reset_storable(); + zbb.config.show(); + zbb.saveExtendedStorageInfo(); + } + else + { + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), + passwords[ 0 ], true ); + zbb.config.show(); + } + } + else { fprintf( stderr, "Error: unknown command line option: %s\n", args[ 0 ] ); return EXIT_FAILURE; @@ -457,5 +390,10 @@ fprintf( stderr, "%s\n", e.what() ); return EXIT_FAILURE; } + catch( ... ) + { + fprintf( stderr, "Unknown exception!\n" ); + return EXIT_FAILURE; + } return EXIT_SUCCESS; } diff -Nru zbackup-1.2/zbackup.hh zbackup-1.4.1+20150403/zbackup.hh --- zbackup-1.2/zbackup.hh 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/zbackup.hh 1970-01-01 00:00:00.000000000 +0000 @@ -1,97 +0,0 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later - -#ifndef ZBACKUP_HH_INCLUDED__ -#define ZBACKUP_HH_INCLUDED__ - -#include -#include -#include -#include - -#include "chunk_id.hh" -#include "chunk_index.hh" -#include "chunk_storage.hh" -#include "encryption_key.hh" -#include "ex.hh" -#include "tmp_mgr.hh" -#include "zbackup.pb.h" - -using std::string; -using std::vector; - -struct Paths -{ - string storageDir; - - Paths( string const & storageDir ); - - string getTmpPath(); - string getRestorePath(); - string getCreatePath(); - string getBundlesPath(); - string getStorageInfoPath(); - string getIndexPath(); - string getBackupsPath(); -}; - -class ZBackupBase: protected Paths -{ -public: - DEF_EX( Ex, "ZBackup exception", std::exception ) - DEF_EX_STR( exWontOverwrite, "Won't overwrite existing file", Ex ) - DEF_EX( exStdinError, "Error reading from standard input", Ex ) - DEF_EX( exWontReadFromTerminal, "Won't read data from a terminal", exStdinError ) - DEF_EX( exStdoutError, "Error writing to standard output", Ex ) - DEF_EX( exWontWriteToTerminal, "Won't write data to a terminal", exStdoutError ) - DEF_EX( exSerializeError, "Failed to serialize data", Ex ) - DEF_EX( exParseError, "Failed to parse data", Ex ) - DEF_EX( exChecksumError, "Checksum error", Ex ) - DEF_EX_STR( exCantDeriveStorageDir, "The path must be within the backups/ dir:", Ex ) - - /// Opens the storage - ZBackupBase( string const & storageDir, string const & password ); - - /// Creates new storage - static void initStorage( string const & storageDir, string const & password, - bool isEncrypted ); - - /// For a given file within the backups/ dir in the storage, returns its - /// storage dir or throws an exception - static string deriveStorageDirFromBackupsFile( string const & backupsFile ); - -protected: - StorageInfo storageInfo; - EncryptionKey encryptionkey; - TmpMgr tmpMgr; - ChunkIndex chunkIndex; - -private: - StorageInfo loadStorageInfo(); -}; - -class ZBackup: public ZBackupBase -{ - ChunkStorage::Writer chunkStorageWriter; - -public: - ZBackup( string const & storageDir, string const & password, - size_t threads ); - - /// Backs up the data from stdin - void backupFromStdin( string const & outputFileName ); -}; - -class ZRestore: public ZBackupBase -{ - ChunkStorage::Reader chunkStorageReader; - -public: - ZRestore( string const & storageDir, string const & password, - size_t cacheSize ); - - /// Restores the data to stdin - void restoreToStdin( string const & inputFileName ); -}; - -#endif diff -Nru zbackup-1.2/zbackup.proto zbackup-1.4.1+20150403/zbackup.proto --- zbackup-1.2/zbackup.proto 2013-08-18 06:59:25.000000000 +0000 +++ zbackup-1.4.1+20150403/zbackup.proto 2015-04-03 09:33:05.000000000 +0000 @@ -1,5 +1,5 @@ -// Copyright (c) 2012-2013 Konstantin Isakov -// Part of ZBackup. Licensed under GNU GPLv2 or later +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE // Protobuffers used in zbackup @@ -30,14 +30,53 @@ message StorageInfo { // Maximum chunk size used when storing chunks - required uint32 chunk_max_size = 1; + optional uint32 chunk_max_size = 1 [deprecated = true]; // Maximum number of bytes a bundle can hold. Only real chunk bytes are // counted, not metadata. Any bundle should be able to contain at least // one arbitrary single chunk, so this should not be smaller than // chunk_max_size - required uint32 bundle_max_payload_size = 2; + optional uint32 bundle_max_payload_size = 2 [deprecated = true]; // If present, used for encryption/decryption of all data optional EncryptionKeyInfo encryption_key = 3; + // Default compression for new bundles + optional string default_compression_method = 4 [default = "lzma", deprecated = true]; +} + +message LZMAConfigInfo +{ + // Compression level for new LZMA-compressed files + optional uint32 compression_level = 1 [default = 6]; +} + +message ChunkConfigInfo +{ + // Maximum chunk size used when storing chunks + required uint32 max_size = 1 [default = 65536]; +} + +message BundleConfigInfo +{ + // Maximum number of bytes a bundle can hold. Only real chunk bytes are + // counted, not metadata. Any bundle should be able to contain at least + // one arbitrary single chunk, so this should not be smaller than + // chunk_max_size + required uint32 max_payload_size = 2 [default = 0x200000]; + // Compression method for new bundles + optional string compression_method = 3 [default = "lzma"]; +} + +// Storable config values should always have default values +message ConfigInfo +{ + required ChunkConfigInfo chunk = 1; + required BundleConfigInfo bundle = 2; + required LZMAConfigInfo lzma = 3; +} + +message ExtendedStorageInfo +{ + // Config data storage + optional ConfigInfo config = 1; } message BundleInfo @@ -61,6 +100,18 @@ required uint32 version = 1; } +message BundleFileHeader +{ + // File format version + required uint32 version = 1; + + // Compression method that is used for this file + // If the program doesn't support that field, it will try LZMA. If it is + // LZMA, that will work. If it isn't, it will have aborted before because + // the version in FileHeader is higher than it can support. + optional string compression_method = 2 [default = "lzma"]; +} + message IndexBundleHeader { // Id of the bundle following in the stream. If not present, indicates the diff -Nru zbackup-1.2/zutils.cc zbackup-1.4.1+20150403/zutils.cc --- zbackup-1.2/zutils.cc 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/zutils.cc 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,369 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include "zutils.hh" +#include "backup_creator.hh" +#include "sha256.hh" +#include "backup_collector.hh" +#include + +using std::vector; +using std::bitset; +using std::iterator; + +ZBackup::ZBackup( string const & storageDir, string const & password, + Config & configIn ): + ZBackupBase( storageDir, password, configIn ), + chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex, + getBundlesPath(), getIndexPath(), config.runtime.threads ) +{ +} + +void ZBackup::backupFromStdin( string const & outputFileName ) +{ + if ( isatty( fileno( stdin ) ) ) + throw exWontReadFromTerminal(); + + if ( File::exists( outputFileName ) ) + throw exWontOverwrite( outputFileName ); + + Sha256 sha256; + BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); + + time_t startTime = time( 0 ); + uint64_t totalDataSize = 0; + + for ( ; ; ) + { + size_t toRead = backupCreator.getInputBufferSize(); +// dPrintf( "Reading up to %u bytes from stdin\n", toRead ); + + void * inputBuffer = backupCreator.getInputBuffer(); + size_t rd = fread( inputBuffer, 1, toRead, stdin ); + + if ( !rd ) + { + if ( feof( stdin ) ) + { + dPrintf( "No more input on stdin\n" ); + break; + } + else + throw exStdinError(); + } + + sha256.add( inputBuffer, rd ); + + backupCreator.handleMoreData( rd ); + + totalDataSize += rd; + } + + // Finish up with the creator + backupCreator.finish(); + + string serialized; + backupCreator.getBackupData( serialized ); + + BackupInfo info; + + info.set_sha256( sha256.finish() ); + info.set_size( totalDataSize ); + + // Shrink the serialized data iteratively until it wouldn't shrink anymore + for ( ; ; ) + { + BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); + char const * ptr = serialized.data(); + size_t left = serialized.size(); + while( left ) + { + size_t bufferSize = backupCreator.getInputBufferSize(); + size_t toCopy = bufferSize > left ? left : bufferSize; + + memcpy( backupCreator.getInputBuffer(), ptr, toCopy ); + backupCreator.handleMoreData( toCopy ); + ptr += toCopy; + left -= toCopy; + } + + backupCreator.finish(); + + string newGen; + backupCreator.getBackupData( newGen ); + + if ( newGen.size() < serialized.size() ) + { + serialized.swap( newGen ); + info.set_iterations( info.iterations() + 1 ); + } + else + break; + } + + dPrintf( "Iterations: %u\n", info.iterations() ); + + info.mutable_backup_data()->swap( serialized ); + + info.set_time( time( 0 ) - startTime ); + + // Commit the bundles to the disk before creating the final output file + chunkStorageWriter.commit(); + + // Now save the resulting BackupInfo + + sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile(); + BackupFile::save( tmpFile->getFileName(), encryptionkey, info ); + tmpFile->moveOverTo( outputFileName ); +} + +ZRestore::ZRestore( string const & storageDir, string const & password, + Config & configIn ): + ZBackupBase( storageDir, password, configIn ), + chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), + config.runtime.cacheSize ) +{ +} + +void ZRestore::restoreToStdin( string const & inputFileName ) +{ + if ( isatty( fileno( stdout ) ) ) + throw exWontWriteToTerminal(); + + BackupInfo backupInfo; + + BackupFile::load( inputFileName, encryptionkey, backupInfo ); + + string backupData; + + // Perform the iterations needed to get to the actual user backup data + BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL ); + + struct StdoutWriter: public DataSink + { + Sha256 sha256; + + virtual void saveData( void const * data, size_t size ) + { + sha256.add( data, size ); + if ( fwrite( data, size, 1, stdout ) != 1 ) + throw exStdoutError(); + } + } stdoutWriter; + + BackupRestorer::restore( chunkStorageReader, backupData, &stdoutWriter, NULL ); + + if ( stdoutWriter.sha256.finish() != backupInfo.sha256() ) + throw exChecksumError(); +} + +ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword, + string const & dstStorageDir, string const & dstPassword, + Config & configIn ): + srcZBackupBase( srcStorageDir, srcPassword, configIn, true ), + dstZBackupBase( dstStorageDir, dstPassword, configIn, true ), + config( configIn ) +{ +} + +void ZExchange::exchange() +{ + vector< BackupExchanger::PendingExchangeRename > pendingExchangeRenames; + + if ( config.runtime.exchange.test( BackupExchanger::bundles ) ) + { + verbosePrintf( "Searching for bundles...\n" ); + + vector< string > bundles = BackupExchanger::findOrRebuild( + srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() ); + + for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it ) + { + verbosePrintf( "Processing bundle file %s... ", it->c_str() ); + string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) ); + if ( !File::exists( outputFileName ) ) + { + sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath ( + srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true ); + sptr< Bundle::Creator > creator = new Bundle::Creator; + sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); + creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader ); + + if ( creator.get() && reader.get() ) + { + creator.reset(); + reader.reset(); + pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( + bundleTempFile, outputFileName ) ); + verbosePrintf( "done.\n" ); + } + } + else + { + verbosePrintf( "file exists - skipped.\n" ); + } + } + + verbosePrintf( "Bundle exchange completed.\n" ); + } + + if ( config.runtime.exchange.test( BackupExchanger::index ) ) + { + verbosePrintf( "Searching for indicies...\n" ); + vector< string > indicies = BackupExchanger::findOrRebuild( + srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() ); + + for ( std::vector< string >::iterator it = indicies.begin(); it != indicies.end(); ++it ) + { + verbosePrintf( "Processing index file %s... ", it->c_str() ); + string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) ); + if ( !File::exists( outputFileName ) ) + { + sptr< IndexFile::Reader > reader = new IndexFile::Reader( srcZBackupBase.encryptionkey, + Dir::addPath( srcZBackupBase.getIndexPath(), *it ) ); + sptr< TemporaryFile > indexTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); + sptr< IndexFile::Writer > writer = new IndexFile::Writer( dstZBackupBase.encryptionkey, + indexTempFile->getFileName() ); + + BundleInfo bundleInfo; + Bundle::Id bundleId; + while( reader->readNextRecord( bundleInfo, bundleId ) ) + { + writer->add( bundleInfo, bundleId ); + } + + if ( writer.get() && reader.get() ) + { + writer.reset(); + reader.reset(); + pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( + indexTempFile, outputFileName ) ); + verbosePrintf( "done.\n" ); + } + } + else + { + verbosePrintf( "file exists - skipped.\n" ); + } + } + + verbosePrintf( "Index exchange completed.\n" ); + } + + if ( config.runtime.exchange.test( BackupExchanger::backups ) ) + { + BackupInfo backupInfo; + + verbosePrintf( "Searching for backups...\n" ); + vector< string > backups = BackupExchanger::findOrRebuild( + srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() ); + + for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) + { + verbosePrintf( "Processing backup file %s... ", it->c_str() ); + string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) ); + if ( !File::exists( outputFileName ) ) + { + BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ), + srcZBackupBase.encryptionkey, backupInfo ); + sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); + BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey, + backupInfo ); + pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( + tmpFile, outputFileName ) ); + verbosePrintf( "done.\n" ); + } + else + { + verbosePrintf( "file exists - skipped.\n" ); + } + } + + verbosePrintf( "Backup exchange completed.\n" ); + } + + if ( pendingExchangeRenames.size() > 0 ) + { + verbosePrintf( "Moving files from temp directory to appropriate places... " ); + for ( size_t x = pendingExchangeRenames.size(); x--; ) + { + BackupExchanger::PendingExchangeRename & r = pendingExchangeRenames[ x ]; + r.first->moveOverTo( r.second ); + if ( r.first.get() ) + { + r.first.reset(); + } + } + pendingExchangeRenames.clear(); + verbosePrintf( "done.\n" ); + } +} + +ZCollector::ZCollector( string const & storageDir, string const & password, + Config & configIn ): + ZBackupBase( storageDir, password, configIn ), + chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), + config.runtime.cacheSize ) +{ +} + +void ZCollector::gc() +{ + ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true ); + + ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr, + chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads ); + + string fileName; + + Dir::Entry entry; + + BundleCollector collector; + collector.bundlesPath = getBundlesPath(); + collector.chunkStorageReader = &this->chunkStorageReader; + collector.chunkStorageWriter = &chunkStorageWriter; + collector.verbose = false; + + verbosePrintf( "Checking used chunks...\n" ); + + verbosePrintf( "Searching for backups...\n" ); + vector< string > backups = BackupExchanger::findOrRebuild( getBackupsPath() ); + + for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) + { + string backup( Dir::addPath( getBackupsPath(), *it ) ); + + verbosePrintf( "Checking backup %s...\n", backup.c_str() ); + + BackupInfo backupInfo; + + BackupFile::load( backup, encryptionkey, backupInfo ); + + string backupData; + + BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, &collector.usedChunkSet ); + + BackupRestorer::restore( chunkStorageReader, backupData, NULL, &collector.usedChunkSet ); + } + + verbosePrintf( "Checking bundles...\n" ); + + chunkIndex.loadIndex( collector ); + + collector.commit(); + + verbosePrintf( "Cleaning up...\n" ); + + string bundlesPath = getBundlesPath(); + Dir::Listing bundleLst( bundlesPath ); + while( bundleLst.getNext( entry ) ) + { + const string dirPath = Dir::addPath( bundlesPath, entry.getFileName()); + if ( entry.isDir() && Dir::isDirEmpty( dirPath ) ) + { + Dir::remove( dirPath ); + } + } + + verbosePrintf( "Garbage collection complete\n" ); +} diff -Nru zbackup-1.2/zutils.hh zbackup-1.4.1+20150403/zutils.hh --- zbackup-1.2/zutils.hh 1970-01-01 00:00:00.000000000 +0000 +++ zbackup-1.4.1+20150403/zutils.hh 2015-04-03 09:33:05.000000000 +0000 @@ -0,0 +1,61 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#ifndef ZUTILS_HH_INCLUDED +#define ZUTILS_HH_INCLUDED + +#include "chunk_storage.hh" +#include "zbackup_base.hh" + +class ZBackup: public ZBackupBase +{ + ChunkStorage::Writer chunkStorageWriter; + +public: + ZBackup( string const & storageDir, string const & password, + Config & configIn ); + + /// Backs up the data from stdin + void backupFromStdin( string const & outputFileName ); +}; + +class ZRestore: public ZBackupBase +{ + ChunkStorage::Reader chunkStorageReader; + +public: + ZRestore( string const & storageDir, string const & password, + Config & configIn ); + + /// Restores the data to stdin + void restoreToStdin( string const & inputFileName ); +}; + +class ZExchange +{ + ZBackupBase srcZBackupBase; + ZBackupBase dstZBackupBase; + +public: + ZExchange( string const & srcStorageDir, string const & srcPassword, + string const & dstStorageDir, string const & dstPassword, + Config & configIn ); + + Config config; + + /// Exchanges the data between storages + void exchange(); +}; + +class ZCollector : public ZBackupBase +{ + ChunkStorage::Reader chunkStorageReader; + +public: + ZCollector( std::string const & storageDir, std::string const & password, + Config & configIn ); + + void gc(); +}; + +#endif