diff -Nru leveldb-1.7.0/AUTHORS leveldb-1.12.0/AUTHORS --- leveldb-1.7.0/AUTHORS 2012-10-16 23:12:49.000000000 +0000 +++ leveldb-1.12.0/AUTHORS 2013-06-18 17:47:42.000000000 +0000 @@ -6,3 +6,6 @@ # Initial version authors: Jeffrey Dean Sanjay Ghemawat + +# Partial list of contributors: +Kevin Regan diff -Nru leveldb-1.7.0/Makefile leveldb-1.12.0/Makefile --- leveldb-1.7.0/Makefile 2012-10-16 23:12:49.000000000 +0000 +++ leveldb-1.12.0/Makefile 2013-06-18 17:47:42.000000000 +0000 @@ -12,7 +12,7 @@ #----------------------------------------------- # detect what platform we're building on -$(shell CC=$(CC) CXX=$(CXX) TARGET_OS=$(TARGET_OS) \ +$(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \ ./build_detect_platform build_config.mk ./) # this file is generated by the previous line to set build flags and sources include build_config.mk @@ -42,6 +42,7 @@ env_test \ filename_test \ filter_block_test \ + issue178_test \ log_test \ memenv_test \ skiplist_test \ @@ -50,7 +51,7 @@ version_set_test \ write_batch_test -PROGRAMS = db_bench $(TESTS) +PROGRAMS = db_bench leveldbutil $(TESTS) BENCHMARKS = db_bench_sqlite3 db_bench_tree_db LIBRARY = libleveldb.a @@ -69,7 +70,7 @@ else # Update db.h if you change these. SHARED_MAJOR = 1 -SHARED_MINOR = 7 +SHARED_MINOR = 12 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) @@ -107,6 +108,9 @@ db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) +leveldbutil: db/leveldb_main.o $(LIBOBJECTS) + $(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS) + arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) @@ -143,6 +147,9 @@ filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) diff -Nru leveldb-1.7.0/build_detect_platform leveldb-1.12.0/build_detect_platform --- leveldb-1.7.0/build_detect_platform 2012-10-16 23:12:49.000000000 +0000 +++ leveldb-1.12.0/build_detect_platform 2013-06-18 17:47:42.000000000 +0000 @@ -44,6 +44,10 @@ CXX=g++ fi +if test -z "$TMPDIR"; then + TMPDIR=/tmp +fi + # Detect OS if test -z "$TARGET_OS"; then TARGET_OS=`uname -s` @@ -141,7 +145,8 @@ set -f # temporarily disable globbing so that our patterns aren't expanded PRUNE_TEST="-name *test*.cc -prune" PRUNE_BENCH="-name *_bench.cc -prune" -PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` +PRUNE_TOOL="-name leveldb_main.cc -prune" +PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` set +f # re-enable globbing @@ -154,8 +159,10 @@ # Cross-compiling; do not try any compilation tests. true else + CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$" + # If -std=c++0x works, use . Otherwise use port_posix.h. - $CXX $CXXFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null </dev/null < int main() {} EOF @@ -168,7 +175,7 @@ # Test whether Snappy library is installed # http://code.google.com/p/snappy/ - $CXX $CXXFLAGS -x c++ - -o /dev/null 2>/dev/null </dev/null < int main() {} EOF @@ -178,12 +185,14 @@ fi # Test whether tcmalloc is available - $CXX $CXXFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null </dev/null </dev/null fi PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" diff -Nru leveldb-1.7.0/db/db_impl.cc leveldb-1.12.0/db/db_impl.cc --- leveldb-1.7.0/db/db_impl.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/db/db_impl.cc 2013-06-18 17:47:42.000000000 +0000 @@ -35,6 +35,8 @@ namespace leveldb { +const int kNumNonTableCacheFiles = 10; + // Information kept for every waiting writer struct DBImpl::Writer { Status status; @@ -92,9 +94,9 @@ Options result = src; result.comparator = icmp; result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL; - ClipToRange(&result.max_open_files, 20, 50000); - ClipToRange(&result.write_buffer_size, 64<<10, 1<<30); - ClipToRange(&result.block_size, 1<<10, 4<<20); + ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); + ClipToRange(&result.write_buffer_size, 64<<10, 1<<30); + ClipToRange(&result.block_size, 1<<10, 4<<20); if (result.info_log == NULL) { // Open a log file in the same directory as the db src.env->CreateDir(dbname); // In case it does not exist @@ -130,12 +132,13 @@ log_(NULL), tmp_batch_(new WriteBatch), bg_compaction_scheduled_(false), - manual_compaction_(NULL) { + manual_compaction_(NULL), + consecutive_compaction_errors_(0) { mem_->Ref(); has_imm_.Release_Store(NULL); // Reserve ten files or so for other uses and give the rest to TableCache. - const int table_cache_size = options.max_open_files - 10; + const int table_cache_size = options.max_open_files - kNumNonTableCacheFiles; table_cache_ = new TableCache(dbname_, &options_, table_cache_size); versions_ = new VersionSet(dbname_, &options_, table_cache_, @@ -310,16 +313,24 @@ if (!s.ok()) { return s; } + std::set expected; + versions_->AddLiveFiles(&expected); uint64_t number; FileType type; std::vector logs; for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type) - && type == kLogFile - && ((number >= min_log) || (number == prev_log))) { - logs.push_back(number); + if (ParseFileName(filenames[i], &number, &type)) { + expected.erase(number); + if (type == kLogFile && ((number >= min_log) || (number == prev_log))) + logs.push_back(number); } } + if (!expected.empty()) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast(expected.size())); + return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); + } // Recover in the order in which the logs were generated std::sort(logs.begin(), logs.end()); @@ -611,6 +622,7 @@ Status s = BackgroundCompaction(); if (s.ok()) { // Success + consecutive_compaction_errors_ = 0; } else if (shutting_down_.Acquire_Load()) { // Error most likely due to shutdown; do not wait } else { @@ -622,7 +634,12 @@ Log(options_.info_log, "Waiting after background compaction error: %s", s.ToString().c_str()); mutex_.Unlock(); - env_->SleepForMicroseconds(1000000); + ++consecutive_compaction_errors_; + int seconds_to_sleep = 1; + for (int i = 0; i < 3 && i < consecutive_compaction_errors_ - 1; ++i) { + seconds_to_sleep *= 2; + } + env_->SleepForMicroseconds(seconds_to_sleep * 1000000); mutex_.Lock(); } } @@ -1268,10 +1285,11 @@ } else if (imm_ != NULL) { // We have filled up the current memtable, but the previous // one is still being compacted, so we wait. + Log(options_.info_log, "Current memtable full; waiting...\n"); bg_cv_.Wait(); } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) { // There are too many level-0 files. - Log(options_.info_log, "waiting...\n"); + Log(options_.info_log, "Too many L0 files; waiting...\n"); bg_cv_.Wait(); } else { // Attempt to switch to a new memtable and trigger compaction of old diff -Nru leveldb-1.7.0/db/db_impl.h leveldb-1.12.0/db/db_impl.h --- leveldb-1.7.0/db/db_impl.h 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/db/db_impl.h 2013-06-18 17:47:42.000000000 +0000 @@ -163,6 +163,7 @@ // Have we encountered a background error in paranoid mode? Status bg_error_; + int consecutive_compaction_errors_; // Per level compaction stats. stats_[level] stores the stats for // compactions that produced data for the specified "level". diff -Nru leveldb-1.7.0/db/db_test.cc leveldb-1.12.0/db/db_test.cc --- leveldb-1.7.0/db/db_test.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/db/db_test.cc 2013-06-18 17:47:43.000000000 +0000 @@ -33,8 +33,11 @@ public: AtomicCounter() : count_(0) { } void Increment() { + IncrementBy(1); + } + void IncrementBy(int count) { MutexLock l(&mu_); - count_++; + count_ += count; } int Read() { MutexLock l(&mu_); @@ -45,6 +48,10 @@ count_ = 0; } }; + +void DelayMilliseconds(int millis) { + Env::Default()->SleepForMicroseconds(millis * 1000); +} } // Special Env used to delay background operations @@ -59,16 +66,25 @@ // Simulate non-writable file system while this pointer is non-NULL port::AtomicPointer non_writable_; + // Force sync of manifest files to fail while this pointer is non-NULL + port::AtomicPointer manifest_sync_error_; + + // Force write to manifest files to fail while this pointer is non-NULL + port::AtomicPointer manifest_write_error_; + bool count_random_reads_; AtomicCounter random_read_counter_; AtomicCounter sleep_counter_; + AtomicCounter sleep_time_counter_; explicit SpecialEnv(Env* base) : EnvWrapper(base) { delay_sstable_sync_.Release_Store(NULL); no_space_.Release_Store(NULL); non_writable_.Release_Store(NULL); count_random_reads_ = false; + manifest_sync_error_.Release_Store(NULL); + manifest_write_error_.Release_Store(NULL); } Status NewWritableFile(const std::string& f, WritableFile** r) { @@ -95,11 +111,35 @@ Status Flush() { return base_->Flush(); } Status Sync() { while (env_->delay_sstable_sync_.Acquire_Load() != NULL) { - env_->SleepForMicroseconds(100000); + DelayMilliseconds(100); } return base_->Sync(); } }; + class ManifestFile : public WritableFile { + private: + SpecialEnv* env_; + WritableFile* base_; + public: + ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { } + ~ManifestFile() { delete base_; } + Status Append(const Slice& data) { + if (env_->manifest_write_error_.Acquire_Load() != NULL) { + return Status::IOError("simulated writer error"); + } else { + return base_->Append(data); + } + } + Status Close() { return base_->Close(); } + Status Flush() { return base_->Flush(); } + Status Sync() { + if (env_->manifest_sync_error_.Acquire_Load() != NULL) { + return Status::IOError("simulated sync error"); + } else { + return base_->Sync(); + } + } + }; if (non_writable_.Acquire_Load() != NULL) { return Status::IOError("simulated write error"); @@ -109,6 +149,8 @@ if (s.ok()) { if (strstr(f.c_str(), ".sst") != NULL) { *r = new SSTableFile(this, *r); + } else if (strstr(f.c_str(), "MANIFEST") != NULL) { + *r = new ManifestFile(this, *r); } } return s; @@ -140,8 +182,9 @@ virtual void SleepForMicroseconds(int micros) { sleep_counter_.Increment(); - target()->SleepForMicroseconds(micros); + sleep_time_counter_.IncrementBy(micros); } + }; class DBTest { @@ -427,6 +470,20 @@ } return result; } + + bool DeleteAnSSTFile() { + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { + ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number))); + return true; + } + } + return false; + } }; TEST(DBTest, Empty) { @@ -577,7 +634,7 @@ } // Step 4: Wait for compaction to finish - env_->SleepForMicroseconds(1000000); + DelayMilliseconds(1000); ASSERT_EQ(NumTableFilesAtLevel(0), 0); } while (ChangeOptions()); @@ -1261,7 +1318,7 @@ Reopen(); Reopen(); ASSERT_EQ("(a->v)", Contents()); - env_->SleepForMicroseconds(1000000); // Wait for compaction to finish + DelayMilliseconds(1000); // Wait for compaction to finish ASSERT_EQ("(a->v)", Contents()); } @@ -1277,7 +1334,7 @@ Put("",""); Reopen(); Put("",""); - env_->SleepForMicroseconds(1000000); // Wait for compaction to finish + DelayMilliseconds(1000); // Wait for compaction to finish Reopen(); Put("d","dv"); Reopen(); @@ -1287,7 +1344,7 @@ Delete("b"); Reopen(); ASSERT_EQ("(->)(c->cv)", Contents()); - env_->SleepForMicroseconds(1000000); // Wait for compaction to finish + DelayMilliseconds(1000); // Wait for compaction to finish ASSERT_EQ("(->)(c->cv)", Contents()); } @@ -1472,6 +1529,30 @@ ASSERT_GE(env_->sleep_counter_.Read(), 5); } +TEST(DBTest, ExponentialBackoff) { + Options options = CurrentOptions(); + options.env = env_; + Reopen(&options); + + ASSERT_OK(Put("foo", "v1")); + ASSERT_EQ("v1", Get("foo")); + Compact("a", "z"); + env_->non_writable_.Release_Store(env_); // Force errors for new files + env_->sleep_counter_.Reset(); + env_->sleep_time_counter_.Reset(); + for (int i = 0; i < 5; i++) { + dbfull()->TEST_CompactRange(2, NULL, NULL); + } + env_->non_writable_.Release_Store(NULL); + + // Wait for compaction to finish + DelayMilliseconds(1000); + + ASSERT_GE(env_->sleep_counter_.Read(), 5); + ASSERT_LT(env_->sleep_counter_.Read(), 10); + ASSERT_GE(env_->sleep_time_counter_.Read(), 10e6); +} + TEST(DBTest, NonWritableFileSystem) { Options options = CurrentOptions(); options.write_buffer_size = 1000; @@ -1485,13 +1566,72 @@ fprintf(stderr, "iter %d; errors %d\n", i, errors); if (!Put("foo", big).ok()) { errors++; - env_->SleepForMicroseconds(100000); + DelayMilliseconds(100); } } ASSERT_GT(errors, 0); env_->non_writable_.Release_Store(NULL); } +TEST(DBTest, ManifestWriteError) { + // Test for the following problem: + // (a) Compaction produces file F + // (b) Log record containing F is written to MANIFEST file, but Sync() fails + // (c) GC deletes F + // (d) After reopening DB, reads fail since deleted F is named in log record + + // We iterate twice. In the second iteration, everything is the + // same except the log record never makes it to the MANIFEST file. + for (int iter = 0; iter < 2; iter++) { + port::AtomicPointer* error_type = (iter == 0) + ? &env_->manifest_sync_error_ + : &env_->manifest_write_error_; + + // Insert foo=>bar mapping + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.error_if_exists = false; + DestroyAndReopen(&options); + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Memtable compaction (will succeed) + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + const int last = config::kMaxMemCompactLevel; + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level + + // Merging compaction (will fail) + error_type->Release_Store(env_); + dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail + ASSERT_EQ("bar", Get("foo")); + + // Recovery: should not lose data + error_type->Release_Store(NULL); + Reopen(&options); + ASSERT_EQ("bar", Get("foo")); + } +} + +TEST(DBTest, MissingSSTFile) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Dump the memtable to disk. + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + + Close(); + ASSERT_TRUE(DeleteAnSSTFile()); + Options options = CurrentOptions(); + options.paranoid_checks = true; + Status s = TryReopen(&options); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) + << s.ToString(); +} + TEST(DBTest, FilesDeletedAfterCompaction) { ASSERT_OK(Put("foo", "v2")); Compact("a", "z"); @@ -1636,13 +1776,13 @@ } // Let them run for a while - env_->SleepForMicroseconds(kTestSeconds * 1000000); + DelayMilliseconds(kTestSeconds * 1000); // Stop the threads and wait for them to finish mt.stop.Release_Store(&mt); for (int id = 0; id < kNumThreads; id++) { while (mt.thread_done[id].Acquire_Load() == NULL) { - env_->SleepForMicroseconds(100000); + DelayMilliseconds(100); } } } while (ChangeOptions()); diff -Nru leveldb-1.7.0/db/dbformat.cc leveldb-1.12.0/db/dbformat.cc --- leveldb-1.7.0/db/dbformat.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/db/dbformat.cc 2013-06-18 17:47:43.000000000 +0000 @@ -26,7 +26,7 @@ (unsigned long long) sequence, int(type)); std::string result = "'"; - result += user_key.ToString(); + result += EscapeString(user_key.ToString()); result += buf; return result; } diff -Nru leveldb-1.7.0/db/filename_test.cc leveldb-1.12.0/db/filename_test.cc --- leveldb-1.7.0/db/filename_test.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/db/filename_test.cc 2013-06-18 17:47:42.000000000 +0000 @@ -70,7 +70,7 @@ for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { std::string f = errors[i]; ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; - }; + } } TEST(FileNameTest, Construction) { diff -Nru leveldb-1.7.0/db/leveldb_main.cc leveldb-1.12.0/db/leveldb_main.cc --- leveldb-1.7.0/db/leveldb_main.cc 1970-01-01 00:00:00.000000000 +0000 +++ leveldb-1.12.0/db/leveldb_main.cc 2013-06-18 17:47:42.000000000 +0000 @@ -0,0 +1,238 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/status.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" + +namespace leveldb { + +namespace { + +bool GuessType(const std::string& fname, FileType* type) { + size_t pos = fname.rfind('/'); + std::string basename; + if (pos == std::string::npos) { + basename = fname; + } else { + basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); + } + uint64_t ignored; + return ParseFileName(basename, &ignored, type); +} + +// Notified when log reader encounters corruption. +class CorruptionReporter : public log::Reader::Reporter { + public: + virtual void Corruption(size_t bytes, const Status& status) { + printf("corruption: %d bytes; %s\n", + static_cast(bytes), + status.ToString().c_str()); + } +}; + +// Print contents of a log file. (*func)() is called on every record. +bool PrintLogContents(Env* env, const std::string& fname, + void (*func)(Slice)) { + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + return false; + } + CorruptionReporter reporter; + log::Reader reader(file, &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + printf("--- offset %llu; ", + static_cast(reader.LastRecordOffset())); + (*func)(record); + } + delete file; + return true; +} + +// Called on every item found in a WriteBatch. +class WriteBatchItemPrinter : public WriteBatch::Handler { + public: + uint64_t offset_; + uint64_t sequence_; + + virtual void Put(const Slice& key, const Slice& value) { + printf(" put '%s' '%s'\n", + EscapeString(key).c_str(), + EscapeString(value).c_str()); + } + virtual void Delete(const Slice& key) { + printf(" del '%s'\n", + EscapeString(key).c_str()); + } +}; + + +// Called on every log record (each one of which is a WriteBatch) +// found in a kLogFile. +static void WriteBatchPrinter(Slice record) { + if (record.size() < 12) { + printf("log record length %d is too small\n", + static_cast(record.size())); + return; + } + WriteBatch batch; + WriteBatchInternal::SetContents(&batch, record); + printf("sequence %llu\n", + static_cast(WriteBatchInternal::Sequence(&batch))); + WriteBatchItemPrinter batch_item_printer; + Status s = batch.Iterate(&batch_item_printer); + if (!s.ok()) { + printf(" error: %s\n", s.ToString().c_str()); + } +} + +bool DumpLog(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, WriteBatchPrinter); +} + +// Called on every log record (each one of which is a WriteBatch) +// found in a kDescriptorFile. +static void VersionEditPrinter(Slice record) { + VersionEdit edit; + Status s = edit.DecodeFrom(record); + if (!s.ok()) { + printf("%s\n", s.ToString().c_str()); + return; + } + printf("%s", edit.DebugString().c_str()); +} + +bool DumpDescriptor(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, VersionEditPrinter); +} + +bool DumpTable(Env* env, const std::string& fname) { + uint64_t file_size; + RandomAccessFile* file = NULL; + Table* table = NULL; + Status s = env->GetFileSize(fname, &file_size); + if (s.ok()) { + s = env->NewRandomAccessFile(fname, &file); + } + if (s.ok()) { + // We use the default comparator, which may or may not match the + // comparator used in this database. However this should not cause + // problems since we only use Table operations that do not require + // any comparisons. In particular, we do not call Seek or Prev. + s = Table::Open(Options(), file, file_size, &table); + } + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + delete table; + delete file; + return false; + } + + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = table->NewIterator(ro); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey key; + if (!ParseInternalKey(iter->key(), &key)) { + printf("badkey '%s' => '%s'\n", + EscapeString(iter->key()).c_str(), + EscapeString(iter->value()).c_str()); + } else { + char kbuf[20]; + const char* type; + if (key.type == kTypeDeletion) { + type = "del"; + } else if (key.type == kTypeValue) { + type = "val"; + } else { + snprintf(kbuf, sizeof(kbuf), "%d", static_cast(key.type)); + type = kbuf; + } + printf("'%s' @ %8llu : %s => '%s'\n", + EscapeString(key.user_key).c_str(), + static_cast(key.sequence), + type, + EscapeString(iter->value()).c_str()); + } + } + s = iter->status(); + if (!s.ok()) { + printf("iterator error: %s\n", s.ToString().c_str()); + } + + delete iter; + delete table; + delete file; + return true; +} + +bool DumpFile(Env* env, const std::string& fname) { + FileType ftype; + if (!GuessType(fname, &ftype)) { + fprintf(stderr, "%s: unknown file type\n", fname.c_str()); + return false; + } + switch (ftype) { + case kLogFile: return DumpLog(env, fname); + case kDescriptorFile: return DumpDescriptor(env, fname); + case kTableFile: return DumpTable(env, fname); + + default: { + fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str()); + break; + } + } + return false; +} + +bool HandleDumpCommand(Env* env, char** files, int num) { + bool ok = true; + for (int i = 0; i < num; i++) { + ok &= DumpFile(env, files[i]); + } + return ok; +} + +} +} // namespace leveldb + +static void Usage() { + fprintf( + stderr, + "Usage: leveldbutil command...\n" + " dump files... -- dump contents of specified files\n" + ); +} + +int main(int argc, char** argv) { + leveldb::Env* env = leveldb::Env::Default(); + bool ok = true; + if (argc < 2) { + Usage(); + ok = false; + } else { + std::string command = argv[1]; + if (command == "dump") { + ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); + } else { + Usage(); + ok = false; + } + } + return (ok ? 0 : 1); +} diff -Nru leveldb-1.7.0/db/version_set.cc leveldb-1.12.0/db/version_set.cc --- leveldb-1.7.0/db/version_set.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/db/version_set.cc 2013-06-18 17:47:42.000000000 +0000 @@ -786,12 +786,23 @@ if (s.ok()) { s = descriptor_file_->Sync(); } + if (!s.ok()) { + Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str()); + if (ManifestContains(record)) { + Log(options_->info_log, + "MANIFEST contains log record despite error; advancing to new " + "version to prevent mismatch between in-memory and logged state"); + s = Status::OK(); + } + } } // If we just created a new descriptor file, install it by writing a // new CURRENT file that points to it. if (s.ok() && !new_manifest_file.empty()) { s = SetCurrentFile(env_, dbname_, manifest_file_number_); + // No need to double-check MANIFEST in case of error since it + // will be discarded below. } mu->Lock(); @@ -1025,6 +1036,31 @@ return scratch->buffer; } +// Return true iff the manifest contains the specified record. +bool VersionSet::ManifestContains(const std::string& record) const { + std::string fname = DescriptorFileName(dbname_, manifest_file_number_); + Log(options_->info_log, "ManifestContains: checking %s\n", fname.c_str()); + SequentialFile* file = NULL; + Status s = env_->NewSequentialFile(fname, &file); + if (!s.ok()) { + Log(options_->info_log, "ManifestContains: %s\n", s.ToString().c_str()); + return false; + } + log::Reader reader(file, NULL, true/*checksum*/, 0); + Slice r; + std::string scratch; + bool result = false; + while (reader.ReadRecord(&r, &scratch)) { + if (r == Slice(record)) { + result = true; + break; + } + } + delete file; + Log(options_->info_log, "ManifestContains: result = %d\n", result ? 1 : 0); + return result; +} + uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { uint64_t result = 0; for (int level = 0; level < config::kNumLevels; level++) { @@ -1295,14 +1331,19 @@ } // Avoid compacting too much in one shot in case the range is large. - const uint64_t limit = MaxFileSizeForLevel(level); - uint64_t total = 0; - for (size_t i = 0; i < inputs.size(); i++) { - uint64_t s = inputs[i]->file_size; - total += s; - if (total >= limit) { - inputs.resize(i + 1); - break; + // But we cannot do this for level-0 since level-0 files can overlap + // and we must not pick one file and drop another older file if the + // two files overlap. + if (level > 0) { + const uint64_t limit = MaxFileSizeForLevel(level); + uint64_t total = 0; + for (size_t i = 0; i < inputs.size(); i++) { + uint64_t s = inputs[i]->file_size; + total += s; + if (total >= limit) { + inputs.resize(i + 1); + break; + } } } diff -Nru leveldb-1.7.0/db/version_set.h leveldb-1.12.0/db/version_set.h --- leveldb-1.7.0/db/version_set.h 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/db/version_set.h 2013-06-18 17:47:42.000000000 +0000 @@ -277,6 +277,8 @@ void AppendVersion(Version* v); + bool ManifestContains(const std::string& record) const; + Env* const env_; const std::string dbname_; const Options* const options_; diff -Nru leveldb-1.7.0/debian/changelog leveldb-1.12.0/debian/changelog --- leveldb-1.7.0/debian/changelog 2013-01-17 13:27:18.000000000 +0000 +++ leveldb-1.12.0/debian/changelog 2013-07-23 12:20:55.000000000 +0000 @@ -1,8 +1,60 @@ -leveldb (1.7.0-1~ubuntu12.04.1~grizzly0) precise; urgency=low +leveldb (1.12.0-1~cloud0) precise-havana; urgency=low - * No-change backport to precise + * New update for the Ubuntu Cloud Archive. - -- Openstack Ubuntu Testing Bot Thu, 17 Jan 2013 08:27:18 -0500 + -- Chuck Short Tue, 23 Jul 2013 08:20:55 -0400 + +leveldb (1.12.0-1) unstable; urgency=low + + [ Alessio Treglia ] + * New upstream release. + * Refresh patches. + + [ Martin Pitt ] + * debian/tests/build: Fix typo in file name passed to g++, + to make the test actually work. + + -- Alessio Treglia Thu, 27 Jun 2013 23:49:32 +0100 + +leveldb (1.10.0-1) unstable; urgency=low + + * New upstream release. + * debian/patches/0113-makefile_memenv.patch: Install missing component + "memenv" (Closes: #701075). Thanks to Dmitry Smirnov for the report + and patch. + * Fix VCS fields. + * Refresh patches. + + -- Alessio Treglia Mon, 27 May 2013 04:55:01 +0200 + +leveldb (1.9.0-3) unstable; urgency=low + + * Upload to unstable. + + -- Alessio Treglia Mon, 06 May 2013 11:11:33 +0200 + +leveldb (1.9.0-2) experimental; urgency=low + + * Improve get-orig-source mechanism. + * Add autopkgtest support. + * Bump Standards. + + -- Alessio Treglia Mon, 25 Mar 2013 18:29:52 +0000 + +leveldb (1.9.0-1) experimental; urgency=low + + * New upstream release. + + -- Alessio Treglia Sat, 12 Jan 2013 14:34:08 +0000 + +leveldb (1.8.0-1) experimental; urgency=low + + * New upstream release (Closes: #696956) + * Refresh 1002-cstdatomic_renaming.patch patch. + * Fix debian/watch file. + Thanks to Bart Martens for the fix. + + -- Alessio Treglia Sat, 29 Dec 2012 23:54:14 +0000 leveldb (1.7.0-1) experimental; urgency=low diff -Nru leveldb-1.7.0/debian/control leveldb-1.12.0/debian/control --- leveldb-1.7.0/debian/control 2012-10-15 10:37:30.000000000 +0000 +++ leveldb-1.12.0/debian/control 2013-05-27 02:35:40.000000000 +0000 @@ -5,10 +5,11 @@ Build-Depends: debhelper (>= 9), libsnappy-dev -Standards-Version: 3.9.3 +XS-Testsuite: autopkgtest +Standards-Version: 3.9.4 Homepage: http://code.google.com/p/leveldb/ -Vcs-Git: git://git.debian.org/collab-maint/leveldb.git -Vcs-Browser: http://git.debian.org/?p=collab-maint/leveldb.git +Vcs-Git: git://anonscm.debian.org/collab-maint/leveldb.git +Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/leveldb.git Package: libleveldb1 Section: libs diff -Nru leveldb-1.7.0/debian/get-git-source.sh leveldb-1.12.0/debian/get-git-source.sh --- leveldb-1.7.0/debian/get-git-source.sh 2012-11-18 14:37:06.000000000 +0000 +++ leveldb-1.12.0/debian/get-git-source.sh 2013-02-14 16:43:43.000000000 +0000 @@ -1,10 +1,11 @@ #!/bin/sh -BASE_REL=$(dpkg-parsechangelog 2>/dev/null | sed -ne 's/Version: $[0-9]$\+.*/\1/p') +PACKAGE=leveldb +BASE_REL=$(dpkg-parsechangelog 2>/dev/null | sed -ne 's/Version: $[0-9.]\+$\+.*/\1/p') OLDDIR=${PWD} GOS_DIR=${OLDDIR}/get-orig-source -GIT_COMMIT='git log --no-color -1 --oneline | cut -d" " -f1' -GIT_DATE='git log --no-color -1 --date=iso | sed -ne "s/Date:\s\+$.*$.*/\1/p" | cut -d" " -f1 | tr -d "-"' +GIT_COMMIT_STR='git log --no-color -1 --oneline | cut -d" " -f1' +GIT_DATE_STR='git log --no-color -1 --date=iso | sed -ne "s/Date:\s\+$.*$.*/\1/p" | cut -d" " -f1 | tr -d "-"' if [ -z ${BASE_REL} ]; then echo 'Please run this script from the sources root directory.' @@ -14,11 +15,11 @@ rm -rf ${GOS_DIR} mkdir ${GOS_DIR} && cd ${GOS_DIR} -git clone http://code.google.com/p/leveldb/ leveldb -cd leveldb/ -LEVELDB_GIT_COMMIT=$(eval "${GIT_COMMIT}") -LEVELDB_GIT_DATE=$(eval "${GIT_DATE}") +git clone http://code.google.com/p/leveldb/ ${PACKAGE} +cd ${PACKAGE}/ +GIT_COMMIT=$(eval "${GIT_COMMIT_STR}") +GIT_DATE=$(eval "${GIT_DATE_STR}") cd .. && tar cjf \ - ${OLDDIR}/leveldb_${BASE_REL}+${LEVELDB_GIT_DATE}.git${LEVELDB_GIT_COMMIT}.orig.tar.bz2 \ - leveldb --exclude-vcs + ${OLDDIR}/${PACKAGE}_${BASE_REL}+${GIT_DATE}.git${GIT_COMMIT}.orig.tar.bz2 \ + ${PACKAGE} --exclude-vcs rm -rf ${GOS_DIR} diff -Nru leveldb-1.7.0/debian/patches/0102-kfreebsd.patch leveldb-1.12.0/debian/patches/0102-kfreebsd.patch --- leveldb-1.7.0/debian/patches/0102-kfreebsd.patch 2012-11-18 14:37:06.000000000 +0000 +++ leveldb-1.12.0/debian/patches/0102-kfreebsd.patch 2013-05-27 02:49:29.000000000 +0000 @@ -7,7 +7,7 @@ --- leveldb.orig/build_detect_platform +++ leveldb/build_detect_platform -@@ -127,6 +127,12 @@ case "$TARGET_OS" in +@@ -131,6 +131,12 @@ case "$TARGET_OS" in # man ld: +h internal_name PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl," ;; diff -Nru leveldb-1.7.0/debian/patches/0103-hurd.patch leveldb-1.12.0/debian/patches/0103-hurd.patch --- leveldb-1.7.0/debian/patches/0103-hurd.patch 2012-11-18 14:37:06.000000000 +0000 +++ leveldb-1.12.0/debian/patches/0103-hurd.patch 2013-05-27 02:49:32.000000000 +0000 @@ -13,7 +13,7 @@ --- leveldb.orig/build_detect_platform +++ leveldb/build_detect_platform -@@ -133,6 +133,12 @@ case "$TARGET_OS" in +@@ -137,6 +137,12 @@ case "$TARGET_OS" in PLATFORM_LIBS="-lpthread -lrt" PORT_FILE=port/port_posix.cc ;; diff -Nru leveldb-1.7.0/debian/patches/0112-makefile_install.patch leveldb-1.12.0/debian/patches/0112-makefile_install.patch --- leveldb-1.7.0/debian/patches/0112-makefile_install.patch 2012-11-18 14:37:06.000000000 +0000 +++ leveldb-1.12.0/debian/patches/0112-makefile_install.patch 2013-06-27 22:47:59.000000000 +0000 @@ -20,7 +20,7 @@ #----------------------------------------------- # Uncomment exactly one of the lines labelled (A), (B), and (C) below # to switch between compilation modes. -@@ -87,6 +91,17 @@ endif # PLATFORM_SHARED_EXT +@@ -88,6 +92,17 @@ endif # PLATFORM_SHARED_EXT all: $(SHARED) $(LIBRARY) diff -Nru leveldb-1.7.0/debian/patches/0113-makefile_memenv.patch leveldb-1.12.0/debian/patches/0113-makefile_memenv.patch --- leveldb-1.7.0/debian/patches/0113-makefile_memenv.patch 1970-01-01 00:00:00.000000000 +0000 +++ leveldb-1.12.0/debian/patches/0113-makefile_memenv.patch 2013-06-27 22:48:02.000000000 +0000 @@ -0,0 +1,37 @@ +Last-Update: 2013-02-21 +Forwarded: no +Author: Dmitry Smirnov +Description: install "memenv" component. + This patch will install two additional files to libleveldb-dev: + + include/helpers/memenv.h + + lib/*/libmemenv.a + +--- + Makefile | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +--- leveldb.orig/Makefile ++++ leveldb/Makefile +@@ -90,18 +90,20 @@ $(SHARED3): + + endif # PLATFORM_SHARED_EXT + +-all: $(SHARED) $(LIBRARY) ++all: $(SHARED) $(LIBRARY) $(MEMENVLIBRARY) + + install: all + install -d $(DESTDIR)$(PREFIX)/$(LIBDIR) +- install -d $(DESTDIR)$(PREFIX)/$(INCLUDEDIR)/leveldb ++ install -d $(DESTDIR)$(PREFIX)/$(INCLUDEDIR)/leveldb/helpers + install -m 0644 $(LIBRARY) $(DESTDIR)$(PREFIX)/$(LIBDIR) ++ install -m 0644 $(MEMENVLIBRARY) $(DESTDIR)$(PREFIX)/$(LIBDIR) + ifneq ($(PLATFORM_SHARED_EXT),) + install -m 0644 $(SHARED3) $(DESTDIR)$(PREFIX)/$(LIBDIR) + cd $(DESTDIR)$(PREFIX)/$(LIBDIR) && ln -sf $(SHARED3) $(SHARED2) + cd $(DESTDIR)$(PREFIX)/$(LIBDIR) && ln -sf $(SHARED3) $(SHARED1) + endif + install -m 0644 include/leveldb/* $(DESTDIR)$(PREFIX)/$(INCLUDEDIR)/leveldb ++ install -m 0644 helpers/memenv/*.h $(DESTDIR)$(PREFIX)/$(INCLUDEDIR)/leveldb/helpers + + check: all $(PROGRAMS) $(TESTS) + for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done diff -Nru leveldb-1.7.0/debian/patches/1002-cstdatomic_renaming.patch leveldb-1.12.0/debian/patches/1002-cstdatomic_renaming.patch --- leveldb-1.7.0/debian/patches/1002-cstdatomic_renaming.patch 2012-11-18 14:37:06.000000000 +0000 +++ leveldb-1.12.0/debian/patches/1002-cstdatomic_renaming.patch 2013-05-27 02:53:31.000000000 +0000 @@ -7,10 +7,10 @@ --- leveldb.orig/build_detect_platform +++ leveldb/build_detect_platform -@@ -168,7 +168,7 @@ if [ "$CROSS_COMPILE" = "true" ]; then - else +@@ -175,7 +175,7 @@ else + # If -std=c++0x works, use . Otherwise use port_posix.h. - $CXX $CXXFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null </dev/null < + #include int main() {} diff -Nru leveldb-1.7.0/debian/patches/series leveldb-1.12.0/debian/patches/series --- leveldb-1.7.0/debian/patches/series 2012-11-18 14:37:06.000000000 +0000 +++ leveldb-1.12.0/debian/patches/series 2013-05-27 02:49:18.000000000 +0000 @@ -2,5 +2,6 @@ 0102-kfreebsd.patch 0103-hurd.patch 0112-makefile_install.patch +0113-makefile_memenv.patch 1001-fix_bloom_test.patch 1002-cstdatomic_renaming.patch diff -Nru leveldb-1.7.0/debian/tests/build leveldb-1.12.0/debian/tests/build --- leveldb-1.7.0/debian/tests/build 1970-01-01 00:00:00.000000000 +0000 +++ leveldb-1.12.0/debian/tests/build 2013-06-27 22:46:47.000000000 +0000 @@ -0,0 +1,64 @@ +#!/bin/sh +# autopkgtest check: Builds a small application against leveldb, checking +# if it compiles, links and runs successfully. +# Author: Alessio Treglia + +set -e + +WORKDIR=$(mktemp -d) +trap "rm -rf $WORKDIR" 0 INT QUIT ABRT PIPE TERM +cd $WORKDIR +cat < build_test.cpp +#include +#include +#include +#include + +#include "leveldb/db.h" + +using namespace std; + +int main(int argc, char **argv) +{ + leveldb::DB *db; + leveldb::Options opts; + leveldb::ReadOptions r_opts; + leveldb::WriteOptions w_opts; + std::string value; + + opts.create_if_missing = true; + // Create a new db + leveldb::Status s = leveldb::DB::Open(opts, + "./build_test_db", + &db); + assert (s.ok() == true); + // Check if the db is empty + s = db->Get(r_opts, "test_key1", &value); + assert (s.IsNotFound() == true); + // Add such new key to the db + s = db->Put(w_opts, "test_key1", "test_value1"); + assert (s.ok() == true); + // Get the new key + s = db->Get(r_opts, "test_key1", &value); + assert (s.ok() == true); + // Check the return value + assert (value.compare("test_value1") == 0); + // Delete the key + s = db->Delete(w_opts, "test_key1"); + assert (s.ok() == true); + // Check if the deletion's gone well + s = db->Get(r_opts, "test_key1", &value); + assert (s.IsNotFound() == true); + + // Close the db + delete db; + + return 0; +} +EOF + +g++ -o build_test build_test.cpp -pthread -lleveldb -lsnappy +echo "build: OK" +[ -x build_test ] +./build_test +echo "run: OK" diff -Nru leveldb-1.7.0/debian/tests/control leveldb-1.12.0/debian/tests/control --- leveldb-1.7.0/debian/tests/control 1970-01-01 00:00:00.000000000 +0000 +++ leveldb-1.12.0/debian/tests/control 2013-03-25 16:16:41.000000000 +0000 @@ -0,0 +1,2 @@ +Tests: build +Depends: build-essential, libleveldb-dev, libsnappy-dev diff -Nru leveldb-1.7.0/debian/watch leveldb-1.12.0/debian/watch --- leveldb-1.7.0/debian/watch 2012-11-18 14:58:26.000000000 +0000 +++ leveldb-1.12.0/debian/watch 2012-11-22 10:49:17.000000000 +0000 @@ -1,2 +1,3 @@ version=3 -http://googlecode.debian.net/p/leveldb/leveldb-(\d+.*)\.tar\.gz +opts=downloadurlmangle=s%.*/%http://leveldb.googlecode.com/files/% \ +http://code.google.com/p/leveldb/downloads/list .*/leveldb-(\d.*)\.(?:tgz|tar\.(?:gz|bz2|xz)) diff -Nru leveldb-1.7.0/include/leveldb/db.h leveldb-1.12.0/include/leveldb/db.h --- leveldb-1.7.0/include/leveldb/db.h 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/include/leveldb/db.h 2013-06-18 17:47:42.000000000 +0000 @@ -14,7 +14,7 @@ // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 7; +static const int kMinorVersion = 12; struct Options; struct ReadOptions; diff -Nru leveldb-1.7.0/issues/issue178_test.cc leveldb-1.12.0/issues/issue178_test.cc --- leveldb-1.7.0/issues/issue178_test.cc 1970-01-01 00:00:00.000000000 +0000 +++ leveldb-1.12.0/issues/issue178_test.cc 2013-06-18 17:47:42.000000000 +0000 @@ -0,0 +1,92 @@ +// Copyright (c) 2013 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Test for issue 178: a manual compaction causes deleted data to reappear. +#include +#include +#include + +#include "leveldb/db.h" +#include "leveldb/write_batch.h" +#include "util/testharness.h" + +namespace { + +const int kNumKeys = 1100000; + +std::string Key1(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "my_key_%d", i); + return buf; +} + +std::string Key2(int i) { + return Key1(i) + "_xxx"; +} + +class Issue178 { }; + +TEST(Issue178, Test) { + // Get rid of any state from an old run. + std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test"; + DestroyDB(dbpath, leveldb::Options()); + + // Open database. Disable compression since it affects the creation + // of layers and the code below is trying to test against a very + // specific scenario. + leveldb::DB* db; + leveldb::Options db_options; + db_options.create_if_missing = true; + db_options.compression = leveldb::kNoCompression; + ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db)); + + // create first key range + leveldb::WriteBatch batch; + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key1(i), "value for range 1 key"); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // create second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key2(i), "value for range 2 key"); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // delete second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Delete(Key2(i)); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // compact database + std::string start_key = Key1(0); + std::string end_key = Key1(kNumKeys - 1); + leveldb::Slice least(start_key.data(), start_key.size()); + leveldb::Slice greatest(end_key.data(), end_key.size()); + + // commenting out the line below causes the example to work correctly + db->CompactRange(&least, &greatest); + + // count the keys + leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions()); + size_t num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + num_keys++; + } + delete iter; + ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; + + // close database + delete db; + DestroyDB(dbpath, leveldb::Options()); +} + +} // anonymous namespace + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff -Nru leveldb-1.7.0/port/port_posix.h leveldb-1.12.0/port/port_posix.h --- leveldb-1.7.0/port/port_posix.h 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/port/port_posix.h 2013-06-18 17:47:42.000000000 +0000 @@ -26,11 +26,17 @@ #include #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\ - defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) + defined(OS_DRAGONFLYBSD) #include #include #elif defined(OS_HPUX) #define PLATFORM_IS_LITTLE_ENDIAN false +#elif defined(OS_ANDROID) + // Due to a bug in the NDK x86 definition, + // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android. + // See http://code.google.com/p/android/issues/detail?id=39824 + #include + #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #else #include #endif diff -Nru leveldb-1.7.0/table/block.cc leveldb-1.12.0/table/block.cc --- leveldb-1.7.0/table/block.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/table/block.cc 2013-06-18 17:47:42.000000000 +0000 @@ -16,7 +16,7 @@ namespace leveldb { inline uint32_t Block::NumRestarts() const { - assert(size_ >= 2*sizeof(uint32_t)); + assert(size_ >= sizeof(uint32_t)); return DecodeFixed32(data_ + size_ - sizeof(uint32_t)); } @@ -27,11 +27,12 @@ if (size_ < sizeof(uint32_t)) { size_ = 0; // Error marker } else { - restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); - if (restart_offset_ > size_ - sizeof(uint32_t)) { - // The size is too small for NumRestarts() and therefore - // restart_offset_ wrapped around. + size_t max_restarts_allowed = (size_-sizeof(uint32_t)) / sizeof(uint32_t); + if (NumRestarts() > max_restarts_allowed) { + // The size is too small for NumRestarts() size_ = 0; + } else { + restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); } } } @@ -253,7 +254,7 @@ }; Iterator* Block::NewIterator(const Comparator* cmp) { - if (size_ < 2*sizeof(uint32_t)) { + if (size_ < sizeof(uint32_t)) { return NewErrorIterator(Status::Corruption("bad block contents")); } const uint32_t num_restarts = NumRestarts(); diff -Nru leveldb-1.7.0/table/table.cc leveldb-1.12.0/table/table.cc --- leveldb-1.7.0/table/table.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/table/table.cc 2013-06-18 17:47:42.000000000 +0000 @@ -228,7 +228,6 @@ !filter->KeyMayMatch(handle.offset(), k)) { // Not found } else { - Slice handle = iiter->value(); Iterator* block_iter = BlockReader(this, options, iiter->value()); block_iter->Seek(k); if (block_iter->Valid()) { diff -Nru leveldb-1.7.0/table/table_test.cc leveldb-1.12.0/table/table_test.cc --- leveldb-1.7.0/table/table_test.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/table/table_test.cc 2013-06-18 17:47:42.000000000 +0000 @@ -644,6 +644,36 @@ Constructor* constructor_; }; +// Test empty table/block. +TEST(Harness, Empty) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 1); + Test(&rnd); + } +} + +// Special test for a block with no restart entries. The C++ leveldb +// code never generates such blocks, but the Java version of leveldb +// seems to. +TEST(Harness, ZeroRestartPointsInBlock) { + char data[sizeof(uint32_t)]; + memset(data, 0, sizeof(data)); + BlockContents contents; + contents.data = Slice(data, sizeof(data)); + contents.cachable = false; + contents.heap_allocated = false; + Block block(contents); + Iterator* iter = block.NewIterator(BytewiseComparator()); + iter->SeekToFirst(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + ASSERT_TRUE(!iter->Valid()); + iter->Seek("foo"); + ASSERT_TRUE(!iter->Valid()); + delete iter; +} + // Test the empty key TEST(Harness, SimpleEmptyKey) { for (int i = 0; i < kNumTestArgs; i++) { diff -Nru leveldb-1.7.0/util/cache.cc leveldb-1.12.0/util/cache.cc --- leveldb-1.7.0/util/cache.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/util/cache.cc 2013-06-18 17:47:43.000000000 +0000 @@ -116,7 +116,6 @@ LRUHandle* h = list_[i]; while (h != NULL) { LRUHandle* next = h->next_hash; - Slice key = h->key(); uint32_t hash = h->hash; LRUHandle** ptr = &new_list[hash & (new_length - 1)]; h->next_hash = *ptr; @@ -160,7 +159,6 @@ // mutex_ protects the following state. port::Mutex mutex_; size_t usage_; - uint64_t last_id_; // Dummy head of LRU list. // lru.prev is newest entry, lru.next is oldest entry. @@ -170,8 +168,7 @@ }; LRUCache::LRUCache() - : usage_(0), - last_id_(0) { + : usage_(0) { // Make empty circular linked list lru_.next = &lru_; lru_.prev = &lru_; diff -Nru leveldb-1.7.0/util/coding_test.cc leveldb-1.12.0/util/coding_test.cc --- leveldb-1.7.0/util/coding_test.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/util/coding_test.cc 2013-06-18 17:47:43.000000000 +0000 @@ -109,7 +109,7 @@ values.push_back(power); values.push_back(power-1); values.push_back(power+1); - }; + } std::string s; for (int i = 0; i < values.size(); i++) { diff -Nru leveldb-1.7.0/util/env_posix.cc leveldb-1.12.0/util/env_posix.cc --- leveldb-1.7.0/util/env_posix.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/util/env_posix.cc 2013-06-18 17:47:43.000000000 +0000 @@ -385,7 +385,7 @@ PosixEnv(); virtual ~PosixEnv() { fprintf(stderr, "Destroying Env::Default()\n"); - exit(1); + abort(); } virtual Status NewSequentialFile(const std::string& fname, @@ -466,7 +466,7 @@ result = IOError(fname, errno); } return result; - }; + } virtual Status CreateDir(const std::string& name) { Status result; @@ -474,7 +474,7 @@ result = IOError(name, errno); } return result; - }; + } virtual Status DeleteDir(const std::string& name) { Status result; @@ -482,7 +482,7 @@ result = IOError(name, errno); } return result; - }; + } virtual Status GetFileSize(const std::string& fname, uint64_t* size) { Status s; @@ -588,7 +588,7 @@ void PthreadCall(const char* label, int result) { if (result != 0) { fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); - exit(1); + abort(); } } diff -Nru leveldb-1.7.0/util/hash.cc leveldb-1.12.0/util/hash.cc --- leveldb-1.7.0/util/hash.cc 2012-10-16 23:12:50.000000000 +0000 +++ leveldb-1.12.0/util/hash.cc 2013-06-18 17:47:43.000000000 +0000 @@ -6,6 +6,13 @@ #include "util/coding.h" #include "util/hash.h" +// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through +// between switch labels. The real definition should be provided externally. +// This one is a fallback version for unsupported compilers. +#ifndef FALLTHROUGH_INTENDED +#define FALLTHROUGH_INTENDED do { } while (0) +#endif + namespace leveldb { uint32_t Hash(const char* data, size_t n, uint32_t seed) { @@ -28,10 +35,10 @@ switch (limit - data) { case 3: h += data[2] << 16; - // fall through + FALLTHROUGH_INTENDED; case 2: h += data[1] << 8; - // fall through + FALLTHROUGH_INTENDED; case 1: h += data[0]; h *= m;