Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 160 additions & 0 deletions db/db_etc2_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@
#include <memory>

#include "db/db_test_util.h"
#include "db/log_format.h"
#include "db/log_writer.h"
#include "db/read_callback.h"
#include "db/version_edit.h"
#include "env/fs_readonly.h"
#include "file/writable_file_writer.h"
#include "options/options_helper.h"
#include "port/port.h"
#include "port/stack_trace.h"
Expand Down Expand Up @@ -9118,6 +9121,163 @@ TEST_F(DBTest2, GetFileChecksumsFromCurrentManifestCRC32) {
}
}

// The DB ID is read from the MANIFEST file of an open DB, from the new MANIFEST
// file that a reopen writes, and from a MANIFEST file of a closed DB.
TEST_F(DBTest2, GetDbIdentityFromManifest) {
Options options = CurrentOptions();
DestroyAndReopen(options);
std::string db_id;
ASSERT_OK(db_->GetDbIdentity(db_id));
ASSERT_FALSE(db_id.empty());
// The function does not write, so it works on a read-only file system.
std::shared_ptr<FileSystem> read_only_fs =
std::make_shared<ReadOnlyFileSystem>(env_->GetFileSystem());
const auto expect_db_id = [&](uint64_t manifest_number) {
std::string read_db_id;
ASSERT_OK(experimental::GetDbIdentityFromManifest(
read_only_fs.get(), DescriptorFileName(dbname_, manifest_number),
&read_db_id));
ASSERT_EQ(db_id, read_db_id);
};

const uint64_t first_manifest = dbfull()->TEST_Current_Manifest_FileNo();
expect_db_id(first_manifest);

Reopen(options);
const uint64_t second_manifest = dbfull()->TEST_Current_Manifest_FileNo();
ASSERT_NE(first_manifest, second_manifest);
expect_db_id(second_manifest);

Close();
expect_db_id(second_manifest);
}

// A MANIFEST file with no DB ID gives NotFound, a record that fails its
// checksum gives Corruption, and a missing file gives the file system error.
TEST_F(DBTest2, GetDbIdentityFromManifestErrors) {
Options options = CurrentOptions();
options.write_dbid_to_manifest = false;
DestroyAndReopen(options);
const std::string manifest_path =
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
Close();
FileSystem* fs = env_->GetFileSystem().get();
std::string db_id;

Status s = experimental::GetDbIdentityFromManifest(fs, manifest_path, &db_id);
ASSERT_EQ(Status::Code::kNotFound, s.code());
ASSERT_TRUE(db_id.empty());

std::string contents;
ASSERT_OK(ReadFileToString(fs, manifest_path, &contents));
ASSERT_GT(contents.size(), size_t{log::kHeaderSize});
contents[log::kHeaderSize] ^= 1;
const std::string damaged_path = manifest_path + ".damaged";
ASSERT_OK(WriteStringToFile(fs, contents, damaged_path));
s = experimental::GetDbIdentityFromManifest(fs, damaged_path, &db_id);
ASSERT_EQ(Status::Code::kCorruption, s.code());

s = experimental::GetDbIdentityFromManifest(
fs, DescriptorFileName(dbname_, 999999), &db_id);
ASSERT_EQ(Status::Code::kIOError, s.code());
ASSERT_EQ(Status::SubCode::kPathNotFound, s.subcode());
}

namespace {
// Returns the contents of a log file in the MANIFEST file format with the given
// records.
std::string LogFileWithRecords(const std::vector<std::string>& records) {
test::StringSink* sink = new test::StringSink();
std::unique_ptr<FSWritableFile> sink_holder(sink);
log::Writer writer(std::make_unique<WritableFileWriter>(
std::move(sink_holder), "manifest", FileOptions()),
0 /* log_number */, false /* recycle_log_files */);
for (const std::string& record : records) {
EXPECT_OK(writer.AddRecord(WriteOptions(), record));
}
EXPECT_OK(writer.WriteBuffer(WriteOptions()));
return sink->contents_;
}

std::string EncodeVersionEdit(const VersionEdit& edit) {
std::string record;
EXPECT_TRUE(edit.EncodeTo(&record));
return record;
}
} // namespace

// The DB ID can be in a later record than the first. When a record in an
// earlier log block fails its checksum, the function returns Corruption, not
// the DB ID of the later record: it does not skip damaged data.
TEST_F(DBTest2, GetDbIdentityFromManifestInLaterRecord) {
const std::string kDbId = "db-id-in-a-later-record";
// log::Reader drops the rest of a log block after a checksum failure and
// continues with the next block. The record with the DB ID must start in a
// block after the damaged one, so that the reader would return it if the
// function ignored the error. The first record has a column family name of
// 2 * log::kBlockSize bytes, so it fills blocks 0 and 1 and ends in block 2,
// where the record with the DB ID follows. The test changes the first payload
// byte of block 0.
VersionEdit first;
first.AddColumnFamily(std::string(2 * log::kBlockSize, 'c'));
VersionEdit second;
second.SetDBId(kDbId);
std::string contents =
LogFileWithRecords({EncodeVersionEdit(first), EncodeVersionEdit(second)});
FileSystem* fs = env_->GetFileSystem().get();
ASSERT_OK(env_->CreateDirIfMissing(dbname_));
const std::string path = dbname_ + "/manifest_with_later_db_id";
std::string db_id;

ASSERT_OK(WriteStringToFile(fs, contents, path));
ASSERT_OK(experimental::GetDbIdentityFromManifest(fs, path, &db_id));
ASSERT_EQ(kDbId, db_id);

// One changed byte in the payload of the first record.
contents[log::kHeaderSize] ^= 1;
ASSERT_OK(WriteStringToFile(fs, contents, path));
db_id.clear();
Status s = experimental::GetDbIdentityFromManifest(fs, path, &db_id);
ASSERT_EQ(Status::Code::kCorruption, s.code());
ASSERT_TRUE(db_id.empty());
}

// An empty file, and a file truncated inside the header or the body of the
// record with the DB ID, give NotFound: a truncated record at the end of the
// file counts as the end of the file. A record that passes its checksum but
// cannot be decoded gives Corruption.
TEST_F(DBTest2, GetDbIdentityFromManifestTruncatedOrUndecodable) {
const std::string kDbId = "db-id";
VersionEdit edit;
edit.SetDBId(kDbId);
const std::string contents = LogFileWithRecords({EncodeVersionEdit(edit)});
ASSERT_GT(contents.size(), size_t{log::kHeaderSize});
FileSystem* fs = env_->GetFileSystem().get();
ASSERT_OK(env_->CreateDirIfMissing(dbname_));
const std::string path = dbname_ + "/manifest_copy";
std::string db_id;

ASSERT_OK(WriteStringToFile(fs, contents, path));
ASSERT_OK(experimental::GetDbIdentityFromManifest(fs, path, &db_id));
ASSERT_EQ(kDbId, db_id);

for (size_t size :
{size_t{0}, size_t{log::kHeaderSize - 1}, contents.size() - 1}) {
SCOPED_TRACE("size=" + std::to_string(size));
ASSERT_OK(WriteStringToFile(fs, contents.substr(0, size), path));
db_id.clear();
Status s = experimental::GetDbIdentityFromManifest(fs, path, &db_id);
ASSERT_EQ(Status::Code::kNotFound, s.code());
ASSERT_TRUE(db_id.empty());
}

// Tag 0 is not a VersionEdit tag.
ASSERT_OK(
WriteStringToFile(fs, LogFileWithRecords({std::string(1, '\0')}), path));
Status s = experimental::GetDbIdentityFromManifest(fs, path, &db_id);
ASSERT_EQ(Status::Code::kCorruption, s.code());
}

// Parameterized by (allow_concurrent_memtable_write,
// min_tombstones_for_range_conversion).
class DBTestConcurrentRangeTombstoneConversions
Expand Down
107 changes: 83 additions & 24 deletions db/experimental.cc
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@
#include <vector>

#include "db/db_impl/db_impl.h"
#include "db/log_reader.h"
#include "db/manifest_ops.h"
#include "db/version_edit.h"
#include "db/version_edit_handler.h"
#include "db/version_util.h"
#include "file/sequence_file_reader.h"
#include "logging/logging.h"
#include "util/atomic.h"

Expand All @@ -41,6 +44,39 @@ Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end) {
return SuggestCompactRange(db, db->DefaultColumnFamily(), begin, end);
}

namespace {
// Keeps the first error that a log::Reader of a MANIFEST file reports, for
// example a read error or a record that fails its checksum. Later reports do
// not change it. The caller sets status_ptr before the first read.
struct ManifestReporter : public log::Reader::Reporter {
Status* status_ptr = nullptr;
void Corruption(size_t /*bytes*/, const Status& st,
uint64_t /*log_number*/ = kMaxSequenceNumber) override {
status_ptr->UpdateIfOk(st);
}
};

// Opens the MANIFEST file at manifest_path for reading with a log::Reader that
// verifies record checksums. reporter must outlive *reader.
Status NewManifestReader(FileSystem* fs, const std::string& manifest_path,
ManifestReporter* reporter,
std::unique_ptr<log::Reader>* reader) {
std::unique_ptr<FSSequentialFile> file;
Status s = fs->NewSequentialFile(manifest_path,
fs->OptimizeForManifestRead(FileOptions()),
&file, nullptr /* dbg */);
if (!s.ok()) {
return s;
}
std::unique_ptr<SequentialFileReader> file_reader =
std::make_unique<SequentialFileReader>(std::move(file), manifest_path);
*reader =
std::make_unique<log::Reader>(nullptr, std::move(file_reader), reporter,
true /* checksum */, 0 /* log_number */);
return Status::OK();
}
} // namespace

Status GetFileChecksumsFromCurrentManifest(FileSystem* fs,
const std::string& dbname,
FileChecksumList* checksum_list) {
Expand All @@ -61,41 +97,64 @@ Status GetFileChecksumsFromCurrentManifest(FileSystem* fs,
Env::IOActivity::kGetFileChecksumsFromCurrentManifest);
checksum_list->reset();

std::unique_ptr<SequentialFileReader> file_reader;
{
std::unique_ptr<FSSequentialFile> file;
s = fs->NewSequentialFile(manifest_path,
fs->OptimizeForManifestRead(FileOptions()), &file,
nullptr /* dbg */);
if (!s.ok()) {
return s;
}
file_reader.reset(new SequentialFileReader(std::move(file), manifest_path));
}

struct LogReporter : public log::Reader::Reporter {
Status* status_ptr;
void Corruption(size_t /*bytes*/, const Status& st,
uint64_t /*log_number*/ = kMaxSequenceNumber) override {
if (status_ptr->ok()) {
*status_ptr = st;
}
}
} reporter;
ManifestReporter reporter;
reporter.status_ptr = &s;
log::Reader reader(nullptr, std::move(file_reader), &reporter,
true /* checksum */, 0 /* log_number */);
std::unique_ptr<log::Reader> reader;
s = NewManifestReader(fs, manifest_path, &reporter, &reader);
if (!s.ok()) {
return s;
}

// Read all records from the manifest file...
uint64_t manifest_file_size = std::numeric_limits<uint64_t>::max();
FileChecksumRetriever retriever(read_options, manifest_file_size);
retriever.Iterate(reader, &s);
retriever.Iterate(*reader, &s);
if (!retriever.status().ok()) {
return retriever.status();
}
return retriever.FetchFileChecksumList(*checksum_list);
}

Status GetDbIdentityFromManifest(FileSystem* fs,
const std::string& manifest_path,
std::string* db_id) {
if (fs == nullptr) {
return Status::InvalidArgument("fs is nullptr");
}
if (db_id == nullptr) {
return Status::InvalidArgument("db_id is nullptr");
}
Status s;
ManifestReporter reporter;
reporter.status_ptr = &s;
std::unique_ptr<log::Reader> reader;
s = NewManifestReader(fs, manifest_path, &reporter, &reader);
if (!s.ok()) {
return s;
}

Slice record;
std::string scratch;
// log::Reader can report a damaged block to the reporter, which sets s, and
// then return a later intact record. Checking s after each ReadRecord() makes
// sure that a DB ID after damaged data is never returned.
while (reader->ReadRecord(&record, &scratch) && s.ok()) {
VersionEdit edit;
s = edit.DecodeFrom(record);
if (!s.ok()) {
return s;
}
if (edit.HasDbId()) {
*db_id = edit.GetDbId();
return Status::OK();
}
}
if (!s.ok()) {
return s;
}
return Status::NotFound("MANIFEST file records no DB ID", manifest_path);
}

Status UpdateManifestForFilesState(
const DBOptions& db_opts, const std::string& db_name,
const std::vector<ColumnFamilyDescriptor>& column_families,
Expand Down
23 changes: 23 additions & 0 deletions include/rocksdb/experimental.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,29 @@ Status GetFileChecksumsFromCurrentManifest(FileSystem* fs,
const std::string& dbname,
FileChecksumList* checksum_list);

// Reads the DB ID (see DB::GetDbIdentity()) that a MANIFEST file records,
// without opening the DB. manifest_path is the path of one MANIFEST file, not a
// DB directory, so no CURRENT file is needed. While
// DBOptions::write_dbid_to_manifest is true, RocksDB writes the DB ID as the
// first record of each new MANIFEST file. The DB ID stays the same when the DB
// reopens or starts a new MANIFEST file, and a physical copy of the DB keeps
// it. RepairDB() gives the DB a new DB ID.
//
// The file is read until the first record that has a DB ID. Returns NotFound
// when no record has one, for example in a MANIFEST file written with
// write_dbid_to_manifest=false. A record that is truncated at the end of the
// file counts as the end of the file, so an empty or truncated file also
// returns NotFound. Returns the first error when the file cannot be read, a
// record fails its checksum, or a record cannot be decoded. Damaged records are
// never skipped.
//
// This function does not write. It can read the MANIFEST file of an open DB:
// the DB ID is in the first record of a new MANIFEST file, so records that the
// DB appends later do not change the result.
Status GetDbIdentityFromManifest(FileSystem* fs,
const std::string& manifest_path,
std::string* db_id);

struct UpdateManifestForFilesStateOptions {
// When true, read current file temperatures from FileSystem and update in
// DB manifest when a temperature other than Unknown is reported and
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added experimental API `GetDbIdentityFromManifest()` to read the DB ID from a MANIFEST file without opening the DB.
Loading