Skip to content

Commit bffa46b

Browse files
committed
fix(inspect): address metadata table review feedback
1 parent 894539e commit bffa46b

4 files changed

Lines changed: 56 additions & 13 deletions

File tree

‎src/iceberg/inspect/metadata_table.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@ class ICEBERG_EXPORT MetadataTable {
7575

7676
/// \brief Scan the metadata table without time travel.
7777
///
78-
/// The caller owns the returned stream and must release it with
79-
/// ArrowArrayStreamRelease.
78+
/// The caller owns the returned stream and must invoke its `release` callback
79+
/// when the stream is no longer needed.
8080
virtual Result<ArrowArrayStream> Scan() = 0;
8181

8282
protected:
@@ -100,6 +100,9 @@ struct SnapshotSelection {
100100
};
101101

102102
/// \brief Base interface for metadata tables that support time travel.
103+
///
104+
/// Keeping snapshot selection on this capability-specific interface prevents it from
105+
/// being exposed by metadata tables that do not support time travel.
103106
class ICEBERG_EXPORT TimeTravelMetadataTable : public MetadataTable {
104107
public:
105108
~TimeTravelMetadataTable() override;

‎src/iceberg/inspect/snapshots_table.cc‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <cstddef>
2424
#include <memory>
2525
#include <optional>
26+
#include <tuple>
2627
#include <utility>
2728
#include <vector>
2829

@@ -36,6 +37,7 @@
3637
#include "iceberg/schema_internal.h"
3738
#include "iceberg/snapshot.h"
3839
#include "iceberg/table.h"
40+
#include "iceberg/table_metadata.h"
3941
#include "iceberg/type.h"
4042
#include "iceberg/util/macros.h"
4143

@@ -78,25 +80,28 @@ Status AppendSnapshot(ArrowRowBuilder& builder, const Snapshot& snapshot) {
7880
class SnapshotsTableStream {
7981
public:
8082
static Result<std::unique_ptr<SnapshotsTableStream>> Make(
81-
std::shared_ptr<Table> table, const iceberg::Schema& schema) {
83+
std::shared_ptr<const TableMetadata> metadata, const iceberg::Schema& schema) {
8284
ArrowSchema arrow_schema{};
8385
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(schema, &arrow_schema));
8486
return std::unique_ptr<SnapshotsTableStream>(
85-
new SnapshotsTableStream(std::move(table), std::move(arrow_schema)));
87+
new SnapshotsTableStream(std::move(metadata), std::move(arrow_schema)));
8688
}
8789

8890
~SnapshotsTableStream() { std::ignore = Close(); }
8991

9092
Status Close() {
91-
table_.reset();
93+
metadata_.reset();
9294
if (arrow_schema_.release != nullptr) {
9395
ArrowSchemaRelease(&arrow_schema_);
9496
}
9597
return {};
9698
}
9799

98100
Result<std::optional<ArrowArray>> Next() {
99-
const auto& snapshots = table_->snapshots();
101+
if (metadata_ == nullptr) [[unlikely]] {
102+
return InvalidArgument("Cannot read from a closed snapshots table stream");
103+
}
104+
const auto& snapshots = metadata_->snapshots;
100105
if (next_snapshot_ == snapshots.size()) {
101106
return std::nullopt;
102107
}
@@ -129,10 +134,11 @@ class SnapshotsTableStream {
129134
}
130135

131136
private:
132-
SnapshotsTableStream(std::shared_ptr<Table> table, ArrowSchema arrow_schema)
133-
: table_(std::move(table)), arrow_schema_(std::move(arrow_schema)) {}
137+
SnapshotsTableStream(std::shared_ptr<const TableMetadata> metadata,
138+
ArrowSchema arrow_schema)
139+
: metadata_(std::move(metadata)), arrow_schema_(std::move(arrow_schema)) {}
134140

135-
std::shared_ptr<Table> table_;
141+
std::shared_ptr<const TableMetadata> metadata_;
136142
ArrowSchema arrow_schema_{};
137143
size_t next_snapshot_ = 0;
138144
};
@@ -165,8 +171,8 @@ Result<std::unique_ptr<SnapshotsTable>> SnapshotsTable::Make(
165171
}
166172

167173
Result<ArrowArrayStream> SnapshotsTable::Scan() {
168-
ICEBERG_ASSIGN_OR_RAISE(auto stream,
169-
SnapshotsTableStream::Make(source_table(), *schema()));
174+
ICEBERG_ASSIGN_OR_RAISE(
175+
auto stream, SnapshotsTableStream::Make(source_table()->metadata(), *schema()));
170176
return MakeArrowArrayStream(std::move(stream));
171177
}
172178

‎src/iceberg/test/metadata_table_test_base.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#pragma once
2828

2929
#include <memory>
30+
#include <string>
3031
#include <unordered_map>
3132
#include <utility>
3233
#include <vector>
@@ -143,7 +144,8 @@ class MetadataTableTestBase : public ::testing::Test {
143144

144145
/// \brief Create a Table with the given snapshots.
145146
Result<std::shared_ptr<Table>> MakeTableWithSnapshots(
146-
std::vector<std::shared_ptr<Snapshot>> snapshots, int64_t current_snapshot_id) {
147+
std::vector<std::shared_ptr<Snapshot>> snapshots, int64_t current_snapshot_id,
148+
std::string metadata_location = "s3://bucket/meta.json") {
147149
auto schema = std::make_shared<Schema>(
148150
std::vector<SchemaField>{SchemaField::MakeRequired(1, "id", int64()),
149151
SchemaField::MakeOptional(2, "name", string())},
@@ -157,7 +159,7 @@ class MetadataTableTestBase : public ::testing::Test {
157159
});
158160

159161
TableIdentifier ident{.ns = Namespace{.levels = {"db"}}, .name = "test_table"};
160-
return Table::Make(ident, metadata, "s3://bucket/meta.json", io_, catalog_);
162+
return Table::Make(ident, metadata, std::move(metadata_location), io_, catalog_);
161163
}
162164

163165
std::shared_ptr<MockFileIO> io_;

‎src/iceberg/test/snapshots_table_test.cc‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,4 +194,36 @@ TEST_F(SnapshotsTableTest, ScanReturnsMultipleBatches) {
194194
EXPECT_EQ(batches[1]->num_rows(), 1);
195195
}
196196

197+
TEST_F(SnapshotsTableTest, ScanUsesMetadataCapturedAtCreation) {
198+
auto snapshot = MakeTestSnapshots().first;
199+
std::vector<std::shared_ptr<Snapshot>> snapshots(1025, snapshot);
200+
ICEBERG_UNWRAP_OR_FAIL(auto table, MakeTableWithSnapshots(std::move(snapshots),
201+
/*current_snapshot_id=*/1));
202+
ICEBERG_UNWRAP_OR_FAIL(auto snapshots_table,
203+
MetadataTable::Make<SnapshotsTable>(table));
204+
205+
ICEBERG_UNWRAP_OR_FAIL(auto stream, snapshots_table->Scan());
206+
auto reader_result = ::arrow::ImportRecordBatchReader(&stream);
207+
ASSERT_TRUE(reader_result.ok()) << reader_result.status().ToString();
208+
auto reader = std::move(reader_result).ValueUnsafe();
209+
210+
auto first_batch_result = reader->Next();
211+
ASSERT_TRUE(first_batch_result.ok()) << first_batch_result.status().ToString();
212+
ASSERT_NE(*first_batch_result, nullptr);
213+
EXPECT_EQ((*first_batch_result)->num_rows(), MetadataTable::kBatchSize);
214+
215+
ICEBERG_UNWRAP_OR_FAIL(
216+
auto refreshed_table,
217+
MakeTableWithSnapshots({}, /*current_snapshot_id=*/kInvalidSnapshotId,
218+
"s3://bucket/refreshed-meta.json"));
219+
EXPECT_CALL(*catalog_, LoadTable(table->name()))
220+
.WillOnce(::testing::Return(refreshed_table));
221+
ASSERT_THAT(table->Refresh(), IsOk());
222+
223+
auto second_batch_result = reader->Next();
224+
ASSERT_TRUE(second_batch_result.ok()) << second_batch_result.status().ToString();
225+
ASSERT_NE(*second_batch_result, nullptr);
226+
EXPECT_EQ((*second_batch_result)->num_rows(), 1);
227+
}
228+
197229
} // namespace iceberg

0 commit comments

Comments
 (0)