Skip to content

Commit 3938983

Browse files
author
yinzhengsun
committed
fix ci
1 parent 64cc97c commit 3938983

2 files changed

Lines changed: 110 additions & 45 deletions

File tree

‎src/iceberg/parquet/parquet_reader.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "iceberg/parquet/parquet_reader.h"
2121

2222
#include <algorithm>
23+
#include <utility>
2324
#include <variant>
2425
#include <vector>
2526

@@ -444,7 +445,7 @@ class ParquetReader::Impl {
444445
if (row_group->num_rows() == 0) {
445446
continue;
446447
}
447-
context_->row_groups_.push_back({i, row_start});
448+
context_->row_groups_.push_back({.index = i, .first_row = row_start});
448449
}
449450
if (context_->row_groups_.empty()) {
450451
context_->record_batch_reader_ = std::make_unique<EmptyRecordBatchReader>();

‎src/iceberg/test/parquet_row_group_filter_test.cc‎

Lines changed: 108 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -283,27 +283,77 @@ TEST_F(ParquetRowGroupFilterTest, FileAndFilterTypeCompatibility) {
283283
R"([["0.00",0],["1.00",1],["10.00",2],["11.00",3],["20.00",4],["21.00",5]])";
284284
const std::string timestamps =
285285
R"([["1970-01-01 00:00:00",0],["1970-01-01 00:00:01",1],["1970-01-01 00:00:10",2],["1970-01-01 00:00:11",3],["1970-01-01 00:00:20",4],["1970-01-01 00:00:21",5]])";
286-
for (const auto& test : std::vector<Case>{
287-
{float32(), float64(), numbers, Literal::Double(10), true},
288-
{float64(), float32(), numbers, Literal::Float(10), false},
289-
{int64(), int32(), numbers, Literal::Int(10), false},
290-
{int32(), date(), numbers, Literal::Date(10), false},
291-
{date(), int32(), numbers, Literal::Int(10), false},
292-
{decimal(9, 2), decimal(18, 2), decimals, Literal::Decimal(1000, 18, 2), true},
293-
{decimal(9, 2), decimal(8, 2), decimals, Literal::Decimal(1000, 8, 2), false},
294-
{decimal(9, 2), decimal(9, 3), decimals, Literal::Decimal(1000, 9, 3), false},
295-
{timestamp(), timestamp_ns(), timestamps, Literal::TimestampNs(10000000),
296-
false},
297-
{timestamp(), timestamp_tz(), timestamps, Literal::TimestampTz(10000000),
298-
false},
299-
{timestamp_ns(), timestamp_ns(), timestamps, Literal::TimestampNs(10000000000),
300-
true},
301-
{timestamp_tz(), timestamp_tz(), timestamps, Literal::TimestampTz(10000000),
302-
true},
303-
{fixed(1), fixed(1), R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
304-
Literal::Fixed({'m'}), true},
305-
{fixed(1), fixed(2), R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
306-
Literal::Fixed({'m', 'm'}), false}}) {
286+
for (const auto& test :
287+
std::vector<Case>{{.file_type = float32(),
288+
.filter_type = float64(),
289+
.json = numbers,
290+
.value = Literal::Double(10),
291+
.compatible = true},
292+
{.file_type = float64(),
293+
.filter_type = float32(),
294+
.json = numbers,
295+
.value = Literal::Float(10),
296+
.compatible = false},
297+
{.file_type = int64(),
298+
.filter_type = int32(),
299+
.json = numbers,
300+
.value = Literal::Int(10),
301+
.compatible = false},
302+
{.file_type = int32(),
303+
.filter_type = date(),
304+
.json = numbers,
305+
.value = Literal::Date(10),
306+
.compatible = false},
307+
{.file_type = date(),
308+
.filter_type = int32(),
309+
.json = numbers,
310+
.value = Literal::Int(10),
311+
.compatible = false},
312+
{.file_type = decimal(9, 2),
313+
.filter_type = decimal(18, 2),
314+
.json = decimals,
315+
.value = Literal::Decimal(1000, 18, 2),
316+
.compatible = true},
317+
{.file_type = decimal(9, 2),
318+
.filter_type = decimal(8, 2),
319+
.json = decimals,
320+
.value = Literal::Decimal(1000, 8, 2),
321+
.compatible = false},
322+
{.file_type = decimal(9, 2),
323+
.filter_type = decimal(9, 3),
324+
.json = decimals,
325+
.value = Literal::Decimal(1000, 9, 3),
326+
.compatible = false},
327+
{.file_type = timestamp(),
328+
.filter_type = timestamp_ns(),
329+
.json = timestamps,
330+
.value = Literal::TimestampNs(10000000),
331+
.compatible = false},
332+
{.file_type = timestamp(),
333+
.filter_type = timestamp_tz(),
334+
.json = timestamps,
335+
.value = Literal::TimestampTz(10000000),
336+
.compatible = false},
337+
{.file_type = timestamp_ns(),
338+
.filter_type = timestamp_ns(),
339+
.json = timestamps,
340+
.value = Literal::TimestampNs(10000000000),
341+
.compatible = true},
342+
{.file_type = timestamp_tz(),
343+
.filter_type = timestamp_tz(),
344+
.json = timestamps,
345+
.value = Literal::TimestampTz(10000000),
346+
.compatible = true},
347+
{.file_type = fixed(1),
348+
.filter_type = fixed(1),
349+
.json = R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
350+
.value = Literal::Fixed({'m'}),
351+
.compatible = true},
352+
{.file_type = fixed(1),
353+
.filter_type = fixed(2),
354+
.json = R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
355+
.value = Literal::Fixed({'m', 'm'}),
356+
.compatible = false}}) {
307357
SCOPED_TRACE(test.file_type->ToString() + " -> " + test.filter_type->ToString());
308358
schema_ = std::make_shared<Schema>(std::vector<SchemaField>{
309359
SchemaField::MakeOptional(1, "key", test.file_type), schema_->fields()[1]});
@@ -367,12 +417,15 @@ TEST_F(ParquetRowGroupFilterTest, BoundComparisonBoundariesAndAllNullGroups) {
367417
std::vector<int64_t> expected;
368418
};
369419
for (const auto& test : std::vector<Case>{
370-
{Expressions::LessThan("key", Literal::Int(10)), {}},
371-
{Expressions::LessThanOrEqual("key", Literal::Int(10)), {2, 3}},
372-
{Expressions::GreaterThan("key", Literal::Int(21)), {}},
373-
{Expressions::GreaterThanOrEqual("key", Literal::Int(21)), {4, 5}},
374-
{Expressions::Equal("key", Literal::Int(11)), {2, 3}},
375-
{Expressions::In("key", {Literal::Int(11), Literal::Int(20)}), {2, 3, 4, 5}},
420+
{.filter = Expressions::LessThan("key", Literal::Int(10)), .expected = {}},
421+
{.filter = Expressions::LessThanOrEqual("key", Literal::Int(10)),
422+
.expected = {2, 3}},
423+
{.filter = Expressions::GreaterThan("key", Literal::Int(21)), .expected = {}},
424+
{.filter = Expressions::GreaterThanOrEqual("key", Literal::Int(21)),
425+
.expected = {4, 5}},
426+
{.filter = Expressions::Equal("key", Literal::Int(11)), .expected = {2, 3}},
427+
{.filter = Expressions::In("key", {Literal::Int(11), Literal::Int(20)}),
428+
.expected = {2, 3, 4, 5}},
376429
}) {
377430
ICEBERG_UNWRAP_OR_FAIL(auto bound, Binder::Bind(*schema_, test.filter, true));
378431
Check(Options(bound), test.expected);
@@ -425,14 +478,14 @@ TEST_F(ParquetRowGroupFilterTest, SplitIntersectionAndEmptySplit) {
425478
auto offset = metadata_->RowGroup(1)->file_offset();
426479
ASSERT_GT(offset, metadata_->RowGroup(0)->file_offset());
427480
auto options = Options(Expressions::GreaterThanOrEqual("key", Literal::Int(20)));
428-
options.split = Split{static_cast<size_t>(offset), 100000};
481+
options.split = Split{.offset = static_cast<size_t>(offset), .length = 100000};
429482
Check(options, {4, 5});
430483
ICEBERG_UNWRAP_OR_FAIL(
431484
options.filter,
432485
Binder::Bind(*schema_, Expressions::LessThan("key", Literal::Int(2)), true));
433486
Check(options, {});
434487
options.filter = nullptr;
435-
options.split = Split{static_cast<size_t>(offset), 0};
488+
options.split = Split{.offset = static_cast<size_t>(offset), .length = 0};
436489
Check(options, {});
437490
}
438491

@@ -443,14 +496,21 @@ TEST_F(ParquetRowGroupFilterTest, PrimitiveComparisonTypes) {
443496
Literal literal;
444497
};
445498
for (const auto& test : std::vector<Case>{
446-
{int64(), "[[0,0],[1,1],[10,2],[11,3],[20,4],[21,5]]", Literal::Long(10)},
447-
{date(), "[[0,0],[1,1],[10,2],[11,3],[20,4],[21,5]]", Literal::Date(10)},
448-
{string(), R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
449-
Literal::String("m")},
450-
{boolean(), "[[false,0],[false,1],[true,2],[true,3],[false,4],[false,5]]",
451-
Literal::Boolean(true)},
452-
{binary(), R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
453-
Literal::Binary({'m'})}}) {
499+
{.type = int64(),
500+
.json = "[[0,0],[1,1],[10,2],[11,3],[20,4],[21,5]]",
501+
.literal = Literal::Long(10)},
502+
{.type = date(),
503+
.json = "[[0,0],[1,1],[10,2],[11,3],[20,4],[21,5]]",
504+
.literal = Literal::Date(10)},
505+
{.type = string(),
506+
.json = R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
507+
.literal = Literal::String("m")},
508+
{.type = boolean(),
509+
.json = "[[false,0],[false,1],[true,2],[true,3],[false,4],[false,5]]",
510+
.literal = Literal::Boolean(true)},
511+
{.type = binary(),
512+
.json = R"([["a",0],["b",1],["m",2],["n",3],["y",4],["z",5]])",
513+
.literal = Literal::Binary({'m'})}}) {
454514
schema_ = std::make_shared<Schema>(
455515
std::vector<SchemaField>{SchemaField::MakeOptional(1, "key", test.type),
456516
SchemaField::MakeRequired(2, "value", int64())});
@@ -510,13 +570,17 @@ TEST_F(ParquetRowGroupFilterTest, DecimalAndTemporalStatistics) {
510570
};
511571
for (
512572
const auto& test : std::vector<Case>{
513-
{decimal(9, 2),
514-
R"([["0.00",0],["1.00",1],["10.00",2],["11.00",3],["20.00",4],["21.00",5]])",
515-
Literal::Decimal(1000, 9, 2)},
516-
{time(), "[[0,0],[1,1],[10,2],[11,3],[20,4],[21,5]]", Literal::Time(10)},
517-
{timestamp(),
518-
R"([["1970-01-01 00:00:00",0],["1970-01-01 00:00:01",1],["1970-01-01 00:00:10",2],["1970-01-01 00:00:11",3],["1970-01-01 00:00:20",4],["1970-01-01 00:00:21",5]])",
519-
Literal::Timestamp(10000000)}}) {
573+
{.type = decimal(9, 2),
574+
.json =
575+
R"([["0.00",0],["1.00",1],["10.00",2],["11.00",3],["20.00",4],["21.00",5]])",
576+
.value = Literal::Decimal(1000, 9, 2)},
577+
{.type = time(),
578+
.json = "[[0,0],[1,1],[10,2],[11,3],[20,4],[21,5]]",
579+
.value = Literal::Time(10)},
580+
{.type = timestamp(),
581+
.json =
582+
R"([["1970-01-01 00:00:00",0],["1970-01-01 00:00:01",1],["1970-01-01 00:00:10",2],["1970-01-01 00:00:11",3],["1970-01-01 00:00:20",4],["1970-01-01 00:00:21",5]])",
583+
.value = Literal::Timestamp(10000000)}}) {
520584
schema_ = std::make_shared<Schema>(
521585
std::vector<SchemaField>{SchemaField::MakeOptional(1, "key", test.type),
522586
SchemaField::MakeRequired(2, "value", int64())});

0 commit comments

Comments
 (0)