From fcc93f3aca67ab614a3f35ff0501f81aca5fcf7c Mon Sep 17 00:00:00 2001 From: jacktengg Date: Tue, 3 Nov 2026 01:22:08 +0800 Subject: [PATCH 1/8] [fix](be) Preserve local time in TIMESTAMPTZ day arithmetic ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: TIMESTAMPTZ day arithmetic changes UTC fields, so adding a day across a DST transition adds a fixed 24 hours and shifts the local clock. In an America/New_York session, adding one day to 2024-03-09 12:00:00-05:00 produces 13:00 instead of 12:00 the following day. Apply DAY intervals to the session-local date in BE and FE constant folding, preserving fractional seconds, preferred offsets in overlaps, existing gap resolution, and date-range errors. Restrict DST-sensitive day arithmetic monotonicity and comparison rearrangement: moving a local-day interval across a comparison can otherwise return incorrect rows at a fall-back. Explicit hour arithmetic keeps elapsed-time semantics. ### Release note TIMESTAMPTZ day addition and subtraction preserve session-local calendar time across daylight-saving transitions, allowing a day to represent 23 or 25 elapsed hours. ### Check List (For Author) - Test: Regression test / Unit Test / Manual test - Regression test: test_timestamptz_days_add_dst, test_timestamptz_dst_fold, test_timestamptz_dst_gap, test_timestamptz_utc_functions; all 4 suites passed. Expected results generated by the regression script; 7 FE/BE result groups match. - Unit Test: 19 FE tests in TimestampTzLiteralTest and SimplifyArithmeticComparisonRuleTest; 10 BE tests in TimeStampTzValueTest and selected VTimestampFunctionsTest cases; all passed. - Manual test: New York spring transition returns local 12:00 and 82800 seconds; adding 24 hours returns 13:00. - ASAN BE build, FE build with Checkstyle, clang-format 16, build hygiene, and clang-tidy passed. - Behavior changed: Yes. TIMESTAMPTZ DAY intervals use session-local calendar dates; unsafe monotonicity and inverse comparison rewrites are disabled for variable-offset zones. - Does this need documentation: No. Existing date arithmetic bug fix; release note included. --- be/src/core/value/timestamptz_value.cpp | 18 +++ be/src/core/value/timestamptz_value.h | 3 + .../function_date_or_datetime_computation.h | 122 +++++++++++------- be/test/exprs/function/function_time_test.cpp | 64 +++++++++ be/test/runtime/timestamptz_value_test.cpp | 72 +++++++++++ .../SimplifyArithmeticComparisonRule.java | 8 ++ .../functions/DateAddSubMonotonic.java | 11 ++ .../literal/TimestampTzLiteral.java | 22 +++- .../SimplifyArithmeticComparisonRuleTest.java | 42 ++++++ .../literal/TimestampTzLiteralTest.java | 94 ++++++++++++++ .../test_timestamptz_days_add_dst.out | 67 ++++++++++ .../test_timestamptz_days_add_dst.groovy | 97 ++++++++++++++ 12 files changed, 572 insertions(+), 48 deletions(-) create mode 100644 regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out create mode 100644 regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy diff --git a/be/src/core/value/timestamptz_value.cpp b/be/src/core/value/timestamptz_value.cpp index 0dd310438e5943..3e473a75e7daa7 100644 --- a/be/src/core/value/timestamptz_value.cpp +++ b/be/src/core/value/timestamptz_value.cpp @@ -88,6 +88,24 @@ int TimestampTzValue::to_buffer(char* buffer, const cctz::time_zone& tz, int sca return len; } +bool TimestampTzValue::date_add_days(int64_t days, const cctz::time_zone& local_time_zone) { + DateV2Value local_dt; + // UTC year 0000/9999 can become an unrepresentable local year after zone conversion. + if (!to_datetime(local_dt, local_time_zone, 6, 6)) { + return false; + } + // For example, adding zero days to the second 01:30 at the autumn transition must + // retain that instant, rather than resolving to the first occurrence of 01:30. + const auto preferred_offset = static_cast(local_dt.datetime_diff_in_seconds(_utc_dt)); + // A day delta can move an otherwise valid local date beyond years [0, 9999]. + if (!local_dt.date_add_days(days)) { + return false; + } + convert_local_to_utc(local_time_zone, local_dt, preferred_offset); + // A valid local result can still fall outside the supported UTC year range. + return is_valid_date(); +} + bool TimestampTzValue::from_datetime(const DateV2Value& origin_dt, const cctz::time_zone& local_time_zone, int dt_scale, int tz_scale) { diff --git a/be/src/core/value/timestamptz_value.h b/be/src/core/value/timestamptz_value.h index 7409e0eb56c682..34d4cb283eb3f3 100644 --- a/be/src/core/value/timestamptz_value.h +++ b/be/src/core/value/timestamptz_value.h @@ -134,6 +134,9 @@ class TimestampTzValue { return _utc_dt.date_add_interval(interval); } + // Add session-local calendar days, preserving the input offset in a DST overlap. + bool date_add_days(int64_t days, const cctz::time_zone& local_time_zone); + // truncate datetime to specified unit template bool datetime_trunc() { diff --git a/be/src/exprs/function/function_date_or_datetime_computation.h b/be/src/exprs/function/function_date_or_datetime_computation.h index 1212fad82b9fdf..856fcdc2c437b5 100644 --- a/be/src/exprs/function/function_date_or_datetime_computation.h +++ b/be/src/exprs/function/function_date_or_datetime_computation.h @@ -76,11 +76,17 @@ namespace doris { template auto date_time_add(const typename PrimitiveTypeTraits::DataType::FieldType& t, - IntervalType delta) { + IntervalType delta, const cctz::time_zone& time_zone) { // e.g.: for DatatypeDatetimeV2, cast from u64 to DateV2Value auto ts_value = t; - if constexpr ((unit == TimeUnit::DAY || unit == TimeUnit::WEEK) && - is_date_v2_or_datetime_v2(ArgType)) { + if constexpr (ArgType == TYPE_TIMESTAMPTZ && unit == TimeUnit::DAY) { + // In New York, 2024-03-09 12:00 + 1 calendar day is only 23 elapsed hours. + if (!ts_value.date_add_days(delta, time_zone)) [[unlikely]] { + throw_out_of_bound_date_int(get_time_unit_name(unit), t, delta); + } + return ts_value; + } else if constexpr ((unit == TimeUnit::DAY || unit == TimeUnit::WEEK) && + is_date_v2_or_datetime_v2(ArgType)) { // DAY / WEEK only move the date part. `date_add_days` is inline, so the whole per-row // computation folds into the caller's loop: no TimeInterval, no second-level arithmetic, // just two L1-resident dictionary lookups. The input comes from a column and is already @@ -102,34 +108,36 @@ auto date_time_add(const typename PrimitiveTypeTraits::DataType::FieldT } } -#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \ - template \ - struct CLASS { \ - /* return type must be same with arg type. cast have already be planned in FE */ \ - static constexpr PrimitiveType ArgPType = PType; \ - static constexpr PrimitiveType ReturnType = PType; \ - /* for interval <= minute, use bigint. otherwise int. */ \ - static constexpr PrimitiveType IntervalPType = \ - (TimeUnit::UNIT == TimeUnit::YEAR || TimeUnit::UNIT == TimeUnit::QUARTER || \ - TimeUnit::UNIT == TimeUnit::MONTH || TimeUnit::UNIT == TimeUnit::WEEK || \ - TimeUnit::UNIT == TimeUnit::DAY || TimeUnit::UNIT == TimeUnit::HOUR) \ - ? PrimitiveType::TYPE_INT \ - : PrimitiveType::TYPE_BIGINT; \ - using InputValueType = typename PrimitiveTypeTraits::DataType::FieldType; \ - using ReturnValueType = InputValueType; \ - using IntervalDataType = typename PrimitiveTypeTraits::DataType; \ - using IntervalValueType = IntervalDataType::FieldType; \ - \ - static constexpr auto name = #NAME; \ - static constexpr auto is_nullable = false; \ - static inline ReturnValueType execute(const InputValueType& t, IntervalValueType delta) { \ - return date_time_add(t, delta); \ - } \ - \ - static DataTypes get_variadic_argument_types() { \ - return {std::make_shared::DataType>(), \ - std::make_shared::DataType>()}; \ - } \ +#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \ + template \ + struct CLASS { \ + /* return type must be same with arg type. cast have already be planned in FE */ \ + static constexpr PrimitiveType ArgPType = PType; \ + static constexpr PrimitiveType ReturnType = PType; \ + /* for interval <= minute, use bigint. otherwise int. */ \ + static constexpr PrimitiveType IntervalPType = \ + (TimeUnit::UNIT == TimeUnit::YEAR || TimeUnit::UNIT == TimeUnit::QUARTER || \ + TimeUnit::UNIT == TimeUnit::MONTH || TimeUnit::UNIT == TimeUnit::WEEK || \ + TimeUnit::UNIT == TimeUnit::DAY || TimeUnit::UNIT == TimeUnit::HOUR) \ + ? PrimitiveType::TYPE_INT \ + : PrimitiveType::TYPE_BIGINT; \ + using InputValueType = typename PrimitiveTypeTraits::DataType::FieldType; \ + using ReturnValueType = InputValueType; \ + using IntervalDataType = typename PrimitiveTypeTraits::DataType; \ + using IntervalValueType = IntervalDataType::FieldType; \ + \ + static constexpr auto name = #NAME; \ + static constexpr auto is_nullable = false; \ + static inline ReturnValueType execute( \ + const InputValueType& t, IntervalValueType delta, \ + const cctz::time_zone& time_zone = cctz::utc_time_zone()) { \ + return date_time_add(t, delta, time_zone); \ + } \ + \ + static DataTypes get_variadic_argument_types() { \ + return {std::make_shared::DataType>(), \ + std::make_shared::DataType>()}; \ + } \ } ADD_TIME_FUNCTION_IMPL(AddMicrosecondsImpl, microseconds_add, MICROSECOND); ADD_TIME_FUNCTION_IMPL(AddMillisecondsImpl, milliseconds_add, MILLISECOND); @@ -422,9 +430,11 @@ struct AddQuartersImpl { static constexpr auto name = "quarters_add"; static constexpr auto is_nullable = false; - static inline ReturnValueType execute(const InputValueType& t, Int64 delta) { + static inline ReturnValueType execute( + const InputValueType& t, Int64 delta, + const cctz::time_zone& time_zone = cctz::utc_time_zone()) { // Preserve both the month offset and the negation of INT_MIN in quarters_sub. - return date_time_add(t, 3 * delta); + return date_time_add(t, 3 * delta, time_zone); } static DataTypes get_variadic_argument_types() { @@ -440,8 +450,10 @@ struct SubtractIntervalImpl { using InputValueType = typename Transform::InputValueType; using ReturnValueType = typename Transform::ReturnValueType; static constexpr auto is_nullable = false; - static inline ReturnValueType execute(const InputValueType& t, Int64 delta) { - return Transform::execute(t, -delta); + static inline ReturnValueType execute( + const InputValueType& t, Int64 delta, + const cctz::time_zone& time_zone = cctz::utc_time_zone()) { + return Transform::execute(t, -delta, time_zone); } static DataTypes get_variadic_argument_types() { @@ -760,6 +772,18 @@ struct DateTimeOp { // arg1 maybe just delta value(e.g. DataTypeInt32, not datelike type) constexpr static bool CastType1 = is_date_type(DataType1); + static ToType execute(const ValueType0& from, const ValueType1& delta, + FunctionContext* context) { + // Date add/sub needs the session zone; diff and extract transforms take two arguments. + if constexpr (requires { + Transform::execute(from, delta, context->state()->timezone_obj()); + }) { + return Transform::execute(from, delta, context->state()->timezone_obj()); + } else { + return Transform::execute(from, delta); + } + } + // Use for compatibility with TimeUnionType interval(string type) static inline ValueType1 get_element(const IntervalColumnType& data, size_t i) { if constexpr (DataType1 == TYPE_STRING) { @@ -772,18 +796,20 @@ struct DateTimeOp { // execute on the null value's nested value may cause false positive exception, so use nullmaps to skip them. static void vector_vector(const PaddedPODArray& vec_from0, const IntervalColumnType& vec_from1, PaddedPODArray& vec_to, - const NullMap* nullmap0, const NullMap* nullmap1) { + const NullMap* nullmap0, const NullMap* nullmap1, + FunctionContext* context) { for (size_t i = 0; i < vec_from0.size(); ++i) { if ((nullmap0 && (*nullmap0)[i]) || (nullmap1 && (*nullmap1)[i])) [[unlikely]] { continue; } - vec_to[i] = Transform::execute(vec_from0[i], get_element(vec_from1, i)); + vec_to[i] = execute(vec_from0[i], get_element(vec_from1, i), context); } } static void vector_constant(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, const ValueType1& delta, - const NullMap* nullmap0, const NullMap* nullmap1) { + const NullMap* nullmap0, const NullMap* nullmap1, + FunctionContext* context) { if (nullmap1 && (*nullmap1)[0]) [[unlikely]] { return; } @@ -792,13 +818,13 @@ struct DateTimeOp { if (nullmap0 && (*nullmap0)[i]) [[unlikely]] { continue; } - vec_to[i] = Transform::execute(vec_from[i], delta); + vec_to[i] = execute(vec_from[i], delta, context); } } static void constant_vector(const ValueType0& from, PaddedPODArray& vec_to, const IntervalColumnType& delta, const NullMap* nullmap0, - const NullMap* nullmap1) { + const NullMap* nullmap1, FunctionContext* context) { if (nullmap0 && (*nullmap0)[0]) [[unlikely]] { return; } @@ -807,7 +833,7 @@ struct DateTimeOp { if (nullmap1 && (*nullmap1)[i]) [[unlikely]] { continue; } - vec_to[i] = Transform::execute(from, get_element(delta, i)); + vec_to[i] = execute(from, get_element(delta, i), context); } } }; @@ -877,11 +903,12 @@ class FunctionTimeDiff : public IFunction { const auto& col1_inside_const = assert_cast(nest_col1_const->get_data_column()); Op::vector_constant(sources->get_data(), res_col->get_data(), - Op::get_element(col1_inside_const, 0), nullmap0, nullmap1); + Op::get_element(col1_inside_const, 0), nullmap0, nullmap1, + context); } else { // vector-vector const auto& concrete_col1 = assert_cast(*nest_col1); Op::vector_vector(sources->get_data(), concrete_col1, res_col->get_data(), nullmap0, - nullmap1); + nullmap1, context); } // update result nullmap with inputs @@ -908,7 +935,7 @@ class FunctionTimeDiff : public IFunction { const ColumnPtr nested_col1 = remove_nullable(col1); const auto& concrete_col1 = assert_cast(*nested_col1); Op::constant_vector(col0_inside_const.get_data()[0], res_col->get_data(), concrete_col1, - nullmap0, nullmap1); + nullmap0, nullmap1, context); // update result nullmap with inputs if (result_nullable) { @@ -1002,11 +1029,12 @@ class FunctionDateOrDateTimeComputation : public IFunction { const auto& col1_inside_const = assert_cast(nest_col1_const->get_data_column()); Op::vector_constant(sources->get_data(), res_col->get_data(), - Op::get_element(col1_inside_const, 0), nullmap0, nullmap1); + Op::get_element(col1_inside_const, 0), nullmap0, nullmap1, + context); } else { // vector-vector const auto& concrete_col1 = assert_cast(*nest_col1); Op::vector_vector(sources->get_data(), concrete_col1, res_col->get_data(), nullmap0, - nullmap1); + nullmap1, context); } // update result nullmap with inputs @@ -1033,7 +1061,7 @@ class FunctionDateOrDateTimeComputation : public IFunction { const ColumnPtr nested_col1 = remove_nullable(col1); const auto& concrete_col1 = assert_cast(*nested_col1); Op::constant_vector(col0_inside_const.get_data()[0], res_col->get_data(), concrete_col1, - nullmap0, nullmap1); + nullmap0, nullmap1, context); // update result nullmap with inputs if (result_nullable) { diff --git a/be/test/exprs/function/function_time_test.cpp b/be/test/exprs/function/function_time_test.cpp index 259675eba2ff04..79fa39c8a797d5 100644 --- a/be/test/exprs/function/function_time_test.cpp +++ b/be/test/exprs/function/function_time_test.cpp @@ -31,6 +31,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamptz.h" #include "core/types.h" #include "core/value/time_value.h" #include "core/value/timestamptz_value.h" @@ -38,12 +39,75 @@ #include "exprs/function/date_time_transforms.h" #include "exprs/function/function_date_or_datetime_computation.h" #include "exprs/function/function_test_util.h" +#include "testutil/datetime_ut_util.h" #include "testutil/mock/mock_runtime_state.h" #include "util/timezone_utils.h" namespace doris { using namespace ut_type; +TEST(VTimestampFunctionsTest, timestamptz_calendar_day_dispatch) { + MockRuntimeState state; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &state._timezone_obj)); + FunctionContext context; + context._state = &state; + const auto spring = make_timestamptz(2024, 3, 9, 17, 0, 0, 123456); + const auto autumn = make_timestamptz(2024, 11, 2, 16, 0, 0, 123456); + using TimestampColumn = ColumnVector; + FunctionDateOrDateTimeComputation> function; + // Exercise vector/vector, vector/constant and constant/vector with nullable inputs. + for (const bool left_const : {false, true}) { + for (const bool right_const : {false, true}) { + // The default IFunction wrapper materializes the constant/constant case. + if (left_const && right_const) { + continue; + } + auto timestamps = TimestampColumn::create(); + timestamps->insert_value(spring); + timestamps->insert_value(autumn); + // An invalid nested timestamp must never be evaluated for a NULL row. + timestamps->insert_value(TimestampTzValue(0)); + auto null_map = ColumnBool::create(3, 0); + null_map->get_data()[2] = 1; + ColumnPtr left = ColumnNullable::create(std::move(timestamps), std::move(null_map)); + auto deltas = ColumnInt32::create(); + deltas->insert_value(1); + deltas->insert_value(-1); + deltas->insert_value(1); + ColumnPtr right = deltas->get_ptr(); + if (left_const) { + left = ColumnConst::create(left->clone_resized(1), 3); + } + if (right_const) { + right = ColumnConst::create(right->clone_resized(1), 3); + } + const auto type = make_nullable(std::make_shared(6)); + Block block {{left, type, "ts"}, + {right, std::make_shared(), "days"}, + {nullptr, type, "result"}}; + ASSERT_TRUE(function.execute_impl(&context, block, {0, 1}, 2, 3).ok()); + const auto& result = + assert_cast(*block.get_by_position(2).column); + const auto& values = assert_cast(result.get_nested_column()); + for (size_t row = 0; row < 3; ++row) { + if (!left_const && row == 2) { + EXPECT_TRUE(result.is_null_at(row)); + continue; + } + auto expected = left_const || row == 0 ? spring : autumn; + ASSERT_TRUE(expected.date_add_days(right_const || row != 1 ? 1 : -1, + state._timezone_obj)); + EXPECT_EQ(values.get_data()[row], expected); + } + } + } + EXPECT_EQ(AddHoursImpl::execute(spring, 24, state._timezone_obj), + make_timestamptz(2024, 3, 10, 17, 0, 0, 123456)); + EXPECT_EQ(SubtractDaysImpl::execute( + make_timestamptz(2024, 3, 10, 16, 0, 0, 123456), 1, state._timezone_obj), + spring); +} + TEST(VTimestampFunctionsTest, iso8601_preserves_negative_subhour_offset) { MockRuntimeState state; state._timezone_obj = cctz::fixed_time_zone(cctz::seconds(-1800)); diff --git a/be/test/runtime/timestamptz_value_test.cpp b/be/test/runtime/timestamptz_value_test.cpp index 35a3e6ec594211..c1bc881a9c1010 100644 --- a/be/test/runtime/timestamptz_value_test.cpp +++ b/be/test/runtime/timestamptz_value_test.cpp @@ -38,6 +38,78 @@ TEST(TimeStampTzValueTest, make_time) { EXPECT_EQ(tz.to_date_int_val(), MIN_DATETIME_V2); } +TEST(TimeStampTzValueTest, AddLocalCalendarDays) { + TimezoneUtils::load_offsets_to_cache(); + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + struct TestCase { + const char* input; + const char* expected; + int days; + int elapsed_seconds; + }; + const TestCase cases[] = { + {.input = "2024-03-09 12:00:00.123456-05:00", + .expected = "2024-03-10 12:00:00.123456-04:00", + .days = 1, + .elapsed_seconds = 82800}, + {.input = "2024-11-02 12:00:00.123456-04:00", + .expected = "2024-11-03 12:00:00.123456-05:00", + .days = 1, + .elapsed_seconds = 90000}, + {.input = "2024-03-10 12:00:00.123456-04:00", + .expected = "2024-03-09 12:00:00.123456-05:00", + .days = -1, + .elapsed_seconds = -82800}, + {.input = "2024-11-03 12:00:00.123456-05:00", + .expected = "2024-11-02 12:00:00.123456-04:00", + .days = -1, + .elapsed_seconds = -90000}, + {.input = "2024-02-01 12:00:00.123456-05:00", + .expected = "2024-02-02 12:00:00.123456-05:00", + .days = 1, + .elapsed_seconds = 86400}, + {.input = "2024-11-03 01:30:00.123456-05:00", + .expected = "2024-11-03 01:30:00.123456-05:00", + .days = 0, + .elapsed_seconds = 0}, + {.input = "2024-11-02 01:30:00.123456-04:00", + .expected = "2024-11-03 01:30:00.123456-04:00", + .days = 1, + .elapsed_seconds = 86400}, + {.input = "2024-11-04 01:30:00.123456-05:00", + .expected = "2024-11-03 01:30:00.123456-05:00", + .days = -1, + .elapsed_seconds = -86400}, + // Use the existing cctz gap policy and preserve the fractional second. + {.input = "2024-03-09 02:30:00.123456-05:00", + .expected = "2024-03-10 03:00:00.123456-04:00", + .days = 1, + .elapsed_seconds = 84600}, + }; + for (const auto& test : cases) { + SCOPED_TRACE(test.input); + CastParameters params; + TimestampTzValue input; + ASSERT_TRUE(input.from_string(StringRef(test.input), &new_york, params, 6)); + auto result = input; + ASSERT_TRUE(result.date_add_days(test.days, new_york)); + EXPECT_EQ(result.to_string(new_york, 6), test.expected); + EXPECT_EQ(result.datetime_diff_in_seconds(input), test.elapsed_seconds); + } + + auto utc = make_timestamptz(2024, 3, 9, 12, 0, 0, 123456); + ASSERT_TRUE(utc.date_add_days(1, cctz::utc_time_zone())); + EXPECT_EQ(utc, make_timestamptz(2024, 3, 10, 12, 0, 0, 123456)); + auto minimum = make_timestamptz(0, 1, 1, 0, 0, 0, 0); + EXPECT_FALSE(minimum.date_add_days(-1, cctz::utc_time_zone())); + auto maximum = make_timestamptz(9999, 12, 31, 12, 0, 0, 0); + EXPECT_FALSE(maximum.date_add_days(1, cctz::utc_time_zone())); + // The local result is valid but its UTC representation would overflow year 9999. + maximum = make_timestamptz(9999, 12, 31, 1, 0, 0, 0); + EXPECT_FALSE(maximum.date_add_days(1, cctz::fixed_time_zone(std::chrono::hours(-8)))); +} + TEST(TimeStampTzValueTest, ToStringPreservesHistoricalOffsetSeconds) { TimezoneUtils::load_offsets_to_cache(); const auto utc = cctz::utc_time_zone(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java index a25f1bb81d4e91..d51dc898fc39b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.functions.DateAddSubMonotonic; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd; @@ -112,6 +113,13 @@ private static boolean couldRearrange(ComparisonPredicate cmp) { return false; } + // Calendar-day addition across DST is not order-preserving or invertible: + // 01:50-04 + 1 day < next-day 01:30-05 is false, but moving the day to the RHS is true. + if (cmp.left() instanceof DateAddSubMonotonic + && !((DateAddSubMonotonic) cmp.left()).isMonotonic(null, null)) { + return false; + } + for (Expression child : cmp.left().children()) { if (child.isConstant()) { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateAddSubMonotonic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateAddSubMonotonic.java index 96265850456387..ff873e58ee8f18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateAddSubMonotonic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateAddSubMonotonic.java @@ -17,12 +17,23 @@ package org.apache.doris.nereids.trees.expressions.functions; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.types.TimeStampTzType; +import org.apache.doris.nereids.util.DateUtils; /** monotonicity for XX_ADD XX_SUB */ public interface DateAddSubMonotonic extends Monotonic { @Override default boolean isMonotonic(Literal lower, Literal upper) { + // At a fall-back, 01:50-04 precedes 01:10-05, but adding one local day + // reverses their UTC order. Fixed-offset sessions retain the original guarantee. + if ((this instanceof DaysAdd || this instanceof DaysSub) + && child(0).getDataType() instanceof TimeStampTzType + && !DateUtils.getTimeZone().getRules().isFixedOffset()) { + return false; + } return MonotonicityUtils.hasConstantOtherArguments(this, 0); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java index b7c89b415a682d..9cebd71b5d68e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.literal; import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.common.util.TimeZoneParser; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.exceptions.UnboundException; @@ -36,6 +37,9 @@ import org.apache.logging.log4j.Logger; import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Objects; /** @@ -192,8 +196,24 @@ dtV2Lit, new StringLiteral("UTC"), throw new AnalysisException(String.format("Cast from %s to %s not supported", this, targetType)); } + /** Add calendar days in the session timezone, retaining the local time of day. */ public Expression plusDays(long days) { - return fromJavaDateType(toJavaDateType().plusDays(days), getDataType().getScale()); + ZoneId zone = TimeZoneParser.parse(getSessionTimeZone(), TimeZoneParser.Usage.SESSION); + ZonedDateTime local = toJavaDateType().atOffset(ZoneOffset.UTC).atZoneSameInstant(zone); + LocalDateTime target = local.toLocalDateTime().plusDays(days); + // Like BE, reject local years outside [0, 9999], even when UTC is representable. + if (isDateOutOfRange(local.toLocalDateTime()) || isDateOutOfRange(target)) { + throw new AnalysisException("datetime out of range: " + target); + } + // Keep the original offset when the target repeats, including adding zero days + // to the second 01:30 during the New York autumn transition. + if (zone.getRules().getValidOffsets(target).contains(local.getOffset())) { + return fromJavaDateType(LocalDateTime.ofInstant( + target.toInstant(local.getOffset()), ZoneOffset.UTC), getScale()); + } + // Match BE cctz's gap policy (the transition instant), retaining microseconds. + return fromJavaDateType(LocalDateTime.ofInstant( + convertLocalToInstantPreservingFraction(target, zone), ZoneOffset.UTC), getScale()); } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java index f2f8dfcf555f86..821d3382498ee0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java @@ -20,7 +20,17 @@ import org.apache.doris.nereids.rules.expression.ExpressionRewrite; import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper; import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; +import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; +import org.apache.doris.nereids.types.TimeStampTzType; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -29,6 +39,38 @@ class SimplifyArithmeticComparisonRuleTest extends ExpressionRewriteTestHelper { + @Test + void testTimestampTzCalendarDaysCannotRearrangeAcrossDst() { + ConnectContext previous = ConnectContext.get(); + ConnectContext connectContext = new ConnectContext(); + connectContext.setThreadLocalInfo(); + try { + SlotReference timestamp = new SlotReference("ts", TimeStampTzType.of(6)); + TimestampTzLiteral bound = new TimestampTzLiteral(TimeStampTzType.of(6), + "2024-11-04 01:30:00-05:00"); + IntegerLiteral one = new IntegerLiteral(1); + connectContext.getSessionVariable().setTimeZone("America/New_York"); + for (Expression arithmetic : new Expression[] {new DaysAdd(timestamp, one), + new DaysSub(timestamp, one)}) { + ComparisonPredicate predicate = new LessThan(arithmetic, bound); + Assertions.assertEquals(predicate, SimplifyArithmeticComparisonRule.simplify(predicate, context)); + for (String zone : new String[] {"UTC", "+08:00"}) { + connectContext.getSessionVariable().setTimeZone(zone); + Assertions.assertNotEquals(predicate, + SimplifyArithmeticComparisonRule.simplify(predicate, context)); + } + connectContext.getSessionVariable().setTimeZone("America/New_York"); + } + ComparisonPredicate hours = new LessThan(new HoursAdd(timestamp, one), bound); + Assertions.assertNotEquals(hours, SimplifyArithmeticComparisonRule.simplify(hours, context)); + } finally { + ConnectContext.remove(); + if (previous != null) { + previous.setThreadLocalInfo(); + } + } + } + @Test public void testNumeric() { executor = new ExpressionRuleExecutor(ImmutableList.of( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java index ac5235fdf34500..9d54d1c1436fcc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java @@ -17,15 +17,109 @@ package org.apache.doris.nereids.trees.expressions.literal; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; +import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd; +import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.qe.ConnectContext; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.time.Duration; + class TimestampTzLiteralTest { + @Test + void testLocalCalendarDayArithmetic() { + ConnectContext previous = ConnectContext.get(); + ConnectContext context = new ConnectContext(); + context.getSessionVariable().setTimeZone("America/New_York"); + context.setThreadLocalInfo(); + try { + String[][] cases = { + {"2024-03-09 12:00:00.123456-05:00", "1", "2024-03-10 12:00:00.123456-04:00", "82800"}, + {"2024-11-02 12:00:00.123456-04:00", "1", "2024-11-03 12:00:00.123456-05:00", "90000"}, + {"2024-03-10 12:00:00.123456-04:00", "-1", "2024-03-09 12:00:00.123456-05:00", "-82800"}, + {"2024-11-03 12:00:00.123456-05:00", "-1", "2024-11-02 12:00:00.123456-04:00", "-90000"}, + {"2024-02-01 12:00:00.123456-05:00", "1", "2024-02-02 12:00:00.123456-05:00", "86400"}, + {"2024-11-03 01:30:00.123456-05:00", "0", "2024-11-03 01:30:00.123456-05:00", "0"}, + {"2024-11-02 01:30:00.123456-04:00", "1", "2024-11-03 01:30:00.123456-04:00", "86400"}, + {"2024-11-04 01:30:00.123456-05:00", "-1", "2024-11-03 01:30:00.123456-05:00", "-86400"}, + {"2024-03-09 02:30:00.123456-05:00", "1", "2024-03-10 03:00:00.123456-04:00", "84600"} + }; + for (String[] testCase : cases) { + TimestampTzLiteral input = new TimestampTzLiteral(TimeStampTzType.of(6), testCase[0]); + IntegerLiteral days = new IntegerLiteral(Integer.parseInt(testCase[1])); + TimestampTzLiteral expected = new TimestampTzLiteral(TimeStampTzType.of(6), testCase[2]); + Expression result = DateTimeArithmetic.daysAdd(input, days); + Assertions.assertEquals(expected, result, testCase[0]); + Assertions.assertEquals(expected, DateTimeArithmetic.dateAdd(input, days)); + Assertions.assertEquals(expected, DateTimeArithmetic.daysSub(input, + new IntegerLiteral(-days.getValue()))); + Assertions.assertEquals(Long.parseLong(testCase[3]), Duration.between( + input.toJavaDateType(), ((TimestampTzLiteral) result).toJavaDateType()).getSeconds()); + } + TimestampTzLiteral input = new TimestampTzLiteral(TimeStampTzType.of(6), cases[0][0]); + Assertions.assertEquals(new TimestampTzLiteral(TimeStampTzType.of(6), + "2024-03-10 13:00:00.123456-04:00"), + DateTimeArithmetic.hoursAdd(input, new IntegerLiteral(24))); + context.getSessionVariable().setTimeZone("UTC"); + Assertions.assertEquals(new TimestampTzLiteral(TimeStampTzType.of(6), + "2024-03-10 17:00:00.123456+00:00"), + DateTimeArithmetic.daysAdd(input, new IntegerLiteral(1))); + TimestampTzLiteral minimum = new TimestampTzLiteral(0, 1, 1, 0, 0, 0); + TimestampTzLiteral maximum = new TimestampTzLiteral(9999, 12, 31, 1, 0, 0); + Assertions.assertThrows(AnalysisException.class, () -> minimum.plusDays(-1)); + Assertions.assertThrows(AnalysisException.class, () -> maximum.plusDays(1)); + context.getSessionVariable().setTimeZone("-08:00"); + Assertions.assertThrows(AnalysisException.class, () -> maximum.plusDays(1)); + } finally { + ConnectContext.remove(); + if (previous != null) { + previous.setThreadLocalInfo(); + } + } + } + + @Test + void testCalendarDayMonotonicity() { + ConnectContext previous = ConnectContext.get(); + ConnectContext context = new ConnectContext(); + context.setThreadLocalInfo(); + try { + SlotReference timestamp = new SlotReference("ts", TimeStampTzType.of(6)); + TimestampTzLiteral lower = new TimestampTzLiteral("2024-11-03 01:50:00-04:00"); + TimestampTzLiteral upper = new TimestampTzLiteral("2024-11-03 01:10:00-05:00"); + DaysAdd add = new DaysAdd(timestamp, new IntegerLiteral(1)); + DaysSub sub = new DaysSub(timestamp, new IntegerLiteral(1)); + context.getSessionVariable().setTimeZone("America/New_York"); + Assertions.assertFalse(add.isMonotonic(lower, upper)); + Assertions.assertFalse(sub.isMonotonic(lower, upper)); + Assertions.assertTrue(lower.toJavaDateType().isBefore(upper.toJavaDateType())); + Assertions.assertTrue(((TimestampTzLiteral) lower.plusDays(1)).toJavaDateType().isAfter( + ((TimestampTzLiteral) upper.plusDays(1)).toJavaDateType())); + Assertions.assertTrue(new HoursAdd(timestamp, new IntegerLiteral(24)).isMonotonic(lower, upper)); + SlotReference datetime = new SlotReference("dt", DateTimeV2Type.of(6)); + Assertions.assertTrue(new DaysAdd(datetime, new IntegerLiteral(1)).isMonotonic(lower, upper)); + for (String zone : new String[] {"UTC", "+08:00"}) { + context.getSessionVariable().setTimeZone(zone); + Assertions.assertTrue(add.isMonotonic(lower, upper)); + Assertions.assertTrue(sub.isMonotonic(lower, upper)); + } + } finally { + ConnectContext.remove(); + if (previous != null) { + previous.setThreadLocalInfo(); + } + } + } + @Test void testConstructorsAndParsing() { TimestampTzLiteral literal; diff --git a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out new file mode 100644 index 00000000000000..1c85b99bbd759f --- /dev/null +++ b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out @@ -0,0 +1,67 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !fe_literal -- +2024-03-09 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 2024-03-10 13:00:00.123456-04:00 82800 2024-03-09 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 + +-- !fe_literal -- +2024-11-02 12:00:00.123456-04:00 2024-11-03 12:00:00.123456-05:00 2024-11-03 11:00:00.123456-05:00 90000 2024-11-02 12:00:00.123456-04:00 2024-11-03 12:00:00.123456-05:00 2024-11-03 12:00:00.123456-05:00 + +-- !fe_literal -- +2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 86400 2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 + +-- !fe_overlap_gap -- +2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-05:00 2024-03-10 03:00:00.123456-04:00 + +-- !fe_columns -- +1 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 82800 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 +10 \N \N \N \N 2024-03-10 12:00:00.123456-04:00 +11 \N \N \N 2024-03-10 12:00:00.123456-04:00 \N +12 2024-11-04 01:50:00.123456-05:00 2024-11-04 01:50:00.123456-05:00 90000 2024-11-04 01:50:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +13 2024-11-04 01:10:00.123456-05:00 2024-11-04 01:10:00.123456-05:00 86400 2024-11-04 01:10:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +2 2024-11-03 12:00:00.123456-05:00 2024-11-03 12:00:00.123456-05:00 90000 2024-11-03 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +3 2024-03-09 12:00:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 -82800 2024-03-11 12:00:00.123456-04:00 2024-03-08 12:00:00.123456-05:00 +4 2024-11-02 12:00:00.123456-04:00 2024-11-02 12:00:00.123456-04:00 -90000 2024-11-04 12:00:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 +5 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 86400 2024-02-02 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +6 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 0 2024-11-04 01:30:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 +7 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-04:00 86400 2024-11-03 01:30:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 +8 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 -86400 2024-11-05 01:30:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 +9 2024-03-10 03:00:00.123456-04:00 2024-03-10 03:00:00.123456-04:00 84600 2024-03-10 03:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 + +-- !fe_non_monotonic -- +13 2024-11-04 01:10:00.123456-05:00 + +-- !fe_utc -- +2024-03-10 17:00:00.123456+00:00 2024-03-10 17:00:00.123456+00:00 86400 + +-- !be_literal -- +2024-03-09 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 2024-03-10 13:00:00.123456-04:00 82800 2024-03-09 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 + +-- !be_literal -- +2024-11-02 12:00:00.123456-04:00 2024-11-03 12:00:00.123456-05:00 2024-11-03 11:00:00.123456-05:00 90000 2024-11-02 12:00:00.123456-04:00 2024-11-03 12:00:00.123456-05:00 2024-11-03 12:00:00.123456-05:00 + +-- !be_literal -- +2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 86400 2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 + +-- !be_overlap_gap -- +2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-05:00 2024-03-10 03:00:00.123456-04:00 + +-- !be_columns -- +1 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 82800 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 +10 \N \N \N \N 2024-03-10 12:00:00.123456-04:00 +11 \N \N \N 2024-03-10 12:00:00.123456-04:00 \N +12 2024-11-04 01:50:00.123456-05:00 2024-11-04 01:50:00.123456-05:00 90000 2024-11-04 01:50:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +13 2024-11-04 01:10:00.123456-05:00 2024-11-04 01:10:00.123456-05:00 86400 2024-11-04 01:10:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +2 2024-11-03 12:00:00.123456-05:00 2024-11-03 12:00:00.123456-05:00 90000 2024-11-03 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +3 2024-03-09 12:00:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 -82800 2024-03-11 12:00:00.123456-04:00 2024-03-08 12:00:00.123456-05:00 +4 2024-11-02 12:00:00.123456-04:00 2024-11-02 12:00:00.123456-04:00 -90000 2024-11-04 12:00:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 +5 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 86400 2024-02-02 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +6 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 0 2024-11-04 01:30:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 +7 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-04:00 86400 2024-11-03 01:30:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 +8 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 -86400 2024-11-05 01:30:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 +9 2024-03-10 03:00:00.123456-04:00 2024-03-10 03:00:00.123456-04:00 84600 2024-03-10 03:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 + +-- !be_non_monotonic -- +13 2024-11-04 01:10:00.123456-05:00 + +-- !be_utc -- +2024-03-10 17:00:00.123456+00:00 2024-03-10 17:00:00.123456+00:00 86400 + diff --git a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy new file mode 100644 index 00000000000000..7fec300f616235 --- /dev/null +++ b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamptz_days_add_dst") { + sql "SET time_zone = 'America/New_York'" + sql "DROP TABLE IF EXISTS timestamptz_days_add_dst" + sql """ + CREATE TABLE timestamptz_days_add_dst (id INT, ts TIMESTAMPTZ(6), delta INT) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + INSERT INTO timestamptz_days_add_dst VALUES + (1, '2024-03-09 12:00:00.123456-05:00', 1), + (2, '2024-11-02 12:00:00.123456-04:00', 1), + (3, '2024-03-10 12:00:00.123456-04:00', -1), + (4, '2024-11-03 12:00:00.123456-05:00', -1), + (5, '2024-02-01 12:00:00.123456-05:00', 1), + (6, '2024-11-03 01:30:00.123456-05:00', 0), + (7, '2024-11-02 01:30:00.123456-04:00', 1), + (8, '2024-11-04 01:30:00.123456-05:00', -1), + (9, '2024-03-09 02:30:00.123456-05:00', 1), + (10, NULL, 1), + (11, '2024-03-09 12:00:00.123456-05:00', NULL), + (12, '2024-11-03 01:50:00.123456-04:00', 1), + (13, '2024-11-03 01:10:00.123456-05:00', 1) + """ + try { + for (def skipFold : [false, true]) { + sql "SET debug_skip_fold_constant = ${skipFold}" + def tag = skipFold ? "be" : "fe" + // Literal expressions exercise FE constant folding and BE constant evaluation separately. + for (def input : ["2024-03-09 12:00:00.123456-05:00", + "2024-11-02 12:00:00.123456-04:00", + "2024-02-01 12:00:00.123456-05:00"]) { + "order_qt_${tag}_literal"(""" + SELECT CAST(ts AS STRING), CAST(days_add(ts, 1) AS STRING), + CAST(hours_add(ts, 24) AS STRING), seconds_diff(days_add(ts, 1), ts), + CAST(days_sub(days_add(ts, 1), 1) AS STRING), + CAST(date_add(ts, INTERVAL 1 DAY) AS STRING), + CAST(date_sub(ts, INTERVAL -1 DAY) AS STRING) + FROM (SELECT CAST('${input}' AS TIMESTAMPTZ(6)) AS ts) t + """) + } + "order_qt_${tag}_overlap_gap"(""" + SELECT CAST(days_add(CAST('2024-11-03 01:30:00.123456-05:00' AS TIMESTAMPTZ(6)), 0) + AS STRING), + CAST(days_add(CAST('2024-11-02 01:30:00.123456-04:00' AS TIMESTAMPTZ(6)), 1) + AS STRING), + CAST(days_sub(CAST('2024-11-04 01:30:00.123456-05:00' AS TIMESTAMPTZ(6)), 1) + AS STRING), + CAST(days_add(CAST('2024-03-09 02:30:00.123456-05:00' AS TIMESTAMPTZ(6)), 1) + AS STRING) + """) + // Cover vector/vector, vector/constant, constant/vector, and NULL propagation. + "order_qt_${tag}_columns"(""" + SELECT id, CAST(days_add(ts, delta) AS STRING), CAST(days_sub(ts, -delta) AS STRING), + seconds_diff(days_add(ts, delta), ts), CAST(days_add(ts, 1) AS STRING), + CAST(days_add(CAST('2024-03-09 12:00:00.123456-05:00' AS TIMESTAMPTZ(6)), delta) + AS STRING) + FROM timestamptz_days_add_dst ORDER BY id + """) + "order_qt_${tag}_non_monotonic"(""" + SELECT id, CAST(days_add(ts, 1) AS STRING) + FROM timestamptz_days_add_dst + WHERE id IN (12, 13) + AND days_add(ts, 1) < CAST('2024-11-04 01:30:00-05:00' AS TIMESTAMPTZ(6)) + ORDER BY id + """) + sql "SET time_zone = 'UTC'" + "order_qt_${tag}_utc"(""" + SELECT CAST(days_add(ts, 1) AS STRING), CAST(hours_add(ts, 24) AS STRING), + seconds_diff(days_add(ts, 1), ts) + FROM (SELECT CAST('2024-03-09 12:00:00.123456-05:00' AS TIMESTAMPTZ(6)) AS ts) t + """) + sql "SET time_zone = 'America/New_York'" + } + } finally { + sql "SET debug_skip_fold_constant = false" + sql "SET time_zone = default" + } +} From 4e4a662c1355be835d5a4e6437bb3e9847d48ed8 Mon Sep 17 00:00:00 2001 From: jacktengg Date: Tue, 3 Nov 2026 02:06:00 +0800 Subject: [PATCH 2/8] [fix](be) Handle TIMESTAMPTZ day arithmetic at UTC boundaries ### What problem does this PR solve? Issue Number: None Related PR: #70003 Problem Summary: The session-local intermediate of a valid UTC TIMESTAMPTZ can be February 29 in year 0. DateV2 rejects that date before applying a DAY interval, although FE uses a civil calendar and can produce valid UTC output. Use the same civil calendar in BE and validate the final UTC value. For example, UTC 0000-02-28 20:00:00 at +08:00 minus one day now consistently yields UTC 0000-02-27 20:00:00. A zero-day interval must also remain an infallible identity for valid UTC boundary values. Resolve zero before local-year validation in both FE and BE so existing predicate movement cannot introduce errors on rows excluded by a join. Add direct boundary unit tests, year-0 projection regressions, and a zero-day join regression with runtime filters disabled. ### Release note Keep TIMESTAMPTZ day arithmetic consistent between FE and BE at year-0 calendar boundaries and preserve zero-day identity at valid UTC boundaries. ### Check List (For Author) - Test: Regression test / Unit Test - Unit Test: 11 selected BE tests and 20 FE tests passed using run-be-ut.sh and run-fe-ut.sh. - Regression test: test_timestamptz_days_add_dst, test_timestamptz_dst_fold, test_timestamptz_dst_gap, test_timestamptz_utc_functions; all 4 suites passed. Expected output generated by the regression script; 10 FE/BE groups match. - ASAN BE build, FE build with Checkstyle, clang-format 16, build hygiene and clang-tidy passed. - Behavior changed: Yes. Valid UTC boundaries retain zero-day identity; civil calendar intermediate dates no longer cause FE/BE divergence. - Does this need documentation: No. Boundary correctness fixes for existing date arithmetic. --- be/src/core/value/timestamptz_value.cpp | 31 ++++++++---- be/test/runtime/timestamptz_value_test.cpp | 23 +++++++++ .../literal/TimestampTzLiteral.java | 9 +++- .../literal/TimestampTzLiteralTest.java | 29 ++++++++++++ .../test_timestamptz_days_add_dst.out | 18 +++++++ .../test_timestamptz_days_add_dst.groovy | 47 +++++++++++++++++++ 6 files changed, 145 insertions(+), 12 deletions(-) diff --git a/be/src/core/value/timestamptz_value.cpp b/be/src/core/value/timestamptz_value.cpp index 3e473a75e7daa7..a0967861714bf3 100644 --- a/be/src/core/value/timestamptz_value.cpp +++ b/be/src/core/value/timestamptz_value.cpp @@ -89,19 +89,30 @@ int TimestampTzValue::to_buffer(char* buffer, const cctz::time_zone& tz, int sca } bool TimestampTzValue::date_add_days(int64_t days, const cctz::time_zone& local_time_zone) { - DateV2Value local_dt; - // UTC year 0000/9999 can become an unrepresentable local year after zone conversion. - if (!to_datetime(local_dt, local_time_zone, 6, 6)) { - return false; + // Predicate movement treats a zero interval as an infallible identity, including + // UTC year 0000/9999 whose local display year can be outside [0, 9999]. + if (days == 0) { + return true; } - // For example, adding zero days to the second 01:30 at the autumn transition must - // retain that instant, rather than resolving to the first occurrence of 01:30. - const auto preferred_offset = static_cast(local_dt.datetime_diff_in_seconds(_utc_dt)); - // A day delta can move an otherwise valid local date beyond years [0, 9999]. - if (!local_dt.date_add_days(days)) { + const cctz::civil_second utc_cs(year(), month(), day(), hour(), minute(), second()); + const auto local = local_time_zone.lookup(cctz::convert(utc_cs, cctz::utc_time_zone())); + const auto target = cctz::civil_day(local.cs) + days; + // UTC boundaries and a day delta can produce an unrepresentable local year. + if (local.cs.year() < 0 || local.cs.year() > 9999 || target.year() < 0 || + target.year() > 9999) { return false; } - convert_local_to_utc(local_time_zone, local_dt, preferred_offset); + // Match Java's civil calendar, which permits February 29 in year 0. For example, + // UTC 0000-02-28 20:00 at +08:00 passes through that local date before subtracting + // a day. Validate the final UTC value rather than this local intermediate as DateV2. + DateV2Value local_dt; + local_dt.unchecked_set_time( + static_cast(target.year()), static_cast(target.month()), + static_cast(target.day()), static_cast(local.cs.hour()), + static_cast(local.cs.minute()), static_cast(local.cs.second()), + microsecond()); + // Preserve the input offset when the target repeats, e.g. the second autumn 01:30. + convert_local_to_utc(local_time_zone, local_dt, local.offset); // A valid local result can still fall outside the supported UTC year range. return is_valid_date(); } diff --git a/be/test/runtime/timestamptz_value_test.cpp b/be/test/runtime/timestamptz_value_test.cpp index c1bc881a9c1010..f0e63000f9b673 100644 --- a/be/test/runtime/timestamptz_value_test.cpp +++ b/be/test/runtime/timestamptz_value_test.cpp @@ -110,6 +110,29 @@ TEST(TimeStampTzValueTest, AddLocalCalendarDays) { EXPECT_FALSE(maximum.date_add_days(1, cctz::fixed_time_zone(std::chrono::hours(-8)))); } +TEST(TimeStampTzValueTest, AddLocalCalendarDaysAtUtcBoundaries) { + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + const auto east = cctz::fixed_time_zone(std::chrono::hours(8)); + const auto west = cctz::fixed_time_zone(std::chrono::hours(-8)); + const auto minimum = make_timestamptz(0, 1, 1, 0, 0, 0, 123456); + const auto maximum = make_timestamptz(9999, 12, 31, 23, 0, 0, 123456); + auto result = minimum; + ASSERT_TRUE(result.date_add_days(0, new_york)); + EXPECT_EQ(result, minimum); + result = maximum; + ASSERT_TRUE(result.date_add_days(0, east)); + EXPECT_EQ(result, maximum); + + // Both conversions pass through local February 29, year 0, and finish in valid UTC. + result = make_timestamptz(0, 2, 28, 20, 0, 0, 123456); + ASSERT_TRUE(result.date_add_days(-1, east)); + EXPECT_EQ(result, make_timestamptz(0, 2, 27, 20, 0, 0, 123456)); + result = make_timestamptz(0, 3, 1, 1, 0, 0, 123456); + ASSERT_TRUE(result.date_add_days(1, west)); + EXPECT_EQ(result, make_timestamptz(0, 3, 2, 1, 0, 0, 123456)); +} + TEST(TimeStampTzValueTest, ToStringPreservesHistoricalOffsetSeconds) { TimezoneUtils::load_offsets_to_cache(); const auto utc = cctz::utc_time_zone(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java index 9cebd71b5d68e7..a980870132cdbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java @@ -198,6 +198,11 @@ dtV2Lit, new StringLiteral("UTC"), /** Add calendar days in the session timezone, retaining the local time of day. */ public Expression plusDays(long days) { + // Predicate movement relies on zero days being an identity even when the local + // display year of a valid UTC boundary value is outside [0, 9999]. + if (days == 0) { + return this; + } ZoneId zone = TimeZoneParser.parse(getSessionTimeZone(), TimeZoneParser.Usage.SESSION); ZonedDateTime local = toJavaDateType().atOffset(ZoneOffset.UTC).atZoneSameInstant(zone); LocalDateTime target = local.toLocalDateTime().plusDays(days); @@ -205,8 +210,8 @@ public Expression plusDays(long days) { if (isDateOutOfRange(local.toLocalDateTime()) || isDateOutOfRange(target)) { throw new AnalysisException("datetime out of range: " + target); } - // Keep the original offset when the target repeats, including adding zero days - // to the second 01:30 during the New York autumn transition. + // Keep the original offset when the target repeats: November 4 01:30-05 minus + // one day selects the second occurrence of November 3 01:30 in New York. if (zone.getRules().getValidOffsets(target).contains(local.getOffset())) { return fromJavaDateType(LocalDateTime.ofInstant( target.toInstant(local.getOffset()), ZoneOffset.UTC), getScale()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java index 9d54d1c1436fcc..14a62791fe3c17 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java @@ -120,6 +120,35 @@ void testCalendarDayMonotonicity() { } } + @Test + void testCalendarDaysAtUtcBoundaries() { + ConnectContext previous = ConnectContext.get(); + ConnectContext context = new ConnectContext(); + context.setThreadLocalInfo(); + try { + TimestampTzLiteral minimum = new TimestampTzLiteral(0, 1, 1, 0, 0, 0); + TimestampTzLiteral maximum = new TimestampTzLiteral(9999, 12, 31, 23, 0, 0); + context.getSessionVariable().setTimeZone("America/New_York"); + Assertions.assertEquals(minimum, minimum.plusDays(0)); + context.getSessionVariable().setTimeZone("+08:00"); + Assertions.assertEquals(maximum, maximum.plusDays(0)); + TimestampTzLiteral east = new TimestampTzLiteral(TimeStampTzType.of(6), + 0, 2, 28, 20, 0, 0, 123456); + Assertions.assertEquals(new TimestampTzLiteral(TimeStampTzType.of(6), + 0, 2, 27, 20, 0, 0, 123456), east.plusDays(-1)); + context.getSessionVariable().setTimeZone("-08:00"); + TimestampTzLiteral west = new TimestampTzLiteral(TimeStampTzType.of(6), + 0, 3, 1, 1, 0, 0, 123456); + Assertions.assertEquals(new TimestampTzLiteral(TimeStampTzType.of(6), + 0, 3, 2, 1, 0, 0, 123456), west.plusDays(1)); + } finally { + ConnectContext.remove(); + if (previous != null) { + previous.setThreadLocalInfo(); + } + } + } + @Test void testConstructorsAndParsing() { TimestampTzLiteral literal; diff --git a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out index 1c85b99bbd759f..3c0bde240a0f6f 100644 --- a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out +++ b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out @@ -29,6 +29,15 @@ -- !fe_non_monotonic -- 13 2024-11-04 01:10:00.123456-05:00 +-- !fe_zero_join -- +1 + +-- !fe_year_zero_east -- +0000-02-28 04:00:00.123456+08:00 0000-02-28 04:00:00.123456+08:00 true + +-- !fe_year_zero_west -- +0000-03-01 17:00:00.123456-08:00 0000-03-01 17:00:00.123456-08:00 + -- !fe_utc -- 2024-03-10 17:00:00.123456+00:00 2024-03-10 17:00:00.123456+00:00 86400 @@ -62,6 +71,15 @@ -- !be_non_monotonic -- 13 2024-11-04 01:10:00.123456-05:00 +-- !be_zero_join -- +1 + +-- !be_year_zero_east -- +0000-02-28 04:00:00.123456+08:00 0000-02-28 04:00:00.123456+08:00 true + +-- !be_year_zero_west -- +0000-03-01 17:00:00.123456-08:00 0000-03-01 17:00:00.123456-08:00 + -- !be_utc -- 2024-03-10 17:00:00.123456+00:00 2024-03-10 17:00:00.123456+00:00 86400 diff --git a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy index 7fec300f616235..ff742edc31800d 100644 --- a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy +++ b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy @@ -40,6 +40,26 @@ suite("test_timestamptz_days_add_dst") { (12, '2024-11-03 01:50:00.123456-04:00', 1), (13, '2024-11-03 01:10:00.123456-05:00', 1) """ + sql "DROP TABLE IF EXISTS timestamptz_days_zero_events" + sql "DROP TABLE IF EXISTS timestamptz_days_zero_ids" + sql """ + CREATE TABLE timestamptz_days_zero_events (id INT, ts TIMESTAMPTZ(6)) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + CREATE TABLE timestamptz_days_zero_ids (id INT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + INSERT INTO timestamptz_days_zero_events VALUES + (1, '2024-01-01 00:00:00+00:00'), + (2, '0000-01-01 00:00:00+00:00'), + (3, '0000-02-28 20:00:00.123456+00:00'), + (4, '0000-03-01 01:00:00.123456+00:00') + """ + sql "INSERT INTO timestamptz_days_zero_ids VALUES (1)" try { for (def skipFold : [false, true]) { sql "SET debug_skip_fold_constant = ${skipFold}" @@ -82,6 +102,32 @@ suite("test_timestamptz_days_add_dst") { AND days_add(ts, 1) < CAST('2024-11-04 01:30:00-05:00' AS TIMESTAMPTZ(6)) ORDER BY id """) + // Zero days must remain safe to push before the join, even for a boundary row + // whose local display year is -1. Return only ids to avoid formatting that row. + sql "SET runtime_filter_mode = 'OFF'" + "order_qt_${tag}_zero_join"(""" + SELECT id FROM ( + SELECT e.id, days_add(e.ts, 0) AS shifted + FROM timestamptz_days_zero_events e JOIN timestamptz_days_zero_ids i ON e.id = i.id + ) s WHERE shifted >= CAST('2000-01-01 00:00:00+00:00' AS TIMESTAMPTZ(6)) ORDER BY id + """) + sql "SET runtime_filter_mode = default" + sql "SET time_zone = '+08:00'" + "order_qt_${tag}_year_zero_east"(""" + SELECT CAST(days_add(ts, -1) AS STRING), + CAST(days_sub(CAST('0000-02-28 20:00:00.123456+00:00' AS TIMESTAMPTZ(6)), 1) + AS STRING), + days_add(CAST('9999-12-31 23:00:00+00:00' AS TIMESTAMPTZ(6)), 0) + = CAST('9999-12-31 23:00:00+00:00' AS TIMESTAMPTZ(6)) + FROM timestamptz_days_zero_events WHERE id = 3 + """) + sql "SET time_zone = '-08:00'" + "order_qt_${tag}_year_zero_west"(""" + SELECT CAST(days_add(ts, 1) AS STRING), + CAST(days_add(CAST('0000-03-01 01:00:00.123456+00:00' AS TIMESTAMPTZ(6)), 1) + AS STRING) + FROM timestamptz_days_zero_events WHERE id = 4 + """) sql "SET time_zone = 'UTC'" "order_qt_${tag}_utc"(""" SELECT CAST(days_add(ts, 1) AS STRING), CAST(hours_add(ts, 24) AS STRING), @@ -92,6 +138,7 @@ suite("test_timestamptz_days_add_dst") { } } finally { sql "SET debug_skip_fold_constant = false" + sql "SET runtime_filter_mode = default" sql "SET time_zone = default" } } From a42bef57381636e4b9e76d67353131347425286e Mon Sep 17 00:00:00 2001 From: jacktengg Date: Wed, 4 Nov 2026 18:26:52 +0800 Subject: [PATCH 3/8] [doc](fe) Explain the guard for date comparison rewrites ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: The date comparison comment omits the session timezone, complete dates, and the reason for passing null range bounds to isMonotonic. Describe the ordering requirement and show how the New York autumn transition changes the comparison result after interval rearrangement. Use short descriptive sentences and consistent technical terms. ### Release note None ### Check List (For Author) - Test: No need to run functional tests because only comments changed. - Targeted Maven Checkstyle passed with zero violations. - Java token comparison confirmed that executable code is unchanged. - Verified the example's UTC conversions and opposite Boolean results. - Comment sentences have at most 18 words; all lines fit the 120-character limit. - Behavior changed: No - Does this need documentation: No. This change updates the source comment. --- .../rules/SimplifyArithmeticComparisonRule.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java index d51dc898fc39b9..861e61a521e0da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java @@ -113,8 +113,18 @@ private static boolean couldRearrange(ComparisonPredicate cmp) { return false; } - // Calendar-day addition across DST is not order-preserving or invertible: - // 01:50-04 + 1 day < next-day 01:30-05 is false, but moving the day to the RHS is true. + // The rule can move a date interval only if the date function keeps the order of input values. + // It calls isMonotonic(null, null) because it has no input range bounds. + // Calendar day addition can change UTC time order for TIMESTAMPTZ values when the UTC offset changes. + // + // Example in America/New_York: + // ts = 2024-11-03 01:50:00-04:00 + // cutoff = 2024-11-04 01:30:00-05:00 + // days_add(ts, 1) < cutoff is false. + // ts < days_sub(cutoff, 1) is true. + // The subtraction selects the second 01:30 on November 3, with UTC offset -05:00. + // + // The check keeps the original comparison if isMonotonic returns false. if (cmp.left() instanceof DateAddSubMonotonic && !((DateAddSubMonotonic) cmp.left()).isMonotonic(null, null)) { return false; From 2a70c4a28950995361e7897ab884d34c8d416cc1 Mon Sep 17 00:00:00 2001 From: jacktengg Date: Tue, 10 Nov 2026 22:21:30 +0800 Subject: [PATCH 4/8] [improvement](be) Reuse the civil lookup in TIMESTAMPTZ day arithmetic ### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: Each nonzero TIMESTAMPTZ DAY operation calls the local-to-UTC conversion overload with a preferred offset. This overload queried the civil time and then called cctz::convert, which repeated the same lookup. Initialize the time point from the existing civil_lookup, using trans for SKIPPED times and pre otherwise, to reduce these civil-time lookups from two to one per conversion. Keep the REPEATED preferred-offset selection and fractional seconds unchanged. This preserves gap and overlap semantics. No end-to-end SQL performance improvement is claimed. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter='TimeStampTzValueTest.*' -j64 (ASAN; all 9 tests passed) - build-support/clang-format.sh and build-support/check-format.sh (clang-format 16) - build-support/run-clang-tidy.sh --build-dir be/ut_build_ASAN (passed; toolchain resource directory supplied) - git diff --check - Behavior changed: No - Does this need documentation: No --- be/src/core/value/timestamptz_value.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/core/value/timestamptz_value.cpp b/be/src/core/value/timestamptz_value.cpp index a0967861714bf3..f1cec3d6514df9 100644 --- a/be/src/core/value/timestamptz_value.cpp +++ b/be/src/core/value/timestamptz_value.cpp @@ -197,7 +197,9 @@ void TimestampTzValue::convert_local_to_utc(const cctz::time_zone& local_time_zo cctz::civil_second local_cs(dt.year(), dt.month(), dt.day(), dt.hour(), dt.minute(), dt.second()); const auto lookup = local_time_zone.lookup(local_cs); - cctz::time_point local_tp = cctz::convert(local_cs, local_time_zone); + // Match cctz::convert's gap policy without repeating the civil-time lookup. + auto local_tp = + lookup.kind == cctz::time_zone::civil_lookup::SKIPPED ? lookup.trans : lookup.pre; if (lookup.kind == cctz::time_zone::civil_lookup::REPEATED) { const auto pre_offset = local_time_zone.lookup(lookup.pre).offset; From 633f33ad39fb1512f6bac6ef00fa66c2804a072a Mon Sep 17 00:00:00 2001 From: jacktengg Date: Tue, 10 Nov 2026 23:46:38 +0800 Subject: [PATCH 5/8] [fix](be) Preserve gap positions in TIMESTAMPTZ day arithmetic ### What problem does this PR solve? Issue Number: N/A Related PR: #70204 Problem Summary: TIMESTAMPTZ day arithmetic clamped a target local time in a DST gap to the transition instant. In America/New_York, adding one day to 2024-03-09 02:30:00.123456-05:00 produced 03:00:00.123456-04:00 instead of 03:30:00.123456-04:00 on March 10. Initialize the preferred-offset local-to-UTC conversion from the existing civil lookup's pre value, aligning its gap policy with #70204 while retaining a single civil-time lookup and the overlap offset selection. Align FE DAY constant folding with the same policy through ZonedDateTime.ofLocal. Update unit and regression coverage for adding and subtracting a day into a gap, and disable SQL caching in the FE/BE comparison. Regenerate the regression output with the test runner. ### Release note TIMESTAMPTZ day arithmetic shifts nonexistent local times forward by the DST gap length and preserves fractional seconds in both FE and BE. ### Check List (For Author) - Test: Unit Test / Regression test - ASAN BE and FE build passed via build.sh --be --fe -j64 - run-be-ut.sh: all 9 TimeStampTzValueTest tests passed - run-fe-ut.sh: all 17 TimestampTzLiteralTest tests passed - run-regression-test.sh: test_timestamptz_days_add_dst and test_timestamptz_dst_fold passed - DAY regression passed with SQL caching disabled; its .out file was generated by the runner - clang-format 16, header hygiene, clang-tidy, FE Checkstyle and git diff --check passed - Behavior changed: Yes. Gap targets retain their position and fraction instead of clamping to the transition. - Does this need documentation: No --- be/src/core/value/timestamptz_value.cpp | 6 +++--- be/test/runtime/timestamptz_value_test.cpp | 10 +++++++--- .../trees/expressions/literal/TimestampTzLiteral.java | 11 +++-------- .../expressions/literal/TimestampTzLiteralTest.java | 3 ++- .../timestamptz/test_timestamptz_days_add_dst.out | 10 ++++++---- .../timestamptz/test_timestamptz_days_add_dst.groovy | 7 ++++++- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/be/src/core/value/timestamptz_value.cpp b/be/src/core/value/timestamptz_value.cpp index f1cec3d6514df9..d7bd2915ea72fa 100644 --- a/be/src/core/value/timestamptz_value.cpp +++ b/be/src/core/value/timestamptz_value.cpp @@ -197,9 +197,9 @@ void TimestampTzValue::convert_local_to_utc(const cctz::time_zone& local_time_zo cctz::civil_second local_cs(dt.year(), dt.month(), dt.day(), dt.hour(), dt.minute(), dt.second()); const auto lookup = local_time_zone.lookup(local_cs); - // Match cctz::convert's gap policy without repeating the civil-time lookup. - auto local_tp = - lookup.kind == cctz::time_zone::civil_lookup::SKIPPED ? lookup.trans : lookup.pre; + // Use the pre-transition offset to shift gap input by the gap length, e.g. New York's + // 2024-03-10 02:30 becomes 03:30. The preferred offset applies only to an overlap. + auto local_tp = lookup.pre; if (lookup.kind == cctz::time_zone::civil_lookup::REPEATED) { const auto pre_offset = local_time_zone.lookup(lookup.pre).offset; diff --git a/be/test/runtime/timestamptz_value_test.cpp b/be/test/runtime/timestamptz_value_test.cpp index f0e63000f9b673..00925f710d87c4 100644 --- a/be/test/runtime/timestamptz_value_test.cpp +++ b/be/test/runtime/timestamptz_value_test.cpp @@ -81,11 +81,15 @@ TEST(TimeStampTzValueTest, AddLocalCalendarDays) { .expected = "2024-11-03 01:30:00.123456-05:00", .days = -1, .elapsed_seconds = -86400}, - // Use the existing cctz gap policy and preserve the fractional second. + // Shift by the gap length and preserve the fractional second in both directions. {.input = "2024-03-09 02:30:00.123456-05:00", - .expected = "2024-03-10 03:00:00.123456-04:00", + .expected = "2024-03-10 03:30:00.123456-04:00", .days = 1, - .elapsed_seconds = 84600}, + .elapsed_seconds = 86400}, + {.input = "2024-03-11 02:30:00.123456-04:00", + .expected = "2024-03-10 03:30:00.123456-04:00", + .days = -1, + .elapsed_seconds = -82800}, }; for (const auto& test : cases) { SCOPED_TRACE(test.input); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java index a980870132cdbd..b3b75ac3cf1b61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java @@ -210,15 +210,10 @@ public Expression plusDays(long days) { if (isDateOutOfRange(local.toLocalDateTime()) || isDateOutOfRange(target)) { throw new AnalysisException("datetime out of range: " + target); } - // Keep the original offset when the target repeats: November 4 01:30-05 minus - // one day selects the second occurrence of November 3 01:30 in New York. - if (zone.getRules().getValidOffsets(target).contains(local.getOffset())) { - return fromJavaDateType(LocalDateTime.ofInstant( - target.toInstant(local.getOffset()), ZoneOffset.UTC), getScale()); - } - // Match BE cctz's gap policy (the transition instant), retaining microseconds. + // Keep the original offset in an overlap and shift gap input by the gap length, + // matching BE lookup.pre: New York's 2024-03-10 02:30 becomes 03:30. return fromJavaDateType(LocalDateTime.ofInstant( - convertLocalToInstantPreservingFraction(target, zone), ZoneOffset.UTC), getScale()); + ZonedDateTime.ofLocal(target, zone, local.getOffset()).toInstant(), ZoneOffset.UTC), getScale()); } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java index 14a62791fe3c17..4c29e6ee49e7b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java @@ -51,7 +51,8 @@ void testLocalCalendarDayArithmetic() { {"2024-11-03 01:30:00.123456-05:00", "0", "2024-11-03 01:30:00.123456-05:00", "0"}, {"2024-11-02 01:30:00.123456-04:00", "1", "2024-11-03 01:30:00.123456-04:00", "86400"}, {"2024-11-04 01:30:00.123456-05:00", "-1", "2024-11-03 01:30:00.123456-05:00", "-86400"}, - {"2024-03-09 02:30:00.123456-05:00", "1", "2024-03-10 03:00:00.123456-04:00", "84600"} + {"2024-03-09 02:30:00.123456-05:00", "1", "2024-03-10 03:30:00.123456-04:00", "86400"}, + {"2024-03-11 02:30:00.123456-04:00", "-1", "2024-03-10 03:30:00.123456-04:00", "-82800"} }; for (String[] testCase : cases) { TimestampTzLiteral input = new TimestampTzLiteral(TimeStampTzType.of(6), testCase[0]); diff --git a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out index 3c0bde240a0f6f..fd3010be4bf87e 100644 --- a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out +++ b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_days_add_dst.out @@ -9,7 +9,7 @@ 2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 86400 2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 -- !fe_overlap_gap -- -2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-05:00 2024-03-10 03:00:00.123456-04:00 +2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-05:00 2024-03-10 03:30:00.123456-04:00 2024-03-10 03:30:00.123456-04:00 -- !fe_columns -- 1 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 82800 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 @@ -17,6 +17,7 @@ 11 \N \N \N 2024-03-10 12:00:00.123456-04:00 \N 12 2024-11-04 01:50:00.123456-05:00 2024-11-04 01:50:00.123456-05:00 90000 2024-11-04 01:50:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 13 2024-11-04 01:10:00.123456-05:00 2024-11-04 01:10:00.123456-05:00 86400 2024-11-04 01:10:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +14 2024-03-10 03:30:00.123456-04:00 2024-03-10 03:30:00.123456-04:00 -82800 2024-03-12 02:30:00.123456-04:00 2024-03-08 12:00:00.123456-05:00 2 2024-11-03 12:00:00.123456-05:00 2024-11-03 12:00:00.123456-05:00 90000 2024-11-03 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 3 2024-03-09 12:00:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 -82800 2024-03-11 12:00:00.123456-04:00 2024-03-08 12:00:00.123456-05:00 4 2024-11-02 12:00:00.123456-04:00 2024-11-02 12:00:00.123456-04:00 -90000 2024-11-04 12:00:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 @@ -24,7 +25,7 @@ 6 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 0 2024-11-04 01:30:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 7 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-04:00 86400 2024-11-03 01:30:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 8 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 -86400 2024-11-05 01:30:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 -9 2024-03-10 03:00:00.123456-04:00 2024-03-10 03:00:00.123456-04:00 84600 2024-03-10 03:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 +9 2024-03-10 03:30:00.123456-04:00 2024-03-10 03:30:00.123456-04:00 86400 2024-03-10 03:30:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 -- !fe_non_monotonic -- 13 2024-11-04 01:10:00.123456-05:00 @@ -51,7 +52,7 @@ 2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 86400 2024-02-01 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 2024-02-02 12:00:00.123456-05:00 -- !be_overlap_gap -- -2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-05:00 2024-03-10 03:00:00.123456-04:00 +2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-05:00 2024-03-10 03:30:00.123456-04:00 2024-03-10 03:30:00.123456-04:00 -- !be_columns -- 1 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 82800 2024-03-10 12:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 @@ -59,6 +60,7 @@ 11 \N \N \N 2024-03-10 12:00:00.123456-04:00 \N 12 2024-11-04 01:50:00.123456-05:00 2024-11-04 01:50:00.123456-05:00 90000 2024-11-04 01:50:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 13 2024-11-04 01:10:00.123456-05:00 2024-11-04 01:10:00.123456-05:00 86400 2024-11-04 01:10:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 +14 2024-03-10 03:30:00.123456-04:00 2024-03-10 03:30:00.123456-04:00 -82800 2024-03-12 02:30:00.123456-04:00 2024-03-08 12:00:00.123456-05:00 2 2024-11-03 12:00:00.123456-05:00 2024-11-03 12:00:00.123456-05:00 90000 2024-11-03 12:00:00.123456-05:00 2024-03-10 12:00:00.123456-04:00 3 2024-03-09 12:00:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 -82800 2024-03-11 12:00:00.123456-04:00 2024-03-08 12:00:00.123456-05:00 4 2024-11-02 12:00:00.123456-04:00 2024-11-02 12:00:00.123456-04:00 -90000 2024-11-04 12:00:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 @@ -66,7 +68,7 @@ 6 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 0 2024-11-04 01:30:00.123456-05:00 2024-03-09 12:00:00.123456-05:00 7 2024-11-03 01:30:00.123456-04:00 2024-11-03 01:30:00.123456-04:00 86400 2024-11-03 01:30:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 8 2024-11-03 01:30:00.123456-05:00 2024-11-03 01:30:00.123456-05:00 -86400 2024-11-05 01:30:00.123456-05:00 2024-03-08 12:00:00.123456-05:00 -9 2024-03-10 03:00:00.123456-04:00 2024-03-10 03:00:00.123456-04:00 84600 2024-03-10 03:00:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 +9 2024-03-10 03:30:00.123456-04:00 2024-03-10 03:30:00.123456-04:00 86400 2024-03-10 03:30:00.123456-04:00 2024-03-10 12:00:00.123456-04:00 -- !be_non_monotonic -- 13 2024-11-04 01:10:00.123456-05:00 diff --git a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy index ff742edc31800d..72f25df67ebbc8 100644 --- a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy +++ b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_days_add_dst.groovy @@ -17,6 +17,7 @@ suite("test_timestamptz_days_add_dst") { sql "SET time_zone = 'America/New_York'" + sql "SET enable_sql_cache = false" sql "DROP TABLE IF EXISTS timestamptz_days_add_dst" sql """ CREATE TABLE timestamptz_days_add_dst (id INT, ts TIMESTAMPTZ(6), delta INT) @@ -38,7 +39,8 @@ suite("test_timestamptz_days_add_dst") { (10, NULL, 1), (11, '2024-03-09 12:00:00.123456-05:00', NULL), (12, '2024-11-03 01:50:00.123456-04:00', 1), - (13, '2024-11-03 01:10:00.123456-05:00', 1) + (13, '2024-11-03 01:10:00.123456-05:00', 1), + (14, '2024-03-11 02:30:00.123456-04:00', -1) """ sql "DROP TABLE IF EXISTS timestamptz_days_zero_events" sql "DROP TABLE IF EXISTS timestamptz_days_zero_ids" @@ -85,6 +87,8 @@ suite("test_timestamptz_days_add_dst") { CAST(days_sub(CAST('2024-11-04 01:30:00.123456-05:00' AS TIMESTAMPTZ(6)), 1) AS STRING), CAST(days_add(CAST('2024-03-09 02:30:00.123456-05:00' AS TIMESTAMPTZ(6)), 1) + AS STRING), + CAST(days_sub(CAST('2024-03-11 02:30:00.123456-04:00' AS TIMESTAMPTZ(6)), 1) AS STRING) """) // Cover vector/vector, vector/constant, constant/vector, and NULL propagation. @@ -137,6 +141,7 @@ suite("test_timestamptz_days_add_dst") { sql "SET time_zone = 'America/New_York'" } } finally { + sql "SET enable_sql_cache = default" sql "SET debug_skip_fold_constant = false" sql "SET runtime_filter_mode = default" sql "SET time_zone = default" From 3b6bca398c96f19e4d36db44f5dbb7535bc05644 Mon Sep 17 00:00:00 2001 From: jacktengg Date: Fri, 13 Nov 2026 14:50:20 +0800 Subject: [PATCH 6/8] [fix](fe) Validate inverse TIMESTAMPTZ day predicates ### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: In a +08:00 session, rewriting days_add(ts, 1) < TIMESTAMPTZ '9999-12-31 23:00:00+00:00' introduces days_sub(cutoff, 1). The cutoff is valid UTC but has session-local year 10000, so its inverse fails even though the original comparison can evaluate successfully. The fixed-offset monotonicity check does not establish that the inverse is valid, and failed FE folding leaves the invalid call for BE execution. Fold the candidate inverse for TIMESTAMPTZ DAY addition and subtraction, and require a TimestampTzLiteral before rearranging the comparison. Keep the original predicate when folding cannot produce a valid literal, and reuse the literal when it succeeds. Cover local and UTC year boundaries, positive and negative intervals, zero-day identities, and valid inverses. ### Release note Prevent optimizer-introduced out-of-range errors in TIMESTAMPTZ DAY comparisons near timezone and year boundaries. ### Check List (For Author) - Test: Unit Test / Regression test / Manual test - 22 FE unit tests passed: SimplifyArithmeticComparisonRuleTest and TimestampTzLiteralTest. - Regression suites test_timestamptz_day_predicate_boundary and test_timestamptz_days_add_dst passed. - Generated the new regression output with run-regression-test.sh -forceGenOut, then compared it in a normal run. - Reproduced the original out-of-range failure and verified the fixed plan retains the original DAY comparison. - FE build via build.sh --fe and Checkstyle passed. - git diff --check passed with the generated .out trailing blank separator allowed. - Behavior changed: Yes; preserve valid DAY comparisons when their inverse cannot be evaluated safely. - Does this need documentation: No --- .../SimplifyArithmeticComparisonRule.java | 17 +++- .../SimplifyArithmeticComparisonRuleTest.java | 73 ++++++++++++++++++ ...est_timestamptz_day_predicate_boundary.out | 37 +++++++++ ..._timestamptz_day_predicate_boundary.groovy | 77 +++++++++++++++++++ 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out create mode 100644 regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java index 861e61a521e0da..287fa1b19beb65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.trees.expressions.literal.ComparableLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.collect.ImmutableList; @@ -100,6 +101,9 @@ public static Expression simplify(ComparisonPredicate comparison, ExpressionRewr } try { List children = tryRearrangeChildren(newComparison.left(), newComparison.right(), context); + if (children == null) { + return comparison; + } newComparison = (ComparisonPredicate) simplify( (ComparisonPredicate) newComparison.withChildren(children), context); return TypeCoercionUtils.processComparisonPredicate(newComparison); @@ -138,7 +142,7 @@ private static boolean couldRearrange(ComparisonPredicate cmp) { return false; } - private static List tryRearrangeChildren(Expression left, Expression right, + private static @Nullable List tryRearrangeChildren(Expression left, Expression right, ExpressionRewriteContext context) throws Exception { if (!left.child(1).isConstant()) { throw new RuntimeException(String.format("Expected literal when arranging children for Expr %s", left)); @@ -150,6 +154,17 @@ private static List tryRearrangeChildren(Expression left, Expression Expression newChild = oppositeOperator.getConstructor(Expression.class, Expression.class) .newInstance(right, leftLiteral); + if ((left instanceof DaysAdd || left instanceof DaysSub) + && left.child(0).getDataType().isTimeStampTzType()) { + // Even a fixed offset can make the inverse fail: at +08:00, the valid UTC cutoff + // 9999-12-31 23:00 has local year 10000. Failed folding leaves the inverse call + // unchanged, so require a valid literal before introducing it into the predicate. + newChild = FoldConstantRule.evaluate(newChild, context); + if (!(newChild instanceof TimestampTzLiteral)) { + return null; + } + } + if (left instanceof Divide && leftLiteral.compareTo(new IntegerLiteral(0)) < 0) { // Multiplying by a negative number will change the operator. return Arrays.asList(newChild, leftExpr); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java index 821d3382498ee0..90d3a1e2d668ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java @@ -71,6 +71,79 @@ void testTimestampTzCalendarDaysCannotRearrangeAcrossDst() { } } + @Test + void testTimestampTzCalendarDaysRequireValidInverseAtLocalBoundaries() { + ConnectContext previous = ConnectContext.get(); + ConnectContext connectContext = new ConnectContext(); + connectContext.setThreadLocalInfo(); + try { + SlotReference timestamp = new SlotReference("ts", TimeStampTzType.of(6)); + String[][] cases = { + {"+08:00", "9999-12-31 23:00:00.123456+00:00"}, + {"-08:00", "0000-01-01 01:00:00.123456+00:00"} + }; + for (String[] testCase : cases) { + connectContext.getSessionVariable().setTimeZone(testCase[0]); + TimestampTzLiteral bound = new TimestampTzLiteral(TimeStampTzType.of(6), testCase[1]); + for (int days : new int[] {-1, 1}) { + IntegerLiteral interval = new IntegerLiteral(days); + for (Expression arithmetic : new Expression[] {new DaysAdd(timestamp, interval), + new DaysSub(timestamp, interval)}) { + ComparisonPredicate predicate = new LessThan(arithmetic, bound); + Assertions.assertSame(predicate, SimplifyArithmeticComparisonRule.simplify(predicate, context)); + } + } + // Zero days is still an identity, even when the local display year is out of range. + IntegerLiteral zero = new IntegerLiteral(0); + for (Expression arithmetic : new Expression[] {new DaysAdd(timestamp, zero), + new DaysSub(timestamp, zero)}) { + Assertions.assertEquals(new LessThan(timestamp, bound), SimplifyArithmeticComparisonRule.simplify( + new LessThan(arithmetic, bound), context)); + } + } + } finally { + ConnectContext.remove(); + if (previous != null) { + previous.setThreadLocalInfo(); + } + } + } + + @Test + void testTimestampTzCalendarDaysRequireValidInverseAtUtcBoundaries() { + ConnectContext previous = ConnectContext.get(); + ConnectContext connectContext = new ConnectContext(); + connectContext.getSessionVariable().setTimeZone("UTC"); + connectContext.setThreadLocalInfo(); + try { + SlotReference timestamp = new SlotReference("ts", TimeStampTzType.of(6)); + TimestampTzLiteral lower = new TimestampTzLiteral(TimeStampTzType.of(6), + "0000-01-01 01:00:00.123456+00:00"); + TimestampTzLiteral upper = new TimestampTzLiteral(TimeStampTzType.of(6), + "9999-12-31 23:00:00.123456+00:00"); + IntegerLiteral one = new IntegerLiteral(1); + for (ComparisonPredicate predicate : new ComparisonPredicate[] { + new LessThan(new DaysAdd(timestamp, one), lower), + new LessThan(new DaysSub(timestamp, one), upper)}) { + Assertions.assertSame(predicate, SimplifyArithmeticComparisonRule.simplify(predicate, context)); + } + // A valid inverse at either boundary must still remove the DAY function from the slot. + Assertions.assertEquals(new LessThan(timestamp, (TimestampTzLiteral) lower.plusDays(1)), + SimplifyArithmeticComparisonRule.simplify(new LessThan(new DaysSub(timestamp, one), lower), context)); + Assertions.assertEquals(new LessThan(timestamp, (TimestampTzLiteral) upper.plusDays(-1)), + SimplifyArithmeticComparisonRule.simplify(new LessThan(new DaysAdd(timestamp, one), upper), context)); + ComparisonPredicate nonConstantBound = new LessThan(new DaysAdd(timestamp, one), + new SlotReference("cutoff", TimeStampTzType.of(6))); + Assertions.assertSame(nonConstantBound, + SimplifyArithmeticComparisonRule.simplify(nonConstantBound, context)); + } finally { + ConnectContext.remove(); + if (previous != null) { + previous.setThreadLocalInfo(); + } + } + } + @Test public void testNumeric() { executor = new ExpressionRuleExecutor(ImmutableList.of( diff --git a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out new file mode 100644 index 00000000000000..5f93c72f9e4cb0 --- /dev/null +++ b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !fe_local_upper -- +1 + +-- !fe_local_upper_negative -- +1 + +-- !fe_local_lower -- +2 + +-- !fe_local_lower_negative -- +2 + +-- !fe_utc_upper -- +1 + +-- !fe_utc_lower -- +2 + +-- !be_local_upper -- +1 + +-- !be_local_upper_negative -- +1 + +-- !be_local_lower -- +2 + +-- !be_local_lower_negative -- +2 + +-- !be_utc_upper -- +1 + +-- !be_utc_lower -- +2 + diff --git a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy new file mode 100644 index 00000000000000..075723e69d4926 --- /dev/null +++ b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamptz_day_predicate_boundary") { + sql "SET enable_sql_cache = false" + sql "DROP TABLE IF EXISTS timestamptz_day_predicate_boundary" + sql """ + CREATE TABLE timestamptz_day_predicate_boundary (id INT, ts TIMESTAMPTZ(6)) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num' = '1') + """ + sql """ + INSERT INTO timestamptz_day_predicate_boundary VALUES + (1, '9999-12-29 23:00:00.123456+00:00'), + (2, '0000-01-03 01:00:00.123456+00:00') + """ + try { + for (def skipFold : [false, true]) { + sql "SET debug_skip_fold_constant = ${skipFold}" + def tag = skipFold ? "be" : "fe" + sql "SET time_zone = '+08:00'" + // The forward operation is valid; the inverse cutoff has local year 10000. + "order_qt_${tag}_local_upper"(""" + SELECT id FROM timestamptz_day_predicate_boundary + WHERE id = 1 AND days_add(ts, 1) + < CAST('9999-12-31 23:00:00.123456+00:00' AS TIMESTAMPTZ(6)) + """) + "order_qt_${tag}_local_upper_negative"(""" + SELECT id FROM timestamptz_day_predicate_boundary + WHERE id = 1 AND days_sub(ts, -1) + < CAST('9999-12-31 23:00:00.123456+00:00' AS TIMESTAMPTZ(6)) + """) + sql "SET time_zone = '-08:00'" + // The lower cutoff is valid UTC, but its local year is -1. + "order_qt_${tag}_local_lower"(""" + SELECT id FROM timestamptz_day_predicate_boundary + WHERE id = 2 AND days_sub(ts, 1) + > CAST('0000-01-01 01:00:00.123456+00:00' AS TIMESTAMPTZ(6)) + """) + "order_qt_${tag}_local_lower_negative"(""" + SELECT id FROM timestamptz_day_predicate_boundary + WHERE id = 2 AND days_add(ts, -1) + > CAST('0000-01-01 01:00:00.123456+00:00' AS TIMESTAMPTZ(6)) + """) + sql "SET time_zone = 'UTC'" + // The inverse can also fail because its target year is outside the UTC range. + "order_qt_${tag}_utc_upper"(""" + SELECT id FROM timestamptz_day_predicate_boundary + WHERE id = 1 AND days_sub(ts, 1) + < CAST('9999-12-31 23:00:00.123456+00:00' AS TIMESTAMPTZ(6)) + """) + "order_qt_${tag}_utc_lower"(""" + SELECT id FROM timestamptz_day_predicate_boundary + WHERE id = 2 AND days_add(ts, 1) + > CAST('0000-01-01 01:00:00.123456+00:00' AS TIMESTAMPTZ(6)) + """) + } + } finally { + sql "SET debug_skip_fold_constant = false" + sql "SET enable_sql_cache = default" + sql "SET time_zone = default" + } +} From da1e1729f6ed812b96a16f480d8894fb622ea92a Mon Sep 17 00:00:00 2001 From: jacktengg Date: Fri, 13 Nov 2026 16:07:54 +0800 Subject: [PATCH 7/8] [refactor](fe) Revert the TIMESTAMPTZ-only inverse guard This reverts commit 3b6bca398c9 in preparation for cherry-picking cd42918848ec2e174902c4b686ea7852a6592ac1 from PR #70556. ### What problem does this PR solve? Issue Number: N/A Related PR: #70556 Problem Summary: Replace the TIMESTAMPTZ DAY-only inverse validation with the shared date arithmetic validation from PR #70556. The replacement covers all date arithmetic functions in the rearrangement map, applies type coercion, and validates the inverse exclusively on FE. Keep the existing DST monotonicity guard from earlier commits. ### Release note None; this preparatory revert is followed by the broader fix. ### Check List (For Author) - Test: Manual test; inspected the reverse patch and replacement code. Run unit and regression tests after applying the replacement. - Behavior changed: Yes; this intermediate commit removes the narrow inverse guard before its replacement. - Does this need documentation: No --- .../SimplifyArithmeticComparisonRule.java | 17 +--- .../SimplifyArithmeticComparisonRuleTest.java | 73 ------------------ ...est_timestamptz_day_predicate_boundary.out | 37 --------- ..._timestamptz_day_predicate_boundary.groovy | 77 ------------------- 4 files changed, 1 insertion(+), 203 deletions(-) delete mode 100644 regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out delete mode 100644 regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java index 287fa1b19beb65..861e61a521e0da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java @@ -41,7 +41,6 @@ import org.apache.doris.nereids.trees.expressions.literal.ComparableLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.collect.ImmutableList; @@ -101,9 +100,6 @@ public static Expression simplify(ComparisonPredicate comparison, ExpressionRewr } try { List children = tryRearrangeChildren(newComparison.left(), newComparison.right(), context); - if (children == null) { - return comparison; - } newComparison = (ComparisonPredicate) simplify( (ComparisonPredicate) newComparison.withChildren(children), context); return TypeCoercionUtils.processComparisonPredicate(newComparison); @@ -142,7 +138,7 @@ private static boolean couldRearrange(ComparisonPredicate cmp) { return false; } - private static @Nullable List tryRearrangeChildren(Expression left, Expression right, + private static List tryRearrangeChildren(Expression left, Expression right, ExpressionRewriteContext context) throws Exception { if (!left.child(1).isConstant()) { throw new RuntimeException(String.format("Expected literal when arranging children for Expr %s", left)); @@ -154,17 +150,6 @@ private static boolean couldRearrange(ComparisonPredicate cmp) { Expression newChild = oppositeOperator.getConstructor(Expression.class, Expression.class) .newInstance(right, leftLiteral); - if ((left instanceof DaysAdd || left instanceof DaysSub) - && left.child(0).getDataType().isTimeStampTzType()) { - // Even a fixed offset can make the inverse fail: at +08:00, the valid UTC cutoff - // 9999-12-31 23:00 has local year 10000. Failed folding leaves the inverse call - // unchanged, so require a valid literal before introducing it into the predicate. - newChild = FoldConstantRule.evaluate(newChild, context); - if (!(newChild instanceof TimestampTzLiteral)) { - return null; - } - } - if (left instanceof Divide && leftLiteral.compareTo(new IntegerLiteral(0)) < 0) { // Multiplying by a negative number will change the operator. return Arrays.asList(newChild, leftExpr); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java index 90d3a1e2d668ec..821d3382498ee0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java @@ -71,79 +71,6 @@ void testTimestampTzCalendarDaysCannotRearrangeAcrossDst() { } } - @Test - void testTimestampTzCalendarDaysRequireValidInverseAtLocalBoundaries() { - ConnectContext previous = ConnectContext.get(); - ConnectContext connectContext = new ConnectContext(); - connectContext.setThreadLocalInfo(); - try { - SlotReference timestamp = new SlotReference("ts", TimeStampTzType.of(6)); - String[][] cases = { - {"+08:00", "9999-12-31 23:00:00.123456+00:00"}, - {"-08:00", "0000-01-01 01:00:00.123456+00:00"} - }; - for (String[] testCase : cases) { - connectContext.getSessionVariable().setTimeZone(testCase[0]); - TimestampTzLiteral bound = new TimestampTzLiteral(TimeStampTzType.of(6), testCase[1]); - for (int days : new int[] {-1, 1}) { - IntegerLiteral interval = new IntegerLiteral(days); - for (Expression arithmetic : new Expression[] {new DaysAdd(timestamp, interval), - new DaysSub(timestamp, interval)}) { - ComparisonPredicate predicate = new LessThan(arithmetic, bound); - Assertions.assertSame(predicate, SimplifyArithmeticComparisonRule.simplify(predicate, context)); - } - } - // Zero days is still an identity, even when the local display year is out of range. - IntegerLiteral zero = new IntegerLiteral(0); - for (Expression arithmetic : new Expression[] {new DaysAdd(timestamp, zero), - new DaysSub(timestamp, zero)}) { - Assertions.assertEquals(new LessThan(timestamp, bound), SimplifyArithmeticComparisonRule.simplify( - new LessThan(arithmetic, bound), context)); - } - } - } finally { - ConnectContext.remove(); - if (previous != null) { - previous.setThreadLocalInfo(); - } - } - } - - @Test - void testTimestampTzCalendarDaysRequireValidInverseAtUtcBoundaries() { - ConnectContext previous = ConnectContext.get(); - ConnectContext connectContext = new ConnectContext(); - connectContext.getSessionVariable().setTimeZone("UTC"); - connectContext.setThreadLocalInfo(); - try { - SlotReference timestamp = new SlotReference("ts", TimeStampTzType.of(6)); - TimestampTzLiteral lower = new TimestampTzLiteral(TimeStampTzType.of(6), - "0000-01-01 01:00:00.123456+00:00"); - TimestampTzLiteral upper = new TimestampTzLiteral(TimeStampTzType.of(6), - "9999-12-31 23:00:00.123456+00:00"); - IntegerLiteral one = new IntegerLiteral(1); - for (ComparisonPredicate predicate : new ComparisonPredicate[] { - new LessThan(new DaysAdd(timestamp, one), lower), - new LessThan(new DaysSub(timestamp, one), upper)}) { - Assertions.assertSame(predicate, SimplifyArithmeticComparisonRule.simplify(predicate, context)); - } - // A valid inverse at either boundary must still remove the DAY function from the slot. - Assertions.assertEquals(new LessThan(timestamp, (TimestampTzLiteral) lower.plusDays(1)), - SimplifyArithmeticComparisonRule.simplify(new LessThan(new DaysSub(timestamp, one), lower), context)); - Assertions.assertEquals(new LessThan(timestamp, (TimestampTzLiteral) upper.plusDays(-1)), - SimplifyArithmeticComparisonRule.simplify(new LessThan(new DaysAdd(timestamp, one), upper), context)); - ComparisonPredicate nonConstantBound = new LessThan(new DaysAdd(timestamp, one), - new SlotReference("cutoff", TimeStampTzType.of(6))); - Assertions.assertSame(nonConstantBound, - SimplifyArithmeticComparisonRule.simplify(nonConstantBound, context)); - } finally { - ConnectContext.remove(); - if (previous != null) { - previous.setThreadLocalInfo(); - } - } - } - @Test public void testNumeric() { executor = new ExpressionRuleExecutor(ImmutableList.of( diff --git a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out deleted file mode 100644 index 5f93c72f9e4cb0..00000000000000 --- a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.out +++ /dev/null @@ -1,37 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !fe_local_upper -- -1 - --- !fe_local_upper_negative -- -1 - --- !fe_local_lower -- -2 - --- !fe_local_lower_negative -- -2 - --- !fe_utc_upper -- -1 - --- !fe_utc_lower -- -2 - --- !be_local_upper -- -1 - --- !be_local_upper_negative -- -1 - --- !be_local_lower -- -2 - --- !be_local_lower_negative -- -2 - --- !be_utc_upper -- -1 - --- !be_utc_lower -- -2 - diff --git a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy deleted file mode 100644 index 075723e69d4926..00000000000000 --- a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_day_predicate_boundary.groovy +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_timestamptz_day_predicate_boundary") { - sql "SET enable_sql_cache = false" - sql "DROP TABLE IF EXISTS timestamptz_day_predicate_boundary" - sql """ - CREATE TABLE timestamptz_day_predicate_boundary (id INT, ts TIMESTAMPTZ(6)) - DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 - PROPERTIES('replication_num' = '1') - """ - sql """ - INSERT INTO timestamptz_day_predicate_boundary VALUES - (1, '9999-12-29 23:00:00.123456+00:00'), - (2, '0000-01-03 01:00:00.123456+00:00') - """ - try { - for (def skipFold : [false, true]) { - sql "SET debug_skip_fold_constant = ${skipFold}" - def tag = skipFold ? "be" : "fe" - sql "SET time_zone = '+08:00'" - // The forward operation is valid; the inverse cutoff has local year 10000. - "order_qt_${tag}_local_upper"(""" - SELECT id FROM timestamptz_day_predicate_boundary - WHERE id = 1 AND days_add(ts, 1) - < CAST('9999-12-31 23:00:00.123456+00:00' AS TIMESTAMPTZ(6)) - """) - "order_qt_${tag}_local_upper_negative"(""" - SELECT id FROM timestamptz_day_predicate_boundary - WHERE id = 1 AND days_sub(ts, -1) - < CAST('9999-12-31 23:00:00.123456+00:00' AS TIMESTAMPTZ(6)) - """) - sql "SET time_zone = '-08:00'" - // The lower cutoff is valid UTC, but its local year is -1. - "order_qt_${tag}_local_lower"(""" - SELECT id FROM timestamptz_day_predicate_boundary - WHERE id = 2 AND days_sub(ts, 1) - > CAST('0000-01-01 01:00:00.123456+00:00' AS TIMESTAMPTZ(6)) - """) - "order_qt_${tag}_local_lower_negative"(""" - SELECT id FROM timestamptz_day_predicate_boundary - WHERE id = 2 AND days_add(ts, -1) - > CAST('0000-01-01 01:00:00.123456+00:00' AS TIMESTAMPTZ(6)) - """) - sql "SET time_zone = 'UTC'" - // The inverse can also fail because its target year is outside the UTC range. - "order_qt_${tag}_utc_upper"(""" - SELECT id FROM timestamptz_day_predicate_boundary - WHERE id = 1 AND days_sub(ts, 1) - < CAST('9999-12-31 23:00:00.123456+00:00' AS TIMESTAMPTZ(6)) - """) - "order_qt_${tag}_utc_lower"(""" - SELECT id FROM timestamptz_day_predicate_boundary - WHERE id = 2 AND days_add(ts, 1) - > CAST('0000-01-01 01:00:00.123456+00:00' AS TIMESTAMPTZ(6)) - """) - } - } finally { - sql "SET debug_skip_fold_constant = false" - sql "SET enable_sql_cache = default" - sql "SET time_zone = default" - } -} From c158d92470f4df95029884b40a0cc7fe18a2bced Mon Sep 17 00:00:00 2001 From: jacktengg Date: Fri, 13 Nov 2026 15:44:11 +0800 Subject: [PATCH 8/8] [fix](nereids) Avoid overflow when rearranging date comparisons ### What problem does this PR solve? Issue Number: N/A Related PR: #70556 Problem Summary: Moving date arithmetic across a comparison can introduce an inverse operation outside the date type's range even when the source operation is valid. Fixed-offset TIMESTAMPTZ sessions can also have a valid UTC cutoff whose local year is outside the supported range. Coerce and evaluate each candidate date inverse on FE, accepting only a DateLiteral and reusing it in the rewritten comparison. Preserve the original comparison for invalid, NULL or nonconstant boundaries. The shared guard covers week, day, hour, minute and second addition and subtraction across the supported date types. Preserve this branch's existing DST monotonicity guard. Resolve import conflicts and explicitly use UTC in the two imported boundary unit tests so their DAY assertions exercise inverse validation in a fixed-offset session. No additional production changes were needed for integration. ### Release note Prevent date comparison errors caused by optimizer-generated arithmetic that exceeds the date type's range. ### Check List (For Author) - Test: Unit Test / Regression test / Manual test - 22 FE tests passed via run-fe-ut.sh: SimplifyArithmeticComparisonRuleTest and TimestampTzLiteralTest. - FE build via build.sh --fe and Checkstyle passed. - Six regression suites passed: test_date_comparison_boundary, test_simplify_arithmetic, test_simplify_comparison_predicate, test_date_arithmetic_rejecting_boundary, test_timestamptz_day_predicate_boundary, and test_timestamptz_days_add_dst. - Temporarily restored the reverted boundary suite to verify all 12 original queries, then removed the temporary copies. - Verified the original +08:00 boundary query returns its row and retains days_add in EXPLAIN with BE constant folding enabled. - git diff --check passed with the generated .out trailing separator allowed. - Behavior changed: Yes; reject unsafe date-comparison rearrangements while preserving valid inverse optimizations. - Does this need documentation: No (cherry picked from commit cd42918848ec2e174902c4b686ea7852a6592ac1) --- .../SimplifyArithmeticComparisonRule.java | 19 +- .../SimplifyArithmeticComparisonRuleTest.java | 79 ++++++++ .../test_date_comparison_boundary.out | 184 ++++++++++++++++++ .../test_date_comparison_boundary.groovy | 122 ++++++++++++ 4 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/nereids_rules_p0/expression/test_date_comparison_boundary.out create mode 100644 regression-test/suites/nereids_rules_p0/expression/test_date_comparison_boundary.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java index 861e61a521e0da..49c79cce0b5bf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRule.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.DateAddSubMonotonic; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; @@ -39,6 +40,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.WeeksAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.WeeksSub; import org.apache.doris.nereids.trees.expressions.literal.ComparableLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.util.TypeCoercionUtils; @@ -100,6 +102,9 @@ public static Expression simplify(ComparisonPredicate comparison, ExpressionRewr } try { List children = tryRearrangeChildren(newComparison.left(), newComparison.right(), context); + if (children == null) { + return comparison; + } newComparison = (ComparisonPredicate) simplify( (ComparisonPredicate) newComparison.withChildren(children), context); return TypeCoercionUtils.processComparisonPredicate(newComparison); @@ -138,7 +143,7 @@ private static boolean couldRearrange(ComparisonPredicate cmp) { return false; } - private static List tryRearrangeChildren(Expression left, Expression right, + private static @Nullable List tryRearrangeChildren(Expression left, Expression right, ExpressionRewriteContext context) throws Exception { if (!left.child(1).isConstant()) { throw new RuntimeException(String.format("Expected literal when arranging children for Expr %s", left)); @@ -150,6 +155,18 @@ private static List tryRearrangeChildren(Expression left, Expression Expression newChild = oppositeOperator.getConstructor(Expression.class, Expression.class) .newInstance(right, leftLiteral); + if (newChild instanceof BoundFunction) { + // Date arithmetic has a finite range. For example, days_sub(d, 1) < MAX must not + // become d < days_add(MAX, 1). Check the inverse on FE before accepting the rewrite. + // NULL and expressions that cannot fold are not valid date boundaries. This also + // excludes a boundary supplied by a column, which can contain MAX at execution time. + newChild = FoldConstantRuleOnFE.evaluate( + TypeCoercionUtils.processBoundFunction((BoundFunction) newChild), context); + if (!(newChild instanceof DateLiteral)) { + return null; + } + } + if (left instanceof Divide && leftLiteral.compareTo(new IntegerLiteral(0)) < 0) { // Multiplying by a negative number will change the operator. return Arrays.asList(newChild, leftExpr); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java index 821d3382498ee0..b807d91eec40f4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java @@ -23,12 +23,19 @@ import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateType; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.qe.ConnectContext; @@ -37,6 +44,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Map; + class SimplifyArithmeticComparisonRuleTest extends ExpressionRewriteTestHelper { @Test @@ -226,6 +235,76 @@ public void testDateLike() { assertRewriteAfterTypeCoercion("seconds_sub(CA, 1) > '2021-01-01 00:00:00'", "(cast(CA as DATETIMEV2(0)) > '2021-01-01 00:00:01')"); } + @Test + public void testDateComparisonBoundaries() { + // Use a fixed-offset session so the DAY inverse guard is exercised. + cascadesContext.getConnectContext().getSessionVariable().setTimeZone("UTC"); + // Each expression needs one slot and one boundary. No table rows are needed. + // The inverse operation exceeds the type range at both ends of that range. + for (DataType type : ImmutableList.of(DateType.INSTANCE, DateTimeType.INSTANCE, + DateV2Type.INSTANCE, DateTimeV2Type.of(0), + DateTimeV2Type.of(6), TimeStampTzType.of(6), TimeStampNsType.INSTANCE)) { + String minimum = "0000-01-01 00:00:00"; + String maximum = "9999-12-31 23:59:59.999999"; + if (type.isDateType() || type.isDateV2Type()) { + minimum = "0000-01-01"; + maximum = "9999-12-31"; + } else if (type.isTimeStampNsType()) { + minimum = "1677-09-21 00:12:43.145224192"; + maximum = "2262-04-11 23:47:16.854775807"; + } else if (type.isTimeStampTzType()) { + minimum += "+00:00"; + maximum += "+00:00"; + } else if (type.isDateTimeType() || type.equals(DateTimeV2Type.of(0))) { + maximum = "9999-12-31 23:59:59"; + } + for (String unit : ImmutableList.of("weeks", "days", "hours", "minutes", "seconds")) { + // Operations on hours, minutes, and seconds return DATETIME for DATE. Use its upper limit. + String upperBoundary = (type.isDateType() || type.isDateV2Type()) + && !unit.equals("weeks") && !unit.equals("days") ? "9999-12-31 23:59:59" : maximum; + for (String operator : ImmutableList.of("<", "<=", ">", ">=", "=", "<=>")) { + assertDateComparisonUnchanged(unit + "_sub(AA, 1) " + operator + " '" + upperBoundary + "'", type); + assertDateComparisonUnchanged(unit + "_add(AA, 1) " + operator + " '" + minimum + "'", type); + // A negative interval changes which inverse operation exceeds the range. + assertDateComparisonUnchanged(unit + "_add(AA, -1) " + operator + " '" + upperBoundary + "'", type); + assertDateComparisonUnchanged(unit + "_sub(AA, -1) " + operator + " '" + minimum + "'", type); + } + } + // A column can contain a boundary value. Planning cannot prove that its inverse is valid. + assertDateComparisonUnchanged("days_sub(AA, 1) < AB", type); + // NULL is not a date boundary. Keep the original null-safe comparison. + assertDateComparisonUnchanged("days_sub(AA, 1) <=> NULL", type); + } + } + + @Test + public void testTimestampComparisonWithinRange() { + // A fixed-offset session permits calendar DAY rearrangement. + cascadesContext.getConnectContext().getSessionVariable().setTimeZone("UTC"); + // Preserve precision when a valid boundary moves by one day. + for (DataType type : ImmutableList.of(TimeStampTzType.of(6), TimeStampNsType.INSTANCE)) { + String fraction = type.isTimeStampNsType() ? ".123456789" : ".123456+00:00"; + Map slots = Maps.newHashMap(); + slots.put("AA", new SlotReference("AA", type)); + ComparisonPredicate original = (ComparisonPredicate) typeCoercion(replaceUnboundSlot( + PARSER.parseExpression("days_sub(AA, 1) < '2026-11-13 15:03:12" + fraction + "'"), slots)); + Expression expected = typeCoercion(replaceUnboundSlot( + PARSER.parseExpression("AA < '2026-11-14 15:03:12" + fraction + "'"), slots)); + Expression actual = SimplifyArithmeticComparisonRule.simplify(original, context); + Assertions.assertEquals(expected, FoldConstantRuleOnFE.evaluate(actual, context)); + } + } + + private void assertDateComparisonUnchanged(String sql, DataType type) { + Map slots = Maps.newHashMap(); + slots.put("AA", new SlotReference("AA", type)); + slots.put("AB", new SlotReference("AB", type)); + ComparisonPredicate original = (ComparisonPredicate) typeCoercion( + replaceUnboundSlot(PARSER.parseExpression(sql), slots)); + Assertions.assertEquals(original, SimplifyArithmeticComparisonRule.simplify(original, context), + type + ": " + sql); + } + private void assertRewriteAfterSimplify(String expr, String expected) { Expression needRewriteExpression = PARSER.parseExpression(expr); needRewriteExpression = replaceUnboundSlot(needRewriteExpression, Maps.newHashMap()); diff --git a/regression-test/data/nereids_rules_p0/expression/test_date_comparison_boundary.out b/regression-test/data/nereids_rules_p0/expression/test_date_comparison_boundary.out new file mode 100644 index 00000000000000..67e437be692e7b --- /dev/null +++ b/regression-test/data/nereids_rules_p0/expression/test_date_comparison_boundary.out @@ -0,0 +1,184 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !reported -- +2026-11-13T15:03:12 + +-- !d_null -- +\N false + +-- !d_weeks_limits -- +1 + +-- !d_weeks_comparisons -- +true true false false false false + +-- !d_days_limits -- +1 + +-- !d_days_comparisons -- +true true false false false false + +-- !d_hours_limits -- +1 + +-- !d_hours_comparisons -- +true true false false false false + +-- !d_minutes_limits -- +1 + +-- !d_minutes_comparisons -- +true true false false false false + +-- !d_seconds_limits -- +1 + +-- !d_seconds_comparisons -- +true true false false false false + +-- !dt_null -- +\N false + +-- !dt_weeks_limits -- +1 + +-- !dt_weeks_comparisons -- +true true false false false false + +-- !dt_days_limits -- +1 + +-- !dt_days_comparisons -- +true true false false false false + +-- !dt_hours_limits -- +1 + +-- !dt_hours_comparisons -- +true true false false false false + +-- !dt_minutes_limits -- +1 + +-- !dt_minutes_comparisons -- +true true false false false false + +-- !dt_seconds_limits -- +1 + +-- !dt_seconds_comparisons -- +true true false false false false + +-- !dv2_null -- +\N false + +-- !dv2_weeks_limits -- +1 + +-- !dv2_weeks_comparisons -- +true true false false false false + +-- !dv2_days_limits -- +1 + +-- !dv2_days_comparisons -- +true true false false false false + +-- !dv2_hours_limits -- +1 + +-- !dv2_hours_comparisons -- +true true false false false false + +-- !dv2_minutes_limits -- +1 + +-- !dv2_minutes_comparisons -- +true true false false false false + +-- !dv2_seconds_limits -- +1 + +-- !dv2_seconds_comparisons -- +true true false false false false + +-- !tz_null -- +\N false + +-- !tz_weeks_limits -- +1 + +-- !tz_weeks_comparisons -- +true true false false false false + +-- !tz_days_limits -- +1 + +-- !tz_days_comparisons -- +true true false false false false + +-- !tz_hours_limits -- +1 + +-- !tz_hours_comparisons -- +true true false false false false + +-- !tz_minutes_limits -- +1 + +-- !tz_minutes_comparisons -- +true true false false false false + +-- !tz_seconds_limits -- +1 + +-- !tz_seconds_comparisons -- +true true false false false false + +-- !ns_null -- +\N false + +-- !ns_weeks_limits -- +1 + +-- !ns_weeks_comparisons -- +true true false false false false + +-- !ns_days_limits -- +1 + +-- !ns_days_comparisons -- +true true false false false false + +-- !ns_hours_limits -- +1 + +-- !ns_hours_comparisons -- +true true false false false false + +-- !ns_minutes_limits -- +1 + +-- !ns_minutes_comparisons -- +true true false false false false + +-- !ns_seconds_limits -- +1 + +-- !ns_seconds_comparisons -- +true true false false false false + +-- !column_boundary -- +1 + +-- !aliases -- +1 + +-- !other_units -- +1 + +-- !nested -- +1 + +-- !be_folding -- +1 + diff --git a/regression-test/suites/nereids_rules_p0/expression/test_date_comparison_boundary.groovy b/regression-test/suites/nereids_rules_p0/expression/test_date_comparison_boundary.groovy new file mode 100644 index 00000000000000..d55b3fc5a6c89e --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/expression/test_date_comparison_boundary.groovy @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_date_comparison_boundary") { + sql "DROP TABLE IF EXISTS test_date_comparison_boundary" + // One row is sufficient. All source values permit each tested date operation. + // The error comes from arithmetic on the comparison boundary, not from this row. + sql """CREATE TABLE test_date_comparison_boundary ( + id INT, d DATE, dt DATETIME(6), dv2 DATETIMEV2(0), + tz TIMESTAMPTZ(6), ns TIMESTAMP_NS, upper_bound DATETIMEV2(0) + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num"="1")""" + sql """INSERT INTO test_date_comparison_boundary VALUES ( + 1, '2026-11-13', '2026-11-13 15:03:12.123456', '2026-11-13 15:03:12', + '2026-11-13 15:03:12.123456+00:00', '2026-11-13 15:03:12.123456789', + '9999-12-31 23:59:59')""" + + // This is the reported query. Moving subtraction to the right adds one day to year 9999. + order_qt_reported """SELECT dv2 FROM test_date_comparison_boundary + WHERE days_sub(dv2, 1) < '9999-12-31 23:59:59'""" + + // Use each type's exact limits. Explicit UTC offsets make the TZ limits independent of the session zone. + def bounds = [ + d: ["0000-01-01", "9999-12-31"], + dt: ["0000-01-01 00:00:00", "9999-12-31 23:59:59.999999"], + dv2: ["0000-01-01 00:00:00", "9999-12-31 23:59:59"], + tz: ["0000-01-01 00:00:00+00:00", "9999-12-31 23:59:59.999999+00:00"], + ns: ["1677-09-21 00:12:43.145224192", "2262-04-11 23:47:16.854775807"] + ] + bounds.each { column, limits -> + // The same stored row supplies a NULL input. No second row is needed. + // A normal comparison must return NULL. Null-safe equality must return false. + "order_qt_${column}_null"("""SELECT + days_sub(if(id = 1, NULL, ${column}), 1) < '${limits[1]}', + days_sub(if(id = 1, NULL, ${column}), 1) <=> '${limits[1]}' + FROM test_date_comparison_boundary""") + ["weeks", "days", "hours", "minutes", "seconds"].each { unit -> + // Operations on hours, minutes, and seconds return DATETIME for DATE. Use its upper limit. + def upperBoundary = column == "d" && !(unit in ["weeks", "days"]) + ? "9999-12-31 23:59:59" : limits[1] + // Check both limits and both interval signs. All four predicates must return the one row. + "order_qt_${column}_${unit}_limits"("""SELECT id FROM test_date_comparison_boundary + WHERE ${unit}_sub(${column}, 1) < '${upperBoundary}' + AND ${unit}_add(${column}, 1) > '${limits[0]}' + AND ${unit}_add(${column}, -1) < '${upperBoundary}' + AND ${unit}_sub(${column}, -1) > '${limits[0]}'""") + // The results must be true, true, false, false, false, false. + // An invalid inverse must not replace these results with NULL. + def comparisons = ["<", "<=", ">", ">=", "=", "<=>"].collect { op -> + "${unit}_sub(${column}, 1) ${op} '${upperBoundary}'" + }.join(", ") + "order_qt_${column}_${unit}_comparisons"( + "SELECT ${comparisons} FROM test_date_comparison_boundary") + } + } + + // A column can supply the upper limit. It cannot be checked by constant folding during planning. + order_qt_column_boundary """SELECT id FROM test_date_comparison_boundary + WHERE days_sub(dv2, 1) < upper_bound""" + + // SQL aliases and INTERVAL expressions bind to the same day arithmetic functions. + order_qt_aliases """SELECT id FROM test_date_comparison_boundary + WHERE date_sub(dv2, 1) < '9999-12-31 23:59:59' + AND date_add(dv2, -1) < '9999-12-31 23:59:59' + AND subdate(dv2, 1) < '9999-12-31 23:59:59' + AND adddate(dv2, -1) < '9999-12-31 23:59:59' + AND dv2 - INTERVAL 1 DAY < '9999-12-31 23:59:59' + AND dv2 + INTERVAL 1 DAY > '0000-01-01 00:00:00' + AND timestampadd(DAY, -1, dv2) < '9999-12-31 23:59:59' + AND '9999-12-31 23:59:59' > days_sub(dv2, 1)""" + + // Month and year arithmetic cannot be reversed because month lengths differ. + // Smaller time units and compound intervals do not use the inverse comparison rule. + order_qt_other_units """SELECT id FROM test_date_comparison_boundary + WHERE years_sub(dv2, 1) < '9999-12-31 23:59:59' + AND quarters_sub(dv2, 1) < '9999-12-31 23:59:59' + AND months_sub(dv2, 1) < '9999-12-31 23:59:59' + AND milliseconds_sub(dt, 1) < '9999-12-31 23:59:59.999999' + AND microseconds_sub(tz, 1) < '9999-12-31 23:59:59.999999+00:00' + AND nanoseconds_sub(ns, 1) < '2262-04-11 23:47:16.854775807' + AND dv2 - INTERVAL '1 00:00:00' DAY_SECOND < '9999-12-31 23:59:59'""" + + // Each step must be checked. The outer inverse is valid; the inner inverse exceeds the range. + order_qt_nested """SELECT id FROM test_date_comparison_boundary + WHERE days_sub(days_sub(dv2, 1), 1) < '9999-12-30 23:59:59'""" + + // Valid inverse boundaries must still allow the existing optimization. + explain { + sql "SELECT id FROM test_date_comparison_boundary WHERE days_sub(dv2, 1) < '2026-11-14 15:03:12'" + notContains "days_sub" + contains "2026-11-15 15:03:12" + } + // Checking an optional rewrite must not send an invalid inverse to BE constant folding. + def originalFoldByBe = sql("SELECT @@enable_fold_constant_by_be")[0][0] + try { + sql "SET enable_fold_constant_by_be = true" + order_qt_be_folding """SELECT id FROM test_date_comparison_boundary + WHERE days_sub(dv2, 1) < '9999-12-31 23:59:59'""" + } finally { + sql "SET enable_fold_constant_by_be = ${originalFoldByBe}" + } + // The fix must not hide an error in date arithmetic that the user actually requested. + test { + sql """SELECT id FROM test_date_comparison_boundary + WHERE days_add(upper_bound, 1) > '0000-01-01 00:00:00'""" + exception "out of range" + } +}