Skip to content

[fix](be) Preserve local time in TIMESTAMPTZ day arithmetic #70003

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jacktengg wants to merge 8 commits into
base: master
Choose a base branch
from
Open

[fix](be) Preserve local time in TIMESTAMPTZ day arithmetic #70003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fcc93f3
[fix](be) Preserve local time in TIMESTAMPTZ day arithmetic
jacktengg Nov 2, 2026
4e4a662
[fix](be) Handle TIMESTAMPTZ day arithmetic at UTC boundaries
jacktengg Nov 2, 2026
a42bef5
[doc](fe) Explain the guard for date comparison rewrites
jacktengg Nov 4, 2026
2a70c4a
[improvement](be) Reuse the civil lookup in TIMESTAMPTZ day arithmetic
jacktengg Nov 10, 2026
633f33a
[fix](be) Preserve gap positions in TIMESTAMPTZ day arithmetic
jacktengg Nov 10, 2026
3b6bca3
[fix](fe) Validate inverse TIMESTAMPTZ day predicates
jacktengg Nov 13, 2026
da1e172
[refactor](fe) Revert the TIMESTAMPTZ-only inverse guard
jacktengg Nov 13, 2026
c158d92
[fix](nereids) Avoid overflow when rearranging date comparisons
jacktengg Nov 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion be/src/core/value/timestamptz_value.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,35 @@ int TimestampTzValue::to_buffer(char* buffer, const cctz::time_zone& tz, int sca
return len;
}

bool TimestampTzValue::date_add_days(int64_t days, const cctz::time_zone& local_time_zone) {
// Predicate movement treats a zero interval as an infallible identity, including
// UTC year 0000/9999 whose local display year can be outside [0, 9999].
if (days == 0) {
return true;
}
const cctz::civil_second utc_cs(year(), month(), day(), hour(), minute(), second());
const auto local = local_time_zone.lookup(cctz::convert(utc_cs, cctz::utc_time_zone()));
const auto target = cctz::civil_day(local.cs) + days;
// UTC boundaries and a day delta can produce an unrepresentable local year.
if (local.cs.year() < 0 || local.cs.year() > 9999 || target.year() < 0 ||
target.year() > 9999) {
return false;
}
// Match Java's civil calendar, which permits February 29 in year 0. For example,
// UTC 0000-02-28 20:00 at +08:00 passes through that local date before subtracting
// a day. Validate the final UTC value rather than this local intermediate as DateV2.
DateV2Value<DateTimeV2ValueType> local_dt;
local_dt.unchecked_set_time(
static_cast<uint16_t>(target.year()), static_cast<uint8_t>(target.month()),
static_cast<uint8_t>(target.day()), static_cast<uint8_t>(local.cs.hour()),
static_cast<uint8_t>(local.cs.minute()), static_cast<uint8_t>(local.cs.second()),
microsecond());
// Preserve the input offset when the target repeats, e.g. the second autumn 01:30.
convert_local_to_utc(local_time_zone, local_dt, local.offset);
Comment thread
jacktengg marked this conversation as resolved.
Resolved
// A valid local result can still fall outside the supported UTC year range.
return is_valid_date();
}

bool TimestampTzValue::from_datetime(const DateV2Value<DateTimeV2ValueType>& origin_dt,
const cctz::time_zone& local_time_zone, int dt_scale,
int tz_scale) {
Expand Down Expand Up @@ -168,7 +197,9 @@ void TimestampTzValue::convert_local_to_utc(const cctz::time_zone& local_time_zo
cctz::civil_second local_cs(dt.year(), dt.month(), dt.day(), dt.hour(), dt.minute(),
dt.second());
const auto lookup = local_time_zone.lookup(local_cs);
cctz::time_point<cctz::seconds> local_tp = cctz::convert(local_cs, local_time_zone);
// Use the pre-transition offset to shift gap input by the gap length, e.g. New York's
// 2024-03-10 02:30 becomes 03:30. The preferred offset applies only to an overlap.
auto local_tp = lookup.pre;
Comment thread
jacktengg marked this conversation as resolved.
Resolved

if (lookup.kind == cctz::time_zone::civil_lookup::REPEATED) {
const auto pre_offset = local_time_zone.lookup(lookup.pre).offset;
Expand Down
3 changes: 3 additions & 0 deletions be/src/core/value/timestamptz_value.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,9 @@ class TimestampTzValue {
return _utc_dt.date_add_interval<unit, need_check>(interval);
}

// Add session-local calendar days, preserving the input offset in a DST overlap.
bool date_add_days(int64_t days, const cctz::time_zone& local_time_zone);

// truncate datetime to specified unit
template <TimeUnit unit>
bool datetime_trunc() {
Expand Down
122 changes: 75 additions & 47 deletions be/src/exprs/function/function_date_or_datetime_computation.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,17 @@ namespace doris {

template <TimeUnit unit, PrimitiveType ArgType, typename IntervalType>
auto date_time_add(const typename PrimitiveTypeTraits<ArgType>::DataType::FieldType& t,
IntervalType delta) {
IntervalType delta, const cctz::time_zone& time_zone) {
// e.g.: for DatatypeDatetimeV2, cast from u64 to DateV2Value<DateTimeV2ValueType>
auto ts_value = t;
if constexpr ((unit == TimeUnit::DAY || unit == TimeUnit::WEEK) &&
is_date_v2_or_datetime_v2(ArgType)) {
if constexpr (ArgType == TYPE_TIMESTAMPTZ && unit == TimeUnit::DAY) {
// In New York, 2024-03-09 12:00 + 1 calendar day is only 23 elapsed hours.
if (!ts_value.date_add_days(delta, time_zone)) [[unlikely]] {
throw_out_of_bound_date_int(get_time_unit_name(unit), t, delta);
}
return ts_value;
} else if constexpr ((unit == TimeUnit::DAY || unit == TimeUnit::WEEK) &&
is_date_v2_or_datetime_v2(ArgType)) {
// DAY / WEEK only move the date part. `date_add_days` is inline, so the whole per-row
// computation folds into the caller's loop: no TimeInterval, no second-level arithmetic,
// just two L1-resident dictionary lookups. The input comes from a column and is already
Expand All @@ -102,34 +108,36 @@ auto date_time_add(const typename PrimitiveTypeTraits<ArgType>::DataType::FieldT
}
}

#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \
template <PrimitiveType PType> \
struct CLASS { \
/* return type must be same with arg type. cast have already be planned in FE */ \
static constexpr PrimitiveType ArgPType = PType; \
static constexpr PrimitiveType ReturnType = PType; \
/* for interval <= minute, use bigint. otherwise int. */ \
static constexpr PrimitiveType IntervalPType = \
(TimeUnit::UNIT == TimeUnit::YEAR || TimeUnit::UNIT == TimeUnit::QUARTER || \
TimeUnit::UNIT == TimeUnit::MONTH || TimeUnit::UNIT == TimeUnit::WEEK || \
TimeUnit::UNIT == TimeUnit::DAY || TimeUnit::UNIT == TimeUnit::HOUR) \
? PrimitiveType::TYPE_INT \
: PrimitiveType::TYPE_BIGINT; \
using InputValueType = typename PrimitiveTypeTraits<PType>::DataType::FieldType; \
using ReturnValueType = InputValueType; \
using IntervalDataType = typename PrimitiveTypeTraits<IntervalPType>::DataType; \
using IntervalValueType = IntervalDataType::FieldType; \
\
static constexpr auto name = #NAME; \
static constexpr auto is_nullable = false; \
static inline ReturnValueType execute(const InputValueType& t, IntervalValueType delta) { \
return date_time_add<TimeUnit::UNIT, PType, IntervalValueType>(t, delta); \
} \
\
static DataTypes get_variadic_argument_types() { \
return {std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>(), \
std::make_shared<typename PrimitiveTypeTraits<IntervalPType>::DataType>()}; \
} \
#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \
template <PrimitiveType PType> \
struct CLASS { \
/* return type must be same with arg type. cast have already be planned in FE */ \
static constexpr PrimitiveType ArgPType = PType; \
static constexpr PrimitiveType ReturnType = PType; \
/* for interval <= minute, use bigint. otherwise int. */ \
static constexpr PrimitiveType IntervalPType = \
(TimeUnit::UNIT == TimeUnit::YEAR || TimeUnit::UNIT == TimeUnit::QUARTER || \
TimeUnit::UNIT == TimeUnit::MONTH || TimeUnit::UNIT == TimeUnit::WEEK || \
TimeUnit::UNIT == TimeUnit::DAY || TimeUnit::UNIT == TimeUnit::HOUR) \
? PrimitiveType::TYPE_INT \
: PrimitiveType::TYPE_BIGINT; \
using InputValueType = typename PrimitiveTypeTraits<PType>::DataType::FieldType; \
using ReturnValueType = InputValueType; \
using IntervalDataType = typename PrimitiveTypeTraits<IntervalPType>::DataType; \
using IntervalValueType = IntervalDataType::FieldType; \
\
static constexpr auto name = #NAME; \
static constexpr auto is_nullable = false; \
static inline ReturnValueType execute( \
const InputValueType& t, IntervalValueType delta, \
const cctz::time_zone& time_zone = cctz::utc_time_zone()) { \
return date_time_add<TimeUnit::UNIT, PType, IntervalValueType>(t, delta, time_zone); \
} \
\
static DataTypes get_variadic_argument_types() { \
return {std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>(), \
std::make_shared<typename PrimitiveTypeTraits<IntervalPType>::DataType>()}; \
} \
}
ADD_TIME_FUNCTION_IMPL(AddMicrosecondsImpl, microseconds_add, MICROSECOND);
ADD_TIME_FUNCTION_IMPL(AddMillisecondsImpl, milliseconds_add, MILLISECOND);
Expand Down Expand Up @@ -422,9 +430,11 @@ struct AddQuartersImpl {

static constexpr auto name = "quarters_add";
static constexpr auto is_nullable = false;
static inline ReturnValueType execute(const InputValueType& t, Int64 delta) {
static inline ReturnValueType execute(
const InputValueType& t, Int64 delta,
const cctz::time_zone& time_zone = cctz::utc_time_zone()) {
// Preserve both the month offset and the negation of INT_MIN in quarters_sub.
return date_time_add<TimeUnit::MONTH, PType, Int64>(t, 3 * delta);
return date_time_add<TimeUnit::MONTH, PType, Int64>(t, 3 * delta, time_zone);
}

static DataTypes get_variadic_argument_types() {
Expand All @@ -440,8 +450,10 @@ struct SubtractIntervalImpl {
using InputValueType = typename Transform::InputValueType;
using ReturnValueType = typename Transform::ReturnValueType;
static constexpr auto is_nullable = false;
static inline ReturnValueType execute(const InputValueType& t, Int64 delta) {
return Transform::execute(t, -delta);
static inline ReturnValueType execute(
const InputValueType& t, Int64 delta,
const cctz::time_zone& time_zone = cctz::utc_time_zone()) {
return Transform::execute(t, -delta, time_zone);
}

static DataTypes get_variadic_argument_types() {
Expand Down Expand Up @@ -760,6 +772,18 @@ struct DateTimeOp {
// arg1 maybe just delta value(e.g. DataTypeInt32, not datelike type)
constexpr static bool CastType1 = is_date_type(DataType1);

static ToType execute(const ValueType0& from, const ValueType1& delta,
FunctionContext* context) {
// Date add/sub needs the session zone; diff and extract transforms take two arguments.
if constexpr (requires {
Transform::execute(from, delta, context->state()->timezone_obj());
}) {
return Transform::execute(from, delta, context->state()->timezone_obj());
} else {
return Transform::execute(from, delta);
}
}

// Use for compatibility with TimeUnionType interval(string type)
static inline ValueType1 get_element(const IntervalColumnType& data, size_t i) {
if constexpr (DataType1 == TYPE_STRING) {
Expand All @@ -772,18 +796,20 @@ struct DateTimeOp {
// execute on the null value's nested value may cause false positive exception, so use nullmaps to skip them.
static void vector_vector(const PaddedPODArray<ValueType0>& vec_from0,
const IntervalColumnType& vec_from1, PaddedPODArray<ToType>& vec_to,
const NullMap* nullmap0, const NullMap* nullmap1) {
const NullMap* nullmap0, const NullMap* nullmap1,
FunctionContext* context) {
for (size_t i = 0; i < vec_from0.size(); ++i) {
if ((nullmap0 && (*nullmap0)[i]) || (nullmap1 && (*nullmap1)[i])) [[unlikely]] {
continue;
}
vec_to[i] = Transform::execute(vec_from0[i], get_element(vec_from1, i));
vec_to[i] = execute(vec_from0[i], get_element(vec_from1, i), context);
}
}

static void vector_constant(const PaddedPODArray<ValueType0>& vec_from,
PaddedPODArray<ToType>& vec_to, const ValueType1& delta,
const NullMap* nullmap0, const NullMap* nullmap1) {
const NullMap* nullmap0, const NullMap* nullmap1,
FunctionContext* context) {
if (nullmap1 && (*nullmap1)[0]) [[unlikely]] {
return;
}
Expand All @@ -792,13 +818,13 @@ struct DateTimeOp {
if (nullmap0 && (*nullmap0)[i]) [[unlikely]] {
continue;
}
vec_to[i] = Transform::execute(vec_from[i], delta);
vec_to[i] = execute(vec_from[i], delta, context);
}
}

static void constant_vector(const ValueType0& from, PaddedPODArray<ToType>& vec_to,
const IntervalColumnType& delta, const NullMap* nullmap0,
const NullMap* nullmap1) {
const NullMap* nullmap1, FunctionContext* context) {
if (nullmap0 && (*nullmap0)[0]) [[unlikely]] {
return;
}
Expand All @@ -807,7 +833,7 @@ struct DateTimeOp {
if (nullmap1 && (*nullmap1)[i]) [[unlikely]] {
continue;
}
vec_to[i] = Transform::execute(from, get_element(delta, i));
vec_to[i] = execute(from, get_element(delta, i), context);
}
}
};
Expand Down Expand Up @@ -877,11 +903,12 @@ class FunctionTimeDiff : public IFunction {
const auto& col1_inside_const =
assert_cast<const RightColumnType&>(nest_col1_const->get_data_column());
Op::vector_constant(sources->get_data(), res_col->get_data(),
Op::get_element(col1_inside_const, 0), nullmap0, nullmap1);
Op::get_element(col1_inside_const, 0), nullmap0, nullmap1,
context);
} else { // vector-vector
const auto& concrete_col1 = assert_cast<const RightColumnType&>(*nest_col1);
Op::vector_vector(sources->get_data(), concrete_col1, res_col->get_data(), nullmap0,
nullmap1);
nullmap1, context);
}

// update result nullmap with inputs
Expand All @@ -908,7 +935,7 @@ class FunctionTimeDiff : public IFunction {
const ColumnPtr nested_col1 = remove_nullable(col1);
const auto& concrete_col1 = assert_cast<const RightColumnType&>(*nested_col1);
Op::constant_vector(col0_inside_const.get_data()[0], res_col->get_data(), concrete_col1,
nullmap0, nullmap1);
nullmap0, nullmap1, context);

// update result nullmap with inputs
if (result_nullable) {
Expand Down Expand Up @@ -1002,11 +1029,12 @@ class FunctionDateOrDateTimeComputation : public IFunction {
const auto& col1_inside_const =
assert_cast<const IntervalColumnType&>(nest_col1_const->get_data_column());
Op::vector_constant(sources->get_data(), res_col->get_data(),
Op::get_element(col1_inside_const, 0), nullmap0, nullmap1);
Op::get_element(col1_inside_const, 0), nullmap0, nullmap1,
context);
} else { // vector-vector
const auto& concrete_col1 = assert_cast<const IntervalColumnType&>(*nest_col1);
Op::vector_vector(sources->get_data(), concrete_col1, res_col->get_data(), nullmap0,
nullmap1);
nullmap1, context);
}

// update result nullmap with inputs
Expand All @@ -1033,7 +1061,7 @@ class FunctionDateOrDateTimeComputation : public IFunction {
const ColumnPtr nested_col1 = remove_nullable(col1);
const auto& concrete_col1 = assert_cast<const IntervalColumnType&>(*nested_col1);
Op::constant_vector(col0_inside_const.get_data()[0], res_col->get_data(), concrete_col1,
nullmap0, nullmap1);
nullmap0, nullmap1, context);

// update result nullmap with inputs
if (result_nullable) {
Expand Down
Loading
Loading