Skip to content

Commit 6530680

Browse files
MisterRaindropMisterRaindrop
committed
feat(hive): render Iceberg schemas as Hive DDL columns
Creating an Iceberg table through HMS means rendering its schema as Hive DDL. Add `TypeToHiveString` for one `Type` and `SchemaToHiveColumns` for a whole `Schema`; nested struct / list / map render recursively. `timestamptz` follows iceberg-java and emits `timestamp with local time zone`, since engines reading through HMS take their column types from this DDL. Java's sub-Hive-3 fallback is not reproduced: there is no Hive-version probe here and the vendored IDL is Hive 4.0.1. Iceberg v3 types that no reference maps to a real Hive type (timestamp_ns, variant, geometry, geography, unknown) return kNotImplemented rather than guessing a spelling.
1 parent 7dcc4ce commit 6530680

5 files changed

Lines changed: 423 additions & 3 deletions

File tree

‎src/iceberg/catalog/hive/CMakeLists.txt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,12 @@ if(NOT TARGET thrift::thrift)
5454
"-DICEBERG_BUNDLE_THRIFT=OFF against a system Thrift install.")
5555
endif()
5656

57-
set(ICEBERG_HIVE_SOURCES hive_catalog.cc hive_catalog_properties.cc hms_client.cc
58-
${ICEBERG_HIVE_THRIFT_GEN_SOURCES})
57+
set(ICEBERG_HIVE_SOURCES
58+
hive_catalog.cc
59+
hive_catalog_properties.cc
60+
hive_schema.cc
61+
hms_client.cc
62+
${ICEBERG_HIVE_THRIFT_GEN_SOURCES})
5963

6064
set(ICEBERG_HIVE_STATIC_BUILD_INTERFACE_LIBS)
6165
set(ICEBERG_HIVE_SHARED_BUILD_INTERFACE_LIBS)

‎src/iceberg/catalog/hive/hive_schema.cc‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/catalog/hive/hive_schema.h"
21+
22+
#include <format>
23+
#include <string>
24+
#include <vector>
25+
26+
#include "iceberg/schema.h"
27+
#include "iceberg/schema_field.h"
28+
#include "iceberg/type.h"
29+
#include "iceberg/util/checked_cast.h"
30+
#include "iceberg/util/macros.h"
31+
32+
namespace iceberg::hive {
33+
34+
namespace {
35+
36+
// Hive doesn't have a precise counterpart to every Iceberg primitive
37+
// type; the closest equivalent is chosen for each case below. Mapping
38+
// matches iceberg-rust's `HiveSchemaBuilder::primitive`.
39+
Result<std::string> PrimitiveToHive(const Type& type) {
40+
switch (type.type_id()) {
41+
case TypeId::kBoolean:
42+
return std::string("boolean");
43+
case TypeId::kInt:
44+
return std::string("int");
45+
case TypeId::kLong:
46+
return std::string("bigint");
47+
case TypeId::kFloat:
48+
return std::string("float");
49+
case TypeId::kDouble:
50+
return std::string("double");
51+
case TypeId::kDate:
52+
return std::string("date");
53+
case TypeId::kTimestamp:
54+
return std::string("timestamp");
55+
case TypeId::kTimestampTz:
56+
// Hive's TIMESTAMPLOCALTZ, the spelling iceberg-java emits on
57+
// Hive 3+. Engines reading through HMS must see the tz-adjusted
58+
// type, otherwise they read an Iceberg timestamptz column as a
59+
// naive timestamp. See the header for why the Hive 2 fallback is
60+
// not reproduced.
61+
return std::string("timestamp with local time zone");
62+
case TypeId::kTime:
63+
case TypeId::kString:
64+
case TypeId::kUuid:
65+
return std::string("string");
66+
case TypeId::kBinary:
67+
case TypeId::kFixed:
68+
return std::string("binary");
69+
case TypeId::kDecimal: {
70+
const auto& decimal = internal::checked_cast<const DecimalType&>(type);
71+
return std::format("decimal({},{})", decimal.precision(), decimal.scale());
72+
}
73+
default:
74+
return NotImplemented("Unhandled primitive Iceberg type id: {}",
75+
static_cast<int>(type.type_id()));
76+
}
77+
}
78+
79+
Result<std::string> StructToHive(const StructType& type) {
80+
std::string out = "struct<";
81+
bool first = true;
82+
for (const auto& field : type.fields()) {
83+
if (!first) {
84+
out += ",";
85+
}
86+
ICEBERG_ASSIGN_OR_RAISE(auto field_type, TypeToHiveString(*field.type()));
87+
out += std::string(field.name());
88+
out += ":";
89+
out += field_type;
90+
first = false;
91+
}
92+
out += ">";
93+
return out;
94+
}
95+
96+
Result<std::string> ListToHive(const ListType& type) {
97+
ICEBERG_ASSIGN_OR_RAISE(auto inner, TypeToHiveString(*type.element().type()));
98+
return std::format("array<{}>", inner);
99+
}
100+
101+
Result<std::string> MapToHive(const MapType& type) {
102+
ICEBERG_ASSIGN_OR_RAISE(auto key_type, TypeToHiveString(*type.key().type()));
103+
ICEBERG_ASSIGN_OR_RAISE(auto value_type, TypeToHiveString(*type.value().type()));
104+
return std::format("map<{},{}>", key_type, value_type);
105+
}
106+
107+
} // namespace
108+
109+
Result<std::string> TypeToHiveString(const Type& type) {
110+
switch (type.type_id()) {
111+
case TypeId::kStruct:
112+
return StructToHive(internal::checked_cast<const StructType&>(type));
113+
case TypeId::kList:
114+
return ListToHive(internal::checked_cast<const ListType&>(type));
115+
case TypeId::kMap:
116+
return MapToHive(internal::checked_cast<const MapType&>(type));
117+
default:
118+
return PrimitiveToHive(type);
119+
}
120+
}
121+
122+
Result<std::vector<HiveColumn>> SchemaToHiveColumns(const Schema& schema) {
123+
std::vector<HiveColumn> columns;
124+
columns.reserve(schema.fields().size());
125+
for (const auto& field : schema.fields()) {
126+
ICEBERG_ASSIGN_OR_RAISE(auto type_string, TypeToHiveString(*field.type()));
127+
columns.push_back(HiveColumn{.name = std::string(field.name()),
128+
.type_string = std::move(type_string),
129+
.comment = std::string(field.doc())});
130+
}
131+
return columns;
132+
}
133+
134+
} // namespace iceberg::hive

‎src/iceberg/catalog/hive/hive_schema.h‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
#include <string>
23+
#include <vector>
24+
25+
#include "iceberg/catalog/hive/iceberg_hive_export.h"
26+
#include "iceberg/result.h"
27+
#include "iceberg/type_fwd.h"
28+
29+
/// \file iceberg/catalog/hive/hive_schema.h
30+
/// \brief Convert an Iceberg `Schema` (or any nested `Type`) into the
31+
/// Hive DDL representation that HMS expects on FieldSchema.type.
32+
///
33+
/// The conversion is the C++ port of iceberg-rust's `HiveSchemaBuilder`
34+
/// (`crates/catalog/hms/src/schema.rs`) and mirrors Java's
35+
/// `HiveSchemaUtil` for the subset of types Hive can represent.
36+
37+
namespace iceberg::hive {
38+
39+
/// \brief A column entry for Hive's HMS table schema.
40+
///
41+
/// HiveCatalog converts each top-level Iceberg `SchemaField` into one of
42+
/// these, then HmsClient further wraps them into Thrift `FieldSchema`
43+
/// records before passing to `create_table` / `alter_table`. Nested
44+
/// types are flattened into the `type_string` field via DDL syntax
45+
/// (e.g. `"struct<a:int,b:string>"`, `"array<int>"`, `"map<string,int>"`).
46+
struct ICEBERG_HIVE_EXPORT HiveColumn {
47+
std::string name;
48+
std::string type_string;
49+
std::string comment;
50+
};
51+
52+
/// \brief Render an Iceberg `Type` as a Hive DDL type string.
53+
///
54+
/// Supported primitive mappings (matching iceberg-rust):
55+
/// boolean -> boolean
56+
/// int -> int
57+
/// long -> bigint
58+
/// float -> float
59+
/// double -> double
60+
/// date -> date
61+
/// timestamp -> timestamp
62+
/// timestamptz -> timestamp with local time zone
63+
/// time / string / uuid -> string
64+
/// binary / fixed -> binary
65+
/// decimal(p, s) -> decimal(p,s)
66+
/// struct<...> -> struct<f:t,...>
67+
/// list<T> -> array<T>
68+
/// map<K, V> -> map<K,V>
69+
///
70+
/// `timestamptz` follows iceberg-java, which emits Hive's
71+
/// `timestamp with local time zone` (TIMESTAMPLOCALTZ). Engines that
72+
/// read the table through HMS -- Hive, Spark, Trino -- take their
73+
/// column types from this DDL, so downgrading to a naive `timestamp`
74+
/// would silently misrepresent a tz-adjusted column to every one of
75+
/// them. iceberg-rust instead rejects zoned timestamps outright, which
76+
/// would make such tables uncreatable; that behaviour is not copied.
77+
///
78+
/// Java picks the spelling at runtime and falls back to plain
79+
/// `timestamp` below Hive 3, where TIMESTAMPLOCALTZ does not exist.
80+
/// That fallback is not reproduced: iceberg-cpp has no Hive-version
81+
/// probe, the vendored Metastore IDL is Hive 4.0.1, and Hive 2 has been
82+
/// end-of-life since 2019.
83+
ICEBERG_HIVE_EXPORT Result<std::string> TypeToHiveString(const Type& type);
84+
85+
/// \brief Convert each top-level field in `schema` to a `HiveColumn`.
86+
///
87+
/// `optional` is intentionally not surfaced — HMS treats every column
88+
/// as nullable. Iceberg `doc` is copied into `comment` so DESCRIBE
89+
/// statements in Hive show the column documentation.
90+
ICEBERG_HIVE_EXPORT Result<std::vector<HiveColumn>> SchemaToHiveColumns(
91+
const Schema& schema);
92+
93+
} // namespace iceberg::hive

‎src/iceberg/test/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,5 +354,5 @@ if(ICEBERG_BUILD_HIVE)
354354
add_test(NAME ${test_name} COMMAND ${test_name})
355355
endfunction()
356356

357-
add_hive_iceberg_test(hive_catalog_test SOURCES hms_client_test.cc)
357+
add_hive_iceberg_test(hive_catalog_test SOURCES hive_schema_test.cc hms_client_test.cc)
358358
endif()

0 commit comments

Comments
 (0)