Skip to content

Commit 69b9934

Browse files
feat(hive): add namespace and table conversion utilities
HMS speaks `Database` and `Table` records; Iceberg speaks `Namespace`, `TableIdentifier` and `Schema`. Add the conversion layer between them, plus the parameter keys (`metadata_location`, `table_type`, `EXTERNAL`) that iceberg-java's HiveCatalog and iceberg-rust's HmsCatalog both use to recognise an Iceberg table in HMS. The helpers deliberately produce plain structs (`HiveDatabase`, `HiveTable`) instead of Thrift objects, so the public iceberg_hive API stays free of Thrift includes and HmsClient remains the only translation unit that touches the generated bindings. Behaviour worth calling out: * A database owner is carried by `hive.metastore.database.owner` and `hive.metastore.database.owner-type`, the keys iceberg-java (HiveCatalog.HMS_DB_OWNER) and iceberg-rust both use. A bare `owner` on a namespace stays an ordinary Hive parameter; it is the *table* owner key. `owner-type` must accompany an owner and must name an HMS PrincipalType -- USER, GROUP or ROLE, compared case-insensitively -- so an unusable value fails here rather than at the Thrift boundary. * `ConvertToHiveDatabase` lifts the reserved properties (`comment`, `location` and the two owner keys) into dedicated `Database` fields and leaves everything else in `parameters`; `ConvertFromHiveDatabase` is its inverse. Namespaces deeper than one level are rejected -- HMS databases are flat. * `ConvertToHiveTable` writes the LazySimpleSerDe storage descriptor alongside the Iceberg parameters so Spark and Trino keep reading tables this catalog creates. * `ValidateIcebergTable` rejects an HMS table whose `table_type` is absent or not `ICEBERG` (compared case-insensitively), so later commits cannot hand a plain Hive table to the Iceberg code paths. * `GetDefaultTableLocation` reproduces the `<warehouse>/<ns>.db/<table>` layout that iceberg-java and iceberg-rust pick.
1 parent 2038d85 commit 69b9934

5 files changed

Lines changed: 594 additions & 1 deletion

File tree

‎src/iceberg/catalog/hive/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ set(ICEBERG_HIVE_SOURCES
5858
hive_catalog.cc
5959
hive_catalog_properties.cc
6060
hive_schema.cc
61+
hive_utils.cc
6162
hms_client.cc
6263
${ICEBERG_HIVE_THRIFT_GEN_SOURCES})
6364

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/catalog/hive/hive_utils.h"
21+
22+
#include <algorithm>
23+
#include <array>
24+
#include <format>
25+
#include <string>
26+
#include <string_view>
27+
#include <vector>
28+
29+
#include "iceberg/util/macros.h"
30+
#include "iceberg/util/string_util.h"
31+
32+
namespace iceberg::hive {
33+
34+
namespace {
35+
36+
// Keys lifted into dedicated `Database` fields, so they must not be
37+
// duplicated into `parameters`.
38+
constexpr std::array<std::string_view, 4> kDatabaseReservedKeys = {
39+
kCommentProperty, kLocationProperty, kHmsDbOwnerProperty, kHmsDbOwnerTypeProperty};
40+
41+
// The table equivalent: `location` and `owner` have dedicated `Table`
42+
// fields, and `comment` is not part of the Iceberg-on-HMS contract.
43+
constexpr std::array<std::string_view, 3> kTableReservedKeys = {
44+
kCommentProperty, kLocationProperty, kOwnerProperty};
45+
46+
// HMS `PrincipalType` names accepted for a database owner.
47+
constexpr std::array<std::string_view, 3> kPrincipalTypes = {"USER", "GROUP", "ROLE"};
48+
49+
bool IsDatabaseReservedKey(std::string_view key) {
50+
return std::ranges::find(kDatabaseReservedKeys, key) != kDatabaseReservedKeys.end();
51+
}
52+
53+
bool IsTableReservedKey(std::string_view key) {
54+
return std::ranges::find(kTableReservedKeys, key) != kTableReservedKeys.end();
55+
}
56+
57+
std::string GetOrEmpty(const std::unordered_map<std::string, std::string>& properties,
58+
std::string_view key) {
59+
auto it = properties.find(std::string(key));
60+
return it == properties.end() ? std::string() : it->second;
61+
}
62+
63+
std::string TrimTrailingSlash(std::string_view path) {
64+
while (path.size() > 1 && path.back() == '/') {
65+
path.remove_suffix(1);
66+
}
67+
return std::string(path);
68+
}
69+
70+
} // namespace
71+
72+
Status ValidateNamespace(const Namespace& ns) {
73+
if (ns.levels.size() != 1) {
74+
return InvalidArgument(
75+
"Hive Metastore only supports single-level namespaces; got {} level(s).",
76+
ns.levels.size());
77+
}
78+
if (ns.levels[0].empty()) {
79+
return InvalidArgument("Hive namespace cannot have an empty name.");
80+
}
81+
return {};
82+
}
83+
84+
Status ValidateOwnerSettings(
85+
const std::unordered_map<std::string, std::string>& properties) {
86+
auto owner_type = properties.find(std::string(kHmsDbOwnerTypeProperty));
87+
if (owner_type == properties.end()) {
88+
return {};
89+
}
90+
if (!properties.contains(std::string(kHmsDbOwnerProperty))) {
91+
return InvalidArgument("Hive namespace property '{}' requires '{}' to also be set.",
92+
kHmsDbOwnerTypeProperty, kHmsDbOwnerProperty);
93+
}
94+
const bool known = std::ranges::any_of(kPrincipalTypes, [&](std::string_view name) {
95+
return StringUtils::EqualsIgnoreCase(owner_type->second, name);
96+
});
97+
if (!known) {
98+
return InvalidArgument(
99+
"Hive namespace property '{}' has value '{}'; expected one of "
100+
"USER, GROUP or ROLE.",
101+
kHmsDbOwnerTypeProperty, owner_type->second);
102+
}
103+
return {};
104+
}
105+
106+
Result<HiveDatabase> ConvertToHiveDatabase(
107+
const Namespace& ns, const std::unordered_map<std::string, std::string>& properties) {
108+
ICEBERG_RETURN_UNEXPECTED(ValidateNamespace(ns));
109+
ICEBERG_RETURN_UNEXPECTED(ValidateOwnerSettings(properties));
110+
111+
HiveDatabase database;
112+
database.name = ns.levels[0];
113+
database.description = GetOrEmpty(properties, kCommentProperty);
114+
database.location_uri = GetOrEmpty(properties, kLocationProperty);
115+
database.owner_name = GetOrEmpty(properties, kHmsDbOwnerProperty);
116+
database.owner_type = GetOrEmpty(properties, kHmsDbOwnerTypeProperty);
117+
118+
for (const auto& [key, value] : properties) {
119+
if (!IsDatabaseReservedKey(key)) {
120+
database.parameters.emplace(key, value);
121+
}
122+
}
123+
return database;
124+
}
125+
126+
HiveNamespace ConvertFromHiveDatabase(const HiveDatabase& database) {
127+
HiveNamespace result;
128+
result.ns.levels = {database.name};
129+
result.properties = database.parameters;
130+
if (!database.description.empty()) {
131+
result.properties[std::string(kCommentProperty)] = database.description;
132+
}
133+
if (!database.location_uri.empty()) {
134+
result.properties[std::string(kLocationProperty)] = database.location_uri;
135+
}
136+
if (!database.owner_name.empty()) {
137+
result.properties[std::string(kHmsDbOwnerProperty)] = database.owner_name;
138+
}
139+
if (!database.owner_type.empty()) {
140+
result.properties[std::string(kHmsDbOwnerTypeProperty)] = database.owner_type;
141+
}
142+
return result;
143+
}
144+
145+
Result<HiveTable> ConvertToHiveTable(
146+
const TableIdentifier& identifier, const std::vector<HiveColumn>& columns,
147+
std::string_view metadata_location, std::string_view location,
148+
const std::unordered_map<std::string, std::string>& table_properties) {
149+
ICEBERG_RETURN_UNEXPECTED(ValidateNamespace(identifier.ns));
150+
ICEBERG_RETURN_UNEXPECTED(identifier.Validate());
151+
152+
HiveTable table;
153+
table.db_name = identifier.ns.levels[0];
154+
table.table_name = identifier.name;
155+
table.owner = GetOrEmpty(table_properties, kOwnerProperty);
156+
table.table_type = "EXTERNAL_TABLE";
157+
table.location = std::string(location);
158+
table.columns = columns;
159+
table.serde = std::string(kLazySimpleSerDe);
160+
table.input_format = std::string(kFileInputFormat);
161+
table.output_format = std::string(kFileOutputFormat);
162+
163+
// Mandatory Iceberg-on-HMS marker parameters.
164+
table.parameters.emplace(std::string(kMetadataLocationKey), metadata_location);
165+
table.parameters.emplace(std::string(kTableTypeKey), std::string(kTableTypeIceberg));
166+
table.parameters.emplace(std::string(kExternalKey), std::string(kExternalTrue));
167+
168+
// Forward any user-supplied table properties that aren't reserved.
169+
for (const auto& [key, value] : table_properties) {
170+
if (!IsTableReservedKey(key) && !table.parameters.contains(key)) {
171+
table.parameters.emplace(key, value);
172+
}
173+
}
174+
return table;
175+
}
176+
177+
Result<std::string> GetMetadataLocation(
178+
const std::unordered_map<std::string, std::string>& table_parameters) {
179+
auto it = table_parameters.find(std::string(kMetadataLocationKey));
180+
if (it == table_parameters.end() || it->second.empty()) {
181+
return NotFound("HMS table is missing '{}' parameter; not an Iceberg table.",
182+
kMetadataLocationKey);
183+
}
184+
return it->second;
185+
}
186+
187+
Status ValidateIcebergTable(
188+
const TableIdentifier& identifier,
189+
const std::unordered_map<std::string, std::string>& table_parameters) {
190+
auto it = table_parameters.find(std::string(kTableTypeKey));
191+
if (it == table_parameters.end() || it->second.empty()) {
192+
return NoSuchTable(
193+
"HMS table {} has no '{}' parameter; refusing to treat it as an Iceberg table.",
194+
identifier.ToString(), kTableTypeKey);
195+
}
196+
if (!StringUtils::EqualsIgnoreCase(it->second, kTableTypeIceberg)) {
197+
return NoSuchTable("HMS table {} has '{}={}'; expected '{}' (case-insensitive).",
198+
identifier.ToString(), kTableTypeKey, it->second,
199+
kTableTypeIceberg);
200+
}
201+
return {};
202+
}
203+
204+
std::string GetDefaultTableLocation(std::string_view warehouse, const Namespace& ns,
205+
std::string_view table_name) {
206+
std::string base = TrimTrailingSlash(warehouse);
207+
std::string ns_part;
208+
for (std::size_t i = 0; i < ns.levels.size(); ++i) {
209+
if (i > 0) {
210+
ns_part += ".";
211+
}
212+
ns_part += ns.levels[i];
213+
}
214+
return std::format("{}/{}.db/{}", base, ns_part, table_name);
215+
}
216+
217+
} // namespace iceberg::hive

0 commit comments

Comments
 (0)