Skip to content

Commit 8a4041c

Browse files
feat(hive): add namespace and table conversion utilities
HMS speaks `Database` and `Table`; Iceberg speaks `Namespace`, `TableIdentifier` and `Schema`. Add the conversion between them, plus the parameter keys (`metadata_location`, `table_type`, `EXTERNAL`) that iceberg-java and iceberg-rust use to recognise an Iceberg table, and the LazySimpleSerDe descriptor that keeps Spark and Trino reading it. The helpers produce plain structs rather than Thrift objects, so HmsClient stays the only translation unit that includes the generated bindings. A database owner is carried by `hive.metastore.database.owner` and `hive.metastore.database.owner-type`, the keys both references use; a bare `owner` is the table owner key. `owner-type` requires an owner and must name an HMS PrincipalType -- USER, GROUP or ROLE, compared case-insensitively.
1 parent 6530680 commit 8a4041c

5 files changed

Lines changed: 594 additions & 1 deletion

File tree

‎src/iceberg/catalog/hive/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ set(ICEBERG_HIVE_SOURCES
5858
hive_catalog.cc
5959
hive_catalog_properties.cc
6060
hive_schema.cc
61+
hive_utils.cc
6162
hms_client.cc
6263
${ICEBERG_HIVE_THRIFT_GEN_SOURCES})
6364

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/catalog/hive/hive_utils.h"
21+
22+
#include <algorithm>
23+
#include <array>
24+
#include <format>
25+
#include <string>
26+
#include <string_view>
27+
#include <vector>
28+
29+
#include "iceberg/util/macros.h"
30+
#include "iceberg/util/string_util.h"
31+
32+
namespace iceberg::hive {
33+
34+
namespace {
35+
36+
// Keys lifted into dedicated `Database` fields, so they must not be
37+
// duplicated into `parameters`.
38+
constexpr std::array<std::string_view, 4> kDatabaseReservedKeys = {
39+
kCommentProperty, kLocationProperty, kHmsDbOwnerProperty, kHmsDbOwnerTypeProperty};
40+
41+
// The table equivalent: `location` and `owner` have dedicated `Table`
42+
// fields, and `comment` is not part of the Iceberg-on-HMS contract.
43+
constexpr std::array<std::string_view, 3> kTableReservedKeys = {
44+
kCommentProperty, kLocationProperty, kOwnerProperty};
45+
46+
// HMS `PrincipalType` names accepted for a database owner.
47+
constexpr std::array<std::string_view, 3> kPrincipalTypes = {"USER", "GROUP", "ROLE"};
48+
49+
bool IsDatabaseReservedKey(std::string_view key) {
50+
return std::ranges::find(kDatabaseReservedKeys, key) != kDatabaseReservedKeys.end();
51+
}
52+
53+
bool IsTableReservedKey(std::string_view key) {
54+
return std::ranges::find(kTableReservedKeys, key) != kTableReservedKeys.end();
55+
}
56+
57+
std::string GetOrEmpty(const std::unordered_map<std::string, std::string>& properties,
58+
std::string_view key) {
59+
auto it = properties.find(std::string(key));
60+
return it == properties.end() ? std::string() : it->second;
61+
}
62+
63+
std::string TrimTrailingSlash(std::string_view path) {
64+
while (path.size() > 1 && path.back() == '/') {
65+
path.remove_suffix(1);
66+
}
67+
return std::string(path);
68+
}
69+
70+
} // namespace
71+
72+
Status ValidateNamespace(const Namespace& ns) {
73+
if (ns.levels.size() != 1) {
74+
return InvalidArgument(
75+
"Hive Metastore only supports single-level namespaces; got {} level(s).",
76+
ns.levels.size());
77+
}
78+
if (ns.levels[0].empty()) {
79+
return InvalidArgument("Hive namespace cannot have an empty name.");
80+
}
81+
return {};
82+
}
83+
84+
Status ValidateOwnerSettings(
85+
const std::unordered_map<std::string, std::string>& properties) {
86+
auto owner_type = properties.find(std::string(kHmsDbOwnerTypeProperty));
87+
if (owner_type == properties.end()) {
88+
return {};
89+
}
90+
if (!properties.contains(std::string(kHmsDbOwnerProperty))) {
91+
return InvalidArgument("Hive namespace property '{}' requires '{}' to also be set.",
92+
kHmsDbOwnerTypeProperty, kHmsDbOwnerProperty);
93+
}
94+
const bool known = std::ranges::any_of(kPrincipalTypes, [&](std::string_view name) {
95+
return StringUtils::EqualsIgnoreCase(owner_type->second, name);
96+
});
97+
if (!known) {
98+
return InvalidArgument(
99+
"Hive namespace property '{}' has value '{}'; expected one of "
100+
"USER, GROUP or ROLE.",
101+
kHmsDbOwnerTypeProperty, owner_type->second);
102+
}
103+
return {};
104+
}
105+
106+
Result<HiveDatabase> ConvertToHiveDatabase(
107+
const Namespace& ns, const std::unordered_map<std::string, std::string>& properties) {
108+
ICEBERG_RETURN_UNEXPECTED(ValidateNamespace(ns));
109+
ICEBERG_RETURN_UNEXPECTED(ValidateOwnerSettings(properties));
110+
111+
HiveDatabase database;
112+
database.name = ns.levels[0];
113+
database.description = GetOrEmpty(properties, kCommentProperty);
114+
database.location_uri = GetOrEmpty(properties, kLocationProperty);
115+
database.owner_name = GetOrEmpty(properties, kHmsDbOwnerProperty);
116+
database.owner_type = GetOrEmpty(properties, kHmsDbOwnerTypeProperty);
117+
118+
for (const auto& [key, value] : properties) {
119+
if (!IsDatabaseReservedKey(key)) {
120+
database.parameters.emplace(key, value);
121+
}
122+
}
123+
return database;
124+
}
125+
126+
HiveNamespace ConvertFromHiveDatabase(const HiveDatabase& database) {
127+
HiveNamespace result;
128+
result.ns.levels = {database.name};
129+
result.properties = database.parameters;
130+
if (!database.description.empty()) {
131+
result.properties[std::string(kCommentProperty)] = database.description;
132+
}
133+
if (!database.location_uri.empty()) {
134+
result.properties[std::string(kLocationProperty)] = database.location_uri;
135+
}
136+
if (!database.owner_name.empty()) {
137+
result.properties[std::string(kHmsDbOwnerProperty)] = database.owner_name;
138+
}
139+
if (!database.owner_type.empty()) {
140+
result.properties[std::string(kHmsDbOwnerTypeProperty)] = database.owner_type;
141+
}
142+
return result;
143+
}
144+
145+
Result<HiveTable> ConvertToHiveTable(
146+
const TableIdentifier& identifier, const std::vector<HiveColumn>& columns,
147+
std::string_view metadata_location, std::string_view location,
148+
const std::unordered_map<std::string, std::string>& table_properties) {
149+
ICEBERG_RETURN_UNEXPECTED(ValidateNamespace(identifier.ns));
150+
ICEBERG_RETURN_UNEXPECTED(identifier.Validate());
151+
152+
HiveTable table;
153+
table.db_name = identifier.ns.levels[0];
154+
table.table_name = identifier.name;
155+
table.owner = GetOrEmpty(table_properties, kOwnerProperty);
156+
table.table_type = "EXTERNAL_TABLE";
157+
table.location = std::string(location);
158+
table.columns = columns;
159+
table.serde = std::string(kLazySimpleSerDe);
160+
table.input_format = std::string(kFileInputFormat);
161+
table.output_format = std::string(kFileOutputFormat);
162+
163+
// Mandatory Iceberg-on-HMS marker parameters.
164+
table.parameters.emplace(std::string(kMetadataLocationKey), metadata_location);
165+
table.parameters.emplace(std::string(kTableTypeKey), std::string(kTableTypeIceberg));
166+
table.parameters.emplace(std::string(kExternalKey), std::string(kExternalTrue));
167+
168+
// Forward any user-supplied table properties that aren't reserved.
169+
for (const auto& [key, value] : table_properties) {
170+
if (!IsTableReservedKey(key) && !table.parameters.contains(key)) {
171+
table.parameters.emplace(key, value);
172+
}
173+
}
174+
return table;
175+
}
176+
177+
Result<std::string> GetMetadataLocation(
178+
const std::unordered_map<std::string, std::string>& table_parameters) {
179+
auto it = table_parameters.find(std::string(kMetadataLocationKey));
180+
if (it == table_parameters.end() || it->second.empty()) {
181+
return NotFound("HMS table is missing '{}' parameter; not an Iceberg table.",
182+
kMetadataLocationKey);
183+
}
184+
return it->second;
185+
}
186+
187+
Status ValidateIcebergTable(
188+
const TableIdentifier& identifier,
189+
const std::unordered_map<std::string, std::string>& table_parameters) {
190+
auto it = table_parameters.find(std::string(kTableTypeKey));
191+
if (it == table_parameters.end() || it->second.empty()) {
192+
return NoSuchTable(
193+
"HMS table {} has no '{}' parameter; refusing to treat it as an Iceberg table.",
194+
identifier.ToString(), kTableTypeKey);
195+
}
196+
if (!StringUtils::EqualsIgnoreCase(it->second, kTableTypeIceberg)) {
197+
return NoSuchTable("HMS table {} has '{}={}'; expected '{}' (case-insensitive).",
198+
identifier.ToString(), kTableTypeKey, it->second,
199+
kTableTypeIceberg);
200+
}
201+
return {};
202+
}
203+
204+
std::string GetDefaultTableLocation(std::string_view warehouse, const Namespace& ns,
205+
std::string_view table_name) {
206+
std::string base = TrimTrailingSlash(warehouse);
207+
std::string ns_part;
208+
for (std::size_t i = 0; i < ns.levels.size(); ++i) {
209+
if (i > 0) {
210+
ns_part += ".";
211+
}
212+
ns_part += ns.levels[i];
213+
}
214+
return std::format("{}/{}.db/{}", base, ns_part, table_name);
215+
}
216+
217+
} // namespace iceberg::hive

0 commit comments

Comments
 (0)