Skip to content

Commit 8f78b13

Browse files
xiaoxmengmeta-codesync[bot]
xiaoxmeng
authored and
meta-codesync[bot]
committed
feat(nimble): Add shared dictionary config
Summary: Add the shared dictionary encoding config object and builder used to describe regular-column and top-level flat-map value dictionary targets. The builder validates configured field paths through Velox `Subfield` before writer integration consumes them. Reviewed By: tanjialiang Differential Revision: D116374169
1 parent 5765183 commit 8f78b13

5 files changed

Lines changed: 603 additions & 0 deletions

File tree

‎velox/dwio/nimble/velox/CMakeLists.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@
1414
add_library(nimble_velox_common SchemaUtils.cpp SchemaUtils.h)
1515
target_link_libraries(nimble_velox_common nimble_common velox_type Folly::folly fmt::fmt)
1616

17+
add_library(nimble_velox_shared_dictionary_config SharedDictionaryConfig.cpp)
18+
target_link_libraries(
19+
nimble_velox_shared_dictionary_config
20+
nimble_common
21+
nimble_encodings
22+
nimble_tablet_reader
23+
velox_type
24+
)
25+
1726
add_library(nimble_velox_schema SchemaTypes.cpp SchemaTypes.h)
1827
target_link_libraries(nimble_velox_schema nimble_common Folly::folly)
1928

‎velox/dwio/nimble/velox/SharedDictionaryConfig.cpp‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "velox/dwio/nimble/velox/SharedDictionaryConfig.h"
18+
19+
#include <algorithm>
20+
#include <iterator>
21+
#include <string>
22+
#include <string_view>
23+
#include <utility>
24+
#include <vector>
25+
26+
#include "velox/dwio/nimble/common/Exceptions.h"
27+
#include "velox/type/Subfield.h"
28+
29+
namespace facebook::nimble {
30+
namespace {
31+
32+
velox::common::Subfield parseFieldPath(
33+
const std::string& fieldPath,
34+
std::string_view configType) {
35+
NIMBLE_USER_CHECK(
36+
!fieldPath.empty(),
37+
"Shared dictionary {} path must not be empty.",
38+
configType);
39+
velox::common::Subfield subfield;
40+
try {
41+
subfield = velox::common::Subfield{fieldPath};
42+
} catch (const velox::VeloxException&) {
43+
NIMBLE_USER_FAIL(
44+
"Shared dictionary {} path '{}' must start with a field name.",
45+
configType,
46+
fieldPath);
47+
}
48+
NIMBLE_USER_CHECK(
49+
subfield.valid(),
50+
"Shared dictionary {} path '{}' must start with a field name.",
51+
configType,
52+
fieldPath);
53+
return subfield;
54+
}
55+
56+
void validateValueStreamPath(
57+
const std::string& fieldPath,
58+
std::string_view configType) {
59+
const auto subfield = parseFieldPath(fieldPath, configType);
60+
for (const auto& pathElement : subfield.path()) {
61+
NIMBLE_USER_CHECK(
62+
pathElement->is(velox::common::SubfieldKind::kNestedField) ||
63+
pathElement->is(velox::common::SubfieldKind::kAllSubscripts),
64+
"Shared dictionary {} path '{}' only supports nested row fields and "
65+
"all-subscript array/map elements.",
66+
configType,
67+
fieldPath);
68+
}
69+
}
70+
71+
velox::common::Subfield validateFlatMapColumnPath(
72+
const std::string& fieldPath) {
73+
auto subfield = parseFieldPath(fieldPath, "flat-map column");
74+
NIMBLE_USER_CHECK_EQ(
75+
subfield.path().size(),
76+
1,
77+
"Shared dictionary flat-map column path '{}' must be a top-level writer "
78+
"input column.",
79+
fieldPath);
80+
return subfield;
81+
}
82+
83+
void validateValueSubfield(const std::string& valueSubfield) {
84+
if (valueSubfield.empty()) {
85+
return;
86+
}
87+
const auto subfield =
88+
parseFieldPath(valueSubfield, "flat-map value subfield");
89+
for (const auto& pathElement : subfield.path()) {
90+
NIMBLE_USER_CHECK(
91+
pathElement->is(velox::common::SubfieldKind::kNestedField) ||
92+
pathElement->is(velox::common::SubfieldKind::kAllSubscripts),
93+
"Shared dictionary flat-map value subfield '{}' only supports nested "
94+
"row fields and all-subscript array/map elements.",
95+
valueSubfield);
96+
}
97+
}
98+
99+
} // namespace
100+
101+
SharedDictionaryConfigBuilder::SharedDictionaryConfigBuilder(
102+
SharedDictionaryEncodingConfig&& config)
103+
: config_{std::move(config)} {}
104+
105+
SharedDictionaryConfigBuilder SharedDictionaryEncodingConfig::builder(
106+
SharedDictionaryEncodingConfig&& config) {
107+
return SharedDictionaryConfigBuilder{std::move(config)};
108+
}
109+
110+
SharedDictionaryConfigBuilder&
111+
SharedDictionaryConfigBuilder::setExternalResolver(
112+
std::shared_ptr<const ExternalDictionaryResolver> externalResolver) {
113+
config_.externalResolver = std::move(externalResolver);
114+
return *this;
115+
}
116+
117+
SharedDictionaryConfigBuilder&
118+
SharedDictionaryConfigBuilder::addColumnDictionary(
119+
std::string fieldPath,
120+
SharedDictionaryConfig dictionary) {
121+
validateValueStreamPath(fieldPath, "column");
122+
123+
const auto duplicate = std::any_of(
124+
config_.columns.begin(),
125+
config_.columns.end(),
126+
[&](const auto& candidate) { return candidate.fieldPath == fieldPath; });
127+
NIMBLE_USER_CHECK(
128+
!duplicate,
129+
"Duplicate shared dictionary column configuration for path '{}'.",
130+
fieldPath);
131+
config_.columns.push_back(
132+
ColumnDictionary{
133+
.fieldPath = std::move(fieldPath),
134+
.dictionary = std::move(dictionary)});
135+
return *this;
136+
}
137+
138+
SharedDictionaryConfigBuilder&
139+
SharedDictionaryConfigBuilder::addFlatmapValueDictionary(
140+
std::string fieldPath,
141+
int64_t key,
142+
SharedDictionaryConfig dictionary,
143+
std::string valueSubfield) {
144+
const auto subfield = validateFlatMapColumnPath(fieldPath);
145+
validateValueSubfield(valueSubfield);
146+
147+
auto column = std::find_if(
148+
config_.flatMapColumns.begin(),
149+
config_.flatMapColumns.end(),
150+
[&](const auto& candidate) { return candidate.fieldPath == fieldPath; });
151+
if (column == config_.flatMapColumns.end()) {
152+
config_.flatMapColumns.push_back(
153+
FlatmapColumnDictionary{.fieldPath = fieldPath, .keys = {}});
154+
column = std::prev(config_.flatMapColumns.end());
155+
}
156+
157+
const auto duplicate = std::any_of(
158+
column->keys.begin(), column->keys.end(), [&](const auto& item) {
159+
return item.key == key && item.valueSubfield == valueSubfield;
160+
});
161+
NIMBLE_USER_CHECK(
162+
!duplicate,
163+
"Duplicate shared dictionary flat-map value configuration for path '{}', "
164+
"key {}, and value subfield '{}'.",
165+
fieldPath,
166+
key,
167+
valueSubfield);
168+
column->keys.push_back(
169+
FlatmapKeyDictionary{
170+
.key = key,
171+
.valueSubfield = std::move(valueSubfield),
172+
.dictionary = std::move(dictionary)});
173+
return *this;
174+
}
175+
176+
SharedDictionaryEncodingConfig SharedDictionaryConfigBuilder::build() {
177+
return std::move(config_);
178+
}
179+
180+
} // namespace facebook::nimble

‎velox/dwio/nimble/velox/SharedDictionaryConfig.h‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#pragma once
17+
18+
#include <cstdint>
19+
#include <memory>
20+
#include <string>
21+
#include <vector>
22+
23+
#include "velox/dwio/nimble/encodings/SharedDictionaryEncoding.h"
24+
#include "velox/dwio/nimble/tablet/SharedDictionaryReader.h"
25+
26+
namespace facebook::nimble {
27+
28+
class SharedDictionaryConfigBuilder;
29+
30+
/// Shared dictionary writer configuration for one value stream.
31+
struct SharedDictionaryConfig {
32+
/// Where the dictionary alphabet is stored or resolved.
33+
SharedDictionaryScope scope{SharedDictionaryScope::Stripe};
34+
/// Dictionary id within File or External scope. Stripe scope assigns this
35+
/// from the generated auxiliary alphabet stream.
36+
uint32_t dictionaryId{};
37+
/// Resolves a provided alphabet instead of building one from written values.
38+
bool useExternalAlphabet{false};
39+
/// Candidate encodings for alphabets stored in this file.
40+
std::vector<EncodingType> alphabetEncodings;
41+
};
42+
43+
/// Shared dictionary settings for one regular column value stream.
44+
struct ColumnDictionary {
45+
/// Velox subfield from the writer input root to the configured value stream.
46+
/// Use `[*]` to traverse array elements or map values.
47+
std::string fieldPath;
48+
49+
/// Dictionary encoding settings for the resolved value stream.
50+
SharedDictionaryConfig dictionary;
51+
};
52+
53+
/// Shared dictionary settings for one flat-map key.
54+
struct FlatmapKeyDictionary {
55+
/// Flat-map key to configure.
56+
int64_t key{};
57+
58+
/// Velox subfield below the keyed flat-map value. Empty selects the key
59+
/// value itself. Non-empty paths start with a field name; use `[*]` to
60+
/// traverse array elements or map values below that field.
61+
std::string valueSubfield;
62+
63+
/// Dictionary encoding settings for the key value stream.
64+
SharedDictionaryConfig dictionary;
65+
};
66+
67+
/// Shared dictionary settings for one top-level flat-map column.
68+
struct FlatmapColumnDictionary {
69+
/// Row-field path from the writer input root to the configured flat-map
70+
/// column. This currently names a top-level writer input column.
71+
std::string fieldPath;
72+
73+
/// Per-key shared dictionary settings.
74+
std::vector<FlatmapKeyDictionary> keys;
75+
};
76+
77+
/// Shared dictionary writer configuration for regular columns and flat-map
78+
/// value streams in a file.
79+
struct SharedDictionaryEncodingConfig {
80+
/// Regular columns eligible for shared dictionary encoding. Paths may use
81+
/// `[*]` to select array elements or map values before a nested field. The
82+
/// resolved value must be an integer scalar, or an array whose element is an
83+
/// integer scalar. For File scope, callers are responsible for assigning a
84+
/// unique dictionaryId per configured value stream.
85+
std::vector<ColumnDictionary> columns;
86+
87+
/// Top-level flat-map columns eligible for shared dictionary encoding.
88+
/// Non-empty valueSubfield paths start with a field name and may use `[*]`
89+
/// below that field to select array elements or map values. The configured
90+
/// value stream must resolve to an integer scalar, or an array whose element
91+
/// is an integer scalar. For File scope, callers are responsible for
92+
/// assigning a unique dictionaryId per configured key value stream.
93+
std::vector<FlatmapColumnDictionary> flatMapColumns;
94+
95+
/// Supplies external alphabets for External shared dictionary configurations
96+
/// and File configurations that set useExternalAlphabet.
97+
std::shared_ptr<const ExternalDictionaryResolver> externalResolver;
98+
99+
/// Returns true when no value streams request shared dictionary encoding.
100+
bool empty() const {
101+
return columns.empty() && flatMapColumns.empty();
102+
}
103+
104+
/// Creates a builder, optionally seeded from an existing config.
105+
static SharedDictionaryConfigBuilder builder(
106+
SharedDictionaryEncodingConfig&& config = {});
107+
};
108+
109+
/// Builder for SharedDictionaryEncodingConfig.
110+
class SharedDictionaryConfigBuilder {
111+
public:
112+
/// Creates a builder seeded from config.
113+
explicit SharedDictionaryConfigBuilder(
114+
SharedDictionaryEncodingConfig&& config = {});
115+
116+
/// Sets the resolver used by external dictionaries and provided file
117+
/// alphabets.
118+
SharedDictionaryConfigBuilder& setExternalResolver(
119+
std::shared_ptr<const ExternalDictionaryResolver> externalResolver);
120+
121+
/// Adds dictionary settings for one regular column value stream.
122+
SharedDictionaryConfigBuilder& addColumnDictionary(
123+
std::string fieldPath,
124+
SharedDictionaryConfig dictionary);
125+
126+
/// Adds dictionary settings for one top-level flat-map key value stream.
127+
SharedDictionaryConfigBuilder& addFlatmapValueDictionary(
128+
std::string fieldPath,
129+
int64_t key,
130+
SharedDictionaryConfig dictionary,
131+
std::string valueSubfield = "");
132+
133+
/// Returns the completed config.
134+
SharedDictionaryEncodingConfig build();
135+
136+
private:
137+
SharedDictionaryEncodingConfig config_;
138+
};
139+
140+
} // namespace facebook::nimble

‎velox/dwio/nimble/velox/tests/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ add_executable(
6161
RowRangeTest.cpp
6262
SchemaTest.cpp
6363
SchemaUtilsTest.cpp
64+
SharedDictionaryConfigTest.cpp
6465
SharedDictionaryWriterTest.cpp
6566
StreamChunkerTest.cpp
6667
StreamDataTest.cpp
@@ -94,6 +95,7 @@ target_link_libraries(
9495
nimble_velox_schema_utils
9596
nimble_writer_test_utils
9697
nimble_velox_reader
98+
nimble_velox_shared_dictionary_config
9799
nimble_writer
98100
nimble_velox_layout_planner
99101
nimble_velox_stats_fb

0 commit comments

Comments
 (0)