Skip to content

Commit 2d87242

Browse files
authored
support data source in transform (#397)
This PR introduces 2 new features for transformation. First is the trim option which will trim ascii whitespace from both the start and end of the value provided. This can be used like so: {{ trim(my_super_cool_value) }} Second is the datasource option. This pulls from a mounted datasource to provide data or an empty string. This can be used like so {{ data_source("my_cool_mounted_file") }} One may often want to use the trim functionality in combination with this new feature.
1 parent 33a74fc commit 2d87242

File tree

10 files changed

+732
-484
lines changed

10 files changed

+732
-484
lines changed

api/envoy/config/filter/http/transformation/v2/transformation_filter.proto

+8
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import "xds/type/matcher/v3/matcher.proto";
1515
import "envoy/config/route/v3/route_components.proto";
1616
import "envoy/type/matcher/v3/string.proto";
1717
import "envoy/config/core/v3/extension.proto";
18+
import "envoy/config/core/v3/base.proto";
1819

1920
message FilterTransformations {
2021
// Specifies transformations based on the route matches. The first matched
@@ -230,6 +231,13 @@ message TransformationTemplate {
230231
// "my-extractor" extractor.
231232
map<string, Extraction> extractors = 2;
232233

234+
// Data sources. use the `data_source`
235+
// function to access data_sources in the template (e.g. '{{
236+
// data_source("my_file") }}').
237+
map<string, envoy.config.core.v3.DataSource> data_sources = 17;
238+
// max size of data source. 4096 bytes by default.
239+
uint32 data_source_max_size = 18;
240+
233241
// Use this attribute to transform request/response headers. It consists of a
234242
// map of strings to templates. The string key determines the name of the
235243
// resulting header, the rendered template will determine the value. Any existing
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
changelog:
2+
- type: NON_USER_FACING
3+
description: >-
4+
Allow usage of subpaths in Metadata funcs within inja templates.

go/config/filter/http/transformation/v2/transformation_filter.pb.go

+450-409
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

go/config/filter/http/transformation/v2/transformation_filter.pb.validate.go

+48
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

go/config/filter/http/transformation/v2/transformation_filter_vtproto.pb.go

+65
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

source/extensions/filters/http/transformation/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ envoy_cc_library(
141141
"@envoy//source/common/common:base64_lib",
142142
"@envoy//source/common/common:macros",
143143
"@envoy//source/common/common:regex_lib",
144+
"@envoy//source/common/config:datasource_lib",
144145
"@envoy//source/common/common:utility_lib",
145146
"@envoy//source/common/protobuf",
146147
"@inja//:inja-lib",

source/extensions/filters/http/transformation/inja_transformer.cc

+43-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include <iterator>
44

55
#include "absl/strings/str_replace.h"
6+
#include "absl/strings/ascii.h"
67

78
#include "source/common/buffer/buffer_impl.h"
89
#include "source/common/common/macros.h"
@@ -230,6 +231,12 @@ TransformerInstance::TransformerInstance(ThreadLocal::Slot &tls, Envoy::Random::
230231
env_.add_callback("extraction", 1, [this](Arguments &args) {
231232
return extracted_callback(args);
232233
});
234+
env_.add_callback("data_source", 1, [this](Arguments &args) {
235+
return data_source_callback(args);
236+
});
237+
env_.add_callback("trim", 1, [this](Arguments &args) {
238+
return trim_callback(args);
239+
});
233240
env_.add_callback("context", 0, [this](Arguments &) { return *tls_.getTyped<ThreadLocalTransformerContext>().context_; });
234241
env_.add_callback("body", 0, [this](Arguments &) { return (*tls_.getTyped<ThreadLocalTransformerContext>().body_)(); });
235242
env_.add_callback("env", 1, [this](Arguments &args) { return env(args); });
@@ -394,6 +401,21 @@ json TransformerInstance::extracted_callback(const inja::Arguments &args) const
394401
return "";
395402
}
396403

404+
json TransformerInstance::data_source_callback(const inja::Arguments &args) const {
405+
const auto& ctx = tls_.getTyped<ThreadLocalTransformerContext>();
406+
const std::string &name = args.at(0)->get_ref<const std::string &>();
407+
const auto value_it = ctx.data_sources_->find(name);
408+
if (value_it != ctx.data_sources_->end()) {
409+
return value_it->second->data();
410+
}
411+
return "";
412+
}
413+
414+
json TransformerInstance::trim_callback(const inja::Arguments &args) const {
415+
const std::string& s = args.at(0)->get_ref<const std::string &>();
416+
return absl::StripAsciiWhitespace(s);
417+
}
418+
397419
json TransformerInstance::env(const inja::Arguments &args) const {
398420
const auto& ctx = tls_.getTyped<ThreadLocalTransformerContext>();
399421
const std::string &key = args.at(0)->get_ref<const std::string &>();
@@ -690,8 +712,9 @@ std::string TransformerInstance::render(const inja::Template &input) {
690712

691713
// An InjaTransformer is constructed on initialization on the main thread
692714
InjaTransformer::InjaTransformer(const TransformationTemplate &transformation,
693-
Envoy::Random::RandomGenerator &rng,
694715
google::protobuf::BoolValue log_request_response_info,
716+
Event::Dispatcher& main_thread_dispatcher,
717+
Envoy::Api::Api& api,
695718
ThreadLocal::SlotAllocator &tls)
696719
: Transformer(log_request_response_info),
697720
advanced_templates_(transformation.advanced_templates()),
@@ -700,7 +723,7 @@ InjaTransformer::InjaTransformer(const TransformationTemplate &transformation,
700723
ignore_error_on_parse_(transformation.ignore_error_on_parse()),
701724
escape_characters_(transformation.escape_characters()),
702725
tls_(tls.allocateSlot()),
703-
instance_(std::make_unique<TransformerInstance>(*tls_, rng)) {
726+
instance_(std::make_unique<TransformerInstance>(*tls_, api.randomGenerator())) {
704727
if (advanced_templates_) {
705728
instance_->set_element_notation(inja::ElementNotation::Pointer);
706729
}
@@ -715,6 +738,23 @@ InjaTransformer::InjaTransformer(const TransformationTemplate &transformation,
715738
for (auto it = extractors.begin(); it != extractors.end(); it++) {
716739
extractors_.emplace_back(std::make_pair(it->first, it->second));
717740
}
741+
uint32_t max_size = 4096;
742+
if (transformation.data_source_max_size() > 0) {
743+
max_size = transformation.data_source_max_size();
744+
}
745+
746+
const auto &data_sources = transformation.data_sources();
747+
for (auto it = data_sources.begin(); it != data_sources.end(); it++) {
748+
auto provider_or_error = Envoy::Config::DataSource::DataSourceProvider::create(
749+
it->second, main_thread_dispatcher, tls, api, true, max_size);
750+
if (provider_or_error.ok()) {
751+
data_sources_.emplace(std::make_pair(it->first, std::move(provider_or_error.value())));
752+
} else {
753+
throw EnvoyException(fmt::format(
754+
"Failed to create data source provider '{}': {}", it->first, provider_or_error.status().message()));
755+
}
756+
}
757+
718758
const auto &headers = transformation.headers();
719759
for (auto it = headers.begin(); it != headers.end(); it++) {
720760
Http::LowerCaseString header_name(it->first);
@@ -980,6 +1020,7 @@ void InjaTransformer::transform(Http::RequestOrResponseHeaderMap &header_map,
9801020
typed_tls_data.request_headers_ = request_headers;
9811021
typed_tls_data.body_ = &get_body;
9821022
typed_tls_data.extractions_ = &extractions;
1023+
typed_tls_data.data_sources_ = &data_sources_;
9831024
typed_tls_data.destructive_extractions_ = &destructive_extractions;
9841025
typed_tls_data.context_ = &json_body;
9851026
typed_tls_data.environ_ = &environ_;

source/extensions/filters/http/transformation/inja_transformer.h

+9-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "envoy/thread_local/thread_local_object.h"
1212
#include "envoy/thread_local/thread_local.h"
1313
#include "source/extensions/filters/http/transformation/transformer.h"
14+
#include "source/common/config/datasource.h"
1415

1516
// clang-format off
1617
#include "nlohmann/json.hpp"
@@ -36,6 +37,8 @@ struct ThreadLocalTransformerContext : public ThreadLocal::ThreadLocalObject {
3637
const GetBodyFunc *body_;
3738
const std::unordered_map<std::string, std::string> *destructive_extractions_;
3839
const std::unordered_map<std::string, absl::string_view> *extractions_;
40+
const std::unordered_map<std::string, Envoy::Config::DataSource::DataSourceProviderPtr>* data_sources_;
41+
3942
const nlohmann::json *context_;
4043
const std::unordered_map<std::string, std::string> *environ_;
4144
const envoy::config::core::v3::Metadata *cluster_metadata_;
@@ -65,6 +68,8 @@ class TransformerInstance {
6568
nlohmann::json request_header_callback(const inja::Arguments &args) const;
6669
// extracted_value(name, index)
6770
nlohmann::json extracted_callback(const inja::Arguments &args) const;
71+
nlohmann::json data_source_callback(const inja::Arguments &args) const;
72+
nlohmann::json trim_callback(const inja::Arguments &args) const;
6873
nlohmann::json dynamic_metadata(const inja::Arguments &args) const;
6974
nlohmann::json env(const inja::Arguments &args) const;
7075
nlohmann::json cluster_metadata_callback(const inja::Arguments &args) const;
@@ -121,9 +126,10 @@ class Extractor : Logger::Loggable<Logger::Id::filter> {
121126
class InjaTransformer : public Transformer, Logger::Loggable<Logger::Id::filter> {
122127
public:
123128
InjaTransformer(const envoy::api::v2::filter::http::TransformationTemplate &transformation,
124-
Envoy::Random::RandomGenerator &rng,
125129
google::protobuf::BoolValue log_request_response_info,
126-
ThreadLocal::SlotAllocator &tls_);
130+
Event::Dispatcher& main_thread_dispatcher,
131+
Envoy::Api::Api& api,
132+
ThreadLocal::SlotAllocator &tls);
127133
~InjaTransformer();
128134

129135
void transform(Http::RequestOrResponseHeaderMap &map,
@@ -143,6 +149,7 @@ class InjaTransformer : public Transformer, Logger::Loggable<Logger::Id::filter>
143149
bool advanced_templates_{};
144150
bool passthrough_body_{};
145151
std::vector<std::pair<std::string, Extractor>> extractors_;
152+
std::unordered_map<std::string, Envoy::Config::DataSource::DataSourceProviderPtr> data_sources_;
146153
std::vector<std::pair<Http::LowerCaseString, inja::Template>> headers_;
147154
std::vector<std::pair<Http::LowerCaseString, inja::Template>> headers_to_append_;
148155
std::vector<Http::LowerCaseString> headers_to_remove_;

source/extensions/filters/http/transformation/transformation_factory.cc

+5-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@ TransformerConstSharedPtr Transformation::getTransformer(
1515
switch (transformation.transformation_type_case()) {
1616
case envoy::api::v2::filter::http::Transformation::kTransformationTemplate:
1717
return std::make_unique<InjaTransformer>(
18-
transformation.transformation_template(), context.api().randomGenerator(), transformation.log_request_response_info(), context.threadLocal());
18+
transformation.transformation_template(),
19+
transformation.log_request_response_info(),
20+
context.mainThreadDispatcher(),
21+
context.api(),
22+
context.threadLocal());
1923
case envoy::api::v2::filter::http::Transformation::kHeaderBodyTransform: {
2024
const auto& header_body_transform = transformation.header_body_transform();
2125
return std::make_unique<BodyHeaderTransformer>(header_body_transform.add_request_metadata(), transformation.log_request_response_info());

0 commit comments

Comments
 (0)