diff --git a/README.md b/README.md index 72459b760..a9228390a 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,6 @@ BUILDIFIER=$GOPATH/bin/buildifier CLANG_FORMAT=clang-format /path/to/envoy/tools # Submit a build ``` -gcloud builds submit --config=cloudbuild.yaml \ +gcloud builds submit --project=solo-public --config=cloudbuild.yaml \ --substitutions=COMMIT_SHA=$(git rev-parse HEAD) . ``` diff --git a/api/envoy/config/filter/http/transformation/v2/transformation_filter.proto b/api/envoy/config/filter/http/transformation/v2/transformation_filter.proto index 6b074d6ba..65bfdf802 100644 --- a/api/envoy/config/filter/http/transformation/v2/transformation_filter.proto +++ b/api/envoy/config/filter/http/transformation/v2/transformation_filter.proto @@ -7,6 +7,7 @@ option java_outer_classname = "TransformationFilterProto"; option java_multiple_files = true; option go_package = "transformation"; +import "google/protobuf/empty.proto"; import "validate/validate.proto"; import "envoy/api/v2/route/route.proto"; @@ -52,7 +53,10 @@ message Transformation { } message Extraction { - string header = 1; + oneof source { + string header = 1; + google.protobuf.Empty body = 4; + } // what information to extract. if extraction fails the result is // an empty value. string regex = 2; @@ -61,7 +65,7 @@ message Extraction { message TransformationTemplate { bool advanced_templates = 1; - // Extractors are in the origin request language domain + // Extractors are in the original request language domain map extractors = 2; map headers = 3; @@ -71,6 +75,22 @@ message TransformationTemplate { Passthrough passthrough = 5; MergeExtractorsToBody merge_extractors_to_body = 6; } + + enum RequestBodyParse { + ParseAsJson = 0; + DontParse = 1; + } + RequestBodyParse parse_body_behavior = 7; + bool ignore_error_on_parse = 8; + + message DynamicMetadataValue { + // optional. if not set filter namespace will be used. + string metadata_namespace = 1; + string key = 2 [(validate.rules).string = {min_bytes: 1}]; + InjaTemplate value = 3; + } + repeated DynamicMetadataValue dynamic_metadata_values = 9; + } /* diff --git a/changelog/v0.1.18/body-transform.yaml b/changelog/v0.1.18/body-transform.yaml new file mode 100644 index 000000000..5acbdadb8 --- /dev/null +++ b/changelog/v0.1.18/body-transform.yaml @@ -0,0 +1,4 @@ +changelog: +- type: NEW_FEATURE + issueLink: https://github.com/solo-io/envoy-gloo/issues/37 + description: Add body() function to template, allow using templates wit dynamic metadata, add env() function to template diff --git a/source/extensions/filters/http/transformation/BUILD b/source/extensions/filters/http/transformation/BUILD index 5077fefe1..a517d4831 100644 --- a/source/extensions/filters/http/transformation/BUILD +++ b/source/extensions/filters/http/transformation/BUILD @@ -80,12 +80,13 @@ envoy_cc_library( deps = [ ":transformer_lib", "//api/envoy/config/filter/http/transformation/v2:pkg_cc_proto", + "//source/extensions/filters/http:solo_well_known_names", "@envoy//include/envoy/buffer:buffer_interface", "@envoy//include/envoy/http:header_map_interface", "@envoy//source/common/common:macros", + "@envoy//source/common/common:regex_lib", "@envoy//source/common/common:utility_lib", "@envoy//source/common/protobuf", - "@envoy//source/common/common:regex_lib", "@inja//:inja-lib", "@json//:json-lib", ], diff --git a/source/extensions/filters/http/transformation/inja_transformer.cc b/source/extensions/filters/http/transformation/inja_transformer.cc index ec6a5475f..33a10d56b 100644 --- a/source/extensions/filters/http/transformation/inja_transformer.cc +++ b/source/extensions/filters/http/transformation/inja_transformer.cc @@ -1,7 +1,9 @@ +#include "extensions/filters/http/solo_well_known_names.h" #include "extensions/filters/http/transformation/inja_transformer.h" #include +#include "common/buffer/buffer_impl.h" #include "common/common/macros.h" #include "common/common/utility.h" #include "common/common/regex.h" @@ -16,6 +18,10 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Transformation { + +using TransformationTemplate = + envoy::api::v2::filter::http::TransformationTemplate; + // TODO: move to common namespace { const Http::HeaderEntry *getHeader(const Http::HeaderMap &header_map, @@ -37,72 +43,105 @@ const Http::HeaderEntry *getHeader(const Http::HeaderMap &header_map, } // namespace Extractor::Extractor(const envoy::api::v2::filter::http::Extraction &extractor) - : headername_(extractor.header()), group_(extractor.subgroup()), - extract_regex_(Regex::Utility::parseStdRegex(extractor.regex())) {} -std::string Extractor::extract(const Http::HeaderMap &header_map) const { - // TODO: should we lowercase them in the config? - const Http::HeaderEntry *header_entry = getHeader(header_map, headername_); - if (!header_entry) { - return ""; - } + : headername_(extractor.header()), body_(extractor.has_body()), + group_(extractor.subgroup()), + extract_regex_(Regex::Utility::parseStdRegex(extractor.regex())) { + // mark count == number of sub groups, and we need to add one for match number 0 + // so we test for < instead of <= + // see: http://www.cplusplus.com/reference/regex/basic_regex/mark_count/ + if (extract_regex_.mark_count() < group_) { + throw EnvoyException(fmt::format("group {} requested for regex with only {} sub groups", group_, extract_regex_.mark_count())); + } + } - std::string value(header_entry->value().getStringView()); +absl::string_view Extractor::extract(Http::StreamFilterCallbacks &callbacks, const Http::HeaderMap &header_map, + GetBodyFunc body) const { + if (body_) { + return extractValue(callbacks, body()); + } else { + const Http::HeaderEntry *header_entry = getHeader(header_map, headername_); + if (!header_entry) { + return ""; + } + return extractValue(callbacks, header_entry->value().getStringView()); + } +} +absl::string_view Extractor::extractValue(Http::StreamFilterCallbacks &callbacks, absl::string_view value) const { // get and regex - std::smatch regex_result; - if (std::regex_match(value, regex_result, extract_regex_)) { - std::smatch::iterator submatch_it = regex_result.begin(); - for (unsigned i = 0; i < group_; i++) { - std::advance(submatch_it, 1); - if (submatch_it == regex_result.end()) { - return ""; - } + std::match_results regex_result; + if (std::regex_match(value.begin(), value.end(), regex_result, + extract_regex_)) { + if (group_ >= regex_result.size()) { + // this should never happen as we test this in the ctor. + ASSERT("no such group in the regex"); + ENVOY_STREAM_LOG(debug, "invalid group specified for regex", callbacks); + return ""; } - return *submatch_it; + const auto &sub_match = regex_result[group_]; + return absl::string_view(sub_match.first, sub_match.length()); + } else { + ENVOY_STREAM_LOG(debug, "extractor regex did not match input", callbacks); } - return ""; } TransformerInstance::TransformerInstance( - const Http::HeaderMap &header_map, - const std::unordered_map &extractions, - const json &context) - : header_map_(header_map), extractions_(extractions), context_(context) { + const Http::HeaderMap &header_map, GetBodyFunc body, + const std::unordered_map &extractions, + const json &context, const std::unordered_map& environ) + : header_map_(header_map), body_(body), extractions_(extractions), + context_(context), environ_(environ) { env_.add_callback("header", 1, - [this](Arguments args) { return header_callback(args); }); - env_.add_callback("extraction", 1, [this](Arguments args) { + [this](Arguments& args) { return header_callback(args); }); + env_.add_callback("extraction", 1, [this](Arguments& args) { return extracted_callback(args); }); + env_.add_callback("context", 0, [this](Arguments&) { return context_; }); + env_.add_callback("body", 0, [this](Arguments&) { return body_(); }); + env_.add_callback("env", 1, [this](Arguments& args) { return env(args); }); } -json TransformerInstance::header_callback(Arguments args) { - std::string headername = args.at(0)->get(); - const Http::HeaderEntry *header_entry = - getHeader(header_map_, std::move(headername)); +json TransformerInstance::header_callback(const inja::Arguments& args) const { + const std::string& headername = args.at(0)->get_ref(); + const Http::HeaderEntry *header_entry = getHeader(header_map_, headername); if (!header_entry) { return ""; } return std::string(header_entry->value().getStringView()); } -json TransformerInstance::extracted_callback(Arguments args) { - std::string name = args.at(0)->get(); +json TransformerInstance::extracted_callback(const inja::Arguments& args) const { + const std::string& name = args.at(0)->get_ref(); const auto value_it = extractions_.find(name); if (value_it != extractions_.end()) { return value_it->second; } return ""; } +json TransformerInstance::env(const inja::Arguments& args) const { + const std::string& key = args.at(0)->get_ref(); + auto it = environ_.find(key); + if (it != environ_.end()) { + return it->second; + } + return ""; +} std::string TransformerInstance::render(const inja::Template &input) { - return env_.render(input, context_); + // inja can't handle context that are not objects correctly, so we give it an empty object in that case + if (context_.is_object()) { + return env_.render(input, context_); + } else { + return env_.render(input, {}); + } } -InjaTransformer::InjaTransformer( - const envoy::api::v2::filter::http::TransformationTemplate &transformation) +InjaTransformer::InjaTransformer(const TransformationTemplate &transformation) : advanced_templates_(transformation.advanced_templates()), - passthrough_body_(transformation.has_passthrough()) { + passthrough_body_(transformation.has_passthrough()), + parse_body_behavior_(transformation.parse_body_behavior()), + ignore_error_on_parse_(transformation.ignore_error_on_parse()) { inja::ParserConfig parser_config; inja::LexerConfig lexer_config; inja::TemplateStorage template_storage; @@ -127,9 +166,25 @@ InjaTransformer::InjaTransformer( "Failed to parse header template '{}': {}", it->first, e.what())); } } + const auto &dynamic_metadata_values = transformation.dynamic_metadata_values(); + for (auto it = dynamic_metadata_values.begin(); it != dynamic_metadata_values.end(); it++) { + try { + DynamicMetadataValue dynamicMetadataValue; + dynamicMetadataValue.namespace_ = it->metadata_namespace(); + if (dynamicMetadataValue.namespace_.empty()){ + dynamicMetadataValue.namespace_ = SoloHttpFilterNames::get().Transformation; + } + dynamicMetadataValue.key_ = it->key(); + dynamicMetadataValue.template_ = parser.parse(it->value().text()); + dynamic_metadata_.emplace_back(std::move(dynamicMetadataValue)); + } catch (const std::runtime_error &e) { + throw EnvoyException(fmt::format( + "Failed to parse header template '{}': {}", it->key(), e.what())); + } + } switch (transformation.body_transformation_case()) { - case envoy::api::v2::filter::http::TransformationTemplate::kBody: { + case TransformationTemplate::kBody: { try { body_template_.emplace(parser.parse(transformation.body().text())); } catch (const std::runtime_error &e) { @@ -138,34 +193,64 @@ InjaTransformer::InjaTransformer( } break; } - case envoy::api::v2::filter::http::TransformationTemplate:: - kMergeExtractorsToBody: { + case TransformationTemplate::kMergeExtractorsToBody: { merged_extractors_to_body_ = true; break; } - case envoy::api::v2::filter::http::TransformationTemplate::kPassthrough: + case TransformationTemplate::kPassthrough: break; - case envoy::api::v2::filter::http::TransformationTemplate:: - BODY_TRANSFORMATION_NOT_SET: { + case TransformationTemplate::BODY_TRANSFORMATION_NOT_SET: { break; } } + + // parse environment + for (char **env = environ; *env != 0; env++) { + std::string current_env(*env); + size_t equals = current_env.find("="); + if (equals > 0) { + std::string key = current_env.substr(0, equals); + std::string value = current_env.substr(equals + 1); + environ_[key] = value; + } + } } InjaTransformer::~InjaTransformer() {} void InjaTransformer::transform(Http::HeaderMap &header_map, Buffer::Instance &body, - Http::StreamFilterCallbacks&) const { + Http::StreamFilterCallbacks &callbacks) const { + absl::optional string_body; + auto get_body = [&string_body, &body]() -> const std::string & { + if (!string_body.has_value()) { + string_body.emplace(body.toString()); + } + return string_body.value(); + }; + json json_body; - if (body.length() > 0) { - const std::string bodystring = body.toString(); + if (parse_body_behavior_ != TransformationTemplate::DontParse && + body.length() > 0) { + const std::string &bodystring = get_body(); // parse the body as json - json_body = json::parse(bodystring); + // TODO: gate this under a parse_body boolean + if (parse_body_behavior_ == TransformationTemplate::ParseAsJson) { + if (ignore_error_on_parse_) { + try { + json_body = json::parse(bodystring); + } catch (std::exception &) { + } + } else { + json_body = json::parse(bodystring); + } + } else { + ASSERT("missing behavior"); + } } // get the extractions - std::unordered_map extractions; + std::unordered_map extractions; if (advanced_templates_) { extractions.reserve(extractors_.size()); } @@ -173,40 +258,43 @@ void InjaTransformer::transform(Http::HeaderMap &header_map, for (const auto &named_extractor : extractors_) { const std::string &name = named_extractor.first; if (advanced_templates_) { - extractions[name] = named_extractor.second.extract(header_map); + extractions[name] = named_extractor.second.extract(callbacks, header_map, get_body); } else { - std::string name_to_split = name; + absl::string_view name_to_split = name; json *current = &json_body; for (size_t pos = name_to_split.find("."); pos != std::string::npos; pos = name_to_split.find(".")) { auto &&field_name = name_to_split.substr(0, pos); - current = &(*current)[field_name]; - name_to_split.erase(0, pos + 1); + current = &(*current)[std::string(field_name)]; + name_to_split = name_to_split.substr(pos + 1); } - (*current)[name_to_split] = named_extractor.second.extract(header_map); + (*current)[std::string(name_to_split)] = + named_extractor.second.extract(callbacks, header_map, get_body); } } // start transforming! - TransformerInstance instance(header_map, extractions, json_body); + TransformerInstance instance(header_map, get_body, extractions, json_body, environ_); // Body transform: - auto replace_body = [&](std::string &output) { - // remove content length, as we have new body. - header_map.removeContentLength(); - // replace body - body.drain(body.length()); - body.add(output); - header_map.insertContentLength().value(body.length()); - }; + absl::optional maybe_body; if (body_template_.has_value()) { std::string output = instance.render(body_template_.value()); - replace_body(output); + maybe_body.emplace(output); } else if (merged_extractors_to_body_) { std::string output = json_body.dump(); - replace_body(output); + maybe_body.emplace(output); } + // DynamicMetadata transform: + for (const auto &templated_dynamic_metadata : dynamic_metadata_) { + std::string output = instance.render(templated_dynamic_metadata.template_); + if (!output.empty()) { + ProtobufWkt::Struct strct(MessageUtil::keyValueStruct(templated_dynamic_metadata.key_, output)); + callbacks.streamInfo().setDynamicMetadata(templated_dynamic_metadata.namespace_, strct); + } + } + // Headers transform: for (const auto &templated_header : headers_) { std::string output = instance.render(templated_header.second); @@ -219,6 +307,18 @@ void InjaTransformer::transform(Http::HeaderMap &header_map, header_map.addReferenceKey(templated_header.first, output); } } + + // replace body. we do it here so that headers and dynamic metadata have the original body. + if (maybe_body.has_value()) { + // remove content length, as we have new body. + header_map.removeContentLength(); + // replace body + body.drain(body.length()); + // prepend is used because it doesn't copy, it drains maybe_body + body.prepend(maybe_body.value()); + header_map.insertContentLength().value(body.length()); + } + } } // namespace Transformation diff --git a/source/extensions/filters/http/transformation/inja_transformer.h b/source/extensions/filters/http/transformation/inja_transformer.h index 12d649a64..df6e06095 100644 --- a/source/extensions/filters/http/transformation/inja_transformer.h +++ b/source/extensions/filters/http/transformation/inja_transformer.h @@ -19,34 +19,45 @@ namespace Extensions { namespace HttpFilters { namespace Transformation { +using GetBodyFunc = std::function; + class TransformerInstance { public: TransformerInstance( - const Http::HeaderMap &header_map, - const std::unordered_map &extractions, - const nlohmann::json &context); - // header_value(name) - // extracted_value(name, index) - nlohmann::json header_callback(inja::Arguments args); - - nlohmann::json extracted_callback(inja::Arguments args); + const Http::HeaderMap &header_map, GetBodyFunc body, + const std::unordered_map &extractions, + const nlohmann::json &context, const std::unordered_map& environ); + std::string render(const inja::Template &input); private: +// header_value(name) + nlohmann::json header_callback(const inja::Arguments& args) const; + // extracted_value(name, index) + nlohmann::json extracted_callback(const inja::Arguments& args) const; + nlohmann::json dynamic_metadata(const inja::Arguments& args) const; + nlohmann::json env(const inja::Arguments& args) const; + inja::Environment env_; const Http::HeaderMap &header_map_; - const std::unordered_map &extractions_; + GetBodyFunc body_; + const std::unordered_map &extractions_; const nlohmann::json &context_; + const std::unordered_map& environ_; }; -class Extractor { +class Extractor : Logger::Loggable { public: Extractor(const envoy::api::v2::filter::http::Extraction &extractor); - std::string extract(const Http::HeaderMap &header_map) const; + absl::string_view extract(Http::StreamFilterCallbacks &callbacks, const Http::HeaderMap &header_map, + GetBodyFunc body) const; private: + absl::string_view extractValue(Http::StreamFilterCallbacks &callbacks, absl::string_view value) const; + const Http::LowerCaseString headername_; + const bool body_; const unsigned int group_; const std::regex extract_regex_; }; @@ -57,25 +68,27 @@ class InjaTransformer : public Transformer { &transformation); ~InjaTransformer(); - void transform(Http::HeaderMap &map, Buffer::Instance &body, - Http::StreamFilterCallbacks&) const override; + void transform(Http::HeaderMap &map, Buffer::Instance &body, + Http::StreamFilterCallbacks &) const override; bool passthrough_body() const override { return passthrough_body_; }; private: - /* - TransformerImpl& impl() { return reinterpret_cast(impl_); - } const TransformerImpl& impl() const { return reinterpret_cast(impl_); } + struct DynamicMetadataValue { + std::string namespace_; + std::string key_; + inja::Template template_; + }; - static const size_t TransformerImplSize = 464; - static const size_t TransformerImplAlign = 8; - - std::aligned_storage::type impl_; - */ bool advanced_templates_{}; bool passthrough_body_{}; std::vector> extractors_; std::vector> headers_; + std::vector dynamic_metadata_; + std::unordered_map environ_; + + envoy::api::v2::filter::http::TransformationTemplate::RequestBodyParse + parse_body_behavior_; + bool ignore_error_on_parse_; absl::optional body_template_; bool merged_extractors_to_body_{}; diff --git a/test/extensions/filters/http/transformation/BUILD b/test/extensions/filters/http/transformation/BUILD index ab3c0da90..1bce7f2d8 100644 --- a/test/extensions/filters/http/transformation/BUILD +++ b/test/extensions/filters/http/transformation/BUILD @@ -2,6 +2,7 @@ licenses(["notice"]) # Apache 2 load( "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_binary", "envoy_package", ) load( @@ -12,9 +13,26 @@ load( envoy_package() envoy_gloo_cc_test( - name = "transformer_test", + name = "inja_transformer_test", srcs = ["inja_transformer_test.cc"], repository = "@envoy", + deps = [ + "//source/extensions/filters/http/transformation:inja_transformer_lib", + "@envoy//test/test_common:environment_lib", + "@envoy//test/mocks/http:http_mocks", + "@envoy//test/mocks/server:server_mocks", + "@envoy//test/mocks/upstream:upstream_mocks", + ], +) + +envoy_cc_binary( + name = "inja_transformer_speed_test", + testonly = 1, + srcs = ["inja_transformer_speed_test.cc"], + external_deps = [ + "benchmark", + ], + repository = "@envoy", deps = [ "//source/extensions/filters/http/transformation:inja_transformer_lib", "@envoy//test/mocks/http:http_mocks", @@ -28,8 +46,8 @@ envoy_gloo_cc_test( srcs = ["transformation_filter_test.cc"], repository = "@envoy", deps = [ - "//source/extensions/filters/http/transformation:transformation_filter_lib", "//source/extensions/filters/http:solo_well_known_names", + "//source/extensions/filters/http/transformation:transformation_filter_lib", "@envoy//test/mocks/http:http_mocks", "@envoy//test/mocks/server:server_mocks", "@envoy//test/mocks/upstream:upstream_mocks", @@ -42,8 +60,8 @@ envoy_gloo_cc_test( repository = "@envoy", deps = [ "//source/extensions/filters/http/transformation:body_header_transformer_lib", - "@envoy//test/test_common:utility_lib", "@envoy//test/mocks/http:http_mocks", "@envoy//test/mocks/server:server_mocks", + "@envoy//test/test_common:utility_lib", ], ) diff --git a/test/extensions/filters/http/transformation/inja_transformer_speed_test.cc b/test/extensions/filters/http/transformation/inja_transformer_speed_test.cc new file mode 100644 index 000000000..76fc777d7 --- /dev/null +++ b/test/extensions/filters/http/transformation/inja_transformer_speed_test.cc @@ -0,0 +1,49 @@ +#include "common/common/empty_string.h" + +#include "extensions/filters/http/transformation/inja_transformer.h" + +#include "test/test_common/utility.h" +#include "test/mocks/http/mocks.h" + +#include "benchmark/benchmark.h" +#include "fmt/format.h" + +using json = nlohmann::json; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Transformation { + +namespace { +std::function empty_body = [] { return EMPTY_STRING; }; +} + +static void BM_ExrtactHeader(benchmark::State &state) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, + {":authority", "www.solo.io"}, + {":path", "/users/123"}}; + envoy::api::v2::filter::http::Extraction extractor; + extractor.set_header(":path"); + extractor.set_regex("/users/(\\d+)"); + extractor.set_subgroup(1); + size_t output_bytes = 0; + NiceMock callbacks; + + Extractor ext(extractor); + for (auto _ : state) { + auto view = ext.extract(callbacks, headers, empty_body); + output_bytes += view.length(); + } + benchmark::DoNotOptimize(output_bytes); +} +// Register the function as a benchmark +BENCHMARK(BM_ExrtactHeader); + +} // namespace Transformation +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy + +// Run the benchmark +BENCHMARK_MAIN(); diff --git a/test/extensions/filters/http/transformation/inja_transformer_test.cc b/test/extensions/filters/http/transformation/inja_transformer_test.cc index 861706fde..67fede71d 100644 --- a/test/extensions/filters/http/transformation/inja_transformer_test.cc +++ b/test/extensions/filters/http/transformation/inja_transformer_test.cc @@ -1,9 +1,10 @@ #include "extensions/filters/http/transformation/inja_transformer.h" +#include "test/test_common/environment.h" #include "test/mocks/common.h" +#include "test/mocks/http/mocks.h" #include "test/mocks/server/mocks.h" #include "test/mocks/upstream/mocks.h" -#include "test/mocks/http/mocks.h" #include "fmt/format.h" #include "gmock/gmock.h" @@ -26,6 +27,13 @@ namespace Extensions { namespace HttpFilters { namespace Transformation { +using TransformationTemplate = + envoy::api::v2::filter::http::TransformationTemplate; + +namespace { +std::function empty_body = [] { return EMPTY_STRING; }; +} + inja::Template parse(std::string s) { inja::ParserConfig parser_config; inja::LexerConfig lexer_config; @@ -39,7 +47,10 @@ TEST(TransformerInstance, ReplacesValueFromContext) { json originalbody; originalbody["field1"] = "value1"; Http::TestHeaderMapImpl headers; - TransformerInstance t(headers, {}, originalbody); + std::unordered_map extractions; + std::unordered_map env; + + TransformerInstance t(headers, empty_body, extractions, originalbody, env); auto res = t.render(parse("{{field1}}")); @@ -52,9 +63,12 @@ TEST(TransformerInstance, ReplacesValueFromInlineHeader) { std::string path = "/getsomething"; Http::TestHeaderMapImpl headers{ - {":method", "GET"}, {":authority", "www.solo.io"}, {":path", path}}; + {":method", "GET"}, {":authority", "www.solo.io"}, {":path", path} + }; + std::unordered_map extractions; + std::unordered_map env; - TransformerInstance t(headers, {}, originalbody); + TransformerInstance t(headers, empty_body, extractions, originalbody, env); auto res = t.render(parse("{{header(\":path\")}}")); @@ -69,7 +83,10 @@ TEST(TransformerInstance, ReplacesValueFromCustomHeader) { {":authority", "www.solo.io"}, {":path", "/getsomething"}, {"x-custom-header", header}}; - TransformerInstance t(headers, {}, originalbody); + std::unordered_map extractions; + std::unordered_map env; + + TransformerInstance t(headers, empty_body, extractions, originalbody, env); auto res = t.render(parse("{{header(\"x-custom-header\")}}")); @@ -78,11 +95,13 @@ TEST(TransformerInstance, ReplacesValueFromCustomHeader) { TEST(TransformerInstance, ReplaceFromExtracted) { json originalbody; - std::unordered_map extractions; - std::string field = "res"; + std::unordered_map extractions; + absl::string_view field = "res"; extractions["f"] = field; Http::TestHeaderMapImpl headers; - TransformerInstance t(headers, extractions, originalbody); + std::unordered_map env; + + TransformerInstance t(headers, empty_body, extractions, originalbody, env); auto res = t.render(parse("{{extraction(\"f\")}}")); @@ -91,17 +110,45 @@ TEST(TransformerInstance, ReplaceFromExtracted) { TEST(TransformerInstance, ReplaceFromNonExistentExtraction) { json originalbody; - std::unordered_map extractions; - extractions["foo"] = "bar"; + std::unordered_map extractions; + extractions["foo"] = absl::string_view("bar"); Http::TestHeaderMapImpl headers; - TransformerInstance t(headers, extractions, originalbody); + std::unordered_map env; + + TransformerInstance t(headers, empty_body, extractions, originalbody, env); auto res = t.render(parse("{{extraction(\"notsuchfield\")}}")); EXPECT_EQ("", res); } -TEST(ExtractorUtil, ExtractIdFromHeader) { +TEST(TransformerInstance, Environment) { + json originalbody; + std::unordered_map extractions; + Http::TestHeaderMapImpl headers; + std::unordered_map env; + env["FOO"] = "BAR"; + + TransformerInstance t(headers, empty_body, extractions, originalbody, env); + + auto res = t.render(parse("{{env(\"FOO\")}}")); + EXPECT_EQ("BAR", res); +} + +TEST(TransformerInstance, EmptyEnvironment) { + json originalbody; + std::unordered_map extractions; + Http::TestHeaderMapImpl headers; + + std::unordered_map env; + + TransformerInstance t(headers, empty_body, extractions, originalbody, env); + + auto res = t.render(parse("{{env(\"FOO\")}}")); + EXPECT_EQ("", res); +} + +TEST(Extraction, ExtractIdFromHeader) { Http::TestHeaderMapImpl headers{{":method", "GET"}, {":authority", "www.solo.io"}, {":path", "/users/123"}}; @@ -109,12 +156,14 @@ TEST(ExtractorUtil, ExtractIdFromHeader) { extractor.set_header(":path"); extractor.set_regex("/users/(\\d+)"); extractor.set_subgroup(1); - auto res = Extractor(extractor).extract(headers); + + NiceMock callbacks; + std::string res(Extractor(extractor).extract(callbacks, headers, empty_body)); EXPECT_EQ("123", res); } -TEST(ExtractorUtil, ExtractorFail) { +TEST(Extraction, ExtractorFail) { Http::TestHeaderMapImpl headers{{":method", "GET"}, {":authority", "www.solo.io"}, {":path", "/users/123"}}; @@ -127,6 +176,18 @@ TEST(ExtractorUtil, ExtractorFail) { "\\a\\ a\\ \\d+)': regex_error"); } +TEST(Extraction, ExtractorFailOnOutOfRangeGroup) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, + {":authority", "www.solo.io"}, + {":path", "/users/123"}}; + envoy::api::v2::filter::http::Extraction extractor; + extractor.set_header(":path"); + extractor.set_regex("(\\d+)"); + extractor.set_subgroup(123); + EXPECT_THROW_WITH_MESSAGE(Extractor a(extractor), EnvoyException, + "group 123 requested for regex with only 1 sub groups"); +} + TEST(Transformer, transform) { Http::TestHeaderMapImpl headers{{":method", "GET"}, {":authority", "www.solo.io"}, @@ -139,7 +200,7 @@ TEST(Transformer, transform) { extractor.set_regex("/users/(\\d+)"); extractor.set_subgroup(1); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; (*transformation.mutable_extractors())["ext1"] = extractor; transformation.mutable_body()->set_text( @@ -150,8 +211,8 @@ TEST(Transformer, transform) { transformation.set_advanced_templates(true); InjaTransformer transformer(transformation); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); std::string res = body.toString(); @@ -171,7 +232,7 @@ TEST(Transformer, transformSimple) { extractor.set_regex("/users/(\\d+)"); extractor.set_subgroup(1); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; (*transformation.mutable_extractors())["ext1"] = extractor; transformation.mutable_body()->set_text( @@ -182,8 +243,8 @@ TEST(Transformer, transformSimple) { transformation.set_advanced_templates(false); InjaTransformer transformer(transformation); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); std::string res = body.toString(); @@ -203,7 +264,7 @@ TEST(Transformer, transformSimpleNestedStructs) { extractor.set_regex("/users/(\\d+)"); extractor.set_subgroup(1); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; (*transformation.mutable_extractors())["ext1.field1"] = extractor; transformation.mutable_body()->set_text( @@ -214,8 +275,8 @@ TEST(Transformer, transformSimpleNestedStructs) { transformation.set_advanced_templates(false); InjaTransformer transformer(transformation); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); std::string res = body.toString(); @@ -232,7 +293,7 @@ TEST(Transformer, transformPassthrough) { std::string emptyBody = ""; Buffer::OwnedImpl body(emptyBody); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; transformation.mutable_passthrough(); (*transformation.mutable_headers())["x-header"].set_text( @@ -241,8 +302,8 @@ TEST(Transformer, transformPassthrough) { transformation.set_advanced_templates(true); InjaTransformer transformer(transformation); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); std::string res = body.toString(); @@ -259,7 +320,7 @@ TEST(Transformer, transformMergeExtractorsToBody) { std::string emptyBody = ""; Buffer::OwnedImpl body(emptyBody); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; transformation.mutable_merge_extractors_to_body(); @@ -272,8 +333,8 @@ TEST(Transformer, transformMergeExtractorsToBody) { transformation.set_advanced_templates(false); InjaTransformer transformer(transformation); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); std::string res = body.toString(); @@ -288,7 +349,7 @@ TEST(Transformer, transformBodyNotSet) { std::string originalBody = "{\"a\":\"456\"}"; Buffer::OwnedImpl body(originalBody); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; // trying to get a value from the body; which should be available in default // mode @@ -297,8 +358,8 @@ TEST(Transformer, transformBodyNotSet) { transformation.set_advanced_templates(true); InjaTransformer transformer(transformation); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); std::string res = body.toString(); @@ -317,7 +378,7 @@ TEST(InjaTransformer, transformWithHyphens) { extractor.set_regex("/accounts/([\\-._[:alnum:]]+)"); extractor.set_subgroup(1); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; (*transformation.mutable_extractors())["id"] = extractor; @@ -325,8 +386,8 @@ TEST(InjaTransformer, transformWithHyphens) { transformation.mutable_merge_extractors_to_body(); InjaTransformer transformer(transformation); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); std::string res = body.toString(); @@ -339,7 +400,7 @@ TEST(InjaTransformer, RemoveHeadersUsingEmptyTemplate) { {":method", "GET"}, {":path", "/foo"}, {content_type, "x-test"}}; Buffer::OwnedImpl body("{}"); - envoy::api::v2::filter::http::TransformationTemplate transformation; + TransformationTemplate transformation; envoy::api::v2::filter::http::InjaTemplate empty; (*transformation.mutable_headers())[content_type] = empty; @@ -347,11 +408,144 @@ TEST(InjaTransformer, RemoveHeadersUsingEmptyTemplate) { InjaTransformer transformer(transformation); EXPECT_TRUE(headers.has(content_type)); - NiceMock filter_callbacks_{}; - transformer.transform(headers, body, filter_callbacks_); + NiceMock callbacks; + transformer.transform(headers, body, callbacks); EXPECT_FALSE(headers.has(content_type)); } +TEST(InjaTransformer, DontParseBodyAndExtractFromIt) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/foo"}}; + Buffer::OwnedImpl body("not json body"); + + TransformationTemplate transformation; + transformation.set_parse_body_behavior(TransformationTemplate::DontParse); + transformation.set_advanced_templates(true); + + envoy::api::v2::filter::http::Extraction extractor; + extractor.mutable_body(); + extractor.set_regex("not ([\\-._[:alnum:]]+) body"); + extractor.set_subgroup(1); + (*transformation.mutable_extractors())["param"] = extractor; + + transformation.mutable_body()->set_text("{{extraction(\"param\")}}"); + + InjaTransformer transformer(transformation); + + NiceMock callbacks; + transformer.transform(headers, body, callbacks); + EXPECT_EQ(body.toString(), "json"); +} + +TEST(InjaTransformer, UseBodyFunction) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/foo"}}; + TransformationTemplate transformation; + transformation.set_parse_body_behavior(TransformationTemplate::DontParse); + transformation.set_advanced_templates(true); + + transformation.mutable_body()->set_text("{{body()}} {{body()}}"); + + InjaTransformer transformer(transformation); + + NiceMock callbacks; + Buffer::OwnedImpl body("1"); + transformer.transform(headers, body, callbacks); + EXPECT_EQ(body.toString(), "1 1"); +} + +TEST(InjaTransformer, UseDefaultNS) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/foo"}}; + TransformationTemplate transformation; + transformation.set_parse_body_behavior(TransformationTemplate::DontParse); + transformation.set_advanced_templates(true); + + auto dynamic_meta = transformation.add_dynamic_metadata_values(); + dynamic_meta->set_key("foo"); + dynamic_meta->mutable_value()->set_text("{{body()}}"); + + InjaTransformer transformer(transformation); + + NiceMock callbacks; + + EXPECT_CALL(callbacks.stream_info_, setDynamicMetadata("io.solo.transformation", _)).Times(1) + .WillOnce(Invoke([](const std::string&, const ProtobufWkt::Struct& value) { + auto field = value.fields().at("foo"); + EXPECT_EQ(field.string_value(), "1"); + })); + Buffer::OwnedImpl body("1"); + transformer.transform(headers, body, callbacks); +} + +TEST(InjaTransformer, UseCustomNS) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/foo"}}; + TransformationTemplate transformation; + transformation.set_parse_body_behavior(TransformationTemplate::DontParse); + transformation.set_advanced_templates(true); + + auto dynamic_meta = transformation.add_dynamic_metadata_values(); + dynamic_meta->set_key("foo"); + dynamic_meta->set_metadata_namespace("foo.ns"); + dynamic_meta->mutable_value()->set_text("123"); + + InjaTransformer transformer(transformation); + + NiceMock callbacks; + + EXPECT_CALL(callbacks.stream_info_, setDynamicMetadata("foo.ns", _)).Times(1); + Buffer::OwnedImpl body; + transformer.transform(headers, body, callbacks); +} + +TEST(InjaTransformer, UseDynamicMetaTwice) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/foo"}}; + TransformationTemplate transformation; + + auto dynamic_meta = transformation.add_dynamic_metadata_values(); + dynamic_meta->set_key("foo"); + dynamic_meta->mutable_value()->set_text("{{body()}}"); + dynamic_meta = transformation.add_dynamic_metadata_values(); + dynamic_meta->set_key("bar"); + dynamic_meta->mutable_value()->set_text("123"); + + InjaTransformer transformer(transformation); + + NiceMock callbacks; + + EXPECT_CALL(callbacks.stream_info_, setDynamicMetadata("io.solo.transformation", _)).Times(2); + Buffer::OwnedImpl body("1"); + transformer.transform(headers, body, callbacks); +} + +TEST(InjaTransformer, UseEnvVar) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/foo"}}; + TransformationTemplate transformation; + transformation.mutable_body()->set_text("{{env(\"FOO\")}}"); + // set env before calling transforer + TestEnvironment::setEnvVar("FOO", "BAR", 1); + TestEnvironment::setEnvVar("EMPTY", "", 1); + + InjaTransformer transformer(transformation); + + NiceMock callbacks; + + Buffer::OwnedImpl body("1"); + transformer.transform(headers, body, callbacks); + EXPECT_EQ(body.toString(), "BAR"); +} + + +TEST(InjaTransformer, ParseBodyListUsingContext) { + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/foo"}}; + TransformationTemplate transformation; + transformation.mutable_body()->set_text("{% for i in context() %}{{ i }}{% endfor %}"); + InjaTransformer transformer(transformation); + + NiceMock callbacks; + + Buffer::OwnedImpl body("[3,2,1]"); + transformer.transform(headers, body, callbacks); + EXPECT_EQ(body.toString(), "321"); +} + } // namespace Transformation } // namespace HttpFilters } // namespace Extensions