diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a55b6caec7b7c..3773d2afd0d8d 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -105,6 +105,13 @@ new_features: change: | Added support for not-equal operator for access log filter rules, in :ref:`ComparisonFilter `. +- area: formatter + change: | + Added support for the following new access log formatters: + - ``%REQUEST_HEADER(X?Y):Z%`` as full name version of ``%REQ(X?Y):Z%`` + - ``%RESPONSE_HEADER(X?Y):Z%`` as full name version of ``%RESP(X?Y):Z%`` + - ``%RESPONSE_TRAILER(X?Y):Z%`` as full name version of ``%TRAILER(X?Y):Z%`` + This mainly to provide a more consistent naming scheme for users to understand and use. - area: lua change: | Added ``drainConnectionUponCompletion()`` to the Lua filter stream info API. This allows Lua scripts diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index f46144b0fb577..bce538e9f38c2 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -44,10 +44,10 @@ If custom format string is not specified, Envoy uses the following default forma .. code-block:: none - [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" + [%START_TIME%] "%REQUEST_HEADER(:METHOD)% %REQUEST_HEADER(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% - %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" - "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + %RESPONSE_HEADER(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQUEST_HEADER(X-FORWARDED-FOR)%" "%REQUEST_HEADER(USER-AGENT)%" + "%REQUEST_HEADER(X-REQUEST-ID)%" "%REQUEST_HEADER(:AUTHORITY)%" "%UPSTREAM_HOST%"\n Example of the default Envoy access log format: @@ -75,7 +75,7 @@ For example, with the following format provided in the configuration as ``json_f "json_format": { "protocol": "%PROTOCOL%", "duration": "%DURATION%", - "my_custom_header": "%REQ(MY_CUSTOM_HEADER)%" + "my_custom_header": "%REQUEST_HEADER(MY_CUSTOM_HEADER)%" } } } @@ -835,8 +835,8 @@ UDP An identifier for the stream (HTTP request, long-live HTTP2 stream, TCP connection, etc.). It can be used to cross-reference TCP access logs across multiple log sinks, or to cross-reference timer-based reports for the same connection. Different with %CONNECTION_ID%, the identifier should be unique across multiple instances or between restarts. - And it's value should be same with %REQ(X-REQUEST-ID)% for HTTP request. - This should be used to replace %CONNECTION_ID% and %REQ(X-REQUEST-ID)% in most cases. + And it's value should be same with %REQUEST_HEADER(X-REQUEST-ID)% for HTTP request. + This should be used to replace %CONNECTION_ID% and %REQUEST_HEADER(X-REQUEST-ID)% in most cases. %GRPC_STATUS(X)% `gRPC status code `_ formatted according to the optional parameter ``X``, which can be ``CAMEL_STRING``, ``SNAKE_STRING`` and ``NUMBER``. @@ -848,7 +848,7 @@ UDP .. _config_access_log_format_req: -%REQ(X?Y):Z% +%REQUEST_HEADER(X?Y):Z%/%REQ(X?Y):Z% HTTP An HTTP request header where X is the main HTTP header, Y is the alternative one, and Z is an optional parameter denoting string truncation up to Z characters long. The value is taken from @@ -858,16 +858,16 @@ UDP TCP/UDP Not implemented ("-"). -%RESP(X?Y):Z% +%RESPONSE_HEADER(X?Y):Z%/%RESP(X?Y):Z% HTTP - Same as **%REQ(X?Y):Z%** but taken from HTTP response headers. + Same as **%REQUEST_HEADER(X?Y):Z%** but taken from HTTP response headers. TCP/UDP Not implemented ("-"). -%TRAILER(X?Y):Z% +%RESPONSE_TRAILER(X?Y):Z%/%TRAILER(X?Y):Z% HTTP - Same as **%REQ(X?Y):Z%** but taken from HTTP response trailers. + Same as **%REQUEST_HEADER(X?Y):Z%** but taken from HTTP response trailers. TCP/UDP Not implemented ("-"). diff --git a/source/common/formatter/http_formatter_context.cc b/source/common/formatter/http_formatter_context.cc index 1d1c29e638d6a..97559f5897e49 100644 --- a/source/common/formatter/http_formatter_context.cc +++ b/source/common/formatter/http_formatter_context.cc @@ -10,11 +10,19 @@ namespace Envoy { namespace Formatter { static constexpr absl::string_view DEFAULT_FORMAT = - "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" " - "%RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% " - "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% " - "\"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" " - "\"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"; + "[%START_TIME%] " + "\"%REQUEST_HEADER(:METHOD)% %REQUEST_HEADER(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" " + "%RESPONSE_CODE% " + "%RESPONSE_FLAGS% " + "%BYTES_RECEIVED% " + "%BYTES_SENT% " + "%DURATION% " + "%RESPONSE_HEADER(X-ENVOY-UPSTREAM-SERVICE-TIME)% " + "\"%REQUEST_HEADER(X-FORWARDED-FOR)%\" " + "\"%REQUEST_HEADER(USER-AGENT)%\" " + "\"%REQUEST_HEADER(X-REQUEST-ID)%\" " + "\"%REQUEST_HEADER(:AUTHORITY)%\" " + "\"%UPSTREAM_HOST%\"\n"; absl::StatusOr HttpSubstitutionFormatUtils::defaultSubstitutionFormatter() { // It is possible that failed to parse the default format string if the required formatters diff --git a/source/common/formatter/http_specific_formatter.cc b/source/common/formatter/http_specific_formatter.cc index 247c03f0c0e6a..9bf02757a65ca 100644 --- a/source/common/formatter/http_specific_formatter.cc +++ b/source/common/formatter/http_specific_formatter.cc @@ -388,7 +388,7 @@ const BuiltInHttpCommandParser::FormatterProviderLookupTbl& BuiltInHttpCommandParser::getKnownFormatters() { CONSTRUCT_ON_FIRST_USE( FormatterProviderLookupTbl, - {{"REQ", + {{"REQ", // Same as REQUEST_HEADER and used for backward compatibility. {CommandSyntaxChecker::PARAMS_REQUIRED | CommandSyntaxChecker::LENGTH_ALLOWED, [](absl::string_view format, absl::optional max_length) { auto result = SubstitutionFormatUtils::parseSubcommandHeaders(format); @@ -396,7 +396,15 @@ BuiltInHttpCommandParser::getKnownFormatters() { return std::make_unique(result.value().first, result.value().second, max_length); }}}, - {"RESP", + {"REQUEST_HEADER", + {CommandSyntaxChecker::PARAMS_REQUIRED | CommandSyntaxChecker::LENGTH_ALLOWED, + [](absl::string_view format, absl::optional max_length) { + auto result = SubstitutionFormatUtils::parseSubcommandHeaders(format); + THROW_IF_NOT_OK_REF(result.status()); + return std::make_unique(result.value().first, + result.value().second, max_length); + }}}, + {"RESP", // Same as RESPONSE_HEADER and used for backward compatibility. {CommandSyntaxChecker::PARAMS_REQUIRED | CommandSyntaxChecker::LENGTH_ALLOWED, [](absl::string_view format, absl::optional max_length) { auto result = SubstitutionFormatUtils::parseSubcommandHeaders(format); @@ -404,7 +412,23 @@ BuiltInHttpCommandParser::getKnownFormatters() { return std::make_unique(result.value().first, result.value().second, max_length); }}}, - {"TRAILER", + {"RESPONSE_HEADER", + {CommandSyntaxChecker::PARAMS_REQUIRED | CommandSyntaxChecker::LENGTH_ALLOWED, + [](absl::string_view format, absl::optional max_length) { + auto result = SubstitutionFormatUtils::parseSubcommandHeaders(format); + THROW_IF_NOT_OK_REF(result.status()); + return std::make_unique(result.value().first, + result.value().second, max_length); + }}}, + {"TRAILER", // Same as RESPONSE_TRAILER and used for backward compatibility. + {CommandSyntaxChecker::PARAMS_REQUIRED | CommandSyntaxChecker::LENGTH_ALLOWED, + [](absl::string_view format, absl::optional max_length) { + auto result = SubstitutionFormatUtils::parseSubcommandHeaders(format); + THROW_IF_NOT_OK_REF(result.status()); + return std::make_unique(result.value().first, + result.value().second, max_length); + }}}, + {"RESPONSE_TRAILER", {CommandSyntaxChecker::PARAMS_REQUIRED | CommandSyntaxChecker::LENGTH_ALLOWED, [](absl::string_view format, absl::optional max_length) { auto result = SubstitutionFormatUtils::parseSubcommandHeaders(format); diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 4e675b083346a..da8786e1a933a 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -4249,14 +4249,16 @@ TEST(SubstitutionFormatterTest, CompositeFormatterSuccess) { { NiceMock stream_info; const std::string format = "{{%PROTOCOL%}} %RESP(not_exist)%++%RESP(test)% " - "%REQ(FIRST?SECOND)% %RESP(FIRST?SECOND)%" - "\t@%TRAILER(THIRD)%@\t%TRAILER(TEST?TEST-2)%[]"; + "%REQ(FIRST?SECOND)%/%REQUEST_HEADER(FIRST?SECOND)% " + "%RESP(FIRST?SECOND)%/%RESPONSE_HEADER(FIRST?SECOND)% " + "\t@%TRAILER(THIRD)%@\t%TRAILER(TEST?TEST-2)%[] " + "%RESPONSE_TRAILER(THIRD)%"; FormatterPtr formatter = *FormatterImpl::create(format, false); absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); - EXPECT_EQ("{{HTTP/1.1}} -++test GET PUT\t@POST@\ttest-2[]", + EXPECT_EQ("{{HTTP/1.1}} -++test GET/GET PUT/PUT \t@POST@\ttest-2[] POST", formatter->format(formatter_context, stream_info)); }