diff --git a/CMakeLists.txt b/CMakeLists.txt index 55181e373a0..692b11985a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,6 +341,33 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) +find_package(zstd CONFIG QUIET) +if(zstd_FOUND) + + # Provide a compatibility target name if the upstream package does not export it + # Our code links against `zstd::zstd`; upstream zstd usually exports + # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed. + if(NOT TARGET zstd::zstd) + if(TARGET zstd::libzstd_shared) + set(_zstd_target zstd::libzstd_shared) + elseif(TARGET zstd::libzstd_static) + set(_zstd_target zstd::libzstd_static) + elseif(TARGET zstd::libzstd) + set(_zstd_target zstd::libzstd) + endif() + if(DEFINED _zstd_target) + add_library(zstd_zstd INTERFACE) + target_link_libraries(zstd_zstd INTERFACE ${_zstd_target}) + add_library(zstd::zstd ALIAS zstd_zstd) + set(HAVE_ZSTD_H TRUE) + else() + set(HAVE_ZSTD_H FALSE) + endif() + endif() +else() + set(HAVE_ZSTD_H FALSE) +endif() + # ncurses is used in traffic_top find_package(Curses) set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H}) diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 337356ca8c3..4e1398d15b7 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -55,7 +55,8 @@ RUN apt-get update; apt-get -y dist-upgrade; \ apt-get -y install libssl-dev libexpat1-dev libpcre3-dev libcap-dev \ libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \ tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \ - libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev; \ + libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \ + libzstd-dev; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \ groupadd -g 665 jenkins && \ diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 85e9a7add64..5a160fb24ff 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y update; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \ - ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel \ + ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 httpd-tools procps-ng nmap-ncat pipenv \ diff --git a/contrib/docker/ubuntu/noble/Dockerfile b/contrib/docker/ubuntu/noble/Dockerfile index 804cbb851ed..8164fb1d59f 100644 --- a/contrib/docker/ubuntu/noble/Dockerfile +++ b/contrib/docker/ubuntu/noble/Dockerfile @@ -48,6 +48,8 @@ RUN apt update \ libpcre3-dev \ hwloc \ libbrotli-dev \ + libzstd-dev \ + luajit \ libluajit-5.1-dev \ libcap-dev \ libmagick++-dev \ diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index 920ec5d7d40..cfa532c8bde 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -2088,10 +2088,14 @@ Proxy User Variables normalize as for value ``1`` ``3`` ``Accept-Encoding: br, gzip`` (if the header has ``br`` and ``gzip`` (with any ``q`` for either) then ``br, gzip``) **ELSE** normalize as for value ``2`` + ``4`` ``Accept-Encoding: zstd`` if the header has ``zstd`` (with any ``q``) **ELSE** + normalize as for value ``2`` + ``5`` ``Accept-Encoding: zstd, br, gzip`` (supports all combinations of ``zstd``, ``br``, and ``gzip``) **ELSE** + normalize as for value ``4`` ===== ====================================================================== This is useful for minimizing cached alternates of documents (e.g. ``gzip, deflate`` vs. ``deflate, gzip``). - Enabling this option is recommended if your origin servers use no encodings other than ``gzip`` or ``br`` (Brotli). + Enabling this option is recommended if your origin servers use no encodings other than ``gzip``, ``br`` (Brotli), or ``zstd`` (Zstandard). Security ======== diff --git a/doc/admin-guide/plugins/compress.en.rst b/doc/admin-guide/plugins/compress.en.rst index 4627adf78b7..77d2bb82868 100644 --- a/doc/admin-guide/plugins/compress.en.rst +++ b/doc/admin-guide/plugins/compress.en.rst @@ -202,12 +202,59 @@ supported-algorithms Provides the compression algorithms that are supported, a comma separate list of values. This will allow |TS| to selectively support ``gzip``, ``deflate``, -and brotli (``br``) compression. The default is ``gzip``. Multiple algorithms can -be selected using ',' delimiter, for instance, ``supported-algorithms -deflate,gzip,br``. Note that this list must **not** contain any white-spaces! +brotli (``br``), and zstd (``zstd``) compression. The default is ``gzip``. +Multiple algorithms can be selected using ',' delimiter, for instance, +``supported-algorithms deflate,gzip,br,zstd``. Note that this list must **not** +contain any white-spaces! + +============== ================================================================= +Algorithm Description +============== ================================================================= +gzip Standard gzip compression (default, widely supported) +deflate Deflate compression (RFC 1951) +br Brotli compression (modern, efficient) +zstd Zstandard compression (fast, high compression ratio) +============== ================================================================= Note that if :ts:cv:`proxy.config.http.normalize_ae` is ``1``, only gzip will -be considered, and if it is ``2``, only br or gzip will be considered. +be considered, if it is ``2``, only br or gzip will be considered, if it is ``4``, +only zstd, br, or gzip will be considered, and if it is ``5``, all combinations +of zstd, br, and gzip will be considered. + +gzip-compression-level +----------------------- + +Sets the compression level for gzip compression. Valid values are 1-9, where +1 is fastest compression (lowest compression ratio) and 9 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-compression-level +------------------------- + +Sets the compression level for Brotli compression. Valid values are 0-11, where +0 is fastest compression (lowest compression ratio) and 11 is slowest compression +(highest compression ratio). The default is 6, which provides a good balance +between compression speed and ratio. + +brotli-lgwin +------------ + +Sets the window size for Brotli compression. Valid values are 10-24, where +larger values provide better compression but use more memory. The default is 16. +This parameter controls the sliding window size used during compression: + +- 10: 1KB window (fastest, least memory) +- 16: 64KB window (default, good balance) +- 24: 16MB window (slowest, most memory, best compression) + +zstd-compression-level +---------------------- + +Sets the compression level for Zstandard compression. Valid values are 1-22, where +1 is fastest compression (lowest compression ratio) and 22 is slowest compression +(highest compression ratio). The default is 12, which provides an excellent +balance between compression speed and ratio for web content. Examples ======== @@ -224,6 +271,10 @@ might create a configuration with the following options:: compressible-status-code 200, 206 minimum-content-length 860 flush false + gzip-compression-level 6 + brotli-compression-level 6 + brotli-lgwin 16 + zstd-compression-level 12 # Now set a configuration for www.example.com [www.example.com] @@ -242,7 +293,7 @@ might create a configuration with the following options:: flush true supported-algorithms gzip,deflate - # Supports brotli compression + # Supports brotli compression with custom settings [brotli.compress.com] enabled true compressible-content-type text/* @@ -250,6 +301,30 @@ might create a configuration with the following options:: content_type_ignore_parameters true flush true supported-algorithms br,gzip + brotli-compression-level 8 + brotli-lgwin 20 + + # Supports zstd compression for high efficiency + [zstd.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + compressible-content-type application/javascript + flush true + supported-algorithms zstd,gzip + zstd-compression-level 15 + + # Supports all compression algorithms with optimized settings + [all.compress.com] + enabled true + compressible-content-type text/* + compressible-content-type application/json + flush true + supported-algorithms zstd,br,gzip,deflate + gzip-compression-level 7 + brotli-compression-level 9 + brotli-lgwin 18 + zstd-compression-level 10 # This origin does it all [bar.example.com] diff --git a/doc/developer-guide/plugins/http-headers/header-functions.en.rst b/doc/developer-guide/plugins/http-headers/header-functions.en.rst index 9a27115434f..47d07459a2c 100644 --- a/doc/developer-guide/plugins/http-headers/header-functions.en.rst +++ b/doc/developer-guide/plugins/http-headers/header-functions.en.rst @@ -92,6 +92,9 @@ headers. ``TS_HTTP_VALUE_GZIP`` "gzip" +``TS_HTTP_VALUE_ZSTD`` + "zstd" + ``TS_HTTP_VALUE_IDENTITY`` "identity" diff --git a/doc/release-notes/whats-new.en.rst b/doc/release-notes/whats-new.en.rst index f00111e5513..0c64321ac8f 100644 --- a/doc/release-notes/whats-new.en.rst +++ b/doc/release-notes/whats-new.en.rst @@ -185,6 +185,7 @@ Plugins * xdebug - ``--enable`` option to selectively enable features has been added * system_stats - Stats about memory have been added * slice plugin - This plugin was promoted to stable. +* compress plugin - Added support for Zstandard (zstd) compression algorithm. JSON-RPC ^^^^^^^^ diff --git a/include/proxy/hdrs/HTTP.h b/include/proxy/hdrs/HTTP.h index eeecba31d74..8759fcc09ab 100644 --- a/include/proxy/hdrs/HTTP.h +++ b/include/proxy/hdrs/HTTP.h @@ -368,6 +368,7 @@ extern c_str_view HTTP_VALUE_COMPRESS; extern c_str_view HTTP_VALUE_DEFLATE; extern c_str_view HTTP_VALUE_GZIP; extern c_str_view HTTP_VALUE_BROTLI; +extern c_str_view HTTP_VALUE_ZSTD; extern c_str_view HTTP_VALUE_IDENTITY; extern c_str_view HTTP_VALUE_KEEP_ALIVE; extern c_str_view HTTP_VALUE_MAX_AGE; diff --git a/include/proxy/hdrs/MIME.h b/include/proxy/hdrs/MIME.h index dcea4893594..4bed80dc13c 100644 --- a/include/proxy/hdrs/MIME.h +++ b/include/proxy/hdrs/MIME.h @@ -602,6 +602,8 @@ extern c_str_view MIME_VALUE_COMPRESS; extern c_str_view MIME_VALUE_DEFLATE; extern c_str_view MIME_VALUE_GZIP; extern c_str_view MIME_VALUE_BROTLI; +extern c_str_view MIME_VALUE_ZSTD; + extern c_str_view MIME_VALUE_IDENTITY; extern c_str_view MIME_VALUE_KEEP_ALIVE; extern c_str_view MIME_VALUE_MAX_AGE; diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 0bc28141907..078ce0eb69c 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -1354,6 +1354,7 @@ extern const char *TS_HTTP_VALUE_COMPRESS; extern const char *TS_HTTP_VALUE_DEFLATE; extern const char *TS_HTTP_VALUE_GZIP; extern const char *TS_HTTP_VALUE_BROTLI; +extern const char *TS_HTTP_VALUE_ZSTD; extern const char *TS_HTTP_VALUE_IDENTITY; extern const char *TS_HTTP_VALUE_KEEP_ALIVE; extern const char *TS_HTTP_VALUE_MAX_AGE; @@ -1378,6 +1379,7 @@ extern int TS_HTTP_LEN_COMPRESS; extern int TS_HTTP_LEN_DEFLATE; extern int TS_HTTP_LEN_GZIP; extern int TS_HTTP_LEN_BROTLI; +extern int TS_HTTP_LEN_ZSTD; extern int TS_HTTP_LEN_IDENTITY; extern int TS_HTTP_LEN_KEEP_ALIVE; extern int TS_HTTP_LEN_MAX_AGE; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index 02cc203422e..300cd5ec26a 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -46,6 +46,7 @@ #cmakedefine HAVE_POSIX_FADVISE 1 #cmakedefine HAVE_POSIX_FALLOCATE 1 #cmakedefine HAVE_POSIX_MADVISE 1 +#cmakedefine HAVE_ZSTD_H 1 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1 diff --git a/plugins/compress/CMakeLists.txt b/plugins/compress/CMakeLists.txt index 6744d69d47b..6cd700c10ca 100644 --- a/plugins/compress/CMakeLists.txt +++ b/plugins/compress/CMakeLists.txt @@ -23,5 +23,11 @@ if(HAVE_BROTLI_ENCODE_H) target_link_libraries(compress PRIVATE brotli::brotlienc) target_compile_definitions(compress PRIVATE HAVE_BROTLI_ENCODE_H=1) endif() + +if(HAVE_ZSTD_H) + target_sources(compress PRIVATE zstd_compress.cc) + target_link_libraries(compress PRIVATE zstd::zstd) +endif() + verify_global_plugin(compress) verify_remap_plugin(compress) diff --git a/plugins/compress/README b/plugins/compress/README index 759add28e03..f963a8e05e9 100644 --- a/plugins/compress/README +++ b/plugins/compress/README @@ -1,7 +1,7 @@ What this plugin does: ===================== -This plugin compresses responses, via gzip or brotli, whichever is applicable +This plugin compresses responses, via gzip, deflate, brotli, or zstd (Zstandard), whichever is applicable it can compress origin responses as well as cached responses installation: @@ -24,4 +24,4 @@ compress.so /sample.compress.config After modifying plugin.config, restart traffic server (sudo traffic_ctl server restart) the configuration is re-read when a management update is given (sudo traffic_ctl config reload) -See sample.config.compress for an example configuration and the options that are available +See sample.compress.config for an example configuration and the options that are available diff --git a/plugins/compress/compress.cc b/plugins/compress/compress.cc index 319059ea567..71dced8e38e 100644 --- a/plugins/compress/compress.cc +++ b/plugins/compress/compress.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -38,6 +38,7 @@ #include "configuration.h" #include "gzip_compress.h" #include "brotli_compress.h" +#include "zstd_compress.h" #include "ts/remap.h" #include "ts/remap_version.h" @@ -133,9 +134,8 @@ namespace static Data * data_alloc(int compression_type, int compression_algorithms, HostConfiguration *hc) { - Data *data; + Data *data = static_cast(TSmalloc(sizeof(Data))); - data = static_cast(TSmalloc(sizeof(Data))); data->downstream_vio = nullptr; data->downstream_buffer = nullptr; data->downstream_reader = nullptr; @@ -156,6 +156,12 @@ data_alloc(int compression_type, int compression_algorithms, HostConfiguration * Brotli::data_alloc(data); } #endif +#if HAVE_ZSTD_H + if ((compression_type & COMPRESSION_TYPE_ZSTD) && (compression_algorithms & ALGORITHM_ZSTD)) { + Zstd::data_alloc(data); + } +#endif + return data; } @@ -179,6 +185,11 @@ data_destroy(Data *data) Brotli::data_destroy(data); } #endif +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { + Zstd::data_destroy(data); + } +#endif TSfree(data); } @@ -191,7 +202,10 @@ content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int compression_ty const char *value = nullptr; int value_len = 0; // Delete Content-Encoding if present??? - if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { + if (compression_type & COMPRESSION_TYPE_ZSTD && (algorithm & ALGORITHM_ZSTD)) { + value = TS_HTTP_VALUE_ZSTD; + value_len = TS_HTTP_LEN_ZSTD; + } else if (compression_type & COMPRESSION_TYPE_BROTLI && (algorithm & ALGORITHM_BROTLI)) { value = TS_HTTP_VALUE_BROTLI; value_len = TS_HTTP_LEN_BROTLI; } else if (compression_type & COMPRESSION_TYPE_GZIP && (algorithm & ALGORITHM_GZIP)) { @@ -323,6 +337,15 @@ compress_transform_init(TSCont contp, Data *data) data->downstream_vio = TSVConnWrite(downstream_conn, contp, data->downstream_reader, INT64_MAX); } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { + if (!Zstd::transform_init(data)) { + error("Failed to configure Zstandard compression context"); + return; + } + } +#endif + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); } @@ -348,8 +371,13 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) upstream_length = amount; } +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && (data->compression_algorithms & ALGORITHM_ZSTD)) { + Zstd::transform_one(data, upstream_buffer, upstream_length); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && (data->compression_algorithms & ALGORITHM_BROTLI)) { Brotli::transform_one(data, upstream_buffer, upstream_length); } else #endif @@ -357,7 +385,13 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) (data->compression_algorithms & (ALGORITHM_GZIP | ALGORITHM_DEFLATE))) { Gzip::transform_one(data, upstream_buffer, upstream_length); } else { - warning("No compression supported. Shouldn't come here."); + warning("No compression supported. Passing data through without transformation."); + int64_t written = TSIOBufferWrite(data->downstream_buffer, upstream_buffer, upstream_length); + if (written == TS_ERROR || written != upstream_length) { + error("Failed to copy upstream data to downstream buffer"); + return; + } + data->downstream_length += written; } TSIOBufferReaderConsume(upstream_reader, upstream_length); @@ -368,8 +402,14 @@ compress_transform_one(Data *data, TSIOBufferReader upstream_reader, int amount) static void compress_transform_finish(Data *data) { +#if HAVE_ZSTD_H + if (data->compression_type & COMPRESSION_TYPE_ZSTD && data->compression_algorithms & ALGORITHM_ZSTD) { + Zstd::transform_finish(data); + debug("compress_transform_finish: zstd compression finish"); + } else +#endif #if HAVE_BROTLI_ENCODE_H - if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { + if (data->compression_type & COMPRESSION_TYPE_BROTLI && data->compression_algorithms & ALGORITHM_BROTLI) { Brotli::transform_finish(data); debug("compress_transform_finish: brotli compression finish"); } else @@ -379,7 +419,7 @@ compress_transform_finish(Data *data) Gzip::transform_finish(data); debug("compress_transform_finish: gzip compression finish"); } else { - error("No Compression matched, shouldn't come here"); + debug("compress_transform_finish: no compression active, passthrough mode"); } } @@ -580,7 +620,14 @@ transformable(TSHttpTxn txnp, bool server, HostConfiguration *host_configuration continue; } - if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { + info("Accept-Encoding value [%.*s]", len, value); + + if (strncasecmp(value, "zstd", sizeof("zstd") - 1) == 0) { + if (*algorithms & ALGORITHM_ZSTD) { + compression_acceptable = 1; + } + *compress_type |= COMPRESSION_TYPE_ZSTD; + } else if (strncasecmp(value, "br", sizeof("br") - 1) == 0) { if (*algorithms & ALGORITHM_BROTLI) { compression_acceptable = 1; } diff --git a/plugins/compress/compress_common.h b/plugins/compress/compress_common.h index 1dccd4aa256..ecb5939dfa4 100644 --- a/plugins/compress/compress_common.h +++ b/plugins/compress/compress_common.h @@ -23,20 +23,29 @@ #pragma once -#include #include +#include "tscore/ink_config.h" + +#include +#include + #if HAVE_BROTLI_ENCODE_H #include #endif +#if HAVE_ZSTD_H +#include +#endif + #include "configuration.h" enum CompressionType { COMPRESSION_TYPE_DEFAULT = 0, COMPRESSION_TYPE_DEFLATE = 1, COMPRESSION_TYPE_GZIP = 2, - COMPRESSION_TYPE_BROTLI = 4 + COMPRESSION_TYPE_BROTLI = 4, + COMPRESSION_TYPE_ZSTD = 8 }; enum transform_state { @@ -57,6 +66,14 @@ struct BrotliStream { }; #endif +#if HAVE_ZSTD_H +struct ZstdStream { + ZSTD_CCtx *cctx; + int64_t total_in; + int64_t total_out; +}; +#endif + struct Data { TSHttpTxn txn; Compress::HostConfiguration *hc; @@ -71,6 +88,9 @@ struct Data { #if HAVE_BROTLI_ENCODE_H BrotliStream bstrm; #endif +#if HAVE_ZSTD_H + ZstdStream zstrm_zstd; +#endif }; void log_compression_ratio(int64_t in, int64_t out); diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index 9b76ffa6191..3ba265d2845 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -62,7 +62,11 @@ enum class ParserState { Flush, Allow, MinimumContentLength, - ContentTypeIgnoreParameters + ContentTypeIgnoreParameters, + GzipCompressionLevel, + BrotliCompressionLevel, + BrotliLGWSize, + ZstdCompressionLevel }; void @@ -197,6 +201,12 @@ HostConfiguration::add_compression_algorithms(swoc::TextView line) auto token = extractFirstToken(line, isCommaOrSpace); if (token.empty()) { break; + } else if (token == "zstd") { +#ifdef HAVE_ZSTD_H + compression_algorithms_ |= ALGORITHM_ZSTD; +#else + error("supported-algorithms: zstd support not compiled in."); +#endif } else if (token == "br") { #ifdef HAVE_BROTLI_ENCODE_H compression_algorithms_ |= ALGORITHM_BROTLI; @@ -208,7 +218,11 @@ HostConfiguration::add_compression_algorithms(swoc::TextView line) } else if (token == "deflate") { compression_algorithms_ |= ALGORITHM_DEFLATE; } else { +#ifdef HAVE_ZSTD_H + error("Unknown compression type. Supported compression-algorithms ."); +#else error("Unknown compression type. Supported compression-algorithms ."); +#endif } } } @@ -261,7 +275,11 @@ static const std::unordered_map KeywordToStateMap {"range-request", ParserState::RangeRequest }, {"flush", ParserState::Flush }, {"allow", ParserState::Allow }, - {"minimum-content-length", ParserState::MinimumContentLength } + {"minimum-content-length", ParserState::MinimumContentLength }, + {"gzip-compression-level", ParserState::GzipCompressionLevel }, + {"brotli-compression-level", ParserState::BrotliCompressionLevel }, + {"brotli-lgwin", ParserState::BrotliLGWSize }, + {"zstd-compression-level", ParserState::ZstdCompressionLevel } }; void @@ -392,6 +410,50 @@ Configuration::Parse(const char *path) state = ParserState::Start; break; } + case ParserState::GzipCompressionLevel: { + swoc::TextView parsed; + intmax_t level = swoc::svtoi(token, &parsed); + if (parsed.size() == token.size() && level >= 1 && level <= 9) { + current_host_configuration->set_gzip_compression_level(level); + } else { + error("gzip-compression-level must be between 1 and 9, got %.*s", static_cast(token.size()), token.data()); + } + state = ParserState::Start; + break; + } + case ParserState::BrotliCompressionLevel: { + swoc::TextView parsed; + intmax_t level = swoc::svtoi(token, &parsed); + if (parsed.size() == token.size() && level >= 0 && level <= 11) { + current_host_configuration->set_brotli_compression_level(level); + } else { + error("brotli-compression-level must be between 0 and 11, got %.*s", static_cast(token.size()), token.data()); + } + state = ParserState::Start; + break; + } + case ParserState::BrotliLGWSize: { + swoc::TextView parsed; + intmax_t lgw = swoc::svtoi(token, &parsed); + if (parsed.size() == token.size() && lgw >= 10 && lgw <= 24) { + current_host_configuration->set_brotli_lgw_size(lgw); + } else { + error("brotli-lgwin must be between 10 and 24, got %.*s", static_cast(token.size()), token.data()); + } + state = ParserState::Start; + break; + } + case ParserState::ZstdCompressionLevel: { + swoc::TextView parsed; + intmax_t level = swoc::svtoi(token, &parsed); + if (parsed.size() == token.size() && level >= 1 && level <= 22) { + current_host_configuration->set_zstd_compression_level(level); + } else { + error("zstd-compression-level must be between 1 and 22, got %.*s", static_cast(token.size()), token.data()); + } + state = ParserState::Start; + break; + } } } } diff --git a/plugins/compress/configuration.h b/plugins/compress/configuration.h index 3af37b10c05..bc54aae63de 100644 --- a/plugins/compress/configuration.h +++ b/plugins/compress/configuration.h @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -39,7 +39,8 @@ enum CompressionAlgorithm { ALGORITHM_DEFAULT = 0, ALGORITHM_DEFLATE = 1, ALGORITHM_GZIP = 2, - ALGORITHM_BROTLI = 4 // For bit manipulations + ALGORITHM_BROTLI = 4, + ALGORITHM_ZSTD = 8 }; enum class RangeRequestCtrl : int { @@ -60,6 +61,10 @@ class HostConfiguration : private atscppapi::noncopyable flush_(false), compression_algorithms_(ALGORITHM_GZIP), minimum_content_length_(1024), + zlib_compression_level_(6), + brotli_compression_level_(6), + brotli_lgw_size_(16), + zstd_compression_level_(12), content_type_ignore_parameters_(false) { } @@ -142,6 +147,51 @@ class HostConfiguration : private atscppapi::noncopyable minimum_content_length_ = x; } + [[nodiscard]] unsigned int + zlib_compression_level() const + { + return zlib_compression_level_; + } + + void + set_gzip_compression_level(int level) + { + zlib_compression_level_ = level; + } + + [[nodiscard]] unsigned int + brotli_compression_level() const + { + return brotli_compression_level_; + } + void + set_brotli_compression_level(int level) + { + brotli_compression_level_ = level; + } + + [[nodiscard]] unsigned int + brotli_lgw_size() const + { + return brotli_lgw_size_; + } + void + set_brotli_lgw_size(unsigned int lgw) + { + brotli_lgw_size_ = lgw; + } + + [[nodiscard]] int + zstd_compression_level() const + { + return zstd_compression_level_; + } + void + set_zstd_compression_level(int level) + { + zstd_compression_level_ = level; + } + void update_defaults(); void add_allow(swoc::TextView allow); void add_compressible_content_type(swoc::TextView content_type); @@ -161,6 +211,10 @@ class HostConfiguration : private atscppapi::noncopyable bool flush_; int compression_algorithms_; unsigned int minimum_content_length_; + unsigned int zlib_compression_level_; + unsigned int brotli_compression_level_; + unsigned int brotli_lgw_size_; + int zstd_compression_level_; bool content_type_ignore_parameters_; RangeRequestCtrl range_request_ctl_ = RangeRequestCtrl::NO_COMPRESSION; diff --git a/plugins/compress/debug_macros.h b/plugins/compress/debug_macros.h index d5ce643a757..8ea10f2f117 100644 --- a/plugins/compress/debug_macros.h +++ b/plugins/compress/debug_macros.h @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License diff --git a/plugins/compress/gzip_compress.cc b/plugins/compress/gzip_compress.cc index 9de74ab0725..04111a47489 100644 --- a/plugins/compress/gzip_compress.cc +++ b/plugins/compress/gzip_compress.cc @@ -25,9 +25,13 @@ #include "debug_macros.h" #include +#include #include -const char *dictionary = nullptr; +namespace Compress +{ +extern const char *dictionary; +} namespace Gzip { @@ -69,8 +73,8 @@ data_alloc(Data *data) fatal("gzip-transform: ERROR: deflateInit (%d)!", err); } - if (dictionary) { - err = deflateSetDictionary(&data->zstrm, reinterpret_cast(dictionary), strlen(dictionary)); + if (Compress::dictionary) { + err = deflateSetDictionary(&data->zstrm, reinterpret_cast(Compress::dictionary), strlen(Compress::dictionary)); if (err != Z_OK) { fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err); } diff --git a/plugins/compress/misc.cc b/plugins/compress/misc.cc index 9a4b7f66852..762efade03f 100644 --- a/plugins/compress/misc.cc +++ b/plugins/compress/misc.cc @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License @@ -72,8 +72,9 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo bool deflate = false; bool gzip = false; bool br = false; + bool zstd = false; // remove the accept encoding field(s), - // while finding out if gzip or deflate is supported. + // while finding out if gzip, brotli, deflate, or zstandard are supported. while (field) { int val_len; const char *values_ = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, -1, &val_len); @@ -88,6 +89,8 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo br = true; } else if (strcasecmp("deflate", next) == 0) { deflate = true; + } else if (strcasecmp("zstd", next) == 0) { + zstd = true; } } } @@ -99,9 +102,13 @@ normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLo } // append a new accept-encoding field in the header - if (deflate || gzip || br) { + if (deflate || gzip || br || zstd) { TSMimeHdrFieldCreate(reqp, hdr_loc, &field); TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); + if (zstd) { + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "zstd", strlen("zstd")); + info("normalized accept encoding to zstd"); + } if (br) { TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "br", strlen("br")); info("normalized accept encoding to br"); diff --git a/plugins/compress/misc.h b/plugins/compress/misc.h index f5d37c1da4e..6c5f4513322 100644 --- a/plugins/compress/misc.h +++ b/plugins/compress/misc.h @@ -1,6 +1,6 @@ /** @file - Transforms content using gzip, deflate or brotli + Transforms content using gzip, deflate, brotli or zstd @section license License diff --git a/plugins/compress/sample.compress.config b/plugins/compress/sample.compress.config index b1431a97794..451f8958349 100644 --- a/plugins/compress/sample.compress.config +++ b/plugins/compress/sample.compress.config @@ -37,6 +37,22 @@ # minimum-content-length: minimum content length for compression to be enabled (in bytes) # - this setting only applies if the origin response has a Content-Length header # +# gzip-compression-level: compression level for gzip (1-9, default 6) +# - 1 is fastest compression (lowest compression ratio) +# - 9 is slowest compression (highest compression ratio) +# +# brotli-compression-level: compression level for brotli (0-11, default 6) +# - 0 is fastest compression (lowest compression ratio) +# - 11 is slowest compression (highest compression ratio) +# +# brotli-lgwin: window size for brotli compression (10-24, default 16) +# - larger values provide better compression but use more memory +# - 10: 1KB window, 16: 64KB window, 24: 16MB window +# +# zstd-compression-level: compression level for zstandard (1-22, default 12) +# - 1 is fastest compression (lowest compression ratio) +# - 22 is slowest compression (highest compression ratio) +# ###################################################################### #first, we configure the default/global plugin behaviour @@ -56,7 +72,13 @@ allow !*/bla* minimum-content-length 1024 #supported algorithms -supported-algorithms br,gzip +supported-algorithms br,gzip,zstd + +# Compression level settings (optional) +gzip-compression-level 6 +brotli-compression-level 6 +brotli-lgwin 16 +zstd-compression-level 12 #override the global configuration for a host. #www.foo.nl does NOT inherit anything @@ -68,6 +90,12 @@ compressible-content-type text/* compressible-status-code 200,206,409 minimum-content-length 1024 +# Custom compression settings for this host +gzip-compression-level 8 +brotli-compression-level 9 +brotli-lgwin 20 +zstd-compression-level 15 + allow /this/*.js allow !/notthis/*.js allow !/notthat* diff --git a/plugins/compress/zstd_compress.cc b/plugins/compress/zstd_compress.cc new file mode 100644 index 00000000000..45c6beb7924 --- /dev/null +++ b/plugins/compress/zstd_compress.cc @@ -0,0 +1,213 @@ +/** @file + + Zstd compression implementation + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file + distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under + the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may + obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS + IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. + */ + +#include "zstd_compress.h" + +#if HAVE_ZSTD_H + +#include "debug_macros.h" + +#include +#include + +namespace +{ +bool +compress_operation(Data *data, const char *upstream_buffer, int64_t upstream_length, ZSTD_EndDirective mode) +{ + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + if (upstream_length < 0) { + error("zstd-transform: negative upstream length (%" PRId64 ")", upstream_length); + return false; + } + + if (upstream_buffer == nullptr && upstream_length > 0) { + error("upstream_buffer is NULL with non-zero length"); + return false; + } + + ZSTD_inBuffer input = {upstream_buffer, static_cast(upstream_length), 0}; + + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + if (downstream_length <= 0) { + error("zstd-transform: downstream block has non-positive length (%" PRId64 ")", downstream_length); + return false; + } + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t result = ZSTD_compressStream2(data->zstrm_zstd.cctx, &output, &input, mode); + + if (ZSTD_isError(result)) { + error("Zstd compression failed (%d): %s", mode, ZSTD_getErrorName(result)); + return false; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (mode == ZSTD_e_continue) { + if (input.pos >= input.size) { + break; + } + if (output.pos == 0 && input.pos < input.size) { + error("zstd-transform: no progress made in compression"); + return false; + } + } else if (result == 0) { + break; + } + } + + return true; +} +} // namespace + +namespace Zstd +{ +void +data_alloc(Data *data) +{ + std::memset(&data->zstrm_zstd, 0, sizeof(data->zstrm_zstd)); + + data->zstrm_zstd.cctx = ZSTD_createCCtx(); + if (!data->zstrm_zstd.cctx) { + fatal("Zstd Compression Context Creation Failed"); + } +} + +void +data_destroy(Data *data) +{ + if (data->zstrm_zstd.cctx) { + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + data->zstrm_zstd.cctx = nullptr; + } +} + +bool +transform_init(Data *data) +{ + if (!data->zstrm_zstd.cctx) { + error("Failed to initialize Zstd compression context"); + return false; + } + + size_t result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_compressionLevel, data->hc->zstd_compression_level()); + if (ZSTD_isError(result)) { + error("Failed to set Zstd compression level: %s", ZSTD_getErrorName(result)); + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + data->zstrm_zstd.cctx = nullptr; + data->zstrm_zstd.total_in = 0; + data->zstrm_zstd.total_out = 0; + return false; + } + + result = ZSTD_CCtx_setParameter(data->zstrm_zstd.cctx, ZSTD_c_checksumFlag, 1); + if (ZSTD_isError(result)) { + error("Failed to enable Zstd checksum: %s", ZSTD_getErrorName(result)); + ZSTD_freeCCtx(data->zstrm_zstd.cctx); + data->zstrm_zstd.cctx = nullptr; + data->zstrm_zstd.total_in = 0; + data->zstrm_zstd.total_out = 0; + return false; + } + + debug("zstd compression context initialized with level %d", data->hc->zstd_compression_level()); + return true; +} + +void +transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) +{ + if (upstream_length < 0) { + error("Zstd compression received negative upstream length (%" PRId64 ")", upstream_length); + return; + } + + if (!compress_operation(data, upstream_buffer, upstream_length, ZSTD_e_continue)) { + error("Zstd compression (CONTINUE) failed"); + return; + } + + data->zstrm_zstd.total_in += upstream_length; + + if (!data->hc->flush()) { + return; + } + + if (!compress_operation(data, nullptr, 0, ZSTD_e_flush)) { + error("Zstd compression (FLUSH) failed"); + } +} + +void +transform_finish(Data *data) +{ + if (data->state != transform_state_output) { + return; + } + + TSIOBufferBlock downstream_blkp; + int64_t downstream_length; + + data->state = transform_state_finished; + + for (;;) { + downstream_blkp = TSIOBufferStart(data->downstream_buffer); + char *downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + + if (downstream_length <= 0) { + error("zstd-transform: downstream block has non-positive length (%" PRId64 ")", downstream_length); + break; + } + + ZSTD_outBuffer output = {downstream_buffer, static_cast(downstream_length), 0}; + + size_t remaining = ZSTD_endStream(data->zstrm_zstd.cctx, &output); + + if (ZSTD_isError(remaining)) { + error("zstd compression finish failed: %s", ZSTD_getErrorName(remaining)); + break; + } + + if (output.pos > 0) { + TSIOBufferProduce(data->downstream_buffer, output.pos); + data->downstream_length += output.pos; + data->zstrm_zstd.total_out += output.pos; + } + + if (remaining == 0) { + break; + } + } + + debug("zstd-transform: Finished zstd compression"); + log_compression_ratio(data->zstrm_zstd.total_in, data->downstream_length); +} +} // namespace Zstd + +#endif // HAVE_ZSTD_H diff --git a/plugins/compress/zstd_compress.h b/plugins/compress/zstd_compress.h new file mode 100644 index 00000000000..3386450a4d0 --- /dev/null +++ b/plugins/compress/zstd_compress.h @@ -0,0 +1,44 @@ +/** @file + + Zstd compression implementation + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file + distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under + the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may + obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS + IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. + */ + +#pragma once + +#include "tscore/ink_config.h" +#include "compress_common.h" + +#if HAVE_ZSTD_H + +namespace Zstd +{ +// Initialize Zstd compression context +void data_alloc(Data *data); + +// Destroy Zstd compression context +void data_destroy(Data *data); + +// Configure the context just before streaming starts. Returns true when ready. +bool transform_init(Data *data); + +// Compress one upstream chunk +void transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length); + +// Finish compression and flush remaining data +void transform_finish(Data *data); +} // namespace Zstd + +#endif // HAVE_ZSTD_H diff --git a/src/api/InkAPIInternal.cc b/src/api/InkAPIInternal.cc index 85f00287c52..8bb20a2b38a 100644 --- a/src/api/InkAPIInternal.cc +++ b/src/api/InkAPIInternal.cc @@ -232,6 +232,7 @@ const char *TS_HTTP_VALUE_COMPRESS; const char *TS_HTTP_VALUE_DEFLATE; const char *TS_HTTP_VALUE_GZIP; const char *TS_HTTP_VALUE_BROTLI; +const char *TS_HTTP_VALUE_ZSTD; const char *TS_HTTP_VALUE_IDENTITY; const char *TS_HTTP_VALUE_KEEP_ALIVE; const char *TS_HTTP_VALUE_MAX_AGE; @@ -256,6 +257,7 @@ int TS_HTTP_LEN_COMPRESS; int TS_HTTP_LEN_DEFLATE; int TS_HTTP_LEN_GZIP; int TS_HTTP_LEN_BROTLI; +int TS_HTTP_LEN_ZSTD; int TS_HTTP_LEN_IDENTITY; int TS_HTTP_LEN_KEEP_ALIVE; int TS_HTTP_LEN_MAX_AGE; @@ -748,6 +750,7 @@ api_init() TS_HTTP_VALUE_DEFLATE = HTTP_VALUE_DEFLATE.c_str(); TS_HTTP_VALUE_GZIP = HTTP_VALUE_GZIP.c_str(); TS_HTTP_VALUE_BROTLI = HTTP_VALUE_BROTLI.c_str(); + TS_HTTP_VALUE_ZSTD = HTTP_VALUE_ZSTD.c_str(); TS_HTTP_VALUE_IDENTITY = HTTP_VALUE_IDENTITY.c_str(); TS_HTTP_VALUE_KEEP_ALIVE = HTTP_VALUE_KEEP_ALIVE.c_str(); TS_HTTP_VALUE_MAX_AGE = HTTP_VALUE_MAX_AGE.c_str(); @@ -771,6 +774,7 @@ api_init() TS_HTTP_LEN_DEFLATE = static_cast(HTTP_VALUE_DEFLATE.length()); TS_HTTP_LEN_GZIP = static_cast(HTTP_VALUE_GZIP.length()); TS_HTTP_LEN_BROTLI = static_cast(HTTP_VALUE_BROTLI.length()); + TS_HTTP_LEN_ZSTD = static_cast(HTTP_VALUE_ZSTD.length()); TS_HTTP_LEN_IDENTITY = static_cast(HTTP_VALUE_IDENTITY.length()); TS_HTTP_LEN_KEEP_ALIVE = static_cast(HTTP_VALUE_KEEP_ALIVE.length()); TS_HTTP_LEN_MAX_AGE = static_cast(HTTP_VALUE_MAX_AGE.length()); diff --git a/src/iocore/cache/HttpTransactCache.cc b/src/iocore/cache/HttpTransactCache.cc index 021985fff47..7b3249af0f4 100644 --- a/src/iocore/cache/HttpTransactCache.cc +++ b/src/iocore/cache/HttpTransactCache.cc @@ -979,60 +979,38 @@ HttpTransactCache::calculate_quality_of_accept_encoding_match(MIMEField *accept_ if (!content_field) { if (!match_accept_content_encoding("identity", accept_field, &wildcard_present, &wildcard_q, &q)) { // CE was not returned, and AE does not have identity - if (match_content_encoding(accept_field, "gzip") and match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } goto encoding_wildcard; } - // use q from identity match - } else { - // "Accept-encoding must correctly handle multiple content encoding" - // The combined quality factor is the product of all quality factors. - // (Note that there may be other possible choice, eg, min(), - // but I think multiplication is the best.) - // For example, if "content-encoding: a, b", and quality factors - // of a and b (in accept-encoding header) are q_a and q_b, resp, - // then the combined quality factor is (q_a * q_b). - // If any one of the content-encoding is not matched, - // then the q value will not be changed. - float combined_q = 1.0; + // Handle multiple content encodings - use minimum quality + float min_q = 1.0; // Start with maximum quality + bool found_match = false; + for (c_value = c_values_list.head; c_value; c_value = c_value->next) { float this_q = -1.0; if (!match_accept_content_encoding(c_value->str, accept_field, &wildcard_present, &wildcard_q, &this_q)) { goto encoding_wildcard; } - combined_q *= this_q; + if (this_q >= 0.0) { + found_match = false; + if (this_q < min_q) { + min_q = this_q; + } + } + } + if (found_match) { + q = min_q; + } else { + q = -1.0; } - q = combined_q; } encoding_wildcard: - // match the wildcard now // if ((q == -1.0) && (wildcard_present == true)) { - q = wildcard_q; - } - ///////////////////////////////////////////////////////////////////////// - // there was an Accept-Encoding, but it didn't match anything, at // - // any quality level --- if this is an identity-coded document, that's // - // still okay, but otherwise, this is just not a match at all. // - ///////////////////////////////////////////////////////////////////////// - if ((q == -1.0) && is_identity_encoding) { - if (match_content_encoding(accept_field, "gzip")) { - if (match_content_encoding(cached_accept_field, "gzip")) { - return 1.0f; - } else { - // always try to fetch GZIP content if we have not tried sending AE before - return -1.0f; - } - } else if (cached_accept_field && !match_content_encoding(cached_accept_field, "gzip")) { - return 0.001f; - } else { - return -1.0f; - } + return wildcard_q; } - // q = (float)-1.0; - return (q); + + return q; } /** @@ -1244,7 +1222,11 @@ HttpTransactCache::CalcVariability(const HttpConfigAccessor *http_config_params, // Disable Vary mismatch checking for Accept-Encoding. This is only safe to // set if you are promising to fix any Accept-Encoding/Content-Encoding mismatches. - if (http_config_params->get_ignore_accept_encoding_mismatch() && + // Only suppress variability checks when the operator explicitly set + // proxy.config.http.cache.ignore_accept_encoding_mismatch to 1. The + // documented default value of 2 should continue to enforce Vary header + // semantics whenever the origin sends one. + if ((http_config_params->get_ignore_accept_encoding_mismatch() == 1) && !strcasecmp(const_cast(field->str), "Accept-Encoding")) { continue; } diff --git a/src/proxy/hdrs/HTTP.cc b/src/proxy/hdrs/HTTP.cc index a6c41c5f4a3..5f80ceab54f 100644 --- a/src/proxy/hdrs/HTTP.cc +++ b/src/proxy/hdrs/HTTP.cc @@ -78,6 +78,7 @@ c_str_view HTTP_VALUE_COMPRESS; c_str_view HTTP_VALUE_DEFLATE; c_str_view HTTP_VALUE_GZIP; c_str_view HTTP_VALUE_BROTLI; +c_str_view HTTP_VALUE_ZSTD; c_str_view HTTP_VALUE_IDENTITY; c_str_view HTTP_VALUE_KEEP_ALIVE; c_str_view HTTP_VALUE_MAX_AGE; @@ -183,6 +184,7 @@ http_init() HTTP_VALUE_DEFLATE = hdrtoken_string_to_wks_sv("deflate"); HTTP_VALUE_GZIP = hdrtoken_string_to_wks_sv("gzip"); HTTP_VALUE_BROTLI = hdrtoken_string_to_wks_sv("br"); + HTTP_VALUE_ZSTD = hdrtoken_string_to_wks_sv("zstd"); HTTP_VALUE_IDENTITY = hdrtoken_string_to_wks_sv("identity"); HTTP_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks_sv("keep-alive"); HTTP_VALUE_MAX_AGE = hdrtoken_string_to_wks_sv("max-age"); diff --git a/src/proxy/hdrs/HdrToken.cc b/src/proxy/hdrs/HdrToken.cc index c57089ced52..5749f8d1d5a 100644 --- a/src/proxy/hdrs/HdrToken.cc +++ b/src/proxy/hdrs/HdrToken.cc @@ -122,7 +122,10 @@ const char *const _hdrtoken_strs[] = { "Early-Data", // RFC-7932 - "br"}; + "br", + + // RFC-8878 + "zstd"}; HdrTokenTypeBinding _hdrtoken_strs_type_initializers[] = { {"file", HdrTokenType::SCHEME }, diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index fe7365a16b4..8eae5d78529 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -210,6 +210,7 @@ c_str_view MIME_VALUE_COMPRESS; c_str_view MIME_VALUE_DEFLATE; c_str_view MIME_VALUE_GZIP; c_str_view MIME_VALUE_BROTLI; +c_str_view MIME_VALUE_ZSTD; c_str_view MIME_VALUE_IDENTITY; c_str_view MIME_VALUE_KEEP_ALIVE; c_str_view MIME_VALUE_MAX_AGE; @@ -807,6 +808,7 @@ mime_init() MIME_VALUE_DEFLATE = hdrtoken_string_to_wks_sv("deflate"); MIME_VALUE_GZIP = hdrtoken_string_to_wks_sv("gzip"); MIME_VALUE_BROTLI = hdrtoken_string_to_wks_sv("br"); + MIME_VALUE_ZSTD = hdrtoken_string_to_wks_sv("zstd"); MIME_VALUE_IDENTITY = hdrtoken_string_to_wks_sv("identity"); MIME_VALUE_KEEP_ALIVE = hdrtoken_string_to_wks_sv("keep-alive"); MIME_VALUE_MAX_AGE = hdrtoken_string_to_wks_sv("max-age"); diff --git a/src/proxy/http/HttpTransactHeaders.cc b/src/proxy/http/HttpTransactHeaders.cc index 4a26790675e..a7ecee2eb32 100644 --- a/src/proxy/http/HttpTransactHeaders.cc +++ b/src/proxy/http/HttpTransactHeaders.cc @@ -1241,6 +1241,53 @@ HttpTransactHeaders::normalize_accept_encoding(const OverridableHttpConfigParams header->field_delete(ae_field); Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-br non-gzip Accept-Encoding"); } + } else if (normalize_ae == 4) { + // Force Accept-Encoding header to zstd or fallback to br/gzip or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } + } else if (normalize_ae == 5) { + // Force Accept-Encoding header to zstd,br,gzip combinations or individual algorithms or no header. + if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, br, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "zstd, br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "zstd, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) { + header->field_value_set(ae_field, "zstd"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br") && + HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "br, gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br, gzip"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "br")) { + header->field_value_set(ae_field, "br"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br"); + } else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) { + header->field_value_set(ae_field, "gzip"sv); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip"); + } else { + header->field_delete(ae_field); + Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding"); + } } else { static bool logged = false; diff --git a/src/proxy/http3/QPACK.cc b/src/proxy/http3/QPACK.cc index bcfa4c70b8d..dfdd2d278b3 100644 --- a/src/proxy/http3/QPACK.cc +++ b/src/proxy/http3/QPACK.cc @@ -69,7 +69,7 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {":status", "503" }, {"accept", "*/*" }, {"accept", "application/dns-message" }, - {"accept-encoding", "gzip, deflate, br" }, + {"accept-encoding", "gzip, deflate, br, zstd" }, {"accept-ranges", "bytes" }, {"access-control-allow-headers", "cache-control" }, {"access-control-allow-headers", "content-type" }, @@ -80,6 +80,7 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {"cache-control", "no-cache" }, {"cache-control", "no-store" }, {"cache-control", "public, max-age=31536000" }, + {"content-encoding", "zstd" }, {"content-encoding", "br" }, {"content-encoding", "gzip" }, {"content-type", "application/dns-message" }, diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index fbc2d3eb653..7890110a219 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -537,7 +537,7 @@ static constexpr RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.http.allow_multi_range", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL} , // This defaults to a special invalid value so the HTTP transaction handling code can tell that it was not explicitly set. - {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-3]", RECA_NULL} + {RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-5]", RECA_NULL} , // #################################################### diff --git a/src/traffic_layout/CMakeLists.txt b/src/traffic_layout/CMakeLists.txt index 56766955c62..b86d4f2573a 100644 --- a/src/traffic_layout/CMakeLists.txt +++ b/src/traffic_layout/CMakeLists.txt @@ -31,6 +31,10 @@ if(HAVE_BROTLI_ENCODE_H) target_link_libraries(traffic_layout PRIVATE brotli::brotlienc) endif() +if(HAVE_ZSTD_H) + target_link_libraries(traffic_layout PRIVATE zstd::zstd) +endif() + install(TARGETS traffic_layout) clang_tidy_check(traffic_layout) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 62732e09473..0a4f44491c0 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -49,6 +49,10 @@ #include #endif +#if HAVE_ZSTD_H +#include +#endif + // Produce output about compile time features, useful for checking how things were built static void print_feature(std::string_view name, int value, bool json, bool last = false) @@ -91,6 +95,11 @@ produce_features(bool json) #else print_feature("TS_HAS_BROTLI", 0, json); #endif +#ifdef HAVE_ZSTD_H + print_feature("TS_HAS_ZSTD", 1, json); +#else + print_feature("TS_HAS_ZSTD", 0, json); +#endif #ifdef F_GETPIPE_SZ print_feature("TS_HAS_PIPE_BUFFER_SIZE_CONFIG", 1, json); #else @@ -203,6 +212,11 @@ produce_versions(bool json) #else print_var("brotli", undef, json); #endif +#ifdef HAVE_ZSTD_H + print_var("zstd", LBW().print("{}", ZSTD_versionString()).view(), json); +#else + print_var("zstd", undef, json); +#endif // This should always be last print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); diff --git a/tests/gold_tests/headers/normalize_ae.gold b/tests/gold_tests/headers/normalize_ae.gold index 5e44f966741..f9b2c675ae9 100644 --- a/tests/gold_tests/headers/normalize_ae.gold +++ b/tests/gold_tests/headers/normalize_ae.gold @@ -11,6 +11,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -24,6 +44,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -37,6 +77,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -50,6 +110,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -63,6 +143,92 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- X-Au-Test: www.no-oride.com ACCEPT-ENCODING MISSING - @@ -76,6 +242,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-0.com ACCEPT-ENCODING MISSING - @@ -89,6 +275,26 @@ gzip, br - gzip;q=0.3, whatever;q=0.666, br;q=0.7 - +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +gzip, zstd, br +- +br, zstd +- +zstd;q=0.8, br;q=0.7, gzip;q=0.6 +- +deflate, zstd +- +identity, zstd, compress +- +br, compress +- X-Au-Test: www.ae-1.com ACCEPT-ENCODING MISSING - @@ -102,6 +308,26 @@ gzip - gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +ACCEPT-ENCODING MISSING +- +gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- X-Au-Test: www.ae-2.com ACCEPT-ENCODING MISSING - @@ -115,6 +341,26 @@ br - br - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br +- +br +- +br +- +br +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- X-Au-Test: www.ae-3.com ACCEPT-ENCODING MISSING - @@ -128,3 +374,89 @@ br, gzip - br, gzip - +ACCEPT-ENCODING MISSING +- +gzip +- +br +- +br, gzip +- +br, gzip +- +br +- +br, gzip +- +ACCEPT-ENCODING MISSING +- +ACCEPT-ENCODING MISSING +- +br +- +X-Au-Test: www.ae-4.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br +- +br +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +zstd +- +br +- +X-Au-Test: www.ae-5.com +ACCEPT-ENCODING MISSING +- +gzip +- +gzip +- +br +- +br, gzip +- +br, gzip +- +zstd +- +zstd, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd, br, gzip +- +zstd, br +- +zstd, br, gzip +- +zstd +- +zstd +- +br +- diff --git a/tests/gold_tests/headers/normalize_ae.test.py b/tests/gold_tests/headers/normalize_ae.test.py index 313023150d6..ca38755bfe9 100644 --- a/tests/gold_tests/headers/normalize_ae.test.py +++ b/tests/gold_tests/headers/normalize_ae.test.py @@ -40,6 +40,10 @@ server.addResponse("sessionlog.json", request_header, response_header) request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-2.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-4.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) +request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-5.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionlog.json", request_header, response_header) # Define first ATS. Disable the cache to make sure each request is sent to the # origin server. @@ -65,6 +69,12 @@ def baselineTsSetup(ts): ts.Disk.remap_config.AddLine( 'map http://www.ae-3.com http://127.0.0.1:{0}'.format(server.Variables.Port) + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-4.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') + ts.Disk.remap_config.AddLine( + 'map http://www.ae-5.com http://127.0.0.1:{0}'.format(server.Variables.Port) + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') baselineTsSetup(ts) @@ -124,6 +134,47 @@ def curlTail(hdrValue): tr.MakeCurlCommand(baseCurl + curlTail('gzip;q=0.3, whatever;q=0.666, br;q=0.7'), ts=ts) tr.Processes.Default.ReturnCode = 0 + # ZSTD-related tests for normalize_ae modes 4 and 5 + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, gzip')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, br')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd, br, gzip')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('gzip, zstd, br')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('br, zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('zstd;q=0.8, br;q=0.7, gzip;q=0.6')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('deflate, zstd')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('identity, zstd, compress')) + tr.Processes.Default.ReturnCode = 0 + + tr = test.AddTestRun() + tr.MakeCurlCommand(baseCurl + curlTail('br, compress')) + tr.Processes.Default.ReturnCode = 0 + def perTsTest(shouldWaitForUServer, ts): allAEHdrs(shouldWaitForUServer, True, ts, 'www.no-oride.com') @@ -131,6 +182,8 @@ def perTsTest(shouldWaitForUServer, ts): allAEHdrs(False, False, ts, 'www.ae-1.com') allAEHdrs(False, False, ts, 'www.ae-2.com') allAEHdrs(False, False, ts, 'www.ae-3.com') + allAEHdrs(False, False, ts, 'www.ae-4.com') + allAEHdrs(False, False, ts, 'www.ae-5.com') perTsTest(True, ts) diff --git a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py index 991d2fda32b..f83ae463fb9 100644 --- a/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py +++ b/tests/gold_tests/headers/normalized_ae_match_vary_cache.test.py @@ -45,6 +45,12 @@ ts.Disk.remap_config.AddLine( f"map http://www.ae-3.com http://127.0.0.1:{server.Variables.http_port}" + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-4.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4') +ts.Disk.remap_config.AddLine( + f"map http://www.ae-5.com http://127.0.0.1:{server.Variables.http_port}" + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5') ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache') ts.Disk.records_config.update( { @@ -53,7 +59,7 @@ 'proxy.config.http.response_via_str': 3, # the following variables could affect the results of alternate cache matching, # define them with their default values explicitly - 'proxy.config.cache.limits.http.max_alts': 5, + 'proxy.config.cache.limits.http.max_alts': 6, 'proxy.config.http.cache.ignore_accept_mismatch': 2, 'proxy.config.http.cache.ignore_accept_language_mismatch': 2, 'proxy.config.http.cache.ignore_accept_encoding_mismatch': 2, diff --git a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml index 57842341a59..4910af81027 100644 --- a/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml +++ b/tests/gold_tests/headers/replays/normalized_ae_varied_transactions.replay.yaml @@ -96,7 +96,7 @@ sessions: - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header deflate would match the alternate of empty Accept-Encoding header + # load an alternate of deflate Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -110,16 +110,25 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, deflate ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Deflate-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Deflate-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress would match the alternate of empty Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -133,14 +142,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Empty-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # load an alternate of gzip Accept-Encoding header - client-request: @@ -162,6 +180,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -176,7 +195,7 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # Accept-Encoding header br, compress, gzip would match the alternate of gzip Accept-Encoding header + # load an alternate of br Accept-Encoding header - client-request: method: "GET" version: "1.1" @@ -190,14 +209,23 @@ sessions: delay: 100ms server-response: - <<: *404_response + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] proxy-response: status: 200 headers: fields: - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Case 2 proxy.config.http.normalize_ae:1 @@ -322,6 +350,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -458,6 +487,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -492,6 +522,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] @@ -651,6 +682,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, br ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Accept-Encoding ] @@ -686,6 +718,7 @@ sessions: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] - [ Vary, Accept-Encoding ] + - [ Content-Encoding, gzip ] - [ Connection, close ] - [ X-Response-Identifier, Gzip-Accept-Encoding ] content: @@ -699,10 +732,6 @@ sessions: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: miss, as: equal } ] - # NOTICE: This case should load an alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -722,6 +751,7 @@ sessions: fields: - [ Transfer-Encoding, chunked ] - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, "br, gzip" ] - [ Vary, Accept-Encoding ] - [ Connection, close ] - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] @@ -733,10 +763,8 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] - # - [ X-Cache, { value: miss, as: equal } ] - - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] - - [ X-Cache, { value: hit-fresh, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header - client-request: @@ -784,11 +812,6 @@ sessions: - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] - # NOTICE: This case should make Accept-Encoding header br, gzip;q=0.8 match - # the alternate of br, gzip Accept-Encoding header. - # However, due to the implementation of calculate_quality_of_match(), - # ATS matches the alternate of gzip Accept-Encoding header. - # The result is DIFFERENT from the description of proxy.config.http.normalize_ae: 3 - client-request: method: "GET" version: "1.1" @@ -808,6 +831,826 @@ sessions: status: 200 headers: fields: - # - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 5 proxy.config.http.normalize_ae:4 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 41 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 42 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 43 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 44 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, zstd ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 45 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Content-Encoding, br ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 46 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, gzip ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # Accept-Encoding header zstd, br, compress, gzip would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 47 ] + - [ Accept-Encoding, "zstd, br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br, compress, gzip would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 48 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 49 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case5 + headers: + fields: + - [ Host, www.ae-4.com ] + - [ X-Debug, x-cache] + - [ uuid, 410 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Case 6 proxy.config.http.normalize_ae:5 + # load an alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 51 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, No-Accept-Encoding ] + content: + encoding: plain + data: "no Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # empty Accept-Encoding header would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, "" ] + - [ uuid, 52 ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header deflate would match the alternate of no Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 53 ] + - [ Accept-Encoding, deflate ] + delay: 100ms + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: No-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # load an alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 54 ] + - [ Accept-Encoding, "zstd, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Content-Encoding, zstd ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Accept-Encoding ] + content: + encoding: plain + data: "zstd Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 55 ] + - [ Accept-Encoding, "br, compress" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Content-Encoding, br ] + - [ Connection, close ] + - [ X-Response-Identifier, Br-Accept-Encoding ] + content: + encoding: plain + data: "br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # load an alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ Accept-Encoding, gzip;q=0.8 ] + - [ uuid, 56 ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ Content-Encoding, gzip ] + - [ X-Response-Identifier, Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 57 ] + - [ Accept-Encoding, "zstd, compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Content-Encoding, zstd ] + - [ Connection, close ] + - [ X-Response-Identifier, Zstd-Br-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Br Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 58 ] + - [ Accept-Encoding, "zstd, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ Content-Encoding, zstd ] + - [ X-Response-Identifier, Zstd-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Zstd, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 59 ] + - [ Accept-Encoding, "br, compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "br, gzip", as: equal }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ Cache-Control, max-age=300 ] + - [ Vary, Accept-Encoding ] + - [ Connection, close ] + - [ Content-Encoding, br ] + - [ X-Response-Identifier, Br-Gzip-Accept-Encoding ] + content: + encoding: plain + data: "Br, Gzip Accept-Encoding" + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: miss, as: equal } ] + + # Accept-Encoding header compress, zstd would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 510 ] + - [ Accept-Encoding, "compress, zstd" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, br would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 511 ] + - [ Accept-Encoding, "compress, br" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header compress, gzip would match the alternate of gzip Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 512 ] + - [ Accept-Encoding, "compress, gzip" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: gzip, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Gzip-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header zstd;q=1.1 would match the alternate of zstd Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 513 ] + - [ Accept-Encoding, "zstd;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: zstd, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + # Accept-Encoding header br;q=1.1 would match the alternate of br Accept-Encoding header + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 514 ] + - [ Accept-Encoding, "br;q=1.1" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: br, as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 515 ] + - [ Accept-Encoding, "zstd, br;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, br", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Br-Accept-Encoding, as: equal } ] + - [ X-Cache, { value: hit-fresh, as: equal } ] + + - client-request: + method: "GET" + version: "1.1" + url: /case6 + headers: + fields: + - [ Host, www.ae-5.com ] + - [ X-Debug, x-cache] + - [ uuid, 516 ] + - [ Accept-Encoding, "zstd, gzip;q=0.8" ] + delay: 100ms + + proxy-request: + headers: + fields: + - [Accept-Encoding, { value: "zstd, gzip", as: equal }] + + server-response: + <<: *404_response + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response-Identifier, { value: Zstd-Gzip-Accept-Encoding, as: equal } ] - [ X-Cache, { value: hit-fresh, as: equal } ] diff --git a/tests/gold_tests/pluginTest/compress/compress.gold b/tests/gold_tests/pluginTest/compress/compress.gold index 6745b65f0ae..76ea2da492e 100644 --- a/tests/gold_tests/pluginTest/compress/compress.gold +++ b/tests/gold_tests/pluginTest/compress/compress.gold @@ -1,13 +1,13 @@ -> GET ``/obj0 HTTP/1.1 -> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK @@ -16,7 +16,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/br > Accept-Encoding: br < HTTP/1.1 200 OK @@ -25,23 +25,30 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj1 HTTP/1.1 -> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-0/obj0 HTTP/1.1 +> X-Ats-Compress-Test: 0/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-1/obj1 HTTP/1.1 +> X-Ats-Compress-Test: 1/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: gzip < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK @@ -50,30 +57,37 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/br > Accept-Encoding: br < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj1 HTTP/1.1 +> GET http://ae-1/obj1 HTTP/1.1 > X-Ats-Compress-Test: 1/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj2 HTTP/1.1 -> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br -> Accept-Encoding: gzip, deflate, sdch, br +> GET http://ae-1/obj1 HTTP/1.1 +> X-Ats-Compress-Test: 1/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-2/obj2 HTTP/1.1 +> X-Ats-Compress-Test: 2/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Encoding: br < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK @@ -82,7 +96,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/br > Accept-Encoding: br < HTTP/1.1 200 OK @@ -91,14 +105,148 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj2 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 > X-Ats-Compress-Test: 2/deflate > Accept-Encoding: deflate < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-2/obj2 HTTP/1.1 +> X-Ats-Compress-Test: 2/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 7`` +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 46 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-3/obj3 HTTP/1.1 +> X-Ats-Compress-Test: 3/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 7`` +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 4`` +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-4/obj4 HTTP/1.1 +> X-Ats-Compress-Test: 4/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip, deflate, sdch, br, zstd +> Accept-Encoding: gzip, deflate, sdch, br, zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/gzip +> Accept-Encoding: gzip +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: gzip +< Vary: Accept-Encoding +< Content-Length: 7`` +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/br +> Accept-Encoding: br +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: br +< Vary: Accept-Encoding +< Content-Length: 4`` +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/deflate +> Accept-Encoding: deflate +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Length: 1049 +=== +> GET http://ae-5/obj5 HTTP/1.1 +> X-Ats-Compress-Test: 5/zstd +> Accept-Encoding: zstd +< HTTP/1.1 200 OK +< Content-Type: text/javascript +< Content-Encoding: zstd +< Vary: Accept-Encoding +< Content-Length: 64 +=== +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666 > Accept-Encoding: gzip;q=0.666 < HTTP/1.1 200 OK @@ -107,7 +255,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.666x > Accept-Encoding: gzip;q=0.666x < HTTP/1.1 200 OK @@ -116,7 +264,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=#0.666 > Accept-Encoding: gzip;q=#0.666 < HTTP/1.1 200 OK @@ -125,7 +273,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip; Q = 0.666 > Accept-Encoding: gzip; Q = 0.666 < HTTP/1.1 200 OK @@ -134,14 +282,14 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=0.0 > Accept-Encoding: gzip;q=0.0 < HTTP/1.1 200 OK < Content-Type: text/javascript < Content-Length: 1049 === -> GET ``obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/gzip;q=-0.1 > Accept-Encoding: gzip;q=-0.1 < HTTP/1.1 200 OK @@ -150,7 +298,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666, bbb > Accept-Encoding: aaa, gzip;q=0.666, bbb < HTTP/1.1 200 OK @@ -159,7 +307,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/ br ; q=0.666, bbb > Accept-Encoding: br ; q=0.666, bbb < HTTP/1.1 200 OK @@ -168,7 +316,7 @@ < Vary: Accept-Encoding < Content-Length: 46 === -> GET ``/obj0 HTTP/1.1 +> GET http://ae-0/obj0 HTTP/1.1 > X-Ats-Compress-Test: 0/aaa, gzip;q=0.666 , > Accept-Encoding: aaa, gzip;q=0.666 , < HTTP/1.1 200 OK @@ -177,7 +325,7 @@ < Vary: Accept-Encoding < Content-Length: 7`` === -> POST ``/obj3 HTTP/1.1 +> POST http://ae-3/obj3 HTTP/1.1 > X-Ats-Compress-Test: 3/gzip > Accept-Encoding: gzip < HTTP/1.1 200 OK diff --git a/tests/gold_tests/pluginTest/compress/compress.test.py b/tests/gold_tests/pluginTest/compress/compress.test.py index fd34c6f7524..4359b64a5cd 100644 --- a/tests/gold_tests/pluginTest/compress/compress.test.py +++ b/tests/gold_tests/pluginTest/compress/compress.test.py @@ -25,7 +25,8 @@ # Skip if plugins not present. # Test.SkipUnless( - Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI')) + Condition.PluginExists('compress.so'), Condition.PluginExists('conf_remap.so'), Condition.HasATSFeature('TS_HAS_BROTLI'), + Condition.HasATSFeature('TS_HAS_ZSTD')) server = Test.MakeOriginServer("server", options={'--load': f'{Test.TestDirectory}/compress_observer.py'}) @@ -43,7 +44,7 @@ "timestamp": "1469733493.993", "body": body } -for i in range(3): +for i in range(6): # add request/response to the server dictionary request_header = {"headers": f"GET /obj{i} HTTP/1.1\r\nHost: just.any.thing\r\n\r\n", "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionfile.log", request_header, response_header) @@ -89,6 +90,7 @@ def curl_post(ts, idx, encodingList, out_path): ts.Setup.Copy("compress.config") ts.Setup.Copy("compress2.config") +ts.Setup.Copy("compress3.config") ts.Disk.remap_config.AddLine( f'map http://ae-0/ http://127.0.0.1:{server.Variables.Port}/' + @@ -103,7 +105,16 @@ def curl_post(ts, idx, encodingList, out_path): f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') ts.Disk.remap_config.AddLine( f'map http://ae-3/ http://127.0.0.1:{server.Variables.Port}/' + - f' @plugin=compress.so @pparam={Test.RunDirectory}/compress.config') + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress2.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-4/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') +ts.Disk.remap_config.AddLine( + f'map http://ae-5/ http://127.0.0.1:{server.Variables.Port}/' + + ' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5' + + f' @plugin=compress.so @pparam={Test.RunDirectory}/compress3.config') out_path_counter = 0 @@ -119,12 +130,12 @@ def get_out_path(): def get_verify_command(out_path, decrompressor): - return f"{decrompressor} -c {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" + return f"{decrompressor} {out_path} > {deflate_path} && diff {deflate_path} {orig_path}" -for i in range(3): +for i in range(6): - tr = Test.AddTestRun(f'gzip, deflate, sdch, br: {i}') + tr = Test.AddTestRun(f'gzip, deflate, sdch, br, zstd: {i}') if (waitForTs): tr.Processes.Default.StartBefore(ts) waitForTs = False @@ -133,15 +144,21 @@ def get_verify_command(out_path, decrompressor): waitForServer = False tr.Processes.Default.ReturnCode = 0 out_path = get_out_path() - tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br', out_path), ts=ts) - tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br: {i}') + tr.MakeCurlCommand(curl(ts, i, 'gzip, deflate, sdch, br, zstd', out_path), ts=ts) + tr = Test.AddTestRun(f'verify gzip, deflate, sdch, br, zstd: {i}') tr.ReturnCode = 0 if i == 0: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") elif i == 1: - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") elif i == 2: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 3: + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") + elif i == 4: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + elif i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") tr = Test.AddTestRun(f'gzip: {i}') tr.Processes.Default.ReturnCode = 0 @@ -149,7 +166,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, i, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip: {i}') tr.ReturnCode = 0 - tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") + tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun(f'br: {i}') tr.Processes.Default.ReturnCode = 0 @@ -160,7 +177,7 @@ def get_verify_command(out_path, decrompressor): if i == 1: tr.Processes.Default.Command = f"diff {out_path} {orig_path}" else: - tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") + tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun(f'deflate: {i}') tr.Processes.Default.ReturnCode = 0 @@ -170,6 +187,17 @@ def get_verify_command(out_path, decrompressor): tr.ReturnCode = 0 tr.Processes.Default.Command = f"diff {out_path} {orig_path}" + tr = Test.AddTestRun(f'zstd: {i}') + tr.Processes.Default.ReturnCode = 0 + out_path = get_out_path() + tr.MakeCurlCommand(curl(ts, i, "zstd", out_path)) + tr = Test.AddTestRun(f'verify zstd: {i}') + tr.ReturnCode = 0 + if i == 4 or i == 5: + tr.Processes.Default.Command = get_verify_command(out_path, "zstd -d -c") + else: + tr.Processes.Default.Command = get_verify_command(out_path, "cat") + # Test Accept-Encoding normalization. tr = Test.AddTestRun() @@ -178,7 +206,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -186,7 +214,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=0.666x", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=0.666x') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -194,7 +222,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=#0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=#0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -202,7 +230,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip; Q = 0.666", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip; Q = 0.666') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -218,7 +246,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "gzip;q=-0.1", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip;q=-0.1') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -226,7 +254,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -234,7 +262,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, " br ; q=0.666, bbb", out_path), ts=ts) tr = Test.AddTestRun(f'verify br ; q=0.666, bbb') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d") +tr.Processes.Default.Command = get_verify_command(out_path, "brotli -d -c") tr = Test.AddTestRun() tr.Processes.Default.ReturnCode = 0 @@ -242,7 +270,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl(ts, 0, "aaa, gzip;q=0.666 , ", out_path), ts=ts) tr = Test.AddTestRun(f'verify aaa, gzip;q=0.666 , ') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # post tr = Test.AddTestRun() @@ -251,7 +279,7 @@ def get_verify_command(out_path, decrompressor): tr.MakeCurlCommand(curl_post(ts, 3, "gzip", out_path), ts=ts) tr = Test.AddTestRun(f'verify gzip post') tr.ReturnCode = 0 -tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k") +tr.Processes.Default.Command = get_verify_command(out_path, "gunzip -k -c") # compress_long.log contains all the output from the curl commands. The tr removes the carriage returns for easier # readability. Curl seems to have a bug, where it will neglect to output an end of line before outputting an HTTP diff --git a/tests/gold_tests/pluginTest/compress/compress2.config b/tests/gold_tests/pluginTest/compress/compress2.config index c808d87fcfd..8fa674c687b 100644 --- a/tests/gold_tests/pluginTest/compress/compress2.config +++ b/tests/gold_tests/pluginTest/compress/compress2.config @@ -5,3 +5,6 @@ compressible-content-type application/x-javascript* compressible-content-type application/javascript* compressible-content-type application/json* supported-algorithms gzip, br +gzip-compression-level 7 +brotli-compression-level 8 +brotli-lgwin 18 diff --git a/tests/gold_tests/pluginTest/compress/compress3.config b/tests/gold_tests/pluginTest/compress/compress3.config new file mode 100644 index 00000000000..54ff0eed2d8 --- /dev/null +++ b/tests/gold_tests/pluginTest/compress/compress3.config @@ -0,0 +1,11 @@ +cache true +remove-accept-encoding true +compressible-content-type text/* +compressible-content-type application/x-javascript* +compressible-content-type application/javascript* +compressible-content-type application/json* +supported-algorithms zstd, br, gzip +gzip-compression-level 5 +brotli-compression-level 7 +brotli-lgwin 17 +zstd-compression-level 10 diff --git a/tests/gold_tests/pluginTest/compress/compress_userver.gold b/tests/gold_tests/pluginTest/compress/compress_userver.gold index 1ddc6a4f3b5..693bd03905e 100644 --- a/tests/gold_tests/pluginTest/compress/compress_userver.gold +++ b/tests/gold_tests/pluginTest/compress/compress_userver.gold @@ -1,15 +1,33 @@ -0/gzip, deflate, sdch, br +0/gzip, deflate, sdch, br, zstd 0/gzip 0/br 0/deflate -1/gzip, deflate, sdch, br +0/zstd +1/gzip, deflate, sdch, br, zstd 1/gzip 1/br 1/deflate -2/gzip, deflate, sdch, br +1/zstd +2/gzip, deflate, sdch, br, zstd 2/gzip 2/br 2/deflate +2/zstd +3/gzip, deflate, sdch, br, zstd +3/gzip +3/br +3/deflate +3/zstd +4/gzip, deflate, sdch, br, zstd +4/gzip +4/br +4/deflate +4/zstd +5/gzip, deflate, sdch, br, zstd +5/gzip +5/br +5/deflate +5/zstd 0/gzip;q=0.666 0/gzip;q=0.666x 0/gzip;q=#0.666 diff --git a/tests/gold_tests/traffic_ctl/gold/test_2.gold b/tests/gold_tests/traffic_ctl/gold/test_2.gold new file mode 100644 index 00000000000..3c75916cf79 --- /dev/null +++ b/tests/gold_tests/traffic_ctl/gold/test_2.gold @@ -0,0 +1 @@ +proxy.config.diags.debug.enabled: 1 diff --git a/tests/gold_tests/traffic_ctl/gold/test_3.gold b/tests/gold_tests/traffic_ctl/gold/test_3.gold new file mode 100644 index 00000000000..e12f994befe --- /dev/null +++ b/tests/gold_tests/traffic_ctl/gold/test_3.gold @@ -0,0 +1 @@ +proxy.config.diags.debug.tags: rpc # default http|dns