Skip to content

Commit 4b8603e

Browse files
JakeChampionJakeChampion
committed
Add support for zstd in Accept-Encoding normalization
Extend the normalize_ae configuration to support Zstandard compression with two new modes: - Mode 4: Prioritize zstd, fallback to br/gzip - Mode 5: Support all combinations of zstd, br, and gzip This allows better cache efficiency when using modern compression algorithms while maintaining backward compatibility with existing deployments.
1 parent bbf1622 commit 4b8603e

File tree

10 files changed

+753
-3
lines changed

10 files changed

+753
-3
lines changed

‎doc/admin-guide/files/records.yaml.en.rst‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2070,10 +2070,14 @@ Proxy User Variables
20702070
normalize as for value ``1``
20712071
``3`` ``Accept-Encoding: br, gzip`` (if the header has ``br`` and ``gzip`` (with any ``q`` for either) then ``br, gzip``) **ELSE**
20722072
normalize as for value ``2``
2073+
``4`` ``Accept-Encoding: zstd`` if the header has ``zstd`` (with any ``q``) **ELSE**
2074+
normalize as for value ``2``
2075+
``5`` ``Accept-Encoding: zstd, br, gzip`` (supports all combinations of ``zstd``, ``br``, and ``gzip``) **ELSE**
2076+
normalize as for value ``4``
20732077
===== ======================================================================
20742078

20752079
This is useful for minimizing cached alternates of documents (e.g. ``gzip, deflate`` vs. ``deflate, gzip``).
2076-
Enabling this option is recommended if your origin servers use no encodings other than ``gzip`` or ``br`` (Brotli).
2080+
Enabling this option is recommended if your origin servers use no encodings other than ``gzip``, ``br`` (Brotli), or ``zstd`` (Zstandard).
20772081

20782082
Security
20792083
========

‎doc/admin-guide/plugins/compress.en.rst‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,9 @@ zstd Zstandard compression (fast, high compression ratio)
201201
============== =================================================================
202202

203203
Note that if :ts:cv:`proxy.config.http.normalize_ae` is ``1``, only gzip will
204-
be considered, and if it is ``2``, only br or gzip will be considered.
204+
be considered, if it is ``2``, only br or gzip will be considered, if it is ``4``,
205+
only zstd, br, or gzip will be considered, and if it is ``5``, all combinations
206+
of zstd, br, and gzip will be considered.
205207

206208
Examples
207209
========

‎src/proxy/http/HttpTransactHeaders.cc‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1233,6 +1233,53 @@ HttpTransactHeaders::normalize_accept_encoding(const OverridableHttpConfigParams
12331233
header->field_delete(ae_field);
12341234
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-br non-gzip Accept-Encoding");
12351235
}
1236+
} else if (normalize_ae == 4) {
1237+
// Force Accept-Encoding header to zstd or fallback to br/gzip or no header.
1238+
if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) {
1239+
header->field_value_set(ae_field, "zstd", 4);
1240+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd");
1241+
} else if (HttpTransactCache::match_content_encoding(ae_field, "br")) {
1242+
header->field_value_set(ae_field, "br", 2);
1243+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br");
1244+
} else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) {
1245+
header->field_value_set(ae_field, "gzip", 4);
1246+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip");
1247+
} else {
1248+
header->field_delete(ae_field);
1249+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding");
1250+
}
1251+
} else if (normalize_ae == 5) {
1252+
// Force Accept-Encoding header to zstd,br,gzip combinations or individual algorithms or no header.
1253+
if (HttpTransactCache::match_content_encoding(ae_field, "zstd") &&
1254+
HttpTransactCache::match_content_encoding(ae_field, "br") &&
1255+
HttpTransactCache::match_content_encoding(ae_field, "gzip")) {
1256+
header->field_value_set(ae_field, "zstd, br, gzip", 14);
1257+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br, gzip");
1258+
} else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") &&
1259+
HttpTransactCache::match_content_encoding(ae_field, "br")) {
1260+
header->field_value_set(ae_field, "zstd, br", 8);
1261+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, br");
1262+
} else if (HttpTransactCache::match_content_encoding(ae_field, "zstd") &&
1263+
HttpTransactCache::match_content_encoding(ae_field, "gzip")) {
1264+
header->field_value_set(ae_field, "zstd, gzip", 10);
1265+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd, gzip");
1266+
} else if (HttpTransactCache::match_content_encoding(ae_field, "br") &&
1267+
HttpTransactCache::match_content_encoding(ae_field, "gzip")) {
1268+
header->field_value_set(ae_field, "br, gzip", 8);
1269+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br, gzip");
1270+
} else if (HttpTransactCache::match_content_encoding(ae_field, "zstd")) {
1271+
header->field_value_set(ae_field, "zstd", 4);
1272+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to zstd");
1273+
} else if (HttpTransactCache::match_content_encoding(ae_field, "br")) {
1274+
header->field_value_set(ae_field, "br", 2);
1275+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to br");
1276+
} else if (HttpTransactCache::match_content_encoding(ae_field, "gzip")) {
1277+
header->field_value_set(ae_field, "gzip", 4);
1278+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] normalized Accept-Encoding to gzip");
1279+
} else {
1280+
header->field_delete(ae_field);
1281+
Dbg(dbg_ctl_http_trans, "[Headers::normalize_accept_encoding] removed non-zstd non-br non-gzip Accept-Encoding");
1282+
}
12361283
} else {
12371284
static bool logged = false;
12381285

‎src/records/RecordsConfig.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ static const RecordElement RecordsConfig[] =
521521
{RECT_CONFIG, "proxy.config.http.allow_multi_range", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-2]", RECA_NULL}
522522
,
523523
// This defaults to a special invalid value so the HTTP transaction handling code can tell that it was not explicitly set.
524-
{RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-3]", RECA_NULL}
524+
{RECT_CONFIG, "proxy.config.http.normalize_ae", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-5]", RECA_NULL}
525525
,
526526

527527
// ####################################################

‎tests/gold_tests/headers/normalize_ae.gold‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,166 @@ br, gzip
6363
-
6464
br, gzip
6565
-
66+
zstd
67+
-
68+
zstd, gzip
69+
-
70+
zstd, br
71+
-
72+
zstd, br, gzip
73+
-
74+
zstd, br, gzip
75+
-
76+
zstd, br
77+
-
78+
zstd, br, gzip
79+
-
80+
zstd
81+
-
82+
zstd
83+
-
84+
X-Au-Test: www.ae-4.com
85+
ACCEPT-ENCODING MISSING
86+
-
87+
gzip
88+
-
89+
gzip
90+
-
91+
br
92+
-
93+
br
94+
-
95+
br
96+
-
97+
zstd
98+
-
99+
zstd
100+
-
101+
zstd
102+
-
103+
zstd
104+
-
105+
zstd
106+
-
107+
zstd
108+
-
109+
zstd
110+
-
111+
zstd
112+
-
113+
zstd
114+
-
115+
X-Au-Test: www.ae-5.com
116+
ACCEPT-ENCODING MISSING
117+
-
118+
gzip
119+
-
120+
gzip
121+
-
122+
br
123+
-
124+
br, gzip
125+
-
126+
br, gzip
127+
-
128+
zstd
129+
-
130+
zstd, gzip
131+
-
132+
zstd, br
133+
-
134+
zstd, br, gzip
135+
-
136+
zstd, br, gzip
137+
-
138+
zstd, br
139+
-
140+
zstd, br, gzip
141+
-
142+
zstd
143+
-
144+
zstd
145+
-
146+
zstd
147+
-
148+
zstd, gzip
149+
-
150+
zstd, br
151+
-
152+
zstd, br, gzip
153+
-
154+
zstd, br, gzip
155+
-
156+
zstd, br
157+
-
158+
zstd, br, gzip
159+
-
160+
zstd
161+
-
162+
zstd
163+
-
164+
X-Au-Test: www.ae-4.com
165+
ACCEPT-ENCODING MISSING
166+
-
167+
gzip
168+
-
169+
gzip
170+
-
171+
br
172+
-
173+
br
174+
-
175+
br
176+
-
177+
zstd
178+
-
179+
zstd
180+
-
181+
zstd
182+
-
183+
zstd
184+
-
185+
zstd
186+
-
187+
zstd
188+
-
189+
zstd
190+
-
191+
zstd
192+
-
193+
zstd
194+
-
195+
X-Au-Test: www.ae-5.com
196+
ACCEPT-ENCODING MISSING
197+
-
198+
gzip
199+
-
200+
gzip
201+
-
202+
br
203+
-
204+
br, gzip
205+
-
206+
br, gzip
207+
-
208+
zstd
209+
-
210+
zstd, gzip
211+
-
212+
zstd, br
213+
-
214+
zstd, br, gzip
215+
-
216+
zstd, br, gzip
217+
-
218+
zstd, br
219+
-
220+
zstd, br, gzip
221+
-
222+
zstd
223+
-
224+
zstd
225+
-
66226
X-Au-Test: www.no-oride.com
67227
ACCEPT-ENCODING MISSING
68228
-

‎tests/gold_tests/headers/normalize_ae.test.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@
4040
server.addResponse("sessionlog.json", request_header, response_header)
4141
request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-2.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
4242
server.addResponse("sessionlog.json", request_header, response_header)
43+
request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-4.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
44+
server.addResponse("sessionlog.json", request_header, response_header)
45+
request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.ae-5.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
46+
server.addResponse("sessionlog.json", request_header, response_header)
4347

4448
# Define first ATS. Disable the cache to make sure each request is sent to the
4549
# origin server.
@@ -65,6 +69,12 @@ def baselineTsSetup(ts):
6569
ts.Disk.remap_config.AddLine(
6670
'map http://www.ae-3.com http://127.0.0.1:{0}'.format(server.Variables.Port) +
6771
' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=3')
72+
ts.Disk.remap_config.AddLine(
73+
'map http://www.ae-4.com http://127.0.0.1:{0}'.format(server.Variables.Port) +
74+
' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=4')
75+
ts.Disk.remap_config.AddLine(
76+
'map http://www.ae-5.com http://127.0.0.1:{0}'.format(server.Variables.Port) +
77+
' @plugin=conf_remap.so @pparam=proxy.config.http.normalize_ae=5')
6878

6979

7080
baselineTsSetup(ts)
@@ -120,13 +130,52 @@ def curlTail(hdrValue):
120130
tr.MakeCurlCommand(baseCurl + curlTail('gzip;q=0.3, whatever;q=0.666, br;q=0.7'))
121131
tr.Processes.Default.ReturnCode = 0
122132

133+
# ZSTD-related tests for normalize_ae modes 4 and 5
134+
tr = test.AddTestRun()
135+
tr.MakeCurlCommand(baseCurl + curlTail('zstd'))
136+
tr.Processes.Default.ReturnCode = 0
137+
138+
tr = test.AddTestRun()
139+
tr.MakeCurlCommand(baseCurl + curlTail('zstd, gzip'))
140+
tr.Processes.Default.ReturnCode = 0
141+
142+
tr = test.AddTestRun()
143+
tr.MakeCurlCommand(baseCurl + curlTail('zstd, br'))
144+
tr.Processes.Default.ReturnCode = 0
145+
146+
tr = test.AddTestRun()
147+
tr.MakeCurlCommand(baseCurl + curlTail('zstd, br, gzip'))
148+
tr.Processes.Default.ReturnCode = 0
149+
150+
tr = test.AddTestRun()
151+
tr.MakeCurlCommand(baseCurl + curlTail('gzip, zstd, br'))
152+
tr.Processes.Default.ReturnCode = 0
153+
154+
tr = test.AddTestRun()
155+
tr.MakeCurlCommand(baseCurl + curlTail('br, zstd'))
156+
tr.Processes.Default.ReturnCode = 0
157+
158+
tr = test.AddTestRun()
159+
tr.MakeCurlCommand(baseCurl + curlTail('zstd;q=0.8, br;q=0.7, gzip;q=0.6'))
160+
tr.Processes.Default.ReturnCode = 0
161+
162+
tr = test.AddTestRun()
163+
tr.MakeCurlCommand(baseCurl + curlTail('deflate, zstd'))
164+
tr.Processes.Default.ReturnCode = 0
165+
166+
tr = test.AddTestRun()
167+
tr.MakeCurlCommand(baseCurl + curlTail('identity, zstd, compress'))
168+
tr.Processes.Default.ReturnCode = 0
169+
123170

124171
def perTsTest(shouldWaitForUServer, ts):
125172
allAEHdrs(shouldWaitForUServer, True, ts, 'www.no-oride.com')
126173
allAEHdrs(False, False, ts, 'www.ae-0.com')
127174
allAEHdrs(False, False, ts, 'www.ae-1.com')
128175
allAEHdrs(False, False, ts, 'www.ae-2.com')
129176
allAEHdrs(False, False, ts, 'www.ae-3.com')
177+
allAEHdrs(False, False, ts, 'www.ae-4.com')
178+
allAEHdrs(False, False, ts, 'www.ae-5.com')
130179

131180

132181
perTsTest(True, ts)

‎tests/gold_tests/headers/normalize_ae_zstd.gold‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
X-Au-Test: www.ae-4.com
2+
ACCEPT-ENCODING MISSING
3+
-
4+
zstd
5+
-
6+
zstd
7+
-
8+
zstd
9+
-
10+
zstd
11+
-
12+
zstd
13+
-
14+
zstd
15+
-
16+
zstd
17+
-
18+
ACCEPT-ENCODING MISSING
19+
-
20+
zstd
21+
-
22+
br
23+
-
24+
gzip
25+
-
26+
X-Au-Test: www.ae-5.com
27+
ACCEPT-ENCODING MISSING
28+
-
29+
zstd
30+
-
31+
zstd, gzip
32+
-
33+
zstd, br
34+
-
35+
zstd, br, gzip
36+
-
37+
zstd, br, gzip
38+
-
39+
zstd, br, gzip
40+
-
41+
zstd
42+
-
43+
ACCEPT-ENCODING MISSING
44+
-
45+
zstd, br, gzip
46+
-
47+
br, gzip
48+
-
49+
gzip
50+
-

0 commit comments

Comments
 (0)