From 82dee8af418a280b5898d15a272d2f976b1fa687 Mon Sep 17 00:00:00 2001 From: Zengyi Wang Date: Thu, 18 Jul 2024 14:58:22 +0800 Subject: [PATCH 1/2] update HTTP caching --- .../io/vertx/httpproxy/impl/CacheControl.java | 108 +++++++- .../vertx/httpproxy/impl/CachingFilter.java | 257 ++++++++++++------ .../vertx/httpproxy/impl/ProxiedResponse.java | 28 +- .../vertx/httpproxy/impl/ProxyTransform.java | 31 +++ .../io/vertx/httpproxy/impl/ReverseProxy.java | 1 + .../vertx/httpproxy/impl/SafeMathUtils.java | 45 +++ .../vertx/httpproxy/spi/cache/Resource.java | 24 +- .../tests/cache/spi/CacheSpiTestBase.java | 1 + .../tests/parsing/ResourceParseTest.java | 2 + 9 files changed, 411 insertions(+), 86 deletions(-) create mode 100644 src/main/java/io/vertx/httpproxy/impl/ProxyTransform.java create mode 100644 src/main/java/io/vertx/httpproxy/impl/SafeMathUtils.java diff --git a/src/main/java/io/vertx/httpproxy/impl/CacheControl.java b/src/main/java/io/vertx/httpproxy/impl/CacheControl.java index a89b94c..1a2bbb6 100644 --- a/src/main/java/io/vertx/httpproxy/impl/CacheControl.java +++ b/src/main/java/io/vertx/httpproxy/impl/CacheControl.java @@ -10,17 +10,38 @@ */ package io.vertx.httpproxy.impl; +import java.math.BigInteger; + /** * @author Julien Viet */ public class CacheControl { private int maxAge; + private int maxStale; + private int minFresh; + private boolean noCache; + private boolean noStore; + private boolean noTransform; + private boolean onlyIfCached; + private boolean mustRevalidate; + private boolean mustUnderstand; + private boolean _private; + private boolean proxyRevalidate; private boolean _public; + private int sMaxage; public CacheControl parse(String header) { - maxAge = -1; + noCache = false; + noStore = false; + noTransform = false; + onlyIfCached = false; + mustRevalidate = false; + mustUnderstand = false; + _private = false; + proxyRevalidate = false; _public = false; + String[] parts = header.split(","); // No regex for (String part : parts) { part = part.trim().toLowerCase(); @@ -28,17 +49,51 @@ public CacheControl parse(String header) { case "public": _public = true; break; + case "no-cache": + noCache = true; + break; + case "no-store": + noStore = true; + break; + case "no-transform": + noTransform = true; + break; + case "only-if-cached": + onlyIfCached = true; + break; + case "must-revalidate": + mustRevalidate = true; + break; + case "must-understand": + mustUnderstand = true; + break; + case "private": + _private = true; + break; + case "proxy-revalidate": + proxyRevalidate = true; + break; default: - if (part.startsWith("max-age=")) { - maxAge = Integer.parseInt(part.substring(8)); - - } + maxAge = loadInt(part, "max-age="); + maxStale = loadInt(part, "max-stale="); + minFresh = loadInt(part, "min-fresh="); + sMaxage = loadInt(part, "s-maxage="); break; } } return this; } + private static int loadInt(String part, String prefix) { + if (part.startsWith(prefix)) { + BigInteger valueRaw = new BigInteger(part.substring(prefix.length())); + return valueRaw + .min(BigInteger.valueOf(Integer.MAX_VALUE)) + .max(BigInteger.ZERO).intValueExact(); + } + return -1; + } + public int maxAge() { return maxAge; } @@ -47,4 +102,47 @@ public boolean isPublic() { return _public; } + public int maxStale() { + return maxStale; + } + + public int minFresh() { + return minFresh; + } + + public boolean isNoCache() { + return noCache; + } + + public boolean isNoStore() { + return noStore; + } + + public boolean isNoTransform() { + return noTransform; + } + + public boolean isOnlyIfCached() { + return onlyIfCached; + } + + public boolean isMustRevalidate() { + return mustRevalidate; + } + + public boolean isMustUnderstand() { + return mustUnderstand; + } + + public boolean isPrivate() { + return _private; + } + + public boolean isProxyRevalidate() { + return proxyRevalidate; + } + + public int sMaxage() { + return sMaxage; + } } diff --git a/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java b/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java index c871476..e003988 100644 --- a/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java +++ b/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java @@ -1,6 +1,7 @@ package io.vertx.httpproxy.impl; import io.vertx.core.Future; +import io.vertx.core.MultiMap; import io.vertx.core.http.HttpHeaders; import io.vertx.core.http.HttpMethod; import io.vertx.core.http.HttpServerRequest; @@ -13,8 +14,8 @@ import io.vertx.httpproxy.spi.cache.Resource; import java.time.Instant; -import java.util.function.BiFunction; -import java.util.function.Predicate; +import java.util.List; +import java.util.Map; class CachingFilter implements ProxyInterceptor { @@ -37,117 +38,221 @@ public Future handleProxyResponse(ProxyContext context) { private Future sendAndTryCacheProxyResponse(ProxyContext context) { ProxyResponse response = context.response(); + ProxyRequest request = response.request(); Resource cached = context.get("cached_resource", Resource.class); + String absoluteUri = request.absoluteURI(); if (cached != null && response.getStatusCode() == 304) { - // Warning: this relies on the fact that HttpServerRequest will not send a body for HEAD - response.release(); - fillResponseFromResource(response, cached); - return context.sendResponse(); + return updateStoredCache(cache, absoluteUri, cached, response.headers()).compose(newCached -> { + response.release(); + newCached.init(response, request.getMethod() == HttpMethod.GET); + return context.sendResponse(); + }); } - ProxyRequest request = response.request(); - if (response.publicCacheControl() && response.maxAge() > 0) { - if (request.getMethod() == HttpMethod.GET) { - String absoluteUri = request.absoluteURI(); - Resource res = new Resource( - absoluteUri, - response.getStatusCode(), - response.getStatusMessage(), - response.headers(), - System.currentTimeMillis(), - response.maxAge()); - Body body = response.getBody(); - response.setBody(Body.body(new BufferingReadStream(body.stream(), res.getContent()), body.length())); - Future fut = context.sendResponse(); - fut.onSuccess(v -> { - cache.put(absoluteUri, res); - }); - return fut; - } else if (request.getMethod() != HttpMethod.HEAD) { - return context.sendResponse(); - } else { - return cache.get(request.absoluteURI()).compose(resource -> { - if (resource != null) { - if (!revalidateResource(response, resource)) { - // Invalidate cache - cache.remove(request.absoluteURI()); - } - } - return context.sendResponse(); - }); + String reqCacheControlStr = request.headers().get(HttpHeaders.CACHE_CONTROL); + CacheControl requestCacheControl = reqCacheControlStr == null ? null : new CacheControl().parse(reqCacheControlStr); + String respCacheControlStr = response.headers().get(HttpHeaders.CACHE_CONTROL); + CacheControl responseCacheControl = respCacheControlStr == null ? null : new CacheControl().parse(respCacheControlStr); + + if (request.getMethod() == HttpMethod.GET || request.getMethod() == HttpMethod.HEAD) { + boolean canCache = response.maxAge() >= 0 || (responseCacheControl != null && responseCacheControl.isPublic()); + if (responseCacheControl != null) { + if (responseCacheControl.isPrivate()) canCache = false; + if (responseCacheControl.isMustUnderstand()) { + if (!statusCodeUnderstandable(response.getStatusCode())) canCache = false; + } else if (responseCacheControl.isNoStore()) { + canCache = false; + } + } + if (response.headers().get(HttpHeaders.AUTHORIZATION) != null) { + if ( + responseCacheControl == null || ( + !responseCacheControl.isMustRevalidate() + && !responseCacheControl.isPublic() + && responseCacheControl.sMaxage() == -1) + ) { + canCache = false; + } + } + if (requestCacheControl != null && requestCacheControl.isNoStore()) { + canCache = false; } - } else { - return context.sendResponse(); + if (canCache) { + if (request.getMethod() == HttpMethod.GET) { + Resource res = new Resource( + absoluteUri, + varyHeaders(request.headers(), response.headers()), + response.getStatusCode(), + response.getStatusMessage(), + response.headers(), + System.currentTimeMillis(), + response.maxAge()); + Body body = response.getBody(); + response.setBody(Body.body(new BufferingReadStream(body.stream(), res.getContent()), body.length())); + Future fut = context.sendResponse(); + fut.onSuccess(v -> { + cache.put(absoluteUri, res); + }); + return fut; + } else { // is HEAD + return cache.remove(absoluteUri).compose(v -> { + return context.sendResponse(); + }); + } + } + + } else if (request.getMethod() != HttpMethod.OPTIONS && request.getMethod() != HttpMethod.TRACE) { + return cache.remove(absoluteUri).compose(v -> { + return context.sendResponse(); + }); } + return context.sendResponse(); } - private static boolean revalidateResource(ProxyResponse response, Resource resource) { - if (resource.getEtag() != null && response.etag() != null) { - return resource.getEtag().equals(response.etag()); + private static MultiMap varyHeaders(MultiMap requestHeaders, MultiMap responseHeaders) { + MultiMap result = MultiMap.caseInsensitiveMultiMap(); + String vary = responseHeaders.get(HttpHeaders.VARY); + if (vary != null) { + if (vary.trim().equals("*")) { + return result.addAll(requestHeaders); + } + for (String toVary : vary.split(",")) { + toVary = toVary.trim(); + String toVaryValue = requestHeaders.get(toVary); + if (toVaryValue != null) { + result.add(toVary, toVaryValue); + } + } } - return true; + return result; + } + + private static boolean statusCodeUnderstandable(int statusCode) { + return statusCode >= 100 && statusCode < 600; // TODO: should investigate } private Future tryHandleProxyRequestFromCache(ProxyContext context) { ProxyRequest proxyRequest = context.request(); - HttpServerRequest response = proxyRequest.proxiedRequest(); + HttpServerRequest inboundRequest = proxyRequest.proxiedRequest(); + String cacheControlHeader = inboundRequest.getHeader(HttpHeaders.CACHE_CONTROL); + CacheControl requestCacheControl = cacheControlHeader == null ? null : new CacheControl().parse(cacheControlHeader); - HttpMethod method = response.method(); + HttpMethod method = inboundRequest.method(); if (method != HttpMethod.GET && method != HttpMethod.HEAD) { return context.sendRequest(); } String cacheKey = proxyRequest.absoluteURI(); return cache.get(cacheKey).compose(resource -> { - if (resource == null) { + if (resource == null || !checkVaryHeaders(proxyRequest.headers(), resource.getRequestVaryHeader())) { + if (requestCacheControl != null && requestCacheControl.isOnlyIfCached()) { + return Future.succeededFuture(proxyRequest.release().response().setStatusCode(504)); + } return context.sendRequest(); } - long now = System.currentTimeMillis(); - long val = resource.getTimestamp() + resource.getMaxAge(); - if (val < now) { - return cache.remove(cacheKey).compose(v -> context.sendRequest()); - } - - String cacheControlHeader = response.getHeader(HttpHeaders.CACHE_CONTROL); - if (cacheControlHeader != null) { - CacheControl cacheControl = new CacheControl().parse(cacheControlHeader); - if (cacheControl.maxAge() >= 0) { - long currentAge = now - resource.getTimestamp(); - if (currentAge > cacheControl.maxAge() * 1000) { - String etag = resource.getHeaders().get(HttpHeaders.ETAG); - if (etag != null) { - proxyRequest.headers().set(HttpHeaders.IF_NONE_MATCH, resource.getEtag()); - context.set("cached_resource", resource); + boolean validInboundCache = false; + String inboundIfModifiedSince = inboundRequest.getHeader(HttpHeaders.IF_MODIFIED_SINCE); + String inboundIfNoneMatch = inboundRequest.getHeader(HttpHeaders.IF_NONE_MATCH); + Instant resourceLastModified = resource.getLastModified(); + String resourceETag = resource.getEtag(); + if (resource.getStatusCode() == 200) { // TODO: status code 206 + if (inboundIfNoneMatch != null && resourceETag != null) { + String[] inboundETags = inboundIfNoneMatch.split(","); + for (String inboundETag : inboundETags) { + inboundETag = inboundETag.trim(); + if (inboundETag.equals(resourceETag)) { + validInboundCache = true; + break; } return context.sendRequest(); } + } else if (inboundIfModifiedSince != null && resourceLastModified != null) { + if (ParseUtils.parseHeaderDate(inboundIfModifiedSince).isAfter(resourceLastModified)) { // TODO: is it wrong??? + validInboundCache = true; + } + } + } + if (validInboundCache) { + MultiMap infoHeaders = MultiMap.caseInsensitiveMultiMap(); + List headersNeeded = List.of( + HttpHeaders.CACHE_CONTROL, + HttpHeaders.CONTENT_LOCATION, + HttpHeaders.DATE, + HttpHeaders.ETAG, + HttpHeaders.EXPIRES, + HttpHeaders.VARY + ); + for (CharSequence header : headersNeeded) { + String value = resource.getHeaders().get(header); + if (value != null) infoHeaders.add(header, value); } + ProxyResponse resp = proxyRequest.release().response(); + resp.headers().setAll(infoHeaders); + resp.setStatusCode(304); + return Future.succeededFuture(resp); } - // - String ifModifiedSinceHeader = response.getHeader(HttpHeaders.IF_MODIFIED_SINCE); - if ((response.method() == HttpMethod.GET || response.method() == HttpMethod.HEAD) && ifModifiedSinceHeader != null && resource.getLastModified() != null) { - Instant ifModifiedSince = ParseUtils.parseHeaderDate(ifModifiedSinceHeader); - if (!ifModifiedSince.isAfter(resource.getLastModified())) { - return Future.succeededFuture(proxyRequest.release().response().setStatusCode(304)); + boolean needValidate = false; + String resourceCacheControlHeader = resource.getHeaders().get(HttpHeaders.CACHE_CONTROL); + CacheControl resourceCacheControl = resourceCacheControlHeader == null ? null : new CacheControl().parse(resourceCacheControlHeader); + if (resourceCacheControl != null && resourceCacheControl.isNoCache()) needValidate = true; + if (requestCacheControl != null && requestCacheControl.isNoCache()) needValidate = true; + long age = Math.subtractExact(System.currentTimeMillis(), resource.getTimestamp()); // in ms + long maxAge = Math.max(0, resource.getMaxAge()); + if (resourceCacheControl != null && (resourceCacheControl.isMustRevalidate() || resourceCacheControl.isProxyRevalidate())) { + if (age > maxAge) needValidate = true; + } else if (requestCacheControl != null) { + if (requestCacheControl.maxAge() != -1) { + maxAge = Math.min(maxAge, SafeMathUtils.safeMultiply(requestCacheControl.maxAge(), 1000)); + } + if (requestCacheControl.minFresh() != -1) { + maxAge -= SafeMathUtils.safeMultiply(requestCacheControl.minFresh(), 1000); + } else if (requestCacheControl.maxStale() != -1) { + maxAge += SafeMathUtils.safeMultiply(requestCacheControl.maxStale(), 1000); + } + if (age > maxAge) needValidate = true; + } + String etag = resource.getHeaders().get(HttpHeaders.ETAG); + String lastModified = resource.getHeaders().get(HttpHeaders.LAST_MODIFIED); + if (needValidate) { + if (etag != null) { + proxyRequest.headers().set(HttpHeaders.IF_NONE_MATCH, etag); + } + if (lastModified != null) { + proxyRequest.headers().set(HttpHeaders.IF_MODIFIED_SINCE, lastModified); } + context.set("cached_resource", resource); + return context.sendRequest(); + } else { + proxyRequest.release(); + ProxyResponse proxyResponse = proxyRequest.response(); + resource.init(proxyResponse, inboundRequest.method() == HttpMethod.GET); + return Future.succeededFuture(proxyResponse); } - proxyRequest.release(); - ProxyResponse proxyResponse = proxyRequest.response(); - fillResponseFromResource(proxyResponse, resource); - return Future.succeededFuture(proxyResponse); + }); } - public void fillResponseFromResource(ProxyResponse proxyResponse, Resource resource) { - proxyResponse.setStatusCode(200); - proxyResponse.setStatusMessage(resource.getStatusMessage()); - proxyResponse.headers().addAll(resource.getHeaders()); - proxyResponse.setBody(Body.body(resource.getContent())); + + private static boolean checkVaryHeaders(MultiMap requestHeaders, MultiMap varyHeaders) { + for (Map.Entry e: varyHeaders) { + String fromVary = e.getValue().toLowerCase(); + String fromRequest = requestHeaders.get(e.getKey()); + if (fromRequest == null) return false; + fromRequest = fromVary.toLowerCase(); + if (!fromRequest.equals(fromVary)) return false; + } + return true; + } + + private static Future updateStoredCache(Cache cache, String key, Resource oldCached, MultiMap newHeaders) { + MultiMap newHeadersInternal = MultiMap.caseInsensitiveMultiMap().addAll(newHeaders).remove(HttpHeaders.CONTENT_LENGTH); + oldCached.getHeaders().setAll(newHeadersInternal); + return cache.put(key, oldCached).compose(v -> Future.succeededFuture(oldCached)); } } diff --git a/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java b/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java index f360daf..ec85038 100644 --- a/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java +++ b/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java @@ -78,13 +78,20 @@ class ProxiedResponse implements ProxyResponse { CacheControl cacheControl = new CacheControl().parse(cacheControlHeader); if (cacheControl.isPublic()) { publicCacheControl = true; - if (cacheControl.maxAge() > 0) { - maxAge = (long)cacheControl.maxAge() * 1000; + if (cacheControl.sMaxage() >= 0) { + maxAge = (long) cacheControl.sMaxage() * 1000; + } else if (cacheControl.maxAge() >= 0) { + maxAge = (long) cacheControl.maxAge() * 1000; } else { String dateHeader = response.getHeader(HttpHeaders.DATE); String expiresHeader = response.getHeader(HttpHeaders.EXPIRES); - if (dateHeader != null && expiresHeader != null) { - maxAge = ParseUtils.parseHeaderDate(expiresHeader).toEpochMilli() - ParseUtils.parseHeaderDate(dateHeader).toEpochMilli(); + if (dateHeader != null) { + if (expiresHeader != null) { + maxAge = Math.max(0, ParseUtils.parseHeaderDate(expiresHeader).toEpochMilli() - ParseUtils.parseHeaderDate(dateHeader).toEpochMilli()); + } else if (heuristicallyCacheable(response)) { + String lastModifiedHeader = response.getHeader(HttpHeaders.LAST_MODIFIED); + maxAge = Math.max(0, (ParseUtils.parseHeaderDate(lastModifiedHeader).toEpochMilli() - ParseUtils.parseHeaderDate(dateHeader).toEpochMilli()) / 10); + } } } } @@ -267,4 +274,17 @@ private Future sendResponse(ReadStream body) { } }); } + + private static boolean heuristicallyCacheable(HttpClientResponse response) { + if (response.getHeader(HttpHeaders.LAST_MODIFIED) == null) return false; + + String cacheControlHeader = response.getHeader(HttpHeaders.CACHE_CONTROL); + if (cacheControlHeader != null) { + CacheControl cacheControl = new CacheControl().parse(cacheControlHeader); + if (cacheControl.isPublic()) return true; + } + + return List.of(200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501) + .contains(response.statusCode()); + } } diff --git a/src/main/java/io/vertx/httpproxy/impl/ProxyTransform.java b/src/main/java/io/vertx/httpproxy/impl/ProxyTransform.java new file mode 100644 index 0000000..19938fe --- /dev/null +++ b/src/main/java/io/vertx/httpproxy/impl/ProxyTransform.java @@ -0,0 +1,31 @@ +package io.vertx.httpproxy.impl; + +import io.vertx.core.Future; +import io.vertx.core.MultiMap; +import io.vertx.core.http.HttpHeaders; +import io.vertx.httpproxy.ProxyContext; +import io.vertx.httpproxy.ProxyInterceptor; +import io.vertx.httpproxy.ProxyResponse; + +public class ProxyTransform implements ProxyInterceptor { + @Override + public Future handleProxyRequest(ProxyContext context) { + operateConnectionHeader(context.request().headers()); + return context.sendRequest(); + } + + @Override + public Future handleProxyResponse(ProxyContext context) { + return context.sendResponse(); + } + + private static void operateConnectionHeader(MultiMap headers) { + String connection = headers.get(HttpHeaders.CONNECTION); + if (connection == null || connection.trim().equals("close")) return; + + String[] toRemoveArr = connection.split(","); + for (String toRemove : toRemoveArr) { + headers.remove(toRemove.trim()); + } + } +} diff --git a/src/main/java/io/vertx/httpproxy/impl/ReverseProxy.java b/src/main/java/io/vertx/httpproxy/impl/ReverseProxy.java index ac1097e..a76ea9f 100644 --- a/src/main/java/io/vertx/httpproxy/impl/ReverseProxy.java +++ b/src/main/java/io/vertx/httpproxy/impl/ReverseProxy.java @@ -55,6 +55,7 @@ public class ReverseProxy implements HttpProxy { private final List interceptors = new ArrayList<>(); public ReverseProxy(ProxyOptions options, HttpClient client) { + addInterceptor(new ProxyTransform()); CacheOptions cacheOptions = options.getCacheOptions(); if (cacheOptions != null) { Cache cache = newCache(cacheOptions, ((HttpClientInternal) client).vertx()); diff --git a/src/main/java/io/vertx/httpproxy/impl/SafeMathUtils.java b/src/main/java/io/vertx/httpproxy/impl/SafeMathUtils.java new file mode 100644 index 0000000..dadfc9b --- /dev/null +++ b/src/main/java/io/vertx/httpproxy/impl/SafeMathUtils.java @@ -0,0 +1,45 @@ +package io.vertx.httpproxy.impl; + +/** + * Math calculation should avoid overflow and represents values as the greatest positive integer. + * Refers to "RFC-9111 1.2.2 Delta Seconds". + */ +public class SafeMathUtils { + + public static int safeAdd(int a, int b) { + if (b > 0 ? a > Integer.MAX_VALUE - b : a < Integer.MIN_VALUE - b) { + return b > 0 ? Integer.MAX_VALUE : Integer.MIN_VALUE; + } else { + return a + b; + } + } + + public static int safeSubtract(int a, int b) { + if (b > 0 ? a < Integer.MIN_VALUE + b : a > Integer.MAX_VALUE + b) { + return b > 0 ? Integer.MIN_VALUE : Integer.MAX_VALUE; + } else { + return a - b; + } + } + + public static int safeMultiply(int a, int b) { + if (a > 0 ? (b > Integer.MAX_VALUE / a || b < Integer.MIN_VALUE / a) + : (a < Integer.MIN_VALUE / b || a > Integer.MAX_VALUE / b)) { + return a > 0 ? Integer.MAX_VALUE : Integer.MIN_VALUE; + } else { + return a * b; + } + } + + public int safeDivide(int a, int b) { + return a / b; + } + + public int castInt(long v) { + if (v > 0) { + return v > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) v; + } else { + return v < Integer.MIN_VALUE ? Integer.MIN_VALUE : (int) v; + } + } +} diff --git a/src/main/java/io/vertx/httpproxy/spi/cache/Resource.java b/src/main/java/io/vertx/httpproxy/spi/cache/Resource.java index 8921f53..849d817 100644 --- a/src/main/java/io/vertx/httpproxy/spi/cache/Resource.java +++ b/src/main/java/io/vertx/httpproxy/spi/cache/Resource.java @@ -30,6 +30,7 @@ public class Resource implements ClusterSerializable { private static final Charset UTF_8 = StandardCharsets.UTF_8; private String absoluteUri; + private MultiMap requestVaryHeader; private int statusCode; private String statusMessage; private MultiMap headers; @@ -43,9 +44,10 @@ public class Resource implements ClusterSerializable { public Resource() { } - public Resource(String absoluteUri, int statusCode, String statusMessage, MultiMap headers, long timestamp, long maxAge) { + public Resource(String absoluteUri, MultiMap requestVaryHeader, int statusCode, String statusMessage, MultiMap headers, long timestamp, long maxAge) { String lastModifiedHeader = headers.get(HttpHeaders.LAST_MODIFIED); this.absoluteUri = absoluteUri; + this.requestVaryHeader = requestVaryHeader; this.statusCode = statusCode; this.statusMessage = statusMessage; this.headers = headers; @@ -55,6 +57,16 @@ public Resource(String absoluteUri, int statusCode, String statusMessage, MultiM this.etag = headers.get(HttpHeaders.ETAG); } + public void init(ProxyResponse proxyResponse, boolean withBody) { + proxyResponse.headers().remove(HttpHeaders.CONTENT_LENGTH); + proxyResponse.setStatusCode(statusCode); + proxyResponse.setStatusMessage(statusMessage); + proxyResponse.headers().addAll(headers); + if (withBody) { + proxyResponse.setBody(Body.body(content)); + } + } + private static class Cursor { int i; } @@ -62,6 +74,7 @@ private static class Cursor { @Override public void writeToBuffer(Buffer buffer) { appendString(buffer, absoluteUri); + appendMultiMap(buffer, requestVaryHeader); appendInt(buffer, statusCode); appendString(buffer, statusMessage); appendMultiMap(buffer, headers); @@ -78,6 +91,7 @@ public int readFromBuffer(int pos, Buffer buffer) { cursor.i = pos; setAbsoluteUri(readString(buffer, cursor)); + setRequestVaryHeader(readMultiMap(buffer, cursor)); setStatusCode(readInt(buffer, cursor)); setStatusMessage(readString(buffer, cursor)); setHeaders(readMultiMap(buffer, cursor)); @@ -178,6 +192,10 @@ public String getAbsoluteUri() { return absoluteUri; } + public MultiMap getRequestVaryHeader() { + return requestVaryHeader; + } + public int getStatusCode() { return statusCode; } @@ -214,6 +232,10 @@ public void setAbsoluteUri(String absoluteUri) { this.absoluteUri = absoluteUri; } + public void setRequestVaryHeader(MultiMap requestVaryHeader) { + this.requestVaryHeader = requestVaryHeader; + } + public void setStatusCode(int statusCode) { this.statusCode = statusCode; } diff --git a/src/test/java/io/vertx/tests/cache/spi/CacheSpiTestBase.java b/src/test/java/io/vertx/tests/cache/spi/CacheSpiTestBase.java index 0f68ff7..f2006f7 100644 --- a/src/test/java/io/vertx/tests/cache/spi/CacheSpiTestBase.java +++ b/src/test/java/io/vertx/tests/cache/spi/CacheSpiTestBase.java @@ -42,6 +42,7 @@ public void tearDown(TestContext context) { private Resource generateResource(String absoluteURI, long maxAge) { return new Resource( absoluteURI, + MultiMap.caseInsensitiveMultiMap(), 200, "OK", MultiMap.caseInsensitiveMultiMap(), diff --git a/src/test/java/io/vertx/tests/parsing/ResourceParseTest.java b/src/test/java/io/vertx/tests/parsing/ResourceParseTest.java index 933c3e7..feb99f0 100644 --- a/src/test/java/io/vertx/tests/parsing/ResourceParseTest.java +++ b/src/test/java/io/vertx/tests/parsing/ResourceParseTest.java @@ -45,6 +45,7 @@ public static boolean resourceEquals(Resource r1, Resource r2) { public void testRegular() { Resource resource = new Resource( "http://www.example.com", + MultiMap.caseInsensitiveMultiMap(), 200, "OK", MultiMap.caseInsensitiveMultiMap() @@ -67,6 +68,7 @@ public void testRegular() { public void testEmpty() { Resource resource = new Resource( "http://www.example.com", + MultiMap.caseInsensitiveMultiMap(), 200, "OK", MultiMap.caseInsensitiveMultiMap(), From 9369f253b01181b95e20d34a22ec8873ea798006 Mon Sep 17 00:00:00 2001 From: Zengyi Wang Date: Wed, 24 Jul 2024 16:34:07 +0800 Subject: [PATCH 2/2] add tests --- .../io/vertx/httpproxy/impl/CacheControl.java | 12 +- .../vertx/httpproxy/impl/CachingFilter.java | 143 ++++++++++------- .../vertx/httpproxy/impl/ProxiedResponse.java | 36 ++--- .../tests/cache/CachePermissionTest.java | 99 ++++++++++++ .../cache/CacheRequestDirectivesTest.java | 115 ++++++++++++++ .../cache/CacheResponseDirectivesTest.java | 95 +++++++++++ .../tests/cache/CacheRevalidateTest.java | 150 ++++++++++++++++++ .../io/vertx/tests/cache/CacheTestBase.java | 58 +++++++ .../io/vertx/tests/cache/CacheVaryTest.java | 115 ++++++++++++++ .../interceptors/HeaderInterceptorTest.java | 19 ++- 10 files changed, 756 insertions(+), 86 deletions(-) create mode 100644 src/test/java/io/vertx/tests/cache/CachePermissionTest.java create mode 100644 src/test/java/io/vertx/tests/cache/CacheRequestDirectivesTest.java create mode 100644 src/test/java/io/vertx/tests/cache/CacheResponseDirectivesTest.java create mode 100644 src/test/java/io/vertx/tests/cache/CacheRevalidateTest.java create mode 100644 src/test/java/io/vertx/tests/cache/CacheVaryTest.java diff --git a/src/main/java/io/vertx/httpproxy/impl/CacheControl.java b/src/main/java/io/vertx/httpproxy/impl/CacheControl.java index 1a2bbb6..e947cd2 100644 --- a/src/main/java/io/vertx/httpproxy/impl/CacheControl.java +++ b/src/main/java/io/vertx/httpproxy/impl/CacheControl.java @@ -41,6 +41,10 @@ public CacheControl parse(String header) { _private = false; proxyRevalidate = false; _public = false; + maxAge = -1; + maxStale = -1; + minFresh = -1; + sMaxage = -1; String[] parts = header.split(","); // No regex for (String part : parts) { @@ -74,10 +78,10 @@ public CacheControl parse(String header) { proxyRevalidate = true; break; default: - maxAge = loadInt(part, "max-age="); - maxStale = loadInt(part, "max-stale="); - minFresh = loadInt(part, "min-fresh="); - sMaxage = loadInt(part, "s-maxage="); + maxAge = Math.max(maxAge, loadInt(part, "max-age=")); + maxStale = Math.max(maxStale, loadInt(part, "max-stale=")); + minFresh = Math.max(minFresh, loadInt(part, "min-fresh=")); + sMaxage = Math.max(sMaxage, loadInt(part, "s-maxage=")); break; } } diff --git a/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java b/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java index e003988..509ba8a 100644 --- a/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java +++ b/src/main/java/io/vertx/httpproxy/impl/CachingFilter.java @@ -14,11 +14,15 @@ import io.vertx.httpproxy.spi.cache.Resource; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Map; class CachingFilter implements ProxyInterceptor { + private static final String SKIP_CACHE_RESPONSE_HANDLING = "skip_cache_response_handling"; + private static final String CACHED_RESOURCE = "cached_resource"; + private final Cache cache; public CachingFilter(Cache cache) { @@ -32,14 +36,19 @@ public Future handleProxyRequest(ProxyContext context) { @Override public Future handleProxyResponse(ProxyContext context) { - return sendAndTryCacheProxyResponse(context); + Boolean skip = context.get(SKIP_CACHE_RESPONSE_HANDLING, Boolean.class); + if (skip != null && skip) { + return context.sendResponse(); + } else { + return sendAndTryCacheProxyResponse(context); + } } private Future sendAndTryCacheProxyResponse(ProxyContext context) { ProxyResponse response = context.response(); ProxyRequest request = response.request(); - Resource cached = context.get("cached_resource", Resource.class); + Resource cached = context.get(CACHED_RESOURCE, Resource.class); String absoluteUri = request.absoluteURI(); if (cached != null && response.getStatusCode() == 304) { @@ -65,7 +74,7 @@ private Future sendAndTryCacheProxyResponse(ProxyContext context) { canCache = false; } } - if (response.headers().get(HttpHeaders.AUTHORIZATION) != null) { + if (request.headers().get(HttpHeaders.AUTHORIZATION) != null) { if ( responseCacheControl == null || ( !responseCacheControl.isMustRevalidate() @@ -78,6 +87,9 @@ private Future sendAndTryCacheProxyResponse(ProxyContext context) { if (requestCacheControl != null && requestCacheControl.isNoStore()) { canCache = false; } + if ("*".equals(response.headers().get(HttpHeaders.VARY))) { + canCache = false; + } if (canCache) { if (request.getMethod() == HttpMethod.GET) { Resource res = new Resource( @@ -114,9 +126,6 @@ private static MultiMap varyHeaders(MultiMap requestHeaders, MultiMap responseHe MultiMap result = MultiMap.caseInsensitiveMultiMap(); String vary = responseHeaders.get(HttpHeaders.VARY); if (vary != null) { - if (vary.trim().equals("*")) { - return result.addAll(requestHeaders); - } for (String toVary : vary.split(",")) { toVary = toVary.trim(); String toVaryValue = requestHeaders.get(toVary); @@ -149,53 +158,13 @@ private Future tryHandleProxyRequestFromCache(ProxyContext contex return cache.get(cacheKey).compose(resource -> { if (resource == null || !checkVaryHeaders(proxyRequest.headers(), resource.getRequestVaryHeader())) { if (requestCacheControl != null && requestCacheControl.isOnlyIfCached()) { + context.set(SKIP_CACHE_RESPONSE_HANDLING, true); return Future.succeededFuture(proxyRequest.release().response().setStatusCode(504)); } return context.sendRequest(); } - boolean validInboundCache = false; - String inboundIfModifiedSince = inboundRequest.getHeader(HttpHeaders.IF_MODIFIED_SINCE); - String inboundIfNoneMatch = inboundRequest.getHeader(HttpHeaders.IF_NONE_MATCH); - Instant resourceLastModified = resource.getLastModified(); - String resourceETag = resource.getEtag(); - if (resource.getStatusCode() == 200) { // TODO: status code 206 - if (inboundIfNoneMatch != null && resourceETag != null) { - String[] inboundETags = inboundIfNoneMatch.split(","); - for (String inboundETag : inboundETags) { - inboundETag = inboundETag.trim(); - if (inboundETag.equals(resourceETag)) { - validInboundCache = true; - break; - } - return context.sendRequest(); - } - } else if (inboundIfModifiedSince != null && resourceLastModified != null) { - if (ParseUtils.parseHeaderDate(inboundIfModifiedSince).isAfter(resourceLastModified)) { // TODO: is it wrong??? - validInboundCache = true; - } - } - } - if (validInboundCache) { - MultiMap infoHeaders = MultiMap.caseInsensitiveMultiMap(); - List headersNeeded = List.of( - HttpHeaders.CACHE_CONTROL, - HttpHeaders.CONTENT_LOCATION, - HttpHeaders.DATE, - HttpHeaders.ETAG, - HttpHeaders.EXPIRES, - HttpHeaders.VARY - ); - for (CharSequence header : headersNeeded) { - String value = resource.getHeaders().get(header); - if (value != null) infoHeaders.add(header, value); - } - ProxyResponse resp = proxyRequest.release().response(); - resp.headers().setAll(infoHeaders); - resp.setStatusCode(304); - return Future.succeededFuture(resp); - } - + // to check if the resource is fresh boolean needValidate = false; String resourceCacheControlHeader = resource.getHeaders().get(HttpHeaders.CACHE_CONTROL); CacheControl resourceCacheControl = resourceCacheControlHeader == null ? null : new CacheControl().parse(resourceCacheControlHeader); @@ -203,9 +172,8 @@ private Future tryHandleProxyRequestFromCache(ProxyContext contex if (requestCacheControl != null && requestCacheControl.isNoCache()) needValidate = true; long age = Math.subtractExact(System.currentTimeMillis(), resource.getTimestamp()); // in ms long maxAge = Math.max(0, resource.getMaxAge()); - if (resourceCacheControl != null && (resourceCacheControl.isMustRevalidate() || resourceCacheControl.isProxyRevalidate())) { - if (age > maxAge) needValidate = true; - } else if (requestCacheControl != null) { + boolean responseValidateOverride = resourceCacheControl != null && (resourceCacheControl.isMustRevalidate() || resourceCacheControl.isProxyRevalidate()); + if (!responseValidateOverride && requestCacheControl != null) { if (requestCacheControl.maxAge() != -1) { maxAge = Math.min(maxAge, SafeMathUtils.safeMultiply(requestCacheControl.maxAge(), 1000)); } @@ -214,8 +182,8 @@ private Future tryHandleProxyRequestFromCache(ProxyContext contex } else if (requestCacheControl.maxStale() != -1) { maxAge += SafeMathUtils.safeMultiply(requestCacheControl.maxStale(), 1000); } - if (age > maxAge) needValidate = true; } + if (age > maxAge) needValidate = true; String etag = resource.getHeaders().get(HttpHeaders.ETAG); String lastModified = resource.getHeaders().get(HttpHeaders.LAST_MODIFIED); if (needValidate) { @@ -225,13 +193,68 @@ private Future tryHandleProxyRequestFromCache(ProxyContext contex if (lastModified != null) { proxyRequest.headers().set(HttpHeaders.IF_MODIFIED_SINCE, lastModified); } - context.set("cached_resource", resource); + context.set(CACHED_RESOURCE, resource); return context.sendRequest(); } else { - proxyRequest.release(); - ProxyResponse proxyResponse = proxyRequest.response(); - resource.init(proxyResponse, inboundRequest.method() == HttpMethod.GET); - return Future.succeededFuture(proxyResponse); + // check if the client already have valid cache using current cache + boolean validInboundCache = false; + Instant inboundIfModifiedSince = ParseUtils.parseHeaderDate(inboundRequest.getHeader(HttpHeaders.IF_MODIFIED_SINCE)); + String inboundIfNoneMatch = inboundRequest.getHeader(HttpHeaders.IF_NONE_MATCH); + Instant resourceLastModified = resource.getLastModified(); + Instant resourceDate = ParseUtils.parseHeaderDate(resource.getHeaders().get(HttpHeaders.DATE)); + String resourceETag = resource.getEtag(); + if (resource.getStatusCode() == 200) { + if (inboundIfNoneMatch != null) { + if (resourceETag != null) { + String[] inboundETags = inboundIfNoneMatch.split(","); + for (String inboundETag : inboundETags) { + inboundETag = inboundETag.trim(); + if (inboundETag.equals(resourceETag)) { + validInboundCache = true; + break; + } + } + } + } else if (inboundIfModifiedSince != null) { + if (resourceLastModified != null) { + if (!inboundIfModifiedSince.isBefore(resourceLastModified)) { + validInboundCache = true; + } + } else if (resourceDate != null) { + if (!inboundIfModifiedSince.isBefore(resourceDate)) { + validInboundCache = true; + } + } + + } + } + if (validInboundCache) { + MultiMap infoHeaders = MultiMap.caseInsensitiveMultiMap(); + List headersNeeded = new ArrayList<>(List.of( + HttpHeaders.CACHE_CONTROL, + HttpHeaders.CONTENT_LOCATION, + HttpHeaders.DATE, + HttpHeaders.ETAG, + HttpHeaders.EXPIRES, + HttpHeaders.VARY + )); + if (inboundIfNoneMatch == null) headersNeeded.add(HttpHeaders.LAST_MODIFIED); + for (CharSequence header : headersNeeded) { + String value = resource.getHeaders().get(header); + if (value != null) infoHeaders.add(header, value); + } + ProxyResponse resp = proxyRequest.release().response(); + resp.headers().setAll(infoHeaders); + resp.setStatusCode(304); + context.set(SKIP_CACHE_RESPONSE_HANDLING, true); + return Future.succeededFuture(resp); + } else { + proxyRequest.release(); + ProxyResponse proxyResponse = proxyRequest.response(); + resource.init(proxyResponse, inboundRequest.method() == HttpMethod.GET); + context.set(SKIP_CACHE_RESPONSE_HANDLING, true); + return Future.succeededFuture(proxyResponse); + } } }); @@ -241,11 +264,9 @@ private Future tryHandleProxyRequestFromCache(ProxyContext contex private static boolean checkVaryHeaders(MultiMap requestHeaders, MultiMap varyHeaders) { for (Map.Entry e: varyHeaders) { - String fromVary = e.getValue().toLowerCase(); + String fromVary = e.getValue(); String fromRequest = requestHeaders.get(e.getKey()); - if (fromRequest == null) return false; - fromRequest = fromVary.toLowerCase(); - if (!fromRequest.equals(fromVary)) return false; + if (fromRequest == null || !fromRequest.equals(fromVary)) return false; } return true; } diff --git a/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java b/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java index ec85038..62da083 100644 --- a/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java +++ b/src/main/java/io/vertx/httpproxy/impl/ProxiedResponse.java @@ -74,25 +74,23 @@ class ProxiedResponse implements ProxyResponse { long maxAge = -1; boolean publicCacheControl = false; String cacheControlHeader = response.getHeader(HttpHeaders.CACHE_CONTROL); - if (cacheControlHeader != null) { - CacheControl cacheControl = new CacheControl().parse(cacheControlHeader); - if (cacheControl.isPublic()) { - publicCacheControl = true; - if (cacheControl.sMaxage() >= 0) { - maxAge = (long) cacheControl.sMaxage() * 1000; - } else if (cacheControl.maxAge() >= 0) { - maxAge = (long) cacheControl.maxAge() * 1000; - } else { - String dateHeader = response.getHeader(HttpHeaders.DATE); - String expiresHeader = response.getHeader(HttpHeaders.EXPIRES); - if (dateHeader != null) { - if (expiresHeader != null) { - maxAge = Math.max(0, ParseUtils.parseHeaderDate(expiresHeader).toEpochMilli() - ParseUtils.parseHeaderDate(dateHeader).toEpochMilli()); - } else if (heuristicallyCacheable(response)) { - String lastModifiedHeader = response.getHeader(HttpHeaders.LAST_MODIFIED); - maxAge = Math.max(0, (ParseUtils.parseHeaderDate(lastModifiedHeader).toEpochMilli() - ParseUtils.parseHeaderDate(dateHeader).toEpochMilli()) / 10); - } - } + CacheControl cacheControl = cacheControlHeader == null ? null : new CacheControl().parse(cacheControlHeader); + if (cacheControl != null && cacheControl.isPublic()) { + publicCacheControl = true; + } + if (cacheControl != null && cacheControl.sMaxage() >= 0) { + maxAge = (long) cacheControl.sMaxage() * 1000; + } else if (cacheControl != null && cacheControl.maxAge() >= 0) { + maxAge = (long) cacheControl.maxAge() * 1000; + } else { + String dateHeader = response.getHeader(HttpHeaders.DATE); + String expiresHeader = response.getHeader(HttpHeaders.EXPIRES); + if (dateHeader != null) { + if (expiresHeader != null) { + maxAge = Math.max(0, ParseUtils.parseHeaderDate(expiresHeader).toEpochMilli() - ParseUtils.parseHeaderDate(dateHeader).toEpochMilli()); + } else if (heuristicallyCacheable(response)) { + String lastModifiedHeader = response.getHeader(HttpHeaders.LAST_MODIFIED); + maxAge = Math.max(0, (ParseUtils.parseHeaderDate(lastModifiedHeader).toEpochMilli() - ParseUtils.parseHeaderDate(dateHeader).toEpochMilli()) / 10); } } } diff --git a/src/test/java/io/vertx/tests/cache/CachePermissionTest.java b/src/test/java/io/vertx/tests/cache/CachePermissionTest.java new file mode 100644 index 0000000..60262c3 --- /dev/null +++ b/src/test/java/io/vertx/tests/cache/CachePermissionTest.java @@ -0,0 +1,99 @@ +package io.vertx.tests.cache; + +import io.vertx.core.Future; +import io.vertx.core.MultiMap; +import io.vertx.core.http.HttpClient; +import io.vertx.core.http.HttpClientResponse; +import io.vertx.core.http.HttpHeaders; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.unit.Async; +import io.vertx.ext.unit.TestContext; +import io.vertx.httpproxy.impl.ParseUtils; +import org.junit.Test; + +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; + +public class CachePermissionTest extends CacheTestBase { + + private HttpClient client; + private AtomicInteger backendResult; + + @Override + public void setUp() { + super.setUp(); + backendResult = new AtomicInteger(INIT); + client = vertx.createHttpClient(); + } + + @Test + public void testUnsafeMethods(TestContext ctx) { + Async latch = ctx.async(); + AtomicInteger hits = new AtomicInteger(0); + SocketAddress backend = startHttpBackend(ctx, 8081, req -> { + hits.incrementAndGet(); + Instant now = Instant.now(); + req.response() + .putHeader(HttpHeaders.ETAG, ETAG_0) + .putHeader(HttpHeaders.CACHE_CONTROL, "max-age=999") + .putHeader(HttpHeaders.DATE, ParseUtils.formatHttpDate(now)) + .end(); + }); + startProxy(backend); + client.request(HttpMethod.GET, 8080, "localhost", "/").compose(req -> req.send()).compose(resp1 -> { + return client.request(HttpMethod.POST, 8080, "localhost", "/").compose(req -> req.send()); + }).compose(resp2 -> { + return client.request(HttpMethod.GET, 8080, "localhost", "/").compose(req -> req.send()); + }).onComplete(ctx.asyncAssertSuccess(resp3 -> { + ctx.assertEquals(hits.get(), 3); + latch.complete(); + })); + } + + @Test + public void testAuth(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, + MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=999")); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add(HttpHeaders.AUTHORIZATION, "Bearer 123")) + .compose(r1 -> call(client)) + .onComplete(ctx.asyncAssertSuccess(r2 -> { + ctx.assertEquals(backendResult.get(), NORMAL); + latch.complete(); + })); + } + + @Test + public void testAuthWithPublic(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, + MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "public, max-age=999")); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add(HttpHeaders.AUTHORIZATION, "Bearer 123")) + .compose(r1 -> call(client)) + .onComplete(ctx.asyncAssertSuccess(r2 -> { + ctx.assertEquals(backendResult.get(), NOT_CALLED); + latch.complete(); + })); + } + + @Test + public void testStrictSemantics(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, + MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "public, private")); + startProxy(backend); + call(client).compose(r1 -> call(client)) + .onComplete(ctx.asyncAssertSuccess(r2 -> { + ctx.assertEquals(backendResult.get(), NORMAL); + latch.complete(); + })); + } + + +} diff --git a/src/test/java/io/vertx/tests/cache/CacheRequestDirectivesTest.java b/src/test/java/io/vertx/tests/cache/CacheRequestDirectivesTest.java new file mode 100644 index 0000000..4a7df28 --- /dev/null +++ b/src/test/java/io/vertx/tests/cache/CacheRequestDirectivesTest.java @@ -0,0 +1,115 @@ +package io.vertx.tests.cache; + +import io.vertx.core.Future; +import io.vertx.core.MultiMap; +import io.vertx.core.http.*; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.unit.Async; +import io.vertx.ext.unit.TestContext; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +public class CacheRequestDirectivesTest extends CacheTestBase { + + private HttpClient client; + private AtomicInteger backendResult; + + @Override + public void setUp() { + super.setUp(); + backendResult = new AtomicInteger(INIT); + client = vertx.createHttpClient(); + } + + private void testWithRequestHeaders(TestContext ctx, MultiMap reqHeaders, int expectedBackendStatus) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap().add( + HttpHeaders.CACHE_CONTROL, "max-age=60" + )); + startProxy(backend); + call(client).onComplete(v -> { + vertx.setTimer(1500, t -> { + call(client, reqHeaders) + .onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(backendResult.get(), expectedBackendStatus); + latch.complete(); + })); + }); + }); + } + + @Test + public void testNoDirectives(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap(); + testWithRequestHeaders(ctx, additionalHeader, NOT_CALLED); + } + + @Test + public void testMaxAgeNotExceed(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=6"); + testWithRequestHeaders(ctx, additionalHeader, NOT_CALLED); + } + + @Test + public void testMaxAgeExceed(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=1"); + testWithRequestHeaders(ctx, additionalHeader, REVALIDATE_SUCCESS); + } + + @Test + public void testMaxStale(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=1, max-stale=999"); + testWithRequestHeaders(ctx, additionalHeader, NOT_CALLED); + } + + @Test + public void testMinFresh(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=6, min-fresh=5"); + testWithRequestHeaders(ctx, additionalHeader, REVALIDATE_SUCCESS); + } + + @Test + public void testNoCache(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "no-cache"); + testWithRequestHeaders(ctx, additionalHeader, REVALIDATE_SUCCESS); + } + + @Test + public void testNoStore(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap().add( + HttpHeaders.CACHE_CONTROL, "max-age=60" + )); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add( + HttpHeaders.CACHE_CONTROL, "no-store" + )).onComplete(v -> { + call(client) + .onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(backendResult.get(), NORMAL); + latch.complete(); + })); + }); + } + + @Test + public void testOnlyIfCached(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap().add( + HttpHeaders.CACHE_CONTROL, "max-age=60" + )); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add( + HttpHeaders.CACHE_CONTROL, "only-if-cached" + )).onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(resp.statusCode(), 504); + latch.complete(); + })); + } +} diff --git a/src/test/java/io/vertx/tests/cache/CacheResponseDirectivesTest.java b/src/test/java/io/vertx/tests/cache/CacheResponseDirectivesTest.java new file mode 100644 index 0000000..112f50c --- /dev/null +++ b/src/test/java/io/vertx/tests/cache/CacheResponseDirectivesTest.java @@ -0,0 +1,95 @@ +package io.vertx.tests.cache; + +import io.vertx.core.MultiMap; +import io.vertx.core.http.HttpClient; +import io.vertx.core.http.HttpHeaders; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.unit.Async; +import io.vertx.ext.unit.TestContext; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +public class CacheResponseDirectivesTest extends CacheTestBase { + + private HttpClient client; + private AtomicInteger backendResult; + + @Override + public void setUp() { + super.setUp(); + backendResult = new AtomicInteger(INIT); + client = vertx.createHttpClient(); + } + + private void testWithResponseHeaders(TestContext ctx, MultiMap respHeaders, int delay, int expectedBackendStatus) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, respHeaders); + startProxy(backend); + call(client).onComplete(v -> { + vertx.setTimer(delay, t -> { + call(client).onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(backendResult.get(), expectedBackendStatus); + latch.complete(); + })); + }); + }); + } + + @Test + public void testMaxAgeNotExceed(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=6"); + testWithResponseHeaders(ctx, additionalHeader, 1500, NOT_CALLED); + } + + @Test + public void testMaxAgeExceed(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=1"); + testWithResponseHeaders(ctx, additionalHeader, 1500, REVALIDATE_SUCCESS); + } + + @Test + public void testNoCache(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=999, no-cache"); + testWithResponseHeaders(ctx, additionalHeader, 100, REVALIDATE_SUCCESS); + } + + @Test + public void testNoStore(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "no-store"); + testWithResponseHeaders(ctx, additionalHeader, 100, NORMAL); + } + + @Test + public void testPrivate(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=999, private"); + testWithResponseHeaders(ctx, additionalHeader, 100, NORMAL); + } + + @Test + public void testPublic(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "public, max-age=0"); + testWithResponseHeaders(ctx, additionalHeader, 100, REVALIDATE_SUCCESS); + } + + @Test + public void testSMaxAgeNotExceed(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=1, s-maxage=999"); + testWithResponseHeaders(ctx, additionalHeader, 1500, NOT_CALLED); + } + + @Test + public void testSMaxAgeExceed(TestContext ctx) { + MultiMap additionalHeader = MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=999, s-maxage=1"); + testWithResponseHeaders(ctx, additionalHeader, 1500, REVALIDATE_SUCCESS); + } + +} diff --git a/src/test/java/io/vertx/tests/cache/CacheRevalidateTest.java b/src/test/java/io/vertx/tests/cache/CacheRevalidateTest.java new file mode 100644 index 0000000..5096158 --- /dev/null +++ b/src/test/java/io/vertx/tests/cache/CacheRevalidateTest.java @@ -0,0 +1,150 @@ +package io.vertx.tests.cache; + +import io.vertx.core.MultiMap; +import io.vertx.core.http.HttpClient; +import io.vertx.core.http.HttpHeaders; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.unit.Async; +import io.vertx.ext.unit.TestContext; +import io.vertx.httpproxy.impl.ParseUtils; +import org.junit.Test; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class CacheRevalidateTest extends CacheTestBase { + + private Instant lastModified; + private HttpClient client; + private AtomicInteger backendResult; + + @Override + public void setUp() { + super.setUp(); + lastModified = Instant.now().minus(1, ChronoUnit.DAYS); + backendResult = new AtomicInteger(INIT); + client = vertx.createHttpClient(); + } + + @Test + public void clientCacheValidETag(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=3600")); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add(HttpHeaders.IF_NONE_MATCH, ETAG_0)).onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(resp.statusCode(), 304); + ctx.assertEquals(resp.headers().get(HttpHeaders.ETAG), ETAG_0); + latch.complete(); + })); + } + + @Test + public void clientCacheInvalidETag(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = etagBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=3600")); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add(HttpHeaders.IF_NONE_MATCH, ETAG_1)).onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(resp.statusCode(), 200); + latch.complete(); + })); + } + + protected SocketAddress lastModifiedBackend(TestContext ctx, AtomicInteger backendResult, MultiMap additionalHeaders) { + return startHttpBackend(ctx, 8081, req -> { + Instant now = Instant.now(); + String ifModifiedSince = req.headers().get(HttpHeaders.IF_MODIFIED_SINCE); + if (ifModifiedSince != null && ParseUtils.parseHeaderDate(ifModifiedSince).isAfter(lastModified)) { + backendResult.set(REVALIDATE_SUCCESS); + req.response().setStatusCode(304); + } else { + if (ifModifiedSince != null) { + backendResult.set(REVALIDATE_FAIL); + } else { + if (backendResult.get() == INIT) { + backendResult.set(NOT_CALLED); + } else { + backendResult.set(NORMAL); + } + } + } + req.response().headers().setAll(additionalHeaders); + req.response() + .putHeader(HttpHeaders.LAST_MODIFIED, ParseUtils.formatHttpDate(lastModified)) + .putHeader(HttpHeaders.DATE, ParseUtils.formatHttpDate(now)) + .end(); + }); + } + + @Test + public void clientCacheValidModifiedSince(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = lastModifiedBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=3600")); + startProxy(backend); + call(client).compose(s -> call(client, MultiMap.caseInsensitiveMultiMap().add(HttpHeaders.IF_MODIFIED_SINCE, ParseUtils.formatHttpDate(lastModified))) + .onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(resp.statusCode(), 304); + ctx.assertNotNull(resp.headers().get(HttpHeaders.LAST_MODIFIED)); + latch.complete(); + }))); + } + + @Test + public void clientCacheInvalidModifiedSince(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = lastModifiedBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=3600")); + startProxy(backend); + call(client).compose(s -> call(client, MultiMap.caseInsensitiveMultiMap().add(HttpHeaders.IF_MODIFIED_SINCE, ParseUtils.formatHttpDate(lastModified.minus(2, ChronoUnit.DAYS)))) + .onComplete(ctx.asyncAssertSuccess(resp -> { + ctx.assertEquals(resp.statusCode(), 200); + latch.complete(); + }))); + } + + protected SocketAddress newEtagBackend(TestContext ctx, AtomicInteger backendResult, MultiMap additionalHeaders) { + return startHttpBackend(ctx, 8081, req -> { + Instant now = Instant.now(); + String ifNoneMatch = req.headers().get(HttpHeaders.IF_NONE_MATCH); + if (ifNoneMatch != null) { + backendResult.set(REVALIDATE_FAIL); + } else { + if (backendResult.get() == INIT) { + backendResult.set(NOT_CALLED); + } else { + backendResult.set(NORMAL); + } + } + req.response().headers().setAll(additionalHeaders); + req.response() + .putHeader(HttpHeaders.ETAG, "\"" + System.currentTimeMillis() + "\"") + .putHeader(HttpHeaders.DATE, ParseUtils.formatHttpDate(now)) + .end(); + }); + } + + @Test + public void serverCacheInvalidETag(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = newEtagBackend(ctx, backendResult, MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "no-cache, max-age=3600")); + startProxy(backend); + call(client).onSuccess(resp1 -> { + vertx.setTimer(500, t -> { + call(client).onComplete(ctx.asyncAssertSuccess(resp2 -> { + ctx.assertEquals(backendResult.get(), REVALIDATE_FAIL); + String oldETag = resp1.getHeader(HttpHeaders.ETAG); + String newETag = resp2.getHeader(HttpHeaders.ETAG); + ctx.assertNotNull(oldETag); + ctx.assertNotNull(newETag); + ctx.assertNotEquals(oldETag, newETag); + latch.complete(); + })); + }); + }); + } + +} diff --git a/src/test/java/io/vertx/tests/cache/CacheTestBase.java b/src/test/java/io/vertx/tests/cache/CacheTestBase.java index 8535a6b..9ca3ca2 100644 --- a/src/test/java/io/vertx/tests/cache/CacheTestBase.java +++ b/src/test/java/io/vertx/tests/cache/CacheTestBase.java @@ -10,16 +10,74 @@ */ package io.vertx.tests.cache; +import io.vertx.core.Future; +import io.vertx.core.MultiMap; +import io.vertx.core.http.HttpClient; +import io.vertx.core.http.HttpClientResponse; +import io.vertx.core.http.HttpHeaders; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.unit.TestContext; import io.vertx.httpproxy.ProxyOptions; import io.vertx.httpproxy.cache.CacheOptions; +import io.vertx.httpproxy.impl.ParseUtils; import io.vertx.tests.TestBase; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; + /** * @author Julien Viet */ public abstract class CacheTestBase extends TestBase { + protected static final String ETAG_0 = "\"etag0\""; + protected static final String ETAG_1 = "\"etag1\""; + protected static final int INIT = -1; + protected static final int NOT_CALLED = 0; // second req not reached to backend + protected static final int NORMAL = 1; // second req reached, but is not revalidate request + protected static final int REVALIDATE_SUCCESS = 2; + protected static final int REVALIDATE_FAIL = 3; + public CacheTestBase() { super(new ProxyOptions().setCacheOptions(new CacheOptions())); } + + protected SocketAddress etagBackend(TestContext ctx, AtomicInteger backendResult, MultiMap additionalHeaders) { + return startHttpBackend(ctx, 8081, req -> { + Instant now = Instant.now(); + String ifNoneMatch = req.headers().get(HttpHeaders.IF_NONE_MATCH); + if (ifNoneMatch != null && ifNoneMatch.equals(ETAG_0)) { + backendResult.set(REVALIDATE_SUCCESS); + req.response().setStatusCode(304); + } else { + if (ifNoneMatch != null) { + backendResult.set(REVALIDATE_FAIL); + } else { + if (backendResult.get() == INIT) { + backendResult.set(NOT_CALLED); + } else { + backendResult.set(NORMAL); + } + } + } + req.response().headers().setAll(additionalHeaders); + req.response() + .putHeader(HttpHeaders.ETAG, ETAG_0) + .putHeader(HttpHeaders.DATE, ParseUtils.formatHttpDate(now)) + .end(); + }); + } + + protected Future call(HttpClient client, MultiMap additionalHeaders) { + return client.request(HttpMethod.GET, 8080, "localhost", "/") + .compose(req -> { + req.headers().setAll(additionalHeaders); + return req.send(); + }); + } + + protected Future call(HttpClient client) { + return call(client, MultiMap.caseInsensitiveMultiMap()); + } } diff --git a/src/test/java/io/vertx/tests/cache/CacheVaryTest.java b/src/test/java/io/vertx/tests/cache/CacheVaryTest.java new file mode 100644 index 0000000..6c9a4d5 --- /dev/null +++ b/src/test/java/io/vertx/tests/cache/CacheVaryTest.java @@ -0,0 +1,115 @@ +package io.vertx.tests.cache; + +import io.vertx.core.MultiMap; +import io.vertx.core.http.HttpClient; +import io.vertx.core.http.HttpHeaders; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.unit.Async; +import io.vertx.ext.unit.TestContext; +import io.vertx.httpproxy.impl.ParseUtils; +import org.junit.Test; + +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; + +public class CacheVaryTest extends CacheTestBase { + private HttpClient client; + private AtomicInteger backendResult; + + private static final String X_CUSTOM_HEADER = "X-Custom-Header"; + + @Override + public void setUp() { + super.setUp(); + backendResult = new AtomicInteger(INIT); + client = vertx.createHttpClient(); + } + + protected SocketAddress varyBackend(TestContext ctx, AtomicInteger backendResult, MultiMap additionalHeaders, String vary) { + return startHttpBackend(ctx, 8081, req -> { + Instant now = Instant.now(); + String custom = req.headers().get(X_CUSTOM_HEADER); + String output = custom; + String ifNoneMatch = req.headers().get(HttpHeaders.IF_NONE_MATCH); + + if (ifNoneMatch != null && ifNoneMatch.equals(ETAG_0)) { + backendResult.set(REVALIDATE_SUCCESS); + output = ""; + req.response().setStatusCode(304); + } else { + if (ifNoneMatch != null) { + backendResult.set(REVALIDATE_FAIL); + } else { + if (backendResult.get() == INIT) { + backendResult.set(NOT_CALLED); + } else { + backendResult.set(NORMAL); + } + } + } + + req.response().headers().setAll(additionalHeaders); + req.response() + .putHeader(HttpHeaders.ETAG, ETAG_0) + .putHeader(HttpHeaders.VARY, vary) + .putHeader(HttpHeaders.DATE, ParseUtils.formatHttpDate(now)) + .end(output); + }); + } + + @Test + public void testCustomVaryMatch(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = varyBackend(ctx, backendResult, + MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=999"), X_CUSTOM_HEADER); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add(X_CUSTOM_HEADER, "A")) + .compose(r1 -> call(client, MultiMap.caseInsensitiveMultiMap().add(X_CUSTOM_HEADER, "A"))) + .onComplete(ctx.asyncAssertSuccess(r2 -> { + r2.body().onSuccess(buffer -> { + ctx.assertEquals(buffer.toString(), "A"); + ctx.assertEquals(backendResult.get(), NOT_CALLED); + latch.complete(); + }); + })); + } + + @Test + public void testCustomVaryNotMatch(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = varyBackend(ctx, backendResult, + MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=999"), X_CUSTOM_HEADER); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add(X_CUSTOM_HEADER, "A")) + .compose(r1 -> call(client, MultiMap.caseInsensitiveMultiMap().add(X_CUSTOM_HEADER, "B"))) + .onComplete(ctx.asyncAssertSuccess(r2 -> { + r2.body().onSuccess(buffer -> { + ctx.assertEquals(buffer.toString(), "B"); + ctx.assertEquals(backendResult.get(), NORMAL); + latch.complete(); + }); + })); + } + + @Test + public void testCustomVaryWildcard(TestContext ctx) { + Async latch = ctx.async(); + SocketAddress backend = varyBackend(ctx, backendResult, + MultiMap.caseInsensitiveMultiMap() + .add(HttpHeaders.CACHE_CONTROL, "max-age=999"), "*"); + startProxy(backend); + call(client, MultiMap.caseInsensitiveMultiMap().add(X_CUSTOM_HEADER, "A")) + .compose(r1 -> call(client, MultiMap.caseInsensitiveMultiMap().add(X_CUSTOM_HEADER, "B"))) + .onComplete(ctx.asyncAssertSuccess(r2 -> { + r2.body().onSuccess(buffer -> { + ctx.assertEquals(buffer.toString(), "B"); + ctx.assertEquals(backendResult.get(), NORMAL); + latch.complete(); + }); + })); + } + + +} diff --git a/src/test/java/io/vertx/tests/interceptors/HeaderInterceptorTest.java b/src/test/java/io/vertx/tests/interceptors/HeaderInterceptorTest.java index 0c80939..d87b765 100644 --- a/src/test/java/io/vertx/tests/interceptors/HeaderInterceptorTest.java +++ b/src/test/java/io/vertx/tests/interceptors/HeaderInterceptorTest.java @@ -11,6 +11,7 @@ package io.vertx.tests.interceptors; +import io.vertx.core.http.HttpClient; import io.vertx.core.http.HttpClientRequest; import io.vertx.core.http.HttpMethod; import io.vertx.core.net.SocketAddress; @@ -28,10 +29,24 @@ */ public class HeaderInterceptorTest extends ProxyTestBase { + HttpClient client = null; + public HeaderInterceptorTest(ProxyOptions options) { super(options); } + @Override + public void setUp() { + super.setUp(); + client = vertx.createHttpClient(); + } + + @Override + public void tearDown(TestContext context) { + if (client != null) client.close(); + super.tearDown(context); + } + @Test public void testFilterRequestHeader(TestContext ctx) { Async latch = ctx.async(); @@ -44,7 +59,7 @@ public void testFilterRequestHeader(TestContext ctx) { startProxy(proxy -> proxy.origin(backend) .addInterceptor(HeadInterceptor.builder().filteringRequestHeaders(Set.of("k2")).build())); - vertx.createHttpClient().request(HttpMethod.GET, 8080, "localhost", "/") + client.request(HttpMethod.GET, 8080, "localhost", "/") .compose(request -> request .putHeader("k1", "v1") .putHeader("k2", "v2") @@ -67,7 +82,7 @@ public void testFilterResponseHeader(TestContext ctx) { startProxy(proxy -> proxy.origin(backend) .addInterceptor(HeadInterceptor.builder().filteringResponseHeaders(Set.of("k2")).build())); - vertx.createHttpClient().request(HttpMethod.GET, 8080, "localhost", "/") + client.request(HttpMethod.GET, 8080, "localhost", "/") .compose(HttpClientRequest::send) .onComplete(ctx.asyncAssertSuccess(resp -> { ctx.assertEquals(resp.headers().get("k1"), "v1");