diff --git a/compatibility-tests/sdk-test-java/src/test/java/io/floci/gcp/test/GcsListFiltersTest.java b/compatibility-tests/sdk-test-java/src/test/java/io/floci/gcp/test/GcsListFiltersTest.java index 08355e51..3b08d912 100644 --- a/compatibility-tests/sdk-test-java/src/test/java/io/floci/gcp/test/GcsListFiltersTest.java +++ b/compatibility-tests/sdk-test-java/src/test/java/io/floci/gcp/test/GcsListFiltersTest.java @@ -97,6 +97,18 @@ void includeTrailingDelimiterAddsThePlaceholderObjectToItems() throws Exception assertThat(itemNames(withFlag)).contains("logs/"); } + @Test + void maxResultsPaginatesDirectoryPrefixes() { + var page = storage.list(BUCKET, + Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.pageSize(1)); + + assertThat(page.getValues()).singleElement() + .extracting(Blob::getName) + .isEqualTo("logs/"); + assertThat(page.getNextPageToken()).isNotNull(); + } + private static String listRaw(String query) throws Exception { URI uri = URI.create(TestFixtures.endpoint() + "/storage/v1/b/" + BUCKET + "/o?" + query); HttpResponse response = HttpClient.newHttpClient().send( diff --git a/src/main/java/io/floci/gcp/services/gcs/GcsObjectController.java b/src/main/java/io/floci/gcp/services/gcs/GcsObjectController.java index 0b64d783..2434c330 100644 --- a/src/main/java/io/floci/gcp/services/gcs/GcsObjectController.java +++ b/src/main/java/io/floci/gcp/services/gcs/GcsObjectController.java @@ -23,7 +23,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; @ApplicationScoped @@ -110,14 +112,48 @@ public Response listObjects(@PathParam("bucket") String bucket, } all = rolledUp; } - PageToken.Page page = PageToken.paginate(all, maxResults, pageToken); + List listedResults; + if (delimiter == null || delimiter.isEmpty()) { + listedResults = all.stream().map(ListedResult::object).toList(); + } else { + TreeMap> objectsByName = new TreeMap<>(); + for (GcsObjectMeta meta : all) { + objectsByName.computeIfAbsent(meta.getName(), ignored -> new ArrayList<>()).add(meta); + } + Comparator generationOrder = Comparator.comparingLong(meta -> Long.parseLong(meta.getGeneration())); + objectsByName.values().forEach(objects -> objects.sort(generationOrder)); + + Set names = new TreeSet<>(prefixes); + names.addAll(objectsByName.keySet()); + listedResults = new ArrayList<>(); + for (String name : names) { + List objects = objectsByName.getOrDefault(name, List.of()); + if (prefixes.contains(name) && !objects.isEmpty()) { + listedResults.add(ListedResult.prefixAndObject(name, objects.getFirst())); + objects.stream().skip(1).map(ListedResult::object).forEach(listedResults::add); + } else if (prefixes.contains(name)) { + listedResults.add(ListedResult.prefix(name)); + } else { + objects.stream().map(ListedResult::object).forEach(listedResults::add); + } + } + } + PageToken.Page page = PageToken.paginate(listedResults, maxResults, pageToken); Map response = new LinkedHashMap<>(); response.put("kind", "storage#objects"); - if (!page.items().isEmpty()) { - response.put("items", page.items()); + List pageItems = page.items().stream() + .map(ListedResult::object) + .filter(Objects::nonNull) + .toList(); + if (!pageItems.isEmpty()) { + response.put("items", pageItems); } - if (!prefixes.isEmpty()) { - response.put("prefixes", new ArrayList<>(prefixes)); + List pagePrefixes = page.items().stream() + .map(ListedResult::prefix) + .filter(Objects::nonNull) + .toList(); + if (!pagePrefixes.isEmpty()) { + response.put("prefixes", pagePrefixes); } if (page.nextPageToken() != null) { response.put("nextPageToken", page.nextPageToken()); @@ -125,6 +161,20 @@ public Response listObjects(@PathParam("bucket") String bucket, return Response.ok(response).build(); } + private record ListedResult(String prefix, GcsObjectMeta object) { + private static ListedResult prefix(String prefix) { + return new ListedResult(prefix, null); + } + + private static ListedResult object(GcsObjectMeta object) { + return new ListedResult(null, object); + } + + private static ListedResult prefixAndObject(String prefix, GcsObjectMeta object) { + return new ListedResult(prefix, object); + } + } + @GET @Path("/{object: .+}/acl") public Response listObjectAcls(@PathParam("bucket") String bucket, diff --git a/src/test/java/io/floci/gcp/services/gcs/GcsListFilterRestIntegrationTest.java b/src/test/java/io/floci/gcp/services/gcs/GcsListFilterRestIntegrationTest.java index bf1b18a6..c065ebfc 100644 --- a/src/test/java/io/floci/gcp/services/gcs/GcsListFilterRestIntegrationTest.java +++ b/src/test/java/io/floci/gcp/services/gcs/GcsListFilterRestIntegrationTest.java @@ -120,6 +120,77 @@ void includeTrailingDelimiterAlsoReturnsThePlaceholderAsAnItem() { .body("prefixes", hasItem("a/")); } + @Test + void maxResultsPaginatesItemsAndPrefixesTogether() { + seed(); + String nextPageToken = given().queryParam("delimiter", "/").queryParam("maxResults", 2) + .when().get("/storage/v1/b/" + BUCKET + "/o") + .then().statusCode(200) + .body("items", org.hamcrest.Matchers.anyOf(org.hamcrest.Matchers.nullValue(), empty())) + .body("prefixes", contains("a/", "b/")) + .extract().path("nextPageToken"); + + given().queryParam("delimiter", "/").queryParam("maxResults", 2) + .queryParam("pageToken", nextPageToken) + .when().get("/storage/v1/b/" + BUCKET + "/o") + .then().statusCode(200) + .body("items.name", contains("c.txt")) + .body("prefixes", contains("logs/")); + } + + @Test + void trailingDelimiterItemAndPrefixShareOneResultSlot() { + seed(); + given().queryParam("delimiter", "/") + .queryParam("includeTrailingDelimiter", true) + .queryParam("maxResults", 1) + .when().get("/storage/v1/b/" + BUCKET + "/o") + .then().statusCode(200) + .body("items.name", contains("a/")) + .body("prefixes", contains("a/")) + .body("nextPageToken", org.hamcrest.Matchers.notNullValue()); + } + + @Test + void versionedTrailingDelimiterGenerationsHaveStablePages() { + String bucket = "versioned-list-filter-bucket"; + given().contentType("application/json") + .body(Map.of("name", bucket, "versioning", Map.of("enabled", true))) + .when().post("/storage/v1/b?project=test-project") + .then().statusCode(200); + String firstGeneration = given().contentType("text/plain").body("first") + .queryParam("uploadType", "media").queryParam("name", "versioned/") + .when().post("/upload/storage/v1/b/" + bucket + "/o") + .then().statusCode(200) + .extract().path("generation"); + String secondGeneration = given().contentType("text/plain").body("second") + .queryParam("uploadType", "media").queryParam("name", "versioned/") + .when().post("/upload/storage/v1/b/" + bucket + "/o") + .then().statusCode(200) + .extract().path("generation"); + + String nextPageToken = given().queryParam("delimiter", "/") + .queryParam("versions", true) + .queryParam("includeTrailingDelimiter", true) + .queryParam("maxResults", 1) + .when().get("/storage/v1/b/" + bucket + "/o") + .then().statusCode(200) + .body("items.name", contains("versioned/")) + .body("items.generation", contains(firstGeneration)) + .body("prefixes", contains("versioned/")) + .extract().path("nextPageToken"); + + given().queryParam("delimiter", "/") + .queryParam("versions", true) + .queryParam("includeTrailingDelimiter", true) + .queryParam("maxResults", 1) + .queryParam("pageToken", nextPageToken) + .when().get("/storage/v1/b/" + bucket + "/o") + .then().statusCode(200) + .body("items.name", contains("versioned/")) + .body("items.generation", contains(secondGeneration)); + } + @Test void repeatedDoubleStarsAreCollapsedRatherThanCompounded() { // "**/**/x" means the same as "**/x"; emitting both groups multiplies the backtracking