From 3166f12c5d0c42d9cf66dcbb0589e5e7a3420c2a Mon Sep 17 00:00:00 2001 From: Harsh Modi Date: Mon, 1 Apr 2024 12:17:38 -0400 Subject: [PATCH 1/3] add pagination query support Signed-off-by: Harsh Modi --- .../controller/AdminController.java | 14 +++-- .../repository/data/StoreDataManager.java | 2 + .../cassandra/CassandraConfiguration.java | 4 ++ .../cassandra/CassandraStoreDataManager.java | 11 +++- .../data/cassandra/CassandraStoreQuery.java | 32 ++++++++++-- .../data/mem/MemoryStoreDataManager.java | 8 ++- .../jaxrs/RepositoryAdminResources.java | 41 ++++++++++++++- .../repository/model/dto/StoreListingDTO.java | 51 ++++++++++++++++++- .../jaxrs/mock/MockAdminController.java | 20 ++++++++ 9 files changed, 170 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java b/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java index 3a0e793..0125b23 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java @@ -113,15 +113,16 @@ public List getAllOfType( final StoreType type ) } } - public List getAllOfType( final String packageType, final StoreType type ) - throws IndyWorkflowException + + public List getAllOfType( final String packageType, final StoreType type, String page ) + throws IndyWorkflowException { try { ArtifactStoreQuery query = storeManager.query().storeTypes( type ); if ( !ALL_PACKAGE_TYPES.equals( packageType ) ) { - return new ArrayList<>( storeManager.getArtifactStoresByPkgAndType( packageType, type ) ); + return new ArrayList<>( storeManager.getArtifactStoresByPkgAndType( packageType, type, page ) ); } else { @@ -135,6 +136,13 @@ public List getAllOfType( final String packageType, final StoreTy } } + + public List getAllOfType( final String packageType, final StoreType type ) + throws IndyWorkflowException + { + return getAllOfType( packageType, type, "" ); + } + public List getAllStores() throws IndyWorkflowException { diff --git a/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java index 2d88af7..7412c42 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java @@ -165,4 +165,6 @@ Set affectedBy( Collection keys ) Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType ); + Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page ); + } diff --git a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraConfiguration.java b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraConfiguration.java index 5ff68b0..d0aec47 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraConfiguration.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraConfiguration.java @@ -33,6 +33,10 @@ public class CassandraConfiguration @ConfigProperty( name = "cassandra.enabled", defaultValue = "false" ) Boolean enabled; + @Inject + @ConfigProperty( name = "cassandra.page_size", defaultValue = "5000" ) + int cassandraPageSize; + @Inject @ConfigProperty( name = "cassandra.host", defaultValue = "localhost" ) String cassandraHost; diff --git a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java index d89aa24..f506e5c 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java @@ -184,17 +184,24 @@ public Set getStoreKeysByPkgAndType( final String pkg, final StoreType } @Override - public Set getArtifactStoresByPkgAndType( final String pkg, final StoreType type ) + public Set getArtifactStoresByPkgAndType( final String pkg, final StoreType type, String page ) { logger.trace( "Get stores: {}/{}", pkg, type ); - Set dtxArtifactStoreSet = storeQuery.getArtifactStoresByPkgAndType( pkg, type ); + Set dtxArtifactStoreSet = storeQuery.getArtifactStoresByPkgAndType( pkg, type, page ); Set storeSet = new HashSet<>(); dtxArtifactStoreSet.forEach( dtxArtifactStore -> storeSet.add( toArtifactStore( dtxArtifactStore ) ) ); return storeSet; } + @Override + public Set getArtifactStoresByPkgAndType( final String pkg, final StoreType type ) + { + + return getArtifactStoresByPkgAndType( pkg, type, "" ); + } + @Override public boolean hasArtifactStore( StoreKey key ) { diff --git a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java index 4100c33..9cbba7b 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java @@ -16,6 +16,7 @@ package org.commonjava.indy.service.repository.data.cassandra; import com.datastax.driver.core.BoundStatement; +import com.datastax.driver.core.PagingState; import com.datastax.driver.core.PreparedStatement; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; @@ -142,19 +143,42 @@ public DtxArtifactStore getArtifactStore( String packageType, StoreType type, St return toDtxArtifactStore( result.one() ); } - public Set getArtifactStoresByPkgAndType( String packageType, StoreType type ) + public Set getArtifactStoresByPkgAndType( String packageType, StoreType type, String page ) { - BoundStatement bound = - preparedArtifactStoresQueryByKeys.bind( CassandraStoreUtil.getTypeKey( packageType, type.name() ) ); + preparedArtifactStoresQueryByKeys.bind( CassandraStoreUtil.getTypeKey( packageType, type.name() ) ); + + if (!page.isEmpty()) { + PagingState currentPage = PagingState.fromString( page ); + bound.setPagingState( currentPage ); + } + ResultSet result = session.execute( bound ); + PagingState nextPage = result.getExecutionInfo().getPagingState(); + int remaining = result.getAvailableWithoutFetching(); + + if (!page.isEmpty()) { + remaining = Math.min( remaining, config.cassandraPageSize ); + } + Set dtxArtifactStoreSet = new HashSet<>(); - result.forEach( row -> dtxArtifactStoreSet.add( toDtxArtifactStore( row ) ) ); + for (Row row: result) + { + dtxArtifactStoreSet.add( toDtxArtifactStore( row ) ); + if (--remaining == 0) { + break; + } + } return dtxArtifactStoreSet; } + public Set getArtifactStoresByPkgAndType( String packageType, StoreType type ) + { + return getArtifactStoresByPkgAndType( packageType, type, "" ); + } + public Set getAllArtifactStores() { diff --git a/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java index 7c9e8bf..9a55ae9 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java @@ -156,7 +156,7 @@ public Stream streamArtifactStoreKeys() } @Override - public Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType ) + public Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page ) { return stores.values() .stream() @@ -165,6 +165,12 @@ public Set getArtifactStoresByPkgAndType( String packageType, Sto .collect( Collectors.toSet() ); } + @Override + public Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType ) + { + return getArtifactStoresByPkgAndType( packageType, storeType, "" ); + } + @Override protected ArtifactStore putArtifactStoreInternal( StoreKey storeKey, ArtifactStore store ) { diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java index 6bf13b6..6bfe0f2 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java @@ -316,7 +316,46 @@ public Response getAll( final @Parameter( @PathParam( "packageType" ) String packageType, @Parameter( name = "type", in = PATH, description = "The type of the repository.", content = @Content( schema = @Schema( implementation = StoreType.class ) ), - required = true ) @PathParam( "type" ) String type ) + required = true ) @PathParam( "type" ) String type, + @QueryParam( "page") String page + ) + { + + final StoreType st = StoreType.get( type ); + + Response response; + try + { + final List stores = adminController.getAllOfType( packageType, st, page ); + + logger.info( "Returning listing containing stores:\n\t{}", new JoinString( "\n\t", stores ) ); + + final StoreListingDTO dto = new StoreListingDTO<>( stores ); + + response = responseHelper.formatOkResponseWithJsonEntity( dto ); + } + catch ( final IndyWorkflowException e ) + { + logger.error( e.getMessage(), e ); + response = responseHelper.formatResponse( e ); + } + + return response; + } + + + @Operation( description = "Retrieve the definitions of all artifact stores of a given type on the system" ) + @APIResponse( responseCode = "200", + content = @Content( schema = @Schema( implementation = StoreListingDTO.class ) ), + description = "The store definitions" ) + @GET + @Produces( APPLICATION_JSON ) + public Response getAllPaginated( final @Parameter( + description = "Filter only stores that support the package type (eg. maven, npm). NOTE: '_all' returns all." ) + @PathParam( "packageType" ) String packageType, + @Parameter( name = "type", in = PATH, description = "The type of the repository.", + content = @Content( schema = @Schema( implementation = StoreType.class ) ), + required = true ) @PathParam( "type" ) String type ) { final StoreType st = StoreType.get( type ); diff --git a/src/main/java/org/commonjava/indy/service/repository/model/dto/StoreListingDTO.java b/src/main/java/org/commonjava/indy/service/repository/model/dto/StoreListingDTO.java index 07e082e..09c40b6 100644 --- a/src/main/java/org/commonjava/indy/service/repository/model/dto/StoreListingDTO.java +++ b/src/main/java/org/commonjava/indy/service/repository/model/dto/StoreListingDTO.java @@ -26,13 +26,19 @@ @Schema( type = SchemaType.OBJECT, discriminatorProperty = "type", description = "List of artifact store definitions" ) public class StoreListingDTO - implements Iterable + implements Iterable { @JsonProperty @Schema( implementation = ArtifactStore.class, description = "The store definition list", required = true ) private List items; + @JsonProperty( "current_page" ) + private String currentPage; + + @JsonProperty( "next_page" ) + private String nextPage; + public StoreListingDTO() { } @@ -42,6 +48,13 @@ public StoreListingDTO( final List items ) this.items = items; } + public StoreListingDTO( final List items, final String currentPage, final String nextPage ) + { + this.items = items; + this.currentPage = currentPage; + this.nextPage = nextPage; + } + public List getItems() { return items == null ? Collections.emptyList() : items; @@ -57,6 +70,18 @@ public String toString() { final StringBuilder sb = new StringBuilder(); sb.append( "StoreListingDTO[" ); + if ( !currentPage.isEmpty() ) + { + sb.append( "\ncurrentPage=" ).append( currentPage ); + } + if ( !nextPage.isEmpty() ) + { + sb.append( "\nnextPage=" ).append( nextPage ); + } + if ( !currentPage.isEmpty() || !nextPage.isEmpty() ) + { + sb.append( "\n [" ); + } if ( items == null || items.isEmpty() ) { sb.append( "NO STORES" ); @@ -68,7 +93,10 @@ public String toString() sb.append( "\n " ).append( item ); } } - + if ( !currentPage.isEmpty() || !nextPage.isEmpty() ) + { + sb.append( "\n ]" ); + } sb.append( "\n]" ); return sb.toString(); } @@ -79,4 +107,23 @@ public Iterator iterator() return getItems().iterator(); } + public String getCurrentPage() + { + return currentPage; + } + + public void setCurrentPage( String currentPage ) + { + this.currentPage = currentPage; + } + + public String getNextPage() + { + return nextPage; + } + + public void setNextPage( String nextPage ) + { + this.nextPage = nextPage; + } } diff --git a/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java b/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java index 4bea60b..af5e82a 100644 --- a/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java +++ b/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java @@ -66,6 +66,26 @@ public List getAllOfType( final String packageType, final StoreTy throw new IndyWorkflowException( Response.Status.NOT_FOUND.getStatusCode(), "Not found" ); } + @Override + public List getAllOfType( final String packageType, final StoreType type, String page ) + throws IndyWorkflowException + { + if ( MAVEN_PKG_KEY.equals( packageType ) ) + { + if ( type == StoreType.remote ) + { + RemoteRepository repo1 = new RemoteRepository( MAVEN_PKG_KEY, "test1", "http://repo.test1" ); + RemoteRepository repo2 = new RemoteRepository( MAVEN_PKG_KEY, "test2", "http://repo.test2" ); + return Arrays.asList( repo1, repo2 ); + } + if ( type == StoreType.hosted ) + { + return Collections.emptyList(); + } + } + throw new IndyWorkflowException( Response.Status.NOT_FOUND.getStatusCode(), "Not found" ); + } + @Override public ArtifactStore get( final StoreKey key ) { From a44620b80b0a7538ec802cc8da078d4f3a2c7891 Mon Sep 17 00:00:00 2001 From: Harsh Modi Date: Thu, 4 Apr 2024 09:10:22 -0400 Subject: [PATCH 2/3] add new DTOs to support pagination Signed-off-by: Harsh Modi --- .../controller/AdminController.java | 12 ++-- .../repository/data/StoreDataManager.java | 3 +- .../cassandra/CassandraStoreDataManager.java | 14 +++-- .../data/cassandra/CassandraStoreQuery.java | 8 ++- .../data/mem/MemoryStoreDataManager.java | 17 +++--- .../jaxrs/RepositoryAdminResources.java | 10 ++- .../model/dto/ListArtifactStoreDTO.java | 61 +++++++++++++++++++ .../model/dto/ListDtxArtifactStoreDTO.java | 56 +++++++++++++++++ .../jaxrs/mock/MockAdminController.java | 8 ++- 9 files changed, 162 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/commonjava/indy/service/repository/model/dto/ListArtifactStoreDTO.java create mode 100644 src/main/java/org/commonjava/indy/service/repository/model/dto/ListDtxArtifactStoreDTO.java diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java b/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java index 0125b23..f857a49 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java @@ -30,6 +30,7 @@ import org.commonjava.indy.service.repository.model.RemoteRepository; import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.StoreType; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; import org.eclipse.microprofile.context.ManagedExecutor; import org.eclipse.microprofile.rest.client.inject.RestClient; import org.slf4j.Logger; @@ -39,6 +40,7 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; @@ -114,7 +116,7 @@ public List getAllOfType( final StoreType type ) } - public List getAllOfType( final String packageType, final StoreType type, String page ) + public ListArtifactStoreDTO getAllOfType( final String packageType, final StoreType type, String page ) throws IndyWorkflowException { try @@ -122,11 +124,12 @@ public List getAllOfType( final String packageType, final StoreTy ArtifactStoreQuery query = storeManager.query().storeTypes( type ); if ( !ALL_PACKAGE_TYPES.equals( packageType ) ) { - return new ArrayList<>( storeManager.getArtifactStoresByPkgAndType( packageType, type, page ) ); + return storeManager.getArtifactStoresByPkgAndType( packageType, type, page ); } else { - return query.getAllByDefaultPackageTypes(); + List result = query.getAllByDefaultPackageTypes(); + return new ListArtifactStoreDTO( new HashSet<>(result) ); } } catch ( final IndyDataException e ) @@ -140,7 +143,8 @@ public List getAllOfType( final String packageType, final StoreTy public List getAllOfType( final String packageType, final StoreType type ) throws IndyWorkflowException { - return getAllOfType( packageType, type, "" ); + ListArtifactStoreDTO result = getAllOfType( packageType, type, "" ); + return new ArrayList<>(result.getItems()); } public List getAllStores() diff --git a/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java index 7412c42..0ad8534 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java @@ -23,6 +23,7 @@ import org.commonjava.indy.service.repository.model.StoreType; import org.commonjava.indy.service.repository.audit.ChangeSummary; import org.commonjava.indy.service.repository.exception.IndyDataException; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; import java.util.Collection; import java.util.Map; @@ -165,6 +166,6 @@ Set affectedBy( Collection keys ) Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType ); - Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page ); + ListArtifactStoreDTO getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page ); } diff --git a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java index f506e5c..4032fc2 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java @@ -31,6 +31,8 @@ import org.commonjava.indy.service.repository.model.RemoteRepository; import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.StoreType; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; +import org.commonjava.indy.service.repository.model.dto.ListDtxArtifactStoreDTO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,22 +186,22 @@ public Set getStoreKeysByPkgAndType( final String pkg, final StoreType } @Override - public Set getArtifactStoresByPkgAndType( final String pkg, final StoreType type, String page ) + public ListArtifactStoreDTO getArtifactStoresByPkgAndType( final String pkg, final StoreType type, String page ) { logger.trace( "Get stores: {}/{}", pkg, type ); - - Set dtxArtifactStoreSet = storeQuery.getArtifactStoresByPkgAndType( pkg, type, page ); + ListDtxArtifactStoreDTO result = storeQuery.getArtifactStoresByPkgAndType( pkg, type, page ); + Set dtxArtifactStoreSet = result.getItems(); Set storeSet = new HashSet<>(); dtxArtifactStoreSet.forEach( dtxArtifactStore -> storeSet.add( toArtifactStore( dtxArtifactStore ) ) ); - return storeSet; + return new ListArtifactStoreDTO( storeSet, result.getCurrentPage(), result.getNextPage()); } @Override public Set getArtifactStoresByPkgAndType( final String pkg, final StoreType type ) { - - return getArtifactStoresByPkgAndType( pkg, type, "" ); + ListArtifactStoreDTO result = getArtifactStoresByPkgAndType( pkg, type, "" ); + return result.getItems(); } @Override diff --git a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java index 9cbba7b..cfb11d8 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java @@ -25,6 +25,7 @@ import com.datastax.driver.mapping.MappingManager; import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.StoreType; +import org.commonjava.indy.service.repository.model.dto.ListDtxArtifactStoreDTO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -143,7 +144,7 @@ public DtxArtifactStore getArtifactStore( String packageType, StoreType type, St return toDtxArtifactStore( result.one() ); } - public Set getArtifactStoresByPkgAndType( String packageType, StoreType type, String page ) + public ListDtxArtifactStoreDTO getArtifactStoresByPkgAndType( String packageType, StoreType type, String page ) { BoundStatement bound = preparedArtifactStoresQueryByKeys.bind( CassandraStoreUtil.getTypeKey( packageType, type.name() ) ); @@ -171,12 +172,13 @@ public Set getArtifactStoresByPkgAndType( String packageType, } } - return dtxArtifactStoreSet; + return new ListDtxArtifactStoreDTO( dtxArtifactStoreSet, page, nextPage.toString() ); } public Set getArtifactStoresByPkgAndType( String packageType, StoreType type ) { - return getArtifactStoresByPkgAndType( packageType, type, "" ); + ListDtxArtifactStoreDTO result = getArtifactStoresByPkgAndType( packageType, type, "" ); + return result.getItems(); } public Set getAllArtifactStores() diff --git a/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java index 9a55ae9..a769dc2 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java @@ -23,6 +23,7 @@ import org.commonjava.indy.service.repository.model.ArtifactStore; import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.StoreType; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -156,19 +157,21 @@ public Stream streamArtifactStoreKeys() } @Override - public Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page ) + public ListArtifactStoreDTO getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page ) { - return stores.values() - .stream() - .filter( item -> packageType.equals( item.getPackageType() ) && storeType.equals( - item.getType() ) ) - .collect( Collectors.toSet() ); + Set resultStores = stores.values() + .stream() + .filter( item -> packageType.equals( item.getPackageType() ) && storeType.equals( + item.getType() ) ) + .collect( Collectors.toSet() ); + return new ListArtifactStoreDTO( resultStores); } @Override public Set getArtifactStoresByPkgAndType( String packageType, StoreType storeType ) { - return getArtifactStoresByPkgAndType( packageType, storeType, "" ); + ListArtifactStoreDTO result = getArtifactStoresByPkgAndType( packageType, storeType, "" ); + return result.getItems(); } @Override diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java index 6bfe0f2..8098193 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryAdminResources.java @@ -25,6 +25,7 @@ import org.commonjava.indy.service.repository.model.RemoteRepository; import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.StoreType; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; import org.commonjava.indy.service.repository.model.dto.StoreListingDTO; import org.commonjava.indy.service.repository.util.jackson.MapperUtil; import org.commonjava.indy.service.security.common.SecurityManager; @@ -59,6 +60,7 @@ import java.io.IOException; import java.net.URI; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -326,11 +328,13 @@ public Response getAll( final @Parameter( Response response; try { - final List stores = adminController.getAllOfType( packageType, st, page ); + final ListArtifactStoreDTO result = adminController.getAllOfType( packageType, st, page ); - logger.info( "Returning listing containing stores:\n\t{}", new JoinString( "\n\t", stores ) ); + logger.info( "Returning listing for page {} containing stores:\n\t{}", result.getCurrentPage(), + new JoinString( "\n\t", result.getItems() ) ); - final StoreListingDTO dto = new StoreListingDTO<>( stores ); + final StoreListingDTO dto = new StoreListingDTO<>( new ArrayList<>(result.getItems()), result.getCurrentPage(), + result.getNextPage() ); response = responseHelper.formatOkResponseWithJsonEntity( dto ); } diff --git a/src/main/java/org/commonjava/indy/service/repository/model/dto/ListArtifactStoreDTO.java b/src/main/java/org/commonjava/indy/service/repository/model/dto/ListArtifactStoreDTO.java new file mode 100644 index 0000000..90f882d --- /dev/null +++ b/src/main/java/org/commonjava/indy/service/repository/model/dto/ListArtifactStoreDTO.java @@ -0,0 +1,61 @@ +package org.commonjava.indy.service.repository.model.dto; + +import org.commonjava.indy.service.repository.model.ArtifactStore; + +import java.util.Set; + +public class ListArtifactStoreDTO +{ + private Set items; + + private String currentPage; + + private String nextPage; + + public ListArtifactStoreDTO() + { + + } + + public ListArtifactStoreDTO( Set items, String currentPage, String nextPage ) + { + this.items = items; + this.currentPage = currentPage; + this.nextPage = nextPage; + } + + public ListArtifactStoreDTO( Set items ) + { + this.items = items; + } + + public Set getItems() + { + return items; + } + + public void setItems( Set items ) + { + this.items = items; + } + + public String getCurrentPage() + { + return currentPage; + } + + public void setCurrentPage( String currentPage ) + { + this.currentPage = currentPage; + } + + public String getNextPage() + { + return nextPage; + } + + public void setNextPage( String nextPage ) + { + this.nextPage = nextPage; + } +} diff --git a/src/main/java/org/commonjava/indy/service/repository/model/dto/ListDtxArtifactStoreDTO.java b/src/main/java/org/commonjava/indy/service/repository/model/dto/ListDtxArtifactStoreDTO.java new file mode 100644 index 0000000..d179c53 --- /dev/null +++ b/src/main/java/org/commonjava/indy/service/repository/model/dto/ListDtxArtifactStoreDTO.java @@ -0,0 +1,56 @@ +package org.commonjava.indy.service.repository.model.dto; + +import org.commonjava.indy.service.repository.data.cassandra.DtxArtifactStore; + +import java.util.Set; + +public class ListDtxArtifactStoreDTO +{ + private Set items; + + private String currentPage; + + private String nextPage; + + public ListDtxArtifactStoreDTO() + { + + } + + public ListDtxArtifactStoreDTO( Set items, String currentPage, String nextPage ) + { + this.items = items; + this.currentPage = currentPage; + this.nextPage = nextPage; + } + + public Set getItems() + { + return items; + } + + public void setItems( Set items ) + { + this.items = items; + } + + public String getCurrentPage() + { + return currentPage; + } + + public void setCurrentPage( String currentPage ) + { + this.currentPage = currentPage; + } + + public String getNextPage() + { + return nextPage; + } + + public void setNextPage( String nextPage ) + { + this.nextPage = nextPage; + } +} diff --git a/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java b/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java index af5e82a..fefb558 100644 --- a/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java +++ b/src/test/java/org/commonjava/indy/service/repository/jaxrs/mock/MockAdminController.java @@ -24,6 +24,7 @@ import org.commonjava.indy.service.repository.model.RemoteRepository; import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.StoreType; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; import org.junit.jupiter.api.Test; import jakarta.enterprise.context.ApplicationScoped; @@ -31,6 +32,7 @@ import jakarta.ws.rs.core.Response; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import static org.commonjava.indy.service.repository.model.pkg.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; @@ -67,7 +69,7 @@ public List getAllOfType( final String packageType, final StoreTy } @Override - public List getAllOfType( final String packageType, final StoreType type, String page ) + public ListArtifactStoreDTO getAllOfType( final String packageType, final StoreType type, String page ) throws IndyWorkflowException { if ( MAVEN_PKG_KEY.equals( packageType ) ) @@ -76,11 +78,11 @@ public List getAllOfType( final String packageType, final StoreTy { RemoteRepository repo1 = new RemoteRepository( MAVEN_PKG_KEY, "test1", "http://repo.test1" ); RemoteRepository repo2 = new RemoteRepository( MAVEN_PKG_KEY, "test2", "http://repo.test2" ); - return Arrays.asList( repo1, repo2 ); + return new ListArtifactStoreDTO( new HashSet<>(Arrays.asList( repo1, repo2 ))); } if ( type == StoreType.hosted ) { - return Collections.emptyList(); + return new ListArtifactStoreDTO( new HashSet<>(Collections.emptyList())); } } throw new IndyWorkflowException( Response.Status.NOT_FOUND.getStatusCode(), "Not found" ); From 89e0a0005a3220e3a293da1e53c0a88829c235ad Mon Sep 17 00:00:00 2001 From: Harsh Modi Date: Thu, 4 Apr 2024 20:55:01 -0400 Subject: [PATCH 3/3] add pagination for getAllArtifactStores Signed-off-by: Harsh Modi --- .../controller/QueryController.java | 24 ++++++++++++-- .../controller/StatsController.java | 14 ++++++-- .../repository/data/StoreDataManager.java | 6 ++++ .../cassandra/CassandraStoreDataManager.java | 13 ++++++-- .../data/cassandra/CassandraStoreQuery.java | 32 ++++++++++++++++--- .../data/mem/MemoryStoreDataManager.java | 6 ++++ .../jaxrs/RepositoryQueryResources.java | 5 +-- .../jaxrs/version/StatsHandler.java | 5 +-- .../model/dto/EndpointViewListing.java | 31 ++++++++++++++++++ 9 files changed, 120 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java b/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java index 90b7999..f1b3a67 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java @@ -28,6 +28,7 @@ import org.commonjava.indy.service.repository.model.StoreType; import org.commonjava.indy.service.repository.model.dto.EndpointView; import org.commonjava.indy.service.repository.model.dto.EndpointViewListing; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; import org.commonjava.indy.service.repository.util.JaxRsUriFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -257,13 +258,16 @@ public Boolean isStoreDataEmpty() } public EndpointViewListing getEndpointsListing( final String pkgType, final String baseUri, - final JaxRsUriFormatter uriFormatter ) + final JaxRsUriFormatter uriFormatter, final String page ) throws IndyWorkflowException { List stores; + String nextPage = ""; try { - stores = new ArrayList<>( storeManager.getAllArtifactStores() ); + ListArtifactStoreDTO result = storeManager.getAllArtifactStores(page); + nextPage = result.getNextPage(); + stores = new ArrayList<>( result.getItems() ); if ( StringUtils.isNotBlank( pkgType ) && !"all".equals( pkgType ) && isValidPackageType( pkgType ) ) { stores = stores.stream() @@ -292,7 +296,21 @@ public EndpointViewListing getEndpointsListing( final String pkgType, final Stri } } - return new EndpointViewListing( points ); + EndpointViewListing result = new EndpointViewListing( points ); + result.setCurrentPage( page ); + + if (!nextPage.isEmpty()) { + result.setNextPage( nextPage ); + } + + return result; + } + + public EndpointViewListing getEndpointsListing( final String pkgType, final String baseUri, + final JaxRsUriFormatter uriFormatter ) + throws IndyWorkflowException + { + return getEndpointsListing( pkgType, baseUri, uriFormatter, "" ); } public Map> getStoreKeysByPackageType( final String pkgType ) diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java b/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java index 3aae35e..cf5648c 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java @@ -22,6 +22,7 @@ import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.dto.EndpointView; import org.commonjava.indy.service.repository.model.dto.EndpointViewListing; +import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO; import org.commonjava.indy.service.repository.model.version.Versioning; import jakarta.enterprise.context.ApplicationScoped; @@ -57,13 +58,14 @@ public Versioning getVersionInfo() return versioning; } - public EndpointViewListing getEndpointsListing( final String baseUri, final JaxRsUriFormatter uriFormatter ) - throws IndyWorkflowException + public EndpointViewListing getEndpointsListing( final String baseUri, final JaxRsUriFormatter uriFormatter, final String page ) + throws IndyWorkflowException { final List stores; try { - stores = new ArrayList<>( dataManager.getAllArtifactStores() ); + ListArtifactStoreDTO result = dataManager.getAllArtifactStores(page); + stores = new ArrayList<>( result.getItems() ); } catch ( final IndyDataException e ) { @@ -89,6 +91,12 @@ public EndpointViewListing getEndpointsListing( final String baseUri, final JaxR return new EndpointViewListing( points ); } + public EndpointViewListing getEndpointsListing( final String baseUri, final JaxRsUriFormatter uriFormatter ) + throws IndyWorkflowException + { + return getEndpointsListing( baseUri, uriFormatter, "" ); + } + public Map> getAllStoreKeys() throws IndyWorkflowException { diff --git a/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java index 0ad8534..769c996 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java @@ -82,6 +82,12 @@ Optional getArtifactStore( StoreKey key ) Set getAllArtifactStores() throws IndyDataException; + /** + * Return the full list of {@link ArtifactStore} instances of a given {@link StoreType} (hosted, remote, or group) available on the system. + */ + ListArtifactStoreDTO getAllArtifactStores(String page) + throws IndyDataException; + /** * Return the {@link ArtifactStore} instances as a {@link Stream}. */ diff --git a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java index 4032fc2..6717d58 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java @@ -143,10 +143,19 @@ public void clear( ChangeSummary summary ) // @WithSpan public Set getAllArtifactStores() { - Set dtxArtifactStoreSet = storeQuery.getAllArtifactStores(); + ListArtifactStoreDTO result = getAllArtifactStores(""); + return result.getItems(); + } + + @Override + // @WithSpan + public ListArtifactStoreDTO getAllArtifactStores(String page) + { + ListDtxArtifactStoreDTO result = storeQuery.getAllArtifactStores(page); + Set dtxArtifactStoreSet = result.getItems(); Set artifactStoreSet = new HashSet<>(); dtxArtifactStoreSet.forEach( dtxArtifactStore -> artifactStoreSet.add( toArtifactStore( dtxArtifactStore ) ) ); - return artifactStoreSet; + return new ListArtifactStoreDTO(artifactStoreSet, page, result.getNextPage()); } @Override diff --git a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java index cfb11d8..dc961ab 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java @@ -181,16 +181,40 @@ public Set getArtifactStoresByPkgAndType( String packageType, return result.getItems(); } - public Set getAllArtifactStores() + public ListDtxArtifactStoreDTO getAllArtifactStores( String page ) { - BoundStatement bound = preparedArtifactStoresQuery.bind(); + + if (!page.isEmpty()) { + PagingState currentPage = PagingState.fromString( page ); + bound.setPagingState( currentPage ); + } + ResultSet result = session.execute( bound ); + PagingState nextPage = result.getExecutionInfo().getPagingState(); + int remaining = result.getAvailableWithoutFetching(); + + if (!page.isEmpty()) { + remaining = Math.min( remaining, config.cassandraPageSize ); + } + Set dtxArtifactStoreSet = new HashSet<>(); - result.forEach( row -> dtxArtifactStoreSet.add( toDtxArtifactStore( row ) ) ); + for (Row row: result) + { + dtxArtifactStoreSet.add( toDtxArtifactStore( row ) ); + if (--remaining == 0) { + break; + } + } + + return new ListDtxArtifactStoreDTO( dtxArtifactStoreSet, page, nextPage.toString() ); + } - return dtxArtifactStoreSet; + public Set getAllArtifactStores() + { + ListDtxArtifactStoreDTO result = getAllArtifactStores(""); + return result.getItems(); } public Boolean isEmpty() diff --git a/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java b/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java index a769dc2..8796f7c 100644 --- a/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java +++ b/src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java @@ -121,6 +121,12 @@ public Set getAllArtifactStores() return new HashSet<>( stores.values() ); } + @Override + public ListArtifactStoreDTO getAllArtifactStores(String page) + { + return new ListArtifactStoreDTO(new HashSet<>( stores.values() ), page, ""); + } + @Override public Map getArtifactStoresByKey() { diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java index 2a94284..0397049 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java @@ -313,14 +313,15 @@ public Response getStoreEmpty() @Path( "/endpoints/{packageType}" ) @GET @Produces( APPLICATION_JSON ) - public Response getEndpoints( @PathParam( "packageType" ) final String pkgType, @Context final UriInfo uriInfo ) + public Response getEndpoints( @PathParam( "packageType" ) final String pkgType, @QueryParam( "page") String page, + @Context final UriInfo uriInfo ) { Response response; try { final String baseUri = uriInfo.getBaseUriBuilder().path( API_PREFIX ).build().toString(); - final EndpointViewListing listing = queryController.getEndpointsListing( pkgType, baseUri, uriFormatter ); + final EndpointViewListing listing = queryController.getEndpointsListing( pkgType, baseUri, uriFormatter, page ); response = responseHelper.formatOkResponseWithJsonEntity( listing ); logger.info( "\n\n\n\n\n\n{} Sent all-endpoints:\n\n{}\n\n\n\n\n\n\n", new Date(), listing ); diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java index d3a9e2d..a7f2356 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java @@ -15,6 +15,7 @@ */ package org.commonjava.indy.service.repository.jaxrs.version; +import jakarta.ws.rs.QueryParam; import org.commonjava.indy.service.repository.controller.StatsController; import org.commonjava.indy.service.repository.exception.IndyWorkflowException; import org.commonjava.indy.service.repository.jaxrs.ResponseHelper; @@ -114,14 +115,14 @@ public Response getPackageTypeNames() @Path( "/all-endpoints" ) @GET @Produces( APPLICATION_JSON ) - public Response getAllEndpoints( @Context final UriInfo uriInfo ) + public Response getAllEndpoints( @QueryParam( "page") String page, @Context final UriInfo uriInfo ) { Response response; try { final String baseUri = uriInfo.getBaseUriBuilder().path( Constants.API_PREFIX ).build().toString(); - final EndpointViewListing listing = statsController.getEndpointsListing( baseUri, uriFormatter ); + final EndpointViewListing listing = statsController.getEndpointsListing( baseUri, uriFormatter, page ); response = responseHelper.formatOkResponseWithJsonEntity( listing ); logger.info( "\n\n\n\n\n\n{} Sent all-endpoints:\n\n{}\n\n\n\n\n\n\n", new Date(), listing ); diff --git a/src/main/java/org/commonjava/indy/service/repository/model/dto/EndpointViewListing.java b/src/main/java/org/commonjava/indy/service/repository/model/dto/EndpointViewListing.java index 0817fe0..979be84 100644 --- a/src/main/java/org/commonjava/indy/service/repository/model/dto/EndpointViewListing.java +++ b/src/main/java/org/commonjava/indy/service/repository/model/dto/EndpointViewListing.java @@ -34,6 +34,10 @@ public class EndpointViewListing private List items; + private String currentPage; + + private String nextPage; + public EndpointViewListing() { } @@ -44,6 +48,14 @@ public EndpointViewListing( final List items ) this.items = items; } + public EndpointViewListing( final List items, final String currentPage, final String nextPage) + { + Collections.sort( items ); + this.items = items; + this.currentPage = currentPage; + this.nextPage = nextPage; + } + public List getItems() { return items; @@ -60,4 +72,23 @@ public void setItems( final List items ) this.items = items; } + public String getCurrentPage() + { + return currentPage; + } + + public void setCurrentPage( String currentPage ) + { + this.currentPage = currentPage; + } + + public String getNextPage() + { + return nextPage; + } + + public void setNextPage( String nextPage ) + { + this.nextPage = nextPage; + } }