Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pagination #146

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Pagination #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3166f12
add pagination query support
hjmodi Apr 1, 2024
a44620b
add new DTOs to support pagination
hjmodi Apr 4, 2024
89e0a00
add pagination for getAllArtifactStores
hjmodi Apr 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions src/main/java/org/commonjava/indy/service/repository/controller/AdminController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.commonjava.indy.service.repository.model.RemoteRepository;
import org.commonjava.indy.service.repository.model.StoreKey;
import org.commonjava.indy.service.repository.model.StoreType;
import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO;
import org.eclipse.microprofile.context.ManagedExecutor;
import org.eclipse.microprofile.rest.client.inject.RestClient;
import org.slf4j.Logger;
Expand All @@ -39,6 +40,7 @@
import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
Expand Down Expand Up @@ -113,19 +115,21 @@ public List<ArtifactStore> getAllOfType( final StoreType type )
}
}

public List<ArtifactStore> getAllOfType( final String packageType, final StoreType type )
throws IndyWorkflowException

public ListArtifactStoreDTO getAllOfType( final String packageType, final StoreType type, String page )
throws IndyWorkflowException
{
try
{
ArtifactStoreQuery<ArtifactStore> query = storeManager.query().storeTypes( type );
if ( !ALL_PACKAGE_TYPES.equals( packageType ) )
{
return new ArrayList<>( storeManager.getArtifactStoresByPkgAndType( packageType, type ) );
return storeManager.getArtifactStoresByPkgAndType( packageType, type, page );
}
else
{
return query.getAllByDefaultPackageTypes();
List<ArtifactStore> result = query.getAllByDefaultPackageTypes();
return new ListArtifactStoreDTO( new HashSet<>(result) );
}
}
catch ( final IndyDataException e )
Expand All @@ -135,6 +139,14 @@ public List<ArtifactStore> getAllOfType( final String packageType, final StoreTy
}
}


public List<ArtifactStore> getAllOfType( final String packageType, final StoreType type )
throws IndyWorkflowException
{
ListArtifactStoreDTO result = getAllOfType( packageType, type, "" );
return new ArrayList<>(result.getItems());
}

public List<ArtifactStore> getAllStores()
throws IndyWorkflowException
{
Expand Down
24 changes: 21 additions & 3 deletions src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.commonjava.indy.service.repository.model.StoreType;
import org.commonjava.indy.service.repository.model.dto.EndpointView;
import org.commonjava.indy.service.repository.model.dto.EndpointViewListing;
import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO;
import org.commonjava.indy.service.repository.util.JaxRsUriFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -257,13 +258,16 @@ public Boolean isStoreDataEmpty()
}

public EndpointViewListing getEndpointsListing( final String pkgType, final String baseUri,
final JaxRsUriFormatter uriFormatter )
final JaxRsUriFormatter uriFormatter, final String page )
throws IndyWorkflowException
{
List<ArtifactStore> stores;
String nextPage = "";
try
{
stores = new ArrayList<>( storeManager.getAllArtifactStores() );
ListArtifactStoreDTO result = storeManager.getAllArtifactStores(page);
nextPage = result.getNextPage();
stores = new ArrayList<>( result.getItems() );
if ( StringUtils.isNotBlank( pkgType ) && !"all".equals( pkgType ) && isValidPackageType( pkgType ) )
{
stores = stores.stream()
Expand Down Expand Up @@ -292,7 +296,21 @@ public EndpointViewListing getEndpointsListing( final String pkgType, final Stri
}
}

return new EndpointViewListing( points );
EndpointViewListing result = new EndpointViewListing( points );
result.setCurrentPage( page );

if (!nextPage.isEmpty()) {
result.setNextPage( nextPage );
}

return result;
}

public EndpointViewListing getEndpointsListing( final String pkgType, final String baseUri,
final JaxRsUriFormatter uriFormatter )
throws IndyWorkflowException
{
return getEndpointsListing( pkgType, baseUri, uriFormatter, "" );
}

public Map<String, List<String>> getStoreKeysByPackageType( final String pkgType )
Expand Down
14 changes: 11 additions & 3 deletions src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.commonjava.indy.service.repository.model.StoreKey;
import org.commonjava.indy.service.repository.model.dto.EndpointView;
import org.commonjava.indy.service.repository.model.dto.EndpointViewListing;
import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO;
import org.commonjava.indy.service.repository.model.version.Versioning;

import jakarta.enterprise.context.ApplicationScoped;
Expand Down Expand Up @@ -57,13 +58,14 @@ public Versioning getVersionInfo()
return versioning;
}

public EndpointViewListing getEndpointsListing( final String baseUri, final JaxRsUriFormatter uriFormatter )
throws IndyWorkflowException
public EndpointViewListing getEndpointsListing( final String baseUri, final JaxRsUriFormatter uriFormatter, final String page )
throws IndyWorkflowException
{
final List<ArtifactStore> stores;
try
{
stores = new ArrayList<>( dataManager.getAllArtifactStores() );
ListArtifactStoreDTO result = dataManager.getAllArtifactStores(page);
stores = new ArrayList<>( result.getItems() );
}
catch ( final IndyDataException e )
{
Expand All @@ -89,6 +91,12 @@ public EndpointViewListing getEndpointsListing( final String baseUri, final JaxR
return new EndpointViewListing( points );
}

public EndpointViewListing getEndpointsListing( final String baseUri, final JaxRsUriFormatter uriFormatter )
throws IndyWorkflowException
{
return getEndpointsListing( baseUri, uriFormatter, "" );
}

public Map<String, List<String>> getAllStoreKeys()
throws IndyWorkflowException
{
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/org/commonjava/indy/service/repository/data/StoreDataManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.commonjava.indy.service.repository.model.StoreType;
import org.commonjava.indy.service.repository.audit.ChangeSummary;
import org.commonjava.indy.service.repository.exception.IndyDataException;
import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO;

import java.util.Collection;
import java.util.Map;
Expand Down Expand Up @@ -81,6 +82,12 @@ Optional<ArtifactStore> getArtifactStore( StoreKey key )
Set<ArtifactStore> getAllArtifactStores()
throws IndyDataException;

/**
* Return the full list of {@link ArtifactStore} instances of a given {@link StoreType} (hosted, remote, or group) available on the system.
*/
ListArtifactStoreDTO getAllArtifactStores(String page)
throws IndyDataException;

/**
* Return the {@link ArtifactStore} instances as a {@link Stream}.
*/
Expand Down Expand Up @@ -165,4 +172,6 @@ Set<Group> affectedBy( Collection<StoreKey> keys )

Set<ArtifactStore> getArtifactStoresByPkgAndType( String packageType, StoreType storeType );

ListArtifactStoreDTO getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page );

}
4 changes: 4 additions & 0 deletions ...in/java/org/commonjava/indy/service/repository/data/cassandra/CassandraConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ public class CassandraConfiguration
@ConfigProperty( name = "cassandra.enabled", defaultValue = "false" )
Boolean enabled;

@Inject
@ConfigProperty( name = "cassandra.page_size", defaultValue = "5000" )
int cassandraPageSize;

@Inject
@ConfigProperty( name = "cassandra.host", defaultValue = "localhost" )
String cassandraHost;
Expand Down
30 changes: 24 additions & 6 deletions ...java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreDataManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import org.commonjava.indy.service.repository.model.RemoteRepository;
import org.commonjava.indy.service.repository.model.StoreKey;
import org.commonjava.indy.service.repository.model.StoreType;
import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO;
import org.commonjava.indy.service.repository.model.dto.ListDtxArtifactStoreDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -141,10 +143,19 @@ public void clear( ChangeSummary summary )
// @WithSpan
public Set<ArtifactStore> getAllArtifactStores()
{
Set<DtxArtifactStore> dtxArtifactStoreSet = storeQuery.getAllArtifactStores();
ListArtifactStoreDTO result = getAllArtifactStores("");
return result.getItems();
}

@Override
// @WithSpan
public ListArtifactStoreDTO getAllArtifactStores(String page)
{
ListDtxArtifactStoreDTO result = storeQuery.getAllArtifactStores(page);
Set<DtxArtifactStore> dtxArtifactStoreSet = result.getItems();
Set<ArtifactStore> artifactStoreSet = new HashSet<>();
dtxArtifactStoreSet.forEach( dtxArtifactStore -> artifactStoreSet.add( toArtifactStore( dtxArtifactStore ) ) );
return artifactStoreSet;
return new ListArtifactStoreDTO(artifactStoreSet, page, result.getNextPage());
}

@Override
Expand Down Expand Up @@ -184,15 +195,22 @@ public Set<StoreKey> getStoreKeysByPkgAndType( final String pkg, final StoreType
}

@Override
public Set<ArtifactStore> getArtifactStoresByPkgAndType( final String pkg, final StoreType type )
public ListArtifactStoreDTO getArtifactStoresByPkgAndType( final String pkg, final StoreType type, String page )
{

logger.trace( "Get stores: {}/{}", pkg, type );

Set<DtxArtifactStore> dtxArtifactStoreSet = storeQuery.getArtifactStoresByPkgAndType( pkg, type );
ListDtxArtifactStoreDTO result = storeQuery.getArtifactStoresByPkgAndType( pkg, type, page );
Set<DtxArtifactStore> dtxArtifactStoreSet = result.getItems();
Set<ArtifactStore> storeSet = new HashSet<>();
dtxArtifactStoreSet.forEach( dtxArtifactStore -> storeSet.add( toArtifactStore( dtxArtifactStore ) ) );
return storeSet;
return new ListArtifactStoreDTO( storeSet, result.getCurrentPage(), result.getNextPage());
}

@Override
public Set<ArtifactStore> getArtifactStoresByPkgAndType( final String pkg, final StoreType type )
{
ListArtifactStoreDTO result = getArtifactStoresByPkgAndType( pkg, type, "" );
return result.getItems();
}

@Override
Expand Down
66 changes: 58 additions & 8 deletions src/main/java/org/commonjava/indy/service/repository/data/cassandra/CassandraStoreQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package org.commonjava.indy.service.repository.data.cassandra;

import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.PagingState;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
Expand All @@ -24,6 +25,7 @@
import com.datastax.driver.mapping.MappingManager;
import org.commonjava.indy.service.repository.model.StoreKey;
import org.commonjava.indy.service.repository.model.StoreType;
import org.commonjava.indy.service.repository.model.dto.ListDtxArtifactStoreDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -142,29 +144,77 @@ public DtxArtifactStore getArtifactStore( String packageType, StoreType type, St
return toDtxArtifactStore( result.one() );
}

public Set<DtxArtifactStore> getArtifactStoresByPkgAndType( String packageType, StoreType type )
public ListDtxArtifactStoreDTO getArtifactStoresByPkgAndType( String packageType, StoreType type, String page )
{

BoundStatement bound =
preparedArtifactStoresQueryByKeys.bind( CassandraStoreUtil.getTypeKey( packageType, type.name() ) );
preparedArtifactStoresQueryByKeys.bind( CassandraStoreUtil.getTypeKey( packageType, type.name() ) );

if (!page.isEmpty()) {
PagingState currentPage = PagingState.fromString( page );
bound.setPagingState( currentPage );
}

ResultSet result = session.execute( bound );

PagingState nextPage = result.getExecutionInfo().getPagingState();
int remaining = result.getAvailableWithoutFetching();

if (!page.isEmpty()) {
remaining = Math.min( remaining, config.cassandraPageSize );
}

Set<DtxArtifactStore> dtxArtifactStoreSet = new HashSet<>();
result.forEach( row -> dtxArtifactStoreSet.add( toDtxArtifactStore( row ) ) );
for (Row row: result)
{
dtxArtifactStoreSet.add( toDtxArtifactStore( row ) );
if (--remaining == 0) {
break;
}
}

return dtxArtifactStoreSet;
return new ListDtxArtifactStoreDTO( dtxArtifactStoreSet, page, nextPage.toString() );
}

public Set<DtxArtifactStore> getAllArtifactStores()
public Set<DtxArtifactStore> getArtifactStoresByPkgAndType( String packageType, StoreType type )
{
ListDtxArtifactStoreDTO result = getArtifactStoresByPkgAndType( packageType, type, "" );
return result.getItems();
}

public ListDtxArtifactStoreDTO getAllArtifactStores( String page )
{
BoundStatement bound = preparedArtifactStoresQuery.bind();

if (!page.isEmpty()) {
PagingState currentPage = PagingState.fromString( page );
bound.setPagingState( currentPage );
}

ResultSet result = session.execute( bound );

PagingState nextPage = result.getExecutionInfo().getPagingState();
int remaining = result.getAvailableWithoutFetching();

if (!page.isEmpty()) {
remaining = Math.min( remaining, config.cassandraPageSize );
}

Set<DtxArtifactStore> dtxArtifactStoreSet = new HashSet<>();
result.forEach( row -> dtxArtifactStoreSet.add( toDtxArtifactStore( row ) ) );
for (Row row: result)
{
dtxArtifactStoreSet.add( toDtxArtifactStore( row ) );
if (--remaining == 0) {
break;
}
}

return new ListDtxArtifactStoreDTO( dtxArtifactStoreSet, page, nextPage.toString() );
}

return dtxArtifactStoreSet;
public Set<DtxArtifactStore> getAllArtifactStores()
{
ListDtxArtifactStoreDTO result = getAllArtifactStores("");
return result.getItems();
}

public Boolean isEmpty()
Expand Down
25 changes: 20 additions & 5 deletions src/main/java/org/commonjava/indy/service/repository/data/mem/MemoryStoreDataManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.commonjava.indy.service.repository.model.ArtifactStore;
import org.commonjava.indy.service.repository.model.StoreKey;
import org.commonjava.indy.service.repository.model.StoreType;
import org.commonjava.indy.service.repository.model.dto.ListArtifactStoreDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -120,6 +121,12 @@ public Set<ArtifactStore> getAllArtifactStores()
return new HashSet<>( stores.values() );
}

@Override
public ListArtifactStoreDTO getAllArtifactStores(String page)
{
return new ListArtifactStoreDTO(new HashSet<>( stores.values() ), page, "");
}

@Override
public Map<StoreKey, ArtifactStore> getArtifactStoresByKey()
{
Expand Down Expand Up @@ -155,14 +162,22 @@ public Stream<StoreKey> streamArtifactStoreKeys()
return stores.keySet().stream();
}

@Override
public ListArtifactStoreDTO getArtifactStoresByPkgAndType( String packageType, StoreType storeType, String page )
{
Set<ArtifactStore> resultStores = stores.values()
.stream()
.filter( item -> packageType.equals( item.getPackageType() ) && storeType.equals(
item.getType() ) )
.collect( Collectors.toSet() );
return new ListArtifactStoreDTO( resultStores);
}

@Override
public Set<ArtifactStore> getArtifactStoresByPkgAndType( String packageType, StoreType storeType )
{
return stores.values()
.stream()
.filter( item -> packageType.equals( item.getPackageType() ) && storeType.equals(
item.getType() ) )
.collect( Collectors.toSet() );
ListArtifactStoreDTO result = getArtifactStoresByPkgAndType( packageType, storeType, "" );
return result.getItems();
}

@Override
Expand Down
Loading
Loading