diff --git a/.github/workflows/merge-build.yml b/.github/workflows/merge-build.yml index bf3cae1..45307e5 100644 --- a/.github/workflows/merge-build.yml +++ b/.github/workflows/merge-build.yml @@ -1,5 +1,5 @@ # -# Copyright (C) 2023 Red Hat, Inc. (https://github.com/Commonjava/indy-ui-service) +# Copyright (C) 2022-2023 Red Hat, Inc. (https://github.com/Commonjava/indy-repository-service) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index fd9daaa..0f5b864 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -1,5 +1,5 @@ # -# Copyright (C) 2023 Red Hat, Inc. (https://github.com/Commonjava/indy-ui-service) +# Copyright (C) 2022-2023 Red Hat, Inc. (https://github.com/Commonjava/indy-repository-service) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pom.xml b/pom.xml index 5ebe5d0..76e277e 100644 --- a/pom.xml +++ b/pom.xml @@ -208,10 +208,10 @@ jackson-dataformat-yaml 2.14.0 - - - - + + org.infinispan + infinispan-component-annotations + org.infinispan infinispan-core-jakarta @@ -265,6 +265,16 @@ commons-lang3 3.12.0 + + org.apache.commons + commons-compress + 1.26.0 + + + commons-io + commons-io + 2.15.1 + com.datastax.cassandra cassandra-driver-core @@ -280,11 +290,6 @@ httpclient 4.5.13 - - commons-io - commons-io - 2.11.0 - org.codehaus.plexus plexus-interpolation diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/MaintenanceController.java b/src/main/java/org/commonjava/indy/service/repository/controller/MaintenanceController.java index dacc15c..488acb6 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/MaintenanceController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/MaintenanceController.java @@ -17,6 +17,9 @@ import com.fasterxml.jackson.core.json.JsonReadFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.commonjava.event.common.EventMetadata; import org.commonjava.indy.service.repository.audit.ChangeSummary; @@ -32,17 +35,18 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.charset.Charset; import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; +import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import static java.util.Map.of; @@ -96,35 +100,40 @@ public File getRepoBundle() public Map> importRepoBundle( final InputStream zipStream ) throws IOException { + File tempRepoZip = createTempFile(); + logger.info( "Saving repo file to {}", tempRepoZip.getPath() ); + try (zipStream) + { + try (OutputStream out = new FileOutputStream( tempRepoZip )) + { + IOUtils.copy( zipStream, out ); + } + } + final List skipped = new ArrayList<>(); final List failed = new ArrayList<>(); final Map payload = new HashMap<>(); - logger.info( "Start extracting repos definitions from bundle!" ); - try (ZipInputStream zip = new ZipInputStream( zipStream )) + + try (ZipFile zipFile = ZipFile.builder().setFile( tempRepoZip ).get()) { - if ( zip.available() > 0 ) + Enumeration entries = zipFile.getEntries(); + while ( entries.hasMoreElements() ) { - ZipEntry entry = zip.getNextEntry(); - while ( entry != null ) + ZipArchiveEntry entry = entries.nextElement(); + if ( !entry.isDirectory() ) { - if ( !entry.isDirectory() ) + try (InputStream in = zipFile.getInputStream( entry )) { - logger.debug( "Processing {}", entry.getName() ); - byte[] buffer = new byte[2048]; - final StringBuilder builder = new StringBuilder(); - while ( zip.read( buffer ) > 0 ) - { - builder.append( new String( buffer, Charset.defaultCharset() ) ); - buffer = new byte[2048]; - } - - payload.put( entry.getName(), builder.toString().trim() ); - + payload.put( entry.getName(), IOUtils.toString( in, Charset.defaultCharset() ) ); } - entry = zip.getNextEntry(); } } } + finally + { + FileUtils.deleteQuietly( tempRepoZip ); + } + logger.info( "Repos definitions extraction from bundle finished.\n\n" ); logger.info( "Start importing repos definitions to data store." ); for ( Map.Entry entry : payload.entrySet() ) diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java b/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java index 0c37136..90b7999 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java @@ -26,6 +26,9 @@ import org.commonjava.indy.service.repository.model.RemoteRepository; import org.commonjava.indy.service.repository.model.StoreKey; import org.commonjava.indy.service.repository.model.StoreType; +import org.commonjava.indy.service.repository.model.dto.EndpointView; +import org.commonjava.indy.service.repository.model.dto.EndpointViewListing; +import org.commonjava.indy.service.repository.util.JaxRsUriFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +37,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -82,7 +87,7 @@ public List getAllArtifactStores( final String packageType, final // when packageType and type are all unique value, use storeManager.getArtifactStoresByPkgAndType to improve performance stores = storeManager.getArtifactStoresByPkgAndType( packageType, typeList.get( 0 ) ); } - else if ( !isValidPackageType( packageType ) && typeList.size() == 0 ) + else if ( !isValidPackageType( packageType ) && typeList.isEmpty() ) { stores = storeManager.getAllArtifactStores(); } @@ -251,6 +256,76 @@ public Boolean isStoreDataEmpty() return storeManager.isEmpty(); } + public EndpointViewListing getEndpointsListing( final String pkgType, final String baseUri, + final JaxRsUriFormatter uriFormatter ) + throws IndyWorkflowException + { + List stores; + try + { + stores = new ArrayList<>( storeManager.getAllArtifactStores() ); + if ( StringUtils.isNotBlank( pkgType ) && !"all".equals( pkgType ) && isValidPackageType( pkgType ) ) + { + stores = stores.stream() + .filter( s -> pkgType.equals( s.getPackageType() ) ) + .collect( Collectors.toList() ); + } + } + catch ( final IndyDataException e ) + { + throw new IndyWorkflowException( INTERNAL_SERVER_ERROR.getStatusCode(), + "Failed to retrieve all endpoints: {}", e, e.getMessage() ); + } + + final List points = new ArrayList<>(); + for ( final ArtifactStore store : stores ) + { + final StoreKey key = store.getKey(); + final String resourceUri = uriFormatter.formatAbsolutePathTo( baseUri, "content", key.getPackageType(), + key.getType().singularEndpointName(), + key.getName() ); + + final EndpointView point = new EndpointView( store, resourceUri ); + if ( !points.contains( point ) ) + { + points.add( point ); + } + } + + return new EndpointViewListing( points ); + } + + public Map> getStoreKeysByPackageType( final String pkgType ) + throws IndyWorkflowException + { + final List stores; + + try + { + final Map> result = new HashMap<>(); + stores = new ArrayList<>( storeManager.getAllArtifactStores() ); + List items; + if ( StringUtils.isNotBlank( pkgType ) && !"all".equals( pkgType ) && isValidPackageType( pkgType ) ) + { + items = stores.stream() + .filter( s -> pkgType.equals( s.getPackageType() ) ) + .map( s -> s.getKey().toString() ) + .collect( Collectors.toList() ); + } + else + { + items = stores.stream().map( s -> s.getKey().toString() ).collect( Collectors.toList() ); + } + result.put( "items", items ); + return result; + } + catch ( final IndyDataException e ) + { + throw new IndyWorkflowException( INTERNAL_SERVER_ERROR.getStatusCode(), + "Failed to retrieve all store keys: {}", e, e.getMessage() ); + } + } + private StoreKey validateStoreKey( final String storeKey ) throws IndyWorkflowException { diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java b/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java index 6900188..3aae35e 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/StatsController.java @@ -23,10 +23,11 @@ import org.commonjava.indy.service.repository.model.dto.EndpointView; import org.commonjava.indy.service.repository.model.dto.EndpointViewListing; import org.commonjava.indy.service.repository.model.version.Versioning; -import org.commonjava.indy.service.repository.util.JaxRsUriFormatter; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import org.commonjava.indy.service.repository.util.JaxRsUriFormatter; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryMaintenanceResources.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryMaintenanceResources.java index e840a95..648cc53 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryMaintenanceResources.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryMaintenanceResources.java @@ -23,7 +23,6 @@ import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.eclipse.microprofile.openapi.annotations.tags.Tag; -import org.jboss.resteasy.spi.HttpRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,10 +36,11 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.WebApplicationException; -import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.MediaType; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Map; @@ -99,13 +99,13 @@ public Response getRepoBundle() @APIResponse( responseCode = "200", description = "All repository definitions which are imported successfully." ) @POST @Path( "/import" ) - @Consumes( MEDIATYPE_APPLICATION_ZIP ) + @Consumes( MediaType.MULTIPART_FORM_DATA ) @Produces( APPLICATION_JSON ) - public Response importRepoBundle( @Context final HttpRequest request ) + public Response importRepoBundle( InputStream input ) { try { - Map> results = maintController.importRepoBundle( request.getInputStream() ); + Map> results = maintController.importRepoBundle( input ); return ok( results ).build(); } catch ( IOException e ) diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java index 11157e8..2a94284 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java @@ -19,8 +19,10 @@ import org.commonjava.indy.service.repository.controller.QueryController; import org.commonjava.indy.service.repository.exception.IndyWorkflowException; import org.commonjava.indy.service.repository.model.ArtifactStore; +import org.commonjava.indy.service.repository.model.dto.EndpointViewListing; import org.commonjava.indy.service.repository.model.dto.SimpleBooleanResultDTO; import org.commonjava.indy.service.repository.model.dto.StoreListingDTO; +import org.commonjava.indy.service.repository.util.JaxRsUriFormatter; import org.commonjava.indy.service.repository.util.UrlUtils; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; @@ -41,12 +43,17 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.UriInfo; +import java.util.Date; import java.util.List; +import java.util.Map; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static jakarta.ws.rs.core.Response.ok; +import static org.commonjava.indy.service.repository.util.Constants.API_PREFIX; import static org.eclipse.microprofile.openapi.annotations.enums.ParameterIn.QUERY; @Tag( name = "Store Querying APIs", description = "Resource for querying artifact store definitions" ) @@ -63,6 +70,9 @@ public class RepositoryQueryResources @Inject QueryController queryController; + @Inject + JaxRsUriFormatter uriFormatter; + public RepositoryQueryResources() { logger.info( "\n\n\n\nStarted Store Querying resources\n\n\n\n" ); @@ -295,6 +305,62 @@ public Response getStoreEmpty() return responseHelper.formatOkResponseWithJsonEntity( dto ); } + @Operation( + summary = "Retrieve a listing of the artifact stores available on the system. This is especially useful for setting up a network of Indy instances that reference one another" ) + @APIResponse( responseCode = "200", + content = @Content( schema = @Schema( implementation = EndpointViewListing.class ) ), + description = "The artifact store listing" ) + @Path( "/endpoints/{packageType}" ) + @GET + @Produces( APPLICATION_JSON ) + public Response getEndpoints( @PathParam( "packageType" ) final String pkgType, @Context final UriInfo uriInfo ) + { + Response response; + try + { + final String baseUri = uriInfo.getBaseUriBuilder().path( API_PREFIX ).build().toString(); + + final EndpointViewListing listing = queryController.getEndpointsListing( pkgType, baseUri, uriFormatter ); + response = responseHelper.formatOkResponseWithJsonEntity( listing ); + + logger.info( "\n\n\n\n\n\n{} Sent all-endpoints:\n\n{}\n\n\n\n\n\n\n", new Date(), listing ); + } + catch ( final IndyWorkflowException e ) + { + logger.error( String.format( "Failed to retrieve endpoint listing: %s", responseHelper.formatEntity( e ) ), + e ); + response = responseHelper.formatResponse( e ); + } + return response; + } + + @Operation( summary = "Retrieve a listing of the artifact stores keys available on the system." ) + @APIResponse( responseCode = "200", content = @Content( schema = @Schema( implementation = Map.class ) ), + description = "The artifact store keys listing" ) + @Path( "/storekeys/{packageType}" ) + @GET + @Produces( APPLICATION_JSON ) + public Response getStoreKeys( @PathParam( "packageType" ) final String pkgType, @Context final UriInfo uriInfo ) + { + Response response; + try + { + + Map> result = queryController.getStoreKeysByPackageType( pkgType ); + + response = responseHelper.formatOkResponseWithJsonEntity( result ); + + logger.debug( "\n\n\n\n\n\n{} Sent store keys:\n\n{}\n\n\n\n\n\n\n", new Date(), result ); + } + catch ( final IndyWorkflowException e ) + { + logger.error( String.format( "Failed to retrieve store keys listing by type %s: %s", pkgType, + responseHelper.formatEntity( e ) ), e ); + response = responseHelper.formatResponse( e ); + } + return response; + } + @SuppressWarnings( { "unchecked", "rawtypes" } ) private Response generateStoreListingResponse( ArtifactStoreListSupplier supplier ) { diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java index f19388b..d3a9e2d 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/version/StatsHandler.java @@ -20,6 +20,7 @@ import org.commonjava.indy.service.repository.jaxrs.ResponseHelper; import org.commonjava.indy.service.repository.model.dto.EndpointViewListing; import org.commonjava.indy.service.repository.model.version.Versioning; +import org.commonjava.indy.service.repository.util.Constants; import org.commonjava.indy.service.repository.util.JaxRsUriFormatter; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; @@ -49,7 +50,6 @@ import static jakarta.ws.rs.core.Response.ok; import static org.commonjava.indy.service.repository.model.PackageTypes.getPackageTypeDescriptorMap; import static org.commonjava.indy.service.repository.model.PackageTypes.getPackageTypes; -import static org.commonjava.indy.service.repository.util.Constants.API_PREFIX; @Tag( name = "Generic Infrastructure Queries (UI Support)", description = "Various read-only operations for retrieving information about the system." ) @@ -119,7 +119,7 @@ public Response getAllEndpoints( @Context final UriInfo uriInfo ) Response response; try { - final String baseUri = uriInfo.getBaseUriBuilder().path( API_PREFIX ).build().toString(); + final String baseUri = uriInfo.getBaseUriBuilder().path( Constants.API_PREFIX ).build().toString(); final EndpointViewListing listing = statsController.getEndpointsListing( baseUri, uriFormatter ); response = responseHelper.formatOkResponseWithJsonEntity( listing ); diff --git a/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java b/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java index 5e26ee0..f00e607 100644 --- a/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java +++ b/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java @@ -56,4 +56,44 @@ public void testGetAllByDefaultPackageTypes() } + @Test + public void testGetAllEndpoints() + { + given().get( normalize( BASE_QUERY_PATH, "endpoints/all" ) ) + .then() + .statusCode( OK.getStatusCode() ) + .body( "size()", is( 1 ) ) + .body( "items.size()", greaterThan( 1 ) ); + } + + @Test + public void testGetMavenEndpoints() + { + given().get( normalize( BASE_QUERY_PATH, "endpoints/maven" ) ) + .then() + .statusCode( OK.getStatusCode() ) + .body( "size()", is( 1 ) ) + .body( "items.size()", greaterThan( 1 ) ); + } + + @Test + public void testGetAllStoreKeys() + { + given().get( normalize( BASE_QUERY_PATH, "storekeys/all" ) ) + .then() + .statusCode( OK.getStatusCode() ) + .body( "size()", is( 1 ) ) + .body( "items.size()", greaterThan( 1 ) ); + } + + @Test + public void testGetMavenStoreKeys() + { + given().get( normalize( BASE_QUERY_PATH, "endpoints/maven" ) ) + .then() + .statusCode( OK.getStatusCode() ) + .body( "size()", is( 1 ) ) + .body( "items.size()", greaterThan( 1 ) ); + } + }