diff --git a/common/src/main/java/org/commonjava/storage/pathmapped/model/ProxySite.java b/common/src/main/java/org/commonjava/storage/pathmapped/model/ProxySite.java new file mode 100644 index 0000000..ebc8a94 --- /dev/null +++ b/common/src/main/java/org/commonjava/storage/pathmapped/model/ProxySite.java @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2019 Red Hat, Inc. (nos-devel@redhat.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.storage.pathmapped.model; + +public interface ProxySite +{ + String getSite(); +} diff --git a/common/src/main/java/org/commonjava/storage/pathmapped/spi/PathDB.java b/common/src/main/java/org/commonjava/storage/pathmapped/spi/PathDB.java index ac16bca..7b45600 100644 --- a/common/src/main/java/org/commonjava/storage/pathmapped/spi/PathDB.java +++ b/common/src/main/java/org/commonjava/storage/pathmapped/spi/PathDB.java @@ -104,4 +104,16 @@ enum FileType { * @param fileType file, dir, or all */ void traverse( String fileSystem, String path, Consumer consumer, int limit, FileType fileType ); + + Set getProxySitesCache(); + + boolean isProxySite( String site ); + + List getProxySiteList(); + + void saveProxySite( String site ); + + void deleteProxySite( String site ); + + void deleteAllProxySite(); } diff --git a/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/CassandraPathDB.java b/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/CassandraPathDB.java index 650ccc2..3c77391 100644 --- a/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/CassandraPathDB.java +++ b/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/CassandraPathDB.java @@ -26,13 +26,21 @@ import com.datastax.driver.mapping.Mapper; import com.datastax.driver.mapping.MappingManager; import com.datastax.driver.mapping.Result; - import com.google.common.collect.TreeTraverser; -import org.commonjava.storage.pathmapped.model.*; -import org.commonjava.storage.pathmapped.pathdb.datastax.model.*; +import org.commonjava.storage.pathmapped.config.PathMappedStorageConfig; +import org.commonjava.storage.pathmapped.model.FileChecksum; +import org.commonjava.storage.pathmapped.model.Filesystem; +import org.commonjava.storage.pathmapped.model.PathMap; +import org.commonjava.storage.pathmapped.model.Reclaim; +import org.commonjava.storage.pathmapped.model.ReverseMap; +import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxFileChecksum; +import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxFilesystem; +import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxPathMap; +import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxProxySite; +import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxReclaim; +import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxReverseMap; import org.commonjava.storage.pathmapped.pathdb.datastax.util.AsyncJobExecutor; import org.commonjava.storage.pathmapped.pathdb.datastax.util.CassandraPathDBUtils; -import org.commonjava.storage.pathmapped.config.PathMappedStorageConfig; import org.commonjava.storage.pathmapped.spi.PathDB; import org.commonjava.storage.pathmapped.spi.PathDBAdmin; import org.commonjava.storage.pathmapped.util.PathMapUtils; @@ -55,11 +63,13 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static com.datastax.driver.core.ConsistencyLevel.*; +import static com.datastax.driver.core.ConsistencyLevel.QUORUM; import static java.util.Collections.emptySet; import static org.apache.commons.lang.StringUtils.isNotBlank; import static org.commonjava.storage.pathmapped.pathdb.datastax.util.CassandraPathDBUtils.getHoursInDay; -import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.*; +import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.all; +import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.dir; +import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.file; import static org.commonjava.storage.pathmapped.util.PathMapUtils.ROOT_DIR; public class CassandraPathDB @@ -83,6 +93,8 @@ public class CassandraPathDB private Mapper filesystemMapper; + protected Set proxySitesCache = new HashSet<>(); + private PathMappedStorageConfig config; private String keyspace; @@ -91,9 +103,12 @@ public class CassandraPathDB private long reconnectDelay = 60000; - private PreparedStatement preparedExistQuery, preparedListQuery, preparedListCheckEmpty, preparedContainingQuery, preparedExistFileQuery, - preparedUpdateExpiration, preparedReverseMapIncrement, preparedReverseMapReduction, - preparedFilesystemIncrement, preparedFilesystemReduction, preparedFilesystemList; + private Mapper proxysiteMapper; + + private PreparedStatement preparedExistQuery, preparedListQuery, preparedListCheckEmpty, preparedContainingQuery, + preparedExistFileQuery, preparedUpdateExpiration, preparedReverseMapIncrement, preparedReverseMapReduction, + preparedFilesystemIncrement, preparedFilesystemReduction, preparedFilesystemList, preparedProxySiteQuery, + preparedProxySiteList, preparedProxySiteTruncate; @Deprecated public CassandraPathDB( PathMappedStorageConfig config, Session session, String keyspace ) @@ -158,6 +173,7 @@ private void prepare( Session session, String keyspace, int replicationFactor ) session.execute( CassandraPathDBUtils.getSchemaCreateTableReclaim( keyspace ) ); session.execute( CassandraPathDBUtils.getSchemaCreateTableFileChecksum( keyspace ) ); session.execute( CassandraPathDBUtils.getSchemaCreateTableFilesystem( keyspace ) ); + session.execute( CassandraPathDBUtils.getSchemaCreateTableProxySites( keyspace ) ); MappingManager manager = new MappingManager( session ); @@ -166,6 +182,7 @@ private void prepare( Session session, String keyspace, int replicationFactor ) reclaimMapper = manager.mapper( DtxReclaim.class, keyspace ); fileChecksumMapper = manager.mapper( DtxFileChecksum.class, keyspace ); filesystemMapper = manager.mapper( DtxFilesystem.class, keyspace ); + proxysiteMapper = manager.mapper( DtxProxySite.class, keyspace ); preparedExistFileQuery = session.prepare( "SELECT count(*) FROM " + keyspace + ".pathmap WHERE filesystem=? and parentpath=? and filename=?;" ); @@ -185,22 +202,28 @@ private void prepare( Session session, String keyspace, int replicationFactor ) "WHERE filesystem=? and parentpath=? and filename=?;" ); preparedContainingQuery = session.prepare( "SELECT filesystem FROM " + keyspace - + ".pathmap WHERE filesystem IN ? and parentpath=? and filename=?;" ); + + ".pathmap WHERE filesystem IN ? and parentpath=? and filename=?;" ); preparedReverseMapIncrement = - session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths + ? WHERE fileid=?;" ); + session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths + ? WHERE fileid=?;" ); preparedReverseMapReduction = - session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths - ? WHERE fileid=?;" ); + session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths - ? WHERE fileid=?;" ); preparedReverseMapReduction.setConsistencyLevel( QUORUM ); - preparedFilesystemIncrement = - session.prepare("UPDATE " + keyspace + ".filesystem SET filecount=filecount+?, size=size+? WHERE filesystem=?;" ); + preparedFilesystemIncrement = session.prepare( + "UPDATE " + keyspace + ".filesystem SET filecount=filecount+?, size=size+? WHERE filesystem=?;" ); - preparedFilesystemReduction = - session.prepare("UPDATE " + keyspace + ".filesystem SET filecount=filecount-?, size=size-? WHERE filesystem=?;" ); + preparedFilesystemReduction = session.prepare( + "UPDATE " + keyspace + ".filesystem SET filecount=filecount-?, size=size-? WHERE filesystem=?;" ); - preparedFilesystemList = session.prepare("SELECT * FROM " + keyspace + ".filesystem;" ); + preparedFilesystemList = session.prepare( "SELECT * FROM " + keyspace + ".filesystem;" ); + + preparedProxySiteQuery = session.prepare( "SELECT * FROM " + keyspace + ".proxysites WHERE site=?;" ); + + preparedProxySiteList = session.prepare( "SELECT * FROM " + keyspace + ".proxysites;" ); + + preparedProxySiteTruncate = session.prepare( "TRUNCATE " + keyspace + ".proxysites;" ); asyncJobExecutor = new AsyncJobExecutor( config ); } @@ -997,4 +1020,48 @@ private ResultSet executeSession ( BoundStatement bind ) } return trackingRecord; } + + @Override + public Set getProxySitesCache() + { + return proxySitesCache; + } + + @Override + public boolean isProxySite( String site ) + { + BoundStatement bound = preparedProxySiteQuery.bind( site ); + ResultSet result = executeSession( bound ); + return notNull( result ); + } + + @Override + public List getProxySiteList() + { + ResultSet result = executeSession( preparedProxySiteList.bind() ); + return proxysiteMapper.map( result ).all().stream().map( DtxProxySite::getSite ).collect( Collectors.toList() ); + } + + @Override + public void saveProxySite( String site ) + { + DtxProxySite proxySite = new DtxProxySite( site ); + logger.debug( "ProxySite, {}", site ); + proxysiteMapper.save( proxySite ); + } + + @Override + public void deleteProxySite( String site ) + { + DtxProxySite proxySite = new DtxProxySite( site ); + logger.debug( "Delete proxySite, {}", site ); + proxysiteMapper.delete( proxySite ); + } + + @Override + public void deleteAllProxySite() + { + BoundStatement bound = preparedProxySiteTruncate.bind(); + executeSession( bound ); + } } diff --git a/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/model/DtxProxySite.java b/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/model/DtxProxySite.java new file mode 100644 index 0000000..58f60d0 --- /dev/null +++ b/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/model/DtxProxySite.java @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2019 Red Hat, Inc. (nos-devel@redhat.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.storage.pathmapped.pathdb.datastax.model; + +import com.datastax.driver.mapping.annotations.PartitionKey; +import com.datastax.driver.mapping.annotations.Table; +import org.commonjava.storage.pathmapped.model.ProxySite; + +import java.util.Objects; + +@Table( name = "proxysites", readConsistency = "QUORUM", writeConsistency = "QUORUM" ) +public class DtxProxySite + implements ProxySite +{ + @PartitionKey + private String site; + + public DtxProxySite() + { + } + + public DtxProxySite( String site ) + { + this.site = site; + } + + @Override + public String getSite() + { + return site; + } + + public void setSite( String site ) + { + this.site = site; + } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( o == null || getClass() != o.getClass() ) + { + return false; + } + DtxProxySite that = (DtxProxySite) o; + return site.equals( that.site ); + } + + @Override + public int hashCode() + { + return Objects.hash( site ); + } + + @Override + public String toString() + { + return "DtxProxySite{" + "site='" + site + '}'; + } +} diff --git a/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/util/CassandraPathDBUtils.java b/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/util/CassandraPathDBUtils.java index f0b2ce8..f7ef18c 100644 --- a/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/util/CassandraPathDBUtils.java +++ b/pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/util/CassandraPathDBUtils.java @@ -96,12 +96,14 @@ public static String getSchemaCreateTableReclaim( String keyspace ) public static String getSchemaCreateTableFileChecksum( String keyspace ) { - return "CREATE TABLE IF NOT EXISTS " + keyspace + ".filechecksum (" - + "checksum varchar," - + "fileid varchar," - + "storage varchar," - + "PRIMARY KEY (checksum)" - + ");"; + return "CREATE TABLE IF NOT EXISTS " + keyspace + ".filechecksum (" + "checksum varchar," + "fileid varchar," + + "storage varchar," + "PRIMARY KEY (checksum)" + ");"; + } + + public static String getSchemaCreateTableProxySites( String keyspace ) + { + return "CREATE TABLE IF NOT EXISTS " + keyspace + ".proxysites (" + "site varchar," + "PRIMARY KEY (site)" + + ");"; } // Since Date.getHours is deprecated, we use this to replace it. diff --git a/pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/JPAPathDB.java b/pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/JPAPathDB.java index a2b603d..b28992d 100644 --- a/pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/JPAPathDB.java +++ b/pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/JPAPathDB.java @@ -15,24 +15,24 @@ */ package org.commonjava.storage.pathmapped.pathdb.jpa; +import jakarta.persistence.EntityManager; +import jakarta.persistence.EntityManagerFactory; +import jakarta.persistence.Persistence; +import jakarta.persistence.Query; import org.commonjava.storage.pathmapped.model.FileChecksum; +import org.commonjava.storage.pathmapped.model.PathMap; +import org.commonjava.storage.pathmapped.model.Reclaim; +import org.commonjava.storage.pathmapped.model.ReverseMap; import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaPathKey; import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaPathMap; import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReclaim; import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReverseKey; import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReverseMap; -import org.commonjava.storage.pathmapped.model.PathMap; -import org.commonjava.storage.pathmapped.model.Reclaim; -import org.commonjava.storage.pathmapped.model.ReverseMap; import org.commonjava.storage.pathmapped.spi.PathDB; import org.commonjava.storage.pathmapped.util.PathMapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.persistence.EntityManager; -import jakarta.persistence.EntityManagerFactory; -import jakarta.persistence.Persistence; -import jakarta.persistence.Query; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -437,4 +437,36 @@ private JpaPathKey getPathKey( String fileSystem, String path ) return new JpaPathKey( fileSystem, parentPath, filename ); } + @Override + public Set getProxySitesCache() + { + return null; + } + + @Override + public boolean isProxySite( String site ) + { + return false; + } + + @Override + public List getProxySiteList() + { + return null; + } + + @Override + public void saveProxySite( String site ) + { + } + + @Override + public void deleteProxySite( String site ) + { + } + + @Override + public void deleteAllProxySite() + { + } } diff --git a/pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/model/JpaProxySite.java b/pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/model/JpaProxySite.java new file mode 100644 index 0000000..b8dec1c --- /dev/null +++ b/pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/model/JpaProxySite.java @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2019 Red Hat, Inc. (nos-devel@redhat.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.storage.pathmapped.pathdb.jpa.model; + +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import org.commonjava.storage.pathmapped.model.ProxySite; + +import java.util.Objects; + +@Entity +@Table( name = "proxysites" ) +public class JpaProxySite + implements ProxySite +{ + @Id + private String site; + + public JpaProxySite() + { + } + + public JpaProxySite( String site ) + { + this.site = site; + } + + @Override + public String getSite() + { + return site; + } + + public void setSite( String site ) + { + this.site = site; + } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( o == null || getClass() != o.getClass() ) + { + return false; + } + JpaProxySite that = (JpaProxySite) o; + return site.equals( that.site ); + } + + @Override + public int hashCode() + { + return Objects.hash( site ); + } + + @Override + public String toString() + { + return "JpaProxySite{" + "site='" + site + '}'; + } +} diff --git a/storage/src/main/java/org/commonjava/storage/pathmapped/core/PathMappedFileManager.java b/storage/src/main/java/org/commonjava/storage/pathmapped/core/PathMappedFileManager.java index 63f99cb..c41c049 100644 --- a/storage/src/main/java/org/commonjava/storage/pathmapped/core/PathMappedFileManager.java +++ b/storage/src/main/java/org/commonjava/storage/pathmapped/core/PathMappedFileManager.java @@ -32,7 +32,12 @@ import java.io.InputStream; import java.io.OutputStream; import java.security.NoSuchAlgorithmException; -import java.util.*; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -476,11 +481,42 @@ public void purgeFilesystem( Filesystem filesystem ) } @Override - public void close() throws IOException + public void close() + throws IOException { if ( pathDB instanceof Closeable ) { ( (Closeable) pathDB ).close(); } } + + public Set getProxySitesCache() + { + return pathDB.getProxySitesCache(); + } + + public boolean isProxySite( String site ) + { + return pathDB.isProxySite( site ); + } + + public List getProxySiteList() + { + return pathDB.getProxySiteList(); + } + + public void saveProxySite( String site ) + { + pathDB.saveProxySite( site ); + } + + public void deleteProxySite( String site ) + { + pathDB.deleteProxySite( site ); + } + + public void deleteAllProxySite() + { + pathDB.deleteAllProxySite(); + } } diff --git a/storage/src/main/java/org/commonjava/storage/pathmapped/metrics/MeasuredPathDB.java b/storage/src/main/java/org/commonjava/storage/pathmapped/metrics/MeasuredPathDB.java index b3f0adb..37e8cc7 100644 --- a/storage/src/main/java/org/commonjava/storage/pathmapped/metrics/MeasuredPathDB.java +++ b/storage/src/main/java/org/commonjava/storage/pathmapped/metrics/MeasuredPathDB.java @@ -221,4 +221,39 @@ protected boolean isMetricEnabled( String metricName ) return true; } + @Override + public Set getProxySitesCache() + { + return measure( () -> decorated.getProxySitesCache(), "getProxySitesCache" ); + } + + @Override + public boolean isProxySite( String site ) + { + return measure( () -> decorated.isProxySite( site ), "isProxySite" ); + } + + @Override + public List getProxySiteList() + { + return measure( () -> decorated.getProxySiteList(), "getProxySiteList" ); + } + + @Override + public void saveProxySite( String site ) + { + measure( () -> decorated.saveProxySite( site ), "saveProxySite" ); + } + + @Override + public void deleteProxySite( String site ) + { + measure( () -> decorated.deleteProxySite( site ), "deleteProxySite" ); + } + + @Override + public void deleteAllProxySite() + { + measure( () -> decorated.deleteAllProxySite(), "deleteAllProxySite" ); + } }