diff --git a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java index a856c720700..5384df6520e 100644 --- a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java +++ b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpRouter.java @@ -45,6 +45,7 @@ public abstract class HttpRouter { public static final String LAST_LOG_MARK = "/api/v1/bookie/last_log_mark"; public static final String LIST_DISK_FILE = "/api/v1/bookie/list_disk_file"; public static final String EXPAND_STORAGE = "/api/v1/bookie/expand_storage"; + public static final String GC = "/api/v1/bookie/gc"; // autorecovery public static final String RECOVERY_BOOKIE = "/api/v1/autorecovery/bookie"; public static final String LIST_UNDER_REPLICATED_LEDGER = "/api/v1/autorecovery/list_under_replicated_ledger"; @@ -73,6 +74,7 @@ public HttpRouter(AbstractHttpHandlerFactory handlerFactory) { this.endpointHandlers.put(LAST_LOG_MARK, handlerFactory.newHandler(HttpServer.ApiType.LAST_LOG_MARK)); this.endpointHandlers.put(LIST_DISK_FILE, handlerFactory.newHandler(HttpServer.ApiType.LIST_DISK_FILE)); this.endpointHandlers.put(EXPAND_STORAGE, handlerFactory.newHandler(HttpServer.ApiType.EXPAND_STORAGE)); + this.endpointHandlers.put(GC, handlerFactory.newHandler(HttpServer.ApiType.GC)); // autorecovery this.endpointHandlers.put(RECOVERY_BOOKIE, handlerFactory.newHandler(HttpServer.ApiType.RECOVERY_BOOKIE)); diff --git a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java index 30e4d05ae0c..5e9f5090274 100644 --- a/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java +++ b/bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http/HttpServer.java @@ -77,6 +77,7 @@ enum ApiType { LAST_LOG_MARK, LIST_DISK_FILE, EXPAND_STORAGE, + GC, // autorecovery RECOVERY_BOOKIE, LIST_UNDER_REPLICATED_LEDGER, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index ad752debdaf..c4d9414f1bc 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -382,6 +382,13 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin lastMinorCompactionTime = System.currentTimeMillis(); minorCompactionCounter.inc(); } + + if (force) { + if (forceGarbageCollection.compareAndSet(true, false)) { + LOG.info("{} Set forceGarbageCollection to false after force GC to make it forceGC-able again.", Thread + .currentThread().getName()); + } + } this.gcThreadRuntime.registerSuccessfulEvent( MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java index 8ab6517b489..d7d4977950d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java @@ -191,6 +191,11 @@ public void diskJustWritable(File disk) { }; } + @Override + public void forceGC() { + gcThread.enableForceGC(); + } + @Override public void start() { gcThread.start(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java index 34e32b9499d..ef64ff986c2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorage.java @@ -175,4 +175,10 @@ default LedgerStorage getUnderlyingLedgerStorage() { return this; } + /** + * Force trigger Garbage Collection. + */ + default void forceGC() { + return; + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java index e4b137ffbb5..dd7b373375a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SortedLedgerStorage.java @@ -327,4 +327,9 @@ public void flushEntriesLocationsIndex() throws IOException { public LedgerStorage getUnderlyingLedgerStorage() { return interleavedLedgerStorage; } + + @Override + public void forceGC() { + interleavedLedgerStorage.forceGC(); + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java index 8753363699f..831de531d91 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java @@ -348,4 +348,8 @@ public static void readLedgerIndexEntries(long ledgerId, ServerConfiguration ser } } + @Override + public void forceGC() { + ledgerStorageList.stream().forEach(SingleDirectoryDbLedgerStorage::forceGC); + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index e31b44e8147..27b4214e1b5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -263,6 +263,11 @@ public void start() { gcThread.start(); } + @Override + public void forceGC() { + gcThread.enableForceGC(); + } + @Override public void shutdown() throws InterruptedException { try { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java index 052b50ecf62..102b0eb57ef 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/BKHttpServiceProvider.java @@ -53,6 +53,7 @@ import org.apache.bookkeeper.server.http.service.ReadLedgerEntryService; import org.apache.bookkeeper.server.http.service.RecoveryBookieService; import org.apache.bookkeeper.server.http.service.TriggerAuditService; +import org.apache.bookkeeper.server.http.service.TriggerGCService; import org.apache.bookkeeper.server.http.service.WhoIsAuditorService; import org.apache.bookkeeper.stats.StatsProvider; import org.apache.bookkeeper.zookeeper.ZooKeeperClient; @@ -208,6 +209,8 @@ public HttpEndpointService provideHttpEndpointService(ApiType type) { return new ListDiskFilesService(configuration); case EXPAND_STORAGE: return new ExpandStorageService(configuration); + case GC: + return new TriggerGCService(configuration, bookieServer); // autorecovery case RECOVERY_BOOKIE: diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/TriggerGCService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/TriggerGCService.java new file mode 100644 index 00000000000..185965afad8 --- /dev/null +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/TriggerGCService.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.server.http.service; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.bookkeeper.common.util.JsonUtil; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.http.HttpServer; +import org.apache.bookkeeper.http.service.HttpEndpointService; +import org.apache.bookkeeper.http.service.HttpServiceRequest; +import org.apache.bookkeeper.http.service.HttpServiceResponse; +import org.apache.bookkeeper.proto.BookieServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * HttpEndpointService that handle force trigger GC requests. + * + *

The PUT method will force trigger GC on current bookie, and make GC run at backend. + */ +public class TriggerGCService implements HttpEndpointService { + + static final Logger LOG = LoggerFactory.getLogger(TriggerGCService.class); + + protected ServerConfiguration conf; + protected BookieServer bookieServer; + + public TriggerGCService(ServerConfiguration conf, BookieServer bookieServer) { + checkNotNull(conf); + checkNotNull(bookieServer); + this.conf = conf; + this.bookieServer = bookieServer; + } + + @Override + public HttpServiceResponse handle(HttpServiceRequest request) throws Exception { + HttpServiceResponse response = new HttpServiceResponse(); + // PUT + if (HttpServer.Method.PUT == request.getMethod()) { + bookieServer.getBookie().getLedgerStorage().forceGC(); + + String output = "Triggered GC on BookieServer: " + bookieServer.toString(); + String jsonResponse = JsonUtil.toJson(output); + LOG.debug("output body:" + jsonResponse); + response.setBody(jsonResponse); + response.setCode(HttpServer.StatusCode.OK); + return response; + } else { + response.setCode(HttpServer.StatusCode.NOT_FOUND); + response.setBody("Not found method. Should be PUT method to trigger GC."); + return response; + } + } +} diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java index 079e08f422f..3f587c75336 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java @@ -741,4 +741,42 @@ public void testDecommissionService() throws Exception { stopAuditorElector(); } + /** + * Create ledgers, then test Delete Ledger service. + */ + @Test + public void testTriggerGCService() throws Exception { + baseConf.setMetadataServiceUri(zkUtil.getMetadataServiceUri()); + BookKeeper.DigestType digestType = BookKeeper.DigestType.CRC32; + int numLedgers = 4; + int numMsgs = 100; + LedgerHandle[] lh = new LedgerHandle[numLedgers]; + // create ledgers + for (int i = 0; i < numLedgers; i++) { + lh[i] = bkc.createLedger(digestType, "".getBytes()); + } + String content = "Apache BookKeeper is cool!"; + // add entries + for (int i = 0; i < numMsgs; i++) { + for (int j = 0; j < numLedgers; j++) { + lh[j].addEntry(content.getBytes()); + } + } + // close ledgers + for (int i = 0; i < numLedgers; i++) { + lh[i].close(); + } + HttpEndpointService triggerGCService = bkHttpServiceProvider + .provideHttpEndpointService(HttpServer.ApiType.GC); + + //1, GET, should return NOT_FOUND + HttpServiceRequest request1 = new HttpServiceRequest(null, HttpServer.Method.GET, null); + HttpServiceResponse response1 = triggerGCService.handle(request1); + assertEquals(HttpServer.StatusCode.NOT_FOUND.getValue(), response1.getStatusCode()); + + //2, PUT, should return OK + HttpServiceRequest request2 = new HttpServiceRequest(null, HttpServer.Method.PUT, null); + HttpServiceResponse response2 = triggerGCService.handle(request2); + assertEquals(HttpServer.StatusCode.OK.getValue(), response2.getStatusCode()); + } } diff --git a/site/docs/latest/admin/http.md b/site/docs/latest/admin/http.md index dc647449621..7c1abea49f2 100644 --- a/site/docs/latest/admin/http.md +++ b/site/docs/latest/admin/http.md @@ -271,6 +271,17 @@ Currently all the HTTP endpoints could be divided into these 4 components: |403 | Permission denied | |404 | Not found | +### Endpoint: /api/v1/bookie/gc +1. Method: PUT + * Description: trigger gc for this bookie. + * Response: + + | Code | Description | + |:-------|:------------| + |200 | Successful operation | + |403 | Permission denied | + |404 | Not found | + ## Auto recovery ### Endpoint: /api/v1/autorecovery/bookie/