HDDS-12785. Add an option to disable lifecycle configuration from OM

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/TestS3SDK.java

@@ -20,13 +20,15 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 
 class TestS3SDK extends OzoneS3SDKTests {
   @Override
   protected OzoneConfiguration createOzoneConfig() {
     OzoneConfiguration conf = super.createOzoneConfig();
     conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, false);
     conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1);
+    conf.setBoolean(OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, true);
     return conf;
   }
 }

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/TestS3SDKWithRatisStreaming.java

@@ -20,6 +20,7 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 
 class TestS3SDKWithRatisStreaming extends OzoneS3SDKTests {
   @Override
@@ -30,6 +31,7 @@ protected OzoneConfiguration createOzoneConfig() {
     conf.setBoolean(OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED, true);
     // Ensure that all writes use datastream
     conf.set(OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD, "0MB");
+    conf.setBoolean(OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, true);
     return conf;
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java

@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.ozone.client.rpc;
 
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED;
+
 import java.io.IOException;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -36,6 +38,7 @@ public static void init() throws Exception {
     conf.setBoolean(OzoneConfigKeys.OZONE_ACL_ENABLED, true);
     conf.set(OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS,
         OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE);
+    conf.setBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, true);
     startCluster(conf);
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java

@@ -24,6 +24,7 @@
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_ROOT;
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
 import static org.apache.hadoop.ozone.TestDataUtil.cleanupDeletedTable;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
 import static org.apache.ozone.test.GenericTestUtils.getTestStartTime;
@@ -125,6 +126,8 @@ public static void init() throws Exception {
     conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
         keyProviderUri);
+    conf.setBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, true);
+
     MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
         .setCertificateClient(certificateClientTest)
         .setSecretKeyClient(new SecretKeyTestClient());

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -364,7 +364,7 @@ public void start(OzoneConfiguration configuration) {
       multipartUploadCleanupService.start();
     }
 
-    if (keyLifecycleService == null) {
+    if (keyLifecycleService == null && ozoneManager.isLifecycleEnabled()) {
       long lifecycleServiceInterval = configuration.getTimeDuration(OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL,
           OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
       long lifecycleServiceTimeout = configuration.getTimeDuration(OZONE_KEY_LIFECYCLE_SERVICE_TIMEOUT,

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -433,6 +433,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   private static final int SHUTDOWN_HOOK_PRIORITY = 30;
   private final File omMetaDir;
   private boolean isAclEnabled;
+  private final boolean isLifecycleEnabled;
   private final boolean isSpnegoEnabled;
   private final SecurityConfig secConfig;
   private S3SecretManager s3SecretManager;
@@ -675,6 +676,9 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
       blockTokenMgr = createBlockTokenSecretManager();
     }
 
+    this.isLifecycleEnabled = conf.getBoolean(
+        OZONE_KEY_LIFECYCLE_SERVICE_ENABLED, OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT);
+
     // Enable S3 multi-tenancy if config keys are set
     this.isS3MultiTenancyEnabled =
         OMMultiTenantManager.checkAndEnableMultiTenancy(this, conf);
@@ -1095,6 +1099,24 @@ public void checkS3MultiTenancyEnabled() throws OMException {
         FEATURE_NOT_ENABLED);
   }
 
+  /**
+   * Returns true if lifecycle is enabled; false otherwise.
+   */
+  public boolean isLifecycleEnabled() {
+    return isLifecycleEnabled;
+  }
+
+  /**
+   * Throws OMException FEATURE_NOT_ENABLED if lifecycle is not enabled.
+   */
+  public void checkLifecycleEnabled() throws OMException {
+    if (!isLifecycleEnabled()) {
+      throw new OMException("OM Lifecycle feature is not enabled. Please "
+          + "set ozone.om.lifecycle.service.enabled to true and restart all OMs.",
+          FEATURE_NOT_ENABLED);
+    }
+  }
+
   /**
    * Return config value of {@link OzoneConfigKeys#OZONE_SECURITY_ENABLED_KEY}.
    */
@@ -3199,8 +3221,7 @@ public GetLifecycleServiceStatusResponse getLifecycleServiceStatus() {
     KeyLifecycleService keyLifecycleService = keyManager.getKeyLifecycleService();
     if (keyLifecycleService == null) {
       return GetLifecycleServiceStatusResponse.newBuilder()
-          .setIsEnabled(getConfiguration().getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED,
-              OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT))
+          .setIsEnabled(isLifecycleEnabled)
           .build();
     }
     return keyLifecycleService.status();

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java

@@ -346,10 +346,13 @@ public static OMClientRequest createClientRequest(OMRequest omRequest,
       bucketName = keyArgs.getBucketName();
       break;
     case SetLifecycleConfiguration:
+      ozoneManager.checkLifecycleEnabled();
       return new OMLifecycleConfigurationSetRequest(omRequest);
     case DeleteLifecycleConfiguration:
+      ozoneManager.checkLifecycleEnabled();
       return new OMLifecycleConfigurationDeleteRequest(omRequest);
     case SetLifecycleServiceStatus:
+      ozoneManager.checkLifecycleEnabled();
       return new OMLifecycleSetServiceStatusRequest(omRequest);
     default:
       throw new OMException("Unrecognized write command type request "

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java

@@ -24,8 +24,6 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT;
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
@@ -138,8 +136,7 @@ public KeyLifecycleService(OzoneManager ozoneManager,
         OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT);
     this.suspended = new AtomicBoolean(false);
     this.metrics = KeyLifecycleServiceMetrics.create();
-    this.isServiceEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_ENABLED,
-        OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT));
+    this.isServiceEnabled = new AtomicBoolean(ozoneManager.isLifecycleEnabled());
     this.moveToTrashEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED,
         OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT));
     this.inFlight = new ConcurrentHashMap();

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java

@@ -396,13 +396,15 @@ public OMResponse handleReadRequest(OMRequest request) {
         responseBuilder.setGetObjectTaggingResponse(getObjectTaggingResponse);
         break;
       case GetLifecycleConfiguration:
+        impl.checkLifecycleEnabled();
         GetLifecycleConfigurationResponse getLifecycleConfigurationResponse =
             infoLifecycleConfiguration(
                 request.getGetLifecycleConfigurationRequest());
         responseBuilder.setGetLifecycleConfigurationResponse(
             getLifecycleConfigurationResponse);
         break;
       case GetLifecycleServiceStatus:
+        impl.checkLifecycleEnabled();
         GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse =
             impl.getLifecycleServiceStatus();
         responseBuilder.setGetLifecycleServiceStatusResponse(

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDisableLifecycle.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FEATURE_NOT_ENABLED;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
+import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Try different configs against OzoneManager#checkLifecycleEnabled and verify its response.
+ */
+public class TestOMDisableLifecycle {
+
+  @Test
+  public void testLifecycleWhenDisabled() throws Exception {
+    final OzoneManager om = mock(OzoneManager.class);
+    doCallRealMethod().when(om).checkLifecycleEnabled();
+    final String volumeName = "vol1";
+    final String bucketName = "bucket1";
+
+    // Check that Lifecycle writes requests are blocked when not enabled
+    expectWriteRequestToFail(om, OMRequestTestUtils.setLifecycleConfigurationRequest(volumeName, bucketName));
+    expectWriteRequestToFail(om, OMRequestTestUtils.deleteLifecycleConfigurationRequest(volumeName, bucketName));
+    expectWriteRequestToFail(om, OMRequestTestUtils.setLifecycleServiceStatus());
+
+    // Check that Lifecycle read requests are blocked when not enabled
+
+    final OzoneManagerRequestHandler ozoneManagerRequestHandler =
+        new OzoneManagerRequestHandler(om);
+
+    expectReadRequestToFail(ozoneManagerRequestHandler,
+        OMRequestTestUtils.getLifecycleConfigurationRequest(volumeName, bucketName));
+    expectReadRequestToFail(ozoneManagerRequestHandler,
+        OMRequestTestUtils.getLifecycleServiceStatus());
+  }
+
+  /**
+   * Helper function for testLifecycleRPCWhenDisabled.
+   */
+  private void expectWriteRequestToFail(OzoneManager om, OMRequest request)
+      throws IOException {
+    OMException e =
+        assertThrows(OMException.class, () -> OzoneManagerRatisUtils.createClientRequest(request, om));
+    assertEquals(FEATURE_NOT_ENABLED, e.getResult());
+  }
+
+  /**
+   * Helper function for tesLifecycleRPCWhenDisabled.
+   */
+  private void expectReadRequestToFail(OzoneManagerRequestHandler handler,
+      OMRequest omRequest) {
+
+    // handleReadRequest does not throw
+    OMResponse omResponse = handler.handleReadRequest(omRequest);
+    assertFalse(omResponse.getSuccess());
+    assertEquals(Status.FEATURE_NOT_ENABLED, omResponse.getStatus());
+    assertTrue(omResponse.getMessage()
+        .startsWith("OM Lifecycle feature is not enabled."));
+  }
+
+}

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java

@@ -79,6 +79,9 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartCommitUploadPartRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartInfoInitiateRequest;
@@ -1492,6 +1495,78 @@ public static OMRequest deleteSnapshotRequest(String volumeName,
         .build();
   }
 
+  public static OMRequest setLifecycleConfigurationRequest(
+      String volumeName, String bucketName) {
+    LifecycleConfiguration.Builder lifecycleConfigurationBuilder = LifecycleConfiguration.newBuilder()
+        .addRules(LifecycleRule.newBuilder()
+            .setId(UUID.randomUUID().toString())
+            .setEnabled(true)
+            .addAction(LifecycleAction.getDefaultInstance())
+            .build())
+        .setVolume(volumeName)
+        .setBucket(bucketName)
+        .setCreationTime(Time.now())
+        .setBucketLayout(getDefaultBucketLayout().toProto());
+    OzoneManagerProtocolProtos.SetLifecycleConfigurationRequest setLifecycleConfigurationRequest =
+        OzoneManagerProtocolProtos.SetLifecycleConfigurationRequest.newBuilder()
+            .setLifecycleConfiguration(lifecycleConfigurationBuilder)
+            .build();
+    return OMRequest.newBuilder()
+        .setSetLifecycleConfigurationRequest(setLifecycleConfigurationRequest)
+        .setCmdType(Type.SetLifecycleConfiguration)
+        .setClientId(UUID.randomUUID().toString())
+        .build();
+  }
+
+  public static OMRequest deleteLifecycleConfigurationRequest(String volumeName, String bucketName) {
+    OzoneManagerProtocolProtos.DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest =
+        OzoneManagerProtocolProtos.DeleteLifecycleConfigurationRequest.newBuilder()
+            .setVolumeName(volumeName)
+            .setBucketName(bucketName)
+            .build();
+    return OMRequest.newBuilder()
+        .setDeleteLifecycleConfigurationRequest(deleteLifecycleConfigurationRequest)
+        .setCmdType(Type.DeleteLifecycleConfiguration)
+        .setClientId(UUID.randomUUID().toString())
+        .build();
+  }
+
+  public static OMRequest setLifecycleServiceStatus() {
+    OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest =
+        OzoneManagerProtocolProtos.SetLifecycleServiceStatusRequest.newBuilder()
+            .setSuspend(true)
+            .build();
+    return OMRequest.newBuilder()
+        .setSetLifecycleServiceStatusRequest(setLifecycleServiceStatusRequest)
+        .setCmdType(Type.SetLifecycleServiceStatus)
+        .setClientId(UUID.randomUUID().toString())
+        .build();
+  }
+
+  public static OMRequest getLifecycleConfigurationRequest(String volumeName, String bucketName) {
+    OzoneManagerProtocolProtos.GetLifecycleConfigurationRequest getLifecycleConfigurationRequest =
+        OzoneManagerProtocolProtos.GetLifecycleConfigurationRequest.newBuilder()
+            .setVolumeName(volumeName)
+            .setBucketName(bucketName)
+            .build();
+    return OMRequest.newBuilder()
+        .setGetLifecycleConfigurationRequest(getLifecycleConfigurationRequest)
+        .setCmdType(Type.GetLifecycleConfiguration)
+        .setClientId(UUID.randomUUID().toString())
+        .build();
+  }
+
+  public static OMRequest getLifecycleServiceStatus() {
+    OzoneManagerProtocolProtos.GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest =
+        OzoneManagerProtocolProtos.GetLifecycleServiceStatusRequest.newBuilder()
+            .build();
+    return OMRequest.newBuilder()
+        .setGetLifecycleServiceStatusRequest(getLifecycleServiceStatusRequest)
+        .setCmdType(Type.GetLifecycleServiceStatus)
+        .setClientId(UUID.randomUUID().toString())
+        .build();
+  }
+
   /**
    * Add the Key information to OzoneManager DB and cache.
    * @param omMetadataManager