HBASE-29568 - Allow for a configurable grace period when using Time Based Priority

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java

@@ -47,6 +47,9 @@ public class DataTieringManager {
     "hbase.regionserver.datatiering.enable";
   public static final boolean DEFAULT_GLOBAL_DATA_TIERING_ENABLED = false; // disabled by default
   public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type";
+  public static final String HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY =
+    "hbase.hstore.datatiering.grace.period.millis";
+  public static final long DEFAULT_DATATIERING_GRACE_PERIOD = 0;
   public static final String DATATIERING_HOT_DATA_AGE_KEY =
     "hbase.hstore.datatiering.hot.age.millis";
   public static final DataTieringType DEFAULT_DATATIERING = DataTieringType.NONE;
@@ -139,6 +142,9 @@ public boolean isHotData(BlockCacheKey key) throws DataTieringException {
    * @return {@code true} if the data is hot, {@code false} otherwise
    */
   public boolean isHotData(long maxTimestamp, Configuration conf) {
+    if (isWithinGracePeriod(maxTimestamp, conf)) {
+      return true;
+    }
     DataTieringType dataTieringType = getDataTieringType(conf);
 
     if (
@@ -170,8 +176,11 @@ public boolean isHotData(Path hFilePath) throws DataTieringException {
         throw new DataTieringException(
           "Store file corresponding to " + hFilePath + " doesn't exist");
       }
-      return hotDataValidator(dataTieringType.getInstance().getTimestamp(getHStoreFile(hFilePath)),
-        getDataTieringHotDataAge(configuration));
+      long maxTimestamp = dataTieringType.getInstance().getTimestamp(hStoreFile);
+      if (isWithinGracePeriod(maxTimestamp, configuration)) {
+        return true;
+      }
+      return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration));
     }
     // DataTieringType.NONE or other types are considered hot by default
     return true;
@@ -189,13 +198,21 @@ public boolean isHotData(Path hFilePath) throws DataTieringException {
   public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) {
     DataTieringType dataTieringType = getDataTieringType(configuration);
     if (hFileInfo != null && !dataTieringType.equals(DataTieringType.NONE)) {
-      return hotDataValidator(dataTieringType.getInstance().getTimestamp(hFileInfo),
-        getDataTieringHotDataAge(configuration));
+      long maxTimestamp = dataTieringType.getInstance().getTimestamp(hFileInfo);
+      if (isWithinGracePeriod(maxTimestamp, configuration)) {
+        return true;
+      }
+      return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration));
     }
     // DataTieringType.NONE or other types are considered hot by default
     return true;
   }
 
+  private boolean isWithinGracePeriod(long maxTimestamp, Configuration conf) {
+    long gracePeriod = getDataTieringGracePeriod(conf);
+    return gracePeriod > 0 && (getCurrentTimestamp() - maxTimestamp) < gracePeriod;
+  }
+
   private boolean hotDataValidator(long maxTimestamp, long hotDataAge) {
     long currentTimestamp = getCurrentTimestamp();
     long diff = currentTimestamp - maxTimestamp;
@@ -275,6 +292,11 @@ private long getDataTieringHotDataAge(Configuration conf) {
       conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE)));
   }
 
+  private long getDataTieringGracePeriod(Configuration conf) {
+    return Long.parseLong(conf.get(HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+      String.valueOf(DEFAULT_DATATIERING_GRACE_PERIOD)));
+  }
+
   /*
    * This API traverses through the list of online regions and returns a subset of these files-names
    * that are cold.

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java

@@ -231,6 +231,59 @@ public void testHotDataWithPath() throws IOException {
       new DataTieringException("Store file corresponding to " + hFilePath + " doesn't exist"));
   }
 
+  @Test
+  public void testGracePeriodMakesColdFileHot() throws IOException, DataTieringException {
+    initializeTestEnvironment();
+
+    long hotAge = 1 * DAY;      
+    long gracePeriod = 3 * DAY;  
+
+    long currentTime = System.currentTimeMillis();
+    long fileTimestamp = currentTime - (2 * DAY);
+
+    Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod);
+    HRegion region = createHRegion("tableGracePeriod", conf);
+    HStore hStore = createHStore(region, "cf1", conf);
+
+    HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(),
+      hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem());
+    file.initReader();
+
+
+    hStore.refreshStoreFiles(); 
+    region.stores.put(Bytes.toBytes("cf1"), hStore);
+    testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region);
+    Path hFilePath = file.getPath();
+    assertTrue("File should be hot due to grace period",
+      dataTieringManager.isHotData(hFilePath));
+  }
+
+  @Test
+  public void testFileIsColdWithoutGracePeriod() throws IOException, DataTieringException {
+    initializeTestEnvironment();
+
+    long hotAge = 1 * DAY;
+    long gracePeriod = 0;
+    long currentTime = System.currentTimeMillis();
+    long fileTimestamp = currentTime - (2 * DAY);
+
+    Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod);
+    HRegion region = createHRegion("tableNoGracePeriod", conf);
+    HStore hStore = createHStore(region, "cf1", conf);
+
+    HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(),
+      hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem());
+    file.initReader();
+
+    hStore.refreshStoreFiles();
+    region.stores.put(Bytes.toBytes("cf1"), hStore);
+    testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region);
+
+    Path hFilePath = file.getPath();
+    assertFalse("File should be cold without grace period",
+      dataTieringManager.isHotData(hFilePath));
+  }
+
   @Test
   public void testPrefetchWhenDataTieringEnabled() throws IOException {
     setPrefetchBlocksOnOpen();
@@ -770,6 +823,8 @@ private static HRegion createHRegion(String table, Configuration conf) throws IO
       .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
       .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
         conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+              .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+        conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY))
       .build();
     RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
 
@@ -797,6 +852,8 @@ private static HStore createHStore(HRegion region, String columnFamily, Configur
         .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
         .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
           conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+          .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+          conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY))
         .build();
 
     return new HStore(region, columnFamilyDescriptor, conf, false);
@@ -809,6 +866,13 @@ private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotData
     return conf;
   }
 
+  private static Configuration getConfWithGracePeriod(long hotDataAge, long gracePeriod) {
+    Configuration conf = getConfWithTimeRangeDataTieringEnabled(hotDataAge);
+    conf.set(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+      String.valueOf(gracePeriod));
+    return conf;
+  }
+
   static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp,
     HRegionFileSystem regionFs) throws IOException {
     String columnFamily = storeDir.getName();
@@ -868,4 +932,4 @@ private static String nextString() {
     }
     return rowKeyString;
   }
-}
+}