feat: Derive pg_partman partition width from ttl unit and partition-divisor setting

CONTRIBUTORS.md

@@ -11,3 +11,4 @@ This file recognizes the people who have made contributions to SQRL.
 | Etienne Chauchot    | echauchot          |
 | Tianchen Wu         | wutianchen         |
 | Brad Bailey         | bradjbailey        |
+| Wellington Mafra    | WellMafra          |

documentation/docs/sqrl-language.md

@@ -329,7 +329,7 @@ Hints live in a `/*+ ... */` comment placed **immediately before** the definitio
 | **query_by_any**            | `query_by_any(col, ...)`                                                   | table          | generate interface with *optional* filter arguments for all listed columns                                                                                                         |
 | **no_query**                | `no_query`                                                                 | table          | hide from interface                                                                                                                                                                |
 | **insert**                  | `insert(type)`                                                             | table          | controls the way how mutations will be written to their target sink. `type` ∈ `SINGLE` (default), `BATCH`, `TRANSACTION`                                                           |
-| **ttl**                     | `ttl(duration)`                                                            | table          | specifies how long the records for this table are retained in the underlying data system before it can be discarded. Expects a duration string like `5 week`. Disabled by default. |
+| **ttl**                     | `ttl(duration [, partition interval])`                                     | table          | specifies how long the records for this table are retained in the underlying data system before it can be discarded. Expects a duration string like `5 week`. An optional second argument sets an explicit partition interval (e.g. `ttl(14 days, 1 day)`) for range-partitioned tables; when omitted, the interval is derived from the duration. Disabled by default. |
 | **cache**                   | `cache(duration)`                                                          | table          | how long the results retrieved from this table can be cached on the server before they are refreshed. Expects a duration string like `10 seconds`. Disabled by default.            |
 | **filtered_distinct_order** | flag                                                                       | DISTINCT table | eliminate updates on order column only before dedup                                                                                                                                |
 | **engine**                  | `enigne(engine_id)`                                                        | table          | pin execution engine (`process`, `database`, `flink`, ...)                                                                                                                         |

sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/AbstractJdbcStatementFactory.java

@@ -160,13 +160,9 @@ public JdbcStatement createTable(JdbcEngineCreateTable createTable) {
     // DagPlanner validates that partition keys are part of primary key
 
     PartitionType partitionType = getPartitionType(createTable, partitionKeys);
-    Duration ttl =
-        createTable
-            .tableAnalysis()
-            .getHints()
-            .getHint(TtlHint.class)
-            .flatMap(TtlHint::getTtl)
-            .orElse(Duration.ZERO);
+    var ttlHint = createTable.tableAnalysis().getHints().getHint(TtlHint.class);
+    Duration ttl = ttlHint.flatMap(TtlHint::getTtl).orElse(Duration.ZERO);
+    String partitionInterval = ttlHint.flatMap(TtlHint::getPartitionInterval).orElse(null);
 
     return new CreateTableJdbcStatement(
         tableName,
@@ -180,6 +176,7 @@ public JdbcStatement createTable(JdbcEngineCreateTable createTable) {
         partitionType,
         partitionType == PartitionType.NONE ? 0 : 1,
         ttl,
+        partitionInterval,
         createTable,
         createTableDdlFactory);
   }

sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/CreateTableJdbcStatement.java

@@ -60,6 +60,9 @@ public class CreateTableJdbcStatement implements JdbcStatement {
   /** The time-to-live of records in this table - ZERO to disable */
   Duration ttl;
 
+  /** Explicit partition interval for range-partitioned tables - null to derive from ttl */
+  String partitionInterval;
+
   /** The engine table - nullable and not set when deserialized */
   @JsonIgnore JdbcEngineCreateTable engineTable;
 
@@ -75,7 +78,8 @@ public CreateTableJdbcStatement(
       @JsonProperty("partitionKey") List<String> partitionKey,
       @JsonProperty("partitionType") PartitionType partitionType,
       @JsonProperty("numPartitions") int numPartitions,
-      @JsonProperty("ttl") Duration ttl) {
+      @JsonProperty("ttl") Duration ttl,
+      @JsonProperty("partitionInterval") String partitionInterval) {
     this.name = name;
     this.description = description;
     this.fields = fields;
@@ -84,6 +88,7 @@ public CreateTableJdbcStatement(
     this.partitionType = partitionType;
     this.numPartitions = numPartitions;
     this.ttl = ttl;
+    this.partitionInterval = partitionInterval;
     this.engineTable = null;
     this.ddlFactory = null;
   }

sqrl-planner/src/main/java/com/datasqrl/function/translation/postgres/extensions/PgPartmanExtension.java

@@ -60,6 +60,10 @@ private void appendTableDdl(StringBuilder sb, CreateTableJdbcStatement createTab
     // 'public."Readings"' never matches relname 'Readings' and create_parent fails.
     var parentTable = "public." + createTable.getName();
     var ttl = createTable.getTtl();
+    var interval =
+        createTable.getPartitionInterval() != null
+            ? createTable.getPartitionInterval()
+            : deriveInterval(ttl);
 
     // p_type was removed in pg_partman 5.x; the default (range) is what we need
     sb.append(
@@ -72,7 +76,7 @@ private void appendTableDdl(StringBuilder sb, CreateTableJdbcStatement createTab
         );
 
         """
-            .formatted(parentTable, createTable.getPartitionKey().get(0), deriveInterval(ttl)));
+            .formatted(parentTable, createTable.getPartitionKey().get(0), interval));
 
     sb.append(
         """

sqrl-planner/src/main/java/com/datasqrl/planner/hint/TtlHint.java

@@ -22,29 +22,58 @@
 import com.google.auto.service.AutoService;
 import java.time.Duration;
 import java.util.Optional;
+import java.util.regex.Pattern;
 import org.apache.flink.util.TimeUtils;
 
 public class TtlHint extends PlannerHint {
 
   public static final String HINT_NAME = "ttl";
 
+  /** Subset of postgres interval syntax accepted as an explicit partition interval */
+  private static final Pattern PARTITION_INTERVAL_PATTERN =
+      Pattern.compile(
+          "^\\d+\\s*(second|minute|hour|day|week|month|year)s?$", Pattern.CASE_INSENSITIVE);
+
   private final Duration ttl;
+  private final String partitionInterval;
 
-  protected TtlHint(ParsedObject<SqrlHint> source, Duration ttlDuration) {
+  protected TtlHint(ParsedObject<SqrlHint> source, Duration ttlDuration, String partitionInterval) {
     super(source, Type.DAG);
     this.ttl = ttlDuration;
+    this.partitionInterval = partitionInterval;
   }
 
   public Optional<Duration> getTtl() {
     return Optional.ofNullable(ttl);
   }
 
+  public Optional<String> getPartitionInterval() {
+    return Optional.ofNullable(partitionInterval);
+  }
+
   @AutoService(Factory.class)
   public static class TtlHintFactory implements Factory {
 
     @Override
     public PlannerHint create(ParsedObject<SqrlHint> source) {
-      return new TtlHint(source, parseDuration(source));
+      var arguments = source.get().options();
+      if (arguments == null || arguments.isEmpty()) {
+        return new TtlHint(source, null, null);
+      }
+      if (arguments.size() > 2 || arguments.get(0) == null) {
+        throw new StatementParserException(
+            ErrorLabel.GENERIC,
+            source.getFileLocation(),
+            "%s hint supports a duration argument and an optional partition interval argument"
+                + " (e.g. `14 days, 1 day`).",
+            source.get().name());
+      }
+      var ttl = parseDurationArgument(source, arguments.get(0));
+      String partitionInterval = null;
+      if (arguments.size() == 2) {
+        partitionInterval = parsePartitionInterval(source, arguments.get(1));
+      }
+      return new TtlHint(source, ttl, partitionInterval);
     }
 
     @Override
@@ -65,16 +94,33 @@ public static Duration parseDuration(ParsedObject<SqrlHint> source) {
           "%s hint only supports one duration argument (e.g. `2 days`).",
           source.get().name());
     }
+    return parseDurationArgument(source, arguments.get(0));
+  }
+
+  private static Duration parseDurationArgument(ParsedObject<SqrlHint> source, String argument) {
     try {
-      return TimeUtils.parseDuration(arguments.get(0));
+      return TimeUtils.parseDuration(argument);
     } catch (Exception e) {
       throw new StatementParserException(
           ErrorLabel.GENERIC,
           source.getFileLocation(),
           "%s hint does not have a valid duration argument: %s. Expected `2 days` or `10 s`. "
               + e.getMessage(),
           source.get().name(),
-          arguments.get(0));
+          argument);
+    }
+  }
+
+  private static String parsePartitionInterval(ParsedObject<SqrlHint> source, String argument) {
+    if (argument == null || !PARTITION_INTERVAL_PATTERN.matcher(argument).matches()) {
+      throw new StatementParserException(
+          ErrorLabel.GENERIC,
+          source.getFileLocation(),
+          "%s hint does not have a valid partition interval argument: %s. Expected an interval"
+              + " like `1 day` or `1 month`.",
+          source.get().name(),
+          argument);
     }
+    return argument;
   }
 }

sqrl-planner/src/test/java/com/datasqrl/engine/database/relational/AbstractJdbcStatementFactoryTest.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright © 2021 DataSQRL (contact@datasqrl.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datasqrl.engine.database.relational;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+import com.datasqrl.error.ErrorCollector;
+import com.datasqrl.error.ErrorLocation.FileLocation;
+import com.datasqrl.planner.analyzer.TableAnalysis;
+import com.datasqrl.planner.hint.PlannerHints;
+import com.datasqrl.planner.parser.ParsedObject;
+import com.datasqrl.planner.parser.SqrlComments;
+import com.datasqrl.planner.parser.SqrlHint;
+import com.datasqrl.planner.tables.FlinkTableBuilder;
+import com.datasqrl.planner.util.Documented;
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import org.apache.calcite.rel.type.RelDataType;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class AbstractJdbcStatementFactoryTest {
+
+  @Mock private TableAnalysis tableAnalysis;
+
+  @Mock private RelDataType datatype;
+
+  @Mock private FlinkTableBuilder tableBuilder;
+
+  private final PostgresStatementFactory factory = new PostgresStatementFactory();
+
+  private CreateTableJdbcStatement createTable(PlannerHints hints) {
+    when(tableAnalysis.getHints()).thenReturn(hints);
+    when(tableAnalysis.getDocumentation()).thenReturn(Documented.EMPTY);
+    when(datatype.getFieldList()).thenReturn(List.of());
+    when(tableBuilder.getPrimaryKey()).thenReturn(Optional.empty());
+
+    var createTable = new JdbcEngineCreateTable("my_table", tableBuilder, datatype, tableAnalysis);
+    return (CreateTableJdbcStatement) factory.createTable(createTable);
+  }
+
+  private static PlannerHints hints(SqrlHint... sqrlHints) {
+    var parsedHints =
+        List.of(sqrlHints).stream()
+            .map(hint -> new ParsedObject<>(hint, FileLocation.START))
+            .toList();
+    var comments = new SqrlComments(List.of(), parsedHints);
+    return PlannerHints.from(comments, Optional.empty(), ErrorCollector.root());
+  }
+
+  @Test
+  void givenTtlHintWithPartitionInterval_whenCreateTable_thenBothPopulated() {
+    var stmt = createTable(hints(new SqrlHint("ttl", List.of("14 days", "1 day"))));
+
+    assertThat(stmt.getTtl()).isEqualTo(Duration.ofDays(14));
+    assertThat(stmt.getPartitionInterval()).isEqualTo("1 day");
+  }
+
+  @Test
+  void givenTtlHintWithoutPartitionInterval_whenCreateTable_thenIntervalNull() {
+    var stmt = createTable(hints(new SqrlHint("ttl", List.of("30 days"))));
+
+    assertThat(stmt.getTtl()).isEqualTo(Duration.ofDays(30));
+    assertThat(stmt.getPartitionInterval()).isNull();
+  }
+
+  @Test
+  void givenNoTtlHint_whenCreateTable_thenZeroTtlAndNullInterval() {
+    var stmt = createTable(PlannerHints.EMPTY);
+
+    assertThat(stmt.getTtl()).isEqualTo(Duration.ZERO);
+    assertThat(stmt.getPartitionInterval()).isNull();
+  }
+}

sqrl-planner/src/test/java/com/datasqrl/engine/database/relational/ddl/PostgresCreateTableDdlFactoryTest.java

@@ -37,7 +37,8 @@ private static CreateTableJdbcStatement stmt(
         partitionKey,
         partitionType,
         1,
-        ttl);
+        ttl,
+        null);
   }
 
   @Test

sqrl-planner/src/test/java/com/datasqrl/function/translation/postgres/extensions/PgPartmanExtensionTest.java

@@ -29,8 +29,25 @@ class PgPartmanExtensionTest {
 
   private static CreateTableJdbcStatement table(
       String name, PartitionType partitionType, List<String> partitionKey, Duration ttl) {
+    return table(name, partitionType, partitionKey, ttl, null);
+  }
+
+  private static CreateTableJdbcStatement table(
+      String name,
+      PartitionType partitionType,
+      List<String> partitionKey,
+      Duration ttl,
+      String partitionInterval) {
     return new CreateTableJdbcStatement(
-        name, null, List.of(), List.of("id", "time"), partitionKey, partitionType, 1, ttl);
+        name,
+        null,
+        List.of(),
+        List.of("id", "time"),
+        partitionKey,
+        partitionType,
+        1,
+        ttl,
+        partitionInterval);
   }
 
   @Test
@@ -82,6 +99,31 @@ void givenMixedTables_whenGetDdl_thenOnlyRangeTtlTablesSortedByName() {
     assertThat(sql).doesNotContain("p_type");
   }
 
+  @Test
+  void givenExplicitPartitionInterval_whenGetDdl_thenIntervalOverridesDerived() {
+    assertThat(
+            extension.getDdl(
+                List.of(
+                    table(
+                        "Metrics",
+                        PartitionType.RANGE,
+                        List.of("time"),
+                        Duration.ofDays(14),
+                        "1 day"))))
+        .contains("p_interval => '1 day'")
+        .contains("retention = '14 days'");
+  }
+
+  @Test
+  void givenNoPartitionInterval_whenGetDdl_thenIntervalDerivedFromTtl() {
+    assertThat(
+            extension.getDdl(
+                List.of(
+                    table("Metrics", PartitionType.RANGE, List.of("time"), Duration.ofDays(14)))))
+        .contains("p_interval => '1 week'")
+        .contains("retention = '14 days'");
+  }
+
   @Test
   void givenTtl_whenDeriveInterval_thenBucketedByDuration() {
     assertThat(PgPartmanExtension.deriveInterval(Duration.ofHours(6))).isEqualTo("1 day");