feat: Derive pg_partman partition width from ttl unit and partition-divisor setting

CONTRIBUTORS.md

@@ -11,3 +11,4 @@ This file recognizes the people who have made contributions to SQRL.
 | Etienne Chauchot    | echauchot          |
 | Tianchen Wu         | wutianchen         |
 | Brad Bailey         | bradjbailey        |
+| Wellington Mafra    | WellMafra          |

documentation/docs/configuration-engine/postgres.md

@@ -6,12 +6,17 @@ PostgreSQL is a realtime database that stores the materialized views and tables
 
 No mandatory configuration keys are required. Physical DDL (tables, indexes, views) is produced automatically by the DataSQRL compiler.
 
+| Key                     | Type        | Default | Description                                                                                                    |
+|-------------------------|-------------|---------|----------------------------------------------------------------------------------------------------------------|
+| `partition-ttl-divisor` | **integer** | `100`   | Controls the number of partitions for range-partitioned tables with a TTL (see [Partitioning](#partitioning)). |
+
 ## Basic Configuration
 
 ```json
 {
   "engines": {
     "postgres": {
+      "partition-ttl-divisor": 100,
       "config": {
         // Optional PostgreSQL-specific settings
       }
@@ -20,6 +25,16 @@ No mandatory configuration keys are required. Physical DDL (tables, indexes, vie
 }
 ```
 
+## Partitioning
+
+Tables annotated with `partition_key` on a timestamp column and a `ttl` hint are range-partitioned with pg_partman, and expired partitions are dropped automatically. The partition width is derived from the TTL:
+
+- The TTL duration divided by `partition-ttl-divisor` caps the number of partitions.
+- The unit the TTL is declared with sets the minimum width: `ttl(14 days)` never produces partitions smaller than 1 day, while `ttl(336 hours)` allows hourly partitions.
+- The result is snapped down to the closest calendar-aligned width from: 15 min, 30 min, 1, 2, 4, 6, 8, 12 hours, 1, 2, 4 days, 1, 2, 4, 8, 12 weeks.
+
+For example, `ttl(14 days)` with the default divisor produces 1-day partitions.
+
 ## Cloud Deployment
 
 For cloud deployment configuration (instance sizes, replica counts), see [Cloud Deployment Configuration](cloud-deployment.md#postgresql-enginespostgresdeployment).

documentation/docs/sqrl-language.md

@@ -329,7 +329,7 @@ Hints live in a `/*+ ... */` comment placed **immediately before** the definitio
 | **query_by_any**            | `query_by_any(col, ...)`                                                   | table          | generate interface with *optional* filter arguments for all listed columns                                                                                                         |
 | **no_query**                | `no_query`                                                                 | table          | hide from interface                                                                                                                                                                |
 | **insert**                  | `insert(type)`                                                             | table          | controls the way how mutations will be written to their target sink. `type` ∈ `SINGLE` (default), `BATCH`, `TRANSACTION`                                                           |
-| **ttl**                     | `ttl(duration)`                                                            | table          | specifies how long the records for this table are retained in the underlying data system before it can be discarded. Expects a duration string like `5 week`. Disabled by default. |
+| **ttl**                     | `ttl(duration)`                                                            | table          | specifies how long the records for this table are retained in the underlying data system before it can be discarded. Expects a duration string like `5 week` with a unit between minute and week (e.g. `30 min`, `36 hours`, `14 days`, `2 weeks`). For range-partitioned tables, the partition width is derived from the duration and its unit (see the [postgres engine configuration](configuration-engine/postgres#partitioning)). Disabled by default. |
 | **cache**                   | `cache(duration)`                                                          | table          | how long the results retrieved from this table can be cached on the server before they are refreshed. Expects a duration string like `10 seconds`. Disabled by default.            |
 | **filtered_distinct_order** | flag                                                                       | DISTINCT table | eliminate updates on order column only before dedup                                                                                                                                |
 | **engine**                  | `enigne(engine_id)`                                                        | table          | pin execution engine (`process`, `database`, `flink`, ...)                                                                                                                         |

sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/AbstractJdbcStatementFactory.java

@@ -39,6 +39,7 @@
 import com.datasqrl.sql.DatabaseTypeExtension;
 import com.datasqrl.util.CalciteUtil;
 import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
@@ -167,13 +168,10 @@ public JdbcStatement createTable(JdbcEngineCreateTable createTable) {
     // DagPlanner validates that partition keys are part of primary key
 
     PartitionType partitionType = getPartitionType(createTable, partitionKeys);
-    Duration ttl =
-        createTable
-            .tableAnalysis()
-            .getHints()
-            .getHint(TtlHint.class)
-            .flatMap(TtlHint::getTtl)
-            .orElse(Duration.ZERO);
+    var ttlHint = createTable.tableAnalysis().getHints().getHint(TtlHint.class);
+    Duration ttl = ttlHint.flatMap(TtlHint::getTtl).orElse(Duration.ZERO);
+    ChronoUnit ttlUnit = ttlHint.flatMap(TtlHint::getTtlUnit).orElse(null);
+    String partitionInterval = derivePartitionInterval(partitionType, ttl, ttlUnit).orElse(null);
 
     return new CreateTableJdbcStatement(
         tableName,
@@ -187,6 +185,7 @@ public JdbcStatement createTable(JdbcEngineCreateTable createTable) {
         partitionType,
         partitionType == PartitionType.NONE ? 0 : 1,
         ttl,
+        partitionInterval,
         createTable,
         createTableDdlFactory);
   }
@@ -196,6 +195,15 @@ protected PartitionType getPartitionType(
     return partitionKey.isEmpty() ? PartitionType.NONE : PartitionType.LIST;
   }
 
+  /**
+   * The width of the partitions for range-partitioned tables with a TTL - empty when the dialect
+   * does not support time-based partitioning.
+   */
+  protected Optional<String> derivePartitionInterval(
+      PartitionType partitionType, Duration ttl, ChronoUnit ttlUnit) {
+    return Optional.empty();
+  }
+
   protected List<Field> getColumns(
       List<RelDataTypeField> fields, PlannerHints hints, Documentation documentation) {
     return fields.stream()

sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/CreateTableJdbcStatement.java

@@ -63,6 +63,9 @@ public class CreateTableJdbcStatement implements JdbcStatement {
   /** The time-to-live of records in this table - ZERO to disable */
   Duration ttl;
 
+  /** Partition width for range-partitioned tables, derived from the ttl - null when not ranged */
+  String partitionInterval;
+
   /** The engine table - nullable and not set when deserialized */
   @JsonIgnore JdbcEngineCreateTable engineTable;
 
@@ -78,7 +81,8 @@ public CreateTableJdbcStatement(
       @JsonProperty("partitionKey") List<String> partitionKey,
       @JsonProperty("partitionType") PartitionType partitionType,
       @JsonProperty("numPartitions") int numPartitions,
-      @JsonProperty("ttl") Duration ttl) {
+      @JsonProperty("ttl") Duration ttl,
+      @JsonProperty("partitionInterval") String partitionInterval) {
     this.name = name;
     this.description = description;
     this.fields = fields;
@@ -87,6 +91,7 @@ public CreateTableJdbcStatement(
     this.partitionType = partitionType;
     this.numPartitions = numPartitions;
     this.ttl = ttl;
+    this.partitionInterval = partitionInterval;
     this.engineTable = null;
     this.ddlFactory = null;
   }

sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/PostgresJdbcEngine.java

@@ -60,7 +60,7 @@ protected DatabaseType getDatabaseType() {
 
   @Override
   public JdbcStatementFactory getStatementFactory() {
-    return new PostgresStatementFactory();
+    return new PostgresStatementFactory(engineConfig);
   }
 
   @Override

sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/PostgresPartitionInterval.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright © 2021 DataSQRL (contact@datasqrl.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datasqrl.engine.database.relational;
+
+import com.google.common.collect.ImmutableMap;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.Map;
+import lombok.Value;
+
+/** The pg_partman partition width for a range-partitioned table with a TTL. */
+@Value
+public class PostgresPartitionInterval {
+
+  /** Calendar-aligned partition widths pg_partman can use, keyed by width in minutes. */
+  private static final Map<Integer, String> PARTITION_MENU =
+      ImmutableMap.<Integer, String>builder()
+          .put(15, "15 minutes")
+          .put(30, "30 minutes")
+          .put(60, "1 hour")
+          .put(120, "2 hours")
+          .put(240, "4 hours")
+          .put(360, "6 hours")
+          .put(480, "8 hours")
+          .put(720, "12 hours")
+          .put(1440, "1 day")
+          .put(2880, "2 days")
+          .put(5760, "4 days")
+          .put(10080, "1 week")
+          .put(20160, "2 weeks")
+          .put(40320, "4 weeks")
+          .put(80640, "8 weeks")
+          .put(120960, "12 weeks")
+          .buildOrThrow();
+
+  String interval;
+
+  /**
+   * Picks the partition width for the given TTL: the TTL divided by the divisor caps the partition
+   * count, while the unit the TTL was declared with sets the floor. The result is snapped down to
+   * the closest calendar-aligned width from the menu.
+   */
+  public static PostgresPartitionInterval of(
+      Duration ttl, ChronoUnit ttlUnit, int partitionTtlDivisor) {
+    var floorMinutes = ttlUnit.getDuration().toMinutes();
+    var targetMinutes = Math.max(ttl.toMinutes() / (double) partitionTtlDivisor, floorMinutes);
+    var interval = PARTITION_MENU.values().iterator().next();
+    for (var entry : PARTITION_MENU.entrySet()) {
+      if (entry.getKey() <= targetMinutes) {
+        interval = entry.getValue();
+      }
+    }
+    return new PostgresPartitionInterval(interval);
+  }
+}

sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/PostgresStatementFactory.java

@@ -20,6 +20,7 @@
 import com.datasqrl.calcite.Dialect;
 import com.datasqrl.calcite.dialect.ExtendedPostgresSqlDialect;
 import com.datasqrl.config.JdbcDialect;
+import com.datasqrl.config.PackageJson.EngineConfig;
 import com.datasqrl.deployment.model.JdbcStatementModel.PartitionType;
 import com.datasqrl.deployment.model.JdbcStatementModel.Type;
 import com.datasqrl.engine.database.relational.ddl.CreateIndexDDL;
@@ -34,6 +35,8 @@
 import com.datasqrl.sql.DatabaseTypeExtension;
 import com.datasqrl.util.CalciteUtil;
 import com.datasqrl.util.ServiceLoaderDiscovery;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -46,8 +49,19 @@
 
 public class PostgresStatementFactory extends AbstractJdbcStatementFactory {
 
-  public PostgresStatementFactory() {
+  public static final String PARTITION_TTL_DIVISOR_KEY = "partition-ttl-divisor";
+
+  private final int partitionTtlDivisor;
+
+  public PostgresStatementFactory(EngineConfig engineConfig) {
+    this(Integer.parseInt(engineConfig.getSetting(PARTITION_TTL_DIVISOR_KEY)));
+  }
+
+  public PostgresStatementFactory(int partitionTtlDivisor) {
     super(Dialect.POSTGRES, new PostgresCreateTableDdlFactory(true));
+    checkArgument(
+        partitionTtlDivisor > 0, "%s must be a positive number", PARTITION_TTL_DIVISOR_KEY);
+    this.partitionTtlDivisor = partitionTtlDivisor;
   }
 
   @Override
@@ -92,6 +106,16 @@ protected PartitionType getPartitionType(
     return CalciteUtil.isTimestamp(colType.getType()) ? PartitionType.RANGE : PartitionType.HASH;
   }
 
+  @Override
+  protected Optional<String> derivePartitionInterval(
+      PartitionType partitionType, Duration ttl, ChronoUnit ttlUnit) {
+    if (partitionType != PartitionType.RANGE || ttl == null || ttl.isZero() || ttlUnit == null) {
+      return Optional.empty();
+    }
+    return Optional.of(
+        PostgresPartitionInterval.of(ttl, ttlUnit, partitionTtlDivisor).getInterval());
+  }
+
   @Override
   public List<JdbcStatement> applyTableExtensions(Collection<CreateTableJdbcStatement> tables) {
     var res = new ArrayList<JdbcStatement>();

sqrl-planner/src/main/java/com/datasqrl/function/translation/postgres/extensions/PgPartmanExtension.java

@@ -19,6 +19,7 @@
 import com.datasqrl.engine.database.relational.CreateTableJdbcStatement;
 import com.datasqrl.sql.DatabaseTableExtension;
 import com.google.auto.service.AutoService;
+import com.google.common.base.Preconditions;
 import java.time.Duration;
 import java.util.Collection;
 import java.util.Comparator;
@@ -60,6 +61,11 @@ private void appendTableDdl(StringBuilder sb, CreateTableJdbcStatement createTab
     // 'public."Readings"' never matches relname 'Readings' and create_parent fails.
     var parentTable = "public." + createTable.getName();
     var ttl = createTable.getTtl();
+    var interval =
+        Preconditions.checkNotNull(
+            createTable.getPartitionInterval(),
+            "Missing partition interval for partitioned table %s with TTL",
+            createTable.getName());
 
     // p_type was removed in pg_partman 5.x; the default (range) is what we need
     sb.append(
@@ -72,7 +78,7 @@ private void appendTableDdl(StringBuilder sb, CreateTableJdbcStatement createTab
         );
 
         """
-            .formatted(parentTable, createTable.getPartitionKey().get(0), deriveInterval(ttl)));
+            .formatted(parentTable, createTable.getPartitionKey().get(0), interval));
 
     sb.append(
         """
@@ -92,18 +98,6 @@ private boolean isNotPartmanTable(CreateTableJdbcStatement createTable) {
         || createTable.getPartitionKey().isEmpty();
   }
 
-  /** Picks a partition interval that keeps the partition count reasonable for the given TTL. */
-  static String deriveInterval(Duration ttl) {
-    long seconds = ttl.getSeconds();
-    if (seconds < 7 * 24 * 3600) {
-      return "1 day";
-    } else if (seconds < 90L * 24 * 3600) {
-      return "1 week";
-    } else {
-      return "1 month";
-    }
-  }
-
   static String formatRetention(Duration ttl) {
     long days = ttl.toDays();
     if (days > 0) {