RLM (Recurrent Language Model) Integration for LCM Compaction

README.md

@@ -137,6 +137,11 @@ Add a `lossless-claw` entry under `plugins.entries` in your OpenClaw config:
 | `LCM_EXPANSION_PROVIDER` | *(from OpenClaw)* | Provider override for `lcm_expand_query` sub-agent |
 | `LCM_AUTOCOMPACT_DISABLED` | `false` | Disable automatic compaction after turns |
 | `LCM_PRUNE_HEARTBEAT_OK` | `false` | Retroactively delete `HEARTBEAT_OK` turn cycles from LCM storage |
+| `LCM_RLM_ENABLED` | `false` | Enable RLM (Recurrent Language Model) pattern-based summarization |
+| `LCM_RLM_PROVIDER` | `""` | Provider for RLM pattern analysis (e.g., 'openai', 'anthropic') |
+| `LCM_RLM_MODEL` | `""` | Model for RLM pattern analysis (e.g., 'gpt-4', 'claude-sonnet-4-20250514') |
+| `LCM_RLM_MIN_DEPTH` | `2` | Minimum compaction depth before using RLM |
+| `LCM_RLM_PATTERN_THRESHOLD` | `0.7` | Confidence threshold for pattern detection (0.0 - 1.0) |
 
 ### Expansion model override requirements
 
@@ -192,6 +197,54 @@ For compaction summarization, lossless-claw resolves the model in this order:
 
 If `summaryModel` already includes a provider prefix such as `anthropic/claude-sonnet-4-20250514`, `summaryProvider` is ignored for that choice. Otherwise, the provider falls back to the matching override, then `OPENCLAW_PROVIDER`, then the provider inferred by the caller.
 
+### RLM (Recurrent Language Model) configuration
+
+RLM adds pattern-based summarization for deep compaction passes. When enabled, RLM analyzes summaries at depth >= `rlmMinDepth` to detect recurring themes, progressions, and task lifecycles. These patterns are compressed into efficient representations, reducing token usage while preserving semantic meaning.
+
+To enable RLM:
+
+```json
+{
+  "plugins": {
+    "entries": {
+      "lossless-claw": {
+        "config": {
+          "rlmEnabled": true,
+          "rlmProvider": "openai",
+          "rlmModel": "gpt-4",
+          "rlmMinDepth": 2,
+          "rlmPatternThreshold": 0.7
+        }
+      }
+    }
+  }
+}
+```
+
+Or via environment variables:
+
+```bash
+LCM_RLM_ENABLED=true
+LCM_RLM_PROVIDER=openai
+LCM_RLM_MODEL=gpt-4
+LCM_RLM_MIN_DEPTH=2
+LCM_RLM_PATTERN_THRESHOLD=0.7
+```
+
+**When to use RLM:**
+- Long-running conversations with recurring topics
+- Multi-session projects with evolving requirements
+- Workflows with clear task lifecycles (planning, execution, review)
+
+**RLM pattern types:**
+- **Recurring themes** — topics that appear across multiple summaries
+- **Progressions** — evolving states or sequences
+- **Decision evolution** — how decisions changed over time
+- **Task lifecycles** — tasks being created, worked on, completed
+- **Constraints** — persistent limitations or requirements
+
+RLM operates as a fallback mechanism: if pattern detection fails or confidence is low, standard summarization is used automatically.
+
 ### Recommended starting configuration
 
 ```
@@ -356,6 +409,10 @@ src/
   engine.ts                 # LcmContextEngine — implements ContextEngine interface
   assembler.ts              # Context assembly (summaries + messages → model context)
   compaction.ts             # CompactionEngine — leaf passes, condensation, sweeps
+  rlm/                       # RLM (Recurrent Language Model) pattern-based summarization
+    types.ts                 # RLM type definitions for patterns and analysis
+    rlm.ts                   # Core RLM engine with heuristic and LLM-based detection
+    index.ts                 # RLM module exports
   summarize.ts              # Depth-aware prompt generation and LLM summarization
   retrieval.ts              # RetrievalEngine — grep, describe, expand operations
   expansion.ts              # DAG expansion logic for lcm_expand_query

openclaw.plugin.json

@@ -44,6 +44,26 @@
     "expansionProvider": {
       "label": "Expansion Provider",
       "help": "Provider override for lcm_expand_query sub-agent (e.g., 'anthropic')"
+    },
+    "rlmEnabled": {
+      "label": "RLM Enabled",
+      "help": "Enable RLM (Recurrent Language Model) pattern-based summarization for deep compaction"
+    },
+    "rlmProvider": {
+      "label": "RLM Provider",
+      "help": "Provider for RLM pattern analysis (e.g., 'openai', 'anthropic')"
+    },
+    "rlmModel": {
+      "label": "RLM Model",
+      "help": "Model for RLM pattern analysis (e.g., 'gpt-4', 'claude-sonnet-4-20250514')"
+    },
+    "rlmMinDepth": {
+      "label": "RLM Minimum Depth",
+      "help": "Minimum compaction depth before using RLM (default: 2)"
+    },
+    "rlmPatternThreshold": {
+      "label": "RLM Pattern Threshold",
+      "help": "Confidence threshold for pattern detection (0.0 - 1.0, default: 0.7)"
     }
   },
   "configSchema": {
@@ -111,6 +131,24 @@
       },
       "expansionProvider": {
         "type": "string"
+      },
+      "rlmEnabled": {
+        "type": "boolean"
+      },
+      "rlmProvider": {
+        "type": "string"
+      },
+      "rlmModel": {
+        "type": "string"
+      },
+      "rlmMinDepth": {
+        "type": "integer",
+        "minimum": 0
+      },
+      "rlmPatternThreshold": {
+        "type": "number",
+        "minimum": 0,
+        "maximum": 1
       }
     }
   }

src/compaction.ts

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import type { ConversationStore, CreateMessagePartInput } from "./store/conversation-store.js";
 import type { SummaryStore, SummaryRecord, ContextItemRecord } from "./store/summary-store.js";
 import { extractFileIdsFromContent } from "./large-files.js";
+import { RlmEngine, type RlmSummaryEntry } from "./rlm/index.js";
 
 // ── Public types ─────────────────────────────────────────────────────────────
 
@@ -49,6 +50,12 @@ export interface CompactionConfig {
   maxRounds: number;
   /** IANA timezone for timestamps in summaries (default: UTC) */
   timezone?: string;
+  /** RLM configuration for pattern-based summarization */
+  rlmEnabled?: boolean;
+  rlmProvider?: string;
+  rlmModel?: string;
+  rlmMinDepth?: number;
+  rlmPatternThreshold?: number;
 }
 
 type CompactionLevel = "normal" | "aggressive" | "fallback";
@@ -57,12 +64,21 @@ type CompactionSummarizeOptions = {
   previousSummary?: string;
   isCondensed?: boolean;
   depth?: number;
+  /** Use RLM (Recurrent Language Model) pattern-based summarization if available */
+  rlmSummarize?: boolean;
 };
 type CompactionSummarizeFn = (
   text: string,
   aggressive?: boolean,
   options?: CompactionSummarizeOptions,
 ) => Promise<string>;
+
+/** LLM completion function for RLM pattern analysis */
+type LlmCompleteFn = (
+  prompt: string,
+  system: string,
+  maxTokens: number,
+) => Promise<string>;
 type PassResult = { summaryId: string; level: CompactionLevel };
 type LeafChunkSelection = {
   items: ContextItemRecord[];
@@ -167,11 +183,25 @@ function dedupeOrderedIds(ids: Iterable<string>): string[] {
 // ── CompactionEngine ─────────────────────────────────────────────────────────
 
 export class CompactionEngine {
+  private rlmEngine?: RlmEngine;
+
   constructor(
     private conversationStore: ConversationStore,
     private summaryStore: SummaryStore,
     private config: CompactionConfig,
-  ) {}
+    llmCompleteFn?: LlmCompleteFn,
+  ) {
+    // Initialize RLM engine if enabled
+    if (config.rlmEnabled) {
+      this.rlmEngine = new RlmEngine({
+        enabled: true,
+        provider: config.rlmProvider ?? "",
+        model: config.rlmModel ?? "",
+        minDepth: config.rlmMinDepth ?? 2,
+        patternThreshold: config.rlmPatternThreshold ?? 0.7,
+      }, llmCompleteFn);
+    }
+  }
 
   // ── evaluate ─────────────────────────────────────────────────────────────
 
@@ -1001,11 +1031,13 @@ export class CompactionEngine {
   /**
    * Run three-level summarization escalation:
    * normal -> aggressive -> deterministic fallback.
+   * For depth >= 1, optionally uses RLM pattern-based summarization.
    */
   private async summarizeWithEscalation(params: {
     sourceText: string;
     summarize: CompactionSummarizeFn;
     options?: CompactionSummarizeOptions;
+    rlmEntries?: RlmSummaryEntry[];
   }): Promise<{ content: string; level: CompactionLevel } | null> {
     const sourceText = params.sourceText.trim();
     if (!sourceText) {
@@ -1027,6 +1059,36 @@ export class CompactionEngine {
       };
     };
 
+    // Check if RLM should be used for this depth
+    const depth = params.options?.depth ?? 0;
+    const useRlm = params.options?.rlmSummarize === true && 
+                   this.rlmEngine?.shouldUseRlm(depth);
+
+    if (useRlm) {
+      // Try RLM pattern-based summarization
+      try {
+        // Use pre-parsed entries if available, otherwise fall back to text extraction
+        const entries: RlmSummaryEntry[] = params.rlmEntries ?? this.extractEntriesFromSourceText(sourceText);
+
+        if (entries.length >= 2) {
+          const rlmResult = await this.rlmEngine!.summarize(entries, {
+            previousSummary: params.options?.previousSummary,
+            depth,
+          });
+
+          if (rlmResult.content && !rlmResult.fallbackToStandard) {
+            // RLM summarization succeeded with viable patterns
+            return { 
+              content: rlmResult.content, 
+              level: rlmResult.confidence > 0.8 ? "normal" : "aggressive" 
+            };
+          }
+        }
+      } catch (error) {
+        console.warn(`[lcm] RLM summarization failed, falling back to standard: ${error}`);
+      }
+    }
+
     const runSummarizer = async (aggressiveMode: boolean): Promise<string | null> => {
       const output = await params.summarize(sourceText, aggressiveMode, params.options);
       const trimmed = output.trim();
@@ -1058,6 +1120,38 @@ export class CompactionEngine {
     return { content: summaryText, level };
   }
 
+  /**
+   * Extract RLM entries from source text for pattern analysis.
+   * Parses the concatenated summary format used in condensed passes.
+   */
+  private extractEntriesFromSourceText(sourceText: string): RlmSummaryEntry[] {
+    const entries: RlmSummaryEntry[] = [];
+
+    // Split by common delimiter patterns in condensed summaries
+    const sections = sourceText.split(/\n\n---\n\n|\n\n(?=\[\d{4}-\d{2}-\d{2})/);
+
+    for (let i = 0; i < sections.length; i++) {
+      const section = sections[i].trim();
+      if (!section) continue;
+
+      // Try to extract timestamp from header like "[2024-01-15 10:30 PST - 2024-01-15 11:00 PST]"
+      const headerMatch = section.match(/^\[(\d{4}-\d{2}-\d{2}[^\]]*)\]/);
+      const createdAt = headerMatch 
+        ? new Date(headerMatch[1].split(" - ")[0].trim()) 
+        : new Date();
+
+      entries.push({
+        summaryId: `extracted_${i}`,
+        content: section,
+        depth: 0, // Will be set by caller
+        createdAt,
+        tokenCount: estimateTokens(section),
+      });
+    }
+
+    return entries;
+  }
+
   // ── Private: Media Annotation ────────────────────────────────────────────
 
   /**
@@ -1245,14 +1339,33 @@ export class CompactionEngine {
       targetDepth === 0
         ? await this.resolvePriorSummaryContextAtDepth(conversationId, summaryItems, targetDepth)
         : undefined;
+
+    // Enable RLM for deeper condensation passes (respecting rlmMinDepth config)
+    const effectiveMinDepth = this.config.rlmMinDepth ?? 2;
+    const nextDepth = targetDepth + 1;
+    const useRlm = this.config.rlmEnabled && nextDepth >= effectiveMinDepth;
+
+    // Build proper RLM entries from summary records when using RLM
+    const rlmEntries: RlmSummaryEntry[] | undefined = useRlm
+      ? summaryRecords.map((rec) => ({
+          summaryId: rec.summaryId,
+          content: rec.content,
+          depth: rec.depth,
+          createdAt: rec.createdAt,
+          tokenCount: rec.tokenCount ?? estimateTokens(rec.content),
+        }))
+      : undefined;
+
     const condensed = await this.summarizeWithEscalation({
       sourceText: concatenated,
       summarize,
       options: {
         previousSummary: previousSummaryContent,
         isCondensed: true,
         depth: targetDepth + 1,
+        rlmSummarize: useRlm,
       },
+      rlmEntries,
     });
     if (!condensed) {
       console.warn(

src/db/config.ts

@@ -29,10 +29,6 @@ export type LcmConfig = {
   largeFileSummaryProvider: string;
   /** Model override for large-file text summarization. */
   largeFileSummaryModel: string;
-  /** Model override for conversation summarization. */
-  summaryModel: string;
-  /** Provider override for conversation summarization. */
-  summaryProvider: string;
   /** Provider override for lcm_expand_query sub-agent. */
   expansionProvider: string;
   /** Model override for lcm_expand_query sub-agent. */
@@ -42,6 +38,16 @@ export type LcmConfig = {
   timezone: string;
   /** When true, retroactively delete HEARTBEAT_OK turn cycles from LCM storage. */
   pruneHeartbeatOk: boolean;
+  /** RLM (Recurrent Language Model) configuration for pattern-based summarization */
+  rlmEnabled: boolean;
+  /** Provider for RLM pattern analysis */
+  rlmProvider: string;
+  /** Model for RLM pattern analysis */
+  rlmModel: string;
+  /** Minimum depth before using RLM (default: 2) */
+  rlmMinDepth: number;
+  /** Confidence threshold for pattern detection (0.0 - 1.0, default: 0.7) */
+  rlmPatternThreshold: number;
 };
 
 /** Safely coerce an unknown value to a finite number, or return undefined. */
@@ -185,5 +191,27 @@ export function resolveLcmConfig(
       env.LCM_PRUNE_HEARTBEAT_OK !== undefined
         ? env.LCM_PRUNE_HEARTBEAT_OK === "true"
         : toBool(pc.pruneHeartbeatOk) ?? false,
+    rlmEnabled:
+      env.LCM_RLM_ENABLED !== undefined
+        ? env.LCM_RLM_ENABLED !== "false"
+        : toBool(pc.rlmEnabled) ?? false,
+    rlmProvider:
+      env.LCM_RLM_PROVIDER?.trim() ?? toStr(pc.rlmProvider) ?? "",
+    rlmModel:
+      env.LCM_RLM_MODEL?.trim() ?? toStr(pc.rlmModel) ?? "",
+    rlmMinDepth:
+      (env.LCM_RLM_MIN_DEPTH !== undefined ? parseInt(env.LCM_RLM_MIN_DEPTH, 10) : undefined)
+        ?? toNumber(pc.rlmMinDepth) ?? 2,
+    rlmPatternThreshold:
+      (env.LCM_RLM_PATTERN_THRESHOLD !== undefined ? parseFloat(env.LCM_RLM_PATTERN_THRESHOLD) : undefined)
+        ?? toNumber(pc.rlmPatternThreshold) ?? 0.7,
   };
 }
+
+/**
+ * Get the effective RLM minimum depth for a given condensation target depth.
+ * Ensures RLM only triggers at or above the configured threshold.
+ */
+export function getEffectiveRlmMinDepth(config: LcmConfig): number {
+  return config.rlmMinDepth ?? 2;
+}