feat(vector): connect in-memory Rust pipeline

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes are recorded here. This project follows Semantic Versioning.
 
+## Unreleased
+
+- Connected typed Rust vector output directly to the chronological simulator, with
+  pair-parallel DAG preparation and no parallel wallet or order mutation.
+- Preserved SHA-verified Feather as the bounded-memory evidence/replay path and added
+  zero-tolerance trade-surface and every-candle full-state transport parity tests.
+- Kept latest-X7 Full Native qualification and the v1.6.0 release behind the remaining
+  M22 source-lowering, dual-mode exactness, performance, storage, and three-OS gates.
+
 ## 1.5.0 - 2026-08-10
 
 - Promoted source-compiled generic state-machine programs to the sole current

README.md

@@ -207,10 +207,14 @@ opens a Draft fixture PR. Neither kind is auto-approved or auto-merged.
 
 For supported X7 stateful callbacks, `run.json` records the
 `x7-generic-stateful` Native lane and every source-compiled primary program. The
-specialized X7 vector transport remains only an input contract. Current manager payloads
-execute generic programs without a handwritten X7 shadow; a missing or retired execution
-mode blocks Native before simulation and leaves the visible official fallback available.
-Historical schema readers remain available for sealed evidence replay.
+Rust vector core can now pass typed pair output directly into the chronological simulator;
+independent pair preparation is parallel, while wallet and order mutation remains one stable
+timestamp stream. SHA-verified Feather remains the durable evidence/replay path, and both transports
+must produce the same trade surface and every-candle state. Current manager payloads execute generic
+programs without a handwritten X7 shadow; a missing or retired execution mode blocks Native before
+simulation and leaves the visible official fallback available. Historical schema readers remain
+available for sealed evidence replay. See
+[Native In-Memory Vector Transport](docs/native-in-memory-vector.md).
 
 ## Confirm with official Freqtrade
 

benchmarks/evidence/native-in-memory-transport-m21-04-2026-08-11.json

@@ -0,0 +1,74 @@
+{
+  "schema_version": "1.0.0",
+  "status": "diagnostic-only",
+  "recorded_at": "2026-08-11",
+  "claim_boundary": "M21-04 typed Rust vector-to-simulator transport. Exactness uses a four-row full-state fixture; performance uses four synthetic 25000-row pairs on one Linux development host. This is not latest-X7 qualification, a five-year benchmark, or a cross-platform release speed claim.",
+  "contract": {
+    "vector_program": "benchmarks/reference/vector-shadow/tag-program.json",
+    "vector_program_sha256": "eea095daeb5329274e3c825026cbed327ff40ada026efded143f6eba4db16e3e",
+    "transport_source": "rust/crates/nfi-vector-io/src/in_memory.rs",
+    "transport_source_sha256": "3cf2e59c50b1d286d21f54907a9c96cd69e54783d92a5e5b9a2b02d68e7894a8",
+    "signal_gate": "exact numeric == 1 or Boolean true",
+    "pair_order": "indexed input order",
+    "wallet_execution": "single global chronological loop",
+    "feather_replay_retained": true
+  },
+  "exact_observation": {
+    "workload_rows": 4,
+    "pair_count": 1,
+    "trade_surface_exact": true,
+    "full_state_exact": true,
+    "comparison": "owned direct input versus SHA-verified file-backed Feather input",
+    "direct": {
+      "pair_prepare_ns": 984131,
+      "estimated_source_column_bytes": 358,
+      "estimated_simulation_owned_bytes": 1216
+    },
+    "feather": {
+      "manifest_ns": 74852,
+      "vector_hash_ns": 23223,
+      "feather_decode_ns": 219656,
+      "file_backed_bytes": 356
+    }
+  },
+  "performance_observation": {
+    "workload_rows": 100000,
+    "rows_per_pair": 25000,
+    "pair_count": 4,
+    "feature_column_count": 4,
+    "direct": {
+      "vector_execute_ns": 901416,
+      "pair_prepare_ns": 11833000,
+      "transport_ns": 12734416,
+      "estimated_source_column_bytes": 8137532,
+      "estimated_simulation_owned_bytes": 29600052,
+      "configured_worker_limit": 16
+    },
+    "feather": {
+      "manifest_ns": 95616,
+      "vector_hash_ns": 7248024,
+      "feather_decode_ns": 18585906,
+      "transport_ns": 25929546,
+      "file_backed_bytes": 8900000
+    },
+    "observed_transport_speedup": 2.036178651616219,
+    "trade_surface_exact": true
+  },
+  "environment": {
+    "kernel": "Linux 7.0.0-28-generic",
+    "architecture": "x86_64",
+    "logical_cpu_limit": 16,
+    "memory_bytes": 50195083264,
+    "cargo_profile": "release"
+  },
+  "commands": [
+    "cargo test --release -p nfi-vector-io --locked in_memory::tests::direct_transport_is_full_state_exact_to_sealed_feather_replay -- --nocapture",
+    "cargo test --release -p nfi-vector-io --locked in_memory::tests::large_pair_transport_profile_is_reported_without_a_speed_assertion -- --ignored --nocapture"
+  ],
+  "limitations": [
+    "Each timing was measured once and includes one local Rayon pool configuration.",
+    "The synthetic workload has no open trades and measures transport, not callback or simulation speed.",
+    "The direct path retains owned simulator rows; Feather remains the bounded-heap replay option.",
+    "M22-01 must still compile and independently qualify the latest NFI Spot and Futures source."
+  ]
+}

docs/architecture.md

@@ -31,7 +31,11 @@ strategy.py + config + candles
 AST preflight + frozen input/data identity
           |
           v
-SHA-bound Feather vectors + projected callback columns
+compiled Vector IR + typed pair columns
+          |
+          +------> parallel Rust pair DAG ------> direct owned simulator input
+          |
+          +------> SHA-bound Feather ----------> verified file-backed replay
           |
           v
 Rust global chronological portfolio loop
@@ -44,9 +48,10 @@ Rust global chronological portfolio loop
                  common canonical state trace -----------------> full exact check
 ```
 
-Python is not called once per candle. Python owns preparation, columnar transforms,
-reports, and proof artifacts. Rust owns mutable trade/order/wallet state and the hot
-chronological loop.
+Python is not called once per candle. Python owns source compilation, reports, and proof artifacts.
+Supported compiled vector work and typed transport can remain inside Rust. Rust owns mutable
+trade/order/wallet state and the hot chronological loop. Latest-X7 entry into this complete path is
+still separately qualified by the roadmap; unsupported source constructs fail closed.
 
 The compact vector manifest stores a relative Feather path, file SHA-256, row count,
 feature names, pair limits, precision, and historical price-step changes. Rust
@@ -79,6 +84,7 @@ competition, shared stake, rebuy timing, rejected signals, and equal-timestamp o
 Parallel work is limited to operations without shared mutable portfolio state:
 
 - indicators by pair;
+- complete compiled vector DAGs and typed simulator-input conversion by pair;
 - independent strategies, timeranges, fixtures, and candidate jobs;
 - offline scoring and report generation.
 

docs/native-in-memory-vector.md

@@ -0,0 +1,55 @@
+# Native In-Memory Vector Transport
+
+M21-04 connects typed Rust vector output to the existing chronological simulator without writing
+an intermediate Feather file. This is a transport change only: it does not change signal timing,
+pair priority, wallet mutation, order IDs, or callback semantics.
+
+## Runtime contract
+
+`nfi-vector-io` accepts one `MutationFrame` per pair through `InMemoryVectorPair`. The frame carries
+the same `date`, OHLCV, callback-feature, and `nfi_exec_*` columns as the sealed Feather manifest.
+The adapter preserves:
+
+- millisecond timestamps and exact Float64 bits, including signed zero and canonical NaN features;
+- Freqtrade's exact numeric-`1`/Boolean-`true` signal gate;
+- raw non-empty entry and exit tags, including trailing whitespace;
+- previous-close, funding-event, precision, stake-limit, and execution-start metadata;
+- caller pair order in the resulting `SimulationInput`.
+
+Missing columns, mixed types, Boolean feature nulls, duplicate pairs, and invalid execution ranges
+fail before simulation. Signal numbers, tags, pairs, strategy names, and source hashes remain data,
+never runtime branches.
+
+## Safe parallel boundary
+
+`execute_in_memory_pair_dag_profiled` runs independent pair vector tasks with Rayon and collects
+them by original index. Pair conversion is also independent and parallel. Both stages finish before
+the returned input enters the simulator. The simulator continues to mutate one shared wallet in one
+global timestamp stream; equal-timestamp pair order is unchanged.
+
+Rayon uses its configured worker limit, including `RAYON_NUM_THREADS`; no machine-specific thread
+count is compiled into the engine. `InMemoryVectorProfile` records vector execution, pair conversion,
+row/column counts, estimated source-buffer and retained simulator bytes, and the configured worker
+limit.
+
+## Feather replay proof
+
+The SHA-verified Feather path remains supported. A focused Rust test writes one typed frame to
+Feather, loads its file-backed representation, and compares it with the direct owned representation.
+Both are run through the simulator with an every-candle observer; `SimulationResult` and every full
+state event must be exactly equal.
+
+```bash
+cargo test --manifest-path rust/Cargo.toml -p nfi-vector-io --locked
+```
+
+The direct path removes vector hashing, IPC decoding, and private spool I/O from a Native pipeline.
+The Feather path remains the durable evidence and replay format and is still preferable when bounded
+heap use matters more than transport latency.
+
+## Claim boundary
+
+This completes the generic Rust vector-to-simulator connection. It does not claim that the latest X7
+source fully compiles: M22-01 still owns remaining source lowering and independent latest-upstream
+Spot/Futures full-state qualification. Until that passes, unsupported X7 constructs remain
+fail-closed and the visible official Freqtrade fallback remains available.

docs/native-signal-program.md

@@ -45,5 +45,6 @@ source location. Signal numbers, strategy names, pairs, timeranges, and source h
 never runtime branches; the source hash is evidence identity only.
 
 The independent Rust execution and exact raw-signal comparison are documented in
-[`native-vector-shadow.md`](native-vector-shadow.md). Production in-memory simulator transport
-remains the separate M21-04 boundary.
+[`native-vector-shadow.md`](native-vector-shadow.md). The completed
+[`native-in-memory-vector.md`](native-in-memory-vector.md) transport carries its shifted output into
+the simulator without changing the Signal contract.

docs/native-vector-core.md

@@ -127,8 +127,8 @@ M20-05 establishes exact multi-timeframe primitives and compiler contracts,
 but their in-memory `VectorEngine` connection is deliberately owned by M21.
 M21-01 and M21-02 establish Native signal assignment and exact tag-generation
 contracts. M21-03 independently proves Python/Rust Indicator, Signal, Tag, and
-execution-index equality. M21-04 owns the in-memory simulator connection. M22 owns
-latest-upstream Spot/Futures
+execution-index equality. M21-04 connects typed pair output to the simulator, preserves
+Feather replay, and keeps wallet mutation single-threaded. M22 owns latest-upstream Spot/Futures
 full-state qualification, removal of Python strategy execution from the Native
 lane, and release certification. Until those proofs pass, unsupported source
 constructs remain fail-closed and the official Freqtrade fallback remains the

docs/native-vector-shadow.md

@@ -35,8 +35,9 @@ unsupported helper behavior, and any exact-value difference.
 
 ## Claim boundary
 
-This proves independent Rust vector semantics for the committed reachable contract. M21-04 still
-owns the production in-memory connection from Rust vector output to the simulator and retained
-Feather replay. The latest X7 source also remains fail-closed at its separately recorded dynamic
-configuration boundary until the remaining source compiler lowering is complete. Full Native
-latest-NFI qualification therefore remains M22-01, not an M21-03 claim.
+This proves independent Rust vector semantics for the committed reachable contract. The M21-04
+[in-memory transport](native-in-memory-vector.md) now connects that typed output to the simulator
+and proves exact equality with retained Feather replay. The latest X7 source still remains
+fail-closed at its separately recorded dynamic configuration boundary until the remaining source
+compiler lowering is complete. Full Native latest-NFI qualification therefore remains M22-01, not
+an M21-03 or transport-only claim.

planning/roadmap-state.json

@@ -1,8 +1,8 @@
 {
   "schema_version": "1.0.0",
   "roadmap_id": "nfi-backtest-engine-post-v1.1.0",
-  "revision": 140,
-  "updated_at": "2026-08-11T20:43:47+09:00",
+  "revision": 142,
+  "updated_at": "2026-08-11T22:09:26+09:00",
   "acceptance_commands": "planning/acceptance-commands.json",
   "active_task_id": null,
   "execution_policy": {
@@ -4917,7 +4917,7 @@
       "order": 2104,
       "milestone": "M21",
       "title": "Connect the in-memory Rust vector and simulation pipeline",
-      "status": "pending",
+      "status": "completed",
       "depends_on": [
         "M21-03"
       ],
@@ -4936,10 +4936,17 @@
         "end-to-end Native vector fixture",
         "transport parity and performance report"
       ],
-      "started_at": null,
-      "completed_at": null,
-      "commit_sha": null,
-      "evidence": [],
+      "started_at": "2026-08-11T21:20:59+09:00",
+      "completed_at": "2026-08-11T22:09:26+09:00",
+      "commit_sha": "d9529131eaa629cf598a97569931eed844e7066a",
+      "evidence": [
+        ".nfi/roadmap-acceptance/M21-04/d9529131eaa629cf598a97569931eed844e7066a/acceptance-report.json",
+        ".nfi/roadmap-acceptance/M21-04/d9529131eaa629cf598a97569931eed844e7066a/SHA256SUMS.txt",
+        ".nfi/roadmap-acceptance/M21-04/d9529131eaa629cf598a97569931eed844e7066a/x7-spot/run.json",
+        ".nfi/roadmap-acceptance/M21-04/d9529131eaa629cf598a97569931eed844e7066a/x7-futures/run.json",
+        "benchmarks/evidence/native-in-memory-transport-m21-04-2026-08-11.json",
+        "docs/native-in-memory-vector.md"
+      ],
       "blocker": null
     },
     {
@@ -8592,6 +8599,52 @@
         "next_eligible_task": "M21-04",
         "next_task_manual_gate": false
       }
+    },
+    {
+      "sequence": 188,
+      "timestamp": "2026-08-11T21:20:59+09:00",
+      "task_id": "M21-04",
+      "event": "task_started",
+      "details": {
+        "objective": "connect generic Rust vector output to the simulator through an exact in-memory transport while preserving sealed Feather replay",
+        "sealed_feather_replay_must_remain": true,
+        "pair_dag_parallelism_allowed": true,
+        "parallel_wallet_mutation_allowed": false,
+        "transport_trade_surface_and_full_state_exact_required": true,
+        "transport_performance_evidence_required": true,
+        "runtime_hardcoding_allowed": false
+      }
+    },
+    {
+      "sequence": 189,
+      "timestamp": "2026-08-11T22:09:26+09:00",
+      "task_id": "M21-04",
+      "event": "task_completed",
+      "details": {
+        "implementation_commit": "d9529131eaa629cf598a97569931eed844e7066a",
+        "in_memory_simulator_connection_complete": true,
+        "sealed_feather_replay_retained": true,
+        "pair_dag_parallel": true,
+        "pair_order_preserved": true,
+        "parallel_wallet_mutation": false,
+        "transport_trade_surface_exact": true,
+        "transport_full_state_exact": true,
+        "transport_diagnostic_rows": 100000,
+        "transport_diagnostic_pairs": 4,
+        "observed_transport_speedup": 2.036178651616219,
+        "performance_claim": "single-host synthetic transport diagnostic only",
+        "python_test_count": 804,
+        "future_compatibility_test_count": 143,
+        "rust_workspace_default_test_count": 224,
+        "spot_trade_surface_and_full_state_exact": true,
+        "futures_trade_surface_and_full_state_exact": true,
+        "runtime_hardcoding_added": false,
+        "latest_x7_fully_compiled": false,
+        "python_strategy_execution_removed_from_native": false,
+        "full_native_strategy_claim": false,
+        "next_eligible_task": "M22-01",
+        "next_task_manual_gate": false
+      }
     }
   ]
 }