From 660304556ee2df4dbd3113525cf8c57fd7467e55 Mon Sep 17 00:00:00 2001 From: vntrevx <20063774+vntrevx@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:26:56 +0900 Subject: [PATCH 1/3] chore(roadmap): start M20-05 --- planning/roadmap-state.json | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/planning/roadmap-state.json b/planning/roadmap-state.json index b1f7f40..c581f8a 100644 --- a/planning/roadmap-state.json +++ b/planning/roadmap-state.json @@ -1,10 +1,10 @@ { "schema_version": "1.0.0", "roadmap_id": "nfi-backtest-engine-post-v1.1.0", - "revision": 131, - "updated_at": "2026-08-11T00:55:12+09:00", + "revision": 132, + "updated_at": "2026-08-11T01:26:42+09:00", "acceptance_commands": "planning/acceptance-commands.json", - "active_task_id": null, + "active_task_id": "M20-05", "execution_policy": { "max_in_progress": 1, "selection": "lowest order pending task whose dependencies are completed", @@ -4767,7 +4767,7 @@ "order": 2005, "milestone": "M20", "title": "Implement multi-timeframe and informative data semantics", - "status": "pending", + "status": "in_progress", "depends_on": [ "M20-04" ], @@ -4786,7 +4786,7 @@ "informative merge exact fixtures", "cross-pair dependency tests" ], - "started_at": null, + "started_at": "2026-08-11T01:26:42+09:00", "completed_at": null, "commit_sha": null, "evidence": [], @@ -8348,6 +8348,20 @@ "next_eligible_task": "M20-05", "next_task_manual_gate": false } + }, + { + "sequence": 179, + "timestamp": "2026-08-11T01:26:42+09:00", + "task_id": "M20-05", + "event": "task_started", + "details": { + "objective": "implement exact Freqtrade multi-timeframe and informative data visibility semantics for latest NFI", + "latest_upstream_commit": "897a1523391b8222ee711eba9714b59a3e77265a", + "fabricating_missing_candles_allowed": false, + "forward_looking_join_allowed": false, + "pair_or_timeframe_hardcoding_allowed": false, + "cross_pair_dependencies_must_be_explicit": true + } } ] } From 68d82daeadcf798e8e82534d470b8709d9191bb9 Mon Sep 17 00:00:00 2001 From: vntrevx <20063774+vntrevx@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:10:41 +0900 Subject: [PATCH 2/3] feat(vector): implement exact informative semantics --- .../informative/freqtrade-2026.5.1.json | 1 + docs/native-vector-core.md | 74 +- .../nfi_backtest_engine/indicator_program.py | 159 +++- .../informative_fixture.py | 542 ++++++++++++++ python/nfi_backtest_engine/strategy_compat.py | 88 ++- python/nfi_backtest_engine/vector_worker.py | 26 +- .../nfi-vector-core/src/alignment/batch.rs | 105 +++ .../nfi-vector-core/src/alignment/fill.rs | 479 ++++++++++++ .../nfi-vector-core/src/alignment/mod.rs | 23 + .../nfi-vector-core/src/alignment/model.rs | 301 ++++++++ .../nfi-vector-core/src/alignment/stream.rs | 235 ++++++ .../nfi-vector-core/src/alignment/support.rs | 248 +++++++ .../nfi-vector-core/src/alignment/tests.rs | 696 ++++++++++++++++++ rust/crates/nfi-vector-core/src/lib.rs | 1 + scripts/generate_informative_fixture.py | 20 + tests/test_indicator_program.py | 134 +++- tests/test_informative_fixture.py | 160 ++++ tests/test_strategy_compat.py | 117 +++ tests/test_vector_worker_resample.py | 109 +++ 19 files changed, 3459 insertions(+), 59 deletions(-) create mode 100644 benchmarks/reference/informative/freqtrade-2026.5.1.json create mode 100644 python/nfi_backtest_engine/informative_fixture.py create mode 100644 rust/crates/nfi-vector-core/src/alignment/batch.rs create mode 100644 rust/crates/nfi-vector-core/src/alignment/fill.rs create mode 100644 rust/crates/nfi-vector-core/src/alignment/mod.rs create mode 100644 rust/crates/nfi-vector-core/src/alignment/model.rs create mode 100644 rust/crates/nfi-vector-core/src/alignment/stream.rs create mode 100644 rust/crates/nfi-vector-core/src/alignment/support.rs create mode 100644 rust/crates/nfi-vector-core/src/alignment/tests.rs create mode 100644 scripts/generate_informative_fixture.py create mode 100644 tests/test_informative_fixture.py create mode 100644 tests/test_strategy_compat.py create mode 100644 tests/test_vector_worker_resample.py diff --git a/benchmarks/reference/informative/freqtrade-2026.5.1.json b/benchmarks/reference/informative/freqtrade-2026.5.1.json new file mode 100644 index 0000000..55ad577 --- /dev/null +++ b/benchmarks/reference/informative/freqtrade-2026.5.1.json @@ -0,0 +1 @@ +{"cases":[{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:45:00Z","f64:0x4046800000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4025000000000000"]]},"informative_pair":"ETH/USDT","name":"boundary_ffill_false","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:45:00Z","f64:0x4046800000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4025000000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000",null,"f64:0x7ff8000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:45:00Z","f64:0x4046800000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4025000000000000"]]},"informative_pair":"ETH/USDT","name":"boundary_ffill_true","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:45:00Z","f64:0x4046800000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4025000000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000","2024-01-01T00:00:00Z","f64:0x4025000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:00:00Z","f64:0x0000000000000000"],["2024-01-01T00:05:00Z","f64:0x4014000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"5m"},"informative":{"columns":["date","same"],"rows":[["2024-01-01T00:05:00Z","f64:0x401d000000000000"]]},"informative_pair":"ETH/USDT","name":"equal_timeframe","output":{"columns":["date","base","date_5m","same_5m"],"rows":[["2024-01-01T00:00:00Z","f64:0x0000000000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:05:00Z","f64:0x4014000000000000","2024-01-01T00:05:00Z","f64:0x401d000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:45:00Z","f64:0x4046800000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[]},"informative_pair":"ETH/USDT","name":"empty_informative","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:45:00Z","f64:0x4046800000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000",null,"f64:0x7ff8000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T02:00:00Z","f64:0x4034000000000000"]]},"informative_pair":"ETH/USDT","name":"missing_informative_rows","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000"],["2024-01-01T00:55:00Z","f64:0x4000000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T00:00:00Z","f64:0x4034000000000000"]]},"informative_pair":"ETH/USDT","name":"duplicate_cartesian","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T00:55:00Z","f64:0x3ff0000000000000","2024-01-01T00:00:00Z","f64:0x4034000000000000"],["2024-01-01T00:55:00Z","f64:0x4000000000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T00:55:00Z","f64:0x4000000000000000","2024-01-01T00:00:00Z","f64:0x4034000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000"],["2024-01-01T00:55:00Z","f64:0x4000000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T00:00:00Z","f64:0x4034000000000000"]]},"informative_pair":"ETH/USDT","name":"duplicate_cartesian_ffill_true","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T00:55:00Z","f64:0x3ff0000000000000","2024-01-01T00:00:00Z","f64:0x4034000000000000"],["2024-01-01T00:55:00Z","f64:0x4000000000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T00:55:00Z","f64:0x4000000000000000","2024-01-01T00:00:00Z","f64:0x4034000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T01:00:00Z","f64:0x404e000000000000"],["2024-01-01T01:05:00Z","f64:0x4050400000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:00:00Z","f64:0x4034000000000000"]]},"informative_pair":"ETH/USDT","name":"leading_repair","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T01:00:00Z","f64:0x404e000000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:05:00Z","f64:0x4050400000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000","2024-01-01T01:00:00Z","f64:0x4034000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T02:00:00Z","f64:0x405e000000000000"],["2024-01-01T02:05:00Z","f64:0x405f400000000000"],["2024-01-01T02:10:00Z","f64:0x4060400000000000"],["2024-01-01T02:55:00Z","f64:0x4065e00000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info","row_marker"],"rows":[["2024-01-01T00:00:00Z","f64:0x4024000000000000","f64:0x4059000000000000"],["2024-01-01T01:00:00Z","f64:0x4034000000000000","f64:0x4069000000000000"],["2024-01-01T02:00:00Z","f64:0x403e000000000000","f64:0x4072c00000000000"]]},"informative_pair":"ETH/USDT","name":"leading_repair_uses_last_historical_row","output":{"columns":["date","base","date_1h","info_1h","row_marker_1h"],"rows":[["2024-01-01T02:00:00Z","f64:0x405e000000000000","2024-01-01T01:00:00Z","f64:0x4034000000000000","f64:0x4069000000000000"],["2024-01-01T02:05:00Z","f64:0x405f400000000000","2024-01-01T01:00:00Z","f64:0x4034000000000000","f64:0x4069000000000000"],["2024-01-01T02:10:00Z","f64:0x4060400000000000","2024-01-01T01:00:00Z","f64:0x4034000000000000","f64:0x4069000000000000"],["2024-01-01T02:55:00Z","f64:0x4065e00000000000","2024-01-01T02:00:00Z","f64:0x403e000000000000","f64:0x4072c00000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T01:00:00Z","f64:0x404e000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4025000000000000"]]},"informative_pair":"ETH/USDT","name":"unsorted_ffill_false","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T01:00:00Z","f64:0x404e000000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4025000000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000",null,"f64:0x7ff8000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T01:00:00Z","f64:0x404e000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T00:50:00Z","f64:0x4049000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4025000000000000"]]},"informative_pair":"ETH/USDT","name":"unsorted_ffill_true","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:50:00Z","f64:0x4049000000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4025000000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000","2024-01-01T00:00:00Z","f64:0x4025000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T01:00:00Z","f64:0x4034000000000000"],["2024-01-01T00:00:00Z","f64:0x4024000000000000"]]},"informative_pair":"ETH/USDT","name":"unsorted_informative_ffill_false","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000","2024-01-01T01:00:00Z","f64:0x4034000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:55:00Z","f64:0x404b800000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":true,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T01:00:00Z","f64:0x4034000000000000"],["2024-01-01T00:00:00Z","f64:0x4024000000000000"]]},"informative_pair":"ETH/USDT","name":"unsorted_informative_ffill_true","output":{"columns":["date","base","date_1h","info_1h"],"rows":[["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:00:00Z","f64:0x404e000000000000","2024-01-01T00:00:00Z","f64:0x4024000000000000"],["2024-01-01T01:55:00Z","f64:0x405cc00000000000","2024-01-01T01:00:00Z","f64:0x4034000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:50:00Z","f64:0x4049000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"date_column":"candle_open","ffill":false,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["candle_open","custom_info"],"rows":[["2024-01-01T00:00:00Z","f64:0x4056c00000000000"]]},"informative_pair":"ETH/USDT","name":"custom_date_column","output":{"columns":["date","base","candle_open_1h","custom_info_1h"],"rows":[["2024-01-01T00:50:00Z","f64:0x4049000000000000",null,"f64:0x7ff8000000000000"],["2024-01-01T00:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x4056c00000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":false,"ffill":false,"suffix":"btc","timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","signal"],"rows":[["2024-01-01T00:00:00Z","f64:0x4056000000000000"]]},"informative_pair":"ETH/USDT","name":"suffix_naming","output":{"columns":["date","base","date_btc","signal_btc"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000","2024-01-01T00:00:00Z","f64:0x4056000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"1h"},"informative":{"columns":["date","btc_sentinel"],"rows":[["2024-01-01T00:00:00Z","f64:0x40b0928000000000"]]},"informative_pair":"BTC/USDT","name":"cross_pair_sentinel","output":{"columns":["date","base","date_1h","btc_sentinel_1h"],"rows":[["2024-01-01T00:55:00Z","f64:0x3ff0000000000000","2024-01-01T00:00:00Z","f64:0x40b0928000000000"]]}},{"base":{"columns":["date","float_nan","positive_infinity","negative_infinity","positive_zero","negative_zero","python_none","pandas_na"],"rows":[["2024-01-01T00:00:00Z","f64:0x7ff8000000000000","f64:0x7ff0000000000000","f64:0xfff0000000000000","f64:0x0000000000000000","f64:0x8000000000000000",null,null]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"5m"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x3ff0000000000000"]]},"informative_pair":"ETH/USDT","name":"f64_and_pandas_null_encoding","output":{"columns":["date","float_nan","positive_infinity","negative_infinity","positive_zero","negative_zero","python_none","pandas_na","date_5m","info_5m"],"rows":[["2024-01-01T00:00:00Z","f64:0x7ff8000000000000","f64:0x7ff0000000000000","f64:0xfff0000000000000","f64:0x0000000000000000","f64:0x8000000000000000",null,null,"2024-01-01T00:00:00Z","f64:0x3ff0000000000000"]]}},{"base":{"columns":["date","base"],"rows":[["2023-12-31T23:55:00Z","f64:0x4037000000000000"],["2024-01-01T00:00:00Z","f64:0x4038000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"5m"},"informative":{"columns":["date","info_float_nan","info_positive_infinity","info_negative_infinity","info_positive_zero","info_negative_zero","info_python_none","info_pandas_na"],"rows":[["2024-01-01T00:00:00Z","f64:0x7ff8000000000000","f64:0x7ff0000000000000","f64:0xfff0000000000000","f64:0x0000000000000000","f64:0x8000000000000000",null,null]]},"informative_pair":"ETH/USDT","name":"informative_f64_and_pandas_null_encoding","output":{"columns":["date","base","date_5m","info_float_nan_5m","info_positive_infinity_5m","info_negative_infinity_5m","info_positive_zero_5m","info_negative_zero_5m","info_python_none_5m","info_pandas_na_5m"],"rows":[["2023-12-31T23:55:00Z","f64:0x4037000000000000",null,"f64:0x7ff8000000000000","f64:0x7ff8000000000000","f64:0x7ff8000000000000","f64:0x7ff8000000000000","f64:0x7ff8000000000000","f64:0x7ff8000000000000","f64:0x7ff8000000000000"],["2024-01-01T00:00:00Z","f64:0x4038000000000000","2024-01-01T00:00:00Z","f64:0x7ff8000000000000","f64:0x7ff0000000000000","f64:0xfff0000000000000","f64:0x0000000000000000","f64:0x8000000000000000",null,null]]}},{"base":{"columns":["date","base"],"rows":[["2024-01-01T00:00:00Z","f64:0x3ff0000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"1h","timeframe_inf":"5m"},"error":{"message":"Tried to merge a faster timeframe to a slower timeframe.This would create new rows, and can throw off your regular indicators.","type":"ValueError"},"informative":{"columns":["date","info"],"rows":[["2024-01-01T00:00:00Z","f64:0x3ff0000000000000"]]},"informative_pair":"ETH/USDT","name":"faster_timeframe_failure"},{"base":{"columns":["date","base"],"rows":[["2024-01-31T23:50:00Z","f64:0x4049000000000000"],["2024-01-31T23:55:00Z","f64:0x404b800000000000"],["2024-02-01T00:00:00Z","f64:0x0000000000000000"]]},"base_pair":"ETH/USDT","call":{"append_timeframe":true,"ffill":false,"timeframe":"5m","timeframe_inf":"1M"},"informative":{"columns":["date","monthly"],"rows":[["2024-01-01T00:00:00Z","f64:0x403f000000000000"]]},"informative_pair":"ETH/USDT","name":"month_boundary","output":{"columns":["date","base","date_1M","monthly_1M"],"rows":[["2024-01-31T23:50:00Z","f64:0x4049000000000000",null,"f64:0x7ff8000000000000"],["2024-01-31T23:55:00Z","f64:0x404b800000000000","2024-01-01T00:00:00Z","f64:0x403f000000000000"],["2024-02-01T00:00:00Z","f64:0x0000000000000000",null,"f64:0x7ff8000000000000"]]}}],"fingerprint":"0c680501981ae7eb4fff1e4494c7c0d27fee5dcfefbf5131a857c47327440939","schema_version":"freqtrade-informative-fixture-v1","source":{"commit":"6fa470939cc74bf0672e0e348a4d9b293072e43c","strategy_helper":"freqtrade/strategy/strategy_helper.py","strategy_helper_sha256":"46a15179738d83a39148ac96f5ee2f2d50c4514332d059c4458a9d7d3d0e4812","timeframe_to_minutes":"ccxt.Exchange.parse_timeframe(timeframe) // 60","version":"2026.5.1"}} diff --git a/docs/native-vector-core.md b/docs/native-vector-core.md index 4bb8363..d770ae0 100644 --- a/docs/native-vector-core.md +++ b/docs/native-vector-core.md @@ -64,15 +64,77 @@ TA-Lib multi-output calls are represented by generic output names such as `aroondown`, `fastk`, or `upperband`; tuple assignment and constant output subscripts compile without Signal- or strategy-specific branches. +## Informative frames and resampling + +M20-05 pins informative alignment to Freqtrade 2026.5.1. Every source frame +has an explicit `(pair, timeframe)` identity; a BTC informative frame cannot be +silently substituted for the traded pair, even when its column names match. +For slower timeframes, visibility begins at: + +```text +informative open + informative merge minutes - base merge minutes +``` + +Consequently, a 1h candle opened at `00:00` first becomes visible to a 5m base +frame at `00:55`, never earlier. Freqtrade obtains both merge durations with +CCXT seconds divided by 60, so sub-minute timeframes deliberately use its +floor-minute behavior while resampling retains their real seconds. +`ffill=False` performs an exact-key left join, +leaving missing matches null and preserving duplicate Cartesian rows. +`ffill=True` carries only an already-visible historical row. Monthly `1M` +visibility uses the next calendar month start minus the base duration. + +The preprocessing contract mirrors the pinned Freqtrade anchors: ordinary +timeframes use fixed-second bins, weekly data uses Monday anchors, months use +month starts (`nMS`), and years use year starts (`nYS`). In particular, the +pinned multiweek branch maps `2w` to `1W-MON`. Cleanup and optional OHLCV gap +filling remain separate from informative merge; the merge does not synthesize +candles. + +Complete-frame alignment reproduces the official missing, duplicate, unsorted, +suffix, monthly, and cross-pair cases. Streaming alignment requires ordered +chunks and explicit identities, retains at most one historical informative row, +and rejects future rows, timestamp regressions, schema drift, unsupported +faster-frame joins, and column collisions at the source location instead of +guessing. Because Freqtrade can repair the whole prefix before its first exact +match, a bounded `ffill=True` stream requires that match in its first non-empty +base chunk; otherwise it stops before emitting provisional rows. + +Latest X7 performs its merges with `ffill=False` and then calls one final +source-order `DataFrame.ffill()`. The Rust fill primitive treats both null and +NaN as missing, preserves the bits of retained finite values, infinities, and +signed zero, and retains at most one value per column across chunks. It never +adds or removes candle rows. + +The committed oracle was generated by the official Freqtrade helper at tag +2026.5.1 (commit `6fa47093`). Regeneration is an evidence-authoring command and +requires that exact checkout at +`.nfi/roadmap-acceptance/M20-05/freqtrade-2026.5.1`: + +```bash +uv run python scripts/generate_informative_fixture.py +``` + +It writes +`benchmarks/reference/informative/freqtrade-2026.5.1.json`; both the Python +compatibility layer and Rust alignment tests replay the same cases exactly. + ## Current claim boundary The latest X7 indicator operation set now has exact Native kernels. This does not -yet claim that the entire latest X7 source compiles: compilation currently stops -at `strategy.py:3168:21` on an informative/config attribute that belongs to the -source and multi-timeframe lowering work in M20-05. EWM remains fail-closed -because it is not reachable in this upstream inventory. Unknown functions, -non-SMA TA-Lib MA types, centered rolling windows, Arrow null indicator inputs, -and unselected multi-output calls also stop rather than being guessed. +yet claim that the entire latest X7 source compiles or executes Full Native. +M20-05 establishes exact multi-timeframe primitives and compiler contracts, +but their in-memory `VectorEngine` connection is deliberately owned by M21. +M21 also owns Native +signal assignments, tag generation, independent Python/Rust vector shadowing, +and the in-memory simulator connection. M22 owns latest-upstream Spot/Futures +full-state qualification, removal of Python strategy execution from the Native +lane, and release certification. Until those proofs pass, unsupported source +constructs remain fail-closed and the official Freqtrade fallback remains the +execution path. EWM also remains fail-closed because it is not reachable in the +current upstream inventory. Unknown functions, non-SMA TA-Lib MA types, centered +rolling windows, Arrow null indicator inputs, and unselected multi-output calls +stop rather than being guessed. The Rust boundary can validate a compiled contract directly: diff --git a/python/nfi_backtest_engine/indicator_program.py b/python/nfi_backtest_engine/indicator_program.py index 82e84a2..f80af8f 100644 --- a/python/nfi_backtest_engine/indicator_program.py +++ b/python/nfi_backtest_engine/indicator_program.py @@ -47,6 +47,22 @@ } _SCALAR_CALLS = {"abs", "bool", "float", "int", "max", "min"} _WINDOW_REDUCERS = {"max", "mean", "min", "std", "sum"} +_INFORMATIVE_MERGE_PARAMETERS = ( + "dataframe", + "informative", + "timeframe", + "timeframe_inf", + "ffill", + "append_timeframe", + "date_column", + "suffix", +) +_INFORMATIVE_MERGE_DEFAULTS: Mapping[str, Any] = { + "ffill": True, + "append_timeframe": True, + "date_column": "date", + "suffix": None, +} class IndicatorProgramCompileError(StrategyAnalysisError): @@ -763,7 +779,12 @@ def method_call(self, node: ast.Call, callable_name: str) -> str: self.node_types[base], inputs=[base], parameters={"direction": "forward"}, - lookback=self.lookback(base), + lookback={ + "kind": "recursive", + "candles": None, + "expression": _safe_expression(node), + "causal": bool(self.lookback(base)["causal"]), + }, ) if method in {"bfill", "backfill"}: self.unsupported(node, "backward fill would look ahead") @@ -816,19 +837,68 @@ def window_call(self, node: ast.Call) -> str | None: ) def informative_merge(self, node: ast.Call) -> str: - if len(node.args) < 4: - self.unsupported(node, "informative merge signature") - base = self.expression(node.args[0]) - informative = self.expression(node.args[1]) + arguments = _bind_informative_merge_arguments(node, self) + base = self.expression(arguments["dataframe"]) + informative = self.expression(arguments["informative"]) if self.node_types[base] != "dataframe" or self.node_types[informative] != "dataframe": self.unsupported(node, "informative merge dataframe inputs") - base_timeframe = _static_value(node.args[2], self.class_constants) - informative_timeframe = _static_value(node.args[3], self.class_constants) + + timeframe_node = arguments["timeframe"] + informative_timeframe_node = arguments["timeframe_inf"] + base_timeframe = _required_static(timeframe_node, self) + informative_timeframe = _required_static(informative_timeframe_node, self) if not isinstance(base_timeframe, str) or not isinstance(informative_timeframe, str): self.unsupported(node, "dynamic informative merge timeframe") - keyword_values = _keyword_value_map(node, self) - if keyword_values.get("ffill", False) is not False: - self.unsupported(node, "informative merge must not fill before source-ordered fill") + + ffill = _static_informative_option(arguments, "ffill", self) + append_timeframe = _static_informative_option(arguments, "append_timeframe", self) + date_column = _static_informative_option(arguments, "date_column", self) + suffix = _static_informative_option(arguments, "suffix", self) + if not isinstance(ffill, bool): + self.unsupported(arguments.get("ffill", node), "non-boolean informative merge ffill") + if not isinstance(append_timeframe, bool): + self.unsupported( + arguments.get("append_timeframe", node), + "non-boolean informative merge append_timeframe", + ) + if not isinstance(date_column, str): + self.unsupported( + arguments.get("date_column", node), + "non-string informative merge date_column", + ) + if suffix is not None and not isinstance(suffix, str): + self.unsupported(arguments.get("suffix", node), "non-string informative merge suffix") + if suffix and append_timeframe: + self.unsupported( + arguments.get("suffix", node), + "informative merge suffix conflicts with append_timeframe", + ) + normalized_suffix = suffix or None + if not append_timeframe and normalized_suffix is None: + self.unsupported( + arguments.get("append_timeframe", node), + "informative merge without an output suffix", + ) + + base_minutes = _freqtrade_timeframe_minutes(timeframe_node, base_timeframe, self) + informative_minutes = _freqtrade_timeframe_minutes( + informative_timeframe_node, + informative_timeframe, + self, + ) + if base_minutes > informative_minutes: + self.unsupported( + informative_timeframe_node, + "faster informative timeframe would create rows", + ) + lookback = self.merged_lookback([base, informative]) + if ffill: + lookback = { + "kind": "recursive", + "candles": None, + "expression": _safe_expression(node), + "causal": bool(lookback["causal"]), + } merged = self.emit( node, "informative-merge", @@ -837,9 +907,12 @@ def informative_merge(self, node: ast.Call) -> str: parameters={ "base_timeframe": base_timeframe, "informative_timeframe": informative_timeframe, - "ffill": False, + "ffill": ffill, + "append_timeframe": append_timeframe, + "date_column": date_column, + "suffix": normalized_suffix, }, - lookback=self.merged_lookback([base, informative]), + lookback=lookback, ) self.informative_nodes.append(merged) return merged @@ -951,6 +1024,68 @@ def _window_parameters( return values +def _bind_informative_merge_arguments( + node: ast.Call, + compiler: _Compiler, +) -> dict[str, ast.expr]: + if len(node.args) > len(_INFORMATIVE_MERGE_PARAMETERS): + compiler.unsupported(node, "informative merge signature") + arguments: dict[str, ast.expr] = dict( + zip(_INFORMATIVE_MERGE_PARAMETERS, node.args, strict=False) + ) + for keyword in node.keywords: + if keyword.arg is None: + compiler.unsupported(keyword.value, "expanded informative merge keyword arguments") + if keyword.arg not in _INFORMATIVE_MERGE_PARAMETERS: + compiler.unsupported( + keyword.value, + f"unknown informative merge keyword {keyword.arg}", + ) + if keyword.arg in arguments: + compiler.unsupported( + keyword.value, + f"duplicate informative merge argument {keyword.arg}", + ) + arguments[keyword.arg] = keyword.value + missing = [name for name in _INFORMATIVE_MERGE_PARAMETERS[:4] if name not in arguments] + if missing: + compiler.unsupported(node, f"informative merge missing arguments: {missing}") + return arguments + + +def _static_informative_option( + arguments: Mapping[str, ast.expr], + name: str, + compiler: _Compiler, +) -> Any: + node = arguments.get(name) + if node is None: + return _INFORMATIVE_MERGE_DEFAULTS[name] + return _required_static(node, compiler) + + +def _freqtrade_timeframe_minutes( + node: ast.expr, + timeframe: str, + compiler: _Compiler, +) -> int: + """Mirror pinned CCXT parsing used by Freqtrade's timeframe helper.""" + try: + amount = int(timeframe[:-1]) + scale = { + "y": 31_536_000, + "M": 2_592_000, + "w": 604_800, + "d": 86_400, + "h": 3_600, + "m": 60, + "s": 1, + }[timeframe[-1]] + except (IndexError, KeyError, ValueError): + compiler.unsupported(node, f"invalid informative merge timeframe {timeframe!r}") + return amount * scale // 60 + + def _literal_keyword_arguments( node: ast.Call, compiler: _Compiler, diff --git a/python/nfi_backtest_engine/informative_fixture.py b/python/nfi_backtest_engine/informative_fixture.py new file mode 100644 index 0000000..71fd862 --- /dev/null +++ b/python/nfi_backtest_engine/informative_fixture.py @@ -0,0 +1,542 @@ +"""Generate deterministic evidence from the pinned Freqtrade informative merge helper.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import math +import struct +import sys +import types +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import cast + +import ccxt +import numpy as np +import pandas as pd + +PINNED_SOURCE = Path(".nfi/roadmap-acceptance/M20-05/freqtrade-2026.5.1") +FIXTURE_PATH = Path("benchmarks/reference/informative/freqtrade-2026.5.1.json") +_Merge = Callable[..., pd.DataFrame] + + +def generate_fixture(source_root: Path | None = None) -> dict[str, object]: + """Execute the official helper for every compact informative-merge case.""" + root = source_root or _repository_root() / PINNED_SOURCE + helper = root / "freqtrade/strategy/strategy_helper.py" + merge = _load_official_merge(helper) + cases = execute_cases(merge) + fixture: dict[str, object] = { + "schema_version": "freqtrade-informative-fixture-v1", + "source": { + "version": _source_version(root), + "commit": _source_commit(root), + "strategy_helper_sha256": _sha256_file(helper), + "strategy_helper": "freqtrade/strategy/strategy_helper.py", + "timeframe_to_minutes": "ccxt.Exchange.parse_timeframe(timeframe) // 60", + }, + "cases": cases, + } + fixture["fingerprint"] = canonical_sha256(fixture) + return fixture + + +def execute_cases(merge: _Merge) -> list[dict[str, object]]: + """Run the canonical case matrix against one merge implementation.""" + return [_execute_case(merge, spec) for spec in _case_specs()] + + +def write_fixture(destination: Path, source_root: Path | None = None) -> dict[str, object]: + """Generate and persist canonical fixture evidence.""" + fixture = generate_fixture(source_root) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(_canonical_json(fixture) + "\n", encoding="utf-8") + return fixture + + +def canonical_sha256(document: Mapping[str, object]) -> str: + """Hash a fixture while excluding its self-referential fingerprint.""" + identity = {key: value for key, value in document.items() if key != "fingerprint"} + return hashlib.sha256(_canonical_json(identity).encode("utf-8")).hexdigest() + + +def _execute_case(merge: _Merge, spec: Mapping[str, object]) -> dict[str, object]: + call = cast(dict[str, object], spec["call"]) + date_column = str(call.get("date_column", "date")) + base = _frame(spec["base"], date_columns=("date",)) + informative = _frame(spec["informative"], date_columns=(date_column,)) + result: dict[str, object] = { + "name": spec["name"], + "base_pair": spec["base_pair"], + "informative_pair": spec["informative_pair"], + "call": spec["call"], + "base": _encode_frame(base), + "informative": _encode_frame(informative), + } + try: + output = merge(base, informative, **call) + except Exception as exc: # The pinned helper's errors are part of the oracle. + result["error"] = {"type": type(exc).__name__, "message": str(exc)} + else: + result["output"] = _encode_frame(output) + return result + + +def _case_specs() -> list[dict[str, object]]: + base_hour = _rows( + ("date", "base"), + ( + ("2024-01-01T00:45:00Z", 45.0), + ("2024-01-01T00:50:00Z", 50.0), + ("2024-01-01T00:55:00Z", 55.0), + ("2024-01-01T01:00:00Z", 60.0), + ), + ) + informative_hour = _rows(("date", "info"), (("2024-01-01T00:00:00Z", 10.5),)) + return [ + _case("boundary_ffill_false", base_hour, informative_hour, "5m", "1h", ffill=False), + _case("boundary_ffill_true", base_hour, informative_hour, "5m", "1h", ffill=True), + _case( + "equal_timeframe", + _rows(("date", "base"), (("2024-01-01T00:00:00Z", 0.0), ("2024-01-01T00:05:00Z", 5.0))), + _rows(("date", "same"), (("2024-01-01T00:05:00Z", 7.25),)), + "5m", + "5m", + ffill=False, + ), + _case( + "empty_informative", + base_hour, + _rows(("date", "info"), ()), + "5m", + "1h", + ffill=True, + ), + _case( + "missing_informative_rows", + _rows( + ("date", "base"), + ( + ("2024-01-01T00:55:00Z", 55.0), + ("2024-01-01T01:00:00Z", 60.0), + ("2024-01-01T01:55:00Z", 115.0), + ), + ), + _rows( + ("date", "info"), + (("2024-01-01T00:00:00Z", 10.0), ("2024-01-01T02:00:00Z", 20.0)), + ), + "5m", + "1h", + ffill=True, + ), + _case( + "duplicate_cartesian", + _rows( + ("date", "base"), + (("2024-01-01T00:55:00Z", 1.0), ("2024-01-01T00:55:00Z", 2.0)), + ), + _rows( + ("date", "info"), + (("2024-01-01T00:00:00Z", 10.0), ("2024-01-01T00:00:00Z", 20.0)), + ), + "5m", + "1h", + ffill=False, + ), + _case( + "duplicate_cartesian_ffill_true", + _rows( + ("date", "base"), + (("2024-01-01T00:55:00Z", 1.0), ("2024-01-01T00:55:00Z", 2.0)), + ), + _rows( + ("date", "info"), + (("2024-01-01T00:00:00Z", 10.0), ("2024-01-01T00:00:00Z", 20.0)), + ), + "5m", + "1h", + ffill=True, + ), + _case( + "leading_repair", + _rows( + ("date", "base"), + ( + ("2024-01-01T01:00:00Z", 60.0), + ("2024-01-01T01:05:00Z", 65.0), + ("2024-01-01T01:55:00Z", 115.0), + ), + ), + _rows( + ("date", "info"), + (("2024-01-01T00:00:00Z", 10.0), ("2024-01-01T01:00:00Z", 20.0)), + ), + "5m", + "1h", + ffill=True, + ), + _case( + "leading_repair_uses_last_historical_row", + _rows( + ("date", "base"), + ( + ("2024-01-01T02:00:00Z", 120.0), + ("2024-01-01T02:05:00Z", 125.0), + ("2024-01-01T02:10:00Z", 130.0), + ("2024-01-01T02:55:00Z", 175.0), + ), + ), + _rows( + ("date", "info", "row_marker"), + ( + ("2024-01-01T00:00:00Z", 10.0, 100.0), + ("2024-01-01T01:00:00Z", 20.0, 200.0), + ("2024-01-01T02:00:00Z", 30.0, 300.0), + ), + ), + "5m", + "1h", + ffill=True, + ), + _case( + "unsorted_ffill_false", + _rows( + ("date", "base"), + ( + ("2024-01-01T01:00:00Z", 60.0), + ("2024-01-01T00:55:00Z", 55.0), + ("2024-01-01T00:50:00Z", 50.0), + ), + ), + informative_hour, + "5m", + "1h", + ffill=False, + ), + _case( + "unsorted_ffill_true", + _rows( + ("date", "base"), + ( + ("2024-01-01T01:00:00Z", 60.0), + ("2024-01-01T00:55:00Z", 55.0), + ("2024-01-01T00:50:00Z", 50.0), + ), + ), + informative_hour, + "5m", + "1h", + ffill=True, + ), + _case( + "unsorted_informative_ffill_false", + _rows( + ("date", "base"), + ( + ("2024-01-01T00:55:00Z", 55.0), + ("2024-01-01T01:55:00Z", 115.0), + ), + ), + _rows( + ("date", "info"), + (("2024-01-01T01:00:00Z", 20.0), ("2024-01-01T00:00:00Z", 10.0)), + ), + "5m", + "1h", + ffill=False, + ), + _case( + "unsorted_informative_ffill_true", + _rows( + ("date", "base"), + ( + ("2024-01-01T00:55:00Z", 55.0), + ("2024-01-01T01:00:00Z", 60.0), + ("2024-01-01T01:55:00Z", 115.0), + ), + ), + _rows( + ("date", "info"), + (("2024-01-01T01:00:00Z", 20.0), ("2024-01-01T00:00:00Z", 10.0)), + ), + "5m", + "1h", + ffill=True, + ), + _case( + "custom_date_column", + _rows( + ("date", "base"), + (("2024-01-01T00:50:00Z", 50.0), ("2024-01-01T00:55:00Z", 55.0)), + ), + _rows(("candle_open", "custom_info"), (("2024-01-01T00:00:00Z", 91.0),)), + "5m", + "1h", + ffill=False, + date_column="candle_open", + ), + _case( + "suffix_naming", + _rows(("date", "base"), (("2024-01-01T00:55:00Z", 1.0),)), + _rows(("date", "signal"), (("2024-01-01T00:00:00Z", 88.0),)), + "5m", + "1h", + ffill=False, + append_timeframe=False, + suffix="btc", + ), + _case( + "cross_pair_sentinel", + _rows(("date", "base"), (("2024-01-01T00:55:00Z", 1.0),)), + _rows(("date", "btc_sentinel"), (("2024-01-01T00:00:00Z", 4242.5),)), + "5m", + "1h", + ffill=False, + base_pair="ETH/USDT", + informative_pair="BTC/USDT", + ), + _case( + "f64_and_pandas_null_encoding", + _rows( + ( + "date", + "float_nan", + "positive_infinity", + "negative_infinity", + "positive_zero", + "negative_zero", + "python_none", + "pandas_na", + ), + ( + ( + "2024-01-01T00:00:00Z", + float("nan"), + float("inf"), + float("-inf"), + 0.0, + -0.0, + None, + pd.NA, + ), + ), + ), + _rows(("date", "info"), (("2024-01-01T00:00:00Z", 1.0),)), + "5m", + "5m", + ffill=False, + ), + _case( + "informative_f64_and_pandas_null_encoding", + _rows( + ("date", "base"), + ( + ("2023-12-31T23:55:00Z", 23.0), + ("2024-01-01T00:00:00Z", 24.0), + ), + ), + _rows( + ( + "date", + "info_float_nan", + "info_positive_infinity", + "info_negative_infinity", + "info_positive_zero", + "info_negative_zero", + "info_python_none", + "info_pandas_na", + ), + ( + ( + "2024-01-01T00:00:00Z", + float("nan"), + float("inf"), + float("-inf"), + 0.0, + -0.0, + None, + pd.NA, + ), + ), + ), + "5m", + "5m", + ffill=False, + ), + _case( + "faster_timeframe_failure", + _rows(("date", "base"), (("2024-01-01T00:00:00Z", 1.0),)), + _rows(("date", "info"), (("2024-01-01T00:00:00Z", 1.0),)), + "1h", + "5m", + ffill=False, + ), + _case( + "month_boundary", + _rows( + ("date", "base"), + ( + ("2024-01-31T23:50:00Z", 50.0), + ("2024-01-31T23:55:00Z", 55.0), + ("2024-02-01T00:00:00Z", 0.0), + ), + ), + _rows(("date", "monthly"), (("2024-01-01T00:00:00Z", 31.0),)), + "5m", + "1M", + ffill=False, + ), + ] + + +def _case( + name: str, + base: dict[str, list[object]], + informative: dict[str, list[object]], + timeframe: str, + informative_timeframe: str, + *, + ffill: bool, + append_timeframe: bool = True, + suffix: str | None = None, + date_column: str | None = None, + base_pair: str = "ETH/USDT", + informative_pair: str = "ETH/USDT", +) -> dict[str, object]: + call: dict[str, object] = { + "timeframe": timeframe, + "timeframe_inf": informative_timeframe, + "ffill": ffill, + "append_timeframe": append_timeframe, + } + if suffix is not None: + call["suffix"] = suffix + if date_column is not None: + call["date_column"] = date_column + return { + "name": name, + "base_pair": base_pair, + "informative_pair": informative_pair, + "call": call, + "base": base, + "informative": informative, + } + + +def _rows( + columns: tuple[str, ...], rows: tuple[tuple[object, ...], ...] +) -> dict[str, list[object]]: + return {column: [row[index] for row in rows] for index, column in enumerate(columns)} + + +def _frame(columns: object, *, date_columns: tuple[str, ...]) -> pd.DataFrame: + if not isinstance(columns, dict): + raise TypeError("fixture frame columns must be a dictionary") + frame = pd.DataFrame(columns) + for column in date_columns: + if column in frame: + frame[column] = pd.to_datetime(frame[column], utc=True) + return frame + + +def _encode_frame(frame: pd.DataFrame) -> dict[str, object]: + return { + "columns": list(frame.columns), + "rows": [ + [_encode_value(value) for value in row] + for row in frame.itertuples(index=False, name=None) + ], + } + + +def _encode_value(value: object) -> object: + if value is None or value is pd.NA or value is pd.NaT: + return None + if isinstance(value, pd.Timestamp): + return _timestamp(value) + if isinstance(value, np.floating | float): + number = float(value) + if math.isnan(number): + return "f64:0x7ff8000000000000" + return f"f64:0x{struct.pack('>d', number).hex()}" + if isinstance(value, np.integer): + return int(value) + return value + + +def _timestamp(value: pd.Timestamp) -> str: + value = value.tz_localize("UTC") if value.tzinfo is None else value.tz_convert("UTC") + return value.isoformat().replace("+00:00", "Z") + + +def _load_official_merge(helper: Path) -> _Merge: + if not helper.is_file(): + raise FileNotFoundError(f"pinned Freqtrade strategy helper is missing: {helper}") + module_name = "_nfi_pinned_freqtrade_strategy_helper" + module_names = ("freqtrade", "freqtrade.exchange", module_name) + saved = {name: sys.modules.get(name) for name in module_names} + freqtrade = types.ModuleType("freqtrade") + exchange = types.ModuleType("freqtrade.exchange") + exchange.timeframe_to_minutes = _official_timeframe_to_minutes # type: ignore[attr-defined] + freqtrade.exchange = exchange # type: ignore[attr-defined] + sys.modules["freqtrade"] = freqtrade + sys.modules["freqtrade.exchange"] = exchange + try: + spec = importlib.util.spec_from_file_location(module_name, helper) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load pinned Freqtrade helper: {helper}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + merge = getattr(module, "merge_informative_pair", None) + if not callable(merge): + raise ImportError("pinned Freqtrade helper has no merge_informative_pair") + return cast(_Merge, merge) + finally: + for name, previous in saved.items(): + if previous is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous + + +def _official_timeframe_to_minutes(timeframe: str) -> int: + return ccxt.Exchange.parse_timeframe(timeframe) // 60 + + +def _source_commit(root: Path) -> str: + head = root / ".git/HEAD" + if not head.is_file(): + raise FileNotFoundError(f"pinned Freqtrade commit metadata is missing: {head}") + reference = head.read_text(encoding="utf-8").strip() + if not reference.startswith("ref: "): + return reference + ref_path = root / ".git" / reference.removeprefix("ref: ") + if not ref_path.is_file(): + raise FileNotFoundError(f"pinned Freqtrade ref is missing: {ref_path}") + return ref_path.read_text(encoding="utf-8").strip() + + +def _source_version(root: Path) -> str: + version_file = root / "freqtrade/__init__.py" + for line in version_file.read_text(encoding="utf-8").splitlines(): + if line.startswith("__version__ = "): + return line.split('"', maxsplit=2)[1] + raise ValueError(f"pinned Freqtrade version is missing: {version_file}") + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _canonical_json(value: object) -> str: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ) + + +def _repository_root() -> Path: + return Path(__file__).resolve().parents[2] diff --git a/python/nfi_backtest_engine/strategy_compat.py b/python/nfi_backtest_engine/strategy_compat.py index 1812d2d..4b90461 100644 --- a/python/nfi_backtest_engine/strategy_compat.py +++ b/python/nfi_backtest_engine/strategy_compat.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +import ccxt import pandas as pd from .errors import StrategyAnalysisError @@ -152,59 +153,86 @@ def merge_informative_pair( informative: pd.DataFrame, timeframe: str, timeframe_inf: str, - *, - ffill: bool = False, + ffill: bool = True, append_timeframe: bool = True, date_column: str = "date", + suffix: str | None = None, ) -> pd.DataFrame: - """Align an informative candle to base candles without lookahead.""" - if date_column not in dataframe or date_column not in informative: - raise StrategyAnalysisError("informative merge requires date columns") + """Mirror Freqtrade's informative merge without exposing future candles.""" base_minutes = timeframe_minutes(timeframe) informative_minutes = timeframe_minutes(timeframe_inf) - if informative_minutes < base_minutes: - raise StrategyAnalysisError("informative timeframe cannot be smaller than base timeframe") prepared = informative.copy(deep=True) - merge_column = "__nfi_merge_date" + merge_column = "date_merge" if informative_minutes == base_minutes: prepared[merge_column] = prepared[date_column] + elif base_minutes < informative_minutes: + if prepared.empty: + prepared[merge_column] = prepared[date_column] + elif timeframe_inf == "1M": + prepared[merge_column] = ( + prepared[date_column] + pd.offsets.MonthBegin(1) + ) - pd.to_timedelta(base_minutes, "m") + else: + prepared[merge_column] = ( + prepared[date_column] + + pd.to_timedelta(informative_minutes, "m") + - pd.to_timedelta(base_minutes, "m") + ) else: - prepared[merge_column] = prepared[date_column] + pd.to_timedelta( - informative_minutes - base_minutes, - unit="m", + raise ValueError( + "Tried to merge a faster timeframe to a slower timeframe." + "This would create new rows, and can throw off your regular indicators." ) + + if suffix and append_timeframe: + raise ValueError("You can not specify `append_timeframe` as True and a `suffix`.") if append_timeframe: - prepared.rename( - columns={ - column: f"{column}_{timeframe_inf}" - for column in prepared.columns - if column != merge_column - }, - inplace=True, + merge_column = f"date_merge_{timeframe_inf}" + prepared.columns = [f"{column}_{timeframe_inf}" for column in prepared.columns] + elif suffix: + merge_column = f"date_merge_{suffix}" + prepared.columns = [f"{column}_{suffix}" for column in prepared.columns] + + if not ffill: + result = pd.merge( + dataframe, + prepared, + left_on="date", + right_on=merge_column, + how="left", ) + return result.drop(merge_column, axis=1) + result = pd.merge_ordered( dataframe, prepared, - left_on=date_column, + fill_method="ffill", + left_on="date", right_on=merge_column, - fill_method="ffill" if ffill else None, how="left", ) - result.drop(columns=[merge_column], inplace=True) - return result + if len(result) > 1 and len(prepared) > 0 and pd.isnull(result.at[0, merge_column]): + first_valid_index = result[merge_column].first_valid_index() + if isinstance(first_valid_index, int) and first_valid_index > 0: + first_valid_date = result.at[first_valid_index, merge_column] + historical = prepared[prepared[merge_column] < first_valid_date] + if not historical.empty: + result.loc[: first_valid_index - 1] = result.loc[ + : first_valid_index - 1 + ].fillna(historical.iloc[-1]) + return result.drop(merge_column, axis=1) def timeframe_minutes(timeframe: str) -> int: - units = {"m": 1, "h": 60, "d": 1440, "w": 10080} - if len(timeframe) < 2 or timeframe[-1] not in units: - raise StrategyAnalysisError(f"unsupported timeframe: {timeframe}") + return timeframe_seconds(timeframe) // 60 + + +def timeframe_seconds(timeframe: str) -> int: + """Use the same CCXT timeframe parser as pinned Freqtrade.""" try: - count = int(timeframe[:-1]) - except ValueError as exc: + return ccxt.Exchange.parse_timeframe(timeframe) + except (TypeError, ValueError) as exc: raise StrategyAnalysisError(f"unsupported timeframe: {timeframe}") from exc - if count <= 0: - raise StrategyAnalysisError(f"unsupported timeframe: {timeframe}") - return count * units[timeframe[-1]] def _copy_json(value: Any) -> Any: diff --git a/python/nfi_backtest_engine/vector_worker.py b/python/nfi_backtest_engine/vector_worker.py index 8a2576d..31535fa 100644 --- a/python/nfi_backtest_engine/vector_worker.py +++ b/python/nfi_backtest_engine/vector_worker.py @@ -23,6 +23,7 @@ load_strategy_class, prepare_worker_config, timeframe_minutes, + timeframe_seconds, ) from .timerange import parse_timerange_milliseconds from .vector_manifest import EMPTY_TAG_TRANSPORT_SENTINEL @@ -424,15 +425,30 @@ def _clean_ohlcv_like_freqtrade( def _freqtrade_resample_frequency(timeframe: str) -> str: """Return the resample anchor used by pinned Freqtrade 2026.5.1.""" - if timeframe.endswith("w"): + if timeframe == "1y": + return "1YS" + + calendar_seconds = {"M": 2_592_000, "y": 31_536_000} + unit = timeframe[-1:] if timeframe else "" + if unit in calendar_seconds: try: - weeks = int(timeframe[:-1]) + count = int(timeframe[:-1]) except ValueError as exc: raise StrategyAnalysisError(f"unsupported timeframe: {timeframe}") from exc - if weeks <= 0: + if count <= 0: raise StrategyAnalysisError(f"unsupported timeframe: {timeframe}") - return f"{weeks}W-MON" - return f"{timeframe_minutes(timeframe) * 60}s" + seconds = count * calendar_seconds[unit] + else: + seconds = timeframe_seconds(timeframe) + + minutes = seconds // 60 + if 10_000 < minutes < 43_200: + return "1W-MON" + if 43_200 <= minutes < 525_600: + return f"{timeframe}S" + if minutes > 43_200: + return timeframe + return f"{seconds}s" def _trim_timerange( diff --git a/rust/crates/nfi-vector-core/src/alignment/batch.rs b/rust/crates/nfi-vector-core/src/alignment/batch.rs new file mode 100644 index 0000000..194e79d --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/batch.rs @@ -0,0 +1,105 @@ +use std::collections::BTreeMap; + +use crate::VectorCoreError; + +use super::model::{MergeSpec, MergedFrame, NumericFrame}; +use super::support::{ + informative_events, ordered_by_timestamp, output_names, validate_frame, validate_identity, + InformativeEvent, OutputBuilder, +}; + +/// Merges one complete informative frame onto a complete base frame. +/// +/// Duplicate informative keys intentionally produce the same Cartesian rows as +/// pandas `merge`/`merge_ordered`; base order is never sorted or rewritten. +/// When `ffill` is enabled, `merge_ordered` instead sorts base timestamps. +/// +/// # Errors +/// +/// Returns a source-located error for an invalid spec, mismatched frame, +/// collision, or unrepresentable timestamp. Informative input is stably +/// ordered by its effective timestamp, matching pandas merge behavior. +pub fn merge( + base: &NumericFrame, + informative: &NumericFrame, + spec: &MergeSpec, +) -> Result { + spec.validate()?; + validate_identity(base, &spec.base, spec)?; + validate_identity(informative, &spec.informative, spec)?; + validate_frame(base, spec)?; + validate_frame(informative, spec)?; + let ordered_base; + let base = if spec.ffill { + ordered_base = ordered_by_timestamp(base); + &ordered_base + } else { + base + }; + let names = output_names(base, informative, spec)?; + let events = informative_events(informative, spec)?; + let mut exact = BTreeMap::>::new(); + for event in &events { + exact + .entry(event.key_ms) + .or_default() + .push(event.source_row); + } + + let mut output = OutputBuilder::new(base, informative, &names); + if spec.ffill { + merge_ffill(base, &events, &exact, &mut output); + } else { + for base_row in 0..base.timestamps_ms.len() { + let matches = exact.get(&base.timestamps_ms[base_row]); + output.extend(base, base_row, matches.map(Vec::as_slice)); + } + } + Ok(output.finish(base.identity.clone())) +} + +fn merge_ffill( + base: &NumericFrame, + events: &[InformativeEvent], + exact: &BTreeMap>, + output: &mut OutputBuilder<'_>, +) { + let first_match = base + .timestamps_ms + .iter() + .find(|timestamp| exact.contains_key(timestamp)) + .copied(); + let leading_history = first_match.and_then(|first| { + events + .iter() + .rev() + .find(|event| event.key_ms < first) + .map(|event| event.source_row) + }); + let mut event_cursor = 0; + let mut last = None; + for base_row in 0..base.timestamps_ms.len() { + let timestamp = base.timestamps_ms[base_row]; + while event_cursor < events.len() && events[event_cursor].key_ms <= timestamp { + last = Some(events[event_cursor].source_row); + event_cursor += 1; + } + if let Some(matches) = exact.get(×tamp) { + output.extend(base, base_row, Some(matches)); + } else if first_match.is_some_and(|first| timestamp < first) { + if let Some(row) = leading_history { + output.extend(base, base_row, Some(&[row])); + } else { + output.extend(base, base_row, None); + } + } else if first_match.is_some() { + if let Some(row) = last { + output.extend(base, base_row, Some(&[row])); + } else { + output.extend(base, base_row, None); + } + } else { + output.extend(base, base_row, None); + } + } +} diff --git a/rust/crates/nfi-vector-core/src/alignment/fill.rs b/rust/crates/nfi-vector-core/src/alignment/fill.rs new file mode 100644 index 0000000..6ae9547 --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/fill.rs @@ -0,0 +1,479 @@ +//! Source-order forward fill for fully aligned frames. +//! +//! This is the final `DataFrame.ffill()` step used by NFI after every +//! informative merge. It deliberately does not inspect timestamps: pandas +//! fills in current row order, and merge is responsible for determining that +//! order before this operation runs. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::VectorCoreError; + +use super::{FrameIdentity, MergedFrame, SourceLocation}; + +/// Forward-fills every numeric and informative-date column in source order. +/// +/// `None` and present `NaN` numeric cells are both missing. Leading missing +/// cells keep their original representation, while a missing cell after a +/// value receives that value bit-for-bit. Timestamps and row count are never +/// changed. +/// +/// # Errors +/// +/// Returns a source-located error when any column length differs from the +/// frame's timestamp length. +pub fn forward_fill( + frame: &MergedFrame, + source: &SourceLocation, +) -> Result { + ForwardFillStream::new(source.clone()).execute(frame) +} + +/// Bounded state for source-order forward fill across arbitrary chunks. +/// +/// At most one non-missing value is retained for each numeric and +/// informative-date column. The first chunk fixes the frame schema; +/// later identity or column changes fail closed instead of reusing values +/// across unrelated frames. +#[derive(Debug)] +pub struct ForwardFillStream { + source: SourceLocation, + schema: Option, + last_numeric: BTreeMap, + last_dates: BTreeMap, +} + +impl ForwardFillStream { + /// Creates a forward-fill stream with no retained values. + #[must_use] + pub fn new(source: SourceLocation) -> Self { + Self { + source, + schema: None, + last_numeric: BTreeMap::new(), + last_dates: BTreeMap::new(), + } + } + + /// Forward-fills one chunk and retains only its last non-missing values. + /// + /// # Errors + /// + /// Returns a source-located error for malformed column lengths or a frame + /// identity/numeric/date schema that differs from the first chunk. + pub fn execute(&mut self, frame: &MergedFrame) -> Result { + validate_shape(frame, &self.source)?; + self.validate_schema(frame)?; + + let mut output = frame.clone(); + for (name, values) in &mut output.columns { + let mut last = self.last_numeric.get(name).copied(); + for value in values { + match *value { + Some(present) if !present.is_nan() => last = Some(present), + Some(_) | None => { + if let Some(previous) = last { + *value = Some(previous); + } + } + } + } + if let Some(last) = last { + self.last_numeric.insert(name.clone(), last); + } + } + for (name, values) in &mut output.informative_dates_ms { + let mut last = self.last_dates.get(name).copied(); + for value in values { + match *value { + Some(present) => last = Some(present), + None => { + if let Some(previous) = last { + *value = Some(previous); + } + } + } + } + if let Some(last) = last { + self.last_dates.insert(name.clone(), last); + } + } + Ok(output) + } + + /// Number of column values retained across calls. + #[must_use] + pub fn retained(&self) -> usize { + self.last_numeric.len() + self.last_dates.len() + } + + /// Maximum values this stream can retain for its established schema. + #[must_use] + pub fn retention_bound(&self) -> usize { + self.schema.as_ref().map_or(0, FillSchema::column_count) + } + + fn validate_schema(&mut self, frame: &MergedFrame) -> Result<(), VectorCoreError> { + let actual = FillSchema::from_frame(frame); + if let Some(expected) = &self.schema { + if actual.identity != expected.identity { + return Err(source_error( + &self.source, + format!( + "forward-fill frame identity changed from {:?} to {:?}", + expected.identity, actual.identity + ), + )); + } + if actual.numeric != expected.numeric { + return Err(source_error( + &self.source, + format!( + "forward-fill numeric columns changed from {:?} to {:?}", + expected.numeric, actual.numeric + ), + )); + } + if actual.dates != expected.dates { + return Err(source_error( + &self.source, + format!( + "forward-fill informative-date columns changed from {:?} to {:?}", + expected.dates, actual.dates + ), + )); + } + } else { + self.schema = Some(actual); + } + Ok(()) + } +} + +#[derive(Debug)] +struct FillSchema { + identity: FrameIdentity, + numeric: BTreeSet, + dates: BTreeSet, +} + +impl FillSchema { + fn from_frame(frame: &MergedFrame) -> Self { + Self { + identity: frame.identity.clone(), + numeric: frame.columns.keys().cloned().collect(), + dates: frame.informative_dates_ms.keys().cloned().collect(), + } + } + + fn column_count(&self) -> usize { + self.numeric.len() + self.dates.len() + } +} + +fn validate_shape(frame: &MergedFrame, source: &SourceLocation) -> Result<(), VectorCoreError> { + let expected = frame.timestamps_ms.len(); + for (name, values) in &frame.columns { + if values.len() != expected { + return Err(source_error( + source, + format!( + "forward-fill numeric column {name:?} has {} rows; expected {expected}", + values.len() + ), + )); + } + } + for (name, values) in &frame.informative_dates_ms { + if values.len() != expected { + return Err(source_error( + source, + format!( + "forward-fill informative-date column {name:?} has {} rows; expected {expected}", + values.len() + ), + )); + } + } + Ok(()) +} + +fn source_error(source: &SourceLocation, message: impl Into) -> VectorCoreError { + VectorCoreError::Execution { + node: source.node.clone(), + message: format!( + "{}:{}:{}: {}", + source.path, + source.line, + source.column, + message.into() + ), + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::{forward_fill, ForwardFillStream}; + use crate::alignment::{FrameIdentity, MergedFrame, SourceLocation, Timeframe}; + + fn source() -> SourceLocation { + SourceLocation::new("fill-final", "strategy.py", 4864, 14) + } + + fn identity(pair: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse("5m").expect("timeframe")).expect("identity") + } + + fn frame() -> MergedFrame { + MergedFrame { + identity: identity("ETH/USDT"), + timestamps_ms: (0_i64..9).map(|row| row * 300_000).collect(), + columns: BTreeMap::from([ + ( + "a".to_owned(), + vec![ + None, + Some(f64::from_bits(0x7ff8_0000_0000_0042)), + Some(-0.0), + None, + Some(f64::INFINITY), + Some(f64::NAN), + Some(f64::NEG_INFINITY), + None, + Some(4.0), + ], + ), + ( + "b".to_owned(), + vec![ + Some(1.0), + None, + Some(f64::NAN), + Some(2.0), + None, + None, + Some(3.0), + None, + None, + ], + ), + ]), + informative_dates_ms: BTreeMap::from([ + ( + "date_1h".to_owned(), + vec![ + None, + Some(10), + None, + Some(20), + None, + None, + Some(30), + None, + None, + ], + ), + ( + "date_4h".to_owned(), + vec![None, None, Some(40), None, None, Some(50), None, None, None], + ), + ]), + } + } + + #[test] + fn batch_preserves_leading_missing_and_fills_every_column_in_source_order() { + let actual = forward_fill(&frame(), &source()).expect("fill"); + assert_eq!(actual.timestamps_ms, frame().timestamps_ms); + assert_numeric_bits( + &actual.columns["a"], + &[ + None, + Some(f64::from_bits(0x7ff8_0000_0000_0042)), + Some(-0.0), + Some(-0.0), + Some(f64::INFINITY), + Some(f64::INFINITY), + Some(f64::NEG_INFINITY), + Some(f64::NEG_INFINITY), + Some(4.0), + ], + ); + assert_numeric_bits( + &actual.columns["b"], + &[ + Some(1.0), + Some(1.0), + Some(1.0), + Some(2.0), + Some(2.0), + Some(2.0), + Some(3.0), + Some(3.0), + Some(3.0), + ], + ); + assert_eq!( + actual.informative_dates_ms["date_1h"], + vec![ + None, + Some(10), + Some(10), + Some(20), + Some(20), + Some(20), + Some(30), + Some(30), + Some(30) + ] + ); + assert_eq!( + actual.informative_dates_ms["date_4h"], + vec![ + None, + None, + Some(40), + Some(40), + Some(40), + Some(50), + Some(50), + Some(50), + Some(50) + ] + ); + } + + #[test] + fn arbitrary_chunks_are_bit_exact_to_batch_and_retention_is_bounded() { + let input = frame(); + let expected = forward_fill(&input, &source()).expect("batch fill"); + let mut stream = ForwardFillStream::new(source()); + let mut parts = Vec::new(); + for range in [0..0, 0..1, 1..3, 3..4, 4..8, 8..9, 9..9] { + let chunk = slice(&input, range.start, range.end); + parts.push(stream.execute(&chunk).expect("stream fill")); + assert!(stream.retained() <= stream.retention_bound()); + assert_eq!(stream.retention_bound(), 4); + } + let actual = concatenate(&parts); + assert_frame_bits(&actual, &expected); + assert_eq!(stream.retained(), 4); + } + + #[test] + fn malformed_shape_and_stream_schema_drift_fail_closed_at_source() { + let mut malformed = frame(); + malformed.columns.get_mut("a").expect("column").pop(); + let error = forward_fill(&malformed, &source()).expect_err("shape rejection"); + assert_eq!( + error.to_string(), + "vector node fill-final failed: strategy.py:4864:14: forward-fill numeric column \ + \"a\" has 8 rows; expected 9" + ); + + let mut stream = ForwardFillStream::new(source()); + stream.execute(&slice(&frame(), 0, 2)).expect("first chunk"); + let mut drifted = slice(&frame(), 2, 4); + drifted.columns.remove("b"); + let error = stream.execute(&drifted).expect_err("schema rejection"); + assert!(error + .to_string() + .contains("strategy.py:4864:14: forward-fill numeric columns changed")); + + let mut date_stream = ForwardFillStream::new(source()); + date_stream + .execute(&slice(&frame(), 0, 2)) + .expect("first date chunk"); + let mut date_drifted = slice(&frame(), 2, 4); + date_drifted.informative_dates_ms.remove("date_4h"); + let error = date_stream + .execute(&date_drifted) + .expect_err("date schema rejection"); + assert!(error + .to_string() + .contains("strategy.py:4864:14: forward-fill informative-date columns changed")); + + let mut identity_drift = slice(&frame(), 2, 4); + identity_drift.identity = identity("BTC/USDT"); + let error = stream + .execute(&identity_drift) + .expect_err("identity rejection"); + assert!(error + .to_string() + .contains("strategy.py:4864:14: forward-fill frame identity changed")); + } + + fn slice(frame: &MergedFrame, start: usize, end: usize) -> MergedFrame { + MergedFrame { + identity: frame.identity.clone(), + timestamps_ms: frame.timestamps_ms[start..end].to_vec(), + columns: frame + .columns + .iter() + .map(|(name, values)| (name.clone(), values[start..end].to_vec())) + .collect(), + informative_dates_ms: frame + .informative_dates_ms + .iter() + .map(|(name, values)| (name.clone(), values[start..end].to_vec())) + .collect(), + } + } + + fn concatenate(parts: &[MergedFrame]) -> MergedFrame { + let mut output = MergedFrame { + identity: parts[0].identity.clone(), + timestamps_ms: Vec::new(), + columns: parts[0] + .columns + .keys() + .map(|name| (name.clone(), Vec::new())) + .collect(), + informative_dates_ms: parts[0] + .informative_dates_ms + .keys() + .map(|name| (name.clone(), Vec::new())) + .collect(), + }; + for part in parts { + output.timestamps_ms.extend_from_slice(&part.timestamps_ms); + for (name, values) in &part.columns { + output.columns.get_mut(name).expect("column").extend(values); + } + for (name, values) in &part.informative_dates_ms { + output + .informative_dates_ms + .get_mut(name) + .expect("date column") + .extend(values); + } + } + output + } + + fn assert_frame_bits(actual: &MergedFrame, expected: &MergedFrame) { + assert_eq!(actual.identity, expected.identity); + assert_eq!(actual.timestamps_ms, expected.timestamps_ms); + assert_eq!(actual.informative_dates_ms, expected.informative_dates_ms); + assert_eq!( + actual.columns.keys().collect::>(), + expected.columns.keys().collect::>() + ); + for name in actual.columns.keys() { + assert_numeric_bits(&actual.columns[name], &expected.columns[name]); + } + } + + fn assert_numeric_bits(actual: &[Option], expected: &[Option]) { + assert_eq!(actual.len(), expected.len()); + for (actual, expected) in actual.iter().zip(expected) { + match (actual, expected) { + (Some(actual), Some(expected)) => assert_eq!(actual.to_bits(), expected.to_bits()), + (None, None) => {} + _ => panic!("option mismatch: {actual:?} != {expected:?}"), + } + } + } +} diff --git a/rust/crates/nfi-vector-core/src/alignment/mod.rs b/rust/crates/nfi-vector-core/src/alignment/mod.rs new file mode 100644 index 0000000..c03114c --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/mod.rs @@ -0,0 +1,23 @@ +//! Generic, causal alignment of base and informative candle frames. +//! +//! This ports the timestamp rule of Freqtrade 2026.5.1's +//! `merge_informative_pair`: an informative candle becomes visible only after +//! it has closed. Frames are deliberately explicit about their `(pair, +//! timeframe)` identity, so equal column names from different markets cannot +//! be joined accidentally. + +mod batch; +mod fill; +mod model; +mod stream; +mod support; + +pub use batch::merge; +pub use fill::{forward_fill, ForwardFillStream}; +pub use model::{FrameIdentity, MergeSpec, MergedFrame, NumericFrame, SourceLocation, Timeframe}; +pub use stream::MergeStream; + +#[cfg(test)] +use model::days_from_civil; +#[cfg(test)] +mod tests; diff --git a/rust/crates/nfi-vector-core/src/alignment/model.rs b/rust/crates/nfi-vector-core/src/alignment/model.rs new file mode 100644 index 0000000..b4bd41b --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/model.rs @@ -0,0 +1,301 @@ +use std::collections::BTreeMap; + +use crate::VectorCoreError; + +/// Identifies the source location that declared an alignment operation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SourceLocation { + pub node: String, + pub path: String, + pub line: u64, + pub column: u64, +} + +impl SourceLocation { + #[must_use] + pub fn new(node: impl Into, path: impl Into, line: u64, column: u64) -> Self { + Self { + node: node.into(), + path: path.into(), + line, + column, + } + } +} + +/// A canonical Freqtrade timeframe token. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Timeframe(String, i64); + +impl Timeframe { + /// Parses a positive `` Freqtrade timeframe. + /// + /// # Errors + /// + /// Returns an invalid-program error for an unsupported, zero, or + /// unrepresentable timeframe. + pub fn parse(value: impl Into) -> Result { + let value = value.into(); + let Some(unit) = value.chars().last() else { + return Err(VectorCoreError::InvalidProgram( + "empty timeframe".to_owned(), + )); + }; + if !matches!(unit, 's' | 'm' | 'h' | 'd' | 'w' | 'M' | 'y') { + return Err(VectorCoreError::InvalidProgram(format!( + "unsupported timeframe {value:?}" + ))); + } + let count = value[..value.len() - unit.len_utf8()] + .parse::() + .ok() + .filter(|count| *count > 0) + .ok_or_else(|| { + VectorCoreError::InvalidProgram(format!("invalid timeframe {value:?}")) + })?; + let multiplier = match unit { + 's' => 1_000_u64, + 'm' => 60_000_u64, + 'h' => 3_600_000, + 'd' => 86_400_000, + 'w' => 7 * 86_400_000, + // Freqtrade's timeframe-to-minutes contract uses 30 days for + // ordering. `1M` gets its calendar MonthBegin merge rule below. + 'M' => 30 * 86_400_000, + 'y' => 365 * 86_400_000, + _ => unreachable!("validated timeframe unit"), + }; + let duration_ms = count.checked_mul(multiplier).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!("timeframe is too large: {value}")) + })?; + let duration_ms = i64::try_from(duration_ms).map_err(|_| { + VectorCoreError::InvalidProgram(format!("timeframe is too large: {value}")) + })?; + Ok(Self(value, duration_ms)) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Duration used for resampling and Arrow timestamp arithmetic. + #[must_use] + pub fn resample_duration_ms(&self) -> i64 { + self.1 + } + + /// Freqtrade merge duration: `ccxt.parse_timeframe(timeframe) // 60`. + #[must_use] + pub fn merge_minutes(&self) -> i64 { + self.resample_duration_ms() / 60_000 + } +} + +/// The concrete pair and timeframe that own one source frame. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct FrameIdentity { + pub pair: String, + pub timeframe: Timeframe, +} + +impl FrameIdentity { + /// Creates a non-empty pair/timeframe identity. + /// + /// # Errors + /// + /// Returns an invalid-program error when `pair` is empty. + pub fn new(pair: impl Into, timeframe: Timeframe) -> Result { + let pair = pair.into(); + if pair.is_empty() { + return Err(VectorCoreError::InvalidProgram( + "frame pair is empty".to_owned(), + )); + } + Ok(Self { pair, timeframe }) + } +} + +/// One timestamp-indexed numeric frame. A null is distinct from a present NaN. +#[derive(Clone, Debug, PartialEq)] +pub struct NumericFrame { + pub identity: FrameIdentity, + pub timestamps_ms: Vec, + pub columns: BTreeMap>>, +} + +impl NumericFrame { + /// Validates column names and row shape. + /// + /// # Errors + /// + /// Returns an invalid-program or column-length error for an invalid frame. + pub fn validate(&self) -> Result<(), VectorCoreError> { + for (name, values) in &self.columns { + if name.is_empty() { + return Err(VectorCoreError::InvalidProgram( + "frame column is empty".to_owned(), + )); + } + if values.len() != self.timestamps_ms.len() { + return Err(VectorCoreError::ColumnLength { + column: name.clone(), + actual: values.len(), + expected: self.timestamps_ms.len(), + }); + } + } + Ok(()) + } +} + +/// Exact options for one `merge_informative_pair` operation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MergeSpec { + pub base: FrameIdentity, + pub informative: FrameIdentity, + pub ffill: bool, + pub append_timeframe: bool, + pub suffix: Option, + /// Informative source timestamp name. The base join key is always `date`. + pub date_column: String, + pub source: SourceLocation, +} + +impl MergeSpec { + /// Validates pair identities and the mutually-exclusive suffix modes. + /// + /// # Errors + /// + /// Returns a source-located error for incompatible merge options. + pub fn validate(&self) -> Result<(), VectorCoreError> { + if self.append_timeframe + && self + .suffix + .as_ref() + .is_some_and(|suffix| !suffix.is_empty()) + { + return Err(self.error("suffix cannot be combined with append_timeframe")); + } + if self.date_column.is_empty() { + return Err(self.error("informative date_column cannot be empty")); + } + if self.informative.timeframe.merge_minutes() < self.base.timeframe.merge_minutes() { + return Err(self.error("informative timeframe cannot be faster than base timeframe")); + } + Ok(()) + } + + pub(super) fn error(&self, message: impl Into) -> VectorCoreError { + VectorCoreError::Execution { + node: self.source.node.clone(), + message: format!( + "{}:{}:{}: {}", + self.source.path, + self.source.line, + self.source.column, + message.into() + ), + } + } + + pub(super) fn output_name(&self, name: &str) -> Result { + let suffix = if self.append_timeframe { + Some(self.informative.timeframe.as_str()) + } else { + self.suffix.as_deref().filter(|suffix| !suffix.is_empty()) + }; + suffix.map_or_else( + || { + Err(self.error(format!( + "informative column {name:?} would collide without a suffix" + ))) + }, + |suffix| Ok(format!("{name}_{suffix}")), + ) + } + + pub(super) fn effective_timestamp(&self, open_ms: i64) -> Result { + let base_minutes = self.base.timeframe.merge_minutes(); + let informative_minutes = self.informative.timeframe.merge_minutes(); + if base_minutes == informative_minutes { + return Ok(open_ms); + } + let base_offset_ms = base_minutes + .checked_mul(60_000) + .ok_or_else(|| self.error("base merge duration is out of range"))?; + if self.informative.timeframe.0 == "1M" { + return month_begin_next(open_ms) + .and_then(|next| next.checked_sub(base_offset_ms)) + .ok_or_else(|| self.error("monthly informative merge timestamp is out of range")); + } + let informative_offset_ms = informative_minutes + .checked_mul(60_000) + .ok_or_else(|| self.error("informative merge duration is out of range"))?; + open_ms + .checked_add(informative_offset_ms) + .and_then(|value| value.checked_sub(base_offset_ms)) + .ok_or_else(|| self.error("informative merge timestamp is out of range")) + } +} + +/// A merged base frame, including the informative source dates returned by Freqtrade. +/// +/// Missing informative numeric join values are canonical `f64::NAN`; an input +/// nullable numeric value remains `None` after an exact match or forward fill. +#[derive(Clone, Debug, PartialEq)] +pub struct MergedFrame { + pub identity: FrameIdentity, + pub timestamps_ms: Vec, + pub columns: BTreeMap>>, + pub informative_dates_ms: BTreeMap>>, +} + +fn month_begin_next(timestamp_ms: i64) -> Option { + const DAY_MS: i64 = 86_400_000; + let days = timestamp_ms.div_euclid(DAY_MS); + let (year, month, day) = civil_from_days(days); + let (next_year, next_month) = if day == 1 && timestamp_ms.rem_euclid(DAY_MS) == 0 { + if month == 12 { + (year + 1, 1) + } else { + (year, month + 1) + } + } else if month == 12 { + (year + 1, 1) + } else { + (year, month + 1) + }; + days_from_civil(next_year, next_month, 1)?.checked_mul(DAY_MS) +} + +fn civil_from_days(days: i64) -> (i64, u8, u8) { + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let day_in_era = z - era * 146_097; + let year_in_era = + (day_in_era - day_in_era / 1_460 + day_in_era / 36_524 - day_in_era / 146_096) / 365; + let year = year_in_era + era * 400; + let ordinal = day_in_era - (365 * year_in_era + year_in_era / 4 - year_in_era / 100); + let month_prime = (5 * ordinal + 2) / 153; + let day = ordinal - (153 * month_prime + 2) / 5 + 1; + let month = month_prime + if month_prime < 10 { 3 } else { -9 }; + ( + year + i64::from(month <= 2), + u8::try_from(month).expect("civil month is in 1..=12"), + u8::try_from(day).expect("civil day is in 1..=31"), + ) +} + +pub(super) fn days_from_civil(year: i64, month: u8, day_of_month: u8) -> Option { + let year = year - i64::from(month <= 2); + let era = if year >= 0 { year } else { year - 399 } / 400; + let yoe = year - era * 400; + let month = i64::from(month); + let day_of_month = i64::from(day_of_month); + let ordinal = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day_of_month - 1; + let day_in_era = yoe * 365 + yoe / 4 - yoe / 100 + ordinal; + era.checked_mul(146_097)? + .checked_add(day_in_era)? + .checked_sub(719_468) +} diff --git a/rust/crates/nfi-vector-core/src/alignment/stream.rs b/rust/crates/nfi-vector-core/src/alignment/stream.rs new file mode 100644 index 0000000..e362032 --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/stream.rs @@ -0,0 +1,235 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::VectorCoreError; + +use super::model::{MergeSpec, MergedFrame, NumericFrame}; +use super::support::{ + informative_events, output_names, validate_frame, validate_identity, validate_ordered, + InformativeEvent, OutputBuilder, OutputNames, StoredInformativeRow, +}; + +/// Bounded cursor for causally ordered alignment chunks. +/// +/// Informative chunks must not contain an effective key after the maximum base +/// timestamp in the same call. This explicit backpressure rule keeps retention +/// to one informative row instead of silently buffering an unbounded future. +#[derive(Debug)] +pub struct MergeStream { + spec: MergeSpec, + schema: Option, + last_base_timestamp: Option, + last_informative_key: Option, + last_informative: Option, + has_exact_match: bool, +} + +impl MergeStream { + /// Creates an empty bounded alignment cursor. + /// + /// # Errors + /// + /// Returns a source-located error for an invalid merge specification. + pub fn new(spec: MergeSpec) -> Result { + spec.validate()?; + Ok(Self { + spec, + schema: None, + last_base_timestamp: None, + last_informative_key: None, + last_informative: None, + has_exact_match: false, + }) + } + + /// Aligns one ordered chunk without retaining historical input batches. + /// + /// # Errors + /// + /// Returns a source-located error for mismatched identities, unordered + /// chunks, or informative rows that have not become visible yet. + pub fn execute( + &mut self, + base: &NumericFrame, + informative: &NumericFrame, + ) -> Result { + validate_identity(base, &self.spec.base, &self.spec)?; + validate_identity(informative, &self.spec.informative, &self.spec)?; + validate_frame(base, &self.spec)?; + validate_frame(informative, &self.spec)?; + validate_ordered(base, &self.spec)?; + validate_ordered(informative, &self.spec)?; + if !self.spec.ffill + && base.timestamps_ms.is_empty() + && !informative.timestamps_ms.is_empty() + { + return Err(self + .spec + .error("non-ffill stream cannot retain informative exact rows without base rows")); + } + let names = output_names(base, informative, &self.spec)?; + let schema = StreamSchema::new(base, informative, &names, &self.spec); + self.validate_schema(&schema)?; + let events = informative_events(informative, &self.spec)?; + self.validate_progress(base, &events)?; + let mut exact = BTreeMap::>::new(); + for event in &events { + exact + .entry(event.key_ms) + .or_default() + .push(event.source_row); + } + self.prepare_ffill(base, informative, &events, &exact)?; + + let mut output = OutputBuilder::new(base, informative, &names); + let mut event_cursor = 0; + for base_row in 0..base.timestamps_ms.len() { + let timestamp = base.timestamps_ms[base_row]; + while event_cursor < events.len() && events[event_cursor].key_ms <= timestamp { + if self.spec.ffill { + self.last_informative = Some(StoredInformativeRow::from_frame( + informative, + events[event_cursor].source_row, + )); + } + event_cursor += 1; + } + let matches = exact.get(×tamp).map(Vec::as_slice); + if let Some(matches) = matches { + self.has_exact_match = true; + output.extend(base, base_row, Some(matches)); + } else if self.spec.ffill && self.has_exact_match { + output.extend_stored(base, base_row, self.last_informative.as_ref()); + } else { + output.extend(base, base_row, None); + } + } + self.last_base_timestamp = base + .timestamps_ms + .last() + .copied() + .or(self.last_base_timestamp); + self.last_informative_key = events + .last() + .map(|event| event.key_ms) + .or(self.last_informative_key); + self.schema = Some(schema); + Ok(output.finish(base.identity.clone())) + } + + fn validate_schema(&self, schema: &StreamSchema) -> Result<(), VectorCoreError> { + if self + .schema + .as_ref() + .is_some_and(|previous| previous != schema) + { + return Err(self + .spec + .error("base or informative numeric/date schema changed across stream calls")); + } + Ok(()) + } + + fn validate_progress( + &self, + base: &NumericFrame, + events: &[InformativeEvent], + ) -> Result<(), VectorCoreError> { + if base.timestamps_ms.first().is_some_and(|timestamp| { + self.last_base_timestamp + .is_some_and(|last| *timestamp <= last) + }) { + return Err(self + .spec + .error("base stream timestamp did not advance across calls")); + } + if events.first().is_some_and(|event| { + self.last_informative_key + .is_some_and(|last| event.key_ms <= last) + }) { + return Err(self + .spec + .error("informative stream effective timestamp did not advance across calls")); + } + if events.iter().any(|event| { + self.last_base_timestamp + .is_some_and(|last| event.key_ms <= last) + }) { + return Err(self.spec.error( + "informative stream contains a late row that could change emitted base output", + )); + } + if let Some(last_base) = base.timestamps_ms.last() { + if events.iter().any(|event| event.key_ms > *last_base) { + return Err(self.spec.error( + "informative stream contains a future row; split before its effective timestamp", + )); + } + } + Ok(()) + } + + fn prepare_ffill( + &mut self, + base: &NumericFrame, + informative: &NumericFrame, + events: &[InformativeEvent], + exact: &BTreeMap>, + ) -> Result<(), VectorCoreError> { + // Freqtrade may repair every row before its first exact match from a + // later historical informative row. No bounded cursor can emit a + // leading base chunk exactly until that first match is present. + if self.spec.ffill && !self.has_exact_match && !base.timestamps_ms.is_empty() { + if !base + .timestamps_ms + .iter() + .any(|timestamp| exact.contains_key(timestamp)) + { + return Err(self.spec.error( + "ffill stream requires an exact informative match in its first nonempty base chunk", + )); + } + self.has_exact_match = true; + } + if base.timestamps_ms.is_empty() && self.spec.ffill { + if let Some(event) = events.last() { + self.last_informative = Some(StoredInformativeRow::from_frame( + informative, + event.source_row, + )); + } + } + Ok(()) + } + + /// Number of informative rows retained across calls. + #[must_use] + pub fn retained(&self) -> usize { + usize::from(self.last_informative.is_some()) + } +} + +#[derive(Debug, Eq, PartialEq)] +struct StreamSchema { + base_numeric: BTreeSet, + informative_numeric: BTreeSet, + base_date: String, + informative_date: String, + output_date: String, +} + +impl StreamSchema { + fn new( + base: &NumericFrame, + informative: &NumericFrame, + names: &OutputNames, + spec: &MergeSpec, + ) -> Self { + Self { + base_numeric: base.columns.keys().cloned().collect(), + informative_numeric: informative.columns.keys().cloned().collect(), + base_date: "date".to_owned(), + informative_date: spec.date_column.clone(), + output_date: names.date.clone(), + } + } +} diff --git a/rust/crates/nfi-vector-core/src/alignment/support.rs b/rust/crates/nfi-vector-core/src/alignment/support.rs new file mode 100644 index 0000000..fc67604 --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/support.rs @@ -0,0 +1,248 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::VectorCoreError; + +use super::model::{FrameIdentity, MergeSpec, MergedFrame, NumericFrame}; + +#[derive(Clone, Debug)] +pub(super) struct InformativeEvent { + pub(super) key_ms: i64, + pub(super) source_row: usize, +} + +#[derive(Clone, Debug)] +pub(super) struct StoredInformativeRow { + pub(super) timestamp_ms: i64, + pub(super) values: BTreeMap>, +} + +impl StoredInformativeRow { + pub(super) fn from_frame(frame: &NumericFrame, source_row: usize) -> Self { + Self { + timestamp_ms: frame.timestamps_ms[source_row], + values: frame + .columns + .iter() + .map(|(name, values)| (name.clone(), values[source_row])) + .collect(), + } + } +} + +pub(super) struct OutputNames { + pub(super) numeric: BTreeMap, + pub(super) date: String, +} + +pub(super) struct OutputBuilder<'frame> { + timestamps_ms: Vec, + columns: BTreeMap>>, + dates: BTreeMap>>, + informative: &'frame NumericFrame, + names: OutputNames, +} + +impl<'frame> OutputBuilder<'frame> { + pub(super) fn new( + base: &NumericFrame, + informative: &'frame NumericFrame, + names: &OutputNames, + ) -> Self { + let mut columns = base + .columns + .keys() + .map(|name| (name.clone(), Vec::new())) + .collect::>(); + for output_name in names.numeric.values() { + columns.insert(output_name.clone(), Vec::new()); + } + let dates = BTreeMap::from([(names.date.clone(), Vec::new())]); + Self { + timestamps_ms: Vec::new(), + columns, + dates, + informative, + names: OutputNames { + numeric: names.numeric.clone(), + date: names.date.clone(), + }, + } + } + + pub(super) fn extend( + &mut self, + base: &NumericFrame, + base_row: usize, + matches: Option<&[usize]>, + ) { + match matches { + Some(rows) => { + for row in rows { + self.extend_one(base, base_row, Some(*row)); + } + } + None => self.extend_one(base, base_row, None), + } + } + + pub(super) fn extend_stored( + &mut self, + base: &NumericFrame, + base_row: usize, + stored: Option<&StoredInformativeRow>, + ) { + for (name, values) in &base.columns { + self.columns + .get_mut(name) + .expect("base column exists") + .push(values[base_row]); + } + for (source, output) in &self.names.numeric { + self.columns + .get_mut(output) + .expect("informative column exists") + .push(match stored { + Some(row) => row.values.get(source).copied().flatten(), + None => Some(f64::NAN), + }); + } + self.dates + .get_mut(&self.names.date) + .expect("date column exists") + .push(stored.map(|row| row.timestamp_ms)); + self.timestamps_ms.push(base.timestamps_ms[base_row]); + } + + fn extend_one(&mut self, base: &NumericFrame, base_row: usize, informative_row: Option) { + for (name, values) in &base.columns { + self.columns + .get_mut(name) + .expect("base column exists") + .push(values[base_row]); + } + for (source, output) in &self.names.numeric { + self.columns + .get_mut(output) + .expect("informative column exists") + .push( + informative_row + .map_or(Some(f64::NAN), |row| self.informative.columns[source][row]), + ); + } + self.dates + .get_mut(&self.names.date) + .expect("date column exists") + .push(informative_row.map(|row| self.informative.timestamps_ms[row])); + self.timestamps_ms.push(base.timestamps_ms[base_row]); + } + + pub(super) fn finish(self, identity: FrameIdentity) -> MergedFrame { + MergedFrame { + identity, + timestamps_ms: self.timestamps_ms, + columns: self.columns, + informative_dates_ms: self.dates, + } + } +} + +pub(super) fn validate_identity( + frame: &NumericFrame, + expected: &FrameIdentity, + spec: &MergeSpec, +) -> Result<(), VectorCoreError> { + if &frame.identity != expected { + return Err(spec.error(format!( + "frame identity is {} {} but merge requires {} {}", + frame.identity.pair, + frame.identity.timeframe.as_str(), + expected.pair, + expected.timeframe.as_str() + ))); + } + Ok(()) +} + +pub(super) fn validate_frame( + frame: &NumericFrame, + spec: &MergeSpec, +) -> Result<(), VectorCoreError> { + frame + .validate() + .map_err(|_| spec.error("frame has invalid column names or row lengths")) +} + +pub(super) fn output_names( + base: &NumericFrame, + informative: &NumericFrame, + spec: &MergeSpec, +) -> Result { + let mut occupied = base.columns.keys().cloned().collect::>(); + let date = spec.output_name(&spec.date_column)?; + if !occupied.insert(date.clone()) { + return Err(spec.error(format!( + "informative date output {date:?} collides with a base column" + ))); + } + let mut numeric = BTreeMap::new(); + for name in informative.columns.keys() { + let output = spec.output_name(name)?; + if !occupied.insert(output.clone()) { + return Err(spec.error(format!("informative output column {output:?} collides"))); + } + numeric.insert(name.clone(), output); + } + Ok(OutputNames { numeric, date }) +} + +pub(super) fn informative_events( + frame: &NumericFrame, + spec: &MergeSpec, +) -> Result, VectorCoreError> { + let mut events = frame + .timestamps_ms + .iter() + .enumerate() + .map(|(source_row, timestamp)| { + Ok(InformativeEvent { + key_ms: spec.effective_timestamp(*timestamp)?, + source_row, + }) + }) + .collect::, _>>()?; + events.sort_by_key(|event| event.key_ms); + Ok(events) +} + +pub(super) fn validate_ordered( + frame: &NumericFrame, + spec: &MergeSpec, +) -> Result<(), VectorCoreError> { + if frame.timestamps_ms.windows(2).any(|pair| pair[0] > pair[1]) { + return Err(spec.error(format!( + "frame {} {} timestamps are not ordered", + frame.identity.pair, + frame.identity.timeframe.as_str() + ))); + } + Ok(()) +} + +pub(super) fn ordered_by_timestamp(frame: &NumericFrame) -> NumericFrame { + let mut rows = (0..frame.timestamps_ms.len()).collect::>(); + rows.sort_by_key(|row| frame.timestamps_ms[*row]); + NumericFrame { + identity: frame.identity.clone(), + timestamps_ms: rows.iter().map(|row| frame.timestamps_ms[*row]).collect(), + columns: frame + .columns + .iter() + .map(|(name, values)| { + ( + name.clone(), + rows.iter().map(|row| values[*row]).collect::>(), + ) + }) + .collect(), + } +} diff --git a/rust/crates/nfi-vector-core/src/alignment/tests.rs b/rust/crates/nfi-vector-core/src/alignment/tests.rs new file mode 100644 index 0000000..62971fb --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/tests.rs @@ -0,0 +1,696 @@ +use std::collections::BTreeMap; + +use serde_json::Value; + +use super::{ + merge, FrameIdentity, MergeSpec, MergeStream, NumericFrame, SourceLocation, Timeframe, +}; +use crate::VectorCoreError; + +const ORACLE: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../benchmarks/reference/informative/freqtrade-2026.5.1.json" +)); + +#[test] +fn pinned_freqtrade_oracle_is_exact() { + let document: Value = serde_json::from_str(ORACLE).expect("valid oracle"); + for case in document["cases"].as_array().expect("oracle cases") { + let name = case["name"].as_str().expect("case name"); + let base = frame( + &case["base"], + case["base_pair"].as_str().expect("base pair"), + case["call"]["timeframe"].as_str().expect("base timeframe"), + ); + let informative = frame( + &case["informative"], + case["informative_pair"].as_str().expect("informative pair"), + case["call"]["timeframe_inf"] + .as_str() + .expect("informative timeframe"), + ); + let spec = spec(case); + if case.get("error").is_some() { + assert!(merge(&base, &informative, &spec).is_err(), "{name}"); + } else { + let actual = merge(&base, &informative, &spec) + .unwrap_or_else(|error| panic!("{name}: {error:?}")); + assert_expected(&actual, &case["output"], name); + } + } +} + +#[test] +fn stream_is_exact_at_the_five_minute_hour_boundary() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![ + ms("2024-01-01T00:50:00Z"), + ms("2024-01-01T00:55:00Z"), + ms("2024-01-01T01:00:00Z"), + ], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0), Some(2.0), Some(3.0)])]), + }; + let informative = NumericFrame { + identity: identity("ETH/USDT", "1h"), + timestamps_ms: vec![ms("2024-01-01T00:00:00Z")], + columns: BTreeMap::from([("info".to_owned(), vec![Some(9.0)])]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: informative.identity.clone(), + ffill: true, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let expected = merge(&base, &informative, &spec).expect("batch result"); + let empty = NumericFrame { + identity: informative.identity.clone(), + timestamps_ms: Vec::new(), + columns: BTreeMap::from([("info".to_owned(), Vec::new())]), + }; + let mut stream = MergeStream::new(spec).expect("stream"); + let first = stream + .execute(&slice(&base, 0, 2), &informative) + .expect("first chunk"); + let second = stream + .execute(&slice(&base, 2, 3), &empty) + .expect("second chunk"); + assert_eq!(stream.retained(), 1); + assert_merged_bits(&concatenate(&first, &second), &expected); +} + +#[test] +fn empty_base_informative_chunk_is_retained_for_the_next_exact_chunk() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![ms("2024-01-01T00:50:00Z"), ms("2024-01-01T00:55:00Z")], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0), Some(2.0)])]), + }; + let historical = NumericFrame { + identity: identity("ETH/USDT", "1h"), + timestamps_ms: vec![ms("2023-12-31T23:00:00Z")], + columns: BTreeMap::from([("info".to_owned(), vec![Some(5.0)])]), + }; + let current = NumericFrame { + identity: historical.identity.clone(), + timestamps_ms: vec![ms("2024-01-01T00:00:00Z")], + columns: BTreeMap::from([("info".to_owned(), vec![Some(9.0)])]), + }; + let all_informative = NumericFrame { + identity: historical.identity.clone(), + timestamps_ms: vec![ms("2023-12-31T23:00:00Z"), ms("2024-01-01T00:00:00Z")], + columns: BTreeMap::from([("info".to_owned(), vec![Some(5.0), Some(9.0)])]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: historical.identity.clone(), + ffill: true, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let expected = merge(&base, &all_informative, &spec).expect("batch result"); + let empty_base = NumericFrame { + identity: base.identity.clone(), + timestamps_ms: Vec::new(), + columns: BTreeMap::from([("base".to_owned(), Vec::new())]), + }; + let mut stream = MergeStream::new(spec).expect("stream"); + let empty = stream + .execute(&empty_base, &historical) + .expect("empty base chunk"); + assert!(empty.timestamps_ms.is_empty()); + assert_eq!(stream.retained(), 1); + let actual = stream.execute(&base, ¤t).expect("exact chunk"); + assert_merged_bits(&actual, &expected); + assert_eq!(stream.retained(), 1); +} + +#[test] +fn ffill_stream_rejects_current_historical_rows_without_an_exact_match() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![ms("2024-01-01T00:50:00Z")], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0)])]), + }; + let historical = NumericFrame { + identity: identity("ETH/USDT", "1h"), + timestamps_ms: vec![ms("2023-12-31T23:00:00Z")], + columns: BTreeMap::from([("info".to_owned(), vec![Some(5.0)])]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: historical.identity.clone(), + ffill: true, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let mut stream = MergeStream::new(spec).expect("stream"); + assert!(matches!( + stream.execute(&base, &historical), + Err(VectorCoreError::Execution { node, .. }) if node == "n7" + )); +} + +#[test] +fn ffill_stream_rejects_a_leading_base_chunk_without_informative_events() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![ms("2024-01-01T00:50:00Z")], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0)])]), + }; + let empty_informative = NumericFrame { + identity: identity("ETH/USDT", "1h"), + timestamps_ms: Vec::new(), + columns: BTreeMap::from([("info".to_owned(), Vec::new())]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: empty_informative.identity.clone(), + ffill: true, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let mut stream = MergeStream::new(spec).expect("stream"); + assert!(matches!( + stream.execute(&base, &empty_informative), + Err(VectorCoreError::Execution { node, .. }) if node == "n7" + )); +} + +#[test] +fn stream_rejects_unresolved_leading_history_and_timestamp_regression() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![ms("2024-01-01T00:00:00Z")], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0)])]), + }; + let historical = NumericFrame { + identity: identity("ETH/USDT", "1h"), + timestamps_ms: vec![ms("2023-12-31T23:00:00Z")], + columns: BTreeMap::from([("info".to_owned(), vec![Some(5.0)])]), + }; + let empty_base = NumericFrame { + identity: base.identity.clone(), + timestamps_ms: Vec::new(), + columns: BTreeMap::from([("base".to_owned(), Vec::new())]), + }; + let empty_info = NumericFrame { + identity: historical.identity.clone(), + timestamps_ms: Vec::new(), + columns: BTreeMap::from([("info".to_owned(), Vec::new())]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: historical.identity.clone(), + ffill: true, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let mut stream = MergeStream::new(spec.clone()).expect("stream"); + stream + .execute(&empty_base, &historical) + .expect("historical seed"); + assert!(matches!( + stream.execute(&base, &empty_info), + Err(VectorCoreError::Execution { node, .. }) if node == "n7" + )); + + let newer = NumericFrame { + identity: historical.identity.clone(), + timestamps_ms: vec![ms("2024-01-01T00:00:00Z")], + columns: BTreeMap::from([("info".to_owned(), vec![Some(9.0)])]), + }; + let mut regression = MergeStream::new(spec).expect("stream"); + regression + .execute(&empty_base, &newer) + .expect("new informative chunk"); + assert!(matches!( + regression.execute(&empty_base, &historical), + Err(VectorCoreError::Execution { node, .. }) if node == "n7" + )); +} + +#[test] +fn ffill_leading_repair_uses_one_latest_historical_row_for_the_whole_prefix() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![ + ms("2024-01-01T00:00:00Z"), + ms("2024-01-01T00:20:00Z"), + ms("2024-01-01T00:30:00Z"), + ms("2024-01-01T00:55:00Z"), + ms("2024-01-01T01:00:00Z"), + ], + columns: BTreeMap::from([("base".to_owned(), vec![Some(0.0); 5])]), + }; + let informative = NumericFrame { + identity: identity("ETH/USDT", "1h"), + timestamps_ms: vec![ + ms("2023-12-31T23:00:00Z"), + ms("2023-12-31T23:30:00Z"), + ms("2024-01-01T00:00:00Z"), + ], + columns: BTreeMap::from([("info".to_owned(), vec![Some(1.0), Some(2.0), Some(3.0)])]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: informative.identity.clone(), + ffill: true, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let merged = merge(&base, &informative, &spec).expect("merge"); + assert_eq!( + merged.columns["info_1h"], + vec![Some(2.0), Some(2.0), Some(2.0), Some(3.0), Some(3.0)] + ); + assert_eq!( + merged.informative_dates_ms["date_1h"], + vec![ + Some(ms("2023-12-31T23:30:00Z")), + Some(ms("2023-12-31T23:30:00Z")), + Some(ms("2023-12-31T23:30:00Z")), + Some(ms("2024-01-01T00:00:00Z")), + Some(ms("2024-01-01T00:00:00Z")), + ] + ); +} + +#[test] +fn stream_retention_is_constant_across_a_long_informative_only_run() { + let base_identity = identity("ETH/USDT", "5m"); + let informative_identity = identity("ETH/USDT", "1h"); + let empty_base = NumericFrame { + identity: base_identity.clone(), + timestamps_ms: Vec::new(), + columns: BTreeMap::from([("base".to_owned(), Vec::new())]), + }; + let spec = MergeSpec { + base: base_identity, + informative: informative_identity.clone(), + ffill: true, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let mut stream = MergeStream::new(spec).expect("stream"); + for hour in 0_i64..256 { + let value = f64::from(u32::try_from(hour).expect("test hour fits u32")); + let informative = NumericFrame { + identity: informative_identity.clone(), + timestamps_ms: vec![hour * 3_600_000], + columns: BTreeMap::from([("info".to_owned(), vec![Some(value)])]), + }; + stream + .execute(&empty_base, &informative) + .expect("ordered informative chunk"); + assert_eq!(stream.retained(), 1); + } +} + +#[test] +fn stream_rejects_numeric_schema_drift_across_calls() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![0], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0)])]), + }; + let informative = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![0], + columns: BTreeMap::from([("info".to_owned(), vec![Some(2.0)])]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: informative.identity.clone(), + ffill: false, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let mut stream = MergeStream::new(spec).expect("stream"); + stream.execute(&base, &informative).expect("first chunk"); + let second_base = NumericFrame { + identity: base.identity.clone(), + timestamps_ms: vec![300_000], + columns: base.columns.clone(), + }; + let drifted_informative = NumericFrame { + identity: informative.identity, + timestamps_ms: vec![300_000], + columns: BTreeMap::from([("renamed".to_owned(), vec![Some(2.0)])]), + }; + assert!(matches!( + stream.execute(&second_base, &drifted_informative), + Err(VectorCoreError::Execution { node, .. }) if node == "n7" + )); +} + +#[test] +fn non_ffill_stream_rejects_informative_only_chunks_without_losing_exact_rows() { + let base_identity = identity("ETH/USDT", "5m"); + let informative_identity = identity("ETH/USDT", "5m"); + let empty_base = NumericFrame { + identity: base_identity.clone(), + timestamps_ms: Vec::new(), + columns: BTreeMap::from([("base".to_owned(), Vec::new())]), + }; + let informative = NumericFrame { + identity: informative_identity.clone(), + timestamps_ms: vec![0], + columns: BTreeMap::from([("info".to_owned(), vec![Some(9.0)])]), + }; + let spec = MergeSpec { + base: base_identity.clone(), + informative: informative_identity, + ffill: false, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let mut stream = MergeStream::new(spec).expect("stream"); + assert!(matches!( + stream.execute(&empty_base, &informative), + Err(VectorCoreError::Execution { node, .. }) if node == "n7" + )); + let base = NumericFrame { + identity: base_identity, + timestamps_ms: vec![0], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0)])]), + }; + let replayed = stream + .execute(&base, &informative) + .expect("replayed exact row"); + assert_eq!(replayed.columns["info_5m"], vec![Some(9.0)]); +} + +#[test] +fn merge_uses_freqtrade_floor_minutes_not_resample_milliseconds() { + let slow_base = NumericFrame { + identity: identity("ETH/USDT", "30s"), + timestamps_ms: vec![60_000], + columns: BTreeMap::from([("base".to_owned(), vec![Some(1.0)])]), + }; + let one_minute = NumericFrame { + identity: identity("ETH/USDT", "1m"), + timestamps_ms: vec![0], + columns: BTreeMap::from([("info".to_owned(), vec![Some(2.0)])]), + }; + let slow_spec = MergeSpec { + base: slow_base.identity.clone(), + informative: one_minute.identity.clone(), + ffill: false, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let visible = merge(&slow_base, &one_minute, &slow_spec).expect("30s <- 1m"); + assert_eq!(visible.columns["info_1m"], vec![Some(2.0)]); + + let second_base = NumericFrame { + identity: identity("ETH/USDT", "1s"), + timestamps_ms: vec![1_000], + columns: BTreeMap::from([("base".to_owned(), vec![Some(3.0)])]), + }; + let thirty_seconds = NumericFrame { + identity: identity("ETH/USDT", "30s"), + timestamps_ms: vec![1_000], + columns: BTreeMap::from([("info".to_owned(), vec![Some(4.0)])]), + }; + let equal_minutes = MergeSpec { + base: second_base.identity.clone(), + informative: thirty_seconds.identity.clone(), + ffill: false, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + let equal = merge(&second_base, &thirty_seconds, &equal_minutes).expect("0m == 0m"); + assert_eq!(equal.columns["info_30s"], vec![Some(4.0)]); + + let faster = MergeSpec { + base: one_minute.identity.clone(), + informative: slow_base.identity.clone(), + ffill: false, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + assert!(matches!( + merge(&one_minute, &slow_base, &faster), + Err(VectorCoreError::Execution { node, .. }) if node == "n7" + )); +} + +#[test] +fn timeframe_supports_ccxt_seconds_years_and_rejects_i64_overflow() { + assert_eq!( + Timeframe::parse("1s") + .expect("seconds") + .resample_duration_ms(), + 1_000 + ); + assert_eq!( + Timeframe::parse("1y") + .expect("years") + .resample_duration_ms(), + 365 * 86_400_000 + ); + assert_eq!(Timeframe::parse("30s").expect("seconds").merge_minutes(), 0); + assert!(matches!( + Timeframe::parse("9223372036854776s"), + Err(VectorCoreError::InvalidProgram(_)) + )); +} + +#[test] +fn collisions_and_cross_pair_identity_fail_at_the_declared_source() { + let base = NumericFrame { + identity: identity("ETH/USDT", "5m"), + timestamps_ms: vec![0], + columns: BTreeMap::from([("info_1h".to_owned(), vec![Some(1.0)])]), + }; + let informative = NumericFrame { + identity: identity("BTC/USDT", "1h"), + timestamps_ms: vec![0], + columns: BTreeMap::from([("info".to_owned(), vec![Some(2.0)])]), + }; + let spec = MergeSpec { + base: base.identity.clone(), + informative: informative.identity.clone(), + ffill: false, + append_timeframe: true, + suffix: None, + date_column: "date".to_owned(), + source: source(), + }; + assert!( + matches!(merge(&base, &informative, &spec), Err(VectorCoreError::Execution { node, .. }) if node == "n7") + ); + let wrong_pair = NumericFrame { + identity: identity("SOL/USDT", "1h"), + ..informative + }; + assert!( + matches!(merge(&base, &wrong_pair, &spec), Err(VectorCoreError::Execution { node, .. }) if node == "n7") + ); +} + +fn spec(case: &Value) -> MergeSpec { + MergeSpec { + base: identity( + case["base_pair"].as_str().expect("base pair"), + case["call"]["timeframe"].as_str().expect("base timeframe"), + ), + informative: identity( + case["informative_pair"].as_str().expect("informative pair"), + case["call"]["timeframe_inf"] + .as_str() + .expect("informative timeframe"), + ), + ffill: case["call"]["ffill"].as_bool().expect("ffill"), + append_timeframe: case["call"]["append_timeframe"] + .as_bool() + .expect("append timeframe"), + suffix: case["call"] + .get("suffix") + .and_then(Value::as_str) + .map(str::to_owned), + date_column: case["call"] + .get("date_column") + .and_then(Value::as_str) + .unwrap_or("date") + .to_owned(), + source: source(), + } +} + +fn source() -> SourceLocation { + SourceLocation::new("n7", "strategy.py", 7, 3) +} + +fn identity(pair: &str, timeframe: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse(timeframe).expect("timeframe")).expect("identity") +} + +fn frame(value: &Value, pair: &str, timeframe: &str) -> NumericFrame { + let names = value["columns"].as_array().expect("columns"); + let rows = value["rows"].as_array().expect("rows"); + let mut columns = names + .iter() + .skip(1) + .map(|name| (name.as_str().expect("name").to_owned(), Vec::new())) + .collect::>(); + let mut timestamps_ms = Vec::new(); + for row in rows { + let row = row.as_array().expect("row"); + timestamps_ms.push(ms(row[0].as_str().expect("date"))); + for (index, name) in names.iter().enumerate().skip(1) { + columns + .get_mut(name.as_str().expect("name")) + .expect("column") + .push(number(&row[index])); + } + } + NumericFrame { + identity: identity(pair, timeframe), + timestamps_ms, + columns, + } +} + +fn assert_expected(actual: &super::MergedFrame, expected: &Value, case: &str) { + let names = expected["columns"].as_array().expect("output columns"); + let rows = expected["rows"].as_array().expect("output rows"); + assert_eq!(actual.timestamps_ms.len(), rows.len(), "{case}"); + for (index, row) in rows.iter().enumerate() { + let row = row.as_array().expect("output row"); + assert_eq!( + actual.timestamps_ms[index], + ms(row[0].as_str().expect("date")), + "{case} row {index}" + ); + for (column_index, name) in names.iter().enumerate().skip(1) { + let name = name.as_str().expect("output name"); + if actual.informative_dates_ms.contains_key(name) { + assert_eq!( + actual.informative_dates_ms[name][index], + row[column_index].as_str().map(ms), + "{case} {name} row {index}" + ); + } else { + assert_number( + actual.columns[name][index], + number(&row[column_index]), + case, + name, + index, + ); + } + } + } +} + +fn concatenate(first: &super::MergedFrame, second: &super::MergedFrame) -> super::MergedFrame { + let mut result = first.clone(); + result + .timestamps_ms + .extend_from_slice(&second.timestamps_ms); + for (name, values) in &second.columns { + result + .columns + .get_mut(name) + .expect("shared column") + .extend_from_slice(values); + } + for (name, values) in &second.informative_dates_ms { + result + .informative_dates_ms + .get_mut(name) + .expect("shared date column") + .extend_from_slice(values); + } + result +} + +fn slice(frame: &NumericFrame, start: usize, end: usize) -> NumericFrame { + NumericFrame { + identity: frame.identity.clone(), + timestamps_ms: frame.timestamps_ms[start..end].to_vec(), + columns: frame + .columns + .iter() + .map(|(name, values)| (name.clone(), values[start..end].to_vec())) + .collect(), + } +} + +fn assert_merged_bits(actual: &super::MergedFrame, expected: &super::MergedFrame) { + assert_eq!(actual.timestamps_ms, expected.timestamps_ms); + assert_eq!(actual.informative_dates_ms, expected.informative_dates_ms); + for (name, expected) in &expected.columns { + for (actual, expected) in actual.columns[name].iter().zip(expected) { + assert_number(*actual, *expected, "stream", name, 0); + } + } +} + +fn assert_number(actual: Option, expected: Option, case: &str, name: &str, row: usize) { + match (actual, expected) { + (None, None) => {} + (Some(actual), Some(expected)) => assert_eq!( + actual.to_bits(), + expected.to_bits(), + "{case} {name} row {row}" + ), + _ => panic!("{case} {name} row {row}: nullable value differs"), + } +} + +fn number(value: &Value) -> Option { + match value { + Value::Null => None, + Value::String(encoded) => Some(match encoded.as_str() { + "f64:nan" => f64::NAN, + "f64:inf" => f64::INFINITY, + "f64:-inf" => f64::NEG_INFINITY, + _ => f64::from_bits( + u64::from_str_radix(encoded.strip_prefix("f64:0x").expect("f64 token"), 16) + .expect("f64 bits"), + ), + }), + _ => panic!("expected nullable f64 token"), + } +} + +fn ms(value: &str) -> i64 { + let year = value[0..4].parse::().expect("year"); + let month = value[5..7].parse::().expect("month"); + let day = value[8..10].parse::().expect("day"); + let hour = value[11..13].parse::().expect("hour"); + let minute = value[14..16].parse::().expect("minute"); + let second = value[17..19].parse::().expect("second"); + super::days_from_civil(year, month, day).expect("date") * 86_400_000 + + hour * 3_600_000 + + minute * 60_000 + + second * 1_000 +} diff --git a/rust/crates/nfi-vector-core/src/lib.rs b/rust/crates/nfi-vector-core/src/lib.rs index 5ed6ba4..8005af2 100644 --- a/rust/crates/nfi-vector-core/src/lib.rs +++ b/rust/crates/nfi-vector-core/src/lib.rs @@ -7,6 +7,7 @@ #![forbid(unsafe_code)] #![allow(clippy::module_name_repetitions)] // Public names stay explicit across crate boundaries. +pub mod alignment; pub mod batch; pub mod column; pub mod engine; diff --git a/scripts/generate_informative_fixture.py b/scripts/generate_informative_fixture.py new file mode 100644 index 0000000..81d2a05 --- /dev/null +++ b/scripts/generate_informative_fixture.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Regenerate the pinned Freqtrade informative-merge oracle fixture.""" + +from __future__ import annotations + +import sys +from importlib import import_module +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "python")) + +def main() -> None: + """Regenerate the fixture from the pinned source tree.""" + module = import_module("nfi_backtest_engine.informative_fixture") + module.write_fixture(ROOT / module.FIXTURE_PATH) + + +if __name__ == "__main__": + main() diff --git a/tests/test_indicator_program.py b/tests/test_indicator_program.py index 10d598e..aff00e1 100644 --- a/tests/test_indicator_program.py +++ b/tests/test_indicator_program.py @@ -231,9 +231,13 @@ def test_indicator_program_records_informative_merge_before_forward_fill(tmp_pat "base_timeframe": "5m", "informative_timeframe": "1h", "ffill": False, + "append_timeframe": True, + "date_column": "date", + "suffix": None, } assert fill["parameters"] == {"direction": "forward"} assert fill["inputs"] == [merge["id"]] + assert fill["lookback"]["kind"] == "recursive" assert merge["source_order"] < fill["source_order"] @@ -265,25 +269,143 @@ def test_indicator_program_rejects_lookahead_capable_source( compile_indicator_program(source, class_name="Lookahead") -def test_indicator_program_rejects_prefilled_informative_merge(tmp_path: Path) -> None: - source = tmp_path / "Prefilled.py" +def test_indicator_program_preserves_informative_merge_defaults(tmp_path: Path) -> None: + source = tmp_path / "Defaults.py" source.write_text( "from freqtrade.strategy import IStrategy, merge_informative_pair\n" - "class Prefilled(IStrategy):\n" + "class Defaults(IStrategy):\n" " timeframe = '5m'\n" " def populate_indicators(self, dataframe, metadata, informative):\n" " dataframe = merge_informative_pair(\n" - " dataframe, informative, self.timeframe, '1h', ffill=True\n" + " dataframe, informative, self.timeframe, '1h'\n" " )\n" " return dataframe\n", encoding="utf-8", ) + program = compile_indicator_program(source, class_name="Defaults") + merge = next(node for node in program["nodes"] if node["op"] == "informative-merge") + + assert merge["parameters"] == { + "base_timeframe": "5m", + "informative_timeframe": "1h", + "ffill": True, + "append_timeframe": True, + "date_column": "date", + "suffix": None, + } + assert merge["lookback"]["kind"] == "recursive" + + +def test_indicator_program_binds_mixed_informative_merge_arguments(tmp_path: Path) -> None: + source = tmp_path / "MixedArguments.py" + source.write_text( + "from freqtrade.strategy import IStrategy, merge_informative_pair\n" + "class MixedArguments(IStrategy):\n" + " timeframe = '5m'\n" + " def populate_indicators(self, dataframe, metadata, informative):\n" + " dataframe = merge_informative_pair(\n" + " dataframe, informative=informative, timeframe=self.timeframe,\n" + " timeframe_inf='1h', ffill=False, append_timeframe=False,\n" + " date_column='opened_at', suffix='btc',\n" + " )\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="MixedArguments") + merge = next(node for node in program["nodes"] if node["op"] == "informative-merge") + + assert merge["parameters"] == { + "base_timeframe": "5m", + "informative_timeframe": "1h", + "ffill": False, + "append_timeframe": False, + "date_column": "opened_at", + "suffix": "btc", + } + + +def test_indicator_program_normalizes_falsy_suffix_like_freqtrade(tmp_path: Path) -> None: + source = tmp_path / "FalsySuffix.py" + source.write_text( + "from freqtrade.strategy import IStrategy, merge_informative_pair\n" + "class FalsySuffix(IStrategy):\n" + " def populate_indicators(self, dataframe, metadata, informative):\n" + " return merge_informative_pair(\n" + " dataframe, informative, '5m', '1h', suffix=''\n" + " )\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="FalsySuffix") + merge = next(node for node in program["nodes"] if node["op"] == "informative-merge") + + assert merge["parameters"]["suffix"] is None + + +@pytest.mark.parametrize( + ("call", "message"), + [ + ( + "merge_informative_pair(dataframe, informative, '5m', '1h', False, ffill=False)", + "duplicate informative merge argument ffill", + ), + ( + "merge_informative_pair(dataframe, informative, '5m', '1h', unknown=False)", + "unknown informative merge keyword unknown", + ), + ( + "merge_informative_pair(dataframe, informative, '5m', '1h', " + "ffill=metadata['ffill'])", + "dynamic indicator parameter", + ), + ( + "merge_informative_pair(dataframe, informative, '5m', '1h', " + "append_timeframe=True, suffix='btc')", + "suffix conflicts with append_timeframe", + ), + ( + "merge_informative_pair(dataframe, informative, '5m', '1h', " + "append_timeframe=False)", + "informative merge without an output suffix", + ), + ( + "merge_informative_pair(dataframe, informative, '1h', '5m', ffill=False)", + "faster informative timeframe would create rows", + ), + ( + "merge_informative_pair(dataframe, informative, '5m', 'nonsense')", + "invalid informative merge timeframe 'nonsense'", + ), + ( + "merge_informative_pair(dataframe, informative, '5m', '1h', " + "False, True, 'date', None, 'extra')", + "informative merge signature", + ), + ], +) +def test_indicator_program_rejects_ambiguous_informative_merge_calls( + tmp_path: Path, + call: str, + message: str, +) -> None: + source = tmp_path / "Rejected.py" + source.write_text( + "from freqtrade.strategy import IStrategy, merge_informative_pair\n" + "class Rejected(IStrategy):\n" + " timeframe = '5m'\n" + " def populate_indicators(self, dataframe, metadata, informative):\n" + f" dataframe = {call}\n" + " return dataframe\n", + encoding="utf-8", + ) + with pytest.raises( IndicatorProgramCompileError, - match="informative merge must not fill before source-ordered fill", + match=rf"strategy\.py:5:\d+:.*{message}", ): - compile_indicator_program(source, class_name="Prefilled") + compile_indicator_program(source, class_name="Rejected") def test_indicator_program_constant_folds_static_control_without_lookahead(tmp_path: Path) -> None: diff --git a/tests/test_informative_fixture.py b/tests/test_informative_fixture.py new file mode 100644 index 0000000..0b1fbee --- /dev/null +++ b/tests/test_informative_fixture.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from nfi_backtest_engine.informative_fixture import ( # isort: skip + FIXTURE_PATH, + canonical_sha256, +) + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_committed_fixture_has_pinned_source_identity_and_valid_fingerprint() -> None: + stored = json.loads((ROOT / FIXTURE_PATH).read_text(encoding="utf-8")) + + assert stored["fingerprint"] == canonical_sha256(stored) + assert stored["source"] == { + "version": "2026.5.1", + "commit": "6fa470939cc74bf0672e0e348a4d9b293072e43c", + "strategy_helper_sha256": ( + "46a15179738d83a39148ac96f5ee2f2d50c4514332d059c4458a9d7d3d0e4812" + ), + "strategy_helper": "freqtrade/strategy/strategy_helper.py", + "timeframe_to_minutes": "ccxt.Exchange.parse_timeframe(timeframe) // 60", + } + + +def test_fixture_covers_the_compact_informative_contract() -> None: + fixture = json.loads((ROOT / FIXTURE_PATH).read_text(encoding="utf-8")) + cases = {str(case["name"]): case for case in fixture["cases"]} + + assert { + "boundary_ffill_false", + "boundary_ffill_true", + "equal_timeframe", + "empty_informative", + "missing_informative_rows", + "duplicate_cartesian", + "duplicate_cartesian_ffill_true", + "leading_repair", + "leading_repair_uses_last_historical_row", + "unsorted_ffill_false", + "unsorted_ffill_true", + "unsorted_informative_ffill_false", + "unsorted_informative_ffill_true", + "custom_date_column", + "suffix_naming", + "cross_pair_sentinel", + "f64_and_pandas_null_encoding", + "informative_f64_and_pandas_null_encoding", + "faster_timeframe_failure", + "month_boundary", + } == cases.keys() + assert cases["faster_timeframe_failure"]["error"]["type"] == "ValueError" + + duplicate_output = cases["duplicate_cartesian_ffill_true"]["output"] + assert len(duplicate_output["rows"]) == 4 + + leading_output = cases["leading_repair"]["output"] + assert [row[3] for row in leading_output["rows"]] == [ + "f64:0x4024000000000000", + "f64:0x4024000000000000", + "f64:0x4034000000000000", + ] + + multi_history = cases["leading_repair_uses_last_historical_row"]["output"] + multi_history_rows = [ + dict(zip(multi_history["columns"], row, strict=True)) for row in multi_history["rows"] + ] + historical_row = { + "date_1h": "2024-01-01T01:00:00Z", + "info_1h": "f64:0x4034000000000000", + "row_marker_1h": "f64:0x4069000000000000", + } + assert [ + {column: row[column] for column in historical_row} for row in multi_history_rows[:3] + ] == [historical_row, historical_row, historical_row] + assert { + column: multi_history_rows[3][column] for column in historical_row + } == { + "date_1h": "2024-01-01T02:00:00Z", + "info_1h": "f64:0x403e000000000000", + "row_marker_1h": "f64:0x4072c00000000000", + } + + unsorted = cases["unsorted_informative_ffill_true"] + assert [row[1] for row in unsorted["informative"]["rows"]] == [ + "f64:0x4034000000000000", + "f64:0x4024000000000000", + ] + assert [row[3] for row in unsorted["output"]["rows"]] == [ + "f64:0x4024000000000000", + "f64:0x4024000000000000", + "f64:0x4034000000000000", + ] + + custom_date = cases["custom_date_column"] + assert custom_date["call"]["date_column"] == "candle_open" + assert custom_date["output"]["columns"] == [ + "date", + "base", + "candle_open_1h", + "custom_info_1h", + ] + + cross_pair = cases["cross_pair_sentinel"] + assert cross_pair["base_pair"] != cross_pair["informative_pair"] + assert cross_pair["output"]["rows"][0][3] == "f64:0x40b0928000000000" + + +def test_fixture_distinguishes_ieee_floats_from_pandas_nulls() -> None: + fixture = json.loads((ROOT / FIXTURE_PATH).read_text(encoding="utf-8")) + cases = {str(case["name"]): case for case in fixture["cases"]} + encoded = cases["f64_and_pandas_null_encoding"] + columns = encoded["base"]["columns"] + row = dict(zip(columns, encoded["base"]["rows"][0], strict=True)) + + assert row == { + "date": "2024-01-01T00:00:00Z", + "float_nan": "f64:0x7ff8000000000000", + "positive_infinity": "f64:0x7ff0000000000000", + "negative_infinity": "f64:0xfff0000000000000", + "positive_zero": "f64:0x0000000000000000", + "negative_zero": "f64:0x8000000000000000", + "python_none": None, + "pandas_na": None, + } + + raw = (ROOT / FIXTURE_PATH).read_text(encoding="utf-8") + assert ":NaN" not in raw + assert ":Infinity" not in raw + + +def test_fixture_preserves_informative_special_values_on_exact_and_unmatched_rows() -> None: + fixture = json.loads((ROOT / FIXTURE_PATH).read_text(encoding="utf-8")) + cases = {str(case["name"]): case for case in fixture["cases"]} + encoded = cases["informative_f64_and_pandas_null_encoding"] + columns = encoded["output"]["columns"] + unmatched = dict(zip(columns, encoded["output"]["rows"][0], strict=True)) + exact = dict(zip(columns, encoded["output"]["rows"][1], strict=True)) + + informative_columns = [column for column in columns if column.startswith("info_")] + assert unmatched["date_5m"] is None + assert {unmatched[column] for column in informative_columns} == { + "f64:0x7ff8000000000000" + } + assert exact == { + "date": "2024-01-01T00:00:00Z", + "base": "f64:0x4038000000000000", + "date_5m": "2024-01-01T00:00:00Z", + "info_float_nan_5m": "f64:0x7ff8000000000000", + "info_positive_infinity_5m": "f64:0x7ff0000000000000", + "info_negative_infinity_5m": "f64:0xfff0000000000000", + "info_positive_zero_5m": "f64:0x0000000000000000", + "info_negative_zero_5m": "f64:0x8000000000000000", + "info_python_none_5m": None, + "info_pandas_na_5m": None, + } diff --git a/tests/test_strategy_compat.py b/tests/test_strategy_compat.py new file mode 100644 index 0000000..243995c --- /dev/null +++ b/tests/test_strategy_compat.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd +import pytest +from nfi_backtest_engine.informative_fixture import FIXTURE_PATH, execute_cases +from nfi_backtest_engine.strategy_compat import ( + merge_informative_pair, + timeframe_minutes, + timeframe_seconds, +) +from pandas.testing import assert_frame_equal + +ROOT = Path(__file__).resolve().parents[1] + + +def _frame(dates: list[str], values: list[float], column: str = "close") -> pd.DataFrame: + return pd.DataFrame( + { + "date": pd.to_datetime(dates, utc=True), + column: values, + } + ) + + +def test_informative_merge_exposes_slow_candle_only_at_its_close_boundary() -> None: + base = _frame( + ["2026-01-01 00:50", "2026-01-01 00:55", "2026-01-01 01:00"], + [1.0, 2.0, 3.0], + ) + informative = _frame(["2026-01-01 00:00"], [100.0], column="signal") + + result = merge_informative_pair(base, informative, "5m", "1h", ffill=False) + + assert pd.isna(result.loc[0, "signal_1h"]) + assert result.loc[1, "signal_1h"] == 100.0 + assert pd.isna(result.loc[2, "signal_1h"]) + + +def test_informative_merge_forward_fills_only_historical_values() -> None: + base = _frame( + ["2026-01-01 01:00", "2026-01-01 01:05", "2026-01-01 01:55"], + [1.0, 2.0, 3.0], + ) + informative = _frame( + ["2026-01-01 00:00", "2026-01-01 01:00"], + [100.0, 200.0], + column="signal", + ) + + result = merge_informative_pair(base, informative, "5m", "1h", ffill=True) + + assert result["signal_1h"].tolist() == [100.0, 100.0, 200.0] + + +def test_informative_merge_preserves_freqtrade_duplicate_cartesian_behavior() -> None: + base = _frame(["2026-01-01 00:55", "2026-01-01 00:55"], [1.0, 2.0]) + informative = _frame( + ["2026-01-01 00:00", "2026-01-01 00:00"], + [100.0, 200.0], + column="signal", + ) + + result = merge_informative_pair(base, informative, "5m", "1h", ffill=False) + + assert len(result) == 4 + assert result["signal_1h"].tolist() == [100.0, 200.0, 100.0, 200.0] + + +def test_informative_merge_supports_suffix_and_month_visibility() -> None: + base = _frame(["2026-02-28 23:55"], [1.0]) + informative = _frame(["2026-02-01 00:00"], [100.0], column="signal") + + result = merge_informative_pair( + base, + informative, + "5m", + "1M", + ffill=False, + append_timeframe=False, + suffix="btc", + ) + + expected = pd.DataFrame( + { + "date": pd.to_datetime(["2026-02-28 23:55"], utc=True), + "close": [1.0], + "date_btc": pd.to_datetime(["2026-02-01 00:00"], utc=True), + "signal_btc": [100.0], + } + ) + assert_frame_equal(result, expected) + + +def test_informative_merge_rejects_faster_frame_and_conflicting_suffix() -> None: + frame = _frame(["2026-01-01 00:00"], [1.0]) + + with pytest.raises(ValueError, match="faster timeframe"): + merge_informative_pair(frame, frame, "1h", "5m") + with pytest.raises(ValueError, match="append_timeframe"): + merge_informative_pair(frame, frame, "5m", "1h", suffix="btc") + + +def test_timeframe_minutes_uses_freqtrade_ccxt_semantics() -> None: + assert timeframe_seconds("30s") == 30 + assert timeframe_minutes("30s") == 0 + assert timeframe_minutes("5m") == 5 + assert timeframe_minutes("1h") == 60 + assert timeframe_minutes("1M") == 43_200 + + +def test_compatibility_merge_is_exact_against_pinned_freqtrade_oracle() -> None: + oracle = json.loads((ROOT / FIXTURE_PATH).read_text(encoding="utf-8")) + + assert execute_cases(merge_informative_pair) == oracle["cases"] diff --git a/tests/test_vector_worker_resample.py b/tests/test_vector_worker_resample.py new file mode 100644 index 0000000..53ce798 --- /dev/null +++ b/tests/test_vector_worker_resample.py @@ -0,0 +1,109 @@ +"""Pinned Freqtrade calendar-frequency regression coverage.""" + +from __future__ import annotations + +import pandas as pd +import pytest +from nfi_backtest_engine.vector_worker import ( + _clean_ohlcv_like_freqtrade, + _freqtrade_resample_frequency, +) + + +def _ohlcv_frame(dates: list[str]) -> pd.DataFrame: + values = list(range(10, 10 + len(dates))) + return pd.DataFrame( + { + "date": pd.to_datetime(dates), + "open": values, + "high": [value + 1 for value in values], + "low": [value - 1 for value in values], + "close": values, + "volume": values, + } + ) + + +@pytest.mark.parametrize( + ("timeframe", "expected"), + [ + ("1s", "1s"), + ("30s", "30s"), + ("2w", "1W-MON"), + ("1M", "1MS"), + ("2M", "2MS"), + ("1y", "1YS"), + ], +) +def test_resample_frequency_matches_pinned_freqtrade_calendar_anchors( + timeframe: str, + expected: str, +) -> None: + assert _freqtrade_resample_frequency(timeframe) == expected + + +def test_weekly_cleanup_uses_monday_anchors_for_multiweek_timeframes() -> None: + result = _clean_ohlcv_like_freqtrade( + _ohlcv_frame( + [ + "2024-01-01T00:00:00Z", + "2024-01-08T00:00:00Z", + "2024-01-15T00:00:00Z", + ] + ), + pair="GENERIC/USDT", + timeframe="2w", + ) + + assert result["date"].tolist() == list( + pd.to_datetime( + [ + "2024-01-01T00:00:00Z", + "2024-01-08T00:00:00Z", + "2024-01-15T00:00:00Z", + ] + ) + ) + + +@pytest.mark.parametrize( + ("timeframe", "dates", "expected_dates"), + [ + ( + "1M", + ["2024-01-15T00:00:00Z", "2024-03-15T00:00:00Z"], + ["2024-01-01T00:00:00Z", "2024-02-01T00:00:00Z", "2024-03-01T00:00:00Z"], + ), + ( + "2M", + ["2024-01-15T00:00:00Z", "2024-03-15T00:00:00Z"], + ["2024-01-01T00:00:00Z", "2024-03-01T00:00:00Z"], + ), + ( + "1y", + ["2024-01-15T00:00:00Z", "2025-01-15T00:00:00Z"], + ["2024-01-01T00:00:00Z", "2025-01-01T00:00:00Z"], + ), + ], +) +def test_calendar_cleanup_uses_period_start_and_keeps_gap_and_final_candle( + timeframe: str, + dates: list[str], + expected_dates: list[str], +) -> None: + result = _clean_ohlcv_like_freqtrade( + _ohlcv_frame(dates), + pair="GENERIC/USDT", + timeframe=timeframe, + ) + + assert result["date"].tolist() == list(pd.to_datetime(expected_dates)) + assert result.iloc[-1]["close"] == 11 + if timeframe == "1M": + assert result.loc[1, ["open", "high", "low", "close", "volume"]].tolist() == [ + 10, + 10, + 10, + 10, + 0, + ] From 19abbddaa7a779fea214f15b77b3ccc90c0552b8 Mon Sep 17 00:00:00 2001 From: vntrevx <20063774+vntrevx@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:31:51 +0900 Subject: [PATCH 3/3] chore(roadmap): complete M20-05 --- planning/roadmap-state.json | 53 ++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/planning/roadmap-state.json b/planning/roadmap-state.json index c581f8a..c02dd7b 100644 --- a/planning/roadmap-state.json +++ b/planning/roadmap-state.json @@ -1,10 +1,10 @@ { "schema_version": "1.0.0", "roadmap_id": "nfi-backtest-engine-post-v1.1.0", - "revision": 132, - "updated_at": "2026-08-11T01:26:42+09:00", + "revision": 133, + "updated_at": "2026-08-11T02:31:11+09:00", "acceptance_commands": "planning/acceptance-commands.json", - "active_task_id": "M20-05", + "active_task_id": null, "execution_policy": { "max_in_progress": 1, "selection": "lowest order pending task whose dependencies are completed", @@ -4767,7 +4767,7 @@ "order": 2005, "milestone": "M20", "title": "Implement multi-timeframe and informative data semantics", - "status": "in_progress", + "status": "completed", "depends_on": [ "M20-04" ], @@ -4787,9 +4787,16 @@ "cross-pair dependency tests" ], "started_at": "2026-08-11T01:26:42+09:00", - "completed_at": null, - "commit_sha": null, - "evidence": [], + "completed_at": "2026-08-11T02:31:11+09:00", + "commit_sha": "68d82daeadcf798e8e82534d470b8709d9191bb9", + "evidence": [ + ".nfi/roadmap-acceptance/M20-05/68d82daeadcf798e8e82534d470b8709d9191bb9/acceptance-report.json", + ".nfi/roadmap-acceptance/M20-05/68d82daeadcf798e8e82534d470b8709d9191bb9/latest-program-gap.json", + ".nfi/roadmap-acceptance/M20-05/68d82daeadcf798e8e82534d470b8709d9191bb9/SHA256SUMS.txt", + ".nfi/roadmap-acceptance/M20-05/68d82daeadcf798e8e82534d470b8709d9191bb9/x7-spot/run.json", + ".nfi/roadmap-acceptance/M20-05/68d82daeadcf798e8e82534d470b8709d9191bb9/x7-futures/run.json", + "benchmarks/reference/informative/freqtrade-2026.5.1.json" + ], "blocker": null }, { @@ -8362,6 +8369,38 @@ "pair_or_timeframe_hardcoding_allowed": false, "cross_pair_dependencies_must_be_explicit": true } + }, + { + "sequence": 180, + "timestamp": "2026-08-11T02:31:11+09:00", + "task_id": "M20-05", + "event": "task_completed", + "details": { + "implementation_commit": "68d82daeadcf798e8e82534d470b8709d9191bb9", + "latest_upstream_commit": "897a1523391b8222ee711eba9714b59a3e77265a", + "latest_strategy_sha256": "e99b4f58ecf507da86d2ba94e641c2994f17a1456133c2cff26946fa6b4a1afb", + "freqtrade_version": "2026.5.1", + "informative_oracle_case_count": 20, + "informative_oracle_fingerprint": "0c680501981ae7eb4fff1e4494c7c0d27fee5dcfefbf5131a857c47327440939", + "resample_anchors_exact": true, + "informative_visibility_exact": true, + "missing_duplicate_unsorted_and_cross_pair_exact": true, + "nan_null_infinity_and_signed_zero_exact": true, + "streaming_fail_closed_and_bounded": true, + "final_forward_fill_bounded": true, + "python_test_count": 763, + "future_compatibility_test_count": 134, + "rust_workspace_test_count": 211, + "spot_trade_surface_and_full_state_exact": true, + "futures_trade_surface_and_full_state_exact": true, + "runtime_hardcoding_added": false, + "latest_x7_fully_compiled": false, + "latest_x7_fail_closed_source": "strategy.py:3168:21", + "vector_engine_multiframe_connection": false, + "full_native_strategy_claim": false, + "next_eligible_task": "M21-01", + "next_task_manual_gate": false + } } ] }