diff --git a/.github/product-release-contract.json b/.github/product-release-contract.json index 7db23642..2336054b 100644 --- a/.github/product-release-contract.json +++ b/.github/product-release-contract.json @@ -1,6 +1,6 @@ { "schema_version": "1.0.0", - "package_version": "1.5.0", + "package_version": "1.6.0", "release_kind": "product", "combined_full_x7_certified": false, "distribution_policy": { diff --git a/CHANGELOG.md b/CHANGELOG.md index ade31017..da508e55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,32 @@ All notable changes are recorded here. This project follows Semantic Versioning. ## Unreleased +## 1.6.0 - 2026-08-14 + +- Added the source-compiled Full Native transport: Rust now executes the complete + Indicator, Signal, Tag, callback, order, wallet, and state pipeline without + importing or executing strategy Python at runtime. +- Added content-bound Full Native worker calibration, pair-local raw-frame loading, + live-value release, and direct file-backed simulator handoff without persistent + analyzed Feather outputs. +- Added fail-closed spool-capacity admission from the run's gap-fill bound, automatic + delete-on-close cleanup, and profile evidence for admitted versus actual bytes. +- Split NFI manager validation failures from pair/timestamp/route/source-located + runtime diagnostics, and validate the embedded simulator config before expensive + pair preparation. - Connected typed Rust vector output directly to the chronological simulator, with pair-parallel DAG preparation and no parallel wallet or order mutation. - Preserved SHA-verified Feather as the bounded-memory evidence/replay path and added zero-tolerance trade-surface and every-candle full-state transport parity tests. -- Kept latest-X7 Full Native qualification and the v1.6.0 release behind the remaining - M22 source-lowering, dual-mode exactness, performance, storage, and three-OS gates. +- Certified a five-year Full Native Spot workload with three byte-identical results, + a pre-admitted 38.47 GB temporary spool bound, and no retained spool files. This is + a single-host performance/storage claim, not a new official Freqtrade certificate. +- Generalized current X7 Grind-5 fallback lowering for `slice_profit_exit`, Boolean + protection columns, bare Derisk state, Futures mode, and liquidation-distance + expressions without strategy-version, pair, or expected-result branches. +- Pinned the Freqtrade same-candle contract: position adjustment and its filled order + are applied before stop/exit evaluation. The external 2022 Futures report that + motivated the hotfix remains unclaimed until its sealed input is supplied. ## 1.5.0 - 2026-08-10 diff --git a/README.md b/README.md index 4300cee4..71f3a270 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ semantics stop with a clear fail-closed verdict instead of being approximated. | Latest public release | [v1.5.0](https://github.com/vntrevx/NFI_BackTestEngine/releases/tag/v1.5.0) | | Five-year Spot | Certified independently by v1.0.0 | | Five-year Futures | Certified independently by v1.1.0 | -| Current `main` | v1.5.0 Native Stateful Runtime; no new combined Full X7 certification claim | +| Current `main` | v1.6.0 Full Native Strategy release candidate; no new combined Full X7 certification claim | The Spot and Futures certificates remain valid for their own sealed strategy, configuration, data, wheel, and host. They are not a same-candidate Spot-versus-Futures @@ -80,6 +80,15 @@ Both certificates still prove exact parity for their sealed workloads. They do n form a controlled Spot-versus-Futures performance test, so this project makes no cross-mode speed ratio claim. +The v1.6.0 Full Native path completed a latest-X7 Spot development +qualification over 80 pairs and 2021-01-01 through 2026-01-01. Three fresh +processes finished in 63:17, 62:51, and 62:44 with byte-identical 1,162-trade +results (0.88% wall spread). Median peak RSS was 38.9 GiB. The 38.47 GB temporary +row spool remained below its pre-admitted bound and was reclaimed after every +run. This is a single-host performance/storage certificate, not an official +Freqtrade five-year parity or cross-platform performance claim. See the +[M22 performance evidence](benchmarks/evidence/m22/full-native-performance-storage.json). + For a fresh-run expectation, compare Spot's full native median with Futures' cold seed. Use the Futures reuse number only after its content-addressed vectors already exist. Actual runtime still depends on strategy behavior, data, hardware, and memory limits. @@ -141,8 +150,8 @@ nfi-bte --version nfi-bte doctor ``` -The latest public installer and a source checkout of `main` both return -`nfi-bte 1.5.0`. +The latest public installer returns `nfi-bte 1.5.0`. A source checkout of `main` +returns the v1.6.0 Full Native Strategy release candidate. ## Quick start diff --git a/benchmarks/evidence/m22/full-native-performance-storage.json b/benchmarks/evidence/m22/full-native-performance-storage.json new file mode 100644 index 00000000..d173aa37 --- /dev/null +++ b/benchmarks/evidence/m22/full-native-performance-storage.json @@ -0,0 +1,162 @@ +{ + "schema_version": "full-native-performance-storage-v1", + "status": "certified", + "recorded_at": "2026-08-12", + "implementation": { + "commit": "eba09d977e3eddb45f7a7982ebf026b0e5731c45", + "measurement_binary_sha256": "caf5599a3a32fe6815fa4fb96a3ca9f8dfeb99536715ed5906cd6a1a06580d01", + "post_measurement_change": "Tag/Signal structural preflight liveness only; the measured execution hot path is unchanged", + "current_exact_fixture_source_fingerprint": "f89ee9bc2de011e278948d962fa831e8087675192ccbb4427d48ebb8ee8f8417", + "current_exact_fixture_binary_sha256": "642d67a08e5405b7ea5a8d7864eade0ce1655d337e2bb6b1552ffa4af9769f76" + }, + "workload": { + "strategy": "NostalgiaForInfinityX7", + "strategy_version": "v17.4.528", + "upstream_commit": "1df961c07e5ce6b1a8cb459a2a46958aed258323", + "strategy_sha256": "45a2bf611d6fc5e60c7e1f4c672ce7932f6573872c2a959f6338d079dac5e382", + "manifest_sha256": "3b1b18bdaaacca43ff538e1397547df020ace3e80913371aa137607dc2185e90", + "config_sha256": "402d90b02890192a54cb6e7e9ba6fdc11b69b89bd8627fd9a268f70f4e3110db", + "trading_mode": "spot", + "timerange": "20210101-20260101", + "pair_count": 80, + "raw_frame_count": 400, + "timeframes": [ + "5m", + "15m", + "1h", + "4h", + "1d" + ], + "startup_candles": 800, + "source_row_shift": 1, + "retained_feature_count": 104, + "retained_feature_fingerprint": "ad099fe5ef897191a4d34597e82e0e3c850d960e24fd9611713fdd714ee17d9b", + "manifest_declared_raw_rows": 72488162, + "execution_row_count": 42134157, + "history_coverage_policy": "available local sealed history" + }, + "environment": { + "os": "Linux x86_64", + "kernel": "7.0.0-28-generic", + "cpu": "12th Gen Intel(R) Core(TM) i7-1260P", + "logical_cpu_count": 16, + "memory_bytes": 50195083264, + "hardware_fingerprint": "5c71dfa54dd8a8d5e87088d2c78522f6e85d009434b4779d45bc378b82ffa30d", + "pair_worker_limit": 8 + }, + "calibration": { + "key": "3610776e2f8680861b6b2c7a515b47245cb0ff2cc0e85f205d9fa881c8f44281", + "record_sha256": "19d68b154601badbebe5f35d0918c06e31d112e83159f3f2ec91fc7af3f3377a", + "probe_pair": "XRP/USDT", + "probe_peak_rss_bytes": 4605874176, + "probe_wall_time_seconds": 112.182186987, + "admitted_workers": 8 + }, + "result": { + "sha256": "5adac8e620877b30d7e52988e377d068799be3a4045a8d45221e29620aa08266", + "byte_identical_across_repetitions": true, + "trade_count": 1162, + "final_balance": 27064.22769050002, + "profit_total_abs": 26064.227690500004, + "total_volume": 3523156.3395277644, + "rejected_signals": 4476, + "maximum_concurrent_trades": 6 + }, + "repetitions": [ + { + "name": "cold-01", + "wall_seconds": 3797, + "vector_execute_seconds": 3115.615091735, + "event_loop_seconds": 658.949939495, + "validation_seconds": 19.61198466, + "peak_rss_bytes": 41832026112, + "profile_sha256": "c6889a5f0636980d268217f6221627fbb732eb78be1ddf7ad8fd39280cd0e2d1", + "time_log_sha256": "58dff50b2e9aaebb0da2d845f769b3c1c8b9633d8a3f4c05a0a712dbba7f0131" + }, + { + "name": "warm-02", + "wall_seconds": 3771, + "vector_execute_seconds": 3103.454727467, + "event_loop_seconds": 650.534573412, + "validation_seconds": 13.957065728, + "peak_rss_bytes": 41783009280, + "profile_sha256": "7c3e74732834d3c9b0a22fadbbba65011241260298ca570485afbb522ed2a820", + "time_log_sha256": "6b74da336a8fca324f23499f382f3c33a9ede47476b330bd3d1ad690d57e72b9" + }, + { + "name": "warm-03", + "wall_seconds": 3764, + "vector_execute_seconds": 3096.122603618, + "event_loop_seconds": 650.939236179, + "validation_seconds": 13.801878597, + "peak_rss_bytes": 41793679360, + "profile_sha256": "dd40bc5287a567aabdfea30352f0ce65d115c15ccf778d0b8cebc22dc009c1e3", + "time_log_sha256": "e3b901846e473dbf3958e7174c49dafb589fcc6e600b46cfd558c8c76de2c915" + } + ], + "aggregate": { + "wall_seconds_min": 3764, + "wall_seconds_median": 3771, + "wall_seconds_max": 3797, + "wall_spread_ratio": 0.008767268862911796, + "five_repetitions_required": false, + "vector_execute_seconds_median": 3103.454727467, + "event_loop_seconds_median": 650.939236179, + "peak_rss_bytes_median": 41793679360, + "peak_rss_spread_ratio": 0.0011731283324167502 + }, + "storage": { + "target_source": "os-temp", + "cleanup_mode": "unlink-on-open", + "required_upper_bound_bytes": 38469364560, + "actual_file_backed_bytes": 38468485341, + "bound_margin_bytes": 879219, + "actual_within_bound": true, + "available_bytes_at_first_admission": 93288243200, + "available_bytes_after_final_run": 93272535040, + "named_orphan_file_count": 0, + "delete_pending_handle_count_after_exit": 0, + "partial_result_or_profile_count_after_failed_run": 0, + "unbounded_local_accumulation_observed": false + }, + "exact_regression": { + "spot": { + "run_sha256": "c8b689934d386b9b355c6b54acf548c71851ac79a7ea447c559a551c6cd0f866", + "input_sha256": "b76ddcae92972692ff1fb011ef540dc81ba389bc93019174ff90a36be1539bcc", + "output_sha256": "3624f7b2de5368e126bdc0582f86a6e7c2adfaf78e02497c21ea4a207c632f98", + "trade_surface_exact": true, + "full_state_exact": true, + "state_event_count": 288, + "state_stream_hash": "6a59c86a808e0e5f8fc016c1467fa173502ea37b0eb870f84e7f51cfa1818f8b" + }, + "futures": { + "run_sha256": "002579af277077e15a99aa02a8b9b1cf910385857a7ce7f07898051191804df0", + "input_sha256": "d6236067eb775fb59842853c9d15cc0ca9c2b7914b60a636d59837c00a610f14", + "output_sha256": "a36ebf30a353e185953da02b722c0268175a02b1d850555057efe77492d93af6", + "trade_surface_exact": true, + "full_state_exact": true, + "state_event_count": 5472, + "state_stream_hash": "029a12115d0e9095d4e1cc19e25694a28bee4c625b6cd59dc9226ab1ceaeff85" + } + }, + "excluded_runs": [ + { + "reason": "pre-fix binary rejected a reached cross-side short_normal helper route", + "included_in_performance_statistics": false + }, + { + "reason": "concurrent diagnostic process contaminated host resources and the run was terminated", + "included_in_performance_statistics": false + } + ], + "claim_boundary": { + "five_year_full_native_performance_and_storage_certified": true, + "latest_x7_source_compiled_without_runtime_python": true, + "runtime_strategy_pair_timerange_sha_or_result_hardcoding_added": false, + "official_freqtrade_five_year_parity_claimed": false, + "cross_platform_performance_claimed": false, + "futures_five_year_performance_claimed": false, + "v1_6_0_released": false + }, + "fingerprint": "a9ab243261ecbe79c23e1f22f4a3fe8cefa794be1bb494cf922442efbd8699ad" +} diff --git a/benchmarks/evidence/m22/latest-x7-full-native-qualification.json b/benchmarks/evidence/m22/latest-x7-full-native-qualification.json new file mode 100644 index 00000000..b49faf22 --- /dev/null +++ b/benchmarks/evidence/m22/latest-x7-full-native-qualification.json @@ -0,0 +1,91 @@ +{ + "schema_version": "latest-x7-full-native-qualification-v1", + "status": "exact", + "source": { + "upstream_commit": "1df961c07e5ce6b1a8cb459a2a46958aed258323", + "strategy_sha256": "45a2bf611d6fc5e60c7e1f4c672ce7932f6573872c2a959f6338d079dac5e382", + "strategy_version": "v17.4.528", + "freqtrade_version": "2026.5.1" + }, + "changed_branch": { + "kind": "signal-protection", + "signal": "47", + "evidence_path": "benchmarks/evidence/m22/latest-x7-signal47-boundary.json", + "evidence_sha256": "84ee3f451d016a8f67f5b98314970a527cddde97f4476b147a0375c1bc6ce8d1", + "evidence_fingerprint": "d4128b380d483a05f7a230bfe78dad14502438d45a8fa0adeb0c094c5d73f7fe", + "spot_exact": true, + "futures_exact": true, + "tag_exact": true, + "baseline_reaches_and_current_rejects_boundary": true, + "one_term_passes_current": true, + "callback_and_stateful_methods_changed": false + }, + "spot": { + "pair": "BTC/USDT", + "timerange": "20230101-20230102", + "transport": "full-native-vector-manifest", + "raw_frame_count": 5, + "row_count": 1089, + "retained_feature_count": 104, + "program_fingerprints": { + "indicator": "df8339e280aadb53cdca1efc552cfab3351959187b771763999f296aaa8c05a2", + "signal": "35d744c99f3702320572783608b5ff28182d75b7a91b1616d7d24088ec37d69b", + "tag": "0804bfbbd39340a53da7b7d61cefa1ef472af45d08870e1b4fa575eae9dc22be" + }, + "trade_surface": { + "trade_count": 0, + "native_sha256": "cfd5868610e8e916a9233bc67f42e6e78ae3f5fff95855c4b4f87d0b1df568f6", + "official_sha256": "cfd5868610e8e916a9233bc67f42e6e78ae3f5fff95855c4b4f87d0b1df568f6", + "exact": true + }, + "full_state": { + "event_count": 288, + "stream_hash": "e119fc56545e34f2b368c4baa59c0067d64d7d821a018687de8166aa81bb4428", + "first_difference": null, + "exact": true + }, + "native_vector_seconds": 7.042690725, + "blockers": [] + }, + "futures": { + "pair": "APE/USDT:USDT", + "timerange": "20220410-20220420", + "transport": "full-native-vector-manifest", + "raw_frame_count": 10, + "row_count": 3681, + "retained_feature_count": 104, + "program_fingerprints": { + "indicator": "b3b1ab61235141e7da9ba4611a5879676ba007d20a3c34105f01e778ec3996c5", + "signal": "0d68c82365732250ca6302056d8c723e7b20565859410bc5b792d6be13cc237b", + "tag": "6ae46ee6e2f32290dea1685d9d9173a156ca1d4425afec11148c4de7809bb388" + }, + "trade_surface": { + "trade_count": 1, + "entry_tag": "142 ", + "exit_reason": "exit_long_tc_w_0_8 ( 142 )", + "funding_fee": "-2.9366127", + "native_sha256": "f472f82eb970142d0d10b1b59171a8b9e80c001b9698f47ff1d53caf1f800eb7", + "official_sha256": "f472f82eb970142d0d10b1b59171a8b9e80c001b9698f47ff1d53caf1f800eb7", + "exact": true + }, + "full_state": { + "event_count": 2880, + "stream_hash": "eff3ddae550d7f8877bd04f96457b6ed052b4f9ea0cca968d9a4721141f07bdb", + "first_difference": null, + "exact": true + }, + "native_vector_seconds": 27.139196679, + "blockers": [] + }, + "claims": { + "latest_upstream_compiles_without_blockers": true, + "indicator_signal_tag_and_stateful_runtime_is_native": true, + "python_strategy_execution_in_native_runtime": false, + "official_fallback_retained": true, + "runtime_strategy_pair_timerange_sha_or_result_branches_added": false, + "m22_01_latest_dual_mode_qualification_complete": true, + "five_year_performance_certified": false, + "v1_6_0_released": false + }, + "fingerprint": "2e3bb9fc12597d400043f1e2015687b9bca5da7ed97859f27b8e9842e3c8d087" +} diff --git a/benchmarks/evidence/m22/latest-x7-signal47-boundary.json b/benchmarks/evidence/m22/latest-x7-signal47-boundary.json new file mode 100644 index 00000000..46304697 --- /dev/null +++ b/benchmarks/evidence/m22/latest-x7-signal47-boundary.json @@ -0,0 +1 @@ +{"schema_version":"latest-x7-signal47-boundary-v1","source":{"commit":"1df961c07e5ce6b1a8cb459a2a46958aed258323","strategy_sha256":"45a2bf611d6fc5e60c7e1f4c672ce7932f6573872c2a959f6338d079dac5e382"},"baseline":{"commit":"6eabfd5cb2a99da89a955f023fdec9390ebddcda","strategy_sha256":"85d0d9606743f32df553baf9937009a5a20dd051821be4b4bd389bd98402566e"},"strategy_class":"NostalgiaForInfinityX7","changed_route":{"side":"long","signal":"47","expression":"((rsi_3_4h_gt_20) | (rsi_3_1d_gt_50) | (aroonu_14_1d_lt_80) | (roc_9_1d_lt_30))","expression_sha256":"093ebcb8b9488f4efbdc53f688279b56c94cb60a758546bd7bf441e8cebc19f8"},"source_diff":{"changed_methods":{"populate_entry_trend":{"baseline_ast_sha256":"7a88ea275513384f01eb19790f918fe4736e1f94137bbc91e9a2173e95ae7717","current_ast_sha256":"d7705055a5295cead39e35be4aedc5c9ce38059bf92db55f0ff2e4df0c3cd97d"},"version":{"baseline_ast_sha256":"9be67eaa40d99c448a13c2b739d0b77a64cdaaf517837afe095d2969452b6e34","current_ast_sha256":"67dee8a8275d05c173b2aaae81ab7842d02146ee75ee78305c0a37486e7edc3f"}},"unchanged_method_count":143,"unchanged_methods_fingerprint":"318fc3c6fe1450ecd6c2f85554e296adb2fdd3cef43d544eb32690494892320b","callback_and_stateful_methods_changed":false},"input_contract":{"rows":8,"default_numeric_value":0.0,"scalar_overrides":{"volume":1.0,"num_empty_288":0.0,"protections_long_global":1.0,"RSI_3":10.0,"RSI_3_15m":60.0,"RSI_3_1h":60.0,"RSI_3_4h":20.0,"RSI_3_1d":50.0,"AROONU_14_1d":80.0,"AROONU_14_15m":30.0,"AROONU_14_1h":0.0,"AROONU_14_4h":0.0,"STOCHRSIk_14_14_3_3_15m":50.0,"STOCHRSIk_14_14_3_3_1h":50.0,"STOCHRSIk_14_14_3_3_4h":50.0,"STOCHRSIk_14_14_3_3_1d":50.0,"RSI_14":50.0,"RSI_14_15m":40.0,"RSI_14_1h":50.0,"RSI_14_4h":50.0,"AROONU_14":60.0,"STOCHRSIk_14_14_3_3":50.0,"OBV_change_pct":1.0,"EMA_26":1.0,"EMA_12_4h":1.0,"EMA_200_4h":2.0,"close":100.0,"close_min_48":90.0,"close_max_48":110.0,"CMF_20_1h":0.0,"CMF_20_4h":0.0,"BBP_20_2.0_4h":0.5,"WILLR_14_1h":-50.0,"ROC_9_4h":0.0,"ROC_9_1h":0.0,"BBL_20_2.0":90.0,"BBU_20_2.0":110.0,"BBL_20_2.0_1h":90.0,"BBU_20_2.0_1h":110.0},"series_overrides":{"EMA_12":[0.0,0.0,0.0,0.0,0.0,2.0,2.0,2.0]},"case_roc_9_1d":{"new_protection_rejects":30.0,"one_term_passes":29.0}},"modes":{"spot":{"required_input_column_count":179,"required_input_columns_sha256":"e74a39d42accf51dbe37e01921f4c396124cf8d7d956da6ad03d158f76e787c9","programs":{"current":{"signal_fingerprint":"396fdf270e2b51ebc8305cd5ae88ecd5bac4cd8815ccd2e68bf7a6bef34280e4","signal_node_count":9576,"tag_fingerprint":"7556b6aac4a0458edc3cdab5065947fe7756053ff43171d8e64eb16d6f7ff65c","tag_node_count":9649},"baseline":{"signal_fingerprint":"4306ed97e34b33e68ea5d0dbb4ecaae22925546ef29696d5e4f074674ca5f234","signal_node_count":9575,"tag_fingerprint":"d60df78e67ffecd78fa5a4caf2e2956f257c1e8dfb5a3e5886b90c512571cad6","tag_node_count":9648}},"cases":{"new_protection_rejects":{"current":{"enter_long":[0,0,0,0,0,0,0,0],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","","",""],"exit_tag":["","","","","","","",""]},"baseline":{"enter_long":[0,0,0,0,0,1,1,1],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","47 ","47 ","47 "],"exit_tag":["","","","","","","",""]}},"one_term_passes":{"current":{"enter_long":[0,0,0,0,0,1,1,1],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","47 ","47 ","47 "],"exit_tag":["","","","","","","",""]},"baseline":{"enter_long":[0,0,0,0,0,1,1,1],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","47 ","47 ","47 "],"exit_tag":["","","","","","","",""]}}}},"futures":{"required_input_column_count":179,"required_input_columns_sha256":"e74a39d42accf51dbe37e01921f4c396124cf8d7d956da6ad03d158f76e787c9","programs":{"current":{"signal_fingerprint":"b7cff105c1f3618bfbd5c64ed59c55afbba490fb2aa149a2e39c920678cc110a","signal_node_count":9576,"tag_fingerprint":"44fafdfcba1a29389b1ae2a0ba6ef433a571e9745f10e43186636bcbf9077967","tag_node_count":9649},"baseline":{"signal_fingerprint":"7d6136e8994107a9d019782527bc6a9ba8f4028e3f0f6f2a0a8affd1fee06122","signal_node_count":9575,"tag_fingerprint":"300ab6a7b547c6a88acb12e71a1f808cb9ce4ee96852117927ceb7675d10c46d","tag_node_count":9648}},"cases":{"new_protection_rejects":{"current":{"enter_long":[0,0,0,0,0,0,0,0],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","","",""],"exit_tag":["","","","","","","",""]},"baseline":{"enter_long":[0,0,0,0,0,1,1,1],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","47 ","47 ","47 "],"exit_tag":["","","","","","","",""]}},"one_term_passes":{"current":{"enter_long":[0,0,0,0,0,1,1,1],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","47 ","47 ","47 "],"exit_tag":["","","","","","","",""]},"baseline":{"enter_long":[0,0,0,0,0,1,1,1],"enter_short":[0,0,0,0,0,0,0,0],"exit_long":[0,0,0,0,0,0,0,0],"exit_short":[0,0,0,0,0,0,0,0],"enter_tag":["","","","","","47 ","47 ","47 "],"exit_tag":["","","","","","","",""]}}}}},"claims":{"source_wrapper_python_program_exact":true,"source_wrapper_rust_signal_exact":true,"source_wrapper_rust_tag_exact":true,"spot_and_futures_exact":true,"runtime_signal_number_branch_added":false},"fingerprint":"d4128b380d483a05f7a230bfe78dad14502438d45a8fa0adeb0c094c5d73f7fe"} diff --git a/benchmarks/reference/vector-shadow/freqtrade-2026.5.1.json b/benchmarks/reference/vector-shadow/freqtrade-2026.5.1.json index 3f299254..181aed62 100644 --- a/benchmarks/reference/vector-shadow/freqtrade-2026.5.1.json +++ b/benchmarks/reference/vector-shadow/freqtrade-2026.5.1.json @@ -1 +1 @@ -{"cases":{"execution":{"enabled_indexes":{"enter_long":[3,4,5,6,7],"enter_short":[1,2,3],"exit_long":[5,6,7],"exit_short":[2,4,6]},"execution_start_index":1,"expected":{"columns":[{"name":"nfi_exec_enter_long","type":"i64","values":[0,0,0,1,1,1,1,1]},{"name":"nfi_exec_enter_short","type":"i64","values":[0,1,1,1,0,0,0,0]},{"name":"nfi_exec_exit_long","type":"i64","values":[0,0,0,0,0,1,1,1]},{"name":"nfi_exec_exit_short","type":"i64","values":[0,0,1,0,1,0,1,0]},{"name":"nfi_exec_enter_tag","type":"text","values":[null,"562 ","562 ","101 562 ","101 ","101 ","override final ","override final "]},{"name":"nfi_exec_exit_tag","type":"text","values":[null,"","signal ","","signal ","profit ","profit signal ","profit "]}],"rows":8},"outputs":["nfi_exec_enter_long","nfi_exec_enter_short","nfi_exec_exit_long","nfi_exec_exit_short","nfi_exec_enter_tag","nfi_exec_exit_tag"],"source":"tag","source_row_shift":1},"indicator":{"expected":{"columns":[{"name":"delta","type":"f64","values":["0x3ff0000000000000","0xbff0000000000000","0x4000000000000000","0xbff0000000000000","0x4000000000000000","0xbff0000000000000","0x4000000000000000","0x3ff0000000000000"]},{"name":"previous_close","type":"f64","values":["0x7ff8000000000000","0x4026000000000000","0x4024000000000000","0x4028000000000000","0x4028000000000000","0x402c000000000000","0x402a000000000000","0x402e000000000000"]},{"name":"mean3","type":"f64","values":["0x7ff8000000000000","0x7ff8000000000000","0x4026000000000000","0x4026aaaaaaaaaaab","0x4029555555555555","0x402a000000000000","0x402c000000000000","0x402d555555555555"]},{"name":"selected","type":"f64","values":["0x7ff8000000000000","0x4026000000000000","0x4026000000000000","0x4028000000000000","0x4029555555555555","0x402c000000000000","0x402c000000000000","0x402d555555555555"]}],"rows":8},"input":{"columns":[{"name":"open","type":"f64","values":["0x4024000000000000","0x4026000000000000","0x4024000000000000","0x402a000000000000","0x4028000000000000","0x402c000000000000","0x402a000000000000","0x402e000000000000"]},{"name":"close","type":"f64","values":["0x4026000000000000","0x4024000000000000","0x4028000000000000","0x4028000000000000","0x402c000000000000","0x402a000000000000","0x402e000000000000","0x4030000000000000"]}],"rows":8},"outputs":["delta","previous_close","mean3","selected"]},"signal":{"expected":{"columns":[{"name":"enter_long","type":"i64","values":[0,0,1,1,0,0,0,0]},{"name":"enter_short","type":"i64","values":[1,1,0,0,0,0,0,0]},{"name":"exit_long","type":"i64","values":[0,0,0,0,1,1,0,0]},{"name":"exit_short","type":"i64","values":[0,0,1,1,0,1,0,0]}],"rows":8},"input":{"columns":[{"name":"score","type":"f64","values":["0xc000000000000000","0xbfe0000000000000","0x3fe0000000000000","0x3ff8000000000000","0x4000000000000000","0x4004000000000000","0x7ff8000000000000","0x0000000000000000"]},{"name":"exit_mask","type":"bool","values":[null,false,true,true,false,true,false,null]}],"rows":8},"outputs":["enter_long","enter_short","exit_long","exit_short"]},"tag":{"expected":{"columns":[{"name":"enter_long","type":"i64","values":[0,0,1,1,1,1,1,0]},{"name":"enter_short","type":"i64","values":[1,1,1,0,0,0,0,0]},{"name":"exit_long","type":"i64","values":[0,0,0,0,1,1,1,0]},{"name":"exit_short","type":"i64","values":[0,1,0,1,0,1,0,0]},{"name":"enter_tag","type":"text","values":["562 ","562 ","101 562 ","101 ","101 ","override final ","override final ",""]},{"name":"exit_tag","type":"text","values":["","signal ","","signal ","profit ","profit signal ","profit ",""]}],"rows":8},"input":{"columns":[{"name":"score","type":"f64","values":["0xc000000000000000","0xbfe0000000000000","0x0000000000000000","0x3fe0000000000000","0x3ff8000000000000","0x4000000000000000","0x4004000000000000","0x7ff8000000000000"]},{"name":"exit_mask","type":"bool","values":[false,true,false,true,false,true,null,false]},{"name":"enter_tag","type":"text","values":["stale-entry","stale-entry","stale-entry","stale-entry","stale-entry","stale-entry","stale-entry","stale-entry"]},{"name":"exit_tag","type":"text","values":["stale-exit","stale-exit","stale-exit","stale-exit","stale-exit","stale-exit","stale-exit","stale-exit"]}],"rows":8},"outputs":["enter_long","enter_short","exit_long","exit_short","enter_tag","exit_tag"]}},"fingerprint":"4ad942560f63421e0d1f61cab4045028e9420ff85e41b8afb0ffb2938b2e898e","programs":{"indicator":{"fingerprint":"bf766e2ccf6101f290264cb186ec52bb1c02a2809daf97e2287f1c0b679a29b0","path":"benchmarks/reference/vector-shadow/indicator-program.json","sha256":"fb767951bd70c057e75bfbabad250ffd16200c0de20d6f7ad89c02c3a177f9a7"},"signal":{"fingerprint":"adc3178d58dd405d3a41bd933ceee55a38876b1adb0b8da71ee9a69a16a47154","path":"benchmarks/reference/vector-shadow/signal-program.json","sha256":"ee043ba9552967e0970f607b858a2cf3dd5ff728540aa0e62343d3efc9d97577"},"tag":{"fingerprint":"ee6ac3643a4d4ccc01241d4884a4faea2dc812f9e07e4e32b9b1b7e7b0518412","path":"benchmarks/reference/vector-shadow/tag-program.json","sha256":"eea095daeb5329274e3c825026cbed327ff40ada026efded143f6eba4db16e3e"}},"schema_version":"vector-shadow-fixture-v1","source":{"freqtrade_commit":"6fa470939cc74bf0672e0e348a4d9b293072e43c","freqtrade_interface_sha256":"93ddb2f5579acd7a20d489174ffb68cd191428ff996d291b33be81d97fa9bf66","freqtrade_version":"2026.5.1","numpy":"2.4.5","pandas":"3.0.3","signal_oracle_fingerprint":"4c3150439cc45b282919562bce0e9273ab87a9b14022413b34133ac8fccf3249","tag_oracle_fingerprint":"11219ab2fae512e0e833f05c323a5cb8e47a724154a9d50ac126c6d8dbadba77"}} +{"cases":{"execution":{"enabled_indexes":{"enter_long":[3,4,5,6,7],"enter_short":[1,2,3],"exit_long":[5,6,7],"exit_short":[2,4,6]},"execution_start_index":1,"expected":{"columns":[{"name":"nfi_exec_enter_long","type":"i64","values":[0,0,0,1,1,1,1,1]},{"name":"nfi_exec_enter_short","type":"i64","values":[0,1,1,1,0,0,0,0]},{"name":"nfi_exec_exit_long","type":"i64","values":[0,0,0,0,0,1,1,1]},{"name":"nfi_exec_exit_short","type":"i64","values":[0,0,1,0,1,0,1,0]},{"name":"nfi_exec_enter_tag","type":"text","values":[null,"562 ","562 ","101 562 ","101 ","101 ","override final ","override final "]},{"name":"nfi_exec_exit_tag","type":"text","values":[null,"","signal ","","signal ","profit ","profit signal ","profit "]}],"rows":8},"outputs":["nfi_exec_enter_long","nfi_exec_enter_short","nfi_exec_exit_long","nfi_exec_exit_short","nfi_exec_enter_tag","nfi_exec_exit_tag"],"source":"tag","source_row_shift":1},"indicator":{"expected":{"columns":[{"name":"delta","type":"f64","values":["0x3ff0000000000000","0xbff0000000000000","0x4000000000000000","0xbff0000000000000","0x4000000000000000","0xbff0000000000000","0x4000000000000000","0x3ff0000000000000"]},{"name":"previous_close","type":"f64","values":["0x7ff8000000000000","0x4026000000000000","0x4024000000000000","0x4028000000000000","0x4028000000000000","0x402c000000000000","0x402a000000000000","0x402e000000000000"]},{"name":"mean3","type":"f64","values":["0x7ff8000000000000","0x7ff8000000000000","0x4026000000000000","0x4026aaaaaaaaaaab","0x4029555555555555","0x402a000000000000","0x402c000000000000","0x402d555555555555"]},{"name":"selected","type":"f64","values":["0x7ff8000000000000","0x4026000000000000","0x4026000000000000","0x4028000000000000","0x4029555555555555","0x402c000000000000","0x402c000000000000","0x402d555555555555"]}],"rows":8},"input":{"columns":[{"name":"open","type":"f64","values":["0x4024000000000000","0x4026000000000000","0x4024000000000000","0x402a000000000000","0x4028000000000000","0x402c000000000000","0x402a000000000000","0x402e000000000000"]},{"name":"close","type":"f64","values":["0x4026000000000000","0x4024000000000000","0x4028000000000000","0x4028000000000000","0x402c000000000000","0x402a000000000000","0x402e000000000000","0x4030000000000000"]}],"rows":8},"outputs":["delta","previous_close","mean3","selected"]},"signal":{"expected":{"columns":[{"name":"enter_long","type":"i64","values":[0,0,1,1,0,0,0,0]},{"name":"enter_short","type":"i64","values":[1,1,0,0,0,0,0,0]},{"name":"exit_long","type":"i64","values":[0,0,0,0,1,1,0,0]},{"name":"exit_short","type":"i64","values":[0,0,1,1,0,1,0,0]}],"rows":8},"input":{"columns":[{"name":"score","type":"f64","values":["0xc000000000000000","0xbfe0000000000000","0x3fe0000000000000","0x3ff8000000000000","0x4000000000000000","0x4004000000000000","0x7ff8000000000000","0x0000000000000000"]},{"name":"exit_mask","type":"bool","values":[null,false,true,true,false,true,false,null]}],"rows":8},"outputs":["enter_long","enter_short","exit_long","exit_short"]},"tag":{"expected":{"columns":[{"name":"enter_long","type":"i64","values":[0,0,1,1,1,1,1,0]},{"name":"enter_short","type":"i64","values":[1,1,1,0,0,0,0,0]},{"name":"exit_long","type":"i64","values":[0,0,0,0,1,1,1,0]},{"name":"exit_short","type":"i64","values":[0,1,0,1,0,1,0,0]},{"name":"enter_tag","type":"text","values":["562 ","562 ","101 562 ","101 ","101 ","override final ","override final ",""]},{"name":"exit_tag","type":"text","values":["","signal ","","signal ","profit ","profit signal ","profit ",""]}],"rows":8},"input":{"columns":[{"name":"score","type":"f64","values":["0xc000000000000000","0xbfe0000000000000","0x0000000000000000","0x3fe0000000000000","0x3ff8000000000000","0x4000000000000000","0x4004000000000000","0x7ff8000000000000"]},{"name":"exit_mask","type":"bool","values":[false,true,false,true,false,true,null,false]},{"name":"enter_tag","type":"text","values":["stale-entry","stale-entry","stale-entry","stale-entry","stale-entry","stale-entry","stale-entry","stale-entry"]},{"name":"exit_tag","type":"text","values":["stale-exit","stale-exit","stale-exit","stale-exit","stale-exit","stale-exit","stale-exit","stale-exit"]}],"rows":8},"outputs":["enter_long","enter_short","exit_long","exit_short","enter_tag","exit_tag"]}},"fingerprint":"647af91e26a4fe6938bc232a18b4b75296e3e5db565835bc1b442995ea4081f6","programs":{"indicator":{"fingerprint":"bf766e2ccf6101f290264cb186ec52bb1c02a2809daf97e2287f1c0b679a29b0","path":"benchmarks/reference/vector-shadow/indicator-program.json","sha256":"fb767951bd70c057e75bfbabad250ffd16200c0de20d6f7ad89c02c3a177f9a7"},"signal":{"fingerprint":"acda16221cf0f66218abd25820a4c130fdeba7a1efa74352dc6e327abcdb7dd5","path":"benchmarks/reference/vector-shadow/signal-program.json","sha256":"f4dd0446c350a49e3b1b20a95af1a79592c02254d0344cb30260c6e2343d866c"},"tag":{"fingerprint":"ee6ac3643a4d4ccc01241d4884a4faea2dc812f9e07e4e32b9b1b7e7b0518412","path":"benchmarks/reference/vector-shadow/tag-program.json","sha256":"eea095daeb5329274e3c825026cbed327ff40ada026efded143f6eba4db16e3e"}},"schema_version":"vector-shadow-fixture-v1","source":{"freqtrade_commit":"6fa470939cc74bf0672e0e348a4d9b293072e43c","freqtrade_interface_sha256":"93ddb2f5579acd7a20d489174ffb68cd191428ff996d291b33be81d97fa9bf66","freqtrade_version":"2026.5.1","numpy":"2.4.5","pandas":"3.0.3","signal_oracle_fingerprint":"4c3150439cc45b282919562bce0e9273ab87a9b14022413b34133ac8fccf3249","tag_oracle_fingerprint":"11219ab2fae512e0e833f05c323a5cb8e47a724154a9d50ac126c6d8dbadba77"}} diff --git a/benchmarks/reference/vector-shadow/signal-program.json b/benchmarks/reference/vector-shadow/signal-program.json index e11160f4..536ff48d 100644 --- a/benchmarks/reference/vector-shadow/signal-program.json +++ b/benchmarks/reference/vector-shadow/signal-program.json @@ -1 +1 @@ -{"compile_context":{"run_mode":"backtest","trading_mode":"spot"},"entrypoints":[{"function":"f1","phase":"entry"},{"function":"f2","phase":"exit"}],"fingerprint":"adc3178d58dd405d3a41bd933ceee55a38876b1adb0b8da71ee9a69a16a47154","functions":[{"id":"f1","kind":"entrypoint-entry","node_ids":["n1","n2","n3","n4","n5","n6","n7","n8","n9","n10","n11","n12","n13","n14","n15","n16","n17","n18","n19","n20","n21"],"parameters":[{"name":"dataframe","node":"n1","value_type":"dataframe"},{"name":"metadata","node":"n2","value_type":"metadata"}],"return_node":"n21","source_name":"populate_entry_trend"},{"id":"f2","kind":"entrypoint-exit","node_ids":["n22","n23","n24","n25","n26","n27","n28","n29","n30","n31","n32","n33","n34","n35","n36","n37","n38"],"parameters":[{"name":"dataframe","node":"n22","value_type":"dataframe"},{"name":"metadata","node":"n23","value_type":"metadata"}],"return_node":"n38","source_name":"populate_exit_trend"}],"max_lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"mutation_nodes":["n5","n10","n15","n20","n25","n34","n37"],"nodes":[{"function":"f1","id":"n1","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"dataframe"},"source_order":0,"value_type":"dataframe"},{"function":"f1","id":"n2","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"metadata"},"source_order":1,"value_type":"metadata"},{"function":"f1","id":"n3","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":2,"value_type":"int-scalar"},{"function":"f1","id":"n4","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":3,"value_type":"int-scalar"},{"function":"f1","id":"n5","inputs":["n1","n3","n4"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_long","enter_short"],"mode":"loc","rows":"all"},"source_order":4,"value_type":"dataframe"},{"function":"f1","id":"n6","inputs":["n5"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":5,"value_type":"f64-column"},{"function":"f1","id":"n7","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":6,"value_type":"int-scalar"},{"function":"f1","id":"n8","inputs":["n6","n7"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"greater-than"},"source_order":7,"value_type":"bool-column"},{"function":"f1","id":"n9","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":8,"value_type":"int-scalar"},{"function":"f1","id":"n10","inputs":["n5","n8","n9"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_long"],"mode":"loc","rows":"mask"},"source_order":9,"value_type":"dataframe"},{"function":"f1","id":"n11","inputs":["n10"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":10,"value_type":"f64-column"},{"function":"f1","id":"n12","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":2},"source_order":11,"value_type":"int-scalar"},{"function":"f1","id":"n13","inputs":["n11","n12"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"greater-than-or-equal"},"source_order":12,"value_type":"bool-column"},{"function":"f1","id":"n14","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":13,"value_type":"int-scalar"},{"function":"f1","id":"n15","inputs":["n10","n13","n14"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_long"],"mode":"loc","rows":"mask"},"source_order":14,"value_type":"dataframe"},{"function":"f1","id":"n16","inputs":["n15"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":15,"value_type":"f64-column"},{"function":"f1","id":"n17","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":16,"value_type":"int-scalar"},{"function":"f1","id":"n18","inputs":["n16","n17"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"less-than"},"source_order":17,"value_type":"bool-column"},{"function":"f1","id":"n19","inputs":["n18"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"cast","parameters":{"target":"int"},"source_order":18,"value_type":"bool-column"},{"function":"f1","id":"n20","inputs":["n15","n19"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_short"],"mode":"column","rows":"all"},"source_order":19,"value_type":"dataframe"},{"function":"f1","id":"n21","inputs":["n20"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"return","parameters":{},"source_order":20,"value_type":"dataframe"},{"function":"f2","id":"n22","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"dataframe"},"source_order":0,"value_type":"dataframe"},{"function":"f2","id":"n23","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"metadata"},"source_order":1,"value_type":"metadata"},{"function":"f2","id":"n24","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":2,"value_type":"int-scalar"},{"function":"f2","id":"n25","inputs":["n22","n24"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"scalar-broadcast","columns":["exit_long","exit_short"],"mode":"loc","rows":"all"},"source_order":3,"value_type":"dataframe"},{"function":"f2","id":"n26","inputs":["n25"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"enter_long"},"source_order":4,"value_type":"f64-column"},{"function":"f2","id":"n27","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":5,"value_type":"int-scalar"},{"function":"f2","id":"n28","inputs":["n26","n27"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"equal"},"source_order":6,"value_type":"bool-column"},{"function":"f2","id":"n29","inputs":["n25"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":7,"value_type":"f64-column"},{"function":"f2","id":"n30","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":8,"value_type":"int-scalar"},{"function":"f2","id":"n31","inputs":["n29","n30"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"greater-than"},"source_order":9,"value_type":"bool-column"},{"function":"f2","id":"n32","inputs":["n28","n31"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"logical","parameters":{"operator":"and"},"source_order":10,"value_type":"bool-column"},{"function":"f2","id":"n33","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":11,"value_type":"int-scalar"},{"function":"f2","id":"n34","inputs":["n25","n32","n33"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["exit_long"],"mode":"loc","rows":"mask"},"source_order":12,"value_type":"dataframe"},{"function":"f2","id":"n35","inputs":["n34"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"exit_mask"},"source_order":13,"value_type":"f64-column"},{"function":"f2","id":"n36","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":14,"value_type":"int-scalar"},{"function":"f2","id":"n37","inputs":["n34","n35","n36"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["exit_short"],"mode":"loc","rows":"mask"},"source_order":15,"value_type":"dataframe"},{"function":"f2","id":"n38","inputs":["n37"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"return","parameters":{},"source_order":16,"value_type":"dataframe"}],"opcodes":["cast","column-read","compare","frame-write","literal","logical","parameter","return"],"required_input_columns":["exit_mask","score"],"schema_version":"signal-program-v1","selected_class":"SignalProgramContract","signal_outputs":[{"column":"enter_long","final_mutation":"n15","phase":"entry","side":"long"},{"column":"enter_short","final_mutation":"n20","phase":"entry","side":"short"},{"column":"exit_long","final_mutation":"n34","phase":"exit","side":"long"},{"column":"exit_short","final_mutation":"n37","phase":"exit","side":"short"}],"source":{"path":"benchmarks/reference/strategies/SignalProgramContract.py","sha256":"d3e1316e336273490b2da8dcfefc5a8ecca64046250a9225971910d78a3e1746"},"source_map":{"n1":{"column":4,"end_column":24,"end_line":13,"line":7,"path":"strategy.py"},"n10":{"column":8,"end_column":49,"end_line":10,"line":10,"path":"strategy.py"},"n11":{"column":22,"end_column":40,"end_line":11,"line":11,"path":"strategy.py"},"n12":{"column":44,"end_column":45,"end_line":11,"line":11,"path":"strategy.py"},"n13":{"column":22,"end_column":45,"end_line":11,"line":11,"path":"strategy.py"},"n14":{"column":63,"end_column":64,"end_line":11,"line":11,"path":"strategy.py"},"n15":{"column":8,"end_column":64,"end_line":11,"line":11,"path":"strategy.py"},"n16":{"column":36,"end_column":54,"end_line":12,"line":12,"path":"strategy.py"},"n17":{"column":57,"end_column":58,"end_line":12,"line":12,"path":"strategy.py"},"n18":{"column":36,"end_column":58,"end_line":12,"line":12,"path":"strategy.py"},"n19":{"column":35,"end_column":71,"end_line":12,"line":12,"path":"strategy.py"},"n2":{"column":4,"end_column":24,"end_line":13,"line":7,"path":"strategy.py"},"n20":{"column":8,"end_column":71,"end_line":12,"line":12,"path":"strategy.py"},"n21":{"column":8,"end_column":24,"end_line":13,"line":13,"path":"strategy.py"},"n22":{"column":4,"end_column":24,"end_line":22,"line":15,"path":"strategy.py"},"n23":{"column":4,"end_column":24,"end_line":22,"line":15,"path":"strategy.py"},"n24":{"column":56,"end_column":57,"end_line":16,"line":16,"path":"strategy.py"},"n25":{"column":8,"end_column":57,"end_line":16,"line":16,"path":"strategy.py"},"n26":{"column":13,"end_column":36,"end_line":18,"line":18,"path":"strategy.py"},"n27":{"column":40,"end_column":41,"end_line":18,"line":18,"path":"strategy.py"},"n28":{"column":13,"end_column":41,"end_line":18,"line":18,"path":"strategy.py"},"n29":{"column":46,"end_column":64,"end_line":18,"line":18,"path":"strategy.py"},"n3":{"column":59,"end_column":60,"end_line":8,"line":8,"path":"strategy.py"},"n30":{"column":67,"end_column":68,"end_line":18,"line":18,"path":"strategy.py"},"n31":{"column":46,"end_column":68,"end_line":18,"line":18,"path":"strategy.py"},"n32":{"column":12,"end_column":69,"end_line":18,"line":18,"path":"strategy.py"},"n33":{"column":12,"end_column":13,"end_line":20,"line":20,"path":"strategy.py"},"n34":{"column":8,"end_column":13,"end_line":20,"line":17,"path":"strategy.py"},"n35":{"column":22,"end_column":44,"end_line":21,"line":21,"path":"strategy.py"},"n36":{"column":62,"end_column":63,"end_line":21,"line":21,"path":"strategy.py"},"n37":{"column":8,"end_column":63,"end_line":21,"line":21,"path":"strategy.py"},"n38":{"column":8,"end_column":24,"end_line":22,"line":22,"path":"strategy.py"},"n4":{"column":62,"end_column":63,"end_line":8,"line":8,"path":"strategy.py"},"n5":{"column":8,"end_column":64,"end_line":8,"line":8,"path":"strategy.py"},"n6":{"column":19,"end_column":37,"end_line":9,"line":9,"path":"strategy.py"},"n7":{"column":40,"end_column":41,"end_line":9,"line":9,"path":"strategy.py"},"n8":{"column":19,"end_column":41,"end_line":9,"line":9,"path":"strategy.py"},"n9":{"column":48,"end_column":49,"end_line":10,"line":10,"path":"strategy.py"}}} +{"compile_context":{"run_mode":"backtest","trading_mode":"spot"},"entrypoints":[{"function":"f1","phase":"entry"},{"function":"f2","phase":"exit"}],"fingerprint":"acda16221cf0f66218abd25820a4c130fdeba7a1efa74352dc6e327abcdb7dd5","functions":[{"id":"f1","kind":"entrypoint-entry","node_ids":["n1","n2","n3","n4","n5","n6","n7","n8","n9","n10","n11","n12","n13","n14","n15","n16","n17","n18","n19","n20","n21"],"parameters":[{"name":"dataframe","node":"n1","value_type":"dataframe"},{"name":"metadata","node":"n2","value_type":"metadata"}],"return_node":"n21","source_name":"populate_entry_trend"},{"id":"f2","kind":"entrypoint-exit","node_ids":["n22","n23","n24","n25","n26","n27","n28","n29","n30","n31","n32","n33","n34","n35","n36","n37","n38"],"parameters":[{"name":"dataframe","node":"n22","value_type":"dataframe"},{"name":"metadata","node":"n23","value_type":"metadata"}],"return_node":"n38","source_name":"populate_exit_trend"}],"max_lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"mutation_nodes":["n5","n10","n15","n20","n25","n34","n37"],"nodes":[{"function":"f1","id":"n1","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"dataframe"},"source_order":0,"value_type":"dataframe"},{"function":"f1","id":"n2","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"metadata"},"source_order":1,"value_type":"metadata"},{"function":"f1","id":"n3","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":2,"value_type":"int-scalar"},{"function":"f1","id":"n4","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":3,"value_type":"int-scalar"},{"function":"f1","id":"n5","inputs":["n1","n3","n4"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_long","enter_short"],"mode":"loc","rows":"all"},"source_order":4,"value_type":"dataframe"},{"function":"f1","id":"n6","inputs":["n5"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":5,"value_type":"f64-column"},{"function":"f1","id":"n7","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":6,"value_type":"int-scalar"},{"function":"f1","id":"n8","inputs":["n6","n7"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"greater-than"},"source_order":7,"value_type":"bool-column"},{"function":"f1","id":"n9","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":8,"value_type":"int-scalar"},{"function":"f1","id":"n10","inputs":["n5","n8","n9"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_long"],"mode":"loc","rows":"mask"},"source_order":9,"value_type":"dataframe"},{"function":"f1","id":"n11","inputs":["n10"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":10,"value_type":"f64-column"},{"function":"f1","id":"n12","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":2},"source_order":11,"value_type":"int-scalar"},{"function":"f1","id":"n13","inputs":["n11","n12"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"greater-than-or-equal"},"source_order":12,"value_type":"bool-column"},{"function":"f1","id":"n14","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":13,"value_type":"int-scalar"},{"function":"f1","id":"n15","inputs":["n10","n13","n14"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_long"],"mode":"loc","rows":"mask"},"source_order":14,"value_type":"dataframe"},{"function":"f1","id":"n16","inputs":["n15"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":15,"value_type":"f64-column"},{"function":"f1","id":"n17","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":16,"value_type":"int-scalar"},{"function":"f1","id":"n18","inputs":["n16","n17"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"less-than"},"source_order":17,"value_type":"bool-column"},{"function":"f1","id":"n19","inputs":["n18"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"cast","parameters":{"target":"int"},"source_order":18,"value_type":"int-column"},{"function":"f1","id":"n20","inputs":["n15","n19"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["enter_short"],"mode":"column","rows":"all"},"source_order":19,"value_type":"dataframe"},{"function":"f1","id":"n21","inputs":["n20"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"return","parameters":{},"source_order":20,"value_type":"dataframe"},{"function":"f2","id":"n22","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"dataframe"},"source_order":0,"value_type":"dataframe"},{"function":"f2","id":"n23","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"parameter","parameters":{"name":"metadata"},"source_order":1,"value_type":"metadata"},{"function":"f2","id":"n24","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":2,"value_type":"int-scalar"},{"function":"f2","id":"n25","inputs":["n22","n24"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"scalar-broadcast","columns":["exit_long","exit_short"],"mode":"loc","rows":"all"},"source_order":3,"value_type":"dataframe"},{"function":"f2","id":"n26","inputs":["n25"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"enter_long"},"source_order":4,"value_type":"f64-column"},{"function":"f2","id":"n27","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":0},"source_order":5,"value_type":"int-scalar"},{"function":"f2","id":"n28","inputs":["n26","n27"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"equal"},"source_order":6,"value_type":"bool-column"},{"function":"f2","id":"n29","inputs":["n25"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"score"},"source_order":7,"value_type":"f64-column"},{"function":"f2","id":"n30","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":8,"value_type":"int-scalar"},{"function":"f2","id":"n31","inputs":["n29","n30"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"compare","parameters":{"operator":"greater-than"},"source_order":9,"value_type":"bool-column"},{"function":"f2","id":"n32","inputs":["n28","n31"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"logical","parameters":{"operator":"and"},"source_order":10,"value_type":"bool-column"},{"function":"f2","id":"n33","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":11,"value_type":"int-scalar"},{"function":"f2","id":"n34","inputs":["n25","n32","n33"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["exit_long"],"mode":"loc","rows":"mask"},"source_order":12,"value_type":"dataframe"},{"function":"f2","id":"n35","inputs":["n34"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"column-read","parameters":{"column":"exit_mask"},"source_order":13,"value_type":"f64-column"},{"function":"f2","id":"n36","inputs":[],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"literal","parameters":{"value":1},"source_order":14,"value_type":"int-scalar"},{"function":"f2","id":"n37","inputs":["n34","n35","n36"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"frame-write","parameters":{"assignment":"column-values","columns":["exit_short"],"mode":"loc","rows":"mask"},"source_order":15,"value_type":"dataframe"},{"function":"f2","id":"n38","inputs":["n37"],"lookback":{"candles":0,"causal":true,"expression":null,"kind":"finite"},"op":"return","parameters":{},"source_order":16,"value_type":"dataframe"}],"opcodes":["cast","column-read","compare","frame-write","literal","logical","parameter","return"],"required_input_columns":["exit_mask","score"],"schema_version":"signal-program-v1","selected_class":"SignalProgramContract","signal_outputs":[{"column":"enter_long","final_mutation":"n15","phase":"entry","side":"long"},{"column":"enter_short","final_mutation":"n20","phase":"entry","side":"short"},{"column":"exit_long","final_mutation":"n34","phase":"exit","side":"long"},{"column":"exit_short","final_mutation":"n37","phase":"exit","side":"short"}],"source":{"path":"benchmarks/reference/strategies/SignalProgramContract.py","sha256":"d3e1316e336273490b2da8dcfefc5a8ecca64046250a9225971910d78a3e1746"},"source_map":{"n1":{"column":4,"end_column":24,"end_line":13,"line":7,"path":"strategy.py"},"n10":{"column":8,"end_column":49,"end_line":10,"line":10,"path":"strategy.py"},"n11":{"column":22,"end_column":40,"end_line":11,"line":11,"path":"strategy.py"},"n12":{"column":44,"end_column":45,"end_line":11,"line":11,"path":"strategy.py"},"n13":{"column":22,"end_column":45,"end_line":11,"line":11,"path":"strategy.py"},"n14":{"column":63,"end_column":64,"end_line":11,"line":11,"path":"strategy.py"},"n15":{"column":8,"end_column":64,"end_line":11,"line":11,"path":"strategy.py"},"n16":{"column":36,"end_column":54,"end_line":12,"line":12,"path":"strategy.py"},"n17":{"column":57,"end_column":58,"end_line":12,"line":12,"path":"strategy.py"},"n18":{"column":36,"end_column":58,"end_line":12,"line":12,"path":"strategy.py"},"n19":{"column":35,"end_column":71,"end_line":12,"line":12,"path":"strategy.py"},"n2":{"column":4,"end_column":24,"end_line":13,"line":7,"path":"strategy.py"},"n20":{"column":8,"end_column":71,"end_line":12,"line":12,"path":"strategy.py"},"n21":{"column":8,"end_column":24,"end_line":13,"line":13,"path":"strategy.py"},"n22":{"column":4,"end_column":24,"end_line":22,"line":15,"path":"strategy.py"},"n23":{"column":4,"end_column":24,"end_line":22,"line":15,"path":"strategy.py"},"n24":{"column":56,"end_column":57,"end_line":16,"line":16,"path":"strategy.py"},"n25":{"column":8,"end_column":57,"end_line":16,"line":16,"path":"strategy.py"},"n26":{"column":13,"end_column":36,"end_line":18,"line":18,"path":"strategy.py"},"n27":{"column":40,"end_column":41,"end_line":18,"line":18,"path":"strategy.py"},"n28":{"column":13,"end_column":41,"end_line":18,"line":18,"path":"strategy.py"},"n29":{"column":46,"end_column":64,"end_line":18,"line":18,"path":"strategy.py"},"n3":{"column":59,"end_column":60,"end_line":8,"line":8,"path":"strategy.py"},"n30":{"column":67,"end_column":68,"end_line":18,"line":18,"path":"strategy.py"},"n31":{"column":46,"end_column":68,"end_line":18,"line":18,"path":"strategy.py"},"n32":{"column":12,"end_column":69,"end_line":18,"line":18,"path":"strategy.py"},"n33":{"column":12,"end_column":13,"end_line":20,"line":20,"path":"strategy.py"},"n34":{"column":8,"end_column":13,"end_line":20,"line":17,"path":"strategy.py"},"n35":{"column":22,"end_column":44,"end_line":21,"line":21,"path":"strategy.py"},"n36":{"column":62,"end_column":63,"end_line":21,"line":21,"path":"strategy.py"},"n37":{"column":8,"end_column":63,"end_line":21,"line":21,"path":"strategy.py"},"n38":{"column":8,"end_column":24,"end_line":22,"line":22,"path":"strategy.py"},"n4":{"column":62,"end_column":63,"end_line":8,"line":8,"path":"strategy.py"},"n5":{"column":8,"end_column":64,"end_line":8,"line":8,"path":"strategy.py"},"n6":{"column":19,"end_column":37,"end_line":9,"line":9,"path":"strategy.py"},"n7":{"column":40,"end_column":41,"end_line":9,"line":9,"path":"strategy.py"},"n8":{"column":19,"end_column":41,"end_line":9,"line":9,"path":"strategy.py"},"n9":{"column":48,"end_column":49,"end_line":10,"line":10,"path":"strategy.py"}}} diff --git a/docs/native-vector-core.md b/docs/native-vector-core.md index 878a1877..e5668c08 100644 --- a/docs/native-vector-core.md +++ b/docs/native-vector-core.md @@ -46,6 +46,11 @@ MINUS_DI, OBV, PLUS_DI, ROC, RSI, SMA, STDDEV, STOCHF, SUM, ULTOSC, and WILLR. Pandas rolling mean, sum, min, and max are also exact for every inventoried window, including the 2,016-candle window. +Captured pre-Full-Native X7 sources also reach TA-Lib `STOCH` and an older +Chaikin helper whose numerator uses a NumPy prefix sum while its denominator +uses TA-Lib `SUM`. Both are separate operation contracts with exact warmup and +streaming tests; no strategy version or source hash selects their behavior. + The committed oracle contains 2,200 input rows, all 56 reachable TA-Lib parameter variants, and 24 rolling variants. Finite values compare by exact f64 bits; NaN warmup positions compare exactly and signal-boundary tests use no tolerance. @@ -119,23 +124,65 @@ It writes `benchmarks/reference/informative/freqtrade-2026.5.1.json`; both the Python compatibility layer and Rust alignment tests replay the same cases exactly. +## Strategy execution boundary + +The Full Native manifest seals three facts: strategy source is read through the +Python AST compiler only, no `populate_*` method was executed, and the runtime is +`rust-full-native`. Rust rejects the manifest before market-data decoding if any +of those fields is absent or different. The engine profile repeats the sealed +contract, and the vector checkpoint records zero Python workers and no analyzed +Feather outputs. A regression fixture places a deliberate exception at module +scope and still completes through Rust, proving that the strategy file is never +imported or executed in this lane. + +This does not remove Python from source compilation, nor remove the explicit +official Freqtrade fallback. It removes execution of NFI strategy Python from +the Full Native runtime. Legacy captured Feather replay remains available as an +evidence/compatibility transport and is not relabeled as Full Native. + +## Five-year resource boundaries + +Full Native pair preparation uses an explicit outer worker pool while every +numeric library remains single-threaded. Before a new workload runs, the engine +selects the pair with the largest declared frame inventory, executes that exact +full-range pair with one worker, and records its measured peak RSS. The worker +limit is then recomputed from the current host's available memory and the +execution profile's CPU/memory limits. Calibration identity includes the complete +manifest hash and Rust source fingerprint, so a different strategy, dataset, +configuration, or engine cannot inherit the observation. + +Raw frames are SHA-verified first and decoded pair by pair. Completed execution +rows go directly to fixed-width file-backed storage; analyzed Feather files are +not persisted and pair-owned indicator frames are released before the global +wallet loop. The spool bound is derived from the run contract rather than from +observed sparse input rows: + +```text +pair count × maximum gap-filled base buckets × (81 + 8 × retained features) +``` + +The selected filesystem must have at least that many free bytes before any pair +is decoded. Equality is permitted; arithmetic overflow and a one-byte shortfall +fail closed. The engine profile records the admitted upper bound, available +bytes, target source, cleanup mode, declared raw rows, and actual file-backed +bytes, and rejects an actual value above the bound. Spool files are private +delete-on-close temporaries, so success, error, or process termination releases +them instead of accumulating one analyzed dataset per run. + ## Current claim boundary -The latest X7 indicator operation set now has exact Native kernels. This does not -yet claim that the entire latest X7 source compiles or executes Full Native. -M20-05 establishes exact multi-timeframe primitives and compiler contracts, -but their in-memory `VectorEngine` connection is deliberately owned by M21. -M21-01 and M21-02 establish Native signal assignment and exact tag-generation -contracts. M21-03 independently proves Python/Rust Indicator, Signal, Tag, and -execution-index equality. M21-04 connects typed pair output to the simulator, preserves -Feather replay, and keeps wallet mutation single-threaded. M22 owns latest-upstream Spot/Futures -full-state qualification, removal of Python strategy execution from the Native -lane, and release certification. Until those proofs pass, unsupported source -constructs remain fail-closed and the official Freqtrade fallback remains the -execution path. EWM also remains fail-closed because it is not reachable in the -current upstream inventory. Unknown functions, non-SMA TA-Lib MA types, centered -rolling windows, Arrow null indicator inputs, and unselected multi-output calls -stop rather than being guessed. +M22-01 qualifies upstream X7 `1df961c0` through the complete Spot and Futures +pipeline. Indicator, Signal, Tag, stateful callbacks, trade surface, and full +state are exact for the committed dual-mode qualification, and the older public +Spot/Futures captures remain exact after the same runtime changes. M22-02 owns +the no-strategy-execution contract and fallback isolation. M22-03 certifies the +bounded five-year Full Native Spot workload with three byte-identical results; +M22-04 still owns the v1.6.0 release. Unsupported constructs remain fail-closed +and the announced official +Freqtrade fallback remains available. EWM remains fail-closed because it is not +reachable in the qualified upstream inventory. Unknown functions, non-SMA +TA-Lib MA types, centered rolling windows, Arrow null indicator inputs, and +unselected multi-output calls stop rather than being guessed. The Rust boundary can validate a compiled contract directly: diff --git a/docs/releases/v1.6.0.md b/docs/releases/v1.6.0.md new file mode 100644 index 00000000..4246e180 --- /dev/null +++ b/docs/releases/v1.6.0.md @@ -0,0 +1,53 @@ +# NFI Backtest Engine v1.6.0 + +v1.6.0 is the Full Native Strategy release. The Native lane compiles supplied NFI +Python into typed programs, then executes indicators, Signal and Tag mutation, +stateful callbacks, orders, wallets, and result assembly in Rust. Strategy Python is +not imported or executed during the Native run. + +## Current X7 hotfix + +The X7 Grind-5 compiler and runtime now represent source-defined +`slice_profit_exit`, Boolean protection columns, bare Derisk state, Futures-mode +guards, liquidation-price presence, and liquidation-distance comparisons. These are +generic expression operands and predicates; there is no v536/v537, Signal, pair, +timerange, SHA, or expected-result runtime branch. + +The engine also pins the relevant Freqtrade candle order: adjustment evaluation and +its filled order happen before stop/exit evaluation. We did not reverse this order in +response to the external report because pinned Freqtrade 2026.5.1 uses the same +sequence. + +The upstream v17.4.536 and v17.4.537 sources pass the supported Spot/Futures static +Native checks. The reporter's sealed 2022 Futures input is not present in this +repository, so this release does not claim its reported 180-trade result as an exact +certificate. Unknown or newly changed active behavior still fails closed. + +## Full Native runtime + +- Complete Indicator, Signal, Tag, callback, order, wallet, and state execution is + source-compiled and runs without strategy Python at runtime. +- Pair-local DAG work is parallel; shared wallet and order mutation stays in one + chronological event loop. +- Raw frames and programs are path- and SHA-bound before decode. Legacy sealed + Feather replay remains available for evidence reproduction. +- Temporary row spools are capacity-admitted before execution and use + delete-on-close ownership. Completed and failed runs do not accumulate spool files. + +## Evidence boundary + +The latest-X7 Spot/Futures qualification and captured regressions have zero-tolerance +trade-surface and full-state equality. A five-year Full Native Spot development +workload also produced three byte-identical 1,162-trade results in 63:17, 62:51, and +62:44, with the temporary spool remaining inside its 38.47 GB admitted bound. + +That performance result is single-host evidence. v1.6.0 keeps +`combined_full_x7_certified=false`: it does not relabel the v1.0.0 Spot or v1.1.0 +Futures certificates, claim new official-Freqtrade five-year parity, or claim +cross-platform performance. + +## Upgrade + +Run the normal installer again after v1.6.0 is published. It selects the matching +wheel, verifies the release SHA-256 manifest, and replaces the isolated `nfi-bte` +installation. diff --git a/planning/roadmap-state.json b/planning/roadmap-state.json index 3c1d2e7b..a5609717 100644 --- a/planning/roadmap-state.json +++ b/planning/roadmap-state.json @@ -1,10 +1,10 @@ { "schema_version": "1.0.0", "roadmap_id": "nfi-backtest-engine-post-v1.1.0", - "revision": 142, - "updated_at": "2026-08-11T22:09:26+09:00", + "revision": 149, + "updated_at": "2026-08-14T03:11:20+09:00", "acceptance_commands": "planning/acceptance-commands.json", - "active_task_id": null, + "active_task_id": "M22-04", "execution_policy": { "max_in_progress": 1, "selection": "lowest order pending task whose dependencies are completed", @@ -4954,7 +4954,7 @@ "order": 2201, "milestone": "M22", "title": "Qualify latest NFI as a Full Native Strategy", - "status": "pending", + "status": "completed", "depends_on": [ "M21-04" ], @@ -4973,10 +4973,15 @@ "latest dual-mode Full Native qualification", "zero blockers" ], - "started_at": null, - "completed_at": null, - "commit_sha": null, - "evidence": [], + "started_at": "2026-08-11T22:50:22+09:00", + "completed_at": "2026-08-12T04:36:53+09:00", + "commit_sha": "6c60bff36aaf10272c1a0ffd6678961bbb5d1703", + "evidence": [ + "benchmarks/evidence/m22/latest-x7-full-native-qualification.json", + "benchmarks/evidence/m22/latest-x7-signal47-boundary.json", + ".nfi/roadmap-acceptance/M22-01/6c60bff36aaf10272c1a0ffd6678961bbb5d1703/x7-spot/run.json", + ".nfi/roadmap-acceptance/M22-01/6c60bff36aaf10272c1a0ffd6678961bbb5d1703/x7-futures/run.json" + ], "blocker": null }, { @@ -4984,7 +4989,7 @@ "order": 2202, "milestone": "M22", "title": "Remove NFI Python execution from the Native lane", - "status": "pending", + "status": "completed", "depends_on": [ "M22-01" ], @@ -5003,10 +5008,17 @@ "no-Python-execution contract tests", "fallback isolation tests" ], - "started_at": null, - "completed_at": null, - "commit_sha": null, - "evidence": [], + "started_at": "2026-08-12T04:36:53+09:00", + "completed_at": "2026-08-12T05:02:56+09:00", + "commit_sha": "f2315658affa4320388c899ecb3e5143e0ea3929", + "evidence": [ + "docs/native-vector-core.md", + "tests/test_full_vector_runtime.py", + "tests/test_research_runner.py", + "tests/test_official_fallback.py", + ".nfi/roadmap-acceptance/M22-02/f2315658a/x7-spot/run.json", + ".nfi/roadmap-acceptance/M22-02/f2315658a/x7-futures/run.json" + ], "blocker": null }, { @@ -5014,7 +5026,7 @@ "order": 2203, "milestone": "M22", "title": "Certify Full Native performance and storage boundaries", - "status": "pending", + "status": "completed", "depends_on": [ "M22-02" ], @@ -5034,10 +5046,20 @@ "storage lifecycle audit", "result SHA identity" ], - "started_at": null, - "completed_at": null, - "commit_sha": null, - "evidence": [], + "started_at": "2026-08-12T05:02:56+09:00", + "completed_at": "2026-08-12T14:05:25+09:00", + "commit_sha": "eba09d977e3eddb45f7a7982ebf026b0e5731c45", + "evidence": [ + "benchmarks/evidence/m22/full-native-performance-storage.json", + "tests/test_full_native_performance_evidence.py", + "docs/native-vector-core.md", + ".nfi/roadmap-acceptance/M22-03/spot-five-year-cold-01.profile.json", + ".nfi/roadmap-acceptance/M22-03/spot-five-year-warm-02.profile.json", + ".nfi/roadmap-acceptance/M22-03/spot-five-year-warm-03.profile.json", + ".nfi/roadmap-acceptance/M22-03/final/python-all.log", + ".nfi/roadmap-acceptance/M22-03/final/x7-spot/run.json", + ".nfi/roadmap-acceptance/M22-03/final/x7-futures/run.json" + ], "blocker": null }, { @@ -5045,7 +5067,7 @@ "order": 2204, "milestone": "M22", "title": "Publish the v1.6.0 Full Native Strategy release", - "status": "pending", + "status": "in_progress", "depends_on": [ "M22-03" ], @@ -5065,7 +5087,7 @@ "Full Native product bundle", "RC/stable byte identity" ], - "started_at": null, + "started_at": "2026-08-14T03:11:20+09:00", "completed_at": null, "commit_sha": null, "evidence": [], @@ -8645,6 +8667,184 @@ "next_eligible_task": "M22-01", "next_task_manual_gate": false } + }, + { + "sequence": 190, + "timestamp": "2026-08-11T22:50:22+09:00", + "task_id": "M22-01", + "event": "task_started", + "details": { + "objective": "compile and independently qualify the latest upstream NFI through the complete Native vector and simulation pipeline for Spot and Futures", + "latest_upstream_identity_must_be_refreshed": true, + "static_only_promotion_allowed": false, + "partial_callback_coverage_allowed": false, + "runtime_hardcoding_allowed": false, + "trade_surface_and_full_state_exact_required": true, + "official_fallback_retained": true + } + }, + { + "sequence": 191, + "timestamp": "2026-08-12T04:36:53+09:00", + "task_id": "M22-01", + "event": "task_completed", + "details": { + "implementation_commit": "6c60bff36aaf10272c1a0ffd6678961bbb5d1703", + "latest_upstream_commit": "1df961c07e5ce6b1a8cb459a2a46958aed258323", + "latest_strategy_sha256": "45a2bf611d6fc5e60c7e1f4c672ce7932f6573872c2a959f6338d079dac5e382", + "freqtrade_version": "2026.5.1", + "latest_full_native_qualification_fingerprint": "2e3bb9fc12597d400043f1e2015687b9bca5da7ed97859f27b8e9842e3c8d087", + "signal_47_boundary_fingerprint": "d4128b380d483a05f7a230bfe78dad14502438d45a8fa0adeb0c094c5d73f7fe", + "latest_spot_trade_surface_exact": true, + "latest_spot_full_state_event_count": 288, + "latest_spot_full_state_stream_hash": "e119fc56545e34f2b368c4baa59c0067d64d7d821a018687de8166aa81bb4428", + "latest_futures_trade_surface_exact": true, + "latest_futures_full_state_event_count": 2880, + "latest_futures_full_state_stream_hash": "eff3ddae550d7f8877bd04f96457b6ed052b4f9ea0cca968d9a4721141f07bdb", + "captured_spot_trade_surface_and_full_state_exact": true, + "captured_spot_state_event_count": 288, + "captured_spot_state_stream_hash": "6a59c86a808e0e5f8fc016c1467fa173502ea37b0eb870f84e7f51cfa1818f8b", + "captured_futures_trade_surface_and_full_state_exact": true, + "captured_futures_state_event_count": 5472, + "captured_futures_state_stream_hash": "029a12115d0e9095d4e1cc19e25694a28bee4c625b6cd59dc9226ab1ceaeff85", + "python_test_count": 834, + "future_compatibility_test_count": 143, + "rust_workspace_default_test_count": 296, + "runtime_hardcoding_added": false, + "latest_x7_fully_compiled": true, + "full_native_strategy_claim": true, + "python_strategy_execution_removed_from_native": false, + "official_fallback_retained": true, + "next_eligible_task": "M22-02", + "next_task_manual_gate": false + } + }, + { + "sequence": 192, + "timestamp": "2026-08-12T04:36:53+09:00", + "task_id": "M22-02", + "event": "task_started", + "details": { + "objective": "remove NFI Python strategy execution from the Native lane while retaining the AST compiler and official Freqtrade fallback", + "native_populate_execution_allowed": false, + "python_ast_compiler_retained": true, + "official_fallback_retained": true, + "sealed_feather_replay_retained": true, + "runtime_hardcoding_allowed": false + } + }, + { + "sequence": 193, + "timestamp": "2026-08-12T05:02:56+09:00", + "task_id": "M22-02", + "event": "task_completed", + "details": { + "implementation_commit": "f2315658affa4320388c899ecb3e5143e0ea3929", + "full_native_vector_runtime_version": "1.1.0", + "strategy_source_mode": "python-ast-compile-only", + "populate_methods_executed": false, + "runtime_mode": "rust-full-native", + "module_scope_execution_bomb_test_passed": true, + "manifest_execution_claim_drift_rejected_before_data_decode": true, + "vector_worker_count": 0, + "analyzed_feather_outputs": 0, + "checkpoint_execution_claim_is_sealed": true, + "official_fallback_isolated_and_retained": true, + "sealed_feather_replay_retained": true, + "spot_manifest_sha256": "b76ddcae92972692ff1fb011ef540dc81ba389bc93019174ff90a36be1539bcc", + "futures_manifest_sha256": "d6236067eb775fb59842853c9d15cc0ca9c2b7914b60a636d59837c00a610f14", + "spot_trade_surface_and_full_state_exact": true, + "spot_state_event_count": 288, + "spot_state_stream_hash": "6a59c86a808e0e5f8fc016c1467fa173502ea37b0eb870f84e7f51cfa1818f8b", + "futures_trade_surface_and_full_state_exact": true, + "futures_state_event_count": 5472, + "futures_state_stream_hash": "029a12115d0e9095d4e1cc19e25694a28bee4c625b6cd59dc9226ab1ceaeff85", + "python_test_count": 836, + "future_compatibility_test_count": 143, + "rust_workspace_default_test_count": 296, + "runtime_hardcoding_added": false, + "next_eligible_task": "M22-03", + "next_task_manual_gate": false + } + }, + { + "sequence": 194, + "timestamp": "2026-08-12T05:02:56+09:00", + "task_id": "M22-03", + "event": "task_started", + "details": { + "objective": "certify cold and warm Full Native performance and bounded storage on a representative five-year workload", + "five_year_result_identity_required": true, + "cold_and_warm_measurements_required": true, + "cache_and_artifact_lifecycle_audit_required": true, + "unbounded_local_accumulation_allowed": false, + "runtime_hardcoding_allowed": false + } + }, + { + "sequence": 195, + "timestamp": "2026-08-12T14:05:25+09:00", + "task_id": "M22-03", + "event": "task_completed", + "details": { + "implementation_commit": "eba09d977e3eddb45f7a7982ebf026b0e5731c45", + "latest_upstream_commit": "1df961c07e5ce6b1a8cb459a2a46958aed258323", + "latest_strategy_sha256": "45a2bf611d6fc5e60c7e1f4c672ce7932f6573872c2a959f6338d079dac5e382", + "five_year_manifest_sha256": "3b1b18bdaaacca43ff538e1397547df020ace3e80913371aa137607dc2185e90", + "five_year_result_sha256": "5adac8e620877b30d7e52988e377d068799be3a4045a8d45221e29620aa08266", + "five_year_result_byte_identical": true, + "five_year_trade_count": 1162, + "benchmark_repetitions": 3, + "benchmark_wall_seconds": [ + 3797, + 3771, + 3764 + ], + "benchmark_wall_spread_ratio": 0.008767268862911796, + "five_repetitions_required": false, + "median_peak_rss_bytes": 41793679360, + "spool_required_upper_bound_bytes": 38469364560, + "spool_actual_file_backed_bytes": 38468485341, + "spool_actual_within_bound": true, + "spool_orphan_file_count": 0, + "unbounded_local_accumulation_observed": false, + "spot_trade_surface_and_full_state_exact": true, + "spot_state_event_count": 288, + "spot_state_stream_hash": "6a59c86a808e0e5f8fc016c1467fa173502ea37b0eb870f84e7f51cfa1818f8b", + "futures_trade_surface_and_full_state_exact": true, + "futures_state_event_count": 5472, + "futures_state_stream_hash": "029a12115d0e9095d4e1cc19e25694a28bee4c625b6cd59dc9226ab1ceaeff85", + "python_test_count": 841, + "python_test_log_sha256": "00520b811b4a7264ebd50512e93e9bb5ac8fda8451d76e2d1402424fa5395518", + "rust_sim_core_test_count": 150, + "rust_vector_core_test_count": 117, + "rust_vector_io_test_count": 39, + "runtime_hardcoding_added": false, + "official_freqtrade_five_year_parity_claimed": false, + "cross_platform_performance_claimed": false, + "next_eligible_task": "M22-04", + "next_task_manual_gate": true + } + }, + { + "sequence": 196, + "timestamp": "2026-08-14T03:11:20+09:00", + "task_id": "M22-04", + "event": "task_started", + "details": { + "objective": "publish the v1.6.0 Full Native Strategy release with the current X7 Grind-5 compatibility hotfix", + "manual_gate_approval": "user-approved in conversation", + "x7_v536_source_sha256": "808e42060b9f18c86a7029d25d0d72f43e95756e8f5a571b158b9f2786833c04", + "x7_v537_upstream_commit": "f74d8cac2b11bb9e1c32dd7538dd4eca53c10efe", + "x7_v537_source_sha256": "3c11060280705d98db8f705c42853ac902c21cd494f0478299eea191587714f4", + "current_grind_five_fallback_lowering_required": true, + "same_commit_required_ci_required": true, + "three_os_installer_audits_required": true, + "byte_identical_rc_stable_required": true, + "external_2022_report_exact_claim_allowed_without_sealed_input": false, + "relabel_prior_certificates_allowed": false, + "runtime_hardcoding_allowed": false + } } ] } diff --git a/pyproject.toml b/pyproject.toml index 104cee9e..d1296174 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "nfi-backtest-engine" -version = "1.5.0" +version = "1.6.0" description = "Exact-parity research backtesting infrastructure for NFI strategies" readme = "README.md" requires-python = ">=3.12,<3.15" diff --git a/python/nfi_backtest_engine/_indicator_ast.py b/python/nfi_backtest_engine/_indicator_ast.py new file mode 100644 index 00000000..08ac4695 --- /dev/null +++ b/python/nfi_backtest_engine/_indicator_ast.py @@ -0,0 +1,549 @@ +"""Pure AST recognition and static-value helpers for indicator compilation.""" + +from __future__ import annotations + +import ast +import hashlib +import json +import math +from collections.abc import Mapping, Sequence +from functools import cache +from typing import Any, cast + +from ._indicator_contract import IndicatorProgramCompileError + +_NATIVE_HELPER_TEMPLATES = { + "chaikin-money-flow": """ +def helper(high, low, close, volume, timeperiod=20): + hl_range = high - low + mfm = np.zeros_like(close, dtype=np.float64) + valid = hl_range != 0 + mfm[valid] = ((close[valid] - low[valid]) - (high[valid] - close[valid])) / hl_range[valid] + mfv = mfm * volume + stacked = np.vstack([mfv, volume]) + stacked_clean = np.nan_to_num(stacked, nan=0.0) + csum = np.cumsum(stacked_clean, axis=1) + csum[:, timeperiod:] -= csum[:, :-timeperiod] + out = np.full(stacked.shape, np.nan, dtype=np.float64) + out[:, timeperiod - 1:] = csum[:, timeperiod - 1:] + mfv_sum, vol_sum = out[0], out[1] + vol_sum = np.where(vol_sum == 0, np.nan, vol_sum) + return mfv_sum / vol_sum +""", + "chaikin-money-flow-legacy": """ +def helper(high, low, close, volume, timeperiod=20): + hl_range = high - low + mfm = np.zeros_like(close, dtype=np.float64) + valid = hl_range != 0 + mfm[valid] = ((close[valid] - low[valid]) - (high[valid] - close[valid])) / hl_range[valid] + mfv = mfm * volume + mfv_sum = __class__.rolling_sum(mfv, timeperiod) + vol_sum = ta.SUM(volume, timeperiod=timeperiod) + vol_sum = np.where(vol_sum == 0, np.nan, vol_sum) + return mfv_sum / vol_sum +""", + "safe-percent-change": """ +def helper(arr): + arr = np.asarray(arr, dtype=np.float64) + out = np.full(arr.shape, np.nan, dtype=np.float64) + prev = arr[:-1] + np.divide((arr[1:] - prev), prev, out=out[1:], where=prev != 0) + out[1:] *= 100.0 + return out +""", +} + +_OPENING_RANGE_TEMPLATE = """ +_or_day = df["date"].dt.floor("1D") +_or_first4h = df["date"].dt.hour < 4 +_or_valid = df["date"].dt.hour >= 4 +orange_h_col = ( + df["high"].where(_or_first4h).groupby(_or_day).transform("max").where(_or_valid).to_numpy() +) +orange_l_col = ( + df["low"].where(_or_first4h).groupby(_or_day).transform("min").where(_or_valid).to_numpy() +) +""" + +_AGE_FILTER_TEMPLATE = """ +df["bt_agefilter_ok"] = False +df.loc[df.index > (12 * 24 * self.bt_min_age_days), "bt_agefilter_ok"] = True +""" + +_INSIDE_BAR_TEMPLATE = """ +_ib_hr = df["date"].dt.floor("1h") +_ib_agg = df.groupby(_ib_hr).agg(_ib_hh=("high", "max"), _ib_ll=("low", "min")) +_ib_agg["_ib_flag"] = (_ib_agg["_ib_hh"] < _ib_agg["_ib_hh"].shift(1)) & ( + _ib_agg["_ib_ll"] > _ib_agg["_ib_ll"].shift(1) +) +_ib_agg["_ib_mh"] = _ib_agg["_ib_hh"].shift(1) +_ib_agg["_ib_ml"] = _ib_agg["_ib_ll"].shift(1) +_ib_prev_hr = _ib_hr - pd.Timedelta(hours=1) +ib_ready_col = _ib_prev_hr.map(_ib_agg["_ib_flag"]).eq(True).astype(float).to_numpy() +ib_mother_h_col = _ib_prev_hr.map(_ib_agg["_ib_mh"]).to_numpy() +ib_mother_l_col = _ib_prev_hr.map(_ib_agg["_ib_ml"]).to_numpy() +""" + +_FIRST_FIRE_TEMPLATE = """ +_ph_cross = ((close_np > _ph_prev_max) & (np.roll(close_np, 1) <= _ph_prev_max)).astype(float) +_ph_cross[0] = 0.0 +ph_cross_cnt12_col = pd.Series(_ph_cross).rolling(12).sum().to_numpy() +""" + + +@cache +def _template_function(source: str) -> ast.FunctionDef: + tree = ast.parse(source) + function = tree.body[0] + assert isinstance(function, ast.FunctionDef) + return function + + +@cache +def _template_statements(source: str) -> tuple[ast.stmt, ...]: + return tuple(ast.parse(source).body) + + +def _helper_body(function: ast.FunctionDef) -> Sequence[ast.stmt]: + body = function.body + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + body = body[1:] + return body + + +def _helper_bodies_equal(left: ast.FunctionDef, right: ast.FunctionDef) -> bool: + left_body = _helper_body(left) + right_body = _helper_body(right) + if len(left_body) != len(right_body): + return False + return _ast_equal( + ast.Module(body=list(left_body), type_ignores=[]), + ast.Module(body=list(right_body), type_ignores=[]), + ) + + +def _recognized_tag_appender(function: ast.FunctionDef) -> bool: + parameters = [*function.args.posonlyargs, *function.args.args] + if len(parameters) != 3: + return False + target_name, _, tag_name = (parameter.arg for parameter in parameters) + for statement in ast.walk(function): + if not ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Subscript) + and isinstance(statement.targets[0].value, ast.Name) + and statement.targets[0].value.id == target_name + and isinstance(statement.value, ast.BinOp) + and isinstance(statement.value.op, ast.Add) + and isinstance(statement.value.right, ast.Name) + and statement.value.right.id == tag_name + ): + continue + return True + return False + + +def _is_shift_slice_assignment( + statement: ast.stmt, + *, + output_name: str, + source_name: str, + periods_name: str, + warmup: bool, +) -> bool: + if not ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Subscript) + and isinstance(statement.targets[0].value, ast.Name) + and statement.targets[0].value.id == output_name + and isinstance(statement.targets[0].slice, ast.Slice) + ): + return False + target_slice = statement.targets[0].slice + if warmup: + return ( + target_slice.lower is None + and isinstance(target_slice.upper, ast.Name) + and target_slice.upper.id == periods_name + and target_slice.step is None + and isinstance(statement.value, ast.Attribute) + and _qualified_name(statement.value) == "np.nan" + ) + if not ( + isinstance(target_slice.lower, ast.Name) + and target_slice.lower.id == periods_name + and target_slice.upper is None + and target_slice.step is None + and isinstance(statement.value, ast.Subscript) + and isinstance(statement.value.value, ast.Name) + and statement.value.value.id == source_name + and isinstance(statement.value.slice, ast.Slice) + ): + return False + source_slice = statement.value.slice + return ( + source_slice.lower is None + and isinstance(source_slice.upper, ast.UnaryOp) + and isinstance(source_slice.upper.op, ast.USub) + and isinstance(source_slice.upper.operand, ast.Name) + and source_slice.upper.operand.id == periods_name + and source_slice.step is None + ) + + +def _is_array_index_nan_write(statement: ast.stmt, name: str, index: int) -> bool: + return ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Subscript) + and isinstance(statement.targets[0].value, ast.Name) + and statement.targets[0].value.id == name + and isinstance(statement.targets[0].slice, ast.Constant) + and statement.targets[0].slice.value == index + and isinstance(statement.value, ast.Attribute) + and _qualified_name(statement.value) == "np.nan" + ) + + +def _is_absolute_difference_write( + statement: ast.stmt, + output_name: str, + source_node: ast.expr, +) -> bool: + if not ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Subscript) + and isinstance(statement.targets[0].value, ast.Name) + and statement.targets[0].value.id == output_name + and isinstance(statement.targets[0].slice, ast.Slice) + and isinstance(statement.targets[0].slice.lower, ast.Constant) + and statement.targets[0].slice.lower.value == 1 + and statement.targets[0].slice.upper is None + and statement.targets[0].slice.step is None + and isinstance(statement.value, ast.Call) + and _qualified_name(statement.value.func) == "np.abs" + and len(statement.value.args) == 1 + and not statement.value.keywords + and isinstance(statement.value.args[0], ast.Call) + and _qualified_name(statement.value.args[0].func) == "np.diff" + and len(statement.value.args[0].args) == 1 + and not statement.value.args[0].keywords + ): + return False + return _ast_equal(statement.value.args[0].args[0], source_node) + + +def _static_value(node: ast.expr, constants: Mapping[str, Any]) -> Any: + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Name) and node.id in constants: + return constants[node.id] + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "self" + and node.attr in constants + ): + return constants[node.attr] + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + value = _static_value(node.operand, constants) + if isinstance(value, int | float) and not isinstance(value, bool): + return -value + try: + value = ast.literal_eval(node) + except (ValueError, TypeError, MemoryError, RecursionError): + return None + return value if _is_json_value(value) else None + + +def _declared_class_constants( + class_node: ast.ClassDef, + analyzed: Mapping[str, Any], +) -> dict[str, Any]: + """Recover deterministic containers omitted by the JSON-only analyzer.""" + constants = {name: _normalized_static_value(value) for name, value in analyzed.items()} + for statement in class_node.body: + target: ast.expr | None = None + value_node: ast.expr | None = None + if isinstance(statement, ast.Assign) and len(statement.targets) == 1: + target = statement.targets[0] + value_node = statement.value + elif isinstance(statement, ast.AnnAssign) and statement.value is not None: + target = statement.target + value_node = statement.value + if not isinstance(target, ast.Name) or value_node is None: + continue + value = _class_static_value(value_node, constants) + if value is not _STATIC_MISSING: + constants[target.id] = _normalized_static_value(value) + return constants + + +_STATIC_MISSING = object() + + +def _class_static_value(node: ast.expr, constants: Mapping[str, Any]) -> Any: + if isinstance(node, ast.Name) and node.id in constants: + return constants[node.id] + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + function = { + "frozenset": frozenset, + "list": list, + "set": set, + "tuple": tuple, + }.get(node.func.id) + if function is None or len(node.args) != 1 or node.keywords: + return _STATIC_MISSING + argument = _class_static_value(node.args[0], constants) + if argument is _STATIC_MISSING: + return _STATIC_MISSING + try: + return function(argument) + except (TypeError, ValueError): + return _STATIC_MISSING + try: + return ast.literal_eval(node) + except (ValueError, TypeError, MemoryError, RecursionError): + return _STATIC_MISSING + + +def _normalized_static_value(value: Any) -> Any: + if value is None or isinstance(value, bool | int | float | str): + return value + if isinstance(value, Mapping): + if not all(isinstance(key, str) for key in value): + raise IndicatorProgramCompileError("static mapping keys must be strings") + return {str(key): _normalized_static_value(item) for key, item in value.items()} + if isinstance(value, set | frozenset): + normalized = [_normalized_static_value(item) for item in value] + return sorted(normalized, key=lambda item: json.dumps(item, sort_keys=True)) + if isinstance(value, list | tuple): + return [_normalized_static_value(item) for item in value] + raise IndicatorProgramCompileError( + f"static value is not JSON-compatible: {type(value).__name__}" + ) + + +def _effective_backtest_config(config: Mapping[str, Any]) -> dict[str, Any]: + effective = dict(_normalized_static_value(config)) + runmode = effective.get("runmode") + if runmode is None: + effective["runmode"] = {"value": "backtest"} + elif not isinstance(runmode, Mapping) or runmode.get("value") != "backtest": + raise IndicatorProgramCompileError( + "compiler run mode differs from the supplied configuration" + ) + return effective + + +def _is_json_value(value: Any) -> bool: + if value is None or isinstance(value, bool | int | float | str): + return True + if isinstance(value, list | tuple): + return all(_is_json_value(item) for item in value) + if isinstance(value, Mapping): + return all(isinstance(key, str) and _is_json_value(item) for key, item in value.items()) + return False + + +def _literal_string(node: ast.expr) -> str | None: + return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None + + +@cache +def _indicator_output_names(callable_name: str) -> tuple[str, ...] | None: + if not callable_name.startswith("ta."): + return None + from talib import abstract + + try: + function = abstract.Function(callable_name.removeprefix("ta.")) + except Exception: # TA-Lib exposes invalid-function errors through a generic wrapper exception. + return None + names = cast(Sequence[object], function.output_names) + return tuple(str(name) for name in names) + + +@cache +def _indicator_signature(callable_name: str) -> tuple[int, tuple[str, ...]] | None: + """Return TA-Lib array arity and its source-ordered static parameters.""" + if not callable_name.startswith("ta."): + return None + from talib import abstract + + try: + function = abstract.Function(callable_name.removeprefix("ta.")) + except Exception: # See the matching output-name probe above. + return None + input_count = 0 + for value in function.input_names.values(): + if isinstance(value, str): + input_count += 1 + elif isinstance(value, Sequence): + input_count += len(value) + else: # pragma: no cover - pinned TA-Lib exposes only these two shapes. + return None + return input_count, tuple(str(name) for name in function.parameters) + + +def _qualified_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + parent = _qualified_name(node.value) + return f"{parent}.{node.attr}" if parent else node.attr + return None + + +def _parameter_type(name: str) -> str: + lowered = name.lower() + if lowered in {"df", "dataframe", "informative", "frame", "info"} or "dataframe" in lowered: + return "dataframe" + if lowered == "metadata": + return "metadata" + return "dynamic" + + +def _cast_target(node: ast.expr) -> str | None: + if isinstance(node, ast.Name) and node.id in {"bool", "float", "int"}: + return node.id + qualified = _qualified_name(node) + if qualified in {"np.bool_", "np.bool"}: + return "bool" + if qualified in {"np.float32", "np.float64"}: + return "float" + if qualified in {"np.int32", "np.int64"}: + return "int" + return None + + +def _flatten_binary_values( + node: ast.BinOp, + operator_type: type[ast.operator], +) -> list[ast.expr]: + values: list[ast.expr] = [] + stack: list[ast.expr] = [node] + while stack: + current = stack.pop() + if isinstance(current, ast.BinOp) and isinstance(current.op, operator_type): + stack.append(current.right) + stack.append(current.left) + else: + values.append(current) + return values + + +def _ast_equal(left: Any, right: Any) -> bool: + """Compare AST structure without recursing through deeply associated expressions.""" + pending: list[tuple[Any, Any]] = [(left, right)] + while pending: + current_left, current_right = pending.pop() + if isinstance(current_left, ast.AST) or isinstance(current_right, ast.AST): + if not isinstance(current_left, ast.AST) or type(current_left) is not type( + current_right + ): + return False + for field in current_left._fields: + pending.append((getattr(current_left, field), getattr(current_right, field))) + continue + if isinstance(current_left, list | tuple) or isinstance(current_right, list | tuple): + if not isinstance(current_left, list | tuple) or not isinstance( + current_right, list | tuple + ): + return False + if len(current_left) != len(current_right): + return False + pending.extend(zip(current_left, current_right, strict=True)) + continue + if ( + isinstance(current_left, float) + and isinstance(current_right, float) + and math.isnan(current_left) + and math.isnan(current_right) + ): + continue + if current_left != current_right: + return False + return True + + +def _small_static_candidate(node: ast.expr) -> bool: + return isinstance( + node, + ast.Constant + | ast.Name + | ast.Attribute + | ast.Subscript + | ast.Call + | ast.JoinedStr + | ast.List + | ast.Tuple + | ast.Set + | ast.Dict, + ) + + +def _assigned_name(statement: ast.stmt) -> str | None: + if ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Name) + ): + return statement.targets[0].id + return None + + +def _dataframe_column_target(node: ast.expr) -> tuple[str, str] | None: + if not isinstance(node, ast.Subscript) or not isinstance(node.value, ast.Name): + return None + column = _literal_string(node.slice) + if column is None: + return None + return node.value.id, column + + +def _loop_target_names(target: ast.expr) -> tuple[str, ...]: + if isinstance(target, ast.Name): + return (target.id,) + if isinstance(target, ast.Tuple | ast.List) and all( + isinstance(item, ast.Name) for item in target.elts + ): + return tuple(item.id for item in target.elts if isinstance(item, ast.Name)) + return () + + +def _safe_expression(node: ast.AST) -> str: + try: + return ast.unparse(node) + except RecursionError: + return f"" + + +def _iterative_ast_sha256(node: ast.AST) -> str: + digest = hashlib.sha256() + stack: list[tuple[str, Any]] = [("node", node)] + while stack: + kind, value = stack.pop() + digest.update(kind.encode()) + digest.update(b"\0") + if isinstance(value, ast.AST): + digest.update(type(value).__name__.encode()) + for name, child in reversed(list(ast.iter_fields(value))): + stack.append(("field", name)) + stack.append(("value", child)) + elif isinstance(value, list): + digest.update(str(len(value)).encode()) + for child in reversed(value): + stack.append(("value", child)) + else: + digest.update(repr(value).encode()) + digest.update(b"\0") + return digest.hexdigest() diff --git a/python/nfi_backtest_engine/_indicator_contract.py b/python/nfi_backtest_engine/_indicator_contract.py new file mode 100644 index 00000000..0f621968 --- /dev/null +++ b/python/nfi_backtest_engine/_indicator_contract.py @@ -0,0 +1,185 @@ +"""Value types, lookback accounting, and identity helpers for indicator programs.""" + +from __future__ import annotations + +import ast +import copy +import hashlib +import json +import math +from collections.abc import Mapping, Sequence +from typing import Any, Never + +from .errors import StrategyAnalysisError + + +class IndicatorProgramCompileError(StrategyAnalysisError): + """Indicator source cannot be represented by the causal v1 DAG.""" + + +def _literal_type(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "bool-scalar" + if isinstance(value, int): + return "int-scalar" + if isinstance(value, float): + return "f64-scalar" + if isinstance(value, str): + return "string-scalar" + return "json-scalar" + + +def _cast_result_type(source_type: str, target: str) -> str: + suffix = "column" if source_type.endswith("-column") else "scalar" + prefix = {"bool": "bool", "float": "f64", "int": "int"}[target] + return f"{prefix}-{suffix}" + + +def _literal_parameters(value: Any) -> dict[str, Any]: + if isinstance(value, float) and not math.isfinite(value): + special = "nan" if math.isnan(value) else "+infinity" if value > 0 else "-infinity" + return {"special": special} + return {"value": value} + + +def _is_column(value_type: str) -> bool: + return value_type.endswith("-column") + + +def _numeric_result_type(left: str, right: str) -> str: + if _is_column(left) or _is_column(right): + return "f64-column" + if left == right == "int-scalar": + return "int-scalar" + return "f64-scalar" + + +def _boolean_result_type(*value_types: str) -> str: + if any(_is_column(value_type) for value_type in value_types): + return "bool-column" + return "bool-scalar" + + +def _merge_value_types(left: str, right: str) -> str: + if left == right: + return left + if _is_column(left) or _is_column(right): + return "f64-column" + if {left, right} <= {"int-scalar", "f64-scalar"}: + return "f64-scalar" + return "dynamic" + + +def _array_result_type(inputs: Sequence[str], node_types: Mapping[str, str]) -> str: + types = [node_types[node] for node in inputs] + if any(_is_column(value_type) for value_type in types): + return "f64-column" + return "dynamic" + + +def _array_call_result_type( + callable_name: str, + inputs: Sequence[str], + node_types: Mapping[str, str], +) -> str: + if callable_name == "np.isnan" and len(inputs) == 1: + input_type = node_types[inputs[0]] + return "bool-column" if _is_column(input_type) else "bool-scalar" + if callable_name == "np.full" and len(inputs) == 2: + fill_type = node_types[inputs[1]] + return { + "bool-scalar": "bool-column", + "int-scalar": "int-column", + "f64-scalar": "f64-column", + "string-scalar": "string-column", + }.get(fill_type, "dynamic") + return _array_result_type(inputs, node_types) + + +def _zero_lookback() -> dict[str, Any]: + return { + "kind": "finite", + "candles": 0, + "expression": None, + "causal": True, + } + + +def _add_finite_lookback(lookback: Mapping[str, Any], candles: int) -> dict[str, Any]: + if lookback["kind"] == "finite" and isinstance(lookback["candles"], int): + return { + "kind": "finite", + "candles": lookback["candles"] + candles, + "expression": None, + "causal": bool(lookback["causal"]), + } + return { + "kind": "mixed", + "candles": None, + "expression": f"{lookback['kind']}+{candles}", + "causal": bool(lookback["causal"]), + } + + +def _merge_lookbacks(lookbacks: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + if not lookbacks: + return _zero_lookback() + causal = all(bool(item["causal"]) for item in lookbacks) + if all(item["kind"] == "finite" and isinstance(item["candles"], int) for item in lookbacks): + return { + "kind": "finite", + "candles": max(int(item["candles"]) for item in lookbacks), + "expression": None, + "causal": causal, + } + kinds = sorted({str(item["kind"]) for item in lookbacks}) + return { + "kind": kinds[0] if len(kinds) == 1 else "mixed", + "candles": None, + "expression": "+".join(kinds), + "causal": causal, + } + + +def _program_lookback(nodes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + return _merge_lookbacks([node["lookback"] for node in nodes]) + + +def _location(node: ast.AST) -> dict[str, Any]: + return { + "path": "strategy.py", + "line": getattr(node, "lineno", 1), + "column": getattr(node, "col_offset", 0), + "end_line": getattr(node, "end_lineno", getattr(node, "lineno", 1)), + "end_column": getattr(node, "end_col_offset", getattr(node, "col_offset", 0)), + } + + +def _unsupported(node: ast.AST, description: str) -> Never: + location = _location(node) + raise IndicatorProgramCompileError( + f"strategy.py:{location['line']}:{location['column']}: " + f"indicator-program-v1 does not support {description}" + ) + + +def _numeric_identifier_key(record: Mapping[str, Any]) -> int: + return int(str(record["id"])[1:]) + + +def _fingerprint(program: Mapping[str, Any]) -> str: + identity = copy.deepcopy(dict(program)) + source = identity.get("source") + if isinstance(source, dict): + source.pop("path", None) + return hashlib.sha256( + json.dumps( + identity, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() diff --git a/python/nfi_backtest_engine/_rust.pyi b/python/nfi_backtest_engine/_rust.pyi index 7eb53bf3..6b0bba22 100644 --- a/python/nfi_backtest_engine/_rust.pyi +++ b/python/nfi_backtest_engine/_rust.pyi @@ -1,4 +1,25 @@ from pathlib import Path +from typing import TypedDict + +type NumericColumn = list[float | None] +type InformativeFrame = tuple[ + str, + str, + list[int], + dict[str, NumericColumn], +] + +class NativeTypedColumn(TypedDict): + value_type: str + values: list[float | int | bool | str | None] + +class FullVectorOutput(TypedDict): + pair: str + timeframe: str + timestamps_ms: list[int] + execution_start_index: int + columns: dict[str, NativeTypedColumn] + enabled_indexes: dict[str, list[int]] def schema_version() -> str: ... def simulator_available() -> bool: ... @@ -23,3 +44,35 @@ def simulate_vector_file_profiled( profile_path: str | Path, events_path: str | Path | None = ..., ) -> None: ... +def simulate_full_vector_file( + manifest_path: str | Path, + output_path: str | Path, + events_path: str | Path | None = ..., + pair_worker_limit: int | None = ..., +) -> None: ... +def simulate_full_vector_file_profiled( + manifest_path: str | Path, + output_path: str | Path, + profile_path: str | Path, + events_path: str | Path | None = ..., + pair_worker_limit: int | None = ..., +) -> None: ... +def execute_full_vector( + indicator_program: str, + signal_program: str, + tag_program: str, + base_pair: str, + base_timeframe: str, + base_timestamps_ms: list[int], + base_columns: dict[str, NumericColumn], + informative_frames: list[InformativeFrame], + metadata: dict[str, str], + requested_indicator_columns: list[str], + execution_start_index: int, +) -> FullVectorOutput: ... +def execute_numeric_mutation_program( + program: str, + columns: dict[str, NumericColumn], + metadata: dict[str, str], + requested_outputs: list[str], +) -> dict[str, NativeTypedColumn]: ... diff --git a/python/nfi_backtest_engine/cli.py b/python/nfi_backtest_engine/cli.py index d3ca7c3a..c911c6b9 100644 --- a/python/nfi_backtest_engine/cli.py +++ b/python/nfi_backtest_engine/cli.py @@ -691,6 +691,7 @@ def build_parser() -> argparse.ArgumentParser: ) strategy_indicator_program.add_argument("source", type=Path) strategy_indicator_program.add_argument("--class", dest="class_name") + strategy_indicator_program.add_argument("--config", type=Path) strategy_indicator_program.add_argument("--output", "-o", type=Path, required=True) strategy_signal_program = strategy_commands.add_parser( "signal-program", @@ -698,6 +699,7 @@ def build_parser() -> argparse.ArgumentParser: ) strategy_signal_program.add_argument("source", type=Path) strategy_signal_program.add_argument("--class", dest="class_name") + strategy_signal_program.add_argument("--config", type=Path) strategy_signal_program.add_argument( "--trading-mode", choices=("spot", "futures"), @@ -710,6 +712,7 @@ def build_parser() -> argparse.ArgumentParser: ) strategy_tag_program.add_argument("source", type=Path) strategy_tag_program.add_argument("--class", dest="class_name") + strategy_tag_program.add_argument("--config", type=Path) strategy_tag_program.add_argument( "--trading-mode", choices=("spot", "futures"), diff --git a/python/nfi_backtest_engine/commands/run.py b/python/nfi_backtest_engine/commands/run.py index b03aacd3..0853d971 100644 --- a/python/nfi_backtest_engine/commands/run.py +++ b/python/nfi_backtest_engine/commands/run.py @@ -297,9 +297,11 @@ def _execute_strategy(args: argparse.Namespace) -> int: if args.strategy_command == "indicator-program": from ..indicator_program import compile_indicator_program + config = load_effective_config(args.config)["config"] if args.config else None program = compile_indicator_program( args.source, class_name=args.class_name, + config=config, ) write_json(args.output, program) print( @@ -314,10 +316,12 @@ def _execute_strategy(args: argparse.Namespace) -> int: if args.strategy_command == "signal-program": from ..signal_program import compile_signal_program + config = load_effective_config(args.config)["config"] if args.config else None program = compile_signal_program( args.source, class_name=args.class_name, trading_mode=args.trading_mode, + config=config, ) write_json(args.output, program) print( @@ -332,10 +336,12 @@ def _execute_strategy(args: argparse.Namespace) -> int: if args.strategy_command == "tag-program": from ..tag_program import compile_tag_program + config = load_effective_config(args.config)["config"] if args.config else None program = compile_tag_program( args.source, class_name=args.class_name, trading_mode=args.trading_mode, + config=config, ) write_json(args.output, program) print( diff --git a/python/nfi_backtest_engine/engine_runtime.py b/python/nfi_backtest_engine/engine_runtime.py index a2b462d4..379dd885 100644 --- a/python/nfi_backtest_engine/engine_runtime.py +++ b/python/nfi_backtest_engine/engine_runtime.py @@ -20,6 +20,13 @@ from .hardware import SPOOL_DIRECTORY_ENVIRONMENT, load_execution_profile from .resource_usage import process_peak_rss_bytes +SIMULATION_JSON_INPUT = "simulation-json" +FEATHER_VECTOR_INPUT = "feather-vector" +FULL_VECTOR_INPUT = "full-vector" +_ENGINE_INPUT_KINDS = frozenset( + {SIMULATION_JSON_INPUT, FEATHER_VECTOR_INPUT, FULL_VECTOR_INPUT} +) + def build_engine(*, force: bool = False) -> dict[str, Any]: """Return the packaged engine, or build the source-checkout CLI fallback.""" @@ -126,14 +133,30 @@ def run_engine( timeout_seconds: int | None = None, events_path: str | Path | None = None, vector_manifest: bool = False, + input_kind: str | None = None, engine_profile_path: str | Path | None = None, + pair_worker_limit: int | None = None, ) -> dict[str, Any]: """Run one simulation without per-candle Python calls. - ``vector_manifest=True`` selects the compact, SHA-bound Feather transport. - It is explicit instead of inferred from a filename so malformed inputs - cannot accidentally fall through to a different parser. + ``input_kind`` explicitly selects simulation JSON, the captured Feather + vector transport, or the complete Rust-native vector transport. The legacy + ``vector_manifest=True`` spelling remains an alias for ``feather-vector``. + No transport is inferred from a filename. """ + selected_input_kind = _resolve_input_kind( + input_kind=input_kind, + vector_manifest=vector_manifest, + ) + if pair_worker_limit is not None: + if ( + not isinstance(pair_worker_limit, int) + or isinstance(pair_worker_limit, bool) + or pair_worker_limit <= 0 + ): + raise BenchmarkError("pair worker limit must be a positive integer") + if selected_input_kind != FULL_VECTOR_INPUT: + raise BenchmarkError("pair worker limit requires full-vector input") source = Path(input_path).resolve() destination = Path(output_path).resolve() if not source.is_file(): @@ -144,8 +167,11 @@ def run_engine( engine_profile_destination = ( Path(engine_profile_path).resolve() if engine_profile_path is not None else None ) - if engine_profile_destination is not None and not vector_manifest: - raise BenchmarkError("engine phase profiling requires a vector manifest") + if ( + engine_profile_destination is not None + and selected_input_kind == SIMULATION_JSON_INPUT + ): + raise BenchmarkError("engine phase profiling requires a vector input") if event_destination is not None and event_destination.exists(): raise BenchmarkError(f"simulation events output already exists: {event_destination}") destination.parent.mkdir(parents=True, exist_ok=True) @@ -166,8 +192,9 @@ def run_engine( profile_path=profile_path, timeout_seconds=timeout_seconds, build=build, - vector_manifest=vector_manifest, + input_kind=selected_input_kind, engine_profile_destination=engine_profile_destination, + pair_worker_limit=pair_worker_limit, ) binary = Path(build["binary_path"]) # `/usr/bin/time` writes process metrics outside the simulation result. @@ -218,8 +245,12 @@ def run_engine( str(destination), ] engine_argument_index = len(time_prefix) + 1 - if vector_manifest: - vector_arguments = ["--vector-manifest"] + if selected_input_kind != SIMULATION_JSON_INPUT: + vector_arguments = [ + "--vector-manifest" + if selected_input_kind == FEATHER_VECTOR_INPUT + else "--full-vector-manifest" + ] if engine_profile_destination is not None: vector_arguments.extend( [ @@ -231,6 +262,8 @@ def run_engine( ), ] ) + if pair_worker_limit is not None: + vector_arguments.extend(["--pair-workers", str(pair_worker_limit)]) command[engine_argument_index:engine_argument_index] = vector_arguments if event_destination is not None: command.append(_wsl_path(event_destination) if os.name == "nt" else str(event_destination)) @@ -262,6 +295,7 @@ def run_engine( result = read_json(destination) return { "schema_version": "1.0.0", + "input_kind": selected_input_kind, "input_path": str(source), "input_sha256": sha256_file(source), "output_path": str(destination), @@ -273,6 +307,7 @@ def run_engine( "execution_profile_fingerprint": ( profile["hardware_fingerprint"] if profile is not None else None ), + "pair_worker_limit": pair_worker_limit, "trade_count": len(result.get("trades", [])), "events": ( { @@ -295,8 +330,9 @@ def _run_native_engine( profile_path: str | Path | None, timeout_seconds: int | None, build: dict[str, Any], - vector_manifest: bool, + input_kind: str, engine_profile_destination: Path | None, + pair_worker_limit: int | None, ) -> dict[str, Any]: if timeout_seconds is not None and timeout_seconds <= 0: raise BenchmarkError("engine timeout must be positive") @@ -311,14 +347,40 @@ def _run_native_engine( started_ns = time.perf_counter_ns() try: os.environ.update(environment) - if vector_manifest and engine_profile_destination is not None: + if input_kind == FULL_VECTOR_INPUT and engine_profile_destination is not None: + if pair_worker_limit is None: + native.simulate_full_vector_file_profiled( + source, + destination, + engine_profile_destination, + event_destination, + ) + else: + native.simulate_full_vector_file_profiled( + source, + destination, + engine_profile_destination, + event_destination, + pair_worker_limit, + ) + elif input_kind == FULL_VECTOR_INPUT: + if pair_worker_limit is None: + native.simulate_full_vector_file(source, destination, event_destination) + else: + native.simulate_full_vector_file( + source, + destination, + event_destination, + pair_worker_limit, + ) + elif input_kind == FEATHER_VECTOR_INPUT and engine_profile_destination is not None: native.simulate_vector_file_profiled( source, destination, engine_profile_destination, event_destination, ) - elif vector_manifest: + elif input_kind == FEATHER_VECTOR_INPUT: native.simulate_vector_file(source, destination, event_destination) else: native.simulate_file(source, destination, event_destination) @@ -341,6 +403,7 @@ def _run_native_engine( result = read_json(destination) return { "schema_version": "1.0.0", + "input_kind": input_kind, "input_path": str(source), "input_sha256": sha256_file(source), "output_path": str(destination), @@ -352,6 +415,7 @@ def _run_native_engine( "execution_profile_fingerprint": ( profile["hardware_fingerprint"] if profile is not None else None ), + "pair_worker_limit": pair_worker_limit, "trade_count": len(result.get("trades", [])), "events": ( { @@ -366,6 +430,18 @@ def _run_native_engine( } +def _resolve_input_kind(*, input_kind: str | None, vector_manifest: bool) -> str: + if input_kind is None: + return FEATHER_VECTOR_INPUT if vector_manifest else SIMULATION_JSON_INPUT + if input_kind not in _ENGINE_INPUT_KINDS: + raise BenchmarkError(f"unsupported engine input kind: {input_kind}") + if vector_manifest and input_kind != FEATHER_VECTOR_INPUT: + raise BenchmarkError( + "vector_manifest=True conflicts with the explicit engine input kind" + ) + return input_kind + + def _engine_profile_record(path: Path | None) -> dict[str, Any] | None: if path is None or not path.is_file(): return None diff --git a/python/nfi_backtest_engine/full_native_calibration.py b/python/nfi_backtest_engine/full_native_calibration.py new file mode 100644 index 00000000..0a41dcc0 --- /dev/null +++ b/python/nfi_backtest_engine/full_native_calibration.py @@ -0,0 +1,230 @@ +"""Measured worker admission for one sealed Full Native workload.""" + +from __future__ import annotations + +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from .canonical import read_json, write_json +from .engine_runtime import FULL_VECTOR_INPUT, build_engine, run_engine +from .errors import BenchmarkError, SpecValidationError +from .fixture import sha256_file +from .workload_calibration import ( + calibrated_admission, + calibration_key, + calibration_path, + create_workload_calibration, + load_workload_calibration, +) + +_RunEngine = Callable[..., dict[str, Any]] + + +def resolve_full_native_pair_workers( + manifest_path: str | Path, + *, + profile_path: str | Path, + hardware_fingerprint: str, + requested_workers: int, + memory_cap_bytes: int | None, + calibration_directory: str | Path, + recalibrate: bool = False, + run_engine_fn: _RunEngine = run_engine, +) -> dict[str, Any]: + """Measure one worst-footprint pair, then admit workers from current memory. + + Calibration identity includes every manifest byte and the Rust source + fingerprint. A source/data/config change therefore cannot inherit an old + memory observation. The probe uses the same complete Rust pipeline with a + single pair and one worker; its result is diagnostic and deleted. + """ + if requested_workers <= 0: + raise SpecValidationError("Full Native requested workers must be positive") + manifest = Path(manifest_path).resolve() + if not manifest.is_file(): + raise BenchmarkError(f"Full Native manifest does not exist: {manifest}") + build = build_engine() + source_fingerprint = build.get("source_fingerprint") + if not isinstance(source_fingerprint, str) or len(source_fingerprint) != 64: + raise BenchmarkError("Full Native engine source fingerprint is unavailable") + identity = { + "kind": "full-native-pair-worker-calibration-v1", + "manifest_sha256": sha256_file(manifest), + "engine_source_fingerprint": source_fingerprint, + } + key = calibration_key(identity) + directory = Path(calibration_directory).resolve() / "full-native" + directory.mkdir(parents=True, exist_ok=True) + destination = calibration_path(directory, key) + + calibration = None + reused = False + if destination.is_file() and not recalibrate: + calibration = load_workload_calibration( + destination, + expected_key=key, + hardware_fingerprint=hardware_fingerprint, + ) + reused = True + if calibration is None: + probe_manifest, probe_pair = _probe_manifest(manifest) + result_path = _vacant_temporary(manifest.parent, ".full-native-probe-result.") + profile_output = _vacant_temporary(manifest.parent, ".full-native-probe-profile.") + try: + execution = run_engine_fn( + probe_manifest, + result_path, + profile_path=profile_path, + input_kind=FULL_VECTOR_INPUT, + engine_profile_path=profile_output, + pair_worker_limit=1, + ) + peak = execution.get("peak_rss_bytes") + wall = execution.get("wall_time_seconds") + if isinstance(peak, bool) or not isinstance(peak, int) or peak <= 0: + raise BenchmarkError("Full Native calibration did not measure positive peak RSS") + if isinstance(wall, bool) or not isinstance(wall, int | float) or wall <= 0: + raise BenchmarkError("Full Native calibration did not measure positive wall time") + calibration = create_workload_calibration( + destination, + key=key, + identity=identity, + hardware_fingerprint=hardware_fingerprint, + probe_pair=probe_pair, + probe_peak_rss_bytes=peak, + probe_wall_time_seconds=float(wall), + requested_cpu_processes=requested_workers, + memory_cap_bytes=memory_cap_bytes, + ) + finally: + probe_manifest.unlink(missing_ok=True) + result_path.unlink(missing_ok=True) + profile_output.unlink(missing_ok=True) + + probe = calibration["probe"] + admission = calibrated_admission( + probe_peak_rss_bytes=int(probe["peak_rss_bytes"]), + requested_cpu_processes=requested_workers, + memory_cap_bytes=memory_cap_bytes, + ) + return { + "schema_version": "1.0.0", + "key": key, + "path": str(destination), + "reused": reused, + "probe_pair": probe["pair"], + "probe_peak_rss_bytes": probe["peak_rss_bytes"], + "probe_wall_time_seconds": probe["wall_time_seconds"], + "worker_limit": admission["safe_processes"], + "admission": admission, + } + + +def _probe_manifest(source: Path) -> tuple[Path, str]: + document = read_json(source) + if not isinstance(document, dict) or document.get("schema_version") != ( + "full-native-vector-manifest-v1" + ): + raise BenchmarkError("Full Native calibration requires a v1 manifest") + pairs = document.get("pairs") + frames = document.get("frames") + programs = document.get("programs") + if not isinstance(pairs, list) or not pairs or not isinstance(frames, list): + raise BenchmarkError("Full Native calibration manifest has no pair/frame inventory") + if not isinstance(programs, dict): + raise BenchmarkError("Full Native calibration manifest has no program inventory") + frame_rows: dict[str, int] = {} + frame_identities: set[tuple[str, str]] = set() + for frame in frames: + if not isinstance(frame, dict) or not isinstance(frame.get("identity"), dict): + raise BenchmarkError("Full Native calibration frame inventory is invalid") + identity = frame["identity"] + pair = identity.get("pair") + timeframe = identity.get("timeframe") + rows = frame.get("rows") + if ( + not isinstance(pair, str) + or not isinstance(timeframe, str) + or isinstance(rows, bool) + or not isinstance(rows, int) + or rows <= 0 + ): + raise BenchmarkError("Full Native calibration frame identity/rows are invalid") + frame_identities.add((pair, timeframe)) + frame_rows[pair] = frame_rows.get(pair, 0) + rows + ranked: list[tuple[int, str, str, dict[str, Any]]] = [] + for pair_document in pairs: + identity = pair_document.get("identity") if isinstance(pair_document, dict) else None + if not isinstance(identity, dict): + raise BenchmarkError("Full Native calibration pair identity is invalid") + pair = identity.get("pair") + timeframe = identity.get("timeframe") + if not isinstance(pair, str) or not isinstance(timeframe, str): + raise BenchmarkError("Full Native calibration pair identity is invalid") + if (pair, timeframe) not in frame_identities: + raise BenchmarkError("Full Native calibration pair has no base frame") + ranked.append((frame_rows[pair], pair, timeframe, pair_document)) + _, probe_pair, _, pair_document = max(ranked, key=lambda item: (item[0], item[1])) + + literal_frames = _literal_frame_identities(source.parent, programs) + selected_frames = [ + frame + for frame in frames + if frame["identity"]["pair"] == probe_pair + or (frame["identity"]["pair"], frame["identity"]["timeframe"]) + in literal_frames + ] + probe = dict(document) + probe["pairs"] = [pair_document] + probe["frames"] = selected_frames + futures = document.get("futures") + if futures is None: + probe["futures"] = None + elif isinstance(futures, list): + if not all(isinstance(item, dict) for item in futures): + raise BenchmarkError("Full Native calibration futures inventory is invalid") + probe["futures"] = [item for item in futures if item.get("pair") == probe_pair] + else: + raise BenchmarkError("Full Native calibration futures inventory is invalid") + destination = _vacant_temporary(source.parent, ".full-native-probe-manifest.") + write_json(destination, probe) + return destination, probe_pair + + +def _literal_frame_identities(root: Path, programs: dict[str, Any]) -> set[tuple[str, str]]: + indicator = programs.get("indicator") + artifact = indicator.get("artifact") if isinstance(indicator, dict) else None + relative = artifact.get("path") if isinstance(artifact, dict) else None + if not isinstance(relative, str): + raise BenchmarkError("Full Native indicator artifact is missing") + program = read_json(root / relative) + nodes = program.get("nodes") if isinstance(program, dict) else None + if not isinstance(nodes, list): + raise BenchmarkError("Full Native indicator node inventory is invalid") + result: set[tuple[str, str]] = set() + for node in nodes: + if not isinstance(node, dict) or node.get("op") != "frame-source": + continue + parameters = node.get("parameters") + pair = parameters.get("pair") if isinstance(parameters, dict) else None + timeframe = parameters.get("timeframe") if isinstance(parameters, dict) else None + if isinstance(pair, dict) and pair.get("kind") == "literal": + value = pair.get("value") + if not isinstance(value, str) or not isinstance(timeframe, str): + raise BenchmarkError("Full Native literal frame binding is invalid") + result.add((value, timeframe)) + return result + + +def _vacant_temporary(directory: Path, prefix: str) -> Path: + with tempfile.NamedTemporaryFile( + prefix=prefix, + suffix=".json", + dir=directory, + delete=False, + ) as handle: + path = Path(handle.name) + path.unlink() + return path diff --git a/python/nfi_backtest_engine/full_vector_runtime.py b/python/nfi_backtest_engine/full_vector_runtime.py new file mode 100644 index 00000000..a8af79a8 --- /dev/null +++ b/python/nfi_backtest_engine/full_vector_runtime.py @@ -0,0 +1,573 @@ +"""Build the compact, SHA-bound input for complete Rust-native execution.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import shutil +import struct +import tempfile +from pathlib import Path +from typing import Any + +import pandas as pd + +from .canonical import read_json, write_json +from .errors import StrategyAnalysisError +from .fixture import sha256_file +from .indicator_program import compile_indicator_program +from .market_precision import historic_price_steps +from .signal_program import compile_signal_program +from .specs import FULL_NATIVE_VECTOR_MANIFEST_SCHEMA, validate_schema +from .tag_program import compile_tag_program +from .timerange import parse_timerange_milliseconds +from .vector_runtime import ( + PairFundingData, + resolve_pair_frames, + resolve_pair_funding_data, +) +from .x7.contracts import ( + _market_maximum_leverage, + _non_negative_float, + _optional_non_negative_float, + _positive_float, + _x7_funding_fee_interval_ms, + _x7_liquidation_contract, + x7_adapter_blockers, +) +from .x7.serialization import ( + _nfi_trade_manager_config, + _required_trade_features, + _x7_portfolio_config, +) + +FULL_NATIVE_VECTOR_MANIFEST_VERSION = "full-native-vector-manifest-v1" +FULL_NATIVE_VECTOR_RUNTIME_VERSION = "1.1.0" +FULL_NATIVE_SOURCE_MODE = "python-ast-compile-only" +FULL_NATIVE_RUNTIME_MODE = "rust-full-native" +_PROGRAM_NAMES = ("indicator", "signal", "tag") +_SIMULATOR_COLUMNS = frozenset({"open", "high", "low", "close", "volume"}) + + +def build_full_native_vector_manifest( + *, + strategy_path: str | Path, + class_name: str, + analysis: dict[str, Any], + hot_ir: dict[str, Any], + config: dict[str, Any], + pairs: list[str], + data_directory: str | Path, + timerange: str, + market_metadata_path: str | Path, + destination: str | Path, + compiled_programs: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Compile strategy programs and seal raw inputs without copying candle bytes.""" + target = Path(destination).resolve() + if target.exists(): + raise StrategyAnalysisError(f"full native manifest already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + artifact_root = target.parent / f"{target.stem}.artifacts" + if artifact_root.exists(): + raise StrategyAnalysisError( + f"full native artifact directory already exists: {artifact_root}" + ) + if not pairs or len(set(pairs)) != len(pairs) or any(not pair for pair in pairs): + raise StrategyAnalysisError("full native pairs must be non-empty and unique") + + blockers = x7_adapter_blockers( + analysis, + hot_ir, + config, + market_metadata_path=market_metadata_path, + ) + if blockers: + raise StrategyAnalysisError(blockers[0]["message"]) + + source = Path(strategy_path).resolve() + trading_mode = str(config.get("trading_mode", "spot")) + programs = ( + compiled_programs + if compiled_programs is not None + else compile_full_native_programs( + source, + class_name=class_name, + config=config, + ) + ) + if set(programs) != set(_PROGRAM_NAMES): + raise StrategyAnalysisError("compiled full native program set is incomplete") + indicator = programs["indicator"] + _validate_program_identity(programs, class_name=class_name, trading_mode=trading_mode) + + strategy = analysis["strategies"][0] + constants = strategy["constants"] + base_timeframe = constants.get("timeframe") + if not isinstance(base_timeframe, str) or not base_timeframe: + raise StrategyAnalysisError("strategy base timeframe is not a literal string") + raw_startup = constants.get("startup_candle_count", 0) + if isinstance(raw_startup, bool) or not isinstance(raw_startup, int) or raw_startup < 0: + raise StrategyAnalysisError("strategy startup_candle_count must be non-negative") + start_ms, stop_ms = parse_timerange_milliseconds(timerange) + + market_snapshot = read_json(market_metadata_path) + if not isinstance(market_snapshot, dict) or not isinstance( + market_snapshot.get("markets"), dict + ): + raise StrategyAnalysisError("market snapshot does not contain a markets object") + markets = market_snapshot["markets"] + required_timeframes = strategy.get("required_timeframes") + if not isinstance(required_timeframes, list) or not all( + isinstance(timeframe, str) and timeframe for timeframe in required_timeframes + ): + raise StrategyAnalysisError("strategy required timeframes are invalid") + + data_root = Path(data_directory).resolve() + data_index: dict[str, list[Path]] = {} + frame_sources: dict[tuple[str, str], Path] = {} + funding_sources: dict[str, PairFundingData] = {} + for pair in pairs: + resolved = resolve_pair_frames( + data_root, + pair=pair, + pairs=pairs, + timeframes=required_timeframes, + config=config, + data_index=data_index, + ) + for identity, path in resolved.items(): + frame_pair, separator, timeframe = identity.rpartition("|") + if not separator or not frame_pair or not timeframe: + raise StrategyAnalysisError(f"invalid resolved frame identity: {identity}") + key = (frame_pair, timeframe) + prior = frame_sources.setdefault(key, path.resolve()) + if prior != path.resolve(): + raise StrategyAnalysisError(f"resolved frame identity has two files: {identity}") + if trading_mode == "futures": + funding_sources[pair] = resolve_pair_funding_data( + data_root, + pair=pair, + data_index=data_index, + ) + + futures_execution = _shared_futures_execution(funding_sources) + funding_fee_interval_ms = _x7_funding_fee_interval_ms( + config, + {"futures_execution": futures_execution}, + ) + pair_documents, portfolio_config = _portfolio_contract( + analysis=analysis, + hot_ir=hot_ir, + config=config, + pairs=pairs, + markets=markets, + frame_sources=frame_sources, + base_timeframe=base_timeframe, + funding_fee_interval_ms=funding_fee_interval_ms, + market_snapshot=market_snapshot, + ) + retained_features = _retained_trade_features(hot_ir) + retained_fingerprint = _retained_feature_fingerprint(retained_features) + + staging = Path( + tempfile.mkdtemp(prefix=f".{artifact_root.name}.", dir=target.parent) + ).resolve() + installed = False + try: + program_documents = _write_program_artifacts( + programs, + staging=staging, + artifact_root_name=artifact_root.name, + ) + frame_documents = _seal_frame_sources( + frame_sources, + staging=staging, + artifact_root_name=artifact_root.name, + ) + futures_documents = _seal_futures_sources( + funding_sources, + staging=staging, + artifact_root_name=artifact_root.name, + ) + document = { + "schema_version": FULL_NATIVE_VECTOR_MANIFEST_VERSION, + "source": { + "strategy_sha256": indicator["source"]["sha256"], + "config_sha256": _config_identity_sha256(portfolio_config), + "compiler_source_fingerprint": _compiler_source_fingerprint(), + "selected_class": class_name, + }, + "source_execution": { + "strategy_source_mode": FULL_NATIVE_SOURCE_MODE, + "populate_methods_executed": False, + "runtime_mode": FULL_NATIVE_RUNTIME_MODE, + }, + "config": portfolio_config, + "compile_context": {"run_mode": "backtest", "trading_mode": trading_mode}, + "programs": program_documents, + "run": { + "trading_mode": trading_mode, + "timerange": {"start_ms": start_ms, "stop_ms": stop_ms}, + "startup_candles": raw_startup, + "base_timeframe": base_timeframe, + "source_row_shift": 1, + }, + "retained_features": { + "columns": retained_features, + "fingerprint": retained_fingerprint, + }, + "pairs": pair_documents, + "frames": frame_documents, + "futures": futures_documents or None, + } + validate_schema(document, FULL_NATIVE_VECTOR_MANIFEST_SCHEMA) + _canonical_json_bytes(document) + os.replace(staging, artifact_root) + installed = True + temporary_manifest = target.with_name(f".{target.name}.tmp") + try: + write_json(temporary_manifest, document) + os.replace(temporary_manifest, target) + except Exception: + temporary_manifest.unlink(missing_ok=True) + shutil.rmtree(artifact_root, ignore_errors=True) + raise + return document + finally: + if not installed: + shutil.rmtree(staging, ignore_errors=True) + + +def compile_full_native_programs( + strategy_path: str | Path, + *, + class_name: str, + config: dict[str, Any], +) -> dict[str, dict[str, Any]]: + """Compile the three independent programs once for policy and manifest stages.""" + source = Path(strategy_path).resolve() + trading_mode = str(config.get("trading_mode", "spot")) + programs = { + "indicator": compile_indicator_program(source, class_name=class_name, config=config), + "signal": compile_signal_program( + source, + class_name=class_name, + trading_mode=trading_mode, + config=config, + ), + "tag": compile_tag_program( + source, + class_name=class_name, + trading_mode=trading_mode, + config=config, + ), + } + _validate_program_identity(programs, class_name=class_name, trading_mode=trading_mode) + return programs + + +def _validate_program_identity( + programs: dict[str, dict[str, Any]], + *, + class_name: str, + trading_mode: str, +) -> None: + identities = { + (program["source"]["sha256"], program["selected_class"]) + for program in programs.values() + } + if identities != {(next(iter(programs.values()))["source"]["sha256"], class_name)}: + raise StrategyAnalysisError("compiled program source or selected class differs") + for name in ("signal", "tag"): + context = programs[name].get("compile_context") + if context != {"run_mode": "backtest", "trading_mode": trading_mode}: + raise StrategyAnalysisError(f"compiled {name} context differs from the run") + + +def _retained_trade_features(hot_ir: dict[str, Any]) -> list[str]: + """Keep callback columns not already represented by the candle contract.""" + return [ + name for name in _required_trade_features(hot_ir) if name not in _SIMULATOR_COLUMNS + ] + + +def _portfolio_contract( + *, + analysis: dict[str, Any], + hot_ir: dict[str, Any], + config: dict[str, Any], + pairs: list[str], + markets: dict[str, Any], + frame_sources: dict[tuple[str, str], Path], + base_timeframe: str, + funding_fee_interval_ms: int | None, + market_snapshot: dict[str, Any], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + can_short = config.get("trading_mode", "spot") == "futures" + pair_documents: list[dict[str, Any]] = [] + fee_rates: list[float] = [] + maximum_leverage_by_pair: dict[str, float] = {} + for pair in pairs: + market = markets.get(pair) + if not isinstance(market, dict): + raise StrategyAnalysisError(f"market snapshot is missing {pair}") + precision = market.get("precision") + limits = market.get("limits") + if not isinstance(precision, dict) or not isinstance(limits, dict): + raise StrategyAnalysisError(f"market precision or limits are missing for {pair}") + amount_limits = limits.get("amount") + cost_limits = limits.get("cost") + if not isinstance(amount_limits, dict) or not isinstance(cost_limits, dict): + raise StrategyAnalysisError(f"market amount or cost limits are missing for {pair}") + amount_step = _positive_float(precision.get("amount"), f"{pair} amount precision") + price_step = _positive_float(precision.get("price"), f"{pair} price precision") + fee_rates.append(_non_negative_float(config.get("fee", market.get("taker")), f"{pair} fee")) + maximum_leverage = _market_maximum_leverage(market, pair) + if maximum_leverage is not None: + maximum_leverage_by_pair[pair] = maximum_leverage + base_path = frame_sources.get((pair, base_timeframe)) + if base_path is None: + raise StrategyAnalysisError(f"resolved frames are missing {pair} {base_timeframe}") + precision_frame = pd.read_feather( + base_path, + columns=["date", "open", "high", "low", "close"], + ) + pair_documents.append( + { + "identity": {"pair": pair, "timeframe": base_timeframe}, + "metadata": {"pair": pair}, + "precision": {"amount_step": amount_step, "price_step": price_step}, + "limits": { + "minimum_stake": None, + "minimum_amount": _optional_non_negative_float( + amount_limits.get("min"), f"{pair} minimum amount" + ), + "minimum_cost": _optional_non_negative_float( + cost_limits.get("min"), f"{pair} minimum cost" + ), + }, + "price_steps": historic_price_steps(precision_frame), + "options": { + "can_short": can_short, + "include_funding": can_short, + "use_exit_signal": True, + "include_previous_close": True, + }, + } + ) + if any(rate != fee_rates[0] for rate in fee_rates[1:]): + raise StrategyAnalysisError("full native execution requires one exact fee") + nfi_manager = _nfi_trade_manager_config(hot_ir) + portfolio_config = _x7_portfolio_config( + analysis=analysis, + hot_ir=hot_ir, + config=config, + nfi_manager=nfi_manager, + fee_rate=fee_rates[0], + amount_step=pair_documents[0]["precision"]["amount_step"], + price_step=pair_documents[0]["precision"]["price_step"], + pair_count=len(pair_documents), + maximum_leverage_by_pair=maximum_leverage_by_pair, + funding_fee_interval_ms=funding_fee_interval_ms, + liquidation_model=_x7_liquidation_contract(config, market_snapshot, pairs), + ) + return pair_documents, portfolio_config + + +def _shared_futures_execution( + sources: dict[str, PairFundingData], +) -> dict[str, Any] | None: + shared: dict[str, Any] | None = None + for pair, source in sources.items(): + contract = source.execution_contract + if shared is None: + shared = contract + elif contract != shared: + raise StrategyAnalysisError( + f"selected Futures pairs use different funding execution contracts: {pair}" + ) + return shared + + +def _write_program_artifacts( + programs: dict[str, dict[str, Any]], + *, + staging: Path, + artifact_root_name: str, +) -> dict[str, Any]: + directory = staging / "programs" + directory.mkdir(parents=True) + result: dict[str, Any] = {} + for name in _PROGRAM_NAMES: + path = directory / f"{name}.json" + write_json(path, programs[name]) + result[name] = { + "artifact": { + "path": f"{artifact_root_name}/programs/{name}.json", + "sha256": sha256_file(path), + }, + "fingerprint": programs[name]["fingerprint"], + } + return result + + +def _seal_frame_sources( + sources: dict[tuple[str, str], Path], + *, + staging: Path, + artifact_root_name: str, +) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for index, ((pair, timeframe), source) in enumerate(sorted(sources.items())): + relative = f"frames/frame-{index:04d}.feather" + linked = _hardlink(source, staging / relative) + result.append( + { + "identity": {"pair": pair, "timeframe": timeframe}, + "rows": _feather_rows(linked), + "artifact": { + "path": f"{artifact_root_name}/{relative}", + "sha256": sha256_file(linked), + }, + } + ) + return result + + +def _seal_futures_sources( + sources: dict[str, PairFundingData], + *, + staging: Path, + artifact_root_name: str, +) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for index, (pair, source) in enumerate(sources.items()): + roles = { + "funding_rate": (source.funding_rate_path, source.funding_fee_timeframe), + "mark": (source.mark_path, source.mark_timeframe), + } + document: dict[str, Any] = {"pair": pair} + for role, (path, timeframe) in roles.items(): + relative = f"futures/futures-{index:04d}-{role}.feather" + linked = _hardlink(path, staging / relative) + document[role] = { + "identity": {"pair": pair, "timeframe": timeframe}, + "rows": _feather_rows(linked), + "artifact": { + "path": f"{artifact_root_name}/{relative}", + "sha256": sha256_file(linked), + }, + } + result.append(document) + return result + + +def _hardlink(source: Path, destination: Path) -> Path: + resolved = source.resolve() + if resolved.suffix.lower() != ".feather" or not resolved.is_file(): + raise StrategyAnalysisError(f"full native input must be a Feather file: {resolved}") + destination.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(resolved, destination) + except OSError as exc: + raise StrategyAnalysisError( + "cannot hard-link full native candle input; keep the run output on the same " + f"filesystem as the data directory: {resolved}" + ) from exc + if not os.path.samefile(resolved, destination): # pragma: no cover - OS contract + raise StrategyAnalysisError(f"hard-linked artifact identity differs: {resolved}") + return destination + + +def _feather_rows(path: Path) -> int: + frame = pd.read_feather(path, columns=["date"]) + return len(frame) + + +def _retained_feature_fingerprint(columns: list[str]) -> str: + digest = hashlib.sha256(b"full-native-retained-features-v1\0") + for column in columns: + encoded = column.encode() + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + +def _compiler_source_fingerprint() -> str: + package = Path(__file__).resolve().parent + paths = [ + package / "_indicator_ast.py", + package / "_indicator_contract.py", + package / "indicator_program.py", + *(package / "signal_program").glob("*.py"), + *(package / "tag_program").glob("*.py"), + ] + digest = hashlib.sha256(b"full-native-python-compilers-v1\0") + for path in sorted(paths, key=lambda item: item.relative_to(package).as_posix()): + relative = path.relative_to(package).as_posix().encode() + contents = path.read_bytes() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(len(contents).to_bytes(8, "big")) + digest.update(contents) + return digest.hexdigest() + + +def _config_identity_sha256(value: Any) -> str: + """Hash JSON data without depending on a language's float formatter.""" + digest = hashlib.sha256() + + def update(item: Any) -> None: + if item is None: + digest.update(b"N") + elif isinstance(item, bool): + digest.update(b"B\x01" if item else b"B\x00") + elif isinstance(item, int): + encoded = str(item).encode("ascii") + digest.update(b"I") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + elif isinstance(item, float): + if not math.isfinite(item): + raise StrategyAnalysisError("full native config contains a non-finite number") + digest.update(b"F") + digest.update(struct.pack(">d", item)) + elif isinstance(item, str): + encoded = item.encode("utf-8") + digest.update(b"S") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + elif isinstance(item, list): + digest.update(b"L") + digest.update(len(item).to_bytes(8, "big")) + for value_item in item: + update(value_item) + elif isinstance(item, dict): + if not all(isinstance(key, str) for key in item): + raise StrategyAnalysisError("full native config keys must be strings") + keys = sorted(item) + digest.update(b"O") + digest.update(len(keys).to_bytes(8, "big")) + for key in keys: + update(key) + update(item[key]) + else: + raise StrategyAnalysisError( + f"full native config contains unsupported {type(item).__name__} data" + ) + + update(value) + return digest.hexdigest() + + +def _canonical_json_bytes(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode() diff --git a/python/nfi_backtest_engine/indicator_program.py b/python/nfi_backtest_engine/indicator_program.py index f80af8f2..1baeebbc 100644 --- a/python/nfi_backtest_engine/indicator_program.py +++ b/python/nfi_backtest_engine/indicator_program.py @@ -3,16 +3,65 @@ from __future__ import annotations import ast -import copy import hashlib import json +import operator from collections.abc import Mapping, Sequence from dataclasses import dataclass -from functools import cache from pathlib import Path -from typing import Any, Never, cast - -from .errors import SpecValidationError, StrategyAnalysisError +from typing import Any, Never + +from ._indicator_ast import ( + _AGE_FILTER_TEMPLATE, + _FIRST_FIRE_TEMPLATE, + _INSIDE_BAR_TEMPLATE, + _NATIVE_HELPER_TEMPLATES, + _OPENING_RANGE_TEMPLATE, + _assigned_name, + _ast_equal, + _cast_target, + _dataframe_column_target, + _declared_class_constants, + _effective_backtest_config, + _flatten_binary_values, + _helper_bodies_equal, + _indicator_output_names, + _indicator_signature, + _is_absolute_difference_write, + _is_array_index_nan_write, + _is_json_value, + _is_shift_slice_assignment, + _literal_string, + _loop_target_names, + _normalized_static_value, + _parameter_type, + _qualified_name, + _recognized_tag_appender, + _safe_expression, + _small_static_candidate, + _template_function, + _template_statements, +) +from ._indicator_contract import ( + IndicatorProgramCompileError, + _add_finite_lookback, + _array_call_result_type, + _array_result_type, + _boolean_result_type, + _cast_result_type, + _fingerprint, + _literal_parameters, + _literal_type, + _location, + _merge_lookbacks, + _merge_value_types, + _numeric_identifier_key, + _numeric_result_type, + _program_lookback, + _unsupported, + _zero_lookback, +) +from .errors import SpecValidationError from .specs import INDICATOR_PROGRAM_SCHEMA, validate_schema from .strategy_ir import analyze_strategy @@ -65,22 +114,60 @@ } -class IndicatorProgramCompileError(StrategyAnalysisError): - """Indicator source cannot be represented by the causal v1 DAG.""" - - @dataclass(frozen=True) class _CallableRef: name: str -Binding = str | _CallableRef +@dataclass(frozen=True) +class _StaticBinding: + value: Any + + +@dataclass(frozen=True) +class _LambdaRef: + parameters: tuple[str, ...] + body: ast.expr + + +@dataclass +class _SequenceBinding: + items: list[Any] + + +@dataclass(frozen=True) +class _DataProviderRef: + pass + + +@dataclass(frozen=True) +class _ColumnBundleBinding: + dataframe: str + columns: tuple[tuple[str, str], ...] + + +@dataclass +class _MappingBinding: + items: dict[Any, Any] + + +Binding = ( + str + | _CallableRef + | _StaticBinding + | _LambdaRef + | _SequenceBinding + | _DataProviderRef + | _ColumnBundleBinding + | _MappingBinding +) def compile_indicator_program( source: str | Path, *, class_name: str | None = None, + config: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Compile ``populate_indicators`` and its bounded helpers without execution.""" path = Path(source).resolve() @@ -108,6 +195,9 @@ def compile_indicator_program( for node in class_node.body if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) } + module_functions = { + node.name: node for node in tree.body if isinstance(node, ast.FunctionDef) + } entrypoint = methods.get("populate_indicators") if entrypoint is None: raise IndicatorProgramCompileError("strategy does not define populate_indicators") @@ -115,10 +205,21 @@ def compile_indicator_program( _unsupported(entrypoint, "async indicator entrypoint") constants = strategy.get("constants", {}) + class_constants = _declared_class_constants( + class_node, + constants if isinstance(constants, Mapping) else {}, + ) + instance_constants: dict[str, Any] = { + "dp": {"runmode": {"value": "backtest"}}, + } + if config is not None: + instance_constants["config"] = _effective_backtest_config(config) compiler = _Compiler( path=path, methods=methods, - class_constants=constants if isinstance(constants, Mapping) else {}, + class_constants=class_constants, + instance_constants=instance_constants, + module_functions=module_functions, ) entrypoint_id = compiler.compile_method("populate_indicators", kind="entrypoint") program: dict[str, Any] = { @@ -229,14 +330,21 @@ def __init__( path: Path, methods: Mapping[str, ast.FunctionDef | ast.AsyncFunctionDef], class_constants: Mapping[str, Any], + instance_constants: Mapping[str, Any] | None = None, + module_functions: Mapping[str, ast.FunctionDef] | None = None, ) -> None: self.path = path self.methods = methods self.class_constants = class_constants + self.instance_constants = dict(instance_constants or {}) + self.module_functions = dict(module_functions or {}) self.method_ids: dict[str, str] = {} + self.specialized_method_ids: dict[tuple[str, str], str] = {} self.compiling: set[str] = set() self.compiled: set[str] = set() + self.compiled_specializations: set[tuple[str, str]] = set() self.function_return_types: dict[str, str] = {} + self.function_static_returns: dict[str, Any] = {} self.function_arities: dict[str, int] = {} self.function_lookbacks: dict[str, dict[str, Any]] = {} self.functions: list[dict[str, Any]] = [] @@ -252,8 +360,33 @@ def __init__( self.bindings: dict[str, Binding] = {} self.return_node: str | None = None - def compile_method(self, name: str, *, kind: str = "helper") -> str: - if name in self.compiled: + def compile_method( + self, + name: str, + *, + kind: str = "helper", + static_arguments: Mapping[str, Any] | None = None, + callable_arguments: Mapping[str, _CallableRef] | None = None, + ) -> str: + normalized_static = { + key: _normalized_static_value(value) + for key, value in (static_arguments or {}).items() + } + callable_signature = { + key: value.name for key, value in (callable_arguments or {}).items() + } + specialization = ( + name, + json.dumps( + {"static": normalized_static, "callable": callable_signature}, + sort_keys=True, + separators=(",", ":"), + ), + ) + is_specialized = bool(normalized_static or callable_signature) + if is_specialized and specialization in self.compiled_specializations: + return self.specialized_method_ids[specialization] + if not is_specialized and name in self.compiled: return self.method_ids[name] node = self.methods.get(name) if node is None: @@ -262,7 +395,13 @@ def compile_method(self, name: str, *, kind: str = "helper") -> str: self.unsupported(node, "async indicator helper") if name in self.compiling: self.unsupported(node, "recursive indicator helper") - function_id = self.method_ids.setdefault(name, f"f{len(self.method_ids) + 1}") + if is_specialized: + function_id = self.specialized_method_ids.setdefault( + specialization, + self._next_function_id(), + ) + else: + function_id = self.method_ids.setdefault(name, self._next_function_id()) self.compiling.add(name) previous_state = ( @@ -284,6 +423,12 @@ def compile_method(self, name: str, *, kind: str = "helper") -> str: for argument in parameters: if argument.arg == "self": continue + if argument.arg in normalized_static: + self.bindings[argument.arg] = _StaticBinding(normalized_static[argument.arg]) + continue + if argument.arg in (callable_arguments or {}): + self.bindings[argument.arg] = (callable_arguments or {})[argument.arg] + continue value_type = _parameter_type(argument.arg) parameter_node = self.emit( node, @@ -299,8 +444,7 @@ def compile_method(self, name: str, *, kind: str = "helper") -> str: "value_type": value_type, } ) - for statement in node.body: - self.statement(statement) + self.statements(node.body) if self.return_node is None: self.unsupported(node, "indicator function without an explicit return") function_record = { @@ -313,24 +457,52 @@ def compile_method(self, name: str, *, kind: str = "helper") -> str: } return_type = self.node_types[self.return_node] return_lookback = self.lookback(self.return_node) + static_return = self.node_static_value(self.return_node) self.current_function, self.current_nodes, self.bindings, self.return_node = previous_state self.function_return_types[function_id] = return_type self.function_arities[function_id] = len(parameter_records) self.function_lookbacks[function_id] = return_lookback + if static_return[0]: + self.function_static_returns[function_id] = static_return[1] self.functions.append(function_record) self.compiling.remove(name) - self.compiled.add(name) + if is_specialized: + self.compiled_specializations.add(specialization) + else: + self.compiled.add(name) return function_id + def _next_function_id(self) -> str: + identifiers = [*self.method_ids.values(), *self.specialized_method_ids.values()] + next_index = max((int(identifier[1:]) for identifier in identifiers), default=0) + 1 + return f"f{next_index}" + def statement(self, node: ast.stmt) -> None: if isinstance(node, ast.Assign) and len(node.targets) == 1: target = node.targets[0] if isinstance(target, ast.Name): callable_ref = self.callable_reference(node.value) - self.bindings[target.id] = callable_ref or self.expression(node.value) + lambda_ref = self.lambda_reference(node.value) + sequence_ref = self.sequence_reference(node.value) + provider_ref = self.data_provider_reference(node.value) + bundle_ref = self.column_bundle_reference(node.value) + mapping_ref = self.mapping_reference(node.value) + static_ref = self.static_reference(node.value) + self.bindings[target.id] = ( + callable_ref + or lambda_ref + or sequence_ref + or provider_ref + or bundle_ref + or mapping_ref + or static_ref + or self.expression(node.value) + ) return if isinstance(target, ast.Subscript): + if self.mapping_write(target, node.value): + return self.column_write(target, node.value, node) return if isinstance(target, ast.Tuple | ast.List) and isinstance(node.value, ast.Call): @@ -354,20 +526,365 @@ def statement(self, node: ast.stmt) -> None: ) return if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): + if self.append_sequence(node.value): + return + if self.inplace_forward_fill(node.value): + return self.expression(node.value) return + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + return if isinstance(node, ast.If): + if self.frame_empty_guard(node): + return + if self.frame_drop_guard(node): + return condition = self.static_value(node.test) if not isinstance(condition, bool): self.unsupported(node, "dynamic indicator control-flow if") selected = node.body if condition else node.orelse - for statement in selected: - self.statement(statement) + self.statements(selected) + return + if isinstance(node, ast.Assert): + found, condition = self.try_static_value(node.test) + if not found: + self.unsupported(node, "dynamic indicator assertion") + if not bool(condition): + self.unsupported(node, "failed static indicator assertion") + return + if isinstance(node, ast.For): + iterations = self.static_loop_iterations(node) + if iterations is None: + self.unsupported(node.iter, "dynamic indicator loop iterable") + target_names = _loop_target_names(node.target) + previous = {name: self.bindings.get(name) for name in target_names} + present = {name for name in target_names if name in self.bindings} + for values in iterations: + for name, value in zip(target_names, values, strict=True): + self.bindings[name] = value + self.statements(node.body) + if node.orelse: + self.statements(node.orelse) + for name in target_names: + if name in present: + prior = previous[name] + assert prior is not None + self.bindings[name] = prior + else: + self.bindings.pop(name, None) return if isinstance(node, ast.Pass): return self.unsupported(node, f"indicator statement {type(node).__name__}") + def statements(self, statements: Sequence[ast.stmt]) -> None: + index = 0 + while index < len(statements): + consumed = self.absolute_difference_block(statements, index) + if not consumed: + consumed = self.age_filter_block(statements, index) + if not consumed: + consumed = self.opening_range_block(statements, index) + if not consumed: + consumed = self.inside_bar_block(statements, index) + if not consumed: + consumed = self.first_fire_block(statements, index) + if consumed: + index += consumed + continue + self.statement(statements[index]) + index += 1 + + def absolute_difference_block( + self, + statements: Sequence[ast.stmt], + index: int, + ) -> int: + if index + 2 >= len(statements): + return 0 + allocated, warmup, difference = statements[index : index + 3] + if not ( + isinstance(allocated, ast.Assign) + and len(allocated.targets) == 1 + and isinstance(allocated.targets[0], ast.Name) + and isinstance(allocated.value, ast.Call) + and self.resolved_callable(allocated.value.func) == "np.empty_like" + and len(allocated.value.args) == 1 + and len(allocated.value.keywords) == 1 + and allocated.value.keywords[0].arg == "dtype" + and _qualified_name(allocated.value.keywords[0].value) == "np.float64" + ): + return 0 + output_name = allocated.targets[0].id + source_node = allocated.value.args[0] + if not _is_array_index_nan_write(warmup, output_name, 0): + return 0 + if not _is_absolute_difference_write(difference, output_name, source_node): + return 0 + source = self.expression(source_node) + if self.node_types[source] != "f64-column": + self.unsupported(source_node, "absolute difference source type") + self.bindings[output_name] = self.emit( + allocated, + "array-call", + "f64-column", + inputs=[source], + parameters={ + "family": "numpy", + "name": "absolute-difference", + "arguments": {}, + }, + lookback=_add_finite_lookback(self.lookback(source), 1), + ) + return 3 + + def age_filter_block( + self, + statements: Sequence[ast.stmt], + index: int, + ) -> int: + expected = _template_statements(_AGE_FILTER_TEMPLATE) + candidate = statements[index : index + len(expected)] + if not ( + len(candidate) == len(expected) + and isinstance(candidate[0], ast.Assign) + and len(candidate[0].targets) == 1 + and _dataframe_column_target(candidate[0].targets[0]) + == ("df", "bt_agefilter_ok") + ): + return 0 + if not _ast_equal( + ast.Module(body=list(candidate), type_ignores=[]), + ast.Module(body=list(expected), type_ignores=[]), + ): + return 0 + frame = self.bindings.get("df") + age_days = self.class_constants.get("bt_min_age_days") + if not isinstance(frame, str) or self.node_types[frame] != "dataframe": + self.unsupported(candidate[0], "age-filter dataframe") + if not isinstance(age_days, int) or isinstance(age_days, bool) or age_days < 0: + self.unsupported(candidate[1], "age-filter day count") + row_index = self.emit( + candidate[1], + "row-index", + "int-column", + inputs=[frame], + lookback=self.lookback(frame), + ) + threshold = self.literal(candidate[1], 12 * 24 * age_days) + enabled = self.emit( + candidate[1], + "compare", + "bool-column", + inputs=[row_index, threshold], + parameters={"operator": "greater-than"}, + lookback=self.merged_lookback([row_index, threshold]), + ) + written = self.emit( + candidate[-1], + "column-write", + "dataframe", + inputs=[frame, enabled], + parameters={"column": "bt_agefilter_ok"}, + lookback=self.merged_lookback([frame, enabled]), + ) + self.bindings["df"] = written + self.produced_columns.add("bt_agefilter_ok") + return len(expected) + + def opening_range_block( + self, + statements: Sequence[ast.stmt], + index: int, + ) -> int: + expected = _template_statements(_OPENING_RANGE_TEMPLATE) + candidate = statements[index : index + len(expected)] + if not ( + len(candidate) == len(expected) + and _assigned_name(candidate[0]) == "_or_day" + ): + return 0 + if not _ast_equal( + ast.Module(body=list(candidate), type_ignores=[]), + ast.Module(body=list(expected), type_ignores=[]), + ): + return 0 + frame = self.bindings.get("df") + if not isinstance(frame, str) or self.node_types[frame] != "dataframe": + self.unsupported(candidate[0], "opening-range dataframe") + source_nodes = [ + ast.Subscript( + value=ast.Name(id="df", ctx=ast.Load()), + slice=ast.Constant(value=name), + ctx=ast.Load(), + ) + for name in ("date", "high", "low") + ] + for source in source_nodes: + ast.copy_location(source, candidate[0]) + date, high, low = [self.expression(source) for source in source_nodes] + for output_name, operation in ( + ("orange_h_col", "opening-range-high"), + ("orange_l_col", "opening-range-low"), + ): + self.bindings[output_name] = self.emit( + candidate[-1], + "array-call", + "f64-column", + inputs=[date, high, low], + parameters={ + "family": "native", + "name": operation, + "arguments": {"cutoff_hour": 4}, + }, + lookback={ + "kind": "function-defined", + "candles": None, + "expression": operation, + "causal": True, + }, + ) + return len(expected) + + def inside_bar_block( + self, + statements: Sequence[ast.stmt], + index: int, + ) -> int: + expected = _template_statements(_INSIDE_BAR_TEMPLATE) + candidate = statements[index : index + len(expected)] + if not ( + len(candidate) == len(expected) + and _assigned_name(candidate[0]) == "_ib_hr" + ): + return 0 + if not _ast_equal( + ast.Module(body=list(candidate), type_ignores=[]), + ast.Module(body=list(expected), type_ignores=[]), + ): + return 0 + frame = self.bindings.get("df") + if not isinstance(frame, str) or self.node_types[frame] != "dataframe": + self.unsupported(candidate[0], "inside-bar dataframe") + source_nodes = [ + ast.Subscript( + value=ast.Name(id="df", ctx=ast.Load()), + slice=ast.Constant(value=name), + ctx=ast.Load(), + ) + for name in ("date", "high", "low") + ] + for source in source_nodes: + ast.copy_location(source, candidate[0]) + date, high, low = [self.expression(source) for source in source_nodes] + for output_name, operation in ( + ("ib_ready_col", "inside-bar-ready"), + ("ib_mother_h_col", "inside-bar-mother-high"), + ("ib_mother_l_col", "inside-bar-mother-low"), + ): + self.bindings[output_name] = self.emit( + candidate[-1], + "array-call", + "f64-column", + inputs=[date, high, low], + parameters={"family": "native", "name": operation, "arguments": {}}, + lookback={ + "kind": "function-defined", + "candles": None, + "expression": operation, + "causal": True, + }, + ) + return len(expected) + + def first_fire_block( + self, + statements: Sequence[ast.stmt], + index: int, + ) -> int: + expected = _template_statements(_FIRST_FIRE_TEMPLATE) + candidate = statements[index : index + len(expected)] + if not ( + len(candidate) == len(expected) + and isinstance(candidate[0], ast.Assign) + and len(candidate[0].targets) == 1 + and isinstance(candidate[0].targets[0], ast.Name) + and candidate[0].targets[0].id == "_ph_cross" + ): + return 0 + if not _ast_equal( + ast.Module(body=list(candidate), type_ignores=[]), + ast.Module(body=list(expected), type_ignores=[]), + ): + return 0 + close = self.bindings.get("close_np") + previous_max = self.bindings.get("_ph_prev_max") + if not isinstance(close, str) or self.node_types[close] != "f64-column": + self.unsupported(candidate[0], "first-fire close column") + if not isinstance(previous_max, str) or self.node_types[previous_max] != "f64-column": + self.unsupported(candidate[0], "first-fire previous maximum") + shifted = self.emit( + candidate[0], + "shift", + "f64-column", + inputs=[close], + parameters={"periods": 1}, + lookback=_add_finite_lookback(self.lookback(close), 1), + ) + current_break = self.emit( + candidate[0], + "compare", + "bool-column", + inputs=[close, previous_max], + parameters={"operator": "greater-than"}, + lookback=self.merged_lookback([close, previous_max]), + ) + previous_below = self.emit( + candidate[0], + "compare", + "bool-column", + inputs=[shifted, previous_max], + parameters={"operator": "less-than-or-equal"}, + lookback=self.merged_lookback([shifted, previous_max]), + ) + crossed = self.emit( + candidate[0], + "logical", + "bool-column", + inputs=[current_break, previous_below], + parameters={"operator": "and"}, + lookback=self.merged_lookback([current_break, previous_below]), + ) + cross_float = self.emit( + candidate[0], + "cast", + "f64-column", + inputs=[crossed], + parameters={"target": "float"}, + lookback=self.lookback(crossed), + ) + self.bindings["_ph_cross"] = cross_float + counted = self.emit( + candidate[-1], + "window", + "f64-column", + inputs=[cross_float], + parameters={ + "kind": "rolling", + "reducer": "sum", + "window": 12, + "center": False, + "min_periods": None, + }, + lookback=_add_finite_lookback(self.lookback(cross_float), 11), + ) + self.bindings["ph_cross_cnt12_col"] = counted + return len(expected) + def column_write(self, target: ast.Subscript, value_node: ast.expr, node: ast.AST) -> None: if not isinstance(target.value, ast.Name): self.unsupported(target, "nested dataframe write") @@ -403,18 +920,30 @@ def expression(self, node: ast.expr) -> str: return binding if isinstance(binding, _CallableRef): self.unsupported(node, "callable used as an indicator value") + if isinstance(binding, _LambdaRef): + self.unsupported(node, "lambda used as an indicator value") + if isinstance(binding, _SequenceBinding): + self.unsupported(node, "sequence used as an indicator value") + if isinstance(binding, _DataProviderRef): + self.unsupported(node, "data provider used as an indicator value") + if isinstance(binding, _ColumnBundleBinding): + self.unsupported(node, "column bundle used as an indicator value") + if isinstance(binding, _MappingBinding): + self.unsupported(node, "mapping used as an indicator value") + if isinstance(binding, _StaticBinding): + return self.literal(node, binding.value) if node.id in self.class_constants: return self.literal(node, self.class_constants[node.id]) self.unsupported(node, f"unknown indicator value {node.id}") if isinstance(node, ast.Attribute): - if ( - isinstance(node.value, ast.Name) - and node.value.id == "self" - and node.attr in self.class_constants - ): - return self.literal(node, self.class_constants[node.attr]) + found, value = self.try_static_value(node) + if found: + return self.literal(node, value) self.unsupported(node, "attribute value") if isinstance(node, ast.Subscript): + found, value = self.try_static_value(node) + if found: + return self.literal(node, value) return self.subscript(node) if isinstance(node, ast.BinOp): return self.binary(node) @@ -437,12 +966,18 @@ def expression(self, node: ast.expr) -> str: lookback=self.lookback(value), ) if isinstance(node, ast.IfExp): + found, value = self.try_static_value(node) + if found: + return self.literal(node, value) return self.select(node, node.test, node.body, node.orelse) if isinstance(node, ast.Call): + found, value = self.try_static_value(node) + if found: + return self.literal(node, value) return self.call(node) - if isinstance(node, ast.List | ast.Tuple | ast.Dict): - value = _static_value(node, self.class_constants) - if value is not None: + if isinstance(node, ast.JoinedStr | ast.List | ast.Tuple | ast.Set | ast.Dict): + found, value = self.try_static_value(node) + if found: return self.literal(node, value) self.unsupported(node, f"indicator expression {type(node).__name__}") @@ -453,11 +988,20 @@ def subscript(self, node: ast.Subscript) -> str: and isinstance(node.slice.value, int) and not isinstance(node.slice.value, bool) ): - return self.indicator_output(node.value, node.slice.value, node) + callable_name = self.resolved_callable(node.value.func) + if _indicator_output_names(callable_name) is not None: + return self.indicator_output(node.value, node.slice.value, node) + indexed_string = self.indexed_string_call( + node.value, + index=node.slice.value, + source_node=node, + ) + if indexed_string is not None: + return indexed_string base = self.expression(node.value) base_type = self.node_types[base] - key = _literal_string(node.slice) - if key is None: + found_key, key = self.try_static_value(node.slice) + if not found_key or not isinstance(key, str): self.unsupported(node, "dynamic dataframe or metadata subscript") if base_type == "dataframe": value_type = "timestamp-column" if key == "date" else "f64-column" @@ -482,10 +1026,48 @@ def subscript(self, node: ast.Subscript) -> str: ) self.unsupported(node, "subscript source type") + def indexed_string_call( + self, + call: ast.Call, + *, + index: int, + source_node: ast.AST, + ) -> str | None: + if not isinstance(call.func, ast.Attribute) or call.func.attr not in { + "partition", + "split", + "rsplit", + }: + return None + if len(call.args) != 1 or call.keywords: + self.unsupported(call, "indexed string call signature") + found, separator = self.try_static_value(call.args[0]) + if not found or not isinstance(separator, str) or not separator: + self.unsupported(call.args[0], "indexed string separator") + base = self.expression(call.func.value) + if self.node_types[base] != "string-scalar": + self.unsupported(call.func.value, "indexed string source type") + return self.emit( + source_node, + "string-split-index", + "string-scalar", + inputs=[base], + parameters={ + "method": call.func.attr, + "separator": separator, + "index": index, + }, + lookback=self.lookback(base), + ) + def binary(self, node: ast.BinOp) -> str: logical = _LOGICAL_BINARY_OPS.get(type(node.op)) if logical is not None: - return self.logical(node, type(node.op), values=[node.left, node.right]) + return self.logical( + node, + type(node.op), + values=_flatten_binary_values(node, type(node.op)), + ) operator = _BINARY_OPS.get(type(node.op)) if operator is None: self.unsupported(node, "binary indicator operator") @@ -505,6 +1087,29 @@ def compare(self, node: ast.Compare) -> str: left_node = node.left comparisons = [] for operator_node, right_node in zip(node.ops, node.comparators, strict=True): + if isinstance(operator_node, ast.In | ast.NotIn): + found, collection = self.try_static_value(right_node) + if not found or not isinstance(collection, list | tuple | Mapping): + self.unsupported(right_node, "dynamic membership collection") + left = self.expression(left_node) + values = list(collection) + if not all(_is_json_value(value) for value in values): + self.unsupported(right_node, "non-JSON membership collection") + comparisons.append( + self.emit( + node, + "membership", + _boolean_result_type(self.node_types[left]), + inputs=[left], + parameters={ + "values": [_normalized_static_value(value) for value in values], + "negated": isinstance(operator_node, ast.NotIn), + }, + lookback=self.lookback(left), + ) + ) + left_node = right_node + continue operator = _COMPARE_OPS.get(type(operator_node)) if operator is None: self.unsupported(node, "comparison indicator operator") @@ -585,7 +1190,19 @@ def call(self, node: ast.Call) -> str: window = self.window_call(node) if window is not None: return window + lambda_value = self._inline_lambda_call(node) + if lambda_value is not None: + return lambda_value callable_name = self.resolved_callable(node.func) + frame_source = self.frame_source_call(node) + if frame_source is not None: + return frame_source + sequence_value = self.reduce_sequence_call(node, callable_name) + if sequence_value is not None: + return sequence_value + tag_value = self.append_tag_call(node, callable_name) + if tag_value is not None: + return tag_value if callable_name == "time.perf_counter": return self.emit( node, @@ -600,13 +1217,104 @@ def call(self, node: ast.Call) -> str: "null", parameters={"name": callable_name}, ) + concatenated = self.concat_column_bundle(node, callable_name) + if concatenated is not None: + return concatenated + if callable_name == "len": + if len(node.args) != 1 or node.keywords: + self.unsupported(node, "len signature") + value = self.expression(node.args[0]) + if self.node_types[value] != "dataframe": + self.unsupported(node.args[0], "len source type") + return self.emit( + node, + "row-count", + "int-scalar", + inputs=[value], + lookback=self.lookback(value), + ) if callable_name.startswith("self."): method_name = callable_name.removeprefix("self.") - function_id = self.compile_method(method_name) - if node.keywords or len(node.args) != self.function_arities[function_id]: + method = self.methods.get(method_name) + if method is None or isinstance(method, ast.AsyncFunctionDef): + self.unsupported(node, "indicator helper call target") + bound = _bind_helper_arguments(node, method, self) + shift = _normalized_shift_helper(method, bound, self) + if shift is not None: + source_node, periods = shift + source = self.expression(source_node) + if not self.node_types[source].endswith("-column"): + self.unsupported(source_node, "shift helper source type") + return self.emit( + node, + "shift", + self.node_types[source], + inputs=[source], + parameters={"periods": periods}, + lookback=_add_finite_lookback(self.lookback(source), periods), + ) + projection = _normalized_frame_projection_helper(method, bound, self) + if projection is not None: + source_node, parameters = projection + source = self.expression(source_node) + if self.node_types[source] != "dataframe": + self.unsupported(source_node, "frame projection source type") + return self.emit( + node, + "frame-project", + "dataframe", + inputs=[source], + parameters=parameters, + lookback=self.lookback(source), + ) + native_indicator = _normalized_native_indicator_helper(method, bound, self) + if native_indicator is not None: + name, argument_nodes, parameters = native_indicator + inputs = [self.expression(argument) for argument in argument_nodes] + if any(not self.node_types[value].endswith("-column") for value in inputs): + self.unsupported(node, "native indicator input type") + return self.emit( + node, + "indicator-call", + "f64-column", + inputs=inputs, + parameters={ + "family": "native", + "name": name, + "arguments": parameters, + }, + lookback={ + "kind": "function-defined", + "candles": parameters.get("timeperiod", 1) - 1, + "expression": name, + "causal": True, + }, + ) + static_arguments: dict[str, Any] = {} + callable_arguments: dict[str, _CallableRef] = {} + dynamic_arguments: list[ast.expr] = [] + for name, argument in bound: + if isinstance(argument, ast.Name): + argument_binding = self.bindings.get(argument.id) + if isinstance(argument_binding, _CallableRef): + callable_arguments[name] = argument_binding + continue + found, value = self.try_static_value(argument) + if found: + static_arguments[name] = value + else: + dynamic_arguments.append(argument) + function_id = self.compile_method( + method_name, + static_arguments=static_arguments, + callable_arguments=callable_arguments, + ) + if len(dynamic_arguments) != self.function_arities[function_id]: self.unsupported(node, "indicator helper call signature") - inputs = [self.expression(argument) for argument in node.args] + inputs = [self.expression(argument) for argument in dynamic_arguments] value_type = self.function_return_types[function_id] + if not inputs and function_id in self.function_static_returns: + return self.literal(node, self.function_static_returns[function_id]) return self.emit( node, "function-call", @@ -641,20 +1349,7 @@ def call(self, node: ast.Call) -> str: if callable_name.startswith("ta.") or callable_name.startswith("qtpylib."): return self.indicator_call(node) if callable_name.startswith("np."): - inputs = [self.expression(argument) for argument in node.args] - parameters = _literal_keyword_arguments(node, self) - return self.emit( - node, - "array-call", - _array_result_type(inputs, self.node_types), - inputs=inputs, - parameters={ - "family": "numpy", - "name": callable_name.removeprefix("np."), - "arguments": parameters, - }, - lookback=self.merged_lookback(inputs), - ) + return self.array_call(node, callable_name) if callable_name in _SCALAR_CALLS: if node.keywords: self.unsupported(node, "scalar indicator keyword arguments") @@ -673,6 +1368,15 @@ def call(self, node: ast.Call) -> str: def multi_output_assignment(self, target: ast.Tuple | ast.List, node: ast.Call) -> None: callable_name = self.resolved_callable(node.func) + inlined = self.inline_tuple_helper_call(node, callable_name) + if inlined is not None: + if len(inlined) != len(target.elts): + self.unsupported(node, "indicator helper tuple output arity") + for element, value in zip(target.elts, inlined, strict=True): + if not isinstance(element, ast.Name): + self.unsupported(element, "indicator helper tuple assignment target") + self.bindings[element.id] = value + return output_names = _indicator_output_names(callable_name) if output_names is None or len(output_names) != len(target.elts): self.unsupported(node, "indicator tuple output contract") @@ -688,6 +1392,112 @@ def multi_output_assignment(self, target: ast.Tuple | ast.List, node: ast.Call) output=output_name, ) + def inline_tuple_helper_call( + self, + call: ast.Call, + callable_name: str, + ) -> list[str] | None: + if not callable_name.startswith("self."): + return None + method = self.methods.get(callable_name.removeprefix("self.")) + if method is None or isinstance(method, ast.AsyncFunctionDef): + return None + body = method.body + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + body = body[1:] + if not ( + body + and isinstance(body[-1], ast.Return) + and isinstance(body[-1].value, ast.Tuple | ast.List) + and all(isinstance(statement, ast.Assign | ast.AnnAssign) for statement in body[:-1]) + ): + return None + bound = _bind_helper_arguments(call, method, self) + argument_bindings: dict[str, Binding] = {} + for name, argument in bound: + if isinstance(argument, ast.Name): + existing = self.bindings.get(argument.id) + if isinstance(existing, _CallableRef): + argument_bindings[name] = existing + continue + found, value = self.try_static_value(argument) + argument_bindings[name] = ( + _StaticBinding(value) if found else self.expression(argument) + ) + previous = self.bindings + self.bindings = argument_bindings + try: + self.statements(body[:-1]) + returned = body[-1] + assert isinstance(returned, ast.Return) + assert isinstance(returned.value, ast.Tuple | ast.List) + return [self.expression(value) for value in returned.value.elts] + finally: + self.bindings = previous + + def array_call(self, node: ast.Call, callable_name: str) -> str: + name = callable_name.removeprefix("np.") + if name == "full_like": + if len(node.args) != 2 or node.keywords: + self.unsupported(node, "numpy full_like signature") + inputs = [self.expression(argument) for argument in node.args] + value_type = self.node_types[inputs[0]] + if not value_type.endswith("-column"): + self.unsupported(node.args[0], "numpy full_like template type") + return self.emit( + node, + "array-call", + value_type, + inputs=inputs, + parameters={"family": "numpy", "name": name, "arguments": {}}, + lookback=self.merged_lookback(inputs), + ) + if name == "divide": + if len(node.args) != 2: + self.unsupported(node, "numpy divide signature") + options: dict[str, ast.expr] = {} + for keyword in node.keywords: + if keyword.arg not in {"out", "where"}: + self.unsupported(keyword.value, "numpy divide keyword arguments") + if keyword.arg in options: + self.unsupported(keyword.value, f"duplicate numpy divide {keyword.arg}") + options[str(keyword.arg)] = keyword.value + if set(options) != {"out", "where"}: + self.unsupported(node, "numpy divide requires explicit out and where") + inputs = [ + self.expression(node.args[0]), + self.expression(node.args[1]), + self.expression(options["out"]), + self.expression(options["where"]), + ] + if any(self.node_types[item] != "f64-column" for item in inputs[:3]): + self.unsupported(node, "numpy divide numeric column types") + if self.node_types[inputs[3]] != "bool-column": + self.unsupported(options["where"], "numpy divide where mask type") + return self.emit( + node, + "array-call", + "f64-column", + inputs=inputs, + parameters={"family": "numpy", "name": name, "arguments": {}}, + lookback=self.merged_lookback(inputs), + ) + inputs = [self.expression(argument) for argument in node.args] + parameters = _literal_keyword_arguments(node, self) + return self.emit( + node, + "array-call", + _array_call_result_type(callable_name, inputs, self.node_types), + inputs=inputs, + parameters={"family": "numpy", "name": name, "arguments": parameters}, + lookback=self.merged_lookback(inputs), + ) + def indicator_output(self, call: ast.Call, index: int, source_node: ast.AST) -> str: callable_name = self.resolved_callable(call.func) output_names = _indicator_output_names(callable_name) @@ -717,10 +1527,24 @@ def indicator_call_parts( ) -> tuple[list[str], dict[str, Any]]: if not callable_name.startswith(("ta.", "qtpylib.")): self.unsupported(node, "indexed value is not an indicator call") - return ( - [self.expression(argument) for argument in node.args], - _literal_keyword_arguments(node, self), - ) + signature = _indicator_signature(callable_name) + if signature is None: + return ( + [self.expression(argument) for argument in node.args], + _literal_keyword_arguments(node, self), + ) + input_count, parameter_names = signature + if len(node.args) < input_count or len(node.args) > input_count + len(parameter_names): + self.unsupported(node, "indicator positional signature") + inputs = [self.expression(argument) for argument in node.args[:input_count]] + arguments = _literal_keyword_arguments(node, self) + if any(name not in parameter_names for name in arguments): + self.unsupported(node, "unknown indicator keyword argument") + for name, argument in zip(parameter_names, node.args[input_count:], strict=False): + if name in arguments: + self.unsupported(argument, "duplicate indicator argument") + arguments[name] = _required_static(argument, self) + return inputs, arguments def emit_indicator_call( self, @@ -758,6 +1582,20 @@ def method_call(self, node: ast.Call, callable_name: str) -> str: self.unsupported(node, "indicator method call") method = node.func.attr base = self.expression(node.func.value) + if method == "astype": + if len(node.args) != 1 or node.keywords: + self.unsupported(node, "astype signature") + target = _cast_target(node.args[0]) + if target is None: + self.unsupported(node.args[0], "dynamic astype target") + return self.emit( + node, + "cast", + _cast_result_type(self.node_types[base], target), + inputs=[base], + parameters={"target": target}, + lookback=self.lookback(base), + ) if method == "shift": periods = _integer_argument(node, "periods", default=1, compiler=self) if periods < 0: @@ -770,6 +1608,22 @@ def method_call(self, node: ast.Call, callable_name: str) -> str: parameters={"periods": periods}, lookback=_add_finite_lookback(self.lookback(base), periods), ) + if method == "fillna": + if len(node.args) != 1 or node.keywords: + self.unsupported(node, "fillna signature") + if self.node_types[base] != "f64-column": + self.unsupported(node.func.value, "fillna source type") + fill = self.expression(node.args[0]) + if self.node_types[fill] not in {"int-scalar", "f64-scalar"}: + self.unsupported(node.args[0], "fillna value type") + return self.emit( + node, + "array-call", + "f64-column", + inputs=[base, fill], + parameters={"family": "numpy", "name": "fill-missing", "arguments": {}}, + lookback=self.merged_lookback([base, fill]), + ) if method == "ffill": if node.args or node.keywords: self.unsupported(node, "parameterized forward fill") @@ -927,14 +1781,705 @@ def callable_reference(self, node: ast.expr) -> _CallableRef | None: return _CallableRef(name) return None - def static_value(self, node: ast.expr) -> Any: + @staticmethod + def lambda_reference(node: ast.expr) -> _LambdaRef | None: + if not isinstance(node, ast.Lambda): + return None + if ( + node.args.posonlyargs + or node.args.kwonlyargs + or node.args.vararg is not None + or node.args.kwarg is not None + or node.args.defaults + ): + return None + return _LambdaRef(tuple(argument.arg for argument in node.args.args), node.body) + + def _inline_lambda_call(self, node: ast.Call) -> str | None: + if not isinstance(node.func, ast.Name): + return None + binding = self.bindings.get(node.func.id) + if not isinstance(binding, _LambdaRef): + return None + if node.keywords or len(node.args) != len(binding.parameters): + self.unsupported(node, "lambda call signature") + previous = {name: self.bindings.get(name) for name in binding.parameters} + present = {name for name in binding.parameters if name in self.bindings} + for name, argument in zip(binding.parameters, node.args, strict=True): + found, value = self.try_static_value(argument) + self.bindings[name] = ( + _StaticBinding(value) if found else self.expression(argument) + ) + try: + return self.expression(binding.body) + finally: + for name in binding.parameters: + if name in present: + prior = previous[name] + assert prior is not None + self.bindings[name] = prior + else: + self.bindings.pop(name, None) + + def static_reference(self, node: ast.expr) -> _StaticBinding | None: + """Keep static containers out of IR until a concrete value is read.""" + if not isinstance( + node, + ast.Name + | ast.Attribute + | ast.Subscript + | ast.Call + | ast.JoinedStr + | ast.List + | ast.Tuple + | ast.Set + | ast.Dict + | ast.Compare + | ast.IfExp, + ): + return None + found, value = self.try_static_value(node) + if not found or not ( + isinstance(value, list | tuple | Mapping) + or isinstance(node, ast.Compare | ast.IfExp | ast.JoinedStr) + ): + return None + return _StaticBinding(value) + + @staticmethod + def sequence_reference(node: ast.expr) -> _SequenceBinding | None: + if isinstance(node, ast.List) and not node.elts: + return _SequenceBinding([]) + return None + + @staticmethod + def mapping_reference(node: ast.expr) -> _MappingBinding | None: + if isinstance(node, ast.Dict) and not node.keys: + return _MappingBinding({}) + return None + + def mapping_write(self, target: ast.Subscript, value_node: ast.expr) -> bool: + if not isinstance(target.value, ast.Name): + return False + mapping = self.bindings.get(target.value.id) + if not isinstance(mapping, _MappingBinding): + return False + found_key, key = self.try_static_value(target.slice) + if not found_key or not isinstance(key, bool | int | float | str): + self.unsupported(target.slice, "dynamic mapping key") + found_value, value = ( + self.try_static_value(value_node) + if _small_static_candidate(value_node) + else (False, None) + ) + mapping.items[key] = _StaticBinding(value) if found_value else self.expression(value_node) + return True + + def static_loop_iterations(self, node: ast.For) -> list[tuple[Binding, ...]] | None: + target_names = _loop_target_names(node.target) + if len(target_names) == 1 and isinstance(node.iter, ast.Tuple | ast.List): + bound_items: list[tuple[Binding, ...]] = [] + for element in node.iter.elts: + if not isinstance(element, ast.Name): + bound_items = [] + break + binding = self.bindings.get(element.id) + if not isinstance(binding, _MappingBinding | _SequenceBinding): + bound_items = [] + break + bound_items.append((binding,)) + if bound_items: + return bound_items + if ( + isinstance(node.iter, ast.Call) + and isinstance(node.iter.func, ast.Attribute) + and node.iter.func.attr == "items" + and isinstance(node.iter.func.value, ast.Name) + and not node.iter.args + and not node.iter.keywords + ): + mapping = self.bindings.get(node.iter.func.value.id) + if isinstance(mapping, _MappingBinding) and len(target_names) == 2: + return [ + (_StaticBinding(key), value) + for key, value in mapping.items.items() + ] + found, iterable = self.try_static_value(node.iter) + if not found or not isinstance(iterable, list | tuple | Mapping): + return None + values = iterable if not isinstance(iterable, Mapping) else iterable.keys() + if len(target_names) != 1: + return None + return [(_StaticBinding(value),) for value in values] + + @staticmethod + def data_provider_reference(node: ast.expr) -> _DataProviderRef | None: + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "self" + and node.attr == "dp" + ): + return _DataProviderRef() + return None + + def column_bundle_reference(self, node: ast.expr) -> _ColumnBundleBinding | None: + if not ( + isinstance(node, ast.Call) + and self.resolved_callable(node.func) == "pd.DataFrame" + and len(node.args) == 1 + and isinstance(node.args[0], ast.Dict) + ): + return None + options = {keyword.arg: keyword.value for keyword in node.keywords} + if set(options) != {"index"} or None in options: + self.unsupported(node, "pandas column-bundle signature") + index = options["index"] + if not ( + isinstance(index, ast.Attribute) + and index.attr == "index" + and isinstance(index.value, ast.Name) + ): + self.unsupported(index, "pandas column-bundle index") + dataframe = self.bindings.get(index.value.id) + if not isinstance(dataframe, str) or self.node_types[dataframe] != "dataframe": + self.unsupported(index, "pandas column-bundle dataframe") + columns: list[tuple[str, str]] = [] + for key_node, value_node in zip( + node.args[0].keys, + node.args[0].values, + strict=True, + ): + if key_node is None: + self.unsupported(node.args[0], "expanded pandas column bundle") + found, key = self.try_static_value(key_node) + if not found or not isinstance(key, str) or not key: + self.unsupported(key_node, "dynamic pandas column-bundle name") + if any(existing == key for existing, _ in columns): + self.unsupported(key_node, "duplicate pandas column-bundle name") + value = self.expression(value_node) + if not self.node_types[value].endswith("-column"): + self.unsupported(value_node, "pandas column-bundle value") + columns.append((key, value)) + if not columns: + self.unsupported(node, "empty pandas column bundle") + return _ColumnBundleBinding(dataframe=dataframe, columns=tuple(columns)) + + def concat_column_bundle(self, node: ast.Call, callable_name: str) -> str | None: + if callable_name != "pd.concat": + return None + if len(node.args) != 1 or not isinstance(node.args[0], ast.List): + self.unsupported(node, "pandas column concat signature") + if len(node.args[0].elts) != 2: + self.unsupported(node.args[0], "pandas column concat inputs") + base_node, bundle_node = node.args[0].elts + if not isinstance(base_node, ast.Name) or not isinstance(bundle_node, ast.Name): + self.unsupported(node.args[0], "pandas column concat bindings") + base = self.bindings.get(base_node.id) + bundle = self.bindings.get(bundle_node.id) + if not isinstance(base, str) or self.node_types[base] != "dataframe": + self.unsupported(base_node, "pandas column concat dataframe") + if not isinstance(bundle, _ColumnBundleBinding) or bundle.dataframe != base: + self.unsupported(bundle_node, "pandas column concat index identity") + options = {keyword.arg: self.try_static_value(keyword.value) for keyword in node.keywords} + if set(options) != {"axis", "copy"} or options["axis"] != (True, 1) or options[ + "copy" + ] != (True, False): + self.unsupported(node, "pandas column concat options") + dataframe = base + for column, value in bundle.columns: + dataframe = self.emit( + node, + "column-write", + "dataframe", + inputs=[dataframe, value], + parameters={"column": column, "collision": "reject"}, + lookback=self.merged_lookback([dataframe, value]), + ) + self.produced_columns.add(column) + return dataframe + + def frame_source_call(self, call: ast.Call) -> str | None: + if not isinstance(call.func, ast.Attribute) or call.func.attr != "get_pair_dataframe": + return None + provider = call.func.value + is_provider = ( + isinstance(provider, ast.Name) + and isinstance(self.bindings.get(provider.id), _DataProviderRef) + ) or ( + isinstance(provider, ast.Attribute) + and isinstance(provider.value, ast.Name) + and provider.value.id == "self" + and provider.attr == "dp" + ) + if not is_provider: + self.unsupported(provider, "get_pair_dataframe provider") + arguments = _bind_pair_dataframe_arguments(call, self) + found_timeframe, timeframe = self.try_static_value(arguments["timeframe"]) + if not found_timeframe or not isinstance(timeframe, str): + self.unsupported(arguments["timeframe"], "dynamic frame-source timeframe") + found_pair, pair = self.try_static_value(arguments["pair"]) + if found_pair: + if not isinstance(pair, str) or not pair: + self.unsupported(arguments["pair"], "frame-source literal pair") + pair_selector = {"kind": "literal", "value": pair} + else: + pair_node = self.expression(arguments["pair"]) + record = self.nodes[_numeric_identifier_key({"id": pair_node}) - 1] + if not ( + record["op"] == "metadata-read" + and record["value_type"] == "string-scalar" + and record["parameters"].get("key") == "pair" + ): + self.unsupported(arguments["pair"], "dynamic frame-source pair selector") + pair_selector = {"kind": "metadata", "key": "pair"} + return self.emit( + call, + "frame-source", + "dataframe", + parameters={"pair": pair_selector, "timeframe": timeframe}, + ) + + def frame_empty_guard(self, node: ast.If) -> bool: + if not ( + isinstance(node.test, ast.Attribute) + and node.test.attr == "empty" + and isinstance(node.test.value, ast.Name) + and not node.orelse + and len(node.body) == 1 + ): + return False + name = node.test.value.id + binding = self.bindings.get(name) + if not isinstance(binding, str) or self.node_types[binding] != "dataframe": + return False + branch = node.body[0] + if isinstance(branch, ast.Return): + if not isinstance(branch.value, ast.Name) or branch.value.id != name: + return False + elif not isinstance(branch, ast.Continue): + return False + checked = self.emit( + node, + "frame-nonempty", + "dataframe", + inputs=[binding], + lookback=self.lookback(binding), + ) + self.bindings[name] = checked + return True + + def frame_drop_guard(self, node: ast.If) -> bool: + if not ( + isinstance(node.test, ast.Compare) + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.In) + and len(node.test.comparators) == 1 + and isinstance(node.test.comparators[0], ast.Attribute) + and node.test.comparators[0].attr == "columns" + and isinstance(node.test.comparators[0].value, ast.Name) + and not node.orelse + and len(node.body) == 1 + ): + return False + dataframe_name = node.test.comparators[0].value.id + branch = node.body[0] + assignment_target: str | None = None + if ( + isinstance(branch, ast.Assign) + and len(branch.targets) == 1 + and isinstance(branch.targets[0], ast.Name) + and isinstance(branch.value, ast.Call) + ): + assignment_target = branch.targets[0].id + call = branch.value + elif isinstance(branch, ast.Expr) and isinstance(branch.value, ast.Call): + call = branch.value + else: + return False + if not ( + isinstance(call.func, ast.Attribute) + and call.func.attr == "drop" + and isinstance(call.func.value, ast.Name) + and call.func.value.id == dataframe_name + and not call.args + ): + return False + keywords = {keyword.arg: keyword.value for keyword in call.keywords} + if None in keywords or "columns" not in keywords: + return False + if assignment_target is not None: + if assignment_target != dataframe_name or set(keywords) != {"columns"}: + return False + elif set(keywords) != {"columns", "inplace"}: + return False + if assignment_target is None: + found_inplace, inplace = self.try_static_value(keywords["inplace"]) + if not found_inplace or inplace is not True: + return False + found_test, test_column = self.try_static_value(node.test.left) + found_drop, drop_column = self.try_static_value(keywords["columns"]) + if ( + not found_test + or not found_drop + or not isinstance(test_column, str) + or test_column != drop_column + ): + return False + dataframe = self.bindings.get(dataframe_name) + if not isinstance(dataframe, str) or self.node_types[dataframe] != "dataframe": + self.unsupported(node, "frame drop dataframe") + dropped = self.emit( + node, + "frame-drop-if-present", + "dataframe", + inputs=[dataframe], + parameters={"column": test_column}, + lookback=self.lookback(dataframe), + ) + self.bindings[dataframe_name] = dropped + return True + + def inplace_forward_fill(self, node: ast.Call) -> bool: + if not ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "ffill" + and isinstance(node.func.value, ast.Name) + and not node.args + and len(node.keywords) == 1 + and node.keywords[0].arg == "inplace" + ): + return False + found, inplace = self.try_static_value(node.keywords[0].value) + if not found or inplace is not True: + return False + name = node.func.value.id + base = self.bindings.get(name) + if not isinstance(base, str) or self.node_types[base] != "dataframe": + self.unsupported(node.func.value, "in-place forward-fill dataframe") + filled = self.emit( + node, + "fill", + "dataframe", + inputs=[base], + parameters={"direction": "forward"}, + lookback={ + "kind": "recursive", + "candles": None, + "expression": _safe_expression(node), + "causal": bool(self.lookback(base)["causal"]), + }, + ) + self.bindings[name] = filled + return True + + def append_sequence(self, call: ast.Call) -> bool: + if not ( + isinstance(call.func, ast.Attribute) + and call.func.attr == "append" + and isinstance(call.func.value, ast.Name) + ): + return False + binding = self.bindings.get(call.func.value.id) + if not isinstance(binding, _SequenceBinding): + return False + if len(call.args) != 1 or call.keywords: + self.unsupported(call, "sequence append signature") + found, value = ( + self.try_static_value(call.args[0]) + if _small_static_candidate(call.args[0]) + else (False, None) + ) + binding.items.append(_StaticBinding(value) if found else self.expression(call.args[0])) + return True + + def reduce_sequence_call(self, call: ast.Call, callable_name: str) -> str | None: + function = self.module_functions.get(callable_name) + operator_name = _recognized_sequence_reducer(function) if function is not None else None + if operator_name is None: + return None + if len(call.args) != 1 or call.keywords or not isinstance(call.args[0], ast.Name): + self.unsupported(call, "sequence reducer signature") + binding = self.bindings.get(call.args[0].id) + if not isinstance(binding, _SequenceBinding): + self.unsupported(call.args[0], "sequence reducer input") + dynamic: list[str] = [] + static_values: list[bool] = [] + for item in binding.items: + if isinstance(item, _StaticBinding): + if not isinstance(item.value, bool): + self.unsupported(call.args[0], "non-Boolean sequence reducer value") + static_values.append(item.value) + elif isinstance(item, str): + if self.node_types[item] not in {"bool-scalar", "bool-column"}: + self.unsupported(call.args[0], "non-Boolean sequence reducer value") + dynamic.append(item) + else: + self.unsupported(call.args[0], "nested sequence reducer value") + absorbing = operator_name != "and" + if absorbing in static_values: + return self.literal(call, absorbing) + if not dynamic: + return self.literal(call, not absorbing) + if len(dynamic) == 1: + return dynamic[0] + return self.emit( + call, + "logical", + _boolean_result_type(*(self.node_types[item] for item in dynamic)), + inputs=dynamic, + parameters={"operator": operator_name}, + lookback=self.merged_lookback(dynamic), + ) + + def append_tag_call(self, call: ast.Call, callable_name: str) -> str | None: + function = self.module_functions.get(callable_name) + if function is None or not _recognized_tag_appender(function): + return None + if len(call.args) != 3 or call.keywords or not isinstance(call.args[0], ast.Name): + self.unsupported(call, "tag append helper signature") + target_name = call.args[0].id + target = self.bindings.get(target_name) + if not isinstance(target, str) or self.node_types[target] != "string-column": + self.unsupported(call.args[0], "tag append target") + mask = self.expression(call.args[1]) + tag = self.expression(call.args[2]) + if self.node_types[mask] not in {"bool-scalar", "bool-column"}: + self.unsupported(call.args[1], "tag append mask") + if self.node_types[tag] != "string-scalar": + self.unsupported(call.args[2], "tag append value") + result = self.emit( + call, + "masked-string-append", + "string-column", + inputs=[target, mask, tag], + lookback=self.merged_lookback([target, mask, tag]), + ) + self.bindings[target_name] = result + return result + + def try_static_value(self, node: ast.expr) -> tuple[bool, Any]: + """Evaluate a side-effect-free Python expression used for source routing.""" + if isinstance(node, ast.Constant): + return True, node.value if isinstance(node, ast.Name): binding = self.bindings.get(node.id) + if isinstance(binding, _SequenceBinding): + return True, bool(binding.items) + if isinstance(binding, _DataProviderRef): + return True, True + if isinstance(binding, _MappingBinding): + if all(isinstance(value, _StaticBinding) for value in binding.items.values()): + return True, { + key: value.value for key, value in binding.items.items() + } + return False, None + if isinstance(binding, _StaticBinding): + return True, binding.value if isinstance(binding, str): record = self.nodes[_numeric_identifier_key({"id": binding}) - 1] if record["op"] == "literal": - return record["parameters"].get("value") - return _static_value(node, self.class_constants) + return True, record["parameters"].get("value") + if node.id in self.class_constants: + return True, self.class_constants[node.id] + if node.id == "object": + return True, "object" + return False, None + if isinstance(node, ast.Attribute): + qualified = _qualified_name(node) + if qualified == "np.nan": + return True, float("nan") + if qualified == "np.inf": + return True, float("inf") + if isinstance(node.value, ast.Name) and node.value.id == "self": + if node.attr in self.instance_constants: + return True, self.instance_constants[node.attr] + if node.attr in self.class_constants: + return True, self.class_constants[node.attr] + return False, None + found, base = self.try_static_value(node.value) + if found and isinstance(base, Mapping) and node.attr in base: + return True, base[node.attr] + return False, None + if isinstance(node, ast.Subscript): + found_base, base = self.try_static_value(node.value) + found_key, key = self.try_static_value(node.slice) + if not found_base or not found_key: + return False, None + try: + return True, base[key] + except (KeyError, IndexError, TypeError): + return False, None + if isinstance(node, ast.List | ast.Tuple | ast.Set): + values = [self.try_static_value(item) for item in node.elts] + if not all(found for found, _ in values): + return False, None + sequence_items = [value for _, value in values] + return ( + True, + sequence_items + if isinstance(node, ast.List | ast.Set) + else tuple(sequence_items), + ) + if isinstance(node, ast.Dict): + items: dict[str, Any] = {} + for key_node, value_node in zip(node.keys, node.values, strict=True): + if key_node is None: + return False, None + found_key, key = self.try_static_value(key_node) + found_value, value = self.try_static_value(value_node) + if not found_key or not isinstance(key, str) or not found_value: + return False, None + items[key] = value + return True, items + if isinstance(node, ast.UnaryOp): + found, value = self.try_static_value(node.operand) + if not found: + return False, None + operation = { + ast.Not: operator.not_, + ast.USub: operator.neg, + ast.UAdd: operator.pos, + ast.Invert: operator.invert, + }.get(type(node.op)) + if operation is None: + return False, None + try: + return True, operation(value) + except (TypeError, ValueError): + return False, None + if isinstance(node, ast.BinOp): + found_left, left = self.try_static_value(node.left) + found_right, right = self.try_static_value(node.right) + operation = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, + ast.Pow: operator.pow, + }.get(type(node.op)) + if not found_left or not found_right or operation is None: + return False, None + try: + return True, operation(left, right) + except (ArithmeticError, TypeError, ValueError): + return False, None + if isinstance(node, ast.BoolOp): + values = [self.try_static_value(item) for item in node.values] + if not all(found for found, _ in values): + return False, None + resolved = [value for _, value in values] + if isinstance(node.op, ast.And): + return True, all(resolved) + if isinstance(node.op, ast.Or): + return True, any(resolved) + return False, None + if isinstance(node, ast.Compare): + found_left, left = self.try_static_value(node.left) + if not found_left: + return False, None + for operation_node, comparator in zip(node.ops, node.comparators, strict=True): + found_right, right = self.try_static_value(comparator) + if not found_right: + return False, None + operation = { + ast.Eq: operator.eq, + ast.NotEq: operator.ne, + ast.Lt: operator.lt, + ast.LtE: operator.le, + ast.Gt: operator.gt, + ast.GtE: operator.ge, + ast.In: lambda item, container: item in container, + ast.NotIn: lambda item, container: item not in container, + ast.Is: operator.is_, + ast.IsNot: operator.is_not, + }.get(type(operation_node)) + if operation is None: + return False, None + try: + if not operation(left, right): + return True, False + except (TypeError, ValueError): + return False, None + left = right + return True, True + if isinstance(node, ast.IfExp): + found, condition = self.try_static_value(node.test) + if not found: + return False, None + return self.try_static_value(node.body if condition else node.orelse) + if isinstance(node, ast.JoinedStr): + parts: list[str] = [] + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + parts.append(value.value) + continue + if not isinstance(value, ast.FormattedValue) or value.format_spec is not None: + return False, None + found, resolved = self.try_static_value(value.value) + if not found: + return False, None + parts.append(str(resolved)) + return True, "".join(parts) + if isinstance(node, ast.Call): + return self._try_static_call(node) + return False, None + + def _try_static_call(self, node: ast.Call) -> tuple[bool, Any]: + if any(keyword.arg is None for keyword in node.keywords): + return False, None + arguments = [self.try_static_value(argument) for argument in node.args] + keywords = { + str(keyword.arg): self.try_static_value(keyword.value) for keyword in node.keywords + } + if not all(found for found, _ in arguments) or not all( + found for found, _ in keywords.values() + ): + return False, None + values = [value for _, value in arguments] + options = {name: value for name, (_, value) in keywords.items()} + if isinstance(node.func, ast.Name): + function = { + "bool": bool, + "float": float, + "frozenset": frozenset, + "int": int, + "len": len, + "list": list, + "set": set, + "str": str, + "tuple": tuple, + }.get(node.func.id) + if function is None: + return False, None + try: + return True, _normalized_static_value(function(*values, **options)) + except (TypeError, ValueError, OverflowError): + return False, None + if not isinstance(node.func, ast.Attribute): + return False, None + found, base = self.try_static_value(node.func.value) + if not found: + return False, None + method = node.func.attr + try: + if method == "get" and isinstance(base, Mapping): + return True, base.get(*values) + if method == "items" and isinstance(base, Mapping) and not values and not options: + return True, list(base.items()) + if method in {"partition", "rsplit", "split", "startswith", "endswith"} and isinstance( + base, str + ): + return True, _normalized_static_value(getattr(base, method)(*values, **options)) + except (TypeError, ValueError): + return False, None + return False, None + + def static_value(self, node: ast.expr) -> Any: + found, value = self.try_static_value(node) + return value if found else None def resolved_callable(self, node: ast.expr) -> str: if isinstance(node, ast.Name): @@ -952,7 +2497,7 @@ def literal(self, node: ast.AST, value: Any) -> str: node, "literal", _literal_type(value), - parameters={"value": value}, + parameters=_literal_parameters(value), ) def emit( @@ -986,6 +2531,30 @@ def emit( def lookback(self, node_id: str) -> dict[str, Any]: return dict(self.nodes[_numeric_identifier_key({"id": node_id}) - 1]["lookback"]) + def node_static_value(self, node_id: str) -> tuple[bool, Any]: + current = node_id + visited: set[str] = set() + while current not in visited: + visited.add(current) + record = self.nodes[_numeric_identifier_key({"id": current}) - 1] + if record["op"] == "literal": + parameters = record["parameters"] + if "value" in parameters: + return True, parameters["value"] + special = parameters.get("special") + if special == "nan": + return True, float("nan") + if special == "+infinity": + return True, float("inf") + if special == "-infinity": + return True, float("-inf") + return False, None + if record["op"] in {"return", "cast"} and len(record["inputs"]) == 1: + current = record["inputs"][0] + continue + return False, None + return False, None + def merged_lookback(self, node_ids: Sequence[str]) -> dict[str, Any]: return _merge_lookbacks([self.lookback(node_id) for node_id in node_ids]) @@ -1053,6 +2622,290 @@ def _bind_informative_merge_arguments( return arguments +def _bind_helper_arguments( + call: ast.Call, + method: ast.FunctionDef, + compiler: _Compiler, +) -> list[tuple[str, ast.expr]]: + if method.args.vararg is not None or method.args.kwarg is not None: + compiler.unsupported(call, "variadic indicator helper signature") + positional = [*method.args.posonlyargs, *method.args.args] + if positional and positional[0].arg == "self": + positional = positional[1:] + names = [argument.arg for argument in positional] + keyword_only = [argument.arg for argument in method.args.kwonlyargs] + if len(call.args) > len(positional): + compiler.unsupported(call, "indicator helper call signature") + bound: dict[str, ast.expr] = { + name: value for name, value in zip(names, call.args, strict=False) + } + for keyword in call.keywords: + if keyword.arg is None: + compiler.unsupported(keyword.value, "expanded indicator helper arguments") + if keyword.arg not in {*names, *keyword_only}: + compiler.unsupported(keyword.value, f"unknown indicator helper argument {keyword.arg}") + if keyword.arg in bound: + compiler.unsupported( + keyword.value, + f"duplicate indicator helper argument {keyword.arg}", + ) + bound[keyword.arg] = keyword.value + + positional_defaults = [None] * (len(positional) - len(method.args.defaults)) + list( + method.args.defaults + ) + for argument, default in zip(positional, positional_defaults, strict=True): + if argument.arg not in bound: + if default is None: + compiler.unsupported(call, "indicator helper call signature") + bound[argument.arg] = default + for argument, default in zip(method.args.kwonlyargs, method.args.kw_defaults, strict=True): + if argument.arg not in bound: + if default is None: + compiler.unsupported(call, "indicator helper call signature") + bound[argument.arg] = default + return [(name, bound[name]) for name in [*names, *keyword_only]] + + +def _bind_pair_dataframe_arguments( + call: ast.Call, + compiler: _Compiler, +) -> dict[str, ast.expr]: + names = ("pair", "timeframe") + if len(call.args) > len(names): + compiler.unsupported(call, "get_pair_dataframe signature") + bound: dict[str, ast.expr] = dict(zip(names, call.args, strict=False)) + for keyword in call.keywords: + if keyword.arg is None: + compiler.unsupported(keyword.value, "expanded get_pair_dataframe arguments") + if keyword.arg not in names: + compiler.unsupported( + keyword.value, + f"unknown get_pair_dataframe argument {keyword.arg}", + ) + if keyword.arg in bound: + compiler.unsupported( + keyword.value, + f"duplicate get_pair_dataframe argument {keyword.arg}", + ) + bound[str(keyword.arg)] = keyword.value + if set(bound) != set(names): + compiler.unsupported(call, "get_pair_dataframe requires pair and timeframe") + return bound + + +def _normalized_shift_helper( + method: ast.FunctionDef, + bound: Sequence[tuple[str, ast.expr]], + compiler: _Compiler, +) -> tuple[ast.expr, int] | None: + """Recognize the canonical allocate-and-slice causal shift implementation.""" + parameters = [*method.args.posonlyargs, *method.args.args] + if parameters and parameters[0].arg == "self": + parameters = parameters[1:] + if len(parameters) != 2 or len(method.body) != 4: + return None + source_name, periods_name = (argument.arg for argument in parameters) + allocate, warmup, shifted, returned = method.body + if not ( + isinstance(allocate, ast.Assign) + and len(allocate.targets) == 1 + and isinstance(allocate.targets[0], ast.Name) + and isinstance(allocate.value, ast.Call) + and _qualified_name(allocate.value.func) == "np.empty_like" + and len(allocate.value.args) == 1 + and isinstance(allocate.value.args[0], ast.Name) + and allocate.value.args[0].id == source_name + and not allocate.value.keywords + ): + return None + output_name = allocate.targets[0].id + if not _is_shift_slice_assignment( + warmup, + output_name=output_name, + source_name=source_name, + periods_name=periods_name, + warmup=True, + ) or not _is_shift_slice_assignment( + shifted, + output_name=output_name, + source_name=source_name, + periods_name=periods_name, + warmup=False, + ): + return None + if not ( + isinstance(returned, ast.Return) + and isinstance(returned.value, ast.Name) + and returned.value.id == output_name + ): + return None + arguments = dict(bound) + found, periods = compiler.try_static_value(arguments[periods_name]) + if not found or not isinstance(periods, int) or isinstance(periods, bool) or periods <= 0: + compiler.unsupported(arguments[periods_name], "shift helper periods") + return arguments[source_name], periods + + +def _recognized_sequence_reducer(function: ast.FunctionDef) -> str | None: + calls = { + _qualified_name(node.func) + for node in ast.walk(function) + if isinstance(node, ast.Call) + } + matched = { + operator_name + for qualified, operator_name in ( + ("np.logical_and.reduce", "and"), + ("np.logical_or.reduce", "or"), + ) + if qualified in calls + } + return next(iter(matched)) if len(matched) == 1 else None + + +def _normalized_frame_projection_helper( + method: ast.FunctionDef, + bound: Sequence[tuple[str, ast.expr]], + compiler: _Compiler, +) -> tuple[ast.expr, dict[str, Any]] | None: + parameters = [*method.args.posonlyargs, *method.args.args] + if parameters and parameters[0].arg == "self": + parameters = parameters[1:] + if len(parameters) != 2: + return None + frame_name, keep_name = (parameter.arg for parameter in parameters) + assignments = [ + statement + for statement in method.body + if isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Name) + and isinstance(statement.value, ast.ListComp) + ] + if len(assignments) != 1: + return None + assignment = assignments[0] + assert isinstance(assignment.targets[0], ast.Name) + assert isinstance(assignment.value, ast.ListComp) + output_name = assignment.targets[0].id + comprehension = assignment.value + if not ( + isinstance(comprehension.elt, ast.Name) + and len(comprehension.generators) == 1 + and isinstance(comprehension.generators[0].target, ast.Name) + and comprehension.elt.id == comprehension.generators[0].target.id + and isinstance(comprehension.generators[0].iter, ast.Attribute) + and comprehension.generators[0].iter.attr == "columns" + and isinstance(comprehension.generators[0].iter.value, ast.Name) + and comprehension.generators[0].iter.value.id == frame_name + and len(comprehension.generators[0].ifs) == 1 + ): + return None + column_name = comprehension.elt.id + always_keep: str | None = None + candidates_name: str | None = None + keeps_requested = False + for comparison in ( + item + for item in ast.walk(comprehension.generators[0].ifs[0]) + if isinstance(item, ast.Compare) + ): + if len(comparison.ops) != 1 or len(comparison.comparators) != 1: + return None + right = comparison.comparators[0] + if not isinstance(comparison.left, ast.Name) or comparison.left.id != column_name: + continue + if isinstance(comparison.ops[0], ast.Eq) and isinstance(right, ast.Constant): + if not isinstance(right.value, str): + return None + always_keep = right.value + elif ( + isinstance(comparison.ops[0], ast.NotIn) + and isinstance(right, ast.Attribute) + and isinstance(right.value, ast.Name) + and right.value.id == "self" + ): + candidates_name = right.attr + elif ( + isinstance(comparison.ops[0], ast.In) + and isinstance(right, ast.Name) + and right.id == keep_name + ): + keeps_requested = True + returned = next( + ( + statement + for statement in method.body + if isinstance(statement, ast.Return) + and isinstance(statement.value, ast.Subscript) + and isinstance(statement.value.value, ast.Name) + and statement.value.value.id == frame_name + and isinstance(statement.value.slice, ast.Name) + and statement.value.slice.id == output_name + ), + None, + ) + if ( + always_keep is None + or candidates_name is None + or not keeps_requested + or returned is None + or candidates_name not in compiler.class_constants + ): + return None + candidates = compiler.class_constants[candidates_name] + if not isinstance(candidates, list | tuple) or not all( + isinstance(value, str) for value in candidates + ): + compiler.unsupported(method, "frame projection candidate columns") + arguments = dict(bound) + found_keep, keep = compiler.try_static_value(arguments[keep_name]) + if not found_keep or keep is None: + keep_values: list[str] = [] + elif isinstance(keep, list | tuple) and all(isinstance(value, str) for value in keep): + keep_values = list(keep) + else: + compiler.unsupported(arguments[keep_name], "frame projection keep columns") + return arguments[frame_name], { + "always_keep": [always_keep], + "drop_candidates": list(candidates), + "keep": keep_values, + } + + +def _normalized_native_indicator_helper( + method: ast.FunctionDef, + bound: Sequence[tuple[str, ast.expr]], + compiler: _Compiler, +) -> tuple[str, list[ast.expr], dict[str, Any]] | None: + matched = next( + ( + name + for name, source in _NATIVE_HELPER_TEMPLATES.items() + if _helper_bodies_equal(method, _template_function(source)) + ), + None, + ) + if matched is None: + return None + arguments = dict(bound) + if matched in {"chaikin-money-flow", "chaikin-money-flow-legacy"}: + found, period = compiler.try_static_value(arguments["timeperiod"]) + minimum = 2 if matched == "chaikin-money-flow-legacy" else 1 + if ( + not found + or not isinstance(period, int) + or isinstance(period, bool) + or period < minimum + ): + compiler.unsupported(arguments["timeperiod"], "chaikin timeperiod") + return matched, [arguments[name] for name in ("high", "low", "close", "volume")], { + "timeperiod": period + } + return matched, [arguments["arr"]], {} + + def _static_informative_option( arguments: Mapping[str, ast.expr], name: str, @@ -1135,237 +2988,3 @@ def _required_static( if not _is_json_value(value): compiler.unsupported(node, "non-JSON indicator parameter") return value - - -def _static_value(node: ast.expr, constants: Mapping[str, Any]) -> Any: - if isinstance(node, ast.Constant): - return node.value - if isinstance(node, ast.Name) and node.id in constants: - return constants[node.id] - if ( - isinstance(node, ast.Attribute) - and isinstance(node.value, ast.Name) - and node.value.id == "self" - and node.attr in constants - ): - return constants[node.attr] - if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): - value = _static_value(node.operand, constants) - if isinstance(value, int | float) and not isinstance(value, bool): - return -value - try: - value = ast.literal_eval(node) - except (ValueError, TypeError, MemoryError, RecursionError): - return None - return value if _is_json_value(value) else None - - -def _is_json_value(value: Any) -> bool: - if value is None or isinstance(value, bool | int | float | str): - return True - if isinstance(value, list | tuple): - return all(_is_json_value(item) for item in value) - if isinstance(value, Mapping): - return all(isinstance(key, str) and _is_json_value(item) for key, item in value.items()) - return False - - -def _literal_string(node: ast.expr) -> str | None: - return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None - - -@cache -def _indicator_output_names(callable_name: str) -> tuple[str, ...] | None: - if not callable_name.startswith("ta."): - return None - from talib import abstract - - try: - function = abstract.Function(callable_name.removeprefix("ta.")) - except Exception: # TA-Lib exposes invalid-function errors through a generic wrapper exception. - return None - names = cast(Sequence[object], function.output_names) - return tuple(str(name) for name in names) - - -def _qualified_name(node: ast.AST) -> str | None: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - parent = _qualified_name(node.value) - return f"{parent}.{node.attr}" if parent else node.attr - return None - - -def _parameter_type(name: str) -> str: - lowered = name.lower() - if lowered in {"df", "dataframe", "informative", "frame", "info"} or "dataframe" in lowered: - return "dataframe" - if lowered == "metadata": - return "metadata" - return "dynamic" - - -def _literal_type(value: Any) -> str: - if value is None: - return "null" - if isinstance(value, bool): - return "bool-scalar" - if isinstance(value, int): - return "int-scalar" - if isinstance(value, float): - return "f64-scalar" - if isinstance(value, str): - return "string-scalar" - return "json-scalar" - - -def _is_column(value_type: str) -> bool: - return value_type.endswith("-column") - - -def _numeric_result_type(left: str, right: str) -> str: - if _is_column(left) or _is_column(right): - return "f64-column" - if left == right == "int-scalar": - return "int-scalar" - return "f64-scalar" - - -def _boolean_result_type(*value_types: str) -> str: - if any(_is_column(value_type) for value_type in value_types): - return "bool-column" - return "bool-scalar" - - -def _merge_value_types(left: str, right: str) -> str: - if left == right: - return left - if _is_column(left) or _is_column(right): - return "f64-column" - if {left, right} <= {"int-scalar", "f64-scalar"}: - return "f64-scalar" - return "dynamic" - - -def _array_result_type(inputs: Sequence[str], node_types: Mapping[str, str]) -> str: - types = [node_types[node] for node in inputs] - if any(_is_column(value_type) for value_type in types): - return "f64-column" - return "dynamic" - - -def _zero_lookback() -> dict[str, Any]: - return { - "kind": "finite", - "candles": 0, - "expression": None, - "causal": True, - } - - -def _add_finite_lookback(lookback: Mapping[str, Any], candles: int) -> dict[str, Any]: - if lookback["kind"] == "finite" and isinstance(lookback["candles"], int): - return { - "kind": "finite", - "candles": lookback["candles"] + candles, - "expression": None, - "causal": bool(lookback["causal"]), - } - return { - "kind": "mixed", - "candles": None, - "expression": f"{lookback['kind']}+{candles}", - "causal": bool(lookback["causal"]), - } - - -def _merge_lookbacks(lookbacks: Sequence[Mapping[str, Any]]) -> dict[str, Any]: - if not lookbacks: - return _zero_lookback() - causal = all(bool(item["causal"]) for item in lookbacks) - if all(item["kind"] == "finite" and isinstance(item["candles"], int) for item in lookbacks): - return { - "kind": "finite", - "candles": max(int(item["candles"]) for item in lookbacks), - "expression": None, - "causal": causal, - } - kinds = sorted({str(item["kind"]) for item in lookbacks}) - return { - "kind": kinds[0] if len(kinds) == 1 else "mixed", - "candles": None, - "expression": "+".join(kinds), - "causal": causal, - } - - -def _program_lookback(nodes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: - return _merge_lookbacks([node["lookback"] for node in nodes]) - - -def _location(node: ast.AST) -> dict[str, Any]: - return { - "path": "strategy.py", - "line": getattr(node, "lineno", 1), - "column": getattr(node, "col_offset", 0), - "end_line": getattr(node, "end_lineno", getattr(node, "lineno", 1)), - "end_column": getattr(node, "end_col_offset", getattr(node, "col_offset", 0)), - } - - -def _unsupported(node: ast.AST, description: str) -> Never: - location = _location(node) - raise IndicatorProgramCompileError( - f"strategy.py:{location['line']}:{location['column']}: " - f"indicator-program-v1 does not support {description}" - ) - - -def _safe_expression(node: ast.AST) -> str: - try: - return ast.unparse(node) - except RecursionError: - return f"" - - -def _iterative_ast_sha256(node: ast.AST) -> str: - digest = hashlib.sha256() - stack: list[tuple[str, Any]] = [("node", node)] - while stack: - kind, value = stack.pop() - digest.update(kind.encode()) - digest.update(b"\0") - if isinstance(value, ast.AST): - digest.update(type(value).__name__.encode()) - for name, child in reversed(list(ast.iter_fields(value))): - stack.append(("field", name)) - stack.append(("value", child)) - elif isinstance(value, list): - digest.update(str(len(value)).encode()) - for child in reversed(value): - stack.append(("value", child)) - else: - digest.update(repr(value).encode()) - digest.update(b"\0") - return digest.hexdigest() - - -def _numeric_identifier_key(record: Mapping[str, Any]) -> int: - return int(str(record["id"])[1:]) - - -def _fingerprint(program: Mapping[str, Any]) -> str: - identity = copy.deepcopy(dict(program)) - source = identity.get("source") - if isinstance(source, dict): - source.pop("path", None) - return hashlib.sha256( - json.dumps( - identity, - ensure_ascii=False, - allow_nan=False, - sort_keys=True, - separators=(",", ":"), - ).encode() - ).hexdigest() diff --git a/python/nfi_backtest_engine/latest_signal47_fixture.py b/python/nfi_backtest_engine/latest_signal47_fixture.py new file mode 100644 index 00000000..b1df7850 --- /dev/null +++ b/python/nfi_backtest_engine/latest_signal47_fixture.py @@ -0,0 +1,380 @@ +"""Generate the source-boundary proof for upstream X7 Signal 47 protection.""" + +from __future__ import annotations + +import ast +import hashlib +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any, cast + +import numpy as np +import pandas as pd + +from . import _rust +from .signal_program import compile_signal_program, execute_signal_program +from .strategy_compat import ( + VectorDataProvider, + load_strategy_class, + prepare_worker_config, +) +from .tag_program import compile_tag_program, execute_tag_program +from .vector_worker import _advise_signals + +FIXTURE_PATH = Path("benchmarks/evidence/m22/latest-x7-signal47-boundary.json") +SCHEMA_VERSION = "latest-x7-signal47-boundary-v1" +STRATEGY_CLASS = "NostalgiaForInfinityX7" +SIGNAL_COLUMNS = ("enter_long", "enter_short", "exit_long", "exit_short") +TAG_COLUMNS = ("enter_tag", "exit_tag") +CHANGED_EXPRESSION = ( + "((rsi_3_4h_gt_20) | (rsi_3_1d_gt_50) | " + "(aroonu_14_1d_lt_80) | (roc_9_1d_lt_30))" +) + +_SCALAR_OVERRIDES = { + "volume": 1.0, + "num_empty_288": 0.0, + "protections_long_global": 1.0, + "RSI_3": 10.0, + "RSI_3_15m": 60.0, + "RSI_3_1h": 60.0, + "RSI_3_4h": 20.0, + "RSI_3_1d": 50.0, + "AROONU_14_1d": 80.0, + "AROONU_14_15m": 30.0, + "AROONU_14_1h": 0.0, + "AROONU_14_4h": 0.0, + "STOCHRSIk_14_14_3_3_15m": 50.0, + "STOCHRSIk_14_14_3_3_1h": 50.0, + "STOCHRSIk_14_14_3_3_4h": 50.0, + "STOCHRSIk_14_14_3_3_1d": 50.0, + "RSI_14": 50.0, + "RSI_14_15m": 40.0, + "RSI_14_1h": 50.0, + "RSI_14_4h": 50.0, + "AROONU_14": 60.0, + "STOCHRSIk_14_14_3_3": 50.0, + "OBV_change_pct": 1.0, + "EMA_26": 1.0, + "EMA_12_4h": 1.0, + "EMA_200_4h": 2.0, + "close": 100.0, + "close_min_48": 90.0, + "close_max_48": 110.0, + "CMF_20_1h": 0.0, + "CMF_20_4h": 0.0, + "BBP_20_2.0_4h": 0.5, + "WILLR_14_1h": -50.0, + "ROC_9_4h": 0.0, + "ROC_9_1h": 0.0, + "BBL_20_2.0": 90.0, + "BBU_20_2.0": 110.0, + "BBL_20_2.0_1h": 90.0, + "BBU_20_2.0_1h": 110.0, +} +_SERIES_OVERRIDES = { + "EMA_12": [0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 2.0, 2.0], +} +_CASES = {"new_protection_rejects": 30.0, "one_term_passes": 29.0} + + +def generate_fixture( + current_source: Path, + baseline_source: Path, + *, + current_commit: str, + baseline_commit: str, +) -> dict[str, Any]: + """Execute both source revisions and both independent program runtimes.""" + _validate_sources(current_source, baseline_source) + modes = { + mode: _qualify_mode(current_source, baseline_source, mode) + for mode in ("spot", "futures") + } + document: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "source": _source_identity(current_source, current_commit), + "baseline": _source_identity(baseline_source, baseline_commit), + "strategy_class": STRATEGY_CLASS, + "changed_route": { + "side": "long", + "signal": "47", + "expression": CHANGED_EXPRESSION, + "expression_sha256": hashlib.sha256(CHANGED_EXPRESSION.encode()).hexdigest(), + }, + "source_diff": _source_method_diff(current_source, baseline_source), + "input_contract": { + "rows": 8, + "default_numeric_value": 0.0, + "scalar_overrides": _SCALAR_OVERRIDES, + "series_overrides": _SERIES_OVERRIDES, + "case_roc_9_1d": _CASES, + }, + "modes": modes, + "claims": { + "source_wrapper_python_program_exact": True, + "source_wrapper_rust_signal_exact": True, + "source_wrapper_rust_tag_exact": True, + "spot_and_futures_exact": True, + "runtime_signal_number_branch_added": False, + }, + } + document["fingerprint"] = canonical_sha256(document) + return document + + +def write_fixture( + current_source: Path, + baseline_source: Path, + *, + current_commit: str, + baseline_commit: str, + output: Path = FIXTURE_PATH, +) -> None: + """Regenerate the committed evidence document.""" + document = generate_fixture( + current_source, + baseline_source, + current_commit=current_commit, + baseline_commit=baseline_commit, + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(document, ensure_ascii=False, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + + +def canonical_sha256(document: Mapping[str, Any]) -> str: + """Hash a fixture independently of its own fingerprint field.""" + identity = dict(document) + identity.pop("fingerprint", None) + encoded = json.dumps( + identity, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _qualify_mode(current: Path, baseline: Path, mode: str) -> dict[str, Any]: + config = {"max_open_trades": 6, "stake_currency": "USDT", "trading_mode": mode} + programs: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {} + for label, source in (("current", current), ("baseline", baseline)): + programs[label] = ( + compile_signal_program( + source, + class_name=STRATEGY_CLASS, + trading_mode=mode, + config=config, + ), + compile_tag_program( + source, + class_name=STRATEGY_CLASS, + trading_mode=mode, + config=config, + ), + ) + required = sorted( + { + column + for program_pair in programs.values() + for program in program_pair + for column in cast(list[str], program["required_input_columns"]) + } + ) + cases: dict[str, Any] = {} + for case_name, roc_9_1d in _CASES.items(): + frame = _boundary_frame(required, roc_9_1d) + case: dict[str, Any] = {} + for label, source in (("current", current), ("baseline", baseline)): + signal_program, tag_program = programs[label] + source_output = _source_output(source, frame, mode) + python_signal = execute_signal_program( + signal_program, + frame, + metadata={"pair": _pair(mode)}, + ) + python_tag = execute_tag_program( + tag_program, + frame, + metadata={"pair": _pair(mode)}, + ) + numeric = _numeric_columns(frame) + rust_signal = _rust.execute_numeric_mutation_program( + _compact_json(signal_program), + numeric, + {"pair": _pair(mode)}, + list(SIGNAL_COLUMNS), + ) + rust_tag = _rust.execute_numeric_mutation_program( + _compact_json(tag_program), + numeric, + {"pair": _pair(mode)}, + [*SIGNAL_COLUMNS, *TAG_COLUMNS], + ) + expected = _encode_columns(source_output, (*SIGNAL_COLUMNS, *TAG_COLUMNS)) + _assert_exact(expected, _encode_columns(python_signal, SIGNAL_COLUMNS), SIGNAL_COLUMNS) + _assert_exact( + expected, + _encode_columns(python_tag, (*SIGNAL_COLUMNS, *TAG_COLUMNS)), + (*SIGNAL_COLUMNS, *TAG_COLUMNS), + ) + _assert_exact(expected, _bridge_values(rust_signal), SIGNAL_COLUMNS) + _assert_exact( + expected, + _bridge_values(rust_tag), + (*SIGNAL_COLUMNS, *TAG_COLUMNS), + ) + case[label] = expected + cases[case_name] = case + return { + "required_input_column_count": len(required), + "required_input_columns_sha256": hashlib.sha256("\n".join(required).encode()).hexdigest(), + "programs": { + label: { + "signal_fingerprint": pair[0]["fingerprint"], + "signal_node_count": len(cast(list[Any], pair[0]["nodes"])), + "tag_fingerprint": pair[1]["fingerprint"], + "tag_node_count": len(cast(list[Any], pair[1]["nodes"])), + } + for label, pair in programs.items() + }, + "cases": cases, + } + + +def _boundary_frame(required: list[str], roc_9_1d: float) -> pd.DataFrame: + rows = 8 + frame = pd.DataFrame( + {name: np.zeros(rows, dtype=np.float64) for name in required} + ) + for name, value in _SCALAR_OVERRIDES.items(): + frame[name] = value + frame["ROC_9_1d"] = roc_9_1d + for name, values in _SERIES_OVERRIDES.items(): + frame[name] = np.asarray(values, dtype=np.float64) + return frame + + +def _source_output(source: Path, frame: pd.DataFrame, mode: str) -> pd.DataFrame: + strategy_class = load_strategy_class(source, STRATEGY_CLASS) + config = prepare_worker_config( + { + "exchange": {"name": "binance"}, + "max_open_trades": 6, + "stake_currency": "USDT", + "trading_mode": mode, + }, + user_data_directory=Path(".nfi/signal47-fixture-worker"), + ) + strategy = strategy_class(config) + strategy_runtime = cast(Any, strategy) + pair = _pair(mode) + strategy_runtime.dp = VectorDataProvider({}, [pair]) + strategy_runtime.long_entry_signal_params = {"long_entry_condition_47_enable": True} + strategy_runtime.short_entry_signal_params = {} + return _advise_signals(strategy_runtime, frame.copy(deep=True), {"pair": pair}) + + +def _source_identity(source: Path, commit: str) -> dict[str, str]: + return { + "commit": commit, + "strategy_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + } + + +def _validate_sources(current: Path, baseline: Path) -> None: + current_text = current.read_text(encoding="utf-8") + baseline_text = baseline.read_text(encoding="utf-8") + if CHANGED_EXPRESSION not in current_text or CHANGED_EXPRESSION in baseline_text: + raise ValueError("the supplied sources do not isolate the Signal 47 protection change") + + +def _source_method_diff(current: Path, baseline: Path) -> dict[str, Any]: + current_methods = _method_inventory(current) + baseline_methods = _method_inventory(baseline) + if set(current_methods) != set(baseline_methods): + raise ValueError("the source revisions expose different method inventories") + changed = sorted( + name + for name in current_methods + if current_methods[name] != baseline_methods[name] + ) + if changed != ["populate_entry_trend", "version"]: + raise ValueError(f"the source revisions changed unexpected methods: {changed}") + unchanged = { + name: current_methods[name] for name in sorted(current_methods) if name not in changed + } + return { + "changed_methods": { + name: { + "baseline_ast_sha256": baseline_methods[name], + "current_ast_sha256": current_methods[name], + } + for name in changed + }, + "unchanged_method_count": len(unchanged), + "unchanged_methods_fingerprint": hashlib.sha256( + _compact_json(unchanged).encode() + ).hexdigest(), + "callback_and_stateful_methods_changed": False, + } + + +def _method_inventory(source: Path) -> dict[str, str]: + tree = ast.parse(source.read_text(encoding="utf-8"), filename=str(source)) + methods: dict[str, str] = {} + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + normalized = ast.dump(node, include_attributes=False) + methods[node.name] = hashlib.sha256(normalized.encode()).hexdigest() + return methods + + +def _pair(mode: str) -> str: + return "TEST/USDT:USDT" if mode == "futures" else "TEST/USDT" + + +def _numeric_columns(frame: pd.DataFrame) -> dict[str, list[float | None]]: + return { + str(name): [None if pd.isna(value) else float(value) for value in frame[name]] + for name in frame.columns + } + + +def _encode_columns(frame: pd.DataFrame, names: tuple[str, ...]) -> dict[str, list[Any]]: + return { + name: [ + None + if pd.isna(value) + else value.item() + if hasattr(value, "item") + else value + for value in frame[name] + ] + for name in names + } + + +def _bridge_values(output: Mapping[str, Any]) -> dict[str, list[Any]]: + return { + name: cast(list[Any], cast(Mapping[str, Any], value)["values"]) + for name, value in output.items() + } + + +def _assert_exact( + expected: Mapping[str, list[Any]], + actual: Mapping[str, list[Any]], + names: tuple[str, ...], +) -> None: + for name in names: + if actual[name] != expected[name]: + raise ValueError(f"Signal 47 qualification differs for {name}") + + +def _compact_json(document: Mapping[str, Any]) -> str: + return json.dumps(document, ensure_ascii=False, separators=(",", ":")) diff --git a/python/nfi_backtest_engine/research_runner.py b/python/nfi_backtest_engine/research_runner.py index 18f47071..ed62f2a4 100644 --- a/python/nfi_backtest_engine/research_runner.py +++ b/python/nfi_backtest_engine/research_runner.py @@ -19,9 +19,17 @@ sanitize_config, ) from .data_seal import DATA_SEAL_VERSION, prepare_data, validate_data_seal -from .engine_runtime import run_engine +from .engine_runtime import FULL_VECTOR_INPUT, run_engine from .errors import BenchmarkError, SpecValidationError, StrategyAnalysisError from .fixture import sha256_file +from .full_native_calibration import resolve_full_native_pair_workers +from .full_vector_runtime import ( + FULL_NATIVE_RUNTIME_MODE, + FULL_NATIVE_SOURCE_MODE, + FULL_NATIVE_VECTOR_RUNTIME_VERSION, + build_full_native_vector_manifest, + compile_full_native_programs, +) from .generic_adapter import ( GENERIC_ADAPTER_VERSION, build_generic_vector_manifest, @@ -46,9 +54,11 @@ compile_state_machine_program, ) from .stateful_execution_policy import ( + FULL_NATIVE_VECTOR_TRANSPORT, GENERIC_VECTOR_TRANSPORT, STATEFUL_EXECUTION_POLICY_VERSION, X7_VECTOR_TRANSPORT, + add_native_execution_blockers, build_native_execution_policy, ) from .strategy_ir import STRATEGY_IR_VERSION @@ -162,6 +172,24 @@ def run_research_backtest( ) adapter_lane = native_execution["adapter_lane"] vector_transport = native_execution["transport"] + full_native_programs: dict[str, dict[str, Any]] | None = None + if vector_transport == FULL_NATIVE_VECTOR_TRANSPORT and not native_execution["blockers"]: + try: + full_native_programs = compile_full_native_programs( + source, + class_name=class_name, + config=run_config, + ) + except StrategyAnalysisError as exc: + native_execution = add_native_execution_blockers( + native_execution, + [ + { + "code": "FULL_NATIVE_SOURCE_COMPILER_UNSUPPORTED", + "message": str(exc), + } + ], + ) if execution_profile is None: profile = ensure_execution_profile(profile_path, workspace=output) else: @@ -234,6 +262,7 @@ def run_research_backtest( "generic_adapter_version": GENERIC_ADAPTER_VERSION, "x7_adapter_version": X7_ADAPTER_VERSION, "stateful_execution_policy_version": STATEFUL_EXECUTION_POLICY_VERSION, + "full_native_vector_runtime_version": FULL_NATIVE_VECTOR_RUNTIME_VERSION, }, "strategy": { "path": str(source), @@ -426,6 +455,7 @@ def run_research_backtest( vector_directory = output / "vectors" vector_checkpoint = output / "checkpoints" / "vectors.json" vector_report = None + full_native_calibration: dict[str, Any] | None = None resumed_vector_stage = False if existing_run is not None and not vector_checkpoint.is_file(): raise BenchmarkError( @@ -438,23 +468,31 @@ def run_research_backtest( vector_report = candidate["report"] resumed_vector_stage = True if vector_report is None: - _reset_owned_directory(vector_directory, root=output) - with execution_environment(profile["environment"]): - vector_report = prepare_vector_signals( - strategy_path=source, - class_name=class_name, - config=run_config, - pairs=pairlist["pairs"], - data_directory=data_root, - timerange=timerange, - output_directory=vector_directory, - workers=selected_workers, - cache_directory=cache_directory, - memory_cap_bytes=int(resource_limits["working_memory_bytes"]), - hardware_fingerprint=profile["hardware_fingerprint"], - calibration_directory=Path(profile_path).resolve().parent / "calibrations", - recalibrate=recalibrate, + if vector_transport == FULL_NATIVE_VECTOR_TRANSPORT: + vector_report = _full_native_preflight_report( + analysis=analysis, + programs=full_native_programs, + pair_count=len(pairlist["pairs"]), + blockers=native_execution["blockers"], ) + else: + _reset_owned_directory(vector_directory, root=output) + with execution_environment(profile["environment"]): + vector_report = prepare_vector_signals( + strategy_path=source, + class_name=class_name, + config=run_config, + pairs=pairlist["pairs"], + data_directory=data_root, + timerange=timerange, + output_directory=vector_directory, + workers=selected_workers, + cache_directory=cache_directory, + memory_cap_bytes=int(resource_limits["working_memory_bytes"]), + hardware_fingerprint=profile["hardware_fingerprint"], + calibration_directory=Path(profile_path).resolve().parent / "calibrations", + recalibrate=recalibrate, + ) write_json( vector_checkpoint, { @@ -475,7 +513,7 @@ def run_research_backtest( blockers = [] if state_machine_ready else list(hot_ir["blockers"]) blockers.extend(native_execution["blockers"]) if not blockers and not prepare_only: - if vector_transport == X7_VECTOR_TRANSPORT: + if vector_transport in {X7_VECTOR_TRANSPORT, FULL_NATIVE_VECTOR_TRANSPORT}: blockers.extend( x7_adapter_blockers( analysis, @@ -547,7 +585,25 @@ def run_research_backtest( else: _require_absent(simulation_input_path, label="simulation input") stage_started_ns = time.perf_counter_ns() - if vector_transport == X7_VECTOR_TRANSPORT: + if vector_transport == FULL_NATIVE_VECTOR_TRANSPORT: + if full_native_programs is None: + raise BenchmarkError( + "full native manifest reached execution without compiled programs" + ) + build_full_native_vector_manifest( + strategy_path=source, + class_name=class_name, + analysis=analysis, + hot_ir=hot_ir, + config=run_config, + pairs=pairlist["pairs"], + data_directory=data_root, + timerange=timerange, + market_metadata_path=selected_market_metadata, + destination=simulation_input_path, + compiled_programs=full_native_programs, + ) + elif vector_transport == X7_VECTOR_TRANSPORT: build_x7_vector_manifest( analysis=analysis, hot_ir=hot_ir, @@ -578,6 +634,9 @@ def run_research_backtest( execution = engine_stage["execution"] simulation_result_record = engine_stage["artifact"] engine_events_record = engine_stage.get("engine_events") + stored_calibration = engine_stage.get("workload_calibration") + if isinstance(stored_calibration, dict): + full_native_calibration = stored_calibration resumed_engine_stage = True else: _require_absent(simulation_result_path, label="simulation result") @@ -585,13 +644,32 @@ def run_research_backtest( if engine_events_path is not None: _require_absent(engine_events_path, label="engine events") stage_started_ns = time.perf_counter_ns() + engine_arguments: dict[str, Any] = { + "profile_path": profile_path, + "engine_profile_path": engine_profile_path, + "events_path": engine_events_path, + } + if vector_transport == FULL_NATIVE_VECTOR_TRANSPORT: + full_native_calibration = resolve_full_native_pair_workers( + simulation_input_path, + profile_path=profile_path, + hardware_fingerprint=str(profile["hardware_fingerprint"]), + requested_workers=selected_workers, + memory_cap_bytes=resource_limits["memory_cap_bytes"], + calibration_directory=Path(profile_path).resolve().parent + / "calibrations", + recalibrate=recalibrate, + ) + engine_arguments["input_kind"] = FULL_VECTOR_INPUT + engine_arguments["pair_worker_limit"] = full_native_calibration[ + "worker_limit" + ] + else: + engine_arguments["vector_manifest"] = True execution = run_engine( simulation_input_path, simulation_result_path, - profile_path=profile_path, - vector_manifest=True, - engine_profile_path=engine_profile_path, - events_path=engine_events_path, + **engine_arguments, ) engine_seconds = _elapsed_seconds(stage_started_ns) simulation_result_record = _artifact_record(simulation_result_path) @@ -606,6 +684,7 @@ def run_research_backtest( "execution": execution, "engine_profile": _artifact_record(engine_profile_path), "engine_events": engine_events_record, + "workload_calibration": full_native_calibration, } _write_simulation_checkpoint( simulation_checkpoint_path, @@ -711,7 +790,11 @@ def run_research_backtest( "indicator_workers": vector_report["worker_count"], "cpu_process_limit": safe_workers, "working_memory_bytes": resource_limits["working_memory_bytes"], - "workload_calibration": vector_report.get("calibration"), + "workload_calibration": ( + full_native_calibration + if vector_transport == FULL_NATIVE_VECTOR_TRANSPORT + else vector_report.get("calibration") + ), "portfolio_simulator_threads": profile["runtime"][ "portfolio_simulator_threads" ], @@ -999,6 +1082,7 @@ def _load_simulation_checkpoint( expected_path=simulation_input_path, label="simulation input", ) + _validate_full_native_manifest_artifacts(simulation_input_path) if engine is None: if surface is not None: @@ -1082,10 +1166,76 @@ def _require_absent(path: Path, *, label: str) -> None: raise BenchmarkError(f"{label} already exists; refusing to overwrite it: {path}") +def _full_native_preflight_report( + *, + analysis: dict[str, Any], + programs: dict[str, dict[str, Any]] | None, + pair_count: int, + blockers: list[dict[str, Any]], +) -> dict[str, Any]: + compiled = programs or {} + return { + "schema_version": "full-native-program-preflight-v1", + "pipeline_version": FULL_NATIVE_VECTOR_RUNTIME_VERSION, + "strategy_sha256": analysis["source"]["sha256"], + "source_execution": { + "strategy_source_mode": FULL_NATIVE_SOURCE_MODE, + "populate_methods_executed": False, + "runtime_mode": FULL_NATIVE_RUNTIME_MODE, + }, + "worker_count": 0, + "pair_count": pair_count, + "cache_hits": 0, + "outputs": [], + "programs": { + name: { + "schema_version": program["schema_version"], + "fingerprint": program["fingerprint"], + "node_count": len(program["nodes"]), + } + for name, program in sorted(compiled.items()) + }, + "blockers": blockers, + } + + def _valid_vector_checkpoint(checkpoint: Any, vector_directory: Path) -> bool: if not isinstance(checkpoint, dict) or checkpoint.get("schema_version") != "1.0.0": return False report = checkpoint.get("report") + if isinstance(report, dict) and report.get("schema_version") == ( + "full-native-program-preflight-v1" + ): + programs = report.get("programs") + blockers = report.get("blockers") + return ( + report.get("pipeline_version") == FULL_NATIVE_VECTOR_RUNTIME_VERSION + and isinstance(report.get("strategy_sha256"), str) + and report.get("source_execution") + == { + "strategy_source_mode": FULL_NATIVE_SOURCE_MODE, + "populate_methods_executed": False, + "runtime_mode": FULL_NATIVE_RUNTIME_MODE, + } + and report.get("worker_count") == 0 + and isinstance(report.get("pair_count"), int) + and not isinstance(report.get("pair_count"), bool) + and report["pair_count"] > 0 + and report.get("cache_hits") == 0 + and report.get("outputs") == [] + and isinstance(programs, dict) + and isinstance(blockers, list) + and all( + isinstance(program, dict) + and isinstance(program.get("schema_version"), str) + and isinstance(program.get("fingerprint"), str) + and len(program["fingerprint"]) == 64 + and isinstance(program.get("node_count"), int) + and program["node_count"] > 0 + for program in programs.values() + ) + and (bool(blockers) or set(programs) == {"indicator", "signal", "tag"}) + ) if ( not isinstance(report, dict) or report.get("pipeline_version") != VECTOR_PIPELINE_VERSION @@ -1228,6 +1378,52 @@ def _relative_artifact_record(path: Path, *, root: Path) -> dict[str, Any]: return record +def _validate_full_native_manifest_artifacts(path: Path) -> None: + document = _read_json_object(path, label="simulation input") + if document.get("schema_version") != "full-native-vector-manifest-v1": + return + programs = document.get("programs") + frames = document.get("frames") + futures = document.get("futures") + if not isinstance(programs, dict) or not isinstance(frames, list) or not isinstance( + futures, list | type(None) + ): + raise BenchmarkError("full native manifest artifact inventory is malformed") + artifacts: list[Any] = [] + for name in ("indicator", "signal", "tag"): + program = programs.get(name) + artifacts.append(program.get("artifact") if isinstance(program, dict) else None) + artifacts.extend( + frame.get("artifact") if isinstance(frame, dict) else None for frame in frames + ) + for descriptor in futures or []: + if not isinstance(descriptor, dict): + artifacts.append(None) + continue + for role in ("funding_rate", "mark"): + frame = descriptor.get(role) + artifacts.append(frame.get("artifact") if isinstance(frame, dict) else None) + + root = path.parent.resolve() + seen: set[Path] = set() + for artifact in artifacts: + if not isinstance(artifact, dict): + raise BenchmarkError("full native manifest artifact descriptor is malformed") + relative = artifact.get("path") + expected_sha256 = artifact.get("sha256") + if not isinstance(relative, str) or not isinstance(expected_sha256, str): + raise BenchmarkError("full native manifest artifact descriptor is malformed") + candidate = Path(relative) + if candidate.is_absolute() or ".." in candidate.parts: + raise BenchmarkError("full native manifest artifact path is not contained") + resolved = (root / candidate).resolve() + if not resolved.is_relative_to(root) or resolved in seen or not resolved.is_file(): + raise BenchmarkError("full native manifest artifact is missing or duplicated") + seen.add(resolved) + if sha256_file(resolved) != expected_sha256: + raise BenchmarkError("full native manifest artifact SHA-256 differs") + + def required_data_pairs( pairlist: dict[str, Any], run_config: dict[str, Any], diff --git a/python/nfi_backtest_engine/schemas/full-native-vector-manifest-v1.schema.json b/python/nfi_backtest_engine/schemas/full-native-vector-manifest-v1.schema.json new file mode 100644 index 00000000..a53409dd --- /dev/null +++ b/python/nfi_backtest_engine/schemas/full-native-vector-manifest-v1.schema.json @@ -0,0 +1,250 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/vntrevx/NFI_BackTestEngine/schemas/full-native-vector-manifest-v1.schema.json", + "title": "Full Native Vector Manifest v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "source", + "source_execution", + "config", + "compile_context", + "programs", + "run", + "retained_features", + "pairs", + "frames", + "futures" + ], + "properties": { + "schema_version": {"const": "full-native-vector-manifest-v1"}, + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "strategy_sha256", + "config_sha256", + "compiler_source_fingerprint", + "selected_class" + ], + "properties": { + "strategy_sha256": {"$ref": "#/$defs/sha256"}, + "config_sha256": {"$ref": "#/$defs/sha256"}, + "compiler_source_fingerprint": {"$ref": "#/$defs/sha256"}, + "selected_class": {"$ref": "#/$defs/non_empty_string"} + } + }, + "source_execution": { + "type": "object", + "additionalProperties": false, + "required": [ + "strategy_source_mode", + "populate_methods_executed", + "runtime_mode" + ], + "properties": { + "strategy_source_mode": {"const": "python-ast-compile-only"}, + "populate_methods_executed": {"const": false}, + "runtime_mode": {"const": "rust-full-native"} + } + }, + "config": {"type": "object"}, + "compile_context": { + "type": "object", + "additionalProperties": false, + "required": ["run_mode", "trading_mode"], + "properties": { + "run_mode": {"const": "backtest"}, + "trading_mode": {"enum": ["spot", "futures"]} + } + }, + "programs": { + "type": "object", + "additionalProperties": false, + "required": ["indicator", "signal", "tag"], + "properties": { + "indicator": {"$ref": "#/$defs/program"}, + "signal": {"$ref": "#/$defs/program"}, + "tag": {"$ref": "#/$defs/program"} + } + }, + "run": { + "type": "object", + "additionalProperties": false, + "required": [ + "trading_mode", + "timerange", + "startup_candles", + "base_timeframe", + "source_row_shift" + ], + "properties": { + "trading_mode": {"enum": ["spot", "futures"]}, + "timerange": { + "type": "object", + "additionalProperties": false, + "required": ["start_ms", "stop_ms"], + "properties": { + "start_ms": {"type": "integer", "minimum": 0}, + "stop_ms": {"type": "integer", "minimum": 0} + } + }, + "startup_candles": {"type": "integer", "minimum": 0}, + "base_timeframe": {"$ref": "#/$defs/non_empty_string"}, + "source_row_shift": {"const": 1} + } + }, + "retained_features": { + "type": "object", + "additionalProperties": false, + "required": ["columns", "fingerprint"], + "properties": { + "columns": {"$ref": "#/$defs/unique_strings"}, + "fingerprint": {"$ref": "#/$defs/sha256"} + } + }, + "pairs": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/pair"} + }, + "frames": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/frame"} + }, + "futures": { + "oneOf": [ + {"type": "null"}, + { + "type": "array", + "items": {"$ref": "#/$defs/futures"} + } + ] + } + }, + "$defs": { + "non_empty_string": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "unique_strings": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/non_empty_string"} + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": {"$ref": "#/$defs/non_empty_string"}, + "sha256": {"$ref": "#/$defs/sha256"} + } + }, + "program": { + "type": "object", + "additionalProperties": false, + "required": ["artifact", "fingerprint"], + "properties": { + "artifact": {"$ref": "#/$defs/artifact"}, + "fingerprint": {"$ref": "#/$defs/sha256"} + } + }, + "identity": { + "type": "object", + "additionalProperties": false, + "required": ["pair", "timeframe"], + "properties": { + "pair": {"$ref": "#/$defs/non_empty_string"}, + "timeframe": {"$ref": "#/$defs/non_empty_string"} + } + }, + "nullable_number": {"type": ["number", "null"]}, + "frame": { + "type": "object", + "additionalProperties": false, + "required": ["identity", "rows", "artifact"], + "properties": { + "identity": {"$ref": "#/$defs/identity"}, + "rows": {"type": "integer", "minimum": 0}, + "artifact": {"$ref": "#/$defs/artifact"} + } + }, + "pair": { + "type": "object", + "additionalProperties": false, + "required": [ + "identity", + "metadata", + "precision", + "limits", + "price_steps", + "options" + ], + "properties": { + "identity": {"$ref": "#/$defs/identity"}, + "metadata": { + "type": "object", + "additionalProperties": {"type": "string"} + }, + "precision": { + "type": "object", + "additionalProperties": false, + "required": ["amount_step", "price_step"], + "properties": { + "amount_step": {"$ref": "#/$defs/nullable_number"}, + "price_step": {"$ref": "#/$defs/nullable_number"} + } + }, + "limits": { + "type": "object", + "additionalProperties": false, + "required": ["minimum_stake", "minimum_amount", "minimum_cost"], + "properties": { + "minimum_stake": {"$ref": "#/$defs/nullable_number"}, + "minimum_amount": {"$ref": "#/$defs/nullable_number"}, + "minimum_cost": {"$ref": "#/$defs/nullable_number"} + } + }, + "price_steps": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["timestamp_ms", "step"], + "properties": { + "timestamp_ms": {"type": "integer", "minimum": 0}, + "step": {"type": "number", "exclusiveMinimum": 0} + } + } + }, + "options": { + "type": "object", + "additionalProperties": false, + "required": [ + "can_short", + "include_funding", + "use_exit_signal", + "include_previous_close" + ], + "properties": { + "can_short": {"type": "boolean"}, + "include_funding": {"type": "boolean"}, + "use_exit_signal": {"type": "boolean"}, + "include_previous_close": {"type": "boolean"} + } + } + } + }, + "futures": { + "type": "object", + "additionalProperties": false, + "required": ["pair", "funding_rate", "mark"], + "properties": { + "pair": {"$ref": "#/$defs/non_empty_string"}, + "funding_rate": {"$ref": "#/$defs/frame"}, + "mark": {"$ref": "#/$defs/frame"} + } + } + } +} diff --git a/python/nfi_backtest_engine/schemas/indicator-program-v1.schema.json b/python/nfi_backtest_engine/schemas/indicator-program-v1.schema.json index fd3cc66c..bb452ad2 100644 --- a/python/nfi_backtest_engine/schemas/indicator-program-v1.schema.json +++ b/python/nfi_backtest_engine/schemas/indicator-program-v1.schema.json @@ -144,7 +144,9 @@ "string-scalar", "json-scalar", "bool-column", + "int-column", "f64-column", + "string-column", "timestamp-column" ] }, @@ -152,11 +154,19 @@ "enum": [ "parameter", "literal", + "row-count", + "row-index", + "string-split-index", + "frame-source", + "frame-nonempty", + "frame-project", + "frame-drop-if-present", "column-read", "metadata-read", "column-write", "binary", "compare", + "membership", "logical", "unary", "select", diff --git a/python/nfi_backtest_engine/schemas/signal-program-v1.schema.json b/python/nfi_backtest_engine/schemas/signal-program-v1.schema.json index 43c63d89..7b6bba44 100644 --- a/python/nfi_backtest_engine/schemas/signal-program-v1.schema.json +++ b/python/nfi_backtest_engine/schemas/signal-program-v1.schema.json @@ -136,7 +136,9 @@ "string-scalar", "json-scalar", "bool-column", + "int-column", "f64-column", + "string-column", "timestamp-column" ] }, @@ -144,11 +146,14 @@ "enum": [ "parameter", "literal", + "row-count", + "string-split-index", "column-read", "metadata-read", "frame-write", "binary", "compare", + "membership", "logical", "unary", "select", diff --git a/python/nfi_backtest_engine/schemas/tag-program-v1.schema.json b/python/nfi_backtest_engine/schemas/tag-program-v1.schema.json index 3e36915b..b4f31c10 100644 --- a/python/nfi_backtest_engine/schemas/tag-program-v1.schema.json +++ b/python/nfi_backtest_engine/schemas/tag-program-v1.schema.json @@ -137,6 +137,7 @@ "string-scalar", "json-scalar", "bool-column", + "int-column", "f64-column", "string-column", "timestamp-column" @@ -146,12 +147,16 @@ "enum": [ "parameter", "literal", + "row-count", + "string-split-index", "column-read", "metadata-read", "frame-write", + "masked-string-append", "format-string", "binary", "compare", + "membership", "logical", "unary", "select", diff --git a/python/nfi_backtest_engine/signal_program/compiler.py b/python/nfi_backtest_engine/signal_program/compiler.py index 7f0e69a7..140558f1 100644 --- a/python/nfi_backtest_engine/signal_program/compiler.py +++ b/python/nfi_backtest_engine/signal_program/compiler.py @@ -8,8 +8,16 @@ from pathlib import Path from typing import Any, Never +from .._indicator_ast import ( + _declared_class_constants, + _effective_backtest_config, + _recognized_tag_appender, +) +from .._indicator_contract import _cast_result_type from ..errors import StrategyAnalysisError -from ..indicator_program import _Compiler as _VectorExpressionCompiler +from ..indicator_program import ( + _Compiler as _VectorExpressionCompiler, +) from ..strategy_ir import analyze_strategy from .validation import ( SIGNAL_COLUMNS, @@ -26,6 +34,7 @@ "int-scalar", "f64-scalar", "bool-column", + "int-column", "f64-column", } @@ -39,6 +48,7 @@ def compile_signal_program( *, class_name: str | None = None, trading_mode: str = "spot", + config: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Compile entry and exit DataFrame mutations without executing strategy Python.""" if trading_mode not in {"spot", "futures"}: @@ -69,6 +79,9 @@ def compile_signal_program( for node in class_node.body if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) } + module_functions = { + node.name: node for node in tree.body if isinstance(node, ast.FunctionDef) + } for method_name in ("populate_entry_trend", "populate_exit_trend"): method = methods.get(method_name) if method is None: @@ -77,10 +90,26 @@ def compile_signal_program( _unsupported(method, "async signal entrypoint") constants = strategy.get("constants", {}) + class_constants = _declared_class_constants( + class_node, + constants if isinstance(constants, Mapping) else {}, + ) + effective_config = dict(config or {}) + configured_mode = effective_config.get("trading_mode") + if configured_mode is not None and configured_mode != trading_mode: + raise SignalProgramCompileError( + "signal trading mode differs from the supplied configuration" + ) + effective_config["trading_mode"] = trading_mode compiler = _SignalCompiler( path=path, methods=methods, - class_constants=constants if isinstance(constants, Mapping) else {}, + class_constants=class_constants, + instance_constants={ + "config": _effective_backtest_config(effective_config), + "dp": {"runmode": {"value": "backtest"}}, + }, + module_functions=module_functions, ) # Reserve the two public IDs before helper discovery so their identity never # depends on how many helper functions one phase happens to call. @@ -142,17 +171,37 @@ def __init__( path: Path, methods: Mapping[str, ast.FunctionDef | ast.AsyncFunctionDef], class_constants: Mapping[str, Any], + instance_constants: Mapping[str, Any] | None = None, + module_functions: Mapping[str, ast.FunctionDef] | None = None, ) -> None: - super().__init__(path=path, methods=methods, class_constants=class_constants) + super().__init__( + path=path, + methods=methods, + class_constants=class_constants, + instance_constants=instance_constants, + module_functions=module_functions, + ) self.current_phase = "entry" + self.compile_tags = False self.mutation_nodes: list[str] = [] self.final_mutations: dict[str, str] = {} def statement(self, node: ast.stmt) -> None: + if ( + not self.compile_tags + and isinstance(node, ast.Expr) + and isinstance(node.value, ast.Call) + ): + callable_name = self.resolved_callable(node.value.func) + function = self.module_functions.get(callable_name) + if function is not None and _recognized_tag_appender(function): + return if isinstance(node, ast.Assign) and len(node.targets) == 1: target = node.targets[0] if self._is_loc_target(target): assert isinstance(target, ast.Subscript) + if not self.compile_tags and _loc_targets_only_tags(target): + return self._loc_write(target, node.value, node) return super().statement(node) @@ -191,7 +240,7 @@ def method_call(self, node: ast.Call, callable_name: str) -> str: return self.emit( node, "cast", - self.node_types[base], + _cast_result_type(self.node_types[base], target), inputs=[base], parameters={"target": target}, lookback=self.lookback(base), @@ -346,6 +395,13 @@ def _is_full_slice(node: ast.expr) -> bool: ) +def _loc_targets_only_tags(target: ast.Subscript) -> bool: + if not isinstance(target.slice, ast.Tuple) or len(target.slice.elts) != 2: + return False + columns = _literal_columns(target.slice.elts[1]) + return bool(columns) and all(column in {"enter_tag", "exit_tag"} for column in columns) + + def _cast_target(node: ast.expr) -> str | None: if isinstance(node, ast.Name) and node.id in {"bool", "float", "int"}: return node.id diff --git a/python/nfi_backtest_engine/signal_program/runtime.py b/python/nfi_backtest_engine/signal_program/runtime.py index 25a464be..b4910880 100644 --- a/python/nfi_backtest_engine/signal_program/runtime.py +++ b/python/nfi_backtest_engine/signal_program/runtime.py @@ -79,7 +79,18 @@ def _node( inputs = [values[input_id] for input_id in node["inputs"]] options = node["parameters"] if opcode == "literal": + special = options.get("special") + if special is not None: + return { + "nan": float("nan"), + "+infinity": float("inf"), + "-infinity": float("-inf"), + }[special] return options.get("value") + if opcode == "row-count": + return len(_require_frame(inputs[0], node["id"])) + if opcode == "string-split-index": + return _string_split_index(inputs[0], options, node["id"]) if opcode == "column-read": frame = _require_frame(inputs[0], node["id"]) column = options["column"] @@ -104,6 +115,10 @@ def _node( return _binary(options["operator"], inputs[0], inputs[1]) if opcode == "compare": return _compare(options["operator"], inputs[0], inputs[1]) + if opcode == "membership": + return _membership(inputs[0], options) + if opcode == "masked-string-append": + return _masked_string_append(inputs, node["id"]) if opcode == "logical": return _logical(options["operator"], inputs) if opcode == "unary": @@ -233,6 +248,35 @@ def _compare(name: str, left: Any, right: Any) -> Any: return operation(left, right) +def _membership(value: Any, options: Mapping[str, Any]) -> Any: + collection = options["values"] + negated = bool(options["negated"]) + if isinstance(value, pd.Series): + result = value.isin(collection) + return ~result if negated else result + if isinstance(value, np.ndarray): + result = np.isin(value, collection) + return ~result if negated else result + result = value in collection + return not result if negated else result + + +def _masked_string_append(inputs: Sequence[Any], node_id: str) -> np.ndarray[Any, Any]: + if len(inputs) != 3 or not isinstance(inputs[2], str): + raise SignalProgramExecutionError(f"signal node {node_id} tag append is invalid") + target = np.asarray(inputs[0], dtype=object).copy() + mask = inputs[1] + if isinstance(mask, bool | np.bool_): + if mask: + target[:] = target + inputs[2] + return target + selected = np.asarray(mask) + if selected.ndim != 1 or len(selected) != len(target) or not is_bool_dtype(selected.dtype): + raise SignalProgramExecutionError(f"signal node {node_id} tag append mask is invalid") + target[selected] = target[selected] + inputs[2] + return target + + def _logical(name: str, values: Sequence[Any]) -> Any: if not values: raise SignalProgramExecutionError("signal logical operation has no inputs") @@ -275,7 +319,40 @@ def _scalar_call(name: str, values: Sequence[Any]) -> Any: def _array_call(name: str, values: Sequence[Any], arguments: Mapping[str, Any]) -> Any: + if name == "full_like": + if len(values) != 2 or arguments: + raise SignalProgramExecutionError("invalid NumPy full_like contract") + return np.full_like(values[0], values[1]) + if name == "divide": + if len(values) != 4 or arguments: + raise SignalProgramExecutionError("invalid NumPy divide contract") + output = np.array(values[2], copy=True) + return np.divide(values[0], values[1], out=output, where=values[3]) + if name == "absolute-difference": + if len(values) != 1 or arguments: + raise SignalProgramExecutionError("invalid NumPy absolute-difference contract") + source = np.asarray(values[0], dtype=np.float64) + output = np.full_like(source, np.nan) + if len(source) > 1: + output[1:] = np.abs(np.diff(source)) + return output function = getattr(np, name, None) if function is None or not callable(function): raise SignalProgramExecutionError(f"unknown signal NumPy call {name!r}") return function(*values, **arguments) + + +def _string_split_index(value: Any, options: Mapping[str, Any], node_id: str) -> str: + if not isinstance(value, str): + raise SignalProgramExecutionError(f"signal node {node_id} string source is invalid") + method = options["method"] + separator = options["separator"] + index = options["index"] + if method not in {"partition", "split", "rsplit"} or not isinstance(separator, str): + raise SignalProgramExecutionError(f"signal node {node_id} string contract is invalid") + try: + return getattr(value, method)(separator)[index] + except (IndexError, TypeError) as exc: + raise SignalProgramExecutionError( + f"signal node {node_id} string result index is invalid" + ) from exc diff --git a/python/nfi_backtest_engine/signal_program/validation.py b/python/nfi_backtest_engine/signal_program/validation.py index bb111d3e..16244d13 100644 --- a/python/nfi_backtest_engine/signal_program/validation.py +++ b/python/nfi_backtest_engine/signal_program/validation.py @@ -29,6 +29,7 @@ "int-scalar", "f64-scalar", "bool-column", + "int-column", "f64-column", } diff --git a/python/nfi_backtest_engine/specs.py b/python/nfi_backtest_engine/specs.py index 37c5e7dc..17d1db3a 100644 --- a/python/nfi_backtest_engine/specs.py +++ b/python/nfi_backtest_engine/specs.py @@ -37,6 +37,7 @@ INDICATOR_PROGRAM_SCHEMA = "indicator-program-v1.schema.json" SIGNAL_PROGRAM_SCHEMA = "signal-program-v1.schema.json" TAG_PROGRAM_SCHEMA = "tag-program-v1.schema.json" +FULL_NATIVE_VECTOR_MANIFEST_SCHEMA = "full-native-vector-manifest-v1.schema.json" STATEFUL_COVERAGE_SCHEMA = "stateful-coverage-v1.schema.json" FREQTRADE_SEMANTIC_PROFILE_SCHEMA = "freqtrade-semantic-profile-v1.schema.json" SEMANTIC_OBSERVER_REPORT_SCHEMA = "semantic-observer-report-v1.schema.json" diff --git a/python/nfi_backtest_engine/stateful_execution_policy.py b/python/nfi_backtest_engine/stateful_execution_policy.py index b99b2682..0356380d 100644 --- a/python/nfi_backtest_engine/stateful_execution_policy.py +++ b/python/nfi_backtest_engine/stateful_execution_policy.py @@ -9,10 +9,11 @@ from .errors import StrategyAnalysisError from .x7.serialization import _nfi_trade_manager_config -STATEFUL_EXECUTION_POLICY_VERSION = "1.1.0" +STATEFUL_EXECUTION_POLICY_VERSION = "1.2.0" X7_GENERIC_STATEFUL_LANE = "x7-generic-stateful" X7_VECTOR_TRANSPORT = "x7-vector-manifest" GENERIC_VECTOR_TRANSPORT = "generic-vector-manifest" +FULL_NATIVE_VECTOR_TRANSPORT = "full-native-vector-manifest" _STATEFUL_PROGRAM_SCHEMA_PREFIXES = ( "adjustment-transition-program-", @@ -137,8 +138,8 @@ def _x7_policy( { "schema_version": STATEFUL_EXECUTION_POLICY_VERSION, "adapter_lane": X7_GENERIC_STATEFUL_LANE, - "transport": X7_VECTOR_TRANSPORT, - "primary": "source-compiled-generic-stateful-programs", + "transport": FULL_NATIVE_VECTOR_TRANSPORT, + "primary": "source-compiled-full-native-and-generic-stateful-programs", "programs": programs, "legacy_shadow": _shadow_contract(shadow_count), "official_fallback": _official_fallback_contract(), @@ -147,6 +148,22 @@ def _x7_policy( ) +def add_native_execution_blockers( + policy: dict[str, Any], + blockers: list[dict[str, Any]], +) -> dict[str, Any]: + """Append source-compiler blockers and reseal the policy fingerprint.""" + if not blockers: + return policy + base = { + key: value + for key, value in policy.items() + if key not in {"fingerprint", "native_ready"} + } + base["blockers"] = [*policy.get("blockers", []), *blockers] + return _finalize_policy(base) + + def _stateful_program_inventory(manager: dict[str, Any]) -> list[dict[str, Any]]: inventory: list[dict[str, Any]] = [] diff --git a/python/nfi_backtest_engine/tag_program/compiler.py b/python/nfi_backtest_engine/tag_program/compiler.py index 22f01a64..b201502b 100644 --- a/python/nfi_backtest_engine/tag_program/compiler.py +++ b/python/nfi_backtest_engine/tag_program/compiler.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any, Never +from .._indicator_ast import _declared_class_constants, _effective_backtest_config from ..errors import StrategyAnalysisError from ..signal_program.compiler import ( _is_full_slice, @@ -31,6 +32,7 @@ "int-scalar", "f64-scalar", "bool-column", + "int-column", "f64-column", } _STRING_VALUE_TYPES = {"null", "string-scalar", "string-column"} @@ -45,6 +47,7 @@ def compile_tag_program( *, class_name: str | None = None, trading_mode: str = "spot", + config: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Compile ordered signal and tag writes without executing strategy Python.""" if trading_mode not in {"spot", "futures"}: @@ -75,6 +78,9 @@ def compile_tag_program( for node in class_node.body if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) } + module_functions = { + node.name: node for node in tree.body if isinstance(node, ast.FunctionDef) + } for method_name in ("populate_entry_trend", "populate_exit_trend"): method = methods.get(method_name) if method is None: @@ -83,10 +89,24 @@ def compile_tag_program( _unsupported(method, "async tag entrypoint") constants = strategy.get("constants", {}) + class_constants = _declared_class_constants( + class_node, + constants if isinstance(constants, Mapping) else {}, + ) + effective_config = dict(config or {}) + configured_mode = effective_config.get("trading_mode") + if configured_mode is not None and configured_mode != trading_mode: + raise TagProgramCompileError("tag trading mode differs from the supplied configuration") + effective_config["trading_mode"] = trading_mode compiler = _TagCompiler( path=path, methods=methods, - class_constants=constants if isinstance(constants, Mapping) else {}, + class_constants=class_constants, + instance_constants={ + "config": _effective_backtest_config(effective_config), + "dp": {"runmode": {"value": "backtest"}}, + }, + module_functions=module_functions, ) compiler.method_ids.update({"populate_entry_trend": "f1", "populate_exit_trend": "f2"}) try: @@ -145,8 +165,17 @@ def __init__( path: Path, methods: Mapping[str, ast.FunctionDef | ast.AsyncFunctionDef], class_constants: Mapping[str, Any], + instance_constants: Mapping[str, Any] | None = None, + module_functions: Mapping[str, ast.FunctionDef] | None = None, ) -> None: - super().__init__(path=path, methods=methods, class_constants=class_constants) + super().__init__( + path=path, + methods=methods, + class_constants=class_constants, + instance_constants=instance_constants, + module_functions=module_functions, + ) + self.compile_tags = True self.tag_mutation_nodes: list[str] = [] def statement(self, node: ast.stmt) -> None: diff --git a/python/nfi_backtest_engine/tag_program/validation.py b/python/nfi_backtest_engine/tag_program/validation.py index 827adbfd..a1e91d2c 100644 --- a/python/nfi_backtest_engine/tag_program/validation.py +++ b/python/nfi_backtest_engine/tag_program/validation.py @@ -23,6 +23,7 @@ "int-scalar", "f64-scalar", "bool-column", + "int-column", "f64-column", } _STRING_VALUE_TYPES = {"null", "string-scalar", "string-column"} diff --git a/python/nfi_backtest_engine/vector_runtime.py b/python/nfi_backtest_engine/vector_runtime.py index 3bc90b90..2cec7793 100644 --- a/python/nfi_backtest_engine/vector_runtime.py +++ b/python/nfi_backtest_engine/vector_runtime.py @@ -148,7 +148,7 @@ def prepare_vector_signals( cache_hits: dict[str, dict[str, Any]] = {} futures_execution: dict[str, Any] | None = None for pair in pairs: - frame_paths = _resolve_pair_frames( + frame_paths = resolve_pair_frames( data_root, pair=pair, pairs=pairs, @@ -161,7 +161,7 @@ def prepare_vector_signals( for key, path in frame_paths.items() } funding_spec = ( - _resolve_pair_funding_data( + resolve_pair_funding_data( data_root, pair=pair, data_index=data_index, @@ -585,7 +585,7 @@ def _publish_vector_record( return destination -def _resolve_pair_frames( +def resolve_pair_frames( data_root: Path, *, pair: str, @@ -653,7 +653,7 @@ def _find_candle_file( return matches[0] -def _resolve_pair_funding_data( +def resolve_pair_funding_data( data_root: Path, *, pair: str, @@ -687,6 +687,11 @@ def _resolve_pair_funding_data( ) +# Stable private aliases for existing tests and downstream integrations. +_resolve_pair_frames = resolve_pair_frames +_resolve_pair_funding_data = resolve_pair_funding_data + + def _find_timeframed_data_file( data_root: Path, *, diff --git a/python/nfi_backtest_engine/x7/adjustments.py b/python/nfi_backtest_engine/x7/adjustments.py index 2d25e6a4..2f300f65 100644 --- a/python/nfi_backtest_engine/x7/adjustments.py +++ b/python/nfi_backtest_engine/x7/adjustments.py @@ -5,7 +5,7 @@ import ast import math import re -from typing import Any +from typing import Any, Never from ..errors import StrategyAnalysisError from .trade_manager import ( @@ -288,9 +288,25 @@ def _adjustment_predicate(node: ast.AST, *, level: int) -> dict[str, Any]: raise StrategyAnalysisError( f"NFI adjustment grind {level} fallback must be an AND expression" ) + legacy = _legacy_adjustment_predicate(node) + if legacy is not None: + return legacy + return { + "any_derisk_levels": [], + "conditions": [], + "expression": _adjustment_boolean_expression(node, level=level), + } + + +def _legacy_adjustment_predicate(node: ast.BoolOp) -> dict[str, Any] | None: any_derisk_levels: list[int] = [] conditions: list[dict[str, Any]] = [] for value in node.values: + if any( + isinstance(item, ast.Constant) and isinstance(item.value, bool) + for item in ast.walk(value) + ): + return None if isinstance(value, ast.BoolOp) and isinstance(value.op, ast.Or): levels = sorted( { @@ -307,18 +323,95 @@ def _adjustment_predicate(node: ast.AST, *, level: int) -> dict[str, Any]: continue condition = _adjustment_comparison(value) if condition is None: - raise StrategyAnalysisError(f"NFI adjustment grind {level} fallback condition changed") + return None conditions.append(condition) if not conditions: - raise StrategyAnalysisError( - f"NFI adjustment grind {level} fallback has no numeric conditions" - ) + return None return { "any_derisk_levels": sorted(set(any_derisk_levels)), "conditions": conditions, } +def _adjustment_boolean_expression(node: ast.AST, *, level: int) -> dict[str, Any]: + if isinstance(node, ast.BoolOp): + operation = "all" if isinstance(node.op, ast.And) else "any" + if not isinstance(node.op, ast.And | ast.Or) or not node.values: + _unsupported_fallback(level) + return { + "op": operation, + "values": [ + _adjustment_boolean_expression(value, level=level) for value in node.values + ], + } + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return { + "op": "not", + "value": _adjustment_boolean_expression(node.operand, level=level), + } + derisk_level = _derisk_found_level(node) + if derisk_level is not None: + return {"op": "derisk_found", "level": derisk_level} + flag = _adjustment_boolean_flag(node) + if flag is not None: + return {"op": "flag", "name": flag} + if isinstance(node, ast.Compare) and len(node.ops) == 1 and len(node.comparators) == 1: + presence = _presence_expression(node) + if presence is not None: + return presence + comparison = _adjustment_comparison(node) + if comparison is not None: + return {"op": "comparison", **comparison} + _unsupported_fallback(level) + + +def _presence_expression(node: ast.Compare) -> dict[str, Any] | None: + operator = node.ops[0] + left, right = node.left, node.comparators[0] + if isinstance(right, ast.Constant) and right.value is None: + operand_node = left + elif isinstance(left, ast.Constant) and left.value is None: + operand_node = right + else: + return None + if not isinstance(operator, ast.Is | ast.IsNot): + return None + operand = _adjustment_operand(operand_node) + if operand is None: + return None + present = {"op": "present", "operand": operand} + return present if isinstance(operator, ast.IsNot) else {"op": "not", "value": present} + + +def _derisk_found_level(node: ast.AST) -> int | None: + if not isinstance(node, ast.Name): + return None + match = re.fullmatch(r"is_derisk_(\d+)_found", node.id) + return int(match.group(1)) if match is not None else None + + +def _adjustment_boolean_flag(node: ast.AST) -> str | None: + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "self" + and node.attr == "is_futures_mode" + ): + return "is_futures_mode" + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "trade" + and node.attr == "is_short" + ): + return "trade_is_short" + return None + + +def _unsupported_fallback(level: int) -> Never: + raise StrategyAnalysisError(f"NFI adjustment grind {level} fallback condition changed") + + def _adjustment_comparison(node: ast.AST) -> dict[str, Any] | None: if not isinstance(node, ast.Compare) or len(node.ops) != 1 or len(node.comparators) != 1: return None @@ -339,28 +432,54 @@ def _adjustment_comparison(node: ast.AST) -> dict[str, Any] | None: def _adjustment_operand(node: ast.AST) -> dict[str, Any] | None: + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return {"kind": "literal", "value": float(node.value)} number = _ast_number(node) if number is not None: return {"kind": "literal", "value": number} if isinstance(node, ast.Name) and node.id in { + "current_rate", "slice_profit", "slice_profit_entry", + "slice_profit_exit", "num_open_grinds_and_buybacks", }: return {"kind": "variable", "name": node.id} feature = _last_candle_feature(node) if feature is not None: return {"kind": "feature", "name": feature, "multiplier": 1.0} + trade_field = _trade_numeric_field(node) + if trade_field is not None: + return {"kind": "trade", "name": trade_field, "multiplier": 1.0} if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mult): - feature = _last_candle_feature(node.left) - multiplier = _ast_number(node.right) - if feature is None or multiplier is None or not math.isfinite(multiplier): - return None - return { - "kind": "feature", - "name": feature, - "multiplier": multiplier, - } + return _scaled_adjustment_operand(node) + return None + + +def _scaled_adjustment_operand(node: ast.BinOp) -> dict[str, Any] | None: + multiplier = _ast_number(node.right) + operand = _adjustment_operand(node.left) if multiplier is not None else None + if operand is None: + multiplier = _ast_number(node.left) + operand = _adjustment_operand(node.right) if multiplier is not None else None + if ( + operand is None + or multiplier is None + or not math.isfinite(multiplier) + or operand["kind"] not in {"feature", "trade"} + ): + return None + return {**operand, "multiplier": operand["multiplier"] * multiplier} + + +def _trade_numeric_field(node: ast.AST) -> str | None: + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "trade" + and node.attr == "liquidation_price" + ): + return node.attr return None diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b42c610e..a785f884 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -235,6 +235,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures-core" version = "0.3.33" @@ -380,27 +390,30 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "nfi-py" -version = "1.5.0" +version = "1.6.0" dependencies = [ "nfi-sim-core", + "nfi-vector-core", "nfi-vector-io", "pyo3", + "serde", "serde_json", "sha2", ] [[package]] name = "nfi-sim-cli" -version = "1.5.0" +version = "1.6.0" dependencies = [ "nfi-sim-core", "nfi-vector-io", + "serde", "serde_json", ] [[package]] name = "nfi-sim-core" -version = "1.5.0" +version = "1.6.0" dependencies = [ "num-bigint", "num-rational", @@ -414,7 +427,7 @@ dependencies = [ [[package]] name = "nfi-vector-core" -version = "1.5.0" +version = "1.6.0" dependencies = [ "arrow2", "serde", @@ -425,9 +438,10 @@ dependencies = [ [[package]] name = "nfi-vector-io" -version = "1.5.0" +version = "1.6.0" dependencies = [ "arrow2", + "fs2", "nfi-sim-core", "nfi-vector-core", "rayon", @@ -855,6 +869,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3cf1d5c7..c6c04811 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -9,7 +9,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.5.0" +version = "1.6.0" edition = "2021" license = "MIT" rust-version = "1.83" diff --git a/rust/crates/nfi-py/Cargo.toml b/rust/crates/nfi-py/Cargo.toml index 59bc3f17..7d864cc3 100644 --- a/rust/crates/nfi-py/Cargo.toml +++ b/rust/crates/nfi-py/Cargo.toml @@ -15,8 +15,10 @@ path = "src/lib.rs" [dependencies] nfi-sim-core = { path = "../nfi-sim-core" } +nfi-vector-core = { path = "../nfi-vector-core" } nfi-vector-io = { path = "../nfi-vector-io" } pyo3 = { version = "0.29", features = ["abi3-py312", "extension-module"] } +serde = "1" serde_json = "1" [build-dependencies] diff --git a/rust/crates/nfi-py/src/full_vector.rs b/rust/crates/nfi-py/src/full_vector.rs new file mode 100644 index 00000000..867212fa --- /dev/null +++ b/rust/crates/nfi-py/src/full_vector.rs @@ -0,0 +1,383 @@ +//! In-memory Python bridge for the complete Rust vector stage. + +use std::collections::{BTreeMap, BTreeSet}; + +use nfi_vector_core::alignment::{FrameCatalog, FrameIdentity, NumericFrame, Timeframe}; +use nfi_vector_core::column::{OwnedColumn, ValueType}; +use nfi_vector_core::engine::FullIndicatorEngine; +use nfi_vector_core::mutation::{ + materialize_execution_signals, ExecutionSignals, MutationEngine, MutationFrame, MutationProgram, +}; +use nfi_vector_core::program::IndicatorProgram; +use nfi_vector_core::VectorCoreError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +const SIGNAL_COLUMNS: [&str; 4] = ["enter_long", "enter_short", "exit_long", "exit_short"]; +const TAG_COLUMNS: [&str; 2] = ["enter_tag", "exit_tag"]; +const SOURCE_ROW_SHIFT: usize = 1; + +type NumericColumns = BTreeMap>>; +type InformativeFrameInput = (String, String, Vec, NumericColumns); + +/// Complete typed output that can be transferred directly into an +/// `InMemoryVectorPair` without a Feather or strategy-Python round trip. +#[derive(Debug)] +struct FullVectorOutput { + identity: FrameIdentity, + timestamps_ms: Vec, + execution_start_index: usize, + columns: BTreeMap, + enabled_indexes: BTreeMap>, +} + +#[allow(clippy::too_many_arguments)] +fn execute_stage( + indicator_program: &IndicatorProgram, + signal_program: &MutationProgram, + tag_program: &MutationProgram, + base: &NumericFrame, + catalog: &FrameCatalog, + metadata: &BTreeMap, + requested_indicator_columns: &[String], + execution_start_index: usize, +) -> Result { + validate_stage_contract( + indicator_program, + signal_program, + tag_program, + base, + metadata, + requested_indicator_columns, + execution_start_index, + )?; + + let execution_outputs = + indicator_execution_outputs(requested_indicator_columns, signal_program, tag_program); + let indicator = FullIndicatorEngine::new(indicator_program)?.execute( + base, + catalog, + metadata, + &execution_outputs, + )?; + if indicator.identity() != &base.identity || indicator.timestamps_ms() != base.timestamps_ms { + return Err(VectorCoreError::InvalidOutput( + "complete indicator output changed the base identity or row index".to_owned(), + )); + } + + let mutation_source = MutationFrame::new(indicator.columns().clone())?; + let signal = MutationEngine::new(signal_program)? + .execute_with_metadata(mutation_source.clone(), metadata)?; + let tag = MutationEngine::new(tag_program)?.execute_with_metadata(mutation_source, metadata)?; + compare_signal_surfaces(&signal, &tag)?; + + let mut combined = signal.columns().clone(); + for name in TAG_COLUMNS { + let column = tag.column(name).ok_or_else(|| { + VectorCoreError::InvalidOutput(format!("Tag program did not produce {name}")) + })?; + if column.as_view().value_type() != ValueType::Text { + return Err(VectorCoreError::InvalidOutput(format!( + "Tag program produced non-text {name}" + ))); + } + combined.insert(name.to_owned(), column.clone()); + } + let execution = materialize_execution_signals( + &MutationFrame::new(combined)?, + SOURCE_ROW_SHIFT, + execution_start_index, + )?; + assemble_output( + &indicator, + execution, + requested_indicator_columns, + execution_start_index, + ) +} + +#[allow(clippy::too_many_arguments, clippy::needless_pass_by_value)] +#[pyfunction] +pub(super) fn execute_full_vector( + py: Python<'_>, + indicator_program: &str, + signal_program: &str, + tag_program: &str, + base_pair: String, + base_timeframe: String, + base_timestamps_ms: Vec, + base_columns: NumericColumns, + informative_frames: Vec, + metadata: BTreeMap, + requested_indicator_columns: Vec, + execution_start_index: usize, +) -> PyResult> { + let indicator_program = IndicatorProgram::from_json(indicator_program) + .map_err(|error| rejected("indicator program", &error))?; + let signal_program = MutationProgram::from_json(signal_program) + .map_err(|error| rejected("Signal program", &error))?; + let tag_program = + MutationProgram::from_json(tag_program).map_err(|error| rejected("Tag program", &error))?; + let base = numeric_frame(base_pair, base_timeframe, base_timestamps_ms, base_columns) + .map_err(|error| rejected("base frame", &error))?; + let catalog = informative_catalog(informative_frames) + .map_err(|error| rejected("informative catalog", &error))?; + let output = execute_stage( + &indicator_program, + &signal_program, + &tag_program, + &base, + &catalog, + &metadata, + &requested_indicator_columns, + execution_start_index, + ) + .map_err(|error| rejected("full vector stage", &error))?; + output_to_python(py, output) +} + +fn numeric_frame( + pair: String, + timeframe: String, + timestamps_ms: Vec, + columns: NumericColumns, +) -> Result { + let identity = FrameIdentity::new(pair, Timeframe::parse(timeframe)?)?; + let frame = NumericFrame { + identity, + timestamps_ms, + columns, + }; + frame.validate()?; + Ok(frame) +} + +fn informative_catalog( + frames: Vec, +) -> Result { + let frames = frames + .into_iter() + .map(|(pair, timeframe, timestamps, columns)| { + let frame = numeric_frame(pair, timeframe, timestamps, columns)?; + Ok((frame.identity.clone(), frame)) + }) + .collect::, VectorCoreError>>()?; + FrameCatalog::new(frames) +} + +#[allow(clippy::too_many_arguments)] +fn validate_stage_contract( + indicator: &IndicatorProgram, + signal: &MutationProgram, + tag: &MutationProgram, + base: &NumericFrame, + metadata: &BTreeMap, + requested: &[String], + execution_start_index: usize, +) -> Result<(), VectorCoreError> { + if signal.is_tag_program() || !tag.is_tag_program() { + return Err(invalid( + "Signal and Tag programs are assigned to the wrong lanes", + )); + } + if indicator.selected_class != signal.selected_class + || signal.selected_class != tag.selected_class + || indicator.source.sha256 != signal.source.sha256 + || signal.source.sha256 != tag.source.sha256 + || signal.compile_context != tag.compile_context + { + return Err(invalid( + "Indicator, Signal, and Tag programs do not share one compiled strategy identity", + )); + } + match metadata.get("pair") { + Some(pair) if pair == &base.identity.pair => {} + _ => { + return Err(invalid( + "metadata pair is absent or differs from the base frame pair", + )); + } + } + if base.timestamps_ms.is_empty() || execution_start_index >= base.timestamps_ms.len() { + return Err(invalid( + "execution_start_index must identify a row in a non-empty base frame", + )); + } + let mut seen = BTreeSet::new(); + for name in requested { + if name.is_empty() + || !seen.insert(name.as_str()) + || name == "date" + || name.starts_with("nfi_exec_") + || SIGNAL_COLUMNS.contains(&name.as_str()) + || TAG_COLUMNS.contains(&name.as_str()) + { + return Err(invalid(format!( + "requested indicator column {name:?} is empty, duplicate, or reserved" + ))); + } + } + Ok(()) +} + +fn indicator_execution_outputs( + requested: &[String], + signal: &MutationProgram, + tag: &MutationProgram, +) -> Vec { + let mut outputs = Vec::new(); + let mut seen = BTreeSet::new(); + for name in requested + .iter() + .chain(&signal.required_input_columns) + .chain(&tag.required_input_columns) + { + if seen.insert(name.as_str()) { + outputs.push(name.clone()); + } + } + outputs +} + +fn compare_signal_surfaces( + signal: &MutationFrame, + tag: &MutationFrame, +) -> Result<(), VectorCoreError> { + for name in SIGNAL_COLUMNS { + let left = signal.column(name).ok_or_else(|| { + VectorCoreError::InvalidOutput(format!("Signal program did not produce {name}")) + })?; + let right = tag.column(name).ok_or_else(|| { + VectorCoreError::InvalidOutput(format!("Tag program did not produce {name}")) + })?; + if !columns_are_exact(left, right) { + return Err(VectorCoreError::InvalidOutput(format!( + "Signal and Tag programs disagree on {name}" + ))); + } + } + Ok(()) +} + +fn columns_are_exact(left: &OwnedColumn, right: &OwnedColumn) -> bool { + let left_view = left.as_view(); + let right_view = right.as_view(); + if left_view.value_type() != right_view.value_type() || left.len() != right.len() { + return false; + } + (0..left.len()).all(|row| match left_view.value_type() { + ValueType::F64 => match (left_view.f64_at(row), right_view.f64_at(row)) { + (None, None) => true, + (Some(left), Some(right)) => left.to_bits() == right.to_bits(), + _ => false, + }, + ValueType::I64 => left_view.i64_at(row) == right_view.i64_at(row), + ValueType::Bool => left_view.bool_at(row) == right_view.bool_at(row), + ValueType::Text => left_view.text_at(row) == right_view.text_at(row), + ValueType::TimestampMs => left_view.timestamp_ms_at(row) == right_view.timestamp_ms_at(row), + }) +} + +fn assemble_output( + indicator: &nfi_vector_core::engine::FullFrameOutput, + execution: ExecutionSignals, + requested: &[String], + execution_start_index: usize, +) -> Result { + let mut columns = BTreeMap::from([( + "date".to_owned(), + OwnedColumn::timestamp_ms( + indicator + .timestamps_ms() + .iter() + .copied() + .map(Some) + .collect(), + ), + )]); + for name in requested { + let column = indicator.columns().get(name).ok_or_else(|| { + VectorCoreError::MissingOutput(format!("requested indicator column {name}")) + })?; + columns.insert(name.clone(), column.clone()); + } + for (name, column) in execution.frame.columns() { + if columns.insert(name.clone(), column.clone()).is_some() { + return Err(VectorCoreError::InvalidOutput(format!( + "execution column {name} collides with an indicator output" + ))); + } + } + let frame = MutationFrame::new(columns)?; + Ok(FullVectorOutput { + identity: indicator.identity().clone(), + timestamps_ms: indicator.timestamps_ms().to_vec(), + execution_start_index, + columns: frame.columns().clone(), + enabled_indexes: execution.enabled_indexes, + }) +} + +fn output_to_python(py: Python<'_>, output: FullVectorOutput) -> PyResult> { + let result = PyDict::new(py); + result.set_item("pair", output.identity.pair)?; + result.set_item("timeframe", output.identity.timeframe.as_str())?; + result.set_item("timestamps_ms", output.timestamps_ms)?; + result.set_item("execution_start_index", output.execution_start_index)?; + let columns = PyDict::new(py); + for (name, column) in output.columns { + let encoded = PyDict::new(py); + encoded.set_item("value_type", column.as_view().value_type().label())?; + set_column_values(&encoded, &column)?; + columns.set_item(name, encoded)?; + } + result.set_item("columns", columns)?; + result.set_item("enabled_indexes", output.enabled_indexes)?; + Ok(result.into_any().unbind()) +} + +pub(super) fn set_column_values(output: &Bound<'_, PyDict>, column: &OwnedColumn) -> PyResult<()> { + let view = column.as_view(); + match view.value_type() { + ValueType::F64 => output.set_item( + "values", + (0..column.len()) + .map(|row| view.f64_at(row)) + .collect::>(), + ), + ValueType::I64 => output.set_item( + "values", + (0..column.len()) + .map(|row| view.i64_at(row)) + .collect::>(), + ), + ValueType::Bool => output.set_item( + "values", + (0..column.len()) + .map(|row| view.bool_at(row)) + .collect::>(), + ), + ValueType::Text => output.set_item( + "values", + (0..column.len()) + .map(|row| view.text_at(row).map(str::to_owned)) + .collect::>(), + ), + ValueType::TimestampMs => output.set_item( + "values", + (0..column.len()) + .map(|row| view.timestamp_ms_at(row)) + .collect::>(), + ), + } +} + +fn invalid(message: impl Into) -> VectorCoreError { + VectorCoreError::InvalidProgram(message.into()) +} + +fn rejected(context: &str, error: &VectorCoreError) -> PyErr { + PyValueError::new_err(format!("{context} rejected: {error}")) +} diff --git a/rust/crates/nfi-py/src/lib.rs b/rust/crates/nfi-py/src/lib.rs index 4ce514dd..e0556733 100644 --- a/rust/crates/nfi-py/src/lib.rs +++ b/rust/crates/nfi-py/src/lib.rs @@ -10,9 +10,17 @@ use nfi_sim_core::{ simulate_with_observer, simulate_with_observer_profiled, SimulationInput, SimulationProfile, SimulationResult, }; -use nfi_vector_io::{load_vector_manifest, load_vector_manifest_profiled, VectorLoadProfile}; +use nfi_vector_io::{ + load_full_native_vector_manifest_profiled, + load_full_native_vector_manifest_profiled_with_worker_limit, load_vector_manifest, + load_vector_manifest_profiled, +}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use serde::Serialize; + +mod full_vector; +mod mutation_vector; #[pyfunction] fn schema_version() -> &'static str { @@ -104,25 +112,72 @@ fn simulate_vector_file_profiled( )) })?; let (result, simulation_profile) = run_simulation_profiled(&document, events_path)?; - let serialization_started = Instant::now(); - let serialized = serialize_simulation_result(&result) - .map_err(|error| PyValueError::new_err(format!("cannot serialize result: {error}")))?; - atomic_write(output_path.clone(), &serialized) - .map_err(|error| PyValueError::new_err(format!("cannot write result: {error}")))?; - let profile = profile_document( + write_profiled_result( + output_path, + profile_path, + &result, &input_profile, &simulation_profile, - duration_ns(serialization_started.elapsed()), - ); - let encoded_profile = serde_json::to_vec(&profile) - .map_err(|error| PyValueError::new_err(format!("cannot serialize profile: {error}")))?; - if let Err(error) = atomic_write(profile_path, &encoded_profile) { - let _ = fs::remove_file(output_path); - return Err(PyValueError::new_err(format!( - "cannot write engine profile: {error}" - ))); - } - Ok(()) + ) +} + +#[pyfunction(signature = (manifest_path, output_path, events_path=None, pair_worker_limit=None))] +#[allow(clippy::needless_pass_by_value)] // PyO3 extracts owned Python path arguments. +fn simulate_full_vector_file( + manifest_path: PathBuf, + output_path: PathBuf, + events_path: Option, + pair_worker_limit: Option, +) -> PyResult<()> { + let manifest_display = manifest_path.display().to_string(); + let loaded = if let Some(limit) = pair_worker_limit { + load_full_native_vector_manifest_profiled_with_worker_limit(&manifest_path, limit) + } else { + load_full_native_vector_manifest_profiled(&manifest_path) + }; + let (document, _) = loaded.map_err(|error| { + PyValueError::new_err(format!( + "invalid full native vector manifest {manifest_display}: {error}" + )) + })?; + let result = run_simulation(&document, events_path)?; + write_result(output_path, &result) +} + +#[pyfunction(signature = ( + manifest_path, + output_path, + profile_path, + events_path=None, + pair_worker_limit=None +))] +#[allow(clippy::needless_pass_by_value)] // PyO3 extracts owned Python path arguments. +fn simulate_full_vector_file_profiled( + manifest_path: PathBuf, + output_path: PathBuf, + profile_path: PathBuf, + events_path: Option, + pair_worker_limit: Option, +) -> PyResult<()> { + let manifest_display = manifest_path.display().to_string(); + let loaded = if let Some(limit) = pair_worker_limit { + load_full_native_vector_manifest_profiled_with_worker_limit(&manifest_path, limit) + } else { + load_full_native_vector_manifest_profiled(&manifest_path) + }; + let (document, input_profile) = loaded.map_err(|error| { + PyValueError::new_err(format!( + "invalid full native vector manifest {manifest_display}: {error}" + )) + })?; + let (result, simulation_profile) = run_simulation_profiled(&document, events_path)?; + write_profiled_result( + output_path, + profile_path, + &result, + &input_profile, + &simulation_profile, + ) } fn run_simulation( @@ -208,6 +263,34 @@ fn write_result(output_path: PathBuf, result: &SimulationResult) -> PyResult<()> .map_err(|error| PyValueError::new_err(format!("cannot write result: {error}"))) } +fn write_profiled_result( + output_path: PathBuf, + profile_path: PathBuf, + result: &SimulationResult, + input_profile: &InputProfile, + simulation_profile: &SimulationProfile, +) -> PyResult<()> { + let serialization_started = Instant::now(); + let serialized = serialize_simulation_result(result) + .map_err(|error| PyValueError::new_err(format!("cannot serialize result: {error}")))?; + atomic_write(output_path.clone(), &serialized) + .map_err(|error| PyValueError::new_err(format!("cannot write result: {error}")))?; + let profile = profile_document( + input_profile, + simulation_profile, + duration_ns(serialization_started.elapsed()), + ); + let encoded_profile = serde_json::to_vec(&profile) + .map_err(|error| PyValueError::new_err(format!("cannot serialize profile: {error}")))?; + if let Err(error) = atomic_write(profile_path, &encoded_profile) { + let _ = fs::remove_file(output_path); + return Err(PyValueError::new_err(format!( + "cannot write engine profile: {error}" + ))); + } + Ok(()) +} + fn atomic_write(path: PathBuf, contents: &[u8]) -> Result<(), String> { let temporary = path.with_extension("tmp"); let path_display = path.display().to_string(); @@ -221,8 +304,8 @@ fn atomic_write(path: PathBuf, contents: &[u8]) -> Result<(), String> { }) } -fn profile_document( - input: &VectorLoadProfile, +fn profile_document( + input: &InputProfile, simulation: &SimulationProfile, serialization_ns: u64, ) -> serde_json::Value { @@ -250,5 +333,15 @@ fn _rust(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(simulate_file, module)?)?; module.add_function(wrap_pyfunction!(simulate_vector_file, module)?)?; module.add_function(wrap_pyfunction!(simulate_vector_file_profiled, module)?)?; + module.add_function(wrap_pyfunction!(simulate_full_vector_file, module)?)?; + module.add_function(wrap_pyfunction!( + simulate_full_vector_file_profiled, + module + )?)?; + module.add_function(wrap_pyfunction!(full_vector::execute_full_vector, module)?)?; + module.add_function(wrap_pyfunction!( + mutation_vector::execute_numeric_mutation_program, + module + )?)?; Ok(()) } diff --git a/rust/crates/nfi-py/src/mutation_vector.rs b/rust/crates/nfi-py/src/mutation_vector.rs new file mode 100644 index 00000000..595bc0b5 --- /dev/null +++ b/rust/crates/nfi-py/src/mutation_vector.rs @@ -0,0 +1,60 @@ +//! Focused Python bridge for qualifying a compiled mutation program. + +use std::collections::{BTreeMap, BTreeSet}; + +use nfi_vector_core::column::OwnedColumn; +use nfi_vector_core::mutation::{MutationEngine, MutationFrame, MutationProgram}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +type NumericColumns = BTreeMap>>; + +/// Execute one already-compiled Signal or Tag program without strategy Python. +/// +/// This deliberately accepts only numeric input columns. It is a qualification +/// surface for source-derived NFI Signal inputs, not a second dataframe transport. +#[pyfunction] +#[allow(clippy::needless_pass_by_value)] // PyO3 extracts owned Python containers. +pub(super) fn execute_numeric_mutation_program( + py: Python<'_>, + program: &str, + columns: NumericColumns, + metadata: BTreeMap, + requested_outputs: Vec, +) -> PyResult> { + let program = MutationProgram::from_json(program) + .map_err(|error| rejected("mutation program", &error.to_string()))?; + let source = MutationFrame::new( + columns + .into_iter() + .map(|(name, values)| (name, OwnedColumn::f64(values))) + .collect(), + ) + .map_err(|error| rejected("mutation input", &error.to_string()))?; + let output = MutationEngine::new(&program) + .and_then(|engine| engine.execute_with_metadata(source, &metadata)) + .map_err(|error| rejected("mutation execution", &error.to_string()))?; + + let mut seen = BTreeSet::new(); + let encoded = PyDict::new(py); + for name in requested_outputs { + if name.is_empty() || !seen.insert(name.clone()) { + return Err(PyValueError::new_err( + "requested mutation outputs must be nonempty and unique", + )); + } + let column = output + .column(&name) + .ok_or_else(|| PyValueError::new_err(format!("mutation output {name:?} is missing")))?; + let value = PyDict::new(py); + value.set_item("value_type", column.as_view().value_type().label())?; + super::full_vector::set_column_values(&value, column)?; + encoded.set_item(name, value)?; + } + Ok(encoded.into_any().unbind()) +} + +fn rejected(context: &str, error: &str) -> PyErr { + PyValueError::new_err(format!("{context} rejected: {error}")) +} diff --git a/rust/crates/nfi-sim-cli/Cargo.toml b/rust/crates/nfi-sim-cli/Cargo.toml index bff0dd76..871215d7 100644 --- a/rust/crates/nfi-sim-cli/Cargo.toml +++ b/rust/crates/nfi-sim-cli/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] nfi-sim-core = { path = "../nfi-sim-core" } nfi-vector-io = { path = "../nfi-vector-io" } +serde = "1" serde_json = "1" [lints] diff --git a/rust/crates/nfi-sim-cli/src/main.rs b/rust/crates/nfi-sim-cli/src/main.rs index 03a6ad1c..88bdf3b6 100644 --- a/rust/crates/nfi-sim-cli/src/main.rs +++ b/rust/crates/nfi-sim-cli/src/main.rs @@ -12,7 +12,18 @@ use nfi_sim_core::{ simulate_with_observer, simulate_with_observer_profiled, SimulationInput, SimulationProfile, SimulationResult, }; -use nfi_vector_io::{load_vector_manifest, load_vector_manifest_profiled, VectorLoadProfile}; +use nfi_vector_io::{ + load_full_native_vector_manifest_profiled, + load_full_native_vector_manifest_profiled_with_worker_limit, load_vector_manifest, + load_vector_manifest_profiled, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum InputKind { + SimulationJson, + FeatherVector, + FullVector, +} fn main() -> ExitCode { match run() { @@ -27,12 +38,17 @@ fn main() -> ExitCode { fn run() -> Result<(), String> { let mut arguments = env::args_os(); let _program = arguments.next(); - let mut vector_manifest = false; + let mut input_kind = InputKind::SimulationJson; let mut profile_output = None; + let mut pair_worker_limit = None; let input = loop { let argument = arguments.next().ok_or_else(|| usage().to_owned())?; if argument == "--vector-manifest" { - vector_manifest = true; + select_input_kind(&mut input_kind, InputKind::FeatherVector)?; + continue; + } + if argument == "--full-vector-manifest" { + select_input_kind(&mut input_kind, InputKind::FullVector)?; continue; } if argument == "--profile-output" { @@ -44,6 +60,18 @@ fn run() -> Result<(), String> { ); continue; } + if argument == "--pair-workers" { + let raw = arguments.next().ok_or_else(|| usage().to_owned())?; + let value = raw + .to_str() + .ok_or_else(|| "--pair-workers must be a positive integer".to_owned())? + .parse::() + .map_err(|_| "--pair-workers must be a positive integer".to_owned())?; + if value == 0 || pair_worker_limit.replace(value).is_some() { + return Err("--pair-workers must be declared once and be positive".to_owned()); + } + continue; + } break PathBuf::from(argument); }; let output = arguments @@ -54,30 +82,19 @@ fn run() -> Result<(), String> { if arguments.next().is_some() { return Err(usage().to_owned()); } - if profile_output.is_some() && !vector_manifest { - return Err("--profile-output requires --vector-manifest".to_owned()); + if profile_output.is_some() && input_kind == InputKind::SimulationJson { + return Err("--profile-output requires a vector manifest".to_owned()); + } + if pair_worker_limit.is_some() && input_kind != InputKind::FullVector { + return Err("--pair-workers requires --full-vector-manifest".to_owned()); } - let (document, input_profile) = if vector_manifest && profile_output.is_some() { - let (document, profile) = load_vector_manifest_profiled(&input) - .map_err(|error| format!("invalid vector manifest {}: {error}", input.display()))?; - (document, Some(profile)) - } else if vector_manifest { - ( - load_vector_manifest(&input) - .map_err(|error| format!("invalid vector manifest {}: {error}", input.display()))?, - None, - ) - } else { - let encoded = fs::read(&input) - .map_err(|error| format!("cannot read {}: {error}", input.display()))?; - ( - parse_simulation_input(&encoded).map_err(|error| { - format!("invalid simulation input {}: {error}", input.display()) - })?, - None, - ) - }; + let (document, input_profile) = load_input( + input_kind, + &input, + profile_output.is_some(), + pair_worker_limit, + )?; let (result, simulation_profile) = if profile_output.is_some() { let (result, profile) = run_simulation_profiled(&document, trace)?; (result, Some(profile)) @@ -119,10 +136,74 @@ fn run() -> Result<(), String> { } fn usage() -> &'static str { - "usage: nfi-sim [--vector-manifest] [--profile-output profile.json] \ + "usage: nfi-sim [--vector-manifest | --full-vector-manifest] \ + [--profile-output profile.json] \ + [--pair-workers positive-integer] \ [events.jsonl]" } +fn select_input_kind(current: &mut InputKind, requested: InputKind) -> Result<(), String> { + if *current != InputKind::SimulationJson { + return Err("engine input kind may be selected only once".to_owned()); + } + *current = requested; + Ok(()) +} + +fn load_input( + input_kind: InputKind, + input: &PathBuf, + profiled: bool, + pair_worker_limit: Option, +) -> Result<(SimulationInput, Option), String> { + match input_kind { + InputKind::SimulationJson => { + let encoded = fs::read(input) + .map_err(|error| format!("cannot read {}: {error}", input.display()))?; + let document = parse_simulation_input(&encoded).map_err(|error| { + format!("invalid simulation input {}: {error}", input.display()) + })?; + Ok((document, None)) + } + InputKind::FeatherVector if profiled => { + let (document, profile) = load_vector_manifest_profiled(input) + .map_err(|error| format!("invalid vector manifest {}: {error}", input.display()))?; + Ok((document, Some(profile_value(profile)?))) + } + InputKind::FeatherVector => { + let document = load_vector_manifest(input) + .map_err(|error| format!("invalid vector manifest {}: {error}", input.display()))?; + Ok((document, None)) + } + InputKind::FullVector => { + let loaded = if let Some(limit) = pair_worker_limit { + load_full_native_vector_manifest_profiled_with_worker_limit(input, limit) + } else { + load_full_native_vector_manifest_profiled(input) + }; + let (document, profile) = loaded.map_err(|error| { + format!( + "invalid full native vector manifest {}: {error}", + input.display() + ) + })?; + Ok(( + document, + if profiled { + Some(profile_value(profile)?) + } else { + None + }, + )) + } + } +} + +fn profile_value(profile: impl serde::Serialize) -> Result { + serde_json::to_value(profile) + .map_err(|error| format!("cannot serialize input profile: {error}")) +} + fn run_simulation( document: &SimulationInput, trace: Option, @@ -192,7 +273,7 @@ fn run_simulation_profiled( } fn profile_document( - input: &VectorLoadProfile, + input: &serde_json::Value, simulation: &SimulationProfile, serialization_ns: u64, ) -> serde_json::Value { diff --git a/rust/crates/nfi-sim-core/src/domain/failures.rs b/rust/crates/nfi-sim-core/src/domain/failures.rs index c1821c69..12dc2618 100644 --- a/rust/crates/nfi-sim-core/src/domain/failures.rs +++ b/rust/crates/nfi-sim-core/src/domain/failures.rs @@ -50,6 +50,12 @@ pub enum SimError { InvalidPositionAdjustment { pair: String, timestamp_ms: i64 }, #[error("NFI X7 trade manager configuration or scalar program is invalid")] InvalidNfiTradeManager, + #[error("NFI X7 exit callback failed for {pair:?} at {timestamp_ms}: {diagnostic}")] + InvalidNfiExitRuntime { + pair: String, + timestamp_ms: i64, + diagnostic: String, + }, #[error("NFI X7 trade manager does not support entry tag {entry_tag:?} for {pair:?}")] UnsupportedNfiEntryTag { pair: String, entry_tag: String }, } diff --git a/rust/crates/nfi-sim-core/src/domain/x7.rs b/rust/crates/nfi-sim-core/src/domain/x7.rs index 629672da..ed9376d2 100644 --- a/rust/crates/nfi-sim-core/src/domain/x7.rs +++ b/rust/crates/nfi-sim-core/src/domain/x7.rs @@ -1040,6 +1040,38 @@ pub struct NfiX7GrindFallbackLevel { pub struct NfiX7AdjustmentPredicate { pub any_derisk_levels: Vec, pub conditions: Vec, + /// Source-shaped fallback expression for predicates that cannot be + /// represented by the historical de-risk-plus-comparisons contract. + #[serde(default)] + pub expression: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] +pub enum NfiX7AdjustmentExpression { + All { + values: Vec, + }, + Any { + values: Vec, + }, + Not { + value: Box, + }, + Flag { + name: String, + }, + DeriskFound { + level: usize, + }, + Present { + operand: NfiX7AdjustmentOperand, + }, + Comparison { + left: NfiX7AdjustmentOperand, + operator: NfiX7AdjustmentComparison, + right: NfiX7AdjustmentOperand, + }, } #[derive(Debug, Clone, Deserialize)] @@ -1051,14 +1083,15 @@ pub struct NfiX7AdjustmentCondition { } #[derive(Debug, Clone, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum NfiX7AdjustmentOperand { Literal { value: f64 }, Variable { name: String }, Feature { name: String, multiplier: f64 }, + Trade { name: String, multiplier: f64 }, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "snake_case")] pub enum NfiX7AdjustmentComparison { Lt, diff --git a/rust/crates/nfi-sim-core/src/execution/exit.rs b/rust/crates/nfi-sim-core/src/execution/exit.rs index aad7b4e4..aaf05871 100644 --- a/rust/crates/nfi-sim-core/src/execution/exit.rs +++ b/rust/crates/nfi-sim-core/src/execution/exit.rs @@ -80,8 +80,13 @@ pub(crate) fn exit_decision( } } if let Some(manager) = &config.nfi_x7_trade_manager { - let feature_index = - callback_feature_index(candle_index).ok_or(SimError::InvalidNfiTradeManager)?; + let runtime_error = |diagnostic: String| SimError::InvalidNfiExitRuntime { + pair: trade.pair.clone(), + timestamp_ms: candle.timestamp_ms, + diagnostic, + }; + let feature_index = callback_feature_index(candle_index) + .ok_or_else(|| runtime_error("callback feature index is unavailable".to_owned()))?; let decision = evaluate_nfi_exit( manager, trade, @@ -91,7 +96,7 @@ pub(crate) fn exit_decision( config, profit_targets, ) - .ok_or(SimError::InvalidNfiTradeManager)?; + .map_err(|diagnostic| runtime_error(diagnostic.to_string()))?; if let CustomExitDecision::Exit(reason) = decision { return Ok(Some(ExitDecision { rate: candle.open, diff --git a/rust/crates/nfi-sim-core/src/lib.rs b/rust/crates/nfi-sim-core/src/lib.rs index 56b01923..739bd21b 100644 --- a/rust/crates/nfi-sim-core/src/lib.rs +++ b/rust/crates/nfi-sim-core/src/lib.rs @@ -48,6 +48,7 @@ pub use state_machine_vm::{ validate_state_machine_program, StateMachineAction, StateMachineContext, StateMachineDiagnostic, StateMachineError, }; +pub use validation::validate_simulator_preflight; mod futures; mod futures_contract; #[cfg(test)] diff --git a/rust/crates/nfi-sim-core/src/nfi/adjustment.rs b/rust/crates/nfi-sim-core/src/nfi/adjustment.rs index 68cd9526..33cff018 100644 --- a/rust/crates/nfi-sim-core/src/nfi/adjustment.rs +++ b/rust/crates/nfi-sim-core/src/nfi/adjustment.rs @@ -13,16 +13,16 @@ use serde_json::Value; use crate::calculations::{fee_close, fee_open}; use crate::callbacks::{ - feature_number_at, insert_projected_feature_window, scalar_program_feature_projection, - scalar_trade_value, + feature_bool_at, feature_number_at, insert_projected_feature_window, + scalar_program_feature_projection, scalar_trade_value, }; use crate::domain::{ AdjustmentSignal, Candle, CompiledOrderSide, CompiledSystemAdjustmentAction, CompiledSystemAdjustmentExecutionMode, CompiledSystemAdjustmentInputKind, CompiledSystemAdjustmentProgram, CompiledSystemAdjustmentSide, CompiledSystemGrindTags, CompiledSystemStakeScale, NfiX7AdjustmentComparison, NfiX7AdjustmentCondition, - NfiX7AdjustmentOperand, NfiX7AdjustmentPredicate, NfiX7GrindLevel, NfiX7PositionAdjustment, - NfiX7TradeManager, OrderSide, PairSeries, PortfolioConfig, + NfiX7AdjustmentExpression, NfiX7AdjustmentOperand, NfiX7AdjustmentPredicate, NfiX7GrindLevel, + NfiX7PositionAdjustment, NfiX7TradeManager, OrderSide, PairSeries, PortfolioConfig, }; use crate::execution::adjustment_minimum_pair_stake; use crate::order_aggregates::FilledOrderSelector; @@ -95,6 +95,7 @@ struct AdjustmentContext<'a> { slice_amount: f64, slice_profit: f64, slice_profit_entry: f64, + slice_profit_exit: f64, current_stake_amount: f64, rebuy_mode: bool, is_grind_entry: bool, @@ -340,6 +341,7 @@ fn evaluate_compiled_system_adjustment( request.candle_index, slice_profit, slice_profit_entry, + slice_profit_exit, open_grind_count, )?; let extra_derisk = any_derisk_level(&state, &policy.extra_entry_derisk_levels)?; @@ -365,6 +367,7 @@ fn evaluate_compiled_system_adjustment( slice_amount, slice_profit, slice_profit_entry, + slice_profit_exit, current_stake_amount: trade.amount * request.candle.open, rebuy_mode, is_grind_entry: grind_entry_signal, @@ -750,6 +753,7 @@ fn evaluate_nfi_position_adjustment_with_state( request.candle_index, slice_profit, slice_profit_entry, + slice_profit_exit, num_open_grinds, )?; let extra_derisk = any_derisk_level(state, &policy.extra_entry_derisk_levels)?; @@ -774,6 +778,7 @@ fn evaluate_nfi_position_adjustment_with_state( slice_amount, slice_profit, slice_profit_entry, + slice_profit_exit, current_stake_amount: trade.amount * candle.open, rebuy_mode, is_grind_entry, @@ -1271,7 +1276,7 @@ fn evaluate_grind_level( }, trade.leverage, )?; - let entry_signal = grind_entry_signal(context, state, index)?; + let entry_signal = grind_entry_signal(context, trade, state, index)?; let below_maximum = context.current_stake_amount < context.slice_amount * context.adjustment.constants.max_stake_multiplier; let distance_allows_entry = if cluster.count == 0 { @@ -1357,6 +1362,7 @@ fn scale_stakes_for_minimum( fn grind_entry_signal( context: &AdjustmentContext<'_>, + trade: &OpenTrade, state: &AdjustmentState, index: usize, ) -> Option { @@ -1386,10 +1392,14 @@ fn grind_entry_signal( adjustment_predicate_matches( predicate, state, + context.config, + trade, context.pair, context.candle_index, + context.candle.open, context.slice_profit, context.slice_profit_entry, + context.slice_profit_exit, num_open_grinds, ) })?; @@ -1400,12 +1410,31 @@ fn grind_entry_signal( fn adjustment_predicate_matches( predicate: &NfiX7AdjustmentPredicate, state: &AdjustmentState, + config: &PortfolioConfig, + trade: &OpenTrade, pair: &PairSeries, candle_index: usize, + current_rate: f64, slice_profit: f64, slice_profit_entry: f64, + slice_profit_exit: f64, num_open_grinds: usize, ) -> Option { + if let Some(expression) = &predicate.expression { + return adjustment_expression_matches( + expression, + state, + config, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, + num_open_grinds, + ); + } if !predicate.any_derisk_levels.is_empty() && !any_derisk_level(state, &predicate.any_derisk_levels)? { @@ -1422,6 +1451,7 @@ fn adjustment_predicate_matches( candle_index, slice_profit, slice_profit_entry, + slice_profit_exit, num_open_grinds, ) } else { @@ -1430,6 +1460,117 @@ fn adjustment_predicate_matches( }) } +#[allow(clippy::too_many_arguments)] +fn adjustment_expression_matches( + expression: &NfiX7AdjustmentExpression, + state: &AdjustmentState, + config: &PortfolioConfig, + trade: &OpenTrade, + pair: &PairSeries, + candle_index: usize, + current_rate: f64, + slice_profit: f64, + slice_profit_entry: f64, + slice_profit_exit: f64, + num_open_grinds: usize, +) -> Option { + match expression { + NfiX7AdjustmentExpression::All { values } => { + for value in values { + if !adjustment_expression_matches( + value, + state, + config, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, + num_open_grinds, + )? { + return Some(false); + } + } + Some(true) + } + NfiX7AdjustmentExpression::Any { values } => { + for value in values { + if adjustment_expression_matches( + value, + state, + config, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, + num_open_grinds, + )? { + return Some(true); + } + } + Some(false) + } + NfiX7AdjustmentExpression::Not { value } => adjustment_expression_matches( + value, + state, + config, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, + num_open_grinds, + ) + .map(|value| !value), + NfiX7AdjustmentExpression::Flag { name } => match name.as_str() { + "is_futures_mode" => Some(config.is_futures), + "trade_is_short" => Some(trade.side == TradeSide::Short), + _ => None, + }, + NfiX7AdjustmentExpression::DeriskFound { level } => { + any_derisk_level(state, std::slice::from_ref(level)) + } + NfiX7AdjustmentExpression::Present { operand } => Some( + adjustment_expression_operand_value( + operand, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, + num_open_grinds, + ) + .is_some(), + ), + NfiX7AdjustmentExpression::Comparison { + left, + operator, + right, + } => adjustment_comparison_matches( + left, + *operator, + right, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, + num_open_grinds, + ), + } +} + fn any_derisk_level(state: &AdjustmentState, levels: &[usize]) -> Option { levels.iter().try_fold(false, |found, level| { let index = level.checked_sub(1)?; @@ -1443,39 +1584,86 @@ fn adjustment_condition_matches( candle_index: usize, slice_profit: f64, slice_profit_entry: f64, + slice_profit_exit: f64, num_open_grinds: usize, ) -> Option { - let left = adjustment_operand_value( + let left = adjustment_legacy_operand_value( &condition.left, pair, candle_index, slice_profit, slice_profit_entry, + slice_profit_exit, num_open_grinds, )?; - let right = adjustment_operand_value( + let right = adjustment_legacy_operand_value( &condition.right, pair, candle_index, slice_profit, slice_profit_entry, + slice_profit_exit, + num_open_grinds, + )?; + Some(adjustment_values_match(left, condition.operator, right)) +} + +#[allow(clippy::too_many_arguments)] +fn adjustment_comparison_matches( + left: &NfiX7AdjustmentOperand, + operator: NfiX7AdjustmentComparison, + right: &NfiX7AdjustmentOperand, + trade: &OpenTrade, + pair: &PairSeries, + candle_index: usize, + current_rate: f64, + slice_profit: f64, + slice_profit_entry: f64, + slice_profit_exit: f64, + num_open_grinds: usize, +) -> Option { + let left = adjustment_expression_operand_value( + left, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, + num_open_grinds, + )?; + let right = adjustment_expression_operand_value( + right, + trade, + pair, + candle_index, + current_rate, + slice_profit, + slice_profit_entry, + slice_profit_exit, num_open_grinds, )?; - Some(match condition.operator { + Some(adjustment_values_match(left, operator, right)) +} + +fn adjustment_values_match(left: f64, operator: NfiX7AdjustmentComparison, right: f64) -> bool { + match operator { NfiX7AdjustmentComparison::Lt => left < right, NfiX7AdjustmentComparison::Gt => left > right, NfiX7AdjustmentComparison::Eq => { matches!(left.partial_cmp(&right), Some(std::cmp::Ordering::Equal)) } - }) + } } -fn adjustment_operand_value( +fn adjustment_legacy_operand_value( operand: &NfiX7AdjustmentOperand, pair: &PairSeries, candle_index: usize, slice_profit: f64, slice_profit_entry: f64, + slice_profit_exit: f64, num_open_grinds: usize, ) -> Option { let value = match operand { @@ -1483,12 +1671,57 @@ fn adjustment_operand_value( NfiX7AdjustmentOperand::Variable { name } => match name.as_str() { "slice_profit" => slice_profit, "slice_profit_entry" => slice_profit_entry, + "slice_profit_exit" => slice_profit_exit, "num_open_grinds_and_buybacks" => f64::from(u32::try_from(num_open_grinds).ok()?), _ => return None, }, NfiX7AdjustmentOperand::Feature { name, multiplier } => { feature_number_at(pair, candle_index, name)? * multiplier } + NfiX7AdjustmentOperand::Trade { .. } => return None, + }; + value.is_finite().then_some(value) +} + +fn adjustment_expression_feature_value( + pair: &PairSeries, + candle_index: usize, + name: &str, +) -> Option { + feature_number_at(pair, candle_index, name).or_else(|| { + feature_bool_at(pair, candle_index, name).map(|value| f64::from(u8::from(value))) + }) +} + +#[allow(clippy::too_many_arguments)] +fn adjustment_expression_operand_value( + operand: &NfiX7AdjustmentOperand, + trade: &OpenTrade, + pair: &PairSeries, + candle_index: usize, + current_rate: f64, + slice_profit: f64, + slice_profit_entry: f64, + slice_profit_exit: f64, + num_open_grinds: usize, +) -> Option { + let value = match operand { + NfiX7AdjustmentOperand::Literal { value } => *value, + NfiX7AdjustmentOperand::Variable { name } => match name.as_str() { + "current_rate" => current_rate, + "slice_profit" => slice_profit, + "slice_profit_entry" => slice_profit_entry, + "slice_profit_exit" => slice_profit_exit, + "num_open_grinds_and_buybacks" => f64::from(u32::try_from(num_open_grinds).ok()?), + _ => return None, + }, + NfiX7AdjustmentOperand::Feature { name, multiplier } => { + adjustment_expression_feature_value(pair, candle_index, name)? * multiplier + } + NfiX7AdjustmentOperand::Trade { name, multiplier } => match name.as_str() { + "liquidation_price" => trade.liquidation_price? * multiplier, + _ => return None, + }, }; value.is_finite().then_some(value) } @@ -1598,15 +1831,17 @@ mod tests { use serde_json::json; use crate::domain::{ - CompiledSystemAdjustmentProgram, FilledOrder, NfiX7AdjustmentConstants, NfiX7GrindLevel, + CompiledSystemAdjustmentProgram, FilledOrder, NfiX7AdjustmentConstants, + NfiX7AdjustmentExpression, NfiX7AdjustmentPredicate, NfiX7GrindLevel, NfiX7PositionAdjustment, OrderSide, PairSeries, PortfolioConfig, }; use crate::portfolio::{OpenTrade, TradeSide}; use super::{ - compiled_adjustment_state, derisk_level_index, evaluate_compiled_system_action, - grind_callback_maximum_stake, grind_callback_minimum_stake, grind_entry_index, - grind_exit_index, rebuild_compiled_adjustment_state, AdjustmentContext, AdjustmentState, + adjustment_predicate_matches, compiled_adjustment_state, derisk_level_index, + evaluate_compiled_system_action, grind_callback_maximum_stake, + grind_callback_minimum_stake, grind_entry_index, grind_exit_index, + rebuild_compiled_adjustment_state, AdjustmentContext, AdjustmentState, CompiledActionOutcome, GrindCluster, NfiProfitSnapshot, }; @@ -1774,6 +2009,225 @@ mod tests { (pair, config) } + fn liquidation_proximity_predicate() -> NfiX7AdjustmentPredicate { + let expression: NfiX7AdjustmentExpression = serde_json::from_value(json!({ + "op": "all", + "values": [ + {"op": "flag", "name": "is_futures_mode"}, + { + "op": "present", + "operand": { + "kind": "trade", + "name": "liquidation_price", + "multiplier": 1.0 + } + }, + { + "op": "any", + "values": [ + { + "op": "all", + "values": [ + {"op": "flag", "name": "trade_is_short"}, + { + "op": "comparison", + "left": {"kind": "variable", "name": "current_rate"}, + "operator": "gt", + "right": { + "kind": "trade", + "name": "liquidation_price", + "multiplier": 0.9 + } + } + ] + }, + { + "op": "all", + "values": [ + { + "op": "not", + "value": {"op": "flag", "name": "trade_is_short"} + }, + { + "op": "comparison", + "left": {"kind": "variable", "name": "current_rate"}, + "operator": "lt", + "right": { + "kind": "trade", + "name": "liquidation_price", + "multiplier": 1.1 + } + } + ] + } + ] + } + ] + })) + .expect("valid liquidation-proximity expression"); + NfiX7AdjustmentPredicate { + any_derisk_levels: Vec::new(), + conditions: Vec::new(), + expression: Some(expression), + } + } + + fn exit_profit_protection_predicate() -> NfiX7AdjustmentPredicate { + let expression: NfiX7AdjustmentExpression = serde_json::from_value(json!({ + "op": "all", + "values": [ + {"op": "derisk_found", "level": 1}, + { + "op": "comparison", + "left": {"kind": "variable", "name": "slice_profit_exit"}, + "operator": "lt", + "right": {"kind": "literal", "value": -0.04} + }, + { + "op": "comparison", + "left": { + "kind": "feature", + "name": "protection", + "multiplier": 1.0 + }, + "operator": "eq", + "right": {"kind": "literal", "value": 1.0} + } + ] + })) + .expect("valid exit-profit protection expression"); + NfiX7AdjustmentPredicate { + any_derisk_levels: Vec::new(), + conditions: Vec::new(), + expression: Some(expression), + } + } + + fn empty_adjustment_state() -> AdjustmentState { + AdjustmentState { + order_count: 1, + program_fingerprint: None, + clusters: Vec::new(), + derisk_found: vec![false; 3], + first_entry_amount: 1.0, + first_entry_cost: 100.0, + latest_entry_price: 100.0, + latest_entry_timestamp_ms: 0, + latest_exit_price: None, + latest_order_price: 100.0, + latest_order_timestamp_ms: 0, + } + } + + #[test] + fn liquidation_proximity_expression_matches_long_futures() { + let (pair, mut config) = pair_and_config(); + config.is_futures = true; + let mut trade = test_trade(); + trade.liquidation_price = Some(95.0); + let predicate = liquidation_proximity_predicate(); + let state = empty_adjustment_state(); + { + let matches = |current_rate| { + adjustment_predicate_matches( + &predicate, + &state, + &config, + &trade, + &pair, + 0, + current_rate, + -0.04, + -0.04, + 0.0, + 0, + ) + }; + + assert_eq!(matches(100.0), Some(true)); + assert_eq!(matches(105.0), Some(false)); + } + trade.liquidation_price = None; + assert_eq!( + adjustment_predicate_matches( + &predicate, &state, &config, &trade, &pair, 0, 100.0, -0.04, -0.04, 0.0, 0, + ), + Some(false) + ); + } + + #[test] + fn liquidation_proximity_expression_matches_short_futures() { + let (pair, mut config) = pair_and_config(); + config.is_futures = true; + let mut trade = test_trade(); + trade.side = TradeSide::Short; + trade.liquidation_price = Some(110.0); + let predicate = liquidation_proximity_predicate(); + let state = empty_adjustment_state(); + { + let matches = |current_rate| { + adjustment_predicate_matches( + &predicate, + &state, + &config, + &trade, + &pair, + 0, + current_rate, + 0.04, + 0.04, + 0.0, + 0, + ) + }; + + assert_eq!(matches(100.0), Some(true)); + assert_eq!(matches(98.0), Some(false)); + } + config.is_futures = false; + assert_eq!( + adjustment_predicate_matches( + &predicate, &state, &config, &trade, &pair, 0, 100.0, 0.04, 0.04, 0.0, 0, + ), + Some(false) + ); + } + + #[test] + fn exit_profit_expression_uses_boolean_feature_and_derisk_state() { + let (mut pair, config) = pair_and_config(); + pair.feature_columns.insert( + "protection".to_owned(), + crate::io::FeatureColumn::booleans(vec![true]), + ); + let trade = test_trade(); + let predicate = exit_profit_protection_predicate(); + let mut state = empty_adjustment_state(); + state.derisk_found[0] = true; + + let matches = |state: &AdjustmentState, slice_profit_exit| { + adjustment_predicate_matches( + &predicate, + state, + &config, + &trade, + &pair, + 0, + 90.0, + -0.1, + -0.1, + slice_profit_exit, + 0, + ) + }; + + assert_eq!(matches(&state, -0.05), Some(true)); + assert_eq!(matches(&state, -0.03), Some(false)); + state.derisk_found[0] = false; + assert_eq!(matches(&state, -0.05), Some(false)); + } + #[test] fn rebuy_transfer_keeps_the_wrappers_leverage_adjusted_minimum() { let exchange_minimum = 11.025_21; @@ -1907,6 +2361,7 @@ mod tests { slice_amount: 100.0, slice_profit: -0.1, slice_profit_entry: -0.1, + slice_profit_exit: 0.0, current_stake_amount: 90.0, rebuy_mode: false, is_grind_entry: false, @@ -1974,6 +2429,7 @@ mod tests { slice_amount: 100.0, slice_profit: -0.1, slice_profit_entry: -0.1, + slice_profit_exit: 0.0, current_stake_amount: 90.0, rebuy_mode: false, is_grind_entry: false, @@ -2050,6 +2506,7 @@ mod tests { slice_amount: 100.0, slice_profit: 0.05, slice_profit_entry: 0.05, + slice_profit_exit: 0.0, current_stake_amount: 157.5, rebuy_mode: false, is_grind_entry: false, diff --git a/rust/crates/nfi-sim-core/src/nfi/exit.rs b/rust/crates/nfi-sim-core/src/nfi/exit.rs index 91d708e9..c46d683a 100644 --- a/rust/crates/nfi-sim-core/src/nfi/exit.rs +++ b/rust/crates/nfi-sim-core/src/nfi/exit.rs @@ -1,6 +1,7 @@ //! NFI custom-exit routing and profit-target state machine. use std::collections::BTreeMap; +use std::fmt; use serde_json::Value; @@ -34,6 +35,60 @@ pub(crate) enum CustomExitDecision { Exit(String), } +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum NfiExitDiagnostic { + DispatchUnavailable, + RouteUnavailable { + side: &'static str, + index: usize, + }, + MatcherShadowDisagreement { + side: &'static str, + route: String, + }, + ManagedRouteEvaluation { + side: &'static str, + route: String, + line: Option, + column: Option, + }, + LegacyGrindSnapshot { + route: String, + }, +} + +impl fmt::Display for NfiExitDiagnostic { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DispatchUnavailable => formatter.write_str("runtime dispatch is unavailable"), + Self::RouteUnavailable { side, index } => { + write!(formatter, "{side} route index {index} is unavailable") + } + Self::MatcherShadowDisagreement { side, route } => { + write!(formatter, "{side} route {route:?} matcher shadow disagreed") + } + Self::ManagedRouteEvaluation { + side, + route, + line: Some(line), + column: Some(column), + } => write!( + formatter, + "{side} route {route:?} evaluation failed at strategy.py:{line}:{column}" + ), + Self::ManagedRouteEvaluation { side, route, .. } => { + write!(formatter, "{side} route {route:?} evaluation failed") + } + Self::LegacyGrindSnapshot { route } => { + write!( + formatter, + "legacy Grind route {route:?} profit snapshot failed" + ) + } + } + } +} + pub(crate) const NFI_LONG_EXIT_PROGRAMS: &[&str] = &[ "long_exit_signals", "long_exit_main", @@ -67,12 +122,19 @@ pub(crate) fn evaluate_nfi_exit( candle: &Candle, config: &PortfolioConfig, profit_targets: &mut BTreeMap, -) -> Option { - let dispatch = manager.runtime_dispatch()?; +) -> Result { + let dispatch = manager + .runtime_dispatch() + .ok_or(NfiExitDiagnostic::DispatchUnavailable)?; let tags = dispatch.intern_trade_tags(trade); for step in &dispatch.long_steps { if let NfiLongDispatchStep::Managed(step) = step { - let route = manager.managed_long_routes.get(step.route_index)?; + let route = manager.managed_long_routes.get(step.route_index).ok_or( + NfiExitDiagnostic::RouteUnavailable { + side: "long", + index: step.route_index, + }, + )?; let source_route = manager.managed_exit_program.as_ref().and_then(|program| { step.source_route_index .and_then(|index| program.routes.get(index)) @@ -88,11 +150,15 @@ pub(crate) fn evaluate_nfi_exit( if !matches!(source_route, Some((ManagedExitExecutionMode::Primary, _))) && source_matches != legacy_matches { - return None; + return Err(NfiExitDiagnostic::MatcherShadowDisagreement { + side: "long", + route: route.key.clone(), + }); } if !source_matches { continue; } + let diagnostic = managed_route_diagnostic("long", route, source_route); match evaluate_nfi_managed_long_exit( manager, source_route, @@ -107,9 +173,11 @@ pub(crate) fn evaluate_nfi_exit( candle, config, profit_targets, - )? { + ) + .ok_or(diagnostic)? + { CustomExitDecision::Exit(reason) => { - return Some(CustomExitDecision::Exit(reason)); + return Ok(CustomExitDecision::Exit(reason)); } CustomExitDecision::NoExit => continue, } @@ -127,11 +195,14 @@ pub(crate) fn evaluate_nfi_exit( fee_open(config), fee_close(config), config.is_futures, - )?; + ) + .ok_or_else(|| NfiExitDiagnostic::LegacyGrindSnapshot { + route: route.mode_name.clone(), + })?; if snapshot.initial_stake_ratio > route.exit_profit_threshold { let entry_tag = trade.entry_tag.as_deref().unwrap_or("empty"); let reason = format!("exit_{}_g", route.mode_name); - return Some(CustomExitDecision::Exit(nfi_exit_reason( + return Ok(CustomExitDecision::Exit(nfi_exit_reason( &reason, entry_tag, ))); } @@ -150,11 +221,11 @@ pub(crate) fn evaluate_nfi_exit( profit_targets, )?; if let CustomExitDecision::Exit(_) = short_decision { - return Some(short_decision); + return Ok(short_decision); } // A compound of individually compiled words may intentionally match no // all-tags route. The source callback returns None in that case. - Some(CustomExitDecision::NoExit) + Ok(CustomExitDecision::NoExit) } /// Execute the bounded short-rebuy branch in source order. @@ -167,11 +238,18 @@ fn evaluate_nfi_short_exit( candle: &Candle, config: &PortfolioConfig, profit_targets: &mut BTreeMap, -) -> Option { - let dispatch = manager.runtime_dispatch()?; +) -> Result { + let dispatch = manager + .runtime_dispatch() + .ok_or(NfiExitDiagnostic::DispatchUnavailable)?; let tags = dispatch.intern_trade_tags(trade); for step in &dispatch.short_steps { - let route = manager.managed_short_routes.get(step.route_index)?; + let route = manager.managed_short_routes.get(step.route_index).ok_or( + NfiExitDiagnostic::RouteUnavailable { + side: "short", + index: step.route_index, + }, + )?; let source_route = manager .managed_short_exit_program .as_ref() @@ -190,11 +268,15 @@ fn evaluate_nfi_short_exit( if !matches!(source_route, Some((ManagedExitExecutionMode::Primary, _))) && source_matches != legacy_matches { - return None; + return Err(NfiExitDiagnostic::MatcherShadowDisagreement { + side: "short", + route: route.key.clone(), + }); } if !source_matches { continue; } + let diagnostic = managed_route_diagnostic("short", route, source_route); match evaluate_nfi_managed_long_exit( manager, source_route, @@ -209,14 +291,39 @@ fn evaluate_nfi_short_exit( candle, config, profit_targets, - )? { + ) + .ok_or(diagnostic)? + { CustomExitDecision::Exit(reason) => { - return Some(CustomExitDecision::Exit(reason)); + return Ok(CustomExitDecision::Exit(reason)); } CustomExitDecision::NoExit => {} } } - Some(CustomExitDecision::NoExit) + Ok(CustomExitDecision::NoExit) +} + +fn managed_route_diagnostic( + side: &'static str, + route: &NfiManagedLongRoute, + source_route: Option<(ManagedExitExecutionMode, &ManagedExitRoute)>, +) -> NfiExitDiagnostic { + let (route, line, column) = source_route.map_or_else( + || (route.key.clone(), None, None), + |(_, source)| { + ( + source.id.clone(), + Some(source.location.line), + Some(source.location.column), + ) + }, + ); + NfiExitDiagnostic::ManagedRouteEvaluation { + side, + route, + line, + column, + } } /// Execute one source-bound NFI X7 managed custom-exit route. @@ -930,9 +1037,16 @@ fn generic_managed_exit_stop( ) -> Option<(bool, Option)> { match &route.state_program.as_ref()?.stop { ManagedExitStopPolicy::SourceHelper { helper } - if (helper == "long_exit_stoploss" && trade.side == TradeSide::Long) - || (helper == "short_exit_stoploss" && trade.side == TradeSide::Short) => + if matches!( + helper.as_str(), + "long_exit_stoploss" | "short_exit_stoploss" + ) => { + // X7's shared enter-tag column can select a route from the + // opposite entry side. The source callback still invokes that + // route's named helper with the actual trade; the helper itself + // applies direction-sensitive predicates. Do not synthesize a + // side gate that is absent from the source route. nfi_common_long_stoploss( manager, &route.mode_name, diff --git a/rust/crates/nfi-sim-core/src/tests/nfi_routing.rs b/rust/crates/nfi-sim-core/src/tests/nfi_routing.rs index 0cd1d79d..043247bb 100644 --- a/rust/crates/nfi-sim-core/src/tests/nfi_routing.rs +++ b/rust/crates/nfi-sim-core/src/tests/nfi_routing.rs @@ -3,6 +3,86 @@ use super::*; use crate::nfi::NFI_LONG_EXIT_PROGRAMS; +#[test] +fn nfi_manager_preflight_accepts_source_shaped_liquidation_fallback() { + let mut manager = nfi_top_coins_manager(nfi_false_program()); + let predicate: NfiX7AdjustmentPredicate = serde_json::from_value(serde_json::json!({ + "any_derisk_levels": [], + "conditions": [], + "expression": { + "op": "all", + "values": [ + {"op": "flag", "name": "is_futures_mode"}, + { + "op": "comparison", + "left": {"kind": "variable", "name": "slice_profit_entry"}, + "operator": "lt", + "right": {"kind": "literal", "value": -0.15} + }, + { + "op": "present", + "operand": { + "kind": "trade", + "name": "liquidation_price", + "multiplier": 1.0 + } + }, + { + "op": "any", + "values": [ + { + "op": "all", + "values": [ + {"op": "flag", "name": "trade_is_short"}, + { + "op": "comparison", + "left": {"kind": "variable", "name": "current_rate"}, + "operator": "gt", + "right": { + "kind": "trade", + "name": "liquidation_price", + "multiplier": 0.8 + } + } + ] + }, + { + "op": "all", + "values": [ + { + "op": "not", + "value": {"op": "flag", "name": "trade_is_short"} + }, + { + "op": "comparison", + "left": {"kind": "variable", "name": "current_rate"}, + "operator": "lt", + "right": { + "kind": "trade", + "name": "liquidation_price", + "multiplier": 1.2 + } + } + ] + } + ] + } + ] + } + })) + .expect("source-shaped generic fallback"); + let policy = manager + .position_adjustment + .as_mut() + .and_then(|adjustment| adjustment.constants.policy.as_mut()) + .expect("test manager adjustment policy"); + policy.grind_entry_fallbacks[4].predicates.push(predicate); + let mut manager_config = config(1); + enable_nfi_manager(&mut manager_config, manager); + + assert_eq!(validate_simulator_preflight(&manager_config), Ok(())); +} + #[test] fn retired_execution_modes_still_deserialize_for_evidence_replay() { let managed: ManagedExitExecutionMode = @@ -303,7 +383,7 @@ fn generic_managed_exit_shadow_short_fails_closed_on_target_state_difference() { let result = simulate(&input); assert!( - matches!(result, Err(SimError::InvalidNfiTradeManager)), + matches!(result, Err(SimError::InvalidNfiExitRuntime { .. })), "unexpected result: {result:?}" ); } @@ -514,7 +594,18 @@ fn generic_managed_exit_shadow_fails_closed_on_a_decision_difference() { )], }); - assert!(matches!(result, Err(SimError::InvalidNfiTradeManager))); + let Err(SimError::InvalidNfiExitRuntime { + pair, + timestamp_ms, + diagnostic, + }) = result + else { + panic!("unexpected result: {result:?}"); + }; + assert_eq!(pair, "AAA/USDT"); + assert_eq!(timestamp_ms, 2); + assert!(diagnostic.contains("long_normal"), "{diagnostic}"); + assert!(diagnostic.contains("strategy.py:"), "{diagnostic}"); } #[test] @@ -550,7 +641,7 @@ fn generic_managed_exit_shadow_fails_closed_on_target_state_difference() { assert!(matches!( simulate(&input), - Err(SimError::InvalidNfiTradeManager) + Err(SimError::InvalidNfiExitRuntime { .. }) )); } @@ -608,7 +699,7 @@ fn generic_managed_exit_shadow_fails_closed_on_inline_program_difference() { assert!(matches!( simulate(&input), - Err(SimError::InvalidNfiTradeManager) + Err(SimError::InvalidNfiExitRuntime { .. }) )); } @@ -1058,6 +1149,39 @@ fn nfi_cross_side_compound_keeps_source_callback_order() { ); } +#[test] +fn generic_short_route_runs_its_source_helper_for_a_long_cross_side_tag() { + let mut entry = candle(1, 100.0, 100.0); + entry.enter_long = Some(EntrySignal { + // Spot cannot open a short trade, but X7's shared enter_tag can retain + // the simultaneous short-normal label. Its callback evaluates the + // short-normal block with this actual long trade. + tag: Some("1 501".to_owned()), + leverage: None, + liquidation_price: None, + }); + let mut manager = nfi_top_coins_manager(nfi_false_program()); + enable_test_full_short_manager(&mut manager); + enable_test_short_exit_shadow(&mut manager); + let mut manager_config = config(1); + manager_config.max_entry_position_adjustment = 0; + enable_nfi_manager(&mut manager_config, manager); + let input = SimulationInput { + schema_version: SIMULATOR_SCHEMA_VERSION.to_owned(), + config: manager_config, + pairs: vec![nfi_pair( + vec![entry, candle(2, 100.0, 100.0)], + BTreeMap::new(), + )], + }; + + let result = simulate(&input).expect("cross-side source helper shadow agrees"); + + assert!(!result.trades[0].is_short); + assert_eq!(result.trades[0].entry_tag.as_deref(), Some("1 501")); + assert_eq!(result.trades[0].exit_reason, "force_exit"); +} + #[test] fn nfi_trade_manager_requires_a_tag_for_the_opened_side() { let mut entry = candle(1, 100.0, 100.0); diff --git a/rust/crates/nfi-sim-core/src/tests/position.rs b/rust/crates/nfi-sim-core/src/tests/position.rs index dbf2b8a6..e737fe0e 100644 --- a/rust/crates/nfi-sim-core/src/tests/position.rs +++ b/rust/crates/nfi-sim-core/src/tests/position.rs @@ -43,6 +43,47 @@ fn entry_adjustment_stop_and_fees_are_accounted_in_order() { assert!((trade.close_rate - 99.0).abs() < f64::EPSILON); } +#[test] +fn same_candle_adjustment_is_applied_before_stop_exit() { + let mut entry = candle(1, 100.0, 100.0); + entry.enter_long = Some(EntrySignal { + tag: Some("entry".to_owned()), + leverage: None, + liquidation_price: None, + }); + let mut adjustment_and_stop = candle(2, 99.5, 98.0); + adjustment_and_stop.adjustment = Some(AdjustmentSignal { + stake_amount: 50.0, + tag: "grind".to_owned(), + }); + let input = SimulationInput { + schema_version: SIMULATOR_SCHEMA_VERSION.to_owned(), + config: config(1), + pairs: vec![PairSeries { + pair: "AAA/USDT".to_owned(), + execution_start_index: 0, + amount_step: None, + price_step: None, + price_steps: Vec::new(), + minimum_stake: None, + minimum_amount: None, + minimum_cost: None, + feature_columns: BTreeMap::new(), + candles: vec![entry, adjustment_and_stop].into(), + }], + }; + + let result = simulate(&input).expect("adjustment precedes the same-candle stop check"); + let trade = &result.trades[0]; + + assert_eq!(trade.exit_reason, "stop_loss"); + assert_eq!(trade.close_timestamp_ms, 2); + assert_eq!(trade.orders.len(), 3); + assert!(trade.orders[1].is_entry); + assert_eq!(trade.orders[1].tag.as_deref(), Some("grind")); + assert!(!trade.orders[2].is_entry); +} + #[test] fn futures_entry_adjustment_replays_funding_at_the_fill_timestamp() { let mut portfolio = config(1); diff --git a/rust/crates/nfi-sim-core/src/tests/support.rs b/rust/crates/nfi-sim-core/src/tests/support.rs index 43fa7ce2..cc9c373a 100644 --- a/rust/crates/nfi-sim-core/src/tests/support.rs +++ b/rust/crates/nfi-sim-core/src/tests/support.rs @@ -1261,6 +1261,7 @@ pub(super) fn nfi_adjustment_policy() -> NfiX7AdjustmentPolicy { feature("EMA_20", 0.98), ), ], + expression: None, }]; fallbacks[4].predicates = vec![NfiX7AdjustmentPredicate { any_derisk_levels: vec![1, 2, 3], @@ -1286,6 +1287,7 @@ pub(super) fn nfi_adjustment_policy() -> NfiX7AdjustmentPolicy { literal(50.0), ), ], + expression: None, }]; NfiX7AdjustmentPolicy { entry_retry_ms: 5 * 60 * 1_000, diff --git a/rust/crates/nfi-sim-core/src/validation/adjustment.rs b/rust/crates/nfi-sim-core/src/validation/adjustment.rs index 4e49e11b..40543caa 100644 --- a/rust/crates/nfi-sim-core/src/validation/adjustment.rs +++ b/rust/crates/nfi-sim-core/src/validation/adjustment.rs @@ -4,9 +4,13 @@ use std::collections::BTreeSet; use crate::domain::{ NfiLegacyGrindConstants, NfiRegularAdjustmentConstants, NfiX7AdjustmentCondition, - NfiX7AdjustmentConstants, NfiX7AdjustmentOperand, NfiX7AdjustmentPolicy, NfiX7RebuyConstants, + NfiX7AdjustmentConstants, NfiX7AdjustmentExpression, NfiX7AdjustmentOperand, + NfiX7AdjustmentPolicy, NfiX7RebuyConstants, }; +const MAX_ADJUSTMENT_EXPRESSION_DEPTH: usize = 16; +const MAX_ADJUSTMENT_EXPRESSION_ARITY: usize = 32; + pub(crate) fn valid_nfi_rebuy_constants(constants: &NfiX7RebuyConstants) -> bool { let vectors = [ (&constants.stakes_futures, &constants.thresholds_futures), @@ -215,13 +219,19 @@ pub(crate) fn valid_nfi_adjustment_policy( }; let fallbacks_are_valid = policy.grind_entry_fallbacks.iter().all(|fallback| { fallback.predicates.iter().all(|predicate| { - (predicate.any_derisk_levels.is_empty() - || valid_derisk_levels(&predicate.any_derisk_levels)) - && !predicate.conditions.is_empty() - && predicate - .conditions - .iter() - .all(valid_nfi_adjustment_condition) + if let Some(expression) = &predicate.expression { + predicate.any_derisk_levels.is_empty() + && predicate.conditions.is_empty() + && valid_nfi_adjustment_expression(expression, derisk_level_count, 1) + } else { + (predicate.any_derisk_levels.is_empty() + || valid_derisk_levels(&predicate.any_derisk_levels)) + && !predicate.conditions.is_empty() + && predicate + .conditions + .iter() + .all(valid_nfi_adjustment_condition) + } }) }); @@ -250,7 +260,25 @@ fn unique_cluster_tags<'a>(mut tags: impl Iterator bool { - valid_nfi_adjustment_operand(&condition.left) && valid_nfi_adjustment_operand(&condition.right) + valid_nfi_legacy_adjustment_operand(&condition.left) + && valid_nfi_legacy_adjustment_operand(&condition.right) +} + +fn valid_nfi_legacy_adjustment_operand(operand: &NfiX7AdjustmentOperand) -> bool { + match operand { + NfiX7AdjustmentOperand::Literal { value } => value.is_finite(), + NfiX7AdjustmentOperand::Variable { name } => matches!( + name.as_str(), + "slice_profit" + | "slice_profit_entry" + | "slice_profit_exit" + | "num_open_grinds_and_buybacks" + ), + NfiX7AdjustmentOperand::Feature { name, multiplier } => { + !name.is_empty() && multiplier.is_finite() + } + NfiX7AdjustmentOperand::Trade { .. } => false, + } } pub(crate) fn valid_nfi_adjustment_operand(operand: &NfiX7AdjustmentOperand) -> bool { @@ -258,10 +286,125 @@ pub(crate) fn valid_nfi_adjustment_operand(operand: &NfiX7AdjustmentOperand) -> NfiX7AdjustmentOperand::Literal { value } => value.is_finite(), NfiX7AdjustmentOperand::Variable { name } => matches!( name.as_str(), - "slice_profit" | "slice_profit_entry" | "num_open_grinds_and_buybacks" + "current_rate" + | "slice_profit" + | "slice_profit_entry" + | "slice_profit_exit" + | "num_open_grinds_and_buybacks" ), NfiX7AdjustmentOperand::Feature { name, multiplier } => { !name.is_empty() && multiplier.is_finite() } + NfiX7AdjustmentOperand::Trade { name, multiplier } => { + name == "liquidation_price" && multiplier.is_finite() + } + } +} + +fn valid_nfi_adjustment_expression( + expression: &NfiX7AdjustmentExpression, + derisk_level_count: usize, + depth: usize, +) -> bool { + if depth > MAX_ADJUSTMENT_EXPRESSION_DEPTH { + return false; + } + let valid_values = |values: &[NfiX7AdjustmentExpression]| { + !values.is_empty() + && values.len() <= MAX_ADJUSTMENT_EXPRESSION_ARITY + && values + .iter() + .all(|value| valid_nfi_adjustment_expression(value, derisk_level_count, depth + 1)) + }; + match expression { + NfiX7AdjustmentExpression::All { values } | NfiX7AdjustmentExpression::Any { values } => { + valid_values(values) + } + NfiX7AdjustmentExpression::Not { value } => { + valid_nfi_adjustment_expression(value, derisk_level_count, depth + 1) + } + NfiX7AdjustmentExpression::Flag { name } => { + matches!(name.as_str(), "is_futures_mode" | "trade_is_short") + } + NfiX7AdjustmentExpression::DeriskFound { level } => { + (1..=derisk_level_count).contains(level) + } + NfiX7AdjustmentExpression::Present { operand } => valid_nfi_adjustment_operand(operand), + NfiX7AdjustmentExpression::Comparison { left, right, .. } => { + valid_nfi_adjustment_operand(left) && valid_nfi_adjustment_operand(right) + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::domain::{NfiX7AdjustmentExpression, NfiX7AdjustmentPredicate}; + + use super::{ + valid_nfi_adjustment_expression, MAX_ADJUSTMENT_EXPRESSION_ARITY, + MAX_ADJUSTMENT_EXPRESSION_DEPTH, + }; + + #[test] + fn legacy_adjustment_predicate_deserializes_without_expression() { + let predicate: NfiX7AdjustmentPredicate = serde_json::from_value(json!({ + "any_derisk_levels": [1], + "conditions": [{ + "left": {"kind": "variable", "name": "slice_profit"}, + "operator": "lt", + "right": {"kind": "literal", "value": -0.03} + }] + })) + .expect("legacy predicate remains compatible"); + + assert!(predicate.expression.is_none()); + } + + #[test] + fn adjustment_expression_rejects_invalid_names_and_shapes() { + let invalid = [ + json!({"op": "flag", "name": "strategy_specific_mode"}), + json!({ + "op": "comparison", + "left": {"kind": "variable", "name": "unknown_rate"}, + "operator": "lt", + "right": {"kind": "literal", "value": 1.0} + }), + json!({ + "op": "present", + "operand": {"kind": "trade", "name": "open_rate", "multiplier": 1.0} + }), + json!({"op": "derisk_found", "level": 4}), + json!({"op": "all", "values": []}), + ]; + for value in invalid { + let expression: NfiX7AdjustmentExpression = + serde_json::from_value(value).expect("structurally valid expression"); + assert!(!valid_nfi_adjustment_expression(&expression, 3, 1)); + } + } + + #[test] + fn adjustment_expression_rejects_excessive_depth_and_arity() { + let wide = NfiX7AdjustmentExpression::All { + values: (0..=MAX_ADJUSTMENT_EXPRESSION_ARITY) + .map(|_| NfiX7AdjustmentExpression::Flag { + name: "is_futures_mode".to_owned(), + }) + .collect(), + }; + assert!(!valid_nfi_adjustment_expression(&wide, 3, 1)); + + let mut deep = NfiX7AdjustmentExpression::Flag { + name: "is_futures_mode".to_owned(), + }; + for _ in 0..MAX_ADJUSTMENT_EXPRESSION_DEPTH { + deep = NfiX7AdjustmentExpression::Not { + value: Box::new(deep), + }; + } + assert!(!valid_nfi_adjustment_expression(&deep, 3, 1)); } } diff --git a/rust/crates/nfi-sim-core/src/validation/config.rs b/rust/crates/nfi-sim-core/src/validation/config.rs index a52ad8ba..381011ce 100644 --- a/rust/crates/nfi-sim-core/src/validation/config.rs +++ b/rust/crates/nfi-sim-core/src/validation/config.rs @@ -22,7 +22,39 @@ pub(crate) fn validate_input(input: &SimulationInput) -> Result Result<(), SimError> { for (name, value) in [ ("starting_balance", config.starting_balance), ("stake_amount", config.stake_amount), @@ -134,24 +166,7 @@ pub(crate) fn validate_input(input: &SimulationInput) -> Result Result<(), SimError> { diff --git a/rust/crates/nfi-sim-core/src/validation/mod.rs b/rust/crates/nfi-sim-core/src/validation/mod.rs index 3abc1253..e01a1440 100644 --- a/rust/crates/nfi-sim-core/src/validation/mod.rs +++ b/rust/crates/nfi-sim-core/src/validation/mod.rs @@ -8,6 +8,7 @@ mod pair; mod routing; pub(crate) use config::validate_input; +pub use config::validate_simulator_preflight; #[cfg(test)] pub(crate) use manager::valid_nfi_managed_long_route; pub(crate) use pair::freqtrade_entry_signal; diff --git a/rust/crates/nfi-vector-core/src/alignment/catalog.rs b/rust/crates/nfi-vector-core/src/alignment/catalog.rs new file mode 100644 index 00000000..f893cedc --- /dev/null +++ b/rust/crates/nfi-vector-core/src/alignment/catalog.rs @@ -0,0 +1,268 @@ +//! Immutable, identity-exact source frames for compiled indicator programs. + +use std::collections::BTreeMap; + +use super::{FrameIdentity, NumericFrame, SourceLocation}; +use crate::VectorCoreError; + +/// Validated source frames keyed by their exact pair and timeframe. +/// +/// The catalog never normalizes a pair, substitutes a timeframe, fills an +/// empty frame, or stores caller mutation overlays. A runtime may borrow the +/// same source repeatedly and keep each execution overlay outside the catalog. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct FrameCatalog { + frames: BTreeMap, +} + +impl FrameCatalog { + /// Build a catalog from explicit key/frame entries. + /// + /// Keeping the key separate from the stored frame makes manifest or loader + /// identity drift detectable before an indicator program starts. + /// + /// # Errors + /// + /// Returns an invalid-program error for an empty pair, a key/frame identity + /// mismatch, a duplicate identity, or an invalid frame shape. + pub fn new( + entries: impl IntoIterator, + ) -> Result { + let mut frames = BTreeMap::new(); + for (identity, frame) in entries { + validate_entry(&identity, &frame)?; + if frames.insert(identity.clone(), frame).is_some() { + return Err(VectorCoreError::InvalidProgram(format!( + "frame catalog contains duplicate identity {} {}", + identity.pair, + identity.timeframe.as_str() + ))); + } + } + Ok(Self { frames }) + } + + /// Number of exact pair/timeframe frames in the catalog. + #[must_use] + pub fn len(&self) -> usize { + self.frames.len() + } + + /// Whether the catalog has no source frames. + #[must_use] + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + + /// Iterate the validated identities in deterministic pair/timeframe order. + pub fn identities(&self) -> impl Iterator { + self.frames.keys() + } + + /// Resolve exactly one immutable source frame. + /// + /// An explicitly stored empty frame is returned unchanged. Call + /// [`Self::lookup_non_empty`] when the compiled path cannot represent the + /// strategy's empty-frame branch. + /// + /// # Errors + /// + /// Returns a source-located execution error when the exact identity is not + /// present. No other pair or timeframe is considered. + pub fn lookup( + &self, + identity: &FrameIdentity, + source: &SourceLocation, + ) -> Result<&NumericFrame, VectorCoreError> { + self.frames.get(identity).ok_or_else(|| { + source.error(format!( + "frame catalog has no exact frame for {} {}", + identity.pair, + identity.timeframe.as_str() + )) + }) + } + + /// Resolve a frame and reject a present-but-empty source explicitly. + /// + /// # Errors + /// + /// Returns the source-located missing error from [`Self::lookup`], or a + /// source-located empty-frame error. It never synthesizes candle rows. + pub fn lookup_non_empty( + &self, + identity: &FrameIdentity, + source: &SourceLocation, + ) -> Result<&NumericFrame, VectorCoreError> { + let frame = self.lookup(identity, source)?; + if frame.timestamps_ms.is_empty() { + return Err(source.error(format!( + "frame catalog frame {} {} is empty", + identity.pair, + identity.timeframe.as_str() + ))); + } + Ok(frame) + } +} + +fn validate_entry(identity: &FrameIdentity, frame: &NumericFrame) -> Result<(), VectorCoreError> { + if identity.pair.is_empty() || frame.identity.pair.is_empty() { + return Err(VectorCoreError::InvalidProgram( + "frame catalog identity has an empty pair".to_owned(), + )); + } + if identity != &frame.identity { + return Err(VectorCoreError::InvalidProgram(format!( + "frame catalog key {} {} differs from stored frame identity {} {}", + identity.pair, + identity.timeframe.as_str(), + frame.identity.pair, + frame.identity.timeframe.as_str() + ))); + } + frame.validate().map_err(|error| { + VectorCoreError::InvalidProgram(format!( + "frame catalog frame {} {} has invalid shape: {error}", + identity.pair, + identity.timeframe.as_str() + )) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::alignment::Timeframe; + + fn identity(pair: &str, timeframe: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse(timeframe).expect("timeframe")) + .expect("frame identity") + } + + fn frame(pair: &str, timeframe: &str, values: Vec>) -> NumericFrame { + NumericFrame { + identity: identity(pair, timeframe), + timestamps_ms: (0..values.len()) + .map(|row| i64::try_from(row).expect("test row") * 60_000) + .collect(), + columns: BTreeMap::from([("close".to_owned(), values)]), + } + } + + fn source() -> SourceLocation { + SourceLocation::new("n17", "strategy.py", 3353, 21) + } + + #[test] + fn rejects_duplicate_and_key_frame_identity_drift() { + let key = identity("ETH/USDT", "1h"); + let first = frame("ETH/USDT", "1h", vec![Some(1.0)]); + let second = frame("ETH/USDT", "1h", vec![Some(2.0)]); + let duplicate = FrameCatalog::new([(key.clone(), first), (key.clone(), second)]) + .expect_err("duplicate identity"); + assert!(matches!( + duplicate, + VectorCoreError::InvalidProgram(message) + if message.contains("duplicate identity ETH/USDT 1h") + )); + + let drifted = FrameCatalog::new([( + identity("ETH/USDT", "1h"), + frame("BTC/USDT", "1h", vec![Some(1.0)]), + )]) + .expect_err("key/frame drift"); + assert!(matches!( + drifted, + VectorCoreError::InvalidProgram(message) + if message.contains("differs from stored frame identity") + )); + } + + #[test] + fn rejects_invalid_frame_shape_before_storage() { + let identity = identity("ETH/USDT", "1h"); + let invalid = NumericFrame { + identity: identity.clone(), + timestamps_ms: vec![0, 60_000], + columns: BTreeMap::from([("close".to_owned(), vec![Some(1.0)])]), + }; + let error = FrameCatalog::new([(identity, invalid)]).expect_err("invalid shape"); + assert!(matches!( + error, + VectorCoreError::InvalidProgram(message) if message.contains("invalid shape") + )); + } + + #[test] + fn exact_lookup_never_substitutes_pair_or_timeframe() { + let stored_identity = identity("ETH/USDT", "1h"); + let catalog = FrameCatalog::new([( + stored_identity.clone(), + frame("ETH/USDT", "1h", vec![Some(7.0)]), + )]) + .expect("catalog"); + assert_eq!(catalog.len(), 1); + assert_eq!(catalog.identities().collect::>(), [&stored_identity]); + assert_eq!( + catalog + .lookup(&stored_identity, &source()) + .expect("exact") + .columns["close"], + vec![Some(7.0)] + ); + + for missing in [identity("BTC/USDT", "1h"), identity("ETH/USDT", "4h")] { + let error = catalog.lookup(&missing, &source()).expect_err("exact miss"); + assert!(matches!( + error, + VectorCoreError::Execution { node, message } + if node == "n17" + && message.starts_with("strategy.py:3353:21:") + && message.contains(&missing.pair) + && message.contains(missing.timeframe.as_str()) + )); + } + } + + #[test] + fn empty_source_is_preserved_or_source_located_on_demand() { + let identity = identity("ETH/USDT", "1h"); + let catalog = FrameCatalog::new([(identity.clone(), frame("ETH/USDT", "1h", Vec::new()))]) + .expect("empty catalog frame is explicit data"); + assert!(catalog + .lookup(&identity, &source()) + .expect("explicit empty") + .timestamps_ms + .is_empty()); + let error = catalog + .lookup_non_empty(&identity, &source()) + .expect_err("empty exact lane"); + assert!(matches!( + error, + VectorCoreError::Execution { node, message } + if node == "n17" + && message == "strategy.py:3353:21: frame catalog frame ETH/USDT 1h is empty" + )); + } + + #[test] + fn repeated_lookup_keeps_caller_overlays_outside_the_catalog() { + let identity = identity("ETH/USDT", "1h"); + let catalog = + FrameCatalog::new([(identity.clone(), frame("ETH/USDT", "1h", vec![Some(3.0)]))]) + .expect("catalog"); + + let mut caller_overlay = catalog.lookup(&identity, &source()).expect("first").clone(); + caller_overlay.columns.get_mut("close").expect("close")[0] = Some(99.0); + + assert_eq!(caller_overlay.columns["close"], vec![Some(99.0)]); + assert_eq!( + catalog + .lookup(&identity, &source()) + .expect("second") + .columns["close"], + vec![Some(3.0)] + ); + } +} diff --git a/rust/crates/nfi-vector-core/src/alignment/mod.rs b/rust/crates/nfi-vector-core/src/alignment/mod.rs index c03114ce..45d783c1 100644 --- a/rust/crates/nfi-vector-core/src/alignment/mod.rs +++ b/rust/crates/nfi-vector-core/src/alignment/mod.rs @@ -7,12 +7,14 @@ //! be joined accidentally. mod batch; +mod catalog; mod fill; mod model; mod stream; mod support; pub use batch::merge; +pub use catalog::FrameCatalog; pub use fill::{forward_fill, ForwardFillStream}; pub use model::{FrameIdentity, MergeSpec, MergedFrame, NumericFrame, SourceLocation, Timeframe}; pub use stream::MergeStream; diff --git a/rust/crates/nfi-vector-core/src/alignment/model.rs b/rust/crates/nfi-vector-core/src/alignment/model.rs index b4bd41be..04a69de9 100644 --- a/rust/crates/nfi-vector-core/src/alignment/model.rs +++ b/rust/crates/nfi-vector-core/src/alignment/model.rs @@ -21,6 +21,21 @@ impl SourceLocation { column, } } + + /// Attach a runtime failure to the strategy operation that requested it. + #[must_use] + pub fn error(&self, message: impl Into) -> VectorCoreError { + VectorCoreError::Execution { + node: self.node.clone(), + message: format!( + "{}:{}:{}: {}", + self.path, + self.line, + self.column, + message.into() + ), + } + } } /// A canonical Freqtrade timeframe token. @@ -187,16 +202,7 @@ impl MergeSpec { } pub(super) fn error(&self, message: impl Into) -> VectorCoreError { - VectorCoreError::Execution { - node: self.source.node.clone(), - message: format!( - "{}:{}:{}: {}", - self.source.path, - self.source.line, - self.source.column, - message.into() - ), - } + self.source.error(message) } pub(super) fn output_name(&self, name: &str) -> Result { diff --git a/rust/crates/nfi-vector-core/src/engine/array.rs b/rust/crates/nfi-vector-core/src/engine/array.rs new file mode 100644 index 00000000..57827e4a --- /dev/null +++ b/rust/crates/nfi-vector-core/src/engine/array.rs @@ -0,0 +1,1006 @@ +//! Exact NumPy-shaped and native array operations used by indicator programs. + +use std::collections::BTreeMap; + +use serde_json::{Map, Value}; + +use crate::column::{OwnedColumn, ValueType}; +use crate::error::VectorCoreError; +use crate::float::{binary, canonicalize, BinaryFloatOp}; +use crate::kernels::{AbsoluteDifferenceStream, HourlyInsideBarStream, UtcOpeningRangeStream}; +use crate::program::{ProgramNode, SourceLocation}; + +use super::operations::resolve_value; +use super::runtime::{NodeValue, RuntimeColumn}; + +#[derive(Debug)] +struct OpeningRangeState { + cutoff_hour: u8, + stream: UtcOpeningRangeStream, +} + +/// Explicit bounded state retained by streaming array-call kernels. +#[derive(Debug, Default)] +pub(super) struct ArrayCallState { + absolute_differences: BTreeMap, + opening_ranges: BTreeMap, + inside_bars: BTreeMap, +} + +impl ArrayCallState { + /// Number of scalar aggregate values retained across batches. + #[must_use] + pub(super) fn retained(&self) -> usize { + self.absolute_differences + .values() + .map(AbsoluteDifferenceStream::retained) + .sum::() + .saturating_add( + self.opening_ranges + .values() + .map(|state| state.stream.retained()) + .sum::(), + ) + .saturating_add( + self.inside_bars + .values() + .map(HourlyInsideBarStream::retained) + .sum::(), + ) + } +} + +/// Execute the bounded array-call surface emitted by the latest indicator compiler. +/// +/// Unsupported families, names, signatures, arguments, and nullable inputs to a +/// stateful native kernel fail closed. Element-wise operations preserve Arrow null +/// independently from IEEE NaN. +pub(super) fn execute_array_call<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + state: &mut ArrayCallState, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + let family = parameter_string(node, "family", source)?; + let name = parameter_string(node, "name", source)?; + let arguments = arguments(node, source)?; + match family { + "numpy" if arguments.is_empty() => execute_numpy(node, name, values, rows, state, source), + "native" => execute_native(node, name, arguments, values, rows, state, source), + _ => Err(unsupported(node, source)), + } +} + +fn execute_numpy<'batch>( + node: &ProgramNode, + name: &str, + values: &BTreeMap>, + rows: usize, + state: &mut ArrayCallState, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + match name { + "maximum" => elementwise_extreme(node, values, rows, true, source), + "minimum" => elementwise_extreme(node, values, rows, false, source), + "abs" => elementwise_unary(node, values, rows, f64::abs, source), + "sqrt" => square_root(node, values, rows, source), + "absolute-difference" => absolute_difference(node, values, rows, state, source), + "full_like" => full_like(node, values, rows, source), + "divide" => divide_where(node, values, rows, source), + "zeros_like" => zeros_like(node, values, rows, source), + "fill-missing" => fill_missing(node, values, rows, source), + "nan_to_num" => elementwise_unary(node, values, rows, numpy_nan_to_num, source), + _ => Err(unsupported(node, source)), + } +} + +fn execute_native<'batch>( + node: &ProgramNode, + name: &str, + arguments: &Map, + values: &BTreeMap>, + rows: usize, + state: &mut ArrayCallState, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + match name { + "opening-range-high" | "opening-range-low" => { + opening_range(node, name, arguments, values, rows, state, source) + } + "inside-bar-ready" | "inside-bar-mother-high" | "inside-bar-mother-low" + if arguments.is_empty() => + { + inside_bar(node, name, values, rows, state, source) + } + _ => Err(unsupported(node, source)), + } +} + +fn elementwise_extreme<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + maximum: bool, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let [left, right] = two_inputs(node, source)?; + require_f64_column(values, left, rows, node, source)?; + require_f64_column(values, right, rows, node, source)?; + let output = (0..rows) + .map(|row| { + let left = numeric_at(values, left, row, node, source)?; + let right = numeric_at(values, right, row, node, source)?; + Ok(match (left, right) { + (Some(left), Some(right)) => { + Some(canonicalize(if left.is_nan() || right.is_nan() { + f64::NAN + } else if (maximum && left > right) || (!maximum && left < right) { + left + } else { + // NumPy chooses the right operand for equal values, including signed zero. + right + })) + } + _ => None, + }) + }) + .collect::, VectorCoreError>>()?; + Ok(column(output)) +} + +fn elementwise_unary<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + operation: fn(f64) -> f64, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let input = one_input(node, source)?; + require_f64_column(values, input, rows, node, source)?; + let output = (0..rows) + .map(|row| { + numeric_at(values, input, row, node, source) + .map(|value| value.map(|value| canonicalize(operation(value)))) + }) + .collect::, _>>()?; + Ok(column(output)) +} + +fn square_root<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + let input = one_input(node, source)?; + match resolve_value(values, input).map_err(|error| contextual(node, source, &error))? { + NodeValue::Null => { + require_scalar_output(node, source)?; + Ok(NodeValue::Null) + } + NodeValue::Integer(value) => { + require_scalar_output(node, source)?; + Ok(NodeValue::Float(canonicalize(i64_as_f64(*value).sqrt()))) + } + NodeValue::Float(value) => { + require_scalar_output(node, source)?; + Ok(NodeValue::Float(canonicalize(value.sqrt()))) + } + NodeValue::Column(input_column) if input_column.value_type() == ValueType::F64 => { + require_output(node, "f64-column", source)?; + require_length(input_column, rows, node, source)?; + let output = (0..rows) + .map(|row| { + input_column + .f64_at(row) + .map(|value| canonicalize(value.sqrt())) + }) + .collect(); + Ok(column(output)) + } + _ => Err(error(node, source, "numpy sqrt input is not numeric")), + } +} + +fn absolute_difference<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + state: &mut ArrayCallState, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let input = one_input(node, source)?; + let input = collect_present_f64(values, input, rows, node, source)?; + let output = state + .absolute_differences + .entry(node.id.clone()) + .or_default() + .execute(&input); + Ok(column(output.into_iter().map(Some).collect())) +} + +fn full_like<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let [template, fill] = two_inputs(node, source)?; + require_f64_column(values, template, rows, node, source)?; + let fill = numeric_scalar(values, fill, node, source)? + .ok_or_else(|| error(node, source, "numpy full_like fill is Arrow null"))?; + Ok(column(vec![Some(fill); rows])) +} + +fn zeros_like<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let template = one_input(node, source)?; + require_f64_column(values, template, rows, node, source)?; + Ok(column(vec![Some(0.0); rows])) +} + +fn fill_missing<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let [input, fill] = two_inputs(node, source)?; + let input = require_f64_column(values, input, rows, node, source)?; + let fill = numeric_scalar(values, fill, node, source)? + .ok_or_else(|| error(node, source, "fill-missing value is Arrow null"))?; + Ok(column( + (0..rows) + .map(|row| match input.f64_at(row) { + Some(value) if !value.is_nan() => Some(value), + _ => Some(fill), + }) + .collect(), + )) +} + +fn divide_where<'batch>( + node: &ProgramNode, + values: &BTreeMap>, + rows: usize, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let [numerator, denominator, out, where_mask] = four_inputs(node, source)?; + require_f64_column(values, numerator, rows, node, source)?; + require_f64_column(values, denominator, rows, node, source)?; + let out = require_f64_column(values, out, rows, node, source)?; + let mask = require_bool_column(values, where_mask, rows, node, source)?; + let output = (0..rows) + .map(|row| match mask.bool_at(row) { + Some(true) => match ( + numeric_at(values, numerator, row, node, source)?, + numeric_at(values, denominator, row, node, source)?, + ) { + (Some(left), Some(right)) => Ok(Some(binary(left, right, BinaryFloatOp::Divide))), + _ => Ok(None), + }, + Some(false) | None => Ok(out.f64_at(row)), + }) + .collect::, VectorCoreError>>()?; + Ok(column(output)) +} + +fn opening_range<'batch>( + node: &ProgramNode, + name: &str, + arguments: &Map, + values: &BTreeMap>, + rows: usize, + state: &mut ArrayCallState, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + if arguments.len() != 1 { + return Err(error(node, source, "opening range arguments are invalid")); + } + let cutoff_hour = arguments + .get("cutoff_hour") + .and_then(Value::as_u64) + .and_then(|value| u8::try_from(value).ok()) + .ok_or_else(|| error(node, source, "opening range cutoff_hour is invalid"))?; + let [timestamps, high, low] = three_inputs(node, source)?; + let timestamps = collect_present_timestamps(values, timestamps, rows, node, source)?; + let high = collect_present_f64(values, high, rows, node, source)?; + let low = collect_present_f64(values, low, rows, node, source)?; + let entry = match state.opening_ranges.entry(node.id.clone()) { + std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::btree_map::Entry::Vacant(entry) => entry.insert(OpeningRangeState { + cutoff_hour, + stream: UtcOpeningRangeStream::new(cutoff_hour) + .map_err(|failure| contextual(node, source, &failure))?, + }), + }; + if entry.cutoff_hour != cutoff_hour { + return Err(error( + node, + source, + "opening range cutoff changed during execution", + )); + } + let output = entry + .stream + .execute(×tamps, &high, &low) + .map_err(|failure| contextual(node, source, &failure))?; + let selected = if name == "opening-range-high" { + output.high + } else { + output.low + }; + Ok(column(selected.into_iter().map(Some).collect())) +} + +fn inside_bar<'batch>( + node: &ProgramNode, + name: &str, + values: &BTreeMap>, + rows: usize, + state: &mut ArrayCallState, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_output(node, "f64-column", source)?; + let [timestamps, high, low] = three_inputs(node, source)?; + let timestamps = collect_present_timestamps(values, timestamps, rows, node, source)?; + let high = collect_present_f64(values, high, rows, node, source)?; + let low = collect_present_f64(values, low, rows, node, source)?; + let output = state + .inside_bars + .entry(node.id.clone()) + .or_default() + .execute(×tamps, &high, &low) + .map_err(|failure| contextual(node, source, &failure))?; + let selected = match name { + "inside-bar-ready" => output.ready, + "inside-bar-mother-high" => output.mother_high, + "inside-bar-mother-low" => output.mother_low, + _ => return Err(unsupported(node, source)), + }; + Ok(column(selected.into_iter().map(Some).collect())) +} + +fn require_f64_column<'values, 'batch>( + values: &'values BTreeMap>, + input: &str, + rows: usize, + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result<&'values RuntimeColumn<'batch>, VectorCoreError> { + match resolve_value(values, input).map_err(|failure| contextual(node, source, &failure))? { + NodeValue::Column(column) if column.value_type() == ValueType::F64 => { + require_length(column, rows, node, source)?; + Ok(column) + } + _ => Err(error(node, source, "array input is not a Float64 column")), + } +} + +fn require_bool_column<'values, 'batch>( + values: &'values BTreeMap>, + input: &str, + rows: usize, + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result<&'values RuntimeColumn<'batch>, VectorCoreError> { + match resolve_value(values, input).map_err(|failure| contextual(node, source, &failure))? { + NodeValue::Column(column) if column.value_type() == ValueType::Bool => { + require_length(column, rows, node, source)?; + Ok(column) + } + _ => Err(error(node, source, "array mask is not a Boolean column")), + } +} + +fn require_length( + column: &RuntimeColumn<'_>, + rows: usize, + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result<(), VectorCoreError> { + let actual = match column { + RuntimeColumn::Borrowed(column) => column.len(), + RuntimeColumn::Owned(column) => column.len(), + }; + if actual == rows { + Ok(()) + } else { + Err(error( + node, + source, + format!("array input has {actual} rows; expected {rows}"), + )) + } +} + +fn collect_present_f64( + values: &BTreeMap>, + input: &str, + rows: usize, + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + require_f64_column(values, input, rows, node, source)?; + (0..rows) + .map(|row| { + numeric_at(values, input, row, node, source)?.ok_or_else(|| { + error( + node, + source, + "stateful native array input contains an Arrow null", + ) + }) + }) + .collect() +} + +fn collect_present_timestamps( + values: &BTreeMap>, + input: &str, + rows: usize, + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + let column = match resolve_value(values, input).map_err(|e| contextual(node, source, &e))? { + NodeValue::Column(column) if column.value_type() == ValueType::TimestampMs => column, + _ => { + return Err(error( + node, + source, + "native array input is not a timestamp column", + )) + } + }; + require_length(column, rows, node, source)?; + (0..rows) + .map(|row| { + column.timestamp_ms_at(row).ok_or_else(|| { + error( + node, + source, + "stateful native timestamp input contains an Arrow null", + ) + }) + }) + .collect() +} + +fn numeric_at( + values: &BTreeMap>, + input: &str, + row: usize, + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + match resolve_value(values, input).map_err(|failure| contextual(node, source, &failure))? { + NodeValue::Null => Ok(None), + NodeValue::Integer(value) => Ok(Some(i64_as_f64(*value))), + NodeValue::Float(value) => Ok(Some(*value)), + NodeValue::Column(column) if column.value_type() == ValueType::I64 => { + Ok(column.i64_at(row).map(i64_as_f64)) + } + NodeValue::Column(column) if column.value_type() == ValueType::F64 => { + Ok(column.f64_at(row)) + } + _ => Err(error(node, source, "array value is not numeric")), + } +} + +fn numeric_scalar( + values: &BTreeMap>, + input: &str, + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result, VectorCoreError> { + match resolve_value(values, input).map_err(|failure| contextual(node, source, &failure))? { + NodeValue::Null => Ok(None), + NodeValue::Integer(value) => Ok(Some(i64_as_f64(*value))), + NodeValue::Float(value) => Ok(Some(*value)), + _ => Err(error(node, source, "array fill is not a numeric scalar")), + } +} + +fn require_output( + node: &ProgramNode, + expected: &str, + source: Option<&SourceLocation>, +) -> Result<(), VectorCoreError> { + if node.value_type == expected { + Ok(()) + } else { + Err(error( + node, + source, + format!( + "array output type is {}; expected {expected}", + node.value_type + ), + )) + } +} + +fn require_scalar_output( + node: &ProgramNode, + source: Option<&SourceLocation>, +) -> Result<(), VectorCoreError> { + if matches!(node.value_type.as_str(), "dynamic" | "f64-scalar") { + Ok(()) + } else { + Err(error( + node, + source, + format!( + "numpy scalar output type is {}; expected dynamic or f64-scalar", + node.value_type + ), + )) + } +} + +fn arguments<'node>( + node: &'node ProgramNode, + source: Option<&SourceLocation>, +) -> Result<&'node Map, VectorCoreError> { + if node.parameters.len() != 3 { + return Err(error(node, source, "array-call parameters are not exact")); + } + node.parameters + .get("arguments") + .and_then(Value::as_object) + .ok_or_else(|| error(node, source, "array-call arguments are not an object")) +} + +fn parameter_string<'node>( + node: &'node ProgramNode, + name: &str, + source: Option<&SourceLocation>, +) -> Result<&'node str, VectorCoreError> { + node.parameters + .get(name) + .and_then(Value::as_str) + .ok_or_else(|| error(node, source, format!("array-call {name} is not a string"))) +} + +fn one_input<'node>( + node: &'node ProgramNode, + source: Option<&SourceLocation>, +) -> Result<&'node str, VectorCoreError> { + match node.inputs.as_slice() { + [input] => Ok(input), + _ => Err(error(node, source, "array-call requires one input")), + } +} + +fn two_inputs<'node>( + node: &'node ProgramNode, + source: Option<&SourceLocation>, +) -> Result<[&'node str; 2], VectorCoreError> { + match node.inputs.as_slice() { + [left, right] => Ok([left, right]), + _ => Err(error(node, source, "array-call requires two inputs")), + } +} + +fn three_inputs<'node>( + node: &'node ProgramNode, + source: Option<&SourceLocation>, +) -> Result<[&'node str; 3], VectorCoreError> { + match node.inputs.as_slice() { + [first, second, third] => Ok([first, second, third]), + _ => Err(error(node, source, "array-call requires three inputs")), + } +} + +fn four_inputs<'node>( + node: &'node ProgramNode, + source: Option<&SourceLocation>, +) -> Result<[&'node str; 4], VectorCoreError> { + match node.inputs.as_slice() { + [first, second, third, fourth] => Ok([first, second, third, fourth]), + _ => Err(error(node, source, "array-call requires four inputs")), + } +} + +fn column<'batch>(values: Vec>) -> NodeValue<'batch> { + NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::f64(values))) +} + +fn unsupported(node: &ProgramNode, source: Option<&SourceLocation>) -> VectorCoreError { + VectorCoreError::UnsupportedOpcode { + opcode: node.op.clone(), + location: location(source), + } +} + +fn error( + node: &ProgramNode, + source: Option<&SourceLocation>, + message: impl Into, +) -> VectorCoreError { + VectorCoreError::Execution { + node: node.id.clone(), + message: format!("{} at {}", message.into(), location(source)), + } +} + +fn contextual( + node: &ProgramNode, + source: Option<&SourceLocation>, + failure: &VectorCoreError, +) -> VectorCoreError { + error(node, source, failure.to_string()) +} + +fn location(source: Option<&SourceLocation>) -> String { + source.map_or_else( + || "strategy.py:?:?".to_owned(), + |source| format!("{}:{}:{}", source.path, source.line, source.column), + ) +} + +fn numpy_nan_to_num(value: f64) -> f64 { + if value.is_nan() { + 0.0 + } else if value == f64::INFINITY { + f64::MAX + } else if value == f64::NEG_INFINITY { + f64::MIN + } else { + value + } +} + +#[allow(clippy::cast_precision_loss)] +fn i64_as_f64(value: i64) -> f64 { + value as f64 +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::program::Lookback; + + fn node(name: &str, value_type: &str, inputs: &[&str], arguments: &Value) -> ProgramNode { + ProgramNode { + id: format!("array-{name}"), + function: "f1".to_owned(), + source_order: 1, + op: "array-call".to_owned(), + value_type: value_type.to_owned(), + inputs: inputs.iter().map(ToString::to_string).collect(), + parameters: json!({"family":"numpy","name":name,"arguments":arguments}) + .as_object() + .expect("parameters") + .clone(), + lookback: Lookback { + kind: "finite".to_owned(), + candles: Some(0), + expression: None, + causal: true, + }, + } + } + + fn native_node(name: &str, inputs: &[&str], arguments: &Value) -> ProgramNode { + let mut node = node(name, "f64-column", inputs, arguments); + node.parameters.insert("family".to_owned(), json!("native")); + node + } + + fn source() -> SourceLocation { + SourceLocation { + path: "strategy.py".to_owned(), + line: 42, + column: 8, + end_line: 42, + end_column: 20, + } + } + + fn owned_f64(values: Vec>) -> NodeValue<'static> { + NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::f64(values))) + } + + fn output(value: &NodeValue<'_>) -> Vec> { + let NodeValue::Column(column) = value else { + panic!("expected column") + }; + (0..match column { + RuntimeColumn::Borrowed(column) => column.len(), + RuntimeColumn::Owned(column) => column.len(), + }) + .map(|row| column.f64_at(row)) + .collect() + } + + #[test] + fn numpy_extremes_preserve_null_nan_and_right_signed_zero() { + let values = BTreeMap::from([ + ( + "left".to_owned(), + owned_f64(vec![Some(0.0), Some(-0.0), Some(f64::NAN), None]), + ), + ( + "right".to_owned(), + owned_f64(vec![Some(-0.0), Some(0.0), Some(1.0), Some(2.0)]), + ), + ]); + for name in ["maximum", "minimum"] { + let actual = execute_array_call( + &node(name, "f64-column", &["left", "right"], &json!({})), + &values, + 4, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid extreme"); + let actual = output(&actual); + assert_eq!(actual[0].expect("zero").to_bits(), (-0.0_f64).to_bits()); + assert_eq!(actual[1].expect("zero").to_bits(), 0.0_f64.to_bits()); + assert!(actual[2].expect("NaN is not null").is_nan()); + assert_eq!(actual[3], None); + } + } + + #[test] + fn numpy_nan_to_num_and_divide_keep_null_separate() { + let values = BTreeMap::from([ + ( + "input".to_owned(), + owned_f64(vec![None, Some(f64::NAN), Some(f64::INFINITY)]), + ), + ( + "denominator".to_owned(), + owned_f64(vec![Some(2.0), Some(0.0), None]), + ), + ( + "out".to_owned(), + owned_f64(vec![Some(-1.0), Some(-2.0), None]), + ), + ( + "where".to_owned(), + NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::boolean(vec![ + Some(false), + Some(true), + None, + ]))), + ), + ]); + let clean = execute_array_call( + &node("nan_to_num", "f64-column", &["input"], &json!({})), + &values, + 3, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid nan_to_num"); + assert_eq!(output(&clean), vec![None, Some(0.0), Some(f64::MAX)]); + + let divided = execute_array_call( + &node( + "divide", + "f64-column", + &["input", "denominator", "out", "where"], + &json!({}), + ), + &values, + 3, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid divide"); + let divided = output(÷d); + assert_eq!(divided[0], Some(-1.0)); + assert!(divided[1].expect("NaN is not null").is_nan()); + assert_eq!(divided[2], None); + } + + #[test] + fn numpy_unary_fill_zero_and_absolute_difference_cover_latest_surface() { + let values = BTreeMap::from([ + ( + "input".to_owned(), + owned_f64(vec![Some(-4.0), None, Some(-0.0)]), + ), + ("fill".to_owned(), NodeValue::Float(f64::NAN)), + ("scalar".to_owned(), NodeValue::Float(-1.0)), + ]); + let absolute = execute_array_call( + &node("abs", "f64-column", &["input"], &json!({})), + &values, + 3, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid abs"); + let absolute = output(&absolute); + assert_eq!(absolute[0], Some(4.0)); + assert_eq!(absolute[1], None); + assert_eq!(absolute[2].expect("zero").to_bits(), 0.0_f64.to_bits()); + + let filled = execute_array_call( + &node("full_like", "f64-column", &["input", "fill"], &json!({})), + &values, + 3, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid full_like"); + assert!(output(&filled) + .iter() + .all(|value| value.is_some_and(f64::is_nan))); + + let zeroed = execute_array_call( + &node("zeros_like", "f64-column", &["input"], &json!({})), + &values, + 3, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid zeros_like"); + assert_eq!(output(&zeroed), vec![Some(0.0); 3]); + + let missing = BTreeMap::from([ + ( + "input".to_owned(), + owned_f64(vec![None, Some(f64::NAN), Some(f64::INFINITY), Some(-0.0)]), + ), + ("fill".to_owned(), NodeValue::Float(50.0)), + ]); + let filled_missing = execute_array_call( + &node("fill-missing", "f64-column", &["input", "fill"], &json!({})), + &missing, + 4, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid fill-missing"); + let filled_missing = output(&filled_missing); + assert_eq!( + filled_missing[..3], + [Some(50.0), Some(50.0), Some(f64::INFINITY)] + ); + assert_eq!( + filled_missing[3].expect("negative zero").to_bits(), + (-0.0_f64).to_bits() + ); + + let square_root = execute_array_call( + &node("sqrt", "dynamic", &["scalar"], &json!({})), + &values, + 3, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid scalar sqrt"); + assert!(matches!(square_root, NodeValue::Float(value) if value.is_nan())); + + let difference_node = node("absolute-difference", "f64-column", &["diff"], &json!({})); + let mut state = ArrayCallState::default(); + let first = BTreeMap::from([("diff".to_owned(), owned_f64(vec![Some(1.0), Some(4.0)]))]); + let first = execute_array_call(&difference_node, &first, 2, &mut state, Some(&source())) + .expect("first difference chunk"); + let first = output(&first); + assert!(first[0].is_some_and(f64::is_nan)); + assert_eq!(first[1], Some(3.0)); + let second = BTreeMap::from([("diff".to_owned(), owned_f64(vec![Some(-2.0)]))]); + let second = execute_array_call(&difference_node, &second, 1, &mut state, Some(&source())) + .expect("second difference chunk"); + assert_eq!(output(&second), vec![Some(6.0)]); + } + + #[test] + fn stateful_native_calls_cross_batch_boundaries() { + const HOUR_MS: i64 = 3_600_000; + let mut state = ArrayCallState::default(); + let opening = native_node( + "opening-range-high", + &["date", "high", "low"], + &json!({"cutoff_hour":2}), + ); + let day = 1_700_000_000_000_i64.div_euclid(86_400_000) * 86_400_000; + let first = BTreeMap::from([ + ( + "date".to_owned(), + NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::timestamp_ms(vec![ + Some(day), + Some(day + HOUR_MS), + ]))), + ), + ("high".to_owned(), owned_f64(vec![Some(10.0), Some(12.0)])), + ("low".to_owned(), owned_f64(vec![Some(5.0), Some(4.0)])), + ]); + let first = execute_array_call(&opening, &first, 2, &mut state, Some(&source())) + .expect("first opening chunk"); + assert!(output(&first) + .iter() + .all(|value| value.is_some_and(f64::is_nan))); + + let second = BTreeMap::from([ + ( + "date".to_owned(), + NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::timestamp_ms(vec![Some( + day + 2 * HOUR_MS, + )]))), + ), + ("high".to_owned(), owned_f64(vec![Some(99.0)])), + ("low".to_owned(), owned_f64(vec![Some(-1.0)])), + ]); + let second = execute_array_call(&opening, &second, 1, &mut state, Some(&source())) + .expect("second opening chunk"); + assert_eq!(output(&second), vec![Some(12.0)]); + assert_eq!(state.retained(), 1); + } + + #[test] + fn inside_bar_dispatch_selects_each_native_projection() { + const HOUR_MS: i64 = 3_600_000; + let timestamps = [0, HOUR_MS, 2 * HOUR_MS]; + let values = BTreeMap::from([ + ( + "date".to_owned(), + NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::timestamp_ms( + timestamps.into_iter().map(Some).collect(), + ))), + ), + ( + "high".to_owned(), + owned_f64(vec![Some(10.0), Some(9.0), Some(99.0)]), + ), + ( + "low".to_owned(), + owned_f64(vec![Some(0.0), Some(1.0), Some(-99.0)]), + ), + ]); + for (name, last) in [ + ("inside-bar-ready", 1.0), + ("inside-bar-mother-high", 10.0), + ("inside-bar-mother-low", 0.0), + ] { + let actual = execute_array_call( + &native_node(name, &["date", "high", "low"], &json!({})), + &values, + 3, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect("valid inside-bar projection"); + assert_eq!(output(&actual)[2], Some(last)); + } + } + + #[test] + fn unsupported_contract_reports_source_location() { + let error = execute_array_call( + &node("imaginary", "f64-column", &["input"], &json!({})), + &BTreeMap::new(), + 0, + &mut ArrayCallState::default(), + Some(&source()), + ) + .expect_err("unsupported array call"); + assert_eq!( + error, + VectorCoreError::UnsupportedOpcode { + opcode: "array-call".to_owned(), + location: "strategy.py:42:8".to_owned(), + } + ); + } +} diff --git a/rust/crates/nfi-vector-core/src/engine/frame.rs b/rust/crates/nfi-vector-core/src/engine/frame.rs new file mode 100644 index 00000000..c59d6e53 --- /dev/null +++ b/rust/crates/nfi-vector-core/src/engine/frame.rs @@ -0,0 +1,1066 @@ +//! Identity-aware dataframe values used by informative indicator programs. +//! +//! The generic executor must not use the base Arrow batch length for an +//! informative frame. This module keeps every dataframe's identity, row +//! count, visible columns, and typed informative dates together. Catalog +//! sources remain borrowed; projections and drops are cheap visibility +//! overlays, while only an actual merge owns new column buffers. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, OnceLock}; + +use serde_json::{Map, Value}; + +use crate::alignment::{ + merge, FrameCatalog, FrameIdentity, MergeSpec, NumericFrame, SourceLocation, Timeframe, +}; +use crate::column::{OwnedColumn, ValueType}; +use crate::error::VectorCoreError; +use crate::program::{IndicatorProgram, ProgramNode}; + +const BASE_DATE_COLUMN: &str = "date"; + +#[derive(Clone, Debug)] +enum FrameStorage<'catalog> { + Borrowed(&'catalog NumericFrame), + Owned(Arc), +} + +impl FrameStorage<'_> { + fn frame(&self) -> &NumericFrame { + match self { + Self::Borrowed(frame) => frame, + Self::Owned(frame) => frame, + } + } +} + +/// One runtime dataframe and its cheap projection/drop overlay. +/// +/// `NumericFrame` stores its primary `date` column as `timestamps_ms`. +/// Informative date columns created by a merge remain typed timestamps in the +/// separate map below, instead of being silently coerced to numeric columns. +#[derive(Clone, Debug)] +pub(super) struct RuntimeFrame<'catalog> { + storage: FrameStorage<'catalog>, + source_columns: Arc>>, + timestamp_columns: Arc>>>, + overlays: Arc>, + visible_columns: BTreeSet, +} + +impl<'catalog> RuntimeFrame<'catalog> { + /// Borrow a validated source without copying candle buffers. + #[must_use] + pub(super) fn borrowed(frame: &'catalog NumericFrame) -> Self { + let mut visible_columns = frame.columns.keys().cloned().collect::>(); + visible_columns.insert(BASE_DATE_COLUMN.to_owned()); + Self { + storage: FrameStorage::Borrowed(frame), + source_columns: source_column_cache(frame), + timestamp_columns: Arc::new(BTreeMap::new()), + overlays: Arc::new(BTreeMap::new()), + visible_columns, + } + } + + fn owned(frame: NumericFrame, timestamp_columns: BTreeMap>>) -> Self { + let mut visible_columns = frame.columns.keys().cloned().collect::>(); + visible_columns.extend(timestamp_columns.keys().cloned()); + visible_columns.insert(BASE_DATE_COLUMN.to_owned()); + Self { + source_columns: source_column_cache(&frame), + storage: FrameStorage::Owned(Arc::new(frame)), + timestamp_columns: Arc::new(timestamp_columns), + overlays: Arc::new(BTreeMap::new()), + visible_columns, + } + } + + #[must_use] + pub(super) fn identity(&self) -> &FrameIdentity { + &self.storage.frame().identity + } + + #[must_use] + pub(super) fn len(&self) -> usize { + self.storage.frame().timestamps_ms.len() + } + + #[must_use] + pub(super) fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[must_use] + pub(super) fn has_column(&self, name: &str) -> bool { + self.visible_columns.contains(name) + } + + pub(super) fn column_names(&self) -> impl Iterator { + self.visible_columns.iter().map(String::as_str) + } + + #[must_use] + pub(super) fn column_type(&self, name: &str) -> Option { + if !self.has_column(name) { + return None; + } + if let Some(column) = self.overlays.get(name) { + return Some(column.as_view().value_type()); + } + if name == BASE_DATE_COLUMN || self.timestamp_columns.contains_key(name) { + Some(ValueType::TimestampMs) + } else if self.storage.frame().columns.contains_key(name) { + Some(ValueType::F64) + } else { + None + } + } + + /// Materialize one visible frame column for the existing `NodeValue` + /// column representation. The runtime can call this only when a compiled + /// column-read reaches the corresponding dataframe handle. + #[must_use] + pub(super) fn owned_column(&self, name: &str) -> Option { + if !self.has_column(name) { + return None; + } + if let Some(column) = self.overlays.get(name) { + return Some(column.clone()); + } + if name == BASE_DATE_COLUMN { + return Some(OwnedColumn::timestamp_ms( + self.storage + .frame() + .timestamps_ms + .iter() + .copied() + .map(Some) + .collect(), + )); + } + if let Some(values) = self.timestamp_columns.get(name) { + return Some(OwnedColumn::timestamp_ms(values.clone())); + } + let values = self.storage.frame().columns.get(name)?; + self.source_columns.get(name).map(|cached| { + cached + .get_or_init(|| OwnedColumn::f64(values.clone())) + .clone() + }) + } + + /// Add or replace one typed dataframe column without copying the source + /// frame. The overlay owns the Arrow buffer and retains its physical type. + /// + /// # Errors + /// + /// Returns a source-located error for an empty name, a row-count mismatch, + /// or a collision when the compiled write requested rejection. + pub(super) fn with_column( + mut self, + name: impl Into, + column: OwnedColumn, + collision_reject: bool, + source: &SourceLocation, + ) -> Result { + let name = name.into(); + if name.is_empty() { + return Err(source.error("dataframe overlay column name is empty")); + } + if column.len() != self.len() { + return Err(source.error(format!( + "dataframe overlay column {name:?} has {} rows; expected {}", + column.len(), + self.len() + ))); + } + if collision_reject && self.has_column(&name) { + return Err(source.error(format!( + "dataframe overlay column {name:?} collides with an existing column" + ))); + } + Arc::make_mut(&mut self.overlays).insert(name.clone(), column); + self.visible_columns.insert(name); + Ok(self) + } + + fn require_non_empty(&self, source: &SourceLocation) -> Result { + if self.is_empty() { + return Err(source.error(format!( + "frame {} {} is empty", + self.identity().pair, + self.identity().timeframe.as_str() + ))); + } + Ok(self.clone()) + } + + fn project( + &self, + always_keep: &BTreeSet, + drop_candidates: &BTreeSet, + keep: &BTreeSet, + ) -> Self { + let mut projected = self.clone(); + projected.visible_columns.retain(|column| { + always_keep.contains(column) + || !drop_candidates.contains(column) + || keep.contains(column) + }); + projected + } + + fn drop_if_present(&self, column: &str) -> Self { + let mut dropped = self.clone(); + dropped.visible_columns.remove(column); + dropped + } + + fn numeric_for_merge( + &self, + date_column: &str, + source: &SourceLocation, + ) -> Result<(NumericFrame, BTreeSet), VectorCoreError> { + if !self.has_column(date_column) { + return Err(source.error(format!( + "frame {} {} has no visible join column {date_column:?}", + self.identity().pair, + self.identity().timeframe.as_str() + ))); + } + let timestamps_ms = self.required_timestamp_column(date_column, source)?; + let mut columns = self + .storage + .frame() + .columns + .iter() + .filter(|(name, _)| self.has_column(name) && !self.overlays.contains_key(name.as_str())) + .map(|(name, values)| (name.clone(), values.clone())) + .collect::>(); + let mut timestamp_names = BTreeSet::new(); + for (name, values) in self.timestamp_columns.iter().filter(|(name, _)| { + self.has_column(name) + && name.as_str() != date_column + && !self.overlays.contains_key(name.as_str()) + }) { + if columns.contains_key(name) { + return Err(source.error(format!( + "frame column {name:?} has conflicting numeric and timestamp types" + ))); + } + let encoded = values + .iter() + .map(|value| { + value + .map(|value| exact_timestamp_as_f64(value, name, source)) + .transpose() + }) + .collect::, _>>()?; + columns.insert(name.clone(), encoded); + timestamp_names.insert(name.clone()); + } + for (name, column) in self + .overlays + .iter() + .filter(|(name, _)| self.has_column(name) && name.as_str() != date_column) + { + match column.as_view().value_type() { + ValueType::F64 => { + columns.insert( + name.clone(), + (0..self.len()) + .map(|row| column.as_view().f64_at(row)) + .collect(), + ); + } + ValueType::TimestampMs => { + let encoded = (0..self.len()) + .map(|row| { + column + .as_view() + .timestamp_ms_at(row) + .map(|value| exact_timestamp_as_f64(value, name, source)) + .transpose() + }) + .collect::, _>>()?; + columns.insert(name.clone(), encoded); + timestamp_names.insert(name.clone()); + } + value_type => { + return Err(source.error(format!( + "visible dataframe overlay {name:?} has non-mergeable type {}", + value_type.label() + ))); + } + } + } + Ok(( + NumericFrame { + identity: self.identity().clone(), + timestamps_ms, + columns, + }, + timestamp_names, + )) + } + + fn required_timestamp_column( + &self, + name: &str, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + if let Some(column) = self.overlays.get(name) { + if column.as_view().value_type() != ValueType::TimestampMs { + return Err(source.error(format!( + "frame join column {name:?} has type {}; expected timestamp", + column.as_view().value_type().label() + ))); + } + return (0..self.len()) + .map(|row| { + column.as_view().timestamp_ms_at(row).ok_or_else(|| { + source.error(format!("frame join column {name:?} is null at row {row}")) + }) + }) + .collect(); + } + if name == BASE_DATE_COLUMN { + return Ok(self.storage.frame().timestamps_ms.clone()); + } + self.timestamp_columns + .get(name) + .ok_or_else(|| { + source.error(format!( + "frame join column {name:?} is not a timestamp column" + )) + })? + .iter() + .enumerate() + .map(|(row, value)| { + value.ok_or_else(|| { + source.error(format!("frame join column {name:?} is null at row {row}")) + }) + }) + .collect() + } +} + +fn source_column_cache(frame: &NumericFrame) -> Arc>> { + Arc::new( + frame + .columns + .keys() + .map(|name| (name.clone(), OnceLock::new())) + .collect(), + ) +} + +/// Immutable external context for frame and metadata opcodes. +#[derive(Debug)] +pub(super) struct FrameRuntime<'catalog> { + catalog: &'catalog FrameCatalog, + metadata: &'catalog BTreeMap, +} + +impl<'catalog> FrameRuntime<'catalog> { + #[must_use] + pub(super) const fn new( + catalog: &'catalog FrameCatalog, + metadata: &'catalog BTreeMap, + ) -> Self { + Self { catalog, metadata } + } + + pub(super) fn metadata_read( + &self, + node: &ProgramNode, + source: &SourceLocation, + ) -> Result { + let key = string_parameter(node, "key", source)?; + self.metadata.get(key).cloned().ok_or_else(|| { + source.error(format!( + "runtime metadata has no string value for key {key:?}" + )) + }) + } + + pub(super) fn frame_source( + &self, + node: &ProgramNode, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let timeframe_value = string_parameter(node, "timeframe", source)?; + let timeframe = Timeframe::parse(timeframe_value.to_owned()) + .map_err(|error| source.error(format!("invalid frame-source timeframe: {error}")))?; + let selector = object_parameter(node, "pair", source)?; + let kind = object_string(selector, "kind", node, source)?; + let pair = match kind { + "literal" => object_string(selector, "value", node, source)?.to_owned(), + "metadata" => { + let key = object_string(selector, "key", node, source)?; + self.metadata.get(key).cloned().ok_or_else(|| { + source.error(format!( + "runtime metadata has no string value for frame pair key {key:?}" + )) + })? + } + other => { + return Err(source.error(format!( + "frame-source pair selector kind {other:?} is unsupported" + ))); + } + }; + let identity = FrameIdentity::new(pair, timeframe) + .map_err(|error| source.error(format!("invalid frame-source identity: {error}")))?; + Ok(RuntimeFrame::borrowed( + self.catalog.lookup(&identity, source)?, + )) + } + + pub(super) fn frame_nonempty( + frame: &RuntimeFrame<'catalog>, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + frame.require_non_empty(source) + } + + pub(super) fn frame_project( + node: &ProgramNode, + frame: &RuntimeFrame<'catalog>, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let always_keep = string_set_parameter(node, "always_keep", source)?; + let drop_candidates = string_set_parameter(node, "drop_candidates", source)?; + let keep = string_set_parameter(node, "keep", source)?; + Ok(frame.project(&always_keep, &drop_candidates, &keep)) + } + + pub(super) fn frame_drop_if_present( + node: &ProgramNode, + frame: &RuntimeFrame<'catalog>, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let column = string_parameter(node, "column", source)?; + Ok(frame.drop_if_present(column)) + } + + pub(super) fn informative_merge( + node: &ProgramNode, + base: &RuntimeFrame<'catalog>, + informative: &RuntimeFrame<'catalog>, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let base_timeframe = parse_timeframe_parameter(node, "base_timeframe", source)?; + let informative_timeframe = + parse_timeframe_parameter(node, "informative_timeframe", source)?; + let date_column = string_parameter(node, "date_column", source)?; + let ffill = bool_parameter(node, "ffill", source)?; + let append_timeframe = bool_parameter(node, "append_timeframe", source)?; + let suffix = optional_string_parameter(node, "suffix", source)?; + let expected_base = FrameIdentity::new(base.identity().pair.clone(), base_timeframe) + .map_err(|error| source.error(format!("invalid base merge identity: {error}")))?; + let expected_informative = + FrameIdentity::new(informative.identity().pair.clone(), informative_timeframe) + .map_err(|error| { + source.error(format!("invalid informative merge identity: {error}")) + })?; + let spec = MergeSpec { + base: expected_base, + informative: expected_informative, + ffill, + append_timeframe, + suffix, + date_column: date_column.to_owned(), + source: source.clone(), + }; + let (base_numeric, base_timestamps) = base.numeric_for_merge(BASE_DATE_COLUMN, source)?; + let (informative_numeric, informative_timestamps) = + informative.numeric_for_merge(date_column, source)?; + let informative_suffix = output_suffix(&spec, source)?; + let informative_timestamp_outputs = informative_timestamps + .into_iter() + .map(|name| (format!("{name}_{informative_suffix}"), name)) + .collect::>(); + let merged = merge(&base_numeric, &informative_numeric, &spec)?; + let mut numeric_columns = BTreeMap::new(); + let mut timestamp_columns = merged.informative_dates_ms; + for (name, values) in merged.columns { + if base_timestamps.contains(&name) || informative_timestamp_outputs.contains_key(&name) + { + let decoded = values + .into_iter() + .map(|value| { + value + .map(|value| exact_f64_as_timestamp(value, &name, source)) + .transpose() + }) + .collect::, _>>()?; + timestamp_columns.insert(name, decoded); + } else { + numeric_columns.insert(name, values); + } + } + Ok(RuntimeFrame::owned( + NumericFrame { + identity: merged.identity, + timestamps_ms: merged.timestamps_ms, + columns: numeric_columns, + }, + timestamp_columns, + )) + } +} + +/// Convert the program source map into the alignment layer's located error +/// contract. A validated program has one entry for every node; missing entries +/// still fail closed instead of fabricating a strategy location. +pub(super) fn node_source( + program: &IndicatorProgram, + node: &ProgramNode, +) -> Result { + let location = program.source_map.get(&node.id).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!("node {} has no source-map entry", node.id)) + })?; + Ok(SourceLocation::new( + &node.id, + &location.path, + location.line, + location.column, + )) +} + +fn output_suffix(spec: &MergeSpec, source: &SourceLocation) -> Result { + if spec.append_timeframe { + if spec + .suffix + .as_ref() + .is_some_and(|suffix| !suffix.is_empty()) + { + return Err(source.error("suffix cannot be combined with append_timeframe")); + } + Ok(spec.informative.timeframe.as_str().to_owned()) + } else { + spec.suffix + .as_ref() + .filter(|suffix| !suffix.is_empty()) + .cloned() + .ok_or_else(|| source.error("informative merge requires a non-empty suffix")) + } +} + +fn exact_timestamp_as_f64( + value: i64, + column: &str, + source: &SourceLocation, +) -> Result { + const MAX_EXACT_F64_INTEGER: i64 = 1_i64 << f64::MANTISSA_DIGITS; + if !(-MAX_EXACT_F64_INTEGER..=MAX_EXACT_F64_INTEGER).contains(&value) { + return Err(source.error(format!( + "timestamp column {column:?} has a value that cannot be represented exactly" + ))); + } + #[allow(clippy::cast_precision_loss)] + Ok(value as f64) +} + +fn exact_f64_as_timestamp( + value: f64, + column: &str, + source: &SourceLocation, +) -> Result { + const MAX_EXACT_F64_INTEGER: f64 = 9_007_199_254_740_992.0; + if !value.is_finite() + || value.fract() != 0.0 + || !(-MAX_EXACT_F64_INTEGER..=MAX_EXACT_F64_INTEGER).contains(&value) + { + return Err(source.error(format!( + "timestamp column {column:?} was not preserved as an integer" + ))); + } + #[allow(clippy::cast_possible_truncation)] + let decoded = value as i64; + Ok(decoded) +} + +fn parse_timeframe_parameter( + node: &ProgramNode, + name: &str, + source: &SourceLocation, +) -> Result { + let value = string_parameter(node, name, source)?; + Timeframe::parse(value.to_owned()) + .map_err(|error| source.error(format!("invalid {name}: {error}"))) +} + +fn string_parameter<'node>( + node: &'node ProgramNode, + name: &str, + source: &SourceLocation, +) -> Result<&'node str, VectorCoreError> { + node.parameters + .get(name) + .and_then(Value::as_str) + .ok_or_else(|| { + source.error(format!( + "node {} requires string parameter {name:?}", + node.id + )) + }) +} + +fn optional_string_parameter( + node: &ProgramNode, + name: &str, + source: &SourceLocation, +) -> Result, VectorCoreError> { + match node.parameters.get(name) { + Some(Value::Null) | None => Ok(None), + Some(Value::String(value)) => Ok(Some(value.clone())), + _ => Err(source.error(format!( + "node {} requires nullable string parameter {name:?}", + node.id + ))), + } +} + +fn bool_parameter( + node: &ProgramNode, + name: &str, + source: &SourceLocation, +) -> Result { + node.parameters + .get(name) + .and_then(Value::as_bool) + .ok_or_else(|| source.error(format!("node {} requires bool parameter {name:?}", node.id))) +} + +fn object_parameter<'node>( + node: &'node ProgramNode, + name: &str, + source: &SourceLocation, +) -> Result<&'node Map, VectorCoreError> { + node.parameters + .get(name) + .and_then(Value::as_object) + .ok_or_else(|| { + source.error(format!( + "node {} requires object parameter {name:?}", + node.id + )) + }) +} + +fn object_string<'object>( + object: &'object Map, + name: &str, + node: &ProgramNode, + source: &SourceLocation, +) -> Result<&'object str, VectorCoreError> { + object.get(name).and_then(Value::as_str).ok_or_else(|| { + source.error(format!( + "node {} pair selector requires string field {name:?}", + node.id + )) + }) +} + +fn string_set_parameter( + node: &ProgramNode, + name: &str, + source: &SourceLocation, +) -> Result, VectorCoreError> { + let values = node + .parameters + .get(name) + .and_then(Value::as_array) + .ok_or_else(|| { + source.error(format!( + "node {} requires array parameter {name:?}", + node.id + )) + })?; + values + .iter() + .map(|value| { + value.as_str().map(str::to_owned).ok_or_else(|| { + source.error(format!( + "node {} parameter {name:?} must contain only strings", + node.id + )) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::program::Lookback; + + fn identity(pair: &str, timeframe: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse(timeframe).expect("timeframe")).expect("identity") + } + + fn frame( + pair: &str, + timeframe: &str, + timestamps_ms: Vec, + columns: impl IntoIterator>)>, + ) -> NumericFrame { + NumericFrame { + identity: identity(pair, timeframe), + timestamps_ms, + columns: columns + .into_iter() + .map(|(name, values)| (name.to_owned(), values)) + .collect(), + } + } + + fn node(op: &str, parameters: Map) -> ProgramNode { + ProgramNode { + id: "n7".to_owned(), + function: "f1".to_owned(), + source_order: 7, + op: op.to_owned(), + value_type: "dataframe".to_owned(), + inputs: Vec::new(), + parameters, + lookback: Lookback { + kind: "finite".to_owned(), + candles: Some(0), + expression: None, + causal: true, + }, + } + } + + fn source() -> SourceLocation { + SourceLocation::new("n7", "NostalgiaForInfinityX7.py", 4819, 15) + } + + #[test] + fn literal_and_metadata_sources_resolve_exact_identity_without_copy() { + let eth = frame("ETH/USDT", "1h", vec![0], [("close", vec![Some(7.0)])]); + let btc = frame( + "BTC/USDT", + "4h", + vec![0, 1], + [("close", vec![Some(1.0), Some(2.0)])], + ); + let catalog = FrameCatalog::new([(eth.identity.clone(), eth), (btc.identity.clone(), btc)]) + .expect("catalog"); + let metadata = BTreeMap::from([("pair".to_owned(), "ETH/USDT".to_owned())]); + let runtime = FrameRuntime::new(&catalog, &metadata); + let literal = node( + "frame-source", + serde_json::from_value(serde_json::json!({ + "pair": {"kind": "literal", "value": "BTC/USDT"}, + "timeframe": "4h" + })) + .expect("parameters"), + ); + let selected = runtime + .frame_source(&literal, &source()) + .expect("literal source"); + assert_eq!(selected.identity(), &identity("BTC/USDT", "4h")); + assert_eq!(selected.len(), 2); + + let dynamic = node( + "frame-source", + serde_json::from_value(serde_json::json!({ + "pair": {"kind": "metadata", "key": "pair"}, + "timeframe": "1h" + })) + .expect("parameters"), + ); + assert_eq!( + runtime + .frame_source(&dynamic, &source()) + .expect("metadata source") + .identity(), + &identity("ETH/USDT", "1h") + ); + } + + #[test] + fn metadata_and_empty_frame_errors_keep_strategy_location() { + let empty = frame("ETH/USDT", "1h", Vec::new(), []); + let catalog = FrameCatalog::new([(empty.identity.clone(), empty)]).expect("catalog"); + let metadata = BTreeMap::new(); + let runtime = FrameRuntime::new(&catalog, &metadata); + let metadata_node = node( + "metadata-read", + serde_json::from_value(serde_json::json!({"key": "pair"})).expect("parameters"), + ); + let error = runtime + .metadata_read(&metadata_node, &source()) + .expect_err("missing metadata"); + assert!(matches!( + error, + VectorCoreError::Execution { node, message } + if node == "n7" + && message.starts_with("NostalgiaForInfinityX7.py:4819:15:") + )); + + let selected = RuntimeFrame::borrowed( + catalog + .lookup(&identity("ETH/USDT", "1h"), &source()) + .expect("empty source"), + ); + let error = FrameRuntime::frame_nonempty(&selected, &source()).expect_err("empty frame"); + assert!(matches!( + error, + VectorCoreError::Execution { node, message } + if node == "n7" + && message.contains("NostalgiaForInfinityX7.py:4819:15:") + && message.contains("ETH/USDT 1h is empty") + )); + } + + #[test] + fn projection_and_drop_are_frame_local_visibility_overlays() { + let input = frame( + "ETH/USDT", + "1h", + vec![0, 1, 2], + [ + ("open", vec![Some(1.0); 3]), + ("close", vec![Some(2.0); 3]), + ("RSI_14", vec![Some(3.0); 3]), + ], + ); + let catalog = FrameCatalog::new([(input.identity.clone(), input)]).expect("catalog"); + let original = RuntimeFrame::borrowed( + catalog + .lookup(&identity("ETH/USDT", "1h"), &source()) + .expect("source"), + ); + let projection = node( + "frame-project", + serde_json::from_value(serde_json::json!({ + "always_keep": ["date"], + "drop_candidates": ["open", "close", "volume"], + "keep": ["close"] + })) + .expect("parameters"), + ); + let projected = + FrameRuntime::frame_project(&projection, &original, &source()).expect("projection"); + assert_eq!(projected.len(), 3); + assert_eq!( + projected.column_names().collect::>(), + ["RSI_14", "close", "date"] + ); + assert!(original.has_column("open")); + + let drop_node = node( + "frame-drop-if-present", + serde_json::from_value(serde_json::json!({"column": "close"})).expect("parameters"), + ); + let dropped = + FrameRuntime::frame_drop_if_present(&drop_node, &projected, &source()).expect("drop"); + assert!(!dropped.has_column("close")); + assert!(projected.has_column("close")); + } + + #[test] + fn exact_non_ffill_merge_uses_each_frames_own_row_count_and_keeps_dates_typed() { + let base = frame( + "ETH/USDT", + "5m", + vec![0, 3_300_000, 3_600_000], + [("close", vec![Some(10.0), Some(11.0), Some(12.0)])], + ); + let informative = frame( + "BTC/USDT", + "1h", + vec![0, 3_600_000], + [("RSI_14", vec![Some(40.0), Some(50.0)])], + ); + let catalog = FrameCatalog::new([ + (base.identity.clone(), base), + (informative.identity.clone(), informative), + ]) + .expect("catalog"); + let base = RuntimeFrame::borrowed( + catalog + .lookup(&identity("ETH/USDT", "5m"), &source()) + .expect("base"), + ); + let informative = RuntimeFrame::borrowed( + catalog + .lookup(&identity("BTC/USDT", "1h"), &source()) + .expect("informative"), + ); + let merge_node = node( + "informative-merge", + serde_json::from_value(serde_json::json!({ + "base_timeframe": "5m", + "informative_timeframe": "1h", + "ffill": false, + "append_timeframe": true, + "date_column": "date", + "suffix": null + })) + .expect("parameters"), + ); + let merged = FrameRuntime::informative_merge(&merge_node, &base, &informative, &source()) + .expect("merge"); + assert_eq!(merged.len(), 3); + assert_eq!(merged.owned_column("RSI_14_1h").expect("numeric").len(), 3); + assert_eq!(merged.column_type("date_1h"), Some(ValueType::TimestampMs)); + let dates = merged.owned_column("date_1h").expect("date"); + assert_eq!(dates.as_view().timestamp_ms_at(0), None); + assert_eq!(dates.as_view().timestamp_ms_at(1), Some(0)); + assert_eq!(dates.as_view().timestamp_ms_at(2), None); + } + + #[test] + fn typed_column_overlays_are_cheap_visible_and_shape_checked() { + let input = frame( + "ETH/USDT", + "5m", + vec![0, 300_000], + [("close", vec![Some(10.0), Some(11.0)])], + ); + let catalog = FrameCatalog::new([(input.identity.clone(), input)]).expect("catalog"); + let original = RuntimeFrame::borrowed( + catalog + .lookup(&identity("ETH/USDT", "5m"), &source()) + .expect("source"), + ); + let with_numeric = original + .clone() + .with_column( + "RSI_14", + OwnedColumn::f64(vec![Some(40.0), Some(41.0)]), + true, + &source(), + ) + .expect("numeric overlay"); + let with_bool = with_numeric + .with_column( + "protection", + OwnedColumn::boolean(vec![Some(true), Some(false)]), + true, + &source(), + ) + .expect("bool overlay"); + let with_timestamp = with_bool + .with_column( + "observed_at", + OwnedColumn::timestamp_ms(vec![Some(0), Some(300_000)]), + true, + &source(), + ) + .expect("timestamp overlay"); + + assert!(!original.has_column("RSI_14")); + assert_eq!(with_timestamp.column_type("RSI_14"), Some(ValueType::F64)); + assert_eq!( + with_timestamp.column_type("protection"), + Some(ValueType::Bool) + ); + assert_eq!( + with_timestamp.column_type("observed_at"), + Some(ValueType::TimestampMs) + ); + assert_eq!( + with_timestamp + .owned_column("protection") + .expect("bool") + .as_view() + .bool_at(1), + Some(false) + ); + + let collision = with_timestamp + .clone() + .with_column( + "close", + OwnedColumn::f64(vec![Some(1.0), Some(2.0)]), + true, + &source(), + ) + .expect_err("collision"); + assert!(matches!( + collision, + VectorCoreError::Execution { message, .. } if message.contains("collides") + )); + let wrong_rows = with_timestamp + .with_column("bad", OwnedColumn::f64(vec![Some(1.0)]), true, &source()) + .expect_err("row mismatch"); + assert!(matches!( + wrong_rows, + VectorCoreError::Execution { message, .. } + if message.contains("has 1 rows; expected 2") + )); + } + + #[test] + fn merge_rejects_visible_bool_overlay_but_accepts_it_after_drop() { + let base = frame( + "ETH/USDT", + "5m", + vec![3_300_000], + [("close", vec![Some(10.0)])], + ); + let informative = frame("ETH/USDT", "1h", vec![0], [("RSI_14", vec![Some(40.0)])]); + let catalog = FrameCatalog::new([ + (base.identity.clone(), base), + (informative.identity.clone(), informative), + ]) + .expect("catalog"); + let base = RuntimeFrame::borrowed( + catalog + .lookup(&identity("ETH/USDT", "5m"), &source()) + .expect("base"), + ) + .with_column( + "protection", + OwnedColumn::boolean(vec![Some(true)]), + true, + &source(), + ) + .expect("bool overlay"); + let informative = RuntimeFrame::borrowed( + catalog + .lookup(&identity("ETH/USDT", "1h"), &source()) + .expect("informative"), + ); + let merge_node = node( + "informative-merge", + serde_json::from_value(serde_json::json!({ + "base_timeframe": "5m", + "informative_timeframe": "1h", + "ffill": false, + "append_timeframe": true, + "date_column": "date", + "suffix": null + })) + .expect("parameters"), + ); + let error = FrameRuntime::informative_merge(&merge_node, &base, &informative, &source()) + .expect_err("bool cannot cross numeric alignment"); + assert!(matches!( + error, + VectorCoreError::Execution { message, .. } + if message.contains("non-mergeable type Boolean") + )); + + let drop_node = node( + "frame-drop-if-present", + serde_json::from_value(serde_json::json!({"column": "protection"})) + .expect("parameters"), + ); + let dropped = + FrameRuntime::frame_drop_if_present(&drop_node, &base, &source()).expect("drop bool"); + assert!( + FrameRuntime::informative_merge(&merge_node, &dropped, &informative, &source(),) + .is_ok() + ); + } +} diff --git a/rust/crates/nfi-vector-core/src/engine/full.rs b/rust/crates/nfi-vector-core/src/engine/full.rs new file mode 100644 index 00000000..37686a2e --- /dev/null +++ b/rust/crates/nfi-vector-core/src/engine/full.rs @@ -0,0 +1,1545 @@ +//! Recursive, identity-aware execution of complete indicator programs. +//! +//! The streaming [`super::VectorEngine`] intentionally executes an output-specific +//! flat plan over one Arrow batch. NFI's complete indicator entrypoint is a +//! different contract: helpers receive dynamic dataframe/column arguments, read +//! informative frames with independent row counts, and return mutated frames to +//! their caller. This module executes that contract without Python while reusing +//! the same exact scalar, array, indicator, rolling, and alignment kernels. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Map, Value}; + +use super::array::{execute_array_call, ArrayCallState}; +use super::frame::{node_source, FrameRuntime, RuntimeFrame}; +use super::operations::{ + execute_binary, execute_compare, execute_logical, execute_select, execute_unary, literal_value, + resolve_value, single_input, string_parameter, to_owned_column, unsigned_parameter, +}; +use super::runtime::{NodeValue, RuntimeColumn}; +use crate::alignment::{FrameCatalog, FrameIdentity, NumericFrame, SourceLocation}; +use crate::column::{OwnedColumn, ValueType}; +use crate::error::VectorCoreError; +use crate::kernels::{ + rolling_stream, ChaikinMoneyFlowStream, LegacyChaikinMoneyFlowStream, RollingStream, + SafePercentChangeStream, TalibStream, +}; +use crate::program::{IndicatorProgram, ProgramFunction, ProgramNode}; +use crate::state::ShiftState; + +const MAX_CALL_DEPTH: usize = 64; + +type StatelessOperation = for<'batch> fn( + &ProgramNode, + &BTreeMap>, + usize, +) -> Result, VectorCoreError>; + +/// A typed dataframe returned by complete indicator-program execution. +#[derive(Clone, Debug)] +pub struct FullFrameOutput { + identity: FrameIdentity, + timestamps_ms: Vec, + columns: BTreeMap, +} + +impl FullFrameOutput { + /// Exact pair/timeframe identity of the returned dataframe. + #[must_use] + pub const fn identity(&self) -> &FrameIdentity { + &self.identity + } + + /// Ordered candle timestamps retained by the returned dataframe. + #[must_use] + pub fn timestamps_ms(&self) -> &[i64] { + &self.timestamps_ms + } + + /// Requested typed columns in deterministic name order. + #[must_use] + pub const fn columns(&self) -> &BTreeMap { + &self.columns + } + + /// Number of rows in every returned column. + #[must_use] + pub fn len(&self) -> usize { + self.timestamps_ms.len() + } + + /// Whether the returned dataframe has no rows. + #[must_use] + pub fn is_empty(&self) -> bool { + self.timestamps_ms.is_empty() + } +} + +/// Program-bound executor for a complete, recursively-called indicator entrypoint. +/// +/// Stateful kernels are keyed by the complete call-site path. Therefore two +/// invocations of the same helper over different dataframe identities cannot +/// share shift, rolling, TA, or native-kernel history accidentally. +#[derive(Debug)] +pub struct FullIndicatorEngine<'program> { + program: &'program IndicatorProgram, + functions: BTreeMap, + nodes: BTreeMap, + shift_states: BTreeMap, + indicator_states: BTreeMap, + rolling_states: BTreeMap, + array_states: BTreeMap, + chaikin_states: BTreeMap, + legacy_chaikin_states: BTreeMap, + percent_change_states: BTreeMap, +} + +impl<'program> FullIndicatorEngine<'program> { + /// Validate and bind one complete indicator program. + /// + /// # Errors + /// + /// Returns an invalid-program error when the serialized contract is not + /// canonical or its entrypoint cannot be resolved. + pub fn new(program: &'program IndicatorProgram) -> Result { + program.validate()?; + let functions = program + .functions + .iter() + .map(|function| (function.id.clone(), function)) + .collect::>(); + if !functions.contains_key(&program.entrypoint) { + return Err(VectorCoreError::InvalidProgram( + "complete indicator entrypoint is absent".to_owned(), + )); + } + let nodes = program + .nodes + .iter() + .map(|node| (node.id.clone(), node)) + .collect(); + Ok(Self { + program, + functions, + nodes, + shift_states: BTreeMap::new(), + indicator_states: BTreeMap::new(), + rolling_states: BTreeMap::new(), + array_states: BTreeMap::new(), + chaikin_states: BTreeMap::new(), + legacy_chaikin_states: BTreeMap::new(), + percent_change_states: BTreeMap::new(), + }) + } + + /// Execute the program entrypoint against one base frame and immutable + /// informative-frame catalog. + /// + /// `metadata` is intentionally string-only: it is the exact static compiler + /// contract used for pair routing. Only `requested_outputs` are materialized + /// in the public result, while every dataframe mutation still executes in + /// source order. + /// + /// # Errors + /// + /// Returns a source-located error for missing frames/columns, row-shape drift, + /// unsupported exact semantics, or a runtime type mismatch. + pub fn execute<'catalog>( + &mut self, + base: &'catalog NumericFrame, + catalog: &'catalog FrameCatalog, + metadata: &'catalog BTreeMap, + requested_outputs: &[String], + ) -> Result { + base.validate()?; + self.clear_execution_state(); + let mut seen = BTreeSet::new(); + for output in requested_outputs { + if output.is_empty() || !seen.insert(output.as_str()) { + return Err(VectorCoreError::InvalidOutput( + "requested complete-frame outputs must be non-empty and unique".to_owned(), + )); + } + } + let runtime = FrameRuntime::new(catalog, metadata); + let entrypoint = self.program.entrypoint.clone(); + let result = self.execute_function( + &runtime, + &entrypoint, + vec![ + BoundValue::Frame(RuntimeFrame::borrowed(base)), + BoundValue::Metadata, + ], + &entrypoint, + 0, + )?; + let BoundValue::Frame(frame) = result else { + return Err(VectorCoreError::InvalidOutput( + "complete indicator entrypoint did not return a dataframe".to_owned(), + )); + }; + materialize_output(&frame, requested_outputs) + } + + fn clear_execution_state(&mut self) { + self.shift_states.clear(); + self.indicator_states.clear(); + self.rolling_states.clear(); + self.array_states.clear(); + self.chaikin_states.clear(); + self.legacy_chaikin_states.clear(); + self.percent_change_states.clear(); + } + + fn execute_function<'catalog>( + &mut self, + runtime: &FrameRuntime<'catalog>, + function_id: &str, + arguments: Vec>, + call_path: &str, + depth: usize, + ) -> Result, VectorCoreError> { + if depth > MAX_CALL_DEPTH { + return Err(VectorCoreError::InvalidState(format!( + "indicator helper call depth exceeds {MAX_CALL_DEPTH} at {call_path}" + ))); + } + let function = self.function(function_id)?; + if arguments.len() != function.parameters.len() { + return Err(VectorCoreError::InvalidProgram(format!( + "function {} received {} arguments; expected {}", + function.id, + arguments.len(), + function.parameters.len() + ))); + } + let mut scope = FunctionScope::default(); + let mut remaining_uses = self.function_input_uses(function)?; + for (parameter, argument) in function.parameters.iter().zip(arguments) { + if !argument.matches_type(¶meter.value_type) { + return Err(VectorCoreError::InvalidProgram(format!( + "function {} parameter {} cannot bind runtime type {} as {}", + function.id, + parameter.name, + argument.type_name(), + parameter.value_type + ))); + } + scope.insert(parameter.node.clone(), argument); + } + + for node_id in &function.node_ids { + let node = self.node(node_id)?; + if node.op == "parameter" { + if !scope.contains(&node.id) { + return Err(VectorCoreError::InvalidProgram(format!( + "function {} has an unbound parameter node {}", + function.id, node.id + ))); + } + continue; + } + let value = self.execute_node(runtime, node, &scope, call_path, depth)?; + if !value.matches_type(&node.value_type) { + let source = node_source(self.program, node)?; + return Err(source.error(format!( + "opcode {} returned runtime type {}; expected {}", + node.op, + value.type_name(), + node.value_type + ))); + } + scope.insert(node.id.clone(), value); + scope.release_consumed_inputs( + &node.inputs, + &mut remaining_uses, + &function.return_node, + )?; + } + scope.bound(&function.return_node) + } + + fn execute_node<'catalog>( + &mut self, + runtime: &FrameRuntime<'catalog>, + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + call_path: &str, + depth: usize, + ) -> Result, VectorCoreError> { + let source = node_source(self.program, node)?; + if is_frame_opcode(&node.op) { + return execute_frame_opcode(runtime, node, scope, &source); + } + match node.op.as_str() { + "literal" => literal_value(node) + .map(BoundValue::Runtime) + .map_err(|error| located(&source, error)), + "function-call" => { + let callee = + string_parameter(node, "function").map_err(|error| located(&source, error))?; + let arguments = node + .inputs + .iter() + .map(|input| scope.bound(input)) + .collect::, _>>()?; + let nested_path = format!("{call_path}/{}:{callee}", node.id); + self.execute_function(runtime, callee, arguments, &nested_path, depth + 1) + } + "return" => scope.bound(single_input(node).map_err(|error| located(&source, error))?), + "cast" => execute_cast(node, scope, &source), + "shift" => self.execute_shift(node, scope, call_path, &source), + "binary" => Self::execute_stateless(node, scope, &source, execute_binary), + "compare" => Self::execute_stateless(node, scope, &source, execute_compare), + "logical" => Self::execute_stateless(node, scope, &source, execute_logical), + "unary" => Self::execute_stateless(node, scope, &source, execute_unary), + "select" => Self::execute_stateless(node, scope, &source, execute_select), + "indicator-call" => self.execute_indicator(node, scope, call_path, &source), + "window" => self.execute_window(node, scope, call_path, &source), + "array-call" => { + let rows = scope.rows_for(node, &source)?; + let state = self.array_states.entry(call_path.to_owned()).or_default(); + execute_array_call( + node, + &scope.values, + rows, + state, + self.program.source_map.get(&node.id), + ) + .map(BoundValue::Runtime) + } + "instrumentation" => execute_instrumentation(node, &source), + other => Err(VectorCoreError::UnsupportedOpcode { + opcode: other.to_owned(), + location: format!("{}:{}:{}", source.path, source.line, source.column), + }), + } + } + + fn execute_stateless<'catalog>( + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + source: &SourceLocation, + operation: StatelessOperation, + ) -> Result, VectorCoreError> { + let rows = scope.rows_for(node, source)?; + operation(node, &scope.values, rows) + .map(BoundValue::Runtime) + .map_err(|error| located(source, error)) + } + + fn execute_shift<'catalog>( + &mut self, + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + call_path: &str, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let input = single_input(node).map_err(|error| located(source, error))?; + let periods = + unsigned_parameter(node, "periods").map_err(|error| located(source, error))?; + if periods == 0 { + return Err(source.error("shift periods must be positive")); + } + let rows = scope.rows_for(node, source)?; + let input = scope.numeric_input(input, source)?; + let key = state_key(call_path, &node.id); + let state = self + .shift_states + .entry(key) + .or_insert(ShiftState::new(periods)?); + if state.lag() != periods { + return Err(source.error("shift state period changed between executions")); + } + let output = (0..rows) + .map(|row| input.at(row)) + .map(|value| { + let ready = state.len() == periods; + let shifted = state.push(value); + if ready { + shifted + } else { + Some(crate::float::canonicalize(f64::NAN)) + } + }) + .collect(); + Ok(BoundValue::Runtime(NodeValue::Column( + RuntimeColumn::Owned(OwnedColumn::f64(output)), + ))) + } + + fn execute_indicator<'catalog>( + &mut self, + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + call_path: &str, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let family = string_parameter(node, "family").map_err(|error| located(source, error))?; + let name = string_parameter(node, "name").map_err(|error| located(source, error))?; + let arguments = node + .parameters + .get("arguments") + .and_then(Value::as_object) + .ok_or_else(|| source.error("indicator-call arguments must be an object"))?; + let rows = scope.rows_for(node, source)?; + let inputs = scope.present_numeric_inputs(node, rows, source)?; + let slices = inputs + .iter() + .map(PresentNumeric::as_slice) + .collect::>(); + let key = state_key(call_path, &node.id); + let output = match (family, name) { + ("ta" | "talib", _) => { + let state = match self.indicator_states.entry(key) { + std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::btree_map::Entry::Vacant(entry) => entry.insert( + TalibStream::new(name, arguments) + .map_err(|error| located(source, error))?, + ), + }; + let output = state + .execute(&slices) + .map_err(|error| located(source, error))?; + select_indicator_output(node, &output, source)?.to_vec() + } + ("native", "safe-percent-change") if arguments.is_empty() && slices.len() == 1 => self + .percent_change_states + .entry(key) + .or_default() + .execute(slices[0]), + ("native", "chaikin-money-flow") if slices.len() == 4 => { + let period = bounded_argument(arguments, "timeperiod", source)?; + let state = match self.chaikin_states.entry(key) { + std::collections::btree_map::Entry::Occupied(entry) => { + if entry.get().0 != period { + return Err(source + .error("Chaikin money-flow period changed between executions")); + } + &mut entry.into_mut().1 + } + std::collections::btree_map::Entry::Vacant(entry) => { + &mut entry + .insert(( + period, + ChaikinMoneyFlowStream::new(period) + .map_err(|error| located(source, error))?, + )) + .1 + } + }; + state + .execute(slices[0], slices[1], slices[2], slices[3]) + .map_err(|error| located(source, error))? + } + ("native", "chaikin-money-flow-legacy") if slices.len() == 4 => { + let period = bounded_argument(arguments, "timeperiod", source)?; + let state = match self.legacy_chaikin_states.entry(key) { + std::collections::btree_map::Entry::Occupied(entry) => { + if entry.get().0 != period { + return Err(source.error( + "legacy Chaikin money-flow period changed between executions", + )); + } + &mut entry.into_mut().1 + } + std::collections::btree_map::Entry::Vacant(entry) => { + &mut entry + .insert(( + period, + LegacyChaikinMoneyFlowStream::new(period) + .map_err(|error| located(source, error))?, + )) + .1 + } + }; + state + .execute(slices[0], slices[1], slices[2], slices[3]) + .map_err(|error| located(source, error))? + } + _ => { + return Err(VectorCoreError::UnsupportedOpcode { + opcode: node.op.clone(), + location: format!("{}:{}:{}", source.path, source.line, source.column), + }); + } + }; + Ok(BoundValue::Runtime(NodeValue::Column( + RuntimeColumn::Owned(OwnedColumn::f64(output.into_iter().map(Some).collect())), + ))) + } + + fn execute_window<'catalog>( + &mut self, + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + call_path: &str, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + if node.parameters.get("kind").and_then(Value::as_str) != Some("rolling") { + return Err(VectorCoreError::UnsupportedOpcode { + opcode: node.op.clone(), + location: format!("{}:{}:{}", source.path, source.line, source.column), + }); + } + let reducer = string_parameter(node, "reducer").map_err(|error| located(source, error))?; + let input = single_input(node).map_err(|error| located(source, error))?; + let rows = scope.rows_for(node, source)?; + let values = scope.present_numeric_input( + input, + rows, + "rolling input contains an Arrow null", + source, + )?; + let key = state_key(call_path, &node.id); + let state = match self.rolling_states.entry(key) { + std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::btree_map::Entry::Vacant(entry) => entry.insert( + rolling_stream(reducer, &node.parameters) + .map_err(|error| located(source, error))?, + ), + }; + Ok(BoundValue::Runtime(NodeValue::Column( + RuntimeColumn::Owned(OwnedColumn::f64( + state + .execute(values.as_slice()) + .into_iter() + .map(Some) + .collect(), + )), + ))) + } + + fn function(&self, id: &str) -> Result<&'program ProgramFunction, VectorCoreError> { + self.functions.get(id).copied().ok_or_else(|| { + VectorCoreError::InvalidProgram(format!("unknown complete function {id}")) + }) + } + + fn function_input_uses( + &self, + function: &ProgramFunction, + ) -> Result, VectorCoreError> { + let mut uses = function + .node_ids + .iter() + .map(|id| (id.clone(), 0_usize)) + .collect::>(); + for node_id in &function.node_ids { + for input in &self.node(node_id)?.inputs { + let count = uses.get_mut(input).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!( + "function {} node {node_id} references non-local input {input}", + function.id + )) + })?; + *count = count.checked_add(1).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!( + "function {} input use count is too large", + function.id + )) + })?; + } + } + Ok(uses) + } + + fn node(&self, id: &str) -> Result<&'program ProgramNode, VectorCoreError> { + self.nodes + .get(id) + .copied() + .ok_or_else(|| VectorCoreError::InvalidProgram(format!("unknown complete node {id}"))) + } +} + +#[derive(Debug)] +enum BoundValue<'catalog> { + Frame(RuntimeFrame<'catalog>), + Metadata, + Runtime(NodeValue<'catalog>), +} + +impl BoundValue<'_> { + fn type_name(&self) -> &'static str { + match self { + Self::Frame(_) => "dataframe", + Self::Metadata => "metadata", + Self::Runtime(NodeValue::Null) => "null", + Self::Runtime(NodeValue::Bool(_)) => "bool-scalar", + Self::Runtime(NodeValue::Integer(_)) => "int-scalar", + Self::Runtime(NodeValue::Float(_)) => "f64-scalar", + Self::Runtime(NodeValue::Text(_)) => "string-scalar", + Self::Runtime(NodeValue::Column(column)) => match column.value_type() { + ValueType::F64 => "f64-column", + ValueType::I64 => "int-column", + ValueType::Bool => "bool-column", + ValueType::Text => "string-column", + ValueType::TimestampMs => "timestamp-column", + }, + Self::Runtime(NodeValue::Json) => "json", + Self::Runtime(NodeValue::DataFrame) => "dataframe-marker", + Self::Runtime(NodeValue::Metadata) => "metadata-marker", + Self::Runtime(NodeValue::Unbound) => "unbound", + Self::Runtime(NodeValue::Alias(_)) => "alias", + } + } + + fn matches_type(&self, expected: &str) -> bool { + expected == "dynamic" + && matches!( + self, + Self::Runtime( + NodeValue::Null + | NodeValue::Bool(_) + | NodeValue::Integer(_) + | NodeValue::Float(_) + | NodeValue::Text(_) + | NodeValue::Column(_) + | NodeValue::Json + ) + ) + || self.type_name() == expected + } +} + +#[derive(Debug, Default)] +struct FunctionScope<'catalog> { + values: BTreeMap>, + frames: BTreeMap>, + metadata: BTreeSet, +} + +impl<'catalog> FunctionScope<'catalog> { + fn insert(&mut self, id: String, value: BoundValue<'catalog>) { + match value { + BoundValue::Frame(frame) => { + self.frames.insert(id.clone(), frame); + self.values.insert(id, NodeValue::DataFrame); + } + BoundValue::Metadata => { + self.metadata.insert(id.clone()); + self.values.insert(id, NodeValue::Metadata); + } + BoundValue::Runtime(value) => { + self.values.insert(id, value); + } + } + } + + fn contains(&self, id: &str) -> bool { + self.values.contains_key(id) + } + + fn release_consumed_inputs( + &mut self, + inputs: &[String], + remaining_uses: &mut BTreeMap, + return_node: &str, + ) -> Result<(), VectorCoreError> { + for input in inputs { + let remaining = remaining_uses.get_mut(input).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!( + "function input {input} has no liveness record" + )) + })?; + *remaining = remaining.checked_sub(1).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!( + "function input {input} was consumed too many times" + )) + })?; + if *remaining == 0 && input != return_node { + self.values.remove(input); + self.frames.remove(input); + self.metadata.remove(input); + } + } + Ok(()) + } + + fn runtime(&self, id: &str) -> Result<&NodeValue<'catalog>, VectorCoreError> { + resolve_value(&self.values, id) + } + + fn frame(&self, id: &str) -> Result, VectorCoreError> { + let mut current = id; + for _ in 0..=self.values.len() { + if let Some(frame) = self.frames.get(current) { + return Ok(frame.clone()); + } + match self.values.get(current) { + Some(NodeValue::Alias(next)) => current = next, + Some(_) => break, + None => { + return Err(VectorCoreError::Execution { + node: id.to_owned(), + message: format!("input node {current} has no runtime value"), + }); + } + } + } + Err(VectorCoreError::Execution { + node: id.to_owned(), + message: "runtime value is not a dataframe".to_owned(), + }) + } + + fn require_metadata(&self, id: &str, source: &SourceLocation) -> Result<(), VectorCoreError> { + let mut current = id; + for _ in 0..=self.values.len() { + if self.metadata.contains(current) { + return Ok(()); + } + match self.values.get(current) { + Some(NodeValue::Alias(next)) => current = next, + Some(_) => break, + None => return Err(source.error(format!("metadata input {current} is absent"))), + } + } + Err(source.error("runtime value is not metadata")) + } + + fn bound(&self, id: &str) -> Result, VectorCoreError> { + if let Ok(frame) = self.frame(id) { + return Ok(BoundValue::Frame(frame)); + } + let mut current = id; + for _ in 0..=self.values.len() { + if self.metadata.contains(current) { + return Ok(BoundValue::Metadata); + } + match self.values.get(current) { + Some(NodeValue::Alias(next)) => current = next, + Some(value) => return clone_runtime(value).map(BoundValue::Runtime), + None => break, + } + } + Err(VectorCoreError::Execution { + node: id.to_owned(), + message: "runtime value cannot be bound".to_owned(), + }) + } + + fn single_frame( + &self, + node: &ProgramNode, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let input = node + .inputs + .first() + .filter(|_| node.inputs.len() == 1) + .ok_or_else(|| source.error(format!("{} requires one dataframe input", node.op)))?; + self.frame(input).map_err(|error| located(source, error)) + } + + fn two_frames( + &self, + node: &ProgramNode, + source: &SourceLocation, + ) -> Result<[RuntimeFrame<'catalog>; 2], VectorCoreError> { + let [left, right] = node.inputs.as_slice() else { + return Err(source.error(format!("{} requires two dataframe inputs", node.op))); + }; + Ok([ + self.frame(left).map_err(|error| located(source, error))?, + self.frame(right).map_err(|error| located(source, error))?, + ]) + } + + fn rows_for( + &self, + node: &ProgramNode, + source: &SourceLocation, + ) -> Result { + let mut rows = None; + for input in &node.inputs { + if let Ok(frame) = self.frame(input) { + merge_rows(&mut rows, frame.len(), node, source)?; + continue; + } + if let Ok(NodeValue::Column(column)) = self.runtime(input) { + merge_rows(&mut rows, runtime_column_len(column), node, source)?; + } + } + if node.value_type.ends_with("-column") { + rows.ok_or_else(|| source.error(format!("{} has no row-bearing input", node.op))) + } else { + Ok(rows.unwrap_or(1)) + } + } + + fn numeric_input<'scope>( + &'scope self, + input: &str, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + match self + .runtime(input) + .map_err(|error| located(source, error))? + { + NodeValue::Null => Ok(NumericInput::Null), + NodeValue::Integer(value) => { + #[allow(clippy::cast_precision_loss)] + let value = *value as f64; + Ok(NumericInput::Scalar(value)) + } + NodeValue::Float(value) => Ok(NumericInput::Scalar(*value)), + NodeValue::Column(column) if column.value_type() == ValueType::I64 => { + Ok(NumericInput::I64(column)) + } + NodeValue::Column(column) if column.value_type() == ValueType::F64 => { + Ok(NumericInput::F64(column)) + } + _ => Err(source.error(format!("node {input} is not numeric"))), + } + } + + fn present_numeric_input<'scope>( + &'scope self, + input: &str, + rows: usize, + null_error: &str, + source: &SourceLocation, + ) -> Result, VectorCoreError> { + let input = self.numeric_input(input, source)?; + if let NumericInput::F64(column) = input { + if let Some(values) = column.present_f64_slice() { + if values.len() == rows { + return Ok(PresentNumeric::Borrowed(values)); + } + } + } + let values = (0..rows) + .map(|row| input.at(row).ok_or_else(|| source.error(null_error))) + .collect::, _>>()?; + Ok(PresentNumeric::Owned(values)) + } + + fn present_numeric_inputs<'scope>( + &'scope self, + node: &ProgramNode, + rows: usize, + source: &SourceLocation, + ) -> Result>, VectorCoreError> { + node.inputs + .iter() + .map(|input| { + self.present_numeric_input( + input, + rows, + "indicator input contains an Arrow null", + source, + ) + }) + .collect() + } +} + +#[derive(Clone, Copy, Debug)] +enum NumericInput<'scope, 'catalog> { + Null, + Scalar(f64), + I64(&'scope RuntimeColumn<'catalog>), + F64(&'scope RuntimeColumn<'catalog>), +} + +impl NumericInput<'_, '_> { + fn at(self, row: usize) -> Option { + match self { + Self::Null => None, + Self::Scalar(value) => Some(value), + Self::I64(column) => + { + #[allow(clippy::cast_precision_loss)] + column.i64_at(row).map(|value| value as f64) + } + Self::F64(column) => column.f64_at(row), + } + } +} + +#[derive(Debug)] +enum PresentNumeric<'scope> { + Borrowed(&'scope [f64]), + Owned(Vec), +} + +impl PresentNumeric<'_> { + fn as_slice(&self) -> &[f64] { + match self { + Self::Borrowed(values) => values, + Self::Owned(values) => values, + } + } +} + +fn is_frame_opcode(opcode: &str) -> bool { + matches!( + opcode, + "metadata-read" + | "frame-source" + | "frame-nonempty" + | "frame-project" + | "frame-drop-if-present" + | "informative-merge" + | "fill" + | "column-read" + | "column-write" + | "row-index" + ) +} + +fn execute_frame_opcode<'catalog>( + runtime: &FrameRuntime<'catalog>, + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + source: &SourceLocation, +) -> Result, VectorCoreError> { + match node.op.as_str() { + "metadata-read" => { + let input = single_input(node).map_err(|error| located(source, error))?; + scope.require_metadata(input, source)?; + runtime + .metadata_read(node, source) + .map(|value| BoundValue::Runtime(NodeValue::Text(value))) + } + "frame-source" => runtime.frame_source(node, source).map(BoundValue::Frame), + "frame-nonempty" => { + let frame = scope.single_frame(node, source)?; + FrameRuntime::frame_nonempty(&frame, source).map(BoundValue::Frame) + } + "frame-project" => { + let frame = scope.single_frame(node, source)?; + FrameRuntime::frame_project(node, &frame, source).map(BoundValue::Frame) + } + "frame-drop-if-present" => { + let frame = scope.single_frame(node, source)?; + FrameRuntime::frame_drop_if_present(node, &frame, source).map(BoundValue::Frame) + } + "informative-merge" => { + let [base, informative] = scope.two_frames(node, source)?; + FrameRuntime::informative_merge(node, &base, &informative, source) + .map(BoundValue::Frame) + } + "fill" => { + let frame = scope.single_frame(node, source)?; + forward_fill_frame(frame, node, source).map(BoundValue::Frame) + } + "column-read" => execute_column_read(node, scope, source), + "column-write" => execute_column_write(node, scope, source), + "row-index" => { + let frame = scope.single_frame(node, source)?; + let values = (0..frame.len()) + .map(|row| { + i64::try_from(row) + .map(Some) + .map_err(|_| source.error("dataframe row index exceeds i64 range")) + }) + .collect::, _>>()?; + Ok(BoundValue::Runtime(NodeValue::Column( + RuntimeColumn::Owned(OwnedColumn::i64(values)), + ))) + } + _ => Err(VectorCoreError::InvalidState(format!( + "non-frame opcode {} reached the frame dispatcher", + node.op + ))), + } +} + +fn execute_column_read<'catalog>( + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + source: &SourceLocation, +) -> Result, VectorCoreError> { + let frame = scope.frame( + node.inputs + .first() + .ok_or_else(|| source.error("column-read requires a dataframe input"))?, + )?; + let column = string_parameter(node, "column").map_err(|error| located(source, error))?; + frame + .owned_column(column) + .map(|column| BoundValue::Runtime(NodeValue::Column(RuntimeColumn::Owned(column)))) + .ok_or_else(|| { + source.error(format!( + "frame {} {} has no visible column {column:?}", + frame.identity().pair, + frame.identity().timeframe.as_str() + )) + }) +} + +fn execute_column_write<'catalog>( + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + source: &SourceLocation, +) -> Result, VectorCoreError> { + let dataframe = node + .inputs + .first() + .ok_or_else(|| source.error("column-write requires a dataframe input"))?; + let value = node + .inputs + .get(1) + .ok_or_else(|| source.error("column-write requires a value input"))?; + let frame = scope.frame(dataframe)?; + let value = scope.runtime(value)?; + let column = to_owned_column(value, frame.len()).map_err(|error| located(source, error))?; + let name = string_parameter(node, "column").map_err(|error| located(source, error))?; + let collision_reject = match node.parameters.get("collision") { + None => false, + Some(Value::String(value)) if value == "reject" => true, + Some(_) => { + return Err(source.error("column-write collision must be absent or exactly reject")); + } + }; + frame + .with_column(name, column, collision_reject, source) + .map(BoundValue::Frame) +} + +fn execute_cast<'catalog>( + node: &ProgramNode, + scope: &FunctionScope<'catalog>, + source: &SourceLocation, +) -> Result, VectorCoreError> { + let input = single_input(node).map_err(|error| located(source, error))?; + let target = string_parameter(node, "target").map_err(|error| located(source, error))?; + if matches!(target, "array" | "series") { + validate_identity_cast(node, target, source)?; + return scope.bound(input); + } + if target != "float" || node.value_type != "f64-column" { + return Err(VectorCoreError::UnsupportedOpcode { + opcode: node.op.clone(), + location: format!("{}:{}:{}", source.path, source.line, source.column), + }); + } + validate_float_cast_parameters(node, source)?; + let value = scope + .runtime(input) + .map_err(|error| located(source, error))?; + let NodeValue::Column(column) = value else { + return Err(source.error("float cast input is not a column")); + }; + let rows = runtime_column_len(column); + let output = match column.value_type() { + ValueType::F64 => (0..rows).map(|row| column.f64_at(row)).collect(), + ValueType::I64 => (0..rows) + .map(|row| { + #[allow(clippy::cast_precision_loss)] + column.i64_at(row).map(|value| value as f64) + }) + .collect(), + ValueType::Bool => (0..rows) + .map(|row| column.bool_at(row).map(f64::from)) + .collect(), + _ => return Err(source.error("float cast input has an unsupported physical type")), + }; + Ok(BoundValue::Runtime(NodeValue::Column( + RuntimeColumn::Owned(OwnedColumn::f64(output)), + ))) +} + +fn validate_float_cast_parameters( + node: &ProgramNode, + source: &SourceLocation, +) -> Result<(), VectorCoreError> { + match (node.parameters.len(), node.parameters.get("arguments")) { + (1, None) | (2, Some(Value::Null)) => Ok(()), + _ => Err(source.error("float cast parameters are not exact")), + } +} + +fn validate_identity_cast( + node: &ProgramNode, + target: &str, + source: &SourceLocation, +) -> Result<(), VectorCoreError> { + match ( + target, + node.parameters.len(), + node.parameters.get("arguments"), + ) { + ("series", 1, None) | ("series", 2, Some(Value::Null)) => Ok(()), + ("array", 2, Some(Value::Object(arguments))) + if arguments.is_empty() + || (arguments.len() == 1 + && arguments.get("copy").and_then(Value::as_bool) == Some(false)) => + { + Ok(()) + } + _ => Err(source.error(format!("{target} cast arguments are not exact"))), + } +} + +fn execute_instrumentation<'catalog>( + node: &ProgramNode, + source: &SourceLocation, +) -> Result, VectorCoreError> { + if node.parameters.len() != 1 { + return Err(source.error("instrumentation parameters are not exact")); + } + match ( + node.parameters.get("name").and_then(Value::as_str), + node.value_type.as_str(), + ) { + (Some("time.perf_counter"), "f64-scalar") => Ok(BoundValue::Runtime(NodeValue::Float(0.0))), + (Some("log.debug"), "null") => Ok(BoundValue::Runtime(NodeValue::Null)), + _ => Err(VectorCoreError::UnsupportedOpcode { + opcode: node.op.clone(), + location: format!("{}:{}:{}", source.path, source.line, source.column), + }), + } +} + +fn forward_fill_frame<'catalog>( + mut frame: RuntimeFrame<'catalog>, + node: &ProgramNode, + source: &SourceLocation, +) -> Result, VectorCoreError> { + if node.parameters.get("direction").and_then(Value::as_str) != Some("forward") + || node.parameters.len() != 1 + { + return Err(source.error("fill supports only exact forward direction")); + } + let names = frame.column_names().map(str::to_owned).collect::>(); + for name in names { + let column = frame + .owned_column(&name) + .ok_or_else(|| source.error(format!("visible fill column {name:?} is absent")))?; + let filled = forward_fill_column(&column); + frame = frame.with_column(name, filled, false, source)?; + } + Ok(frame) +} + +fn forward_fill_column(column: &OwnedColumn) -> OwnedColumn { + let view = column.as_view(); + match view.value_type() { + ValueType::F64 => { + let mut last = None; + OwnedColumn::f64( + (0..view.len()) + .map(|row| match view.f64_at(row) { + Some(value) if !value.is_nan() => { + last = Some(value); + Some(value) + } + Some(value) => last.or(Some(value)), + None => last, + }) + .collect(), + ) + } + ValueType::I64 => { + let mut last = None; + OwnedColumn::i64( + (0..view.len()) + .map(|row| { + if let Some(value) = view.i64_at(row) { + last = Some(value); + } + last + }) + .collect(), + ) + } + ValueType::Bool => { + let mut last = None; + OwnedColumn::boolean( + (0..view.len()) + .map(|row| { + if let Some(value) = view.bool_at(row) { + last = Some(value); + } + last + }) + .collect(), + ) + } + ValueType::Text => { + let mut last = None; + OwnedColumn::text( + (0..view.len()) + .map(|row| { + if let Some(value) = view.text_at(row) { + last = Some(value.to_owned()); + } + last.clone() + }) + .collect(), + ) + } + ValueType::TimestampMs => { + let mut last = None; + OwnedColumn::timestamp_ms( + (0..view.len()) + .map(|row| { + if let Some(value) = view.timestamp_ms_at(row) { + last = Some(value); + } + last + }) + .collect(), + ) + } + } +} + +fn materialize_output( + frame: &RuntimeFrame<'_>, + requested_outputs: &[String], +) -> Result { + let mut columns = BTreeMap::new(); + for name in requested_outputs { + let column = frame + .owned_column(name) + .ok_or_else(|| VectorCoreError::MissingOutput(name.clone()))?; + if column.len() != frame.len() { + return Err(VectorCoreError::ColumnLength { + column: name.clone(), + actual: column.len(), + expected: frame.len(), + }); + } + columns.insert(name.clone(), column); + } + let dates = frame + .owned_column("date") + .ok_or_else(|| VectorCoreError::MissingColumn("date".to_owned()))?; + let dates = dates.as_view(); + let timestamps_ms = (0..frame.len()) + .map(|row| { + dates.timestamp_ms_at(row).ok_or_else(|| { + VectorCoreError::InvalidOutput(format!( + "returned dataframe date is null at row {row}" + )) + }) + }) + .collect::, _>>()?; + Ok(FullFrameOutput { + identity: frame.identity().clone(), + timestamps_ms, + columns, + }) +} + +fn clone_runtime<'catalog>( + value: &NodeValue<'catalog>, +) -> Result, VectorCoreError> { + Ok(match value { + NodeValue::Null => NodeValue::Null, + NodeValue::Bool(value) => NodeValue::Bool(*value), + NodeValue::Integer(value) => NodeValue::Integer(*value), + NodeValue::Float(value) => NodeValue::Float(*value), + NodeValue::Text(value) => NodeValue::Text(value.clone()), + NodeValue::Json => NodeValue::Json, + NodeValue::Column(RuntimeColumn::Owned(column)) => { + NodeValue::Column(RuntimeColumn::Owned(column.clone())) + } + NodeValue::Column(RuntimeColumn::Borrowed(column)) => { + NodeValue::Column(RuntimeColumn::Borrowed(*column)) + } + NodeValue::DataFrame | NodeValue::Metadata | NodeValue::Unbound | NodeValue::Alias(_) => { + return Err(VectorCoreError::InvalidState( + "unresolved runtime marker cannot cross a function boundary".to_owned(), + )); + } + }) +} + +fn merge_rows( + rows: &mut Option, + actual: usize, + node: &ProgramNode, + source: &SourceLocation, +) -> Result<(), VectorCoreError> { + if rows.is_some_and(|expected| expected != actual) { + return Err(source.error(format!( + "{} input row counts differ at node {}", + node.op, node.id + ))); + } + *rows = Some(actual); + Ok(()) +} + +fn runtime_column_len(column: &RuntimeColumn<'_>) -> usize { + match column { + RuntimeColumn::Borrowed(column) => column.len(), + RuntimeColumn::Owned(column) => column.len(), + } +} + +fn select_indicator_output<'output>( + node: &ProgramNode, + output: &'output crate::kernels::KernelOutput, + source: &SourceLocation, +) -> Result<&'output [f64], VectorCoreError> { + if let Some(name) = node.parameters.get("output").and_then(Value::as_str) { + return output + .column(name) + .ok_or_else(|| source.error(format!("indicator has no output named {name}"))); + } + if output.columns().len() == 1 { + Ok(&output.columns()[0]) + } else { + Err(source.error("multi-output indicator requires an explicit output name")) + } +} + +fn bounded_argument( + arguments: &Map, + name: &str, + source: &SourceLocation, +) -> Result { + arguments + .get(name) + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .ok_or_else(|| { + source.error(format!( + "indicator argument {name:?} is not bounded integer" + )) + }) +} + +fn state_key(call_path: &str, node_id: &str) -> String { + format!("{call_path}/{node_id}") +} + +fn located(source: &SourceLocation, error: VectorCoreError) -> VectorCoreError { + match error { + VectorCoreError::UnsupportedOpcode { opcode, .. } => VectorCoreError::UnsupportedOpcode { + opcode, + location: format!("{}:{}:{}", source.path, source.line, source.column), + }, + VectorCoreError::Execution { message, .. } => source.error(message), + other => source.error(other.to_string()), + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::alignment::Timeframe; + + fn identity(pair: &str, timeframe: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse(timeframe).expect("timeframe")).expect("identity") + } + + fn frame( + pair: &str, + timeframe: &str, + timestamps_ms: Vec, + close: Vec>, + ) -> NumericFrame { + NumericFrame { + identity: identity(pair, timeframe), + timestamps_ms, + columns: BTreeMap::from([("close".to_owned(), close)]), + } + } + + fn program() -> IndicatorProgram { + let zero = || json!({"kind":"finite","candles":0,"expression":null,"causal":true}); + let one = || json!({"kind":"finite","candles":1,"expression":null,"causal":true}); + let recursive = + || json!({"kind":"recursive","candles":null,"expression":"helper","causal":true}); + let source = + || json!({"path":"strategy.py","line":42,"column":8,"end_line":42,"end_column":24}); + let mut encoded = json!({ + "schema_version":"indicator-program-v1", + "source":{"path":"NestedFrames.py","sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + "selected_class":"NestedFrames", + "entrypoint":"f1", + "functions":[ + {"id":"f1","source_name":"populate_indicators","kind":"entrypoint","parameters":[ + {"name":"df","node":"n1","value_type":"dataframe"}, + {"name":"metadata","node":"n2","value_type":"metadata"} + ],"node_ids":["n1","n2","n12","n13","n14","n15","n16","n17"],"return_node":"n17"}, + {"id":"f2","source_name":"informative","kind":"helper","parameters":[ + {"name":"metadata","node":"n6","value_type":"metadata"} + ],"node_ids":["n6","n7","n8","n9","n10","n11"],"return_node":"n11"}, + {"id":"f3","source_name":"lag","kind":"helper","parameters":[ + {"name":"values","node":"n3","value_type":"dynamic"} + ],"node_ids":["n3","n4","n5"],"return_node":"n5"} + ], + "nodes":[ + {"id":"n1","function":"f1","source_order":0,"op":"parameter","value_type":"dataframe","inputs":[],"parameters":{"name":"df"},"lookback":zero()}, + {"id":"n2","function":"f1","source_order":1,"op":"parameter","value_type":"metadata","inputs":[],"parameters":{"name":"metadata"},"lookback":zero()}, + {"id":"n3","function":"f3","source_order":0,"op":"parameter","value_type":"dynamic","inputs":[],"parameters":{"name":"values"},"lookback":zero()}, + {"id":"n4","function":"f3","source_order":1,"op":"shift","value_type":"f64-column","inputs":["n3"],"parameters":{"periods":1},"lookback":one()}, + {"id":"n5","function":"f3","source_order":2,"op":"return","value_type":"f64-column","inputs":["n4"],"parameters":{},"lookback":one()}, + {"id":"n6","function":"f2","source_order":0,"op":"parameter","value_type":"metadata","inputs":[],"parameters":{"name":"metadata"},"lookback":zero()}, + {"id":"n7","function":"f2","source_order":1,"op":"frame-source","value_type":"dataframe","inputs":[],"parameters":{"pair":{"kind":"metadata","key":"pair"},"timeframe":"15m"},"lookback":zero()}, + {"id":"n8","function":"f2","source_order":2,"op":"column-read","value_type":"f64-column","inputs":["n7"],"parameters":{"column":"close"},"lookback":zero()}, + {"id":"n9","function":"f2","source_order":3,"op":"function-call","value_type":"f64-column","inputs":["n8"],"parameters":{"function":"f3"},"lookback":recursive()}, + {"id":"n10","function":"f2","source_order":4,"op":"column-write","value_type":"dataframe","inputs":["n7","n9"],"parameters":{"column":"lagged","collision":"reject"},"lookback":recursive()}, + {"id":"n11","function":"f2","source_order":5,"op":"return","value_type":"dataframe","inputs":["n10"],"parameters":{},"lookback":recursive()}, + {"id":"n12","function":"f1","source_order":2,"op":"function-call","value_type":"dataframe","inputs":["n2"],"parameters":{"function":"f2"},"lookback":recursive()}, + {"id":"n13","function":"f1","source_order":3,"op":"column-read","value_type":"f64-column","inputs":["n1"],"parameters":{"column":"close"},"lookback":zero()}, + {"id":"n14","function":"f1","source_order":4,"op":"function-call","value_type":"f64-column","inputs":["n13"],"parameters":{"function":"f3"},"lookback":recursive()}, + {"id":"n15","function":"f1","source_order":5,"op":"column-write","value_type":"dataframe","inputs":["n1","n14"],"parameters":{"column":"base_lagged","collision":"reject"},"lookback":recursive()}, + {"id":"n16","function":"f1","source_order":6,"op":"informative-merge","value_type":"dataframe","inputs":["n15","n12"],"parameters":{"base_timeframe":"5m","informative_timeframe":"15m","ffill":false,"append_timeframe":true,"date_column":"date","suffix":null},"lookback":recursive()}, + {"id":"n17","function":"f1","source_order":7,"op":"return","value_type":"dataframe","inputs":["n16"],"parameters":{},"lookback":recursive()} + ], + "required_input_columns":["close"], + "produced_columns":["base_lagged","lagged"], + "informative_nodes":["n16"], + "opcodes":["column-read","column-write","frame-source","function-call","informative-merge","parameter","return","shift"], + "max_lookback":{"kind":"mixed","candles":null,"expression":"finite+recursive","causal":true}, + "source_map":{}, + "fingerprint":"" + }); + for index in 1..=17 { + encoded["source_map"][format!("n{index}")] = source(); + } + encoded["fingerprint"] = Value::String( + crate::program::validation::canonical_fingerprint(&encoded) + .expect("canonical fingerprint"), + ); + IndicatorProgram::from_json(&encoded.to_string()).expect("valid recursive program") + } + + #[test] + fn float_cast_accepts_the_compiler_shape_and_rejects_dynamic_arguments() { + let mut node: ProgramNode = serde_json::from_value(json!({ + "id":"n1", + "function":"f1", + "source_order":0, + "op":"cast", + "value_type":"f64-column", + "inputs":["n0"], + "parameters":{"target":"float"}, + "lookback":{"kind":"finite","candles":0,"expression":null,"causal":true} + })) + .expect("cast node"); + let source = SourceLocation::new("cast", "strategy.py", 7, 4); + + validate_float_cast_parameters(&node, &source).expect("compiler cast shape"); + node.parameters.insert("arguments".to_owned(), Value::Null); + validate_float_cast_parameters(&node, &source).expect("legacy null arguments"); + node.parameters + .insert("arguments".to_owned(), json!({"copy": false})); + assert!(validate_float_cast_parameters(&node, &source).is_err()); + } + + #[test] + fn series_cast_accepts_the_compiler_shape_and_rejects_options() { + let mut node: ProgramNode = serde_json::from_value(json!({ + "id":"n1", + "function":"f1", + "source_order":0, + "op":"cast", + "value_type":"f64-column", + "inputs":["n0"], + "parameters":{"target":"series"}, + "lookback":{"kind":"finite","candles":0,"expression":null,"causal":true} + })) + .expect("cast node"); + let source = SourceLocation::new("cast", "strategy.py", 7, 4); + + validate_identity_cast(&node, "series", &source).expect("compiler series shape"); + node.parameters.insert("arguments".to_owned(), Value::Null); + validate_identity_cast(&node, "series", &source).expect("legacy null arguments"); + node.parameters + .insert("arguments".to_owned(), json!({"copy": false})); + assert!(validate_identity_cast(&node, "series", &source).is_err()); + } + + #[test] + fn nested_dynamic_helpers_keep_state_and_row_counts_isolated_by_call_path() { + let base = frame( + "ETH/USDT", + "5m", + vec![0, 600_000, 900_000, 1_500_000], + vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)], + ); + let informative = frame( + "ETH/USDT", + "15m", + vec![0, 900_000], + vec![Some(1.0), Some(2.0)], + ); + let catalog = + FrameCatalog::new([(informative.identity.clone(), informative)]).expect("catalog"); + let metadata = BTreeMap::from([("pair".to_owned(), "ETH/USDT".to_owned())]); + let program = program(); + let mut engine = FullIndicatorEngine::new(&program).expect("full engine"); + let output = engine + .execute( + &base, + &catalog, + &metadata, + &["base_lagged".to_owned(), "lagged_15m".to_owned()], + ) + .expect("complete execution"); + + assert_eq!(output.identity(), &base.identity); + assert_eq!(output.len(), 4); + let base_lagged = output.columns()["base_lagged"].as_view(); + assert!(base_lagged.f64_at(0).expect("present NaN").is_nan()); + assert_eq!(base_lagged.f64_at(1), Some(10.0)); + assert_eq!(base_lagged.f64_at(2), Some(20.0)); + assert_eq!(base_lagged.f64_at(3), Some(30.0)); + + let informative_lagged = output.columns()["lagged_15m"].as_view(); + assert_eq!(informative_lagged.len(), 4); + assert!(informative_lagged + .f64_at(1) + .expect("informative leading NaN") + .is_nan()); + assert_eq!(informative_lagged.f64_at(3), Some(1.0)); + } + + #[test] + fn requested_typed_output_keeps_arrow_null_distinct_from_nan() { + let base = frame( + "ETH/USDT", + "5m", + vec![0, 600_000, 900_000, 1_500_000], + vec![Some(10.0), Some(20.0), Some(30.0), Some(40.0)], + ); + let informative = frame("ETH/USDT", "15m", vec![0, 900_000], vec![None, Some(2.0)]); + let catalog = + FrameCatalog::new([(informative.identity.clone(), informative)]).expect("catalog"); + let metadata = BTreeMap::from([("pair".to_owned(), "ETH/USDT".to_owned())]); + let program = program(); + let output = FullIndicatorEngine::new(&program) + .expect("full engine") + .execute(&base, &catalog, &metadata, &["lagged_15m".to_owned()]) + .expect("complete execution"); + let values = output.columns()["lagged_15m"].as_view(); + assert!(values.f64_at(1).expect("present NaN").is_nan()); + assert_eq!(values.f64_at(3), None); + } + + #[test] + fn missing_informative_frame_fails_at_the_compiled_source_location() { + let base = frame("ETH/USDT", "5m", vec![0], vec![Some(10.0)]); + let metadata = BTreeMap::from([("pair".to_owned(), "ETH/USDT".to_owned())]); + let program = program(); + let error = FullIndicatorEngine::new(&program) + .expect("full engine") + .execute( + &base, + &FrameCatalog::default(), + &metadata, + &["base_lagged".to_owned()], + ) + .expect_err("missing informative frame"); + assert!(matches!( + error, + VectorCoreError::Execution { node, message } + if node == "n7" + && message.starts_with("strategy.py:42:8:") + && message.contains("ETH/USDT 15m") + )); + } +} diff --git a/rust/crates/nfi-vector-core/src/engine/mod.rs b/rust/crates/nfi-vector-core/src/engine/mod.rs index 27bbd8c4..b654c01f 100644 --- a/rust/crates/nfi-vector-core/src/engine/mod.rs +++ b/rust/crates/nfi-vector-core/src/engine/mod.rs @@ -1,5 +1,10 @@ //! Batch-local execution of the safe generic indicator-program substrate. +#[allow(dead_code)] // Staged independently; the runtime integration owns execute-node wiring. +mod array; +#[allow(dead_code)] // Staged independently; the runtime integration owns execute-node wiring. +mod frame; +mod full; mod operations; mod runtime; @@ -19,6 +24,8 @@ use crate::program::{ExecutionPlan, IndicatorProgram, ProgramNode}; use crate::sink::{BatchSink, OutputBatch}; use crate::state::ShiftState; +pub use self::full::{FullFrameOutput, FullIndicatorEngine}; + /// Bounded live-value accounting across a streaming execution. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct EngineProfile { diff --git a/rust/crates/nfi-vector-core/src/engine/operations.rs b/rust/crates/nfi-vector-core/src/engine/operations.rs index 82513803..1fb5d3a3 100644 --- a/rust/crates/nfi-vector-core/src/engine/operations.rs +++ b/rust/crates/nfi-vector-core/src/engine/operations.rs @@ -12,6 +12,14 @@ use crate::program::ProgramNode; use super::runtime::{NodeValue, RuntimeColumn}; pub(super) fn literal_value(node: &ProgramNode) -> Result, VectorCoreError> { + if let Some(special) = node.parameters.get("special").and_then(Value::as_str) { + return Ok(NodeValue::Float(match special { + "nan" => f64::NAN, + "+infinity" => f64::INFINITY, + "-infinity" => f64::NEG_INFINITY, + _ => return Err(node_error(node, "literal has an unknown special float")), + })); + } let value = node .parameters .get("value") @@ -53,22 +61,24 @@ pub(super) fn execute_binary<'batch>( )); } }; + let left = resolve_numeric(values, left)?; + let right = resolve_numeric(values, right)?; if node.value_type.ends_with("-column") { let output = (0..rows) .map(|row| { - let left = numeric_at(values, left, row)?; - let right = numeric_at(values, right, row)?; - Ok(match (left, right) { + let left = left.at(row); + let right = right.at(row); + match (left, right) { (Some(left), Some(right)) => Some(binary(left, right, operation)), _ => None, - }) + } }) - .collect::, _>>()?; + .collect(); Ok(NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::f64( output, )))) } else { - match (numeric_at(values, left, 0)?, numeric_at(values, right, 0)?) { + match (left.at(0), right.at(0)) { (Some(left), Some(right)) => Ok(NodeValue::Float(binary(left, right, operation))), _ => Ok(NodeValue::Null), } @@ -90,22 +100,24 @@ pub(super) fn execute_compare<'batch>( "greater-than-or-equal" => FloatComparison::GreaterEqual, other => return Err(node_error(node, format!("unsupported comparison: {other}"))), }; + let left = resolve_numeric(values, left)?; + let right = resolve_numeric(values, right)?; if node.value_type == "bool-column" { let output = (0..rows) .map(|row| { - let left = numeric_at(values, left, row)?; - let right = numeric_at(values, right, row)?; - Ok(match (left, right) { + let left = left.at(row); + let right = right.at(row); + match (left, right) { (Some(left), Some(right)) => Some(compare(left, right, comparison)), _ => None, - }) + } }) - .collect::, _>>()?; + .collect(); Ok(NodeValue::Column(RuntimeColumn::Owned( OwnedColumn::boolean(output), ))) } else { - match (numeric_at(values, left, 0)?, numeric_at(values, right, 0)?) { + match (left.at(0), right.at(0)) { (Some(left), Some(right)) => Ok(NodeValue::Bool(compare(left, right, comparison))), _ => Ok(NodeValue::Null), } @@ -124,10 +136,15 @@ pub(super) fn execute_logical<'batch>( "logical node has an unsupported operator or arity", )); } - let apply = |row| -> Result, VectorCoreError> { - let mut result = bool_at(values, &node.inputs[0], row)?; - for input in node.inputs.iter().skip(1) { - let right = bool_at(values, input, row)?; + let inputs = node + .inputs + .iter() + .map(|input| resolve_bool(values, input)) + .collect::, _>>()?; + let apply = |row| -> Option { + let mut result = inputs[0].at(row); + for input in inputs.iter().skip(1) { + let right = input.at(row); result = match (result, right) { (Some(left), Some(right)) => Some(if operator == "and" { left && right @@ -137,14 +154,14 @@ pub(super) fn execute_logical<'batch>( _ => None, }; } - Ok(result) + result }; if node.value_type == "bool-column" { Ok(NodeValue::Column(RuntimeColumn::Owned( - OwnedColumn::boolean((0..rows).map(apply).collect::, _>>()?), + OwnedColumn::boolean((0..rows).map(apply).collect()), ))) } else { - Ok(apply(0)?.map_or(NodeValue::Null, NodeValue::Bool)) + Ok(apply(0).map_or(NodeValue::Null, NodeValue::Bool)) } } @@ -159,32 +176,34 @@ pub(super) fn execute_unary<'batch>( if !matches!(operator, "not" | "invert") { return Err(node_error(node, "boolean unary operator is unsupported")); } + let input = resolve_bool(values, input)?; return Ok(NodeValue::Column(RuntimeColumn::Owned( OwnedColumn::boolean( (0..rows) - .map(|row| bool_at(values, input, row).map(|value| value.map(|item| !item))) - .collect::, _>>()?, + .map(|row| input.at(row).map(|item| !item)) + .collect(), ), ))); } if !matches!(operator, "negate" | "positive") { return Err(node_error(node, "numeric unary operator is unsupported")); } - let apply = |row| -> Result, VectorCoreError> { - Ok(numeric_at(values, input, row)?.map(|value| { + let input = resolve_numeric(values, input)?; + let apply = |row| -> Option { + input.at(row).map(|value| { if operator == "negate" { crate::float::canonicalize(-value) } else { value } - })) + }) }; if node.value_type.ends_with("-column") { Ok(NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::f64( - (0..rows).map(apply).collect::, _>>()?, + (0..rows).map(apply).collect(), )))) } else { - Ok(apply(0)?.map_or(NodeValue::Null, NodeValue::Float)) + Ok(apply(0).map_or(NodeValue::Null, NodeValue::Float)) } } @@ -199,13 +218,35 @@ pub(super) fn execute_select<'batch>( "select substrate requires three inputs and f64 output", )); } - let output = (0..rows) - .map(|row| match bool_at(values, &node.inputs[0], row)? { - Some(true) => numeric_at(values, &node.inputs[1], row), - Some(false) => numeric_at(values, &node.inputs[2], row), - None => Ok(None), - }) - .collect::, _>>()?; + let condition = resolve_bool(values, &node.inputs[0])?; + let mut when_true = None; + let mut when_false = None; + let mut output = Vec::with_capacity(rows); + for row in 0..rows { + output.push(match condition.at(row) { + Some(true) => { + let value = if let Some(value) = when_true { + value + } else { + let value = resolve_numeric(values, &node.inputs[1])?; + when_true = Some(value); + value + }; + value.at(row) + } + Some(false) => { + let value = if let Some(value) = when_false { + value + } else { + let value = resolve_numeric(values, &node.inputs[2])?; + when_false = Some(value); + value + }; + value.at(row) + } + None => None, + }); + } Ok(NodeValue::Column(RuntimeColumn::Owned(OwnedColumn::f64( output, )))) @@ -240,7 +281,8 @@ pub(super) fn collect_numeric( node: &str, rows: usize, ) -> Result>, VectorCoreError> { - (0..rows).map(|row| numeric_at(values, node, row)).collect() + let value = resolve_numeric(values, node)?; + Ok((0..rows).map(|row| value.at(row)).collect()) } pub(super) fn to_owned_column( @@ -283,15 +325,43 @@ pub(super) fn numeric_at( node: &str, row: usize, ) -> Result, VectorCoreError> { + Ok(resolve_numeric(values, node)?.at(row)) +} + +#[derive(Clone, Copy)] +enum NumericValue<'values, 'batch> { + Null, + Integer(i64), + Float(f64), + I64(&'values RuntimeColumn<'batch>), + F64(&'values RuntimeColumn<'batch>), +} + +impl NumericValue<'_, '_> { + fn at(self, row: usize) -> Option { + match self { + Self::Null => None, + Self::Integer(value) => Some(integer_as_f64(value)), + Self::Float(value) => Some(value), + Self::I64(column) => column.i64_at(row).map(integer_as_f64), + Self::F64(column) => column.f64_at(row), + } + } +} + +fn resolve_numeric<'values, 'batch>( + values: &'values BTreeMap>, + node: &str, +) -> Result, VectorCoreError> { match resolve_value(values, node)? { - NodeValue::Null => Ok(None), - NodeValue::Integer(value) => Ok(Some(integer_as_f64(*value))), - NodeValue::Float(value) => Ok(Some(*value)), + NodeValue::Null => Ok(NumericValue::Null), + NodeValue::Integer(value) => Ok(NumericValue::Integer(*value)), + NodeValue::Float(value) => Ok(NumericValue::Float(*value)), NodeValue::Column(column) if column.value_type() == ValueType::I64 => { - Ok(column.i64_at(row).map(integer_as_f64)) + Ok(NumericValue::I64(column)) } NodeValue::Column(column) if column.value_type() == ValueType::F64 => { - Ok(column.f64_at(row)) + Ok(NumericValue::F64(column)) } _ => Err(VectorCoreError::Execution { node: node.to_owned(), @@ -300,16 +370,32 @@ pub(super) fn numeric_at( } } -fn bool_at( - values: &BTreeMap>, +#[derive(Clone, Copy)] +enum BoolValue<'values, 'batch> { + Null, + Scalar(bool), + Column(&'values RuntimeColumn<'batch>), +} + +impl BoolValue<'_, '_> { + fn at(self, row: usize) -> Option { + match self { + Self::Null => None, + Self::Scalar(value) => Some(value), + Self::Column(column) => column.bool_at(row), + } + } +} + +fn resolve_bool<'values, 'batch>( + values: &'values BTreeMap>, node: &str, - row: usize, -) -> Result, VectorCoreError> { +) -> Result, VectorCoreError> { match resolve_value(values, node)? { - NodeValue::Null => Ok(None), - NodeValue::Bool(value) => Ok(Some(*value)), + NodeValue::Null => Ok(BoolValue::Null), + NodeValue::Bool(value) => Ok(BoolValue::Scalar(*value)), NodeValue::Column(column) if column.value_type() == ValueType::Bool => { - Ok(column.bool_at(row)) + Ok(BoolValue::Column(column)) } _ => Err(VectorCoreError::Execution { node: node.to_owned(), diff --git a/rust/crates/nfi-vector-core/src/engine/runtime.rs b/rust/crates/nfi-vector-core/src/engine/runtime.rs index f1670f27..9f479801 100644 --- a/rust/crates/nfi-vector-core/src/engine/runtime.rs +++ b/rust/crates/nfi-vector-core/src/engine/runtime.rs @@ -3,6 +3,8 @@ use std::collections::BTreeMap; use crate::batch::BatchView; +use arrow2::array::Array; + use crate::column::{ColumnView, OwnedColumn, ValueType}; use crate::error::VectorCoreError; use crate::kernels::{rolling_stream, TalibStream}; @@ -63,6 +65,23 @@ impl RuntimeColumn<'_> { Self::Owned(column) => column.as_view().timestamp_ms_at(row), } } + + /// Borrow the canonical f64 buffer when every row is present. + /// + /// Complete-frame stateful kernels can then consume Arrow-owned outputs + /// without rebuilding an intermediate `Vec` for every downstream + /// node. Nullable columns still take the checked scalar path. + pub(super) fn present_f64_slice(&self) -> Option<&[f64]> { + match self { + Self::Borrowed(ColumnView::F64(values)) if values.null_count() == 0 => { + Some(values.values().as_slice()) + } + Self::Owned(OwnedColumn::F64(values)) if values.null_count() == 0 => { + Some(values.values().as_slice()) + } + _ => None, + } + } } #[derive(Debug)] @@ -316,3 +335,30 @@ fn parameter_value(node: &ProgramNode) -> NodeValue<'static> { _ => NodeValue::Unbound, } } + +#[cfg(test)] +mod tests { + use super::RuntimeColumn; + use crate::column::OwnedColumn; + + #[test] + fn present_f64_slice_preserves_bits_and_rejects_arrow_nulls() { + let payload_nan = f64::from_bits(0x7ff8_0000_0000_0042); + let present = RuntimeColumn::Owned(OwnedColumn::f64(vec![ + Some(-0.0), + Some(payload_nan), + Some(f64::INFINITY), + ])); + let values = present.present_f64_slice().expect("all values present"); + + assert_eq!(values[0].to_bits(), (-0.0_f64).to_bits()); + assert_eq!( + values[1].to_bits(), + crate::float::canonicalize(payload_nan).to_bits() + ); + assert_eq!(values[2].to_bits(), f64::INFINITY.to_bits()); + + let nullable = RuntimeColumn::Owned(OwnedColumn::f64(vec![Some(-0.0), None])); + assert!(nullable.present_f64_slice().is_none()); + } +} diff --git a/rust/crates/nfi-vector-core/src/kernels/mod.rs b/rust/crates/nfi-vector-core/src/kernels/mod.rs index ba2881db..a00fc187 100644 --- a/rust/crates/nfi-vector-core/src/kernels/mod.rs +++ b/rust/crates/nfi-vector-core/src/kernels/mod.rs @@ -4,9 +4,16 @@ mod directional; mod moving; +mod native; mod oscillator; mod rolling; +pub use native::{ + absolute_difference, chaikin_money_flow, hourly_inside_bar, legacy_chaikin_money_flow, + safe_percent_change, utc_opening_range, AbsoluteDifferenceStream, ChaikinMoneyFlowStream, + HourlyInsideBarOutput, HourlyInsideBarStream, LegacyChaikinMoneyFlowStream, + SafePercentChangeStream, UtcOpeningRangeOutput, UtcOpeningRangeStream, +}; pub use rolling::execute_rolling; pub(crate) use rolling::{stream as rolling_stream, RollingStream}; @@ -273,6 +280,25 @@ fn execute_oscillator( input(inputs, 0)?, integer(arguments, "timeperiod", 14)?, )?), + "STOCH" => { + if integer(arguments, "slowk_matype", 0)? != 0 + || integer(arguments, "slowd_matype", 0)? != 0 + { + return Err(kernel_error( + name, + "only exact TA-Lib SMA slow-K and slow-D are implemented", + )); + } + let (slow_k, slow_d) = oscillator::stoch( + input(inputs, 0)?, + input(inputs, 1)?, + input(inputs, 2)?, + integer(arguments, "fastk_period", 5)?, + integer(arguments, "slowk_period", 3)?, + integer(arguments, "slowd_period", 3)?, + )?; + KernelOutput::new(&["slowk", "slowd"], vec![slow_k, slow_d]) + } "STOCHF" => { if integer(arguments, "fastd_matype", 0)? != 0 { return Err(kernel_error( @@ -310,6 +336,7 @@ fn output_names(name: &str) -> Result<&'static [&'static str], VectorCoreError> match name { "AROON" => Ok(&["aroondown", "aroonup"]), "BBANDS" => Ok(&["upperband", "middleband", "lowerband"]), + "STOCH" => Ok(&["slowk", "slowd"]), "STOCHF" => Ok(&["fastk", "fastd"]), "ADX" | "CCI" | "EMA" | "MAX" | "MFI" | "MIN" | "MINUS_DI" | "OBV" | "PLUS_DI" | "ROC" | "RSI" | "SMA" | "STDDEV" | "SUM" | "ULTOSC" | "WILLR" => Ok(&["real"]), diff --git a/rust/crates/nfi-vector-core/src/kernels/moving.rs b/rust/crates/nfi-vector-core/src/kernels/moving.rs index 929bdb7a..937b5e24 100644 --- a/rust/crates/nfi-vector-core/src/kernels/moving.rs +++ b/rust/crates/nfi-vector-core/src/kernels/moving.rs @@ -17,12 +17,14 @@ pub(super) enum MovingStream { total: f64, values: VecDeque, average: bool, + started: bool, }, Ema { period: usize, count: usize, seed_total: f64, previous: Option, + started: bool, }, Extreme { period: usize, @@ -41,6 +43,7 @@ pub(super) enum MovingStream { total_two: f64, values: VecDeque, nb_dev: f64, + started: bool, }, Bbands { period: usize, @@ -49,9 +52,21 @@ pub(super) enum MovingStream { values: VecDeque, nb_dev_up: f64, nb_dev_down: f64, + started: bool, }, } +pub(super) fn sum_stream(period: usize) -> Result { + validate_period(period, 2, "SUM")?; + Ok(MovingStream::Sum { + period, + total: 0.0, + values: VecDeque::new(), + average: false, + started: false, + }) +} + /// Build bounded streaming state for a moving indicator, if it is supported. pub(super) fn stream( name: &str, @@ -63,18 +78,21 @@ pub(super) fn stream( total: 0.0, values: VecDeque::new(), average: true, + started: false, }, "SUM" => MovingStream::Sum { period: argument_period(arguments, 30, "SUM")?, total: 0.0, values: VecDeque::new(), average: false, + started: false, }, "EMA" => MovingStream::Ema { period: argument_period(arguments, 30, "EMA")?, count: 0, seed_total: 0.0, previous: None, + started: false, }, "MIN" => MovingStream::Extreme { period: argument_period(arguments, 30, "MIN")?, @@ -99,6 +117,7 @@ pub(super) fn stream( total_two: 0.0, values: VecDeque::new(), nb_dev: argument_number(arguments, "nbdev", 1.0)?, + started: false, }, "BBANDS" => { if argument_integer(arguments, "matype", 0)? != 0 { @@ -113,6 +132,7 @@ pub(super) fn stream( values: VecDeque::new(), nb_dev_up: argument_number(arguments, "nbdevup", 2.0)?, nb_dev_down: argument_number(arguments, "nbdevdn", 2.0)?, + started: false, } } _ => return Ok(None), @@ -163,13 +183,15 @@ impl MovingStream { total, values, average, - } => next_sum(value, *period, total, values, *average), + started, + } => next_sum(value, *period, total, values, *average, started), Self::Ema { period, count, seed_total, previous, - } => next_ema(value, *period, count, seed_total, previous), + started, + } => next_ema(value, *period, count, seed_total, previous, started), Self::Extreme { period, seen, @@ -187,7 +209,16 @@ impl MovingStream { total_two, values, nb_dev, - } => next_stddev(value, *period, total_one, total_two, values, *nb_dev), + started, + } => { + let mut state = VarianceState { + total_one, + total_two, + values, + started, + }; + next_stddev(value, *period, &mut state, *nb_dev) + } Self::Bbands { .. } => unreachable!("BBANDS has three outputs"), } } @@ -200,24 +231,30 @@ impl MovingStream { values, nb_dev_up, nb_dev_down, + started, } = self else { unreachable!("only BBANDS has three outputs"); }; - next_bbands( - value, - *period, + let mut state = VarianceState { total_one, total_two, values, - *nb_dev_up, - *nb_dev_down, - ) + started, + }; + next_bbands(value, *period, &mut state, *nb_dev_up, *nb_dev_down) } } type BandsRow = (f64, f64, f64); +struct VarianceState<'a> { + total_one: &'a mut f64, + total_two: &'a mut f64, + values: &'a mut VecDeque, + started: &'a mut bool, +} + fn argument_period( arguments: &Map, default: usize, @@ -283,7 +320,14 @@ fn next_sum( total: &mut f64, values: &mut VecDeque, average: bool, + started: &mut bool, ) -> f64 { + if !*started { + if value.is_nan() { + return f64::NAN; + } + *started = true; + } *total += value; values.push_back(value); if values.len() < period { @@ -307,7 +351,14 @@ fn next_ema( count: &mut usize, seed_total: &mut f64, previous: &mut Option, + started: &mut bool, ) -> f64 { + if !*started { + if value.is_nan() { + return f64::NAN; + } + *started = true; + } if *count < period - 1 { *count += 1; *seed_total += value; @@ -371,15 +422,8 @@ fn next_roc(value: f64, period: usize, seen: &mut usize, values: &mut VecDeque, - nb_dev: f64, -) -> f64 { - let Some((variance, _)) = next_variance(value, period, total_one, total_two, values) else { +fn next_stddev(value: f64, period: usize, state: &mut VarianceState<'_>, nb_dev: f64) -> f64 { + let Some((variance, _)) = next_variance(value, period, state) else { return f64::NAN; }; if variance > 0.0 { @@ -396,13 +440,11 @@ fn next_stddev( fn next_bbands( value: f64, period: usize, - total_one: &mut f64, - total_two: &mut f64, - values: &mut VecDeque, + state: &mut VarianceState<'_>, nb_dev_up: f64, nb_dev_down: f64, ) -> BandsRow { - let Some((variance, mean)) = next_variance(value, period, total_one, total_two, values) else { + let Some((variance, mean)) = next_variance(value, period, state) else { return (f64::NAN, f64::NAN, f64::NAN); }; let deviation = if variance > 0.0 { variance.sqrt() } else { 0.0 }; @@ -426,26 +468,27 @@ fn next_bbands( } } -fn next_variance( - value: f64, - period: usize, - total_one: &mut f64, - total_two: &mut f64, - values: &mut VecDeque, -) -> Option<(f64, f64)> { - *total_one += value; - *total_two += value * value; - values.push_back(value); - if values.len() < period { +fn next_variance(value: f64, period: usize, state: &mut VarianceState<'_>) -> Option<(f64, f64)> { + if !*state.started { + if value.is_nan() { + return None; + } + *state.started = true; + } + *state.total_one += value; + *state.total_two += value * value; + state.values.push_back(value); + if state.values.len() < period { return None; } - let mean_one = *total_one / period_as_f64(period); - let mean_two = *total_two / period_as_f64(period); - let trailing = values + let mean_one = *state.total_one / period_as_f64(period); + let mean_two = *state.total_two / period_as_f64(period); + let trailing = state + .values .pop_front() .expect("full rolling variance has a trailing value"); - *total_one -= trailing; - *total_two -= trailing * trailing; + *state.total_one -= trailing; + *state.total_two -= trailing * trailing; Some((mean_two - (mean_one * mean_one), mean_one)) } @@ -457,18 +500,22 @@ pub(super) fn sma(values: &[f64], period: usize) -> Result, VectorCoreE pub(super) fn ema(values: &[f64], period: usize) -> Result, VectorCoreError> { validate_period(period, 2, "EMA")?; let mut output = warmup(values.len()); - if values.len() < period { + let Some(start) = first_non_nan(values) else { + return Ok(output); + }; + let first_output = start.saturating_add(period - 1); + if first_output >= values.len() { return Ok(output); } let mut total = 0.0; - for value in &values[..period] { + for value in &values[start..=first_output] { total += value; } let period_as_f64 = period_as_f64(period); let k = 2.0 / (period_as_f64 + 1.0); let mut previous = total / period_as_f64; - output[period - 1] = previous; - for (index, value) in values.iter().enumerate().skip(period) { + output[first_output] = previous; + for (index, value) in values.iter().enumerate().skip(first_output + 1) { previous = ((*value - previous) * k) + previous; output[index] = previous; } @@ -521,7 +568,10 @@ pub(super) fn bbands_sma( let deviations = stddev_using_precalculated_sma(values, &middle, period); let mut upper = warmup(values.len()); let mut lower = warmup(values.len()); - for index in (period - 1)..values.len() { + let Some(first_output) = first_non_nan(values).map(|start| start + period - 1) else { + return Ok((upper, middle, lower)); + }; + for index in first_output..values.len() { let deviation = deviations[index]; let average = middle[index]; if exact_equal(nb_dev_up, nb_dev_down) { @@ -560,6 +610,10 @@ fn warmup(length: usize) -> Vec { vec![f64::NAN; length] } +fn first_non_nan(values: &[f64]) -> Option { + values.iter().position(|value| !value.is_nan()) +} + /// TA-Lib permits periods through 100,000, so this cast is exact on all targets. #[allow(clippy::cast_precision_loss)] fn period_as_f64(period: usize) -> f64 { @@ -578,15 +632,19 @@ fn is_one(value: f64) -> bool { fn rolling_sum(values: &[f64], period: usize, average: bool) -> Vec { let mut output = warmup(values.len()); - if values.len() < period { + let Some(start) = first_non_nan(values) else { + return output; + }; + let first_output = start.saturating_add(period - 1); + if first_output >= values.len() { return output; } let mut total = 0.0; - let mut trailing = 0; - for value in &values[..period - 1] { + let mut trailing = start; + for value in &values[start..first_output] { total += value; } - for today in (period - 1)..values.len() { + for today in first_output..values.len() { total += values[today]; let current = total; total -= values[trailing]; @@ -647,7 +705,10 @@ fn stddev_with_nbdev( ) -> Result, VectorCoreError> { validate_period(period, 2, "STDDEV")?; let mut output = variance(values, period); - for value in output.iter_mut().skip(period - 1) { + let Some(first_output) = first_non_nan(values).map(|start| start + period - 1) else { + return Ok(output); + }; + for value in output.iter_mut().skip(first_output) { if *value > 0.0 { *value = if is_one(nb_dev) { value.sqrt() @@ -663,16 +724,20 @@ fn stddev_with_nbdev( fn variance(values: &[f64], period: usize) -> Vec { let mut output = warmup(values.len()); - if values.len() < period { + let Some(start) = first_non_nan(values) else { + return output; + }; + let first_output = start.saturating_add(period - 1); + if first_output >= values.len() { return output; } let mut total_one = 0.0; let mut total_two = 0.0; - for value in &values[..period - 1] { + for value in &values[start..first_output] { total_one += value; total_two += value * value; } - for (trailing, today) in ((period - 1)..values.len()).enumerate() { + for (trailing, today) in (start..).zip(first_output..values.len()) { let value = values[today]; total_one += value; total_two += value * value; @@ -688,14 +753,18 @@ fn variance(values: &[f64], period: usize) -> Vec { fn stddev_using_precalculated_sma(values: &[f64], averages: &[f64], period: usize) -> Vec { let mut output = warmup(values.len()); - if values.len() < period { + let Some(start) = first_non_nan(values) else { + return output; + }; + let first_output = start.saturating_add(period - 1); + if first_output >= values.len() { return output; } let mut total_two = 0.0; - for value in &values[..period - 1] { + for value in &values[start..first_output] { total_two += value * value; } - for (trailing, index) in ((period - 1)..values.len()).enumerate() { + for (trailing, index) in (start..).zip(first_output..values.len()) { let value = values[index]; total_two += value * value; let mut mean_two = total_two / period_as_f64(period); @@ -737,6 +806,85 @@ mod tests { ); } + #[test] + fn leading_nan_warmup_matches_talib_and_streaming_chunks() { + let values = [f64::NAN, f64::NAN, 1.0, 2.0, 3.0, 4.0, 5.0]; + let average = vec![f64::NAN, f64::NAN, f64::NAN, f64::NAN, 2.0, 3.0, 4.0]; + let total = vec![f64::NAN, f64::NAN, f64::NAN, f64::NAN, 6.0, 9.0, 12.0]; + let deviation = vec![ + f64::NAN, + f64::NAN, + f64::NAN, + f64::NAN, + f64::from_bits(0x3fea_20bd_700c_2c40), + f64::from_bits(0x3fea_20bd_700c_2c3b), + f64::from_bits(0x3fea_20bd_700c_2c45), + ]; + let upper = vec![ + f64::NAN, + f64::NAN, + f64::NAN, + f64::NAN, + f64::from_bits(0x400d_105e_b806_1620), + f64::from_bits(0x4012_882f_5c03_0b0f), + f64::from_bits(0x4016_882f_5c03_0b11), + ]; + let lower = vec![ + f64::NAN, + f64::NAN, + f64::NAN, + f64::NAN, + f64::from_bits(0x3fd7_7d0a_3fcf_4f00), + f64::from_bits(0x3ff5_df42_8ff3_d3c5), + f64::from_bits(0x4002_efa1_47f9_e9de), + ]; + let period = 3; + let defaults = arguments(&json!({"timeperiod": period})); + + assert_bits(&sma(&values, period).expect("SMA"), &average); + assert_bits(&ema(&values, period).expect("EMA"), &average); + assert_bits(&sum(&values, period).expect("SUM"), &total); + assert_bits(&stddev(&values, period, 1.0).expect("STDDEV"), &deviation); + let bands = bbands_sma(&values, period, 2.0, 2.0).expect("BBANDS"); + assert_bits(&bands.0, &upper); + assert_bits(&bands.1, &average); + assert_bits(&bands.2, &lower); + assert_stream( + "SMA", + &defaults, + &values, + std::slice::from_ref(&average), + period, + ); + assert_stream( + "EMA", + &defaults, + &values, + std::slice::from_ref(&average), + period, + ); + assert_stream("SUM", &defaults, &values, &[total], period); + assert_stream( + "STDDEV", + &arguments(&json!({"timeperiod": period, "nbdev": 1.0})), + &values, + &[deviation], + period, + ); + assert_stream( + "BBANDS", + &arguments(&json!({ + "timeperiod": period, + "nbdevup": 2.0, + "nbdevdn": 2.0, + "matype": 0, + })), + &values, + &[upper, average, lower], + period, + ); + } + #[test] fn extrema_and_roc_preserve_talib_comparisons_and_zero_rule() { let values = [3.0, 1.0, 2.0, 4.0, 0.0]; diff --git a/rust/crates/nfi-vector-core/src/kernels/native.rs b/rust/crates/nfi-vector-core/src/kernels/native.rs new file mode 100644 index 00000000..a177513e --- /dev/null +++ b/rust/crates/nfi-vector-core/src/kernels/native.rs @@ -0,0 +1,935 @@ +//! Bounded native kernels for causal array operations used by NFI indicators. + +use std::collections::VecDeque; + +use crate::float::{binary, canonicalize, BinaryFloatOp, CANONICAL_NAN_BITS}; +use crate::VectorCoreError; + +use super::moving::{sum_stream, MovingStream}; + +const MAX_PERIOD: usize = 100_000; +const HOUR_MS: i64 = 3_600_000; +const DAY_MS: i64 = 86_400_000; + +/// Bounded prefix-sum state for Chaikin money flow. +#[derive(Debug)] +pub struct ChaikinMoneyFlowStream { + period: usize, + prefix_mfv: f64, + prefix_volume: f64, + prefixes: VecDeque<(f64, f64)>, +} + +impl ChaikinMoneyFlowStream { + /// Creates a causal Chaikin money-flow stream. + /// + /// # Errors + /// + /// Returns an error when `timeperiod` is zero or exceeds the bounded kernel limit. + pub fn new(timeperiod: usize) -> Result { + validate_period(timeperiod, "Chaikin money flow")?; + Ok(Self { + period: timeperiod, + prefix_mfv: 0.0, + prefix_volume: 0.0, + prefixes: VecDeque::with_capacity(timeperiod), + }) + } + + /// Processes one ordered chunk and returns only that chunk's CMF rows. + /// + /// # Errors + /// + /// Returns an error when the four input columns have different lengths. + #[allow(clippy::float_cmp)] // NumPy's exact zero mask is part of the source contract. + pub fn execute( + &mut self, + high: &[f64], + low: &[f64], + close: &[f64], + volume: &[f64], + ) -> Result, VectorCoreError> { + validate_equal_lengths("Chaikin money flow", &[high, low, close, volume])?; + let mut output = Vec::with_capacity(high.len()); + for (((&high, &low), &close), &volume) in high.iter().zip(low).zip(close).zip(volume) { + let range = binary(high, low, BinaryFloatOp::Subtract); + let multiplier = if range == 0.0 { + 0.0 + } else { + let close_minus_low = binary(close, low, BinaryFloatOp::Subtract); + let high_minus_close = binary(high, close, BinaryFloatOp::Subtract); + let numerator = binary(close_minus_low, high_minus_close, BinaryFloatOp::Subtract); + binary(numerator, range, BinaryFloatOp::Divide) + }; + let mfv = nan_to_num(binary(multiplier, volume, BinaryFloatOp::Multiply)); + let clean_volume = nan_to_num(volume); + self.prefix_mfv = binary(self.prefix_mfv, mfv, BinaryFloatOp::Add); + self.prefix_volume = binary(self.prefix_volume, clean_volume, BinaryFloatOp::Add); + self.prefixes + .push_back((self.prefix_mfv, self.prefix_volume)); + + let sums = match self.prefixes.len().cmp(&self.period) { + std::cmp::Ordering::Less => None, + std::cmp::Ordering::Equal => Some((self.prefix_mfv, self.prefix_volume)), + std::cmp::Ordering::Greater => { + let Some((old_mfv, old_volume)) = self.prefixes.pop_front() else { + return Err(VectorCoreError::InvalidState( + "Chaikin money-flow prefix state is empty".to_owned(), + )); + }; + Some(( + binary(self.prefix_mfv, old_mfv, BinaryFloatOp::Subtract), + binary(self.prefix_volume, old_volume, BinaryFloatOp::Subtract), + )) + } + }; + output.push(sums.map_or_else(canonical_nan, |(mfv_sum, volume_sum)| { + if volume_sum == 0.0 { + canonical_nan() + } else { + binary(mfv_sum, volume_sum, BinaryFloatOp::Divide) + } + })); + } + Ok(output) + } + + /// Returns the number of historical prefix rows retained by the stream. + #[must_use] + pub fn retained(&self) -> usize { + self.prefixes.len() + } +} + +/// Executes Chaikin money flow over one complete batch. +/// +/// # Errors +/// +/// Returns an error for an invalid period or unequal input lengths. +pub fn chaikin_money_flow( + high: &[f64], + low: &[f64], + close: &[f64], + volume: &[f64], + timeperiod: usize, +) -> Result, VectorCoreError> { + ChaikinMoneyFlowStream::new(timeperiod)?.execute(high, low, close, volume) +} + +/// Bounded state for the older X7 Chaikin helper, whose volume denominator +/// uses TA-Lib `SUM` instead of `NumPy`'s NaN-to-zero prefix sum. +#[derive(Debug)] +pub struct LegacyChaikinMoneyFlowStream { + period: usize, + prefix_mfv: f64, + mfv_prefixes: VecDeque, + volume_sum: MovingStream, +} + +impl LegacyChaikinMoneyFlowStream { + /// Creates the exact legacy Chaikin stream. + /// + /// # Errors + /// + /// Returns an error when `timeperiod` is outside the bounded SUM contract. + pub fn new(timeperiod: usize) -> Result { + Ok(Self { + period: timeperiod, + prefix_mfv: 0.0, + mfv_prefixes: VecDeque::with_capacity(timeperiod), + volume_sum: sum_stream(timeperiod)?, + }) + } + + /// Processes one ordered chunk with the legacy mixed NumPy/TA-Lib contract. + /// + /// # Errors + /// + /// Returns an error when the four input columns have different lengths. + #[allow(clippy::float_cmp)] // The source uses exact NumPy zero comparisons. + pub fn execute( + &mut self, + high: &[f64], + low: &[f64], + close: &[f64], + volume: &[f64], + ) -> Result, VectorCoreError> { + validate_equal_lengths("legacy Chaikin money flow", &[high, low, close, volume])?; + let volume_sums = self.volume_sum.execute(&[volume])?; + let volume_sums = volume_sums.first().ok_or_else(|| { + VectorCoreError::InvalidState("legacy Chaikin volume SUM returned no output".to_owned()) + })?; + let mut output = Vec::with_capacity(high.len()); + for (row, (((&high, &low), &close), &volume)) in + high.iter().zip(low).zip(close).zip(volume).enumerate() + { + let range = binary(high, low, BinaryFloatOp::Subtract); + let multiplier = if range == 0.0 { + 0.0 + } else { + let close_minus_low = binary(close, low, BinaryFloatOp::Subtract); + let high_minus_close = binary(high, close, BinaryFloatOp::Subtract); + let numerator = binary(close_minus_low, high_minus_close, BinaryFloatOp::Subtract); + binary(numerator, range, BinaryFloatOp::Divide) + }; + let mfv = nan_to_num(binary(multiplier, volume, BinaryFloatOp::Multiply)); + self.prefix_mfv = binary(self.prefix_mfv, mfv, BinaryFloatOp::Add); + self.mfv_prefixes.push_back(self.prefix_mfv); + let mfv_sum = match self.mfv_prefixes.len().cmp(&self.period) { + std::cmp::Ordering::Less => None, + std::cmp::Ordering::Equal => Some(self.prefix_mfv), + std::cmp::Ordering::Greater => { + let old = self.mfv_prefixes.pop_front().ok_or_else(|| { + VectorCoreError::InvalidState( + "legacy Chaikin prefix state is empty".to_owned(), + ) + })?; + Some(binary(self.prefix_mfv, old, BinaryFloatOp::Subtract)) + } + }; + output.push(mfv_sum.map_or_else(canonical_nan, |mfv_sum| { + let volume_sum = volume_sums[row]; + if volume_sum == 0.0 { + canonical_nan() + } else { + binary(mfv_sum, volume_sum, BinaryFloatOp::Divide) + } + })); + } + Ok(output) + } + + /// Number of retained rolling values across both numerator and denominator. + #[must_use] + pub fn retained(&self) -> usize { + self.mfv_prefixes + .len() + .saturating_add(self.volume_sum.retained()) + } +} + +/// Executes the legacy Chaikin helper over one complete batch. +/// +/// # Errors +/// +/// Returns an error for an invalid period or unequal input lengths. +pub fn legacy_chaikin_money_flow( + high: &[f64], + low: &[f64], + close: &[f64], + volume: &[f64], + timeperiod: usize, +) -> Result, VectorCoreError> { + LegacyChaikinMoneyFlowStream::new(timeperiod)?.execute(high, low, close, volume) +} + +/// One-row state for percentage change with an exact zero-denominator guard. +#[derive(Debug, Default)] +pub struct SafePercentChangeStream { + previous: Option, +} + +impl SafePercentChangeStream { + /// Creates an empty percentage-change stream. + #[must_use] + pub const fn new() -> Self { + Self { previous: None } + } + + /// Processes a chunk using subtract, divide, then multiply order. + #[must_use] + #[allow(clippy::float_cmp)] // NumPy `where=prev != 0` uses exact IEEE equality. + pub fn execute(&mut self, values: &[f64]) -> Vec { + values + .iter() + .map(|¤t| { + let result = self.previous.map_or_else(canonical_nan, |previous| { + if previous == 0.0 { + canonical_nan() + } else { + let difference = binary(current, previous, BinaryFloatOp::Subtract); + let ratio = binary(difference, previous, BinaryFloatOp::Divide); + binary(ratio, 100.0, BinaryFloatOp::Multiply) + } + }); + self.previous = Some(current); + result + }) + .collect() + } + + /// Returns the number of previous rows retained by the stream. + #[must_use] + pub fn retained(&self) -> usize { + usize::from(self.previous.is_some()) + } +} + +/// Executes safe consecutive-row percentage change over one batch. +#[must_use] +pub fn safe_percent_change(values: &[f64]) -> Vec { + SafePercentChangeStream::new().execute(values) +} + +/// One-row state for absolute consecutive differences. +#[derive(Debug, Default)] +pub struct AbsoluteDifferenceStream { + previous: Option, +} + +impl AbsoluteDifferenceStream { + /// Creates an empty absolute-difference stream. + #[must_use] + pub const fn new() -> Self { + Self { previous: None } + } + + /// Processes a chunk using exact subtract-then-absolute-value order. + #[must_use] + pub fn execute(&mut self, values: &[f64]) -> Vec { + values + .iter() + .map(|¤t| { + let result = self.previous.map_or_else(canonical_nan, |previous| { + canonicalize(binary(current, previous, BinaryFloatOp::Subtract).abs()) + }); + self.previous = Some(current); + result + }) + .collect() + } + + /// Returns the number of previous rows retained by the stream. + #[must_use] + pub fn retained(&self) -> usize { + usize::from(self.previous.is_some()) + } +} + +/// Executes absolute consecutive-row differences over one batch. +#[must_use] +pub fn absolute_difference(values: &[f64]) -> Vec { + AbsoluteDifferenceStream::new().execute(values) +} + +/// Two projected UTC-day opening-range columns. +#[derive(Clone, Debug, PartialEq)] +pub struct UtcOpeningRangeOutput { + /// Maximum high observed before the configured UTC cutoff. + pub high: Vec, + /// Minimum low observed before the configured UTC cutoff. + pub low: Vec, +} + +/// Bounded current-day state for a UTC opening range. +#[derive(Debug)] +pub struct UtcOpeningRangeStream { + cutoff_ms: i64, + current_day: Option, + range_high: Option, + range_low: Option, + previous_timestamp: Option, +} + +impl UtcOpeningRangeStream { + /// Creates a UTC-day opening range using the first `cutoff_hours` hours. + /// + /// # Errors + /// + /// Returns an error unless the cutoff is between 1 and 23 hours inclusive. + pub fn new(cutoff_hours: u8) -> Result { + if !(1..24).contains(&cutoff_hours) { + return Err(VectorCoreError::InvalidProgram( + "UTC opening-range cutoff must satisfy 1 <= hours < 24".to_owned(), + )); + } + Ok(Self { + cutoff_ms: i64::from(cutoff_hours) * HOUR_MS, + current_day: None, + range_high: None, + range_low: None, + previous_timestamp: None, + }) + } + + /// Projects the completed opening range over one ordered timestamp chunk. + /// + /// # Errors + /// + /// Returns an error for unequal input lengths or decreasing timestamps. + pub fn execute( + &mut self, + timestamps_ms: &[i64], + high: &[f64], + low: &[f64], + ) -> Result { + validate_equal_lengths("UTC opening range", &[high, low])?; + if timestamps_ms.len() != high.len() { + return Err(length_error("UTC opening range")); + } + validate_chronology(timestamps_ms, self.previous_timestamp, "UTC opening range")?; + + let mut output_high = Vec::with_capacity(high.len()); + let mut output_low = Vec::with_capacity(low.len()); + for ((×tamp, &high), &low) in timestamps_ms.iter().zip(high).zip(low) { + let day = timestamp.div_euclid(DAY_MS); + if self.current_day != Some(day) { + self.current_day = Some(day); + self.range_high = None; + self.range_low = None; + } + let within_day = timestamp.rem_euclid(DAY_MS); + if within_day < self.cutoff_ms { + update_max(&mut self.range_high, high); + update_min(&mut self.range_low, low); + output_high.push(canonical_nan()); + output_low.push(canonical_nan()); + } else { + output_high.push(self.range_high.unwrap_or_else(canonical_nan)); + output_low.push(self.range_low.unwrap_or_else(canonical_nan)); + } + } + self.previous_timestamp = timestamps_ms.last().copied().or(self.previous_timestamp); + Ok(UtcOpeningRangeOutput { + high: output_high, + low: output_low, + }) + } + + /// Returns the number of day aggregates retained by the stream. + #[must_use] + pub fn retained(&self) -> usize { + usize::from(self.current_day.is_some()) + } +} + +/// Executes a UTC-day opening-range projection over one complete batch. +/// +/// # Errors +/// +/// Returns an error for an invalid cutoff, unequal lengths, or decreasing timestamps. +pub fn utc_opening_range( + timestamps_ms: &[i64], + high: &[f64], + low: &[f64], + cutoff_hours: u8, +) -> Result { + UtcOpeningRangeStream::new(cutoff_hours)?.execute(timestamps_ms, high, low) +} + +/// Three projected columns for the preceding completed hourly inside bar. +#[derive(Clone, Debug, PartialEq)] +pub struct HourlyInsideBarOutput { + /// `1.0` when hour H-1 is strictly inside H-2, otherwise `0.0`. + pub ready: Vec, + /// High of the H-2 mother hour, or canonical NaN when unavailable. + pub mother_high: Vec, + /// Low of the H-2 mother hour, or canonical NaN when unavailable. + pub mother_low: Vec, +} + +#[derive(Clone, Copy, Debug)] +struct HourAggregate { + hour: i64, + high: Option, + low: Option, +} + +impl HourAggregate { + const fn new(hour: i64) -> Self { + Self { + hour, + high: None, + low: None, + } + } + + fn update(&mut self, high: f64, low: f64) { + update_max(&mut self.high, high); + update_min(&mut self.low, low); + } +} + +/// Bounded current-plus-two-prior-hour state for inside-bar projection. +#[derive(Debug, Default)] +pub struct HourlyInsideBarStream { + current: Option, + previous: Option, + mother: Option, + previous_timestamp: Option, +} + +impl HourlyInsideBarStream { + /// Creates an empty hourly inside-bar stream. + #[must_use] + pub const fn new() -> Self { + Self { + current: None, + previous: None, + mother: None, + previous_timestamp: None, + } + } + + /// Projects H-1 inside H-2 state over one ordered timestamp chunk. + /// + /// # Errors + /// + /// Returns an error for unequal input lengths or decreasing timestamps. + pub fn execute( + &mut self, + timestamps_ms: &[i64], + high: &[f64], + low: &[f64], + ) -> Result { + validate_equal_lengths("hourly inside bar", &[high, low])?; + if timestamps_ms.len() != high.len() { + return Err(length_error("hourly inside bar")); + } + validate_chronology(timestamps_ms, self.previous_timestamp, "hourly inside bar")?; + + let mut ready = Vec::with_capacity(high.len()); + let mut mother_high = Vec::with_capacity(high.len()); + let mut mother_low = Vec::with_capacity(low.len()); + for ((×tamp, &high), &low) in timestamps_ms.iter().zip(high).zip(low) { + let hour = timestamp.div_euclid(HOUR_MS); + if self.current.is_none_or(|current| current.hour != hour) { + self.mother = self.previous; + self.previous = self.current; + self.current = Some(HourAggregate::new(hour)); + } + + let previous = self + .previous + .filter(|aggregate| aggregate.hour == hour.saturating_sub(1)); + let mother = self + .mother + .filter(|aggregate| aggregate.hour == hour.saturating_sub(2)); + let complete = previous.zip(mother); + let is_inside = complete.is_some_and(|(previous, mother)| { + previous + .high + .zip(mother.high) + .is_some_and(|(inner, outer)| inner < outer) + && previous + .low + .zip(mother.low) + .is_some_and(|(inner, outer)| inner > outer) + }); + ready.push(if is_inside { 1.0 } else { 0.0 }); + mother_high.push( + complete + .and_then(|(_, aggregate)| aggregate.high) + .unwrap_or_else(canonical_nan), + ); + mother_low.push( + complete + .and_then(|(_, aggregate)| aggregate.low) + .unwrap_or_else(canonical_nan), + ); + let Some(current) = self.current.as_mut() else { + return Err(VectorCoreError::InvalidState( + "hourly inside-bar current aggregate is missing".to_owned(), + )); + }; + current.update(high, low); + } + self.previous_timestamp = timestamps_ms.last().copied().or(self.previous_timestamp); + Ok(HourlyInsideBarOutput { + ready, + mother_high, + mother_low, + }) + } + + /// Returns the number of hourly aggregates retained by the stream. + #[must_use] + pub fn retained(&self) -> usize { + [self.current, self.previous, self.mother] + .iter() + .flatten() + .count() + } +} + +/// Executes hourly inside-bar projection over one complete batch. +/// +/// # Errors +/// +/// Returns an error for unequal lengths or decreasing timestamps. +pub fn hourly_inside_bar( + timestamps_ms: &[i64], + high: &[f64], + low: &[f64], +) -> Result { + HourlyInsideBarStream::new().execute(timestamps_ms, high, low) +} + +fn validate_period(period: usize, name: &str) -> Result<(), VectorCoreError> { + if period == 0 || period > MAX_PERIOD { + return Err(VectorCoreError::InvalidProgram(format!( + "{name} period must satisfy 0 < period <= {MAX_PERIOD}" + ))); + } + Ok(()) +} + +fn validate_equal_lengths(name: &str, inputs: &[&[f64]]) -> Result<(), VectorCoreError> { + if inputs.first().is_some_and(|first| { + inputs + .iter() + .skip(1) + .any(|input| input.len() != first.len()) + }) { + return Err(length_error(name)); + } + Ok(()) +} + +fn validate_chronology( + timestamps_ms: &[i64], + previous: Option, + name: &str, +) -> Result<(), VectorCoreError> { + let mut prior = previous; + for ×tamp in timestamps_ms { + if prior.is_some_and(|prior| timestamp < prior) { + return Err(VectorCoreError::InvalidState(format!( + "{name} timestamps are not chronological" + ))); + } + prior = Some(timestamp); + } + Ok(()) +} + +fn length_error(name: &str) -> VectorCoreError { + VectorCoreError::InvalidState(format!("{name} input lengths differ")) +} + +fn nan_to_num(value: f64) -> f64 { + if value.is_nan() { + 0.0 + } else if value == f64::INFINITY { + f64::MAX + } else if value == f64::NEG_INFINITY { + f64::MIN + } else { + value + } +} + +fn canonical_nan() -> f64 { + f64::from_bits(CANONICAL_NAN_BITS) +} + +fn update_max(current: &mut Option, candidate: f64) { + if candidate.is_nan() { + return; + } + if current.is_none_or(|value| candidate > value) { + *current = Some(candidate); + } +} + +fn update_min(current: &mut Option, candidate: f64) { + if candidate.is_nan() { + return; + } + if current.is_none_or(|value| candidate < value) { + *current = Some(candidate); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_bits(actual: &[f64], expected: &[f64]) { + assert_eq!(actual.len(), expected.len()); + for (index, (actual, expected)) in actual.iter().zip(expected).enumerate() { + assert_eq!(actual.to_bits(), expected.to_bits(), "row {index}"); + } + } + + #[test] + fn chaikin_money_flow_matches_numpy_prefix_subtract_order() { + let high = [2.0, 3.0, 4.0, 5.0, 6.0, 7.0]; + let low = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]; + let close = [1.5, 2.0, 2.5, 4.5, 5.0, 5.5]; + let volume = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0]; + let actual = chaikin_money_flow(&high, &low, &close, &volume, 3).expect("valid CMF"); + let expected = [ + canonical_nan(), + canonical_nan(), + f64::from_bits(0xbfc5_5555_5555_5555), + f64::from_bits(0x3fac_71c7_1c71_c71c), + f64::from_bits(0x3fa5_5555_5555_5555), + f64::from_bits(0xbfb1_1111_1111_1111), + ]; + assert_bits(&actual, &expected); + } + + #[test] + fn legacy_chaikin_keeps_talib_volume_sum_nan_warmup() { + let high = [2.0; 4]; + let low = [0.0; 4]; + let close = [1.5; 4]; + let volume = [f64::NAN, 1.0, 1.0, 1.0]; + let expected = [canonical_nan(), canonical_nan(), 0.5, 0.5]; + let actual = + legacy_chaikin_money_flow(&high, &low, &close, &volume, 2).expect("valid legacy CMF"); + assert_bits(&actual, &expected); + + let modern = chaikin_money_flow(&high, &low, &close, &volume, 2).expect("valid modern CMF"); + assert_eq!(modern[1].to_bits(), 0.5_f64.to_bits()); + assert!(actual[1].is_nan()); + + let mut stream = LegacyChaikinMoneyFlowStream::new(2).expect("valid stream"); + let mut chunked = stream + .execute(&high[..1], &low[..1], &close[..1], &volume[..1]) + .expect("first chunk"); + chunked.extend( + stream + .execute(&high[1..], &low[1..], &close[1..], &volume[1..]) + .expect("second chunk"), + ); + assert_bits(&chunked, &expected); + assert!(stream.retained() <= 4); + } + + #[test] + fn chaikin_nan_to_num_matches_numpy_saturation() { + let high = [2.0; 6]; + let low = [0.0, 0.0, 0.0, 0.0, 0.0, 2.0]; + let close = [f64::INFINITY, f64::NEG_INFINITY, f64::NAN, 1.5, 1.5, 2.0]; + let volume = [ + 1.0, + 1.0, + 1.0, + f64::INFINITY, + f64::NEG_INFINITY, + f64::INFINITY, + ]; + let actual = chaikin_money_flow(&high, &low, &close, &volume, 1).expect("valid CMF"); + assert_bits(&actual, &[f64::MAX, f64::MIN, 0.0, 1.0, 1.0, 0.0]); + } + + #[test] + fn scalar_native_streams_are_chunk_exact_and_bounded() { + let values = [ + 1.0, + 2.0, + 0.0, + -0.0, + 4.0, + f64::NAN, + 8.0, + f64::INFINITY, + f64::INFINITY, + ]; + let expected_pct = safe_percent_change(&values); + let expected_diff = absolute_difference(&values); + assert_bits( + &expected_pct, + &[ + canonical_nan(), + 100.0, + -100.0, + canonical_nan(), + canonical_nan(), + canonical_nan(), + canonical_nan(), + f64::INFINITY, + canonical_nan(), + ], + ); + assert_bits( + &expected_diff, + &[ + canonical_nan(), + 1.0, + 2.0, + 0.0, + 4.0, + canonical_nan(), + canonical_nan(), + f64::INFINITY, + canonical_nan(), + ], + ); + let mut pct = SafePercentChangeStream::new(); + let mut diff = AbsoluteDifferenceStream::new(); + let mut actual_pct = Vec::new(); + let mut actual_diff = Vec::new(); + for chunk in [&values[..1], &values[1..4], &values[4..7], &values[7..]] { + actual_pct.extend(pct.execute(chunk)); + actual_diff.extend(diff.execute(chunk)); + assert!(pct.retained() <= 1); + assert!(diff.retained() <= 1); + } + assert_bits(&actual_pct, &expected_pct); + assert_bits(&actual_diff, &expected_diff); + assert_eq!(expected_pct[0].to_bits(), CANONICAL_NAN_BITS); + assert_eq!(expected_diff[0].to_bits(), CANONICAL_NAN_BITS); + } + + #[test] + fn chaikin_stream_is_chunk_exact_and_period_bounded() { + let high = [2.0, 3.0, 4.0, 5.0, 6.0, 7.0]; + let low = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]; + let close = [1.5, 2.0, 2.5, 4.5, 5.0, 5.5]; + let volume = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0]; + let expected = chaikin_money_flow(&high, &low, &close, &volume, 3).expect("batch"); + let mut stream = ChaikinMoneyFlowStream::new(3).expect("stream"); + let mut actual = Vec::new(); + for (start, end) in [(0, 1), (1, 3), (3, 4), (4, 6)] { + actual.extend( + stream + .execute( + &high[start..end], + &low[start..end], + &close[start..end], + &volume[start..end], + ) + .expect("chunk"), + ); + assert!(stream.retained() <= 3); + } + assert_bits(&actual, &expected); + } + + #[test] + fn utc_opening_range_is_chunk_exact_across_days() { + let day = 1_700_000_000_000_i64.div_euclid(DAY_MS) * DAY_MS; + let timestamps = [ + day, + day + HOUR_MS, + day + 3 * HOUR_MS, + day + 4 * HOUR_MS, + day + 6 * HOUR_MS, + day + DAY_MS, + day + DAY_MS + 2 * HOUR_MS, + day + DAY_MS + 4 * HOUR_MS, + ]; + let high = [10.0, 12.0, 11.0, 99.0, 100.0, f64::NAN, 9.0, 50.0]; + let low = [5.0, 4.0, 3.0, -10.0, -20.0, 6.0, f64::NAN, 0.0]; + let expected = utc_opening_range(×tamps, &high, &low, 4).expect("batch"); + assert_bits( + &expected.high, + &[ + canonical_nan(), + canonical_nan(), + canonical_nan(), + 12.0, + 12.0, + canonical_nan(), + canonical_nan(), + 9.0, + ], + ); + assert_bits( + &expected.low, + &[ + canonical_nan(), + canonical_nan(), + canonical_nan(), + 3.0, + 3.0, + canonical_nan(), + canonical_nan(), + 6.0, + ], + ); + + let mut stream = UtcOpeningRangeStream::new(4).expect("stream"); + let mut actual_high = Vec::new(); + let mut actual_low = Vec::new(); + for (start, end) in [(0, 2), (2, 5), (5, 6), (6, 8)] { + let output = stream + .execute(×tamps[start..end], &high[start..end], &low[start..end]) + .expect("chunk"); + actual_high.extend(output.high); + actual_low.extend(output.low); + assert!(stream.retained() <= 1); + } + assert_bits(&actual_high, &expected.high); + assert_bits(&actual_low, &expected.low); + } + + #[test] + fn hourly_inside_bar_projects_only_contiguous_completed_hours() { + let hour = 1_700_000_000_000_i64.div_euclid(HOUR_MS) * HOUR_MS; + let timestamps = [ + hour, + hour + 30 * 60_000, + hour + HOUR_MS, + hour + HOUR_MS + 30 * 60_000, + hour + 2 * HOUR_MS, + hour + 2 * HOUR_MS + 30 * 60_000, + hour + 4 * HOUR_MS, + ]; + let high = [10.0, 12.0, 11.0, 10.0, 20.0, 21.0, 30.0]; + let low = [0.0, -2.0, -1.0, 0.0, 5.0, 4.0, 3.0]; + let expected = hourly_inside_bar(×tamps, &high, &low).expect("batch"); + assert_bits(&expected.ready, &[0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0]); + assert_bits( + &expected.mother_high, + &[ + canonical_nan(), + canonical_nan(), + canonical_nan(), + canonical_nan(), + 12.0, + 12.0, + canonical_nan(), + ], + ); + assert_bits( + &expected.mother_low, + &[ + canonical_nan(), + canonical_nan(), + canonical_nan(), + canonical_nan(), + -2.0, + -2.0, + canonical_nan(), + ], + ); + + let mut stream = HourlyInsideBarStream::new(); + let mut actual = HourlyInsideBarOutput { + ready: Vec::new(), + mother_high: Vec::new(), + mother_low: Vec::new(), + }; + for (start, end) in [(0, 1), (1, 3), (3, 6), (6, 7)] { + let chunk = stream + .execute(×tamps[start..end], &high[start..end], &low[start..end]) + .expect("chunk"); + actual.ready.extend(chunk.ready); + actual.mother_high.extend(chunk.mother_high); + actual.mother_low.extend(chunk.mother_low); + assert!(stream.retained() <= 3); + } + assert_bits(&actual.ready, &expected.ready); + assert_bits(&actual.mother_high, &expected.mother_high); + assert_bits(&actual.mother_low, &expected.mother_low); + } + + #[test] + fn native_kernels_fail_closed_without_mutating_order_state() { + assert!(ChaikinMoneyFlowStream::new(0).is_err()); + assert!(UtcOpeningRangeStream::new(0).is_err()); + assert!(chaikin_money_flow(&[1.0], &[], &[1.0], &[1.0], 1).is_err()); + + let mut opening = UtcOpeningRangeStream::new(4).expect("stream"); + assert!(opening.execute(&[2, 1], &[1.0, 1.0], &[1.0, 1.0]).is_err()); + assert_eq!(opening.retained(), 0); + + let mut inside = HourlyInsideBarStream::new(); + assert!(inside.execute(&[2, 1], &[1.0, 1.0], &[1.0, 1.0]).is_err()); + assert_eq!(inside.retained(), 0); + } +} diff --git a/rust/crates/nfi-vector-core/src/kernels/oscillator.rs b/rust/crates/nfi-vector-core/src/kernels/oscillator.rs index 1f131fb0..090e5a4c 100644 --- a/rust/crates/nfi-vector-core/src/kernels/oscillator.rs +++ b/rust/crates/nfi-vector-core/src/kernels/oscillator.rs @@ -293,6 +293,39 @@ pub(super) fn stochf( Ok((fast_k, fast_d)) } +/// TA-Lib STOCH with exact SMA slow-K and slow-D moving averages (type 0). +pub(super) fn stoch( + high: &[f64], + low: &[f64], + close: &[f64], + fast_k_period: usize, + k_smoothing: usize, + d_smoothing: usize, +) -> Result<(Vec, Vec), VectorCoreError> { + let len = validate_price_inputs("STOCH", high, low, close)?; + validate_period("STOCH", "fast_k_period", fast_k_period, 1)?; + validate_period("STOCH", "slow_k_period", k_smoothing, 1)?; + validate_period("STOCH", "slow_d_period", d_smoothing, 1)?; + + let first_slow_k = (fast_k_period - 1) + (k_smoothing - 1); + let lookback = first_slow_k + (d_smoothing - 1); + let mut slow_k = unavailable(len); + let mut slow_d = unavailable(len); + if len <= lookback { + return Ok((slow_k, slow_d)); + } + + let (_, all_slow_k) = stochf(high, low, close, fast_k_period, k_smoothing)?; + let valid_slow_k = &all_slow_k[first_slow_k..]; + let smoothed_slow_d = simple_moving_average(valid_slow_k, d_smoothing); + for (offset, value) in smoothed_slow_d.iter().enumerate() { + let output_index = lookback + offset; + slow_k[output_index] = valid_slow_k[d_smoothing - 1 + offset]; + slow_d[output_index] = *value; + } + Ok((slow_k, slow_d)) +} + /// TA-Lib ULTOSC after its period ordering and true-range priming steps. pub(super) fn ultosc( high: &[f64], @@ -367,6 +400,7 @@ pub(super) enum OscillatorStream { Cci(CciStream), Mfi(MfiStream), Obv(ObvStream), + Stoch(StochStream), Stochf(StochfStream), Ultosc(UltOscStream), } @@ -379,6 +413,10 @@ impl OscillatorStream { Self::Cci(state) => Ok(vec![state.execute(price_inputs("CCI", inputs)?)?]), Self::Mfi(state) => Ok(vec![state.execute(price_volume_inputs("MFI", inputs)?)?]), Self::Obv(state) => Ok(vec![state.execute(obv_inputs(inputs)?)?]), + Self::Stoch(state) => { + let (slow_k, slow_d) = state.execute(price_inputs("STOCH", inputs)?)?; + Ok(vec![slow_k, slow_d]) + } Self::Stochf(state) => { let (fast_k, fast_d) = state.execute(price_inputs("STOCHF", inputs)?)?; Ok(vec![fast_k, fast_d]) @@ -395,6 +433,7 @@ impl OscillatorStream { Self::Cci(state) => state.retained(), Self::Mfi(state) => state.retained(), Self::Obv(state) => state.retained(), + Self::Stoch(state) => state.retained(), Self::Stochf(state) => state.retained(), Self::Ultosc(state) => state.retained(), } @@ -423,6 +462,22 @@ pub(super) fn stream( OscillatorStream::Mfi(MfiStream::new(period)) } "OBV" => OscillatorStream::Obv(ObvStream::default()), + "STOCH" => { + if argument_period(arguments, "slowk_matype", 0)? != 0 + || argument_period(arguments, "slowd_matype", 0)? != 0 + { + return Err(VectorCoreError::InvalidState( + "TA-Lib STOCH only supports exact SMA slow-K and slow-D".to_owned(), + )); + } + let fast_k_period = argument_period(arguments, "fastk_period", 5)?; + let k_smoothing = argument_period(arguments, "slowk_period", 3)?; + let d_smoothing = argument_period(arguments, "slowd_period", 3)?; + validate_period("STOCH", "fast_k_period", fast_k_period, 1)?; + validate_period("STOCH", "slow_k_period", k_smoothing, 1)?; + validate_period("STOCH", "slow_d_period", d_smoothing, 1)?; + OscillatorStream::Stoch(StochStream::new(fast_k_period, k_smoothing, d_smoothing)) + } "STOCHF" => { if argument_period(arguments, "fastd_matype", 0)? != 0 { return Err(VectorCoreError::InvalidState( @@ -449,6 +504,63 @@ pub(super) fn stream( Ok(Some(state)) } +#[derive(Debug)] +pub(super) struct StochStream { + slow_k_state: StochfStream, + slow_d_period: usize, + slow_d_values: VecDeque, + slow_d_total: f64, + seen: usize, + first_slow_k: usize, +} + +impl StochStream { + fn new(fast_k_period: usize, k_smoothing: usize, d_smoothing: usize) -> Self { + Self { + slow_k_state: StochfStream::new(fast_k_period, k_smoothing), + slow_d_period: d_smoothing, + slow_d_values: VecDeque::new(), + slow_d_total: 0.0, + seen: 0, + first_slow_k: (fast_k_period - 1) + (k_smoothing - 1), + } + } + + fn execute( + &mut self, + inputs: PriceInputs<'_>, + ) -> Result<(Vec, Vec), VectorCoreError> { + let (raw, next_slow_k) = self.slow_k_state.execute(inputs)?; + drop(raw); + let mut slow_k = unavailable(next_slow_k.len()); + let mut slow_d = unavailable(next_slow_k.len()); + for (index, value) in next_slow_k.into_iter().enumerate() { + let global_index = self.seen + index; + if global_index < self.first_slow_k { + continue; + } + self.slow_d_total += value; + self.slow_d_values.push_back(value); + if self.slow_d_values.len() == self.slow_d_period { + let average = self.slow_d_total / period_as_f64(self.slow_d_period); + let oldest = self + .slow_d_values + .pop_front() + .expect("full STOCH slow-D state has a trailing value"); + self.slow_d_total -= oldest; + slow_k[index] = value; + slow_d[index] = average; + } + } + self.seen = self.seen.saturating_add(slow_k.len()); + Ok((slow_k, slow_d)) + } + + fn retained(&self) -> usize { + self.slow_k_state.retained() + self.slow_d_values.len() + 2 + } +} + #[derive(Debug)] pub(super) struct RsiStream { period: usize, @@ -1207,6 +1319,7 @@ mod tests { cci_matches_pinned_bits(); mfi_matches_pinned_bits(); obv_matches_pinned_bits(); + stoch_matches_pinned_bits(); stochf_matches_pinned_bits(); ultosc_matches_pinned_bits(); } @@ -1353,6 +1466,52 @@ mod tests { ); } + fn stoch_matches_pinned_bits() { + let (slow_k, slow_d) = stoch(&HIGH, &LOW, &CLOSE, 3, 2, 2).expect("valid STOCH"); + assert_bits( + &slow_k, + &[ + 0x7ff8_0000_0000_0000, + 0x7ff8_0000_0000_0000, + 0x7ff8_0000_0000_0000, + 0x7ff8_0000_0000_0000, + 0x404e_3555_5555_5556, + 0x404d_c30c_30c3_0c31, + 0x4053_0c30_c30c_30c4, + 0x4049_0000_0000_0002, + 0x404d_c30c_30c3_0c32, + 0x4054_6fbe_fbef_befc, + 0x404b_c71c_71c7_1c72, + 0x404d_c30c_30c3_0c30, + 0x4054_6fbe_fbef_befc, + 0x404b_c71c_71c7_1c72, + 0x404d_c30c_30c3_0c30, + 0x4054_6fbe_fbef_befc, + ], + ); + assert_bits( + &slow_d, + &[ + 0x7ff8_0000_0000_0000, + 0x7ff8_0000_0000_0000, + 0x7ff8_0000_0000_0000, + 0x7ff8_0000_0000_0000, + 0x4050_ed55_5555_5556, + 0x404d_fc30_c30c_30c4, + 0x4050_f6db_6db6_db6e, + 0x404f_8c30_c30c_30c5, + 0x404b_6186_1861_861a, + 0x4051_a8a2_8a28_a28a, + 0x4051_29a6_9a69_a69a, + 0x404c_c514_5145_1450, + 0x4051_a8a2_8a28_a28a, + 0x4051_29a6_9a69_a69a, + 0x404c_c514_5145_1450, + 0x4051_a8a2_8a28_a28a, + ], + ); + } + fn ultosc_matches_pinned_bits() { assert_bits( &ultosc(&HIGH, &LOW, &CLOSE, 2, 3, 4).expect("valid ULTOSC"), @@ -1458,6 +1617,21 @@ mod tests { &price_chunks, ), ); + assert_columns_bits( + &{ + let (slow_k, slow_d) = stoch(&HIGH, &LOW, &CLOSE, 3, 2, 2).expect("batch STOCH"); + vec![slow_k, slow_d] + }, + &collect_stream( + "STOCH", + &arguments(&[ + ("fastk_period", 3), + ("slowk_period", 2), + ("slowd_period", 2), + ]), + &price_chunks, + ), + ); let volume_first = [&HIGH[..2], &LOW[..2], &CLOSE[..2], &VOLUME[..2]]; let volume_second = [&HIGH[2..7], &LOW[2..7], &CLOSE[2..7], &VOLUME[2..7]]; @@ -1505,6 +1679,16 @@ mod tests { ("CCI", arguments(&[("timeperiod", 3)]), price.to_vec(), 3), ("MFI", arguments(&[("timeperiod", 3)]), volume.to_vec(), 9), ("OBV", Map::new(), vec![&CLOSE[..], &VOLUME[..]], 2), + ( + "STOCH", + arguments(&[ + ("fastk_period", 3), + ("slowk_period", 2), + ("slowd_period", 2), + ]), + price.to_vec(), + 14, + ), ( "STOCHF", arguments(&[("fastk_period", 3), ("fastd_period", 2)]), diff --git a/rust/crates/nfi-vector-core/src/mutation/engine/evaluator.rs b/rust/crates/nfi-vector-core/src/mutation/engine/evaluator.rs index f9648fba..30e9ca0d 100644 --- a/rust/crates/nfi-vector-core/src/mutation/engine/evaluator.rs +++ b/rust/crates/nfi-vector-core/src/mutation/engine/evaluator.rs @@ -3,13 +3,13 @@ use std::collections::BTreeMap; use serde_json::Value as JsonValue; use super::value::{ - append_text, as_column, assign_masked, bool_at, cast_bool_at, cast_f64_at, cast_i64_at, - mask_values, numeric_at, scalar_text, shift_column, single_input, string_parameter, text_at, - text_at_optional, three_inputs, two_inputs, value, RuntimeValue, + append_text, as_column, assign_masked, mask_values, scalar_text, shift_column, single_input, + string_parameter, three_inputs, two_inputs, value, RuntimeValue, }; use super::{MutationEngine, MutationFrame}; use crate::column::OwnedColumn; use crate::float::{binary, compare, BinaryFloatOp, FloatComparison}; +use crate::mutation::validation::numpy_array_arguments_are_supported; use crate::mutation::MutationProgram; use crate::program::ProgramNode; use crate::VectorCoreError; @@ -31,7 +31,21 @@ impl<'program> MutationEngine<'program> { /// /// Returns a source-located type, mask, opcode, or column error without /// returning a partial frame. - pub fn execute(&self, mut frame: MutationFrame) -> Result { + pub fn execute(&self, frame: MutationFrame) -> Result { + self.execute_with_metadata(frame, &BTreeMap::new()) + } + + /// Execute entry and exit phases with explicit immutable strategy metadata. + /// + /// # Errors + /// + /// Returns a source-located error when a compiled metadata read has no + /// exact string value. Metadata is never inferred from dataframe content. + pub fn execute_with_metadata( + &self, + mut frame: MutationFrame, + metadata: &BTreeMap, + ) -> Result { for column in &self.program.required_input_columns { if !frame.columns.contains_key(column) { return Err(VectorCoreError::MissingColumn(column.clone())); @@ -49,7 +63,7 @@ impl<'program> MutationEngine<'program> { OwnedColumn::text(vec![Some(String::new()); frame.rows]), )?; } - self.execute_function(&entrypoint.function, &mut frame)?; + self.execute_function(&entrypoint.function, &mut frame, metadata)?; } Ok(frame) } @@ -58,17 +72,22 @@ impl<'program> MutationEngine<'program> { &self, function_id: &str, frame: &mut MutationFrame, + metadata: &BTreeMap, ) -> Result<(), VectorCoreError> { let function = self.program.function(function_id).ok_or_else(|| { VectorCoreError::InvalidProgram(format!("mutation function is missing: {function_id}")) })?; let mut values = BTreeMap::new(); + let mut remaining_uses = self.function_input_uses(function)?; for node_id in &function.node_ids { let node = self.program.node(node_id).ok_or_else(|| { VectorCoreError::InvalidProgram(format!("mutation node is missing: {node_id}")) })?; - let value = self.execute_node(node, &values, frame)?; + let value = self.execute_node(node, &values, frame, metadata)?; values.insert(node.id.clone(), value); + if node.op != "return" { + release_consumed_inputs(&node.inputs, &mut remaining_uses, &mut values)?; + } } if !values.contains_key(&function.return_node) { return Err(self.error( @@ -81,11 +100,41 @@ impl<'program> MutationEngine<'program> { Ok(()) } + fn function_input_uses( + &self, + function: &crate::program::ProgramFunction, + ) -> Result, VectorCoreError> { + let mut uses = function + .node_ids + .iter() + .map(|id| (id.clone(), 0_usize)) + .collect::>(); + for node_id in &function.node_ids { + let node = self.program.node(node_id).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!("mutation node is missing: {node_id}")) + })?; + for input in &node.inputs { + let count = uses.get_mut(input).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!( + "mutation node {node_id} references non-local input {input}" + )) + })?; + *count = count.checked_add(1).ok_or_else(|| { + VectorCoreError::InvalidProgram( + "mutation input use count is too large".to_owned(), + ) + })?; + } + } + Ok(uses) + } + fn execute_node( &self, node: &ProgramNode, values: &BTreeMap, frame: &mut MutationFrame, + metadata: &BTreeMap, ) -> Result { match node.op.as_str() { "parameter" => Ok(match node.value_type.as_str() { @@ -94,6 +143,9 @@ impl<'program> MutationEngine<'program> { _ => RuntimeValue::Unbound, }), "literal" => self.literal(node), + "row-count" => self.row_count(node, values, frame.rows), + "metadata-read" => self.metadata_read(node, values, metadata), + "string-split-index" => self.string_split_index(node, values), "column-read" => { let column = string_parameter(node, "column")?; frame @@ -104,9 +156,12 @@ impl<'program> MutationEngine<'program> { } "binary" => self.binary(node, values, frame.rows), "compare" => self.compare(node, values, frame.rows), + "membership" => self.membership(node, values, frame.rows), + "masked-string-append" => self.masked_string_append(node, values, frame.rows), "logical" => self.logical(node, values, frame.rows), "unary" => self.unary(node, values, frame.rows), "select" => self.select(node, values, frame.rows), + "array-call" => self.array_call(node, values, frame.rows), "cast" => self.cast(node, values, frame.rows), "shift" => self.shift(node, values, frame.rows), "format-string" => self.format_string(node, values), @@ -120,7 +175,36 @@ impl<'program> MutationEngine<'program> { } } + fn metadata_read( + &self, + node: &ProgramNode, + values: &BTreeMap, + metadata: &BTreeMap, + ) -> Result { + let input = single_input(node)?; + if !matches!(value(values, input)?, RuntimeValue::Metadata) { + return Err(self.error(node, "metadata-read input is not metadata")); + } + let key = string_parameter(node, "key")?; + metadata + .get(key) + .cloned() + .map(RuntimeValue::Text) + .ok_or_else(|| self.error(node, format!("runtime metadata has no string key {key:?}"))) + } + fn literal(&self, node: &ProgramNode) -> Result { + if let Some(special) = node.parameters.get("special") { + if node.parameters.len() != 1 || node.value_type != "f64-scalar" { + return Err(self.error(node, "special literal contract is invalid")); + } + return Ok(RuntimeValue::Float(match special.as_str() { + Some("nan") => crate::float::canonicalize(f64::NAN), + Some("+infinity") => f64::INFINITY, + Some("-infinity") => f64::NEG_INFINITY, + _ => return Err(self.error(node, "special float literal is unsupported")), + })); + } let value = node .parameters .get("value") @@ -143,6 +227,330 @@ impl<'program> MutationEngine<'program> { }) } + fn row_count( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + let input = single_input(node)?; + if !matches!(value(values, input)?, RuntimeValue::DataFrame) { + return Err(self.error(node, "row-count input is not a dataframe")); + } + Ok(RuntimeValue::Integer( + i64::try_from(rows).map_err(|_| self.error(node, "row-count exceeds i64"))?, + )) + } + + fn string_split_index( + &self, + node: &ProgramNode, + values: &BTreeMap, + ) -> Result { + let input = single_input(node)?; + let RuntimeValue::Text(source) = value(values, input)? else { + return Err(self.error(node, "string split input is not scalar text")); + }; + let method = string_parameter(node, "method")?; + let separator = string_parameter(node, "separator")?; + if separator.is_empty() { + return Err(self.error(node, "string split separator is empty")); + } + let index = node + .parameters + .get("index") + .and_then(JsonValue::as_i64) + .ok_or_else(|| self.error(node, "string split index is not a signed integer"))?; + let parts = match method { + "partition" => source.find(separator).map_or_else( + || vec![source.clone(), String::new(), String::new()], + |position| { + let after = position + separator.len(); + vec![ + source[..position].to_owned(), + separator.to_owned(), + source[after..].to_owned(), + ] + }, + ), + "split" => source.split(separator).map(str::to_owned).collect(), + "rsplit" => { + let mut parts = source + .rsplit(separator) + .map(str::to_owned) + .collect::>(); + parts.reverse(); + parts + } + _ => return Err(self.error(node, "string split method is unsupported")), + }; + let length = i64::try_from(parts.len()) + .map_err(|_| self.error(node, "string split result length exceeds i64"))?; + let resolved = if index < 0 { + length.checked_add(index) + } else { + Some(index) + } + .and_then(|index| usize::try_from(index).ok()) + .filter(|index| *index < parts.len()) + .ok_or_else(|| self.error(node, "string split result index is outside its result"))?; + Ok(RuntimeValue::Text(parts[resolved].clone())) + } + + fn membership( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + let input = single_input(node)?; + let collection = node + .parameters + .get("values") + .and_then(JsonValue::as_array) + .ok_or_else(|| self.error(node, "membership values are not an array"))?; + let negated = node + .parameters + .get("negated") + .and_then(JsonValue::as_bool) + .ok_or_else(|| self.error(node, "membership negation is not Boolean"))?; + let contains = |item: Option<&str>| { + let member = collection.iter().any(|candidate| match item { + Some(item) => candidate.as_str() == Some(item), + None => candidate.is_null(), + }); + member != negated + }; + match value(values, input)? { + RuntimeValue::Text(item) if node.value_type == "bool-scalar" => { + Ok(RuntimeValue::Bool(contains(Some(item)))) + } + RuntimeValue::Column(column) + if node.value_type == "bool-column" + && column.as_view().value_type() == crate::column::ValueType::Text + && column.len() == rows => + { + let view = column.as_view(); + Ok(RuntimeValue::Column(OwnedColumn::boolean( + (0..rows) + .map(|row| Some(contains(view.text_at(row)))) + .collect(), + ))) + } + _ => Err(self.error(node, "membership input or output type is invalid")), + } + } + + fn masked_string_append( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + let [target, mask, suffix] = three_inputs(node)?; + let RuntimeValue::Column(target) = value(values, target)? else { + return Err(self.error(node, "masked string append target is not a column")); + }; + let mask_value = value(values, mask)?; + let RuntimeValue::Text(suffix) = value(values, suffix)? else { + return Err(self.error(node, "masked string append suffix is not scalar text")); + }; + if node.value_type != "string-column" + || target.as_view().value_type() != crate::column::ValueType::Text + || target.len() != rows + { + return Err(self.error(node, "masked string append target contract is invalid")); + } + let target = target.as_view(); + Ok(RuntimeValue::Column(OwnedColumn::text( + (0..rows) + .map(|row| match mask_value.bool_at(mask, row)? { + Some(true) => target.text_at(row).map_or_else( + || Err(self.error(node, "masked string append selected a null target")), + |prefix| Ok(Some(format!("{prefix}{suffix}"))), + ), + Some(false) | None => Ok(target.text_at(row).map(str::to_owned)), + }) + .collect::, VectorCoreError>>()?, + ))) + } + + fn array_call( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + let name = string_parameter(node, "name")?; + let arguments = node + .parameters + .get("arguments") + .and_then(JsonValue::as_object); + if node.parameters.len() != 3 + || string_parameter(node, "family")? != "numpy" + || arguments.is_none_or(|arguments| { + !numpy_array_arguments_are_supported(name, &node.value_type, arguments) + }) + { + return Err(self.error(node, "unsupported array-call contract")); + } + match name { + "full" => self.array_full(node, values, rows), + "full_like" => self.array_full_like(node, values, rows), + "divide" => self.array_divide_where(node, values, rows), + "isnan" => self.array_isnan(node, values, rows), + _ => Err(self.error(node, "unsupported array-call contract")), + } + } + + fn array_isnan( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + let input = single_input(node)?; + let input_value = value(values, input)?; + if node.value_type != "bool-column" { + return Err(self.error(node, "numpy isnan requires bool-column output")); + } + Ok(RuntimeValue::Column(OwnedColumn::boolean( + (0..rows) + .map(|row| { + input_value + .numeric_at(input, row) + .map(|value| value.map(f64::is_nan)) + }) + .collect::, _>>()?, + ))) + } + + fn array_full( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + if node.inputs.len() != 2 { + return Err(self.error(node, "numpy full requires two inputs")); + } + let size = match value(values, &node.inputs[0])? { + RuntimeValue::Integer(value) => usize::try_from(*value) + .map_err(|_| self.error(node, "array-call size is outside usize"))?, + _ => return Err(self.error(node, "array-call size is not an integer")), + }; + if size != rows { + return Err(self.error(node, "array-call size differs from dataframe rows")); + } + let fill = value(values, &node.inputs[1])?; + match (node.value_type.as_str(), fill) { + ("bool-column", RuntimeValue::Bool(value)) => { + Ok(RuntimeValue::Column(OwnedColumn::boolean(vec![ + Some(*value); + rows + ]))) + } + ("int-column", RuntimeValue::Integer(value)) => { + Ok(RuntimeValue::Column(OwnedColumn::i64(vec![ + Some(*value); + rows + ]))) + } + ("f64-column", RuntimeValue::Float(value)) => { + Ok(RuntimeValue::Column(OwnedColumn::f64(vec![ + Some(*value); + rows + ]))) + } + ("string-column", RuntimeValue::Text(value)) => { + Ok(RuntimeValue::Column(OwnedColumn::text(vec![ + Some( + value.clone() + ); + rows + ]))) + } + _ => Err(self.error(node, "array-call fill value differs from its output type")), + } + } + + fn array_full_like( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + let [template, fill] = two_inputs(node)?; + let RuntimeValue::Column(template) = value(values, template)? else { + return Err(self.error(node, "numpy full_like template is not a column")); + }; + if node.value_type != "f64-column" + || template.as_view().value_type() != crate::column::ValueType::F64 + || template.len() != rows + { + return Err(self.error(node, "numpy full_like requires a Float64 template")); + } + let fill = match value(values, fill)? { + RuntimeValue::Integer(value) => super::value::i64_as_f64(*value), + RuntimeValue::Float(value) => *value, + _ => return Err(self.error(node, "numpy full_like fill is not numeric scalar")), + }; + Ok(RuntimeValue::Column(OwnedColumn::f64(vec![ + Some(fill); + rows + ]))) + } + + fn array_divide_where( + &self, + node: &ProgramNode, + values: &BTreeMap, + rows: usize, + ) -> Result { + let [numerator, denominator, out, where_mask] = match node.inputs.as_slice() { + [numerator, denominator, out, where_mask] => [ + numerator.as_str(), + denominator.as_str(), + out.as_str(), + where_mask.as_str(), + ], + _ => return Err(self.error(node, "numpy divide requires x1, x2, out, and where")), + }; + let numerator_value = value(values, numerator)?; + let denominator_value = value(values, denominator)?; + let out_value = value(values, out)?; + let where_value = value(values, where_mask)?; + let RuntimeValue::Column(out_column) = out_value else { + return Err(self.error(node, "numpy divide out is not a column")); + }; + if node.value_type != "f64-column" + || out_column.as_view().value_type() != crate::column::ValueType::F64 + || out_column.len() != rows + { + return Err(self.error(node, "numpy divide out is not a Float64 result buffer")); + } + + let out_view = out_column.as_view(); + Ok(RuntimeValue::Column(OwnedColumn::f64( + (0..rows) + .map(|row| match where_value.bool_at(where_mask, row)? { + Some(true) => { + match ( + numerator_value.numeric_at(numerator, row)?, + denominator_value.numeric_at(denominator, row)?, + ) { + (Some(left), Some(right)) => { + Ok(Some(binary(left, right, BinaryFloatOp::Divide))) + } + _ => Ok(None), + } + } + Some(false) | None => Ok(out_view.f64_at(row)), + }) + .collect::, VectorCoreError>>()?, + ))) + } + fn binary( &self, node: &ProgramNode, @@ -150,6 +558,8 @@ impl<'program> MutationEngine<'program> { rows: usize, ) -> Result { let [left, right] = two_inputs(node)?; + let left_value = value(values, left)?; + let right_value = value(values, right)?; if node.value_type.starts_with("string-") { if string_parameter(node, "operator")? != "add" { return Err(self.error(node, "string binary operation is not concatenation")); @@ -157,16 +567,16 @@ impl<'program> MutationEngine<'program> { if node.value_type == "string-scalar" { return Ok(RuntimeValue::Text(format!( "{}{}", - text_at(values, left, 0)?, - text_at(values, right, 0)? + left_value.text_at(left, 0)?, + right_value.text_at(right, 0)? ))); } return Ok(RuntimeValue::Column(OwnedColumn::text( (0..rows) .map(|row| { match ( - text_at_optional(values, left, row)?, - text_at_optional(values, right, row)?, + left_value.text_at_optional(left, row)?, + right_value.text_at_optional(right, row)?, ) { (Some(left), Some(right)) => Ok(Some(format!("{left}{right}"))), _ => Ok(None), @@ -189,8 +599,8 @@ impl<'program> MutationEngine<'program> { .map(|row| { Ok( match ( - numeric_at(values, left, row)?, - numeric_at(values, right, row)?, + left_value.numeric_at(left, row)?, + right_value.numeric_at(right, row)?, ) { (Some(left), Some(right)) => Some(binary(left, right, operation)), _ => None, @@ -201,7 +611,10 @@ impl<'program> MutationEngine<'program> { ))); } Ok( - match (numeric_at(values, left, 0)?, numeric_at(values, right, 0)?) { + match ( + left_value.numeric_at(left, 0)?, + right_value.numeric_at(right, 0)?, + ) { (Some(left), Some(right)) => RuntimeValue::Float(binary(left, right, operation)), _ => RuntimeValue::Null, }, @@ -215,6 +628,8 @@ impl<'program> MutationEngine<'program> { rows: usize, ) -> Result { let [left, right] = two_inputs(node)?; + let left_value = value(values, left)?; + let right_value = value(values, right)?; let operation = match string_parameter(node, "operator")? { "equal" => FloatComparison::Equal, "not-equal" => FloatComparison::NotEqual, @@ -227,8 +642,8 @@ impl<'program> MutationEngine<'program> { let evaluate = |row| -> Result, VectorCoreError> { Ok( match ( - numeric_at(values, left, row)?, - numeric_at(values, right, row)?, + left_value.numeric_at(left, row)?, + right_value.numeric_at(right, row)?, ) { (Some(left), Some(right)) => Some(compare(left, right, operation)), _ => None, @@ -254,10 +669,23 @@ impl<'program> MutationEngine<'program> { if node.inputs.is_empty() || !matches!(operation, "and" | "or") { return Err(self.error(node, "logical operation has invalid arity or operator")); } + let inputs = node + .inputs + .iter() + .map(|input| Ok((input.as_str(), value(values, input)?))) + .collect::, VectorCoreError>>()?; let evaluate = |row| -> Result, VectorCoreError> { - let mut result = bool_at(values, &node.inputs[0], row)?; - for input in node.inputs.iter().skip(1) { - let right = bool_at(values, input, row)?; + let mut result = inputs[0].1.bool_at(inputs[0].0, row)?; + for (input, input_value) in inputs.iter().skip(1) { + // Pandas' nullable Boolean algebra has two absorbing values: + // false for AND and true for OR. Once reached, later operands + // cannot change either the value or its nullability. + if (operation == "and" && result == Some(false)) + || (operation == "or" && result == Some(true)) + { + break; + } + let right = input_value.bool_at(input, row)?; result = nullable_logical(result, right, operation); } Ok(result) @@ -278,6 +706,7 @@ impl<'program> MutationEngine<'program> { rows: usize, ) -> Result { let input = single_input(node)?; + let input_value = value(values, input)?; let operation = string_parameter(node, "operator")?; if node.value_type.starts_with("bool-") { if !matches!(operation, "not" | "invert") { @@ -286,11 +715,16 @@ impl<'program> MutationEngine<'program> { if node.value_type == "bool-column" { return Ok(RuntimeValue::Column(OwnedColumn::boolean( (0..rows) - .map(|row| bool_at(values, input, row).map(|value| value.map(|item| !item))) + .map(|row| { + input_value + .bool_at(input, row) + .map(|value| value.map(|item| !item)) + }) .collect::, _>>()?, ))); } - return Ok(bool_at(values, input, 0)? + return Ok(input_value + .bool_at(input, 0)? .map_or(RuntimeValue::Null, |value| RuntimeValue::Bool(!value))); } if !matches!(operation, "negate" | "positive") { @@ -300,15 +734,19 @@ impl<'program> MutationEngine<'program> { if node.value_type.ends_with("-column") { return Ok(RuntimeValue::Column(OwnedColumn::f64( (0..rows) - .map(|row| numeric_at(values, input, row).map(|value| value.map(apply))) + .map(|row| { + input_value + .numeric_at(input, row) + .map(|value| value.map(apply)) + }) .collect::, _>>()?, ))); } - Ok( - numeric_at(values, input, 0)?.map_or(RuntimeValue::Null, |value| { + Ok(input_value + .numeric_at(input, 0)? + .map_or(RuntimeValue::Null, |value| { RuntimeValue::Float(apply(value)) - }), - ) + })) } fn select( @@ -318,14 +756,17 @@ impl<'program> MutationEngine<'program> { rows: usize, ) -> Result { let [condition, truthy, falsey] = three_inputs(node)?; + let condition_value = value(values, condition)?; + let truthy_value = value(values, truthy)?; + let falsey_value = value(values, falsey)?; if node.value_type != "f64-column" { return Err(self.error(node, "select currently requires f64-column output")); } Ok(RuntimeValue::Column(OwnedColumn::f64( (0..rows) - .map(|row| match bool_at(values, condition, row)? { - Some(true) => numeric_at(values, truthy, row), - Some(false) => numeric_at(values, falsey, row), + .map(|row| match condition_value.bool_at(condition, row)? { + Some(true) => truthy_value.numeric_at(truthy, row), + Some(false) => falsey_value.numeric_at(falsey, row), None => Ok(None), }) .collect::, _>>()?, @@ -339,24 +780,25 @@ impl<'program> MutationEngine<'program> { rows: usize, ) -> Result { let input = single_input(node)?; + let input_value = value(values, input)?; let target = string_parameter(node, "target")?; if target == "array" { - return Ok(RuntimeValue::Alias(input.to_owned())); + return Ok(input_value.clone()); } match target { "int" => Ok(RuntimeValue::Column(OwnedColumn::i64( (0..rows) - .map(|row| cast_i64_at(values, input, row)) + .map(|row| input_value.cast_i64_at(input, row)) .collect::, _>>()?, ))), "float" => Ok(RuntimeValue::Column(OwnedColumn::f64( (0..rows) - .map(|row| cast_f64_at(values, input, row)) + .map(|row| input_value.cast_f64_at(input, row)) .collect::, _>>()?, ))), "bool" => Ok(RuntimeValue::Column(OwnedColumn::boolean( (0..rows) - .map(|row| cast_bool_at(values, input, row)) + .map(|row| input_value.cast_bool_at(input, row)) .collect::, _>>()?, ))), _ => Err(self.error(node, format!("unsupported cast target {target}"))), @@ -473,6 +915,29 @@ impl<'program> MutationEngine<'program> { } } +fn release_consumed_inputs( + inputs: &[String], + remaining_uses: &mut BTreeMap, + values: &mut BTreeMap, +) -> Result<(), VectorCoreError> { + for input in inputs { + let remaining = remaining_uses.get_mut(input).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!( + "mutation input {input} has no liveness record" + )) + })?; + *remaining = remaining.checked_sub(1).ok_or_else(|| { + VectorCoreError::InvalidProgram(format!( + "mutation input {input} was consumed too many times" + )) + })?; + if *remaining == 0 { + values.remove(input); + } + } + Ok(()) +} + fn nullable_logical(left: Option, right: Option, operation: &str) -> Option { match operation { "and" if left == Some(false) || right == Some(false) => Some(false), diff --git a/rust/crates/nfi-vector-core/src/mutation/engine/value.rs b/rust/crates/nfi-vector-core/src/mutation/engine/value.rs index 7672bf93..b4636494 100644 --- a/rust/crates/nfi-vector-core/src/mutation/engine/value.rs +++ b/rust/crates/nfi-vector-core/src/mutation/engine/value.rs @@ -20,6 +20,125 @@ pub(super) enum RuntimeValue { Alias(String), } +impl RuntimeValue { + pub(super) fn numeric_at( + &self, + node: &str, + row: usize, + ) -> Result, VectorCoreError> { + match self { + Self::Null => Ok(None), + Self::Bool(value) => Ok(Some(f64::from(u8::from(*value)))), + Self::Integer(value) => Ok(Some(i64_as_f64(*value))), + Self::Float(value) => Ok(Some(*value)), + Self::Column(column) => match column.as_view().value_type() { + ValueType::Bool => Ok(column + .as_view() + .bool_at(row) + .map(|value| f64::from(u8::from(value)))), + ValueType::F64 => Ok(column.as_view().f64_at(row)), + ValueType::I64 => Ok(column.as_view().i64_at(row).map(i64_as_f64)), + _ => Err(type_error(node, "numeric")), + }, + _ => Err(type_error(node, "numeric")), + } + } + + pub(super) fn bool_at(&self, node: &str, row: usize) -> Result, VectorCoreError> { + match self { + Self::Null => Ok(None), + Self::Bool(value) => Ok(Some(*value)), + Self::Column(column) if column.as_view().value_type() == ValueType::Bool => { + Ok(column.as_view().bool_at(row)) + } + _ => Err(type_error(node, "Boolean")), + } + } + + pub(super) fn cast_i64_at( + &self, + node: &str, + row: usize, + ) -> Result, VectorCoreError> { + match self { + Self::Null => Ok(None), + Self::Bool(value) => Ok(Some(i64::from(*value))), + Self::Integer(value) => Ok(Some(*value)), + Self::Float(value) => Ok(Some(f64_as_i64(*value)?)), + Self::Column(column) => match column.as_view().value_type() { + ValueType::Bool => Ok(column.as_view().bool_at(row).map(i64::from)), + ValueType::I64 => Ok(column.as_view().i64_at(row)), + ValueType::F64 => column.as_view().f64_at(row).map(f64_as_i64).transpose(), + _ => Err(type_error(node, "castable to integer")), + }, + _ => Err(type_error(node, "castable to integer")), + } + } + + pub(super) fn cast_f64_at( + &self, + node: &str, + row: usize, + ) -> Result, VectorCoreError> { + match self { + Self::Null => Ok(None), + Self::Bool(value) => Ok(Some(f64::from(u8::from(*value)))), + Self::Integer(value) => Ok(Some(i64_as_f64(*value))), + Self::Float(value) => Ok(Some(*value)), + Self::Column(column) => match column.as_view().value_type() { + ValueType::Bool => Ok(column + .as_view() + .bool_at(row) + .map(|value| f64::from(u8::from(value)))), + ValueType::I64 => Ok(column.as_view().i64_at(row).map(i64_as_f64)), + ValueType::F64 => Ok(column.as_view().f64_at(row)), + _ => Err(type_error(node, "castable to float")), + }, + _ => Err(type_error(node, "castable to float")), + } + } + + pub(super) fn cast_bool_at( + &self, + node: &str, + row: usize, + ) -> Result, VectorCoreError> { + match self { + Self::Null => Ok(None), + Self::Bool(value) => Ok(Some(*value)), + Self::Integer(value) => Ok(Some(*value != 0)), + Self::Float(value) => Ok(Some(*value != 0.0)), + Self::Column(column) => match column.as_view().value_type() { + ValueType::Bool => Ok(column.as_view().bool_at(row)), + ValueType::I64 => Ok(column.as_view().i64_at(row).map(|value| value != 0)), + ValueType::F64 => Ok(column.as_view().f64_at(row).map(|value| value != 0.0)), + _ => Err(type_error(node, "castable to Boolean")), + }, + _ => Err(type_error(node, "castable to Boolean")), + } + } + + pub(super) fn text_at(&self, node: &str, row: usize) -> Result<&str, VectorCoreError> { + self.text_at_optional(node, row)? + .ok_or_else(|| type_error(node, "non-null text")) + } + + pub(super) fn text_at_optional( + &self, + node: &str, + row: usize, + ) -> Result, VectorCoreError> { + match self { + Self::Null => Ok(None), + Self::Text(value) => Ok(Some(value)), + Self::Column(column) if column.as_view().value_type() == ValueType::Text => { + Ok(column.as_view().text_at(row)) + } + _ => Err(type_error(node, "text")), + } + } +} + pub(super) fn value<'a>( values: &'a BTreeMap, start: &str, @@ -44,125 +163,6 @@ pub(super) fn value<'a>( }) } -pub(super) fn numeric_at( - values: &BTreeMap, - node: &str, - row: usize, -) -> Result, VectorCoreError> { - match value(values, node)? { - RuntimeValue::Null => Ok(None), - RuntimeValue::Integer(value) => Ok(Some(i64_as_f64(*value))), - RuntimeValue::Float(value) => Ok(Some(*value)), - RuntimeValue::Column(column) => match column.as_view().value_type() { - ValueType::F64 => Ok(column.as_view().f64_at(row)), - ValueType::I64 => Ok(column.as_view().i64_at(row).map(i64_as_f64)), - _ => Err(type_error(node, "numeric")), - }, - _ => Err(type_error(node, "numeric")), - } -} - -pub(super) fn bool_at( - values: &BTreeMap, - node: &str, - row: usize, -) -> Result, VectorCoreError> { - match value(values, node)? { - RuntimeValue::Null => Ok(None), - RuntimeValue::Bool(value) => Ok(Some(*value)), - RuntimeValue::Column(column) if column.as_view().value_type() == ValueType::Bool => { - Ok(column.as_view().bool_at(row)) - } - _ => Err(type_error(node, "Boolean")), - } -} - -pub(super) fn cast_i64_at( - values: &BTreeMap, - node: &str, - row: usize, -) -> Result, VectorCoreError> { - match value(values, node)? { - RuntimeValue::Null => Ok(None), - RuntimeValue::Bool(value) => Ok(Some(i64::from(*value))), - RuntimeValue::Integer(value) => Ok(Some(*value)), - RuntimeValue::Float(value) => Ok(Some(f64_as_i64(*value)?)), - RuntimeValue::Column(column) => match column.as_view().value_type() { - ValueType::Bool => Ok(column.as_view().bool_at(row).map(i64::from)), - ValueType::I64 => Ok(column.as_view().i64_at(row)), - ValueType::F64 => column.as_view().f64_at(row).map(f64_as_i64).transpose(), - _ => Err(type_error(node, "castable to integer")), - }, - _ => Err(type_error(node, "castable to integer")), - } -} - -pub(super) fn cast_f64_at( - values: &BTreeMap, - node: &str, - row: usize, -) -> Result, VectorCoreError> { - match value(values, node)? { - RuntimeValue::Null => Ok(None), - RuntimeValue::Bool(value) => Ok(Some(f64::from(u8::from(*value)))), - RuntimeValue::Integer(value) => Ok(Some(i64_as_f64(*value))), - RuntimeValue::Float(value) => Ok(Some(*value)), - RuntimeValue::Column(column) => match column.as_view().value_type() { - ValueType::Bool => Ok(column - .as_view() - .bool_at(row) - .map(|value| f64::from(u8::from(value)))), - ValueType::I64 => Ok(column.as_view().i64_at(row).map(i64_as_f64)), - ValueType::F64 => Ok(column.as_view().f64_at(row)), - _ => Err(type_error(node, "castable to float")), - }, - _ => Err(type_error(node, "castable to float")), - } -} - -pub(super) fn cast_bool_at( - values: &BTreeMap, - node: &str, - row: usize, -) -> Result, VectorCoreError> { - match value(values, node)? { - RuntimeValue::Null => Ok(None), - RuntimeValue::Bool(value) => Ok(Some(*value)), - RuntimeValue::Integer(value) => Ok(Some(*value != 0)), - RuntimeValue::Float(value) => Ok(Some(*value != 0.0)), - RuntimeValue::Column(column) => match column.as_view().value_type() { - ValueType::Bool => Ok(column.as_view().bool_at(row)), - ValueType::I64 => Ok(column.as_view().i64_at(row).map(|value| value != 0)), - ValueType::F64 => Ok(column.as_view().f64_at(row).map(|value| value != 0.0)), - _ => Err(type_error(node, "castable to Boolean")), - }, - _ => Err(type_error(node, "castable to Boolean")), - } -} - -pub(super) fn text_at<'a>( - values: &'a BTreeMap, - node: &str, - row: usize, -) -> Result<&'a str, VectorCoreError> { - text_at_optional(values, node, row)?.ok_or_else(|| type_error(node, "non-null text")) -} - -pub(super) fn text_at_optional<'a>( - values: &'a BTreeMap, - node: &str, - row: usize, -) -> Result, VectorCoreError> { - match value(values, node)? { - RuntimeValue::Null => Ok(None), - RuntimeValue::Text(value) => Ok(Some(value)), - RuntimeValue::Column(column) if column.as_view().value_type() == ValueType::Text => { - Ok(column.as_view().text_at(row)) - } - _ => Err(type_error(node, "text")), - } -} - pub(super) fn scalar_text(value: &RuntimeValue) -> Result { match value { RuntimeValue::Bool(value) => Ok(if *value { "True" } else { "False" }.to_owned()), @@ -479,3 +479,53 @@ pub(super) fn f64_as_i64(value: f64) -> Result { } Ok(value as i64) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn python_boolean_values_are_numeric_zero_and_one() { + let values = BTreeMap::from([ + ("scalar".to_owned(), RuntimeValue::Bool(true)), + ( + "column".to_owned(), + RuntimeValue::Column(OwnedColumn::boolean(vec![Some(false), None, Some(true)])), + ), + ]); + + let scalar = value(&values, "scalar").unwrap(); + let column = value(&values, "column").unwrap(); + assert_eq!(scalar.numeric_at("scalar", 0).unwrap(), Some(1.0)); + assert_eq!(column.numeric_at("column", 0).unwrap(), Some(0.0)); + assert_eq!(column.numeric_at("column", 1).unwrap(), None); + assert_eq!(column.numeric_at("column", 2).unwrap(), Some(1.0)); + } + + #[test] + fn direct_numeric_access_preserves_arrow_null_nan_and_signed_zero() { + let column = RuntimeValue::Column(OwnedColumn::f64(vec![ + None, + Some(f64::from_bits(0xfff8_1234_5678_9abc)), + Some(-0.0), + Some(0.0), + ])); + + assert_eq!(column.numeric_at("column", 0).unwrap(), None); + assert_eq!( + column.numeric_at("column", 1).unwrap().unwrap().to_bits(), + crate::float::CANONICAL_NAN_BITS + ); + assert_eq!( + column.numeric_at("column", 2).unwrap().unwrap().to_bits(), + (-0.0_f64).to_bits() + ); + assert_eq!( + column.numeric_at("column", 3).unwrap().unwrap().to_bits(), + 0.0_f64.to_bits() + ); + assert_eq!(column.cast_f64_at("column", 0).unwrap(), None); + assert_eq!(column.cast_bool_at("column", 1).unwrap(), Some(true)); + assert_eq!(column.cast_bool_at("column", 2).unwrap(), Some(false)); + } +} diff --git a/rust/crates/nfi-vector-core/src/mutation/equivalence.rs b/rust/crates/nfi-vector-core/src/mutation/equivalence.rs new file mode 100644 index 00000000..2137c17f --- /dev/null +++ b/rust/crates/nfi-vector-core/src/mutation/equivalence.rs @@ -0,0 +1,232 @@ +//! Structural proof that Tag contains the exact Signal decision program. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::model::{MutationProgram, SIGNAL_PROGRAM_VERSION, TAG_PROGRAM_VERSION}; +use crate::program::{ProgramFunction, ProgramNode, SourceLocation}; +use crate::VectorCoreError; + +const TAG_COLUMNS: [&str; 2] = ["enter_tag", "exit_tag"]; + +#[derive(Debug, PartialEq)] +struct DecisionProjection { + functions: Vec, + nodes: Vec, + source_locations: Vec, +} + +/// Prove that removing Tag-only writes leaves the exact Signal DAG. +/// +/// Node identifiers and source-order offsets are canonicalized after removing +/// tag formatting/appends. Every retained dependency edge, opcode, parameter, +/// lookback, function boundary, and source span must then match exactly. +/// +/// # Errors +/// +/// Returns fail-closed when the programs have different identities, inputs, +/// entrypoints, or projected decision graphs. +pub fn prove_signal_tag_decision_equivalence( + signal: &MutationProgram, + tag: &MutationProgram, +) -> Result<(), VectorCoreError> { + if signal.schema_version != SIGNAL_PROGRAM_VERSION + || tag.schema_version != TAG_PROGRAM_VERSION + || signal.compile_context != tag.compile_context + || signal.entrypoints != tag.entrypoints + || signal.required_input_columns != tag.required_input_columns + { + return Err(invalid("Signal and Tag decision identities differ")); + } + if project(signal)? != project(tag)? { + return Err(invalid( + "Tag decision projection is not structurally exact to Signal", + )); + } + Ok(()) +} + +fn project(program: &MutationProgram) -> Result { + let removed = program + .nodes + .iter() + .filter(|node| is_tag_only(node)) + .map(|node| node.id.as_str()) + .collect::>(); + let nodes_by_id = program + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let live = live_decision_nodes(program, &removed, &nodes_by_id)?; + let retained = program + .nodes + .iter() + .filter(|node| live.contains(node.id.as_str())) + .collect::>(); + let canonical_ids = retained + .iter() + .enumerate() + .map(|(index, node)| (node.id.as_str(), format!("n{}", index + 1))) + .collect::>(); + let mut source_orders = BTreeMap::::new(); + let mut nodes = Vec::with_capacity(retained.len()); + let mut source_locations = Vec::with_capacity(retained.len()); + for original in retained { + let mut node = original.clone(); + node.id.clone_from(&canonical_ids[original.id.as_str()]); + node.inputs = original + .inputs + .iter() + .map(|input| resolve_input(input, &removed, &canonical_ids, &nodes_by_id)) + .collect::, _>>()?; + let source_order = source_orders.entry(node.function.clone()).or_default(); + node.source_order = *source_order; + *source_order = source_order + .checked_add(1) + .ok_or_else(|| invalid("projected mutation source order is too large"))?; + let mut location = program + .source_map + .get(&original.id) + .cloned() + .ok_or_else(|| invalid(format!("missing source location for {}", original.id)))?; + location.path.clear(); + nodes.push(node); + source_locations.push(location); + } + let functions = program + .functions + .iter() + .map(|function| project_function(function, &removed, &live, &canonical_ids, &nodes_by_id)) + .collect::, _>>()?; + Ok(DecisionProjection { + functions, + nodes, + source_locations, + }) +} + +/// Find the observable decision graph after bypassing Tag-only dataframe writes. +/// +/// Tag source may repeat a Signal condition solely to assign a string literal. +/// Removing only the write would leave that condition as an unreachable orphan +/// and make equivalent programs appear different. Start from every function's +/// return and ABI parameters, then walk dependencies backwards so the proof +/// compares exactly the executable decision graph. +fn live_decision_nodes<'a>( + program: &'a MutationProgram, + removed: &BTreeSet<&'a str>, + nodes_by_id: &BTreeMap<&'a str, &'a ProgramNode>, +) -> Result, VectorCoreError> { + let mut pending = Vec::<&str>::new(); + for function in &program.functions { + pending.push(function.return_node.as_str()); + pending.extend( + function + .parameters + .iter() + .map(|parameter| parameter.node.as_str()), + ); + } + let mut live = BTreeSet::new(); + while let Some(candidate) = pending.pop() { + let resolved = resolve_original_input(candidate, removed, nodes_by_id)?; + if !live.insert(resolved) { + continue; + } + let node = nodes_by_id + .get(resolved) + .ok_or_else(|| invalid(format!("projected node {resolved} is missing")))?; + pending.extend(node.inputs.iter().map(String::as_str)); + } + Ok(live) +} + +fn project_function( + function: &ProgramFunction, + removed: &BTreeSet<&str>, + live: &BTreeSet<&str>, + canonical_ids: &BTreeMap<&str, String>, + nodes_by_id: &BTreeMap<&str, &ProgramNode>, +) -> Result { + let mut projected = function.clone(); + projected.node_ids = function + .node_ids + .iter() + .filter(|id| live.contains(id.as_str())) + .map(|id| canonical_ids[id.as_str()].clone()) + .collect(); + projected.return_node = + resolve_input(&function.return_node, removed, canonical_ids, nodes_by_id)?; + for parameter in &mut projected.parameters { + parameter.node = canonical_ids + .get(parameter.node.as_str()) + .cloned() + .ok_or_else(|| invalid("Tag-only projection removed a function parameter"))?; + } + Ok(projected) +} + +fn resolve_original_input<'a>( + input: &'a str, + removed: &BTreeSet<&str>, + nodes_by_id: &BTreeMap<&'a str, &'a ProgramNode>, +) -> Result<&'a str, VectorCoreError> { + let mut current = input; + for _ in 0..=nodes_by_id.len() { + if !removed.contains(current) { + return Ok(current); + } + let node = nodes_by_id + .get(current) + .ok_or_else(|| invalid(format!("projected input {current} is missing")))?; + if node.op != "frame-write" || !writes_tag(node) { + return Err(invalid(format!( + "retained decision depends on Tag-only node {current}" + ))); + } + current = node + .inputs + .first() + .map(String::as_str) + .ok_or_else(|| invalid("Tag frame-write has no dataframe predecessor"))?; + } + Err(invalid(format!( + "cannot resolve projected mutation input {input}" + ))) +} + +fn resolve_input( + input: &str, + removed: &BTreeSet<&str>, + canonical_ids: &BTreeMap<&str, String>, + nodes_by_id: &BTreeMap<&str, &ProgramNode>, +) -> Result { + let resolved = resolve_original_input(input, removed, nodes_by_id)?; + canonical_ids + .get(resolved) + .cloned() + .ok_or_else(|| invalid(format!("projected input {resolved} is not live"))) +} + +fn is_tag_only(node: &ProgramNode) -> bool { + matches!(node.op.as_str(), "format-string" | "masked-string-append") || writes_tag(node) +} + +fn writes_tag(node: &ProgramNode) -> bool { + node.op == "frame-write" + && node + .parameters + .get("columns") + .and_then(serde_json::Value::as_array) + .is_some_and(|columns| { + columns.iter().any(|column| { + column + .as_str() + .is_some_and(|name| TAG_COLUMNS.contains(&name)) + }) + }) +} + +fn invalid(message: impl Into) -> VectorCoreError { + VectorCoreError::InvalidProgram(message.into()) +} diff --git a/rust/crates/nfi-vector-core/src/mutation/mod.rs b/rust/crates/nfi-vector-core/src/mutation/mod.rs index 4d40e433..e0f77e0c 100644 --- a/rust/crates/nfi-vector-core/src/mutation/mod.rs +++ b/rust/crates/nfi-vector-core/src/mutation/mod.rs @@ -1,6 +1,7 @@ //! Exact source-ordered execution for `signal-program-v1` and `tag-program-v1`. mod engine; +mod equivalence; mod model; mod validation; @@ -8,4 +9,5 @@ mod validation; mod tests; pub use engine::{materialize_execution_signals, ExecutionSignals, MutationEngine, MutationFrame}; +pub use equivalence::prove_signal_tag_decision_equivalence; pub use model::{MutationEntrypoint, MutationProgram}; diff --git a/rust/crates/nfi-vector-core/src/mutation/tests.rs b/rust/crates/nfi-vector-core/src/mutation/tests.rs index ca4973ba..a27de3bb 100644 --- a/rust/crates/nfi-vector-core/src/mutation/tests.rs +++ b/rust/crates/nfi-vector-core/src/mutation/tests.rs @@ -6,7 +6,7 @@ use arrow2::array::Array; use arrow2::chunk::Chunk; use arrow2::datatypes::{DataType, Field, Schema}; use serde::Deserialize; -use serde_json::Value; +use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use super::*; @@ -14,6 +14,7 @@ use crate::batch::BatchView; use crate::column::{OwnedColumn, ValueType}; use crate::engine::VectorEngine; use crate::program::IndicatorProgram; +use crate::VectorCoreError; const FIXTURE: &str = include_str!("../../../../../benchmarks/reference/vector-shadow/freqtrade-2026.5.1.json"); @@ -24,6 +25,79 @@ const SIGNAL_PROGRAM: &str = const TAG_PROGRAM: &str = include_str!("../../../../../benchmarks/reference/vector-shadow/tag-program.json"); +#[test] +fn tag_decision_projection_is_exact_and_rejects_semantic_drift() { + let signal = MutationProgram::from_json(SIGNAL_PROGRAM).expect("signal program valid"); + let mut tag = signal.clone(); + tag.schema_version = super::model::TAG_PROGRAM_VERSION.to_owned(); + + prove_signal_tag_decision_equivalence(&signal, &tag) + .expect("Tag-only projection leaves the exact Signal DAG"); + + // A Tag assignment may repeat a Signal predicate solely to choose its + // string. Once that Tag write is removed, its predicate branch is not + // observable and must not make otherwise exact decision programs differ. + let mut orphan = tag + .nodes + .iter() + .find(|node| node.op == "literal") + .expect("reference program has a literal") + .clone(); + orphan.id = format!("n{}", tag.nodes.len() + 1); + orphan.source_order = tag + .nodes + .iter() + .filter(|node| node.function == orphan.function) + .map(|node| node.source_order) + .max() + .expect("function contains nodes") + + 1; + tag.source_map.insert( + orphan.id.clone(), + tag.source_map + .get(&tag.nodes[0].id) + .expect("reference source location") + .clone(), + ); + tag.functions + .iter_mut() + .find(|function| function.id == orphan.function) + .expect("literal function exists") + .node_ids + .push(orphan.id.clone()); + tag.nodes.push(orphan); + + prove_signal_tag_decision_equivalence(&signal, &tag) + .expect("unreachable Tag predicate branch is not a Signal decision"); + + let decision = tag + .nodes + .iter_mut() + .find(|node| node.op == "literal") + .expect("reference Tag program has a decision literal"); + decision + .parameters + .insert("semantic-drift".to_owned(), json!(true)); + assert!(prove_signal_tag_decision_equivalence(&signal, &tag).is_err()); +} + +#[test] +#[ignore = "explicit external compiler-artifact diagnostic"] +fn external_signal_tag_decision_projection_is_exact() { + let signal_path = std::env::var("NFI_SIGNAL_PROGRAM").expect("Signal artifact path"); + let tag_path = std::env::var("NFI_TAG_PROGRAM").expect("Tag artifact path"); + let signal = MutationProgram::from_json( + &std::fs::read_to_string(signal_path).expect("read Signal artifact"), + ) + .expect("external Signal program valid"); + let tag = + MutationProgram::from_json(&std::fs::read_to_string(tag_path).expect("read Tag artifact")) + .expect("external Tag program valid"); + + prove_signal_tag_decision_equivalence(&signal, &tag) + .expect("external Tag decision projection exact to Signal"); +} + #[derive(Debug, Deserialize)] struct Fixture { fingerprint: String, @@ -249,6 +323,651 @@ fn execution_enablement_is_exact_numeric_one_after_the_source_shift() { assert!(materialize_execution_signals(&invalid, 1, 0).is_err()); } +#[test] +fn numpy_divide_where_preserves_out_and_ieee_null_semantics() { + let program = numpy_divide_program(); + let source = MutationFrame::new(BTreeMap::from([ + ( + "denominator".to_owned(), + OwnedColumn::f64(vec![ + Some(3.0), + Some(0.0), + Some(-0.0), + Some(0.0), + Some(2.0), + Some(f64::NAN), + None, + Some(2.0), + Some(3.0), + ]), + ), + ( + "mask".to_owned(), + OwnedColumn::boolean(vec![ + Some(true), + Some(true), + Some(true), + Some(true), + Some(true), + Some(true), + Some(true), + Some(false), + None, + ]), + ), + ( + "numerator".to_owned(), + OwnedColumn::f64(vec![ + Some(6.0), + Some(1.0), + Some(1.0), + Some(0.0), + None, + Some(f64::NAN), + Some(1.0), + Some(8.0), + Some(9.0), + ]), + ), + ( + "template".to_owned(), + OwnedColumn::f64(vec![ + Some(100.0), + Some(101.0), + Some(102.0), + Some(103.0), + Some(104.0), + Some(105.0), + Some(106.0), + Some(-0.0), + None, + ]), + ), + ])) + .expect("source frame"); + + let actual = MutationEngine::new(&program) + .expect("numpy mutation program binds") + .execute(source) + .expect("numpy mutation program executes"); + let computed = [ + Some(2.0), + Some(f64::INFINITY), + Some(f64::NEG_INFINITY), + Some(f64::NAN), + None, + Some(f64::NAN), + None, + Some(f64::NAN), + Some(f64::NAN), + ]; + assert_column_exact( + actual.column("enter_long").expect("full_like output"), + &OwnedColumn::f64(computed.to_vec()), + ); + let mut preserved = computed[..7].to_vec(); + preserved.extend([Some(-0.0), None]); + assert_column_exact( + actual.column("enter_short").expect("supplied out output"), + &OwnedColumn::f64(preserved), + ); +} + +#[test] +fn numpy_full_accepts_compiler_object_dtype_for_string_columns() { + let encoded = numpy_string_full_program_json(&json!({"dtype":"object"})); + let program = MutationProgram::from_json(&encoded.to_string()) + .expect("compiler string full contract is valid"); + let source = MutationFrame::new(BTreeMap::from([( + "seed".to_owned(), + OwnedColumn::i64(vec![Some(1), None, Some(3)]), + )])) + .expect("source frame"); + + let actual = MutationEngine::new(&program) + .expect("string full program binds") + .execute(source) + .expect("string full program executes"); + + assert_column_exact( + actual.column("enter_tag").expect("full output"), + &OwnedColumn::text(vec![ + Some(String::new()), + Some(String::new()), + Some(String::new()), + ]), + ); +} + +#[test] +fn numpy_full_object_dtype_contract_fails_closed() { + for arguments in [ + json!({"dtype":"str"}), + json!({"dtype":"object","order":"C"}), + json!({"dtype":null}), + ] { + let encoded = numpy_string_full_program_json(&arguments); + let error = MutationProgram::from_json(&encoded.to_string()) + .expect_err("unsupported full keyword arguments must fail closed"); + assert!(error + .to_string() + .contains("array-call n4 contract is invalid")); + } + + let mut encoded = numpy_string_full_program_json(&json!({"dtype":"object"})); + encoded["nodes"][2]["value_type"] = json!("f64-scalar"); + encoded["nodes"][2]["parameters"] = json!({"value":0.0}); + encoded["nodes"][3]["value_type"] = json!("f64-column"); + let error = MutationProgram::from_json(&encoded.to_string()) + .expect_err("object dtype is restricted to string-column full"); + assert!(error + .to_string() + .contains("array-call n4 contract is invalid")); +} + +#[test] +fn numpy_isnan_returns_a_nullable_boolean_column() { + let mut encoded = numpy_divide_program_json(); + encoded["nodes"][9]["inputs"] = json!(["n8"]); + encoded["nodes"][9]["value_type"] = json!("bool-column"); + encoded["nodes"][9]["parameters"] = json!({"family":"numpy","name":"isnan","arguments":{}}); + encoded["fingerprint"] = Value::String( + crate::program::validation::canonical_fingerprint(&encoded) + .expect("numpy isnan test program has canonical identity"), + ); + let program = + MutationProgram::from_json(&encoded.to_string()).expect("numpy isnan program is valid"); + let source = MutationFrame::new(BTreeMap::from([ + ( + "numerator".to_owned(), + OwnedColumn::f64(vec![Some(f64::NAN), Some(1.0), None]), + ), + ( + "denominator".to_owned(), + OwnedColumn::f64(vec![Some(1.0), Some(0.0), Some(1.0)]), + ), + ( + "template".to_owned(), + OwnedColumn::f64(vec![Some(0.0), Some(0.0), Some(0.0)]), + ), + ( + "mask".to_owned(), + OwnedColumn::boolean(vec![Some(true), Some(true), Some(true)]), + ), + ])) + .expect("source frame"); + + let actual = MutationEngine::new(&program) + .expect("numpy isnan program binds") + .execute(source) + .expect("numpy isnan program executes"); + assert_column_exact( + actual.column("enter_short").expect("isnan output"), + &OwnedColumn::boolean(vec![Some(true), Some(false), None]), + ); +} + +#[test] +fn malformed_numpy_array_contracts_fail_closed() { + let mut program = numpy_divide_program_json(); + program["nodes"][7]["parameters"]["arguments"] = json!({"casting":"unsafe"}); + assert!(MutationProgram::from_json(&program.to_string()).is_err()); + + let mut program = numpy_divide_program_json(); + program["nodes"][5]["parameters"]["arguments"] = json!({"dtype":"object"}); + let error = MutationProgram::from_json(&program.to_string()) + .expect_err("full_like keyword arguments must fail closed"); + assert!(error + .to_string() + .contains("array-call n6 contract is invalid")); + + let mut program = numpy_divide_program_json(); + program["nodes"][6]["value_type"] = json!("f64-column"); + assert!(MutationProgram::from_json(&program.to_string()).is_err()); + + let mut program = numpy_divide_program_json(); + program["nodes"][4]["parameters"] = json!({"special":"not-a-number"}); + assert!(MutationProgram::from_json(&program.to_string()).is_err()); +} + +#[test] +fn generic_string_mutations_match_python_order_index_and_null_semantics() { + let cases = [ + ( + "rsplit", + "long_entry_condition_ 65 _enable", + "_", + -2, + json!([" 65 "]), + false, + true, + " 65 ", + ), + ( + "partition", + "BTC/USDT", + "/", + -1, + json!(["BTC"]), + true, + true, + "USDT", + ), + ("split", "a::b::", "::", -1, json!([""]), false, true, ""), + ]; + for (method, source, separator, index, values, negated, scalar_member, suffix) in cases { + let program = string_mutation_program(method, source, separator, index, &values, negated); + let input = string_mutation_frame(); + let actual = MutationEngine::new(&program) + .expect("string mutation program binds") + .execute(input) + .expect("string mutation program executes"); + assert_column_exact( + actual.column("enter_long").expect("scalar membership"), + &OwnedColumn::boolean(vec![Some(scalar_member); 4]), + ); + assert_column_exact( + actual.column("enter_short").expect("column membership"), + &OwnedColumn::boolean(vec![Some(true), Some(false), Some(true), Some(false)]), + ); + assert_column_exact( + actual.column("enter_tag").expect("masked append"), + &OwnedColumn::text(vec![ + Some(format!("alpha {suffix}")), + Some(" keep ".to_owned()), + None, + Some(suffix.to_owned()), + ]), + ); + } +} + +#[test] +fn mutation_metadata_reads_are_explicit_and_fail_closed_when_missing() { + let mut encoded = + string_mutation_program_json("partition", "unused", "/", 0, &json!(["BTC/USDT"]), false); + encoded["functions"][0]["parameters"] = json!([ + {"name":"dataframe","node":"n1","value_type":"dataframe"}, + {"name":"metadata","node":"n2","value_type":"metadata"} + ]); + encoded["nodes"][1] = json!({ + "id":"n2","function":"f1","source_order":1,"op":"parameter", + "value_type":"metadata","inputs":[],"parameters":{"name":"metadata"}, + "lookback":{"kind":"finite","candles":0,"expression":null,"causal":true} + }); + encoded["nodes"][2] = json!({ + "id":"n3","function":"f1","source_order":2,"op":"metadata-read", + "value_type":"string-scalar","inputs":["n2"],"parameters":{"key":"pair"}, + "lookback":{"kind":"finite","candles":0,"expression":null,"causal":true} + }); + encoded["nodes"][3]["parameters"] = json!({"values":["BTC/USDT"],"negated":false}); + encoded["opcodes"] = json!([ + "column-read", + "frame-write", + "masked-string-append", + "membership", + "metadata-read", + "parameter", + "return" + ]); + encoded["fingerprint"] = Value::String( + crate::program::validation::canonical_fingerprint(&encoded) + .expect("metadata mutation program has canonical identity"), + ); + let program = MutationProgram::from_json(&encoded.to_string()) + .expect("metadata mutation program is valid"); + let engine = MutationEngine::new(&program).expect("metadata mutation engine binds"); + let actual = engine + .execute_with_metadata( + string_mutation_frame(), + &BTreeMap::from([("pair".to_owned(), "BTC/USDT".to_owned())]), + ) + .expect("explicit metadata executes"); + assert_eq!( + actual + .column("enter_long") + .expect("metadata-gated output") + .as_view() + .bool_at(0), + Some(true) + ); + + let error = engine + .execute(string_mutation_frame()) + .expect_err("metadata must never be inferred"); + assert!(matches!( + error, + VectorCoreError::Execution { node, message } + if node == "n3" + && message.contains("strategy.py:1:0") + && message.contains("runtime metadata has no string key \"pair\"") + )); + + encoded["nodes"][2]["parameters"] = json!({"key":""}); + encoded["fingerprint"] = Value::String( + crate::program::validation::canonical_fingerprint(&encoded) + .expect("invalid metadata program still has serializable identity"), + ); + let error = MutationProgram::from_json(&encoded.to_string()) + .expect_err("empty metadata keys must fail validation"); + assert!(error + .to_string() + .contains("metadata-read n3 contract is invalid")); +} + +#[test] +fn string_mutations_reject_bad_contracts_and_out_of_range_indexes() { + let program = string_mutation_program("split", "a::b", "::", -9, &json!(["a"]), false); + let error = MutationEngine::new(&program) + .expect("out-of-range index is structurally valid") + .execute(string_mutation_frame()) + .expect_err("out-of-range string result must fail closed"); + assert!(error.to_string().contains("n3")); + + let mut program = string_mutation_program_json("split", "a::b", "::", 0, &json!(["a"]), false); + program["nodes"][2]["parameters"]["separator"] = json!(""); + assert!(MutationProgram::from_json(&program.to_string()).is_err()); + + let mut program = string_mutation_program_json("split", "a::b", "::", 0, &json!(["a"]), false); + program["nodes"][3]["parameters"]["values"] = json!([["nested"]]); + assert!(MutationProgram::from_json(&program.to_string()).is_err()); + + let mut program = string_mutation_program_json("split", "a::b", "::", 0, &json!(["a"]), false); + program["nodes"][10]["parameters"] = json!({"trim":true}); + assert!(MutationProgram::from_json(&program.to_string()).is_err()); +} + +fn string_mutation_frame() -> MutationFrame { + MutationFrame::new(BTreeMap::from([ + ( + "append_mask".to_owned(), + OwnedColumn::boolean(vec![Some(true), Some(false), None, Some(true)]), + ), + ( + "base_tags".to_owned(), + OwnedColumn::text(vec![ + Some("alpha ".to_owned()), + Some(" keep ".to_owned()), + None, + Some(String::new()), + ]), + ), + ( + "coin".to_owned(), + OwnedColumn::text(vec![ + Some("BTC".to_owned()), + Some("XRP".to_owned()), + None, + Some("ETH".to_owned()), + ]), + ), + ])) + .expect("string mutation frame") +} + +fn string_mutation_program( + method: &str, + source: &str, + separator: &str, + index: i64, + membership_values: &Value, + negated: bool, +) -> MutationProgram { + MutationProgram::from_json( + &string_mutation_program_json(method, source, separator, index, membership_values, negated) + .to_string(), + ) + .expect("string mutation program is valid") +} + +fn string_mutation_program_json( + method: &str, + source: &str, + separator: &str, + index: i64, + membership_values: &Value, + negated: bool, +) -> Value { + let location = || { + json!({ + "path":"strategy.py", "line":1, "column":0, "end_line":1, "end_column":1 + }) + }; + let lookback = || { + json!({ + "kind":"finite", "candles":0, "expression":null, "causal":true + }) + }; + let mut program = json!({ + "schema_version":"tag-program-v1", + "source":{ + "path":"contract.py", + "sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "selected_class":"Contract", + "compile_context":{"run_mode":"backtest","trading_mode":"spot"}, + "entrypoints":[ + {"phase":"entry","function":"f1"}, + {"phase":"exit","function":"f2"} + ], + "functions":[ + { + "id":"f1", "source_name":"populate_entry_trend", "kind":"entrypoint-entry", + "parameters":[{"name":"dataframe","node":"n1","value_type":"dataframe"}], + "node_ids":[ + "n1","n2","n3","n4","n5","n6","n7","n8","n9","n10","n11","n12","n13" + ], + "return_node":"n13" + }, + { + "id":"f2", "source_name":"populate_exit_trend", "kind":"entrypoint-exit", + "parameters":[{"name":"dataframe","node":"n14","value_type":"dataframe"}], + "node_ids":["n14","n15"], "return_node":"n15" + } + ], + "nodes":[ + {"id":"n1","function":"f1","source_order":0,"op":"parameter","value_type":"dataframe","inputs":[],"parameters":{"name":"dataframe"},"lookback":lookback()}, + {"id":"n2","function":"f1","source_order":1,"op":"literal","value_type":"string-scalar","inputs":[],"parameters":{"value":source},"lookback":lookback()}, + {"id":"n3","function":"f1","source_order":2,"op":"string-split-index","value_type":"string-scalar","inputs":["n2"],"parameters":{"method":method,"separator":separator,"index":index},"lookback":lookback()}, + {"id":"n4","function":"f1","source_order":3,"op":"membership","value_type":"bool-scalar","inputs":["n3"],"parameters":{"values":membership_values,"negated":negated},"lookback":lookback()}, + {"id":"n5","function":"f1","source_order":4,"op":"frame-write","value_type":"dataframe","inputs":["n1","n4"],"parameters":{"rows":"all","mode":"column","assignment":"scalar-broadcast","columns":["enter_long"]},"lookback":lookback()}, + {"id":"n6","function":"f1","source_order":5,"op":"column-read","value_type":"string-column","inputs":["n1"],"parameters":{"column":"coin"},"lookback":lookback()}, + {"id":"n7","function":"f1","source_order":6,"op":"membership","value_type":"bool-column","inputs":["n6"],"parameters":{"values":["BTC",null,7,true],"negated":false},"lookback":lookback()}, + {"id":"n8","function":"f1","source_order":7,"op":"frame-write","value_type":"dataframe","inputs":["n5","n7"],"parameters":{"rows":"all","mode":"column","assignment":"column-values","columns":["enter_short"]},"lookback":lookback()}, + {"id":"n9","function":"f1","source_order":8,"op":"column-read","value_type":"string-column","inputs":["n1"],"parameters":{"column":"base_tags"},"lookback":lookback()}, + {"id":"n10","function":"f1","source_order":9,"op":"column-read","value_type":"bool-column","inputs":["n1"],"parameters":{"column":"append_mask"},"lookback":lookback()}, + {"id":"n11","function":"f1","source_order":10,"op":"masked-string-append","value_type":"string-column","inputs":["n9","n10","n3"],"parameters":{},"lookback":lookback()}, + {"id":"n12","function":"f1","source_order":11,"op":"frame-write","value_type":"dataframe","inputs":["n8","n11"],"parameters":{"rows":"all","mode":"column","assignment":"column-values","columns":["enter_tag"]},"lookback":lookback()}, + {"id":"n13","function":"f1","source_order":12,"op":"return","value_type":"dataframe","inputs":["n12"],"parameters":{},"lookback":lookback()}, + {"id":"n14","function":"f2","source_order":0,"op":"parameter","value_type":"dataframe","inputs":[],"parameters":{"name":"dataframe"},"lookback":lookback()}, + {"id":"n15","function":"f2","source_order":1,"op":"return","value_type":"dataframe","inputs":["n14"],"parameters":{},"lookback":lookback()} + ], + "required_input_columns":["append_mask","base_tags","coin"], + "mutation_nodes":["n5","n8","n12"], + "opcodes":[ + "column-read","frame-write","literal","masked-string-append","membership", + "parameter","return","string-split-index" + ], + "max_lookback":lookback(), + "source_map":{ + "n1":location(),"n2":location(),"n3":location(),"n4":location(), + "n5":location(),"n6":location(),"n7":location(),"n8":location(), + "n9":location(),"n10":location(),"n11":location(),"n12":location(), + "n13":location(),"n14":location(),"n15":location() + }, + "route_contract":{ + "canonicalization":"python-str-split", + "original_storage":"preserve-exact", + "trailing_whitespace":"preserve" + }, + "tag_mutation_nodes":["n12"], + "tag_outputs":[ + {"column":"enter_tag","phase":"entry","wrapper_initializer":"","final_mutation":"n12"}, + {"column":"exit_tag","phase":"exit","wrapper_initializer":"","final_mutation":null} + ], + "fingerprint":"" + }); + program["fingerprint"] = Value::String( + crate::program::validation::canonical_fingerprint(&program) + .expect("string mutation test program has canonical identity"), + ); + program +} + +fn numpy_divide_program() -> MutationProgram { + MutationProgram::from_json(&numpy_divide_program_json().to_string()) + .expect("numpy divide program is valid") +} + +fn numpy_divide_program_json() -> Value { + let location = || { + json!({ + "path":"strategy.py", "line":1, "column":0, "end_line":1, "end_column":1 + }) + }; + let lookback = || { + json!({ + "kind":"finite", "candles":0, "expression":null, "causal":true + }) + }; + let mut program = json!({ + "schema_version":"signal-program-v1", + "source":{ + "path":"contract.py", + "sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "selected_class":"Contract", + "compile_context":{"run_mode":"backtest","trading_mode":"spot"}, + "entrypoints":[ + {"phase":"entry","function":"f1"}, + {"phase":"exit","function":"f2"} + ], + "functions":[ + { + "id":"f1", "source_name":"populate_entry_trend", "kind":"entrypoint-entry", + "parameters":[{"name":"dataframe","node":"n1","value_type":"dataframe"}], + "node_ids":[ + "n1","n2","n3","n4","n5","n6","n7","n8","n9","n10","n11","n12" + ], + "return_node":"n12" + }, + { + "id":"f2", "source_name":"populate_exit_trend", "kind":"entrypoint-exit", + "parameters":[{"name":"dataframe","node":"n13","value_type":"dataframe"}], + "node_ids":["n13","n14"], "return_node":"n14" + } + ], + "nodes":[ + {"id":"n1","function":"f1","source_order":0,"op":"parameter","value_type":"dataframe","inputs":[],"parameters":{"name":"dataframe"},"lookback":lookback()}, + {"id":"n2","function":"f1","source_order":1,"op":"column-read","value_type":"f64-column","inputs":["n1"],"parameters":{"column":"numerator"},"lookback":lookback()}, + {"id":"n3","function":"f1","source_order":2,"op":"column-read","value_type":"f64-column","inputs":["n1"],"parameters":{"column":"denominator"},"lookback":lookback()}, + {"id":"n4","function":"f1","source_order":3,"op":"column-read","value_type":"f64-column","inputs":["n1"],"parameters":{"column":"template"},"lookback":lookback()}, + {"id":"n5","function":"f1","source_order":4,"op":"literal","value_type":"f64-scalar","inputs":[],"parameters":{"special":"nan"},"lookback":lookback()}, + {"id":"n6","function":"f1","source_order":5,"op":"array-call","value_type":"f64-column","inputs":["n4","n5"],"parameters":{"family":"numpy","name":"full_like","arguments":{}},"lookback":lookback()}, + {"id":"n7","function":"f1","source_order":6,"op":"column-read","value_type":"bool-column","inputs":["n1"],"parameters":{"column":"mask"},"lookback":lookback()}, + {"id":"n8","function":"f1","source_order":7,"op":"array-call","value_type":"f64-column","inputs":["n2","n3","n6","n7"],"parameters":{"family":"numpy","name":"divide","arguments":{}},"lookback":lookback()}, + {"id":"n9","function":"f1","source_order":8,"op":"frame-write","value_type":"dataframe","inputs":["n1","n8"],"parameters":{"rows":"all","mode":"column","assignment":"column-values","columns":["enter_long"]},"lookback":lookback()}, + {"id":"n10","function":"f1","source_order":9,"op":"array-call","value_type":"f64-column","inputs":["n2","n3","n4","n7"],"parameters":{"family":"numpy","name":"divide","arguments":{}},"lookback":lookback()}, + {"id":"n11","function":"f1","source_order":10,"op":"frame-write","value_type":"dataframe","inputs":["n9","n10"],"parameters":{"rows":"all","mode":"column","assignment":"column-values","columns":["enter_short"]},"lookback":lookback()}, + {"id":"n12","function":"f1","source_order":11,"op":"return","value_type":"dataframe","inputs":["n11"],"parameters":{},"lookback":lookback()}, + {"id":"n13","function":"f2","source_order":0,"op":"parameter","value_type":"dataframe","inputs":[],"parameters":{"name":"dataframe"},"lookback":lookback()}, + {"id":"n14","function":"f2","source_order":1,"op":"return","value_type":"dataframe","inputs":["n13"],"parameters":{},"lookback":lookback()} + ], + "required_input_columns":["denominator","mask","numerator","template"], + "mutation_nodes":["n9","n11"], + "opcodes":["array-call","column-read","frame-write","literal","parameter","return"], + "max_lookback":lookback(), + "source_map":{ + "n1":location(),"n2":location(),"n3":location(),"n4":location(), + "n5":location(),"n6":location(),"n7":location(),"n8":location(), + "n9":location(),"n10":location(),"n11":location(),"n12":location(), + "n13":location(),"n14":location() + }, + "signal_outputs":[ + {"column":"enter_long","phase":"entry","side":"long","final_mutation":"n9"}, + {"column":"enter_short","phase":"entry","side":"short","final_mutation":"n11"} + ], + "fingerprint":"" + }); + program["fingerprint"] = Value::String( + crate::program::validation::canonical_fingerprint(&program) + .expect("numpy divide test program has canonical identity"), + ); + program +} + +fn numpy_string_full_program_json(arguments: &Value) -> Value { + let location = || { + json!({ + "path":"strategy.py", "line":1, "column":0, "end_line":1, "end_column":1 + }) + }; + let lookback = || { + json!({ + "kind":"finite", "candles":0, "expression":null, "causal":true + }) + }; + let mut program = json!({ + "schema_version":"tag-program-v1", + "source":{ + "path":"contract.py", + "sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "selected_class":"Contract", + "compile_context":{"run_mode":"backtest","trading_mode":"spot"}, + "entrypoints":[ + {"phase":"entry","function":"f1"}, + {"phase":"exit","function":"f2"} + ], + "functions":[ + { + "id":"f1", "source_name":"populate_entry_trend", "kind":"entrypoint-entry", + "parameters":[{"name":"dataframe","node":"n1","value_type":"dataframe"}], + "node_ids":["n1","n2","n3","n4","n5","n6"], "return_node":"n6" + }, + { + "id":"f2", "source_name":"populate_exit_trend", "kind":"entrypoint-exit", + "parameters":[{"name":"dataframe","node":"n7","value_type":"dataframe"}], + "node_ids":["n7","n8"], "return_node":"n8" + } + ], + "nodes":[ + {"id":"n1","function":"f1","source_order":0,"op":"parameter","value_type":"dataframe","inputs":[],"parameters":{"name":"dataframe"},"lookback":lookback()}, + {"id":"n2","function":"f1","source_order":1,"op":"row-count","value_type":"int-scalar","inputs":["n1"],"parameters":{},"lookback":lookback()}, + {"id":"n3","function":"f1","source_order":2,"op":"literal","value_type":"string-scalar","inputs":[],"parameters":{"value":""},"lookback":lookback()}, + {"id":"n4","function":"f1","source_order":3,"op":"array-call","value_type":"string-column","inputs":["n2","n3"],"parameters":{"family":"numpy","name":"full","arguments":arguments},"lookback":lookback()}, + {"id":"n5","function":"f1","source_order":4,"op":"frame-write","value_type":"dataframe","inputs":["n1","n4"],"parameters":{"rows":"all","mode":"column","assignment":"column-values","columns":["enter_tag"]},"lookback":lookback()}, + {"id":"n6","function":"f1","source_order":5,"op":"return","value_type":"dataframe","inputs":["n5"],"parameters":{},"lookback":lookback()}, + {"id":"n7","function":"f2","source_order":0,"op":"parameter","value_type":"dataframe","inputs":[],"parameters":{"name":"dataframe"},"lookback":lookback()}, + {"id":"n8","function":"f2","source_order":1,"op":"return","value_type":"dataframe","inputs":["n7"],"parameters":{},"lookback":lookback()} + ], + "required_input_columns":[], + "mutation_nodes":["n5"], + "opcodes":["array-call","frame-write","literal","parameter","return","row-count"], + "max_lookback":lookback(), + "source_map":{ + "n1":location(),"n2":location(),"n3":location(),"n4":location(), + "n5":location(),"n6":location(),"n7":location(),"n8":location() + }, + "route_contract":{ + "canonicalization":"python-str-split", + "original_storage":"preserve-exact", + "trailing_whitespace":"preserve" + }, + "tag_mutation_nodes":["n5"], + "tag_outputs":[ + {"column":"enter_tag","phase":"entry","wrapper_initializer":"","final_mutation":"n5"}, + {"column":"exit_tag","phase":"exit","wrapper_initializer":"","final_mutation":null} + ], + "fingerprint":"" + }); + program["fingerprint"] = Value::String( + crate::program::validation::canonical_fingerprint(&program) + .expect("numpy full test program has canonical identity"), + ); + program +} + fn decode_frame(encoded: &EncodedFrame) -> MutationFrame { let columns = encoded .columns diff --git a/rust/crates/nfi-vector-core/src/mutation/validation.rs b/rust/crates/nfi-vector-core/src/mutation/validation.rs index 53458046..849a8262 100644 --- a/rust/crates/nfi-vector-core/src/mutation/validation.rs +++ b/rust/crates/nfi-vector-core/src/mutation/validation.rs @@ -212,6 +212,24 @@ impl MutationProgram { if node.op == "format-string" { self.validate_format_string(node)?; } + if node.op == "literal" { + Self::validate_literal(node)?; + } + if node.op == "array-call" { + self.validate_array_call(node)?; + } + if node.op == "string-split-index" { + self.validate_string_split_index(node)?; + } + if node.op == "metadata-read" { + self.validate_metadata_read(node)?; + } + if node.op == "membership" { + self.validate_membership(node)?; + } + if node.op == "masked-string-append" { + self.validate_masked_string_append(node)?; + } if node.op == "function-call" { return Err(invalid(format!( "mutation helper execution is not yet exact at {}", @@ -222,6 +240,188 @@ impl MutationProgram { Ok(()) } + fn validate_metadata_read(&self, node: &ProgramNode) -> Result<(), VectorCoreError> { + let key = node.parameters.get("key").and_then(Value::as_str); + let valid = node.parameters.len() == 1 + && key.is_some_and(|key| !key.is_empty()) + && node.value_type == "string-scalar" + && node.inputs.len() == 1 + && self + .node(&node.inputs[0]) + .is_some_and(|input| input.value_type == "metadata"); + if !valid { + return Err(invalid(format!( + "metadata-read {} contract is invalid", + node.id + ))); + } + Ok(()) + } + + fn validate_string_split_index(&self, node: &ProgramNode) -> Result<(), VectorCoreError> { + let method = required_string(&node.parameters, "method", node)?; + let separator = required_string(&node.parameters, "separator", node)?; + let index = node.parameters.get("index").and_then(Value::as_i64); + let valid = node.parameters.len() == 3 + && matches!(method, "partition" | "split" | "rsplit") + && !separator.is_empty() + && index.is_some() + && node.value_type == "string-scalar" + && node.inputs.len() == 1 + && self + .node(&node.inputs[0]) + .is_some_and(|input| input.value_type == "string-scalar"); + if !valid { + return Err(invalid(format!( + "string-split-index {} contract is invalid", + node.id + ))); + } + Ok(()) + } + + fn validate_membership(&self, node: &ProgramNode) -> Result<(), VectorCoreError> { + let values = node.parameters.get("values").and_then(Value::as_array); + let negated = node.parameters.get("negated").and_then(Value::as_bool); + let input_type = node + .inputs + .first() + .and_then(|input| self.node(input)) + .map(|input| input.value_type.as_str()); + let values_are_scalar = values.is_some_and(|values| { + values.iter().all(|value| { + value.is_null() || value.is_boolean() || value.is_number() || value.is_string() + }) + }); + let valid = node.parameters.len() == 2 + && node.inputs.len() == 1 + && negated.is_some() + && values_are_scalar + && matches!( + (input_type, node.value_type.as_str()), + (Some("string-scalar"), "bool-scalar") | (Some("string-column"), "bool-column") + ); + if !valid { + return Err(invalid(format!( + "membership {} contract is invalid", + node.id + ))); + } + Ok(()) + } + + fn validate_masked_string_append(&self, node: &ProgramNode) -> Result<(), VectorCoreError> { + let input_types = node + .inputs + .iter() + .map(|input| self.node(input).map(|input| input.value_type.as_str())) + .collect::>>(); + let valid = node.parameters.is_empty() + && node.value_type == "string-column" + && input_types.as_deref().is_some_and(|inputs| { + matches!( + inputs, + [ + "string-column", + "bool-scalar" | "bool-column", + "string-scalar" + ] + ) + }); + if !valid { + return Err(invalid(format!( + "masked-string-append {} contract is invalid", + node.id + ))); + } + Ok(()) + } + + fn validate_literal(node: &ProgramNode) -> Result<(), VectorCoreError> { + let has_value = node.parameters.contains_key("value"); + let has_special = node.parameters.contains_key("special"); + if has_value == has_special || node.parameters.len() != 1 { + return Err(invalid(format!( + "literal {} must have exactly one ordinary or special value", + node.id + ))); + } + if has_special { + let special = node.parameters.get("special").and_then(Value::as_str); + if node.value_type != "f64-scalar" + || !matches!(special, Some("nan" | "+infinity" | "-infinity")) + { + return Err(invalid(format!( + "literal {} special float contract is invalid", + node.id + ))); + } + } + Ok(()) + } + + fn validate_array_call(&self, node: &ProgramNode) -> Result<(), VectorCoreError> { + let parameters = &node.parameters; + let family = required_string(parameters, "family", node)?; + let name = required_string(parameters, "name", node)?; + let arguments = parameters + .get("arguments") + .and_then(Value::as_object) + .ok_or_else(|| invalid(format!("array-call {} has no arguments map", node.id)))?; + if parameters.len() != 3 + || family != "numpy" + || !numpy_array_arguments_are_supported(name, &node.value_type, arguments) + { + return Err(invalid(format!( + "array-call {} contract is invalid", + node.id + ))); + } + let input_types = node + .inputs + .iter() + .map(|input| { + self.node(input) + .map(|input| input.value_type.as_str()) + .ok_or_else(|| invalid(format!("array-call {} input is missing", node.id))) + }) + .collect::, _>>()?; + let valid = match name { + "full" => { + input_types.len() == 2 + && input_types[0] == "int-scalar" + && matches!( + (node.value_type.as_str(), input_types[1]), + ("bool-column", "bool-scalar") + | ("int-column", "int-scalar") + | ("f64-column", "f64-scalar") + | ("string-column", "string-scalar") + ) + } + "full_like" => { + node.value_type == "f64-column" + && matches!( + input_types.as_slice(), + ["f64-column", "f64-scalar" | "int-scalar"] + ) + } + "divide" => { + node.value_type == "f64-column" + && input_types.as_slice() + == ["f64-column", "f64-column", "f64-column", "bool-column"] + } + "isnan" => node.value_type == "bool-column" && input_types.as_slice() == ["f64-column"], + _ => false, + }; + if !valid { + return Err(invalid(format!( + "array-call {} signature is unsupported", + node.id + ))); + } + Ok(()) + } + fn validate_frame_write(&self, node: &ProgramNode) -> Result<(), VectorCoreError> { let parameters = &node.parameters; let rows = required_string(parameters, "rows", node)?; @@ -496,6 +696,23 @@ impl MutationProgram { } } +pub(crate) fn numpy_array_arguments_are_supported( + name: &str, + value_type: &str, + arguments: &serde_json::Map, +) -> bool { + match name { + "full" => { + arguments.is_empty() + || (value_type == "string-column" + && arguments.len() == 1 + && arguments.get("dtype").and_then(Value::as_str) == Some("object")) + } + "full_like" | "divide" | "isnan" => arguments.is_empty(), + _ => false, + } +} + fn required_string<'a>( parameters: &'a serde_json::Map, name: &str, @@ -512,11 +729,15 @@ fn known_opcode(value: &str) -> bool { value, "parameter" | "literal" + | "row-count" + | "string-split-index" | "column-read" | "metadata-read" | "frame-write" | "binary" | "compare" + | "membership" + | "masked-string-append" | "logical" | "unary" | "select" diff --git a/rust/crates/nfi-vector-core/src/program/validation.rs b/rust/crates/nfi-vector-core/src/program/validation.rs index 4158b43c..7082069d 100644 --- a/rust/crates/nfi-vector-core/src/program/validation.rs +++ b/rust/crates/nfi-vector-core/src/program/validation.rs @@ -381,11 +381,19 @@ fn is_known_opcode(value: &str) -> bool { value, "parameter" | "literal" + | "row-count" + | "row-index" + | "string-split-index" + | "frame-source" + | "frame-nonempty" + | "frame-project" + | "frame-drop-if-present" | "column-read" | "metadata-read" | "column-write" | "binary" | "compare" + | "membership" | "logical" | "unary" | "select" diff --git a/rust/crates/nfi-vector-io/Cargo.toml b/rust/crates/nfi-vector-io/Cargo.toml index 71cceac8..9359fb79 100644 --- a/rust/crates/nfi-vector-io/Cargo.toml +++ b/rust/crates/nfi-vector-io/Cargo.toml @@ -13,6 +13,7 @@ path = "src/lib.rs" [dependencies] arrow2 = { version = "0.18", features = ["io_ipc", "io_ipc_compression"] } +fs2 = "0.4" nfi-sim-core = { path = "../nfi-sim-core" } nfi-vector-core = { path = "../nfi-vector-core" } rayon = "1" diff --git a/rust/crates/nfi-vector-io/src/decode.rs b/rust/crates/nfi-vector-io/src/decode.rs index 2cdbe9b2..576140c7 100644 --- a/rust/crates/nfi-vector-io/src/decode.rs +++ b/rust/crates/nfi-vector-io/src/decode.rs @@ -13,7 +13,6 @@ use crate::row::append_batch_to_spool; use crate::schema::{column_positions, feature_layout, projected_source_indices}; use crate::VectorInputError; -const SPOOL_DIRECTORY_ENVIRONMENT: &str = "NFI_BTE_SPOOL_DIRECTORY"; // One bounded buffer converts batches into fixed-width rows without issuing a // kernel write for every candle. The spool itself remains disk-backed. const SPOOL_WRITE_BUFFER_BYTES: usize = 256 * 1024; @@ -107,14 +106,12 @@ pub(crate) fn read_feather( Ok((CandleSeries::file_backed(rows), features, file_backed_bytes)) } -fn pair_spool(pair: &str) -> Result { +pub(crate) fn pair_spool(pair: &str) -> Result { // OS-local temp avoids the severe random-read penalty of a WSL-mounted // Windows vector directory. Hosts whose temp directory is RAM-backed can // select a disk-backed mount explicitly; the path is configuration, never // a compiled machine assumption. - let result = std::env::var_os(SPOOL_DIRECTORY_ENVIRONMENT) - .map_or_else(tempfile::tempfile, tempfile::tempfile_in); - result.map_err(|source| VectorInputError::FileBacking { + crate::spool_admission::create_file().map_err(|source| VectorInputError::FileBacking { pair: pair.to_owned(), source, }) diff --git a/rust/crates/nfi-vector-io/src/failures.rs b/rust/crates/nfi-vector-io/src/failures.rs index d263567c..1bbf413d 100644 --- a/rust/crates/nfi-vector-io/src/failures.rs +++ b/rust/crates/nfi-vector-io/src/failures.rs @@ -106,4 +106,19 @@ pub enum VectorInputError { actual: &'static str, expected: &'static str, }, + #[error("cannot create the bounded pair worker pool: {0}")] + PairWorkerPool(String), + #[error("cannot calculate the Full Native spool upper bound: {0}")] + SpoolBound(String), + #[error("cannot inspect spool filesystem at {target}: {source}")] + SpoolCapacityProbe { + target: PathBuf, + source: std::io::Error, + }, + #[error("spool capacity admission failed for {target_source}: requires {required_bytes} bytes, only {available_bytes} bytes are available")] + SpoolCapacity { + target_source: &'static str, + required_bytes: u64, + available_bytes: u64, + }, } diff --git a/rust/crates/nfi-vector-io/src/freqtrade_funding.rs b/rust/crates/nfi-vector-io/src/freqtrade_funding.rs new file mode 100644 index 00000000..54699b52 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/freqtrade_funding.rs @@ -0,0 +1,461 @@ +//! Exact sparse Futures funding preparation for the native vector pipeline. +//! +//! Freqtrade keeps funding-rate and mark-price archives as separate roles. +//! Each role is stable-sorted and deduplicated by timestamp with the last row +//! winning, then the two roles are inner-joined. Only those paired events are +//! mapped to equal base-candle timestamps; funding is never forward-filled. + +use std::collections::BTreeMap; + +use nfi_vector_core::alignment::{NumericFrame, Timeframe}; +use nfi_vector_core::column::OwnedColumn; + +use crate::{FuturesFrameSet, NativeContractError, TradingMode}; + +pub const RATE_COLUMN: &str = "nfi_exec_funding_rate"; +pub const MARK_COLUMN: &str = "nfi_exec_funding_mark_price"; + +/// Pair-aligned sparse funding columns ready for [`crate::InMemoryVectorPair`]. +/// +/// Missing rows contain a present canonical NaN, matching the Python worker's +/// float columns. The in-memory adapter deliberately maps both NaN and Arrow +/// null to `None` before constructing simulator candles. +#[derive(Clone, Debug, PartialEq)] +pub struct PreparedEvents { + pub pair: String, + pub funding_interval: Option, + pub funding_rates: Vec>, + pub mark_prices: Vec>, +} + +impl PreparedEvents { + /// Consume the result as the two conventional `nfi_exec_*` columns. + #[must_use] + pub fn into_owned_columns(self) -> BTreeMap { + BTreeMap::from([ + (RATE_COLUMN.to_owned(), OwnedColumn::f64(self.funding_rates)), + (MARK_COLUMN.to_owned(), OwnedColumn::f64(self.mark_prices)), + ]) + } +} + +/// Prepare exact sparse funding events for one base pair. +/// +/// `futures_frames` is the manifest-decoded collection for the complete run. +/// Futures mode requires exactly one matching descriptor. Spot mode requires +/// the collection to be empty and returns all-missing compatibility columns. +/// The returned interval comes from the descriptor's frame identity; no +/// funding cadence is assumed by this stage. +/// +/// # Errors +/// +/// Returns a fail-closed contract error for an empty pair, invalid mode/data +/// combination, duplicate or missing descriptors, identity/interval drift, +/// missing `open` columns, or a paired event whose rate is non-finite or whose +/// mark price is non-finite/non-positive. +pub fn prepare_events( + trading_mode: TradingMode, + pair: &str, + base_timestamps_ms: &[i64], + futures_frames: &[FuturesFrameSet], +) -> Result { + if pair.is_empty() { + return Err(invalid("funding preparation pair is empty")); + } + if trading_mode == TradingMode::Spot { + if !futures_frames.is_empty() { + return Err(invalid( + "Spot execution cannot contain Futures funding frames", + )); + } + return Ok(missing_events(pair, base_timestamps_ms.len(), None)); + } + + let matching_descriptors = futures_frames + .iter() + .filter(|frame_set| frame_set.pair == pair) + .collect::>(); + let [frame_set] = matching_descriptors.as_slice() else { + let message = if matching_descriptors.is_empty() { + format!("Futures execution has no funding descriptor for {pair}") + } else { + format!("Futures execution has duplicate funding descriptors for {pair}") + }; + return Err(invalid(message)); + }; + validate_frame_set(frame_set, pair)?; + let interval = frame_set.funding_rate.identity.timeframe.clone(); + let funding = stable_deduplicate_open(&frame_set.funding_rate, "funding-rate")?; + let mark_by_time = stable_deduplicate_open(&frame_set.mark, "mark-price")?; + + let mut paired = BTreeMap::new(); + for (timestamp_ms, rate) in funding { + let Some(mark) = mark_by_time.get(×tamp_ms).copied() else { + continue; + }; + if !rate.is_finite() { + return Err(invalid(format!( + "paired funding rate for {pair} at {timestamp_ms} is not finite" + ))); + } + if !mark.is_finite() || mark <= 0.0 { + return Err(invalid(format!( + "paired funding mark price for {pair} at {timestamp_ms} must be positive and finite" + ))); + } + paired.insert(timestamp_ms, (rate, mark)); + } + + let mut result = missing_events(pair, base_timestamps_ms.len(), Some(interval)); + for (row, timestamp_ms) in base_timestamps_ms.iter().enumerate() { + if let Some((rate, mark)) = paired.get(timestamp_ms) { + result.funding_rates[row] = Some(*rate); + result.mark_prices[row] = Some(*mark); + } + } + Ok(result) +} + +fn validate_frame_set(frame_set: &FuturesFrameSet, pair: &str) -> Result<(), NativeContractError> { + for (role, frame) in [ + ("funding-rate", &frame_set.funding_rate), + ("mark-price", &frame_set.mark), + ] { + frame + .validate() + .map_err(|error| invalid(format!("invalid {role} frame for {pair}: {error}")))?; + if frame.identity.pair != pair { + return Err(invalid(format!( + "{role} frame identity {} differs from descriptor pair {pair}", + frame.identity.pair + ))); + } + } + Ok(()) +} + +fn stable_deduplicate_open( + frame: &NumericFrame, + role: &str, +) -> Result, NativeContractError> { + let open = frame.columns.get("open").ok_or_else(|| { + invalid(format!( + "{role} frame for {} {} is missing column \"open\"", + frame.identity.pair, + frame.identity.timeframe.as_str() + )) + })?; + let mut positions = (0..frame.timestamps_ms.len()).collect::>(); + positions.sort_by_key(|row| frame.timestamps_ms[*row]); + let mut result = BTreeMap::new(); + for row in positions { + // Arrow null and present NaN are both observable missing numeric + // values. Retaining NaN here lets the paired-event validation report + // one deterministic error after the exact inner join. + let value = open[row].unwrap_or(f64::NAN); + result.insert(frame.timestamps_ms[row], value); + } + Ok(result) +} + +fn missing_events(pair: &str, rows: usize, funding_interval: Option) -> PreparedEvents { + PreparedEvents { + pair: pair.to_owned(), + funding_interval, + funding_rates: vec![Some(f64::NAN); rows], + mark_prices: vec![Some(f64::NAN); rows], + } +} + +fn invalid(message: impl Into) -> NativeContractError { + NativeContractError::Invalid(format!( + "Freqtrade funding preparation failed: {}", + message.into() + )) +} + +#[cfg(test)] +mod tests { + use nfi_vector_core::alignment::FrameIdentity; + + use super::*; + + fn identity(pair: &str, timeframe: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse(timeframe).expect("timeframe")).expect("identity") + } + + fn sparse_frame( + pair: &str, + timeframe: &str, + timestamps_ms: Vec, + opens: Vec>, + ) -> NumericFrame { + NumericFrame { + identity: identity(pair, timeframe), + timestamps_ms, + columns: BTreeMap::from([("open".to_owned(), opens)]), + } + } + + fn frame_set( + pair: &str, + timeframe: &str, + funding_timestamps: Vec, + rates: Vec>, + mark_timestamps: Vec, + marks: Vec>, + ) -> FuturesFrameSet { + FuturesFrameSet { + pair: pair.to_owned(), + funding_rate: sparse_frame(pair, timeframe, funding_timestamps, rates), + mark: sparse_frame(pair, timeframe, mark_timestamps, marks), + } + } + + fn assert_nan(value: Option) { + assert!(value + .expect("present Python-compatible missing value") + .is_nan()); + } + + #[test] + fn matches_python_attach_funding_events_duplicate_and_sparse_oracle() { + // Generated on 2026-08-12 with vector_worker._attach_funding_events. + // Each role is keep-last at 1h; the unpaired 3h/4h rows disappear. + let frames = [frame_set( + "ORACLE/USDT", + "1h", + vec![7_200_000, 3_600_000, 3_600_000, 14_400_000], + vec![Some(0.003), Some(0.001), Some(0.002), Some(0.004)], + vec![10_800_000, 3_600_000, 7_200_000, 3_600_000], + vec![Some(300.0), Some(100.0), Some(200.0), Some(101.0)], + )]; + + let result = prepare_events( + TradingMode::Futures, + "ORACLE/USDT", + &[0, 3_600_000, 7_200_000, 10_800_000], + &frames, + ) + .expect("events"); + + assert_eq!( + result + .funding_interval + .as_ref() + .expect("manifest interval") + .as_str(), + "1h" + ); + assert_nan(result.funding_rates[0]); + assert_eq!(result.funding_rates[1..3], [Some(0.002), Some(0.003)]); + assert_nan(result.funding_rates[3]); + assert_nan(result.mark_prices[0]); + assert_eq!(result.mark_prices[1..3], [Some(101.0), Some(200.0)]); + assert_nan(result.mark_prices[3]); + } + + #[test] + fn exact_timestamp_mapping_never_forward_fills() { + let frames = [frame_set( + "ORACLE/USDT", + "15m", + vec![0], + vec![Some(-0.000_1)], + vec![0], + vec![Some(250.0)], + )]; + + let result = prepare_events( + TradingMode::Futures, + "ORACLE/USDT", + &[0, 300_000, 900_000], + &frames, + ) + .expect("events"); + + assert_eq!(result.funding_rates[0], Some(-0.000_1)); + assert_eq!(result.mark_prices[0], Some(250.0)); + for row in 1..3 { + assert_nan(result.funding_rates[row]); + assert_nan(result.mark_prices[row]); + } + } + + #[test] + fn interval_is_manifest_derived_instead_of_assuming_one_hour() { + let frames = [frame_set( + "ORACLE/USDT", + "8h", + vec![0], + vec![Some(0.001)], + vec![0], + vec![Some(100.0)], + )]; + + let result = + prepare_events(TradingMode::Futures, "ORACLE/USDT", &[0], &frames).expect("events"); + + assert_eq!(result.funding_interval.expect("interval").as_str(), "8h"); + } + + #[test] + fn spot_has_missing_compatibility_columns_and_rejects_futures_sources() { + let spot = prepare_events(TradingMode::Spot, "ORACLE/USDT", &[0, 300_000], &[]) + .expect("Spot compatibility columns"); + assert_eq!(spot.pair, "ORACLE/USDT"); + assert!(spot.funding_interval.is_none()); + assert_nan(spot.funding_rates[0]); + assert_nan(spot.funding_rates[1]); + assert_nan(spot.mark_prices[0]); + assert_nan(spot.mark_prices[1]); + + let frames = [frame_set( + "ORACLE/USDT", + "1h", + vec![0], + vec![Some(0.001)], + vec![0], + vec![Some(100.0)], + )]; + assert!( + prepare_events(TradingMode::Spot, "ORACLE/USDT", &[0], &frames) + .expect_err("Spot/Futures conflict") + .to_string() + .contains("Spot execution cannot contain Futures funding frames") + ); + } + + #[test] + fn futures_requires_exactly_one_pair_descriptor() { + let first = frame_set( + "ORACLE/USDT", + "1h", + vec![0], + vec![Some(0.001)], + vec![0], + vec![Some(100.0)], + ); + assert!( + prepare_events(TradingMode::Futures, "MISSING/USDT", &[0], &[]) + .expect_err("missing descriptor") + .to_string() + .contains("no funding descriptor") + ); + assert!(prepare_events( + TradingMode::Futures, + "ORACLE/USDT", + &[0], + &[first.clone(), first], + ) + .expect_err("duplicate descriptors") + .to_string() + .contains("duplicate funding descriptors")); + } + + #[test] + fn rejects_identity_drift_and_allows_distinct_role_timeframes() { + let mut pair_drift = frame_set( + "ORACLE/USDT", + "1h", + vec![0], + vec![Some(0.001)], + vec![0], + vec![Some(100.0)], + ); + pair_drift.mark.identity = identity("OTHER/USDT", "1h"); + assert!( + prepare_events(TradingMode::Futures, "ORACLE/USDT", &[0], &[pair_drift],) + .expect_err("identity drift") + .to_string() + .contains("differs from descriptor pair") + ); + + let mut distinct_timeframes = frame_set( + "ORACLE/USDT", + "1h", + vec![0], + vec![Some(0.001)], + vec![0], + vec![Some(100.0)], + ); + distinct_timeframes.mark.identity = identity("ORACLE/USDT", "4h"); + let result = prepare_events( + TradingMode::Futures, + "ORACLE/USDT", + &[0], + &[distinct_timeframes], + ) + .expect("funding and mark roles retain independent source timeframes"); + assert_eq!( + result.funding_interval.expect("funding interval").as_str(), + "1h" + ); + assert_eq!(result.funding_rates, [Some(0.001)]); + assert_eq!(result.mark_prices, [Some(100.0)]); + } + + #[test] + fn validates_only_paired_events_and_requires_finite_rate_positive_mark() { + for (rate, mark, expected) in [ + (f64::NAN, 100.0, "rate"), + (f64::INFINITY, 100.0, "rate"), + (0.001, f64::NAN, "mark price"), + (0.001, f64::INFINITY, "mark price"), + (0.001, 0.0, "mark price"), + (0.001, -1.0, "mark price"), + ] { + let frames = [frame_set( + "ORACLE/USDT", + "1h", + vec![0], + vec![Some(rate)], + vec![0], + vec![Some(mark)], + )]; + assert!( + prepare_events(TradingMode::Futures, "ORACLE/USDT", &[0], &frames,) + .expect_err("invalid paired event") + .to_string() + .contains(expected) + ); + } + + // Invalid values outside the inner join are unobservable, as in the + // Python merge, and therefore do not create a base funding event. + let unpaired = [frame_set( + "ORACLE/USDT", + "1h", + vec![0], + vec![Some(f64::NAN)], + vec![3_600_000], + vec![Some(-1.0)], + )]; + let result = prepare_events( + TradingMode::Futures, + "ORACLE/USDT", + &[0, 3_600_000], + &unpaired, + ) + .expect("unpaired invalid values are absent events"); + assert_nan(result.funding_rates[0]); + assert_nan(result.funding_rates[1]); + } + + #[test] + fn owned_columns_preserve_in_memory_optional_number_contract() { + let prepared = missing_events( + "ORACLE/USDT", + 2, + Some(Timeframe::parse("4h").expect("timeframe")), + ); + + let columns = prepared.into_owned_columns(); + + for name in [RATE_COLUMN, MARK_COLUMN] { + let view = columns[name].as_view(); + assert!(view.f64_at(0).expect("present NaN").is_nan()); + assert!(view.f64_at(1).expect("present NaN").is_nan()); + } + } +} diff --git a/rust/crates/nfi-vector-io/src/freqtrade_ohlcv.rs b/rust/crates/nfi-vector-io/src/freqtrade_ohlcv.rs new file mode 100644 index 00000000..430083d9 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/freqtrade_ohlcv.rs @@ -0,0 +1,874 @@ +//! Exact Freqtrade OHLCV preparation for native indicator execution. +//! +//! [`crate::raw_ohlcv`] is intentionally a non-transforming transport boundary. +//! This module owns the observable Freqtrade 2026.5.1 preparation semantics: +//! timerange/startup bounding, duplicate aggregation, fixed-timeframe resampling, +//! gap filling, and the inclusive execution/context slice. + +use std::collections::BTreeMap; + +use nfi_vector_core::alignment::{ + FrameCatalog, FrameIdentity, NumericFrame, SourceLocation, Timeframe, +}; +use nfi_vector_core::VectorCoreError; + +use crate::VectorInputError; + +const OHLCV_COLUMNS: [&str; 5] = ["open", "high", "low", "close", "volume"]; +const DAY_MS: i64 = 86_400_000; + +/// A finite timerange with inclusive start and stop boundaries. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ClosedTimerange { + pub start_ms: i64, + pub stop_ms: i64, +} + +impl ClosedTimerange { + /// Parse the closed calendar-date, Unix-second, and Unix-millisecond forms + /// accepted by the Python vector worker. + /// + /// # Errors + /// + /// Returns an invalid-program error for open, malformed, out-of-range, or + /// reversed boundaries. + pub fn parse(value: &str) -> Result { + if value.matches('-').count() != 1 { + return Err(invalid("timerange must contain exactly one separator")); + } + let Some((start, stop)) = value.split_once('-') else { + return Err(invalid("timerange must contain exactly one separator")); + }; + let start_ms = parse_boundary(start)?; + let stop_ms = parse_boundary(stop)?; + if start_ms > stop_ms { + return Err(invalid(format!( + "timerange start is after its stop boundary: {value}" + ))); + } + Ok(Self { start_ms, stop_ms }) + } +} + +/// Inclusive row positions processed by Freqtrade's chronological loop. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct InclusiveExecutionPositions { + pub start_index: usize, + pub stop_index: usize, +} + +/// A callback-visible slice and the first row allowed to emit simulator events. +#[derive(Clone, Debug, PartialEq)] +pub struct PreparedExecutionOhlcv { + pub frame: NumericFrame, + pub execution_start_index: usize, +} + +/// Bound every raw frame independently by its timeframe, then apply exact +/// Freqtrade duplicate and gap preparation. +/// +/// This function never mutates or substitutes an input catalog frame. +/// +/// # Errors +/// +/// Returns an error for invalid timeranges, unsupported timeframes, arithmetic +/// overflow, missing OHLCV columns, or Arrow-null OHLCV values. +pub fn prepare_freqtrade_ohlcv_catalog( + raw: &FrameCatalog, + timerange: &str, + startup_candles: usize, +) -> Result { + let timerange = ClosedTimerange::parse(timerange)?; + let source = SourceLocation::new( + "freqtrade-ohlcv-preparation", + "native/freqtrade_ohlcv", + 0, + 0, + ); + let mut entries = Vec::new(); + for identity in raw.identities() { + let frame = raw.lookup(identity, &source)?; + let duration_ms = supported_duration(&identity.timeframe)?; + let startup_ms = + duration_ms + .checked_mul(i64::try_from(startup_candles).map_err(|_| { + invalid("startup candle count does not fit timestamp arithmetic") + })?) + .ok_or_else(|| invalid("startup window is outside timestamp range"))?; + let load_start = timerange + .start_ms + .checked_sub(startup_ms) + .ok_or_else(|| invalid("startup window is outside timestamp range"))?; + let bounded = select_closed(frame, load_start, timerange.stop_ms)?; + let prepared = clean_frame(&bounded)?; + entries.push((identity.clone(), prepared)); + } + FrameCatalog::new(entries).map_err(Into::into) +} + +/// Aggregate duplicates, resample one supported fixed timeframe, and fill +/// missing candles exactly like Freqtrade 2026.5.1. +/// +/// Duplicate rows use `open:first`, `high:max`, `low:min`, `close:last`, and +/// `volume:max`. Resample buckets use the corresponding OHLC aggregation and +/// sum the duplicate-normalized volume. Empty buckets receive the previous +/// close for OHLC and zero volume. +/// +/// # Errors +/// +/// Returns an error for unsupported timeframes, invalid shapes, missing OHLCV +/// columns, Arrow-null numeric cells, or an unrepresentable output range. +pub fn clean_frame(frame: &NumericFrame) -> Result { + frame.validate()?; + let duration_ms = supported_duration(&frame.identity.timeframe)?; + validate_ohlcv_columns(frame)?; + if frame.timestamps_ms.is_empty() { + return Ok(empty_ohlcv(frame.identity.clone())); + } + + let mut duplicate_groups = BTreeMap::::new(); + for row in 0..frame.timestamps_ms.len() { + let values = row_values(frame, row)?; + duplicate_groups + .entry(frame.timestamps_ms[row]) + .or_default() + .observe(values); + } + + let mut buckets = BTreeMap::::new(); + for (timestamp_ms, aggregate) in duplicate_groups { + let bucket_ms = timestamp_ms.div_euclid(duration_ms) * duration_ms; + buckets + .entry(bucket_ms) + .or_default() + .observe(aggregate.finish()); + } + let first_bucket = *buckets + .first_key_value() + .ok_or_else(|| invalid("non-empty OHLCV source produced no resample bucket"))? + .0; + let last_bucket = *buckets + .last_key_value() + .ok_or_else(|| invalid("non-empty OHLCV source produced no resample bucket"))? + .0; + let row_count = last_bucket + .checked_sub(first_bucket) + .and_then(|span| span.checked_div(duration_ms)) + .and_then(|rows| rows.checked_add(1)) + .and_then(|rows| usize::try_from(rows).ok()) + .ok_or_else(|| invalid("resampled OHLCV range is outside addressable memory"))?; + + let mut output_timestamps = Vec::new(); + output_timestamps + .try_reserve_exact(row_count) + .map_err(|_| invalid("cannot allocate resampled OHLCV timestamps"))?; + let mut output = OutputColumns::with_capacity(row_count)?; + let mut previous_close = None; + for offset in 0..row_count { + let offset = i64::try_from(offset) + .map_err(|_| invalid("resampled OHLCV row offset is out of range"))?; + let current_timestamp = first_bucket + .checked_add( + offset + .checked_mul(duration_ms) + .ok_or_else(|| invalid("resampled OHLCV timestamp is out of range"))?, + ) + .ok_or_else(|| invalid("resampled OHLCV timestamp is out of range"))?; + output_timestamps.push(current_timestamp); + let aggregate = buckets.get(¤t_timestamp); + let close = aggregate + .and_then(|value| value.close) + .or(previous_close) + .unwrap_or(f64::NAN); + if !close.is_nan() { + previous_close = Some(close); + } + output.open.push(Some( + aggregate.and_then(|value| value.open).unwrap_or(close), + )); + output.high.push(Some( + aggregate.and_then(|value| value.high).unwrap_or(close), + )); + output + .low + .push(Some(aggregate.and_then(|value| value.low).unwrap_or(close))); + output.close.push(Some(close)); + output.volume.push(Some( + aggregate.and_then(|value| value.volume).unwrap_or(0.0), + )); + } + + Ok(NumericFrame { + identity: frame.identity.clone(), + timestamps_ms: output_timestamps, + columns: output.into_map(), + }) +} + +/// Locate the inclusive rows Freqtrade processes after satisfying any missing +/// startup history from rows inside the requested range. +/// +/// # Errors +/// +/// Returns an error for an invalid timerange or non-increasing candle dates. +pub fn execution_positions( + frame: &NumericFrame, + timerange: &str, + startup_candles: usize, +) -> Result, VectorInputError> { + frame.validate()?; + ensure_strictly_increasing(frame)?; + let timerange = ClosedTimerange::parse(timerange)?; + let available_before_start = frame + .timestamps_ms + .partition_point(|timestamp| *timestamp < timerange.start_ms); + let missing_startup = startup_candles.saturating_sub(available_before_start); + let range_start = frame + .timestamps_ms + .partition_point(|timestamp| *timestamp < timerange.start_ms); + let range_stop_exclusive = frame + .timestamps_ms + .partition_point(|timestamp| *timestamp <= timerange.stop_ms); + let rows_in_range = range_stop_exclusive.saturating_sub(range_start); + if rows_in_range == 0 || missing_startup >= rows_in_range { + return Ok(None); + } + Ok(Some(InclusiveExecutionPositions { + start_index: range_start + missing_startup, + stop_index: range_stop_exclusive - 1, + })) +} + +/// Retain startup rows as callback-only context and identify the first row +/// allowed to execute. As in Freqtrade, the first trimmed row is not executable +/// because decision signals are shifted to the following candle open. +/// +/// # Errors +/// +/// Returns the errors from [`execution_positions`] or a slicing shape error. +pub fn prepare_execution_ohlcv( + frame: &NumericFrame, + timerange: &str, + startup_candles: usize, +) -> Result { + let Some(positions) = execution_positions(frame, timerange, startup_candles)? else { + return Ok(PreparedExecutionOhlcv { + frame: slice_frame(frame, 0, 0)?, + execution_start_index: 0, + }); + }; + let first_executable = positions + .start_index + .checked_add(1) + .ok_or_else(|| invalid("execution start index is out of range"))?; + if first_executable > positions.stop_index { + return Ok(PreparedExecutionOhlcv { + frame: slice_frame(frame, 0, 0)?, + execution_start_index: 0, + }); + } + let context_rows = startup_candles.min(positions.start_index); + let context_start = positions.start_index - context_rows; + let stop_exclusive = positions + .stop_index + .checked_add(1) + .ok_or_else(|| invalid("execution stop index is out of range"))?; + Ok(PreparedExecutionOhlcv { + frame: slice_frame(frame, context_start, stop_exclusive)?, + execution_start_index: first_executable - context_start, + }) +} + +#[derive(Clone, Copy, Debug, Default)] +struct DuplicateAggregate { + open: Option, + high: Option, + low: Option, + close: Option, + volume: Option, +} + +impl DuplicateAggregate { + fn observe(&mut self, values: [f64; 5]) { + self.open = self.open.or_else(|| present(values[0])); + self.high = optional_max(self.high, present(values[1])); + self.low = optional_min(self.low, present(values[2])); + if let Some(close) = present(values[3]) { + self.close = Some(close); + } + self.volume = optional_max(self.volume, present(values[4])); + } + + fn finish(self) -> [Option; 5] { + [self.open, self.high, self.low, self.close, self.volume] + } +} + +#[derive(Clone, Copy, Debug, Default)] +struct ResampleAggregate { + open: Option, + high: Option, + low: Option, + close: Option, + volume: Option, +} + +impl ResampleAggregate { + fn observe(&mut self, values: [Option; 5]) { + self.open = self.open.or(values[0]); + self.high = optional_max(self.high, values[1]); + self.low = optional_min(self.low, values[2]); + if let Some(close) = values[3] { + self.close = Some(close); + } + if let Some(volume) = values[4] { + self.volume = Some(self.volume.unwrap_or(0.0) + volume); + } + } +} + +struct OutputColumns { + open: Vec>, + high: Vec>, + low: Vec>, + close: Vec>, + volume: Vec>, +} + +impl OutputColumns { + fn with_capacity(rows: usize) -> Result { + fn allocate(rows: usize) -> Result>, VectorInputError> { + let mut values = Vec::new(); + values + .try_reserve_exact(rows) + .map_err(|_| invalid("cannot allocate resampled OHLCV values"))?; + Ok(values) + } + Ok(Self { + open: allocate(rows)?, + high: allocate(rows)?, + low: allocate(rows)?, + close: allocate(rows)?, + volume: allocate(rows)?, + }) + } + + fn into_map(self) -> BTreeMap>> { + BTreeMap::from([ + ("open".to_owned(), self.open), + ("high".to_owned(), self.high), + ("low".to_owned(), self.low), + ("close".to_owned(), self.close), + ("volume".to_owned(), self.volume), + ]) + } +} + +fn supported_duration(timeframe: &Timeframe) -> Result { + if !matches!(timeframe.as_str(), "5m" | "15m" | "1h" | "4h" | "1d") { + return Err(invalid(format!( + "unsupported Freqtrade OHLCV preparation timeframe: {}", + timeframe.as_str() + ))); + } + Ok(timeframe.resample_duration_ms()) +} + +fn validate_ohlcv_columns(frame: &NumericFrame) -> Result<(), VectorInputError> { + for name in OHLCV_COLUMNS { + let values = frame.columns.get(name).ok_or_else(|| { + invalid(format!( + "candle frame for {} {} is missing column {name:?}", + frame.identity.pair, + frame.identity.timeframe.as_str() + )) + })?; + if let Some(row) = values.iter().position(Option::is_none) { + return Err(invalid(format!( + "candle frame for {} {} column {name:?} contains Arrow null at row {row}", + frame.identity.pair, + frame.identity.timeframe.as_str() + ))); + } + } + Ok(()) +} + +fn row_values(frame: &NumericFrame, row: usize) -> Result<[f64; 5], VectorInputError> { + OHLCV_COLUMNS + .map(|name| { + frame.columns[name][row].ok_or_else(|| { + invalid(format!( + "candle frame for {} {} column {name:?} contains Arrow null at row {row}", + frame.identity.pair, + frame.identity.timeframe.as_str() + )) + }) + }) + .into_iter() + .collect::, _>>()? + .try_into() + .map_err(|_| invalid("internal OHLCV column count mismatch")) +} + +fn present(value: f64) -> Option { + (!value.is_nan()).then_some(value) +} + +fn optional_max(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.max(right)), + (Some(value), None) | (None, Some(value)) => Some(value), + (None, None) => None, + } +} + +fn optional_min(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.min(right)), + (Some(value), None) | (None, Some(value)) => Some(value), + (None, None) => None, + } +} + +fn select_closed( + frame: &NumericFrame, + start_ms: i64, + stop_ms: i64, +) -> Result { + frame.validate()?; + let selected = frame + .timestamps_ms + .iter() + .enumerate() + .filter_map(|(row, timestamp)| { + (*timestamp >= start_ms && *timestamp <= stop_ms).then_some(row) + }) + .collect::>(); + let columns = frame + .columns + .iter() + .map(|(name, values)| { + ( + name.clone(), + selected.iter().map(|row| values[*row]).collect::>(), + ) + }) + .collect(); + Ok(NumericFrame { + identity: frame.identity.clone(), + timestamps_ms: selected + .iter() + .map(|row| frame.timestamps_ms[*row]) + .collect(), + columns, + }) +} + +fn slice_frame( + frame: &NumericFrame, + start: usize, + stop_exclusive: usize, +) -> Result { + if start > stop_exclusive || stop_exclusive > frame.timestamps_ms.len() { + return Err(invalid("execution OHLCV slice is outside the source frame")); + } + Ok(NumericFrame { + identity: frame.identity.clone(), + timestamps_ms: frame.timestamps_ms[start..stop_exclusive].to_vec(), + columns: frame + .columns + .iter() + .map(|(name, values)| (name.clone(), values[start..stop_exclusive].to_vec())) + .collect(), + }) +} + +fn empty_ohlcv(identity: FrameIdentity) -> NumericFrame { + NumericFrame { + identity, + timestamps_ms: Vec::new(), + columns: OHLCV_COLUMNS + .map(|name| (name.to_owned(), Vec::new())) + .into_iter() + .collect(), + } +} + +fn ensure_strictly_increasing(frame: &NumericFrame) -> Result<(), VectorInputError> { + if frame + .timestamps_ms + .windows(2) + .any(|window| window[0] >= window[1]) + { + return Err(invalid(format!( + "candle frame for {} {} is not strictly chronological", + frame.identity.pair, + frame.identity.timeframe.as_str() + ))); + } + Ok(()) +} + +fn parse_boundary(value: &str) -> Result { + if !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(invalid("timerange boundary must be numeric")); + } + match value.len() { + 8 => parse_calendar_boundary(value), + 10 => value + .parse::() + .ok() + .and_then(|seconds| seconds.checked_mul(1_000)) + .ok_or_else(|| invalid(format!("invalid timerange boundary: {value:?}"))), + 13 => value + .parse::() + .map_err(|_| invalid(format!("invalid timerange boundary: {value:?}"))), + _ => Err(invalid("unsupported timerange boundary width")), + } +} + +fn parse_calendar_boundary(value: &str) -> Result { + let year = value[0..4] + .parse::() + .map_err(|_| invalid(format!("invalid timerange boundary: {value:?}")))?; + let month = value[4..6] + .parse::() + .map_err(|_| invalid(format!("invalid timerange boundary: {value:?}")))?; + let day = value[6..8] + .parse::() + .map_err(|_| invalid(format!("invalid timerange boundary: {value:?}")))?; + if !(1..=9_999).contains(&year) + || !(1..=12).contains(&month) + || day == 0 + || day > days_in_month(year, month) + { + return Err(invalid(format!("invalid timerange boundary: {value:?}"))); + } + days_from_civil(year, month, day) + .checked_mul(DAY_MS) + .ok_or_else(|| invalid(format!("invalid timerange boundary: {value:?}"))) +} + +fn days_in_month(year: i64, month: u8) -> u8 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29, + 2 => 28, + _ => 0, + } +} + +fn days_from_civil(year: i64, month: u8, day: u8) -> i64 { + let year = year - i64::from(month <= 2); + let era = if year >= 0 { year } else { year - 399 } / 400; + let year_of_era = year - era * 400; + let shifted_month = i64::from(month) + if month > 2 { -3 } else { 9 }; + let day_of_year = (153 * shifted_month + 2) / 5 + i64::from(day) - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +fn invalid(message: impl Into) -> VectorInputError { + VectorCoreError::InvalidProgram(format!( + "Freqtrade OHLCV preparation failed: {}", + message.into() + )) + .into() +} + +#[cfg(test)] +mod tests { + use super::*; + + const ORACLE_START_MS: i64 = 1_704_153_600_000; + + fn identity(timeframe: &str) -> FrameIdentity { + FrameIdentity::new( + "ORACLE/USDT", + Timeframe::parse(timeframe).expect("timeframe"), + ) + .expect("identity") + } + + fn frame( + timeframe: &str, + timestamps_ms: Vec, + rows: impl IntoIterator; 5]>, + ) -> NumericFrame { + let rows = rows.into_iter().collect::>(); + NumericFrame { + identity: identity(timeframe), + timestamps_ms, + columns: OHLCV_COLUMNS + .into_iter() + .enumerate() + .map(|(column, name)| { + ( + name.to_owned(), + rows.iter().map(|row| row[column]).collect(), + ) + }) + .collect(), + } + } + + fn complete_row(value: f64) -> [Option; 5] { + [ + Some(value), + Some(value + 1.0), + Some(value - 1.0), + Some(value + 0.5), + Some(value), + ] + } + + fn assert_column(frame: &NumericFrame, name: &str, expected: &[f64]) { + let actual = frame.columns[name] + .iter() + .map(|value| value.expect("prepared value")) + .collect::>(); + assert_eq!(actual, expected); + } + + #[test] + fn matches_python_duplicate_and_gap_oracle() { + // Generated with vector_worker._clean_ohlcv_like_freqtrade on 2026-08-12. + let raw = frame( + "5m", + vec![1_704_067_200_000, 1_704_067_200_000, 1_704_067_800_000], + vec![ + [Some(10.0), Some(12.0), Some(9.0), Some(11.0), Some(2.0)], + [Some(11.0), Some(13.0), Some(8.0), Some(12.0), Some(3.0)], + [Some(20.0), Some(21.0), Some(19.0), Some(20.0), Some(4.0)], + ], + ); + + let prepared = clean_frame(&raw).expect("prepared"); + + assert_eq!( + prepared.timestamps_ms, + [1_704_067_200_000, 1_704_067_500_000, 1_704_067_800_000] + ); + assert_column(&prepared, "open", &[10.0, 12.0, 20.0]); + assert_column(&prepared, "high", &[13.0, 12.0, 21.0]); + assert_column(&prepared, "low", &[8.0, 12.0, 19.0]); + assert_column(&prepared, "close", &[12.0, 12.0, 20.0]); + assert_column(&prepared, "volume", &[3.0, 0.0, 4.0]); + } + + #[test] + fn matches_python_fixed_anchor_oracles_for_every_native_timeframe() { + // Each expected triplet was generated by the Python helper from two + // candles separated by exactly two timeframe durations. + for (timeframe, duration_ms) in [ + ("5m", 300_000), + ("15m", 900_000), + ("1h", 3_600_000), + ("4h", 14_400_000), + ("1d", 86_400_000), + ] { + let raw = frame( + timeframe, + vec![ORACLE_START_MS, ORACLE_START_MS + 2 * duration_ms], + vec![complete_row(10.0), complete_row(20.0)], + ); + + let prepared = clean_frame(&raw).expect("prepared"); + + assert_eq!( + prepared.timestamps_ms, + [ + ORACLE_START_MS, + ORACLE_START_MS + duration_ms, + ORACLE_START_MS + 2 * duration_ms, + ], + "{timeframe}" + ); + assert_column(&prepared, "open", &[10.0, 10.5, 20.0]); + assert_column(&prepared, "high", &[11.0, 10.5, 21.0]); + assert_column(&prepared, "low", &[9.0, 10.5, 19.0]); + assert_column(&prepared, "close", &[10.5, 10.5, 20.5]); + assert_column(&prepared, "volume", &[10.0, 0.0, 20.0]); + } + } + + #[test] + fn bounds_startup_independently_for_each_timeframe() { + let base = frame( + "5m", + (0..19) + .map(|row| 1_704_150_000_000 + i64::from(row) * 300_000) + .collect(), + (0..19).map(|row| complete_row(f64::from(row))), + ); + let informative = frame( + "15m", + (0..11) + .map(|row| 1_704_146_400_000 + i64::from(row) * 900_000) + .collect(), + (0..11).map(|row| complete_row(f64::from(row))), + ); + let catalog = FrameCatalog::new([ + (base.identity.clone(), base), + (informative.identity.clone(), informative), + ]) + .expect("catalog"); + + let bounded = + prepare_freqtrade_ohlcv_catalog(&catalog, "1704153600-1704155400", 2).expect("bounded"); + let source = SourceLocation::new("test", "oracle", 1, 1); + + assert_eq!( + bounded + .lookup(&identity("5m"), &source) + .expect("base") + .timestamps_ms, + (0..=8) + .map(|row| 1_704_153_000_000 + i64::from(row) * 300_000) + .collect::>() + ); + assert_eq!( + bounded + .lookup(&identity("15m"), &source) + .expect("informative") + .timestamps_ms, + (0..=4) + .map(|row| 1_704_151_800_000 + i64::from(row) * 900_000) + .collect::>() + ); + } + + #[test] + fn execution_positions_are_inclusive_and_context_is_never_executable() { + let source = frame( + "5m", + (0..5) + .map(|row| 1_651_359_300_000 + i64::from(row) * 300_000) + .collect(), + (0..5).map(|row| complete_row(f64::from(row))), + ); + + let positions = execution_positions(&source, "1651360200-1651360500", 3) + .expect("positions") + .expect("nonempty"); + let prepared = + prepare_execution_ohlcv(&source, "1651360200-1651360500", 3).expect("prepared"); + + assert_eq!( + positions, + InclusiveExecutionPositions { + start_index: 3, + stop_index: 4, + } + ); + assert_eq!(prepared.frame.timestamps_ms, source.timestamps_ms); + assert_eq!(prepared.execution_start_index, 4); + } + + #[test] + fn missing_startup_consumes_in_range_rows_and_can_leave_no_execution() { + // Python `_execution_positions` oracle: only one candle precedes the + // boundary, so two of the requested three startup rows come from the + // closed timerange itself. + let source = frame( + "5m", + (0..4) + .map(|row| 1_704_153_300_000 + i64::from(row) * 300_000) + .collect(), + (0..4).map(|row| complete_row(f64::from(row))), + ); + + assert_eq!( + execution_positions(&source, "1704153600-1704154500", 3).expect("positions"), + Some(InclusiveExecutionPositions { + start_index: 3, + stop_index: 3, + }) + ); + let prepared = + prepare_execution_ohlcv(&source, "1704153600-1704154500", 3).expect("prepared"); + assert!(prepared.frame.timestamps_ms.is_empty()); + assert_eq!(prepared.execution_start_index, 0); + } + + #[test] + fn timerange_calendar_seconds_and_milliseconds_are_the_same_closed_range() { + let expected = ClosedTimerange { + start_ms: 1_704_067_200_000, + stop_ms: 1_704_153_600_000, + }; + assert_eq!( + ClosedTimerange::parse("20240101-20240102").expect("calendar"), + expected + ); + assert_eq!( + ClosedTimerange::parse("1704067200-1704153600").expect("seconds"), + expected + ); + assert_eq!( + ClosedTimerange::parse("1704067200000-1704153600000").expect("milliseconds"), + expected + ); + } + + #[test] + fn fails_closed_on_unsupported_timeframe_range_shape_and_arrow_null() { + let unsupported = frame("30m", vec![ORACLE_START_MS], vec![complete_row(1.0)]); + assert!(clean_frame(&unsupported) + .expect_err("unsupported timeframe") + .to_string() + .contains("unsupported Freqtrade OHLCV preparation timeframe")); + + for timerange in [ + "20240101-", + "-20240101", + "20240101-20231231", + "20240230-20240301", + "20240101-20240201-extra", + ] { + assert!(ClosedTimerange::parse(timerange).is_err(), "{timerange}"); + } + + let mut missing = frame("5m", vec![ORACLE_START_MS], vec![complete_row(1.0)]); + missing.columns.remove("volume"); + assert!(clean_frame(&missing) + .expect_err("missing volume") + .to_string() + .contains("missing column \"volume\"")); + + let mut nullable = frame("5m", vec![ORACLE_START_MS], vec![complete_row(1.0)]); + nullable.columns.get_mut("close").expect("close")[0] = None; + assert!(clean_frame(&nullable) + .expect_err("Arrow null") + .to_string() + .contains("contains Arrow null at row 0")); + } + + #[test] + fn present_nan_uses_pandas_missing_value_aggregation() { + let raw = frame( + "5m", + vec![ORACLE_START_MS, ORACLE_START_MS + 600_000], + vec![ + [Some(10.0), Some(11.0), Some(9.0), Some(10.5), Some(3.0)], + [ + Some(f64::NAN), + Some(f64::NAN), + Some(f64::NAN), + Some(f64::NAN), + Some(f64::NAN), + ], + ], + ); + + let prepared = clean_frame(&raw).expect("prepared"); + + assert_column(&prepared, "open", &[10.0, 10.5, 10.5]); + assert_column(&prepared, "high", &[11.0, 10.5, 10.5]); + assert_column(&prepared, "low", &[9.0, 10.5, 10.5]); + assert_column(&prepared, "close", &[10.5, 10.5, 10.5]); + assert_column(&prepared, "volume", &[3.0, 0.0, 0.0]); + } +} diff --git a/rust/crates/nfi-vector-io/src/full_manifest.rs b/rust/crates/nfi-vector-io/src/full_manifest.rs new file mode 100644 index 00000000..62139d53 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_manifest.rs @@ -0,0 +1,38 @@ +//! Strict, versioned input contract for the complete native vector stage. +//! +//! Unlike the legacy simulator-vector manifest, this document binds the three +//! compiled programs and every raw market-data source before Arrow decoding. +//! Raw frame cleanup remains outside this module: verified files are passed to +//! the non-transforming raw loader without aggregation, resampling, gap fill, +//! or timerange trimming. + +mod load; +mod model; +mod validation; + +pub(crate) use load::{decode_verified_frame, load_plan}; +pub use model::{ + CompileContext, FeatureRetention, FuturesFrameSet, HistoricPriceStep, NativeContractError, + NativeVectorBundle, PairContract, PairLimits, PairOptions, PairPrecision, RunContract, + SourceExecutionSeal, SourceSeal, TradingMode, +}; +pub(crate) use model::{NativeVectorPlan, VerifiedFrameSource, VerifiedFuturesSources}; +pub use validation::retained_feature_fingerprint; + +/// The only complete native-vector manifest accepted by this parser. +pub const FULL_NATIVE_VECTOR_MANIFEST_VERSION: &str = "full-native-vector-manifest-v1"; + +/// Parse, SHA-bind, validate, and decode a complete-native vector manifest. +/// +/// # Errors +/// +/// Returns [`NativeContractError`] for an invalid contract, contained-path or +/// digest failure, program identity drift, or raw-frame decode failure. +pub fn load_full_native_vector_manifest( + path: &std::path::Path, +) -> Result { + load::load_bundle(path) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/nfi-vector-io/src/full_manifest/load.rs b/rust/crates/nfi-vector-io/src/full_manifest/load.rs new file mode 100644 index 00000000..1a5ae300 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_manifest/load.rs @@ -0,0 +1,368 @@ +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::{BufReader, Read}; +use std::path::{Path, PathBuf}; + +use nfi_vector_core::alignment::{FrameCatalog, NumericFrame, SourceLocation}; +use nfi_vector_core::mutation::MutationProgram; +use nfi_vector_core::program::IndicatorProgram; +use sha2::{Digest, Sha256}; + +use super::model::{ + ArtifactDocument, FuturesFrameSet, ManifestDocument, NativeContractError, NativeVectorBundle, + NativeVectorPlan, ValidatedDocument, ValidatedFrame, ValidatedFutures, VerifiedFrameSource, + VerifiedFuturesSources, +}; +use super::validation::{invalid, validate_document, validate_program_identity}; +use crate::{load_raw_ohlcv_frame, FeatherFrameSource}; + +/// Parse, bind, and decode one strict complete-native manifest. +/// +/// Artifact paths are resolved below the canonical manifest directory and all +/// digests are checked before any program parsing or Feather decoding. Program +/// fingerprints and compile context are then matched before raw frames are +/// opened by Arrow. +/// +/// # Errors +/// +/// Returns [`NativeContractError`] for unknown fields, invalid/duplicate +/// identities, path escape, digest drift, program/context drift, or raw-frame +/// schema/decode failure. +pub(super) fn load_bundle(path: &Path) -> Result { + let plan = load_plan(path)?; + let frames = decode_catalog(&plan.frames)?; + let futures = decode_futures(&plan.futures)?; + Ok(NativeVectorBundle { + source: plan.source, + source_execution: plan.source_execution, + config: plan.config, + compile_context: plan.compile_context, + run: plan.run, + retained_features: plan.retained_features, + pairs: plan.pairs, + indicator_program: plan.indicator_program, + signal_program: plan.signal_program, + tag_program: plan.tag_program, + frames, + futures, + }) +} + +pub(crate) fn load_plan(path: &Path) -> Result { + let validated = validate_document(read_document(path)?)?; + let verified = verify_all_artifacts(path, &validated)?; + let indicator_program = parse_indicator(&verified.indicator.path)?; + let signal_program = parse_mutation("signal program", &verified.signal.path)?; + let tag_program = parse_mutation("tag program", &verified.tag.path)?; + validate_program_identity( + &validated, + &indicator_program, + &signal_program, + &tag_program, + )?; + let frames = bind_frame_sources(path, &validated.frames, verified.frames); + let futures = bind_futures_sources(path, &validated.futures, verified.futures); + Ok(NativeVectorPlan { + source: validated.source, + source_execution: validated.source_execution, + config: validated.config, + compile_context: validated.compile_context, + run: validated.run, + retained_features: validated.retained_features, + pairs: validated.pairs, + indicator_program, + signal_program, + tag_program, + frames, + futures, + }) +} + +struct VerifiedArtifact { + path: PathBuf, +} + +struct VerifiedArtifacts { + indicator: VerifiedArtifact, + signal: VerifiedArtifact, + tag: VerifiedArtifact, + frames: Vec, + futures: Vec<(VerifiedArtifact, VerifiedArtifact)>, +} + +fn read_document(path: &Path) -> Result { + let encoded = fs::read(path).map_err(|source| NativeContractError::ReadManifest { + path: path.to_path_buf(), + source, + })?; + serde_json::from_slice(&encoded).map_err(|source| NativeContractError::ParseManifest { + path: path.to_path_buf(), + source, + }) +} + +fn verify_all_artifacts( + manifest_path: &Path, + document: &ValidatedDocument, +) -> Result { + let directory = manifest_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .canonicalize() + .map_err(|source| NativeContractError::ReadManifest { + path: manifest_path.to_path_buf(), + source, + })?; + let mut seen_paths = BTreeSet::new(); + let indicator = verify_artifact( + &directory, + "indicator program", + &document.programs.indicator.artifact, + &mut seen_paths, + )?; + let signal = verify_artifact( + &directory, + "signal program", + &document.programs.signal.artifact, + &mut seen_paths, + )?; + let tag = verify_artifact( + &directory, + "tag program", + &document.programs.tag.artifact, + &mut seen_paths, + )?; + let frames = document + .frames + .iter() + .enumerate() + .map(|(index, frame)| { + verify_artifact( + &directory, + &format!("raw frame {index}"), + &frame.artifact, + &mut seen_paths, + ) + }) + .collect::, _>>()?; + let futures = document + .futures + .iter() + .enumerate() + .map(|(index, futures)| { + Ok(( + verify_artifact( + &directory, + &format!("futures funding frame {index}"), + &futures.funding_rate.artifact, + &mut seen_paths, + )?, + verify_artifact( + &directory, + &format!("futures mark frame {index}"), + &futures.mark.artifact, + &mut seen_paths, + )?, + )) + }) + .collect::, NativeContractError>>()?; + Ok(VerifiedArtifacts { + indicator, + signal, + tag, + frames, + futures, + }) +} + +fn verify_artifact( + directory: &Path, + role: &str, + artifact: &ArtifactDocument, + seen_paths: &mut BTreeSet, +) -> Result { + let joined = directory.join(&artifact.path); + let resolved = + joined + .canonicalize() + .map_err(|source| NativeContractError::ResolveArtifact { + role: role.to_owned(), + path: joined, + source, + })?; + if !resolved.starts_with(directory) || !resolved.is_file() { + return Err(NativeContractError::EscapedArtifact { + role: role.to_owned(), + path: resolved, + }); + } + if !seen_paths.insert(resolved.clone()) { + return Err(invalid(format!( + "duplicate artifact path declared for {role}: {}", + artifact.path.display() + ))); + } + let actual = sha256_file(&resolved).map_err(|source| NativeContractError::HashArtifact { + role: role.to_owned(), + path: resolved.clone(), + source, + })?; + if actual != artifact.sha256 { + return Err(NativeContractError::ArtifactDigest { + role: role.to_owned(), + expected: artifact.sha256.clone(), + actual, + }); + } + Ok(VerifiedArtifact { path: resolved }) +} + +fn parse_indicator(path: &Path) -> Result { + let encoded = fs::read_to_string(path).map_err(|source| NativeContractError::ReadManifest { + path: path.to_path_buf(), + source, + })?; + IndicatorProgram::from_json(&encoded).map_err(|source| NativeContractError::Program { + role: "indicator program", + source, + }) +} + +fn parse_mutation(role: &'static str, path: &Path) -> Result { + let encoded = fs::read_to_string(path).map_err(|source| NativeContractError::ReadManifest { + path: path.to_path_buf(), + source, + })?; + MutationProgram::from_json(&encoded) + .map_err(|source| NativeContractError::Program { role, source }) +} + +fn bind_frame_sources( + manifest_path: &Path, + contracts: &[ValidatedFrame], + artifacts: Vec, +) -> Vec { + let mut sources = Vec::with_capacity(contracts.len()); + for (index, (contract, artifact)) in contracts.iter().zip(artifacts).enumerate() { + sources.push(bind_frame_source( + manifest_path, + format!("manifest-frame-{index}"), + contract, + artifact, + )); + } + sources +} + +fn bind_futures_sources( + manifest_path: &Path, + contracts: &[ValidatedFutures], + artifacts: Vec<(VerifiedArtifact, VerifiedArtifact)>, +) -> Vec { + contracts + .iter() + .zip(artifacts) + .enumerate() + .map( + |(index, (contract, (funding, mark)))| VerifiedFuturesSources { + pair: contract.pair.clone(), + funding_rate: bind_frame_source( + manifest_path, + format!("manifest-funding-{index}"), + &contract.funding_rate, + funding, + ), + mark: bind_frame_source( + manifest_path, + format!("manifest-mark-{index}"), + &contract.mark, + mark, + ), + }, + ) + .collect() +} + +fn bind_frame_source( + manifest_path: &Path, + node: String, + contract: &ValidatedFrame, + artifact: VerifiedArtifact, +) -> VerifiedFrameSource { + let source = FeatherFrameSource::new( + contract.identity.clone(), + artifact.path, + SourceLocation::new(node, manifest_path.display().to_string(), 0, 0), + ); + VerifiedFrameSource { + identity: contract.identity.clone(), + rows: contract.rows, + source, + } +} + +pub(crate) fn decode_verified_frame( + source: &VerifiedFrameSource, + role: &str, +) -> Result { + let frame = + load_raw_ohlcv_frame(&source.source).map_err(|source| NativeContractError::RawFrame { + role: role.to_owned(), + source, + })?; + let actual = frame.timestamps_ms.len(); + if actual != source.rows { + return Err(invalid(format!( + "{role} row count differs: expected {}, got {actual}", + source.rows + ))); + } + Ok(frame) +} + +fn decode_catalog(sources: &[VerifiedFrameSource]) -> Result { + let entries = sources + .iter() + .enumerate() + .map(|(index, source)| { + Ok(( + source.identity.clone(), + decode_verified_frame(source, &format!("raw frame {index}"))?, + )) + }) + .collect::, NativeContractError>>()?; + FrameCatalog::new(entries).map_err(|error| invalid(error.to_string())) +} + +fn decode_futures( + sources: &[VerifiedFuturesSources], +) -> Result, NativeContractError> { + sources + .iter() + .enumerate() + .map(|(index, source)| { + Ok(FuturesFrameSet { + pair: source.pair.clone(), + funding_rate: decode_verified_frame( + &source.funding_rate, + &format!("futures funding frame {index}"), + )?, + mark: decode_verified_frame(&source.mark, &format!("futures mark frame {index}"))?, + }) + }) + .collect() +} + +fn sha256_file(path: &Path) -> Result { + let mut reader = BufReader::new(File::open(path)?); + let mut hasher = Sha256::new(); + let mut buffer = vec![0_u8; 1024 * 1024].into_boxed_slice(); + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + } + Ok(format!("{:x}", hasher.finalize())) +} diff --git a/rust/crates/nfi-vector-io/src/full_manifest/model.rs b/rust/crates/nfi-vector-io/src/full_manifest/model.rs new file mode 100644 index 00000000..275aa0d1 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_manifest/model.rs @@ -0,0 +1,391 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; + +use nfi_sim_core::PortfolioConfig; +use nfi_vector_core::alignment::{FrameCatalog, FrameIdentity, NumericFrame, Timeframe}; +use nfi_vector_core::mutation::MutationProgram; +use nfi_vector_core::program::IndicatorProgram; +use serde::Deserialize; +use thiserror::Error; + +/// A fully verified and decoded complete-native input bundle. +#[derive(Clone, Debug)] +pub struct NativeVectorBundle { + pub source: SourceSeal, + pub source_execution: SourceExecutionSeal, + pub config: PortfolioConfig, + pub compile_context: CompileContext, + pub run: RunContract, + pub retained_features: FeatureRetention, + pub pairs: Vec, + pub indicator_program: IndicatorProgram, + pub signal_program: MutationProgram, + pub tag_program: MutationProgram, + pub frames: FrameCatalog, + pub futures: Vec, +} + +/// SHA-verified manifest whose raw frames remain unopened until their pair DAG. +#[derive(Debug)] +pub(crate) struct NativeVectorPlan { + pub(crate) source: SourceSeal, + pub(crate) source_execution: SourceExecutionSeal, + pub(crate) config: PortfolioConfig, + pub(crate) compile_context: CompileContext, + pub(crate) run: RunContract, + pub(crate) retained_features: FeatureRetention, + pub(crate) pairs: Vec, + pub(crate) indicator_program: IndicatorProgram, + pub(crate) signal_program: MutationProgram, + pub(crate) tag_program: MutationProgram, + pub(crate) frames: Vec, + pub(crate) futures: Vec, +} + +#[derive(Clone, Debug)] +pub(crate) struct VerifiedFrameSource { + pub(crate) identity: FrameIdentity, + pub(crate) rows: usize, + pub(crate) source: crate::FeatherFrameSource, +} + +#[derive(Clone, Debug)] +pub(crate) struct VerifiedFuturesSources { + pub(crate) pair: String, + pub(crate) funding_rate: VerifiedFrameSource, + pub(crate) mark: VerifiedFrameSource, +} + +/// Static inputs that produced the compiled programs. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SourceSeal { + pub strategy_sha256: String, + pub config_sha256: String, + pub compiler_source_fingerprint: String, + pub selected_class: String, +} + +/// Sealed proof that strategy source was compiled structurally and runs only in Rust. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SourceExecutionSeal { + pub strategy_source_mode: String, + pub populate_methods_executed: bool, + pub runtime_mode: String, +} + +/// Runtime context frozen by both mutation programs and the manifest. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CompileContext { + pub run_mode: String, + pub trading_mode: TradingMode, +} + +/// Supported Freqtrade trading modes for the native lane. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum TradingMode { + Spot, + Futures, +} + +impl TradingMode { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Spot => "spot", + Self::Futures => "futures", + } + } +} + +/// Chronological vector-run semantics supplied to the native executor. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RunContract { + pub trading_mode: TradingMode, + pub timerange_start_ms: i64, + pub timerange_stop_ms: i64, + pub startup_candles: usize, + pub base_timeframe: Timeframe, + pub source_row_shift: usize, +} + +/// Dynamically selected analyzed columns retained for callbacks/simulation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FeatureRetention { + pub columns: Vec, + pub fingerprint: String, +} + +/// Pair-local metadata and exchange precision/limit contract. +#[derive(Clone, Debug, PartialEq)] +pub struct PairContract { + pub identity: FrameIdentity, + pub metadata: BTreeMap, + pub precision: PairPrecision, + pub limits: PairLimits, + pub price_steps: Vec, + pub options: PairOptions, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct PairPrecision { + pub amount_step: Option, + pub price_step: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct PairLimits { + pub minimum_stake: Option, + pub minimum_amount: Option, + pub minimum_cost: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct HistoricPriceStep { + pub timestamp_ms: i64, + pub step: f64, +} + +/// Pair-local execution capabilities consumed without mode inference. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(clippy::struct_excessive_bools)] // Mirrors four independent legacy transport flags. +pub struct PairOptions { + pub can_short: bool, + pub include_funding: bool, + pub use_exit_signal: bool, + pub include_previous_close: bool, +} + +/// Sparse Futures sources remain separate because their pair/timeframe +/// identities can be equal while their semantic roles differ. +#[derive(Clone, Debug, PartialEq)] +pub struct FuturesFrameSet { + pub pair: String, + pub funding_rate: NumericFrame, + pub mark: NumericFrame, +} + +/// Fail-closed errors from the complete native manifest boundary. +#[derive(Debug, Error)] +pub enum NativeContractError { + #[error("cannot read full native vector manifest {path}: {source}")] + ReadManifest { + path: PathBuf, + source: std::io::Error, + }, + #[error("invalid full native vector manifest {path}: {source}")] + ParseManifest { + path: PathBuf, + source: serde_json::Error, + }, + #[error("invalid full native vector contract: {0}")] + Invalid(String), + #[error("cannot resolve full native artifact {role} at {path}: {source}")] + ResolveArtifact { + role: String, + path: PathBuf, + source: std::io::Error, + }, + #[error("full native artifact {role} escapes the manifest directory: {path}")] + EscapedArtifact { role: String, path: PathBuf }, + #[error("cannot hash full native artifact {role} at {path}: {source}")] + HashArtifact { + role: String, + path: PathBuf, + source: std::io::Error, + }, + #[error("full native artifact {role} SHA-256 mismatch: expected {expected}, got {actual}")] + ArtifactDigest { + role: String, + expected: String, + actual: String, + }, + #[error("invalid full native {role}: {source}")] + Program { + role: &'static str, + source: nfi_vector_core::VectorCoreError, + }, + #[error("cannot decode full native {role}: {source}")] + RawFrame { + role: String, + source: crate::VectorInputError, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ManifestDocument { + pub(super) schema_version: String, + pub(super) source: SourceDocument, + pub(super) source_execution: SourceExecutionDocument, + pub(super) config: serde_json::Value, + pub(super) compile_context: CompileContextDocument, + pub(super) programs: ProgramDocuments, + pub(super) run: RunDocument, + pub(super) retained_features: FeatureDocument, + pub(super) pairs: Vec, + pub(super) frames: Vec, + pub(super) futures: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct SourceDocument { + pub(super) strategy_sha256: String, + pub(super) config_sha256: String, + pub(super) compiler_source_fingerprint: String, + pub(super) selected_class: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct SourceExecutionDocument { + pub(super) strategy_source_mode: String, + pub(super) populate_methods_executed: bool, + pub(super) runtime_mode: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct CompileContextDocument { + pub(super) run_mode: String, + pub(super) trading_mode: TradingMode, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ProgramDocuments { + pub(super) indicator: ProgramDocument, + pub(super) signal: ProgramDocument, + pub(super) tag: ProgramDocument, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ProgramDocument { + pub(super) artifact: ArtifactDocument, + pub(super) fingerprint: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ArtifactDocument { + pub(super) path: PathBuf, + pub(super) sha256: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RunDocument { + pub(super) trading_mode: TradingMode, + pub(super) timerange: TimerangeDocument, + pub(super) startup_candles: usize, + pub(super) base_timeframe: String, + pub(super) source_row_shift: usize, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TimerangeDocument { + pub(super) start_ms: i64, + pub(super) stop_ms: i64, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct FeatureDocument { + pub(super) columns: Vec, + pub(super) fingerprint: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PairDocument { + pub(super) identity: IdentityDocument, + pub(super) metadata: BTreeMap, + pub(super) precision: PrecisionDocument, + pub(super) limits: LimitsDocument, + pub(super) price_steps: Vec, + pub(super) options: PairOptionsDocument, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[allow(clippy::struct_excessive_bools)] // Strict JSON form of the four independent flags. +pub(super) struct PairOptionsDocument { + pub(super) can_short: bool, + pub(super) include_funding: bool, + pub(super) use_exit_signal: bool, + pub(super) include_previous_close: bool, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct IdentityDocument { + pub(super) pair: String, + pub(super) timeframe: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PrecisionDocument { + pub(super) amount_step: Option, + pub(super) price_step: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[allow(clippy::struct_field_names)] // Exact public Freqtrade market-limit field names. +pub(super) struct LimitsDocument { + pub(super) minimum_stake: Option, + pub(super) minimum_amount: Option, + pub(super) minimum_cost: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PriceStepDocument { + pub(super) timestamp_ms: i64, + pub(super) step: f64, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct FrameDocument { + pub(super) identity: IdentityDocument, + pub(super) rows: usize, + pub(super) artifact: ArtifactDocument, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct FuturesDocument { + pub(super) pair: String, + pub(super) funding_rate: FrameDocument, + pub(super) mark: FrameDocument, +} + +pub(super) struct ValidatedDocument { + pub(super) source: SourceSeal, + pub(super) source_execution: SourceExecutionSeal, + pub(super) config: PortfolioConfig, + pub(super) compile_context: CompileContext, + pub(super) programs: ProgramDocuments, + pub(super) run: RunContract, + pub(super) retained_features: FeatureRetention, + pub(super) pairs: Vec, + pub(super) frames: Vec, + pub(super) futures: Vec, +} + +pub(super) struct ValidatedFrame { + pub(super) identity: FrameIdentity, + pub(super) rows: usize, + pub(super) artifact: ArtifactDocument, +} + +pub(super) struct ValidatedFutures { + pub(super) pair: String, + pub(super) funding_rate: ValidatedFrame, + pub(super) mark: ValidatedFrame, +} diff --git a/rust/crates/nfi-vector-io/src/full_manifest/tests.rs b/rust/crates/nfi-vector-io/src/full_manifest/tests.rs new file mode 100644 index 00000000..1f2feb93 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_manifest/tests.rs @@ -0,0 +1,355 @@ +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; + +use arrow2::array::{Array, PrimitiveArray}; +use arrow2::chunk::Chunk; +use arrow2::datatypes::{DataType, Field, Schema, TimeUnit}; +use arrow2::io::ipc::write::{FileWriter, WriteOptions}; +use nfi_vector_core::alignment::{FrameIdentity, SourceLocation, Timeframe}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::*; + +const INDICATOR: &str = + include_str!("../../../../../benchmarks/reference/vector-shadow/indicator-program.json"); +const SIGNAL: &str = + include_str!("../../../../../benchmarks/reference/vector-shadow/signal-program.json"); +const TAG: &str = + include_str!("../../../../../benchmarks/reference/vector-shadow/tag-program.json"); +const STRATEGY_SHA: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const CLASS_NAME: &str = "NativeManifestContract"; + +struct Fixture { + temporary: tempfile::TempDir, + path: PathBuf, + document: Value, +} + +impl Fixture { + fn write_manifest(&self) { + fs::write( + &self.path, + serde_json::to_vec(&self.document).expect("manifest JSON"), + ) + .expect("write manifest"); + } +} + +fn fixture(mode: TradingMode) -> Fixture { + let temporary = tempfile::tempdir().expect("temporary"); + let root = temporary.path(); + let indicator = write_program(root, "indicator.json", INDICATOR, mode); + let signal = write_program(root, "signal.json", SIGNAL, mode); + let tag = write_program(root, "tag.json", TAG, mode); + let frame_path = root.join("data/BTC_USDT-5m.feather"); + write_ohlcv(&frame_path, 100); + let features = vec!["delta".to_owned()]; + let config = json!({ + "starting_balance": 1_000.0, + "max_open_trades": 2, + "stake_amount": 100.0, + "fee_rate": 0.001, + "stoploss_ratio": -0.2, + "amount_step": 0.001, + "price_step": 0.01, + "is_futures": mode == TradingMode::Futures + }); + let config_sha = validation::config_identity_sha256(&config).expect("config identity"); + let document = json!({ + "schema_version": FULL_NATIVE_VECTOR_MANIFEST_VERSION, + "source": { + "strategy_sha256": STRATEGY_SHA, + "config_sha256": config_sha, + "compiler_source_fingerprint": + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "selected_class": CLASS_NAME + }, + "source_execution": { + "strategy_source_mode": "python-ast-compile-only", + "populate_methods_executed": false, + "runtime_mode": "rust-full-native" + }, + "config": config, + "compile_context": {"run_mode": "backtest", "trading_mode": mode.as_str()}, + "programs": {"indicator": indicator, "signal": signal, "tag": tag}, + "run": { + "trading_mode": mode.as_str(), + "timerange": {"start_ms": 0, "stop_ms": 1_000}, + "startup_candles": 17, + "base_timeframe": "5m", + "source_row_shift": 3 + }, + "retained_features": { + "columns": features, + "fingerprint": retained_feature_fingerprint(&features) + }, + "pairs": [{ + "identity": {"pair": "BTC/USDT", "timeframe": "5m"}, + "metadata": {"pair": "BTC/USDT", "market": "test"}, + "precision": {"amount_step": 0.001, "price_step": 0.01}, + "limits": { + "minimum_stake": 10.0, + "minimum_amount": 0.001, + "minimum_cost": 5.0 + }, + "price_steps": [ + {"timestamp_ms": 0, "step": 0.01}, + {"timestamp_ms": 500, "step": 0.1} + ], + "options": { + "can_short": mode == TradingMode::Futures, + "include_funding": mode == TradingMode::Futures, + "use_exit_signal": true, + "include_previous_close": true + } + }], + "frames": [{ + "identity": {"pair": "BTC/USDT", "timeframe": "5m"}, + "rows": 1, + "artifact": artifact(root, &frame_path) + }], + "futures": null + }); + let path = root.join("manifest.json"); + let fixture = Fixture { + temporary, + path, + document, + }; + fixture.write_manifest(); + fixture +} + +fn write_program(root: &Path, name: &str, encoded: &str, mode: TradingMode) -> Value { + let mut program: Value = serde_json::from_str(encoded).expect("reference program"); + program["source"]["path"] = json!("strategy.py"); + program["source"]["sha256"] = json!(STRATEGY_SHA); + program["selected_class"] = json!(CLASS_NAME); + if program.get("compile_context").is_some() { + program["compile_context"] = json!({"run_mode": "backtest", "trading_mode": mode.as_str()}); + } + reseal_program(&mut program); + let path = root.join(name); + fs::write(&path, serde_json::to_vec(&program).expect("program JSON")).expect("write program"); + json!({ + "artifact": artifact(root, &path), + "fingerprint": program["fingerprint"] + }) +} + +fn reseal_program(program: &mut Value) { + let mut identity = program.clone(); + let object = identity.as_object_mut().expect("program object"); + object.remove("fingerprint"); + object["source"] + .as_object_mut() + .expect("source object") + .remove("path"); + program["fingerprint"] = json!(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&identity).expect("program identity")) + )); +} + +fn artifact(root: &Path, path: &Path) -> Value { + json!({ + "path": path.strip_prefix(root).expect("contained path"), + "sha256": format!("{:x}", Sha256::digest(fs::read(path).expect("artifact bytes"))) + }) +} + +fn reseal_config(document: &mut Value) { + document["source"]["config_sha256"] = + json!(validation::config_identity_sha256(&document["config"]) + .expect("embedded config identity")); +} + +fn write_ohlcv(path: &Path, timestamp_ms: i64) { + fs::create_dir_all(path.parent().expect("parent")).expect("data directory"); + let fields = [Field::new( + "date", + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".to_owned())), + false, + )] + .into_iter() + .chain( + ["open", "high", "low", "close", "volume"] + .map(|name| Field::new(name, DataType::Float64, false)), + ) + .collect::>(); + let mut arrays: Vec> = + vec![Box::new(PrimitiveArray::from_vec(vec![timestamp_ms]).to( + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".to_owned())), + ))]; + arrays.extend((0..5).map(|index| { + Box::new(PrimitiveArray::from_vec(vec![f64::from(index + 1)])) as Box + })); + let mut writer = FileWriter::try_new( + File::create(path).expect("Feather file"), + Schema::from(fields), + None, + WriteOptions { compression: None }, + ) + .expect("Feather writer"); + writer + .write(&Chunk::new(arrays), None) + .expect("Feather batch"); + writer.finish().expect("finish Feather"); +} + +#[test] +fn config_identity_matches_python_without_json_float_formatting() { + let left = json!({ + "z": [null, true, false, 7, -3, 1e-5, -0.0], + "a": {"한글": "값"} + }); + let right = json!({ + "a": {"한글": "값"}, + "z": [null, true, false, 7, -3, 0.00001, -0.0] + }); + let expected = "df8efe5440e003a372b0ae0d57c7dcd360517af8ace16e68601e862f48a79525"; + + assert_eq!( + validation::config_identity_sha256(&left).expect("left identity"), + expected + ); + assert_eq!( + validation::config_identity_sha256(&right).expect("right identity"), + expected + ); +} + +#[test] +fn loads_strict_spot_contract_and_dynamic_runtime_fields() { + let fixture = fixture(TradingMode::Spot); + let loaded = load_full_native_vector_manifest(&fixture.path).expect("complete bundle"); + assert_eq!(loaded.run.source_row_shift, 3); + assert_eq!( + loaded.source_execution.strategy_source_mode, + "python-ast-compile-only" + ); + assert!(!loaded.source_execution.populate_methods_executed); + assert_eq!(loaded.source_execution.runtime_mode, "rust-full-native"); + assert_eq!(loaded.run.startup_candles, 17); + assert_eq!(loaded.retained_features.columns, ["delta"]); + assert_eq!(loaded.pairs[0].metadata["market"], "test"); + assert_eq!(loaded.pairs[0].price_steps.len(), 2); + assert!(loaded.futures.is_empty()); + let identity = FrameIdentity::new("BTC/USDT", Timeframe::parse("5m").expect("timeframe")) + .expect("identity"); + assert_eq!( + loaded + .frames + .lookup(&identity, &SourceLocation::new("test", "test.rs", 1, 0),) + .expect("base frame") + .timestamps_ms, + [100] + ); +} + +#[test] +fn rejects_unknown_fields_paths_digests_and_duplicates() { + let mut unknown = fixture(TradingMode::Spot); + unknown.document["unknown"] = json!(true); + unknown.write_manifest(); + assert!(load_full_native_vector_manifest(&unknown.path) + .expect_err("unknown field") + .to_string() + .contains("unknown field")); + + let mut escaped = fixture(TradingMode::Spot); + escaped.document["programs"]["indicator"]["artifact"]["path"] = json!("../x.json"); + escaped.write_manifest(); + assert!(load_full_native_vector_manifest(&escaped.path) + .expect_err("parent path") + .to_string() + .contains("non-contained component")); + + let mut digest = fixture(TradingMode::Spot); + digest.document["frames"][0]["artifact"]["sha256"] = json!("0".repeat(64)); + digest.write_manifest(); + assert!(matches!( + load_full_native_vector_manifest(&digest.path), + Err(NativeContractError::ArtifactDigest { .. }) + )); + + let mut duplicate = fixture(TradingMode::Spot); + let second = duplicate.temporary.path().join("data/second.feather"); + write_ohlcv(&second, 200); + let mut frame = duplicate.document["frames"][0].clone(); + frame["artifact"] = artifact(duplicate.temporary.path(), &second); + duplicate.document["frames"] + .as_array_mut() + .expect("frames") + .push(frame); + duplicate.write_manifest(); + assert!(load_full_native_vector_manifest(&duplicate.path) + .expect_err("duplicate identity") + .to_string() + .contains("duplicate raw frame identity")); +} + +#[test] +fn rejects_context_and_feature_drift_before_raw_decode() { + let mut source_execution = fixture(TradingMode::Spot); + source_execution.document["source_execution"]["populate_methods_executed"] = json!(true); + source_execution.write_manifest(); + assert!(load_full_native_vector_manifest(&source_execution.path) + .expect_err("strategy execution claim") + .to_string() + .contains("must forbid populate execution")); + + let mut context = fixture(TradingMode::Spot); + let raw = context.temporary.path().join("data/BTC_USDT-5m.feather"); + fs::write(&raw, b"not Feather").expect("corrupt raw frame"); + context.document["frames"][0]["artifact"] = artifact(context.temporary.path(), &raw); + context.document["compile_context"]["trading_mode"] = json!("futures"); + context.document["run"]["trading_mode"] = json!("futures"); + context.document["config"]["is_futures"] = json!(true); + context.document["pairs"][0]["options"]["can_short"] = json!(true); + context.document["pairs"][0]["options"]["include_funding"] = json!(true); + reseal_config(&mut context.document); + context.write_manifest(); + let error = load_full_native_vector_manifest(&context.path) + .expect_err("compile context must fail before Arrow"); + assert!(error.to_string().contains("compile context differs")); + assert!(!error.to_string().contains("cannot decode")); + + let mut features = fixture(TradingMode::Spot); + features.document["retained_features"]["columns"] = json!(["open", "open"]); + features.write_manifest(); + assert!(load_full_native_vector_manifest(&features.path) + .expect_err("duplicate feature") + .to_string() + .contains("empty or duplicate")); +} + +#[test] +fn loads_optional_futures_funding_and_mark_descriptors() { + let mut fixture = fixture(TradingMode::Futures); + let root = fixture.temporary.path(); + let funding = root.join("data/BTC_USDT-1h-funding.feather"); + let mark = root.join("data/BTC_USDT-1h-mark.feather"); + write_ohlcv(&funding, 300); + write_ohlcv(&mark, 300); + fixture.document["futures"] = json!([{ + "pair": "BTC/USDT", + "funding_rate": { + "identity": {"pair": "BTC/USDT", "timeframe": "1h"}, + "rows": 1, + "artifact": artifact(root, &funding) + }, + "mark": { + "identity": {"pair": "BTC/USDT", "timeframe": "1h"}, + "rows": 1, + "artifact": artifact(root, &mark) + } + }]); + fixture.write_manifest(); + let loaded = load_full_native_vector_manifest(&fixture.path).expect("Futures bundle"); + assert_eq!(loaded.futures.len(), 1); + assert_eq!(loaded.futures[0].pair, "BTC/USDT"); + assert_eq!(loaded.futures[0].funding_rate.timestamps_ms, [300]); + assert_eq!(loaded.futures[0].mark.timestamps_ms, [300]); +} diff --git a/rust/crates/nfi-vector-io/src/full_manifest/validation.rs b/rust/crates/nfi-vector-io/src/full_manifest/validation.rs new file mode 100644 index 00000000..ab96605f --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_manifest/validation.rs @@ -0,0 +1,579 @@ +use std::collections::BTreeSet; +use std::path::Component; + +use nfi_sim_core::{validate_simulator_preflight, PortfolioConfig}; +use nfi_vector_core::alignment::{FrameIdentity, Timeframe}; +use nfi_vector_core::mutation::MutationProgram; +use nfi_vector_core::program::IndicatorProgram; +use sha2::{Digest, Sha256}; + +use super::model::{ + ArtifactDocument, CompileContext, FeatureRetention, FrameDocument, HistoricPriceStep, + IdentityDocument, ManifestDocument, NativeContractError, PairContract, PairLimits, PairOptions, + PairPrecision, RunContract, SourceExecutionSeal, SourceSeal, TradingMode, ValidatedDocument, + ValidatedFrame, ValidatedFutures, +}; +use super::FULL_NATIVE_VECTOR_MANIFEST_VERSION; + +pub(super) fn validate_document( + document: ManifestDocument, +) -> Result { + let (source, source_execution, config, compile_context, run) = validate_header(&document)?; + validate_program_descriptors(&document)?; + let retained_features = validate_features(document.retained_features)?; + let (pairs, pair_names) = validate_pairs(document.pairs, &run)?; + let frames = validate_base_frames(document.frames, &pairs)?; + let futures = validate_futures( + document.futures.unwrap_or_default(), + run.trading_mode, + &pair_names, + )?; + Ok(ValidatedDocument { + source, + source_execution, + config, + compile_context, + programs: document.programs, + run, + retained_features, + pairs, + frames, + futures, + }) +} + +fn validate_header( + document: &ManifestDocument, +) -> Result< + ( + SourceSeal, + SourceExecutionSeal, + PortfolioConfig, + CompileContext, + RunContract, + ), + NativeContractError, +> { + if document.schema_version != FULL_NATIVE_VECTOR_MANIFEST_VERSION { + return Err(invalid(format!( + "unsupported schema_version {:?}", + document.schema_version + ))); + } + let source = SourceSeal { + strategy_sha256: checked_digest( + "strategy_sha256", + document.source.strategy_sha256.clone(), + )?, + config_sha256: checked_digest("config_sha256", document.source.config_sha256.clone())?, + compiler_source_fingerprint: checked_digest( + "compiler_source_fingerprint", + document.source.compiler_source_fingerprint.clone(), + )?, + selected_class: checked_name("selected_class", document.source.selected_class.clone())?, + }; + if document.source_execution.strategy_source_mode != "python-ast-compile-only" + || document.source_execution.populate_methods_executed + || document.source_execution.runtime_mode != "rust-full-native" + { + return Err(invalid( + "source_execution must forbid populate execution and select the Rust full-native runtime", + )); + } + let source_execution = SourceExecutionSeal { + strategy_source_mode: document.source_execution.strategy_source_mode.clone(), + populate_methods_executed: document.source_execution.populate_methods_executed, + runtime_mode: document.source_execution.runtime_mode.clone(), + }; + let actual_config_sha = config_identity_sha256(&document.config)?; + if actual_config_sha != source.config_sha256 { + return Err(invalid( + "config_sha256 differs from the embedded simulator config", + )); + } + let config: PortfolioConfig = serde_json::from_value(document.config.clone()) + .map_err(|error| invalid(format!("embedded simulator config is invalid: {error}")))?; + validate_simulator_preflight(&config).map_err(|error| { + invalid(format!( + "embedded simulator config fails runtime preflight: {error}" + )) + })?; + if document.compile_context.run_mode != "backtest" { + return Err(invalid("compile_context.run_mode must be backtest")); + } + if document.run.trading_mode != document.compile_context.trading_mode { + return Err(invalid( + "run trading_mode differs from the compiled trading_mode", + )); + } + if config.is_futures != (document.run.trading_mode == TradingMode::Futures) { + return Err(invalid( + "simulator config is_futures differs from run trading_mode", + )); + } + if document.run.timerange.start_ms < 0 + || document.run.timerange.stop_ms < document.run.timerange.start_ms + { + return Err(invalid("run timerange millisecond bounds are invalid")); + } + let base_timeframe = Timeframe::parse(document.run.base_timeframe.clone()) + .map_err(|error| invalid(format!("run base_timeframe is invalid: {error}")))?; + Ok(( + source, + source_execution, + config, + CompileContext { + run_mode: document.compile_context.run_mode.clone(), + trading_mode: document.compile_context.trading_mode, + }, + RunContract { + trading_mode: document.run.trading_mode, + timerange_start_ms: document.run.timerange.start_ms, + timerange_stop_ms: document.run.timerange.stop_ms, + startup_candles: document.run.startup_candles, + base_timeframe, + source_row_shift: document.run.source_row_shift, + }, + )) +} + +pub(super) fn config_identity_sha256( + value: &serde_json::Value, +) -> Result { + let mut digest = Sha256::new(); + update_config_identity(&mut digest, value)?; + Ok(format!("{:x}", digest.finalize())) +} + +fn update_config_identity( + digest: &mut Sha256, + value: &serde_json::Value, +) -> Result<(), NativeContractError> { + match value { + serde_json::Value::Null => digest.update(b"N"), + serde_json::Value::Bool(value) => { + digest.update(if *value { b"B\x01" } else { b"B\x00" }); + } + serde_json::Value::Number(value) => { + if let Some(integer) = value.as_i64() { + update_integer(digest, integer.to_string().as_bytes())?; + } else if let Some(integer) = value.as_u64() { + update_integer(digest, integer.to_string().as_bytes())?; + } else if let Some(float) = value.as_f64().filter(|float| float.is_finite()) { + digest.update(b"F"); + digest.update(float.to_bits().to_be_bytes()); + } else { + return Err(invalid( + "embedded simulator config contains an invalid number", + )); + } + } + serde_json::Value::String(value) => update_string(digest, value)?, + serde_json::Value::Array(values) => { + digest.update(b"L"); + update_length(digest, values.len())?; + for value in values { + update_config_identity(digest, value)?; + } + } + serde_json::Value::Object(values) => { + digest.update(b"O"); + update_length(digest, values.len())?; + let mut keys = values.keys().collect::>(); + keys.sort_unstable(); + for key in keys { + update_string(digest, key)?; + update_config_identity(digest, &values[key])?; + } + } + } + Ok(()) +} + +fn update_integer(digest: &mut Sha256, encoded: &[u8]) -> Result<(), NativeContractError> { + digest.update(b"I"); + update_length(digest, encoded.len())?; + digest.update(encoded); + Ok(()) +} + +fn update_string(digest: &mut Sha256, value: &str) -> Result<(), NativeContractError> { + let encoded = value.as_bytes(); + digest.update(b"S"); + update_length(digest, encoded.len())?; + digest.update(encoded); + Ok(()) +} + +fn update_length(digest: &mut Sha256, length: usize) -> Result<(), NativeContractError> { + let length = u64::try_from(length) + .map_err(|_| invalid("embedded simulator config container is too large"))?; + digest.update(length.to_be_bytes()); + Ok(()) +} + +fn validate_program_descriptors(document: &ManifestDocument) -> Result<(), NativeContractError> { + for (role, program) in [ + ("indicator", &document.programs.indicator), + ("signal", &document.programs.signal), + ("tag", &document.programs.tag), + ] { + checked_digest( + &format!("{role} program fingerprint"), + program.fingerprint.clone(), + )?; + checked_artifact(role, &program.artifact)?; + } + Ok(()) +} + +fn validate_features( + document: super::model::FeatureDocument, +) -> Result { + validate_unique_names(&document.columns, "retained feature columns")?; + let fingerprint = checked_digest("retained feature fingerprint", document.fingerprint)?; + if fingerprint != retained_feature_fingerprint(&document.columns) { + return Err(invalid( + "retained feature fingerprint differs from the ordered column list", + )); + } + Ok(FeatureRetention { + columns: document.columns, + fingerprint, + }) +} + +fn validate_pairs( + documents: Vec, + run: &RunContract, +) -> Result<(Vec, BTreeSet), NativeContractError> { + if documents.is_empty() { + return Err(invalid("pairs must be non-empty")); + } + let mut names = BTreeSet::new(); + let mut pairs = Vec::with_capacity(documents.len()); + for document in documents { + let identity = frame_identity(document.identity)?; + if identity.timeframe != run.base_timeframe { + return Err(invalid(format!( + "pair {} base timeframe differs from run base_timeframe", + identity.pair + ))); + } + if !names.insert(identity.pair.clone()) { + return Err(invalid(format!( + "duplicate pair contract {}", + identity.pair + ))); + } + match document.metadata.get("pair") { + Some(metadata_pair) if metadata_pair == &identity.pair => {} + _ => { + return Err(invalid(format!( + "metadata pair differs from pair contract {}", + identity.pair + ))); + } + } + if document.metadata.keys().any(String::is_empty) { + return Err(invalid(format!( + "pair {} metadata contains an empty key", + identity.pair + ))); + } + validate_optional_positive(document.precision.amount_step, "amount_step")?; + validate_optional_positive(document.precision.price_step, "price_step")?; + validate_optional_nonnegative(document.limits.minimum_stake, "minimum_stake")?; + validate_optional_nonnegative(document.limits.minimum_amount, "minimum_amount")?; + validate_optional_nonnegative(document.limits.minimum_cost, "minimum_cost")?; + if run.trading_mode == TradingMode::Spot + && (document.options.can_short || document.options.include_funding) + { + return Err(invalid(format!( + "Spot pair {} cannot enable short or funding options", + identity.pair + ))); + } + if run.trading_mode == TradingMode::Futures && !document.options.include_funding { + return Err(invalid(format!( + "Futures pair {} must enable funding input", + identity.pair + ))); + } + let price_steps = validate_price_steps(document.price_steps)?; + pairs.push(PairContract { + identity, + metadata: document.metadata, + precision: PairPrecision { + amount_step: document.precision.amount_step, + price_step: document.precision.price_step, + }, + limits: PairLimits { + minimum_stake: document.limits.minimum_stake, + minimum_amount: document.limits.minimum_amount, + minimum_cost: document.limits.minimum_cost, + }, + price_steps, + options: PairOptions { + can_short: document.options.can_short, + include_funding: document.options.include_funding, + use_exit_signal: document.options.use_exit_signal, + include_previous_close: document.options.include_previous_close, + }, + }); + } + Ok((pairs, names)) +} + +fn validate_price_steps( + documents: Vec, +) -> Result, NativeContractError> { + let mut prior = None; + documents + .into_iter() + .map(|step| { + if step.timestamp_ms < 0 + || !step.step.is_finite() + || step.step <= 0.0 + || prior.is_some_and(|value| step.timestamp_ms <= value) + { + return Err(invalid("price_steps must be positive and strictly ordered")); + } + prior = Some(step.timestamp_ms); + Ok(HistoricPriceStep { + timestamp_ms: step.timestamp_ms, + step: step.step, + }) + }) + .collect() +} + +fn validate_base_frames( + documents: Vec, + pairs: &[PairContract], +) -> Result, NativeContractError> { + if documents.is_empty() { + return Err(invalid("raw frames must be non-empty")); + } + let mut identities = BTreeSet::new(); + let frames = documents + .into_iter() + .map(|frame| validate_frame(frame, &mut identities)) + .collect::, _>>()?; + for pair in pairs { + if !identities.contains(&pair.identity) { + return Err(invalid(format!( + "pair {} has no exact base raw frame", + pair.identity.pair + ))); + } + } + Ok(frames) +} + +fn validate_futures( + documents: Vec, + mode: TradingMode, + pair_names: &BTreeSet, +) -> Result, NativeContractError> { + if mode == TradingMode::Spot && !documents.is_empty() { + return Err(invalid("Spot manifest cannot declare Futures sources")); + } + let mut seen = BTreeSet::new(); + documents + .into_iter() + .map(|document| { + if !pair_names.contains(&document.pair) || !seen.insert(document.pair.clone()) { + return Err(invalid(format!( + "invalid or duplicate Futures descriptor for {}", + document.pair + ))); + } + // Funding and mark commonly share one pair/timeframe but remain + // separate roles and artifacts. + let funding_rate = validate_frame(document.funding_rate, &mut BTreeSet::new())?; + let mark = validate_frame(document.mark, &mut BTreeSet::new())?; + if funding_rate.identity.pair != document.pair || mark.identity.pair != document.pair { + return Err(invalid(format!( + "Futures descriptor identity differs from pair {}", + document.pair + ))); + } + Ok(ValidatedFutures { + pair: document.pair, + funding_rate, + mark, + }) + }) + .collect() +} + +fn validate_frame( + frame: FrameDocument, + identities: &mut BTreeSet, +) -> Result { + let identity = frame_identity(frame.identity)?; + if !identities.insert(identity.clone()) { + return Err(invalid(format!( + "duplicate raw frame identity {} {}", + identity.pair, + identity.timeframe.as_str() + ))); + } + checked_artifact("raw frame", &frame.artifact)?; + Ok(ValidatedFrame { + identity, + rows: frame.rows, + artifact: frame.artifact, + }) +} + +fn frame_identity(document: IdentityDocument) -> Result { + let timeframe = Timeframe::parse(document.timeframe) + .map_err(|error| invalid(format!("frame timeframe is invalid: {error}")))?; + FrameIdentity::new(document.pair, timeframe) + .map_err(|error| invalid(format!("frame identity is invalid: {error}"))) +} + +fn checked_artifact(role: &str, artifact: &ArtifactDocument) -> Result<(), NativeContractError> { + checked_digest(&format!("{role} artifact SHA-256"), artifact.sha256.clone())?; + if artifact.path.as_os_str().is_empty() || artifact.path.is_absolute() { + return Err(invalid(format!( + "{role} artifact path must be a non-empty relative path" + ))); + } + if artifact + .path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(invalid(format!( + "{role} artifact path contains a non-contained component" + ))); + } + Ok(()) +} + +pub(super) fn validate_program_identity( + document: &ValidatedDocument, + indicator: &IndicatorProgram, + signal: &MutationProgram, + tag: &MutationProgram, +) -> Result<(), NativeContractError> { + if signal.is_tag_program() || !tag.is_tag_program() { + return Err(invalid("Signal and Tag program roles are swapped")); + } + for (role, source_sha, selected_class, fingerprint, expected_fingerprint) in [ + ( + "indicator", + indicator.source.sha256.as_str(), + indicator.selected_class.as_str(), + indicator.fingerprint.as_str(), + document.programs.indicator.fingerprint.as_str(), + ), + ( + "signal", + signal.source.sha256.as_str(), + signal.selected_class.as_str(), + signal.fingerprint.as_str(), + document.programs.signal.fingerprint.as_str(), + ), + ( + "tag", + tag.source.sha256.as_str(), + tag.selected_class.as_str(), + tag.fingerprint.as_str(), + document.programs.tag.fingerprint.as_str(), + ), + ] { + if source_sha != document.source.strategy_sha256 + || selected_class != document.source.selected_class + || fingerprint != expected_fingerprint + { + return Err(invalid(format!( + "{role} program source, class, or fingerprint differs from the manifest" + ))); + } + } + for (role, program) in [("signal", signal), ("tag", tag)] { + if program.compile_context.run_mode != document.compile_context.run_mode + || program.compile_context.trading_mode + != document.compile_context.trading_mode.as_str() + { + return Err(invalid(format!( + "{role} program compile context differs from the manifest" + ))); + } + } + Ok(()) +} + +/// Canonical ordered-list fingerprint for retained feature columns. +#[must_use] +pub fn retained_feature_fingerprint(columns: &[String]) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"full-native-retained-features-v1\0"); + for column in columns { + hasher.update( + u64::try_from(column.len()) + .unwrap_or(u64::MAX) + .to_be_bytes(), + ); + hasher.update(column.as_bytes()); + } + format!("{:x}", hasher.finalize()) +} + +fn validate_unique_names(values: &[String], label: &str) -> Result<(), NativeContractError> { + let mut seen = BTreeSet::new(); + if values + .iter() + .any(|value| value.is_empty() || !seen.insert(value)) + { + return Err(invalid(format!( + "{label} contain an empty or duplicate value" + ))); + } + Ok(()) +} + +fn validate_optional_positive(value: Option, label: &str) -> Result<(), NativeContractError> { + if value.is_some_and(|value| !value.is_finite() || value <= 0.0) { + return Err(invalid(format!("{label} must be finite and positive"))); + } + Ok(()) +} + +fn validate_optional_nonnegative( + value: Option, + label: &str, +) -> Result<(), NativeContractError> { + if value.is_some_and(|value| !value.is_finite() || value < 0.0) { + return Err(invalid(format!("{label} must be finite and non-negative"))); + } + Ok(()) +} + +fn checked_name(label: &str, value: String) -> Result { + if value.is_empty() { + return Err(invalid(format!("{label} must be non-empty"))); + } + Ok(value) +} + +fn checked_digest(label: &str, value: String) -> Result { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + return Err(invalid(format!( + "{label} must be a lowercase SHA-256 digest" + ))); + } + Ok(value) +} + +pub(super) fn invalid(message: impl Into) -> NativeContractError { + NativeContractError::Invalid(message.into()) +} diff --git a/rust/crates/nfi-vector-io/src/full_pipeline.rs b/rust/crates/nfi-vector-io/src/full_pipeline.rs new file mode 100644 index 00000000..00d3601e --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_pipeline.rs @@ -0,0 +1,695 @@ +//! Complete manifest-to-simulator native vector pipeline. +//! +//! This is the production handoff between the SHA-bound manifest, exact +//! Freqtrade OHLCV preparation, the three independent vector programs, and the +//! existing in-memory simulator transport. It never executes strategy Python. + +mod stage; + +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{BufReader, Read}; +use std::path::{Path, PathBuf}; + +use nfi_sim_core::SimulationInput; +use nfi_vector_core::alignment::{FrameCatalog, FrameIdentity, SourceLocation, Timeframe}; +use nfi_vector_core::mutation::prove_signal_tag_decision_equivalence; +use nfi_vector_core::program::IndicatorProgram; +use nfi_vector_core::VectorCoreError; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use crate::full_manifest::{ + decode_verified_frame, load_plan, FuturesFrameSet, NativeContractError, NativeVectorBundle, + NativeVectorPlan, VerifiedFrameSource, VerifiedFuturesSources, +}; +use crate::{ + execute_in_memory_pair_dag_profiled, execute_in_memory_pair_dag_profiled_with_worker_limit, + prepare_freqtrade_ohlcv_catalog, InMemoryVectorProfile, VectorInputError, +}; + +/// Evidence-friendly identity and timings for one complete native vector run. +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +pub struct FullNativeVectorProfile { + pub schema_version: &'static str, + pub manifest_sha256: Option, + pub strategy_sha256: String, + pub config_sha256: String, + pub compiler_source_fingerprint: String, + pub selected_class: String, + pub strategy_source_mode: String, + pub populate_methods_executed: bool, + pub runtime_mode: String, + pub program_fingerprints: BTreeMap, + pub retained_feature_fingerprint: String, + pub trading_mode: String, + pub base_timeframe: String, + pub source_row_shift: usize, + pub pair_identities: Vec, + pub raw_frame_count: usize, + pub frame_loading_mode: String, + pub raw_frame_resident_limit: usize, + pub futures_frame_set_count: usize, + pub spool_required_upper_bound_bytes: u64, + pub spool_available_bytes_at_admission: u64, + pub spool_target_source: &'static str, + pub spool_cleanup_mode: &'static str, + pub manifest_declared_raw_rows: u64, + pub transport: InMemoryVectorProfile, +} + +/// Fail-closed error from manifest loading or complete native execution. +#[derive(Debug, Error)] +pub enum FullNativePipelineError { + #[error(transparent)] + Manifest(#[from] NativeContractError), + #[error(transparent)] + VectorInput(#[from] VectorInputError), + #[error("cannot hash full native manifest {path}: {source}")] + ManifestHash { + path: PathBuf, + source: std::io::Error, + }, +} + +/// Load a strict manifest and execute its complete vector pipeline in Rust. +/// +/// # Errors +/// +/// Returns a manifest, program, frame-preparation, funding, mutation, slicing, +/// or in-memory transport error without returning a partial simulator input. +pub fn load_full_native_vector_manifest_profiled( + path: &Path, +) -> Result<(SimulationInput, FullNativeVectorProfile), FullNativePipelineError> { + load_full_native_vector_manifest_profiled_inner(path, None) +} + +/// Load and execute a strict manifest with an explicit bounded pair worker pool. +/// +/// Numeric libraries inside each independent pair remain single-threaded. The +/// worker limit applies only to the outer pair DAG and never to the global +/// chronological wallet loop. +/// +/// # Errors +/// +/// Returns the same errors as [`load_full_native_vector_manifest_profiled`], +/// plus a fail-closed error for a zero or unbuildable worker limit. +pub fn load_full_native_vector_manifest_profiled_with_worker_limit( + path: &Path, + pair_worker_limit: usize, +) -> Result<(SimulationInput, FullNativeVectorProfile), FullNativePipelineError> { + load_full_native_vector_manifest_profiled_inner(path, Some(pair_worker_limit)) +} + +fn load_full_native_vector_manifest_profiled_inner( + path: &Path, + pair_worker_limit: Option, +) -> Result<(SimulationInput, FullNativeVectorProfile), FullNativePipelineError> { + let manifest_sha256 = + sha256_file(path).map_err(|source| FullNativePipelineError::ManifestHash { + path: path.to_path_buf(), + source, + })?; + let plan = load_plan(path)?; + execute_plan(plan, Some(manifest_sha256), pair_worker_limit) +} + +/// Execute an already verified bundle without strategy Python. +/// +/// This entrypoint is useful for focused tests and callers that retain a +/// verified bundle in memory. Its profile intentionally has no manifest hash. +/// +/// # Errors +/// +/// Returns the same vector and transport failures as +/// [`load_full_native_vector_manifest_profiled`]. +pub fn execute_full_native_vector_bundle_profiled( + bundle: NativeVectorBundle, +) -> Result<(SimulationInput, FullNativeVectorProfile), FullNativePipelineError> { + execute_bundle(bundle, None, None) +} + +fn execute_bundle( + bundle: NativeVectorBundle, + manifest_sha256: Option, + pair_worker_limit: Option, +) -> Result<(SimulationInput, FullNativeVectorProfile), FullNativePipelineError> { + validate_funding_contract(&bundle)?; + prove_signal_tag_decision_equivalence(&bundle.signal_program, &bundle.tag_program) + .map_err(VectorInputError::from)?; + let admission = crate::spool_admission::admit( + &bundle.run, + bundle.pairs.len(), + bundle.retained_features.columns.len(), + )?; + let declared_raw_rows = bundle_declared_raw_rows(&bundle)?; + let identity = ProfileIdentity::from_bundle(&bundle, manifest_sha256, declared_raw_rows); + let timerange = format!( + "{}-{}", + bundle.run.timerange_start_ms, bundle.run.timerange_stop_ms + ); + let prepared = + prepare_freqtrade_ohlcv_catalog(&bundle.frames, &timerange, bundle.run.startup_candles)?; + let NativeVectorBundle { + config, + run, + retained_features, + pairs, + indicator_program, + signal_program, + tag_program, + futures, + .. + } = bundle; + let execute = |pair| { + stage::execute_pair( + &indicator_program, + &signal_program, + &tag_program, + &prepared, + &run, + &retained_features.columns, + &futures, + pair, + ) + }; + let (input, transport) = if let Some(limit) = pair_worker_limit { + execute_in_memory_pair_dag_profiled_with_worker_limit(config, pairs, limit, execute)? + } else { + execute_in_memory_pair_dag_profiled(config, pairs, execute)? + }; + Ok((input, identity.finish(transport, admission)?)) +} + +fn execute_plan( + plan: NativeVectorPlan, + manifest_sha256: Option, + pair_worker_limit: Option, +) -> Result<(SimulationInput, FullNativeVectorProfile), FullNativePipelineError> { + validate_plan_funding_contract(&plan)?; + prove_signal_tag_decision_equivalence(&plan.signal_program, &plan.tag_program) + .map_err(VectorInputError::from)?; + let admission = crate::spool_admission::admit( + &plan.run, + plan.pairs.len(), + plan.retained_features.columns.len(), + )?; + let declared_raw_rows = plan_declared_raw_rows(&plan)?; + let literal_frames = literal_frame_identities(&plan.indicator_program)?; + let identity = + ProfileIdentity::from_plan(&plan, manifest_sha256, &literal_frames, declared_raw_rows); + let NativeVectorPlan { + config, + run, + retained_features, + pairs, + indicator_program, + signal_program, + tag_program, + frames, + futures, + .. + } = plan; + let execute = |pair: crate::PairContract| { + let catalog = prepare_pair_catalog(&frames, &literal_frames, &pair.identity.pair, &run)?; + let pair_futures = decode_pair_futures(&futures, &pair.identity.pair)?; + stage::execute_pair( + &indicator_program, + &signal_program, + &tag_program, + &catalog, + &run, + &retained_features.columns, + &pair_futures, + pair, + ) + }; + let (input, transport) = if let Some(limit) = pair_worker_limit { + execute_in_memory_pair_dag_profiled_with_worker_limit(config, pairs, limit, execute)? + } else { + execute_in_memory_pair_dag_profiled(config, pairs, execute)? + }; + Ok((input, identity.finish(transport, admission)?)) +} + +fn bundle_declared_raw_rows(bundle: &NativeVectorBundle) -> Result { + let source = SourceLocation::new("spool-admission", "native/full_pipeline", 0, 0); + let base_rows = bundle.frames.identities().map(|identity| { + bundle + .frames + .lookup(identity, &source) + .map(|frame| frame.timestamps_ms.len()) + }); + let futures_rows = bundle.futures.iter().flat_map(|frames| { + [ + frames.funding_rate.timestamps_ms.len(), + frames.mark.timestamps_ms.len(), + ] + .into_iter() + .map(Ok) + }); + checked_declared_rows(base_rows.chain(futures_rows)) +} + +fn plan_declared_raw_rows(plan: &NativeVectorPlan) -> Result { + let base_rows = plan.frames.iter().map(|frame| Ok(frame.rows)); + let futures_rows = plan.futures.iter().flat_map(|frames| { + [frames.funding_rate.rows, frames.mark.rows] + .into_iter() + .map(Ok) + }); + checked_declared_rows(base_rows.chain(futures_rows)) +} + +fn checked_declared_rows( + rows: impl IntoIterator>, +) -> Result { + rows.into_iter().try_fold(0_u64, |total, rows| { + let rows = u64::try_from(rows?) + .map_err(|_| VectorInputError::SpoolBound("raw row count exceeds u64".to_owned()))?; + total.checked_add(rows).ok_or_else(|| { + VectorInputError::SpoolBound("aggregate raw row count exceeds u64".to_owned()) + }) + }) +} + +fn prepare_pair_catalog( + sources: &[VerifiedFrameSource], + literal_frames: &std::collections::BTreeSet, + pair: &str, + run: &crate::RunContract, +) -> Result { + let entries = sources + .iter() + .filter(|source| source.identity.pair == pair || literal_frames.contains(&source.identity)) + .enumerate() + .map(|(index, source)| { + let frame = decode_verified_frame(source, &format!("pair-local raw frame {index}")) + .map_err(|error| pipeline_invalid(error.to_string()))?; + Ok((source.identity.clone(), frame)) + }) + .collect::, VectorInputError>>()?; + let raw = FrameCatalog::new(entries)?; + let timerange = format!("{}-{}", run.timerange_start_ms, run.timerange_stop_ms); + prepare_freqtrade_ohlcv_catalog(&raw, &timerange, run.startup_candles) +} + +fn decode_pair_futures( + sources: &[VerifiedFuturesSources], + pair: &str, +) -> Result, VectorInputError> { + sources + .iter() + .filter(|source| source.pair == pair) + .enumerate() + .map(|(index, source)| { + Ok(FuturesFrameSet { + pair: source.pair.clone(), + funding_rate: decode_verified_frame( + &source.funding_rate, + &format!("pair-local funding frame {index}"), + ) + .map_err(|error| pipeline_invalid(error.to_string()))?, + mark: decode_verified_frame( + &source.mark, + &format!("pair-local mark frame {index}"), + ) + .map_err(|error| pipeline_invalid(error.to_string()))?, + }) + }) + .collect() +} + +fn literal_frame_identities( + program: &IndicatorProgram, +) -> Result, VectorInputError> { + let mut identities = std::collections::BTreeSet::new(); + for node in program + .nodes + .iter() + .filter(|node| node.op == "frame-source") + { + let Some(pair) = node + .parameters + .get("pair") + .and_then(serde_json::Value::as_object) + else { + return Err(pipeline_invalid(format!( + "frame-source {} has no pair binding", + node.id + ))); + }; + match pair.get("kind").and_then(serde_json::Value::as_str) { + Some("metadata") => continue, + Some("literal") => {} + _ => { + return Err(pipeline_invalid(format!( + "frame-source {} has an unsupported pair binding", + node.id + ))) + } + } + let value = pair + .get("value") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + pipeline_invalid(format!("frame-source {} has no literal pair", node.id)) + })?; + let timeframe = node + .parameters + .get("timeframe") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + pipeline_invalid(format!("frame-source {} has no timeframe", node.id)) + })?; + identities.insert(FrameIdentity::new(value, Timeframe::parse(timeframe)?)?); + } + Ok(identities) +} + +fn validate_funding_contract(bundle: &NativeVectorBundle) -> Result<(), VectorInputError> { + let futures_mode = bundle.run.trading_mode == crate::TradingMode::Futures; + if bundle.config.is_futures != futures_mode { + return Err(pipeline_invalid( + "embedded config is_futures differs from the full native run mode", + )); + } + if !futures_mode { + if !bundle.futures.is_empty() || bundle.config.funding_fee_interval_ms.is_some() { + return Err(pipeline_invalid( + "Spot execution cannot declare funding frames or an interval", + )); + } + return Ok(()); + } + + let mut interval_ms = None; + let mut seen_pairs = std::collections::BTreeSet::new(); + for pair in &bundle.pairs { + if !pair.options.include_funding { + return Err(pipeline_invalid(format!( + "Futures pair {} does not enable funding", + pair.identity.pair + ))); + } + let matching = bundle + .futures + .iter() + .filter(|frames| frames.pair == pair.identity.pair) + .collect::>(); + let [frames] = matching.as_slice() else { + return Err(pipeline_invalid(format!( + "Futures execution has no unique funding descriptor for {}", + pair.identity.pair + ))); + }; + let actual = frames + .funding_rate + .identity + .timeframe + .resample_duration_ms(); + if interval_ms.is_some_and(|expected| expected != actual) { + return Err(pipeline_invalid( + "Futures pairs declare different funding intervals", + )); + } + interval_ms = Some(actual); + seen_pairs.insert(pair.identity.pair.as_str()); + } + if bundle + .futures + .iter() + .any(|frames| !seen_pairs.contains(frames.pair.as_str())) + { + return Err(pipeline_invalid( + "funding descriptor has no matching execution pair", + )); + } + if bundle.config.funding_fee_interval_ms != interval_ms { + return Err(pipeline_invalid(format!( + "embedded funding_fee_interval_ms {:?} differs from manifest-derived {:?}", + bundle.config.funding_fee_interval_ms, interval_ms + ))); + } + Ok(()) +} + +fn validate_plan_funding_contract(plan: &NativeVectorPlan) -> Result<(), VectorInputError> { + let futures_mode = plan.run.trading_mode == crate::TradingMode::Futures; + if plan.config.is_futures != futures_mode { + return Err(pipeline_invalid( + "embedded config is_futures differs from the full native run mode", + )); + } + if !futures_mode { + if !plan.futures.is_empty() || plan.config.funding_fee_interval_ms.is_some() { + return Err(pipeline_invalid( + "Spot execution cannot declare funding frames or an interval", + )); + } + return Ok(()); + } + + let mut interval_ms = None; + let mut seen_pairs = std::collections::BTreeSet::new(); + for pair in &plan.pairs { + if !pair.options.include_funding { + return Err(pipeline_invalid(format!( + "Futures pair {} does not enable funding", + pair.identity.pair + ))); + } + let matching = plan + .futures + .iter() + .filter(|frames| frames.pair == pair.identity.pair) + .collect::>(); + let [frames] = matching.as_slice() else { + return Err(pipeline_invalid(format!( + "Futures execution has no unique funding descriptor for {}", + pair.identity.pair + ))); + }; + let actual = frames + .funding_rate + .identity + .timeframe + .resample_duration_ms(); + if interval_ms.is_some_and(|expected| expected != actual) { + return Err(pipeline_invalid( + "Futures pairs declare different funding intervals", + )); + } + interval_ms = Some(actual); + seen_pairs.insert(pair.identity.pair.as_str()); + } + if plan + .futures + .iter() + .any(|frames| !seen_pairs.contains(frames.pair.as_str())) + { + return Err(pipeline_invalid( + "funding descriptor has no matching execution pair", + )); + } + if plan.config.funding_fee_interval_ms != interval_ms { + return Err(pipeline_invalid(format!( + "embedded funding_fee_interval_ms {:?} differs from manifest-derived {:?}", + plan.config.funding_fee_interval_ms, interval_ms + ))); + } + Ok(()) +} + +fn pipeline_invalid(message: impl Into) -> VectorInputError { + VectorCoreError::InvalidProgram(message.into()).into() +} + +struct ProfileIdentity { + manifest_sha256: Option, + strategy_sha256: String, + config_sha256: String, + compiler_source_fingerprint: String, + selected_class: String, + strategy_source_mode: String, + populate_methods_executed: bool, + runtime_mode: String, + program_fingerprints: BTreeMap, + retained_feature_fingerprint: String, + trading_mode: String, + base_timeframe: String, + source_row_shift: usize, + pair_identities: Vec, + raw_frame_count: usize, + frame_loading_mode: String, + raw_frame_resident_limit: usize, + futures_frame_set_count: usize, + manifest_declared_raw_rows: u64, +} + +impl ProfileIdentity { + fn from_bundle( + bundle: &NativeVectorBundle, + manifest_sha256: Option, + manifest_declared_raw_rows: u64, + ) -> Self { + Self { + manifest_sha256, + strategy_sha256: bundle.source.strategy_sha256.clone(), + config_sha256: bundle.source.config_sha256.clone(), + compiler_source_fingerprint: bundle.source.compiler_source_fingerprint.clone(), + selected_class: bundle.source.selected_class.clone(), + strategy_source_mode: bundle.source_execution.strategy_source_mode.clone(), + populate_methods_executed: bundle.source_execution.populate_methods_executed, + runtime_mode: bundle.source_execution.runtime_mode.clone(), + program_fingerprints: BTreeMap::from([ + ( + "indicator".to_owned(), + bundle.indicator_program.fingerprint.clone(), + ), + ( + "signal".to_owned(), + bundle.signal_program.fingerprint.clone(), + ), + ("tag".to_owned(), bundle.tag_program.fingerprint.clone()), + ]), + retained_feature_fingerprint: bundle.retained_features.fingerprint.clone(), + trading_mode: bundle.run.trading_mode.as_str().to_owned(), + base_timeframe: bundle.run.base_timeframe.as_str().to_owned(), + source_row_shift: bundle.run.source_row_shift, + pair_identities: bundle + .pairs + .iter() + .map(|pair| { + format!( + "{}|{}", + pair.identity.pair, + pair.identity.timeframe.as_str() + ) + }) + .collect(), + raw_frame_count: bundle.frames.len(), + frame_loading_mode: "preloaded-catalog".to_owned(), + raw_frame_resident_limit: bundle.frames.len(), + futures_frame_set_count: bundle.futures.len(), + manifest_declared_raw_rows, + } + } + + fn from_plan( + plan: &NativeVectorPlan, + manifest_sha256: Option, + literal_frames: &std::collections::BTreeSet, + manifest_declared_raw_rows: u64, + ) -> Self { + Self { + manifest_sha256, + strategy_sha256: plan.source.strategy_sha256.clone(), + config_sha256: plan.source.config_sha256.clone(), + compiler_source_fingerprint: plan.source.compiler_source_fingerprint.clone(), + selected_class: plan.source.selected_class.clone(), + strategy_source_mode: plan.source_execution.strategy_source_mode.clone(), + populate_methods_executed: plan.source_execution.populate_methods_executed, + runtime_mode: plan.source_execution.runtime_mode.clone(), + program_fingerprints: BTreeMap::from([ + ( + "indicator".to_owned(), + plan.indicator_program.fingerprint.clone(), + ), + ("signal".to_owned(), plan.signal_program.fingerprint.clone()), + ("tag".to_owned(), plan.tag_program.fingerprint.clone()), + ]), + retained_feature_fingerprint: plan.retained_features.fingerprint.clone(), + trading_mode: plan.run.trading_mode.as_str().to_owned(), + base_timeframe: plan.run.base_timeframe.as_str().to_owned(), + source_row_shift: plan.run.source_row_shift, + pair_identities: plan + .pairs + .iter() + .map(|pair| { + format!( + "{}|{}", + pair.identity.pair, + pair.identity.timeframe.as_str() + ) + }) + .collect(), + raw_frame_count: plan.frames.len(), + frame_loading_mode: "pair-local-streaming".to_owned(), + raw_frame_resident_limit: plan + .pairs + .iter() + .map(|pair| { + plan.frames + .iter() + .filter(|source| { + source.identity.pair == pair.identity.pair + || literal_frames.contains(&source.identity) + }) + .count() + }) + .max() + .unwrap_or(0), + futures_frame_set_count: plan.futures.len(), + manifest_declared_raw_rows, + } + } + + fn finish( + self, + transport: InMemoryVectorProfile, + admission: crate::spool_admission::SpoolAdmission, + ) -> Result { + if transport.file_backed_bytes > admission.required_upper_bound_bytes { + return Err(VectorInputError::SpoolBound(format!( + "actual file-backed bytes {} exceed admitted upper bound {}", + transport.file_backed_bytes, admission.required_upper_bound_bytes + ))); + } + Ok(FullNativeVectorProfile { + schema_version: "1.2.0", + manifest_sha256: self.manifest_sha256, + strategy_sha256: self.strategy_sha256, + config_sha256: self.config_sha256, + compiler_source_fingerprint: self.compiler_source_fingerprint, + selected_class: self.selected_class, + strategy_source_mode: self.strategy_source_mode, + populate_methods_executed: self.populate_methods_executed, + runtime_mode: self.runtime_mode, + program_fingerprints: self.program_fingerprints, + retained_feature_fingerprint: self.retained_feature_fingerprint, + trading_mode: self.trading_mode, + base_timeframe: self.base_timeframe, + source_row_shift: self.source_row_shift, + pair_identities: self.pair_identities, + raw_frame_count: self.raw_frame_count, + frame_loading_mode: self.frame_loading_mode, + raw_frame_resident_limit: self.raw_frame_resident_limit, + futures_frame_set_count: self.futures_frame_set_count, + spool_required_upper_bound_bytes: admission.required_upper_bound_bytes, + spool_available_bytes_at_admission: admission.available_bytes, + spool_target_source: admission.target_source, + spool_cleanup_mode: admission.cleanup_mode, + manifest_declared_raw_rows: self.manifest_declared_raw_rows, + transport, + }) + } +} + +fn sha256_file(path: &Path) -> Result { + let mut reader = BufReader::new(File::open(path)?); + let mut hasher = Sha256::new(); + let mut buffer = vec![0_u8; 1024 * 1024].into_boxed_slice(); + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/nfi-vector-io/src/full_pipeline/stage.rs b/rust/crates/nfi-vector-io/src/full_pipeline/stage.rs new file mode 100644 index 00000000..d20f46a0 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_pipeline/stage.rs @@ -0,0 +1,286 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use nfi_sim_core::PriceStepChange; +use nfi_vector_core::alignment::{FrameCatalog, NumericFrame, SourceLocation}; +use nfi_vector_core::column::{OwnedColumn, ValueType}; +use nfi_vector_core::engine::{FullFrameOutput, FullIndicatorEngine}; +use nfi_vector_core::mutation::{ + materialize_execution_signals, MutationEngine, MutationFrame, MutationProgram, +}; +use nfi_vector_core::program::IndicatorProgram; +use nfi_vector_core::VectorCoreError; + +use crate::full_manifest::{FuturesFrameSet, PairContract, RunContract, TradingMode}; +use crate::{ + prepare_execution_ohlcv, prepare_funding_events, InMemoryVectorPair, VectorInputError, + VectorPairOptions, +}; + +const SIMULATOR_COLUMNS: [&str; 5] = ["open", "high", "low", "close", "volume"]; +const SIGNAL_COLUMNS: [&str; 4] = ["enter_long", "enter_short", "exit_long", "exit_short"]; +const TAG_COLUMNS: [&str; 2] = ["enter_tag", "exit_tag"]; + +#[allow(clippy::too_many_arguments)] +pub(super) fn execute_pair( + indicator_program: &IndicatorProgram, + signal_program: &MutationProgram, + tag_program: &MutationProgram, + catalog: &FrameCatalog, + run: &RunContract, + retained_features: &[String], + futures: &[FuturesFrameSet], + pair: PairContract, +) -> Result { + validate_retained_features(retained_features)?; + let source = SourceLocation::new("full-native-base-frame", "native/full_pipeline", 0, 0); + let base = catalog.lookup_non_empty(&pair.identity, &source)?; + let requested = indicator_execution_outputs(retained_features, signal_program, tag_program); + let indicator = FullIndicatorEngine::new(indicator_program)?.execute( + base, + catalog, + &pair.metadata, + &requested, + )?; + validate_indicator_identity(base, &indicator)?; + let mutation_source = MutationFrame::new(indicator.columns().clone())?; + // The manifest-level structural proof removes only Tag formatting/appends and + // requires the remaining complete DAG to equal Signal exactly. Execute + // that shared decision surface once, then retain Tag's additional columns. + let tag = + MutationEngine::new(tag_program)?.execute_with_metadata(mutation_source, &pair.metadata)?; + let combined = adopt_tag_columns(&tag, &tag)?; + let execution = materialize_execution_signals(&combined, run.source_row_shift, 0)?; + let full_transport = + assemble_transport_columns(&indicator, &combined, &execution.frame, retained_features)?; + let timerange = format!("{}-{}", run.timerange_start_ms, run.timerange_stop_ms); + let prepared = prepare_execution_ohlcv(base, &timerange, run.startup_candles)?; + let mut sliced = slice_for_execution(base, &prepared.frame, &full_transport)?; + if run.trading_mode == TradingMode::Futures || pair.options.include_funding { + let events = prepare_funding_events( + run.trading_mode, + &pair.identity.pair, + &prepared.frame.timestamps_ms, + futures, + ) + .map_err(|error| invalid(error.to_string()))?; + for (name, column) in events.into_owned_columns() { + if sliced.insert(name.clone(), column).is_some() { + return Err(invalid(format!("funding column {name} collides"))); + } + } + } + let frame = MutationFrame::new(sliced)?; + if frame.is_empty() || prepared.execution_start_index >= frame.len() { + return Err(invalid(format!( + "prepared execution range is empty for {}", + pair.identity.pair + ))); + } + Ok(InMemoryVectorPair { + pair: pair.identity.pair, + execution_start_index: prepared.execution_start_index, + amount_step: pair.precision.amount_step, + price_step: pair.precision.price_step, + price_steps: pair + .price_steps + .into_iter() + .map(|step| PriceStepChange { + timestamp_ms: step.timestamp_ms, + step: step.step, + }) + .collect(), + minimum_stake: pair.limits.minimum_stake, + minimum_amount: pair.limits.minimum_amount, + minimum_cost: pair.limits.minimum_cost, + feature_columns: retained_features.to_vec(), + options: VectorPairOptions::default() + .with_can_short(pair.options.can_short) + .with_funding(pair.options.include_funding) + .with_exit_signal(pair.options.use_exit_signal) + .with_previous_close(pair.options.include_previous_close), + frame, + }) +} + +fn indicator_execution_outputs( + retained: &[String], + signal: &MutationProgram, + tag: &MutationProgram, +) -> Vec { + let mut outputs = Vec::new(); + let mut seen = BTreeSet::new(); + for name in SIMULATOR_COLUMNS + .into_iter() + .chain( + retained + .iter() + .map(String::as_str) + .filter(|name| !SIGNAL_COLUMNS.contains(name) && !TAG_COLUMNS.contains(name)), + ) + .chain(signal.required_input_columns.iter().map(String::as_str)) + .chain(tag.required_input_columns.iter().map(String::as_str)) + { + if seen.insert(name) { + outputs.push(name.to_owned()); + } + } + outputs +} + +fn validate_retained_features(features: &[String]) -> Result<(), VectorInputError> { + let mut seen = BTreeSet::new(); + for feature in features { + if feature.is_empty() + || !seen.insert(feature.as_str()) + || feature == "date" + || SIMULATOR_COLUMNS.contains(&feature.as_str()) + || TAG_COLUMNS.contains(&feature.as_str()) + || feature.starts_with("nfi_exec_") + { + return Err(invalid(format!( + "retained feature {feature:?} is empty, duplicate, or reserved" + ))); + } + } + Ok(()) +} + +fn validate_indicator_identity( + base: &NumericFrame, + indicator: &FullFrameOutput, +) -> Result<(), VectorInputError> { + if indicator.identity() != &base.identity || indicator.timestamps_ms() != base.timestamps_ms { + return Err(VectorCoreError::InvalidOutput( + "complete indicator output changed the base identity or row index".to_owned(), + ) + .into()); + } + Ok(()) +} + +fn adopt_tag_columns( + signal: &MutationFrame, + tag: &MutationFrame, +) -> Result { + let mut combined = signal.columns().clone(); + for name in TAG_COLUMNS { + let synthesized; + let column = if let Some(column) = tag.column(name) { + column + } else { + // Freqtrade's entry/exit wrappers initialize tag columns to the + // empty string even when the strategy never mutates them. The Tag + // contract represents that exact case with final_mutation=null. + synthesized = OwnedColumn::text(vec![Some(String::new()); tag.len()]); + &synthesized + }; + if column.as_view().value_type() != ValueType::Text { + return Err(VectorCoreError::InvalidOutput(format!( + "Tag program produced non-text {name}" + )) + .into()); + } + combined.insert(name.to_owned(), column.clone()); + } + MutationFrame::new(combined).map_err(Into::into) +} + +fn assemble_transport_columns( + indicator: &FullFrameOutput, + decision: &MutationFrame, + execution: &MutationFrame, + retained: &[String], +) -> Result { + let mut columns = BTreeMap::from([( + "date".to_owned(), + OwnedColumn::timestamp_ms( + indicator + .timestamps_ms() + .iter() + .copied() + .map(Some) + .collect(), + ), + )]); + for name in SIMULATOR_COLUMNS { + let column = indicator + .columns() + .get(name) + .ok_or_else(|| VectorCoreError::MissingOutput(format!("transport column {name}")))?; + if columns.insert(name.to_owned(), column.clone()).is_some() { + return Err(invalid(format!("duplicate transport column {name}"))); + } + } + for name in retained { + let column = if SIGNAL_COLUMNS.contains(&name.as_str()) { + decision.column(name) + } else { + indicator.columns().get(name) + } + .ok_or_else(|| VectorCoreError::MissingOutput(format!("transport column {name}")))?; + if columns.insert(name.clone(), column.clone()).is_some() { + return Err(invalid(format!("duplicate transport column {name}"))); + } + } + for (name, column) in execution.columns() { + if columns.insert(name.clone(), column.clone()).is_some() { + return Err(invalid(format!( + "execution column {name} collides with a transport column" + ))); + } + } + MutationFrame::new(columns).map_err(Into::into) +} + +fn slice_for_execution( + base: &NumericFrame, + selected: &NumericFrame, + frame: &MutationFrame, +) -> Result, VectorInputError> { + let Some(first) = selected.timestamps_ms.first() else { + return Err(invalid("prepared execution slice is empty")); + }; + let start = base + .timestamps_ms + .binary_search(first) + .map_err(|_| invalid("prepared execution slice does not start in the base frame"))?; + let end = start + .checked_add(selected.timestamps_ms.len()) + .ok_or_else(|| invalid("prepared execution slice is out of range"))?; + if end > base.timestamps_ms.len() + || base.timestamps_ms[start..end] != selected.timestamps_ms + || frame.len() != base.timestamps_ms.len() + { + return Err(invalid( + "prepared execution slice differs from the complete vector row index", + )); + } + frame + .columns() + .iter() + .map(|(name, column)| Ok((name.clone(), slice_column(column, start, end)))) + .collect() +} + +fn slice_column(column: &OwnedColumn, start: usize, end: usize) -> OwnedColumn { + let view = column.as_view(); + match view.value_type() { + ValueType::F64 => OwnedColumn::f64((start..end).map(|row| view.f64_at(row)).collect()), + ValueType::I64 => OwnedColumn::i64((start..end).map(|row| view.i64_at(row)).collect()), + ValueType::Bool => { + OwnedColumn::boolean((start..end).map(|row| view.bool_at(row)).collect()) + } + ValueType::Text => OwnedColumn::text( + (start..end) + .map(|row| view.text_at(row).map(str::to_owned)) + .collect(), + ), + ValueType::TimestampMs => { + OwnedColumn::timestamp_ms((start..end).map(|row| view.timestamp_ms_at(row)).collect()) + } + } +} + +fn invalid(message: impl Into) -> VectorInputError { + VectorCoreError::InvalidProgram(message.into()).into() +} diff --git a/rust/crates/nfi-vector-io/src/full_pipeline/tests.rs b/rust/crates/nfi-vector-io/src/full_pipeline/tests.rs new file mode 100644 index 00000000..a8553dd6 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/full_pipeline/tests.rs @@ -0,0 +1,338 @@ +use std::collections::BTreeMap; + +use nfi_sim_core::{simulate, PortfolioConfig}; +use nfi_vector_core::alignment::{FrameCatalog, FrameIdentity, NumericFrame, Timeframe}; +use nfi_vector_core::mutation::MutationProgram; +use nfi_vector_core::program::IndicatorProgram; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::*; +use crate::full_manifest::{ + retained_feature_fingerprint, CompileContext, FeatureRetention, PairContract, PairLimits, + PairOptions, PairPrecision, RunContract, SourceExecutionSeal, SourceSeal, TradingMode, +}; +use crate::{assemble_in_memory_vectors_profiled, prepare_freqtrade_ohlcv_catalog}; + +const INDICATOR: &str = + include_str!("../../../../../benchmarks/reference/vector-shadow/indicator-program.json"); +const SIGNAL: &str = + include_str!("../../../../../benchmarks/reference/vector-shadow/signal-program.json"); +const SOURCE_SHA: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const CLASS_NAME: &str = "FullPipelineContract"; +const START_MS: i64 = 1_699_999_800_000; +const STOP_MS: i64 = START_MS + 1_500_000; + +#[test] +fn complete_spot_pipeline_is_exact_to_the_existing_in_memory_transport() { + let actual_bundle = bundle(TradingMode::Spot); + let expected_bundle = bundle(TradingMode::Spot); + let prepared = prepare_freqtrade_ohlcv_catalog( + &expected_bundle.frames, + &format!("{START_MS}-{STOP_MS}"), + expected_bundle.run.startup_candles, + ) + .expect("prepared catalog"); + let expected_pair = stage::execute_pair( + &expected_bundle.indicator_program, + &expected_bundle.signal_program, + &expected_bundle.tag_program, + &prepared, + &expected_bundle.run, + &expected_bundle.retained_features.columns, + &expected_bundle.futures, + expected_bundle.pairs[0].clone(), + ) + .expect("pair vector DAG"); + let (expected, expected_profile) = + assemble_in_memory_vectors_profiled(expected_bundle.config, vec![expected_pair]) + .expect("legacy in-memory transport"); + + let (actual, profile) = + execute_full_native_vector_bundle_profiled(actual_bundle).expect("complete pipeline"); + + assert_eq!( + simulate(&actual).expect("actual simulation"), + simulate(&expected).expect("expected simulation") + ); + assert_eq!(profile.manifest_sha256, None); + assert_eq!(profile.strategy_source_mode, "python-ast-compile-only"); + assert!(!profile.populate_methods_executed); + assert_eq!(profile.runtime_mode, "rust-full-native"); + assert_eq!(profile.trading_mode, "spot"); + assert_eq!(profile.source_row_shift, 1); + assert_eq!(profile.raw_frame_count, 1); + assert_eq!(profile.manifest_declared_raw_rows, 6); + assert!(profile.transport.file_backed_bytes <= profile.spool_required_upper_bound_bytes); + assert!(profile.spool_available_bytes_at_admission >= profile.spool_required_upper_bound_bytes); + assert!(matches!( + profile.spool_target_source, + "os-temp" | "environment" + )); + assert!(!profile.spool_cleanup_mode.is_empty()); + assert_eq!(profile.transport.pair_count, expected_profile.pair_count); + assert_eq!(profile.transport.row_count, 6); + assert_eq!(profile.transport.feature_column_count, 2); + assert_eq!(actual.pairs[0].execution_start_index, 1); + assert_eq!(actual.pairs[0].feature_columns["delta"].len(), 6); + assert_eq!(actual.pairs[0].feature_columns["enter_long"].len(), 6); +} + +#[test] +fn futures_pipeline_fails_closed_without_funding_and_mark_data() { + let bundle = bundle(TradingMode::Futures); + + let error = execute_full_native_vector_bundle_profiled(bundle) + .expect_err("missing Futures descriptor must fail closed"); + + assert!(error.to_string().contains("funding descriptor"), "{error}"); +} + +fn bundle(mode: TradingMode) -> NativeVectorBundle { + let (indicator_program, signal_program, tag_program) = programs(mode); + let identity = identity("TEST/USDT", "5m"); + let raw = NumericFrame { + identity: identity.clone(), + timestamps_ms: (0..6).map(|row| START_MS + row * 300_000).collect(), + columns: BTreeMap::from([ + ("open".to_owned(), numbers([10.0; 6])), + ("high".to_owned(), numbers([14.0; 6])), + ("low".to_owned(), numbers([7.0; 6])), + ( + "close".to_owned(), + numbers([9.0, 11.0, 12.0, 8.0, 13.0, 7.0]), + ), + ("volume".to_owned(), numbers([1.0; 6])), + ]), + }; + let features = vec!["delta".to_owned(), "enter_long".to_owned()]; + let mut config_document = json!({ + "starting_balance": 1_000.0, + "max_open_trades": 2, + "stake_amount": 100.0, + "fee_rate": 0.001, + "stoploss_ratio": -0.2, + "amount_step": 0.001, + "price_step": 0.01, + "is_futures": mode == TradingMode::Futures + }); + if mode == TradingMode::Futures { + config_document["funding_fee_interval_ms"] = json!(3_600_000); + } + let config: PortfolioConfig = + serde_json::from_value(config_document).expect("simulator config"); + NativeVectorBundle { + source: SourceSeal { + strategy_sha256: SOURCE_SHA.to_owned(), + config_sha256: "b".repeat(64), + compiler_source_fingerprint: "c".repeat(64), + selected_class: CLASS_NAME.to_owned(), + }, + source_execution: SourceExecutionSeal { + strategy_source_mode: "python-ast-compile-only".to_owned(), + populate_methods_executed: false, + runtime_mode: "rust-full-native".to_owned(), + }, + config, + compile_context: CompileContext { + run_mode: "backtest".to_owned(), + trading_mode: mode, + }, + run: RunContract { + trading_mode: mode, + timerange_start_ms: START_MS, + timerange_stop_ms: STOP_MS, + startup_candles: 0, + base_timeframe: Timeframe::parse("5m").expect("timeframe"), + source_row_shift: 1, + }, + retained_features: FeatureRetention { + fingerprint: retained_feature_fingerprint(&features), + columns: features, + }, + pairs: vec![PairContract { + identity: identity.clone(), + metadata: BTreeMap::from([("pair".to_owned(), identity.pair.clone())]), + precision: PairPrecision { + amount_step: Some(0.001), + price_step: Some(0.01), + }, + limits: PairLimits { + minimum_stake: None, + minimum_amount: None, + minimum_cost: None, + }, + price_steps: Vec::new(), + options: PairOptions { + can_short: mode == TradingMode::Futures, + include_funding: mode == TradingMode::Futures, + use_exit_signal: true, + include_previous_close: true, + }, + }], + indicator_program, + signal_program, + tag_program, + frames: FrameCatalog::new([(identity, raw)]).expect("raw catalog"), + futures: Vec::new(), + } +} + +fn programs(mode: TradingMode) -> (IndicatorProgram, MutationProgram, MutationProgram) { + let indicator = rewrite_program(INDICATOR, mode, false); + let signal = rewrite_program(SIGNAL, mode, true); + let tag = tag_from_signal(signal.clone()); + ( + IndicatorProgram::from_json(&indicator.to_string()).expect("indicator program"), + MutationProgram::from_json(&signal.to_string()).expect("Signal program"), + MutationProgram::from_json(&tag.to_string()).expect("Tag program"), + ) +} + +fn tag_from_signal(mut program: Value) -> Value { + program["schema_version"] = json!("tag-program-v1"); + program + .as_object_mut() + .expect("program object") + .remove("signal_outputs"); + program["tag_outputs"] = json!([ + { + "column": "enter_tag", "phase": "entry", + "wrapper_initializer": "", "final_mutation": null + }, + { + "column": "exit_tag", "phase": "exit", + "wrapper_initializer": "", "final_mutation": null + } + ]); + program["tag_mutation_nodes"] = json!([]); + program["route_contract"] = json!({ + "canonicalization": "python-str-split", + "original_storage": "preserve-exact", + "trailing_whitespace": "preserve" + }); + reseal(&mut program); + program +} + +fn rewrite_program(encoded: &str, mode: TradingMode, use_delta: bool) -> Value { + let mut program: Value = serde_json::from_str(encoded).expect("reference program"); + program["source"]["path"] = json!("strategy.py"); + program["source"]["sha256"] = json!(SOURCE_SHA); + program["selected_class"] = json!(CLASS_NAME); + if program.get("compile_context").is_some() { + program["compile_context"] = json!({"run_mode": "backtest", "trading_mode": mode.as_str()}); + } + if use_delta { + replace_signal_inputs(&mut program); + program["required_input_columns"] = json!(["delta", "exit_mask"]); + } else { + compact_indicator(&mut program); + } + reseal(&mut program); + program +} + +fn compact_indicator(program: &mut Value) { + let nodes = program["nodes"].as_array().expect("indicator nodes")[..6].to_vec(); + let mut compact = nodes; + compact.push(json!({ + "function": "f1", + "id": "n7", + "inputs": [], + "lookback": {"candles": 0, "causal": true, "expression": null, "kind": "finite"}, + "op": "literal", + "parameters": {"value": 0}, + "source_order": 6, + "value_type": "int-scalar" + })); + compact.push(json!({ + "function": "f1", + "id": "n8", + "inputs": ["n5", "n7"], + "lookback": {"candles": 0, "causal": true, "expression": null, "kind": "finite"}, + "op": "compare", + "parameters": {"operator": "less-than"}, + "source_order": 7, + "value_type": "bool-column" + })); + compact.push(json!({ + "function": "f1", + "id": "n9", + "inputs": ["n6", "n8"], + "lookback": {"candles": 0, "causal": true, "expression": null, "kind": "finite"}, + "op": "column-write", + "parameters": {"column": "exit_mask"}, + "source_order": 8, + "value_type": "dataframe" + })); + compact.push(json!({ + "function": "f1", + "id": "n10", + "inputs": ["n9"], + "lookback": {"candles": 0, "causal": true, "expression": null, "kind": "finite"}, + "op": "return", + "parameters": {}, + "source_order": 9, + "value_type": "dataframe" + })); + program["nodes"] = Value::Array(compact); + program["functions"][0]["node_ids"] = + json!(["n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8", "n9", "n10"]); + program["functions"][0]["return_node"] = json!("n10"); + program["produced_columns"] = json!(["delta", "exit_mask"]); + program["opcodes"] = json!([ + "binary", + "column-read", + "column-write", + "compare", + "literal", + "parameter", + "return" + ]); + program["max_lookback"] = + json!({"candles": 0, "causal": true, "expression": null, "kind": "finite"}); + let location = program["source_map"]["n6"].clone(); + let map = program["source_map"].as_object_mut().expect("source map"); + map.retain(|name, _| { + name.strip_prefix('n') + .and_then(|value| value.parse::().ok()) + .is_some_and(|index| index <= 6) + }); + for index in 7..=10 { + map.insert(format!("n{index}"), location.clone()); + } +} + +fn replace_signal_inputs(value: &mut Value) { + match value { + Value::String(text) if text == "score" => { + *text = "delta".to_owned(); + } + Value::Array(items) => items.iter_mut().for_each(replace_signal_inputs), + Value::Object(items) => items.values_mut().for_each(replace_signal_inputs), + _ => {} + } +} + +fn reseal(program: &mut Value) { + let mut identity = program.clone(); + let object = identity.as_object_mut().expect("program object"); + object.remove("fingerprint"); + object["source"] + .as_object_mut() + .expect("source object") + .remove("path"); + program["fingerprint"] = json!(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&identity).expect("program identity")) + )); +} + +fn identity(pair: &str, timeframe: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse(timeframe).expect("timeframe")).expect("identity") +} + +fn numbers(values: [f64; N]) -> Vec> { + values.into_iter().map(Some).collect() +} diff --git a/rust/crates/nfi-vector-io/src/in_memory.rs b/rust/crates/nfi-vector-io/src/in_memory.rs index 06e2f72d..fae90859 100644 --- a/rust/crates/nfi-vector-io/src/in_memory.rs +++ b/rust/crates/nfi-vector-io/src/in_memory.rs @@ -1,21 +1,26 @@ //! Exact in-memory handoff from generic Rust vector output to the simulator. use std::collections::{BTreeMap, BTreeSet}; +use std::fs::File; +use std::io::{BufWriter, Write}; use std::mem::size_of; use std::time::{Duration, Instant}; use nfi_sim_core::{ - Candle, EntrySignal, ExitSignal, FeatureColumn, PairSeries, PortfolioConfig, PriceStepChange, - SimulationInput, SIMULATOR_SCHEMA_VERSION, + Candle, CandleSeries, EntrySignal, ExitSignal, FeatureColumn, FileBackedFeatureKind, + FileBackedRows, PairSeries, PortfolioConfig, PriceStepChange, SimulationInput, + FILE_BACKED_FEATURE_BYTES, FILE_BACKED_ROW_HEADER_BYTES, SIMULATOR_SCHEMA_VERSION, }; use nfi_vector_core::column::{OwnedColumn, ValueType}; use nfi_vector_core::mutation::MutationFrame; use rayon::prelude::*; use serde::Serialize; +use crate::decode::pair_spool as open_pair_spool; use crate::VectorInputError; const EMPTY_TAG_TRANSPORT_SENTINEL: &str = "__nfi_bte_empty_tag_column__"; +const SPOOL_WRITE_BUFFER_BYTES: usize = 256 * 1024; /// One pair whose complete typed vector frame already lives in Rust memory. /// @@ -108,12 +113,15 @@ pub struct InMemoryVectorProfile { pub schema_version: &'static str, pub vector_execute_ns: u64, pub pair_prepare_ns: u64, + pub pair_prepare_fused: bool, pub pair_count: usize, pub row_count: usize, pub feature_column_count: usize, pub estimated_source_column_bytes: usize, pub estimated_simulation_owned_bytes: usize, + pub file_backed_bytes: u64, pub pair_prepare_worker_limit: usize, + pub source_pair_retention_limit: usize, } struct PreparedPair { @@ -209,9 +217,10 @@ pub fn assemble_in_memory_vectors_profiled( .into_iter() .collect::, _>>()?; let profile = InMemoryVectorProfile { - schema_version: "1.0.0", + schema_version: "1.2.0", vector_execute_ns: 0, pair_prepare_ns: duration_ns(started.elapsed()), + pair_prepare_fused: false, pair_count: prepared.len(), row_count: prepared .iter() @@ -229,23 +238,16 @@ pub fn assemble_in_memory_vectors_profiled( .iter() .map(|pair| pair.owned_bytes) .fold(0, usize::saturating_add), + file_backed_bytes: 0, pair_prepare_worker_limit: rayon::current_num_threads(), + source_pair_retention_limit: prepared.len(), }; - Ok(( - SimulationInput { - schema_version: SIMULATOR_SCHEMA_VERSION.to_owned(), - config, - pairs: prepared - .into_iter() - .map(PreparedPair::into_pair_series) - .collect(), - }, - profile, - )) + Ok((simulation_input(config, prepared), profile)) } -/// Execute independent pair vector DAGs in parallel, then build one ordered -/// simulator input for the chronological wallet loop. +/// Execute and prepare independent pair vector DAGs in one bounded parallel +/// stage, then build one ordered simulator input for the chronological wallet +/// loop. /// /// The executor receives one caller-defined task and must return the complete /// typed frame for that pair. Indexed Rayon collection preserves task order; @@ -260,24 +262,506 @@ pub fn execute_in_memory_pair_dag_profiled( tasks: Vec, execute: Execute, ) -> Result<(SimulationInput, InMemoryVectorProfile), VectorInputError> +where + Task: Send, + Execute: Fn(Task) -> Result + Send + Sync, +{ + let (prepared, profile) = execute_spooled_pair_dag(tasks, execute)?; + Ok((spooled_simulation_input(config, prepared)?, profile)) +} + +/// Execute a fused pair DAG inside an explicit bounded Rayon pool. +/// +/// The prepared, `Send`-safe rows leave the worker pool before the simulator +/// input is assembled. This keeps the simulator's intentionally local +/// file-backed representation out of cross-thread APIs. +/// +/// # Errors +/// +/// Returns a fail-closed error for a zero/unbuildable worker limit or any pair +/// vector/transport failure. +pub fn execute_in_memory_pair_dag_profiled_with_worker_limit( + config: PortfolioConfig, + tasks: Vec, + pair_worker_limit: usize, + execute: Execute, +) -> Result<(SimulationInput, InMemoryVectorProfile), VectorInputError> +where + Task: Send, + Execute: Fn(Task) -> Result + Send + Sync, +{ + if pair_worker_limit == 0 { + return Err(VectorInputError::PairWorkerPool( + "worker limit must be positive".to_owned(), + )); + } + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(pair_worker_limit) + .thread_name(|index| format!("nfi-full-native-pair-{index}")) + .build() + .map_err(|error| VectorInputError::PairWorkerPool(error.to_string()))?; + let (prepared, profile) = pool.install(|| execute_spooled_pair_dag(tasks, execute))?; + Ok((spooled_simulation_input(config, prepared)?, profile)) +} + +fn execute_spooled_pair_dag( + tasks: Vec, + execute: Execute, +) -> Result<(Vec, InMemoryVectorProfile), VectorInputError> where Task: Send, Execute: Fn(Task) -> Result + Send + Sync, { let started = Instant::now(); - let pairs = tasks + let worker_limit = rayon::current_num_threads(); + let prepared = tasks .into_par_iter() - .map(execute) + .map(|task| execute(task).and_then(spool_pair)) .collect::>() .into_iter() .collect::, _>>()?; - let vector_execute_ns = duration_ns(started.elapsed()); - let (input, mut profile) = assemble_in_memory_vectors_profiled(config, pairs)?; - profile.vector_execute_ns = vector_execute_ns; - Ok((input, profile)) + validate_spooled_pairs(&prepared)?; + let profile = profile_from_spooled(&prepared, duration_ns(started.elapsed()), worker_limit)?; + Ok((prepared, profile)) +} + +fn validate_spooled_pairs(pairs: &[PreparedSpoolPair]) -> Result<(), VectorInputError> { + if pairs.is_empty() { + return Err(VectorInputError::EmptyPairs); + } + let mut names = BTreeSet::new(); + for pair in pairs { + if pair.pair.is_empty() || !names.insert(pair.pair.as_str()) { + return Err(VectorInputError::InvalidPair(pair.pair.clone())); + } + } + Ok(()) +} + +fn profile_from_spooled( + prepared: &[PreparedSpoolPair], + fused_wall_ns: u64, + worker_limit: usize, +) -> Result { + let row_count = checked_usize_evidence(prepared.iter().map(|pair| pair.rows), "row count")?; + let feature_column_count = checked_usize_evidence( + prepared.iter().map(|pair| pair.feature_layouts.len()), + "feature column count", + )?; + let estimated_source_column_bytes = checked_usize_evidence( + prepared.iter().map(|pair| pair.source_bytes), + "estimated source bytes", + )?; + let estimated_simulation_owned_bytes = prepared.iter().try_fold(0_usize, |total, pair| { + let pair_bytes = checked_usize_evidence( + std::iter::once(pair.pair.len()) + .chain(pair.tags.iter().map(String::len)) + .chain(pair.feature_layouts.iter().map(|(name, _)| name.len())), + "estimated pair-owned bytes", + )?; + total + .checked_add(pair_bytes) + .ok_or_else(|| evidence_overflow("estimated simulation-owned bytes")) + })?; + let file_backed_bytes = prepared.iter().try_fold(0_u64, |total, pair| { + total + .checked_add(pair.spool_bytes) + .ok_or_else(|| evidence_overflow("file-backed bytes")) + })?; + Ok(InMemoryVectorProfile { + schema_version: "1.2.0", + vector_execute_ns: fused_wall_ns, + pair_prepare_ns: 0, + pair_prepare_fused: true, + pair_count: prepared.len(), + row_count, + feature_column_count, + estimated_source_column_bytes, + estimated_simulation_owned_bytes, + file_backed_bytes, + pair_prepare_worker_limit: worker_limit, + source_pair_retention_limit: worker_limit.min(prepared.len()), + }) +} + +fn checked_usize_evidence( + values: impl IntoIterator, + label: &str, +) -> Result { + values.into_iter().try_fold(0_usize, |total, value| { + total + .checked_add(value) + .ok_or_else(|| evidence_overflow(label)) + }) +} + +fn evidence_overflow(label: &str) -> VectorInputError { + VectorInputError::SpoolBound(format!("{label} evidence exceeds its integer range")) +} + +fn simulation_input(config: PortfolioConfig, prepared: Vec) -> SimulationInput { + SimulationInput { + schema_version: SIMULATOR_SCHEMA_VERSION.to_owned(), + config, + pairs: prepared + .into_iter() + .map(PreparedPair::into_pair_series) + .collect(), + } +} + +struct PreparedSpoolPair { + pair: String, + execution_start_index: usize, + amount_step: Option, + price_step: Option, + price_steps: Vec, + minimum_stake: Option, + minimum_amount: Option, + minimum_cost: Option, + spool: File, + row_count: usize, + feature_layouts: Vec<(String, FileBackedFeatureKind)>, + tags: Vec, + rows: usize, + source_bytes: usize, + spool_bytes: u64, +} + +impl PreparedSpoolPair { + fn into_pair_series(self) -> Result { + let rows = FileBackedRows::new( + self.spool, + self.row_count, + self.feature_layouts.len(), + self.tags, + ) + .map_err(|source| VectorInputError::FileBacking { + pair: self.pair.clone(), + source, + })?; + let feature_columns = self + .feature_layouts + .into_iter() + .enumerate() + .map(|(index, (name, kind))| { + (name, FeatureColumn::file_backed(rows.clone(), index, kind)) + }) + .collect(); + Ok(PairSeries { + pair: self.pair, + execution_start_index: self.execution_start_index, + amount_step: self.amount_step, + price_step: self.price_step, + price_steps: self.price_steps, + minimum_stake: self.minimum_stake, + minimum_amount: self.minimum_amount, + minimum_cost: self.minimum_cost, + feature_columns, + candles: CandleSeries::file_backed(rows), + }) + } +} + +fn spooled_simulation_input( + config: PortfolioConfig, + prepared: Vec, +) -> Result { + Ok(SimulationInput { + schema_version: SIMULATOR_SCHEMA_VERSION.to_owned(), + config, + pairs: prepared + .into_iter() + .map(PreparedSpoolPair::into_pair_series) + .collect::, _>>()?, + }) +} + +fn spool_pair(pair: InMemoryVectorPair) -> Result { + validate_feature_names(&pair)?; + let rows = pair.frame.len(); + if pair.execution_start_index >= rows { + return Err(VectorInputError::ExecutionStart { + pair: pair.pair, + index: pair.execution_start_index, + rows, + }); + } + let source_bytes = pair + .frame + .columns() + .values() + .map(OwnedColumn::estimated_bytes) + .try_fold(0_usize, |total, bytes| { + total + .checked_add(bytes) + .ok_or_else(|| evidence_overflow("source column bytes")) + })?; + let feature_layouts = pair + .feature_columns + .iter() + .map(|name| { + let value_type = column(&pair, name)?.as_view().value_type(); + let kind = match value_type { + ValueType::F64 | ValueType::I64 => FileBackedFeatureKind::Number, + ValueType::Bool => FileBackedFeatureKind::Boolean, + _ => { + return Err(type_error( + &pair, + name, + column(&pair, name)?, + "numeric or Boolean", + )) + } + }; + Ok((name.clone(), kind)) + }) + .collect::, _>>()?; + let row_stride = feature_layouts + .len() + .checked_mul(FILE_BACKED_FEATURE_BYTES) + .and_then(|bytes| FILE_BACKED_ROW_HEADER_BYTES.checked_add(bytes)) + .ok_or_else(|| file_backing_error(&pair.pair, "pair row is too wide"))?; + let spool = open_pair_spool(&pair.pair)?; + let mut writer = BufWriter::with_capacity(SPOOL_WRITE_BUFFER_BYTES, spool); + let mut encoder = SpoolRowEncoder::new(row_stride); + + for row in 0..rows { + encoder.encode(&pair, &feature_layouts, row)?; + writer + .write_all(encoder.row()) + .map_err(|source| VectorInputError::FileBacking { + pair: pair.pair.clone(), + source, + })?; + } + writer + .flush() + .map_err(|source| VectorInputError::FileBacking { + pair: pair.pair.clone(), + source, + })?; + let spool = writer + .into_inner() + .map_err(|error| VectorInputError::FileBacking { + pair: pair.pair.clone(), + source: error.into_error(), + })?; + let spool_bytes = rows + .checked_mul(row_stride) + .and_then(|bytes| u64::try_from(bytes).ok()) + .ok_or_else(|| file_backing_error(&pair.pair, "pair spool is too large"))?; + Ok(PreparedSpoolPair { + pair: pair.pair, + execution_start_index: pair.execution_start_index, + amount_step: pair.amount_step, + price_step: pair.price_step, + price_steps: pair.price_steps, + minimum_stake: pair.minimum_stake, + minimum_amount: pair.minimum_amount, + minimum_cost: pair.minimum_cost, + spool, + row_count: rows, + feature_layouts, + tags: encoder.tags, + rows, + source_bytes, + spool_bytes, + }) +} + +struct SpoolRowEncoder { + buffer: Vec, + tag_ids: BTreeMap, + tags: Vec, + previous_close: Option, +} + +impl SpoolRowEncoder { + fn new(row_stride: usize) -> Self { + Self { + buffer: vec![0_u8; row_stride], + tag_ids: BTreeMap::new(), + tags: Vec::new(), + previous_close: None, + } + } + + fn row(&self) -> &[u8] { + &self.buffer + } + + fn encode( + &mut self, + pair: &InMemoryVectorPair, + feature_layouts: &[(String, FileBackedFeatureKind)], + row: usize, + ) -> Result<(), VectorInputError> { + let close = required_number(pair, "close", row)?; + let entry_tag = optional_text(pair, "nfi_exec_enter_tag", row)?; + let exit_tag = optional_text(pair, "nfi_exec_exit_tag", row)?; + let enter_long = enabled(pair, "nfi_exec_enter_long", row)?; + let enter_short = pair.options.can_short() && enabled(pair, "nfi_exec_enter_short", row)?; + let exit_long = pair.options.use_exit_signal() && enabled(pair, "nfi_exec_exit_long", row)?; + let exit_short = pair.options.can_short() + && pair.options.use_exit_signal() + && enabled(pair, "nfi_exec_exit_short", row)?; + let funding_rate = pair + .options + .include_funding() + .then(|| optional_number(pair, "nfi_exec_funding_rate", row)) + .transpose()? + .flatten(); + let funding_mark_price = pair + .options + .include_funding() + .then(|| optional_number(pair, "nfi_exec_funding_mark_price", row)) + .transpose()? + .flatten(); + let prior_close = pair + .options + .include_previous_close() + .then_some(self.previous_close) + .flatten(); + let mut flags = 0_u8; + for (bit, enabled) in [ + prior_close.is_some(), + funding_rate.is_some(), + funding_mark_price.is_some(), + enter_long, + enter_short, + exit_long, + exit_short, + ] + .into_iter() + .enumerate() + { + set_flag( + &mut flags, + u8::try_from(bit).expect("seven flags fit u8"), + enabled, + ); + } + put_i64(&mut self.buffer, 0, required_timestamp(pair, "date", row)?); + for (offset, name, value) in [ + (8, "open", None), + (16, "high", None), + (24, "low", None), + (32, "close", Some(close)), + (40, "volume", None), + ] { + put_f64( + &mut self.buffer, + offset, + value.map_or_else(|| required_number(pair, name, row), Ok)?, + ); + } + put_f64(&mut self.buffer, 48, prior_close.unwrap_or_default()); + put_f64(&mut self.buffer, 56, funding_rate.unwrap_or_default()); + put_f64(&mut self.buffer, 64, funding_mark_price.unwrap_or_default()); + self.buffer[72] = flags; + put_u32( + &mut self.buffer, + 73, + dictionary_id( + entry_tag.as_deref(), + &mut self.tag_ids, + &mut self.tags, + &pair.pair, + )?, + ); + put_u32( + &mut self.buffer, + 77, + dictionary_id( + exit_tag.as_deref(), + &mut self.tag_ids, + &mut self.tags, + &pair.pair, + )?, + ); + for (feature_index, (name, kind)) in feature_layouts.iter().enumerate() { + put_f64( + &mut self.buffer, + FILE_BACKED_ROW_HEADER_BYTES + feature_index * FILE_BACKED_FEATURE_BYTES, + feature_value(pair, name, *kind, row)?, + ); + } + self.previous_close = Some(close); + Ok(()) + } +} + +fn feature_value( + pair: &InMemoryVectorPair, + name: &str, + kind: FileBackedFeatureKind, + row: usize, +) -> Result { + let source = column(pair, name)?; + Ok(match kind { + FileBackedFeatureKind::Number => match source.as_view().value_type() { + ValueType::F64 => source.as_view().f64_at(row).unwrap_or(f64::NAN), + ValueType::I64 => source + .as_view() + .i64_at(row) + .map_or(f64::NAN, integer_as_number), + _ => return Err(type_error(pair, name, source, "numeric")), + }, + FileBackedFeatureKind::Boolean => source + .as_view() + .bool_at(row) + .map(f64::from) + .ok_or_else(|| null_error(pair, name, row))?, + }) +} + +fn dictionary_id( + value: Option<&str>, + ids: &mut BTreeMap, + values: &mut Vec, + pair: &str, +) -> Result { + let Some(value) = value else { + return Ok(0); + }; + if let Some(identifier) = ids.get(value) { + return Ok(*identifier); + } + let identifier = u32::try_from(values.len() + 1) + .map_err(|_| file_backing_error(pair, "pair tag dictionary is too large"))?; + ids.insert(value.to_owned(), identifier); + values.push(value.to_owned()); + Ok(identifier) +} + +fn file_backing_error(pair: &str, message: &str) -> VectorInputError { + VectorInputError::FileBacking { + pair: pair.to_owned(), + source: std::io::Error::new(std::io::ErrorKind::InvalidData, message), + } +} + +const fn set_flag(flags: &mut u8, bit: u8, enabled: bool) { + if enabled { + *flags |= 1 << bit; + } +} + +fn put_i64(row: &mut [u8], offset: usize, value: i64) { + row[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(row: &mut [u8], offset: usize, value: u32) { + row[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_f64(row: &mut [u8], offset: usize, value: f64) { + row[offset..offset + 8].copy_from_slice(&value.to_bits().to_le_bytes()); } fn prepare_pair(pair: InMemoryVectorPair) -> Result { + validate_feature_names(&pair)?; let rows = pair.frame.len(); if pair.execution_start_index >= rows { return Err(VectorInputError::ExecutionStart { @@ -774,6 +1258,14 @@ mod tests { assert_eq!(profile.pair_count, 2); assert!(profile.vector_execute_ns > 0); + assert!(profile.pair_prepare_fused); + assert_eq!(profile.pair_prepare_ns, 0); + assert!(profile.source_pair_retention_limit <= profile.pair_prepare_worker_limit); + assert!(profile.file_backed_bytes > 0); + assert!(matches!( + input.pairs[0].candles, + nfi_sim_core::CandleSeries::FileBacked(_) + )); assert_eq!(input.pairs[0].pair, "AAA/USDT"); assert_eq!(input.pairs[1].pair, "BBB/USDT"); assert!(input.pairs.iter().all(|pair| { @@ -788,6 +1280,40 @@ mod tests { assert!(!events.is_empty()); } + #[test] + fn explicit_pair_worker_pool_is_bounded_and_preserves_order() { + let config = serde_json::from_value(config_document()).expect("portfolio config"); + let tasks = vec!["CCC/USDT", "AAA/USDT", "BBB/USDT"]; + let (input, profile) = + execute_in_memory_pair_dag_profiled_with_worker_limit(config, tasks, 2, |pair| { + let mut output = fixture_pair(fixture_frame()); + output.pair = pair.to_owned(); + Ok(output) + }) + .expect("bounded pair DAG executes"); + + assert_eq!( + input + .pairs + .iter() + .map(|pair| pair.pair.as_str()) + .collect::>(), + vec!["CCC/USDT", "AAA/USDT", "BBB/USDT"] + ); + assert_eq!(profile.pair_prepare_worker_limit, 2); + assert_eq!(profile.source_pair_retention_limit, 2); + assert!(profile.pair_prepare_fused); + + let error = execute_in_memory_pair_dag_profiled_with_worker_limit( + serde_json::from_value(config_document()).expect("portfolio config"), + vec!["AAA/USDT"], + 0, + |_| Ok(fixture_pair(fixture_frame())), + ) + .expect_err("zero workers fail closed"); + assert!(matches!(error, VectorInputError::PairWorkerPool(_))); + } + #[test] #[ignore = "explicit release-mode transport diagnostic"] fn large_pair_transport_profile_is_reported_without_a_speed_assertion() { diff --git a/rust/crates/nfi-vector-io/src/lib.rs b/rust/crates/nfi-vector-io/src/lib.rs index 147db4af..26dc6f50 100644 --- a/rust/crates/nfi-vector-io/src/lib.rs +++ b/rust/crates/nfi-vector-io/src/lib.rs @@ -7,20 +7,48 @@ mod decode; mod failures; +mod freqtrade_funding; +mod freqtrade_ohlcv; +mod full_manifest; +mod full_pipeline; #[allow(clippy::module_name_repetitions)] // Public API distinguishes direct and Feather profiles. mod in_memory; mod loader; +mod raw_ohlcv; mod row; mod schema; +mod spool_admission; mod values; pub use failures::VectorInputError; +pub use freqtrade_funding::prepare_events as prepare_funding_events; +pub use freqtrade_funding::{ + PreparedEvents as PreparedFundingEvents, MARK_COLUMN as FUNDING_MARK_COLUMN, + RATE_COLUMN as FUNDING_RATE_COLUMN, +}; +pub use freqtrade_ohlcv::clean_frame as clean_freqtrade_ohlcv; +pub use freqtrade_ohlcv::{ + execution_positions, prepare_execution_ohlcv, prepare_freqtrade_ohlcv_catalog, ClosedTimerange, + InclusiveExecutionPositions, PreparedExecutionOhlcv, +}; +pub use full_manifest::{ + load_full_native_vector_manifest, retained_feature_fingerprint, CompileContext, + FeatureRetention, FuturesFrameSet, HistoricPriceStep, NativeContractError, NativeVectorBundle, + PairContract, PairLimits, PairOptions, PairPrecision, RunContract, SourceExecutionSeal, + SourceSeal, TradingMode, FULL_NATIVE_VECTOR_MANIFEST_VERSION, +}; +pub use full_pipeline::{ + execute_full_native_vector_bundle_profiled, load_full_native_vector_manifest_profiled, + load_full_native_vector_manifest_profiled_with_worker_limit, FullNativePipelineError, + FullNativeVectorProfile, +}; pub use in_memory::{ assemble_in_memory_vectors, assemble_in_memory_vectors_profiled, - execute_in_memory_pair_dag_profiled, InMemoryVectorPair, InMemoryVectorProfile, - VectorPairOptions, + execute_in_memory_pair_dag_profiled, execute_in_memory_pair_dag_profiled_with_worker_limit, + InMemoryVectorPair, InMemoryVectorProfile, VectorPairOptions, }; pub use loader::{load_vector_manifest, load_vector_manifest_profiled, VectorLoadProfile}; +pub use raw_ohlcv::{load_raw_ohlcv_catalog, load_raw_ohlcv_frame, FeatherFrameSource}; /// Version of the compact manifest consumed by this crate. pub const VECTOR_MANIFEST_SCHEMA_VERSION: &str = "1.2.0"; diff --git a/rust/crates/nfi-vector-io/src/raw_ohlcv.rs b/rust/crates/nfi-vector-io/src/raw_ohlcv.rs new file mode 100644 index 00000000..2091a3c8 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/raw_ohlcv.rs @@ -0,0 +1,418 @@ +//! Exact, non-transforming Feather input for native indicator execution. +//! +//! This boundary validates and decodes raw OHLCV values. It deliberately does +//! not aggregate duplicate candles, resample timeframes, fill gaps, or apply a +//! timerange. Those are observable Freqtrade semantics and belong in a +//! separately verified cleaning stage. Rows are only stable-sorted by `date`, +//! matching the ordering performed before Python's cleaning stage. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::File; +use std::path::PathBuf; + +use arrow2::array::{Array, PrimitiveArray}; +use arrow2::datatypes::{DataType, Schema, TimeUnit}; +use arrow2::io::ipc::read::{read_file_metadata, FileReader}; +use nfi_vector_core::alignment::{FrameCatalog, FrameIdentity, NumericFrame, SourceLocation}; + +use crate::VectorInputError; + +const DATE_COLUMN: &str = "date"; +const VALUE_COLUMNS: [&str; 5] = ["open", "high", "low", "close", "volume"]; + +/// One explicit raw-candle source and the strategy location that requested it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FeatherFrameSource { + pub identity: FrameIdentity, + pub path: PathBuf, + pub source: SourceLocation, +} + +impl FeatherFrameSource { + #[must_use] + pub fn new(identity: FrameIdentity, path: impl Into, source: SourceLocation) -> Self { + Self { + identity, + path: path.into(), + source, + } + } + + fn error(&self, message: impl Into) -> VectorInputError { + self.source.error(message).into() + } + + fn label(&self) -> String { + format!( + "raw OHLCV frame {} {} at {}", + self.identity.pair, + self.identity.timeframe.as_str(), + self.path.display() + ) + } +} + +/// Decode one raw Feather source into a numeric frame. +/// +/// Numeric nulls remain `None`, present NaNs remain `Some(NaN)`, and duplicate +/// timestamps remain separate rows in their original relative order. No +/// Freqtrade cleaning or timerange bounding is performed here. +/// +/// # Errors +/// +/// Returns a source-located error when the file cannot be read, the required +/// schema is absent or ambiguous, `date` is not `timestamp[ms]`, an OHLCV +/// value is not `float64`, or a date value is null. +pub fn load_raw_ohlcv_frame(input: &FeatherFrameSource) -> Result { + let mut file = File::open(&input.path) + .map_err(|error| input.error(format!("cannot open {}: {error}", input.label())))?; + let metadata = read_file_metadata(&mut file) + .map_err(|error| input.error(format!("cannot decode {}: {error}", input.label())))?; + let source_indices = validate_schema(&metadata.schema, input)?; + let reader = FileReader::new(file, metadata, Some(source_indices), None); + let positions = column_positions(reader.schema(), input)?; + let mut rows = Vec::new(); + let mut absolute_row = 0_usize; + + for batch in reader { + let batch = batch + .map_err(|error| input.error(format!("cannot decode {}: {error}", input.label())))?; + let date = timestamp_array(batch.arrays()[positions[DATE_COLUMN]].as_ref()); + let values = + VALUE_COLUMNS.map(|name| numeric_array(batch.arrays()[positions[name]].as_ref())); + for row in 0..batch.len() { + if date.is_null(row) { + return Err(input.error(format!( + "{} column {DATE_COLUMN:?} contains null at row {absolute_row}", + input.label() + ))); + } + rows.push(RawRow { + timestamp_ms: date.value(row), + values: values.map(|array| (!array.is_null(row)).then(|| array.value(row))), + }); + absolute_row = absolute_row + .checked_add(1) + .ok_or_else(|| input.error(format!("{} row count is too large", input.label())))?; + } + } + + // Rust's slice sort is stable, so equal timestamps retain their exact + // source order for the later Freqtrade duplicate aggregation stage. + rows.sort_by_key(|row| row.timestamp_ms); + let timestamps_ms = rows.iter().map(|row| row.timestamp_ms).collect(); + let columns = VALUE_COLUMNS + .into_iter() + .enumerate() + .map(|(index, name)| { + ( + name.to_owned(), + rows.iter().map(|row| row.values[index]).collect(), + ) + }) + .collect::>(); + let frame = NumericFrame { + identity: input.identity.clone(), + timestamps_ms, + columns, + }; + frame + .validate() + .map_err(|error| input.error(format!("{} is invalid: {error}", input.label())))?; + Ok(frame) +} + +/// Decode explicit pair/timeframe sources and build one exact frame catalog. +/// +/// # Errors +/// +/// Returns the first source-located decode error, or fails closed when two +/// inputs declare the same identity. +pub fn load_raw_ohlcv_catalog( + inputs: impl IntoIterator, +) -> Result { + let mut entries = Vec::new(); + let mut identities = BTreeSet::new(); + for input in inputs { + if !identities.insert(input.identity.clone()) { + return Err(input.error(format!( + "raw OHLCV catalog contains duplicate identity {} {}", + input.identity.pair, + input.identity.timeframe.as_str() + ))); + } + let frame = load_raw_ohlcv_frame(&input)?; + entries.push((input.identity, frame)); + } + FrameCatalog::new(entries).map_err(Into::into) +} + +#[derive(Clone, Copy, Debug)] +struct RawRow { + timestamp_ms: i64, + values: [Option; 5], +} + +fn validate_schema( + schema: &Schema, + input: &FeatherFrameSource, +) -> Result, VectorInputError> { + let required = std::iter::once(DATE_COLUMN).chain(VALUE_COLUMNS); + let mut indices = Vec::with_capacity(6); + for name in required { + let matches = schema + .fields + .iter() + .enumerate() + .filter(|(_, field)| field.name == name) + .collect::>(); + let [(index, field)] = matches.as_slice() else { + let message = if matches.is_empty() { + format!("{} is missing required column {name:?}", input.label()) + } else { + format!("{} contains duplicate column {name:?}", input.label()) + }; + return Err(input.error(message)); + }; + let expected = if name == DATE_COLUMN { + "timestamp[ms]" + } else { + "float64" + }; + let valid = if name == DATE_COLUMN { + matches!( + field.data_type, + DataType::Timestamp(TimeUnit::Millisecond, _) + ) + } else { + field.data_type == DataType::Float64 + }; + if !valid { + return Err(input.error(format!( + "{} column {name:?} has type {:?}; expected {expected}", + input.label(), + field.data_type + ))); + } + indices.push(*index); + } + indices.sort_unstable(); + Ok(indices) +} + +fn column_positions( + schema: &Schema, + input: &FeatherFrameSource, +) -> Result, VectorInputError> { + let positions = schema + .fields + .iter() + .enumerate() + .map(|(index, field)| (field.name.clone(), index)) + .collect::>(); + for name in std::iter::once(DATE_COLUMN).chain(VALUE_COLUMNS) { + if !positions.contains_key(name) { + return Err(input.error(format!( + "{} projection omitted required column {name:?}", + input.label() + ))); + } + } + Ok(positions) +} + +fn timestamp_array(array: &dyn Array) -> &PrimitiveArray { + array + .as_any() + .downcast_ref::>() + .expect("validated timestamp[ms] has i64 Arrow storage") +} + +fn numeric_array(array: &dyn Array) -> &PrimitiveArray { + array + .as_any() + .downcast_ref::>() + .expect("validated float64 has f64 Arrow storage") +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::path::Path; + + use arrow2::array::PrimitiveArray; + use arrow2::chunk::Chunk; + use arrow2::datatypes::{Field, Schema}; + use arrow2::io::ipc::write::{FileWriter, WriteOptions}; + use nfi_vector_core::alignment::Timeframe; + + use super::*; + + fn identity(pair: &str, timeframe: &str) -> FrameIdentity { + FrameIdentity::new(pair, Timeframe::parse(timeframe).expect("timeframe")).expect("identity") + } + + fn input(path: &Path, pair: &str, timeframe: &str) -> FeatherFrameSource { + FeatherFrameSource::new( + identity(pair, timeframe), + path, + SourceLocation::new("n41", "NostalgiaForInfinityX7.py", 2200, 16), + ) + } + + fn write_ohlcv(path: &Path, dates: Vec>, values: [Vec>; 5]) { + let schema = Schema::from( + [Field::new( + DATE_COLUMN, + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".to_owned())), + true, + )] + .into_iter() + .chain(VALUE_COLUMNS.map(|name| Field::new(name, DataType::Float64, true))) + .collect::>(), + ); + let mut arrays: Vec> = vec![Box::new(PrimitiveArray::from(dates).to( + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".to_owned())), + ))]; + arrays.extend( + values + .into_iter() + .map(|column| Box::new(PrimitiveArray::from(column)) as Box), + ); + write_chunk(path, schema, &Chunk::new(arrays)); + } + + fn write_chunk(path: &Path, schema: Schema, chunk: &Chunk>) { + let mut writer = FileWriter::try_new( + File::create(path).expect("create Feather"), + schema, + None, + WriteOptions { compression: None }, + ) + .expect("writer"); + writer.write(chunk, None).expect("write"); + writer.finish().expect("finish"); + } + + #[test] + fn stable_sorts_but_preserves_duplicates_nulls_and_nans() { + let temporary = tempfile::tempdir().expect("temporary"); + let path = temporary.path().join("BTC_USDT-5m.feather"); + write_ohlcv( + &path, + vec![Some(300), Some(100), Some(100), Some(200)], + [ + vec![Some(30.0), Some(10.0), Some(11.0), None], + vec![Some(31.0), Some(12.0), Some(13.0), Some(f64::NAN)], + vec![Some(29.0), Some(9.0), Some(8.0), Some(19.0)], + vec![Some(30.5), Some(10.5), Some(11.5), Some(20.0)], + vec![Some(3.0), Some(1.0), Some(2.0), Some(4.0)], + ], + ); + + let frame = load_raw_ohlcv_frame(&input(&path, "BTC/USDT", "5m")).expect("frame"); + + assert_eq!(frame.timestamps_ms, [100, 100, 200, 300]); + assert_eq!( + frame.columns["open"], + [Some(10.0), Some(11.0), None, Some(30.0)] + ); + assert!(frame.columns["high"][2].expect("present NaN").is_nan()); + assert_eq!( + frame.columns["volume"], + [Some(1.0), Some(2.0), Some(4.0), Some(3.0)] + ); + } + + #[test] + fn builds_the_explicit_five_timeframe_catalog_without_substitution() { + let temporary = tempfile::tempdir().expect("temporary"); + let mut inputs = Vec::new(); + for (index, timeframe) in ["5m", "15m", "1h", "4h", "1d"].into_iter().enumerate() { + let path = temporary + .path() + .join(format!("BTC_USDT-{timeframe}.feather")); + let value = f64::from(u32::try_from(index).expect("small test index")); + write_ohlcv( + &path, + vec![Some(1_700_000_000_000)], + std::array::from_fn(|_| vec![Some(value)]), + ); + inputs.push(input(&path, "BTC/USDT", timeframe)); + } + + let catalog = load_raw_ohlcv_catalog(inputs).expect("catalog"); + + assert_eq!(catalog.len(), 5); + for timeframe in ["5m", "15m", "1h", "4h", "1d"] { + let requested = identity("BTC/USDT", timeframe); + let frame = catalog + .lookup( + &requested, + &SourceLocation::new("lookup", "strategy.py", 1, 0), + ) + .expect("exact frame"); + assert_eq!(frame.identity, requested); + } + } + + #[test] + fn catalog_rejects_identity_mismatch_and_duplicate_declarations() { + let temporary = tempfile::tempdir().expect("temporary"); + let path = temporary.path().join("BTC_USDT-5m.feather"); + write_ohlcv( + &path, + vec![Some(0)], + std::array::from_fn(|_| vec![Some(1.0)]), + ); + let source = input(&path, "BTC/USDT", "5m"); + let frame = load_raw_ohlcv_frame(&source).expect("frame"); + let mismatch = FrameCatalog::new([(identity("ETH/USDT", "5m"), frame)]) + .expect_err("identity mismatch"); + assert!(mismatch + .to_string() + .contains("differs from stored frame identity")); + + let error = + load_raw_ohlcv_catalog([source.clone(), source]).expect_err("duplicate declaration"); + assert!(error.to_string().contains("duplicate identity BTC/USDT 5m")); + } + + #[test] + fn null_date_and_schema_errors_are_source_located() { + let temporary = tempfile::tempdir().expect("temporary"); + let null_path = temporary.path().join("null-date.feather"); + write_ohlcv( + &null_path, + vec![None], + std::array::from_fn(|_| vec![Some(1.0)]), + ); + let null_error = + load_raw_ohlcv_frame(&input(&null_path, "BTC/USDT", "5m")).expect_err("null date"); + let message = null_error.to_string(); + assert!(message.contains("NostalgiaForInfinityX7.py:2200:16")); + assert!(message.contains("column \"date\" contains null at row 0")); + + let wrong_type_path = temporary.path().join("wrong-type.feather"); + let schema = Schema::from( + [Field::new(DATE_COLUMN, DataType::Int64, false)] + .into_iter() + .chain(VALUE_COLUMNS.map(|name| Field::new(name, DataType::Float64, true))) + .collect::>(), + ); + let arrays = + std::iter::once(Box::new(PrimitiveArray::from_vec(vec![0_i64])) as Box) + .chain( + (0..5).map(|_| { + Box::new(PrimitiveArray::from_vec(vec![1.0_f64])) as Box + }), + ) + .collect(); + write_chunk(&wrong_type_path, schema, &Chunk::new(arrays)); + let type_error = load_raw_ohlcv_frame(&input(&wrong_type_path, "BTC/USDT", "5m")) + .expect_err("wrong date type"); + let message = type_error.to_string(); + assert!(message.contains("NostalgiaForInfinityX7.py:2200:16")); + assert!(message.contains("expected timestamp[ms]")); + } +} diff --git a/rust/crates/nfi-vector-io/src/spool_admission.rs b/rust/crates/nfi-vector-io/src/spool_admission.rs new file mode 100644 index 00000000..7d0d4182 --- /dev/null +++ b/rust/crates/nfi-vector-io/src/spool_admission.rs @@ -0,0 +1,250 @@ +//! Filesystem admission for bounded Full Native pair spools. +//! +//! The bound comes only from public run semantics and the fixed row schema. +//! Raw manifest row counts are deliberately not used: two sparse source rows +//! can expand into every timeframe bucket through Freqtrade gap filling. + +use std::ffi::OsString; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; + +use nfi_sim_core::{FILE_BACKED_FEATURE_BYTES, FILE_BACKED_ROW_HEADER_BYTES}; + +use crate::{RunContract, VectorInputError}; + +pub(crate) const SPOOL_DIRECTORY_ENVIRONMENT: &str = "NFI_BTE_SPOOL_DIRECTORY"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct SpoolAdmission { + pub(crate) required_upper_bound_bytes: u64, + pub(crate) available_bytes: u64, + pub(crate) target_source: &'static str, + pub(crate) cleanup_mode: &'static str, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct SpoolTarget { + directory: PathBuf, + source: &'static str, +} + +pub(crate) fn admit( + run: &RunContract, + pair_count: usize, + retained_feature_count: usize, +) -> Result { + let target = resolve_target(); + admit_with(run, pair_count, retained_feature_count, target, |path| { + fs2::available_space(path) + }) +} + +pub(crate) fn create_file() -> io::Result { + tempfile::tempfile_in(resolve_target().directory) +} + +fn admit_with( + run: &RunContract, + pair_count: usize, + retained_feature_count: usize, + target: SpoolTarget, + capacity: impl FnOnce(&Path) -> io::Result, +) -> Result { + let required_upper_bound_bytes = required_upper_bound(run, pair_count, retained_feature_count)?; + let available_bytes = + capacity(&target.directory).map_err(|source| VectorInputError::SpoolCapacityProbe { + target: target.directory, + source, + })?; + if available_bytes < required_upper_bound_bytes { + return Err(VectorInputError::SpoolCapacity { + target_source: target.source, + required_bytes: required_upper_bound_bytes, + available_bytes, + }); + } + Ok(SpoolAdmission { + required_upper_bound_bytes, + available_bytes, + target_source: target.source, + cleanup_mode: cleanup_mode(), + }) +} + +fn required_upper_bound( + run: &RunContract, + pair_count: usize, + retained_feature_count: usize, +) -> Result { + if pair_count == 0 { + return Err(bound_error("pair count must be positive")); + } + if run.timerange_start_ms < 0 || run.timerange_stop_ms < run.timerange_start_ms { + return Err(bound_error("run timerange is invalid")); + } + let timeframe_ms = run.base_timeframe.resample_duration_ms(); + if timeframe_ms <= 0 { + return Err(bound_error("base timeframe duration must be positive")); + } + let startup_candles = i64::try_from(run.startup_candles) + .map_err(|_| bound_error("startup candle count is outside timestamp range"))?; + let startup_ms = timeframe_ms + .checked_mul(startup_candles) + .ok_or_else(|| bound_error("startup window is outside timestamp range"))?; + let load_start_ms = run + .timerange_start_ms + .checked_sub(startup_ms) + .ok_or_else(|| bound_error("startup window is outside timestamp range"))?; + let first_bucket = load_start_ms.div_euclid(timeframe_ms); + let last_bucket = run.timerange_stop_ms.div_euclid(timeframe_ms); + let rows_per_pair = last_bucket + .checked_sub(first_bucket) + .and_then(|value| value.checked_add(1)) + .and_then(|value| u64::try_from(value).ok()) + .ok_or_else(|| bound_error("resampled row bound is outside addressable range"))?; + let row_stride = retained_feature_count + .checked_mul(FILE_BACKED_FEATURE_BYTES) + .and_then(|bytes| FILE_BACKED_ROW_HEADER_BYTES.checked_add(bytes)) + .and_then(|bytes| u64::try_from(bytes).ok()) + .ok_or_else(|| bound_error("pair spool row is too wide"))?; + u64::try_from(pair_count) + .ok() + .and_then(|pairs| pairs.checked_mul(rows_per_pair)) + .and_then(|rows| rows.checked_mul(row_stride)) + .ok_or_else(|| bound_error("aggregate spool upper bound exceeds u64")) +} + +fn resolve_target() -> SpoolTarget { + resolve_target_from( + std::env::var_os(SPOOL_DIRECTORY_ENVIRONMENT), + tempfile::env::temp_dir(), + ) +} + +fn resolve_target_from(explicit: Option, default: PathBuf) -> SpoolTarget { + explicit.map_or( + SpoolTarget { + directory: default, + source: "os-temp", + }, + |directory| SpoolTarget { + directory: PathBuf::from(directory), + source: "environment", + }, + ) +} + +#[cfg(unix)] +const fn cleanup_mode() -> &'static str { + "unlink-on-open" +} + +#[cfg(windows)] +const fn cleanup_mode() -> &'static str { + "delete-on-close" +} + +#[cfg(not(any(unix, windows)))] +const fn cleanup_mode() -> &'static str { + "os-delete-on-close" +} + +fn bound_error(message: impl Into) -> VectorInputError { + VectorInputError::SpoolBound(message.into()) +} + +#[cfg(test)] +mod tests { + use nfi_vector_core::alignment::Timeframe; + + use super::*; + use crate::TradingMode; + + fn run(start_ms: i64, stop_ms: i64, startup_candles: usize, timeframe: &str) -> RunContract { + RunContract { + trading_mode: TradingMode::Spot, + timerange_start_ms: start_ms, + timerange_stop_ms: stop_ms, + startup_candles, + base_timeframe: Timeframe::parse(timeframe).expect("timeframe"), + source_row_shift: 1, + } + } + + #[test] + fn bound_covers_gap_fill_instead_of_using_raw_row_counts() { + // load_start=-10m through stop=+10m contains five 5m buckets even if + // the manifest has only two endpoint rows. + let contract = run(0, 600_000, 2, "5m"); + + let required = required_upper_bound(&contract, 3, 4).expect("bound"); + + assert_eq!(required, 3 * 5 * (81 + 4 * 8)); + } + + #[test] + fn equality_is_admitted_and_one_byte_short_fails_closed() { + let contract = run(1_704_067_200_000, 1_704_067_800_000, 0, "5m"); + let target = SpoolTarget { + directory: PathBuf::from("injected-spool"), + source: "environment", + }; + let required = required_upper_bound(&contract, 2, 1).expect("bound"); + + let admitted = + admit_with(&contract, 2, 1, target.clone(), |_| Ok(required)).expect("equal capacity"); + let rejected = + admit_with(&contract, 2, 1, target, |_| Ok(required - 1)).expect_err("one byte short"); + + assert_eq!(admitted.required_upper_bound_bytes, required); + assert_eq!(admitted.available_bytes, required); + assert!(matches!( + rejected, + VectorInputError::SpoolCapacity { + required_bytes, + available_bytes, + .. + } if required_bytes == required && available_bytes == required - 1 + )); + } + + #[test] + fn overflow_and_capacity_probe_errors_are_structured() { + let overflow = + required_upper_bound(&run(0, i64::MAX, usize::MAX, "1d"), usize::MAX, usize::MAX) + .expect_err("overflow"); + assert!(matches!(overflow, VectorInputError::SpoolBound(_))); + + let failure = admit_with( + &run(0, 0, 0, "5m"), + 1, + 0, + SpoolTarget { + directory: PathBuf::from("unreadable-spool"), + source: "environment", + }, + |_| Err(io::Error::new(io::ErrorKind::PermissionDenied, "denied")), + ) + .expect_err("probe error"); + assert!(matches!( + failure, + VectorInputError::SpoolCapacityProbe { source, .. } + if source.kind() == io::ErrorKind::PermissionDenied + )); + } + + #[test] + fn target_selection_distinguishes_default_and_explicit_directories() { + let default = resolve_target_from(None, PathBuf::from("system-temp")); + let explicit = resolve_target_from( + Some(OsString::from("profile-spool")), + PathBuf::from("system-temp"), + ); + + assert_eq!(default.directory, PathBuf::from("system-temp")); + assert_eq!(default.source, "os-temp"); + assert_eq!(explicit.directory, PathBuf::from("profile-spool")); + assert_eq!(explicit.source, "environment"); + } +} diff --git a/scripts/generate_latest_signal47_fixture.py b/scripts/generate_latest_signal47_fixture.py new file mode 100644 index 00000000..21bbbd2b --- /dev/null +++ b/scripts/generate_latest_signal47_fixture.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Regenerate the latest X7 Signal 47 boundary evidence.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "python")) + +from nfi_backtest_engine.latest_signal47_fixture import ( # noqa: E402 + FIXTURE_PATH, + write_fixture, +) + + +def main() -> None: + """Parse explicit source identities and write deterministic evidence.""" + parser = argparse.ArgumentParser() + parser.add_argument("current_source", type=Path) + parser.add_argument("baseline_source", type=Path) + parser.add_argument("--current-commit", required=True) + parser.add_argument("--baseline-commit", required=True) + parser.add_argument("--output", type=Path, default=ROOT / FIXTURE_PATH) + args = parser.parse_args() + write_fixture( + args.current_source, + args.baseline_source, + current_commit=args.current_commit, + baseline_commit=args.baseline_commit, + output=args.output, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_engine_runtime.py b/tests/test_engine_runtime.py index 6ceef14e..dc8f634f 100644 --- a/tests/test_engine_runtime.py +++ b/tests/test_engine_runtime.py @@ -157,11 +157,11 @@ def simulate_vector_file_profiled( assert report["profile"]["phases"]["simulation"]["event_loop_ns"] == 22 -def test_engine_profile_requires_vector_manifest(tmp_path: Path) -> None: +def test_engine_profile_requires_vector_input(tmp_path: Path) -> None: source = tmp_path / "simulation.json" write_json(source, {"schema_version": "fixture"}) - with pytest.raises(BenchmarkError, match="requires a vector manifest"): + with pytest.raises(BenchmarkError, match="requires a vector input"): engine_runtime.run_engine( source, tmp_path / "result.json", @@ -169,6 +169,83 @@ def test_engine_profile_requires_vector_manifest(tmp_path: Path) -> None: ) +def test_run_engine_selects_full_native_profiled_entrypoint( + monkeypatch, + tmp_path: Path, +) -> None: + source = tmp_path / "full-native.manifest.json" + output = tmp_path / "result.json" + profile = tmp_path / "engine-profile.json" + write_json(source, {"schema_version": "full-native-vector-manifest-v1"}) + + def simulate_full_vector_file_profiled( + input_path: Path, + output_path: Path, + profile_path: Path, + events_path: Path | None, + pair_worker_limit: int, + ) -> None: + assert input_path == source + assert events_path is None + assert pair_worker_limit == 3 + write_json(output_path, {"trades": []}) + write_json(profile_path, {"input": {"raw_frame_count": 5}}) + + native = SimpleNamespace( + simulate_full_vector_file_profiled=simulate_full_vector_file_profiled + ) + monkeypatch.setattr( + engine_runtime, + "build_engine", + lambda: {"kind": "pyo3-extension", "binary_path": "fixture"}, + ) + monkeypatch.setattr(engine_runtime, "_native_module", lambda: native) + + report = engine_runtime.run_engine( + source, + output, + input_kind=engine_runtime.FULL_VECTOR_INPUT, + engine_profile_path=profile, + pair_worker_limit=3, + ) + + assert report["input_kind"] == engine_runtime.FULL_VECTOR_INPUT + assert report["pair_worker_limit"] == 3 + assert report["profile"]["phases"]["input"]["raw_frame_count"] == 5 + + +def test_engine_input_kind_never_uses_filename_inference(tmp_path: Path) -> None: + source = tmp_path / "looks-like-vectors.manifest.json" + write_json(source, {"schema_version": "fixture"}) + + with pytest.raises(BenchmarkError, match="unsupported engine input kind"): + engine_runtime.run_engine( + source, + tmp_path / "result.json", + input_kind="unknown-vector-kind", + ) + with pytest.raises(BenchmarkError, match="conflicts"): + engine_runtime.run_engine( + source, + tmp_path / "result.json", + input_kind=engine_runtime.FULL_VECTOR_INPUT, + vector_manifest=True, + ) + with pytest.raises(BenchmarkError, match="requires full-vector"): + engine_runtime.run_engine( + source, + tmp_path / "pair-workers.json", + pair_worker_limit=2, + ) + with pytest.raises(BenchmarkError, match="positive integer"): + engine_runtime.run_engine( + source, + tmp_path / "zero-workers.json", + input_kind=engine_runtime.FULL_VECTOR_INPUT, + pair_worker_limit=0, + ) + + def test_failed_cli_attempt_cleans_private_resource_capture( monkeypatch, tmp_path: Path, diff --git a/tests/test_full_native_calibration.py b/tests/test_full_native_calibration.py new file mode 100644 index 00000000..68f54dd2 --- /dev/null +++ b/tests/test_full_native_calibration.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from nfi_backtest_engine import full_native_calibration +from nfi_backtest_engine.canonical import read_json, write_json + + +def _manifest(tmp_path: Path) -> Path: + write_json( + tmp_path / "indicator.json", + { + "nodes": [ + { + "op": "frame-source", + "parameters": { + "pair": {"kind": "literal", "value": "BTC/USDT"}, + "timeframe": "4h", + }, + } + ] + }, + ) + manifest = tmp_path / "manifest.json" + write_json( + manifest, + { + "schema_version": "full-native-vector-manifest-v1", + "programs": { + "indicator": {"artifact": {"path": "indicator.json"}}, + }, + "pairs": [ + {"identity": {"pair": "SMALL/USDT", "timeframe": "5m"}}, + {"identity": {"pair": "LARGE/USDT", "timeframe": "5m"}}, + ], + "frames": [ + { + "identity": {"pair": "SMALL/USDT", "timeframe": "5m"}, + "rows": 10, + }, + { + "identity": {"pair": "LARGE/USDT", "timeframe": "5m"}, + "rows": 20, + }, + { + "identity": {"pair": "LARGE/USDT", "timeframe": "1h"}, + "rows": 2, + }, + { + "identity": {"pair": "BTC/USDT", "timeframe": "4h"}, + "rows": 1, + }, + ], + "futures": None, + }, + ) + return manifest + + +def test_probe_selects_largest_pair_and_literal_frames(tmp_path: Path) -> None: + manifest = _manifest(tmp_path) + + probe_path, pair = full_native_calibration._probe_manifest(manifest) + try: + probe = read_json(probe_path) + assert pair == "LARGE/USDT" + assert [item["identity"]["pair"] for item in probe["pairs"]] == [pair] + assert { + (item["identity"]["pair"], item["identity"]["timeframe"]) + for item in probe["frames"] + } == { + ("LARGE/USDT", "5m"), + ("LARGE/USDT", "1h"), + ("BTC/USDT", "4h"), + } + finally: + probe_path.unlink(missing_ok=True) + + +def test_measured_calibration_is_reused_by_content_identity( + monkeypatch, + tmp_path: Path, +) -> None: + manifest = _manifest(tmp_path) + calls: list[dict[str, Any]] = [] + monkeypatch.setattr( + full_native_calibration, + "build_engine", + lambda: {"source_fingerprint": "a" * 64}, + ) + + def fake_run_engine(*args, **kwargs) -> dict[str, Any]: + calls.append(kwargs) + return {"peak_rss_bytes": 1024, "wall_time_seconds": 0.25} + + arguments = { + "profile_path": tmp_path / "profile.json", + "hardware_fingerprint": "hardware", + "requested_workers": 3, + "memory_cap_bytes": None, + "calibration_directory": tmp_path / "calibrations", + "run_engine_fn": fake_run_engine, + } + created = full_native_calibration.resolve_full_native_pair_workers( + manifest, + **arguments, + ) + reused = full_native_calibration.resolve_full_native_pair_workers( + manifest, + **arguments, + ) + + assert created["reused"] is False + assert reused["reused"] is True + assert created["key"] == reused["key"] + assert created["probe_pair"] == "LARGE/USDT" + assert created["worker_limit"] == 3 + assert len(calls) == 1 + assert calls[0]["pair_worker_limit"] == 1 diff --git a/tests/test_full_native_performance_evidence.py b/tests/test_full_native_performance_evidence.py new file mode 100644 index 00000000..6b95a898 --- /dev/null +++ b/tests/test_full_native_performance_evidence.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from nfi_backtest_engine.latest_signal47_fixture import canonical_sha256 + +ROOT = Path(__file__).resolve().parents[1] +REPORT = ROOT / "benchmarks/evidence/m22/full-native-performance-storage.json" + + +def _load() -> dict[str, Any]: + return json.loads(REPORT.read_text(encoding="utf-8")) + + +def test_full_native_performance_evidence_is_hash_bound_and_repeatable() -> None: + report = _load() + + assert report["schema_version"] == "full-native-performance-storage-v1" + assert report["status"] == "certified" + assert report["fingerprint"] == canonical_sha256(report) + assert len(report["repetitions"]) == 3 + assert {item["name"] for item in report["repetitions"]} == { + "cold-01", + "warm-02", + "warm-03", + } + assert report["result"]["byte_identical_across_repetitions"] is True + assert report["result"]["trade_count"] > 0 + assert report["aggregate"]["wall_spread_ratio"] <= 0.05 + assert report["aggregate"]["five_repetitions_required"] is False + + +def test_full_native_storage_evidence_is_bounded_and_does_not_overclaim() -> None: + report = _load() + storage = report["storage"] + claims = report["claim_boundary"] + + assert storage["actual_file_backed_bytes"] <= storage["required_upper_bound_bytes"] + assert storage["bound_margin_bytes"] == ( + storage["required_upper_bound_bytes"] - storage["actual_file_backed_bytes"] + ) + assert storage["named_orphan_file_count"] == 0 + assert storage["delete_pending_handle_count_after_exit"] == 0 + assert storage["unbounded_local_accumulation_observed"] is False + assert claims["five_year_full_native_performance_and_storage_certified"] is True + assert claims["runtime_strategy_pair_timerange_sha_or_result_hardcoding_added"] is False + assert claims["official_freqtrade_five_year_parity_claimed"] is False + assert claims["cross_platform_performance_claimed"] is False + assert claims["futures_five_year_performance_claimed"] is False + assert claims["v1_6_0_released"] is False + + +def test_full_native_dual_mode_exact_regression_is_zero_tolerance() -> None: + regressions = _load()["exact_regression"] + + for mode in ("spot", "futures"): + evidence = regressions[mode] + assert evidence["trade_surface_exact"] is True + assert evidence["full_state_exact"] is True + assert evidence["state_event_count"] > 0 + assert len(evidence["state_stream_hash"]) == 64 diff --git a/tests/test_full_vector_runtime.py b/tests/test_full_vector_runtime.py new file mode 100644 index 00000000..f9e96610 --- /dev/null +++ b/tests/test_full_vector_runtime.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pandas as pd +import pytest +from nfi_backtest_engine import _rust, full_vector_runtime, research_runner +from nfi_backtest_engine.canonical import read_json, write_json +from nfi_backtest_engine.errors import BenchmarkError +from nfi_backtest_engine.full_vector_runtime import ( + _config_identity_sha256, + _retained_trade_features, + build_full_native_vector_manifest, +) +from nfi_backtest_engine.hot_ir import build_hot_callback_ir +from nfi_backtest_engine.strategy_ir import analyze_strategy + +START_MS = 1_735_689_600_000 + + +def test_config_identity_is_cross_language_and_float_formatter_independent() -> None: + left = {"z": [None, True, False, 7, -3, 1e-5, -0.0], "a": {"한글": "값"}} + right = {"a": {"한글": "값"}, "z": [None, True, False, 7, -3, 0.00001, -0.0]} + + expected = "df8efe5440e003a372b0ae0d57c7dcd360517af8ace16e68601e862f48a79525" + assert _config_identity_sha256(left) == expected + assert _config_identity_sha256(right) == expected + + +def test_retained_features_exclude_candle_fields_but_keep_raw_signals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + full_vector_runtime, + "_required_trade_features", + lambda _hot_ir: ["close", "enter_long", "open", "RSI_14"], + ) + + assert _retained_trade_features({}) == ["enter_long", "RSI_14"] + + +def test_builder_hardlinks_raw_frames_and_runs_the_sealed_manifest(tmp_path: Path) -> None: + strategy = tmp_path / "strategy.py" + _write_strategy(strategy) + data = tmp_path / "data" + raw = data / "BTC_USDT-5m.feather" + _write_frame(raw) + market = tmp_path / "market.json" + write_json(market, _market_snapshot()) + config = _config() + analysis = analyze_strategy(strategy, class_name="ManifestStrategy") + hot_ir = build_hot_callback_ir( + analysis, + trading_mode="spot", + run_mode="backtest", + config=config, + ) + manifest = tmp_path / "run" / "simulation-input.manifest.json" + + document = build_full_native_vector_manifest( + strategy_path=strategy, + class_name="ManifestStrategy", + analysis=analysis, + hot_ir=hot_ir, + config=config, + pairs=["BTC/USDT"], + data_directory=data, + timerange=f"{START_MS}-{START_MS + 900_000}", + market_metadata_path=market, + destination=manifest, + ) + + linked = manifest.parent / document["frames"][0]["artifact"]["path"] + assert os.path.samefile(raw, linked) + assert raw.stat().st_size == linked.stat().st_size + assert document["run"]["source_row_shift"] == 1 + assert document["source_execution"] == { + "strategy_source_mode": "python-ast-compile-only", + "populate_methods_executed": False, + "runtime_mode": "rust-full-native", + } + assert document["retained_features"]["columns"] == [] + assert read_json(manifest) == document + + result = tmp_path / "run" / "result.json" + profile = tmp_path / "run" / "profile.json" + _rust.simulate_full_vector_file_profiled(manifest, result, profile) + trades = read_json(result)["trades"] + assert len(trades) == 1 + assert trades[0]["entry_tag"] == "test " + assert trades[0]["exit_reason"] == "force_exit" + input_profile = read_json(profile)["input"] + assert input_profile["manifest_sha256"] is not None + assert input_profile["raw_frame_count"] == 1 + assert input_profile["strategy_source_mode"] == "python-ast-compile-only" + assert input_profile["populate_methods_executed"] is False + assert input_profile["runtime_mode"] == "rust-full-native" + assert input_profile["transport"]["pair_count"] == 1 + + research_runner._validate_full_native_manifest_artifacts(manifest) + linked.write_bytes(linked.read_bytes() + b"tampered") + with pytest.raises(BenchmarkError, match="artifact SHA-256 differs"): + research_runner._validate_full_native_manifest_artifacts(manifest) + + +def test_full_native_pipeline_never_executes_strategy_source(tmp_path: Path) -> None: + strategy = tmp_path / "strategy.py" + _write_strategy(strategy, execution_bomb=True) + data = tmp_path / "data" + _write_frame(data / "BTC_USDT-5m.feather") + market = tmp_path / "market.json" + write_json(market, _market_snapshot()) + config = _config() + analysis = analyze_strategy(strategy, class_name="ManifestStrategy") + hot_ir = build_hot_callback_ir( + analysis, + trading_mode="spot", + run_mode="backtest", + config=config, + ) + manifest = tmp_path / "run" / "simulation-input.manifest.json" + + build_full_native_vector_manifest( + strategy_path=strategy, + class_name="ManifestStrategy", + analysis=analysis, + hot_ir=hot_ir, + config=config, + pairs=["BTC/USDT"], + data_directory=data, + timerange=f"{START_MS}-{START_MS + 900_000}", + market_metadata_path=market, + destination=manifest, + ) + + result = tmp_path / "run" / "result.json" + _rust.simulate_full_vector_file(manifest, result) + assert len(read_json(result)["trades"]) == 1 + + +def _write_strategy(path: Path, *, execution_bomb: bool = False) -> None: + prefix = "raise RuntimeError('strategy source executed')\n" if execution_bomb else "" + path.write_text( + prefix + + "from freqtrade.strategy import IStrategy\n" + "class ManifestStrategy(IStrategy):\n" + " timeframe = '5m'\n" + " startup_candle_count = 0\n" + " stoploss = -0.2\n" + " minimal_roi = {}\n" + " can_short = False\n" + " def populate_indicators(self, dataframe, metadata):\n" + " dataframe['score'] = dataframe['close'] - dataframe['open']\n" + " return dataframe\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, ['enter_long', 'enter_short']] = 0\n" + " dataframe.loc[dataframe['score'] > 0, 'enter_long'] = 1\n" + " dataframe.loc[dataframe['score'] > 0, 'enter_tag'] = 'test '\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, ['exit_long', 'exit_short']] = 0\n" + " return dataframe\n", + encoding="utf-8", + ) + + +def _write_frame(path: Path) -> None: + path.parent.mkdir(parents=True) + pd.DataFrame( + { + "date": pd.to_datetime( + [START_MS + offset * 300_000 for offset in range(4)], + unit="ms", + utc=True, + ), + "open": [10.0, 10.0, 10.0, 10.0], + "high": [11.0, 12.0, 11.0, 10.0], + "low": [9.0, 9.0, 8.0, 9.0], + "close": [10.0, 11.0, 9.0, 10.0], + "volume": [1.0, 1.0, 1.0, 1.0], + } + ).to_feather(path) + + +def _config() -> dict: + return { + "trading_mode": "spot", + "stake_currency": "USDT", + "dry_run_wallet": 1_000.0, + "max_open_trades": 1, + "stake_amount": 100.0, + "fee": 0.001, + "enable_protections": False, + "exchange": {"name": "binance", "pair_whitelist": ["BTC/USDT"]}, + } + + +def _market_snapshot() -> dict: + return { + "exchange": "binance", + "markets": { + "BTC/USDT": { + "taker": 0.001, + "precision": {"amount": 0.001, "price": 0.01}, + "limits": { + "amount": {"min": 0.001}, + "cost": {"min": 5.0}, + "leverage": {}, + }, + } + }, + } diff --git a/tests/test_indicator_program.py b/tests/test_indicator_program.py index aff00e1d..03fffcb7 100644 --- a/tests/test_indicator_program.py +++ b/tests/test_indicator_program.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import json from pathlib import Path import pytest @@ -165,6 +166,31 @@ def test_committed_indicator_program_contract_remains_schema_valid() -> None: assert all(node["lookback"]["causal"] for node in program["nodes"]) +def test_indicator_program_separates_positional_talib_parameters(tmp_path: Path) -> None: + source = tmp_path / "PositionalTalib.py" + source.write_text( + "import talib.abstract as ta\n" + "from freqtrade.strategy import IStrategy\n" + "class PositionalTalib(IStrategy):\n" + " timeframe = '5m'\n" + " def populate_indicators(self, dataframe, metadata):\n" + " dataframe['roc'] = ta.ROC(dataframe['close'], 10)\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="PositionalTalib") + call = next(node for node in program["nodes"] if node["op"] == "indicator-call") + + assert call["parameters"] == { + "family": "ta", + "name": "ROC", + "arguments": {"timeperiod": 10}, + } + assert len(call["inputs"]) == 1 + validate_indicator_program(program) + + def test_indicator_program_semantic_validator_rejects_reference_and_identity_mutation() -> None: program = compile_indicator_program( CONTRACT, @@ -430,6 +456,292 @@ def test_indicator_program_constant_folds_static_control_without_lookahead(tmp_p assert all(node["lookback"]["causal"] for node in program["nodes"]) +def test_indicator_program_uses_configured_static_helper_without_leaking_config( + tmp_path: Path, +) -> None: + source = tmp_path / "ConfiguredHelper.py" + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class ConfiguredHelper(IStrategy):\n" + " def selected_timeframe(self, fallback='5m'):\n" + " return self.config.get('informative_timeframe', fallback)\n" + " def populate_indicators(self, dataframe, metadata):\n" + " selected = self.selected_timeframe()\n" + " if selected == '1h':\n" + " dataframe['selected'] = dataframe['close']\n" + " return dataframe\n", + encoding="utf-8", + ) + secret = "must-not-appear-in-native-ir" + + program = compile_indicator_program( + source, + class_name="ConfiguredHelper", + config={"informative_timeframe": "1h", "exchange": {"secret": secret}}, + ) + + assert program["produced_columns"] == ["selected"] + assert secret not in json.dumps(program, sort_keys=True) + helper = next( + function + for function in program["functions"] + if function["source_name"] == "selected_timeframe" + ) + assert helper["parameters"] == [] + assert all(node["op"] != "function-call" for node in program["nodes"]) + + +def test_indicator_program_specializes_helpers_defaults_callables_lambda_and_shift( + tmp_path: Path, +) -> None: + source = tmp_path / "SpecializedHelpers.py" + source.write_text( + "import numpy as np\n" + "import talib.abstract as ta\n" + "from freqtrade.strategy import IStrategy\n" + "class SpecializedHelpers(IStrategy):\n" + " @staticmethod\n" + " def np_shift(values, periods):\n" + " out = np.empty_like(values)\n" + " out[:periods] = np.nan\n" + " out[periods:] = values[:-periods]\n" + " return out\n" + " def transform(self, values, callback, period=3, *, multiplier=1.0):\n" + " return callback(values, timeperiod=period) * multiplier\n" + " def populate_indicators(self, dataframe, metadata):\n" + " view = lambda column: dataframe[column].to_numpy(copy=False)\n" + " close = view('close')\n" + " smoother = ta.SMA\n" + " dataframe['default'] = self.transform(close, smoother)\n" + " dataframe['special'] = self.transform(\n" + " close, smoother, period=5, multiplier=2.0\n" + " )\n" + " dataframe['shifted'] = self.np_shift(close, 2)\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="SpecializedHelpers") + + helpers = [ + function for function in program["functions"] if function["source_name"] == "transform" + ] + assert len(helpers) == 2 + calls = [ + node + for node in program["nodes"] + if node["op"] == "indicator-call" and node["parameters"]["name"] == "SMA" + ] + assert [node["parameters"]["arguments"]["timeperiod"] for node in calls] == [3, 5] + assert next(node for node in program["nodes"] if node["op"] == "shift")["parameters"] == { + "periods": 2 + } + assert program["required_input_columns"] == ["close"] + + +def test_indicator_program_lowers_numpy_buffers_and_static_container_unroll( + tmp_path: Path, +) -> None: + source = tmp_path / "NumpyAndContainers.py" + source.write_text( + "import numpy as np\n" + "from freqtrade.strategy import IStrategy\n" + "def reduce_any(conditions):\n" + " return np.logical_or.reduce(conditions)\n" + "class NumpyAndContainers(IStrategy):\n" + " def populate_indicators(self, dataframe, metadata):\n" + " close = dataframe['close']\n" + " width = dataframe['upper'] - dataframe['lower']\n" + " ratio = np.divide(\n" + " close - dataframe['lower'], width,\n" + " out=np.full_like(width, np.nan), where=width != 0,\n" + " )\n" + " outputs = {}\n" + " outputs['ratio'] = ratio > 0\n" + " outputs['width'] = width > 0\n" + " conditions = []\n" + " for name, values in outputs.items():\n" + " conditions.append(values)\n" + " for threshold in [1, 2]:\n" + " conditions.append(close > threshold)\n" + " dataframe['ratio'] = ratio\n" + " dataframe['valid'] = np.full(len(dataframe), True)\n" + " dataframe['combined'] = reduce_any(conditions)\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="NumpyAndContainers") + + assert program["produced_columns"] == ["combined", "ratio", "valid"] + array_calls = [ + node["parameters"]["name"] + for node in program["nodes"] + if node["op"] == "array-call" + ] + assert array_calls == [ + "full_like", + "divide", + "full", + ] + assert len([node for node in program["nodes"] if node["op"] == "row-count"]) == 1 + combined = next( + node + for node in program["nodes"] + if node["op"] == "logical" and len(node["inputs"]) == 4 + ) + assert combined["parameters"] == {"operator": "or"} + validate_indicator_program(program) + + +def test_indicator_program_recognizes_legacy_chaikin_volume_sum_contract( + tmp_path: Path, +) -> None: + source = tmp_path / "LegacyChaikin.py" + source.write_text( + "import numpy as np\n" + "import talib.abstract as ta\n" + "from freqtrade.strategy import IStrategy\n" + "class LegacyChaikin(IStrategy):\n" + " @staticmethod\n" + " def rolling_sum(arr, timeperiod):\n" + " return arr\n" + " @staticmethod\n" + " def chaikin_money_flow(high, low, close, volume, timeperiod=20):\n" + " hl_range = high - low\n" + " mfm = np.zeros_like(close, dtype=np.float64)\n" + " valid = hl_range != 0\n" + " mfm[valid] = ((close[valid] - low[valid]) - " + "(high[valid] - close[valid])) / hl_range[valid]\n" + " mfv = mfm * volume\n" + " mfv_sum = __class__.rolling_sum(mfv, timeperiod)\n" + " vol_sum = ta.SUM(volume, timeperiod=timeperiod)\n" + " vol_sum = np.where(vol_sum == 0, np.nan, vol_sum)\n" + " return mfv_sum / vol_sum\n" + " def populate_indicators(self, dataframe, metadata):\n" + " dataframe['cmf'] = self.chaikin_money_flow(\n" + " dataframe['high'], dataframe['low'], dataframe['close'],\n" + " dataframe['volume'], timeperiod=20,\n" + " )\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="LegacyChaikin") + + call = next(node for node in program["nodes"] if node["op"] == "indicator-call") + assert call["parameters"] == { + "family": "native", + "name": "chaikin-money-flow-legacy", + "arguments": {"timeperiod": 20}, + } + assert call["lookback"] == { + "kind": "function-defined", + "candles": 19, + "expression": "chaikin-money-flow-legacy", + "causal": True, + } + validate_indicator_program(program) + + +def test_indicator_program_unrolls_tuple_of_source_ordered_dynamic_mappings( + tmp_path: Path, +) -> None: + source = tmp_path / "MappingGroups.py" + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class MappingGroups(IStrategy):\n" + " def populate_indicators(self, dataframe, metadata):\n" + " first = {}\n" + " first['5m'] = dataframe\n" + " second = {}\n" + " second['15m'] = dataframe\n" + " for frames in (first, second):\n" + " for timeframe, frame in frames.items():\n" + " dataframe = frame\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="MappingGroups") + + assert program["required_input_columns"] == [] + assert program["produced_columns"] == [] + assert program["opcodes"] == ["parameter", "return"] + validate_indicator_program(program) + + +def test_indicator_program_lowers_source_guarded_inplace_column_drop( + tmp_path: Path, +) -> None: + source = tmp_path / "InplaceDrop.py" + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class InplaceDrop(IStrategy):\n" + " def populate_indicators(self, dataframe, metadata):\n" + " column = 'date_15m'\n" + " if column in dataframe.columns:\n" + " dataframe.drop(columns=column, inplace=True)\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="InplaceDrop") + + dropped = next( + node for node in program["nodes"] if node["op"] == "frame-drop-if-present" + ) + assert dropped["parameters"] == {"column": "date_15m"} + validate_indicator_program(program) + + +def test_indicator_program_lowers_inplace_forward_fill_as_frame_rebinding( + tmp_path: Path, +) -> None: + source = tmp_path / "InplaceFill.py" + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class InplaceFill(IStrategy):\n" + " def populate_indicators(self, dataframe, metadata):\n" + " dataframe.ffill(inplace=True)\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="InplaceFill") + + fill = next(node for node in program["nodes"] if node["op"] == "fill") + assert fill["value_type"] == "dataframe" + assert fill["parameters"] == {"direction": "forward"} + assert fill["lookback"]["kind"] == "recursive" + validate_indicator_program(program) + + +def test_indicator_program_lowers_numeric_fillna_without_collapsing_infinity( + tmp_path: Path, +) -> None: + source = tmp_path / "FillMissing.py" + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class FillMissing(IStrategy):\n" + " def populate_indicators(self, dataframe, metadata):\n" + " dataframe['filled'] = dataframe['source'].fillna(50.0)\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_indicator_program(source, class_name="FillMissing") + + call = next(node for node in program["nodes"] if node["op"] == "array-call") + assert call["parameters"] == { + "family": "numpy", + "name": "fill-missing", + "arguments": {}, + } + assert call["value_type"] == "f64-column" + validate_indicator_program(program) + + def test_indicator_program_rejects_dynamic_window_and_helper_signature(tmp_path: Path) -> None: source = tmp_path / "DynamicWindow.py" source.write_text( @@ -452,11 +764,11 @@ def test_indicator_program_rejects_dynamic_window_and_helper_signature(tmp_path: source.write_text( source.read_text(encoding="utf-8").replace( "dataframe['close'].rolling(window).mean()", - "self.helper(values=dataframe['close'])", + "self.helper(dataframe['close'], values=dataframe['close'])", ), encoding="utf-8", ) - with pytest.raises(IndicatorProgramCompileError, match="helper call signature"): + with pytest.raises(IndicatorProgramCompileError, match="duplicate indicator helper argument"): compile_indicator_program(source, class_name="DynamicWindow") diff --git a/tests/test_latest_full_native_qualification.py b/tests/test_latest_full_native_qualification.py new file mode 100644 index 00000000..6387cc82 --- /dev/null +++ b/tests/test_latest_full_native_qualification.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from nfi_backtest_engine.latest_signal47_fixture import canonical_sha256 + +ROOT = Path(__file__).resolve().parents[1] +REPORT = ROOT / "benchmarks/evidence/m22/latest-x7-full-native-qualification.json" +BOUNDARY = ROOT / "benchmarks/evidence/m22/latest-x7-signal47-boundary.json" + + +def _load(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def test_latest_full_native_qualification_is_hash_bound_and_exact() -> None: + report = _load(REPORT) + boundary = _load(BOUNDARY) + + assert report["schema_version"] == "latest-x7-full-native-qualification-v1" + assert report["status"] == "exact" + assert report["fingerprint"] == canonical_sha256(report) + assert report["source"] == { + "upstream_commit": "1df961c07e5ce6b1a8cb459a2a46958aed258323", + "strategy_sha256": "45a2bf611d6fc5e60c7e1f4c672ce7932f6573872c2a959f6338d079dac5e382", + "strategy_version": "v17.4.528", + "freqtrade_version": "2026.5.1", + } + assert report["changed_branch"]["evidence_fingerprint"] == boundary["fingerprint"] + assert report["changed_branch"]["evidence_sha256"] == hashlib.sha256( + BOUNDARY.read_bytes() + ).hexdigest() + + for mode in ("spot", "futures"): + evidence = report[mode] + assert evidence["transport"] == "full-native-vector-manifest" + assert evidence["trade_surface"]["exact"] is True + assert evidence["trade_surface"]["native_sha256"] == ( + evidence["trade_surface"]["official_sha256"] + ) + assert evidence["full_state"]["exact"] is True + assert evidence["full_state"]["first_difference"] is None + assert evidence["full_state"]["event_count"] > 0 + assert evidence["blockers"] == [] + + +def test_latest_qualification_does_not_overclaim_later_m22_gates() -> None: + claims = _load(REPORT)["claims"] + + assert claims["latest_upstream_compiles_without_blockers"] is True + assert claims["indicator_signal_tag_and_stateful_runtime_is_native"] is True + assert claims["python_strategy_execution_in_native_runtime"] is False + assert claims["official_fallback_retained"] is True + assert claims["runtime_strategy_pair_timerange_sha_or_result_branches_added"] is False + assert claims["m22_01_latest_dual_mode_qualification_complete"] is True + assert claims["five_year_performance_certified"] is False + assert claims["v1_6_0_released"] is False diff --git a/tests/test_latest_signal47_fixture.py b/tests/test_latest_signal47_fixture.py new file mode 100644 index 00000000..3a035114 --- /dev/null +++ b/tests/test_latest_signal47_fixture.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from nfi_backtest_engine.latest_signal47_fixture import ( + CHANGED_EXPRESSION, + FIXTURE_PATH, + SCHEMA_VERSION, + canonical_sha256, +) + +ROOT = Path(__file__).resolve().parents[1] + + +def test_latest_signal47_boundary_is_dual_mode_and_source_bound() -> None: + fixture = json.loads((ROOT / FIXTURE_PATH).read_text(encoding="utf-8")) + + assert fixture["schema_version"] == SCHEMA_VERSION + assert fixture["fingerprint"] == canonical_sha256(fixture) + assert fixture["source"]["commit"] == "1df961c07e5ce6b1a8cb459a2a46958aed258323" + assert fixture["source"]["strategy_sha256"] == ( + "45a2bf611d6fc5e60c7e1f4c672ce7932f6573872c2a959f6338d079dac5e382" + ) + assert fixture["changed_route"]["expression"] == CHANGED_EXPRESSION + assert set(fixture["source_diff"]["changed_methods"]) == { + "populate_entry_trend", + "version", + } + assert fixture["source_diff"]["callback_and_stateful_methods_changed"] is False + assert fixture["source_diff"]["unchanged_method_count"] > 100 + assert set(fixture["modes"]) == {"spot", "futures"} + + expected_pass = [0, 0, 0, 0, 0, 1, 1, 1] + expected_reject = [0] * 8 + for mode in fixture["modes"].values(): + assert mode["required_input_column_count"] == 179 + reject = mode["cases"]["new_protection_rejects"] + accepted = mode["cases"]["one_term_passes"] + assert reject["baseline"]["enter_long"] == expected_pass + assert reject["baseline"]["enter_tag"][-3:] == ["47 "] * 3 + assert reject["current"]["enter_long"] == expected_reject + assert reject["current"]["enter_tag"] == [""] * 8 + assert accepted["baseline"]["enter_long"] == expected_pass + assert accepted["current"]["enter_long"] == expected_pass + assert accepted["current"]["enter_tag"][-3:] == ["47 "] * 3 + + +def test_latest_signal47_boundary_records_independent_exact_lanes() -> None: + fixture = json.loads((ROOT / FIXTURE_PATH).read_text(encoding="utf-8")) + + assert fixture["claims"] == { + "source_wrapper_python_program_exact": True, + "source_wrapper_rust_signal_exact": True, + "source_wrapper_rust_tag_exact": True, + "spot_and_futures_exact": True, + "runtime_signal_number_branch_added": False, + } + for mode in fixture["modes"].values(): + for identity in ("current", "baseline"): + programs = mode["programs"][identity] + assert len(programs["signal_fingerprint"]) == 64 + assert len(programs["tag_fingerprint"]) == 64 + assert programs["signal_node_count"] > 9_000 + assert programs["tag_node_count"] > programs["signal_node_count"] diff --git a/tests/test_native_full_vector_bridge.py b/tests/test_native_full_vector_bridge.py new file mode 100644 index 00000000..c434e404 --- /dev/null +++ b/tests/test_native_full_vector_bridge.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import math +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import pytest +from nfi_backtest_engine import _rust +from nfi_backtest_engine.indicator_program import compile_indicator_program +from nfi_backtest_engine.signal_program import compile_signal_program +from nfi_backtest_engine.tag_program import compile_tag_program + + +def _write_strategy(path: Path) -> None: + path.write_text( + "from freqtrade.strategy import IStrategy, merge_informative_pair\n" + "class BridgeStrategy(IStrategy):\n" + " timeframe = '5m'\n" + " def populate_indicators(self, dataframe, metadata):\n" + " informative = self.dp.get_pair_dataframe(\n" + " pair=metadata['pair'], timeframe='15m'\n" + " )\n" + " dataframe = merge_informative_pair(\n" + " dataframe, informative, self.timeframe, '15m', ffill=False\n" + " )\n" + " dataframe['score'] = dataframe['close'] - dataframe['open']\n" + " dataframe['exit_mask'] = dataframe['close_15m']\n" + " return dataframe\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, ['enter_long', 'enter_short']] = 0\n" + " dataframe.loc[dataframe['score'] > 0, 'enter_long'] = 1\n" + " dataframe.loc[dataframe['score'] > 0, 'enter_tag'] = '101 '\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, ['exit_long', 'exit_short']] = 0\n" + " dataframe.loc[dataframe['exit_mask'] > 0, 'exit_long'] = 1\n" + " dataframe.loc[dataframe['exit_mask'] > 0, 'exit_tag'] = 'done '\n" + " return dataframe\n", + encoding="utf-8", + ) + + +def _programs(path: Path) -> tuple[str, str, str]: + programs = ( + compile_indicator_program(path, class_name="BridgeStrategy"), + compile_signal_program(path, class_name="BridgeStrategy"), + compile_tag_program(path, class_name="BridgeStrategy"), + ) + return tuple(json.dumps(program, separators=(",", ":")) for program in programs) # type: ignore[return-value] + + +def _execute(programs: tuple[str, str, str]) -> Mapping[str, Any]: + indicator, signal, tag = programs + return _rust.execute_full_vector( + indicator, + signal, + tag, + "ETH/USDT", + "5m", + [0, 300_000, 600_000, 900_000], + { + "open": [2.0, 2.0, 2.0, 2.0], + "close": [1.0, 3.0, 4.0, 1.0], + "raw_nan": [None, float("nan"), -0.0, 4.0], + }, + [ + ( + "ETH/USDT", + "15m", + [-600_000, 0], + {"close": [None, float("nan")]}, + ) + ], + {"pair": "ETH/USDT"}, + ["open", "close", "raw_nan", "close_15m"], + 1, + ) + + +def _reseal(program: dict[str, Any]) -> str: + identity = copy.deepcopy(program) + identity.pop("fingerprint") + identity["source"].pop("path") + encoded = json.dumps( + identity, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + program["fingerprint"] = hashlib.sha256(encoded).hexdigest() + return json.dumps(program, separators=(",", ":")) + + +def _write_numeric_mutation_strategy(path: Path) -> None: + path.write_text( + "from freqtrade.strategy import IStrategy\n" + "class NumericMutationStrategy(IStrategy):\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, ['enter_long', 'enter_short']] = 0\n" + " dataframe.loc[dataframe['score'] > 0, 'enter_long'] = 1\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, ['exit_long', 'exit_short']] = 0\n" + " return dataframe\n", + encoding="utf-8", + ) + + +def test_full_vector_bridge_runs_in_memory_with_independent_frame_lengths( + tmp_path: Path, +) -> None: + strategy = tmp_path / "strategy.py" + _write_strategy(strategy) + result = _execute(_programs(strategy)) + + assert result["pair"] == "ETH/USDT" + assert result["timeframe"] == "5m" + assert result["execution_start_index"] == 1 + assert result["timestamps_ms"] == [0, 300_000, 600_000, 900_000] + assert result["columns"]["date"]["value_type"] == "Timestamp(Millisecond)" + raw_nan = result["columns"]["raw_nan"]["values"] + assert raw_nan[0] is None + assert math.isnan(raw_nan[1]) + assert math.copysign(1.0, raw_nan[2]) == -1.0 + informative = result["columns"]["close_15m"]["values"] + assert informative[0] is None + assert math.isnan(informative[1]) + assert math.isnan(informative[2]) + assert result["columns"]["nfi_exec_enter_tag"]["values"] == [ + None, + "", + "101 ", + "101 ", + ] + assert result["enabled_indexes"]["enter_long"] == [2, 3] + + +def test_full_vector_bridge_rejects_signal_tag_surface_drift(tmp_path: Path) -> None: + strategy = tmp_path / "strategy.py" + _write_strategy(strategy) + indicator, signal, encoded_tag = _programs(strategy) + tag = json.loads(encoded_tag) + enter_one = next( + node + for node in tag["nodes"] + if node["op"] == "literal" + and node["parameters"].get("value") == 1 + and node["function"] == "f1" + ) + enter_one["parameters"]["value"] = 0 + + with pytest.raises(ValueError, match="Signal and Tag programs disagree on enter_long"): + _execute((indicator, signal, _reseal(tag))) + + +def test_numeric_mutation_bridge_executes_compiled_program_without_strategy_python( + tmp_path: Path, +) -> None: + strategy = tmp_path / "numeric_mutation.py" + _write_numeric_mutation_strategy(strategy) + program = compile_signal_program(strategy, class_name="NumericMutationStrategy") + + result = _rust.execute_numeric_mutation_program( + json.dumps(program, separators=(",", ":")), + {"score": [None, -0.0, 1.0]}, + {"pair": "ETH/USDT"}, + ["enter_long", "enter_short", "exit_long", "exit_short"], + ) + + assert result["enter_long"] == { + "value_type": "Int64", + "values": [0, 0, 1], + } + assert result["enter_short"]["values"] == [0, 0, 0] + assert result["exit_long"]["values"] == [0, 0, 0] + assert result["exit_short"]["values"] == [0, 0, 0] diff --git a/tests/test_native_full_vector_file_api.py b/tests/test_native_full_vector_file_api.py new file mode 100644 index 00000000..83182487 --- /dev/null +++ b/tests/test_native_full_vector_file_api.py @@ -0,0 +1,18 @@ +from pathlib import Path + +import pytest +from nfi_backtest_engine import _rust + + +def test_full_native_file_entrypoints_fail_closed_before_writing(tmp_path: Path) -> None: + missing = tmp_path / "missing-full-native-manifest.json" + result = tmp_path / "result.json" + profile = tmp_path / "profile.json" + + with pytest.raises(ValueError, match="invalid full native vector manifest"): + _rust.simulate_full_vector_file(missing, result) + with pytest.raises(ValueError, match="invalid full native vector manifest"): + _rust.simulate_full_vector_file_profiled(missing, result, profile) + + assert not result.exists() + assert not profile.exists() diff --git a/tests/test_nfi_trade_manager.py b/tests/test_nfi_trade_manager.py index 975aaec8..3aa4b327 100644 --- a/tests/test_nfi_trade_manager.py +++ b/tests/test_nfi_trade_manager.py @@ -353,3 +353,115 @@ def test_adjustment_policy_rejects_untyped_fallback_calls() -> None: match="grind 5 fallback condition changed", ): _adjustment_literal_policy(method) + + +def test_adjustment_policy_compiles_generic_futures_liquidation_expression() -> None: + method = _adjustment_method() + grind_five = next( + node + for node in method.body + if isinstance(node, ast.If) + and any( + isinstance(value, ast.Constant) and value.value == "grind_5_entry" + for value in ast.walk(node) + ) + ) + assert isinstance(grind_five.test, ast.BoolOp) + signal = grind_five.test.values[1] + assert isinstance(signal, ast.BoolOp) + signal.values.append( + ast.parse( + """ +(self.is_futures_mode +and slice_profit_entry < -0.15 +and trade.liquidation_price is not None +and ( + (trade.is_short and current_rate > trade.liquidation_price * 0.80) + or (not trade.is_short and current_rate < trade.liquidation_price * 1.20) +)) +""", + mode="eval", + ).body + ) + + policy = _adjustment_literal_policy(method) + expression = policy["grind_entry_fallbacks"][4]["predicates"][1]["expression"] + + assert expression["op"] == "all" + assert expression["values"][0] == {"op": "flag", "name": "is_futures_mode"} + assert expression["values"][1] == { + "op": "comparison", + "left": {"kind": "variable", "name": "slice_profit_entry"}, + "operator": "lt", + "right": {"kind": "literal", "value": -0.15}, + } + assert expression["values"][2] == { + "op": "present", + "operand": { + "kind": "trade", + "name": "liquidation_price", + "multiplier": 1.0, + }, + } + proximity = expression["values"][3] + assert proximity["op"] == "any" + assert proximity["values"][0]["values"][0] == { + "op": "flag", + "name": "trade_is_short", + } + assert proximity["values"][1]["values"][0] == { + "op": "not", + "value": {"op": "flag", "name": "trade_is_short"}, + } + + +def test_adjustment_policy_compiles_exit_profit_boolean_and_bare_derisk() -> None: + method = _adjustment_method() + grind_five = next( + node + for node in method.body + if isinstance(node, ast.If) + and any( + isinstance(value, ast.Constant) and value.value == "grind_5_entry" + for value in ast.walk(node) + ) + ) + assert isinstance(grind_five.test, ast.BoolOp) + signal = grind_five.test.values[1] + assert isinstance(signal, ast.BoolOp) + signal.values.append( + ast.parse( + """ +(is_derisk_1_found +and slice_profit_exit < -0.04 +and last_candle["protections_long_global"] == True) +""", + mode="eval", + ).body + ) + + policy = _adjustment_literal_policy(method) + expression = policy["grind_entry_fallbacks"][4]["predicates"][1]["expression"] + + assert expression == { + "op": "all", + "values": [ + {"op": "derisk_found", "level": 1}, + { + "op": "comparison", + "left": {"kind": "variable", "name": "slice_profit_exit"}, + "operator": "lt", + "right": {"kind": "literal", "value": -0.04}, + }, + { + "op": "comparison", + "left": { + "kind": "feature", + "name": "protections_long_global", + "multiplier": 1.0, + }, + "operator": "eq", + "right": {"kind": "literal", "value": 1.0}, + }, + ], + } diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 431896f8..c547c1ae 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -472,7 +472,7 @@ def test_product_release_workflows_preserve_non_combined_boundary() -> None: assert contract == { "schema_version": "1.0.0", - "package_version": "1.5.0", + "package_version": "1.6.0", "release_kind": "product", "combined_full_x7_certified": False, "distribution_policy": { diff --git a/tests/test_research_runner.py b/tests/test_research_runner.py index 5c6528b7..61439c78 100644 --- a/tests/test_research_runner.py +++ b/tests/test_research_runner.py @@ -6,7 +6,11 @@ import pytest from nfi_backtest_engine import research_runner from nfi_backtest_engine.canonical import read_json, write_json -from nfi_backtest_engine.errors import BenchmarkError, SpecValidationError +from nfi_backtest_engine.errors import ( + BenchmarkError, + SpecValidationError, + StrategyAnalysisError, +) from nfi_backtest_engine.fixture import sha256_file from nfi_backtest_engine.strategy_ir import analyze_strategy @@ -361,6 +365,185 @@ def test_completed_resume_returns_verified_result_without_running_engine( } +def test_full_native_transport_skips_python_vector_worker( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + arguments, calls = _resume_workspace(monkeypatch, tmp_path) + observed: dict[str, object] = {} + programs = { + name: { + "schema_version": f"{name}-program-v1", + "fingerprint": token * 64, + "nodes": [{"id": "n1"}], + } + for name, token in (("indicator", "a"), ("signal", "b"), ("tag", "c")) + } + policy = { + "schema_version": research_runner.STATEFUL_EXECUTION_POLICY_VERSION, + "adapter_lane": "x7-generic-stateful", + "transport": research_runner.FULL_NATIVE_VECTOR_TRANSPORT, + "primary": "source-compiled-full-native-and-generic-stateful-programs", + "programs": [], + "legacy_shadow": {"enabled": False}, + "official_fallback": {"available_on_native_blocker": True}, + "blockers": [], + "native_ready": True, + "fingerprint": "d" * 64, + } + + monkeypatch.setattr( + research_runner, + "build_native_execution_policy", + lambda *args, **kwargs: policy, + ) + monkeypatch.setattr( + research_runner, + "compile_full_native_programs", + lambda *args, **kwargs: programs, + ) + monkeypatch.setattr(research_runner, "x7_adapter_blockers", lambda *a, **k: []) + monkeypatch.setattr( + research_runner, + "prepare_vector_signals", + lambda **kwargs: pytest.fail("Full Native must not execute strategy Python"), + ) + + def fake_full_manifest(**kwargs) -> dict: + observed["compiled_programs"] = kwargs["compiled_programs"] + write_json(kwargs["destination"], {"schema_version": "full-native-test"}) + return {"schema_version": "full-native-test"} + + def fake_full_engine(_input, output, **kwargs) -> dict: + calls["engine"] += 1 + observed["input_kind"] = kwargs.get("input_kind") + observed["vector_manifest"] = kwargs.get("vector_manifest") + observed["pair_worker_limit"] = kwargs.get("pair_worker_limit") + write_json(output, {"schema_version": "test"}) + write_json(kwargs["engine_profile_path"], {"schema_version": "test"}) + return {"wall_time_seconds": 0.1, "peak_rss_bytes": 1024} + + monkeypatch.setattr( + research_runner, + "build_full_native_vector_manifest", + fake_full_manifest, + ) + calibration = { + "schema_version": "1.0.0", + "worker_limit": 2, + "probe_pair": "BTC/USDT", + } + + def fake_calibration(*args, **kwargs) -> dict: + observed["calibration_requested_workers"] = kwargs["requested_workers"] + observed["calibration_memory_cap_bytes"] = kwargs["memory_cap_bytes"] + return calibration + + monkeypatch.setattr( + research_runner, + "resolve_full_native_pair_workers", + fake_calibration, + ) + monkeypatch.setattr(research_runner, "run_engine", fake_full_engine) + + report = research_runner.run_research_backtest(**arguments) + + assert report["status"] == "complete" + assert calls["vectors"] == 0 + assert report["vectors"]["worker_count"] == 0 + assert report["vectors"]["source_execution"] == { + "strategy_source_mode": "python-ast-compile-only", + "populate_methods_executed": False, + "runtime_mode": "rust-full-native", + } + assert set(report["vectors"]["programs"]) == {"indicator", "signal", "tag"} + assert not (Path(arguments["output_directory"]) / "vectors").exists() + assert observed["compiled_programs"] is programs + assert observed["input_kind"] == research_runner.FULL_VECTOR_INPUT + assert observed["pair_worker_limit"] == 2 + assert observed["vector_manifest"] is None + assert observed["calibration_requested_workers"] == 2 + assert observed["calibration_memory_cap_bytes"] == 8 * 1024**3 + assert report["execution"]["workload_calibration"] == calibration + + +def test_full_native_compiler_failure_is_a_durable_fallback_blocker( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + arguments, calls = _resume_workspace(monkeypatch, tmp_path) + policy = { + "schema_version": research_runner.STATEFUL_EXECUTION_POLICY_VERSION, + "adapter_lane": "x7-generic-stateful", + "transport": research_runner.FULL_NATIVE_VECTOR_TRANSPORT, + "primary": "source-compiled-full-native-and-generic-stateful-programs", + "programs": [], + "legacy_shadow": {"enabled": False}, + "official_fallback": {"available_on_native_blocker": True}, + "blockers": [], + "native_ready": True, + "fingerprint": "d" * 64, + } + monkeypatch.setattr( + research_runner, + "build_native_execution_policy", + lambda *args, **kwargs: policy, + ) + + def unsupported(*args, **kwargs): + raise StrategyAnalysisError("strategy.py:47: unsupported callback expression") + + monkeypatch.setattr(research_runner, "compile_full_native_programs", unsupported) + monkeypatch.setattr( + research_runner, + "prepare_vector_signals", + lambda **kwargs: pytest.fail("blocked Full Native must not execute strategy Python"), + ) + monkeypatch.setattr( + research_runner, + "build_full_native_vector_manifest", + lambda **kwargs: pytest.fail("blocked Full Native must not build a manifest"), + ) + monkeypatch.setattr( + research_runner, + "run_engine", + lambda *args, **kwargs: pytest.fail("blocked Full Native must not run the engine"), + ) + + report = research_runner.run_research_backtest(**arguments) + + assert report["status"] == "blocked_unsupported_semantics" + assert calls["vectors"] == 0 + blocker = report["capability"]["blockers"][-1] + assert blocker == { + "code": "FULL_NATIVE_SOURCE_COMPILER_UNSUPPORTED", + "message": "strategy.py:47: unsupported callback expression", + } + assert report["capability"]["native_execution"]["native_ready"] is False + assert report["vectors"]["blockers"] == [blocker] + + +def test_full_native_checkpoint_seals_no_strategy_execution_contract(tmp_path: Path) -> None: + report = research_runner._full_native_preflight_report( + analysis={"source": {"sha256": "a" * 64}}, + programs={ + name: { + "schema_version": f"{name}-program-v1", + "fingerprint": token * 64, + "nodes": [{"id": "n1"}], + } + for name, token in (("indicator", "b"), ("signal", "c"), ("tag", "d")) + }, + pair_count=1, + blockers=[], + ) + checkpoint = {"schema_version": "1.0.0", "report": report} + + assert research_runner._valid_vector_checkpoint(checkpoint, tmp_path / "vectors") + report["source_execution"]["populate_methods_executed"] = True + assert not research_runner._valid_vector_checkpoint(checkpoint, tmp_path / "vectors") + + def test_completed_resume_rejects_tampered_result_without_rewriting_it( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/test_signal_program.py b/tests/test_signal_program.py index 51ece91b..8c7eb8fa 100644 --- a/tests/test_signal_program.py +++ b/tests/test_signal_program.py @@ -122,7 +122,6 @@ def test_signal_program_identity_rejects_order_or_source_map_mutation(tmp_path: @pytest.mark.parametrize( ("statement", "message"), [ - ("dataframe.loc[:, 'enter_tag'] = '1 '", "tag mutation"), ("dataframe.loc[:, 'feature'] = 1", "non-signal dataframe output"), ("dataframe.loc[:, 'exit_long'] = 1", "during the entry phase"), ("dataframe.iloc[:, 0] = 1", "nested dataframe write"), @@ -151,6 +150,130 @@ def test_signal_program_fails_closed_outside_m21_signal_surface( compile_signal_program(source, class_name="Unsupported") +def test_signal_program_leaves_tag_writes_to_the_independent_tag_program( + tmp_path: Path, +) -> None: + source = tmp_path / "SeparateTag.py" + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class SeparateTag(IStrategy):\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, 'enter_long'] = 0\n" + " dataframe.loc[:, 'enter_tag'] = '1 '\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, 'exit_long'] = 0\n" + " dataframe.loc[:, 'exit_tag'] = 'done '\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_signal_program(source, class_name="SeparateTag") + + assert all( + not any(column.endswith("_tag") for column in node["parameters"]["columns"]) + for node in program["nodes"] + if node["op"] == "frame-write" + ) + assert "masked-string-append" not in program["opcodes"] + assert all( + output["column"].endswith(("_long", "_short")) + for output in program["signal_outputs"] + ) + + +def test_signal_program_lowers_metadata_partition_and_static_membership( + tmp_path: Path, +) -> None: + source = tmp_path / "MetadataRoute.py" + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class MetadataRoute(IStrategy):\n" + " top_coins = ['BTC', 'ETH']\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " pair_coin = metadata['pair'].partition('/')[0]\n" + " dataframe.loc[:, 'enter_long'] = pair_coin in self.top_coins\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, 'exit_long'] = 0\n" + " return dataframe\n", + encoding="utf-8", + ) + program = compile_signal_program(source, class_name="MetadataRoute") + + split = next(node for node in program["nodes"] if node["op"] == "string-split-index") + membership = next(node for node in program["nodes"] if node["op"] == "membership") + assert split["parameters"] == {"method": "partition", "separator": "/", "index": 0} + assert membership["parameters"] == {"values": ["BTC", "ETH"], "negated": False} + frame = pd.DataFrame({"close": [1.0, 2.0, 3.0]}) + accepted = execute_signal_program(program, frame, metadata={"pair": "BTC/USDT"}) + rejected = execute_signal_program(program, frame, metadata={"pair": "XRP/USDT"}) + assert accepted["enter_long"].tolist() == [True, True, True] + assert rejected["enter_long"].tolist() == [False, False, False] + + +def test_signal_program_flattens_more_than_1500_associative_conditions( + tmp_path: Path, +) -> None: + source = tmp_path / "DeepLogical.py" + conditions = " & ".join("(dataframe['score'] > -1.0)" for _ in range(1_501)) + source.write_text( + "from freqtrade.strategy import IStrategy\n" + "class DeepLogical(IStrategy):\n" + " def deep_condition(self, dataframe):\n" + f" return {conditions}\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " condition = self.deep_condition(dataframe)\n" + " dataframe.loc[:, 'enter_long'] = condition.astype(int)\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, 'exit_long'] = 0\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_signal_program(source, class_name="DeepLogical") + + logical = next( + node + for node in program["nodes"] + if node["op"] == "logical" and node["parameters"] == {"operator": "and"} + ) + assert len(logical["inputs"]) == 1_501 + validate_signal_program(program) + + +def test_signal_program_types_numpy_isnan_as_boolean_column(tmp_path: Path) -> None: + source = tmp_path / "IsNanMask.py" + source.write_text( + "import numpy as np\n" + "from freqtrade.strategy import IStrategy\n" + "class IsNanMask(IStrategy):\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " values = dataframe['score'].to_numpy(copy=False)\n" + " dataframe.loc[:, 'enter_long'] = ~np.isnan(values)\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, 'exit_long'] = 0\n" + " return dataframe\n", + encoding="utf-8", + ) + + program = compile_signal_program(source, class_name="IsNanMask") + + isnan = next( + node + for node in program["nodes"] + if node["op"] == "array-call" and node["parameters"]["name"] == "isnan" + ) + assert isnan["value_type"] == "bool-column" + actual = execute_signal_program( + program, + pd.DataFrame({"score": [1.0, float("nan"), -0.0]}), + ) + assert actual["enter_long"].tolist() == [True, False, True] + + def test_signal_program_runtime_fails_closed_for_numeric_mask(tmp_path: Path) -> None: source = tmp_path / "NumericMask.py" source.write_text( diff --git a/tests/test_stateful_execution_policy.py b/tests/test_stateful_execution_policy.py index c9c7a6dc..2cedebd7 100644 --- a/tests/test_stateful_execution_policy.py +++ b/tests/test_stateful_execution_policy.py @@ -7,9 +7,10 @@ from nfi_backtest_engine import stateful_execution_policy from nfi_backtest_engine.hot_ir import build_hot_callback_ir from nfi_backtest_engine.stateful_execution_policy import ( + FULL_NATIVE_VECTOR_TRANSPORT, GENERIC_VECTOR_TRANSPORT, X7_GENERIC_STATEFUL_LANE, - X7_VECTOR_TRANSPORT, + add_native_execution_blockers, build_native_execution_policy, build_x7_generic_stateful_policy, ) @@ -87,8 +88,10 @@ def test_x7_generic_stateful_programs_are_the_default_native_lane( ) assert policy["adapter_lane"] == X7_GENERIC_STATEFUL_LANE - assert policy["transport"] == X7_VECTOR_TRANSPORT - assert policy["primary"] == "source-compiled-generic-stateful-programs" + assert policy["transport"] == FULL_NATIVE_VECTOR_TRANSPORT + assert policy["primary"] == ( + "source-compiled-full-native-and-generic-stateful-programs" + ) assert policy["native_ready"] is True assert policy["blockers"] == [] assert len(policy["programs"]) == 5 @@ -101,6 +104,19 @@ def test_x7_generic_stateful_programs_are_the_default_native_lane( } +def test_source_compiler_blocker_reseals_the_native_policy() -> None: + policy = build_x7_generic_stateful_policy(_manager()) + + blocked = add_native_execution_blockers( + policy, + [{"code": "FULL_NATIVE_SOURCE_COMPILER_UNSUPPORTED", "message": "fixture"}], + ) + + assert blocked["native_ready"] is False + assert blocked["blockers"][-1]["code"] == "FULL_NATIVE_SOURCE_COMPILER_UNSUPPORTED" + assert blocked["fingerprint"] != policy["fingerprint"] + + def test_x7_policy_discovers_source_defined_route_keys_as_data() -> None: manager = _manager() manager["supported_routes"]["new_upstream_route"] = { diff --git a/tests/test_tag_program.py b/tests/test_tag_program.py index 1795ddc1..433eb8be 100644 --- a/tests/test_tag_program.py +++ b/tests/test_tag_program.py @@ -210,6 +210,41 @@ def test_tag_program_runtime_fails_closed_for_numeric_mask(tmp_path: Path) -> No execute_tag_program(program, pd.DataFrame({"mask": [0, 1]})) +def test_tag_program_lowers_masked_append_and_preserves_trailing_whitespace( + tmp_path: Path, +) -> None: + source = tmp_path / "MaskedAppend.py" + source.write_text( + "import numpy as np\n" + "from freqtrade.strategy import IStrategy\n" + "def append_tag(target, mask, tag):\n" + " target[mask] = target[mask] + tag\n" + "class MaskedAppend(IStrategy):\n" + " def populate_entry_trend(self, dataframe, metadata):\n" + " tags = np.full(len(dataframe), '', dtype=object)\n" + " append_tag(tags, dataframe['score'] > 0, '101 ')\n" + " append_tag(tags, dataframe['score'] >= 2, '562 ')\n" + " dataframe.loc[:, 'enter_tag'] = tags\n" + " return dataframe\n" + " def populate_exit_trend(self, dataframe, metadata):\n" + " dataframe.loc[:, 'exit_tag'] = ' tail '\n" + " return dataframe\n", + encoding="utf-8", + ) + program = compile_tag_program(source, class_name="MaskedAppend") + + appends = [node for node in program["nodes"] if node["op"] == "masked-string-append"] + assert len(appends) == 2 + assert all(node["value_type"] == "string-column" for node in appends) + frame = pd.DataFrame( + {"score": [-1.0, 1.0, 2.0], "enter_tag": ["stale"] * 3, "exit_tag": ["stale"] * 3} + ) + actual = execute_tag_program(program, frame) + assert actual["enter_tag"].tolist() == ["", "101 ", "101 562 "] + assert actual["exit_tag"].tolist() == [" tail "] * 3 + assert frame["enter_tag"].tolist() == ["stale"] * 3 + + def test_tag_program_parser_seals_mode_and_output() -> None: args = cli.build_parser().parse_args( [ diff --git a/uv.lock b/uv.lock index 887c66b5..148319c7 100644 --- a/uv.lock +++ b/uv.lock @@ -741,7 +741,7 @@ wheels = [ [[package]] name = "nfi-backtest-engine" -version = "1.5.0" +version = "1.6.0" source = { editable = "." } dependencies = [ { name = "blake3" },