diff --git a/.agents/skills/joinquant-archive-sync/requirements.txt b/.agents/skills/joinquant-archive-sync/requirements.txt index 24f21e4..289d6c5 100644 --- a/.agents/skills/joinquant-archive-sync/requirements.txt +++ b/.agents/skills/joinquant-archive-sync/requirements.txt @@ -1,5 +1,5 @@ -numpy==2.4.4 -pandas==3.0.2 +numpy==2.4.6 +pandas==3.0.3 requests beautifulsoup4 pyyaml==6.0.3 diff --git a/.agents/skills/run-local-quant-research/SKILL.md b/.agents/skills/run-local-quant-research/SKILL.md index 6f9ed2d..c32d9a8 100644 --- a/.agents/skills/run-local-quant-research/SKILL.md +++ b/.agents/skills/run-local-quant-research/SKILL.md @@ -1,25 +1,26 @@ --- name: run-local-quant-research -description: Use when 用户或 Agent 需要依据项目配置与共享行情快照执行本地量化研究、复算研究结果、核验必需输出，或固化可追溯证据。 +description: Use when 用户或 Agent 需要基于仓库项目配置与共享行情快照执行或复核一次本地量化研究场景。 --- # 本地量化研究流程 -只编排通用研究入口。让共享脚本校验配置、行情身份、项目结果和摘要；不要在 Skill（技能）中解释策略字段或复制项目逻辑。 +只编排一次通用本地研究调用。共享脚本负责校验配置、行情身份、单场景结果和摘要；Skill（技能）不解释策略字段，不复制项目逻辑，不读取复数分析计划。 ## 执行前提 - 从仓库根目录运行项目 `.venv`（虚拟环境），不回退系统 Python（编程语言），不静默安装依赖。 -- 配置必须声明项目入口、共享 `snapshot_id`（快照标识）、参数集合、仓库内路径和必需输出。 +- 配置必须声明项目入口、共享 `snapshot_id`（快照标识）、一个场景、仓库内路径和一份必需结果。 - 配置或快照缺失时停止，不猜测路径、不改用旧快照、不缩减资产范围。 ## 固定流程 -1. 校验行情快照：只读取配置声明的 `snapshot_id`，从共享行情中心验证来源、范围、字段、价格口径和内容摘要。 -2. 校验项目配置：检查其余结构、仓库边界、项目入口、参数集合、三态和必需输出声明。 -3. 运行项目入口：由通用 CLI（命令行接口）调用配置声明的项目进程；项目自行解释策略参数。 -4. 校验必需输出：重新读取输出文件并核对状态、运行身份、输入摘要和输出摘要。 -5. 固化运行证据：原子写入不可变清单；相同输入和相同输出复用既有运行，摘要冲突则失败。 +1. 校验行情快照：只读取配置声明的 `snapshot_id`，验证共享行情中心的 `market-data.parquet`、来源、范围、字段、价格口径和内容摘要；查询只使用内存 DuckDB（嵌入式分析数据库），不创建持久数据库。 +2. 校验单场景配置：拒绝候选数组、场景数组、分析计划和流程内循环，只接受一个项目场景及其必需输出声明。 +3. 运行项目入口：由通用 CLI（命令行接口）调用配置声明的项目进程；项目自行解释策略参数，并在同一全新进程内分别完成一次冷启动和一次预热执行。 +4. 校验单场景结果：重新读取唯一结果目录，核对状态、运行身份、输入摘要、性能证据、临时产物清理和输出摘要。 +5. 固化运行证据：通过后原子写入不可变清单；相同输入和相同输出复用既有运行，摘要冲突则失败。 +6. 返回调用者：完整运行固定输出 `next_action=return_to_caller`。复数场景由主 agent（代理）多次调用本 Skill，不在 Skill 内聚合、分析、报告或推荐。 统一公开命令： @@ -29,9 +30,9 @@ description: Use when 用户或 Agent 需要依据项目配置与共享行情快 ## 结果判断 -- `complete`（完整）：配置、快照、项目进程、全部必需输出和证据摘要均通过。 -- `evidence_insufficient`（证据不足）：执行前缺少身份、快照、范围或声明输入，或项目按自身规则判定可信输入不足以形成研究建议。 -- `failed`（失败）：既有证据被篡改或摘要不一致，或执行、输出、硬约束、确定性和原子固化任一环节失败。 +- `complete`（完整）：配置、快照、项目进程、单场景性能门禁、唯一必需结果和证据摘要全部通过，然后返回调用者。 +- `evidence_insufficient`（证据不足）：执行前缺少身份、快照、范围或声明输入，包括缺失单场景配置。 +- `failed`（失败）：既有证据被篡改或摘要不一致，或执行、输出、180 秒门槛、确定性、清理、原子固化任一环节失败。 只接受这三种最终状态。缺失证据与既有证据损坏必须严格区分。 @@ -40,3 +41,4 @@ description: Use when 用户或 Agent 需要依据项目配置与共享行情快 - 本地结果只属于探索性研究，不代表正式回测、稳健性通过或实盘准入。正式回测和模拟交易只在 JoinQuant（聚宽）云端运行。 - 不读取、打印或保存账号、密码、Cookie（浏览器凭证）或 Token（访问令牌）。需要认证或远端数据时，调用仓库已有的专用能力。 - 不在 Skill（技能）目录沉淀行情、策略代码或运行结果；共用能力保留在仓库脚本，项目能力保留在项目目录。 +- 策略分析、Vibe-Trading（AI 研究助理）、完整报告、推荐和人工确认属于独立流程，不在本 Skill 内调用。 diff --git a/.build-and-verify/config.json b/.build-and-verify/config.json index a2422d1..1a69314 100644 --- a/.build-and-verify/config.json +++ b/.build-and-verify/config.json @@ -298,15 +298,18 @@ "pytest", "tests\\local_quant_research", "-k", - "not test_skill_public_command_runs_complete_turtle_workflow and not test_non_strategy_project_completes_through_shared_market_and_runner" + "not test_non_strategy_project_completes_through_shared_market_and_runner" ], "paths": [ ".agents/skills/run-local-quant-research/**", ".claude/skills/run-local-quant-research", "scripts/research/market_data/**", "scripts/research/local_quant_research/**", + "scripts/research/analysis_data/**", + "scripts/research/quant_analysis/**", "joinquant/strategies/strategy-003/research/**", "tests/local_quant_research/**", + "tests/quant_analysis/**", "tests/test_skill_layout.py", "requirements*.txt" ], @@ -315,8 +318,11 @@ ".claude/skills/run-local-quant-research", "scripts/research/market_data/**", "scripts/research/local_quant_research/**", + "scripts/research/analysis_data/**", + "scripts/research/quant_analysis/**", "joinquant/strategies/strategy-003/research/**", "tests/local_quant_research/**", + "tests/quant_analysis/**", "tests/test_skill_layout.py", "requirements.txt", "requirements-dev.txt" @@ -330,18 +336,19 @@ ".\\.venv\\Scripts\\python.exe", "-m", "pytest", - "tests\\local_quant_research\\test_turtle_e2e.py", "tests\\local_quant_research\\test_generic_e2e.py", - "-k", - "skill_public_command_runs_complete_turtle_workflow or non_strategy_project_completes_through_shared_market_and_runner" + "tests\\local_quant_research\\test_turtle_e2e.py" ], "paths": [ ".agents/skills/run-local-quant-research/**", ".claude/skills/run-local-quant-research", "scripts/research/market_data/**", "scripts/research/local_quant_research/**", + "scripts/research/analysis_data/**", + "scripts/research/quant_analysis/**", "joinquant/strategies/strategy-003/research/**", "tests/local_quant_research/**", + "tests/quant_analysis/**", "tests/test_skill_layout.py", "requirements*.txt" ], @@ -350,15 +357,18 @@ ".claude/skills/run-local-quant-research", "scripts/research/market_data/**", "scripts/research/local_quant_research/**", + "scripts/research/analysis_data/**", + "scripts/research/quant_analysis/**", "joinquant/strategies/strategy-003/research/**", "tests/local_quant_research/**", + "tests/quant_analysis/**", "tests/test_skill_layout.py", "requirements.txt", "requirements-dev.txt" ], "checkParallel": false, "pytestXdistWorkers": 1, - "timeoutSeconds": 180 + "timeoutSeconds": 300 }, { "id": "verify.openspec", diff --git a/.github/workflows/full-verify.yml b/.github/workflows/full-verify.yml index 31479e8..72d94a4 100644 --- a/.github/workflows/full-verify.yml +++ b/.github/workflows/full-verify.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + lfs: true - name: Set up Node uses: actions/setup-node@v4 diff --git a/docs/research/2026-07-13-turtle-etf-system-final-plan.md b/docs/research/2026-07-13-turtle-etf-system-final-plan.md index df5abf4..d555a85 100644 --- a/docs/research/2026-07-13-turtle-etf-system-final-plan.md +++ b/docs/research/2026-07-13-turtle-etf-system-final-plan.md @@ -1,598 +1,194 @@ -# 海龟式 ETF 趋势跟踪系统完整方案 +# 海龟 ETF 系统最终方案 -更新日期：2026-07-13 -方案状态：研究设计已确认，尚未实施代码，尚未运行 JoinQuant（聚宽）正式回测 -详细决策来源：[海龟式 ETF 交易系统讨论检查点](./2026-07-13-turtle-etf-system-checkpoint.md) +**状态：** 已确认 -## 一、方案摘要 +**最后更新：** 2026-07-16 -本方案建立一套独立、只做多、日线级的海龟式 ETF（交易型开放式指数基金）趋势跟踪系统。 -系统以中期趋势延续为核心 Edge（交易优势），使用55日价格突破入场、20日反向突破退出、20日 -海龟N值统一波动风险尺度，并在趋势有利发展时按0.5N固定档位顺势加仓。 +**适用范围：** `strategy-003` 本地探索性研究；不修改 JoinQuant（聚宽）正式策略、正式回测或模拟交易 -系统不依靠高胜率，而是通过小额可控亏损和少数大趋势形成正偏态收益。运行时同时使用保护性 -止损、计划止损风险、资金仓位、同类资产集中度和组合目标波动率控制风险，不设置账户回撤减仓 -阶梯。正式回测最大回撤、稳健性、尾部风险和压力测试属于验收门槛，不直接产生交易信号。 +## 一、结论摘要 -核心结构如下： +当前基线固定使用 11 只 ETF、6 个资产组和 55/20/20 参数：55 日突破入场、20 日通道退出、20 日海龟 N。资金分配采用事件驱动的全量仓位再分配，风险预算采用 4/6/12 N 单位：单标的最多 4 个逻辑单位、资产组最多 6 个有效单位、组合最多 12 个有效单位。 -```text -55日收盘突破入场 -→ 按20日N值和0.5%初始风险确定标准单位 -→ 每上涨0.5N检查一次顺势加仓 -→ 共用只上移的2N保护性止损 -→ 20日反向突破或保护性止损全仓退出 -→ 全程受单ETF、资产组、组合风险和10%目标波动率约束 -``` - -## 二、系统定位与边界 - -| 项目 | 已确认方案 | -|---|---| -| 系统类型 | 独立海龟式ETF趋势跟踪系统 | -| 交易市场 | 仅交易中国内地上市ETF，允许QDII（合格境内机构投资者）跨境ETF | -| 交易方向 | 只做多，不做空 | -| 数据周期 | 日线 | -| 信号时点 | 当日收盘确认 | -| 执行时点 | 下一交易日开盘 | -| 无趋势状态 | 保留现金，不强制切换防御资产 | -| 杠杆 | 不使用融资或其他杠杆补足仓位 | -| 正式裁决 | 正式回测和模拟交易仅在JoinQuant（聚宽）运行 | -| 比较对象 | 沪深300、纳斯达克100、`strategy-001`、`strategy-002` | - -`strategy-001`和`strategy-002`只作为Benchmark（比较基线），本方案不改造两套已有策略。 - -首版Baseline（基线）只使用纯价格突破。以下方法保留为Challenger（挑战方案），不写入首版 -核心规则： - -- 双简单移动平均线：入场信号挑战方案； -- 20日横截面动量：资产排序挑战方案； -- 成交量和非极端波动：信号质量实验。 - -## 三、最终ETF资产池 - -| 代码 | 中文名称 | 主要角色 | 固定资产组 | 管理费与托管费合计 | -|---|---|---|---|---:| -| `510300.SH` | 沪深300ETF华泰柏瑞 | A股大盘宽基 | A股高同步权益组 | 0.20% | -| `512100.SH` | 中证1000ETF南方 | A股小盘宽基 | A股高同步权益组 | 0.20% | -| `512480.SH` | 半导体ETF国联安 | 半导体行业 | A股高同步权益组 | 0.60% | -| `159819.SZ` | 人工智能ETF易方达 | 人工智能主题 | A股高同步权益组 | 0.20% | -| `516160.SH` | 新能源ETF南方 | 新能源行业 | A股高同步权益组 | 0.20% | -| `513100.SH` | 纳指ETF国泰 | 美国大型成长权益 | 跨境科技权益组 | 0.80% | -| `513180.SH` | 恒生科技ETF华夏 | 港股科技权益 | 跨境科技权益组 | 0.65% | -| `515180.SH` | 红利ETF易方达 | A股红利风格 | A股红利组 | 0.20% | -| `516080.SH` | 创新药ETF易方达 | A股创新药行业 | A股创新药组 | 0.20% | -| `518880.SH` | 黄金ETF华安 | 黄金资产 | 黄金组 | 0.60% | -| `511010.SH` | 国债ETF国泰 | 5年期国债资产 | 国债组 | 0.20% | - -明确不加入`159985.SZ`豆粕ETF华夏。 - -资产池约束： - -- 首版固定为六个资产组，不按滚动相关性动态合并或拆分； -- 相关性只用于组合波动率计算、监控和压力测试； -- 动态分组只能作为挑战方案，必须重新通过正式回测和完整验收后才能升级； -- 管理费和托管费已经反映在ETF净值和市场价格中，不作为交易佣金重复扣除； -- 正式回测前重新核对上市状态、流动性、费率和QDII折溢价风险； -- 截至2026-07-13，11只ETF均可融资买入、可充抵保证金，充抵折算率均为90%、等级均为R1， - 但本系统仍不使用融资杠杆；资格以正式运行前国投证券当日清单为准。 - -## 四、入场规则 - -首版采用单一55日突破系统，不并行运行快慢双系统。 - -```text -入场通道 = 此前55个交易日盘中最高价的最大值 -入场条件 = 当日收盘价 > 入场通道 -执行时间 = 下一交易日开盘 -``` - -具体口径： - -- 55日通道不包含信号当日； -- 使用此前盘中最高价，不使用最高收盘价代替； -- 盘中突破但收盘跌回通道内，不产生信号； -- 仅突破此前最高收盘价但未突破此前盘中最高价，不产生信号； -- 40日和60日突破只用于参数邻域稳健性对照，不根据历史最优结果替换55日基线。 - -## 五、20日海龟N值 - -### 1. True Range（真实波幅） - -```text -TR = max( - 当日最高价－当日最低价, - |当日最高价－前一日收盘价|, - |当日最低价－前一日收盘价| -) -``` - -### 2. N值 - -```text -初始N = 最初20个有效TR的简单平均 -今日N =（昨日N × 19 + 今日TR）÷ 20 -``` - -N以价格为单位，用于统一计算仓位、初始止损、共同止损和加仓档位。N不产生入场方向，20日 -横截面动量不能替代N。 - -## 六、初始仓位与保护性止损 - -### 1. 初始止损 - -```text -初始止损价 = 下一交易日实际成交价－2 × 信号日N -``` - -- 止损以实际成交价为起点； -- N使用信号日收盘后已经确定的数值； -- 1.5N和2.5N只作为稳健性对照； -- 跳空、涨跌停或流动性不足可能使实际损失超过计划损失，必须在正式回测和压力测试中记录。 - -### 2. 初始标准单位 - -```text -U0 = 入场时账户净值 × 0.5% ÷（2 × 信号日N） -``` - -- U0是理论标准股数； -- 初次建仓触发2N止损时，计划损失为入场时账户净值的0.5%； -- U0在一轮完整趋势交易中固定，不随净值或后续N变化重算； -- 实际股数还要服从整手、流动性、风险、资金和组合波动率限制。 - -## 七、顺势加仓 - -### 1. 固定理论档位 - -```text -第k档加仓价 = 初始实际成交价 + k × 0.5 × 信号日N -``` - -- 只对盈利方向加仓，禁止亏损摊平； -- 后续N值变化和实际成交跳空不移动剩余理论档位； -- 每越过一个0.5N档位，产生一次候选加仓风险检查； -- 同一ETF每个交易日最多执行一次加仓； -- 一天跨过多个档位时不集中补齐，其余档位只在后续收盘仍满足条件时逐日检查； -- 每次检查都重新计算全部剩余风险和仓位预算。 - -### 2. 每档数量 - -```text -实际加仓股数 = min（U0，各项剩余预算允许股数） -``` - -- 每个档位最多申请一个U0； -- 按ETF交易整手向下取整； -- 不足一个整手时跳过； -- 被预算裁剪的档位不在同一交易日追加第二笔； -- 不给每个加仓批次独立增加0.5%风险预算。 - -### 3. 不设置固定次数上限 - -只要新的固定档位触发且全部预算允许，就可以继续逐日加仓。满足以下任一条件时自然停止： - -1. 单ETF计划止损风险预算不足； -2. 单ETF资金仓位预算不足； -3. 同类资产组风险或资金仓位预算不足； -4. 组合计划止损风险不足； -5. 组合年化10%目标波动率不允许新增风险； -6. 组合资金或可用现金不足； -7. 流动性不合格或数量不足一个交易整手； -8. 触发保护性止损或趋势退出。 - -## 八、加仓后的共同止损 - -所有批次共用一个有效保护性止损： - -```text -初始共同止损 = 初始实际成交价－2 × 信号日N -候选新止损 = 最新实际加仓价－2 × 信号日N -更新后止损 = max（更新前止损，候选新止损） -``` - -- 只有真实成交的加仓才能推动止损； -- 信号日N在整轮交易中固定用于保护性止损； -- 止损只能保持或上移，不能下调； -- 每次加仓后重新计算全部批次在共同止损价退出时的合计计划损益； -- 早期批次的锁定利润可抵减同一ETF新增批次的计划损失，但不能突破资金仓位、资产组和组合约束。 - -## 九、趋势退出 - -```text -退出通道 = 此前20个交易日盘中最低价的最小值 -退出条件 = 当日收盘价 < 退出通道 -执行时间 = 下一交易日开盘 -``` - -- 20日通道不包含信号当日； -- 盘中跌破但收盘重新站回，不产生趋势退出； -- 触发后退出该ETF全部批次； -- 共同保护性止损和20日趋势退出是并列条件，任一先触发都全仓退出。 +本方案删除旧的仅分配当日新增订单机制、资金仓位上限、计划风险上限、交易用协方差门槛和目标波动率控制。协方差、实现波动率、集中度和计划损失只由独立策略分析事后计算，不产生订单。 -## 十、风险与资金仓位体系 +## 二、整体架构 -### 1. 分层硬上限 +仓库长期架构分为三个独立 Skill（技能）： -| 层级 | 计划止损风险上限 | 资金仓位上限 | 超限处理 | -|---|---:|---:|---| -| 初次建仓 | 净值0.5% | 同时服从单ETF上限 | 缩小或跳过 | -| 单ETF全仓 | 净值1.25% | 净值30% | 新订单缩小；被动超过30%只停止该ETF加仓 | -| 同类资产组 | 净值2.5% | 净值50% | 组内同比例缩减至上限 | -| 全组合 | 净值5% | 净值100% | 正计划风险ETF同比例缩减；资金不足时停止新增 | +1. 本地研究流程：一次只执行一个明确场景，输出标准分析数据包； +2. 聚宽回测流程：在聚宽云端执行正式回测，沿用现有归档结果结构； +3. 策略分析：统一读取本地或聚宽结果，计算收益、风险、基准、归因、稳健性和报告。 -计划止损风险统一定义为： +本次变更实现本地研究流程、标准分析数据包和可真实读取该数据包的独立分析能力。聚宽正式复核、聚宽回测 Skill、策略分析 Skill 的创建、规则冻结、前向模拟和实盘不在本次范围。 -```text -单ETF计划止损风险 -= max（0，该ETF全部批次在共同止损价退出时的计划净损失） - -组合计划止损风险 -= Σ max（0，各ETF在各自共同止损价退出时的计划净损失） -``` - -不允许用一只ETF的锁定利润抵减另一只ETF的计划损失。 - -### 2. 100%组合资金使用上限 - -```text -全部ETF预计成交后总市值 ≤ 预计账户净值 × 100% -预计成交后可用现金 ≥ 0 -``` +## 三、资产池与分组 -- 取消强制10%现金缓冲； -- 预计成交金额使用包含基准或压力滑点的预计成交价； -- 可用现金还要扣除预计佣金和其他适用交易费用； -- 超过100%或预计形成负现金时缩小新订单，不足一个整手时跳过； -- 不使用融资、透支或其他杠杆突破100%； -- 100%是上限而不是必须达到的目标，风险预算、目标波动率、整手和费用可以自然留下现金； -- 资金上限本身只约束新增买入和加仓，不单独触发已有趋势仓位减仓。 +| 资产组 | ETF | +|---|---| +| `china_sync_equity` | `510300.XSHG`、`512100.XSHG`、`512480.XSHG`、`159819.XSHE`、`516160.XSHG` | +| `cross_border_tech_equity` | `513100.XSHG`、`513180.XSHG` | +| `china_dividend` | `515180.XSHG` | +| `china_innovative_drug` | `516080.XSHG` | +| `gold` | `518880.XSHG` | +| `treasury_bond` | `511010.XSHG` | -### 3. 同日订单优先级与A1共享预算 +资产组只用于相关风险聚合，不用于趋势强弱预测。所有满足突破条件的 ETF 都获得风险预算，不按信号时间、代码顺序、动量排名或主观评分选择。 -- 下一交易日开盘依次处理：全仓退出、强制风险减仓、有效新建仓与加仓； -- 新建仓和加仓处于同一优先级，共同申请退出和减仓后释放的资金与风险预算； -- 同一ETF当日出现退出时，取消该ETF全部新建仓和加仓候选； -- 每个买入候选先按既定规则生成最多一个U0的标准请求量； -- 当总请求超过资金、单ETF、资产组、组合计划风险或目标波动率预算时，对全部仍可行候选使用同一完成比例缩减； -- 候选被自身或所属资产组上限卡住后，未用预算可继续分配给其他仍可增仓候选； -- 缩放后先按ETF交易整手向下取整，剩余预算按小数余额从大到小逐手补分； -- 每补一手都重新检查全部硬门槛，完全同分时按ETF代码升序确定，候选输入顺序不得改变结果。 +17 ETF 扩展不是当前基线，也不进入本次真实运行。 -### 4. 风险缩放后的状态 +## 四、行情与公司行动 -- 资产组或组合风险缩放不视为趋势退出； -- 剩余仓位继续使用原信号日N、固定档位、共同止损和20日退出线； -- 风险缩放卖出的股数不形成欠仓； -- 预算恢复不自动买回； -- 只有新的有效入场信号或更高未执行固定档位才能重新申请风险预算。 +- 行情来自 JoinQuant（聚宽），以未复权日线保存；`fq=null`、`skip_paused=false`、`use_real_price=false`。 +- 权威行情以 Parquet（列式文件）保存在 `.local/market-data/`，查询时使用内存 DuckDB（嵌入式分析数据库），不创建持久数据库副本。 +- CSV（逗号分隔文件）只用于聚宽传输，校验转换后删除。 +- 本地执行使用按应用日可见的 `上一交易日原始 close / 当日 pre_close` 派生连续经济价格，统一用于信号、N、成交和估值。 +- 公司行动元数据用于授权和审计价格基准变化；晚公布记录标为事后核对，不回写未来知识。 +- 该口径是研究级总回报近似，不声称精确还原真实份额、派息日现金或聚宽逐日账户。 -### 5. 不增加账户回撤交易规则 +成交额和流动性只属于上游 ETF 池初筛，不进入本策略订单数量、入场、加仓、退出或止损规则。不存在单笔成交额 1% 限制。 -- 不设置账户回撤减仓阶梯； -- 不因账户回撤单独暂停开仓、缩小风险预算或退出仍有效的趋势； -- 15%最大回撤只作为正式回测和前向模拟的验收淘汰线。 +## 五、信号规则 -## 十一、组合目标波动率 +### 1. 入场 -### 1. 估算方法 +- 信号日收盘价严格高于此前 55 个交易日盘中最高价时确认突破； +- 通道不包含信号日； +- 盘中突破但收盘未确认不入场。 -使用最近60个完整、有效且日期对齐的日收益率计算滚动Covariance Matrix（协方差矩阵）： +### 2. N ```text -候选组合年化波动率 = sqrt（wᵀ × Σ × w）× sqrt（252） -``` - -### 2. 单向上限 - -- 年化10%是风险硬上限，不是必须主动达到的目标； -- 每个交易日收盘后更新协方差矩阵和候选组合波动率； -- 不超过10%时不因波动率控制交易，也不加杠杆； -- 超过10%时，在下一交易日开盘把全部持仓同比例缩减至9.5%目标： - -```text -缩放系数 = 9.5% ÷ 调整前候选组合年化波动率 -全部持仓目标仓位 = 调整前持仓 × 缩放系数 -``` - -- 减仓后重新检查整手、资金和其他风险上限； -- 卖出的部分不形成欠仓，不自动买回； -- 120日滚动协方差和30日半衰期EWMA（指数加权移动平均）仅作稳健性对照。 - -### 3. 数据冷启动与故障安全 - -- 新ETF必须取得60个完整、有效且日期对齐的收益率样本后才可建仓或加仓； -- 不使用代理相关性、零相关或临时短窗口； -- 任一持仓ETF缺少有效价格并导致协方差无法可靠更新时，暂停整个组合新增风险； -- 不填零，不使用旧协方差批准新交易； -- 保留已有仓位，只执行能够成交的止损、趋势退出和风险减仓； -- 数据恢复后重新计算协方差并复核全部风险门槛，再恢复新增风险。 - -## 十二、交易执行与成本 - -| 项目 | 基线口径 | -|---|---| -| 正式回测初始资金 | 人民币1,500,000元 | -| 买入佣金 | 成交额万分之0.85，每笔最低5元 | -| 卖出佣金 | 成交额万分之0.85，每笔最低5元 | -| ETF印花税 | 0 | -| 基准滑点 | 单边0.05%，买入上调、卖出下调 | -| 压力滑点 | 单边0.10% | -| 信号成交 | 收盘确认，下一交易日开盘 | - -万分之0.85为全佣，已经包含交易所经手费，不再重复叠加。次日开盘跳空属于真实价格变化， -不能与滑点重复计算。 - -新增建仓和加仓必须同时满足： +TR = max( + 当日最高价 - 当日最低价, + |当日最高价 - 前收盘价|, + |当日最低价 - 前收盘价| +) -```text -最近20个交易日成交额中位数 ≥ 100,000,000元 -单笔新增订单金额 ≤ 最近20个交易日成交额中位数 × 1% +初始 N = 最初 20 个有效 TR 的平均值 +今日 N =（昨日 N × 19 + 今日 TR）÷ 20 ``` -已有仓位不会仅因流动性下降而强制退出，但在重新满足门槛前禁止继续加仓，并在报告中记录无法 -按理论价格成交的风险。 +### 3. 加仓 -## 十三、比较基准 +- 首次入场为第 1 个逻辑单位； +- 固定加仓档位为首次实际成交价加 `0.5N`、`1.0N`、`1.5N`； +- N 使用首次信号日 N，档位建立后冻结； +- 每只 ETF 每个交易日最多增加 1 个单位； +- 单标的最多 4 个逻辑单位； +- 只顺势加仓，不亏损摊平。 -正式回测并列展示： +### 4. 退出 -1. 沪深300人民币Total Return（总回报），包含分红再投资； -2. 纳斯达克100人民币总回报，包含分红再投资和美元兑人民币汇率变化； -3. `strategy-001`； -4. `strategy-002`。 +- 收盘价严格低于此前 20 个交易日盘中最低价时趋势退出； +- 收盘价小于或等于共同止损时保护性退出； +- 任一条件触发都退出该 ETF 全部实际持仓并清除全部逻辑单位。 -海龟策略、`strategy-001`和`strategy-002`必须使用相同的150万元初始资金、回测区间和适用 -费用口径重新运行。旧回测只作历史资料，不直接拼接。四个比较基准只用于横向展示和归因， -不直接决定本系统是否通过验收。 +### 5. 成交时点 -## 十四、正式回测核心验收门槛 +所有信号每日收盘检查一次，下一交易日开盘成交。当前不采用收盘前半小时成交。停牌、涨跌停、缺少有效开盘价或官方拒单时不得假设成交，逻辑状态只按真实成交更新。 -正式基线必须同时满足： +## 六、逐单位 N 风险 -```text -扣除成本后的CAGR（复合年增长率） > 0 -最大回撤 ≤ 15% -Calmar Ratio（卡玛比率） ≥ 0.50 -``` +每个候选单位在信号日计算： ```text -Calmar = 同一次正式回测的CAGR ÷ 最大回撤绝对值 +base_quantity = floor_to_100_shares( + signal_equity × 1% ÷ signal_n +) ``` -任一稳健性、压力或尾部风险硬门槛失败，均不能由高收益或高卡玛比率补偿。 - -## 十五、五个必选稳健性维度 - -### 1. 参数邻域 - -每次只替换一个参数，其他规则保持不变： - -- 入场周期：40日、60日分别替换55日； -- 初始止损：1.5N、2.5N分别替换2N； -- 协方差：120日滚动、30日半衰期指数加权分别替换60日滚动。 - -每个变体必须满足净CAGR大于0、最大回撤不超过20%。每个参数家族还必须满足： +单位真实成交后冻结自己的 `signal_n`、`base_quantity` 和实际成交价。每个单位生成候选止损： ```text -最差挑战变体卡玛比率 ≥ 基线卡玛比率 × 50% -两个挑战变体平均卡玛比率 ≥ 基线卡玛比率 × 75% +candidate_stop_i = actual_fill_price_i - 2 × frozen_signal_n_i +common_stop = max(previous_common_stop, candidate_stop_i) ``` -不得删除较差变体，也不得用挑战变体替换冻结的首版基线。 - -### 2. 跨时期与样本外 - -固定历史区间： - -- 2015-01-01至2018-12-31； -- 2019-01-01至2022-12-31； -- 2023-01-01至正式数据截止日。 +共同止损只能保持或上移。每日新 N 不重算既有单位止损；全量再分配买卖也不建立单位、不删除单位、不推进加仓档位、不改变止损。 -每个固定区间必须净CAGR大于0、最大回撤不超过20%。另使用3年滚动窗口，每季度移动一次： +## 七、4/6/12 风险预算 -- 至少70%的滚动窗口净CAGR大于0； -- 任一滚动窗口最大回撤不超过20%。 - -### 3. 跨资产稳定性 - -- 从完整资产池逐只删除一只ETF，共11个变体； -- 从六个固定资产组逐组删除一组，共6个变体； -- 删除变体保持其余参数、成本和回测区间不变； -- 每个变体净CAGR大于0、最大回撤不超过20%； -- 逐ETF层和逐资产组层分别满足： +设标的 `i` 的逻辑单位数为 `u_i`： ```text -该层最差变体卡玛比率 ≥ 完整基线卡玛比率 × 50% -该层全部变体平均卡玛比率 ≥ 完整基线卡玛比率 × 75% -``` - -删除测试只用于识别依赖性，不用于事后剔除表现较差的资产。 - -### 4. 成本与执行压力 - -除基准场景外，必须运行： - -1. 双倍佣金、单边0.05%滑点、原定次日开盘； -2. 实际佣金、单边0.10%滑点、原定次日开盘； -3. 双倍佣金、单边0.10%滑点、原定次日开盘； -4. 实际佣金、单边0.05%滑点、所有原信号再延迟一个交易日； -5. 双倍佣金、单边0.10%滑点、所有原信号再延迟一个交易日。 +group_scale_g = min(1, 6 / sum(u_i in group g)) -每个场景必须净CAGR大于0、最大回撤不超过20%，并满足： +effective_units = sum( + u_i × group_scale_group(i) +) -```text -最差压力场景卡玛比率 ≥ 基线卡玛比率 × 50% -五个压力场景平均卡玛比率 ≥ 基线卡玛比率 × 75% +portfolio_scale = min(1, 12 / effective_units) ``` -延迟场景只推迟原始订单，不使用延迟日的新信息重新决定是否交易。 - -### 5. 风险路径与压力情景 - -详见下一节的区块抽样、历史压力、假设冲击和CVaR（条件风险价值）门槛。 - -## 十六、风险路径与尾部风险 - -### 1. Block Bootstrap（区块自助抽样） - -使用正式基线扣除成本后的每日组合收益： +目标数量为该标的所有冻结基础单位之和，依次乘资产组比例、组合比例和统一现金比例，再按 100 股向下取整。12 单位代表 1N 暴露预算，不代表账户最多亏损 12%；2N 止损、跳空和无法成交仍可能造成更大损失。 -- 每条路径756个交易日，即3年； -- 主测试使用20日连续区块； -- 生成10,000条路径； -- 固定并归档随机种子、输入收益序列和配置； -- 不删除极端日、亏损日或亏损区块； -- 使用5日和60日区块分别重新生成10,000条路径复核。 +## 八、全量仓位再分配 -三种区块长度均必须满足： +只在以下事件出现时重算全部目标：有效入场候选、有效加仓候选、保护性止损、20 日趋势退出。没有事件时不调仓。 -```text -P（最大回撤 > 20%）≤ 5% -P（最大回撤 > 30%）≤ 1% -三年期末净收益中位数 > 0 -``` +每次事件按以下顺序执行： -### 2. 固定历史压力窗口 +1. 完整退出； +2. 再分配卖出； +3. 入场或加仓买入； +4. 再分配买入。 -| 情景 | 固定区间 | 门槛 | -|---|---|---| -| 2015年A股异常波动 | 2015-06-12至2015-09-30 | 窗口回撤≤15% | -| 2018年全球风险与贸易摩擦 | 2018-01-01至2018-12-31 | 窗口回撤≤15% | -| 2020年疫情冲击 | 2020-02-03至2020-04-30 | 窗口回撤≤15% | -| 2022年全球加息 | 2022-01-04至2022-10-31 | 窗口回撤≤15% | -| 2024年A股流动性冲击 | 2024-01-02至2024-02-08 | 窗口回撤≤15% | +系统先把可成交候选加入临时单位簿，再计算 4/6/12 风险比例和统一现金比例。目标统一向下取整，不按代码或信号先后分配剩余现金。不能产生至少一个整手净新增成交的候选被移除并重新计算，直到候选集合稳定。 -同时报告最低点、恢复时间、无法成交事件和风险规则执行记录。 +因此，后出现的趋势可以同比例挤占先出现趋势的资金；信号时间不构成持仓优先权。再分配只改变实际持仓，逻辑单位与止损保持不变。 -### 3. 四个假设冲击 +## 九、成本和市场约束 -对正式回测每个交易日收盘后的真实持仓分别施加： +- 初始资金：150 万元； +- 买卖佣金：成交额万分之 0.85，每笔最低 5 元； +- ETF 印花税：0； +- 单边滑点：0.05%； +- 现金不足时对全部可调整目标使用同一个最大可行比例； +- 禁止融资，预计费用后现金必须非负； +- 不做余额补仓或最大余数分配。 -1. 同步权益暴跌：全部权益ETF -20%，黄金ETF -5%，国债ETF 0%； -2. 全市场流动性冲击：全部权益ETF -15%，黄金ETF -10%，国债ETF -8%； -3. 跨境ETF跳空：纳指ETF和恒生科技ETF -25%，其他权益ETF -8%，黄金和国债ETF 0%； -4. 止损失效：所有持仓按各自共同止损价再下跌2N成交。 +## 十、本地执行与标准结果包 -每个情景在全历史持仓中的最差单次账户损失均不得超过15%。压力测试不产生新的运行时账户 -回撤规则。 +- 本地模拟使用 vectorbt（向量化回测框架）官方 `Portfolio.from_order_func()`； +- Numba（即时编译）回调只负责策略状态和订单决策；vectorbt 负责时间遍历、共享现金、成交、费用、持仓和权益； +- 额外延迟研究继续使用冻结订单的 `Portfolio.from_orders()` 路径，基线 `additional_delay_days=0` 不进入额外延迟； +- 旧自研逐日引擎、旧分配函数和旧风险路径不保留兼容层。 -### 4. Historical CVaR（历史条件风险价值） +本地结果尽量对齐聚宽现有目录，物理输出 `results`、`balances`、`positions`、`orders` 四类共同事实及海龟归因扩展。聚宽现有归档 0 改动直读。归因必须包含逻辑单位、候选基础数量、冻结 N、实际成交价、共同止损、资产组比例、组合比例、现金比例和再分配状态隔离证据。 -使用扣除实际佣金和滑点后的非参数历史收益： +## 十一、独立策略分析 -```text -单日95% CVaR ≤ 2.5% -单日99% CVaR ≤ 4.0% -连续5日复合收益95% CVaR ≤ 5.0% -``` +策略分析不进入交易回调，统一读取标准结果包并计算： -- 分位点相同收益全部纳入尾部样本； -- 不删除、缩尾、替换或修改极端收益； -- 5日收益使用实际连续复利，不使用平方根时间近似； -- CVaR只作为验收和报告指标，不新增CVaR减仓、暂停或恢复规则。 +- 累计收益、CAGR（复合年化收益率）、波动率、Sharpe（夏普比率）、Sortino（索提诺比率）、最大回撤、回撤期和 Calmar（卡玛比率）； +- 相对沪深300人民币总回报和纳斯达克100人民币总回报的超额收益、Alpha（阿尔法）、Beta（贝塔）和相关性； +- 平均、中位、最高仓位，现金、换手、费用、单标的和资产组实际权重； +- 最高计划损失比例、最高有效 N 单位、组合单位预算利用率、再分配事件、止损和趋势退出； +- ETF、资产组、时期和交易原因归因。 -## 十七、前向模拟与实盘准入 +Vibe-Trading（AI 研究助理）只允许使用无已知缺陷的单体公开能力；已知有缺陷的群体分析不得作为证据。若没有安全可用的单体入口，确定性分析和报告仍正常完成，并记录未使用原因。 -规则、参数、代码和正式历史证据冻结后，在JoinQuant（聚宽）启动正式模拟交易。验证必须同时 -满足： +## 十二、当前实施与验收范围 -- 连续运行至少12个月； -- 至少完成20轮完整趋势交易； -- 两个样本条件取较晚满足者，未达到时继续运行； -- 模拟期最大回撤不超过15%； -- 模拟期实现年化波动率不超过12%； -- 扣除实际费用后的净收益不低于预先冻结的历史同期限滚动收益第10百分位； -- 不存在无法解释的信号、仓位、止损、费用、风险预算或成交偏差； -- 不要求模拟期收益必须大于0，避免用单个弱趋势样本否定正偏态趋势系统。 +本次只运行一次真实 11 ETF 新基线： -一轮完整趋势交易从某ETF首次建仓开始，到该ETF全部退出结束；同一轮中的加仓不单独计数。 +- 不运行旧方案对照； +- 不运行 17 ETF 扩展； +- 不运行 7 个场景或稳健性矩阵； +- `run-local-quant-research` Skill 仍只执行一个场景； +- 单次冷启动和预热都必须小于 180 秒，规范化结果摘要一致； +- 最终生成本地研究报告、明确推荐结论并等待人工确认； +- 结果是本地探索性证据，不代替聚宽正式裁决。 -任何会改变信号、仓位、止损、订单或交易结果的规则与代码变更，都必须重新开始前向模拟计时和 -交易计数。纯报告或展示修改只有在证明不影响账户结果时才不重置。未通过前向模拟前，系统不得 -进入真实资金交易。 +## 十三、完成标准 -## 十八、研究闭环 - -```text -研究假设 -→ Vibe-Trading进行AI探索与证据挑战 -→ 形成候选方案 -→ 聚宽研究环境导出11只ETF日线并写入共享行情中心 -→ 共享行情中心登记不可变批次和snapshot_id -→ strategy-003通过Data Bridge引用权威CSV和内存DuckDB视图 -→ Vibe-Trading进行方向性粗略模拟和预设参数挑战 -→ Codex执行确定性事件、风险和稳健性计算 -→ 输出本地研究报告、研究建议和固定候选策略包 -→ JoinQuant正式回测并裁决 -→ 下载并归档聚宽正式回测快照 -→ Codex基于正式回测快照做本地稳健性复核 -→ Vibe-Trading辅助归因、解释和报告 -→ 冻结规则、代码、参数和证据 -→ JoinQuant前向模拟 -→ 最终验收 -→ 实盘 -``` +只有以下条件全部满足才算完成： -### 1. Data Bridge（数据桥） - -- `.local/market-data/` 是与任何策略解耦的共享日线行情中心，完整行情不得进入公开仓库； -- 不可变批次中的 `market-data.csv`（逗号分隔文件）是精确聚宽原始导出和唯一行情事实源； -- `snapshot_id`只引用已验证批次及明确证券、区间、字段、来源和未复权口径，不复制行情； -- DuckDB（嵌入式分析数据库）只建立读取权威CSV的内存查询视图，不保存持久数据库副本； -- `strategy-003`和其他策略只引用共享快照，不拥有或复制行情； -- 行情字段固定包含日期、证券、OHLCV（开高低收量）、前收盘、成交额、复权因子、停牌和涨跌停价； -- Vibe-Trading只读取其支持的OHLCV（开高低收量）字段；暂停、涨跌停、复权和成交额等扩展 - 字段由Codex或聚宽处理； -- 本地行情快照只用于探索性研究；聚宽正式回测快照才是后续正式收益和稳健性裁决证据。 - -### 2. 工具职责 - -| 工具 | 职责 | 禁止越界 | -|---|---|---| -| Vibe-Trading（AI研究助理） | 研究探索、方向性粗筛、参数方向、归因和报告 | 不作为正式回测或验收裁判 | -| Codex（代码代理） | 编排、状态记录、确定性计算、正式结果复核 | 不在本地宣称完整正式回测 | -| JoinQuant（聚宽） | 完整交易路径、正式回测、模拟交易和最终裁决 | 不用旧回测拼接最终结论 | -| 共享行情中心 | 保存不可变行情批次和快照引用，向所有策略提供日线查询 | 不保存持久DuckDB，不替代聚宽正式回测数据 | -| 策略研究证据 | 保存代码、配置、报告、结论、候选和运行摘要 | 不复制共享行情，不宣称正式收益 | -| 正式回测归档 | 保存聚宽代码、参数、订单、持仓、净值、费用、日志和报告 | 不用本地行情快照替代正式回测快照 | - -允许升级Vibe-Trading官方发布版，但不直接修改上游源码。当前稳定版0.1.11尚未包含已知组合 -优化器前视偏差修复；官方发布版包含修复前，跳过受影响的组合优化器，不阻塞其他研究步骤。 - -## 十九、规则冻结与变更管理 - -- 首版基线固定为55日入场、2N初始止损、0.5N加仓档位、20日退出和60日协方差； -- 40/60日、1.5N/2.5N、120日和指数加权方法只作预设挑战变体； -- 本地候选包固定为一项冻结基线加上述六项预设单项挑战，共用同一代码； -- 本地粗筛只记录方向性证据，不按本地收益删除候选、替换基线或新增参数； -- 不读取正式结果后修改门槛、删除不利时期或增加有利参数； -- 不根据单项历史最优结果替换基线； -- 资产删除测试不用于事后修改资产池； -- 任何基线升级必须另行提出，并重新通过正式回测、五维稳健性和前向模拟； -- 正式验收使用同一次回测的代码、参数、数据、成本和结果快照。 - -## 二十、当前状态与下一步 - -当前已经完成： - -- 66项系统设计参数确认； -- 研究闭环和工具分工确认； -- 11只ETF资产池及固定资产组确认； -- 组合资金使用上限调整为100%； -- 稳健性、尾部风险、压力测试和前向模拟门槛确认。 - -尚未完成： - -- 策略实施代码； -- 共享行情中心和Data Bridge配置； -- 11只ETF的聚宽研究行情快照； -- Vibe-Trading方向性粗筛； -- JoinQuant正式历史回测； -- 正式结果归档和本地稳健性复核； -- 规则冻结后的聚宽前向模拟； -- 最终实盘验收。 - -下一步先创建并同步真实`strategy-003`空壳但不启动正式回测，再实施共享行情中心、本地研究流程和 -策略代码；之后依次进入正式回测、结果归档、稳健性复核和前向模拟。在执行计划确认前，不新增 -策略参数，也不启动正式回测。 +1. 配置、代码、测试、设计和 OpenSpec（开放规格）使用同一套规则； +2. 旧资金仓位控制、计划风险上限、协方差交易门槛、目标波动率交易控制和旧分配实现已从生产路径删除； +3. 逐单位 N、固定 0.5N 加仓、只上移 2N 止损、4/6/12 和全量仓位再分配通过测试； +4. 标准四表与聚宽零改动读取契约保持不变； +5. 完整入口 E2E（端到端）回归通过； +6. 真实 11 ETF 单场景在 180 秒内完成，并产出可追溯报告； +7. 临时文件已删除，历史 `.local` 证据未被改写。 diff --git a/docs/research/2026-07-16-turtle-full-position-redistribution-baseline-report.md b/docs/research/2026-07-16-turtle-full-position-redistribution-baseline-report.md new file mode 100644 index 0000000..3a7913c --- /dev/null +++ b/docs/research/2026-07-16-turtle-full-position-redistribution-baseline-report.md @@ -0,0 +1,213 @@ +# 海龟 ETF 全量仓位再分配基线研究报告 + +**研究日期：** 2026-07-16 + +**研究范围：** 11 只 ETF、6 个资产组、单次本地基线 + +**结论状态：** 不推荐按当前规则进入聚宽复核；等待人工确认 + +**证据权限：** 本地探索性模拟，不是聚宽正式回测 + +## 一、推荐结论 + +当前基线不建议继续进入聚宽复核，主要原因不是趋势没有触发，也不是长期大部分时间空仓，而是全量仓位再分配与冻结止损之间存在风险口径错位：再分配可以在更高价格增加实际持仓，却按已确认规则不增加逻辑单位、不改变共同止损。结果是 4/6/12 N 单位仍满足配置，但真实持仓到旧止损的计划损失可明显超过单位预算所表达的风险。 + +确定性结果为：累计收益 120.07%，CAGR（复合年化收益率）5.96%，最大回撤 -34.66%，Calmar（卡玛比率）0.172。基线虽然获得正收益，但最大回撤超过分析计划的 20%上限，Calmar 低于 0.50 门槛，因此基线状态为不通过。 + +最主要的反对证据是： + +- 2015-05-07，`510300.XSHG` 接近账户满仓，计划止损损失达到权益的 31.18%； +- 单标的最高实际权重 100.00%，资产组最高实际权重 100.00%； +- 收益高度集中于 `510300.XSHG` 和 `518880.XSHG`，其他 7 只 ETF 为负贡献； +- 本次按范围约定没有运行 7 场景和稳健性矩阵，不能宣称规则已经稳健。 + +推荐先重新确认“再分配增量如何保持 N 风险含义”，再决定是否运行下一次基线。当前不应优先优化入场、动量或标的扩展。 + +## 二、运行身份与可复现性 + +| 项目 | 结果 | +|---|---:| +| run id（运行标识） | `d10991ad7e0ec53841a73d70accafc197871b563fb47b8ec5038a839b9b98e79` | +| 行情快照 | `e88238cca420a8ae66b90adb6cda4dd6c38a07390a13b8ac2f471e534742e33e` | +| 代码身份摘要 | `c6aa53d15499551efb59fb2babfa38152e1362cc6bed350ca4da3cfe2ad0f015` | +| 参数摘要 | `b291c12e4c462292ac599268b0d0532a7df7661b687655c96c1434412584c087` | +| 研究区间 | 2012-05-28 至 2026-07-13 | +| 交易日 | 3,432 | +| 初始 / 期末权益 | 150.00 万元 / 330.11 万元 | +| 冷启动 / 预热 | 26.00 秒 / 2.40 秒 | +| 180 秒性能门禁 | 通过 | +| 冷热结果摘要一致 | 通过 | +| 标准结果包门禁 | 通过，无例外 | + +标准结果包包含 `results`、`balances`、`positions`、`orders` 四类共同事实，行数分别为 3,432、3,432、4,847、1,799；海龟归因扩展为 8,272 行。冷启动与预热的结果摘要均为 `fdfc70c4e7e656eaf41835c1cec2adaca69c9b10f7a6aa293982d45599f0a4cb`。 + +## 三、策略与执行规则 + +本次实际基线使用 55 日突破入场、20 日通道退出、20 日 N、每上涨 0.5N 增加一个单位、单标的最多 4 个逻辑单位和 2N 共同止损。每个候选单位按信号日权益的 1%/N 计算基础数量；信号在收盘确认，下一交易日开盘成交，每只 ETF 每日最多增加一个单位。 + +风险分配采用单标的 4、资产组 6、组合 12 个有效 N 单位。入场、加仓、止损或趋势退出事件触发全部持仓重新计算；先退出、再分配卖出、入场或加仓、最后再分配买入。现金不足时统一缩放，整手向下取整，不做余额补仓。 + +没有单标的资金上限、资产组资金上限、目标波动率控制、协方差门槛、流动性门槛或单笔成交额 1%限制。 + +## 四、收益与回撤 + +| 指标 | 基线 | +|---|---:| +| 累计收益 | 120.07% | +| CAGR（复合年化收益率） | 5.96% | +| 年化波动率 | 13.11% | +| Sharpe（夏普比率） | 0.508 | +| Sortino（索提诺比率） | 0.715 | +| 最大回撤 | -34.66% | +| 最大回撤持续期 | 1,138 个交易日 | +| Calmar（卡玛比率） | 0.172 | +| 非零收益日胜率 | 53.68% | +| 当前回撤 | -11.10% | + +最大回撤从 2015-06-08 的高点开始，2017-06-16 到达低点，直到 2020-02-12 才恢复。最大回撤深度和恢复时间都说明当前风险不是短期噪声。 + +### 年度收益 + +| 年份 | 收益 | 年份 | 收益 | 年份 | 收益 | +|---|---:|---|---:|---|---:| +| 2012 | 6.20% | 2017 | 5.46% | 2022 | 3.33% | +| 2013 | 6.70% | 2018 | -2.20% | 2023 | 6.48% | +| 2014 | 21.96% | 2019 | 15.83% | 2024 | 4.86% | +| 2015 | -0.44% | 2020 | 29.10% | 2025 | 12.98% | +| 2016 | -6.03% | 2021 | -12.03% | 2026 至 7 月 13 日 | -3.75% | + +完整年度中 2016、2018、2021 为负收益；2021 年最差，为 -12.03%。最佳月份是 2015-04，收益 17.39%；最差月份是 2015-08，收益 -12.78%。 + +### 近期滚动表现 + +| 窗口 | CAGR | 最大回撤 | Calmar | +|---|---:|---:|---:| +| 最近 1 年 | 3.30% | -11.85% | 0.278 | +| 最近 3 年 | 3.30% | -11.85% | 0.278 | +| 最近 5 年 | 1.89% | -14.24% | 0.132 | +| 最近 10 年 | 4.83% | -24.97% | 0.194 | + +近期收益没有显示基线正在改善：最近 5 年 CAGR 仅 1.89%。这些是同一路径切片，不是独立起始资金回测，因此只用于描述时间分布，不等于稳健性验证。 + +## 五、双基准分析 + +基准采用官方沪深300人民币总收益和纳斯达克100人民币总收益。对齐只保留双方都有真实收益的共同日期，不补零、不前向填充。Alpha（阿尔法）按日收益回归截距年化，未扣无风险利率；它不能替代累计相对收益。 + +| 指标 | 沪深300人民币总收益 | 纳斯达克100人民币总收益 | +|---|---:|---:| +| 共同样本 | 3,432 日 | 3,276 日 | +| 共同样本策略累计收益 | 120.07% | 83.91% | +| 基准累计收益 | 153.78% | 1,049.24% | +| 累计收益差 | -33.70 个百分点 | -965.33 个百分点 | +| 相对财富差 | -13.28% | -84.00% | +| 策略 / 基准 CAGR | 5.96% / 7.08% | 4.80% / 20.66% | +| CAGR 差 | -1.11 个百分点/年 | -15.86 个百分点/年 | +| Alpha（阿尔法） | 4.47% | 4.85% | +| Beta（贝塔） | 0.240 | 0.033 | +| 相关性 | 0.390 | 0.052 | +| Tracking Error（跟踪误差） | 20.23% | 23.87% | +| Information Ratio（信息比率） | -0.122 | -0.645 | + +Alpha 为正而累计相对收益为负并不矛盾：策略对两个基准的 Beta 很低，Alpha 表示低市场暴露条件下的回归截距，不表示最终财富超过基准。两个基准都构成反对证据，尤其纳斯达克100基准长期差距很大。 + +## 六、仓位、风险与交易 + +| 指标 | 结果 | +|---|---:| +| 平均仓位 / 平均现金 | 78.91% / 21.09% | +| 中位仓位 | 99.97% | +| 低于半仓天数比例 | 20.69% | +| 接近满仓天数比例 | 76.89% | +| 有仓位时平均 / 最多持有 ETF 数 | 1.77 / 8 | +| 最高单标的权重 | 100.00% | +| 最高资产组权重 | 100.00% | +| 最高有效 N 单位 | 12.00 | +| 组合单位预算最高利用率 | 100.00% | +| 最高计划损失比例 | 31.18% | +| 最高 60 日已实现年化波动率 | 37.45% | +| 成交订单 / 非完成订单 | 1,175 / 624 | +| 全量再分配 / 保护止损 / 趋势退出 | 675 / 98 / 45 | +| 入场 / 加仓事件 | 144 / 213 | +| 累计换手 | 184.35 倍平均权益 | +| 累计费用 | 39,108.73 元 | + +平均仓位已经达到 78.91%，2014 年以后多数年份平均仓位超过 75%，2024 年达到 99.75%。因此当前 5.96%的年化收益不能主要归因于“大部分时间没有仓位”。系统已经利用了大量资金，但风险收益转换效率不足。 + +资产组缩放在 1,800 个有决策记录的交易日中绑定 60 日，最低比例 37.50%；组合缩放绑定 71 日，最低比例 57.14%；现金缩放绑定 778 日，最低比例 1.85%。现金缩放绑定约 43.22%的决策日，说明资金不足经常成为最终目标数量的主导约束。 + +2026-07-13 期末仅持有 `511010.XSHG`，权重 99.99%。这说明 N 风险单位和逻辑单位上限不是资金集中度上限；低波动标的可以占用几乎全部名义资金。 + +## 七、收益归因 + +归因采用“每日标的损益 / 前一日权益”的算术贡献，总和为 90.63%。它与 120.07%的复利累计收益不同，不能直接相减或作为反事实收益。 + +### ETF 贡献 + +| ETF | 算术贡献 | ETF | 算术贡献 | +|---|---:|---|---:| +| `510300.XSHG` | 72.27 个百分点 | `516080.XSHG` | -5.64 个百分点 | +| `518880.XSHG` | 26.85 个百分点 | `512480.XSHG` | -4.98 个百分点 | +| `511010.XSHG` | 7.91 个百分点 | `516160.XSHG` | -3.14 个百分点 | +| `512100.XSHG` | 5.54 个百分点 | `513180.XSHG` | -2.97 个百分点 | +| `515180.XSHG` | -0.92 个百分点 | `159819.XSHE` | -2.95 个百分点 | +| `513100.XSHG` | -1.35 个百分点 | | | + +`510300.XSHG` 与 `518880.XSHG` 合计贡献 99.12 个百分点，超过组合 90.63 个百分点的算术总收益，说明其他标的整体抵消了一部分收益。11 只 ETF 提供了信号机会，但最终收益来源并不分散。 + +### 资产组贡献 + +| 资产组 | 算术贡献 | +|---|---:| +| 中国同步权益 | 66.74 个百分点 | +| 黄金 | 26.85 个百分点 | +| 国债 | 7.91 个百分点 | +| 红利 | -0.92 个百分点 | +| 跨境科技权益 | -4.31 个百分点 | +| 创新药 | -5.64 个百分点 | + +交易原因归因中，全量再分配标签对应 -21.15 个百分点，保护止损对应 -7.22 个百分点。但该标签只说明当日估值事实关联的最近原因，不是“取消再分配后会增加 21.15%收益”的反事实实验，不能据此直接归因成本。 + +## 八、风险失真根因 + +风险失真的直接证据出现在 `510300.XSHG`： + +1. 2014-11-03 至 2014-11-13 建立 4 个逻辑单位，共同止损最终为 2.6536； +2. 趋势继续上涨后，全量再分配多次改变实际持仓，但按规则不改变单位数和共同止损； +3. 2015-05-07，再分配以 4.7278 买入，持仓增至 505,700 股，平均成本升至 4.1097； +4. 共同止损仍为 2.6536，持仓市值占权益 99.99%； +5. 按“平均成本减共同止损”计算的计划损失为 736,352.20 元，占当日权益 31.18%。 + +这不是输入顺序或信号先后造成的分配错误，而是两个已确认规则组合后的语义冲突： + +```text +全量再分配可以买入更多实际份额 ++ 再分配不更新逻辑单位与止损 += 实际止损风险可能脱离 N 单位预算 +``` + +4/6/12 仍正确限制逻辑单位和分组比例，但它没有持续约束“当前实际数量 × 当前价格到旧止损的距离”。当只有一个趋势可用时，统一现金缩放还会让该趋势接近满仓，从而放大这个偏差。 + +因此，当前收益瓶颈的上游根因不是缺少趋势强弱预测，而是风险预算的计量单位与再分配后的实际持仓风险不再一致。进入下一轮前，需要人工选择新的契约，例如：再分配增量按当前价格到共同止损的距离重新限制数量，或把再分配买入视为带独立风险依据的新执行批次。任何方案都会改变当前已确认规则，本报告不自行实施。 + +## 九、Vibe 单体审计 + +Vibe-Trading（AI 研究助理）通过公开单体入口实际运行，运行标识为 `20260716_192533_82_da3181`，加载了 `performance-attribution`、`risk-analysis` 和 `report-generate`。没有调用群体分析、回测、行情下载或优化器。 + +Vibe 给出的状态是 `insufficient_evidence`（证据不足），建议暂不确认进入聚宽复核。其文档读取器没有获得标准结果目录权限，未可靠展开 Parquet 行级数据，因此 Vibe 只确认了运行可重复、参数边界和稳健性缺失，不能提供数值裁判。本报告的全部收益、风险、基准和归因数字仍以仓库确定性分析为准。 + +## 十、限制与未覆盖范围 + +- 本地公司行动使用连续经济价格和除权日隐含再投资近似，不精确模拟派息日现金、税费、零碎份额或聚宽真实份额； +- 本地 `vectorbt`（向量化回测库）撮合不代替聚宽正式撮合，停牌、涨跌停和开盘跳空仍需平台复核； +- Alpha 未扣无风险利率；双基准只按共同真实日期对齐； +- 归因为日度算术贡献，不是几何链式归因； +- 本次没有运行旧方案对照、17 ETF 扩展、7 个参数场景、成本压力、标的删除、区块抽样、历史压力、冲击测试或 CVaR（条件风险价值）； +- 因此只能判断当前单一基线，不得宣称策略稳健、可部署或优于其他参数。 + +## 十一、人工确认项 + +本报告推荐停止在当前基线，不进入聚宽复核。下一步应只确认一个问题：是否修改“再分配不改变止损”的规则，使每次再分配后的实际计划损失重新与 N 风险预算对齐。 + +在该问题解决前，不建议继续做动量排序、趋势强弱预测、资产扩展或稳健性矩阵，因为这些下游研究无法修复当前风险单位失真。 + +**最终状态：等待人工确认。** diff --git a/docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md b/docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md index 73d0acd..b713de4 100644 --- a/docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md +++ b/docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md @@ -1,636 +1,530 @@ --- change: build-turtle-etf-local-research-workflow design-doc: docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md -base-ref: a61a53b6852b8dd8ad111693145e40d5555e99d8 +base-ref: 4400fec8149f02bc7d42f0294be65e9dacc9b639 --- -# 海龟 ETF 本地研究流程实施计划 +# 海龟 ETF 本地研究流程、聚宽原生分析数据与 vectorbt 执行内核实施计划 -> **给执行 Agent（代理）：** 必须按任务逐项使用 `superpowers:subagent-driven-development`（子代理驱动开发，推荐）或 `superpowers:executing-plans`（计划执行）实施；所有代码任务遵循 TDD（测试驱动开发）的 RED（失败）→ GREEN（通过）→ REFACTOR（重构）循环。 +> **给执行 Agent（代理）：** 必须按任务逐项使用 `superpowers:executing-plans`（计划执行），代码任务遵循 TDD（测试驱动开发）的 RED（失败）→ GREEN（通过）→ REFACTOR（重构）循环。不得执行或恢复已废弃的旧逐日方案。 -**目标：** 建立与具体策略解耦的本地日线行情中心和研究运行器，并为 `strategy-003` 实现可复算的海龟 ETF 探索性研究、报告、结论和固定候选包。 +## 目标与边界 -**架构：** `.agents/skills/run-local-quant-research/` 只编排；`scripts/research/market_data/` 管理不可变 CSV（逗号分隔文件）批次、快照和内存 DuckDB（嵌入式分析数据库）视图；`scripts/research/local_quant_research/` 负责配置、运行身份、项目子进程和原子证据；`joinquant/strategies/strategy-003/research/` 只保存海龟项目配置、纯计算模块和报告逻辑。正式回测不在本计划执行。 +- 本地研究 Skill（技能）每次只编排一个快照、一个场景、一份聚宽口径兼容结果和不可变证据，并以 `next_action=return_to_caller` 停止；它不接收候选数组，也不知道冻结基线和六个挑战这一组合。 +- JoinQuant（聚宽）现有回测目录、清单和归档流程保持零改动；它们是标准物理基准。 +- 本地 vectorbt（向量化回测框架）结果从 `/backtests//` 向内尽量对齐聚宽目录，但使用独立本地 Schema（结构约束）明确身份。 +- 本地物理落盘只有 `results`、`balances`、`positions`、`orders` 四类共同执行事实；`risk`、`period_risks` 只作为缺失的来源参考条目。统一读取器提供六类逻辑视图。 +- 策略分析 Skill 另立变更。本变更只在本地 Skill 外使用通用确定性分析脚本完成一次真实、完整的独立验收，不创建策略分析 Skill；Vibe-Trading（氛围量化）仅记录安全单体能力与边界审计，禁止群体分析。 +- 单次回测性能门槛为180秒。主 agent（代理）每调用一次 Skill，都必须为该单场景提供冷启动与预热证据；两次执行和摘要一致性在暂存区通过后才发布一份权威结果。 +- 聚宽正式复核、聚宽回测 Skill、策略分析 Skill、规则冻结、模拟交易和实盘不在本变更范围。 -**技术栈：** Python 3.12、pytest（测试框架）、DuckDB 1.5.4、Pandas 3.0.2、NumPy 2.4.4、JSON（结构化清单）、CSV、PowerShell、JoinQuant（聚宽）研究环境。 +仍有效的现有基础只有三项：真实 `strategy-003` 身份、共享 Parquet（列式文件）行情中心、通用本地研究 Skill 与三态证据。旧逐日执行模块、旧八表物理契约、流程内分析/报告/人工门禁不属于当前方案，不能作为任务、兼容接口或验收依据保留。 -## 全局约束 - -- 所有本地 Python（编程语言）命令必须使用 `.\.venv\Scripts\python.exe`，不得回退系统 Python 或静默安装依赖。 -- 完整行情只能写入已忽略的 `.local/market-data/`；不得提交行情值、账号、Token（访问令牌）或 Cookie（浏览器凭证）。 -- `market-data.csv` 是唯一行情事实源；DuckDB 只能使用 `duckdb.connect(':memory:')`，不得生成持久 `.duckdb` 文件。 -- 首版只实现 `source=joinquant`、`asset_type=etf`、`frequency=1d`，但共享模块不得包含海龟参数、ETF 资产池或交易规则。 -- 运行状态且只能是 `complete`、`evidence_insufficient`、`failed`；项目建议使用独立枚举，不得混用。 -- 本地研究只生成探索性证据，不能宣称正式回测通过、稳健性通过或实盘准入。 -- 不修改 `strategy-001`、`strategy-002`；`strategy-003` 必须先绑定真实聚宽策略对象，且本计划不启动正式回测。 -- 每个任务完成后运行该任务的定向测试，勾选对应 OpenSpec（开放规格）任务并用简体中文提交说明提交。 +## 固定输出 -## 文件结构 +本地基础研究： ```text -.agents/skills/run-local-quant-research/ -├── SKILL.md -└── agents/openai.yaml -.claude/skills/run-local-quant-research -> ../../.agents/skills/run-local-quant-research -scripts/research/ -├── market_data/ -│ ├── __init__.py -│ ├── contracts.py -│ ├── storage.py -│ ├── query.py -│ └── joinquant_export.py -└── local_quant_research/ - ├── __init__.py - ├── contracts.py - ├── evidence.py - ├── runner.py - └── cli.py -joinquant/strategies/strategy-003/ -├── default_code.py -├── manifest.json -└── research/ - ├── project-run.json - ├── baseline.json - ├── candidates.json - └── turtle_etf/ - ├── __init__.py - ├── indicators.py - ├── signals.py - ├── state.py - ├── risk.py - ├── allocation.py - ├── execution.py - ├── reporting.py - └── cli.py -tests/local_quant_research/ -├── test_skill_contract.py -├── test_market_data_storage.py -├── test_market_data_query.py -├── test_joinquant_export.py -├── test_runner.py -├── test_evidence.py -├── test_turtle_indicators.py -├── test_turtle_risk.py -├── test_turtle_allocation.py -├── test_turtle_e2e.py -├── test_generic_e2e.py -└── fixtures/daily-bars.csv +.local/quant-research/strategy-003// +└── backtests/ + └── / + ├── manifest.json + ├── code.py + ├── params.json + ├── params_versions/.json + ├── performance.json + └── data/ + ├── results.parquet + ├── balances.parquet + ├── positions.parquet + ├── orders.parquet + └── attribution_log-.parquet（strategy-003 必需扩展） ``` -## OpenSpec 覆盖映射 - -| 计划任务 | 覆盖 OpenSpec 子项 | -|---|---| -| 任务 1 | 1.1、1.2、1.3 | -| 任务 2 | 2.1、2.2 | -| 任务 3 | 3.1、3.2、3.3 | -| 任务 4 | 3.4、3.5 | -| 任务 5 | 2.3、2.4、2.5、2.6 | -| 任务 6 | 4.1、4.2、4.3、4.4 | -| 任务 7 | 4.5、4.6 | -| 任务 8 | 5.1、5.2、5.3 | -| 任务 9 | 6.1、6.2、6.3、6.4、7.1 | -| 任务 10 | 7.2、7.3 | - -### 任务 1：建立真实 `strategy-003` 身份和冻结契约夹具 - -**文件：** - -- 创建：`joinquant/strategies/strategy-003/default_code.py` -- 创建：`joinquant/strategies/strategy-003/manifest.json` -- 修改：`joinquant/strategies/strategy_index.csv` -- 创建：`joinquant/strategies/strategy-003/research/baseline.json` -- 创建：`joinquant/strategies/strategy-003/research/candidates.json` -- 创建：`tests/local_quant_research/test_strategy_identity.py` -- 创建：`tests/local_quant_research/test_contract_fixtures.py` - -**接口：** +共享分析基准： -- 产出：真实聚宽详情 URL、远端名称、`default_code.py` SHA256（文件摘要）和本地 `strategy-003` 唯一映射。 -- 产出：`baseline.json` 固定资产池、资产组、55/20 通道、20 日 N、0.5N 加仓、2N 止损、风险和价格口径。 -- 产出：`candidates.json` 恰好为冻结基线加六个单参数挑战。 - -- [x] **步骤 1：先写身份与冻结契约失败测试** - -```python -def test_strategy_003_is_real_and_unique(repo_root): - rows = list(csv.DictReader((repo_root / "joinquant/strategies/strategy_index.csv").open(encoding="utf-8"))) - row = next(item for item in rows if item["strategy_id"] == "strategy-003") - assert row["joinquant_strategy_url"].startswith("https://www.joinquant.com/algorithm/index/edit?") - assert (repo_root / row["current_default_code"]).is_file() - -def test_candidates_are_frozen_single_factor_challenges(repo_root): - document = json.loads((repo_root / "joinquant/strategies/strategy-003/research/candidates.json").read_text(encoding="utf-8")) - items = document["candidates"] - assert [item["id"] for item in items] == [ - "baseline", "entry-40", "entry-60", "stop-1.5n", "stop-2.5n", - "covariance-120d", "covariance-ewma-30d", - ] - assert all(len(item["overrides"]) <= 1 for item in items) +```text +.local/market-data/benchmark-sets// +├── manifest.json +└── benchmark-returns.parquet ``` -- [x] **步骤 2：运行测试并确认因 `strategy-003` 尚不存在而失败** - -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_strategy_identity.py tests\local_quant_research\test_contract_fixtures.py -q` - -预期：FAIL（失败），明确指出缺少 `strategy-003` 行或配置文件。 - -- [x] **步骤 3：通过已登录的聚宽页面创建真实策略空壳并同步身份** - -使用 Chrome（浏览器）现有登录状态创建名为 `turtle_etf_local_research` 的策略，仅保存最小 `initialize(context): pass` 代码，不创建回测。记录详情 URL，按现有 manifest（清单）结构写入远端身份、观察时间、代码路径和摘要；不得读取或保存 Cookie。 - -- [x] **步骤 4：写入精确冻结配置** +独立分析验收： -`baseline.json` 必须包含 11 个聚宽代码（`.XSHG`/`.XSHE`）、六个固定资产组、`entry_days=55`、`exit_days=20`、`n_days=20`、`risk_per_unit=0.005`、`add_step_n=0.5`、`stop_n=2.0`、`covariance_days=60`、`target_volatility=0.10`、`fq=null`、`use_real_price=false`。`candidates.json` 只覆盖对应单一字段。 - -- [x] **步骤 5：重跑定向测试并保护既有策略** - -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_strategy_identity.py tests\local_quant_research\test_contract_fixtures.py -q` - -预期：PASS（通过）；另运行 `git diff -- joinquant/strategies/strategy-001 joinquant/strategies/strategy-002`，预期无输出。 - -- [x] **步骤 6：提交任务 1** - -```powershell -git add joinquant/strategies/strategy-003 joinquant/strategies/strategy_index.csv tests/local_quant_research/test_strategy_identity.py tests/local_quant_research/test_contract_fixtures.py -git commit -m "功能：建立海龟ETF研究项目身份与冻结契约" +```text +.local/strategy-analysis-preparations// +├── analysis-scenarios.json +├── preparation.json +└── scenario-configs// + ├── params.json + └── run.json + +.local/strategy-analysis// +├── analysis-scenarios.json +├── preparation.json +├── source-results.json +├── deterministic-analysis.json +├── evidence-matrix.parquet +├── local-strategy-analysis-report.md +├── vibe-evidence.json +└── recommendation.json ``` -### 任务 2：初始化薄 Skill（技能）并锁定公开入口 - -**文件：** - -- 创建：`.agents/skills/run-local-quant-research/SKILL.md` -- 创建：`.agents/skills/run-local-quant-research/agents/openai.yaml` -- 创建：`.claude/skills/run-local-quant-research`（目录链接） -- 修改：`tests/test_skill_layout.py` -- 创建：`tests/local_quant_research/test_skill_contract.py` +## 全局约束 -**接口：** +- 所有本地 Python（编程语言）命令使用 `.\.venv\Scripts\python.exe`；依赖变更必须先明确记录并经用户授权，不得静默安装。 +- 完整行情和研究结果只写入已忽略的 `.local/`；不得提交行情值、账号、Token（访问令牌）或 Cookie（浏览器凭证）。 +- `market-data.parquet` 是行情事实源；DuckDB（嵌入式分析数据库）只连接 `:memory:`。 +- 共享批次以原始 `market-data.parquet` 和 `corporate-actions.parquet` 共同构成行情事实；连续信号价格只在内存派生。无法解释的除权差异必须关闭运行。 +- 海龟策略层完全不读取成交额，不设置最低成交额、单笔成交额占比、订单参与率或其他流动性规则；共享成交额字段只供上游 ETF 池筛选和其他策略使用。 +- vectorbt 只属于 `strategy-003` 项目执行层；共享行情、统一读取、通用运行器和分析算法不得依赖 vectorbt 对象。 +- 所有 T 日收盘信号和滚动输入必须显式错位到 T+1 执行行；海龟状态只按实际成交更新。 +- 当日顺序固定为退出、强制风险减仓、A1 买入；卖出实际成交后才能按最新现金和持仓计算一次 A1。 +- 冻结基线与六个预设挑战全部保留，由主 agent 从策略自有机器可读 `analysis-plan.json` 读取并分别调用 Skill 七次；Skill 不读取该计划，本身不得包含数量、顺序或循环逻辑。 +- `preparation_id` 只绑定分析计划、基准集、运行模板和七份待执行配置；七次调用完成后，主 agent 必须显式登记 `scenario_id -> run_id`，校验七份结果共享快照、代码和执行后端，再由全部来源摘要派生不可变 `analysis_id`。不得扫描目录猜测来源或覆盖同计划下的旧分析。 +- 旧实现通过新路径验收后直接删除；不运行旧完整流程，不新旧双跑，不保留兼容模块、导入别名、双引擎开关、回退或死代码。 +- 确定性指标、稳健性数值、证据挑战、报告和推荐均由统一读取与 `quant_analysis`（量化分析）完成；Vibe 不执行回测、不替代数值裁判。加载方法文档不算实际分析，已知缺陷的群体分析不得调用或进入结论。 -- 产出：唯一公开命令 `.\.venv\Scripts\python.exe scripts\research\local_quant_research\cli.py run --config `。 -- 约束：Skill 只描述顺序、输入、三态、正式回测边界和凭证边界，不包含海龟参数。 +--- -- [x] **步骤 1：添加失败的布局和内容测试** +## Task 1：锁定 vectorbt 依赖、输入与官方回调 -```python -def test_local_research_skill_is_thin(repo_root): - skill = repo_root / ".agents/skills/run-local-quant-research/SKILL.md" - text = skill.read_text(encoding="utf-8") - assert "scripts/research/local_quant_research/cli.py" in text - assert "complete" in text and "evidence_insufficient" in text and "failed" in text - for forbidden in ("55日", "0.5N", "strategy-003", "510300"): - assert forbidden not in text -``` +**对应 OpenSpec：** 2.1—2.4 -- [x] **步骤 2：运行测试并确认缺少 Skill** +**Files：** -运行：`.\.venv\Scripts\python.exe -m pytest tests\test_skill_layout.py tests\local_quant_research\test_skill_contract.py -q` +- Modify: `pyproject.toml` 或仓库现有依赖锁定文件 +- Create: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py` +- Create: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py` +- Create: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py` +- Create: `tests/local_quant_research/test_turtle_vectorbt_inputs.py` +- Create: `tests/local_quant_research/test_turtle_vectorbt_callbacks.py` +- Create: `tests/local_quant_research/test_turtle_vectorbt_engine.py` +- Modify: `joinquant/strategies/strategy-003/research/code-identity.json` +- Modify only if affected mapping requires: `.build-and-verify/config.json` -预期：FAIL，缺少 `run-local-quant-research`。 +**接口：** -- [x] **步骤 3：用官方生成器初始化 Skill** +- `prepare_simulation_inputs(frames, config) -> SimulationInputs` +- `run_vectorbt_simulation(inputs, config) -> VectorbtSimulationResult` +- 回调固定使用 `pre_sim_func_nb`、`pre_segment_func_nb`、`order_func_nb`、`post_order_func_nb` -运行：`.\.venv\Scripts\python.exe C:\Users\liuli\.codex\skills\.system\skill-creator\scripts\init_skill.py run-local-quant-research --path .agents\skills` +### Step 1：RED -编辑 `SKILL.md` 和 `agents/openai.yaml` 后，在仓库根目录运行： +先写失败测试，覆盖依赖版本与许可记录、稳定数组类型、T→T+1 错位、无未来数据、单一共享现金组、普通订单函数、卖出优先、成交后 A1、成交后状态更新、停牌/涨跌停/拒单和 `nopython`（无 Python 模式）。 ```powershell -New-Item -ItemType SymbolicLink -Path '.claude\skills\run-local-quant-research' -Target '..\..\.agents\skills\run-local-quant-research' +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_inputs.py tests\local_quant_research\test_turtle_vectorbt_callbacks.py tests\local_quant_research\test_turtle_vectorbt_engine.py -q ``` -随后用 `Get-Item '.claude\skills\run-local-quant-research' | Format-List LinkType,Target` 确认 `LinkType=SymbolicLink` 且目标为 `..\..\.agents\skills\run-local-quant-research`。 +Expected: FAIL，原因是 vectorbt 新入口尚未形成唯一执行路径。 -- [x] **步骤 4：运行结构验证和定向测试** +### Step 2：GREEN -运行：`.\.venv\Scripts\python.exe C:\Users\liuli\.codex\skills\.system\skill-creator\scripts\quick_validate.py .agents\skills\run-local-quant-research` +在获得依赖变更授权后，使用项目 `.venv` 固定实测兼容版本。所有 ETF 使用一个 `cash_sharing=True`（共享现金）组；不得启用 `flexible=True`（灵活多订单）或私有模拟函数。实现已确认的海龟状态、共同止损、风险门槛与 A1，并用固定合成夹具直接验证订单、成交、现金、持仓、批次和原因码。 -运行：`.\.venv\Scripts\python.exe -m pytest tests\test_skill_layout.py tests\local_quant_research\test_skill_contract.py -q` +### Step 3：回归 -预期：全部 PASS。 +运行本任务测试和现有海龟规则测试。测试预期只来自规格夹具，不调用旧 `process_day` 或旧 `_simulate` 生成答案。 -- [x] **步骤 5：提交任务 2** +--- -```powershell -git add .agents/skills/run-local-quant-research .claude/skills/run-local-quant-research tests/test_skill_layout.py tests/local_quant_research/test_skill_contract.py -git commit -m "功能：增加通用本地量化研究技能入口" -``` +## Task 2：建立双 Schema、统一读取和双基准契约 -### 任务 3：实现不可变行情批次与快照 +**对应 OpenSpec：** 3.1—3.4 -**文件：** +**Files：** -- 创建：`scripts/research/market_data/__init__.py` -- 创建：`scripts/research/market_data/contracts.py` -- 创建：`scripts/research/market_data/storage.py` -- 创建：`tests/local_quant_research/test_market_data_storage.py` -- 创建：`tests/local_quant_research/fixtures/daily-bars.csv` +- Create: `scripts/research/analysis_data/schemas/local-backtest-manifest.schema.json` +- Create: `scripts/research/analysis_data/__init__.py` +- Create: `scripts/research/analysis_data/manifest.py` +- Create: `scripts/research/analysis_data/views.py` +- Create: `scripts/research/analysis_data/derived.py` +- Create: `scripts/research/analysis_data/cli.py` +- Create: `scripts/research/market_data/benchmark_sets.py` +- Modify: `scripts/research/quant_analysis/benchmarks.py` +- Delete: `scripts/research/quant_analysis/contracts.py` +- Create: `tests/local_quant_research/test_analysis_manifest_schemas.py` +- Create: `tests/local_quant_research/test_analysis_data_contract.py` +- Create: `tests/local_quant_research/test_benchmark_set_contract.py` +- Read only fixture: `joinquant/strategies/strategy-001/backtests/111/` +- Read only fixture: `joinquant/strategies/strategy-001/backtests/109/` +- Read only fixture: `joinquant/strategies/strategy-002/backtests/9/` **接口：** -```python -def import_batch(*, csv_path: Path, manifest: Mapping[str, object], root: Path) -> BatchRecord: ... -def create_snapshot(*, batch_ids: Sequence[str], selection: SnapshotSelection, root: Path) -> SnapshotRecord: ... -def validate_snapshot(snapshot_id: str, *, root: Path) -> SnapshotRecord: ... -``` +- `open_analysis_source(result_dir: Path) -> AnalysisSource` +- `validate_analysis_source(source: AnalysisSource) -> ValidationResult` +- `register_core_views(connection, source) -> CoreViews` +- `build_derived_views(connection, views) -> DerivedViews` +- `build_benchmark_set(definitions, source_snapshots, output_root) -> BenchmarkSet` -- [x] **步骤 1：为批次身份、去重和冲突写失败测试** +### Step 1：RED——本地清单 Schema -测试必须断言：批次目录恰好包含 `manifest.json`、`market-data.csv`、`validation.json`；相同来源和字节摘要复用同一 `batch_id`；重叠键值不同抛出 `MarketDataConflict`；新证券追加不改变旧批次摘要。 +本地清单必须使用： -- [x] **步骤 2：为快照引用和旧快照稳定性写失败测试** - -```python -snapshot = create_snapshot(batch_ids=[first.batch_id], selection=selection, root=tmp_path) -before = sha256((tmp_path / "snapshots" / f"{snapshot.snapshot_id}.json").read_bytes()).hexdigest() -import_batch(csv_path=second_csv, manifest=second_manifest, root=tmp_path) -assert sha256((tmp_path / "snapshots" / f"{snapshot.snapshot_id}.json").read_bytes()).hexdigest() == before +```text +schema_version = "local-backtest/1" +object.kind = "local_backtest" +source.kind = "local_vectorbt" +authority = "local_research" ``` -- [x] **步骤 3：运行失败测试** - -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_market_data_storage.py -q` +Schema 必须固定代码、参数、运行、场景、快照、引擎版本、六类数据集条目、`performance.json` 路径/字节数/SHA256（文件摘要）、文件摘要、行数、空表和门禁。四类共同事实必须 `complete`；`risk` 与 `period_risks` 必须 `required=false`、`status=missing_at_source`、`reason=computed_by_strategy_analysis`。本地清单禁止出现聚宽 URL、`research_response`、`research_lineage`、`collection_fence` 或 `official_summary`。 -预期：FAIL，模块不存在。 +测试还要覆盖未知版本、`local_backtest` 冒充聚宽版本、聚宽字段混入本地清单、本地字段混入聚宽清单、Schema 失败后回退另一分支等拒绝场景。 -- [x] **步骤 4：实现规范化 JSON、SHA256、原子写入和冲突索引** +### Step 2：RED——聚宽零改动与六类逻辑视图 -`batch_id` 使用来源身份、导出契约和 CSV 字节摘要的规范化 JSON 摘要；`snapshot_id` 使用快照清单规范化 JSON 摘要。写入必须先进入同文件系统临时目录，再以原子替换固化；目标已存在时只允许内容完全相同。 +读取器只按顶层 `schema_version` 选择契约：整数 `1` 使用现有聚宽 Schema；字符串 `local-backtest/1` 使用本地 Schema；其他值直接拒绝。聚宽路径验证原 `manifest.json`、对象、来源、门禁、文件摘要和合法空表，运行前后目录摘要必须一致。 -- [x] **步骤 5：验证所有存储不变量** +聚宽来源读取六类物理数据集；本地来源读取四类物理事实，并为 `risk`、`period_risks` 建立带来源缺失状态的空参考视图。两个来源最终都暴露六类逻辑视图；权益、完整往返交易和事件只在查询期派生。 -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_market_data_storage.py -q` +真实只读核对已确认聚宽 `results` 的 `time:string`、`returns:double`、`benchmark_returns:double` 均表示累计序列。本地 `results.parquet` 也固定这三个字段：`returns` 为从初始资金起算的累计净收益，`benchmark_returns` 全列为空但物理类型必须保持 `double`，清单记录 `source_benchmark_returns.status=missing_at_source`、`reason=independent_benchmark_set` 和空值行数。禁止填零或任选双基准之一冒充聚宽单基准。 -预期：PASS；测试临时目录结束后不存在 `.tmp`、`.duckdb` 或未被快照引用的伪完成目录。 +统一读取器把 `results.time` 规范化为 Asia/Shanghai（亚洲/上海）交易日，并按 `(1 + cumulative_return_t) / (1 + cumulative_return_t-1) - 1` 在查询期派生策略单日收益；首样本累计收益不为零时因缺少前值而排除。双基准文件保存单日人民币总回报，分析只能在共同有效交易日比较单日序列，不得把来源累计收益直接与基准单日收益比较。 -- [x] **步骤 6：提交任务 3** +### Step 3：RED——双基准集 -```powershell -git add scripts/research/market_data tests/local_quant_research/test_market_data_storage.py tests/local_quant_research/fixtures/daily-bars.csv -git commit -m "功能：实现共享行情批次与快照" -``` +固定仅有： -### 任务 4：实现内存 DuckDB 查询和聚宽导出契约 +- `CSI300_CNY_TOTAL_RETURN`：沪深300人民币总回报； +- `NASDAQ100_CNY_TOTAL_RETURN`：纳斯达克100人民币总回报，包含美元兑人民币变化。 -**文件：** +`benchmark-returns.parquet` 至少包含 `time`、`benchmark_id`、`returns`。清单记录币种、总回报定义、汇率公式、源标识、实际日期范围、底层快照与文件摘要。实现前必须对配置的数据源做真实最小可行性验证；覆盖不足、来源不明或汇率口径不完整时输出 `evidence_insufficient`，禁止使用 ETF 代理或零收益补齐。 -- 创建：`scripts/research/market_data/query.py` -- 创建：`scripts/research/market_data/joinquant_export.py` -- 创建：`tests/local_quant_research/test_market_data_query.py` -- 创建：`tests/local_quant_research/test_joinquant_export.py` +聚宽 `results.benchmark_returns` 只注册为 `source_benchmark_returns` 官方参考；除非其清单能证明与目标基准身份和口径完全一致，否则不能代替上述两条分析基准。 -**接口：** +### Step 4：GREEN 与回归 -```python -def open_snapshot(snapshot_id: str, *, root: Path) -> SnapshotView: ... -def normalized_digest(rows: Iterable[Mapping[str, object]]) -> str: ... -def render_export_program(request: ExportRequest) -> str: ... -def verify_transfer(*, local_file: Path, remote_sha256: str, remote_cleaned: bool) -> TransferEvidence: ... +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_analysis_manifest_schemas.py tests\local_quant_research\test_analysis_data_contract.py tests\local_quant_research\test_benchmark_set_contract.py tests\local_quant_research\test_analysis_contracts.py -q ``` -- [x] **步骤 1：写 CSV 与内存视图一致性失败测试** - -覆盖固定 13 字段、`date/security` 唯一键、排序、空值、`paused` 数值到布尔规范化、行差异和内容摘要差异；使用 `monkeypatch` 断言连接字符串恰好为 `:memory:`。 - -- [x] **步骤 2：写聚宽导出程序失败测试** - -断言生成程序包含 `get_price(..., fq=None, skip_paused=False)`、13 个字段、`line_terminator='\n'`、远端回读 SHA256 和清理入口；断言没有 `from jqdata import get_price`，没有凭证字面量。 +Expected: PASS；聚宽目录零改动，本地清单有独立可执行契约，双基准可复算且不污染来源回测目录。 -- [x] **步骤 3：运行失败测试** +--- -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_market_data_query.py tests\local_quant_research\test_joinquant_export.py -q` +## Task 3：适配本地结果、切换唯一入口并删除旧方案 -预期：FAIL，查询和导出模块尚未实现。 +**对应 OpenSpec：** 4.1—4.4 -- [x] **步骤 4：实现最小查询层和可渲染导出程序** +**Files：** -查询层从快照引用的一个或多个权威 CSV 建内存视图，返回只读 `SnapshotView`；导出模块只接受 `ExportRequest(securities, fields, snapshot_end_date, fq=None, skip_paused=False)`，不内置资产池或交易规则。 +- Create: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_adapter.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/cli.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/__init__.py` +- Modify: `scripts/research/local_quant_research/contract.py` +- Modify: `scripts/research/local_quant_research/runner.py` +- Create: `tests/local_quant_research/test_turtle_vectorbt_contract_execution.py` +- Create: `tests/local_quant_research/test_turtle_attribution_contract.py` +- Delete: `joinquant/strategies/strategy-003/research/turtle_etf/execution.py` +- Delete: `joinquant/strategies/strategy-003/research/turtle_etf/state.py` +- Delete: `joinquant/strategies/strategy-003/research/turtle_etf/signals.py` +- Delete: `joinquant/strategies/strategy-003/research/turtle_etf/risk.py` +- Delete: `joinquant/strategies/strategy-003/research/turtle_etf/allocation.py` +- Delete: `joinquant/strategies/strategy-003/research/turtle_etf/reporting.py` +- Delete old-only tests after replacement coverage exists -- [x] **步骤 5：验证失败门禁和无持久数据库** +**接口：** -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_market_data_query.py tests\local_quant_research\test_joinquant_export.py -q` +- `to_joinquant_facts(inputs, simulation, scenario_id) -> LocalExecutionFacts` +- `write_local_result(backtest_dir, manifest, performance, facts, attribution) -> LocalResultPackage` +- `run_candidate_with_vectorbt(frames, config, candidate_id, output_dir) -> LocalResultPackage` -预期：PASS；摘要不一致或 `remote_cleaned=False` 返回 `failed`；测试树中不存在 `*.duckdb`。 +### Step 1：RED -- [x] **步骤 6：提交任务 4** +测试目录、清单 Schema、代码与参数摘要、`performance.json`、四类物理事实字段和跨表勾稽。对 `strategy-003` 强制 `attribution_log-.parquet`，固定 `event_id` 唯一主键、字段、`turtle-etf-attribution/2` 原因码版本和最小原因码集合，并记录公司行动应用；缺失、摘要错误、未知原因码或无法覆盖实际订单、风险状态变化及公司行动应用时拒绝完成。明确断言不存在 `data/risk.parquet`、`data/period_risks.parquet`、`data/equity.parquet`、`data/trades.parquet` 或本地 `raw/` 伪证据。 -```powershell -git add scripts/research/market_data/query.py scripts/research/market_data/joinquant_export.py tests/local_quant_research/test_market_data_query.py tests/local_quant_research/test_joinquant_export.py -git commit -m "功能：实现行情查询与聚宽日线导出契约" -``` +### Step 2：GREEN -### 任务 5：实现通用运行器、三态和原子证据 +每次调用只执行传入的一个场景，通过 vectorbt 唯一入口把一份结果写入 `/backtests//`。完成后输出 `next_action=return_to_caller`，不得读取其他候选、循环调用自身、生成候选/聚合清单或调用 `quant_analysis`、Vibe、报告和推荐。冻结基线与六个挑战由主 agent 分别调用七次。 -**文件：** +### Step 3：删除旧方案 -- 创建：`scripts/research/local_quant_research/__init__.py` -- 创建：`scripts/research/local_quant_research/contracts.py` -- 创建：`scripts/research/local_quant_research/evidence.py` -- 创建：`scripts/research/local_quant_research/runner.py` -- 创建：`scripts/research/local_quant_research/cli.py` -- 创建：`tests/local_quant_research/test_runner.py` -- 创建：`tests/local_quant_research/test_evidence.py` +新规则夹具、适配和公开入口通过后，在同一任务中删除旧模块、旧导出与旧专用测试。扫描必须确认不存在： -**接口：** - -```python -RunStatus = Literal["complete", "evidence_insufficient", "failed"] -def load_run_config(path: Path, *, repo_root: Path) -> RunConfig: ... -def compute_run_id(snapshot_digest: str, config_digest: str, code_digest: str) -> str: ... -def run_project(config_path: Path, *, repo_root: Path) -> RunResult: ... +```text +process_day +def _simulate +旧 execution/state/signals/risk/allocation/reporting 导入 +旧八表物理契约 +流程内 quant_analysis 或 Vibe 调用 +兼容层、双引擎和回退 ``` -- [x] **步骤 1：写不安全配置和三态失败测试** +### Step 4：回归 -覆盖 Shell（命令解释器）字符串、仓库外路径、缺少 `snapshot_id`、缺少必需输出、未知状态、系统 Python、隐式安装和凭证字段。缺失声明输入必须在项目子进程前返回 `evidence_insufficient`。 +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_contract_execution.py tests\local_quant_research\test_analysis_manifest_schemas.py tests\local_quant_research\test_analysis_data_contract.py tests\local_quant_research\test_runner.py -q +``` -- [x] **步骤 2：写 `run_id`、幂等和原子固化失败测试** +--- -测试相同输入复用、快照/配置/代码任一变化产生新 ID、相同 ID 输出变化返回 `failed`、失败尝试保留紧凑诊断但不创建完成目录、成功目录恰好位于 `.local/quant-research///`。 +## Task 4：逐场景验证单次回测不超过180秒 -- [x] **步骤 3：运行失败测试** +**对应 OpenSpec：** 5.1—5.3 -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_runner.py tests\local_quant_research\test_evidence.py -q` +**Files：** -预期：FAIL，通用运行器不存在。 +- Create: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_benchmark.py` +- Create: `tests/local_quant_research/test_turtle_vectorbt_performance_contract.py` +- Create: `tests/local_quant_research/test_turtle_vectorbt_e2e.py` +- Runtime failure evidence: `.local/quant-research/strategy-003/.attempts/.json` -- [x] **步骤 4：实现固定运行阶段** +**接口：** -阶段严格为：配置和路径校验 → 快照与 CSV 摘要 → 内存 DuckDB 同源校验 → 同文件系统暂存目录 → `subprocess.run([...], shell=False)` 调用项目适配器 → 输出结构和摘要 → 原子固化唯一状态。 +- `benchmark_scenario(scenario_config, prepared_inputs, output_dir) -> PerformanceEvidence` -- [x] **步骤 5：验证安全边界和三态矩阵** +### Step 1：固定计时方法 -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_runner.py tests\local_quant_research\test_evidence.py -q` +每个场景通过一个全新子进程启动。子进程中： -预期：PASS；测试日志不含环境凭证值，项目适配器不能写出暂存目录。 +1. 读取已经准备好的输入，并创建一个尚不可见为完成结果的暂存区； +2. 第一次执行 vectorbt、写四类事实与海龟归因日志并完成校验，记为 `cold_seconds`； +3. 在同一已编译进程再次执行同一场景到暂存子目录，完成相同校验，记为 `warm_seconds`； +4. 比较冷/热规范化结果摘要，要求完全一致且两次均不超过180秒； +5. 保留冷启动权威候选目录，先删除预热副本和所有可丢弃暂存并验证清理结果； +6. 把已验证清理结果写入 `performance.json`，再生成并校验最终本地清单和全部摘要； +7. 只把已整理好的权威候选目录原子发布；发布后不再依赖任何写入或清理动作。任一前置门禁失败只留下 attempt（尝试）证据。 -- [x] **步骤 6：提交任务 5** +每次计时都从“准备后输入交给 vectorbt 执行内核”开始，到“交易执行、四类共同事实、海龟必需归因日志及其结构、摘要和跨表勾稽校验完成”时停止。停止计时后才写入并校验 `performance.json` 和最终本地清单；二者属于原子完成门禁，但不计入 `cold_seconds`、`warm_seconds`。行情导出、输入准备和独立策略分析同样不计入单次回测门槛。 -```powershell -git add scripts/research/local_quant_research tests/local_quant_research/test_runner.py tests/local_quant_research/test_evidence.py -git commit -m "功能：实现通用研究运行器与不可变证据" -``` +### Step 2：逐场景证据 -### 任务 6：实现海龟指标、状态和风险门禁 +主 agent 只对冻结基线和六个挑战分别调用 Skill，共七次；每次调用都必须保存环境、依赖、输入、代码、参数、场景、冷/热结果摘要，以及 `cold_seconds`、`warm_seconds`。摘要不一致或任一数值超过180秒即失败，不得使用多次调用的整体墙钟时间替代，也不得先发布 `complete` 目录后补性能或清理证据。 -**文件：** +### Step 3：公开入口 E2E -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/__init__.py` -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/indicators.py` -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/signals.py` -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/state.py` -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/risk.py` -- 创建：`tests/local_quant_research/test_turtle_indicators.py` -- 创建：`tests/local_quant_research/test_turtle_risk.py` +从 Skill 文档命令启动一个单场景完整 E2E，验证只产生一份结果、一次运行证据、逐场景性能证据、原子固化、`next_action=return_to_caller` 和临时产物清理，并明确断言没有候选数组或内部循环。另由主 agent 的集成 E2E 连续调用七次并在独立分析目录生成 `source-results.json`；聚宽归档只读 E2E 前后全部文件摘要与 Git 状态必须一致。 -**接口：** - -```python -def true_range(frame: pd.DataFrame) -> pd.Series: ... -def turtle_n(frame: pd.DataFrame, days: int = 20) -> pd.Series: ... -def breakout_levels(frame: pd.DataFrame, entry_days: int, exit_days: int) -> pd.DataFrame: ... -def initial_unit(equity: Decimal, n_value: Decimal, risk_fraction: Decimal = Decimal("0.005")) -> int: ... -def evaluate_risk(requests: Sequence[OrderIntent], state: PortfolioState, inputs: RiskInputs) -> RiskDecision: ... -``` - -- [x] **步骤 1：写指标、信号和次日执行失败测试** +--- -固定夹具必须证明 55/20 通道排除信号当日、TR 使用 `max(high-low, abs(high-pre_close), abs(low-pre_close))`、N 为 20 日均值、收盘确认信号在下一交易日开盘才成为订单。 +## Task 5：在本地 Skill 外完成完整稳健性、归因与确定性报告 -- [x] **步骤 2：写批次和共同止损失败测试** +**对应 OpenSpec：** 6.1—6.5 -覆盖固定信号日 N、0.5N 理论档位、同一 ETF 每日最多一次加仓、实际成交才改变批次、共同止损只上移、保护止损和 20 日退出均生成全仓退出意图。 +**Files：** -- [x] **步骤 3：写风险和故障安全失败测试** +- Modify: `scripts/research/quant_analysis/benchmarks.py` +- Modify: `scripts/research/quant_analysis/robustness.py` +- Modify: `scripts/research/quant_analysis/cvar.py` +- Modify: `scripts/research/quant_analysis/evidence.py` +- Create: `scripts/research/quant_analysis/analysis_plan.py` +- Create: `scripts/research/quant_analysis/orchestration.py` +- Create: `scripts/research/quant_analysis/unified_analysis.py` +- Create: `scripts/research/quant_analysis/reporting.py` +- Create: `scripts/research/quant_analysis/schemas/analysis-plan.schema.json` +- Create: `joinquant/strategies/strategy-003/research/analysis-plan.json` +- Create: `tests/quant_analysis/test_analysis_plan.py` +- Create: `tests/quant_analysis/test_unified_analysis.py` +- Create: `tests/quant_analysis/test_reporting.py` +- Create: `tests/quant_analysis/test_statistics.py` +- Runtime only: `.local/strategy-analysis-preparations//`、`.local/strategy-analysis//` -覆盖整手、现金、流动性、单 ETF、资产组、计划风险、60 个对齐样本、60 日协方差、10% 目标波动率；持仓价格或风险输入缺失时 `allow_new_risk=False`，但退出和强制减仓仍保留。 +### Step 1：生成完整且封闭的场景矩阵 -- [x] **步骤 4：运行失败测试** +`strategy-003/research/analysis-plan.json` 是策略自有的机器可读分析定义，固定 `schema_version=strategy-analysis-plan/1`。顶层包含 `strategy_id`、`baseline_config`、七个 `scenarios`、`universe`、`analyses`、`expected` 和 `thresholds`；每个基础场景包含唯一 `scenario_id`、`dimension` 和结构化 `overrides`，其余分析包含日期、资产、成本、抽样、冲击、门槛或固定种子等确定性配置。通用 `quant_analysis` 只按 `analysis-plan.schema.json` 校验并展开 `analysis-scenarios.json`，不得解析 Markdown、导入海龟模块或硬编码海龟资产、参数、分组、数量和门槛。本地研究 Skill 不读取该文件。 -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_indicators.py tests\local_quant_research\test_turtle_risk.py -q` +展开后的 `analysis-scenarios.json` 必须记录计划摘要和版本，逐项分类： -预期：FAIL，海龟纯计算模块尚未实现。 +| 分析维度 | 执行方式 | +|---|---| +| 冻结基线与六个参数邻域 | 引用主 agent 七次独立调用结果 | +| 三个固定时期 | 基线既有路径切片 | +| 三年滚动窗口、每季度移动 | 基线既有路径滚动切片 | +| 逐只删除11只 ETF | 删除对应收益贡献，不重新分配资金 | +| 逐组删除资产组 | 删除对应收益贡献，不重新分配资金 | +| 成本与延迟执行场景 | 一阶订单级敏感性估算 | +| 5/20/60日区块抽样各10,000条 | 从基线收益确定性计算 | +| 五个历史压力窗口 | 从基线收益、持仓和事件视图计算 | +| 四个持仓冲击 | 从每日实际持仓确定性计算 | +| 95%/99%及5日 CVaR | 从基线收益确定性计算 | -- [x] **步骤 5：实现最小纯计算模块并通过测试** +`analysis-plan.json` 与场景矩阵必须给出七个来源的期望数量、实际数量、参数摘要和输入范围，以及每项派生稳健性的计算方法。缺一项或摘要不一致即 `evidence_insufficient`，不得用 Vibe 补齐。 -模块只接收 DataFrame（数据表）和不可变记录对象，不读取全局目录、环境变量或行情中心；计算只使用未复权 `open/high/low/close/pre_close`。 +### Step 2：执行七个基础场景 -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_indicators.py tests\local_quant_research\test_turtle_risk.py -q` +独立分析准备入口只校验策略分析计划并在 `preparation_id` 下生成七份通用单场景配置，不导入海龟模块或直接循环本地运行器。主 agent 对每份配置调用一次本地研究 Skill；配置由 `strategy-003` 项目入口解释并使用同一 vectorbt 回调执行，结果保存在新的 `.local/quant-research/strategy-003//backtests//`。七次调用后，主 agent 显式传入七组 `scenario_id=run_id`；分析入口逐项校验唯一运行、参数摘要、结果摘要、性能证据、同一 `snapshot_id`、同一代码身份及结果清单后端。所有来源 `run_id` 保持不可变，`source-results.json` 只保存路径、摘要引用、登记表摘要和共享执行身份。 -预期：PASS。 +七个场景都执行Task 4的冷/热性能门禁。其余稳健性不再调用 Skill；报告必须声明时期是既有路径切片、资产删除不重新分配资金、成本和延迟是一阶估算。 -- [x] **步骤 6：提交任务 6** +### Step 3：确定性分析 -```powershell -git add joinquant/strategies/strategy-003/research/turtle_etf tests/local_quant_research/test_turtle_indicators.py tests/local_quant_research/test_turtle_risk.py -git commit -m "功能：实现海龟指标状态与风险门禁" -``` +通过统一读取器和双基准集计算收益、CAGR（复合年增长率）、波动率、回撤、Sharpe（夏普比率）、Sortino（索提诺比率）、Calmar（卡玛比率）、Alpha/Beta、上下行捕获、持仓与现金分布、风险预算使用、按 ETF/资产组/时期/交易原因归因，以及最终方案规定的全部稳健性、压力、CVaR和挑战门槛。 -### 任务 7：实现执行状态流与 A1 共享预算分配 +分析输出必须区分：来源事实、确定性派生值、门槛判断和 Vibe 安全边界审计。所有数值和结论由本地算法固化，Vibe 不得重新定义计算口径。 -**文件：** +### Step 4：Vibe 安全边界审计 -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/allocation.py` -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/execution.py` -- 创建：`tests/local_quant_research/test_turtle_allocation.py` -- 创建：`tests/local_quant_research/test_turtle_e2e.py` +最终 `analysis_id` 由 `preparation_id`、七份显式来源清单/结果摘要和共享执行身份共同派生，在其目录下记录来源、基准集、配置和确定性结果摘要；相同计划下的另一批运行得到不同身份，不能覆盖旧证据。Vibe 的研究目标和证据登记只作审计编排；只允许调用无已知缺陷的单体公开分析入口。禁止 `run_swarm`（运行群体分析）、Vibe 回测和有前视偏差风险的组合优化器；没有安全单体入口时记录 `evidence_insufficient`。若安全入口只能传 CSV（逗号分隔文件），必须由统一视图按明确字段与日期临时物化，确认读取后删除并记录清理结果。 -**接口：** +### Step 5：完整报告与推荐 -```python -def allocate_a1(candidates: Sequence[BuyRequest], constraints: PortfolioConstraints) -> AllocationResult: ... -def process_day(day: TradingDay, state: PortfolioState, market: DailyMarket) -> DayResult: ... -``` +必须生成收益、回撤、Alpha/Beta、仓位与风险控制、归因、六个挑战、完整稳健性、压力和尾部风险、反对证据、不确定性、推荐和工具证据。输出 `next_action=human_confirmation_required`，等待用户人工确认；不得启动聚宽、改参数、替换基线、冻结或模拟交易。 -- [x] **步骤 1：写 A1 公平分配失败测试** +--- -测试所有可行候选先按同一完成比例缩放；自身或资产组上限释放的预算可流向其他候选；先向下取整手，再按小数余额降序逐手补分；完全同分按证券代码升序；每补一手重查全部硬门槛。 +## Task 6：全量验证与交付 -- [x] **步骤 2：写输入顺序不变量和约束失败测试** +**对应 OpenSpec：** 7.1—7.3 -对同一候选集合的全部排列运行 `allocate_a1`，断言分配摘要相同、现金非负、单 ETF/资产组/计划风险/目标波动率均不突破。 +### Step 1：删除与边界扫描 -- [x] **步骤 3：写每日完整顺序失败测试** +扫描并确认没有旧执行符号、旧模块、旧八表物理契约、流程内分析调用、兼容层、双引擎、回退、死测试、聚宽归档修改或转换副本。历史原因只允许存在于说明本次迁移原因的非执行文档，不能形成任务、接口或验收要求。 -固定场景同时产生退出、强制减仓、新建仓和加仓，断言顺序严格为“全仓退出 → 强制风险减仓 → 同级新建仓/加仓”；同一 ETF 退出取消当日全部买入；停牌、跳空、涨跌停和不可成交不虚构成交。 +### Step 2：完整业务回归 -- [x] **步骤 4：运行失败测试** +使用项目 `.venv` 运行： -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_allocation.py tests\local_quant_research\test_turtle_e2e.py -q` +- 全量单元与集成测试； +- 从 Skill 用户入口贯通一次单场景行情快照、兼容结果、性能和 `return_to_caller` 状态的完整 E2E；另由主 agent 连续调用七次，证明 Skill 不含七方案耦合并聚合为独立分析来源； +- 独立分析入口贯通场景矩阵、路径重跑、双基准、确定性报告、Vibe 安全边界审计和人工确认前停止状态的完整 E2E； +- OpenSpec（开放规格）严格校验； +- Build and Verify（构建与验证）完整门禁； +- 敏感数据、临时文件和持久 DuckDB 扫描； +- 现有聚宽回测归档零改动断言； +- 独立前向验证。 -预期：FAIL，分配和执行模块尚未实现。 +不能用几个单元测试拼接代替完整入口。 -- [x] **步骤 5：实现最小分配和日状态机并通过测试** +### Step 3：完成报告 -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_allocation.py tests\local_quant_research\test_turtle_e2e.py -q` +逐项记录已验证、无法验证、性能结果、实际分析结果、Vibe 安全边界证据和临时产物清理。只有阻断项为零才进入完成审查；本任务不提交、不推送、不创建 PR（拉取请求），除非用户另行授权。 -预期：PASS，重复运行产生相同审计摘要。 +--- -- [x] **步骤 6：提交任务 7** +## Task 7：实现行情时点可知的公司行动近似核算并重建研究证据 + +**对应 OpenSpec：** 8.1—8.7；8.4、8.5 已完成，本任务只实施 8.2、8.3、8.6、8.7。 + +**Files：** + +- Modify: `scripts/research/market_data/contracts.py` +- Modify: `scripts/research/market_data/storage.py` +- Modify: `scripts/research/market_data/query.py` +- Modify: `scripts/research/market_data/joinquant_export.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_cli.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/single_scenario.py` +- Modify: `scripts/research/analysis_data/schemas/local-backtest-manifest.schema.json` +- Modify: `scripts/research/analysis_data/manifest.py` +- Modify: `tests/local_quant_research/test_market_data_storage.py` +- Modify: `tests/local_quant_research/test_market_data_query.py` +- Modify: `tests/local_quant_research/test_joinquant_export.py` +- Modify: `tests/local_quant_research/test_turtle_vectorbt_inputs.py` +- Modify: `tests/local_quant_research/test_turtle_result_adapter.py` +- Modify: `tests/local_quant_research/test_turtle_single_scenario.py` +- Modify: `tests/local_quant_research/test_analysis_manifest_schemas.py` +- Runtime only: `.local/market-data/`、`.local/quant-research/strategy-003/`、`.local/strategy-analysis-preparations/`、`.local/strategy-analysis/` + +**固定接口与口径：** + +- `normalize_corporate_action_rows(rows) -> list[dict[str, object]]` 固定事件字段、类型、主键、状态与日期语义；`corporate_action_digest(rows) -> str` 生成规范化内容摘要。 +- `import_batch(csv_path, corporate_actions_csv_path, manifest, root) -> BatchRecord` 原子固化 `market-data.parquet` 与 `corporate-actions.parquet`；两类规范化内容摘要共同决定 `batch_id`，两类批次摘要共同决定 `snapshot_id`。 +- `SnapshotView.corporate_actions` 与 `SnapshotView.corporate_actions_digest` 返回与行情同一快照身份的只读事件和摘要。 +- `corporate-actions.parquet` 至少保存来源事件主键、证券、事件类型、公告日、登记日、除权日、生效日、支付日、状态、知识截止日、拆分比例、每份现金、来源身份和来源摘要；空事件集也必须以固定 Schema（结构约束）落盘。 +- 导出器必须以取消日期相对 `snapshot_end_date` 重建事件状态；截止日后的取消在该快照中仍为有效，当前状态显示已取消但缺少取消日期时以 `evidence_insufficient` 停止，不得用当前 `process_id` 回写历史状态。 +- `prepare_simulation_inputs(frames, config, corporate_actions) -> SimulationInputs` 只应用公告日在生效日之前或当日、状态有效且知识截止日完整的事件；以 `上一交易日原始 close / 当日原始 pre_close` 决定连续因子。公告日晚于应用日、事件取消或数值不能勾稽时停止为 `evidence_insufficient`。连续因子从应用日向未来累乘，绝不回写过去。 +- 当公司行动能解释价格基准变化时，连续因子使用 `上一交易日原始 close / 当日原始 pre_close`；同一因子应用于当日及以后原始 OHLC（开高低收）与 `pre_close`。 +- vectorbt（向量化回测框架）的信号、突破、N 值、协方差、风险、成交、估值全部使用连续经济价格与经济单位。现金分红按除权日隐含再投资，不在支付日另加现金。 +- 本地结果只能声明时点可知的总回报近似，不得宣称真实拆分后份额、支付日现金、税费、真实再投资份额、零碎份额现金或聚宽订单路径精确一致。 + +### Step 1：8.2 RED——双事实批次与快照 + +先在 `test_market_data_storage.py` 添加失败测试，覆盖： + +1. 有事件与空事件两种导入都生成固定 Schema 的 `corporate-actions.parquet`； +2. 事件行主键、日期先后、状态、知识截止日、拆分比例和每份现金校验； +3. 行情相同而公司行动不同会得到不同 `batch_id` 和 `snapshot_id`； +4. 两类 Parquet（列式文件）任一被篡改均拒绝读取； +5. DuckDB（嵌入式分析数据库）只用 `:memory:` 回读两类事实； +6. 没有对应有效事件授权的价格基准变化返回 `evidence_insufficient`，不使用价格阈值猜测； +7. 成功与失败路径都不遗留 CSV（逗号分隔文件）暂存或持久数据库。 + +Run： ```powershell -git add joinquant/strategies/strategy-003/research/turtle_etf/allocation.py joinquant/strategies/strategy-003/research/turtle_etf/execution.py tests/local_quant_research/test_turtle_allocation.py tests/local_quant_research/test_turtle_e2e.py -git commit -m "功能：实现A1共享预算与每日执行状态流" +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_market_data_storage.py -q ``` -### 任务 8：实现项目适配器、报告、结论和候选包 +Expected：RED，原因是当前批次只认识 `market-data.parquet`，批次与快照身份没有公司行动证据。 -**文件：** +### Step 2：8.2 GREEN——实现双事实原子存储 -- 创建：`joinquant/strategies/strategy-003/research/project-run.json` -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/reporting.py` -- 创建：`joinquant/strategies/strategy-003/research/turtle_etf/cli.py` -- 修改：`tests/local_quant_research/test_turtle_e2e.py` +在 `storage.py` 增加最小公司行动规范化、Arrow（列式内存格式）Schema、内容摘要、原子写入、完整性校验和快照引用。保留现有行情接口语义；需要兼容无公司行动的通用夹具时，调用方必须显式提供合法空事件集，不能静默假设“没有事件”。实现后重新运行 Step 1，并运行市场数据相关回归： -**接口：** - -```python -def run_research(config_path: Path, snapshot_path: Path, output_dir: Path) -> ProjectResult: ... -def write_outputs(result: ResearchResult, output_dir: Path) -> Mapping[str, str]: ... +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_market_data_storage.py tests\local_quant_research\test_market_data_query.py tests\local_quant_research\test_joinquant_export.py -q ``` -- [x] **步骤 1：写三类必需输出失败测试** +### Step 3：8.3 RED——行情时点可知的连续经济价格 -断言运行生成 `research-report.md`、`conclusion.json`、`candidate-strategies.json`，以及 `daily-audit.csv`、`trades.csv`、`positions.csv`、`risk.csv`。任一文件缺失、JSON 结构错误或摘要不匹配时项目不得报告 `complete`。 +在输入、结果清单和统一读取测试中先加入失败用例： -- [x] **步骤 2：写研究建议和候选包失败测试** +- 512480 原始 `close=2.700`、次日原始 `pre_close=1.350` 的 1:2 拆分样例，连续因子从应用日开始为 2，过去行保持不变； +- 公告日晚于应用日时保留并标记为事后核对；取消事件不能授权价格基准变化，未知状态、关键字段缺失、重复冲突、知识截止日无效或没有有效事件解释的价格基准变化一律 `evidence_insufficient`； +- 当前已取消但取消日期晚于快照截止日时仍按截止日有效处理；已取消但缺少取消日期时关闭导出，防止把未来状态带回历史快照； +- 有效事件未发生价格基准变化时只记审计，不强制改变连续因子；官方拆分比例或每份现金必须在声明容差内与价格基准变化勾稽； +- 连续 OHLC、连续 `pre_close`、经济单位、突破、N 值和 `continuous_close / continuous_pre_close - 1` 协方差收益保持同一价格基准； +- 现金分红只通过除权日连续总回报隐含再投资，不在支付日增加现金或生成虚假订单； +- 公司行动前后权益连续，原始机械跳变不产生虚假突破、止损或风险放大； +- 本地清单缺少或篡改 `source.accounting` 时拒绝，统一读取器原样暴露精度限制。 -`conclusion.json.recommendation` 只能为 `proceed_to_joinquant`、`revise_and_reassess`、`stop_evidence_insufficient`；候选恰好七项、共用代码摘要与 `snapshot_id`，且没有按收益排名删除候选或新增参数。 - -- [x] **步骤 3：写报告内容失败测试** - -报告必须列出方法、输入身份、事件/交易、实际仓位分布、现金占比、留现原因、资产组和组合风险使用率、限制、产物摘要，以及“不是正式回测或最终验收结论”。设计期 63.7%/55.7% 代理值不得作为本次运行结果。 - -- [x] **步骤 4：运行失败测试** - -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_e2e.py -q` - -预期：FAIL，报告和项目 CLI（命令行接口）尚未实现。 - -- [x] **步骤 5：实现报告与项目入口并通过测试** - -`project-run.json` 只引用共享 `snapshot_id`，不得复制 CSV。Vibe-Trading（AI 研究助理）组合优化器配置固定为 `enabled=false` 并在报告写明跳过原因；方向性粗筛只消费确定性结果，不反向修改配置。 - -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_e2e.py -q` - -预期：PASS。 - -- [x] **步骤 6：提交任务 8** +Run： ```powershell -git add joinquant/strategies/strategy-003/research tests/local_quant_research/test_turtle_e2e.py -git commit -m "功能：生成海龟ETF本地研究报告与候选包" +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_inputs.py tests\local_quant_research\test_turtle_result_adapter.py tests\local_quant_research\test_turtle_single_scenario.py tests\local_quant_research\test_analysis_manifest_schemas.py -q ``` -### 任务 9：贯通 Skill 用户入口、非海龟 E2E 和验证映射 - -**文件：** - -- 创建：`tests/local_quant_research/test_generic_e2e.py` -- 修改：`tests/local_quant_research/test_skill_contract.py` -- 修改：`tests/local_quant_research/test_turtle_e2e.py` -- 修改：`.build-and-verify/config.json` - -**接口：** +Expected：RED，原因是当前输入直接使用原始未复权价格，清单没有核算口径。 -- 产出：从 Skill 文档公开命令启动的完整离线 E2E（端到端）。 -- 产出：不加载 `strategy-003`、海龟参数或海龟资产的最小项目 E2E。 +### Step 4：8.3 GREEN——接入 vectorbt 与结果精度元数据 -- [x] **步骤 1：写 Skill 用户入口失败测试** +最小实现只修改输入准备和结果契约： -测试从 `SKILL.md` 提取公开命令，使用临时 `.local` 根目录和固定日线夹具执行：批次 → 快照 → CSV 校验 → 内存 DuckDB → 通用运行器 → 海龟项目 → 审计/三类输出 → 不可变证据。 +1. 从快照公司行动事实派生逐证券前向累计连续因子； +2. 用连续经济 OHLC、`pre_close` 和经济单位构造 `SimulationInputs`（模拟输入）； +3. 协方差日收益改为 `continuous_close / continuous_pre_close - 1`； +4. 不新增公司行动订单、支付日现金或私有回测引擎； +5. `manifest.json` 的 `source.accounting` 固定写入： + - `corporate_action_mode=point_in_time_total_return_approximation` + - `continuity_factor_basis=raw_previous_close_over_current_pre_close` + - `corporate_action_metadata_timing=point_in_time_known` + - `price_basis=continuous_economic_price` + - `quantity_basis=economic_units` + - `cash_dividend_mode=implicit_reinvestment_on_ex_date` + - `pay_date_cash_supported=false` + - `exact_joinquant_reconciliation=false` + - 公司行动来源摘要与核算版本； +6. 归因日志记录事件身份、应用日、`evidence_timing=point_in_time`、连续因子与限制，不改变订单事实。 -- [x] **步骤 2：写非海龟前向失败测试** - -在临时目录生成只输出 `result.json` 的最小项目适配器，环境中不加入 `joinquant/strategies/strategy-003/research`；断言同一行情中心和运行器可返回 `complete`，通用源码中不出现 `turtle`、`55日` 或 11 个 ETF 代码。 - -- [x] **步骤 3：运行失败测试** - -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_generic_e2e.py tests\local_quant_research\test_turtle_e2e.py tests\local_quant_research\test_skill_contract.py -q` - -预期：FAIL，公开入口和验证映射尚未贯通。 - -- [x] **步骤 4：补齐 CLI 入口与 Build and Verify（构建与验证）检查** - -在 `.build-and-verify/config.json` 新增 `verify.local-quant-research-unit` 和 `verify.local-quant-research-e2e`，路径覆盖新 Skill、共享行情脚本、通用运行器、`strategy-003/research` 和测试，但 `inputs` 不得包含 `.local/**`。 - -- [x] **步骤 5：运行离线完整回归** - -运行：`.\.venv\Scripts\python.exe -m pytest tests\local_quant_research -q` - -运行：`.\.venv\Scripts\python.exe -m pytest tests\test_skill_layout.py -q` - -预期：全部 PASS；测试结束后临时目录自动清理。 - -- [x] **步骤 6：提交任务 9** +完成后重跑 Step 3，并加跑引擎与统一读取回归： ```powershell -git add tests/local_quant_research tests/test_skill_layout.py .build-and-verify/config.json scripts/research .agents/skills/run-local-quant-research .claude/skills/run-local-quant-research -git commit -m "测试：贯通本地研究技能端到端流程" +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_inputs.py tests\local_quant_research\test_turtle_vectorbt_engine.py tests\local_quant_research\test_turtle_result_adapter.py tests\local_quant_research\test_turtle_single_scenario.py tests\local_quant_research\test_analysis_manifest_schemas.py tests\local_quant_research\test_analysis_data_views.py -q ``` -### 任务 10：真实导出 11 只 ETF、执行本地研究并完成仓库验证 - -**文件：** - -- 仅本地生成：`.local/market-data/batches//manifest.json` -- 仅本地生成：`.local/market-data/batches//market-data.csv` -- 仅本地生成：`.local/market-data/batches//validation.json` -- 仅本地生成：`.local/market-data/snapshots/.json` -- 仅本地生成：`.local/quant-research/strategy-003//...` -- 修改：`openspec/changes/build-turtle-etf-local-research-workflow/tasks.md` - -**验收输入：** - -- 证券：`510300.XSHG`、`512100.XSHG`、`512480.XSHG`、`159819.XSHE`、`516160.XSHG`、`513100.XSHG`、`513180.XSHG`、`515180.XSHG`、`516080.XSHG`、`518880.XSHG`、`511010.XSHG`。 -- 字段：`date, security, open, high, low, close, pre_close, volume, money, factor, paused, high_limit, low_limit`。 -- 区间：每只 ETF 自身首个可用完整交易日至显式 `snapshot_end_date`；2015-01-01 前仅作预热；新增风险需 60 个有效对齐样本。 - -- [x] **步骤 1：在真实聚宽研究环境执行导出** - -使用任务 4 生成的程序，确认内置 `get_price`/`write_file`/`read_file`、`fq=None`、`skip_paused=False`、Pandas 0.23.4 `line_terminator` 和 `paused` 原始类型；远端文件回读字节摘要必须与下载文件一致。 - -- [x] **步骤 2：导入共享中心并清理远端中转文件** - -先验证 13 字段、唯一键、日期、空值、实际起止日、CSV 字节摘要和规范化内容摘要，再固化批次及快照；删除聚宽端中转文件并复查不存在。若清理无法确认，运行状态必须为 `failed`。 - -- [x] **步骤 3：从 Skill 用户入口执行真实本地研究** - -运行：`.\.venv\Scripts\python.exe scripts\research\local_quant_research\cli.py run --config joinquant\strategies\strategy-003\research\project-run.json` - -预期：输出唯一三态之一；只有 11 只 ETF 快照、运行清单、全部审计和三类必需输出均通过摘要校验时才允许 `complete`。 - -- [x] **步骤 4：人工复核本地报告和临时产物** - -确认报告给出实际平均/中位仓位、低于 50% 与接近满仓占比、现金占比和留现原因；确认没有把本地结果描述为正式回测；确认聚宽远端中转、本地下载暂存、隐藏 staging（暂存）目录均不存在，已固化批次/快照/完整运行证据保留。 - -- [x] **步骤 5：运行完整验证** - -运行：`.\.venv\Scripts\python.exe C:\Users\liuli\.codex\skills\.system\skill-creator\scripts\quick_validate.py .agents\skills\run-local-quant-research` - -运行：`.\.venv\Scripts\python.exe -m pytest -q` +### Step 5：8.6——重建七个场景与完整报告 -运行：`openspec validate --all --strict --no-interactive` +先解析并打印将要清理的绝对路径，确认都位于本仓库 `.local` 且只属于本变更，再删除旧公司行动口径污染的共享快照、七份单场景结果和派生分析；不得触碰 `strategy-001/002` 聚宽归档。随后： -在已获授权的 PR Flow hotfix（拉取请求热修复流程）收尾前运行：`.\.venv\Scripts\python.exe .build-and-verify\runtime\build_and_verify.py verify --project . --full`，并确认新检查命中；再运行 `git ls-files | rg "(^|/)\.local/|market-data\.csv$|\.duckdb$"`，预期无输出。 +1. 用真实 11 只 ETF 行情和权威公司行动事实生成新不可变快照； +2. 主 agent 从 `analysis-plan.json` 生成七份配置，并复数调用单场景 Skill 七次；Skill 本身不包含七方案数量、循环或聚合； +3. 每个场景冷启动和预热都小于等于 180 秒，规范化结果摘要一致； +4. 显式登记七组 `scenario_id -> run_id`，重新生成收益、回撤、Alpha/Beta（超额收益/市场暴露）、归因、仓位、风险、六个挑战、全部稳健性、压力、CVaR（条件风险价值）、报告和推荐； +5. 检查报告明确披露近似核算限制，不保留兼容副本，不运行新旧方案对照。 -- [x] **步骤 6：逐项勾选 OpenSpec 任务并提交验证证据** +### Step 6：8.7——完整用户入口与仓库门禁 -只有在对应测试、真实集成和清理证据均通过后，才把 `tasks.md` 的 30 项全部改为 `[x]`。无法验证的项保持未勾选并记录精确原因，不得用说明文字代替完成证据。 +依次运行： ```powershell -git add openspec/changes/build-turtle-etf-local-research-workflow/tasks.md -git commit -m "验证：完成海龟ETF本地研究流程回归" +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_market_data_storage.py tests\local_quant_research\test_turtle_vectorbt_inputs.py tests\local_quant_research\test_turtle_vectorbt_engine.py tests\local_quant_research\test_turtle_result_adapter.py tests\local_quant_research\test_turtle_single_scenario.py tests\local_quant_research\test_analysis_manifest_schemas.py -q +.\.venv\Scripts\python.exe -m pytest -q +openspec validate build-turtle-etf-local-research-workflow --strict ``` -## 最终完成门禁 - -- `git diff --check` 无输出。 -- `.\.venv\Scripts\python.exe -m pytest -q` 全部通过。 -- `openspec validate --all --strict --no-interactive` 全部通过。 -- Build and Verify（构建与验证）full（完整）检查通过。 -- Skill `quick_validate.py` 通过，`.claude` 目录链接解析正确。 -- `strategy-001`、`strategy-002` 无改动；没有正式回测或模拟交易被启动。 -- Git（版本管理）跟踪文件不含 `.local`、完整行情、持久 DuckDB、账号、Token 或 Cookie。 -- 真实 11 ETF 导出中转与本地暂存已清理；不可变行情批次、快照和完整运行证据保留在 `.local/`。 -- `research-report.md`、`conclusion.json`、`candidate-strategies.json` 均绑定同一 `run_id`、`snapshot_id`、代码摘要和配置摘要。 +再从公开 `run-local-quant-research` Skill 用户入口完成一次公司行动单场景 E2E（端到端），由主 agent 完成七次复数调用和独立分析 E2E，运行 Build and Verify（构建与验证）完整门禁、全仓旧精确记账/流动性规则扫描与全面代码审查。确认 `.local` 下没有 CSV 暂存、预热副本、测试临时产物或持久 DuckDB 文件；阻断项为零后才能进入 Comet verify（验证）阶段。 diff --git a/docs/superpowers/plans/2026-07-16-turtle-full-position-redistribution.md b/docs/superpowers/plans/2026-07-16-turtle-full-position-redistribution.md new file mode 100644 index 0000000..118dcfc --- /dev/null +++ b/docs/superpowers/plans/2026-07-16-turtle-full-position-redistribution.md @@ -0,0 +1,836 @@ +# Turtle ETF Full-Position Redistribution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 把 `strategy-003` 的本地研究基线改成经典海龟逐单位 N 风险模型与事件驱动的全量仓位再分配，并产出一个可由 Vibe-Trading（AI 研究助理）真实读取的新基线结果和研究报告。 + +**Architecture:** 行情和信号仍由共享 Parquet（列式存储）快照、DuckDB（内存数据库）和现有 `vectorbt`（向量化回测库）输入层提供；`strategy-003` 内部用固定大小的 Numba（即时编译器）数组保存最多四个逻辑单位，并在入场、加仓、止损或退出事件出现时统一计算 4/6/12 风险缩放、现金缩放和整手目标。标准四表不变，海龟归因扩展增加单位和缩放证据；独立 Vibe 分析只读标准结果包，不进入交易回调。 + +**Tech Stack:** Python 3.12、vectorbt 1.1.0、Numba 0.66.0、NumPy 2.4.6、Pandas 3.0.3、PyArrow（列式数据）、DuckDB（内存查询）、Pytest（测试框架）、OpenSpec（开放规格） + +## Global Constraints + +- 只改本计划列出的 `strategy-003` 本地研究、标准结果适配、独立分析兼容、规格和测试；不改聚宽正式策略、正式回测或模拟交易。 +- 所有 Python 命令使用 `.\.venv\Scripts\python.exe`；不安装或升级依赖。 +- 当前工作区已有未提交改动。每次提交前执行 `git diff --cached --name-only`，只暂存本任务文件；不得覆盖、清理或提交无关改动。 +- 不运行旧基线对照，不运行 17 ETF 扩展，不运行稳健性矩阵；本次只运行一个 11 ETF 新基线。 +- 不重写、不迁移、不删除既有 `.local/` 研究证据；新运行由现有不可变运行目录生成。 +- 删除旧 A1、资金仓位上限、计划风险上限、协方差交易门槛和目标波动率交易路径，不保留开关、极大值、`null`、兼容分支或回退实现。 +- 不增加成交额 1%、流动性门槛或任何未经确认的策略外规则。 +- `run-local-quant-research` Skill（技能）仍只编排一次单场景；不得把 7 个场景、海龟规则、Vibe 分析或报告写入 Skill。 +- 每个生产改动先有失败测试，再做最小实现；行为测试通过后再提交。 +- 完整业务验收必须从 `scripts/research/local_quant_research/cli.py run --config joinquant/strategies/strategy-003/research/project-run.json` 用户入口执行，单元测试不能替代。 + +## File Map + +- `joinquant/strategies/strategy-003/research/baseline.json`：唯一新基线机器契约。 +- `joinquant/strategies/strategy-003/research/analysis-plan.json`：保留 7 个单因子声明，但本次不执行；移除已失效协方差变体。 +- `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py`：只准备行情、信号、交易约束和分组，不再计算交易用协方差。 +- `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py`：逻辑单位、4/6/12 缩放、现金缩放、事件目标和成交后状态的唯一实现。 +- `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py`：严格解析新参数、分配固定状态数组、连接官方 vectorbt 回调。 +- `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py`：保留已存在的额外延迟研究能力，只同步新的再分配买卖动作；基线仍为次日开盘。 +- `joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py`：保持标准四表，输出逐单位与全量再分配归因证据。 +- `scripts/research/quant_analysis/unified_analysis.py`、`scripts/research/quant_analysis/reporting.py`：移除对旧仓位上限和目标波动率字段的硬依赖，使现有 Vibe 分析可读取新结果。 +- `tests/local_quant_research/`、`tests/quant_analysis/`：规则、结果包、分析和完整 E2E（端到端）回归。 +- `openspec/changes/build-turtle-etf-local-research-workflow/`、`docs/research/2026-07-13-turtle-etf-system-final-plan.md`：同步最新规则和范围。 +- `joinquant/strategies/strategy-003/research/code-identity.json`：最后更新实际执行文件摘要。 +- `.local/quant-research/strategy-003/`：新基线不可变运行证据，仅运行时写入，不提交。 +- `docs/research/2026-07-16-turtle-full-position-redistribution-baseline-report.md`：真实新基线研究报告。 + +--- + +### Task 1: 冻结唯一基线配置并删除失效挑战契约 + +**Files:** +- Modify: `tests/local_quant_research/test_contract_fixtures.py` +- Modify: `joinquant/strategies/strategy-003/research/baseline.json` +- Modify: `joinquant/strategies/strategy-003/research/analysis-plan.json` +- Delete: `joinquant/strategies/strategy-003/research/challenge-analysis-plan.json` +- Delete: `docs/superpowers/specs/2026-07-16-turtle-volatility-rearm-design.md` +- Delete: `docs/superpowers/plans/2026-07-16-classic-turtle-unit-challenge.md` + +- [ ] **Step 1: 先把配置测试改成新契约** + +将基线断言改为以下精确值： + +```python +assert baseline["signal"] == { + "entry_days": 55, + "exit_days": 20, + "n_days": 20, + "add_step_n": 0.5, + "stop_n": 2.0, + "max_units": 4, +} +assert baseline["risk"] == { + "unit_risk_per_n": 0.01, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, +} +assert baseline["execution"] == { + "additional_delay_days": 0, + "order_priority": [ + "full_exit", + "redistribution_sell", + "entry_or_addition", + "redistribution_buy", + ], + "allocation": "full_position_redistribution", + "acceptance_fixture": { + "same_security_exit_cancels_buys": True, + "candidate_requires_net_buy_lot": True, + "group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + "cash_scaling": "uniform", + "lot_rounding": "floor", + "residual_cash_redistribution": False, + "input_order_invariant": True, + }, +} +assert not (_research_dir(repo_root) / "challenge-analysis-plan.json").exists() +``` + +把原协方差两个单因子替换为尚不执行的 `group-unit-cap-5` 和 `portfolio-unit-cap-10`，其覆盖分别为 `{"risk": {"asset_group_unit_cap": 5.0}}`、`{"risk": {"portfolio_unit_cap": 10.0}}`；保持总场景声明为 7，删除挑战计划测试。 + +- [ ] **Step 2: 运行配置测试并确认失败** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_contract_fixtures.py -q +``` + +Expected: FAIL，显示旧风险字段、`max_units=null`、旧分配名称和挑战文件仍存在。 + +- [ ] **Step 3: 最小修改配置与删除文件** + +按失败测试修改两个 JSON（结构化配置）。保留 11 ETF、6 分组、行情口径、费用和 150 万初始资金不变；物理删除三个失效文件，不创建替代兼容文件。 + +- [ ] **Step 4: 重新运行配置测试** + +Run: + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_contract_fixtures.py -q +``` + +Expected: PASS。 + +- [ ] **Step 5: 只提交本任务文件** + +```powershell +Test-Path joinquant/strategies/strategy-003/research/challenge-analysis-plan.json +Test-Path docs/superpowers/specs/2026-07-16-turtle-volatility-rearm-design.md +Test-Path docs/superpowers/plans/2026-07-16-classic-turtle-unit-challenge.md +git add -- tests/local_quant_research/test_contract_fixtures.py joinquant/strategies/strategy-003/research/baseline.json joinquant/strategies/strategy-003/research/analysis-plan.json +git diff --cached --name-only +git commit -m "配置：切换海龟N风险单位基线" +``` + +三个 `Test-Path` 必须都返回 `False`。这些被删除文件当前均未纳入 Git（版本管理），不得为提交记录重新创建空文件。 + +--- + +### Task 2: 删除交易用协方差输入并严格解析新参数 + +**Files:** +- Modify: `tests/local_quant_research/test_turtle_vectorbt_inputs.py` +- Modify: `tests/local_quant_research/test_turtle_vectorbt_engine.py` +- Modify: `tests/local_quant_research/test_turtle_result_adapter.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py` + +- [ ] **Step 1: 写输入和参数失败测试** + +将 `SimulationInputs` 的精确字段断言收敛为 `signal_n` 结束，不再包含 `covariance`、`covariance_eligible`。将引擎参数测试改为： + +```python +assert CallbackParams._fields == ( + "lot_size", + "unit_risk_per_n", + "add_step_n", + "stop_n", + "max_units", + "asset_group_unit_cap", + "portfolio_unit_cap", + "commission_multiplier", + "one_way_slippage", +) +assert params.max_units == 4 +assert params.asset_group_unit_cap == 6.0 +assert params.portfolio_unit_cap == 12.0 +``` + +增加参数化拒绝测试，逐个向 `risk` 注入以下字段并断言 `ValueError("legacy risk fields are not supported")`： + +```python +LEGACY = ( + "security_risk_cap", + "security_value_cap", + "asset_group_risk_cap", + "asset_group_value_cap", + "portfolio_risk_cap", + "portfolio_value_cap", + "covariance", + "target_volatility", + "risk_reduction_target_volatility", + "minimum_aligned_samples", +) +``` + +另断言 `max_units` 缺失、`null`、布尔值、非 4 正整数均被拒绝。 + +- [ ] **Step 2: 运行聚焦测试并确认失败** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_inputs.py tests\local_quant_research\test_turtle_vectorbt_engine.py tests\local_quant_research\test_turtle_result_adapter.py -q +``` + +Expected: FAIL，旧协方差数组和旧 `CallbackParams` 仍存在。 + +- [ ] **Step 3: 从输入层物理删除协方差计算** + +删除 `SimulationInputs.covariance`、`SimulationInputs.covariance_eligible`、`_covariance_matrix`、风险配置读取、滚动收益矩阵和相关 `math` 导入。`prepare_simulation_inputs` 仍输出排序稳定、连续只读的行情、公司行动、信号和分组数组；`additional_delay_days` 不得移动信号源行。 + +- [ ] **Step 4: 用新参数替换引擎解析** + +在 `vectorbt_engine.py` 定义并调用： + +```python +_LEGACY_RISK_FIELDS = frozenset({ + "security_risk_cap", "security_value_cap", + "asset_group_risk_cap", "asset_group_value_cap", + "portfolio_risk_cap", "portfolio_value_cap", + "covariance", "target_volatility", + "risk_reduction_target_volatility", "minimum_aligned_samples", +}) + +def _reject_legacy_risk_fields(risk: Mapping[str, object]) -> None: + found = sorted(set(risk) & _LEGACY_RISK_FIELDS) + if found: + raise ValueError( + "legacy risk fields are not supported: " + ", ".join(found) + ) +``` + +`_params` 只构造 `lot_size`、`unit_risk_per_n`、`add_step_n`、`stop_n`、固定为 4 的 `max_units`、`asset_group_unit_cap`、`portfolio_unit_cap`、佣金倍数和滑点。`CallbackInputs` 只传交易所需数组和 `asset_group_ids`。 + +- [ ] **Step 5: 运行聚焦测试并确认通过** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_inputs.py tests\local_quant_research\test_turtle_vectorbt_engine.py tests\local_quant_research\test_turtle_result_adapter.py -q +``` + +Expected: PASS。 + +- [ ] **Step 6: 提交输入和引擎契约** + +```powershell +git add -- tests/local_quant_research/test_turtle_vectorbt_inputs.py tests/local_quant_research/test_turtle_vectorbt_engine.py tests/local_quant_research/test_turtle_result_adapter.py joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py +git diff --cached --name-only +git commit -m "重构：删除海龟交易用协方差门槛" +``` + +--- + +### Task 3: 先实现可独立验证的 4/6/12 与现金缩放内核 + +**Files:** +- Modify: `tests/local_quant_research/test_turtle_vectorbt_callbacks.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py` + +- [ ] **Step 1: 写缩放公式失败测试** + +直接调用 Numba 函数的 `.py_func`，至少增加以下四组断言： + +```python +group_scales, portfolio_scale = _risk_scales_nb.py_func( + np.asarray([4, 4, 4]), + np.asarray([0, 0, 1]), + 2, + 6.0, + 12.0, +) +assert group_scales.tolist() == pytest.approx([0.75, 1.0]) +assert portfolio_scale == pytest.approx(1.0) + +group_scales, portfolio_scale = _risk_scales_nb.py_func( + np.asarray([4, 4, 4, 4]), + np.asarray([0, 1, 2, 3]), + 4, + 6.0, + 12.0, +) +assert group_scales.tolist() == pytest.approx([1.0, 1.0, 1.0, 1.0]) +assert portfolio_scale == pytest.approx(0.75) +``` + +再构造基础数量 `[1000, 2000]`、相同开盘价、现金不足和每笔最低佣金场景，断言：目标都是 100 的整数倍、预计成交后现金非负、两者使用同一现金比例向下取整、没有把剩余现金按代码补给任一标的。最后对输入列做排列并还原，断言目标、组缩放、组合缩放和现金缩放完全一致。 + +- [ ] **Step 2: 运行缩放测试并确认失败** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_callbacks.py -k "group_unit_scale or portfolio_unit_scale or uniform_cash_scale or permutation" -q +``` + +Expected: FAIL，新函数尚不存在。 + +- [ ] **Step 3: 实现纯缩放函数并删除 A1/Hamilton（最大余数）函数** + +在 `vectorbt_callbacks.py` 实现以下精确接口： + +```python +@njit +def _risk_scales_nb( + unit_counts: np.ndarray, + asset_group_ids: np.ndarray, + group_count: int, + asset_group_unit_cap: float, + portfolio_unit_cap: float, +) -> tuple[np.ndarray, float]: + group_units = np.zeros(group_count, dtype=np.float64) + for column in range(unit_counts.shape[0]): + group_units[asset_group_ids[column]] += unit_counts[column] + group_scales = np.ones(group_count, dtype=np.float64) + for group in range(group_count): + if group_units[group] > asset_group_unit_cap: + group_scales[group] = asset_group_unit_cap / group_units[group] + effective_units = 0.0 + for column in range(unit_counts.shape[0]): + effective_units += ( + unit_counts[column] * group_scales[asset_group_ids[column]] + ) + portfolio_scale = 1.0 + if effective_units > portfolio_unit_cap: + portfolio_scale = portfolio_unit_cap / effective_units + return group_scales, portfolio_scale + +@njit +def _targets_for_scale_nb( + unit_base_quantities: np.ndarray, + unit_counts: np.ndarray, + asset_group_ids: np.ndarray, + group_scales: np.ndarray, + portfolio_scale: float, + cash_scale: float, + locked_quantities: np.ndarray, + lot_size: int, +) -> np.ndarray: + targets = np.zeros(unit_counts.shape[0], dtype=np.int64) + for column in range(unit_counts.shape[0]): + if locked_quantities[column] >= 0: + targets[column] = locked_quantities[column] + continue + raw_quantity = 0 + for unit in range(unit_counts[column]): + raw_quantity += unit_base_quantities[column, unit] + scaled = ( + raw_quantity + * group_scales[asset_group_ids[column]] + * portfolio_scale + * cash_scale + ) + targets[column] = int(scaled // lot_size) * lot_size + return targets + +@njit +def _cash_after_targets_nb( + row: int, + targets: np.ndarray, + positions: np.ndarray, + cash: float, + inputs: CallbackInputs, + params: CallbackParams, +) -> float: + projected_cash = cash + for column in range(targets.shape[0]): + current = int(round(positions[column])) + if targets[column] >= current: + continue + quantity = current - targets[column] + price = _sell_price( + inputs.execution_open[row, column], params.one_way_slippage + ) + projected_cash += price * quantity - _commission( + price, quantity, params.commission_multiplier + ) + for column in range(targets.shape[0]): + current = int(round(positions[column])) + if targets[column] <= current: + continue + quantity = targets[column] - current + price = _buy_price( + inputs.execution_open[row, column], params.one_way_slippage + ) + projected_cash -= price * quantity + _commission( + price, quantity, params.commission_multiplier + ) + return projected_cash + +@njit +def _cash_feasible_targets_nb( + row: int, + raw_unit_base_quantities: np.ndarray, + unit_counts: np.ndarray, + positions: np.ndarray, + cash: float, + group_scales: np.ndarray, + portfolio_scale: float, + locked_quantities: np.ndarray, + inputs: CallbackInputs, + params: CallbackParams, +) -> tuple[np.ndarray, float]: + full_targets = _targets_for_scale_nb( + raw_unit_base_quantities, + unit_counts, + inputs.asset_group_ids, + group_scales, + portfolio_scale, + 1.0, + locked_quantities, + params.lot_size, + ) + if _cash_after_targets_nb( + row, full_targets, positions, cash, inputs, params + ) >= -1e-9: + return full_targets, 1.0 + lower = 0.0 + upper = 1.0 + best = _targets_for_scale_nb( + raw_unit_base_quantities, + unit_counts, + inputs.asset_group_ids, + group_scales, + portfolio_scale, + lower, + locked_quantities, + params.lot_size, + ) + for _ in range(64): + candidate_scale = (lower + upper) / 2.0 + candidate = _targets_for_scale_nb( + raw_unit_base_quantities, + unit_counts, + inputs.asset_group_ids, + group_scales, + portfolio_scale, + candidate_scale, + locked_quantities, + params.lot_size, + ) + if _cash_after_targets_nb( + row, candidate, positions, cash, inputs, params + ) >= -1e-9: + lower = candidate_scale + best = candidate + else: + upper = candidate_scale + return best, lower +``` + +实现要求：`_risk_scales_nb` 先组后组合；`_targets_for_scale_nb` 只统一乘比例并按整手向下取整；`locked_quantities` 以 `-1` 表示可调整，以非负实际持仓固定不可交易标的；`_cash_feasible_targets_nb` 把可成交卖出净收入、买入价、滑点和每笔佣金纳入现金，若比例 1 不可行则在 `[0, 1]` 上二分 64 次，返回最大共同可行比例对应目标。禁止最大余数分配和残余现金补仓。 + +同时物理删除 `_feasibility_mask_nb`、`_hamilton_quantities_nb`、`_maximum_hamilton_allocation_nb`、`_allocate_a1_nb`、旧掩码常量和组合波动率函数。 + +- [ ] **Step 4: 运行缩放测试并确认通过** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_callbacks.py -k "group_unit_scale or portfolio_unit_scale or uniform_cash_scale or permutation" -q +``` + +Expected: PASS。 + +- [ ] **Step 5: 提交纯分配内核** + +```powershell +git add -- tests/local_quant_research/test_turtle_vectorbt_callbacks.py joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py +git diff --cached --name-only +git commit -m "实现：增加海龟全量风险缩放内核" +``` + +--- + +### Task 4: 用逐单位状态实现事件驱动全量仓位再分配 + +**Files:** +- Modify: `tests/local_quant_research/test_turtle_vectorbt_callbacks.py` +- Modify: `tests/local_quant_research/test_turtle_vectorbt_engine.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py` + +- [ ] **Step 1: 写逐单位状态与止损失败测试** + +增加测试覆盖以下精确行为： + +1. 入场候选数量为 `floor_to_100(signal_equity * 0.01 / signal_n)`；成交后保存单位的 `signal_n`、`base_quantity` 和实际成交价。 +2. 首次成交价 10、冻结 N=1 时初始止损为 8；后续加仓实际成交价 13、该单位冻结 N=2 时候选止损为 9，共同止损变为 9。 +3. 后续每日 N 改为 999 且没有新单位成交时，共同止损仍为 9。 +4. 固定档位只使用首次实际成交价和首次 N；同日跨越多个档位仍只新增一个单位；第四单位后不再产生候选。 +5. 停牌、涨停、整手不足、现金缩放为零或订单拒绝时，单位数、下一档和共同止损均不变化。 +6. 再分配买卖改变实际持仓，但不改变单位数、单位数组、下一档或共同止损。 + +- [ ] **Step 2: 写全量再分配失败测试** + +增加以下组合测试： + +- 三个早期标的各有 4 单位、组合已达 12 单位；第四个标的产生 1 单位候选后，组合缩放为 `12/13`，三个早期目标同时按相同比例下降，晚到标的获得至少一手，证明不按时间占用预算。 +- 同组两只标的合计 8 单位时组缩放为 `6/8`；组外标的不受组缩放影响。 +- 同日多个候选使用统一临时单位集合计算；交换证券列顺序后还原，订单方向和目标数量一致。 +- 没有入场、加仓、止损或退出事件时，即使权益、价格或 N 变化，也没有再平衡订单。 +- 同一 ETF 同日退出与加仓同时满足时只保留完整退出；完整退出和再分配卖出先于所有买入。 +- 候选因统一目标不能形成至少一手净买入时被移除，重新计算后其状态不推进；直到候选集合稳定。 + +- [ ] **Step 3: 运行行为测试并确认失败** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_callbacks.py tests\local_quant_research\test_turtle_vectorbt_engine.py -k "unit or stop or redistribution or late_signal or no_event or candidate" -q +``` + +Expected: FAIL，旧状态只有一个 `standard_unit/signal_n`，且只分配当日新增订单。 + +- [ ] **Step 4: 替换回调状态结构** + +`CallbackState` 必须改为固定四单位数组，至少包含： + +```python +( + "unit_count", + "unit_signal_n", + "unit_base_quantities", + "unit_fill_prices", + "initial_fill_price", + "initial_signal_n", + "common_stop", + "next_add_index", + "candidate_signal_n", + "candidate_base_quantity", + "action_codes", + "reason_codes", + "requested_quantities", + "planned_quantities", + "filled_quantities", + "fill_prices", + "fees", + "state_quantities", + "state_common_stop", + "state_next_add_index", + "state_unit_counts", + "event_group_scales", + "event_portfolio_scales", + "event_cash_scales", + "day_equity", + "allocation_ready", +) +``` + +其中单位数组形状固定为 `(columns, 4)`；候选和状态证据按 `(rows, columns)`；组合与现金缩放按 `rows`。删除旧 `standard_unit`、单一 `signal_n` 和批次数组。 + +- [ ] **Step 5: 实现事件规划的稳定重算** + +在 `pre_segment_func_nb` 中按以下唯一顺序实现：先识别并冻结退出与至多一个单位候选；退出覆盖同标的候选；把可交易候选加入临时单位簿；调用 4/6/12 和现金缩放；删除不能形成至少一手净新增买入的候选并循环重算；生成每只 ETF 的目标差额和动作。 + +动作常量只保留： + +```python +ACTION_NONE = 0 +ACTION_FULL_EXIT = 1 +ACTION_REDISTRIBUTION_SELL = 2 +ACTION_ENTRY = 3 +ACTION_ADDITION = 4 +ACTION_REDISTRIBUTION_BUY = 5 +``` + +调用顺序固定为完整退出、再分配卖出、入场或加仓、再分配买入。`order_func_nb` 只执行已统一计算的差额，不再二次分配。 + +- [ ] **Step 6: 实现真实成交后的状态隔离** + +`post_order_func_nb` 必须满足：完整退出成交才清除全部单位；再分配买卖永不改单位和止损；入场或加仓真实买入成交才把候选写入下一空单位槽，并用 `actual_fill_price - 2 * frozen_signal_n` 只上移共同止损；拒绝或零成交不推进。每日最后一列调用后写入所有状态与缩放证据。 + +- [ ] **Step 7: 运行回调与引擎测试** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_callbacks.py tests\local_quant_research\test_turtle_vectorbt_engine.py -q +``` + +Expected: PASS；四个官方回调均产生 `nopython_signatures`，现金始终非负。 + +- [ ] **Step 8: 提交核心交易实现** + +```powershell +git add -- tests/local_quant_research/test_turtle_vectorbt_callbacks.py tests/local_quant_research/test_turtle_vectorbt_engine.py joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py +git diff --cached --name-only +git commit -m "实现：完成海龟全量仓位再分配" +``` + +--- + +### Task 5: 同步额外延迟执行器且不改变基线成交时点 + +**Files:** +- Modify: `tests/local_quant_research/test_turtle_vectorbt_delayed.py` +- Modify: `tests/local_quant_research/test_turtle_result_adapter.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py` + +- [ ] **Step 1: 写新动作的延迟执行失败测试** + +保留既有冻结计划测试，并增加：再分配卖出优先于入场/加仓和再分配买入；再分配买卖成交不改共同止损与下一档；只有入场/加仓成交使用冻结 N 更新止损；完整退出才清空状态。基线 `additional_delay_days=0` 时仍在信号次日开盘执行，不进入额外延迟后端。 + +- [ ] **Step 2: 运行延迟测试并确认失败** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_delayed.py tests\local_quant_research\test_turtle_result_adapter.py -k "delayed or redistribution" -q +``` + +Expected: FAIL，延迟执行器仍引用 `ACTION_RISK_REDUCTION`。 + +- [ ] **Step 3: 最小同步新动作语义** + +删除 `ACTION_RISK_REDUCTION`；卖出集合改为 `ACTION_FULL_EXIT`、`ACTION_REDISTRIBUTION_SELL`，买入集合包含 `ACTION_ENTRY`、`ACTION_ADDITION`、`ACTION_REDISTRIBUTION_BUY`。冻结目标、费用、整手现金截断和不可交易证据保持不变；再分配动作不能调用单位止损更新分支。 + +- [ ] **Step 4: 运行延迟测试并确认通过** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_vectorbt_delayed.py tests\local_quant_research\test_turtle_result_adapter.py -k "delayed or redistribution" -q +``` + +Expected: PASS。 + +- [ ] **Step 5: 提交延迟执行同步** + +```powershell +git add -- tests/local_quant_research/test_turtle_vectorbt_delayed.py tests/local_quant_research/test_turtle_result_adapter.py joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py +git diff --cached --name-only +git commit -m "修复：同步全量再分配延迟执行语义" +``` + +--- + +### Task 6: 保持标准四表并增加可分析的单位与缩放归因 + +**Files:** +- Modify: `tests/local_quant_research/test_turtle_result_adapter.py` +- Modify: `tests/quant_analysis/test_unified_analysis.py` +- Modify: `tests/quant_analysis/test_reporting.py` +- Modify: `joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py` +- Modify: `scripts/research/quant_analysis/unified_analysis.py` +- Modify: `scripts/research/quant_analysis/reporting.py` + +- [ ] **Step 1: 写结果包归因失败测试** + +保持 `results`、`balances`、`positions`、`orders` 字段完全不变。把动作映射改为 `full_exit`、`redistribution_sell`、`entry`、`addition`、`redistribution_buy`；删除 `risk_reduction` 和 `target_volatility_reduction`。增加断言： + +```python +details = json.loads(redistribution_event["details_json"]) +assert details["unit_count_after"] == 4 +assert details["group_scale"] == pytest.approx(0.75) +assert details["portfolio_scale"] == pytest.approx(12 / 13) +assert 0.0 < details["cash_scale"] <= 1.0 +assert details["redistribution_state_changed"] is False +``` + +入场/加仓决策还要包含 `candidate_base_quantity`、`frozen_signal_n`、`actual_fill_price` 和成交后的共同止损。归因顶层原因码加入 `full_position_redistribution`，删除 `forced_risk_reduction` 和 `risk_gate_block` 的生产路径。 + +- [ ] **Step 2: 写 Vibe 风险指标失败测试** + +将 `_risk_metrics` 测试配置替换为新风险字段，断言分析不再读取旧仓位上限或目标波动率，并输出： + +```python +assert metrics["maximum_security_weight"] == pytest.approx(0.4) +assert metrics["maximum_asset_group_weight"] == pytest.approx(0.4) +assert metrics["maximum_planned_loss_ratio"] == pytest.approx(0.01) +assert metrics["maximum_effective_risk_units"] == pytest.approx(12.0) +assert metrics["maximum_portfolio_unit_utilization"] == pytest.approx(1.0) +assert metrics["redistribution_event_count"] == 1 +``` + +没有海龟扩展字段的聚宽结果应返回 `None/0`，而不是失败，以保持现有聚宽结果零改动可读。 + +- [ ] **Step 3: 运行结果与分析测试并确认失败** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_result_adapter.py tests\quant_analysis\test_unified_analysis.py tests\quant_analysis\test_reporting.py -q +``` + +Expected: FAIL，适配器和分析仍依赖旧风险字段。 + +- [ ] **Step 4: 更新结果适配器** + +读取 `state_unit_counts`、`event_group_scales`、`event_portfolio_scales`、`event_cash_scales` 和候选基础数量，写入现有 `details_json`；不新增第五张标准表，不改变标准 Schema（结构约束）。再分配订单的买卖方向仅由新动作集合决定；再分配成交后的 `state_changed` 只表示实际持仓变化，另以 `redistribution_state_changed=False` 明确海龟单位状态未变。 + +- [ ] **Step 5: 把 Vibe 风险分析改成通用暴露与可选单位证据** + +在 `_risk_metrics` 中删除对 `security_value_cap`、`asset_group_value_cap`、`portfolio_value_cap`、`portfolio_risk_cap`、`target_volatility` 的索引。保留实际平均仓位、现金、最大单标的/资产组权重、60 日已实现波动率、订单、换手、费用、退出收益和止损事件；把计划风险改成 `planned_loss / equity` 的 `maximum_planned_loss_ratio`。从归因 `details_json` 可选读取 `effective_risk_units`、`portfolio_unit_cap` 和再分配标记，生成上述三个单位指标；缺失时返回 `None/0`。 + +同步 `reporting.py` 的风险表标签，删除“超过旧上限”“超过目标波动率”“强制风险约束”行，增加“最高计划损失比例”“最高有效 N 风险单位”“组合单位预算最高利用率”“全量再分配事件”。 + +- [ ] **Step 6: 运行结果、分析和报告测试** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_turtle_result_adapter.py tests\quant_analysis\test_unified_analysis.py tests\quant_analysis\test_reporting.py -q +``` + +Expected: PASS，标准四表字段集合不变，聚宽形状夹具仍可读取。 + +- [ ] **Step 7: 提交结果与分析兼容** + +```powershell +git add -- tests/local_quant_research/test_turtle_result_adapter.py tests/quant_analysis/test_unified_analysis.py tests/quant_analysis/test_reporting.py joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py scripts/research/quant_analysis/unified_analysis.py scripts/research/quant_analysis/reporting.py +git diff --cached --name-only +git commit -m "分析：支持海龟N风险单位归因" +``` + +--- + +### Task 7: 同步规格、研究方案和执行身份并清除旧生产路径 + +**Files:** +- Modify: `docs/superpowers/specs/2026-07-16-turtle-full-position-redistribution-design.md` +- Modify: `docs/research/2026-07-13-turtle-etf-system-final-plan.md` +- Modify: `openspec/changes/build-turtle-etf-local-research-workflow/proposal.md` +- Modify: `openspec/changes/build-turtle-etf-local-research-workflow/design.md` +- Modify: `openspec/changes/build-turtle-etf-local-research-workflow/tasks.md` +- Modify: `openspec/changes/build-turtle-etf-local-research-workflow/specs/turtle-etf-local-research/spec.md` +- Modify: `openspec/changes/build-turtle-etf-local-research-workflow/specs/standard-strategy-analysis-data/spec.md` +- Modify: `joinquant/strategies/strategy-003/research/code-identity.json` + +- [ ] **Step 1: 先写/更新文档契约测试或严格规格断言** + +在现有 `test_contract_fixtures.py` 和 `test_turtle_vectorbt_engine.py` 中断言当前设计状态为“已确认”，OpenSpec 明确 11 ETF、55/20/20、0.5N、2N、4/6/12、事件驱动全量再分配、单场景小于 180 秒、无旧对照和无本次稳健性运行；执行身份文件中的回调摘要必须等于实际文件 SHA256（摘要）。 + +- [ ] **Step 2: 运行契约与 OpenSpec 校验并确认失败** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_contract_fixtures.py tests\local_quant_research\test_turtle_vectorbt_engine.py -q +openspec validate build-turtle-etf-local-research-workflow --strict +``` + +Expected: 至少一项 FAIL，文档和摘要仍描述旧实现。 + +- [ ] **Step 3: 同步权威文档** + +把设计状态改为“已确认”；原始研究方案和 OpenSpec 使用同一规则。明确通用 Skill 仍单次运行、策略分析独立、聚宽正式复核不在范围、历史 `.local` 不改、17 ETF 不进基线、此次只运行一个新基线。 + +- [ ] **Step 4: 核对旧生产符号已物理删除** + +Run: + +```powershell +rg -n "_allocate_a1_nb|_hamilton_quantities_nb|ACTION_RISK_REDUCTION|REASON_TARGET_VOLATILITY_REDUCTION|_portfolio_volatility_nb|mandatory_risk_reduction|a1_uniform_completion" joinquant/strategies/strategy-003/research/turtle_etf joinquant/strategies/strategy-003/research/baseline.json scripts/research/quant_analysis +``` + +Expected: 无输出。旧字段名称只允许出现在“拒绝旧字段”的测试常量和已确认设计的删除说明中，不得存在于生产配置、参数、回调、结果适配或分析执行分支。 + +- [ ] **Step 5: 更新执行身份摘要** + +用 `Get-FileHash -Algorithm SHA256` 计算 `code-identity.json.files` 中每个实际文件，并用 `apply_patch` 更新对应摘要；`execution.callbacks_sha256` 必须等于 `vectorbt_callbacks.py` 摘要。删除已经不存在的文件条目，新增本变更实际进入执行身份但尚未登记的文件。 + +- [ ] **Step 6: 运行契约与严格规格校验** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_contract_fixtures.py tests\local_quant_research\test_turtle_vectorbt_engine.py -q +openspec validate build-turtle-etf-local-research-workflow --strict +``` + +Expected: PASS。 + +- [ ] **Step 7: 提交文档和执行身份** + +```powershell +git add -- docs/superpowers/specs/2026-07-16-turtle-full-position-redistribution-design.md docs/research/2026-07-13-turtle-etf-system-final-plan.md openspec/changes/build-turtle-etf-local-research-workflow joinquant/strategies/strategy-003/research/code-identity.json tests/local_quant_research/test_contract_fixtures.py tests/local_quant_research/test_turtle_vectorbt_engine.py +git diff --cached --name-only +git commit -m "文档：同步海龟全量再分配规格" +``` + +--- + +### Task 8: 完整验证、真实 11 ETF 基线与 Vibe 研究报告 + +**Files:** +- Create: `tests/local_quant_research/test_turtle_e2e.py` +- Modify: `tests/local_quant_research/test_turtle_single_scenario.py` +- Modify: `tests/test_skill_layout.py` +- Modify: `.build-and-verify/config.json` +- Create: `docs/research/2026-07-16-turtle-full-position-redistribution-baseline-report.md` +- Runtime only: `.local/quant-research/strategy-003/` 下由运行器生成的新不可变 run 目录 + +- [ ] **Step 1: 扩充发布入口 E2E 断言** + +新增 `test_turtle_e2e.py`：用真实共享行情存储接口在临时目录建立足够覆盖 55/20/20 窗口的小型 Parquet 快照，经通用 CLI 启动真实 `strategy-003` 入口，并断言只有一个场景、目标包包含标准四表和一份海龟归因、至少发生一次后到趋势触发的再分配、单位状态不被再分配改写、冷/热摘要一致、两次均小于 180 秒、基准工作目录已清理、最终 `next_action=return_to_caller`。保留 `test_generic_e2e.py` 作为非海龟通用前向验证，不把海龟规则写入其中。 + +同步 `.build-and-verify/config.json` 的 `verify.local-quant-research-e2e` 命令，使它同时运行 `test_generic_e2e.py` 与 `test_turtle_e2e.py`；先更新 `tests/test_skill_layout.py` 的精确命令断言。 + +- [ ] **Step 2: 运行 E2E 并修到通过** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research\test_generic_e2e.py tests\local_quant_research\test_turtle_e2e.py tests\local_quant_research\test_turtle_single_scenario.py tests\test_skill_layout.py -q +``` + +Expected: PASS。若失败，只修复本计划规则，不放宽断言、不切换回旧路径。 + +- [ ] **Step 3: 运行本地研究相关完整回归** + +```powershell +.\.venv\Scripts\python.exe -m pytest tests\local_quant_research tests\quant_analysis -q +.\.venv\Scripts\python.exe .build-and-verify\runtime\build_and_verify.py verify --project . +openspec validate --all --strict --no-interactive +``` + +Expected: 全部 PASS。`verify` 使用快速受影响检查；不运行未经用户额外授权的 `--full`。 + +- [ ] **Step 4: 从正式 Skill 用户入口运行唯一真实新基线** + +确认 `project-run.json` 指向 11 ETF 快照和新 `baseline.json` 后运行： + +```powershell +.\.venv\Scripts\python.exe scripts\research\local_quant_research\cli.py run --config joinquant\strategies\strategy-003\research\project-run.json +``` + +Expected: `complete`、`next_action=return_to_caller`；生成一个新不可变 run id（运行标识）；`performance.json` 显示 `result_match=true`、冷/热均小于 180 秒、临时目录全部删除。不得复用旧代码身份的历史结果，也不得运行旧基线对照。 + +- [ ] **Step 5: 用现有 Vibe 分析函数只读新标准结果包** + +从新运行的 `backtests/local-baseline` 读取标准四表和海龟归因，调用 `scripts.research.quant_analysis.unified_analysis` 现有的收益、风险、基准、持仓和贡献计算函数；基准固定为 `CSI300_CNY_TOTAL_RETURN`（沪深300人民币总收益）和 `NASDAQ100_CNY_TOTAL_RETURN`（纳斯达克100人民币总收益）。分析产物写入该 run 的独立 `.local` 分析目录，至少包含： + +- 累计收益、CAGR（复合年化收益率）、年化波动率、Sharpe（夏普比率）、Sortino（索提诺比率）、最大回撤、回撤持续期、Calmar（卡玛比率）； +- 对两个基准的累计超额、年化超额、Beta（贝塔）、Alpha（阿尔法）、相关性和共同样本说明； +- 平均/中位/最高仓位、低于半仓比例、接近满仓比例、现金比例、换手和费用； +- 逻辑单位、有效 N 单位、组/组合/现金缩放利用率、再分配次数、止损与趋势退出次数； +- 逐 ETF、逐资产组、逐时间段贡献和现金/费用残差； +- 最主要正面证据、反对证据、数据与公司行动近似限制； +- 明确写出“本次未运行稳健性矩阵，不能宣称稳健性通过”。 + +- [ ] **Step 6: 编写并核对完整研究报告** + +用 `apply_patch` 创建报告，引用真实 run id、快照摘要、代码身份、参数摘要、性能门禁和结果包路径。所有数字必须来自新标准结果包或 Vibe 输出，不手算、不猜测；结论必须给出“推荐/不推荐继续进入聚宽复核”的明确建议，但最终状态为“等待人工确认”，不得代替聚宽正式裁决。 + +- [ ] **Step 7: 清理临时产物并复核工作区** + +删除本次手工分析产生的临时脚本、临时 JSON 和非不可变工作目录；保留正式 `.local` run 和分析证据。运行： + +```powershell +git status --short +Get-ChildItem -Recurse -Force -Include *.tmp,.benchmark-work -Path .local,joinquant\strategies\strategy-003\research -ErrorAction SilentlyContinue +``` + +Expected: 没有本次临时残留；既有无关脏文件保持原样。 + +- [ ] **Step 8: 提交 E2E 与研究报告** + +```powershell +git add -- tests/local_quant_research/test_turtle_e2e.py tests/local_quant_research/test_turtle_single_scenario.py tests/test_skill_layout.py .build-and-verify/config.json docs/research/2026-07-16-turtle-full-position-redistribution-baseline-report.md +git diff --cached --name-only +git commit -m "报告：完成海龟全量再分配本地研究" +``` + +## Final Self-Review Checklist + +- [ ] 逐条对照已确认设计的 1 至 10 节，没有遗漏 11 ETF、6 分组、55/20/20、0.5N、2N、4/6/12、次日开盘、一天一单位和只上移止损。 +- [ ] 所有生产代码只存在一条全量再分配路径；没有 A1、Hamilton（最大余数）、资金仓位上限、协方差交易门槛或目标波动率交易残留。 +- [ ] 计划中的每个测试先失败后通过，命令和预期均明确，每个代码片段都能直接实现。 +- [ ] 回调、引擎、延迟执行器、结果适配器和分析器的动作名、字段名、数组形状一致。 +- [ ] 标准四表和聚宽现有结果读取契约没有改变；海龟专用信息只在归因扩展中增加。 +- [ ] 本地研究 Skill 仍只执行一个场景；实际只运行一个 11 ETF 新基线，没有旧方案对照、17 ETF 或稳健性矩阵。 +- [ ] 冷/热运行都小于 180 秒且结果摘要一致；完整用户入口 E2E 已通过。 +- [ ] 报告的收益、回撤、Alpha/Beta（阿尔法/贝塔）、仓位、风险预算和归因数字均能追溯到新 run 证据。 +- [ ] `.local` 历史证据未被改写或删除，临时产物已删除，无关工作区改动未被提交。 diff --git a/docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md b/docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md index 5b33be2..30d03c4 100644 --- a/docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md +++ b/docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md @@ -1,41 +1,130 @@ # 海龟 ETF 本地研究流程验证报告 +- 日期：2026-07-15 +- Change（变更）：`build-turtle-etf-local-research-workflow` +- 行情批次：`1923c902f5692d35bd84e2745620a06cb6c18666c4a4add724ce80d261d5f4e1` +- 行情快照：`e88238cca420a8ae66b90adb6cda4dd6c38a07390a13b8ac2f471e534742e33e` +- 准备标识：`ea78cf53997ad9e9db4e9bff473ae5ecfde67e42cca50a5084dadf19020263ba` +- 分析标识：`b76821272f792bafe2557b72988d505d3c5d0e166ddf5337fd70c23ffcd06942` +- 当前停止状态：`human_confirmation_required` + ## 结论 -本变更验证通过，可以进入归档确认阶段。本地研究流程、共享行情中心、海龟项目适配、真实数据研究和不可变证据均已完成；本地结论只建议进入 JoinQuant（聚宽）正式回测，不代表正式回测、稳健性或实盘准入通过。 +实现门禁通过，但当前策略研究结论未通过： + +- vectorbt（向量化回测框架）本地单场景研究、主 agent（代理）七次编排、聚宽现有结构兼容的标准分析数据包、完整确定性分析、Vibe-Trading（氛围量化）单体复核和报告均已真实运行。 +- 冻结基线累计收益 `117.44%`、年化收益 `5.87%`、最大回撤 `-12.07%`，Calmar（年化收益/最大回撤）为 `0.486`，低于 `0.5` 门槛。 +- 90 项证据中 52 项通过、38 项失败、0 项证据不足。推荐 `revise_and_reassess`，即修改后再评估。 +- `covariance-ewma-30d` 是唯一通过基础门槛的挑战场景，只能作为下一轮研究起点，不能自动替换、冻结或送入聚宽正式回测。 +- 本结果属于本地探索性研究，不是 JoinQuant（聚宽）正式回测、模拟交易或最终策略验收。 + +完整研究报告：[local-strategy-analysis-report.md](../../../.local/strategy-analysis/b76821272f792bafe2557b72988d505d3c5d0e166ddf5337fd70c23ffcd06942/local-strategy-analysis-report.md) + +## 公司行动与行情证据 + +| 验证项 | 实际结果 | +| --- | --- | +| 行情事实 | 23,938 行原始未复权日线，固化为 `market-data.parquet` | +| 公司行动事实 | 37 行聚宽 `finance.FUND_DIVIDEND` 记录，固化为 `corporate-actions.parquet` | +| 查询 | DuckDB（嵌入式分析数据库）只使用 `:memory:`，没有持久数据库副本 | +| 连续因子 | 只由实际应用日的“上一交易日原始收盘 / 当日原始前收盘”计算 | +| 元数据权限 | 公司行动元数据只授权和审计价格基准变化；官方拆分比例、每份现金不决定因子 | +| 时点边界 | 晚公布记录标为 `retrospective_reconciliation`；生效日停牌时延后到首个复牌行情日 | +| 取消状态 | 按取消日期与快照截止日重建；截止日后的取消不回写历史，缺少取消日期时停止导出 | +| 精度边界 | 连续经济价格、经济单位、除权日隐含再投资；不模拟支付日现金、税费、真实份额和碎股 | +| 失败关闭 | 未知状态、无效知识截止日或没有有效事件解释的价格基准变化会停止运行 | + +结果与报告明确声明 `point_in_time_total_return_approximation`，不能与聚宽逐日账户精确对账。 + +## 实现与职责边界 + +| 验证项 | 实际结果 | +| --- | --- | +| 执行内核 | vectorbt 1.1.0 官方 `Portfolio.from_order_func()` 路径，Numba（即时编译器）0.66.0 | +| 单场景 Skill（技能） | 每次只接受一个场景，输出一份兼容结果，以 `next_action=return_to_caller` 停止 | +| 主 agent 编排 | 冻结基线加六个挑战，恰好七次；Skill 内没有候选循环或分析逻辑 | +| 标准分析数据包 | 本地目录和清单尽量对齐聚宽现有 `backtests//` 结果；聚宽现有结果无需修改 | +| 聚宽合法例外 | `gate.status=pass` 时仅放行既有 `attribution_log:missing_at_source`；其他未知例外仍拒绝 | +| 策略分析 | 独立读取本地或聚宽来源，生成收益、回撤、双基准、归因、仓位、风险、稳健性、压力和 CVaR（条件风险价值） | +| 归因契约 | `attribution_log` 使用 `turtle-etf-attribution/2`，覆盖订单、风险状态和公司行动应用 | +| 流动性边界 | 最低成交额、订单参与率、单笔成交额占比和“成交额 1%”规则均已删除 | +| 资金上限 | 被动超限采用“不得恶化”；不冻结其他证券，不新增策略外强制退出 | +| 旧方案 | 旧逐日引擎、兼容层、专用报告和旧测试已删除，没有新旧双跑 | + +## 七次真实本地回测 + +每个场景都从公开 Skill 入口单独调用；冷启动包含首次 JIT（即时编译），预热在同一进程运行。冷/热规范化摘要一致，全部低于 180 秒。 + +| 场景 | `run_id` 前缀 | 累计收益 | 年化收益 | 最大回撤 | Calmar | 平均仓位 | 冷启动 | 预热 | 结论 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| baseline | `ad988e8a` | 117.44% | 5.87% | -12.07% | 0.486 | 44.07% | 30.94秒 | 5.29秒 | fail | +| entry-40 | `fdf89621` | 100.37% | 5.24% | -15.32% | 0.342 | 48.43% | 31.21秒 | 5.70秒 | fail | +| entry-60 | `d848d01e` | 106.49% | 5.47% | -13.44% | 0.407 | 43.47% | 30.72秒 | 5.21秒 | fail | +| stop-1-5n | `205b098f` | 97.36% | 5.12% | -13.01% | 0.394 | 41.39% | 30.02秒 | 5.04秒 | fail | +| stop-2-5n | `c2066eb2` | 133.99% | 6.44% | -13.69% | 0.471 | 47.81% | 30.31秒 | 5.51秒 | fail | +| covariance-120d | `03ae9ca9` | 88.25% | 4.75% | -13.39% | 0.355 | 42.21% | 30.09秒 | 5.03秒 | fail | +| covariance-ewma-30d | `50bcc27c` | 111.96% | 5.67% | -11.07% | 0.512 | 42.71% | 29.92秒 | 5.36秒 | pass | + +## 冻结基线结果 | 维度 | 结果 | -|---|---| -| 完整性 | 30/30 项 OpenSpec（开放规格）任务完成；14/14 项需求、59/59 个场景已核对 | -| 正确性 | 共享流程、行情中心、海龟规则、A1 分配和三类输出均有自动测试及端到端证据 | -| 一致性 | 实现遵循四层依赖、未复权行情、不可变 CSV（逗号分隔文件）批次/快照及项目层交易语义设计 | -| 代码审查 | Critical（严重）0，Important（重要）0 | +| --- | ---: | +| 累计收益 / 年化收益 | 117.44% / 5.87% | +| 最大回撤 / 最长回撤期 | -12.07% / 915 个观察日 | +| 年化波动率 | 6.89% | +| Sharpe（夏普比率） / Sortino（索提诺比率） | 0.863 / 1.252 | +| Calmar（卡玛比率） | 0.486，低于 0.5 门槛 | +| 平均仓位 / 中位仓位 | 44.07% / 34.67% | +| 低于半仓 / 接近满仓日期占比 | 55.19% / 11.36% | +| 平均现金 / 最高仓位 | 55.93% / 100.00% | +| 最高组合计划风险 / 风险预算 | 40.42% | +| 最高 60 日已实现波动率 | 15.77% | +| 高于目标波动率天数 | 440 天 | +| 成交订单 / 平仓订单 / 胜率 | 1,270 / 570 / 60.53% | +| 费用 | 22,417.34 元 | +| 保护止损 / 风险约束事件 | 175 / 4,009 | + +持有期存在单证券或资产组收盘权重高于入场上限的观察记录，但组合权重超限日期为 0。这是成交后市值诊断，价格上涨可以造成被动超限，不等同于下单风险门禁失效。 + +## 双基准与归因 + +双基准只使用策略、沪深300人民币总回报和纳斯达克100人民币总回报的三方共同交易日： + +| 基准 | 同期策略收益 | 基准收益 | 主动收益 | Alpha（超额收益） | Beta（市场暴露） | 相关性 | 信息比率 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 沪深300人民币总回报 | 90.51% | 87.66% | 2.85% | 4.26% | 0.1317 | 0.4075 | -0.0984 | +| 纳斯达克100人民币总回报 | 90.51% | 1,049.24% | -958.73% | 4.97% | 0.0109 | 0.0328 | -0.7283 | -## 验证证据 +归因为日度算术损益贡献，勾稽误差为 0；它不是几何链式归因，不能直接按复利累计收益逐项相加解释。 -- 项目测试：`345 passed`。 -- Build and Verify（构建与验证）完整门禁：`status: passed`，`full-not-run: false`；包含 110 项本地研究单元测试和 2 项公开入口端到端回归。 -- Skill（技能）结构：`quick_validate.py` 返回 `Skill is valid!`。 -- OpenSpec 严格校验：5 项通过、0 项失败；当前 change（变更）单独严格校验通过。 -- 公开仓库敏感数据扫描：Git 跟踪的 `.local/`、`market-data.csv` 和 `.duckdb` 文件数量为 0。 -- 临时产物检查：`.tmp`、`.inputs`、E2E（端到端）和 transfer（中转）残留数量为 0。 -- 真实 JoinQuant 日线：11 只 ETF（交易型开放式指数基金）、13 个字段、`fq=None`、截至 2026-07-13；快照 `64785f1607d90cf58f3db4545c2a718796659df818e283a4ec0614b0dfd12d8a`。 -- 最终本地运行：`b3649407660db998c823c9387ff861d09f52e73e1029859e917399985b5b48f1`，五阶段全部 `complete`（完整），第二次调用复用同一不可变运行。 -- 最终运行检查：3,432 条风险记录、492 笔成交、负现金 0、最低现金 310.4479815、最终冷启动证券为空。 -- A1 回归：覆盖负协方差分散和共同止损组合降险；最终复审确认硬阻断与非单调风险原因边界正确。 -- 分支处理：用户明确选择 PR Flow hotfix（拉取请求流程热修复）直推；首次推送后 `origin/main` 与实现提交均为 `5a9486af48a9ee8b2760ffbf04c43b3b58a90eac`。本报告及 Comet（彗星工作流）运行态作为验证收口提交后，使用同一门禁再次推送。 +## 稳健性、挑战与 Vibe 复核 -Comet 构建守卫只自动识别 npm、Maven 和 Cargo，不识别本仓库的 Python（编程语言）构建入口。本次先实际运行仓库 `Build and Verify --full` 并取得通过结果，再使用守卫提供的 `COMET_SKIP_BUILD=1` 跳过重复自动探测；没有跳过真实构建、测试或端到端验证。 +- 证据矩阵共 90 项：52 项通过、38 项失败、0 项证据不足。 +- 参数、时期、资产和资产组删除、成本与延迟、区块抽样、历史压力、持仓冲击和 CVaR 均已进入报告。 +- `covariance-ewma-30d` 通过基础门槛，但不能覆盖基线与其他场景的失败证据。 +- Vibe 0.1.10 通过公开单体 CLI（命令行入口）真实运行，运行标识 `20260715_203438_33_09ea02`，加载 `performance-attribution`、`risk-analysis`、`report-generate`。 +- Vibe 单体复核同意“保留研究价值、修改后再评估”；它只作定性审计，不改变任何确定性数值或门槛。 +- 已知有缺陷的 Vibe 群体分析、Vibe 回测和有前视偏差风险的组合优化器均未调用。 -## 已接受偏差 +## 自动验证 -- WARNING（警告）：提交区间包含 2026-07-14 04:00 自动同步产生的 `strategy-001`、`strategy-002` 模拟交易归档更新，而本 change 的设计要求海龟实现不得修改这两个策略。该更新不是海龟实现依赖或行为修改，已由 `joinquant-archive-sync`（聚宽归档同步）分别验证门禁通过，并按用户“全部提交”的明确要求以独立提交 `befc1e1` 纳入。影响限于独立远端归档快照和来源别名，不改变本地研究规格或结果。 +- 公司行动、行情、输入、引擎、结果适配、清单和报告首轮定向回归：`95 passed`；聚宽合法例外与取消状态修复定向回归：`21 passed`。 +- 项目 `.venv` 最终全仓测试：`445 passed`。 +- Build and Verify（构建与验证）最终完整门禁：`status=passed`、`full-not-run=false`；11 组检查全部通过，其中本地研究单元测试 `166 passed`，公开入口 E2E（端到端）`2 passed`。 +- OpenSpec（开放规格）严格校验：当前变更通过；全仓规格 `5 passed, 0 failed`。 +- 代码身份文件中的全部 SHA256（文件摘要）与当前实现一致。 +- 旧执行模块、旧引擎符号、流动性规则扫描为 0。 +- `.local` 下暂存 CSV、持久 DuckDB、失败 `.attempts`、预热副本和测试临时产物为 0；行情传输目录为空。 +- 独立审查先发现“聚宽合法例外误拒绝”和“取消状态可能回写未来”两个 IMPORTANT（重要阻断）；均按 TDD 修复并重建七次结果，复审为 `No findings`。规格对齐审查同样为 `No findings`。 +- 策略 001/002 的外部同步归档未被本变更编辑、删除或纳入结论。 -## 未验证边界 +## 未验证与禁止外推 -- 本 change 不运行 JoinQuant 正式回测或模拟交易。 -- 本地结果不验证正式收益、滑点、分红撮合、稳健性或实盘表现；这些内容属于后续独立 change。 +- 没有运行聚宽正式回测或模拟交易。 +- 没有精确还原公司行动后的真实 ETF 份额、支付日现金、税费、碎股和聚宽订单路径。 +- 没有接受候选参数，没有冻结策略。 +- 没有提交、推送、创建 PR（拉取请求）或合入主干。 -## 最终判断 +## 当前推荐 -CRITICAL（关键）0，WARNING（警告）1（已接受并限定范围），SUGGESTION（建议）0。无阻止归档确认的问题。 +保持 `candidate_accepted=false`，停止在 `human_confirmation_required`。建议人工确认是否以 `covariance-ewma-30d` 为下一轮“修改后再评估”的研究起点；确认不等于接受该候选，更不等于批准聚宽正式回测。 diff --git a/docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md b/docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md index 33286c2..4b96b98 100644 --- a/docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md +++ b/docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md @@ -4,31 +4,44 @@ role: technical-design canonical_spec: openspec --- -# 海龟 ETF 本地研究流程技术设计 +# 海龟 ETF 本地研究流程与聚宽原生分析数据技术设计（vectorbt 执行内核修订） ## 1. 设计边界 需求和验收场景以以下 OpenSpec（开放规格）增量规格为唯一事实源： - `openspec/changes/build-turtle-etf-local-research-workflow/specs/local-quant-research-workflow/spec.md` +- `openspec/changes/build-turtle-etf-local-research-workflow/specs/standard-strategy-analysis-data/spec.md` - `openspec/changes/build-turtle-etf-local-research-workflow/specs/turtle-etf-local-research/spec.md` 本文只说明实现结构、接口、数据流、错误处理和测试方法，不建立第二份需求规格。 -正式回测和模拟交易仍只在 JoinQuant（聚宽）云端运行。本地流程只验证规则、生成探索性研究证据并向后续聚宽回测交付固定候选包。`complete` 只表示本地流程完整执行，不代表策略通过正式回测、稳健性验收或实盘准入。 +整体架构固定为三个独立 Skill（技能）：本地研究流程、JoinQuant（聚宽）回测流程、策略分析。标准分析数据以仓库现有聚宽回测归档为物理基准：聚宽现有目录 0 改动直读，本地 vectorbt（向量化回测框架）结果生成聚宽同名的四类共同执行事实；聚宽官方 `risk` 和 `period_risks` 作为来源参考，不要求本地流程预先计算。本变更主要实现本地研究流程和统一分析读取能力；聚宽回测流程与策略分析 Skill 另立变更。 + +2026-07-14 的真实行情运行表明，旧 Python（编程语言）逐日内核在 A1（同日共享预算分配）和组合风险可行性复算处连续触发 10、30 和 60 分钟超时。性能剖析已把主要耗时定位到 `process_day → allocate_a1 → _maximum_hamilton_allocation → evaluate_risk`。本次修订不改变策略规则和行情口径，只把本地模拟内核迁移到 vectorbt 官方 `Portfolio.from_order_func()`（自定义订单函数）。单次冷/热计时均从已准备输入进入 vectorbt 开始，到交易执行、四类共同事实、海龟必需归因日志及其结构/摘要/勾稽校验完成时停止，均限制在 180 秒内；停止计时后才写入 `performance.json` 和最终清单。独立确定性分析耗时单独记录，不计入该门禁。 ## 2. 发布结构与依赖方向 -实现采用四层单向依赖： +实现采用单向依赖，并以聚宽原生结果读取契约隔离三个流程： ```text -run-local-quant-research Skill（流程编排） - ↓ -仓库共用脚本（行情中心、运行器、证据） - ↓ -strategy-003 项目适配器与海龟纯计算模块 - ↓ -.local 共享行情与策略运行证据 +run-local-quant-research Skill（本变更） + → 一次只接收一个场景 + → 行情中心、通用运行器、strategy-003 适配器 + → vectorbt 官方 Portfolio.from_order_func + → .local/quant-research/strategy-003//backtests// + → next_action=return_to_caller + +run-joinquant-backtest Skill（后续变更） + → 保持现有聚宽回测与归档产物，不增加适配或转换步骤 + +strategy-analysis Skill（后续变更） + ← 统一读取现有聚宽目录或本地聚宽口径兼容目录 + +本变更独立确定性策略分析 + ← 主 agent 多次调用 Skill 得到的独立兼容结果 + ← strategy-003 自有 analysis-plan.json 展开的场景、双基准集与完整稳健性结果 + → .local/strategy-analysis// ``` 建议目录如下： @@ -51,6 +64,23 @@ scripts/research/ │ ├── query.py │ ├── joinquant_export.py │ └── cli.py +├── analysis_data/ +│ ├── __init__.py +│ ├── manifest.py +│ ├── views.py +│ ├── derived.py +│ ├── cli.py +│ └── schemas/local-backtest-manifest.schema.json +├── quant_analysis/ +│ ├── metrics.py +│ ├── benchmarks.py +│ ├── attribution.py +│ ├── robustness.py +│ ├── stress.py +│ ├── cvar.py +│ ├── evidence.py +│ ├── cli.py +│ └── schemas/analysis-plan.schema.json └── local_quant_research/ ├── __init__.py ├── contract.py @@ -64,16 +94,15 @@ joinquant/strategies/strategy-003/ ├── manifest.json └── research/ ├── project.json + ├── analysis-plan.json ├── adapter.py ├── export-request.json - ├── turtle/ + ├── turtle_etf/ │ ├── indicators.py - │ ├── signals.py - │ ├── state.py - │ ├── risk.py - │ ├── allocation.py - │ ├── execution.py - │ └── reporting.py + │ ├── vectorbt_inputs.py + │ ├── vectorbt_callbacks.py + │ ├── vectorbt_engine.py + │ └── vectorbt_adapter.py └── fixtures/ tests/ @@ -82,7 +111,9 @@ tests/ └── strategy_003/ ``` -Skill（技能）目录不保存 Python 实现、行情、海龟参数或测试数据，只描述调用顺序和停止条件。共用脚本不导入 `strategy-003`；项目适配器可以调用共用行情读取接口，但共用层不得反向解释海龟字段。 +Skill（技能）目录不保存 Python 实现、行情、海龟参数或测试数据，只描述一次单场景调用顺序和停止条件。`analysis_data` 只负责读取现有聚宽归档、建立六表内存视图、校验和派生查询；它不导入 `strategy-003`、vectorbt 或分析算法，也不写回聚宽目录。`quant_analysis` 是独立策略分析能力的算法实现，只按通用 Schema（结构约束）校验和展开策略自有 `analysis-plan.json`；本地研究运行器和海龟项目不得导入。项目适配器只负责把本地执行事实适配为聚宽口径结果。 + +共享行情中心另在 `.local/market-data/benchmark-sets//` 保存双基准清单和 `benchmark-returns.parquet`。该基准集与策略、执行引擎和来源回测目录解耦；聚宽现有单基准收益只作来源参考。 不建设 Provider（数据提供方）插件框架、后台服务、持久数据库或第四个行情 Skill。以后新增来源或频率时，以新的明确变更扩展共用契约。 @@ -109,7 +140,8 @@ Skill（技能）目录不保存 Python 实现、行情、海龟参数或测试 .local/market-data/ ├── batches// │ ├── manifest.json -│ ├── market-data.csv +│ ├── market-data.parquet +│ ├── corporate-actions.parquet │ └── validation.json └── snapshots/.json ``` @@ -118,17 +150,19 @@ Skill（技能）目录不保存 Python 实现、行情、海龟参数或测试 ### 4.2 批次身份 -`market-data.csv` 是从来源精确保留的原始导出，也是唯一行情事实源。`batch_id` 由规范化来源身份、导出契约和 CSV 字节 SHA256（文件摘要）计算；清单至少记录： +`market-data.parquet` 保存原始未复权日线，`corporate-actions.parquet` 保存经来源核验且保留版本语义的拆分、现金分红等公司行动；两者共同构成本地行情事实。连续总回报价格只在查询或输入构造时用生效日可见的原始 `close/pre_close` 行情事实派生；公司行动元数据只用于核对，晚公布记录必须标为事后核对，不回写此前历史，也不固化成第二份行情。聚宽 CSV（逗号分隔文件）只存在于远端传输和本地隐藏暂存目录，转换与验证完成后删除。`batch_id` 由规范化来源身份、结构版本、导出契约和两类规范化逻辑内容摘要计算，不直接依赖 Parquet 编码字节；清单至少记录： - schema version（结构版本）； - `source`、`asset_type`（标的类型）、`frequency=1d`； - 证券列表和每只证券实际起止日； - 字段顺序、时区、交易日历和价格口径； - `snapshot_end_date`（快照截止日）； -- 导出代码摘要、CSV 字节摘要、行数和规范化内容摘要； +- 导出代码摘要、远端与本地传输 CSV 字节摘要、Parquet 字节摘要、行数和规范化内容摘要； +- Parquet 写入器、DuckDB 和结构版本； +- 公司行动来源事件标识、类型、公告日、登记日/除权日/生效日/支付日、拆分比例或每份现金、状态、知识截止日、来源身份与摘要； - 创建时间、验证状态和验证器版本。 -同一来源身份和 CSV 字节摘要完全相同时复用已有批次。新标的或新日期可以追加新批次。相同来源、频率、证券和日期出现不同值或不同价格口径时拒绝导入；首版不实现自动修订、覆盖或 `supersedes`（取代关系）。 +同一来源身份、结构版本和规范化逻辑内容摘要完全相同时复用已有批次。Parquet 字节摘要用于完整性验证，但写入器版本造成的无语义字节变化不能产生两套逻辑事实。新标的或新日期可以追加新批次。相同来源、频率、证券和日期出现不同值或不同价格口径时拒绝导入；首版不实现自动修订、覆盖或 `supersedes`（取代关系）。 ### 4.3 快照身份 @@ -138,9 +172,9 @@ Skill（技能）目录不保存 Python 实现、行情、海龟参数或测试 ### 4.4 DuckDB 查询层 -查询进程使用 `duckdb.connect(':memory:')` 从快照引用的权威 CSV 建视图。查询层统一字段顺序、数据类型、空值、日期、证券排序，并把聚宽返回的 `paused` 从数值规范化为布尔值；规范化结果摘要必须与 CSV 规范化摘要一致。 +查询进程使用 `duckdb.connect(':memory:')` 和 `read_parquet` 从快照引用的权威 Parquet 建视图。查询层统一字段顺序、数据类型、空值、日期、证券排序，并把聚宽返回的 `paused` 从数值规范化为布尔值；规范化结果摘要必须与批次清单的逻辑内容摘要一致。 -`.local/market-data/` 不保存 `.duckdb` 文件。DuckDB（嵌入式分析数据库）视图可以随时仅凭快照清单和权威 CSV 重建，不构成第二事实源。 +`.local/market-data/` 不保存 `.duckdb` 文件。DuckDB（嵌入式分析数据库）视图可以随时仅凭快照清单和权威 Parquet 重建，不构成第二事实源。 首版只实现日线行情。通用清单预留来源、标的类型、频率和显式字段能力，但分钟线、基本面、财务和因子请求直接报告不支持，不自动降级成日线。 @@ -162,9 +196,9 @@ volume, money, factor, paused, high_limit, low_limit 每只 ETF 从自身首个可用完整交易日导出到显式 `snapshot_end_date`。2015-01-01 之前数据只用于指标预热；未满 60 个完整、有效且日期对齐样本的 ETF 可以存在于快照中，但不能新增风险。 -CSV 保存未复权实际价格与 `factor`（复权因子），本地研究不生成或使用复权价。供后续聚宽功能校验的策略信号同样显式使用 `fq=None`，含场内基金的策略设置 `use_real_price=False`。报告必须说明该模式的撮合价使用聚宽固定基准日前复权行为，不能把它描述为未复权实际成交价。 +传输 CSV 与固化 Parquet 保存原始未复权实际价格、`pre_close`（前收盘参考价）与 `factor`（复权因子），只作为行情事实和审计依据。本地 vectorbt 只按实际应用日可见的 `上一交易日原始 close / 当日 pre_close` 更新从该日生效的累计连续因子，并将同一因子应用于当日及以后 OHLC 与 `pre_close`；生效日停牌时应用日延后到首个复牌行情日。公司行动元数据只用于授权和审计价格基准变化：公告日在生效日之前或当日标为 `point_in_time`，晚于生效日标为 `retrospective_reconciliation`；取消状态按取消日期与快照截止日重建，截止日后的取消不得回写历史，当前显示已取消却缺少取消日期时停止导出；官方拆分比例或每份现金均不决定因子幅度或订单，有效事件未出现价格基准变化时只记录审计。突破、N 值、协方差、波动率、vectorbt 成交与估值统一使用连续经济价格和经济单位；现金分红按除权日隐含再投资近似，不在支付日另增现金。公司行动来源必须通过真实接口与真实事件最小验证；取消事件不得授权变化，若状态未知、知识截止日无效，或 `pre_close` 与前一交易日收盘的价格基准变化没有对应有效事件，则批次或运行以 `evidence_insufficient`（证据不足）停止。供后续聚宽功能校验的策略信号同样显式使用 `fq=None`，含场内基金的策略设置 `use_real_price=False`。报告必须分别说明本地研究近似与聚宽撮合限制，不能把两者描述成逐日账户精确对账。 -远端文件只是传输中转。本地收到文件后先与远端回读字节 SHA256 比较，再导入批次并删除远端文件。任何一项摘要不一致或无法确认远端清理都输出 `failed`。实现不得保存或打印账号、密码、Token（访问令牌）或 Cookie（浏览器凭证）。 +远端文件和本地 CSV 都只是传输中转。本地收到文件后先与远端回读字节 SHA256 比较，再在隐藏暂存目录执行固定结构解析、Parquet 转换、规范化内容摘要和 DuckDB 回读复核；随后先删除两端 CSV 并确认清理，再把清理结果写入批次清单，最后只原子发布已整理的 Parquet 批次。发布后不再依赖清理动作。任何一项摘要不一致、转换失败或无法确认清理都输出 `failed`。实现不得保存或打印账号、密码、Token（访问令牌）或 Cookie（浏览器凭证）。 ## 6. 通用运行器接口 @@ -186,12 +220,14 @@ run --config ```text 配置与路径校验 -→ 快照和 CSV 摘要校验 +→ 快照、Parquet 和逻辑内容摘要校验 → 内存 DuckDB 视图同源校验 → 创建隔离暂存目录 → 以参数数组调用项目适配器 -→ 校验项目输出结构与摘要 -→ 固化运行清单和唯一状态 +→ 对同一场景完成冷启动与预热执行、结果摘要一致性和180秒门禁 +→ 用聚宽原生结果读取器校验一份本地兼容结果与 performance.json +→ 全部门禁通过后原子固化单场景清单、唯一权威结果和状态 +→ 输出 next_action=return_to_caller 后停止 ``` 项目适配器接收经过验证的快照清单路径、项目配置路径和暂存输出目录，并通过共用 `market_data` Python 接口读取数据。适配器不能写出暂存目录，也不能要求共用运行器解释策略字段。 @@ -201,85 +237,165 @@ run --config `run_id` 由以下规范化摘要计算： ```text -snapshot manifest + project config + declared code identity +snapshot manifest + project config + normalized scenario config + declared code/backend identity ``` 策略成功证据位于： ```text .local/quant-research/strategy-003// +└── backtests/ + └── / + ├── manifest.json + ├── code.py + ├── params.json + ├── params_versions/.json + ├── performance.json + └── data/ + ├── results.parquet + ├── balances.parquet + ├── positions.parquet + ├── orders.parquet + └── attribution_log-.parquet（strategy-003 必需扩展） ``` -项目先在同一文件系统的隐藏暂存目录生成全部产物。输入、进程、必需文件、JSON 结构和所有摘要通过后，运行器才使用原子目录替换固化 ``。失败尝试只保留紧凑 attempt manifest（尝试清单）和诊断，不留下可被误认为完成的运行目录。 +从 `backtests//` 向内，文件位置、名称和数据集清单结构尽量镜像聚宽现有 `backtests//`。本地不存在真实聚宽详情页、远端响应和官方计算结果，因此不伪造 `raw/`、`research_response`、`collection_fence`、`official-summary.csv`、`risk.parquet` 或 `period_risks.parquet`；本地清单固定 `schema_version=local-backtest/1`、`object.kind=local_backtest`、`source.kind=local_vectorbt`、`authority=local_research`，并由独立本地 Schema 校验。项目先在同一文件系统的隐藏暂存目录生成同一场景的冷启动和预热产物并比较规范化结果摘要；通过后保留冷启动权威候选，删除预热副本和可丢弃暂存并确认清理，再把清理结果写入 `performance.json`、生成最终清单并校验全部摘要。运行器最后只使用原子目录替换固化已整理的一份权威 ``，发布后不依赖写入或清理。失败尝试只保留紧凑 attempt manifest（尝试清单）和诊断，不留下可被误认为完成的运行目录。 + +同一 `run_id` 已有 `complete` 证据时先重新校验：全部通过则复用，任何输出摘要差异都视为确定性冲突并输出 `failed`。快照、项目配置、单场景配置、代码或后端身份任一变化都会生成新 `run_id`。失败重试使用新的 `attempt_id`，不覆盖前次尝试或既有完整运行。 + +独立策略分析证据不写回不可变本地研究运行目录，而保存在： -同一 `run_id` 已有 `complete` 证据时先重新校验：全部通过则复用，任何输出摘要差异都视为确定性冲突并输出 `failed`。快照、配置或代码变化会生成新 `run_id`。失败重试使用新的 `attempt_id`，不覆盖前次尝试或既有完整运行。 +```text +.local/strategy-analysis// +``` + +每个 `run_id` 只绑定一个场景。调用前先在 `.local/strategy-analysis-preparations//` 固化分析计划、基准集、运行模板和七份单场景配置。主 agent（代理）分别调用 Skill 后，显式登记七组 `scenario_id -> run_id`；分析入口不得扫描目录猜测来源，必须校验运行唯一、同一快照、同一代码身份与执行后端，再由准备身份和全部来源摘要派生最终 `.local/strategy-analysis//source-results.json`。`analysis_id` 绑定全部基础/稳健性来源、双基准集、分析配置和可选 Vibe-Trading（氛围量化）审计身份，同计划下的另一批来源不得覆盖旧证据。分析完成、失败或证据不足都不得修改任何来源 `run_id`、结果或本地流程状态。后续人工决定绑定 `analysis_id`、完整报告和 `recommendation.json` 摘要，属于独立策略分析流程的门禁。 -## 8. 海龟项目模块 +## 8. 海龟项目模块与 vectorbt 边界 -海龟项目保持小型纯计算模块： +项目层保留海龟规则，vectorbt 只接管模拟与账户记账： | 模块 | 责任 | 主要不变量 | |---|---|---| -| `indicators.py` | TR（真实波幅）、N 值、突破通道、收益率和协方差输入 | 通道排除当日；不使用复权价 | -| `signals.py` | 55 日入场、20 日退出、0.5N 加仓档位 | 收盘确认；同一 ETF 每日最多一次加仓 | -| `state.py` | 批次、固定信号日 N、理论档位和共同止损状态 | 实际成交才改变状态；共同止损只上移 | -| `risk.py` | U0、资金、流动性、单 ETF、资产组、计划风险和目标波动率 | 现金不为负；所有硬上限不突破 | -| `allocation.py` | A1 等比分配、整手取整和余额补分 | 输入顺序不改变结果；同分按代码升序 | -| `execution.py` | 次日开盘订单、成交夹具、退出和强制减仓 | 不虚构成交；退出优先 | -| `reporting.py` | 审计、指标、报告、研究建议和候选清单 | 所有输出绑定运行身份和摘要 | +| `indicators.py` | 以连续总回报经济价格计算 TR（真实波幅）、N 值、突破通道、收益率和协方差输入 | 通道排除当日；协方差收益使用连续 `close / pre_close - 1`；原始行情只作审计 | +| `vectorbt_inputs.py` | 把 DuckDB 查询结果、原始行情事实、公司行动核对元数据、累计连续因子、指标和 55 日入场/20 日退出/0.5N 加仓规则转为日期 × ETF 的只读 NumPy 数组 | 因子只依赖生效日可见行情；晚公布元数据只标记事后核对；不回写事件前历史；收盘信息显式错位；不含成交额或流动性参数；稳定类型与排序 | +| `vectorbt_callbacks.py` | 唯一的 Numba 数值状态、A1、风险和四类官方回调实现 | `nopython`（无 Python 模式）；不创建账户或交易记录；不保留 Python 兼容实现 | +| `vectorbt_engine.py` | 固定 vectorbt 参数并调用官方 `Portfolio.from_order_func()` | 单一共享现金组；普通订单函数；不直接调用私有内核 | +| `vectorbt_adapter.py` | 把 `Portfolio`（组合）记录转换为 `results`、`balances`、`positions`、`orders`、本地清单和海龟必需归因日志 | 清单声明连续经济价格、经济单位、隐含再投资及不能精确对账聚宽；不生成聚宽官方风险摘要；不计算分析结论 | + +`vectorbt_inputs.py` 输出不可变 `SimulationInputs`，至少包含日期、证券、原始开高低收、原始前收盘、连续经济开高低收、连续前收盘、累计连续因子、公司行动有效掩码与摘要、成交量、停牌、涨跌停、已错位信号、N 值、协方差、资产组和成本数组。它不得包含成交额、最低成交额、订单参与率或任何流动性参数。输入构造可以使用 Pandas（数据处理）与 NumPy，但回调只接收连续数值数组、标量和预分配状态，不接收 DataFrame（数据表）、Decimal（高精度小数）、字典或 Python 对象。 + +公司行动在进入 vectorbt 前由输入构造器处理。拆分与现金分红都通过从应用日开始生效的连续总回报因子进入连续经济 OHLC，因子幅度只来自当时可见的原始 `close/pre_close`；经济单位在 vectorbt 内保持稳定；现金分红等价于除权日隐含再投资，不在支付日增加共享现金，避免重复计入。归因日志记录事件标识、类型、应用日、官方比例或每份现金、`evidence_timing=point_in_time|retrospective_reconciliation`、累计连续因子和来源摘要，但不伪造订单、现金流或真实份额变更。代码身份与本地清单必须固定 `corporate_action_mode=point_in_time_total_return_approximation`、`continuity_factor_basis=raw_previous_close_over_current_pre_close`、`corporate_action_metadata_timing=audit_only_may_be_retrospective`、`price_basis=continuous_economic_price`、`quantity_basis=economic_units`、`cash_dividend_mode=implicit_reinvestment_on_ex_date`、`pay_date_cash_supported=false` 和 `exact_joinquant_reconciliation=false`。这是研究级收益近似，不是聚宽账户复刻，也不声称全部事件元数据在生效日已知。 -模块之间使用明确记录对象，不读取全局路径或环境变量。报告模块只消费确定性结果，不能反向修改信号、参数或资产池。 +海龟策略完全不执行流动性判断。共享行情中的 `money`（成交额）可以被 ETF 池筛选或其他策略使用，但海龟配置、输入、回调和原因码均不读取它；成交额为空、极低或极高时，相同价格与状态输入必须生成完全相同的订单和结果。 -## 9. 每日状态流与 A1 分配 +项目固定 vectorbt 1.1.0。其兼容要求会推动项目 `.venv` 中 NumPy 与 Pandas 的小版本升级，因此实施第一步必须锁定依赖并运行全仓回归。代码身份增加 `execution_backend=vectorbt.from_order_func`、vectorbt/Numba/NumPy/Pandas 版本、回调摘要和输出适配器版本。依赖审计同时记录 vectorbt 的 Apache 2.0 + Commons Clause（商业销售限制条款）许可；本变更只用于仓库内部研究，不把 vectorbt 打包为对外销售的回测产品或服务。通用 Skill、行情中心、标准数据契约、量化分析和非海龟夹具不导入 vectorbt。 -每日处理顺序固定为： +实施过程中先从现有代码提取已经确认的信号、A1、风险和状态规则，迁入 `vectorbt_inputs.py` 与 `vectorbt_callbacks.py`，再由规格夹具直接声明预期订单、成交、状态和风险结果。新路径通过后，删除旧 `execution.py`、`state.py`、`signals.py`、`risk.py`、`allocation.py`、直接耦合分析算法的 `reporting.py`，以及只服务这些模块的测试和公开导出；不得保留兼容模块、别名、双引擎开关、运行时回退或无效代码。`indicators.py` 仅在新输入路径实际使用时保留。 + +## 9. 官方回调生命周期、每日状态流与 A1 + +所有 ETF 设置为一个 vectorbt 组合组并启用 `cash_sharing=True`（共享现金）。现行规则每只 ETF 每个交易日最多一笔最终订单，使用普通 `Portfolio.from_order_func()`；不启用 `flexible=True`（灵活多订单）。官方回调生命周期固定为： ```text -T 日收盘更新指标与信号 -→ T+1 日开盘处理全仓退出 -→ 处理强制风险减仓 -→ 汇总同级的新建仓与加仓候选 -→ A1 分配资金和风险预算 -→ 应用整手与成交约束 -→ 更新批次、现金、持仓、共同止损和审计 +prepare_simulation_inputs +→ 把 T 日收盘信号和滚动统计错位到 T+1 执行行 +→ pre_sim_func_nb：分配批次、止损、原因码、候选和订单临时数组 +→ pre_segment_func_nb：读取当日开盘前状态，分类退出、强制减仓和买入候选，设置调用顺序 +→ order_func_nb：先返回退出和风险减仓订单 +→ 第一笔买单前：按实际卖出后的现金与持仓只运行一次 allocate_a1_nb +→ order_func_nb：为各买入候选返回最多一笔订单 +→ post_order_func_nb：只按实际成交更新海龟状态和审计 +→ vectorbt：固化订单、交易、持仓、现金、费用和权益记录 +→ vectorbt_adapter：转换为四类共同执行事实、本地清单和海龟必需归因日志 ``` -同一 ETF 当日出现退出时取消它的所有买入候选。每个有效建仓或加仓候选先产生最多一个 U0（标准单位）的请求量。A1 对所有仍可行候选使用同一完成比例，直到资金、单 ETF、资产组、组合计划风险和目标波动率全部满足；候选被自身或所属组上限卡住后，未用预算可以流向其他可行候选。 +`pre_segment_func_nb` 可以查看整个共享现金组并改写当日 `call_seq`（调用顺序），但不能提前假定卖单会成交。它只分类候选和排序。`order_func_nb` 处理完退出与风险减仓后，在首个买入列到达时读取 vectorbt 当前现金和持仓，调用一次 `allocate_a1_nb`，把所有买入数量写入预分配数组；后续买入列只读取结果。这样既保留“卖出实际成交后再分配”的原规则，也避免每个 ETF 重算 A1。 + +A1 仍执行同一完成比例、整手向下取整、Hamilton 余额补分和 ETF 代码同分规则。Numba 内核使用连续数组、预计算的当前敞口、资产组映射和协方差矩阵复用不变量；不得通过修改风险公式或假定组合波动率单调来换取速度。每个候选补一手时仍复核资金、单 ETF、资产组、组合计划风险和目标波动率硬门槛。单 ETF 或资产组因价格变化被动超过资金上限时采用“不得恶化”判断：不强制卖出，不冻结其他证券，只禁止增加同一超限证券或同一超限组；退出、止损和风险降低订单始终允许。 -缩放结果先向下取整到交易整手。剩余预算按小数余额从大到小逐手补分，每补一手重新检查全部硬门槛，完全同分时按 ETF 代码升序。算法不得依赖输入列表顺序。 +市场可交易性由项目回调判断：停牌不下单；买入开盘触及不可买上限时拒绝；卖出开盘触及不可卖下限时拒绝；退出取消同一 ETF 当日买入。订单价格为次日开盘价，费用和滑点交给 vectorbt 官方订单处理。`post_order_func_nb` 读取 `order_result`（订单结果），只有实际成交才更新固定信号日 N、理论档位、共同止损和批次审计；拒单、无订单和未成交不推进状态。 停牌是正常市场状态，不伪造成交。未满 60 个样本只禁止对应 ETF 新增风险。任一持仓 ETF 缺少可用价格或风险输入时暂停整个组合新增风险，不填零、不使用陈旧协方差；其他可交易 ETF 仍允许退出和强制减仓。 -## 10. 项目输出契约 +执行语义直接按 OpenSpec（开放规格）场景和固定小型合成夹具验证逐日订单、成交、现金、持仓、批次、共同止损、风险原因和最终摘要。所有夹具通过后，`cli.py` 每次只把传入的一个场景交给 `run_vectorbt_simulation`，拒绝候选数组和内部循环，随后删除旧执行与报告模块和专用测试。删除后以全仓扫描和公开入口 E2E（端到端）证明 `process_day`、旧 `_simulate`、旧对象导入、分析算法导入和兼容路径均不存在。 + +## 10. 聚宽原生分析数据与独立策略分析验证 + +### 10.1 现有归档是物理基准 -一个完整海龟运行至少输出： +2026-07-14 对仓库的只读核对覆盖 120 个 `gate.status=pass` 的聚宽回测清单：全部拥有同一组数据集。125 份核心 Parquet 中，`results` 与 `period_risks` 结构完全一致；其他表的差异只涉及列顺序、`cancel_time` 的空类型/字符串表现，以及少量风险数值的整数/浮点表现。基于该事实，核心分析输入固定为聚宽现有六表： ```text -run-manifest.json -daily-audit.csv -trades.csv -positions.csv -risk.csv -research-report.md -conclusion.json -candidate-strategies.json +data/results.parquet +data/balances.parquet +data/positions.parquet +data/orders.parquet +data/risk.parquet +data/period_risks.parquet ``` -`research-report.md` 包含方法、快照/配置/代码身份、事件和交易结果、实际仓位分布、现金占比、留现原因、资产组和组合风险使用率、限制及产物摘要。设计期 9 只 ETF 的 63.7% 和 55.7% 代理仓位只用于说明研究必要性，不得冒充最终 11 只 ETF + 完整规则结果。 +`results` 提供策略和聚宽单基准的累计收益；`balances` 提供总资产、净值和现金；`positions` 与 `orders` 提供持仓、成交、费用和完整往返交易重建输入；`risk` 与 `period_risks` 提供聚宽官方风险摘要。只读实测确认聚宽 `results` 为 `time:string`、`returns:double`、`benchmark_returns:double`。统一读取器把时间规范化为 Asia/Shanghai（亚洲/上海）交易日，并按相邻累计净值比在查询期派生单日收益；双基准文件保存单日人民币总回报，只在共同有效交易日比较单日序列，不得把来源累计收益直接与基准单日收益比较。权益、收益、双基准、完整往返交易和事件等分析表只作为 DuckDB（嵌入式分析数据库）查询期派生视图，不再要求物理八表。 + +### 10.2 聚宽结果 0 改动直读 + +聚宽标准输入就是现有 `joinquant/strategies//backtests//`。读取器先验证原 `manifest.json`、对象身份、`gate.status=pass`、数据集状态、文件摘要与行数，然后直接读取现有 Parquet。不得新增 `analysis-data-manifest.json`、转换后目录、八表副本或回写字段；聚宽回测与归档流程不调用本变更的新接口。 + +清单中 `positions` 或 `orders` 合法声明 `status=complete`、`rows=0`、`verified_empty=true` 时，可以没有对应 Parquet；读取器在内存建立固定字段空视图。物理列顺序、`cancel_time` 的 Arrow（列式内存格式）空类型/字符串差异，以及兼容风险数值类型只在 DuckDB 内存查询中按字段名规范化，源文件和摘要保持不变。 + +`data/official-summary.csv` 继续作为聚宽页面展示口径的交叉校验证据，不替代六表高精度明细，也不进入核心分析计算。`attribution_log-.parquet`、`records`、日志、性能剖析和 `raw/` 保持可选扩展或归档证据，不要求所有策略共享字段。 -`conclusion.json` 的项目建议与流程状态分离。建议值固定为： +### 10.3 本地结果向聚宽口径适配 + +本地 vectorbt 适配器承担全部兼容成本：每个场景写入 `.local/quant-research/strategy-003//backtests//`，从该层向内镜像聚宽现有 `manifest.json`、`code.py`、`params.json`、`params_versions/` 与 `data/` 布局。`results`、`balances`、`positions`、`orders` 的文件位置、字段名称、时间含义、方向、数量、价格、费用、现金和持仓语义对齐现有聚宽结果。 + +本地清单沿用聚宽顶层概念和数据集条目的组织方式，但使用独立 `local-backtest-manifest.schema.json`，固定 `schema_version=local-backtest/1`、`object.kind=local_backtest`、`source.kind=local_vectorbt`、`authority=local_research`。读取器按 `schema_version` 严格选择聚宽或本地 Schema，未知版本、混合字段或失败后回退一律拒绝。清单把 `risk` 与 `period_risks` 标为 `required=false`、`status=missing_at_source`、`reason=computed_by_strategy_analysis`，避免本地流程计算 Alpha/Beta、Sharpe、回撤或分期风险。本地 `results.parquet` 保留与聚宽一致的三个物理字段：`returns` 为累计净收益，`benchmark_returns` 为全空但保持 `double` 的来源缺失参考；清单记录 `missing_at_source/independent_benchmark_set`，禁止填零或选择任一双基准冒充。海龟专属状态、风险原因、公司行动应用记录和事件必须写入 `data/attribution_log-.parquet`；其 `event_id` 为唯一主键，字段和 `turtle-etf-attribution/2` 原因码受项目契约约束，缺失或无法覆盖实际订单、风险变化及公司行动应用时阻止完成。没有真实来源的聚宽 URL、远端原始响应、围栏、官方摘要、官方风险文件和 `raw/` 一律不生成。 + +### 10.4 单场景 Skill 与主代理复数调用 + +本地研究 Skill 每次只运行一个明确场景，并产生一份聚宽口径兼容结果与单场景冷/热性能证据；完成后输出 `complete` 与 `next_action=return_to_caller`。它不接收候选数组、不知道七方案数量、不生成聚合清单，也不生成绩效、Alpha/Beta（超额收益/市场暴露）、归因、稳健性矩阵、挑战筛选、报告、推荐或人工决定。 + +冻结基线和六项预设挑战先在 `preparation_id` 下展开，再由主 agent 从策略自有 `analysis-plan.json` 读取后分别调用 Skill 七次。主 agent 显式登记七个 `run_id`；分析入口验证同一代码摘要、同一 `snapshot_id`、同一 vectorbt 执行后端、七个独立场景身份、结果摘要和性能证据，再派生独立 `analysis_id` 并生成 `source-results.json`。增加、删除或重排场景只改变分析计划和主 agent 调用列表，不改变 Skill 契约。 + +一次本地研究调用至少输出： ```text -proceed_to_joinquant 进入聚宽回测 -revise_and_reassess 修订后再评估 -stop_evidence_insufficient 证据不足而停止 +backtests// + manifest.json + code.py + params.json + params_versions/ + data/<四类共同事实 Parquet> ``` -它必须列出确定性理由、阻断项、证据摘要和“不是正式回测或最终验收结论”的声明。 +### 10.5 本次独立确定性分析、Vibe 安全边界与人工确认 + +本变更由 `strategy-003/research/analysis-plan.json` 以 `strategy-analysis-plan/1` 机器可读定义一个冻结基线、六个挑战和完整稳健性矩阵；通用 `quant_analysis` 只按 `analysis-plan.schema.json` 校验和展开，不解析 Markdown、不导入海龟模块、不硬编码海龟资产、参数、分组、数量和门槛。本地研究 Skill 不读取该计划。主 agent 先创建 `preparation_id`，从计划读取七个基础场景并分别调用 Skill；七份来源明确登记并通过共享身份校验后才创建最终 `analysis_id`。固定时期与季度滚动窗口使用基线既有路径切片；逐 ETF 和逐资产组删除使用贡献删除且不重新分配资金；成本和延迟使用一阶订单级敏感性；区块抽样、历史压力、持仓冲击和 CVaR 从来源事实确定性计算。所有七个来源只通过 `source-results.json` 聚合，不写回单场景运行。 + +双基准独立保存在 `.local/market-data/benchmark-sets//`，只包含沪深300人民币总回报和纳斯达克100人民币总回报，并记录币种、汇率、来源、日期和摘要；聚宽 `results.benchmark_returns` 仅作平台单基准参考。确定性分析完成收益、回撤、仓位与风险、Alpha/Beta、多维归因、完整稳健性、挑战结果、反对证据、不确定性、报告和推荐。Vibe-Trading（氛围量化）的研究目标和证据登记只作审计编排；加载 `performance-attribution`（绩效归因）、`risk-analysis`（风险分析）和 `report-generate`（报告生成）方法文档不等于实际分析。只允许调用无已知缺陷的单体公开入口；禁止 `run_swarm`（运行群体分析）、Vibe 回测和存在前视偏差风险的组合优化器。 -`candidate-strategies.json` 恰好包含七项：一项冻结基线，以及 40/60 日入场、1.5N/2.5N 止损、120 日滚动/30 日半衰期 EWMA（指数加权移动平均）协方差六项单参数挑战。七项共用同一策略代码摘要和 `snapshot_id`，只由配置区分。本地结果可以附方向性证据和风险提示，但不能按收益排名删除候选、替换基线或新增参数。 +若安全单体 Vibe 入口只能读取 CSV（逗号分隔文件），独立分析步骤只能从六表内存查询按明确字段和日期范围临时物化，记录查询版本和字节摘要；确认读取后立即删除。临时 CSV 不形成第二事实源。Vibe 单体能力不可用时记录 `evidence_insufficient`，不得转用群体分析，也不得反向改变本地 `run_id`、来源清单、报告或推荐状态。 -三类必需输出任一缺失、结构无效或摘要不匹配时不得输出 `complete`。 +独立策略分析至少输出： + +```text +.local/strategy-analysis-preparations// +├── analysis-scenarios.json +├── preparation.json +└── scenario-configs// + ├── params.json + └── run.json + +.local/strategy-analysis// +├── analysis-scenarios.json +├── preparation.json +├── source-results.json +├── deterministic-analysis.json +├── evidence-matrix.parquet +├── local-strategy-analysis-report.md +├── vibe-evidence.json +└── recommendation.json +``` + +`vibe-evidence.json` 记录 Vibe 研究目标标识、实际调用能力、安全边界、证据状态和临时 CSV 清理结果。任何群体分析调用必须标记 `valid_evidence=false` 和 `excluded_from_conclusions=true`。`recommendation.json` 的建议值固定为 `proceed_to_joinquant`、`revise_and_reassess` 或 `stop_evidence_insufficient`，并列出推荐基线行动、挑战关注项、确定性理由、反对证据、不确定性、阻断项及“不是聚宽正式回测或最终验收结论”的声明。 + +完整独立分析通过后输出 `next_action=human_confirmation_required`，等待用户人工确认。确认前不得启动聚宽、替换基线、修改参数、冻结策略或启动模拟交易。分析产物缺失或摘要不匹配不能改变已完成的本地数据包交付，只能把独立分析标记为 `evidence_insufficient` 或 `failed`。 ## 11. 状态与错误处理 @@ -289,18 +405,26 @@ stop_evidence_insufficient 证据不足而停止 |---|---| | `evidence_insufficient` | 真实策略身份、快照、来源、字段、范围或声明输入在项目执行前不完整 | | `failed` | 已有文件摘要或内容不一致、结构/类型/重复键违规、批次冲突、项目进程异常、硬约束突破、同输入不同输出或远端临时文件清理不可确认 | -| `complete` | 输入门禁、项目流程、必需输出、摘要和原子固化全部通过 | +| `complete` | 本次单场景输入门禁、冷/热两次执行及摘要一致性、180秒门槛、`performance.json`、一份聚宽口径兼容结果、清单和原子固化全部通过 | -不得把部分结果标记为完成，也不得用零值、旧数据或默认口径继续。项目建议为“修订后再评估”时流程仍可 `complete`；两套状态不合并。 +不得把部分结果标记为完成，也不得用零值、旧数据或默认口径继续。本地运行状态与后续策略分析状态相互独立，不得合并或反向改写。 ## 12. 测试与验收 ### 12.1 共用脚本单元测试 - 配置、仓库路径、参数数组和输出边界； -- 批次身份、字节摘要、字段、日期、空值、重复键和布尔规范化； +- CSV 暂存导入、Parquet 批次身份、逻辑与字节摘要、字段、日期、空值、重复键和布尔规范化； +- 公司行动结构、版本状态、公告与应用时间语义、知识截止日、来源摘要、原始价格/`pre_close`/连续因子勾稽，以及无法解释除权时关闭运行； - 相同内容去重、冲突重叠拒绝、新标的追加和旧快照不变； - 快照清单、内存 DuckDB 视图和规范化摘要； +- 现有聚宽 `manifest.json`、六类核心 Parquet、合法空表、文件摘要和跨表勾稽的 0 改动读取； +- 物理列顺序、`cancel_time` 空类型/字符串和兼容风险数值类型只在内存规范化，不改源文件； +- 本地 vectorbt 输出四类共同事实，官方风险参考显式缺失，字段和业务语义与聚宽现有结果兼容； +- 本地 `results.benchmark_returns` 为全空 `double` 且清单明确来源缺失；双基准只来自独立基准集； +- 通用分析计划 Schema 与 `strategy-003` 的机器可读计划可以校验、展开和复算，Skill 不读取计划； +- 海龟归因日志字段、唯一主键、原因码版本、覆盖范围和必需清单条目； +- 本地研究依赖图不导入 `quant_analysis` 算法或策略分析 Skill； - `run_id`、原子固化、幂等复用、失败重试和确定性冲突； - 三态唯一收口和敏感信息清理。 @@ -310,32 +434,60 @@ stop_evidence_insufficient 证据不足而停止 - 55/20 日通道排除当日、TR、20 日 N、U0 和次日执行； - 固定 0.5N 档位、每日一次、共同止损只上移； -- 资金、流动性、单 ETF、资产组、计划风险和波动率上限； +- 资金、单 ETF、资产组、计划风险和波动率上限； +- 海龟输入、配置、回调和原因码不含成交额或流动性规则；不同成交额输入不改变任何海龟订单； +- 拆分与现金分红前后连续经济权益不产生机械跳变；经济单位保持稳定，现金分红按除权日隐含再投资且支付日不重复增加现金； +- 本地清单完整声明研究级近似，统一读取器和报告不得把经济单位、近似现金或订单路径描述成聚宽精确账户； +- 单 ETF 或资产组被动超限不冻结其他证券，只禁止同一证券或同一组继续增加； - 退出、强制减仓、同级买入顺序； - A1 同比例、整手、小数余额、代码同分和输入乱序； - 现金不为负、硬上限不突破、相同输入输出摘要一致； - 停牌、涨跌停、不可成交、60 样本冷启动和风险输入故障安全。 -### 12.3 完整 E2E +### 12.3 vectorbt 接线、规则夹具与性能测试 + +- 依赖锁定、代码身份和 vectorbt 1.1.0 兼容版本； +- `SimulationInputs` 的日期/证券对齐、稳定类型、只读数组、原始与连续经济 OHLC、公司行动核对时点、累计连续因子和 T 日到 T+1 显式错位，并证明因子不读取晚公布元数据且不存在成交额或流动性输入； +- `Portfolio.from_order_func()`、单一共享现金组和四类官方回调实际被调用； +- 回调在 Numba `nopython`（无 Python 模式）下编译，不回落对象模式； +- 退出、风险减仓、卖出实际成交后 A1、买入和 `post_order_func_nb` 成交回填顺序； +- 普通订单函数每 ETF 每日最多一单，未启用 `flexible=True`； +- 逐日订单、成交、现金、持仓、批次、共同止损、风险原因和摘要满足规格夹具的明确预期； +- vectorbt 官方记录转换后满足四类共同事实字段语义、海龟必需归因日志、清单证据与跨表勾稽，不预计算策略分析指标； +- 旧 `execution.py`、`state.py`、`signals.py`、`risk.py`、`allocation.py`、`reporting.py`、专用测试和公开导出已经删除；全仓不存在 `process_day`、旧 `_simulate`、旧模块导入、兼容层、双引擎开关、回退或无效代码； +- 分别记录 vectorbt 冷启动 JIT 和预热执行时间，不运行旧完整流程作为性能比较。 + +日常小夹具测试不以微秒级波动判定失败。真实性能验收在同一暂存区保存 vectorbt 的环境、准备后输入摘要、首次 JIT、预热执行、四类共同事实、海龟归因日志校验和冷/热结果摘要；主 agent 的每次真实单场景调用都在全新进程先测冷启动，再在同一已编译进程测预热。两次均从已准备输入进入 vectorbt 开始，到交易执行、四类共同事实、海龟必需归因日志及其结构/摘要/勾稽校验完成时停止，均不得超过 180 秒且规范化结果摘要必须一致。停止计时后先删除预热副本和可丢弃暂存并确认清理，再把清理结果写入 `performance.json`、生成最终清单并校验全部摘要；最后只原子发布已整理的权威结果目录，发布后不依赖写入或清理。复数调用整体耗时和独立分析时间另记，但不替代单次门禁。 -正式 E2E（端到端）从 Skill 用户入口启动，使用固定小型日线夹具，经过共享批次、快照、CSV 校验、内存 DuckDB、通用运行器、`strategy-003` 适配器、收盘信号、次日订单、持仓/现金/风险、三类输出和不可变证据收口。不能以若干单元测试拼接替代。 +### 12.4 完整 E2E + +本地流程完整 E2E（端到端）从 Skill 用户入口启动，使用固定小型日线夹具，经过 CSV 暂存、Parquet 批次、快照、内存 DuckDB、通用运行器、`strategy-003` 适配器、vectorbt 官方回调、收盘信号、次日订单、持仓/现金/风险、一份聚宽口径兼容结果、`next_action=return_to_caller` 和不可变证据收口。测试明确断言 Skill 不接收候选数组、不内部循环，也未调用绩效归因、稳健性、报告、Vibe 或人工决定。主 agent 集成 E2E 另行调用七次并在分析目录聚合。另以仓库现有聚宽回测目录执行只读 E2E，断言运行前后 Git（版本管理）状态与全部文件摘要不变。 另用不含海龟词汇和资产的最小项目适配器执行同一 E2E，证明 Skill、行情中心和运行器未反向依赖海龟。常规自动测试不访问网络、不加载历史 `.local` 数据，并在临时目录结束后清理。 -### 12.4 真实集成验收 +### 12.5 真实集成验收 -使用聚宽真实研究环境导出最终 11 只 ETF 全历史日线，验证固定字段、未复权口径、字节摘要、共享批次、快照、内存视图和一次完整本地研究。完成后复查聚宽远端临时文件和本地暂存产物均不存在。 +使用已经验证的 11 只 ETF 全历史 Parquet 快照；只有数据或公司行动不完整时才重新调用聚宽研究环境导出。先验收固定字段、原始未复权口径、公司行动来源/元数据核对时点/勾稽、连续因子只依赖当日行情事实、连续总回报经济价格、研究级近似声明、传输摘要、Parquet 共享批次、快照、内存视图、零海龟流动性规则、vectorbt 代码身份、单场景聚宽口径兼容结果和每次冷/热不超过 180 秒。随后由主 agent 在 `preparation_id` 下分别完成七个基础调用，显式登记并校验七份共享身份，在本地流程之外派生不可变 `analysis_id`，通过同一读取入口和双基准集完成确定性绩效、归因、稳健性挑战、报告和推荐；报告必须把收益与回撤视为研究级总回报近似，披露晚公布公司行动只用于事后核对，并把现金、仓位、订单和聚宽差异列为不确定性。最后确认 Vibe 安全边界证据、输入摘要与临时 CSV 清理结果，并停在人工确认门禁。 最终执行 `quick_validate.py`、仓库布局测试、Build and Verify（构建与验证）完整检查、OpenSpec 严格校验和公开仓库敏感数据扫描。 ## 13. 实施顺序与回滚 -1. 建立并同步真实 `strategy-003` 身份，不启动正式回测。 -2. 以 TDD（测试驱动开发）实现共享日线行情中心和内存 DuckDB 查询。 -3. 初始化薄 Skill，建立共用运行器、证据和三态收口。 -4. 以 TDD 实现海龟纯计算模块、项目适配器和固定夹具。 -5. 完成离线海龟 E2E、非海龟 E2E 和三类输出。 -6. 导入真实 11 只 ETF 快照并执行完整本地研究。 -7. 运行仓库完整验证、独立前向验证和敏感数据扫描。 +1. 保留已验证的 `strategy-003` 身份、Parquet 行情和历史运行证据；先以现有聚宽清单和六表建立只读 `analysis_data` 契约，并固定聚宽目录 0 改动门禁。旧逐日实现仅在规则迁入期间作为只读来源，不执行完整对照。 +2. 先用真实最小样例验证公司行动来源，并保存 vectorbt 1.1.0 开源公开接口不能原生处理拆分与现金分红的实测证据；按用户确认采用只依赖当日原始行情事实的连续总回报近似，公司行动元数据仅作时点可知或事后核对，不使用私有引擎、伪订单或价格阈值猜测。 +3. 以 TDD（测试驱动开发）固定 vectorbt 1.1.0 和兼容依赖，建立不含任何成交额或流动性参数的 `SimulationInputs`、连续经济价格/经济单位、Numba 数值内核和官方四类回调，逐项通过公司行动、精度声明、前视偏差、顺序、A1、被动超限、风险和成交状态测试。 +4. 将 vectorbt 记录适配为四类共同执行事实、本地清单、`performance.json` 和海龟必需归因日志，显式声明官方风险参考缺失；规则夹具全部通过后删除旧五个执行/规则模块、流动性执行规则、`reporting.py`、旧专用测试和公开导出，不建立兼容层。 +5. 全仓扫描旧符号、旧导入、分析耦合、兼容开关和回退，确认项目只剩 vectorbt 唯一执行入口；从 Skill 公开入口执行单场景 E2E，并验证一次调用只产生一个结果且冷启动和预热都不超过 180 秒。 +6. 建立通用分析计划 Schema 和策略自有 `analysis-plan.json`；主 agent 使用真实 11 只 ETF 快照读取计划并分别调用 Skill 七次。其他稳健性场景只从基线已有事实确定性计算，不再调用 Skill；七份回测来源和派生证据只在独立分析目录聚合，Skill 不读取计划且契约不包含数量或循环。 +7. 在本地流程之外使用确定性分析处理上述数据包，生成独立报告、Vibe 安全边界证据和推荐，清理临时 CSV，等待人工确认。 +8. 完成非海龟 E2E、仓库完整验证、独立前向验证、OpenSpec 严格校验和敏感数据扫描。 + +迁移失败时停止在当前分支并保留性能、语义或依赖失败证据，不通过提高超时、恢复旧源码、增加兼容层或维护双生产引擎掩盖问题。不得删除已验证不可变批次、快照或历史运行证据，不得修改或删除 `strategy-001`、`strategy-002`，不得把功能分支本地合入主干。未固化暂存和未引用文件只有在确认不属于任何清单后才可清理。 + +## 14. 官方接口依据 + +- vectorbt `Portfolio`（组合）官方文档： +- vectorbt Numba 回调官方文档： +- vectorbt 1.1.0 官方许可： -回滚只停用新增 Skill 和共用脚本，不删除已验证不可变批次、快照或完整运行证据。不得修改或删除 `strategy-001`、`strategy-002`，不得把功能分支本地合入主干。未固化暂存和未引用文件只有在确认不属于任何清单后才可清理。 +官方文档把 `from_orders()`（预生成订单）、`from_signals()`（信号模拟）和 `from_order_func()` 列为三种主要模拟方式，并把 `from_order_func()` 定义为支持任意回调逻辑的最强模式。本设计只使用这些公开扩展点，不依赖未声明的私有实现。 diff --git a/docs/superpowers/specs/2026-07-16-turtle-full-position-redistribution-design.md b/docs/superpowers/specs/2026-07-16-turtle-full-position-redistribution-design.md new file mode 100644 index 0000000..6fe1c98 --- /dev/null +++ b/docs/superpowers/specs/2026-07-16-turtle-full-position-redistribution-design.md @@ -0,0 +1,274 @@ +# 海龟 ETF 全量仓位再分配与 N 风险单位设计 + +**状态：** 已确认 + +**日期：** 2026-07-16 + +**范围：** `strategy-003` 本地研究基线与标准分析数据包；不修改聚宽正式策略、回测或模拟交易 + +**冻结参数摘要：** 11 只 ETF、6 个资产组、55/20/20、0.5N 加仓、2N 止损、4/6/12 N 风险单位、单场景 180 秒门禁。 + +## 1. 目标与边界 + +当前基线已经定义 11 只 ETF、6 个固定资产组、55 日突破、20 日退出、20 日海龟 N、0.5N 加仓、2N 保护性止损、收盘检查与次日开盘成交。本变更不重新设计这些既有规则，只解决当前收益瓶颈对应的资金分配问题：旧分配只处理当日新增订单，先进入的趋势长期占用资金，后进入的有效趋势不能公平获得风险预算。 + +本变更完成三项核心调整： + +1. 用全量仓位再分配机制替换旧的同日新增订单分配； +2. 删除资金仓位上限、计划止损风险上限、协方差交易门槛和目标波动率控制； +3. 用海龟 N 风险单位统一定义单标的、资产组和全组合风险预算。 + +本次只实现并验证一个新基线，不运行旧基线对照，不把稳健性扫描耦合进单场景本地研究 Skill（技能），也不修改聚宽现有结果目录或正式回测流程。历史 `.local/` 结果作为不可变研究证据保留，不充当兼容执行路径。 + +## 2. 资产池与固定分组 + +基线固定为 11 只 ETF、6 个资产组： + +| 资产组 | ETF | +|---|---| +| `china_sync_equity` | `510300.XSHG`、`512100.XSHG`、`512480.XSHG`、`159819.XSHE`、`516160.XSHG` | +| `cross_border_tech_equity` | `513100.XSHG`、`513180.XSHG` | +| `china_dividend` | `515180.XSHG` | +| `china_innovative_drug` | `516080.XSHG` | +| `gold` | `518880.XSHG` | +| `treasury_bond` | `511010.XSHG` | + +资产组是固定相关风险分类，不是趋势强弱评分。系统不使用动量排名、预测分数或信号先后顺序选择标的；所有有效突破都进入统一风险缩放。 + +## 3. 保留的信号与执行规则 + +### 3.1 入场与退出 + +- 入场通道为信号日前 55 个交易日盘中最高价的最大值；当日收盘价严格高于通道时产生入场信号。 +- 退出通道为信号日前 20 个交易日盘中最低价的最小值；当日收盘价严格低于通道时产生趋势退出信号。 +- 通道均不包含信号当日，盘中越过但收盘未确认不产生信号。 +- 所有信号每天收盘检查一次，订单在下一交易日开盘执行。 +- 同一 ETF 同日退出优先，退出信号取消该 ETF 的入场或加仓候选。 + +### 3.2 N 与固定加仓档位 + +N 继续使用 20 日海龟递推： + +```text +TR = max( + 当日最高价 - 当日最低价, + |当日最高价 - 前收盘价|, + |当日最低价 - 前收盘价| +) + +初始 N = 最初 20 个有效 TR 的简单平均 +今日 N =（昨日 N × 19 + 今日 TR）÷ 20 +``` + +固定加仓档位继续以首次实际成交价和首次信号日 N 计算： + +```text +第 k 档 = 首次实际成交价 + k × 0.5 × 首次信号日 N +k = 1、2、3 +``` + +- 初始入场为第 1 个逻辑单位，最多再增加 3 个单位，单标的最多 4 个逻辑单位。 +- 每只 ETF 每个交易日最多新增 1 个单位；一天越过多个档位不得在次日开盘集中补齐。 +- 后续交易日收盘仍满足下一档时，才生成下一单位候选。 +- 只顺势加仓，不允许亏损摊平。 + +## 4. 单位状态与 2N 保护性止损 + +### 4.1 单位记录 + +每个成功建立的逻辑单位保存独立、不可变的状态： + +```text +unit_id +signal_date +signal_equity +signal_n +base_quantity +actual_fill_date +actual_fill_price +``` + +单位候选在信号日按当日账户权益和当日 N 计算基础数量： + +```text +base_quantity = floor_to_lot(signal_equity × 1% ÷ signal_n) +``` + +`signal_n` 和 `base_quantity` 在单位建立后冻结。全量再分配只改变实际目标持仓，不回写单位基础数量，也不因账户权益或每日 N 漂移重建既有单位。 + +单位候选只有在下一交易日可交易、全量目标计算后该标的至少产生一个整手的净新增买入并真实成交时才建立。若停牌、涨跌停、现金缩放或整手取整导致没有净新增成交，则候选不建立、不移动档位、不更新止损；下一交易日收盘仍满足条件时重新检查。多个候选同时出现时，移除未能产生净新增成交的候选后重新计算统一目标，直到候选集合稳定。 + +### 4.2 共同止损 + +每个真实成交的入场或加仓单位使用自己的冻结 N 生成候选止损： + +```text +候选止损_i = 实际成交价_i - 2 × signal_n_i +共同止损 = max（原共同止损，候选止损_i） +``` + +- 第一单位的候选止损直接成为初始共同止损。 +- 共同止损只能保持或上移，绝不允许下移。 +- 每日最新 N 不重新计算共同止损，避免把保护性止损变成每日波动率跟踪止损。 +- 全量再分配产生的增减仓不建立单位、不改变固定加仓档位、不更新共同止损。 +- 保护性止损在收盘价小于或等于共同止损时确认，下一交易日开盘退出全部持仓。 +- 保护性止损和 20 日趋势退出并列，任一先触发即清除该 ETF 的全部单位状态。 +- 第四单位建立后不再有加仓候选；共同止损保持不变，20 日退出通道继续每日更新。 + +## 5. N 风险单位上限 + +`1N` 单位在建立时使一个 N 的价格波动约对应信号日账户权益的 1%。风险约束只使用逻辑单位和统一缩放： + +- 单只 ETF 最多 4 个逻辑单位； +- 同一资产组最多 6 个有效风险单位； +- 全组合最多 12 个有效风险单位。 + +设标的 `i` 的逻辑单位数为 `u_i`，所属资产组为 `a`。资产组缩放为： + +```text +group_scale_a = min(1, 6 ÷ Σ(u_i), i 属于资产组 a) +``` + +经过资产组缩放后的全组合有效单位数为： + +```text +effective_units = Σ(u_i × group_scale_asset_group(i)) +portfolio_scale = min(1, 12 ÷ effective_units) +``` + +每只 ETF 的未考虑现金目标为： + +```text +raw_quantity_i = Σ(base_quantity_ij), j 为标的 i 的已建立单位 +risk_scaled_quantity_i = raw_quantity_i + × group_scale_asset_group(i) + × portfolio_scale +``` + +同组和全组合缩放均对受影响标的使用相同比例。逻辑单位状态不因缩放被删除；当后续组合事件释放风险预算时，既有趋势可以在下一次全量再分配中恢复到新的统一目标。 + +12 单位是 `1N` 暴露上限，不是最大亏损 12%。新单位的初始保护距离是 2N；多个单位同时止损、跳空或无法成交时，实际损失可以显著超过 12%。 + +## 6. 全量仓位再分配 + +### 6.1 触发条件 + +系统每天检查信号，但只在以下组合事件发生时重新计算全部目标持仓： + +- 有效入场候选； +- 有效加仓候选； +- 保护性止损； +- 20 日趋势退出。 + +没有上述事件时不调仓，不因账户权益、每日 N、波动率或相关性变化产生交易。 + +### 6.2 计算与执行顺序 + +每次事件按以下顺序处理： + +1. 收盘后冻结当日信号、单位候选、退出集合和信号日 N； +2. 下一交易日开盘先处理完整退出，并从目标状态删除对应单位； +3. 把可成交的入场和加仓候选加入临时单位状态； +4. 按单标的 4、资产组 6、组合 12 计算统一风险缩放； +5. 按预计开盘成交价、佣金和可用现金求全体目标共同的最大现金缩放比例； +6. 将全部目标统一按 100 股整手向下取整，不按代码或信号时间分配剩余现金； +7. 移除不能产生至少一个整手净新增成交的单位候选，并重新计算，直到候选集合稳定； +8. 先卖出退出和超配仓位，再买入欠配仓位； +9. 仅用真实成交结果建立新单位、更新共同止损和最终持仓状态。 + +现金缩放为组合共同因子，必须满足成交后现金不为负并覆盖预计费用。禁止按 ETF 顺序买到现金耗尽，也禁止融资补足 N 风险目标。 + +### 6.3 状态隔离 + +全量再分配订单与海龟信号状态严格分离： + +- 再分配卖出不视为趋势退出，不删除单位； +- 再分配买入不视为新增单位，不推进下一加仓档位； +- 再分配成交不更新共同止损； +- 只有有效入场或 0.5N 加仓候选的净新增成交建立单位； +- 后进入趋势可以通过统一缩放同比例挤占已有趋势资金，先后顺序不构成持仓优先权。 + +## 7. 删除的旧交易控制 + +以下字段、执行分支、原因码生产路径、挑战配置和专用测试从当前基线实现中删除，不保留关闭开关、兼容模式或双路径： + +- `security_value_cap=30%`； +- `asset_group_value_cap=50%`； +- `portfolio_value_cap=100%` 作为策略风险参数； +- `security_risk_cap=1.25%`； +- `asset_group_risk_cap=2.5%`； +- `portfolio_risk_cap=5%`； +- `target_volatility=10%`； +- `risk_reduction_target_volatility=9.5%`； +- 60/120 日协方差和指数加权协方差交易门槛； +- `minimum_aligned_samples` 对交易的阻断； +- `mandatory_risk_reduction` 和目标波动率强制减仓； +- `a1_uniform_completion` 及只分配当日新增订单的逻辑； +- 成交额 1%、流动性门槛或其他不属于海龟规则的限制。 + +不得用极大数、`null` 或隐藏默认值模拟删除。配置解析、回调状态、结果适配和测试契约都必须移除这些旧字段。协方差、实现波动率、集中度和计划损失仍可由独立策略分析事后计算，但不能反向产生订单。 + +`2026-07-16-turtle-volatility-rearm-design.md` 和 `2026-07-16-classic-turtle-unit-challenge.md` 依赖已删除的目标波动率或旧基线，实施时作为被本设计取代的未落地文档删除，不保留兼容实现。17 ETF 扩展不是本基线范围，不进入配置或本次验收。 + +## 8. 数据、成本与结果包边界 + +- 行情继续使用共享 Parquet（列式存储）快照和 DuckDB（内存数据库）查询；不复制行情，不创建持久数据库。 +- 继续使用现有公司行动连续经济价格与经济单位近似，不改变原始未复权行情存储口径。 +- 本地研究初始资金为 150 万元；买卖佣金为成交额万分之 0.85、每笔最低 5 元；ETF 印花税为 0；单边滑点为 0.05%。 +- 停牌、涨跌停、无法成交和整手不足不得假定成交。 +- 标准分析数据包的目录、`results`、`balances`、`positions`、`orders` 与归因扩展结构保持不变；聚宽现有结果继续 0 改动直读。 +- 归因日志需要区分信号单位建立、全量再分配增减仓、组缩放、组合缩放、现金缩放、止损和趋势退出，使独立策略分析能够解释风险预算利用率与仓位变化。 +- 通用本地研究 Skill 仍然一次只运行一个明确场景，不知道海龟规则，不内置 7 个场景或稳健性循环。 + +## 9. 实施结构 + +本变更只修改 `strategy-003` 项目能力及其契约接线： + +- `baseline.json`：替换为 N 风险单位与全量再分配配置； +- `vectorbt_engine.py`：解析新配置、建立单位数组与全量目标状态； +- `vectorbt_callbacks.py`：实现单位候选、4/6/12 缩放、现金缩放、事件驱动全量再分配和只上移共同止损； +- `result_adapter.py`：适配新的原因码和全量再分配证据，不改变标准四表结构； +- OpenSpec（开放规格）、原始研究方案、代码身份与测试：同步删除旧控制和旧命名； +- 通用行情中心、本地研究 Skill、聚宽归档读取器和独立策略分析算法不承载海龟专用逻辑。 + +实现不得保留旧分配函数、旧风险字段、旧目标波动率回调、兼容开关或回退路径。历史 `.local` 结果不迁移、不重写、不删除。 + +## 10. TDD 与验收 + +### 10.1 单元与规则测试 + +实施必须先写失败测试，再写代码，至少覆盖： + +- 新基线只接受 N 风险单位字段，旧资金、风险、协方差和目标波动率字段被拒绝； +- 55/20 日通道、20 日 N、收盘检查和次日开盘成交保持不变； +- 单位按信号日权益和 N 计算基础数量并冻结； +- 固定 0.5N 档位、每天最多一单位、最多四单位； +- 每单位候选 2N 止损、共同止损只上移、每日 N 不改止损； +- 再分配买卖不建立或删除单位，也不更新止损； +- 单标的 4、资产组 6、组合 12 的缩放公式和边界值； +- 后进入趋势使已有趋势同比例让出风险预算，不受输入顺序影响； +- 多个同日信号统一处理，不能产生按代码或时间先后买入； +- 现金不足时统一缩放，交易费用后现金不为负； +- 整手取整后不能成交的候选不推进状态，并触发稳定重算； +- 退出优先、卖出优先于买入、停牌与涨跌停失败安全； +- 旧字段、旧函数、旧原因码生产路径和旧挑战配置在全仓扫描中不存在。 + +### 10.2 结果包与完整 E2E + +- 标准四表和海龟归因扩展通过现有结构、摘要和跨表勾稽； +- 从 `run-local-quant-research` Skill 用户入口执行一个固定小型场景，完整经过快照、内存 DuckDB、vectorbt 回调、全量再分配、标准结果包和不可变清单； +- Skill 仍只运行一个场景，不调用稳健性、Vibe-Trading（AI 研究助理）、报告或人工决策； +- 使用真实 11 ETF 快照运行一次新基线，冷启动与预热结果一致，二者均小于 180 秒； +- 不运行旧基线性能或收益对照，不以旧引擎作为验收条件； +- 实际基线结果完成后，独立读取标准数据包汇总收益、回撤、仓位、风险预算利用率、逐标的贡献和主要反对证据，明确标记为本地探索性研究，不代替聚宽正式裁决。 + +### 10.3 完成标准 + +只有以下条件全部满足才算完成： + +1. 设计、OpenSpec、原始方案、机器配置、代码和测试使用同一套最新规则； +2. 旧资金控制、目标波动率控制、协方差交易门槛和旧分配实现已物理删除； +3. 全量仓位再分配、4/6/12 N 风险单位和逐单位 2N 止损通过规则测试； +4. 标准分析数据包兼容契约保持不变； +5. 完整 E2E 与真实 11 ETF 单场景性能门禁通过； +6. 工作区没有临时产物、兼容残留或无效旧方案。 diff --git a/joinquant/strategies/strategy-001/manifest.json b/joinquant/strategies/strategy-001/manifest.json index c289035..be4a782 100644 --- a/joinquant/strategies/strategy-001/manifest.json +++ b/joinquant/strategies/strategy-001/manifest.json @@ -7,8 +7,11 @@ "name": "etf_factor_rotation" }, "source": { - "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=8550fc0ab51897750ae63709679a9c45&backtest=115", + "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=2632adf656bc9797f82e2748c2ce928e&backtest=115", "aliases": [ + { + "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=2632adf656bc9797f82e2748c2ce928e&backtest=115" + }, { "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=8550fc0ab51897750ae63709679a9c45&backtest=115" }, @@ -565,7 +568,7 @@ "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=05d8070463bd0c212db54eed317288a1&backtest=115" } ], - "observed_at": "2026-07-14T04:00:05+08:00" + "observed_at": "2026-07-15T04:00:06+08:00" }, "fence": { "before_sha256": "79b2bdb2c82de35a722741b34292df7ed012f8348a19f1b0acfb209020322857", diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/raw/code-history-8b5d3e6fdb561d4a34100f76.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/raw/code-history-8b5d3e6fdb561d4a34100f76.json.gz new file mode 100644 index 0000000..bc5fefc --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/raw/code-history-8b5d3e6fdb561d4a34100f76.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acc119ec63045f88036851fe883ceead4c1f575fe16ee8982e5341c645089337 +size 828 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/attribution_log-0dcd6b38e3665bab5046f63f.parquet b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/attribution_log-0dcd6b38e3665bab5046f63f.parquet new file mode 100644 index 0000000..a5af3dc --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/attribution_log-0dcd6b38e3665bab5046f63f.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43c3663350069f1d4f859d4e792dc77cb0a1d38b5c9f36f8a86be38caccbb788 +size 29118 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/balances.parquet b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/balances.parquet new file mode 100644 index 0000000..814a838 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/balances.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad39ba65329f42a426edeba954c73661486e7f6d944267fc87ed508e1b1ac749 +size 2598 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/orders.parquet b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/orders.parquet new file mode 100644 index 0000000..104de73 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/orders.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d34dbe36c990af3f9f7300077bea1c5f8cbe54741dec148558306c74990ae39f +size 6339 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/positions.parquet b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/positions.parquet new file mode 100644 index 0000000..c318ed2 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/positions.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0dc39c12cebc13a6454bbb8c9bdee381a1b49405a2853968922c15b3e9e1634 +size 6961 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/results.parquet b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/results.parquet new file mode 100644 index 0000000..14af8eb --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/results.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f4230d6ae45da3951bf873b37350733984171b04bd93328b7a0bcefcc45e26a +size 2006 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/risk.parquet b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/risk.parquet new file mode 100644 index 0000000..9276dbf --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/data/risk.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:778584e78920e0f19b4a9b34d7ed8e4069e015be1454e0c4fee6c79afca81627 +size 11160 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/attribution-log-0dcd6b38e3665bab5046f63f.jsonl.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/attribution-log-0dcd6b38e3665bab5046f63f.jsonl.gz new file mode 100644 index 0000000..6db08ac --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/attribution-log-0dcd6b38e3665bab5046f63f.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1db439cfdd969da596b71692647ef60647a2767e7a0a4d1a39c7f9b8c5fae016 +size 3855 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/balances.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/balances.json.gz new file mode 100644 index 0000000..77a0571 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/balances.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79ed6eb8a63c93f0c4bd6d0e25e570bf8c799d0bb756c088b790aa5744bb68c +size 670 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/normal-log-pages.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/normal-log-pages.json.gz new file mode 100644 index 0000000..a6cfe15 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/normal-log-pages.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad46898c2a314a6b1e4e84d3ab2a0cc0d53d1a954fc2d55612df605f280eba52 +size 7473 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/normal-log.jsonl.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/normal-log.jsonl.gz new file mode 100644 index 0000000..369dc00 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/normal-log.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e92fbe870c27e19554ee6e9e147bb5e65de895307de6642aae071e4497b4486 +size 9266 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/orders.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/orders.json.gz new file mode 100644 index 0000000..7c03c6c --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/orders.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8009cb8260ceeb1f9c0248513ae013970180319d5bcffbd153f1a215ccfaa4b7 +size 681 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/period_risks.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/period_risks.json.gz new file mode 100644 index 0000000..f851046 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/period_risks.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c14bfcf1832a180081ce0abd2061eeb1dc780074f0c2cdfd923be805f2407368 +size 22 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/positions.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/positions.json.gz new file mode 100644 index 0000000..7b85ec2 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/positions.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb36ee958ac515a032314944ed933b830528e9131a72596ff7712208d9edcea8 +size 1954 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/records.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/records.json.gz new file mode 100644 index 0000000..c7f68c1 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/records.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64406e753d82e11c6bd63d1d04bb6aa083eefa0a46949bbfce994d556b989f0f +size 22 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/research-response-79c4851c2d0dbc27f0dc88f8.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/research-response-79c4851c2d0dbc27f0dc88f8.json.gz new file mode 100644 index 0000000..7fe9100 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/research-response-79c4851c2d0dbc27f0dc88f8.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f52b7ddff4f81f5bbff78045e544419b20f1025c837208618b6a1959e6e837e +size 1692 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/results.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/results.json.gz new file mode 100644 index 0000000..674197e --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/results.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c386c422b8b690d718e229b4bf5f4e8c380691ce246dbcac2fe3271745c17a44 +size 1015 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/risk.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/risk.json.gz new file mode 100644 index 0000000..a459e7c --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/raw/risk.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73affd0a708f336735b8dd263bd3a0f0e5f98ce0516bd589ece3716d037dc87a +size 567 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/reports/live-summary.json.gz b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/reports/live-summary.json.gz new file mode 100644 index 0000000..6c8a15f --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/snapshots/ddf7c320ed210995ad251e0a/reports/live-summary.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89e465802c9b5d94e0aa0c9bda712f49a0afba13d6e098299d8a3d95051d124 +size 665 diff --git a/joinquant/strategies/strategy-001/simulations/simulation-001/source_versions/eba29467755991738957a3eaff55ebf49e9076e5e5644541f63cf19667dd9ec5.json b/joinquant/strategies/strategy-001/simulations/simulation-001/source_versions/eba29467755991738957a3eaff55ebf49e9076e5e5644541f63cf19667dd9ec5.json new file mode 100644 index 0000000..18125a8 --- /dev/null +++ b/joinquant/strategies/strategy-001/simulations/simulation-001/source_versions/eba29467755991738957a3eaff55ebf49e9076e5e5644541f63cf19667dd9ec5.json @@ -0,0 +1,3 @@ +{ + "backtest_id": "c326681766e36c52d7f1fd858447a08b" +} diff --git a/joinquant/strategies/strategy-002/manifest.json b/joinquant/strategies/strategy-002/manifest.json index c4d2950..9bba774 100644 --- a/joinquant/strategies/strategy-002/manifest.json +++ b/joinquant/strategies/strategy-002/manifest.json @@ -7,8 +7,11 @@ "name": "ETF动态调仓" }, "source": { - "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=11b6f741da9c68283f9e1f01cdefb119&backtest=12", + "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=89bd88cca11d4ed59fd77462ee6addcb&backtest=12", "aliases": [ + { + "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=89bd88cca11d4ed59fd77462ee6addcb&backtest=12" + }, { "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=11b6f741da9c68283f9e1f01cdefb119&backtest=12" }, @@ -133,7 +136,7 @@ "url": "https://www.joinquant.com/algorithm/index/edit?algorithmId=c57fcc9eeb34b39428c46f5c87bfe77f&backtest=12" } ], - "observed_at": "2026-07-14T04:00:28+08:00" + "observed_at": "2026-07-15T04:00:33+08:00" }, "fence": { "before_sha256": "9f25381bdf78a8b0a42a71d09fa8f359bfa1e53855c829526607f6e4b254b731", diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/raw/code-history-233b4e8f26bc9c8affb490d0.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/raw/code-history-233b4e8f26bc9c8affb490d0.json.gz new file mode 100644 index 0000000..6618240 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/raw/code-history-233b4e8f26bc9c8affb490d0.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9f27cdeaca2f56bd41cf907e83c71a8811b8381829aa2e44f7cdb7be5ae95f4 +size 322 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/balances.parquet b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/balances.parquet new file mode 100644 index 0000000..5af6bd5 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/balances.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6f170f5a59b479a4c97360b702dfa9921ca42497b865f31f2b26d1e1299da5c +size 2695 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/orders.parquet b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/orders.parquet new file mode 100644 index 0000000..2a7c14b --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/orders.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1acbe363de1ac0f741e1c8bb4149bb95ec4ea70106c8ca5eb343391c6a9686c9 +size 6771 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/positions.parquet b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/positions.parquet new file mode 100644 index 0000000..5b5ca85 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/positions.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2a6b41aae9d015eb14edd5cd43c7f5f3014027c2029876ecd0de4aa46b6aa03 +size 8493 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/results.parquet b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/results.parquet new file mode 100644 index 0000000..bee1046 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/results.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c76c2beb363970cae1cc8e9d7423bc483bb8406a4dd1c93c1ba71b56cd467d0 +size 2006 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/risk.parquet b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/risk.parquet new file mode 100644 index 0000000..d19c779 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/data/risk.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:825d6bad801f40fe2b9285f16ff9e2d7192fef6000b489ffa3ce305eac630458 +size 11160 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/balances.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/balances.json.gz new file mode 100644 index 0000000..6048809 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/balances.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:121698166466a11847452691e7a336a7050e8e4e248eed9c00721bd6bfa0c2d7 +size 662 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/normal-log-pages.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/normal-log-pages.json.gz new file mode 100644 index 0000000..594c5b2 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/normal-log-pages.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66be31e340a0b417c8aa432120353c7f6e308df5680fcb98cce59e5cfde192f1 +size 9323 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/normal-log.jsonl.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/normal-log.jsonl.gz new file mode 100644 index 0000000..fbc2c11 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/normal-log.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c32b32953bb0ba106b507ab91a74c78bf281d947f5db555897ef322313212ff2 +size 12420 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/orders.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/orders.json.gz new file mode 100644 index 0000000..325553e --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/orders.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aa4f9f9e619aec30d2e49e8c60e3e520857adfaddd01399ae48af8e610d42ec +size 986 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/period_risks.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/period_risks.json.gz new file mode 100644 index 0000000..f851046 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/period_risks.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c14bfcf1832a180081ce0abd2061eeb1dc780074f0c2cdfd923be805f2407368 +size 22 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/positions.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/positions.json.gz new file mode 100644 index 0000000..786e9c9 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/positions.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34acc55ae3f3e0a3c7fc53e288188557475e18ea59195b48f3ac8db055ba33b7 +size 3363 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/records.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/records.json.gz new file mode 100644 index 0000000..c7f68c1 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/records.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64406e753d82e11c6bd63d1d04bb6aa083eefa0a46949bbfce994d556b989f0f +size 22 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/research-response-d8d210476ee7180bac7c83bc.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/research-response-d8d210476ee7180bac7c83bc.json.gz new file mode 100644 index 0000000..c4ee922 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/research-response-d8d210476ee7180bac7c83bc.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9945aae842335a3bee892c08449612b6a301f71687490cf42312dbf1074c27f2 +size 1888 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/results.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/results.json.gz new file mode 100644 index 0000000..c9a8e85 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/results.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc32a2e16e807576b643772a953cdcc9c23cf5a9816c7d17cf376a8ebe34f72b +size 1004 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/risk.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/risk.json.gz new file mode 100644 index 0000000..0034666 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/raw/risk.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd457e7d32b58a82dd9007bebec00c1c3f4514c83c1f967f21e46236efeae3bf +size 574 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/reports/live-summary.json.gz b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/reports/live-summary.json.gz new file mode 100644 index 0000000..3b531e0 --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/snapshots/687da9f0da3ff8e07d44e553/reports/live-summary.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06cc88396aa8e8e65d071a5855634b4d69ad77cda84f1fae77da107c64d86854 +size 671 diff --git a/joinquant/strategies/strategy-002/simulations/simulation-001/source_versions/3f4b2c3584dd8c943b84c229cba4f54594754e7713a24e3356722344aba67f55.json b/joinquant/strategies/strategy-002/simulations/simulation-001/source_versions/3f4b2c3584dd8c943b84c229cba4f54594754e7713a24e3356722344aba67f55.json new file mode 100644 index 0000000..3cc6cdb --- /dev/null +++ b/joinquant/strategies/strategy-002/simulations/simulation-001/source_versions/3f4b2c3584dd8c943b84c229cba4f54594754e7713a24e3356722344aba67f55.json @@ -0,0 +1,3 @@ +{ + "backtest_id": "75c1798d947e8764d8079bf66e1df45e" +} diff --git a/joinquant/strategies/strategy-003/research/analysis-plan.json b/joinquant/strategies/strategy-003/research/analysis-plan.json new file mode 100644 index 0000000..ec36f18 --- /dev/null +++ b/joinquant/strategies/strategy-003/research/analysis-plan.json @@ -0,0 +1,85 @@ +{ + "schema_version": "strategy-analysis-plan/1", + "strategy_id": "strategy-003", + "baseline_config": "joinquant/strategies/strategy-003/research/baseline.json", + "scenarios": [ + {"scenario_id": "baseline", "dimension": "baseline", "overrides": {}}, + {"scenario_id": "entry-40", "dimension": "parameter", "overrides": {"signal": {"entry_days": 40}}}, + {"scenario_id": "entry-60", "dimension": "parameter", "overrides": {"signal": {"entry_days": 60}}}, + {"scenario_id": "stop-1-5n", "dimension": "parameter", "overrides": {"signal": {"stop_n": 1.5}}}, + {"scenario_id": "stop-2-5n", "dimension": "parameter", "overrides": {"signal": {"stop_n": 2.5}}}, + {"scenario_id": "group-unit-cap-5", "dimension": "parameter", "overrides": {"risk": {"asset_group_unit_cap": 5.0}}}, + {"scenario_id": "portfolio-unit-cap-10", "dimension": "parameter", "overrides": {"risk": {"portfolio_unit_cap": 10.0}}} + ], + "universe": { + "159819.XSHE": "china_sync_equity", + "510300.XSHG": "china_sync_equity", + "511010.XSHG": "treasury_bond", + "512100.XSHG": "china_sync_equity", + "512480.XSHG": "china_sync_equity", + "513100.XSHG": "cross_border_tech_equity", + "513180.XSHG": "cross_border_tech_equity", + "515180.XSHG": "china_dividend", + "516080.XSHG": "china_innovative_drug", + "516160.XSHG": "china_sync_equity", + "518880.XSHG": "gold" + }, + "analyses": { + "fixed_periods": [ + {"id": "2015-2018", "start": "2015-01-01", "end": "2018-12-31"}, + {"id": "2019-2022", "start": "2019-01-01", "end": "2022-12-31"}, + {"id": "2023-end", "start": "2023-01-01", "end": "2026-07-13"} + ], + "rolling": {"window_years": 3, "step_months": 3}, + "deletions": {"each_security": true, "each_asset_group": true}, + "cost_execution": [ + {"id": "double-commission", "commission_multiplier": 2.0, "slippage": 0.0005, "delay_days": 0}, + {"id": "high-slippage", "commission_multiplier": 1.0, "slippage": 0.001, "delay_days": 0}, + {"id": "delay-one-day", "commission_multiplier": 1.0, "slippage": 0.0005, "delay_days": 1} + ], + "bootstrap": { + "block_sizes": [5, 20, 60], + "paths": 10000, + "horizon_days": 756, + "seed": 20260714, + "thresholds": { + "probability_drawdown_over_20pct_max": 0.05, + "probability_drawdown_over_30pct_max": 0.01, + "median_terminal_return_min_exclusive": 0.0 + } + }, + "historical_stress": [ + {"id": "history-2015-a-share-volatility", "start": "2015-06-12", "end": "2015-09-30", "max_drawdown_abs_max": 0.15}, + {"id": "history-2018-global-risk", "start": "2018-01-01", "end": "2018-12-31", "max_drawdown_abs_max": 0.15}, + {"id": "history-2020-pandemic", "start": "2020-02-03", "end": "2020-04-30", "max_drawdown_abs_max": 0.15}, + {"id": "history-2022-rate-hikes", "start": "2022-01-04", "end": "2022-10-31", "max_drawdown_abs_max": 0.15}, + {"id": "history-2024-liquidity", "start": "2024-01-02", "end": "2024-02-08", "max_drawdown_abs_max": 0.15} + ], + "position_shocks": [ + { + "id": "shock-synchronous-equity-crash", + "asset_group_shocks": {"china_sync_equity": -0.20, "cross_border_tech_equity": -0.20, "china_dividend": -0.20, "china_innovative_drug": -0.20, "gold": -0.05, "treasury_bond": 0.0}, + "maximum_loss_abs_max": 0.15 + }, + { + "id": "shock-market-liquidity", + "asset_group_shocks": {"china_sync_equity": -0.15, "cross_border_tech_equity": -0.15, "china_dividend": -0.15, "china_innovative_drug": -0.15, "gold": -0.10, "treasury_bond": -0.08}, + "maximum_loss_abs_max": 0.15 + }, + { + "id": "shock-cross-border-gap", + "asset_group_shocks": {"china_sync_equity": -0.08, "cross_border_tech_equity": -0.08, "china_dividend": -0.08, "china_innovative_drug": -0.08, "gold": 0.0, "treasury_bond": 0.0}, + "security_shocks": {"513100.XSHG": -0.25, "513180.XSHG": -0.25}, + "maximum_loss_abs_max": 0.15 + }, + {"id": "shock-stop-failure", "use_stop_failure_loss": true, "maximum_loss_abs_max": 0.15} + ], + "cvar": [ + {"id": "cvar-1d-95", "horizon_days": 1, "confidence": 0.95, "maximum_loss_abs_max": 0.025, "minimum_tail_observations": 20}, + {"id": "cvar-1d-99", "horizon_days": 1, "confidence": 0.99, "maximum_loss_abs_max": 0.04, "minimum_tail_observations": 20}, + {"id": "cvar-5d-95", "horizon_days": 5, "confidence": 0.95, "maximum_loss_abs_max": 0.05, "minimum_tail_observations": 20} + ] + }, + "expected": {"scenario_runs": 7, "benchmarks": 2, "bootstrap_paths": 10000, "seed": 20260714}, + "thresholds": {"cagr_min_exclusive": 0.0, "max_drawdown_abs_max": 0.2, "calmar_min": 0.5} +} diff --git a/joinquant/strategies/strategy-003/research/baseline.json b/joinquant/strategies/strategy-003/research/baseline.json index bc80457..0015bba 100644 --- a/joinquant/strategies/strategy-003/research/baseline.json +++ b/joinquant/strategies/strategy-003/research/baseline.json @@ -1,6 +1,7 @@ { "schema_version": 1, "project_id": "strategy-003", + "scenario_id": "baseline", "universe": [ { "security": "510300.XSHG", @@ -52,23 +53,13 @@ "exit_days": 20, "n_days": 20, "add_step_n": 0.5, - "stop_n": 2.0 + "stop_n": 2.0, + "max_units": 4 }, "risk": { - "risk_per_unit": 0.005, - "security_risk_cap": 0.0125, - "security_value_cap": 0.3, - "asset_group_risk_cap": 0.025, - "asset_group_value_cap": 0.5, - "portfolio_risk_cap": 0.05, - "portfolio_value_cap": 1.0, - "covariance": { - "method": "sample", - "window_days": 60 - }, - "target_volatility": 0.1, - "risk_reduction_target_volatility": 0.095, - "minimum_aligned_samples": 60 + "unit_risk_per_n": 0.01, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0 }, "market_data": { "source": "joinquant", @@ -86,7 +77,6 @@ "close", "pre_close", "volume", - "money", "factor", "paused", "high_limit", @@ -94,9 +84,11 @@ ] }, "joinquant_export": { - "api_source": "research_runtime_injected", + "api_source": "research_runtime_and_jqdata", "apis": [ "get_price", + "query", + "finance.FUND_DIVIDEND", "write_file", "read_file" ], @@ -110,23 +102,29 @@ "remote_cleanup_required": true }, "execution": { + "additional_delay_days": 0, "order_priority": [ "full_exit", - "mandatory_risk_reduction", - "entry_or_addition" + "redistribution_sell", + "entry_or_addition", + "redistribution_buy" ], - "allocation": "a1_uniform_completion", + "allocation": "full_position_redistribution", "acceptance_fixture": { "same_security_exit_cancels_buys": true, - "standard_request_limit": "one_u0", - "uniform_completion_ratio": true, - "capped_budget_redistribution": true, - "lot_rounding": "floor_then_largest_remainder", - "recheck_hard_caps_each_lot": true, - "tie_breaker": "security_code_ascending", + "candidate_requires_net_buy_lot": true, + "group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + "cash_scaling": "uniform", + "lot_rounding": "floor", + "residual_cash_redistribution": false, "input_order_invariant": true } }, + "costs": { + "commission_multiplier": 1.0, + "one_way_slippage": 0.0005 + }, "research": { "initial_cash": 1500000, "vibe_optimizer": { diff --git a/joinquant/strategies/strategy-003/research/candidates.json b/joinquant/strategies/strategy-003/research/candidates.json deleted file mode 100644 index 059e419..0000000 --- a/joinquant/strategies/strategy-003/research/candidates.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "schema_version": 1, - "baseline_config": "baseline.json", - "candidates": [ - { - "id": "baseline", - "overrides": {} - }, - { - "id": "entry-40", - "overrides": { - "signal.entry_days": 40 - } - }, - { - "id": "entry-60", - "overrides": { - "signal.entry_days": 60 - } - }, - { - "id": "stop-1.5n", - "overrides": { - "signal.stop_n": 1.5 - } - }, - { - "id": "stop-2.5n", - "overrides": { - "signal.stop_n": 2.5 - } - }, - { - "id": "covariance-120d", - "overrides": { - "risk.covariance": { - "method": "sample", - "window_days": 120 - } - } - }, - { - "id": "covariance-ewma-30d", - "overrides": { - "risk.covariance": { - "method": "ewma", - "half_life_days": 30 - } - } - } - ] -} diff --git a/joinquant/strategies/strategy-003/research/code-identity.json b/joinquant/strategies/strategy-003/research/code-identity.json index c8b4fbe..54f823c 100644 --- a/joinquant/strategies/strategy-003/research/code-identity.json +++ b/joinquant/strategies/strategy-003/research/code-identity.json @@ -1,41 +1,77 @@ { "schema_version": 1, + "execution": { + "backend": "vectorbt.Portfolio.from_order_func", + "delayed_backend": "vectorbt.Portfolio.from_orders", + "adapter_version": "local-vectorbt-adapter/2", + "dependencies": { + "vectorbt": "1.1.0", + "numba": "0.66.0", + "numpy": "2.4.6", + "pandas": "3.0.3" + }, + "callbacks_sha256": "585246dea896f4353c3074df52fa358bc0ed73f5af22e1a217e1b8e80e63efc1", + "accounting": { + "version": "turtle-etf-corporate-actions/1", + "corporate_action_mode": "point_in_time_total_return_approximation", + "continuity_factor_basis": "raw_previous_close_over_current_pre_close", + "corporate_action_metadata_timing": "audit_only_may_be_retrospective", + "price_basis": "continuous_economic_price", + "quantity_basis": "economic_units", + "cash_dividend_mode": "implicit_reinvestment_on_ex_date", + "pay_date_cash_supported": false, + "exact_joinquant_reconciliation": false + }, + "license": { + "expression": "Apache-2.0 WITH Commons-Clause", + "usage": "internal_research_only", + "resale_prohibited": true + } + }, "files": [ + { + "path": ".agents/skills/joinquant-archive-sync/references/manifest.schema.json", + "sha256": "4600fb1f5a1ba8a2a7f0e0080af55d2fb85a17ac524ba86e1b75770f830d05fc" + }, { "path": "joinquant/strategies/strategy-003/research/turtle_etf/__init__.py", - "sha256": "3a42f837172c7975176d4d0a627b557a2bb3daf2d3af8b5c19be959eefbea7d8" + "sha256": "57f64f5c7258a78bd721335e1b18ffa9c9de6f8c04634b055b0e4461640b2f0c" + }, + { + "path": "joinquant/strategies/strategy-003/research/turtle_etf/indicators.py", + "sha256": "4e83a58a80550b31395a98b983389582a3fcf001f692b2ac860445b899fe7bb6" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/allocation.py", - "sha256": "7f075e23ad067be83bd7218e4fe8f30a9026307959001582500a6b96558bf29d" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py", + "sha256": "41f2013ea3b7cbab10167147bea9b60ccea41e63a5c6bb354338d48b9d2c6b5b" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/cli.py", - "sha256": "6b0757619dd1b9d67e486a4e4fe2a882f617b6ed306357294b6a3ad0e6778e9d" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/single_scenario.py", + "sha256": "eab9c339d12c3aabf093bb22b98ef51ed5637b4187c98295b18c6e8d2cbd9264" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/execution.py", - "sha256": "9ffab26424a70afb74a3baad2dcb9036180e56fba851b8ef93567eed71d14ebb" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_benchmark.py", + "sha256": "8734280e8285e0b7df12146f7db3571d5e1e96c91adfe1f6f3717ef9c06f2842" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/indicators.py", - "sha256": "4e83a58a80550b31395a98b983389582a3fcf001f692b2ac860445b899fe7bb6" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py", + "sha256": "585246dea896f4353c3074df52fa358bc0ed73f5af22e1a217e1b8e80e63efc1" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/reporting.py", - "sha256": "6ababafcf615308cd0bac0a018826e471225cdb5b926acbc01f931d4cfe75a71" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_cli.py", + "sha256": "f09e51da53d6ffab74efed93bf8ffd7e3e1bd153e68e7992f09b20b631c3caf1" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/risk.py", - "sha256": "d82841db39ddd3d28a598614b81181bb31d72072d30adf0f682e1f8f2b5d9917" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py", + "sha256": "1298a8d914eec26224d08f01d1cd6e348495ce0f44ecfdf327912abbf624b49c" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/signals.py", - "sha256": "4f54b9110c6a8dde00ba36a3bfed23de933797e260fe0644264578e531e9009d" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py", + "sha256": "557d3832725fc4cb0175d822e8d7fd08e620f61992bf217558004327a8356e5a" }, { - "path": "joinquant/strategies/strategy-003/research/turtle_etf/state.py", - "sha256": "50f3655b54737b118e392b8645d94160de93674fcf213066431a46f1072ff6a9" + "path": "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py", + "sha256": "92b0673856408f3ac23eb2236c71c8ab9afa70bfdc550b1e13bfe739765b558f" }, { "path": "scripts/__init__.py", @@ -45,21 +81,45 @@ "path": "scripts/research/__init__.py", "sha256": "db573e2ee117f1994a4b7ce24e91147d71bcc9a9597e89ecae4b8b90af350bc9" }, + { + "path": "scripts/research/analysis_data/__init__.py", + "sha256": "f6e40eceacd8589ae642ed19aa97ca05d41ca397d58a3d35dba1fed33546aa64" + }, + { + "path": "scripts/research/analysis_data/derived.py", + "sha256": "02eec15326539a96319c8d346d3df9c4a9f5a805ca18f88ae1091e32ef1a91b9" + }, + { + "path": "scripts/research/analysis_data/manifest.py", + "sha256": "0c402c0b253c9b9834d7cacdc7e8d3fd448d86d9bbfdfab9ca9577a93a762d62" + }, + { + "path": "scripts/research/analysis_data/schemas/local-backtest-manifest.schema.json", + "sha256": "65cf1556a702173cb20101a713ec27b61d32ca9ea47d77b5a984312fed9c66fb" + }, + { + "path": "scripts/research/analysis_data/views.py", + "sha256": "b561963a55389042a2a11b9e65f997dabeb646878be6ba22616f116a0a60bce3" + }, { "path": "scripts/research/market_data/__init__.py", - "sha256": "6f0c8a6a3ea176856ed43cb49514eb21be1aab4d514ff81e8a84177ddb0da0fb" + "sha256": "80a69e67a47749f1fa8eeb536056499eb1f3805cedfacbf78952291d50f7693f" }, { "path": "scripts/research/market_data/contracts.py", - "sha256": "768c311e21becc130e7d188f511f75a135aec0548272a5cbdb94f945f4b4fd70" + "sha256": "50d5c2fb015b3c6a5e5bb800df99345de67e60c677a6e65202f5676974c54204" + }, + { + "path": "scripts/research/market_data/economic_returns.py", + "sha256": "13969fc9ee6bbec08eff096bd53709c54e4007aa1a5f960c977d923c3e2883e3" }, { "path": "scripts/research/market_data/query.py", - "sha256": "0b52344130dbf94ab27e5c42b99a35bce042a57fea454043c672d3fba0a3bb45" + "sha256": "87fb5bad20cdce7719434a45a1373b89196e1d1963c2161d01b5193fd7d864e4" }, { "path": "scripts/research/market_data/storage.py", - "sha256": "15b9c09dd535ccdbe7b9273bcf7876806ff6edda58e3b52272f1fe49a0d25982" + "sha256": "09bfb3ee07594d6899f065268d06a0fe33f60c852128a7c67fb392e4eb3c34b9" } ] } diff --git a/joinquant/strategies/strategy-003/research/project-run.json b/joinquant/strategies/strategy-003/research/project-run.json index 72ce430..bfe6d2b 100644 --- a/joinquant/strategies/strategy-003/research/project-run.json +++ b/joinquant/strategies/strategy-003/research/project-run.json @@ -1,7 +1,7 @@ { "schema_version": 1, "project_id": "strategy-003", - "snapshot_id": "64785f1607d90cf58f3db4545c2a718796659df818e283a4ec0614b0dfd12d8a", + "snapshot_id": "e88238cca420a8ae66b90adb6cda4dd6c38a07390a13b8ac2f471e534742e33e", "snapshot_requirements": { "source": { "name": "joinquant", @@ -44,45 +44,20 @@ "skip_paused": false } }, - "project_entry": "joinquant/strategies/strategy-003/research/turtle_etf/cli.py", + "project_entry": "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_cli.py", "command": [ ".venv/Scripts/python.exe", - "joinquant/strategies/strategy-003/research/turtle_etf/cli.py" + "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_cli.py" ], "project_config": "joinquant/strategies/strategy-003/research/baseline.json", "code_identity": "joinquant/strategies/strategy-003/research/code-identity.json", "declared_inputs": [ - "joinquant/strategies/strategy-003/manifest.json", - "joinquant/strategies/strategy-003/research/candidates.json" + "joinquant/strategies/strategy-003/manifest.json" ], "required_outputs": [ { - "path": "candidate-strategies.json", - "format": "json" - }, - { - "path": "conclusion.json", - "format": "json" - }, - { - "path": "daily-audit.csv", - "format": "csv" - }, - { - "path": "positions.csv", - "format": "csv" - }, - { - "path": "research-report.md", - "format": "markdown" - }, - { - "path": "risk.csv", - "format": "csv" - }, - { - "path": "trades.csv", - "format": "csv" + "path": "backtests/local-baseline", + "format": "directory" } ], "output_root": ".local/quant-research", diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/__init__.py b/joinquant/strategies/strategy-003/research/turtle_etf/__init__.py index 13f2b84..ee49162 100644 --- a/joinquant/strategies/strategy-003/research/turtle_etf/__init__.py +++ b/joinquant/strategies/strategy-003/research/turtle_etf/__init__.py @@ -1,17 +1,9 @@ -"""Deterministic, unadjusted-price Turtle ETF research primitives.""" +"""Unadjusted-price Turtle ETF vectorbt research entry points.""" from .indicators import breakout_levels, true_range, turtle_n -from .risk import PortfolioState, RiskInputs, evaluate_risk, initial_unit -from .state import OrderIntent, TrendState __all__ = [ - "OrderIntent", - "PortfolioState", - "RiskInputs", - "TrendState", "breakout_levels", - "evaluate_risk", - "initial_unit", "true_range", "turtle_n", ] diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/allocation.py b/joinquant/strategies/strategy-003/research/turtle_etf/allocation.py deleted file mode 100644 index 8de1b23..0000000 --- a/joinquant/strategies/strategy-003/research/turtle_etf/allocation.py +++ /dev/null @@ -1,294 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass, replace -from decimal import Decimal -from types import MappingProxyType -from typing import Mapping, Sequence - -from .risk import PortfolioState, RiskInputs, evaluate_risk -from .state import OrderIntent, commission_fee - - -_NON_MONOTONIC_REASONS = frozenset( - { - "security_risk_cap", - "group_risk_cap", - "portfolio_risk_cap", - "target_volatility", - } -) - - -@dataclass(frozen=True) -class BuyRequest: - intent: OrderIntent - - def __post_init__(self) -> None: - if self.intent.action not in {"entry", "addition"}: - raise ValueError("A1 candidates must be entry or addition requests") - - -@dataclass(frozen=True) -class PortfolioConstraints: - state: PortfolioState - risk_inputs: RiskInputs - - -@dataclass(frozen=True) -class AllocationResult: - allocations: tuple[OrderIntent, ...] - rejected: tuple[BuyRequest, ...] - quantities: Mapping[str, int] - completion_ratios: Mapping[str, Decimal] - remaining_cash: Decimal - audit_sha256: str - - def __post_init__(self) -> None: - object.__setattr__(self, "allocations", tuple(self.allocations)) - object.__setattr__(self, "rejected", tuple(self.rejected)) - object.__setattr__( - self, - "quantities", - MappingProxyType(dict(self.quantities)), - ) - object.__setattr__( - self, - "completion_ratios", - MappingProxyType(dict(self.completion_ratios)), - ) - - -def _allocated_intents( - candidates: Sequence[BuyRequest], - quantities: Mapping[str, int], -) -> tuple[OrderIntent, ...]: - return tuple( - replace( - candidate.intent, - quantity=quantities[candidate.intent.security], - estimated_fee=commission_fee( - candidate.intent.expected_price, - quantities[candidate.intent.security], - ), - ) - for candidate in candidates - if quantities[candidate.intent.security] > 0 - ) - - -def _reason_codes( - candidates: Sequence[BuyRequest], - quantities: Mapping[str, int], - constraints: PortfolioConstraints, -) -> tuple[str, ...]: - intents = _allocated_intents(candidates, quantities) - if not intents: - return () - decision = evaluate_risk( - intents, - constraints.state, - constraints.risk_inputs, - ) - if decision.approved != intents and not decision.reason_codes: - return ("allocation_not_approved",) - return decision.reason_codes - - -def _is_feasible( - candidates: Sequence[BuyRequest], - quantities: Mapping[str, int], - constraints: PortfolioConstraints, -) -> bool: - return not _reason_codes(candidates, quantities, constraints) - - -def _hamilton_quantities( - *, - base: Mapping[str, int], - active: set[str], - extra_lots: int, - lot: int, - requests: Mapping[str, BuyRequest], -) -> dict[str, int]: - quantities = dict(base) - remaining_lots = { - security: (requests[security].intent.quantity - quantities[security]) // lot - for security in active - } - total_remaining = sum(remaining_lots.values()) - if extra_lots < 0 or extra_lots > total_remaining: - raise ValueError("A1 extra lots exceed remaining requests") - if not extra_lots or not total_remaining: - return quantities - - floor_lots: dict[str, int] = {} - remainders: dict[str, int] = {} - for security in active: - numerator = extra_lots * remaining_lots[security] - floor_lots[security], remainders[security] = divmod( - numerator, - total_remaining, - ) - remainder_lots = extra_lots - sum(floor_lots.values()) - for security in sorted(active, key=lambda item: (-remainders[item], item)): - if not remainder_lots: - break - if floor_lots[security] < remaining_lots[security]: - floor_lots[security] += 1 - remainder_lots -= 1 - if remainder_lots: - raise ValueError("A1 remainder allocation did not converge") - for security, added_lots in floor_lots.items(): - quantities[security] += added_lots * lot - return quantities - - -def _maximum_hamilton_allocation( - *, - candidates: Sequence[BuyRequest], - base: Mapping[str, int], - active: set[str], - lot: int, - requests: Mapping[str, BuyRequest], - constraints: PortfolioConstraints, -) -> tuple[int, dict[str, int]]: - maximum_lots = sum( - (requests[security].intent.quantity - base[security]) // lot - for security in active - ) - base_intents = _allocated_intents(candidates, base) - base_spend = sum( - ( - intent.expected_price * intent.quantity + intent.estimated_fee - for intent in base_intents - ), - Decimal("0"), - ) - available_cash = max(Decimal("0"), constraints.state.cash - base_spend) - cheapest_lot = min( - requests[security].intent.expected_price * lot for security in active - ) - maximum_lots = min(maximum_lots, int(available_cash // cheapest_lot)) - # Portfolio volatility is not monotonic when candidates diversify each - # other. Search actual Hamilton portfolios from largest to smallest so a - # feasible bundle cannot be discarded merely because its single-lot - # members are infeasible in isolation. - for candidate_lots in range(maximum_lots, 0, -1): - proposed = _hamilton_quantities( - base=base, - active=active, - extra_lots=candidate_lots, - lot=lot, - requests=requests, - ) - if _is_feasible(candidates, proposed, constraints): - return candidate_lots, proposed - return 0, dict(base) - - -def _audit_digest( - candidates: Sequence[BuyRequest], - quantities: Mapping[str, int], - ratios: Mapping[str, Decimal], - remaining_cash: Decimal, -) -> str: - document = { - "allocations": [ - { - "security": candidate.intent.security, - "action": candidate.intent.action, - "requested_quantity": candidate.intent.quantity, - "allocated_quantity": quantities[candidate.intent.security], - "completion_ratio": str(ratios[candidate.intent.security]), - } - for candidate in candidates - ], - "remaining_cash": str(remaining_cash), - } - payload = json.dumps( - document, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - return hashlib.sha256(payload).hexdigest() - - -def allocate_a1( - candidates: Sequence[BuyRequest], - constraints: PortfolioConstraints, -) -> AllocationResult: - ordered = tuple(sorted(candidates, key=lambda item: item.intent.security)) - securities = tuple(candidate.intent.security for candidate in ordered) - if len(securities) != len(set(securities)): - raise ValueError("A1 candidates must be unique by security") - lot = constraints.risk_inputs.lot_size - quantities = {security: 0 for security in securities} - active = set(securities) - by_security = {candidate.intent.security: candidate for candidate in ordered} - - while active: - active = { - security - for security in active - if quantities[security] < by_security[security].intent.quantity - } - if not active: - break - _, proposed = _maximum_hamilton_allocation( - candidates=ordered, - base=quantities, - active=active, - lot=lot, - requests=by_security, - constraints=constraints, - ) - quantities = proposed - remaining_lots = sum( - (by_security[security].intent.quantity - quantities[security]) // lot - for security in active - ) - if not remaining_lots: - break - blocked: set[str] = set() - for security in active: - next_quantities = dict(quantities) - next_quantities[security] += lot - reasons = set(_reason_codes(ordered, next_quantities, constraints)) - if reasons - _NON_MONOTONIC_REASONS: - blocked.add(security) - if blocked: - active.difference_update(blocked) - continue - break - - allocations = _allocated_intents(ordered, quantities) - ratios = { - security: Decimal(quantities[security]) - / Decimal(by_security[security].intent.quantity) - for security in securities - } - spent = sum( - ( - intent.expected_price * intent.quantity + intent.estimated_fee - for intent in allocations - ), - Decimal("0"), - ) - remaining_cash = constraints.state.cash - spent - rejected = tuple( - candidate - for candidate in ordered - if quantities[candidate.intent.security] == 0 - ) - return AllocationResult( - allocations=allocations, - rejected=rejected, - quantities=quantities, - completion_ratios=ratios, - remaining_cash=remaining_cash, - audit_sha256=_audit_digest(ordered, quantities, ratios, remaining_cash), - ) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/cli.py b/joinquant/strategies/strategy-003/research/turtle_etf/cli.py deleted file mode 100644 index 2344137..0000000 --- a/joinquant/strategies/strategy-003/research/turtle_etf/cli.py +++ /dev/null @@ -1,633 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import math -import statistics -import sys -from collections import Counter -from dataclasses import dataclass, replace -from decimal import Decimal -from pathlib import Path -from typing import Mapping, Sequence - -import pandas as pd - -if __package__ in {None, ""}: - RESEARCH_ROOT = Path(__file__).resolve().parent.parent - sys.path.insert(0, str(RESEARCH_ROOT)) - from turtle_etf.execution import DailyMarket, MarketQuote, TradingDay, process_day - from turtle_etf.indicators import breakout_levels, turtle_n - from turtle_etf.reporting import ( - OutputValidationError, - ResearchResult, - RunIdentity, - decimal_text, - validate_project_outputs, - write_outputs, - ) - from turtle_etf.risk import ( - PortfolioState, - RiskInputs, - estimate_covariance, - initial_unit, - portfolio_volatility, - target_volatility_reductions, - ) - from turtle_etf.signals import entry_signal, make_entry_intent - from turtle_etf.state import commission_fee, request_addition, request_full_exit -else: - from .execution import DailyMarket, MarketQuote, TradingDay, process_day - from .indicators import breakout_levels, turtle_n - from .reporting import ( - OutputValidationError, - ResearchResult, - RunIdentity, - decimal_text, - validate_project_outputs, - write_outputs, - ) - from .risk import ( - PortfolioState, - RiskInputs, - estimate_covariance, - initial_unit, - portfolio_volatility, - target_volatility_reductions, - ) - from .signals import entry_signal, make_entry_intent - from .state import commission_fee, request_addition, request_full_exit - -from scripts.research.market_data.query import open_snapshot - - -class ResearchEvidenceInsufficient(RuntimeError): - pass - - -@dataclass(frozen=True) -class ProjectResult: - status: str - reason_codes: tuple[str, ...] - output_dir: Path - - -def _load_object(path: Path, label: str) -> dict[str, object]: - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ResearchEvidenceInsufficient(f"invalid_{label}") from exc - if not isinstance(value, dict): - raise ResearchEvidenceInsufficient(f"invalid_{label}") - return value - - -def _decimal(value: object) -> Decimal | None: - if value is None: - return None - result = Decimal(str(value)) - return result if result.is_finite() else None - - -def _risk_inputs( - *, - config: Mapping[str, object], - securities: tuple[str, ...], - frames: Mapping[str, pd.DataFrame], - returns: pd.DataFrame, - through_date: str, - prices: Mapping[str, Decimal | None], -) -> RiskInputs: - risk = config["risk"] - window_days = int(risk["covariance"]["window_days"]) - through_returns = returns.loc[:through_date] - eligible_securities = tuple( - security - for security in securities - if sum( - pd.notna(value) and math.isfinite(float(value)) - for value in pd.to_numeric( - through_returns[security], - errors="coerce", - ) - ) - >= window_days - ) - covariance = ( - None - if not eligible_securities - else estimate_covariance( - through_returns, - securities=eligible_securities, - days=window_days, - ) - ) - turnover: dict[str, Decimal] = {} - for security in securities: - values = pd.to_numeric( - frames[security].loc[:through_date, "money"], - errors="coerce", - ).dropna() - if len(values) >= 20: - turnover[security] = Decimal(str(values.tail(20).median())) - return RiskInputs( - prices=prices, - median_turnover_20d=turnover, - covariance=covariance, - minimum_aligned_samples=int(risk["minimum_aligned_samples"]), - security_risk_cap=Decimal(str(risk["security_risk_cap"])), - security_value_cap=Decimal(str(risk["security_value_cap"])), - asset_group_risk_cap=Decimal(str(risk["asset_group_risk_cap"])), - asset_group_value_cap=Decimal(str(risk["asset_group_value_cap"])), - portfolio_risk_cap=Decimal(str(risk["portfolio_risk_cap"])), - portfolio_value_cap=Decimal(str(risk["portfolio_value_cap"])), - target_volatility=Decimal(str(risk["target_volatility"])), - ) - - -def _prepare_frames( - rows: Sequence[Mapping[str, object]], - config: Mapping[str, object], -) -> tuple[dict[str, pd.DataFrame], pd.DataFrame, tuple[str, ...]]: - universe = tuple(str(item["security"]) for item in config["universe"]) - frame = pd.DataFrame([dict(row) for row in rows]) - if frame.empty or set(frame["security"].unique()) != set(universe): - raise ResearchEvidenceInsufficient("snapshot_universe_mismatch") - frames: dict[str, pd.DataFrame] = {} - signal = config["signal"] - for security in universe: - selected = frame.loc[frame["security"] == security].copy() - selected = selected.sort_values("date").set_index("date", drop=False) - selected["n"] = turtle_n(selected, days=int(signal["n_days"])) - levels = breakout_levels( - selected, - entry_days=int(signal["entry_days"]), - exit_days=int(signal["exit_days"]), - ) - selected["entry_high"] = levels["entry_high"] - selected["exit_low"] = levels["exit_low"] - frames[security] = selected - close = frame.pivot(index="date", columns="security", values="close") - close = close.sort_index().apply(pd.to_numeric, errors="coerce") - returns = close.pct_change(fill_method=None) - return frames, returns, universe - - -def _quote(row: pd.Series) -> MarketQuote: - return MarketQuote( - open=_decimal(row["open"]), - paused=bool(row["paused"]), - high_limit=_decimal(row["high_limit"]), - low_limit=_decimal(row["low_limit"]), - ) - - -def _percent(value: Decimal) -> str: - return f"{(value * Decimal('100')).quantize(Decimal('0.01'))}%" - - -def _simulate( - *, - config: Mapping[str, object], - candidates: Sequence[Mapping[str, object]], - identity: RunIdentity, - snapshot_normalized_sha256: str, - rows: Sequence[Mapping[str, object]], -) -> ResearchResult: - frames, returns, securities = _prepare_frames(rows, config) - groups = { - str(item["security"]): str(item["asset_group"]) - for item in config["universe"] - } - dates = tuple(sorted(set(str(row["date"]) for row in rows))) - initial_cash = Decimal(str(config["research"]["initial_cash"])) - portfolio = PortfolioState(initial_cash, initial_cash) - pending: TradingDay | None = None - audit_rows: list[dict[str, object]] = [] - trade_rows: list[dict[str, object]] = [] - position_rows: list[dict[str, object]] = [] - risk_rows: list[dict[str, object]] = [] - leave_cash = Counter() - invested_ratios: list[Decimal] = [] - cash_ratios: list[Decimal] = [] - portfolio_risk_usage: list[Decimal] = [] - target_volatility_usage: list[Decimal] = [] - maximum_group_value: dict[str, Decimal] = {} - maximum_group_risk: dict[str, Decimal] = {} - last_close: dict[str, Decimal] = {} - - for index, current_date in enumerate(dates): - rows_today = { - security: frames[security].loc[current_date] - for security in securities - if current_date in frames[security].index - } - previous_date = dates[max(0, index - 1)] - open_prices = { - security: ( - None - if security not in rows_today - else _decimal(rows_today[security]["open"]) - ) - for security in securities - } - open_risk = _risk_inputs( - config=config, - securities=securities, - frames=frames, - returns=returns, - through_date=previous_date, - prices=open_prices, - ) - if pending is not None: - day_result = process_day( - pending, - portfolio, - DailyMarket( - quotes={ - security: _quote(row) for security, row in rows_today.items() - }, - risk_inputs=open_risk, - ), - ) - portfolio = day_result.portfolio - for item in day_result.audit: - row = { - "date": current_date, - **item.to_document(), - "allocation_sha256": day_result.allocation.audit_sha256, - } - audit_rows.append(row) - if item.status == "filled": - trade_rows.append( - { - "date": current_date, - "sequence": item.sequence, - "security": item.security, - "action": item.action, - "quantity": item.filled_quantity, - "fill_price": decimal_text(item.fill_price), - "reason": item.reason, - } - ) - else: - leave_cash[item.reason] += 1 - - close_prices = { - security: ( - None - if security not in rows_today - else _decimal(rows_today[security]["close"]) - ) - for security in securities - } - last_close.update( - { - security: value - for security, value in close_prices.items() - if value is not None - } - ) - equity = portfolio.cash + sum( - ( - last_close[position.security] * position.quantity - for position in portfolio.positions - if position.security in last_close - ), - Decimal("0"), - ) - portfolio = PortfolioState(equity, portfolio.cash, portfolio.positions) - close_risk = _risk_inputs( - config=config, - securities=securities, - frames=frames, - returns=returns, - through_date=current_date, - prices=close_prices, - ) - group_values: dict[str, Decimal] = {} - group_risks: dict[str, Decimal] = {} - for position in portfolio.positions: - close_price = last_close.get(position.security) - if close_price is None: - continue - value = close_price * position.quantity - group = position.asset_group - group_values[group] = group_values.get(group, Decimal("0")) + value - group_risks[group] = group_risks.get(group, Decimal("0")) + position.planned_loss - position_rows.append( - { - "date": current_date, - "security": position.security, - "asset_group": group, - "quantity": position.quantity, - "close": decimal_text(close_price), - "market_value": decimal_text(value), - "common_stop": decimal_text(position.common_stop), - "planned_loss": decimal_text(position.planned_loss), - } - ) - invested = sum(group_values.values(), Decimal("0")) / equity - cash_ratio = portfolio.cash / equity - planned_risk = sum(group_risks.values(), Decimal("0")) - risk_usage = planned_risk / ( - equity * Decimal(str(config["risk"]["portfolio_risk_cap"])) - ) - volatility = portfolio_volatility(portfolio, close_risk) - volatility_usage = ( - Decimal("0") - if volatility is None - else volatility / Decimal(str(config["risk"]["target_volatility"])) - ) - group_value_usage = { - group: value - / (equity * Decimal(str(config["risk"]["asset_group_value_cap"]))) - for group, value in group_values.items() - } - group_risk_usage = { - group: value - / (equity * Decimal(str(config["risk"]["asset_group_risk_cap"]))) - for group, value in group_risks.items() - } - for group, value in group_value_usage.items(): - maximum_group_value[group] = max(maximum_group_value.get(group, Decimal("0")), value) - for group, value in group_risk_usage.items(): - maximum_group_risk[group] = max(maximum_group_risk.get(group, Decimal("0")), value) - if invested < Decimal("1"): - leave_cash["risk_or_no_active_trend"] += 1 - invested_ratios.append(invested) - cash_ratios.append(cash_ratio) - portfolio_risk_usage.append(risk_usage) - target_volatility_usage.append(volatility_usage) - risk_rows.append( - { - "date": current_date, - "equity": decimal_text(equity), - "cash": decimal_text(portfolio.cash), - "invested_ratio": decimal_text(invested), - "cash_ratio": decimal_text(cash_ratio), - "portfolio_planned_risk": decimal_text(planned_risk), - "portfolio_risk_usage": decimal_text(risk_usage), - "portfolio_volatility": decimal_text(volatility), - "target_volatility_usage": decimal_text(volatility_usage), - "asset_group_value_usage": json.dumps( - {key: decimal_text(value) for key, value in group_value_usage.items()}, - sort_keys=True, - ), - "asset_group_risk_usage": json.dumps( - {key: decimal_text(value) for key, value in group_risk_usage.items()}, - sort_keys=True, - ), - "eligible_securities": json.dumps( - list( - () - if close_risk.covariance is None - else close_risk.covariance.securities - ), - sort_keys=True, - ), - "cold_start_securities": json.dumps( - [ - security - for security in securities - if close_risk.covariance is None - or security not in close_risk.covariance.securities - ], - sort_keys=True, - ), - "leave_cash_reasons": json.dumps(dict(sorted(leave_cash.items())), sort_keys=True), - } - ) - - pending = None - if index + 1 >= len(dates): - continue - next_date = dates[index + 1] - intents = list( - target_volatility_reductions( - portfolio, - close_risk, - signal_date=current_date, - execution_date=next_date, - reduction_target=Decimal( - str(config["risk"]["risk_reduction_target_volatility"]) - ), - ) - ) - positions = {position.security: position for position in portfolio.positions} - for security in securities: - if security not in rows_today: - continue - row = rows_today[security] - close = _decimal(row["close"]) - n_value = _decimal(row["n"]) - entry_high = _decimal(row["entry_high"]) - exit_low = _decimal(row["exit_low"]) - position = positions.get(security) - if close is None: - continue - if position is not None: - exit_intent = request_full_exit( - position, - signal_date=current_date, - execution_date=next_date, - close=close, - exit_level=exit_low, - expected_price=close, - ) - if exit_intent is not None: - intents.append( - replace( - exit_intent, - estimated_fee=commission_fee(close, exit_intent.quantity), - ) - ) - continue - requested, addition = request_addition( - position, - signal_date=current_date, - execution_date=next_date, - close=close, - expected_price=close, - ) - positions[security] = requested - if addition is not None: - intents.append( - replace( - addition, - estimated_fee=commission_fee(close, addition.quantity), - ) - ) - elif n_value is not None and entry_signal(close, entry_high): - quantity = initial_unit( - portfolio.equity, - n_value, - Decimal(str(config["risk"]["risk_per_unit"])), - ) - if quantity > 0: - entry = make_entry_intent( - security=security, - asset_group=groups[security], - signal_date=current_date, - execution_date=next_date, - expected_price=close, - quantity=quantity, - signal_n=n_value, - standard_unit=quantity, - stop_n=Decimal(str(config["signal"]["stop_n"])), - ) - intents.append( - replace( - entry, - estimated_fee=commission_fee(close, quantity), - ) - ) - portfolio = PortfolioState( - portfolio.equity, - portfolio.cash, - tuple(positions[security] for security in sorted(positions)), - ) - pending = TradingDay(date=next_date, intents=tuple(intents)) - - count = Decimal(len(invested_ratios)) - metrics = { - "audit_events": len(audit_rows), - "filled_trades": len(trade_rows), - "average_invested_ratio": _percent(sum(invested_ratios) / count), - "median_invested_ratio": _percent(statistics.median(invested_ratios)), - "below_half_ratio": _percent( - Decimal(sum(value < Decimal("0.5") for value in invested_ratios)) / count - ), - "near_full_ratio": _percent( - Decimal(sum(value >= Decimal("0.9") for value in invested_ratios)) / count - ), - "average_cash_ratio": _percent(sum(cash_ratios) / count), - "leave_cash_reasons": dict(sorted(leave_cash.items())), - "maximum_asset_group_value_usage": { - key: _percent(value) for key, value in sorted(maximum_group_value.items()) - }, - "maximum_asset_group_risk_usage": { - key: _percent(value) for key, value in sorted(maximum_group_risk.items()) - }, - "maximum_portfolio_risk_usage": _percent(max(portfolio_risk_usage)), - "maximum_target_volatility_usage": _percent(max(target_volatility_usage)), - } - return ResearchResult( - identity=identity, - snapshot_normalized_sha256=snapshot_normalized_sha256, - config=config, - candidates=tuple(candidates), - audit_rows=tuple(audit_rows), - trade_rows=tuple(trade_rows), - position_rows=tuple(position_rows), - risk_rows=tuple(risk_rows), - metrics=metrics, - recommendation="proceed_to_joinquant", - reasons=( - "deterministic_local_flow_completed", - "fixed_candidate_package_preserved", - "joinquant_formal_backtest_required", - ), - ) - - -def _write_status(output_dir: Path, status: str, reasons: Sequence[str]) -> None: - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "project-status.json").write_text( - json.dumps( - { - "schema_version": 1, - "status": status, - "reason_codes": list(reasons), - }, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - + "\n", - encoding="utf-8", - ) - - -def run_research( - config_path: Path, - snapshot_path: Path, - output_dir: Path, - *, - market_data_root: Path | None = None, - identity: RunIdentity | None = None, -) -> ProjectResult: - output_dir = Path(output_dir) - try: - if identity is None: - raise ResearchEvidenceInsufficient("missing_run_identity") - config = _load_object(config_path, "project_config") - candidates_document = _load_object( - Path(config_path).with_name("candidates.json"), - "candidate_config", - ) - candidates = candidates_document.get("candidates") - if not isinstance(candidates, list) or len(candidates) != 7: - raise ResearchEvidenceInsufficient("invalid_candidate_config") - snapshot_document = _load_object(snapshot_path, "snapshot") - if snapshot_document.get("snapshot_id") != identity.snapshot_id: - raise ResearchEvidenceInsufficient("snapshot_identity_mismatch") - root = ( - Path(market_data_root) - if market_data_root is not None - else Path(snapshot_path).resolve().parents[1] - ) - snapshot_view = open_snapshot(identity.snapshot_id, root=root) - result = _simulate( - config=config, - candidates=candidates, - identity=identity, - snapshot_normalized_sha256=snapshot_view.digest, - rows=snapshot_view.rows, - ) - write_outputs(result, output_dir) - validate_project_outputs(output_dir, identity) - _write_status(output_dir, "complete", ()) - return ProjectResult("complete", (), output_dir) - except ResearchEvidenceInsufficient as exc: - reason = str(exc) - _write_status(output_dir, "evidence_insufficient", (reason,)) - return ProjectResult("evidence_insufficient", (reason,), output_dir) - except (OutputValidationError, ValueError, KeyError, TypeError, ArithmeticError): - _write_status(output_dir, "failed", ("deterministic_research_failed",)) - return ProjectResult("failed", ("deterministic_research_failed",), output_dir) - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument("--snapshot-manifest", type=Path, required=True) - parser.add_argument("--market-data-root", type=Path, required=True) - parser.add_argument("--project-config", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--run-id", required=True) - parser.add_argument("--snapshot-id", required=True) - parser.add_argument("--code-sha256", required=True) - parser.add_argument("--config-sha256", required=True) - return parser - - -def main(argv: list[str] | None = None) -> int: - args = _parser().parse_args(argv) - identity = RunIdentity( - run_id=args.run_id, - snapshot_id=args.snapshot_id, - code_sha256=args.code_sha256, - config_sha256=args.config_sha256, - ) - result = run_research( - args.project_config, - args.snapshot_manifest, - args.output_dir, - market_data_root=args.market_data_root, - identity=identity, - ) - return {"complete": 0, "failed": 1, "evidence_insufficient": 2}[result.status] - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/execution.py b/joinquant/strategies/strategy-003/research/turtle_etf/execution.py deleted file mode 100644 index 8346ad3..0000000 --- a/joinquant/strategies/strategy-003/research/turtle_etf/execution.py +++ /dev/null @@ -1,351 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass, replace -from decimal import Decimal -from types import MappingProxyType -from typing import Literal, Mapping - -from .allocation import ( - AllocationResult, - BuyRequest, - PortfolioConstraints, - allocate_a1, -) -from .risk import PortfolioState, RiskInputs -from .state import ( - Batch, - OrderIntent, - TrendState, - _date, - _decimal, - apply_addition_fill, - apply_entry_fill, - commission_fee, -) - - -@dataclass(frozen=True) -class MarketQuote: - open: Decimal | None - paused: bool = False - high_limit: Decimal | None = None - low_limit: Decimal | None = None - - def __post_init__(self) -> None: - object.__setattr__( - self, - "open", - None if self.open is None else _decimal(self.open, "open", positive=True), - ) - object.__setattr__( - self, - "high_limit", - None - if self.high_limit is None - else _decimal(self.high_limit, "high_limit", positive=True), - ) - object.__setattr__( - self, - "low_limit", - None - if self.low_limit is None - else _decimal(self.low_limit, "low_limit", positive=True), - ) - if not isinstance(self.paused, bool): - raise ValueError("paused must be boolean") - - -@dataclass(frozen=True) -class TradingDay: - date: str - intents: tuple[OrderIntent, ...] - - def __post_init__(self) -> None: - object.__setattr__(self, "date", _date(self.date, "date")) - object.__setattr__(self, "intents", tuple(self.intents)) - if any(intent.execution_date != self.date for intent in self.intents): - raise ValueError("all intents must execute on the trading day") - - -@dataclass(frozen=True) -class DailyMarket: - quotes: Mapping[str, MarketQuote] - risk_inputs: RiskInputs - - def __post_init__(self) -> None: - normalized = {str(security): quote for security, quote in self.quotes.items()} - if any(not security or not isinstance(quote, MarketQuote) for security, quote in normalized.items()): - raise ValueError("daily market quotes are invalid") - object.__setattr__(self, "quotes", MappingProxyType(normalized)) - - -ExecutionStatus = Literal["filled", "unfilled", "cancelled"] - - -@dataclass(frozen=True) -class ExecutionRecord: - sequence: int - security: str - action: str - status: ExecutionStatus - requested_quantity: int - filled_quantity: int - fill_price: Decimal | None - reason: str - - def to_document(self) -> dict[str, object]: - return { - "sequence": self.sequence, - "security": self.security, - "action": self.action, - "status": self.status, - "requested_quantity": self.requested_quantity, - "filled_quantity": self.filled_quantity, - "fill_price": None if self.fill_price is None else str(self.fill_price), - "reason": self.reason, - } - - -@dataclass(frozen=True) -class DayResult: - portfolio: PortfolioState - audit: tuple[ExecutionRecord, ...] - allocation: AllocationResult - audit_sha256: str - - -def _sell_fill_reason(quote: MarketQuote | None) -> str | None: - if quote is None or quote.open is None: - return "missing_open" - if quote.paused: - return "paused" - if quote.low_limit is not None and quote.open <= quote.low_limit: - return "low_limit" - return None - - -def _buy_fill_reason(quote: MarketQuote | None) -> str | None: - if quote is None or quote.open is None: - return "missing_open" - if quote.paused: - return "paused" - if quote.high_limit is not None and quote.open >= quote.high_limit: - return "high_limit" - return None - - -def _reduce_position(position: TrendState, quantity: int) -> TrendState | None: - remaining = min(quantity, position.quantity) - kept_reversed: list[Batch] = [] - for batch in reversed(position.batches): - removed = min(remaining, batch.quantity) - kept = batch.quantity - removed - remaining -= removed - if kept: - kept_reversed.append(replace(batch, quantity=kept)) - batches = tuple(reversed(kept_reversed)) - return None if not batches else replace(position, batches=batches) - - -def _adjust_buy_to_open( - intent: OrderIntent, - quote: MarketQuote, - positions: Mapping[str, TrendState], -) -> OrderIntent: - distance = intent.expected_price - intent.common_stop_after - common_stop = quote.open - distance - existing = positions.get(intent.security) - if existing is not None: - common_stop = max(existing.common_stop, common_stop) - return replace( - intent, - expected_price=quote.open, - common_stop_after=common_stop, - estimated_fee=commission_fee(quote.open, intent.quantity), - ) - - -def _audit_digest(records: tuple[ExecutionRecord, ...]) -> str: - payload = json.dumps( - [record.to_document() for record in records], - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - return hashlib.sha256(payload).hexdigest() - - -def process_day( - day: TradingDay, - state: PortfolioState, - market: DailyMarket, -) -> DayResult: - positions = {position.security: position for position in state.positions} - cash = state.cash - audit: list[ExecutionRecord] = [] - - def record( - intent: OrderIntent, - status: ExecutionStatus, - *, - filled_quantity: int = 0, - fill_price: Decimal | None = None, - reason: str, - ) -> None: - audit.append( - ExecutionRecord( - sequence=len(audit) + 1, - security=intent.security, - action=intent.action, - status=status, - requested_quantity=intent.quantity, - filled_quantity=filled_quantity, - fill_price=fill_price, - reason=reason, - ) - ) - - exits = tuple( - sorted( - (intent for intent in day.intents if intent.action == "full_exit"), - key=lambda item: item.security, - ) - ) - exit_securities = {intent.security for intent in exits} - for intent in exits: - quote = market.quotes.get(intent.security) - reason = _sell_fill_reason(quote) - position = positions.get(intent.security) - if reason is not None or position is None: - record(intent, "unfilled", reason=reason or "position_missing") - continue - quantity = position.quantity - cash += quote.open * quantity - commission_fee(quote.open, quantity) - del positions[intent.security] - record( - intent, - "filled", - filled_quantity=quantity, - fill_price=quote.open, - reason="full_exit", - ) - - reductions = tuple( - sorted( - ( - intent - for intent in day.intents - if intent.action == "mandatory_risk_reduction" - ), - key=lambda item: item.security, - ) - ) - for intent in reductions: - if intent.security in exit_securities: - record(intent, "cancelled", reason="full_exit_precedence") - continue - quote = market.quotes.get(intent.security) - reason = _sell_fill_reason(quote) - position = positions.get(intent.security) - if reason is not None or position is None: - record(intent, "unfilled", reason=reason or "position_missing") - continue - quantity = min(intent.quantity, position.quantity) - reduced = _reduce_position(position, quantity) - if reduced is None: - del positions[intent.security] - else: - positions[intent.security] = reduced - cash += quote.open * quantity - commission_fee(quote.open, quantity) - record( - intent, - "filled", - filled_quantity=quantity, - fill_price=quote.open, - reason="mandatory_risk_reduction", - ) - - candidate_intents: list[OrderIntent] = [] - original_buys = tuple( - sorted( - ( - intent - for intent in day.intents - if intent.action in {"entry", "addition"} - ), - key=lambda item: (item.security, item.action), - ) - ) - for intent in original_buys: - if intent.security in exit_securities: - record(intent, "cancelled", reason="full_exit_precedence") - continue - quote = market.quotes.get(intent.security) - reason = _buy_fill_reason(quote) - if reason is not None: - record(intent, "unfilled", reason=reason) - continue - candidate_intents.append(_adjust_buy_to_open(intent, quote, positions)) - - current_state = PortfolioState( - equity=state.equity, - cash=cash, - positions=tuple(positions[security] for security in sorted(positions)), - ) - prices = dict(market.risk_inputs.prices) - for security in set(positions) | {intent.security for intent in candidate_intents}: - quote = market.quotes.get(security) - prices[security] = None if quote is None else quote.open - risk_inputs = replace(market.risk_inputs, prices=prices) - allocation = allocate_a1( - tuple(BuyRequest(intent) for intent in candidate_intents), - PortfolioConstraints(state=current_state, risk_inputs=risk_inputs), - ) - allocated = {intent.security: intent for intent in allocation.allocations} - for intent in candidate_intents: - fill = allocated.get(intent.security) - if fill is None: - record(intent, "unfilled", reason="allocation_constraint") - continue - if fill.action == "entry": - positions[fill.security] = apply_entry_fill( - security=fill.security, - asset_group=fill.asset_group, - execution_date=day.date, - fill_price=fill.expected_price, - quantity=fill.quantity, - signal_n=fill.signal_n, - standard_unit=fill.standard_unit, - ) - else: - positions[fill.security] = apply_addition_fill( - positions[fill.security], - fill, - execution_date=day.date, - fill_price=fill.expected_price, - quantity=fill.quantity, - ) - cash -= fill.expected_price * fill.quantity + fill.estimated_fee - record( - fill, - "filled", - filled_quantity=fill.quantity, - fill_price=fill.expected_price, - reason="a1_allocation", - ) - - portfolio = PortfolioState( - equity=state.equity, - cash=cash, - positions=tuple(positions[security] for security in sorted(positions)), - ) - records = tuple(audit) - return DayResult( - portfolio=portfolio, - audit=records, - allocation=allocation, - audit_sha256=_audit_digest(records), - ) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/reporting.py b/joinquant/strategies/strategy-003/research/turtle_etf/reporting.py deleted file mode 100644 index 5e8efc5..0000000 --- a/joinquant/strategies/strategy-003/research/turtle_etf/reporting.py +++ /dev/null @@ -1,395 +0,0 @@ -from __future__ import annotations - -import csv -import hashlib -import json -import re -from dataclasses import dataclass -from decimal import Decimal -from pathlib import Path -from types import MappingProxyType -from typing import Mapping, Sequence - - -_SHA256 = re.compile(r"[0-9a-f]{64}") -_RECOMMENDATIONS = { - "proceed_to_joinquant", - "revise_and_reassess", - "stop_evidence_insufficient", -} -_CANDIDATE_IDS = ( - "baseline", - "entry-40", - "entry-60", - "stop-1.5n", - "stop-2.5n", - "covariance-120d", - "covariance-ewma-30d", -) -_DISCLAIMER = "本地结果不是正式回测或最终验收结论。" -_REPORT_DIGEST_PREFIX = "\n", - encoding="utf-8", - ) - return MappingProxyType( - { - name: _file_digest(output_dir / name) - for name in (*digests, "research-report.md") - } - ) - - -def _read_json_object(path: Path) -> dict[str, object]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise OutputValidationError(f"invalid project output: {path.name}") from exc - if not isinstance(value, dict): - raise OutputValidationError(f"project output must be an object: {path.name}") - digest = value.get("document_sha256") - if not isinstance(digest, str) or digest != _semantic_digest(value): - raise OutputValidationError(f"project output digest mismatch: {path.name}") - return value - - -def validate_project_outputs(output_dir: Path, identity: RunIdentity) -> None: - output_dir = Path(output_dir) - conclusion = _read_json_object(output_dir / "conclusion.json") - candidates = _read_json_object(output_dir / "candidate-strategies.json") - expected_identity = identity.to_document() - if conclusion.get("identity") != expected_identity: - raise OutputValidationError("conclusion identity mismatch") - if conclusion.get("recommendation") not in _RECOMMENDATIONS: - raise OutputValidationError("conclusion recommendation is invalid") - if conclusion.get("disclaimer") != _DISCLAIMER: - raise OutputValidationError("conclusion disclaimer is missing") - if candidates.get("identity") != expected_identity: - raise OutputValidationError("candidate identity mismatch") - items = candidates.get("candidates") - if ( - not isinstance(items, list) - or any(not isinstance(item, dict) for item in items) - or tuple(item.get("id") for item in items) != _CANDIDATE_IDS - ): - raise OutputValidationError("candidate set differs from the frozen seven") - for item in items: - if ( - not isinstance(item, dict) - or item.get("snapshot_id") != identity.snapshot_id - or item.get("code_sha256") != identity.code_sha256 - or item.get("config_sha256") != identity.config_sha256 - or "rank" in item - or "score" in item - ): - raise OutputValidationError("candidate evidence is invalid") - try: - report = (output_dir / "research-report.md").read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError) as exc: - raise OutputValidationError("research report is missing or invalid") from exc - marker_start = report.rfind("\n" + _REPORT_DIGEST_PREFIX) - if marker_start < 0 or not report.endswith(" -->\n"): - raise OutputValidationError("research report digest marker is missing") - body = report[:marker_start] - declared = report[ - marker_start + len("\n" + _REPORT_DIGEST_PREFIX) : -len(" -->\n") - ] - if hashlib.sha256(body.encode("utf-8")).hexdigest() != declared: - raise OutputValidationError("research report digest mismatch") - if _DISCLAIMER not in body: - raise OutputValidationError("research report boundary is missing") - - -def decimal_text(value: Decimal | float | int | None) -> str: - if value is None: - return "" - return format(Decimal(str(value)), "f") diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py b/joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py new file mode 100644 index 0000000..1d86a8e --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/result_adapter.py @@ -0,0 +1,1650 @@ +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import os +import shutil +import uuid +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Mapping + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +from scripts.research.analysis_data import open_analysis_source + +from .vectorbt_callbacks import ( + ACTION_ADDITION, + ACTION_ENTRY, + ACTION_FULL_EXIT, + ACTION_NONE, + ACTION_REDISTRIBUTION_BUY, + ACTION_REDISTRIBUTION_SELL, + REASON_ALLOCATION_CONSTRAINT, + REASON_ENTRY_BREAKOUT, + REASON_FIXED_ADDITION_LEVEL, + REASON_FULL_POSITION_REDISTRIBUTION, + REASON_HIGH_LIMIT, + REASON_LOW_LIMIT, + REASON_MISSING_OPEN, + REASON_NONE, + REASON_ORDER_REJECTED, + REASON_PAUSED, + REASON_PROTECTIVE_STOP, + REASON_TREND_EXIT, +) +from .vectorbt_delayed import ( + ADJUST_CASH_TRUNCATED, + ADJUST_HOLDING_TRUNCATED, + ADJUST_HORIZON_EXPIRED, + ADJUST_NONE, + ADJUST_UNTRADABLE, +) + + +ATTRIBUTION_SCHEMA_VERSION = "turtle-etf-attribution/2" +ATTRIBUTION_FIELDS = ( + "time", + "event_id", + "scope", + "security", + "event_type", + "reason_code", + "requested_amount", + "executed_amount", + "reference_price", + "risk_before", + "risk_after", + "details_json", +) + +_REASON_CODES = { + "signal_entry", + "signal_add", + "signal_exit", + "protective_stop", + "full_position_redistribution", + "allocation_constraint", + "untradeable", + "order_rejected", + "state_update", + "corporate_action_applied", +} +_EVENT_TYPES = {"decision", "state", "valuation", "corporate_action"} + +_ACCOUNTING_CONTRACT = { + "version": "turtle-etf-corporate-actions/1", + "corporate_action_mode": "point_in_time_total_return_approximation", + "continuity_factor_basis": "raw_previous_close_over_current_pre_close", + "corporate_action_metadata_timing": "audit_only_may_be_retrospective", + "price_basis": "continuous_economic_price", + "quantity_basis": "economic_units", + "cash_dividend_mode": "implicit_reinvestment_on_ex_date", + "pay_date_cash_supported": False, + "exact_joinquant_reconciliation": False, +} + +_ACTION_NAMES = { + ACTION_NONE: "none", + ACTION_FULL_EXIT: "full_exit", + ACTION_REDISTRIBUTION_SELL: "redistribution_sell", + ACTION_ENTRY: "entry", + ACTION_ADDITION: "addition", + ACTION_REDISTRIBUTION_BUY: "redistribution_buy", +} +_REASON_NAMES = { + REASON_NONE: "none", + REASON_ENTRY_BREAKOUT: "entry_breakout", + REASON_FIXED_ADDITION_LEVEL: "fixed_addition_level", + REASON_PROTECTIVE_STOP: "protective_stop", + REASON_TREND_EXIT: "trend_exit", + REASON_FULL_POSITION_REDISTRIBUTION: "full_position_redistribution", + REASON_MISSING_OPEN: "missing_open", + REASON_PAUSED: "paused", + REASON_HIGH_LIMIT: "high_limit", + REASON_LOW_LIMIT: "low_limit", + REASON_ALLOCATION_CONSTRAINT: "allocation_constraint", + REASON_ORDER_REJECTED: "order_rejected", +} +_SELL_ACTIONS = {"full_exit", "redistribution_sell"} +_ADJUSTMENT_NAMES = { + ADJUST_NONE: "none", + ADJUST_CASH_TRUNCATED: "cash_truncated", + ADJUST_HOLDING_TRUNCATED: "holding_truncated", + ADJUST_UNTRADABLE: "untradable", + ADJUST_HORIZON_EXPIRED: "horizon_expired", +} + +_RESULTS_SCHEMA = pa.schema( + [ + pa.field("benchmark_returns", pa.float64()), + pa.field("returns", pa.float64(), nullable=False), + pa.field("time", pa.string(), nullable=False), + ] +) +_BALANCES_SCHEMA = pa.schema( + [ + pa.field("total_value", pa.float64(), nullable=False), + pa.field("net_value", pa.float64(), nullable=False), + pa.field("cash", pa.float64(), nullable=False), + pa.field("aval_cash", pa.float64(), nullable=False), + pa.field("time", pa.string(), nullable=False), + ] +) +_POSITIONS_SCHEMA = pa.schema( + [ + pa.field("pindex", pa.int64(), nullable=False), + pa.field("avg_cost", pa.float64(), nullable=False), + pa.field("margin", pa.float64(), nullable=False), + pa.field("amount", pa.float64(), nullable=False), + pa.field("today_amount", pa.int64(), nullable=False), + pa.field("hold_cost", pa.float64(), nullable=False), + pa.field("side", pa.string(), nullable=False), + pa.field("price", pa.float64(), nullable=False), + pa.field("gains", pa.float64(), nullable=False), + pa.field("daily_gains", pa.float64(), nullable=False), + pa.field("closeable_amount", pa.int64(), nullable=False), + pa.field("time", pa.string(), nullable=False), + pa.field("security_name", pa.string(), nullable=False), + pa.field("security", pa.string(), nullable=False), + ] +) +_ORDERS_SCHEMA = pa.schema( + [ + pa.field("match_time", pa.string()), + pa.field("pindex", pa.int64(), nullable=False), + pa.field("cancel_time", pa.string()), + pa.field("action", pa.string(), nullable=False), + pa.field("limit_price", pa.float64(), nullable=False), + pa.field("comment", pa.string(), nullable=False), + pa.field("entrust_time", pa.string(), nullable=False), + pa.field("finish_time", pa.string()), + pa.field("side", pa.string(), nullable=False), + pa.field("price", pa.float64(), nullable=False), + pa.field("commission", pa.float64(), nullable=False), + pa.field("gains", pa.float64(), nullable=False), + pa.field("type", pa.string(), nullable=False), + pa.field("time", pa.string(), nullable=False), + pa.field("security_name", pa.string(), nullable=False), + pa.field("security", pa.string(), nullable=False), + pa.field("filled", pa.int64(), nullable=False), + pa.field("amount", pa.int64(), nullable=False), + pa.field("status", pa.string(), nullable=False), + ] +) +_ATTRIBUTION_SCHEMA = pa.schema( + [ + pa.field("time", pa.string(), nullable=False), + pa.field("event_id", pa.string(), nullable=False), + pa.field("scope", pa.string(), nullable=False), + pa.field("security", pa.string()), + pa.field("event_type", pa.string(), nullable=False), + pa.field("reason_code", pa.string(), nullable=False), + pa.field("requested_amount", pa.float64()), + pa.field("executed_amount", pa.float64()), + pa.field("reference_price", pa.float64()), + pa.field("risk_before", pa.float64()), + pa.field("risk_after", pa.float64()), + pa.field("details_json", pa.string(), nullable=False), + ] +) + + +class ResultContractError(ValueError): + """Raised when vectorbt facts cannot prove the local result contract.""" + + +@dataclass(frozen=True) +class LocalExecutionFacts: + results: pa.Table + balances: pa.Table + positions: pa.Table + orders: pa.Table + attribution: pa.Table + + def with_attribution(self, document: Mapping[str, object]) -> LocalExecutionFacts: + return replace( + self, + attribution=pa.Table.from_pydict(dict(document), schema=_ATTRIBUTION_SCHEMA), + ) + + +@dataclass(frozen=True) +class LocalResultPackage: + root: Path + params_sha256: str + attribution_sha256: str + + +def _table(rows: list[dict[str, object]], schema: pa.Schema) -> pa.Table: + if not rows: + return pa.Table.from_pylist([], schema=schema) + return pa.Table.from_pylist(rows, schema=schema) + + +def _vector(value: object, rows: int, field: str) -> np.ndarray: + array = np.asarray(value, dtype=np.float64).reshape(-1) + if array.shape != (rows,) or not np.all(np.isfinite(array)): + raise ResultContractError(f"portfolio {field} is invalid") + return array + + +def _matrix(value: object, shape: tuple[int, int], field: str) -> np.ndarray: + array = np.asarray(value) + if array.shape != shape: + raise ResultContractError(f"simulation {field} shape is invalid") + return array + + +def _date_text(value: object) -> str: + return str(np.datetime64(value, "D")) + + +def _nullable(value: float) -> float | None: + return float(value) if np.isfinite(value) else None + + +def _safe_simulation_number(value: object) -> float | None: + if value is None: + return None + try: + numeric = float(value) + except (TypeError, ValueError): + return None + return numeric if np.isfinite(numeric) else None + + +def _changed(left: float, right: float) -> bool: + if np.isnan(left) and np.isnan(right): + return False + return bool(left != right) + + +def _event_id( + scenario_id: str, + date_text: str, + security: str, + event_type: str, + reason_code: str, +) -> str: + identity = "|".join( + ( + ATTRIBUTION_SCHEMA_VERSION, + scenario_id, + date_text, + security, + event_type, + reason_code, + ) + ) + return hashlib.sha256(identity.encode("utf-8")).hexdigest() + + +def _reason_code(action: str, source_reason: str) -> str: + if source_reason == "order_rejected": + return "order_rejected" + if source_reason in {"missing_open", "paused", "high_limit", "low_limit"}: + return "untradeable" + if source_reason == "allocation_constraint": + return "allocation_constraint" + if source_reason == "protective_stop": + return "protective_stop" + if action in {"redistribution_sell", "redistribution_buy"} or ( + source_reason == "full_position_redistribution" + ): + return "full_position_redistribution" + if action == "full_exit" or source_reason == "trend_exit": + return "signal_exit" + if action == "entry": + return "signal_entry" + if action == "addition": + return "signal_add" + return "state_update" + + +def _planned_risk(quantity: int, average_cost: float, common_stop: float) -> float | None: + if quantity <= 0: + return 0.0 + if not np.isfinite(average_cost) or not np.isfinite(common_stop): + return None + return max(float(average_cost) - float(common_stop), 0.0) * int(quantity) + + +def _json_value(value: object) -> object: + if isinstance(value, (np.bool_, bool)): + return bool(value) + if isinstance(value, (np.integer, int)): + return int(value) + if isinstance(value, (np.floating, float)): + return float(value) if np.isfinite(value) else None + return value + + +def _details(**values: object) -> str: + return json.dumps( + {key: _json_value(value) for key, value in values.items()}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + +def to_joinquant_facts( + inputs: object, + simulation: object, + scenario_id: str, +) -> LocalExecutionFacts: + if not isinstance(scenario_id, str) or not scenario_id: + raise ResultContractError("scenario_id is required") + dates = np.asarray(getattr(inputs, "dates"), dtype="datetime64[D]") + securities = tuple(str(item) for item in getattr(inputs, "securities")) + close = np.asarray(getattr(inputs, "close"), dtype=np.float64) + rows, columns = close.shape + if dates.shape != (rows,) or len(securities) != columns: + raise ResultContractError("input identities are inconsistent") + shape = (rows, columns) + actions = _matrix(simulation.action_codes, shape, "action_codes").astype(np.int64) + reasons = _matrix(simulation.reason_codes, shape, "reason_codes").astype(np.int64) + requested = _matrix( + simulation.requested_quantities, shape, "requested_quantities" + ).astype(np.int64) + planned = _matrix(simulation.planned_quantities, shape, "planned_quantities").astype( + np.int64 + ) + filled = _matrix(simulation.filled_quantities, shape, "filled_quantities").astype( + np.int64 + ) + fill_prices = _matrix(simulation.fill_prices, shape, "fill_prices").astype( + np.float64 + ) + fees = _matrix(simulation.fees, shape, "fees").astype(np.float64) + state_quantities = _matrix( + simulation.state_quantities, shape, "state_quantities" + ).astype(np.int64) + common_stops = _matrix( + simulation.state_common_stop, shape, "state_common_stop" + ).astype(np.float64) + next_add = _matrix( + simulation.state_next_add_index, shape, "state_next_add_index" + ).astype(np.int64) + unit_counts = _matrix( + getattr(simulation, "state_unit_counts", next_add), + shape, + "state_unit_counts", + ).astype(np.int64) + candidate_base_quantities = _matrix( + getattr( + simulation, + "candidate_base_quantities", + np.where( + np.isin(actions, [ACTION_ENTRY, ACTION_ADDITION]), + requested, + 0, + ), + ), + shape, + "candidate_base_quantities", + ).astype(np.int64) + event_group_scales = _matrix( + getattr( + simulation, + "event_group_scales", + np.ones(shape, dtype=np.float64), + ), + shape, + "event_group_scales", + ).astype(np.float64) + event_portfolio_scales = _vector( + getattr( + simulation, + "event_portfolio_scales", + np.ones(rows, dtype=np.float64), + ), + rows, + "event_portfolio_scales", + ) + event_cash_scales = _vector( + getattr( + simulation, + "event_cash_scales", + np.ones(rows, dtype=np.float64), + ), + rows, + "event_cash_scales", + ) + portfolio_unit_cap = _safe_simulation_number( + getattr(simulation, "portfolio_unit_cap", None) + ) + raw_signal_n = getattr(inputs, "signal_n", None) + signal_n = ( + np.full(shape, np.nan, dtype=np.float64) + if raw_signal_n is None + else _matrix(raw_signal_n, shape, "signal_n").astype(np.float64) + ) + row_indices = np.broadcast_to( + np.arange(rows, dtype=np.int64)[:, None], shape + ) + planned_row_indices = _matrix( + getattr( + simulation, + "planned_row_indices", + np.where(actions != ACTION_NONE, row_indices, -1), + ), + shape, + "planned_row_indices", + ).astype(np.int64) + adjustments = _matrix( + getattr( + simulation, + "execution_adjustment_codes", + np.zeros(shape, dtype=np.int16), + ), + shape, + "execution_adjustment_codes", + ).astype(np.int64) + frozen_signal_n = _matrix( + getattr(simulation, "frozen_signal_n", signal_n), + shape, + "frozen_signal_n", + ).astype(np.float64) + execution_delay_days = int(getattr(simulation, "execution_delay_days", 0)) + if np.any(requested < 0) or np.any(planned < 0) or np.any(filled < 0): + raise ResultContractError("simulation quantities must be non-negative") + if not np.all(np.isfinite(fees)) or np.any(fees < 0.0): + raise ResultContractError("simulation commissions must be finite and non-negative") + if any(int(value) not in _ACTION_NAMES for value in np.unique(actions)): + raise ResultContractError("simulation action code is unknown") + if any(int(value) not in _REASON_NAMES for value in np.unique(reasons)): + raise ResultContractError("simulation reason code is unknown") + if any(int(value) not in _ADJUSTMENT_NAMES for value in np.unique(adjustments)): + raise ResultContractError("simulation execution adjustment code is unknown") + if execution_delay_days < 0: + raise ResultContractError("simulation execution delay is invalid") + active_plan_rows = planned_row_indices[actions != ACTION_NONE] + if np.any(active_plan_rows < 0) or np.any(active_plan_rows >= rows): + raise ResultContractError("simulation planned row is invalid") + + values = _vector(simulation.portfolio.value(), rows, "value") + cash = _vector(simulation.portfolio.cash(), rows, "cash") + initial_cash = float(getattr(simulation, "initial_cash", np.nan)) + if initial_cash <= 0.0 or not np.isfinite(initial_cash): + raise ResultContractError("initial cash could not be reconciled") + average_cost = np.zeros(columns, dtype=np.float64) + previous_quantity = np.zeros(columns, dtype=np.int64) + previous_close = np.full(columns, np.nan, dtype=np.float64) + previous_stop = np.full(columns, np.nan, dtype=np.float64) + previous_next_add = np.zeros(columns, dtype=np.int64) + previous_unit_count = np.zeros(columns, dtype=np.int64) + order_rows: list[dict[str, object]] = [] + position_rows: list[dict[str, object]] = [] + attribution_rows: list[dict[str, object]] = [] + valid_dates = {_date_text(value) for value in dates} + for application in getattr(inputs, "corporate_action_applications", ()): + effective_date = str(getattr(application, "effective_date", "")) + application_date = str( + getattr(application, "application_date", effective_date) + ) + security = str(getattr(application, "security", "")) + source_event_id = str(getattr(application, "source_event_id", "")) + if ( + effective_date not in valid_dates + or application_date not in valid_dates + or security not in securities + or not source_event_id + ): + raise ResultContractError( + "corporate-action attribution identity is invalid" + ) + attribution_rows.append( + { + "time": f"{application_date} 00:00:00", + "event_id": _event_id( + scenario_id, + application_date, + security, + "corporate_action", + f"corporate_action_applied:{source_event_id}", + ), + "scope": "security", + "security": security, + "event_type": "corporate_action", + "reason_code": "corporate_action_applied", + "requested_amount": None, + "executed_amount": None, + "reference_price": None, + "risk_before": None, + "risk_after": None, + "details_json": _details( + source_event_id=source_event_id, + event_type=str(getattr(application, "event_type", "")), + effective_date=effective_date, + application_date=application_date, + announcement_date=str( + getattr(application, "announcement_date", "") + ), + knowledge_cutoff_date=str( + getattr(application, "knowledge_cutoff_date", "") + ), + evidence_timing=str( + getattr( + application, + "evidence_timing", + "retrospective_reconciliation" + if str(getattr(application, "announcement_date", "")) + > effective_date + else "point_in_time", + ) + ), + split_ratio=getattr(application, "split_ratio", None), + cash_per_share=getattr(application, "cash_per_share", None), + cumulative_factor=float( + getattr(application, "cumulative_factor", np.nan) + ), + price_basis_changed=bool( + getattr(application, "price_basis_changed", True) + ), + source=str(getattr(application, "source", "")), + source_record_sha256=str( + getattr(application, "source_record_sha256", "") + ), + corporate_action_mode=( + "point_in_time_total_return_approximation" + ), + ), + } + ) + + for expired in getattr(simulation, "horizon_expired_orders", ()): + planned_row = int(getattr(expired, "planned_row_index")) + column = int(getattr(expired, "column")) + if not 0 <= planned_row < rows or not 0 <= column < columns: + raise ResultContractError("horizon-expired order identity is invalid") + date_text = _date_text(dates[planned_row]) + security = securities[column] + action = _ACTION_NAMES[int(getattr(expired, "action_code"))] + source_reason = _REASON_NAMES[int(getattr(expired, "reason_code"))] + reason_code = _reason_code(action, source_reason) + target = int(getattr(expired, "target_quantity")) + requested_quantity = int(getattr(expired, "requested_quantity")) + delay_days = int(getattr(expired, "delay_days")) + attribution_rows.append( + { + "time": f"{date_text} 09:30:00", + "event_id": _event_id( + scenario_id, + date_text, + security, + "decision", + reason_code + ":horizon_expired", + ), + "scope": "security", + "security": security, + "event_type": "decision", + "reason_code": reason_code, + "requested_amount": float(requested_quantity), + "executed_amount": 0.0, + "reference_price": _nullable(float(close[planned_row, column])), + "risk_before": None, + "risk_after": None, + "details_json": _details( + action=action, + source_reason=source_reason, + planned_date=date_text, + execution_date=None, + delay_days=delay_days, + frozen_reason=source_reason, + frozen_target_amount=target, + frozen_signal_n=float(getattr(expired, "signal_n")), + execution_adjustment="horizon_expired", + planned_amount=target, + state_changed=False, + ), + } + ) + + for row in range(rows): + date_text = _date_text(dates[row]) + execution_time = f"{date_text} 09:30:00" + balance_time = f"{date_text} 16:00:00" + today_buys = np.zeros(columns, dtype=np.int64) + valuation_rows: list[tuple[dict[str, object], dict[str, object]]] = [] + daily_security_pnl_total = 0.0 + for column, security in enumerate(securities): + action = _ACTION_NAMES[int(actions[row, column])] + source_reason = _REASON_NAMES[int(reasons[row, column])] + reason_code = _reason_code(action, source_reason) + adjustment = _ADJUSTMENT_NAMES[int(adjustments[row, column])] + planned_row = int(planned_row_indices[row, column]) + planned_date = ( + _date_text(dates[planned_row]) if planned_row >= 0 else date_text + ) + entrust_time = f"{planned_date} 09:30:00" + quantity = int(filled[row, column]) + frozen_target = max(int(planned[row, column]), quantity) + before = int(previous_quantity[column]) + before_cost = float(average_cost[column]) + fee = float(fees[row, column]) + close_price = float(close[row, column]) + if quantity > 0 and action == "none": + raise ResultContractError("filled order has no action") + is_sell = action in _SELL_ACTIONS + realized_gains = 0.0 + if quantity > 0: + price = float(fill_prices[row, column]) + if not np.isfinite(price) or price <= 0.0 or fee < 0.0: + raise ResultContractError("filled order price or commission is invalid") + if is_sell: + if quantity > before: + raise ResultContractError("sell order exceeds the held position") + realized_gains = (price - before_cost) * quantity - fee + expected_after = before - quantity + if expected_after == 0: + average_cost[column] = 0.0 + else: + expected_after = before + quantity + average_cost[column] = ( + before * before_cost + quantity * price + ) / expected_after + today_buys[column] = quantity + if int(state_quantities[row, column]) != expected_after: + raise ResultContractError("filled order and position state do not reconcile") + order_rows.append( + { + "match_time": execution_time, + "pindex": 0, + "cancel_time": None, + "action": "close" if is_sell else "open", + "limit_price": 0.0, + "comment": "" if adjustment == "none" else adjustment, + "entrust_time": entrust_time, + "finish_time": execution_time, + "side": "long", + "price": price, + "commission": fee, + "gains": realized_gains, + "type": "market", + "time": execution_time, + "security_name": security, + "security": security, + "filled": quantity, + "amount": frozen_target, + "status": "done", + } + ) + elif int(state_quantities[row, column]) != before: + raise ResultContractError("position changed without a filled order") + elif action != "none" and ( + source_reason == "order_rejected" or adjustment != "none" + ): + amount = max( + int(requested[row, column]), int(planned[row, column]) + ) + if amount <= 0: + raise ResultContractError("rejected order has no requested amount") + order_rows.append( + { + "match_time": None, + "pindex": 0, + "cancel_time": execution_time, + "action": "close" if is_sell else "open", + "limit_price": 0.0, + "comment": ( + source_reason + if adjustment == "none" + else adjustment + ), + "entrust_time": entrust_time, + "finish_time": None, + "side": "long", + "price": 0.0, + "commission": 0.0, + "gains": 0.0, + "type": "market", + "time": execution_time, + "security_name": security, + "security": security, + "filled": 0, + "amount": amount, + "status": "canceled", + } + ) + + after = int(state_quantities[row, column]) + stop_after = float(common_stops[row, column]) + next_after = int(next_add[row, column]) + units_after = int(unit_counts[row, column]) + state_changed = ( + before != after + or _changed(float(previous_stop[column]), stop_after) + or int(previous_next_add[column]) != next_after + ) + logical_state_changed = ( + _changed(float(previous_stop[column]), stop_after) + or int(previous_next_add[column]) != next_after + or int(previous_unit_count[column]) != units_after + ) + effective_risk_units = float( + np.sum( + unit_counts[row].astype(np.float64) + * event_group_scales[row] + ) + * event_portfolio_scales[row] + ) + risk_before = _planned_risk( + before, before_cost, float(previous_stop[column]) + ) + risk_after = _planned_risk( + after, float(average_cost[column]), stop_after + ) + if action != "none" or source_reason != "none" or state_changed: + event_type = ( + "decision" + if action != "none" or source_reason != "none" + else "state" + ) + attribution_rows.append( + { + "time": execution_time, + "event_id": _event_id( + scenario_id, date_text, security, event_type, reason_code + ), + "scope": "security", + "security": security, + "event_type": event_type, + "reason_code": reason_code, + "requested_amount": float(requested[row, column]), + "executed_amount": float(quantity), + "reference_price": _nullable( + float(fill_prices[row, column]) + if quantity > 0 + else close_price + ), + "risk_before": risk_before, + "risk_after": risk_after, + "details_json": _details( + action=action, + source_reason=source_reason, + planned_amount=int(planned[row, column]), + commission=fee, + position_before=before, + position_after=after, + average_cost_before=before_cost, + average_cost_after=float(average_cost[column]), + common_stop_before=float(previous_stop[column]), + common_stop_after=stop_after, + next_add_before=int(previous_next_add[column]), + next_add_after=next_after, + unit_count_before=int(previous_unit_count[column]), + unit_count_after=units_after, + candidate_base_quantity=int( + candidate_base_quantities[row, column] + ), + frozen_signal_n=float( + frozen_signal_n[row, column] + ), + actual_fill_price=( + float(fill_prices[row, column]) + if quantity > 0 + else np.nan + ), + group_scale=float( + event_group_scales[row, column] + ), + portfolio_scale=float( + event_portfolio_scales[row] + ), + cash_scale=float(event_cash_scales[row]), + effective_risk_units=effective_risk_units, + portfolio_unit_cap=portfolio_unit_cap, + redistribution_state_changed=( + logical_state_changed + if action + in { + "redistribution_sell", + "redistribution_buy", + } + else None + ), + state_changed=state_changed, + **( + { + "planned_date": planned_date, + "execution_date": date_text, + "delay_days": execution_delay_days, + "frozen_reason": source_reason, + "frozen_target_amount": frozen_target, + "execution_adjustment": adjustment, + } + if execution_delay_days > 0 + or adjustment != "none" + else {} + ), + ), + } + ) + + active = before > 0 or after > 0 or quantity > 0 or fee > 0.0 + security_daily_pnl = 0.0 + if active: + if not np.isfinite(close_price) or close_price <= 0.0: + raise ResultContractError("active position has no valid close price") + if before > 0 and ( + not np.isfinite(previous_close[column]) + or float(previous_close[column]) <= 0.0 + ): + raise ResultContractError("held position has no previous close price") + previous_price = ( + float(previous_close[column]) if before > 0 else close_price + ) + if quantity > 0 and is_sell: + security_daily_pnl = ( + after * (close_price - previous_price) + + quantity + * (float(fill_prices[row, column]) - previous_price) + - fee + ) + elif quantity > 0: + security_daily_pnl = ( + before * (close_price - previous_price) + + quantity + * (close_price - float(fill_prices[row, column])) + - fee + ) + else: + security_daily_pnl = before * (close_price - previous_price) - fee + daily_security_pnl_total += security_daily_pnl + + n_value = float(signal_n[row, column]) + stop_failure_price = ( + stop_after - 2.0 * n_value + if after > 0 and np.isfinite(stop_after) and np.isfinite(n_value) + else np.nan + ) + stop_failure_loss = ( + max(close_price - stop_failure_price, 0.0) * after + if np.isfinite(stop_failure_price) + else np.nan + ) + valuation_rows.append( + ( + { + "time": balance_time, + "event_id": _event_id( + scenario_id, + date_text, + security, + "valuation", + reason_code, + ), + "scope": "security", + "security": security, + "event_type": "valuation", + "reason_code": reason_code, + "requested_amount": None, + "executed_amount": None, + "reference_price": close_price, + "risk_before": risk_before, + "risk_after": risk_after, + }, + { + "source_reason": source_reason, + "action": action, + "position_before": before, + "position_after": after, + "previous_close": previous_price, + "close": close_price, + "fill_price": ( + float(fill_prices[row, column]) + if quantity > 0 + else np.nan + ), + "filled_amount": quantity, + "commission": fee, + "average_cost_before": before_cost, + "average_cost_after": float(average_cost[column]), + "common_stop_before": float(previous_stop[column]), + "common_stop_after": stop_after, + "n": n_value, + "stop_failure_price": stop_failure_price, + "stop_failure_loss": stop_failure_loss, + "security_daily_pnl": security_daily_pnl, + }, + ) + ) + + if after > 0: + position_rows.append( + { + "pindex": 0, + "avg_cost": float(average_cost[column]), + "margin": 0.0, + "amount": float(after), + "today_amount": int(today_buys[column]), + "hold_cost": float(average_cost[column]), + "side": "long", + "price": close_price, + "gains": (close_price - float(average_cost[column])) * after, + "daily_gains": security_daily_pnl, + "closeable_amount": max( + after - int(today_buys[column]), 0 + ), + "time": balance_time, + "security_name": security, + "security": security, + } + ) + previous_quantity[column] = after + previous_stop[column] = stop_after + previous_next_add[column] = next_after + previous_unit_count[column] = units_after + + portfolio_daily_pnl = float(values[row]) - ( + initial_cash if row == 0 else float(values[row - 1]) + ) + reconciliation_difference = daily_security_pnl_total - portfolio_daily_pnl + if abs(reconciliation_difference) > 0.02: + raise ResultContractError( + "security daily PnL does not reconcile with portfolio change" + ) + for event, valuation in valuation_rows: + event["details_json"] = _details( + **valuation, + daily_security_pnl_total=daily_security_pnl_total, + portfolio_daily_pnl=portfolio_daily_pnl, + reconciliation_difference=reconciliation_difference, + ) + attribution_rows.append(event) + previous_close = close[row].copy() + + result_rows = [] + balance_rows = [] + for row in range(rows): + time_text = f"{_date_text(dates[row])} 16:00:00" + result_rows.append( + { + "benchmark_returns": None, + "returns": float(values[row] / initial_cash - 1.0), + "time": time_text, + } + ) + balance_rows.append( + { + "total_value": float(values[row]), + "net_value": float(values[row]), + "cash": float(cash[row]), + "aval_cash": float(cash[row]), + "time": time_text, + } + ) + + facts = LocalExecutionFacts( + results=_table(result_rows, _RESULTS_SCHEMA), + balances=_table(balance_rows, _BALANCES_SCHEMA), + positions=_table(position_rows, _POSITIONS_SCHEMA), + orders=_table(order_rows, _ORDERS_SCHEMA), + attribution=_table(attribution_rows, _ATTRIBUTION_SCHEMA), + ) + validate_turtle_attribution(facts) + _validate_common_facts(facts) + return facts + + +def _validate_common_facts(facts: LocalExecutionFacts) -> None: + expected = { + "results": _RESULTS_SCHEMA, + "balances": _BALANCES_SCHEMA, + "positions": _POSITIONS_SCHEMA, + "orders": _ORDERS_SCHEMA, + } + for name, schema in expected.items(): + table = getattr(facts, name) + if table.schema != schema: + raise ResultContractError(f"{name} fields do not match the contract") + if facts.results.num_rows != facts.balances.num_rows: + raise ResultContractError("results and balances do not reconcile") + if facts.results["benchmark_returns"].null_count != facts.results.num_rows: + raise ResultContractError("source benchmark returns must remain null") + result_rows = facts.results.to_pylist() + balance_rows = facts.balances.to_pylist() + result_times = [str(item["time"]) for item in result_rows] + if result_times != [str(item["time"]) for item in balance_rows]: + raise ResultContractError("results and balances times do not reconcile") + implied_initial_cash = [] + for result, balance in zip(result_rows, balance_rows, strict=True): + denominator = 1.0 + float(result["returns"]) + if denominator <= 0.0: + raise ResultContractError("cumulative return is invalid") + implied_initial_cash.append(float(balance["total_value"]) / denominator) + if implied_initial_cash and max(implied_initial_cash) - min(implied_initial_cash) > 0.01: + raise ResultContractError("returns do not use one configured initial cash value") + + position_rows = facts.positions.to_pylist() + position_keys = [ + (item["time"], item["pindex"], item["security"], item["side"]) + for item in position_rows + ] + if len(position_keys) != len(set(position_keys)): + raise ResultContractError("position identity is not unique") + if any(str(item["time"]) not in set(result_times) for item in position_rows): + raise ResultContractError("position time is absent from results") + position_value_by_time: dict[str, float] = {} + for item in position_rows: + time_text = str(item["time"]) + position_value_by_time[time_text] = position_value_by_time.get( + time_text, 0.0 + ) + float(item["amount"]) * float(item["price"]) + for balance in balance_rows: + time_text = str(balance["time"]) + reconciled = float(balance["cash"]) + position_value_by_time.get(time_text, 0.0) + if abs(float(balance["total_value"]) - reconciled) > 0.02: + raise ResultContractError("balance does not reconcile with cash and positions") + + result_dates = {time_text[:10] for time_text in result_times} + for item in facts.orders.to_pylist(): + if str(item["time"])[:10] not in result_dates: + raise ResultContractError("order date is absent from results") + if not 0 <= int(item["filled"]) <= int(item["amount"]): + raise ResultContractError("order filled amount is invalid") + order_keys = list( + zip( + facts.orders["time"].to_pylist(), + facts.orders["pindex"].to_pylist(), + facts.orders["security"].to_pylist(), + ) + ) + if len(order_keys) != len(set(order_keys)): + raise ResultContractError("each security may have at most one order per day") + + +def validate_turtle_attribution(facts: LocalExecutionFacts) -> None: + table = facts.attribution + if table.schema != _ATTRIBUTION_SCHEMA: + raise ResultContractError("attribution fields do not match the contract") + rows = table.to_pylist() + event_ids = [item["event_id"] for item in rows] + if any(not isinstance(value, str) or not value for value in event_ids): + raise ResultContractError("attribution event_id must be non-empty") + if len(event_ids) != len(set(event_ids)): + raise ResultContractError("attribution event_id must be unique") + parsed_details: dict[str, dict[str, object]] = {} + for item in rows: + if item["scope"] != "security": + raise ResultContractError("attribution scope is unknown") + if item["event_type"] not in _EVENT_TYPES: + raise ResultContractError("attribution event type is unknown") + if item["reason_code"] not in _REASON_CODES: + raise ResultContractError("attribution reason code is unknown") + for name in ( + "requested_amount", + "executed_amount", + "reference_price", + "risk_before", + "risk_after", + ): + value = item[name] + if value is not None and not np.isfinite(float(value)): + raise ResultContractError(f"attribution {name} is invalid") + for name in ("requested_amount", "executed_amount", "risk_before", "risk_after"): + value = item[name] + if value is not None and float(value) < 0.0: + raise ResultContractError(f"attribution {name} is negative") + try: + details = json.loads(str(item["details_json"])) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise ResultContractError("attribution details_json is invalid") from exc + if not isinstance(details, dict): + raise ResultContractError("attribution details_json must be an object") + parsed_details[str(item["event_id"])] = details + if item["event_type"] == "valuation": + required = { + "source_reason", + "position_before", + "position_after", + "previous_close", + "close", + "filled_amount", + "commission", + "average_cost_before", + "average_cost_after", + "common_stop_before", + "common_stop_after", + "n", + "stop_failure_price", + "stop_failure_loss", + "security_daily_pnl", + "daily_security_pnl_total", + "portfolio_daily_pnl", + "reconciliation_difference", + } + if not required.issubset(details): + raise ResultContractError("valuation attribution evidence is incomplete") + if item["event_type"] == "corporate_action": + required = { + "source_event_id", + "event_type", + "effective_date", + "application_date", + "announcement_date", + "knowledge_cutoff_date", + "evidence_timing", + "split_ratio", + "cash_per_share", + "cumulative_factor", + "price_basis_changed", + "source", + "source_record_sha256", + "corporate_action_mode", + } + if not required.issubset(details): + raise ResultContractError( + "corporate-action attribution evidence is incomplete" + ) + coverage = { + (str(item["time"])[:10], str(item["security"])) + for item in rows + if item["event_type"] == "decision" + and item["executed_amount"] is not None + and float(item["executed_amount"]) > 0.0 + } + filled_orders = { + (str(item["time"])[:10], str(item["security"])) + for item in facts.orders.to_pylist() + if int(item["filled"]) > 0 + } + if coverage != filled_orders: + raise ResultContractError("attribution does not cover every order") + rejected_coverage = set() + for item in rows: + if item["event_type"] != "decision": + continue + details = parsed_details[str(item["event_id"])] + adjustment = details.get("execution_adjustment", "none") + is_canceled_adjustment = ( + adjustment in {"cash_truncated", "holding_truncated", "untradable"} + and float(item["executed_amount"] or 0.0) == 0.0 + ) + if item["reason_code"] == "order_rejected" or is_canceled_adjustment: + rejected_coverage.add( + (str(item["time"])[:10], str(item["security"])) + ) + canceled_orders = { + (str(item["time"])[:10], str(item["security"])) + for item in facts.orders.to_pylist() + if item["status"] == "canceled" + } + if rejected_coverage != canceled_orders: + raise ResultContractError("attribution does not cover every canceled order") + + valuation_totals: dict[str, float] = {} + for item in rows: + if item["event_type"] != "valuation": + continue + date_text = str(item["time"])[:10] + details = parsed_details[str(item["event_id"])] + try: + pnl = float(details["security_daily_pnl"]) + declared_total = float(details["daily_security_pnl_total"]) + declared_portfolio = float(details["portfolio_daily_pnl"]) + declared_difference = float(details["reconciliation_difference"]) + except (TypeError, ValueError) as exc: + raise ResultContractError("valuation daily PnL evidence is invalid") from exc + if not all( + np.isfinite(value) + for value in (pnl, declared_total, declared_portfolio, declared_difference) + ): + raise ResultContractError("valuation daily PnL evidence is invalid") + if abs((declared_total - declared_portfolio) - declared_difference) > 1e-9: + raise ResultContractError("valuation daily PnL evidence is inconsistent") + valuation_totals[date_text] = valuation_totals.get(date_text, 0.0) + pnl + + result_rows = facts.results.to_pylist() + balance_rows = facts.balances.to_pylist() + if result_rows and len(result_rows) != len(balance_rows): + raise ResultContractError("daily PnL cannot reconcile unmatched result rows") + initial_cash = None + if result_rows: + denominator = 1.0 + float(result_rows[0]["returns"]) + if denominator <= 0.0: + raise ResultContractError("daily PnL initial equity is invalid") + initial_cash = float(balance_rows[0]["total_value"]) / denominator + previous_value = initial_cash + for balance in balance_rows: + date_text = str(balance["time"])[:10] + current_value = float(balance["total_value"]) + portfolio_pnl = current_value - float(previous_value) + if abs(valuation_totals.get(date_text, 0.0) - portfolio_pnl) > 0.02: + raise ResultContractError( + "attribution security daily PnL does not reconcile with portfolio change" + ) + previous_value = current_value + + +def validate_turtle_result(result_dir: Path) -> None: + root = Path(result_dir).resolve() + try: + source = open_analysis_source(root) + except Exception as exc: + raise ResultContractError("local result failed the common contract") from exc + if source.kind != "local_backtest": + raise ResultContractError("turtle result must be a local backtest") + try: + extensions = source.manifest["extensions"] + turtle = extensions["turtle_etf"] + entry = turtle["attribution_log"] + reference = entry["files"][0] + except (KeyError, IndexError, TypeError) as exc: + raise ResultContractError("turtle attribution declaration is missing") from exc + expected_entry_fields = { + "required", + "status", + "schema_version", + "reason_code_version", + "rows", + "verified_empty", + "time_range", + "files", + "evidence", + } + if ( + not isinstance(extensions, Mapping) + or set(extensions) != {"turtle_etf"} + or not isinstance(turtle, Mapping) + or set(turtle) != {"attribution_log"} + or not isinstance(entry, Mapping) + or set(entry) != expected_entry_fields + or entry["required"] is not True + or entry["status"] != "complete" + or entry["schema_version"] != ATTRIBUTION_SCHEMA_VERSION + or entry["reason_code_version"] != ATTRIBUTION_SCHEMA_VERSION + or not isinstance(reference, Mapping) + ): + raise ResultContractError("turtle attribution declaration is invalid") + path_text = reference.get("path") + digest = reference.get("sha256") + if ( + not isinstance(path_text, str) + or not isinstance(digest, str) + or path_text != f"data/attribution_log-{digest}.parquet" + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise ResultContractError("turtle attribution file identity is invalid") + path = root / path_text + if ( + not path.is_file() + or path.stat().st_size != reference.get("bytes") + or _sha256_file(path) != digest + or reference.get("rows") != entry["rows"] + or reference.get("format") != "parquet" + or reference.get("compression") != "zstd" + ): + raise ResultContractError("turtle attribution file evidence is invalid") + evidence = entry.get("evidence") + if ( + not isinstance(evidence, Mapping) + or evidence.get("fields") != list(ATTRIBUTION_FIELDS) + or evidence.get("unique_key") != ["event_id"] + or evidence.get("reason_codes") != sorted(_REASON_CODES) + ): + raise ResultContractError("turtle attribution evidence is invalid") + try: + facts = LocalExecutionFacts( + results=pq.read_table(root / "data/results.parquet"), + balances=pq.read_table(root / "data/balances.parquet"), + positions=pq.read_table(root / "data/positions.parquet"), + orders=pq.read_table(root / "data/orders.parquet"), + attribution=pq.read_table(path), + ) + except Exception as exc: + raise ResultContractError("turtle result Parquet is unreadable") from exc + if facts.attribution.num_rows != entry["rows"]: + raise ResultContractError("turtle attribution row count is invalid") + if entry["verified_empty"] is not (facts.attribution.num_rows == 0): + raise ResultContractError("turtle attribution empty evidence is invalid") + if entry["time_range"] != _time_range(facts.attribution): + raise ResultContractError("turtle attribution time range is invalid") + _validate_common_facts(facts) + validate_turtle_attribution(facts) + + +def _json_bytes(document: Mapping[str, object]) -> bytes: + return ( + json.dumps( + dict(document), ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False + ) + + "\n" + ).encode("utf-8") + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def parameter_document_digest(document: Mapping[str, object]) -> str: + return _sha256_bytes(_json_bytes(document)) + + +def _sha256_file(path: Path) -> str: + return _sha256_bytes(path.read_bytes()) + + +def _file_ref(root: Path, path: Path) -> dict[str, object]: + return { + "path": path.relative_to(root).as_posix(), + "sha256": _sha256_file(path), + "bytes": path.stat().st_size, + } + + +def _parquet_ref(root: Path, path: Path, rows: int) -> dict[str, object]: + return { + **_file_ref(root, path), + "rows": rows, + "format": "parquet", + "compression": "zstd", + } + + +def _time_range(table: pa.Table) -> dict[str, str | None]: + if table.num_rows == 0: + return {"start": None, "end": None} + dates = [str(value)[:10] for value in table["time"].to_pylist()] + return {"start": min(dates), "end": max(dates)} + + +def _dataset_entry( + root: Path, + name: str, + table: pa.Table, + path: Path, + unique_key: list[str], +) -> dict[str, object]: + return { + "required": True, + "status": "complete", + "rows": table.num_rows, + "verified_empty": table.num_rows == 0, + "time_range": _time_range(table), + "files": [_parquet_ref(root, path, table.num_rows)], + "evidence": {"fields": table.schema.names, "unique_key": unique_key}, + } + + +def _engine() -> dict[str, str]: + return { + "backend": "vectorbt.Portfolio.from_order_func", + "adapter_version": "local-vectorbt-adapter/1", + "vectorbt": importlib.metadata.version("vectorbt"), + "numba": importlib.metadata.version("numba"), + "numpy": importlib.metadata.version("numpy"), + "pandas": importlib.metadata.version("pandas"), + } + + +def execution_facts_digest(facts: LocalExecutionFacts) -> str: + document: dict[str, object] = {} + for name in ("results", "balances", "positions", "orders", "attribution"): + table = getattr(facts, name) + document[name] = { + "fields": [ + { + "name": field.name, + "type": str(field.type), + "nullable": field.nullable, + } + for field in table.schema + ], + "rows": table.to_pylist(), + } + return hashlib.sha256( + json.dumps( + document, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + + +def _read_materialized_facts(data_dir: Path) -> LocalExecutionFacts: + attribution_paths = sorted(Path(data_dir).glob("attribution_log-*.parquet")) + expected_files = { + "results.parquet", + "balances.parquet", + "positions.parquet", + "orders.parquet", + } + actual_files = {path.name for path in Path(data_dir).iterdir() if path.is_file()} + if len(attribution_paths) != 1 or actual_files != { + *expected_files, + attribution_paths[0].name, + }: + raise ResultContractError("materialized execution fact file set is invalid") + attribution_path = attribution_paths[0] + digest = _sha256_file(attribution_path) + if attribution_path.name != f"attribution_log-{digest}.parquet": + raise ResultContractError("materialized attribution filename is invalid") + try: + return LocalExecutionFacts( + results=pq.read_table(Path(data_dir) / "results.parquet"), + balances=pq.read_table(Path(data_dir) / "balances.parquet"), + positions=pq.read_table(Path(data_dir) / "positions.parquet"), + orders=pq.read_table(Path(data_dir) / "orders.parquet"), + attribution=pq.read_table(attribution_path), + ) + except Exception as exc: + raise ResultContractError("materialized execution facts are unreadable") from exc + + +def materialize_execution_facts(data_dir: Path, facts: LocalExecutionFacts) -> str: + target = Path(data_dir) + if target.exists(): + raise ResultContractError("execution fact directory already exists") + _validate_common_facts(facts) + validate_turtle_attribution(facts) + try: + target.mkdir(parents=True) + for name in ("results", "balances", "positions", "orders"): + pq.write_table( + getattr(facts, name), target / f"{name}.parquet", compression="zstd" + ) + temporary = target / ".attribution.parquet" + pq.write_table(facts.attribution, temporary, compression="zstd") + attribution_digest = _sha256_file(temporary) + os.replace( + temporary, + target / f"attribution_log-{attribution_digest}.parquet", + ) + written = _read_materialized_facts(target) + _validate_common_facts(written) + validate_turtle_attribution(written) + expected_digest = execution_facts_digest(facts) + if execution_facts_digest(written) != expected_digest: + raise ResultContractError("materialized execution fact digest changed") + return expected_digest + except Exception: + if target.exists(): + shutil.rmtree(target) + raise + + +def write_local_result( + backtest_dir: Path, + *, + facts: LocalExecutionFacts, + run_id: str, + local_backtest_id: str, + scenario_id: str, + snapshot_id: str, + corporate_actions_sha256: str, + code_path: Path, + params: Mapping[str, object], + performance: Mapping[str, object], +) -> LocalResultPackage: + target = Path(backtest_dir).resolve() + if target.exists(): + raise ResultContractError("local backtest directory already exists") + if not all(isinstance(value, str) and value for value in (run_id, local_backtest_id, scenario_id)): + raise ResultContractError("local result identity is incomplete") + if len(snapshot_id) != 64 or any(character not in "0123456789abcdef" for character in snapshot_id): + raise ResultContractError("snapshot_id must be a lowercase SHA256") + if len(corporate_actions_sha256) != 64 or any( + character not in "0123456789abcdef" + for character in corporate_actions_sha256 + ): + raise ResultContractError( + "corporate_actions_sha256 must be a lowercase SHA256" + ) + source_code = Path(code_path) + if not source_code.is_file(): + raise ResultContractError("code source is missing") + _validate_common_facts(facts) + validate_turtle_attribution(facts) + + target.parent.mkdir(parents=True, exist_ok=True) + staging = target.parent / f".{target.name}.{uuid.uuid4().hex}.tmp" + try: + data_dir = staging / "data" + params_dir = staging / "params_versions" + staging.mkdir() + params_dir.mkdir() + (staging / "code.py").write_bytes(source_code.read_bytes()) + params_bytes = _json_bytes(params) + params_sha256 = parameter_document_digest(params) + (staging / "params.json").write_bytes(params_bytes) + (params_dir / f"{params_sha256}.json").write_bytes(params_bytes) + (staging / "performance.json").write_bytes(_json_bytes(performance)) + + materialize_execution_facts(data_dir, facts) + paths = { + name: data_dir / f"{name}.parquet" + for name in ("results", "balances", "positions", "orders") + } + attribution_path = next(data_dir.glob("attribution_log-*.parquet")) + attribution_sha256 = _sha256_file(attribution_path) + + datasets = { + "results": _dataset_entry( + staging, "results", facts.results, paths["results"], ["time"] + ), + "balances": _dataset_entry( + staging, "balances", facts.balances, paths["balances"], ["time"] + ), + "positions": _dataset_entry( + staging, + "positions", + facts.positions, + paths["positions"], + ["time", "pindex", "security", "side"], + ), + "orders": _dataset_entry( + staging, + "orders", + facts.orders, + paths["orders"], + ["time", "pindex", "security"], + ), + "risk": { + "required": False, + "status": "missing_at_source", + "reason": "computed_by_strategy_analysis", + "rows": 0, + "verified_empty": True, + "files": [], + }, + "period_risks": { + "required": False, + "status": "missing_at_source", + "reason": "computed_by_strategy_analysis", + "rows": 0, + "verified_empty": True, + "files": [], + }, + } + attribution_entry = { + "required": True, + "status": "complete", + "schema_version": ATTRIBUTION_SCHEMA_VERSION, + "reason_code_version": ATTRIBUTION_SCHEMA_VERSION, + "rows": facts.attribution.num_rows, + "verified_empty": facts.attribution.num_rows == 0, + "time_range": _time_range(facts.attribution), + "files": [ + _parquet_ref( + staging, + attribution_path, + facts.attribution.num_rows, + ) + ], + "evidence": { + "fields": list(ATTRIBUTION_FIELDS), + "unique_key": ["event_id"], + "reason_codes": sorted(_REASON_CODES), + }, + } + code_ref = _file_ref(staging, staging / "code.py") + current_params_ref = _file_ref(staging, staging / "params.json") + version_params_ref = _file_ref( + staging, params_dir / f"{params_sha256}.json" + ) + manifest = { + "schema_version": "local-backtest/1", + "object": { + "kind": "local_backtest", + "local_id": local_backtest_id, + "status": "complete", + }, + "source": { + "kind": "local_vectorbt", + "engine": _engine(), + "accounting": { + **_ACCOUNTING_CONTRACT, + "corporate_actions_sha256": corporate_actions_sha256, + }, + }, + "authority": "local_research", + "run": { + "run_id": run_id, + "scenario_id": scenario_id, + "snapshot_id": snapshot_id, + }, + "code": code_ref, + "params": {"current": current_params_ref, "version": version_params_ref}, + "performance": _file_ref(staging, staging / "performance.json"), + "datasets": datasets, + "source_benchmark_returns": { + "status": "missing_at_source", + "reason": "independent_benchmark_set", + "null_rows": facts.results.num_rows, + }, + "gate": { + "status": "pass", + "exceptions": [], + "checks": [ + "local_schema", + "common_fact_fields", + "cross_table_reconciliation", + "turtle_attribution_coverage", + ], + }, + "extensions": {"turtle_etf": {"attribution_log": attribution_entry}}, + } + (staging / "manifest.json").write_bytes(_json_bytes(manifest)) + validate_turtle_result(staging) + if _sha256_file(attribution_path) != attribution_sha256: + raise ResultContractError("attribution digest changed after writing") + os.replace(staging, target) + return LocalResultPackage( + root=target, + params_sha256=params_sha256, + attribution_sha256=attribution_sha256, + ) + except Exception: + if staging.exists(): + shutil.rmtree(staging) + raise diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/risk.py b/joinquant/strategies/strategy-003/research/turtle_etf/risk.py deleted file mode 100644 index b9d8e18..0000000 --- a/joinquant/strategies/strategy-003/research/turtle_etf/risk.py +++ /dev/null @@ -1,428 +0,0 @@ -from __future__ import annotations - -import math -from dataclasses import dataclass -from decimal import Decimal, ROUND_FLOOR -from types import MappingProxyType -from typing import Mapping, Sequence - -import numpy as np -import pandas as pd - -from .state import Batch, OrderIntent, TrendState, _decimal - - -@dataclass(frozen=True) -class CovarianceEstimate: - securities: tuple[str, ...] - matrix: tuple[tuple[Decimal, ...], ...] - aligned_samples: int - window_days: int - - def __post_init__(self) -> None: - size = len(self.securities) - if size == 0 or len(set(self.securities)) != size: - raise ValueError("covariance securities must be non-empty and unique") - if len(self.matrix) != size or any(len(row) != size for row in self.matrix): - raise ValueError("covariance matrix dimensions are invalid") - if self.aligned_samples < self.window_days or self.window_days < 2: - raise ValueError("covariance sample evidence is invalid") - - def covers(self, securities: Sequence[str]) -> bool: - return set(securities).issubset(self.securities) - - -@dataclass(frozen=True) -class PortfolioState: - equity: Decimal - cash: Decimal - positions: tuple[TrendState, ...] = () - - def __post_init__(self) -> None: - object.__setattr__(self, "equity", _decimal(self.equity, "equity", positive=True)) - object.__setattr__(self, "cash", _decimal(self.cash, "cash")) - object.__setattr__(self, "positions", tuple(self.positions)) - if self.cash < 0: - raise ValueError("cash must not be negative") - securities = [position.security for position in self.positions] - if len(securities) != len(set(securities)): - raise ValueError("portfolio positions must be unique by security") - - -@dataclass(frozen=True) -class RiskInputs: - prices: Mapping[str, Decimal | None] - median_turnover_20d: Mapping[str, Decimal] - covariance: CovarianceEstimate | None - lot_size: int = 100 - minimum_aligned_samples: int = 60 - minimum_turnover: Decimal = Decimal("100000000") - maximum_order_turnover_fraction: Decimal = Decimal("0.01") - security_risk_cap: Decimal = Decimal("0.0125") - security_value_cap: Decimal = Decimal("0.30") - asset_group_risk_cap: Decimal = Decimal("0.025") - asset_group_value_cap: Decimal = Decimal("0.50") - portfolio_risk_cap: Decimal = Decimal("0.05") - portfolio_value_cap: Decimal = Decimal("1.00") - target_volatility: Decimal = Decimal("0.10") - - def __post_init__(self) -> None: - prices = { - str(security): None - if value is None - else _decimal(value, "price", positive=True) - for security, value in self.prices.items() - } - turnover = { - str(security): _decimal(value, "median_turnover_20d") - for security, value in self.median_turnover_20d.items() - } - object.__setattr__(self, "prices", MappingProxyType(prices)) - object.__setattr__(self, "median_turnover_20d", MappingProxyType(turnover)) - if not isinstance(self.lot_size, int) or self.lot_size <= 0: - raise ValueError("lot_size must be positive") - if not isinstance(self.minimum_aligned_samples, int) or self.minimum_aligned_samples < 2: - raise ValueError("minimum_aligned_samples must be at least 2") - for field in ( - "minimum_turnover", - "maximum_order_turnover_fraction", - "security_risk_cap", - "security_value_cap", - "asset_group_risk_cap", - "asset_group_value_cap", - "portfolio_risk_cap", - "portfolio_value_cap", - "target_volatility", - ): - object.__setattr__(self, field, _decimal(getattr(self, field), field, positive=True)) - - -@dataclass(frozen=True) -class RiskDecision: - allow_new_risk: bool - approved: tuple[OrderIntent, ...] - rejected: tuple[OrderIntent, ...] - reason_codes: tuple[str, ...] - projected_volatility: Decimal | None - - -def initial_unit( - equity: Decimal, - n_value: Decimal, - risk_fraction: Decimal = Decimal("0.005"), -) -> int: - equity_value = _decimal(equity, "equity", positive=True) - n = _decimal(n_value, "n_value", positive=True) - fraction = _decimal(risk_fraction, "risk_fraction", positive=True) - return int( - (equity_value * fraction / (Decimal("2") * n)).to_integral_value( - rounding=ROUND_FLOOR - ) - ) - - -def estimate_covariance( - returns: pd.DataFrame, - *, - securities: Sequence[str], - days: int = 60, -) -> CovarianceEstimate | None: - securities = tuple(securities) - if not isinstance(days, int) or days < 2: - raise ValueError("days must be at least 2") - if not securities or len(securities) != len(set(securities)): - raise ValueError("securities must be non-empty and unique") - if any(security not in returns.columns for security in securities): - return None - numeric = returns.loc[:, list(securities)].apply(pd.to_numeric, errors="coerce") - aligned = numeric.replace([np.inf, -np.inf], np.nan).dropna(how="any") - if len(aligned) < days: - return None - window = aligned.tail(days) - covariance = window.cov() - matrix_values = covariance.to_numpy(dtype=float) - if not np.isfinite(matrix_values).all(): - return None - matrix = tuple( - tuple(Decimal(str(value)) for value in row) for row in matrix_values - ) - return CovarianceEstimate( - securities=securities, - matrix=matrix, - aligned_samples=len(window), - window_days=days, - ) - - -def _annualized_volatility( - values: Mapping[str, Decimal], - *, - equity: Decimal, - covariance: CovarianceEstimate, -) -> Decimal: - weights = { - security: float(value / equity) for security, value in values.items() - } - positions = {security: index for index, security in enumerate(covariance.securities)} - variance = 0.0 - for left, left_weight in weights.items(): - for right, right_weight in weights.items(): - variance += ( - left_weight - * float(covariance.matrix[positions[left]][positions[right]]) - * right_weight - ) - if variance < -1e-15: - raise ValueError("projected covariance variance is negative") - annualized = math.sqrt(max(0.0, variance)) * math.sqrt(252.0) - return Decimal(str(annualized)) - - -def portfolio_volatility( - state: PortfolioState, - inputs: RiskInputs, -) -> Decimal | None: - if not state.positions: - return Decimal("0") - securities = tuple(position.security for position in state.positions) - covariance = inputs.covariance - if ( - covariance is None - or covariance.aligned_samples < inputs.minimum_aligned_samples - or not covariance.covers(securities) - ): - return None - values: dict[str, Decimal] = {} - for position in state.positions: - price = inputs.prices.get(position.security) - if price is None: - return None - values[position.security] = price * position.quantity - return _annualized_volatility( - values, - equity=state.equity, - covariance=covariance, - ) - - -def target_volatility_reductions( - state: PortfolioState, - inputs: RiskInputs, - *, - signal_date: str, - execution_date: str, - reduction_target: Decimal = Decimal("0.095"), -) -> tuple[OrderIntent, ...]: - target = _decimal(reduction_target, "reduction_target", positive=True) - if target >= inputs.target_volatility: - raise ValueError("reduction_target must be below target_volatility") - current = portfolio_volatility(state, inputs) - if current is None or current <= inputs.target_volatility: - return () - scale = target / current - reductions: list[OrderIntent] = [] - lot = Decimal(inputs.lot_size) - for position in sorted(state.positions, key=lambda item: item.security): - target_quantity = int( - (Decimal(position.quantity) * scale / lot).to_integral_value( - rounding=ROUND_FLOOR - ) - * inputs.lot_size - ) - reduction = position.quantity - target_quantity - if reduction <= 0: - continue - price = inputs.prices.get(position.security) - if price is None: - return () - reductions.append( - OrderIntent( - security=position.security, - asset_group=position.asset_group, - action="mandatory_risk_reduction", - quantity=reduction, - expected_price=price, - signal_date=signal_date, - execution_date=execution_date, - reason="target_volatility_reduction", - ) - ) - return tuple(reductions) - - -def _position_projection( - state: PortfolioState, - exits: Sequence[OrderIntent], -) -> tuple[dict[str, dict[str, object]], Decimal]: - full_exits = {intent.security for intent in exits if intent.action == "full_exit"} - projected: dict[str, dict[str, object]] = {} - cash = state.cash - for position in state.positions: - if position.security in full_exits: - exit_intent = next(intent for intent in exits if intent.security == position.security) - cash += exit_intent.expected_price * min(position.quantity, exit_intent.quantity) - cash -= exit_intent.estimated_fee - continue - projected[position.security] = { - "asset_group": position.asset_group, - "batches": list(position.batches), - "common_stop": position.common_stop, - } - return projected, cash - - -def evaluate_risk( - requests: Sequence[OrderIntent], - state: PortfolioState, - inputs: RiskInputs, -) -> RiskDecision: - requests = tuple(requests) - exits = tuple( - intent - for intent in requests - if intent.action in {"full_exit", "mandatory_risk_reduction"} - ) - buys = tuple( - intent for intent in requests if intent.action in {"entry", "addition"} - ) - held = tuple(position.security for position in state.positions) - held_inputs_valid = all(inputs.prices.get(security) is not None for security in held) - held_inputs_valid = held_inputs_valid and ( - not held - or ( - inputs.covariance is not None - and inputs.covariance.aligned_samples >= inputs.minimum_aligned_samples - and inputs.covariance.covers(held) - ) - ) - if not held_inputs_valid: - return RiskDecision( - allow_new_risk=False, - approved=exits, - rejected=buys, - reason_codes=("held_risk_input_missing",), - projected_volatility=None, - ) - if not buys: - return RiskDecision(True, exits, (), (), None) - - reasons: list[str] = [] - projected, available_cash = _position_projection(state, exits) - for intent in buys: - if intent.quantity % inputs.lot_size: - reasons.append("invalid_lot") - if intent.standard_unit is None or intent.quantity > intent.standard_unit: - reasons.append("standard_unit_cap") - turnover = inputs.median_turnover_20d.get(intent.security) - if turnover is None or turnover < inputs.minimum_turnover: - reasons.append("liquidity_floor") - elif ( - intent.expected_price * intent.quantity - > turnover * inputs.maximum_order_turnover_fraction - ): - reasons.append("order_liquidity_cap") - existing = projected.get(intent.security) - if intent.action == "entry" and existing is not None: - reasons.append("invalid_position_transition") - continue - if intent.action == "addition" and existing is None: - reasons.append("invalid_position_transition") - continue - if intent.common_stop_after is None: - reasons.append("invalid_common_stop") - continue - if existing is not None and intent.common_stop_after < existing["common_stop"]: - reasons.append("common_stop_decrease") - continue - if existing is None: - existing = { - "asset_group": intent.asset_group, - "batches": [], - "common_stop": intent.common_stop_after, - } - projected[intent.security] = existing - existing["common_stop"] = intent.common_stop_after - existing["batches"].append( - Batch( - execution_date=intent.execution_date, - quantity=intent.quantity, - fill_price=intent.expected_price, - ) - ) - available_cash -= intent.expected_price * intent.quantity + intent.estimated_fee - - if available_cash < 0: - reasons.append("insufficient_cash") - - projected_securities = tuple(sorted(projected)) - covariance = inputs.covariance - covariance_valid = ( - covariance is not None - and covariance.aligned_samples >= inputs.minimum_aligned_samples - and covariance.covers(projected_securities) - ) - if not covariance_valid: - reasons.append("covariance_unavailable") - - values: dict[str, Decimal] = {} - risks: dict[str, Decimal] = {} - groups: dict[str, str] = {} - for security, position in projected.items(): - batches = position["batches"] - price = next( - ( - intent.expected_price - for intent in reversed(buys) - if intent.security == security - ), - inputs.prices.get(security), - ) - if price is None: - reasons.append("price_unavailable") - continue - quantity = sum(batch.quantity for batch in batches) - stop = position["common_stop"] - values[security] = _decimal(price, "price", positive=True) * quantity - risks[security] = max( - Decimal("0"), - sum((batch.fill_price - stop) * batch.quantity for batch in batches), - ) - groups[security] = str(position["asset_group"]) - - group_values: dict[str, Decimal] = {} - group_risks: dict[str, Decimal] = {} - for security in values: - group = groups[security] - group_values[group] = group_values.get(group, Decimal("0")) + values[security] - group_risks[group] = group_risks.get(group, Decimal("0")) + risks[security] - if values[security] > state.equity * inputs.security_value_cap: - reasons.append("security_value_cap") - if risks[security] > state.equity * inputs.security_risk_cap: - reasons.append("security_risk_cap") - if any(value > state.equity * inputs.asset_group_value_cap for value in group_values.values()): - reasons.append("group_value_cap") - if any(value > state.equity * inputs.asset_group_risk_cap for value in group_risks.values()): - reasons.append("group_risk_cap") - if sum(values.values()) > state.equity * inputs.portfolio_value_cap: - reasons.append("portfolio_value_cap") - if sum(risks.values()) > state.equity * inputs.portfolio_risk_cap: - reasons.append("portfolio_risk_cap") - - projected_volatility: Decimal | None = None - if covariance_valid and len(values) == len(projected): - projected_volatility = _annualized_volatility( - values, - equity=state.equity, - covariance=covariance, - ) - if projected_volatility > inputs.target_volatility: - reasons.append("target_volatility") - - unique_reasons = tuple(dict.fromkeys(reasons)) - return RiskDecision( - allow_new_risk=True, - approved=exits if unique_reasons else requests, - rejected=buys if unique_reasons else (), - reason_codes=unique_reasons, - projected_volatility=projected_volatility, - ) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/signals.py b/joinquant/strategies/strategy-003/research/turtle_etf/signals.py deleted file mode 100644 index 33b2e59..0000000 --- a/joinquant/strategies/strategy-003/research/turtle_etf/signals.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -from decimal import Decimal - -from .state import OrderIntent, _decimal - - -def entry_signal(close: Decimal, entry_level: Decimal | None) -> bool: - if entry_level is None: - return False - return _decimal(close, "close", positive=True) > _decimal( - entry_level, "entry_level", positive=True - ) - - -def trend_exit_signal(close: Decimal, exit_level: Decimal | None) -> bool: - if exit_level is None: - return False - return _decimal(close, "close", positive=True) < _decimal( - exit_level, "exit_level", positive=True - ) - - -def make_entry_intent( - *, - security: str, - asset_group: str, - signal_date: str, - execution_date: str, - expected_price: Decimal, - quantity: int, - signal_n: Decimal, - standard_unit: int, - stop_n: Decimal = Decimal("2"), -) -> OrderIntent: - price = _decimal(expected_price, "expected_price", positive=True) - n_value = _decimal(signal_n, "signal_n", positive=True) - stop_multiple = _decimal(stop_n, "stop_n", positive=True) - return OrderIntent( - security=security, - asset_group=asset_group, - action="entry", - quantity=quantity, - expected_price=price, - signal_date=signal_date, - execution_date=execution_date, - signal_n=n_value, - standard_unit=standard_unit, - common_stop_after=price - stop_multiple * n_value, - reason="entry_breakout", - ) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/single_scenario.py b/joinquant/strategies/strategy-003/research/turtle_etf/single_scenario.py new file mode 100644 index 0000000..8295011 --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/single_scenario.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Sequence + +from .result_adapter import LocalResultPackage, write_local_result +from .vectorbt_benchmark import benchmark_scenario + + +_FORBIDDEN_FLOW_FIELDS = {"candidates", "scenarios", "analysis_plan"} +_SCENARIO_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,63}") + + +class SingleScenarioError(ValueError): + """Raised when a project request attempts more than one local scenario.""" + + +@dataclass(frozen=True) +class SingleScenarioOutcome: + scenario_id: str + local_backtest_id: str + result_path: Path + next_action: str = "return_to_caller" + + +def validate_single_scenario_config(config: Mapping[str, object]) -> str: + if not isinstance(config, Mapping): + raise SingleScenarioError("single scenario config must be an object") + if set(config) & _FORBIDDEN_FLOW_FIELDS: + raise SingleScenarioError("single scenario config cannot contain batch or analysis inputs") + scenario_id = config.get("scenario_id") + if not isinstance(scenario_id, str) or _SCENARIO_ID.fullmatch(scenario_id) is None: + raise SingleScenarioError("single scenario config requires scenario_id") + if config.get("project_id") != "strategy-003" or config.get("schema_version") != 1: + raise SingleScenarioError("single scenario project identity is invalid") + return scenario_id + + +def execute_prepared_scenario( + *, + prepared_inputs: object, + config: Mapping[str, object], + output_dir: Path, + run_id: str, + snapshot_id: str, + code_sha256: str, + config_sha256: str, + code_path: Path, +) -> SingleScenarioOutcome: + scenario_id = validate_single_scenario_config(config) + if len(config_sha256) != 64: + raise SingleScenarioError("config_sha256 is invalid") + local_backtest_id = f"local-{scenario_id}" + output_root = Path(output_dir) + benchmark = benchmark_scenario( + prepared_inputs=prepared_inputs, + config=config, + scenario_id=scenario_id, + work_dir=output_root / ".benchmark-work", + code_sha256=code_sha256, + config_sha256=config_sha256, + ) + target = Path(output_dir) / "backtests" / local_backtest_id + package: LocalResultPackage = write_local_result( + target, + facts=benchmark.facts, + run_id=run_id, + local_backtest_id=local_backtest_id, + scenario_id=scenario_id, + snapshot_id=snapshot_id, + corporate_actions_sha256=str( + getattr(prepared_inputs, "corporate_actions_digest", "") + ), + code_path=code_path, + params=config, + performance=benchmark.performance, + ) + return SingleScenarioOutcome( + scenario_id=scenario_id, + local_backtest_id=local_backtest_id, + result_path=package.root, + ) + + +def write_project_status( + output_dir: Path, + *, + status: str, + reason_codes: Sequence[str], + next_action: str | None = None, +) -> Path: + if status not in {"complete", "evidence_insufficient", "failed"}: + raise SingleScenarioError("project status is invalid") + if status == "complete" and reason_codes: + raise SingleScenarioError("complete project status cannot contain reasons") + if next_action is not None and ( + status != "complete" or next_action != "return_to_caller" + ): + raise SingleScenarioError("single scenario next action is invalid") + document: dict[str, object] = { + "schema_version": 1, + "status": status, + "reason_codes": list(reason_codes), + } + if next_action is not None: + document["next_action"] = next_action + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + path = root / "project-status.json" + path.write_text( + json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + return path diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/state.py b/joinquant/strategies/strategy-003/research/turtle_etf/state.py deleted file mode 100644 index a16ce4d..0000000 --- a/joinquant/strategies/strategy-003/research/turtle_etf/state.py +++ /dev/null @@ -1,296 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, replace -from datetime import date -from decimal import Decimal -from typing import Literal - - -OrderAction = Literal[ - "entry", - "addition", - "full_exit", - "mandatory_risk_reduction", -] - - -def commission_fee(price: Decimal, quantity: int) -> Decimal: - normalized_price = _decimal(price, "commission_price", positive=True) - if not isinstance(quantity, int) or quantity <= 0: - raise ValueError("commission quantity must be positive") - return max( - Decimal("5"), - normalized_price * quantity * Decimal("0.000085"), - ) - - -def _decimal(value: object, field: str, *, positive: bool = False) -> Decimal: - try: - result = Decimal(str(value)) - except Exception as exc: - raise ValueError(f"{field} must be numeric") from exc - if not result.is_finite() or (positive and result <= 0): - raise ValueError(f"{field} must be finite and positive") - return result - - -def _date(value: str, field: str) -> str: - try: - date.fromisoformat(value) - except (TypeError, ValueError) as exc: - raise ValueError(f"{field} must use YYYY-MM-DD") from exc - return value - - -@dataclass(frozen=True) -class Batch: - execution_date: str - quantity: int - fill_price: Decimal - - def __post_init__(self) -> None: - object.__setattr__(self, "execution_date", _date(self.execution_date, "execution_date")) - if not isinstance(self.quantity, int) or self.quantity <= 0: - raise ValueError("batch quantity must be positive") - object.__setattr__(self, "fill_price", _decimal(self.fill_price, "fill_price", positive=True)) - - -@dataclass(frozen=True) -class OrderIntent: - security: str - asset_group: str - action: OrderAction - quantity: int - expected_price: Decimal - signal_date: str - execution_date: str - signal_n: Decimal | None = None - standard_unit: int | None = None - common_stop_after: Decimal | None = None - estimated_fee: Decimal = Decimal("0") - reason: str = "" - - def __post_init__(self) -> None: - if not self.security or not self.asset_group: - raise ValueError("security and asset_group must be non-empty") - if self.action not in { - "entry", - "addition", - "full_exit", - "mandatory_risk_reduction", - }: - raise ValueError("unsupported order action") - if not isinstance(self.quantity, int) or self.quantity <= 0: - raise ValueError("order quantity must be positive") - object.__setattr__( - self, - "expected_price", - _decimal(self.expected_price, "expected_price", positive=True), - ) - object.__setattr__(self, "signal_date", _date(self.signal_date, "signal_date")) - object.__setattr__( - self, - "execution_date", - _date(self.execution_date, "execution_date"), - ) - if self.execution_date <= self.signal_date: - raise ValueError("execution_date must be after signal_date") - object.__setattr__( - self, - "estimated_fee", - _decimal(self.estimated_fee, "estimated_fee"), - ) - if self.estimated_fee < 0: - raise ValueError("estimated_fee must not be negative") - if self.action in {"entry", "addition"}: - if self.signal_n is None or self.common_stop_after is None: - raise ValueError("buy intent requires signal_n and common_stop_after") - if not isinstance(self.standard_unit, int) or self.standard_unit <= 0: - raise ValueError("buy intent requires a positive standard_unit") - object.__setattr__(self, "signal_n", _decimal(self.signal_n, "signal_n", positive=True)) - object.__setattr__( - self, - "common_stop_after", - _decimal(self.common_stop_after, "common_stop_after"), - ) - - -@dataclass(frozen=True) -class TrendState: - security: str - asset_group: str - signal_n: Decimal - standard_unit: int - initial_fill_price: Decimal - batches: tuple[Batch, ...] - common_stop: Decimal - next_add_index: int = 1 - add_step_n: Decimal = Decimal("0.5") - stop_n: Decimal = Decimal("2") - last_add_request_date: str | None = None - - def __post_init__(self) -> None: - if not self.security or not self.asset_group or not self.batches: - raise ValueError("trend state identity and batches are required") - object.__setattr__(self, "signal_n", _decimal(self.signal_n, "signal_n", positive=True)) - object.__setattr__( - self, - "initial_fill_price", - _decimal(self.initial_fill_price, "initial_fill_price", positive=True), - ) - object.__setattr__(self, "common_stop", _decimal(self.common_stop, "common_stop")) - object.__setattr__(self, "add_step_n", _decimal(self.add_step_n, "add_step_n", positive=True)) - object.__setattr__(self, "stop_n", _decimal(self.stop_n, "stop_n", positive=True)) - object.__setattr__(self, "batches", tuple(self.batches)) - if not isinstance(self.standard_unit, int) or self.standard_unit <= 0: - raise ValueError("standard_unit must be positive") - if not isinstance(self.next_add_index, int) or self.next_add_index < 1: - raise ValueError("next_add_index must be positive") - if self.last_add_request_date is not None: - object.__setattr__( - self, - "last_add_request_date", - _date(self.last_add_request_date, "last_add_request_date"), - ) - - @property - def quantity(self) -> int: - return sum(batch.quantity for batch in self.batches) - - @property - def next_add_level(self) -> Decimal: - return self.initial_fill_price + ( - Decimal(self.next_add_index) * self.add_step_n * self.signal_n - ) - - @property - def planned_loss(self) -> Decimal: - net_loss = sum( - (batch.fill_price - self.common_stop) * batch.quantity - for batch in self.batches - ) - return max(Decimal("0"), net_loss) - - -def apply_entry_fill( - *, - security: str, - asset_group: str, - execution_date: str, - fill_price: Decimal, - quantity: int, - signal_n: Decimal, - standard_unit: int, - add_step_n: Decimal = Decimal("0.5"), - stop_n: Decimal = Decimal("2"), -) -> TrendState: - price = _decimal(fill_price, "fill_price", positive=True) - n_value = _decimal(signal_n, "signal_n", positive=True) - stop_multiple = _decimal(stop_n, "stop_n", positive=True) - batch = Batch(execution_date=execution_date, quantity=quantity, fill_price=price) - return TrendState( - security=security, - asset_group=asset_group, - signal_n=n_value, - standard_unit=standard_unit, - initial_fill_price=price, - batches=(batch,), - common_stop=price - stop_multiple * n_value, - add_step_n=add_step_n, - stop_n=stop_multiple, - ) - - -def request_addition( - state: TrendState, - *, - signal_date: str, - execution_date: str, - close: Decimal, - expected_price: Decimal, -) -> tuple[TrendState, OrderIntent | None]: - signal_date = _date(signal_date, "signal_date") - close_value = _decimal(close, "close", positive=True) - expected = _decimal(expected_price, "expected_price", positive=True) - if state.last_add_request_date == signal_date or close_value < state.next_add_level: - return state, None - candidate_stop = max( - state.common_stop, - expected - state.stop_n * state.signal_n, - ) - requested_state = replace(state, last_add_request_date=signal_date) - return requested_state, OrderIntent( - security=state.security, - asset_group=state.asset_group, - action="addition", - quantity=state.standard_unit, - expected_price=expected, - signal_date=signal_date, - execution_date=execution_date, - signal_n=state.signal_n, - standard_unit=state.standard_unit, - common_stop_after=candidate_stop, - reason="fixed_addition_level", - ) - - -def apply_addition_fill( - state: TrendState, - intent: OrderIntent, - *, - execution_date: str, - fill_price: Decimal, - quantity: int, -) -> TrendState: - if intent is None or intent.action != "addition" or intent.security != state.security: - raise ValueError("fill does not match an addition intent") - if execution_date != intent.execution_date: - raise ValueError("fill execution date does not match the addition intent") - if ( - intent.asset_group != state.asset_group - or intent.signal_n != state.signal_n - or intent.standard_unit != state.standard_unit - or state.last_add_request_date != intent.signal_date - ): - raise ValueError("fill identity does not match the addition state") - if not isinstance(quantity, int) or quantity <= 0 or quantity > intent.quantity: - raise ValueError("filled quantity exceeds the addition intent") - price = _decimal(fill_price, "fill_price", positive=True) - new_stop = max(state.common_stop, price - state.stop_n * state.signal_n) - batch = Batch(execution_date=execution_date, quantity=quantity, fill_price=price) - return replace( - state, - batches=(*state.batches, batch), - common_stop=new_stop, - next_add_index=state.next_add_index + 1, - ) - - -def request_full_exit( - state: TrendState, - *, - signal_date: str, - execution_date: str, - close: Decimal, - exit_level: Decimal | None, - expected_price: Decimal, -) -> OrderIntent | None: - close_value = _decimal(close, "close", positive=True) - level = None if exit_level is None else _decimal(exit_level, "exit_level") - if close_value <= state.common_stop: - reason = "protective_stop" - elif level is not None and close_value < level: - reason = "trend_exit" - else: - return None - return OrderIntent( - security=state.security, - asset_group=state.asset_group, - action="full_exit", - quantity=state.quantity, - expected_price=expected_price, - signal_date=signal_date, - execution_date=execution_date, - signal_n=state.signal_n, - reason=reason, - ) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_benchmark.py b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_benchmark.py new file mode 100644 index 0000000..02da53a --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_benchmark.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import platform +import shutil +import sys +import time +from dataclasses import asdict, dataclass, is_dataclass +from pathlib import Path +from typing import Mapping + +import numpy as np + +from .result_adapter import ( + LocalExecutionFacts, + materialize_execution_facts, + parameter_document_digest, + to_joinquant_facts, +) +from .vectorbt_engine import run_vectorbt_simulation + + +class PerformanceGateError(RuntimeError): + """Raised when a single local scenario misses its execution gate.""" + + +@dataclass(frozen=True) +class BenchmarkResult: + facts: LocalExecutionFacts + performance: Mapping[str, object] + + +def _canonical_bytes(value: object) -> bytes: + def default(item: object) -> object: + if is_dataclass(item) and not isinstance(item, type): + return asdict(item) + if isinstance(item, np.generic): + return item.item() + raise TypeError(f"unsupported prepared input identity: {type(item).__name__}") + + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + default=default, + ).encode("utf-8") + + +def _prepared_inputs_digest(prepared_inputs: object) -> str: + digest = hashlib.sha256() + try: + fields = vars(prepared_inputs) + except TypeError as exc: + raise PerformanceGateError("prepared inputs do not expose an identity") from exc + for name in sorted(fields): + value = fields[name] + digest.update(name.encode("utf-8")) + digest.update(b"\0") + if isinstance(value, np.ndarray): + contiguous = np.ascontiguousarray(value) + digest.update(str(contiguous.dtype).encode("ascii")) + digest.update(_canonical_bytes(list(contiguous.shape))) + digest.update(contiguous.tobytes()) + else: + digest.update(_canonical_bytes(value)) + return digest.hexdigest() + + +def _environment() -> dict[str, object]: + return { + "python": platform.python_version(), + "implementation": platform.python_implementation(), + "platform": sys.platform, + "dependencies": { + name: importlib.metadata.version(name) + for name in ("vectorbt", "numba", "numpy", "pandas", "pyarrow") + }, + } + + +def _run_once( + *, + prepared_inputs: object, + config: Mapping[str, object], + scenario_id: str, + data_dir: Path, +) -> tuple[LocalExecutionFacts, str, float]: + started = time.perf_counter() + simulation = run_vectorbt_simulation(prepared_inputs, config) + facts = to_joinquant_facts(prepared_inputs, simulation, scenario_id) + result_digest = materialize_execution_facts(data_dir, facts) + elapsed = time.perf_counter() - started + return facts, result_digest, elapsed + + +def benchmark_scenario( + *, + prepared_inputs: object, + config: Mapping[str, object], + scenario_id: str, + work_dir: Path, + code_sha256: str, + config_sha256: str, + limit_seconds: float = 180.0, +) -> BenchmarkResult: + if not isinstance(scenario_id, str) or not scenario_id: + raise PerformanceGateError("scenario identity is missing") + for value in (code_sha256, config_sha256): + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise PerformanceGateError("execution identity is invalid") + if not np.isfinite(limit_seconds) or limit_seconds <= 0.0: + raise PerformanceGateError("performance limit is invalid") + root = Path(work_dir) + if root.exists(): + raise PerformanceGateError("benchmark work directory already exists") + root.mkdir(parents=True) + cold_dir = root / "cold-data" + warm_dir = root / "warm-data" + try: + cold_facts, cold_digest, cold_seconds = _run_once( + prepared_inputs=prepared_inputs, + config=config, + scenario_id=scenario_id, + data_dir=cold_dir, + ) + _, warm_digest, warm_seconds = _run_once( + prepared_inputs=prepared_inputs, + config=config, + scenario_id=scenario_id, + data_dir=warm_dir, + ) + if cold_digest != warm_digest: + raise PerformanceGateError("cold and warm results are not deterministic") + if cold_seconds > limit_seconds or warm_seconds > limit_seconds: + raise PerformanceGateError( + f"single scenario execution exceeded {limit_seconds:g} seconds" + ) + except Exception: + shutil.rmtree(root, ignore_errors=True) + raise + + shutil.rmtree(cold_dir) + shutil.rmtree(warm_dir) + root.rmdir() + cleanup = { + "cold_temporary_result_removed": not cold_dir.exists(), + "warm_temporary_result_removed": not warm_dir.exists(), + "work_directory_removed": not root.exists(), + "verified": not root.exists() and not cold_dir.exists() and not warm_dir.exists(), + } + if not cleanup["verified"]: + raise PerformanceGateError("benchmark temporary artifacts could not be cleaned") + params_sha256 = parameter_document_digest(config) + scenario_sha256 = hashlib.sha256( + _canonical_bytes({"scenario_id": scenario_id, "params_sha256": params_sha256}) + ).hexdigest() + performance = { + "schema_version": "local-backtest-performance/1", + "status": "pass", + "limit_seconds": float(limit_seconds), + "environment": _environment(), + "prepared_inputs_sha256": _prepared_inputs_digest(prepared_inputs), + "code_sha256": code_sha256, + "config_sha256": config_sha256, + "params_sha256": params_sha256, + "scenario": {"scenario_id": scenario_id, "sha256": scenario_sha256}, + "cold_seconds": float(cold_seconds), + "warm_seconds": float(warm_seconds), + "cold_result_sha256": cold_digest, + "warm_result_sha256": warm_digest, + "result_match": True, + "cleanup": cleanup, + } + return BenchmarkResult(facts=cold_facts, performance=performance) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py new file mode 100644 index 0000000..a3e0df1 --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py @@ -0,0 +1,728 @@ +from __future__ import annotations + +from collections import namedtuple + +import numpy as np +from numba import njit +from vectorbt.portfolio import nb +from vectorbt.portfolio.enums import Direction, OrderSide, OrderStatus + + +ACTION_NONE = 0 +ACTION_FULL_EXIT = 1 +ACTION_REDISTRIBUTION_SELL = 2 +ACTION_ENTRY = 3 +ACTION_ADDITION = 4 +ACTION_REDISTRIBUTION_BUY = 5 + +REASON_NONE = 0 +REASON_ENTRY_BREAKOUT = 1 +REASON_FIXED_ADDITION_LEVEL = 2 +REASON_PROTECTIVE_STOP = 3 +REASON_TREND_EXIT = 4 +REASON_MISSING_OPEN = 5 +REASON_PAUSED = 6 +REASON_HIGH_LIMIT = 7 +REASON_LOW_LIMIT = 8 +REASON_ALLOCATION_CONSTRAINT = 9 +REASON_ORDER_REJECTED = 10 +REASON_FULL_POSITION_REDISTRIBUTION = 11 + + +CallbackInputs = namedtuple( + "CallbackInputs", + ( + "execution_open", + "signal_close", + "signal_entry_high", + "signal_exit_low", + "signal_n", + "paused", + "high_limit", + "low_limit", + "asset_group_ids", + ), +) + +CallbackParams = namedtuple( + "CallbackParams", + ( + "lot_size", + "unit_risk_per_n", + "add_step_n", + "stop_n", + "max_units", + "asset_group_unit_cap", + "portfolio_unit_cap", + "commission_multiplier", + "one_way_slippage", + ), +) + +CallbackState = namedtuple( + "CallbackState", + ( + "unit_count", + "unit_signal_n", + "unit_base_quantities", + "unit_fill_prices", + "initial_fill_price", + "initial_signal_n", + "common_stop", + "next_add_index", + "candidate_signal_n", + "candidate_base_quantity", + "action_codes", + "reason_codes", + "requested_quantities", + "planned_quantities", + "filled_quantities", + "fill_prices", + "fees", + "state_quantities", + "state_common_stop", + "state_next_add_index", + "state_unit_counts", + "event_group_scales", + "event_portfolio_scales", + "event_cash_scales", + "day_equity", + "allocation_ready", + ), +) + + +@njit +def _finite_positive(value: float) -> bool: + return np.isfinite(value) and value > 0.0 + + +@njit +def _commission(price: float, quantity: int, multiplier: float) -> float: + return max(5.0, price * quantity * 0.000085) * multiplier + + +@njit +def _buy_price(open_price: float, slippage: float) -> float: + return open_price * (1.0 + slippage) + + +@njit +def _sell_price(open_price: float, slippage: float) -> float: + return open_price * (1.0 - slippage) + + +@njit +def _buy_tradability_reason_nb( + row: int, column: int, inputs: CallbackInputs +) -> int: + open_price = inputs.execution_open[row, column] + if not _finite_positive(open_price): + return REASON_MISSING_OPEN + if inputs.paused[row, column]: + return REASON_PAUSED + high_limit = inputs.high_limit[row, column] + if np.isfinite(high_limit) and open_price >= high_limit: + return REASON_HIGH_LIMIT + return REASON_NONE + + +@njit +def _sell_tradability_reason_nb( + row: int, column: int, inputs: CallbackInputs +) -> int: + open_price = inputs.execution_open[row, column] + if not _finite_positive(open_price): + return REASON_MISSING_OPEN + if inputs.paused[row, column]: + return REASON_PAUSED + low_limit = inputs.low_limit[row, column] + if np.isfinite(low_limit) and open_price <= low_limit: + return REASON_LOW_LIMIT + return REASON_NONE + + +@njit +def _risk_scales_nb( + unit_counts: np.ndarray, + asset_group_ids: np.ndarray, + group_count: int, + asset_group_unit_cap: float, + portfolio_unit_cap: float, +): + group_units = np.zeros(group_count, dtype=np.float64) + for column in range(unit_counts.shape[0]): + group_units[asset_group_ids[column]] += unit_counts[column] + group_scales = np.ones(group_count, dtype=np.float64) + for group in range(group_count): + if group_units[group] > asset_group_unit_cap: + group_scales[group] = asset_group_unit_cap / group_units[group] + effective_units = 0.0 + for column in range(unit_counts.shape[0]): + effective_units += ( + unit_counts[column] * group_scales[asset_group_ids[column]] + ) + portfolio_scale = 1.0 + if effective_units > portfolio_unit_cap: + portfolio_scale = portfolio_unit_cap / effective_units + return group_scales, portfolio_scale + + +@njit +def _targets_for_scale_nb( + unit_base_quantities: np.ndarray, + unit_counts: np.ndarray, + asset_group_ids: np.ndarray, + group_scales: np.ndarray, + portfolio_scale: float, + cash_scale: float, + locked_quantities: np.ndarray, + lot_size: int, +) -> np.ndarray: + targets = np.zeros(unit_counts.shape[0], dtype=np.int64) + for column in range(unit_counts.shape[0]): + if locked_quantities[column] >= 0: + targets[column] = locked_quantities[column] + continue + raw_quantity = 0 + for unit in range(unit_counts[column]): + raw_quantity += unit_base_quantities[column, unit] + scaled = ( + raw_quantity + * group_scales[asset_group_ids[column]] + * portfolio_scale + * cash_scale + ) + targets[column] = int(scaled // lot_size) * lot_size + return targets + + +@njit +def _cash_after_targets_nb( + row: int, + targets: np.ndarray, + positions: np.ndarray, + cash: float, + inputs: CallbackInputs, + params: CallbackParams, +) -> float: + projected_cash = cash + for column in range(targets.shape[0]): + current = int(round(positions[column])) + if targets[column] >= current: + continue + quantity = current - targets[column] + price = _sell_price( + inputs.execution_open[row, column], params.one_way_slippage + ) + projected_cash += price * quantity - _commission( + price, quantity, params.commission_multiplier + ) + for column in range(targets.shape[0]): + current = int(round(positions[column])) + if targets[column] <= current: + continue + quantity = targets[column] - current + price = _buy_price( + inputs.execution_open[row, column], params.one_way_slippage + ) + projected_cash -= price * quantity + _commission( + price, quantity, params.commission_multiplier + ) + return projected_cash + + +@njit +def _cash_feasible_targets_nb( + row: int, + unit_base_quantities: np.ndarray, + unit_counts: np.ndarray, + positions: np.ndarray, + cash: float, + group_scales: np.ndarray, + portfolio_scale: float, + locked_quantities: np.ndarray, + inputs: CallbackInputs, + params: CallbackParams, +): + full_targets = _targets_for_scale_nb( + unit_base_quantities, + unit_counts, + inputs.asset_group_ids, + group_scales, + portfolio_scale, + 1.0, + locked_quantities, + params.lot_size, + ) + if _cash_after_targets_nb( + row, full_targets, positions, cash, inputs, params + ) >= -1e-9: + return full_targets, 1.0 + lower = 0.0 + upper = 1.0 + best = _targets_for_scale_nb( + unit_base_quantities, + unit_counts, + inputs.asset_group_ids, + group_scales, + portfolio_scale, + lower, + locked_quantities, + params.lot_size, + ) + for _ in range(64): + candidate_scale = (lower + upper) / 2.0 + candidate = _targets_for_scale_nb( + unit_base_quantities, + unit_counts, + inputs.asset_group_ids, + group_scales, + portfolio_scale, + candidate_scale, + locked_quantities, + params.lot_size, + ) + if _cash_after_targets_nb( + row, candidate, positions, cash, inputs, params + ) >= -1e-9: + lower = candidate_scale + best = candidate + else: + upper = candidate_scale + return best, lower + + +@njit +def _clear_position_state_nb(column: int, state: CallbackState) -> None: + state.unit_count[column] = 0 + for unit in range(state.unit_signal_n.shape[1]): + state.unit_signal_n[column, unit] = np.nan + state.unit_base_quantities[column, unit] = 0 + state.unit_fill_prices[column, unit] = np.nan + state.initial_fill_price[column] = np.nan + state.initial_signal_n[column] = np.nan + state.common_stop[column] = np.nan + state.next_add_index[column] = 0 + + +@njit +def pre_sim_func_nb( + c, + state: CallbackState, + inputs: CallbackInputs, + params: CallbackParams, +): + state.unit_count[:] = 0 + state.unit_signal_n[:, :] = np.nan + state.unit_base_quantities[:, :] = 0 + state.unit_fill_prices[:, :] = np.nan + state.initial_fill_price[:] = np.nan + state.initial_signal_n[:] = np.nan + state.common_stop[:] = np.nan + state.next_add_index[:] = 0 + state.candidate_signal_n[:, :] = np.nan + state.candidate_base_quantity[:, :] = 0 + state.action_codes[:, :] = ACTION_NONE + state.reason_codes[:, :] = REASON_NONE + state.requested_quantities[:, :] = 0 + state.planned_quantities[:, :] = 0 + state.filled_quantities[:, :] = 0 + state.fill_prices[:, :] = np.nan + state.fees[:, :] = 0.0 + state.state_quantities[:, :] = 0 + state.state_common_stop[:, :] = np.nan + state.state_next_add_index[:, :] = 0 + state.state_unit_counts[:, :] = 0 + state.event_group_scales[:, :] = 1.0 + state.event_portfolio_scales[:] = 1.0 + state.event_cash_scales[:] = 1.0 + state.day_equity[:] = np.nan + state.allocation_ready[:] = False + return state, inputs, params + + +@njit +def _set_call_sequence_nb(c, state: CallbackState) -> None: + row = c.i + call_index = 0 + for category in range(5): + for column in range(c.from_col, c.to_col): + action = state.action_codes[row, column] + actual_category = 4 + if action == ACTION_FULL_EXIT: + actual_category = 0 + elif action == ACTION_REDISTRIBUTION_SELL: + actual_category = 1 + elif action == ACTION_ENTRY or action == ACTION_ADDITION: + actual_category = 2 + elif action == ACTION_REDISTRIBUTION_BUY: + actual_category = 3 + if actual_category == category: + c.call_seq_now[call_index] = column - c.from_col + call_index += 1 + + +@njit +def pre_segment_func_nb( + c, + state: CallbackState, + inputs: CallbackInputs, + params: CallbackParams, +): + row = c.i + column_count = c.to_col - c.from_col + equity = c.last_value[c.group] + state.day_equity[row] = equity + state.allocation_ready[row] = False + for column in range(c.from_col, c.to_col): + state.action_codes[row, column] = ACTION_NONE + state.reason_codes[row, column] = REASON_NONE + state.requested_quantities[row, column] = 0 + state.planned_quantities[row, column] = 0 + state.candidate_signal_n[row, column] = np.nan + state.candidate_base_quantity[row, column] = 0 + + exit_active = np.zeros(column_count, dtype=np.bool_) + candidate_active = np.zeros(column_count, dtype=np.bool_) + any_decision = False + + for offset in range(column_count): + column = c.from_col + offset + position = c.last_position[column] + close = inputs.signal_close[row, column] + if position <= 0.0 or not _finite_positive(close): + continue + reason = REASON_NONE + if np.isfinite(state.common_stop[column]) and close <= state.common_stop[column]: + reason = REASON_PROTECTIVE_STOP + elif ( + np.isfinite(inputs.signal_exit_low[row, column]) + and close < inputs.signal_exit_low[row, column] + ): + reason = REASON_TREND_EXIT + if reason != REASON_NONE: + state.action_codes[row, column] = ACTION_FULL_EXIT + state.reason_codes[row, column] = reason + state.requested_quantities[row, column] = int(round(position)) + tradability = _sell_tradability_reason_nb(row, column, inputs) + if tradability == REASON_NONE: + exit_active[offset] = True + else: + state.reason_codes[row, column] = tradability + any_decision = True + + for offset in range(column_count): + column = c.from_col + offset + if state.action_codes[row, column] == ACTION_FULL_EXIT: + continue + close = inputs.signal_close[row, column] + signal_n = inputs.signal_n[row, column] + if not _finite_positive(close) or not _finite_positive(signal_n): + continue + position = c.last_position[column] + action = ACTION_NONE + reason = REASON_NONE + if position > 0.0 and state.unit_count[column] > 0: + next_index = state.next_add_index[column] + if ( + next_index < params.max_units + and close + >= state.initial_fill_price[column] + + next_index * params.add_step_n * state.initial_signal_n[column] + ): + action = ACTION_ADDITION + reason = REASON_FIXED_ADDITION_LEVEL + elif ( + position <= 0.0 + and np.isfinite(inputs.signal_entry_high[row, column]) + and close > inputs.signal_entry_high[row, column] + ): + action = ACTION_ENTRY + reason = REASON_ENTRY_BREAKOUT + if action == ACTION_NONE: + continue + quantity = int(equity * params.unit_risk_per_n / signal_n) + quantity = (quantity // params.lot_size) * params.lot_size + if quantity <= 0: + continue + state.action_codes[row, column] = action + state.reason_codes[row, column] = reason + state.requested_quantities[row, column] = quantity + state.candidate_signal_n[row, column] = signal_n + state.candidate_base_quantity[row, column] = quantity + tradability = _buy_tradability_reason_nb(row, column, inputs) + if tradability == REASON_NONE: + candidate_active[offset] = True + else: + state.reason_codes[row, column] = tradability + any_decision = True + + if not any_decision: + _set_call_sequence_nb(c, state) + for column in range(c.from_col, c.to_col): + open_price = inputs.execution_open[row, column] + if _finite_positive(open_price): + c.last_val_price[column] = open_price + state.allocation_ready[row] = True + return state, inputs, params + + locked = np.full(column_count, -1, dtype=np.int64) + for offset in range(column_count): + column = c.from_col + offset + if ( + not _finite_positive(inputs.execution_open[row, column]) + or inputs.paused[row, column] + or ( + state.action_codes[row, column] == ACTION_FULL_EXIT + and not exit_active[offset] + ) + ): + locked[offset] = int(round(c.last_position[column])) + + targets = np.asarray(c.last_position[c.from_col : c.to_col], dtype=np.int64) + group_count = 1 + for offset in range(column_count): + group_count = max( + group_count, int(inputs.asset_group_ids[c.from_col + offset]) + 1 + ) + group_scales = np.ones(group_count, dtype=np.float64) + portfolio_scale = 1.0 + cash_scale = 1.0 + for _ in range(column_count * 3 + 1): + counts = state.unit_count[c.from_col : c.to_col].copy() + bases = state.unit_base_quantities[c.from_col : c.to_col].copy() + for offset in range(column_count): + column = c.from_col + offset + if exit_active[offset]: + counts[offset] = 0 + for unit in range(params.max_units): + bases[offset, unit] = 0 + elif candidate_active[offset]: + slot = counts[offset] + bases[offset, slot] = state.candidate_base_quantity[row, column] + counts[offset] = slot + 1 + group_scales, portfolio_scale = _risk_scales_nb( + counts, + inputs.asset_group_ids[c.from_col : c.to_col], + group_count, + params.asset_group_unit_cap, + params.portfolio_unit_cap, + ) + local_inputs = CallbackInputs( + inputs.execution_open[:, c.from_col : c.to_col], + inputs.signal_close[:, c.from_col : c.to_col], + inputs.signal_entry_high[:, c.from_col : c.to_col], + inputs.signal_exit_low[:, c.from_col : c.to_col], + inputs.signal_n[:, c.from_col : c.to_col], + inputs.paused[:, c.from_col : c.to_col], + inputs.high_limit[:, c.from_col : c.to_col], + inputs.low_limit[:, c.from_col : c.to_col], + inputs.asset_group_ids[c.from_col : c.to_col], + ) + targets, cash_scale = _cash_feasible_targets_nb( + row, + bases, + counts, + c.last_position[c.from_col : c.to_col], + c.last_cash[c.group], + group_scales, + portfolio_scale, + locked, + local_inputs, + params, + ) + changed = False + for offset in range(column_count): + column = c.from_col + offset + current = int(round(c.last_position[column])) + if locked[offset] < 0 and targets[offset] < current: + if _sell_tradability_reason_nb(row, column, inputs) != REASON_NONE: + locked[offset] = current + changed = True + elif locked[offset] < 0 and targets[offset] > current: + if _buy_tradability_reason_nb(row, column, inputs) != REASON_NONE: + locked[offset] = current + changed = True + if ( + candidate_active[offset] + and targets[offset] - current < params.lot_size + ): + candidate_active[offset] = False + state.reason_codes[row, column] = REASON_ALLOCATION_CONSTRAINT + changed = True + if not changed: + break + + has_effective_event = False + for offset in range(column_count): + if exit_active[offset] or candidate_active[offset]: + has_effective_event = True + if has_effective_event: + for offset in range(column_count): + column = c.from_col + offset + group = inputs.asset_group_ids[column] + state.event_group_scales[row, column] = group_scales[group] + state.event_portfolio_scales[row] = portfolio_scale + state.event_cash_scales[row] = cash_scale + for offset in range(column_count): + column = c.from_col + offset + current = int(round(c.last_position[column])) + action = state.action_codes[row, column] + if action == ACTION_FULL_EXIT: + if exit_active[offset]: + state.planned_quantities[row, column] = current + continue + delta = targets[offset] - current + if action == ACTION_ENTRY or action == ACTION_ADDITION: + if candidate_active[offset] and delta >= params.lot_size: + state.planned_quantities[row, column] = delta + continue + if delta <= -params.lot_size: + state.action_codes[row, column] = ACTION_REDISTRIBUTION_SELL + state.reason_codes[row, column] = ( + REASON_FULL_POSITION_REDISTRIBUTION + ) + state.requested_quantities[row, column] = -delta + state.planned_quantities[row, column] = -delta + elif delta >= params.lot_size: + state.action_codes[row, column] = ACTION_REDISTRIBUTION_BUY + state.reason_codes[row, column] = ( + REASON_FULL_POSITION_REDISTRIBUTION + ) + state.requested_quantities[row, column] = delta + state.planned_quantities[row, column] = delta + + _set_call_sequence_nb(c, state) + for column in range(c.from_col, c.to_col): + open_price = inputs.execution_open[row, column] + if _finite_positive(open_price): + c.last_val_price[column] = open_price + state.allocation_ready[row] = True + return state, inputs, params + + +@njit +def order_func_nb( + c, + state: CallbackState, + inputs: CallbackInputs, + params: CallbackParams, +): + row = c.i + column = c.col + action = state.action_codes[row, column] + reason = state.reason_codes[row, column] + if action == ACTION_NONE or reason in ( + REASON_MISSING_OPEN, + REASON_PAUSED, + REASON_HIGH_LIMIT, + REASON_LOW_LIMIT, + REASON_ALLOCATION_CONSTRAINT, + ): + return nb.NoOrder + quantity = state.planned_quantities[row, column] + if quantity <= 0: + return nb.NoOrder + open_price = inputs.execution_open[row, column] + if action == ACTION_FULL_EXIT or action == ACTION_REDISTRIBUTION_SELL: + quantity = min(quantity, int(round(c.position_now))) + if quantity <= 0: + return nb.NoOrder + price = _sell_price(open_price, params.one_way_slippage) + return nb.order_nb( + size=-float(quantity), + price=price, + direction=Direction.LongOnly, + fixed_fees=_commission(price, quantity, params.commission_multiplier), + allow_partial=False, + ) + price = _buy_price(open_price, params.one_way_slippage) + return nb.order_nb( + size=float(quantity), + price=price, + direction=Direction.LongOnly, + fixed_fees=_commission(price, quantity, params.commission_multiplier), + size_granularity=float(params.lot_size), + allow_partial=False, + ) + + +@njit +def _record_candidate_unit_nb( + row: int, + column: int, + fill_price: float, + state: CallbackState, + params: CallbackParams, +) -> None: + slot = state.unit_count[column] + if slot >= params.max_units: + return + signal_n = state.candidate_signal_n[row, column] + state.unit_signal_n[column, slot] = signal_n + state.unit_base_quantities[column, slot] = state.candidate_base_quantity[ + row, column + ] + state.unit_fill_prices[column, slot] = fill_price + if slot == 0: + state.initial_fill_price[column] = fill_price + state.initial_signal_n[column] = signal_n + state.common_stop[column] = fill_price - params.stop_n * signal_n + else: + candidate_stop = fill_price - params.stop_n * signal_n + state.common_stop[column] = max( + state.common_stop[column], candidate_stop + ) + state.unit_count[column] = slot + 1 + state.next_add_index[column] = slot + 1 + + +@njit +def post_order_func_nb( + c, + state: CallbackState, + inputs: CallbackInputs, + params: CallbackParams, +) -> None: + row = c.i + column = c.col + action = state.action_codes[row, column] + result = c.order_result + if result.status == OrderStatus.Filled: + quantity = int(round(result.size)) + state.filled_quantities[row, column] = quantity + state.fill_prices[row, column] = result.price + state.fees[row, column] = result.fees + if result.side == OrderSide.Sell: + if action == ACTION_FULL_EXIT and c.position_now <= 1e-9: + _clear_position_state_nb(column, state) + elif action == ACTION_ENTRY: + _clear_position_state_nb(column, state) + _record_candidate_unit_nb( + row, column, result.price, state, params + ) + elif action == ACTION_ADDITION: + _record_candidate_unit_nb( + row, column, result.price, state, params + ) + elif result.status == OrderStatus.Rejected: + state.reason_codes[row, column] = REASON_ORDER_REJECTED + + + +@njit +def post_segment_func_nb( + c, + state: CallbackState, + inputs: CallbackInputs, + params: CallbackParams, +) -> None: + row = c.i + for column in range(c.from_col, c.to_col): + state.state_quantities[row, column] = int( + round(c.last_position[column]) + ) + state.state_common_stop[row, column] = state.common_stop[column] + state.state_next_add_index[row, column] = state.next_add_index[column] + state.state_unit_counts[row, column] = state.unit_count[column] diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_cli.py b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_cli.py new file mode 100644 index 0000000..32d1da6 --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_cli.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Mapping + +import pandas as pd + +if __package__ in {None, ""}: + REPOSITORY_ROOT = Path(__file__).resolve().parents[5] + sys.path.insert(0, str(REPOSITORY_ROOT)) +else: + REPOSITORY_ROOT = Path(__file__).resolve().parents[5] + +from scripts.research.market_data.query import open_snapshot +from scripts.research.market_data.storage import MarketDataError + +if __package__ in {None, ""}: + RESEARCH_ROOT = Path(__file__).resolve().parent.parent + sys.path.insert(0, str(RESEARCH_ROOT)) + from turtle_etf.result_adapter import ResultContractError + from turtle_etf.single_scenario import ( + SingleScenarioError, + execute_prepared_scenario, + validate_single_scenario_config, + write_project_status, + ) + from turtle_etf.vectorbt_benchmark import PerformanceGateError + from turtle_etf.vectorbt_inputs import prepare_simulation_inputs +else: + from .result_adapter import ResultContractError + from .single_scenario import ( + SingleScenarioError, + execute_prepared_scenario, + validate_single_scenario_config, + write_project_status, + ) + from .vectorbt_benchmark import PerformanceGateError + from .vectorbt_inputs import prepare_simulation_inputs + + +class ProjectInputError(ValueError): + """Raised when frozen project inputs cannot identify one scenario.""" + + +def _load_json(path: Path, name: str) -> dict[str, object]: + try: + value = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ProjectInputError(f"invalid_{name}") from exc + if not isinstance(value, dict): + raise ProjectInputError(f"invalid_{name}") + return value + + +def _market_frames( + rows: tuple[Mapping[str, object], ...], + config: Mapping[str, object], +) -> dict[str, pd.DataFrame]: + universe = config.get("universe") + if not isinstance(universe, list) or not universe: + raise ProjectInputError("invalid_universe") + securities = tuple(str(item.get("security", "")) for item in universe if isinstance(item, Mapping)) + if len(securities) != len(universe) or any(not security for security in securities): + raise ProjectInputError("invalid_universe") + frame = pd.DataFrame([dict(row) for row in rows]) + if frame.empty or set(frame["security"].astype(str)) != set(securities): + raise ProjectInputError("snapshot_universe_mismatch") + return { + security: frame.loc[frame["security"].astype(str) == security].copy() + for security in securities + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run one local vectorbt scenario") + parser.add_argument("--snapshot-manifest", type=Path, required=True) + parser.add_argument("--market-data-root", type=Path, required=True) + parser.add_argument("--project-config", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--snapshot-id", required=True) + parser.add_argument("--code-sha256", required=True) + parser.add_argument("--config-sha256", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + expected_manifest = ( + args.market_data_root / "snapshots" / f"{args.snapshot_id}.json" + ).resolve() + if args.snapshot_manifest.resolve() != expected_manifest: + raise ProjectInputError("snapshot_manifest_identity_mismatch") + config = _load_json(args.project_config, "project_config") + validate_single_scenario_config(config) + snapshot = open_snapshot(args.snapshot_id, root=args.market_data_root) + frames = _market_frames(snapshot.rows, config) + prepared = prepare_simulation_inputs( + frames, + config, + corporate_actions=snapshot.corporate_actions, + corporate_actions_digest=snapshot.corporate_actions_digest, + ) + execute_prepared_scenario( + prepared_inputs=prepared, + config=config, + output_dir=args.output_dir, + run_id=args.run_id, + snapshot_id=args.snapshot_id, + code_sha256=args.code_sha256, + config_sha256=args.config_sha256, + code_path=Path(__file__), + ) + write_project_status( + args.output_dir, + status="complete", + reason_codes=(), + next_action="return_to_caller", + ) + return 0 + except (PerformanceGateError, ResultContractError, OSError): + write_project_status( + args.output_dir, + status="failed", + reason_codes=("local_vectorbt_execution_failed",), + ) + return 1 + except (ProjectInputError, SingleScenarioError, MarketDataError, ValueError): + write_project_status( + args.output_dir, + status="evidence_insufficient", + reason_codes=("project_input_invalid",), + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py new file mode 100644 index 0000000..b2d8f95 --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import vectorbt as vbt + +from .vectorbt_callbacks import ( + ACTION_ADDITION, + ACTION_ENTRY, + ACTION_FULL_EXIT, + ACTION_NONE, + ACTION_REDISTRIBUTION_BUY, + ACTION_REDISTRIBUTION_SELL, + REASON_NONE, +) + + +ADJUST_NONE = 0 +ADJUST_CASH_TRUNCATED = 1 +ADJUST_HOLDING_TRUNCATED = 2 +ADJUST_UNTRADABLE = 3 +ADJUST_HORIZON_EXPIRED = 4 + + +@dataclass(frozen=True) +class FrozenOrderPlan: + planned_row_indices: np.ndarray + action_codes: np.ndarray + reason_codes: np.ndarray + requested_quantities: np.ndarray + target_quantities: np.ndarray + signal_n: np.ndarray + + +@dataclass(frozen=True) +class HorizonExpiredOrder: + planned_row_index: int + column: int + action_code: int + reason_code: int + requested_quantity: int + target_quantity: int + signal_n: float + delay_days: int + + +@dataclass(frozen=True) +class DelayedExecutionResult: + portfolio: object + action_codes: np.ndarray + reason_codes: np.ndarray + requested_quantities: np.ndarray + planned_quantities: np.ndarray + filled_quantities: np.ndarray + fill_prices: np.ndarray + fees: np.ndarray + state_quantities: np.ndarray + state_common_stop: np.ndarray + state_next_add_index: np.ndarray + state_unit_counts: np.ndarray + day_equity: np.ndarray + planned_row_indices: np.ndarray + execution_adjustment_codes: np.ndarray + frozen_signal_n: np.ndarray + execution_sequence: tuple[tuple[str, ...], ...] + horizon_expired_orders: tuple[HorizonExpiredOrder, ...] + + +def _readonly(values: np.ndarray, dtype: str) -> np.ndarray: + result = np.ascontiguousarray(values, dtype=dtype) + result.setflags(write=False) + return result + + +def _matrix(value: object, shape: tuple[int, int], name: str) -> np.ndarray: + result = np.asarray(value) + if result.shape != shape: + raise ValueError(f"invalid frozen order source shape: {name}") + return result + + +def freeze_order_plan(inputs: object, immediate: object) -> FrozenOrderPlan: + signal_n = np.asarray(getattr(inputs, "signal_n"), dtype=np.float64) + shape = signal_n.shape + actions = _matrix(immediate.action_codes, shape, "action_codes") + reasons = _matrix(immediate.reason_codes, shape, "reason_codes") + requested = _matrix( + immediate.requested_quantities, shape, "requested_quantities" + ) + planned = _matrix(immediate.planned_quantities, shape, "planned_quantities") + filled = _matrix(immediate.filled_quantities, shape, "filled_quantities") + valid = filled > 0 + row_indices = np.broadcast_to( + np.arange(shape[0], dtype=np.int64)[:, None], shape + ) + targets = np.where(valid, np.where(planned > 0, planned, filled), 0) + return FrozenOrderPlan( + planned_row_indices=_readonly( + np.where(valid, row_indices, -1), "int64" + ), + action_codes=_readonly(np.where(valid, actions, ACTION_NONE), "int16"), + reason_codes=_readonly(np.where(valid, reasons, REASON_NONE), "int16"), + requested_quantities=_readonly(np.where(valid, requested, 0), "int64"), + target_quantities=_readonly(targets, "int64"), + signal_n=_readonly(np.where(valid, signal_n, np.nan), "float64"), + ) + + +def _commission(price: float, quantity: int, multiplier: float) -> float: + return max(5.0, price * quantity * 0.000085) * multiplier + + +def _priority(action: int) -> int: + if action == ACTION_FULL_EXIT: + return 0 + if action == ACTION_REDISTRIBUTION_SELL: + return 1 + if action in (ACTION_ENTRY, ACTION_ADDITION): + return 2 + if action == ACTION_REDISTRIBUTION_BUY: + return 3 + return 4 + + +def _is_tradable(inputs: object, row: int, column: int, action: int) -> bool: + open_price = float(inputs.execution_open[row, column]) + if not np.isfinite(open_price) or open_price <= 0.0: + return False + if bool(inputs.paused[row, column]): + return False + if action in (ACTION_FULL_EXIT, ACTION_REDISTRIBUTION_SELL): + low_limit = float(inputs.low_limit[row, column]) + return not np.isfinite(low_limit) or open_price > low_limit + high_limit = float(inputs.high_limit[row, column]) + return not np.isfinite(high_limit) or open_price < high_limit + + +def _affordable_quantity( + *, + cash: float, + target: int, + lot_size: int, + price: float, + commission_multiplier: float, +) -> int: + quantity = (target // lot_size) * lot_size + while quantity > 0: + fee = _commission(price, quantity, commission_multiplier) + if price * quantity + fee <= cash + 1e-9: + return quantity + quantity -= lot_size + return 0 + + +def run_delayed_execution( + inputs: object, + plan: FrozenOrderPlan, + *, + initial_cash: float, + lot_size: int, + stop_n: float, + commission_multiplier: float, + one_way_slippage: float, + delay_days: int, +) -> DelayedExecutionResult: + if delay_days <= 0: + raise ValueError("delayed execution requires a positive delay") + if lot_size <= 0 or initial_cash <= 0.0: + raise ValueError("delayed execution cash and lot size must be positive") + opens = np.asarray(inputs.execution_open, dtype=np.float64) + close = np.asarray(inputs.close, dtype=np.float64) + if opens.shape != close.shape or opens.shape != plan.action_codes.shape: + raise ValueError("delayed execution input shapes differ") + rows, columns = opens.shape + securities = tuple(str(value) for value in inputs.securities) + if len(securities) != columns: + raise ValueError("delayed execution securities do not match columns") + + actions = np.zeros((rows, columns), dtype=np.int16) + reasons = np.zeros((rows, columns), dtype=np.int16) + requested = np.zeros((rows, columns), dtype=np.int64) + planned = np.zeros((rows, columns), dtype=np.int64) + filled = np.zeros((rows, columns), dtype=np.int64) + fill_prices = np.full((rows, columns), np.nan, dtype=np.float64) + fees = np.zeros((rows, columns), dtype=np.float64) + state_quantities = np.zeros((rows, columns), dtype=np.int64) + state_common_stop = np.full((rows, columns), np.nan, dtype=np.float64) + state_next_add = np.zeros((rows, columns), dtype=np.int64) + state_unit_counts = np.zeros((rows, columns), dtype=np.int64) + planned_rows = np.full((rows, columns), -1, dtype=np.int64) + adjustments = np.zeros((rows, columns), dtype=np.int16) + frozen_n = np.full((rows, columns), np.nan, dtype=np.float64) + sequences: list[tuple[str, ...]] = [] + expired: list[HorizonExpiredOrder] = [] + + cash_now = float(initial_cash) + positions = np.zeros(columns, dtype=np.int64) + common_stop = np.full(columns, np.nan, dtype=np.float64) + next_add = np.zeros(columns, dtype=np.int64) + unit_counts = np.zeros(columns, dtype=np.int64) + last_close = np.full(columns, np.nan, dtype=np.float64) + values = np.full(rows, np.nan, dtype=np.float64) + cash_values = np.full(rows, np.nan, dtype=np.float64) + call_seq = np.empty((rows, columns), dtype=np.int64) + + for planned_row, column in zip(*np.nonzero(plan.action_codes != ACTION_NONE)): + if planned_row + delay_days >= rows: + expired.append( + HorizonExpiredOrder( + planned_row_index=int(planned_row), + column=int(column), + action_code=int(plan.action_codes[planned_row, column]), + reason_code=int(plan.reason_codes[planned_row, column]), + requested_quantity=int( + plan.requested_quantities[planned_row, column] + ), + target_quantity=int(plan.target_quantities[planned_row, column]), + signal_n=float(plan.signal_n[planned_row, column]), + delay_days=delay_days, + ) + ) + + for execution_row in range(rows): + source_row = execution_row - delay_days + queue: list[int] = [] + if source_row >= 0: + queue = [ + column + for column in range(columns) + if plan.action_codes[source_row, column] != ACTION_NONE + ] + queue.sort( + key=lambda column: ( + _priority(int(plan.action_codes[source_row, column])), + securities[column], + ) + ) + sequences.append( + tuple( + f"queued-from-row-{source_row}:{securities[column]}" + for column in queue + ) + ) + ordered_columns = queue + sorted( + (column for column in range(columns) if column not in queue), + key=lambda column: securities[column], + ) + for rank, column in enumerate(ordered_columns): + call_seq[execution_row, rank] = column + for column in queue: + action = int(plan.action_codes[source_row, column]) + target = int(plan.target_quantities[source_row, column]) + actions[execution_row, column] = action + reasons[execution_row, column] = int( + plan.reason_codes[source_row, column] + ) + requested[execution_row, column] = int( + plan.requested_quantities[source_row, column] + ) + planned[execution_row, column] = target + planned_rows[execution_row, column] = source_row + frozen_n[execution_row, column] = float(plan.signal_n[source_row, column]) + if not _is_tradable(inputs, execution_row, column, action): + adjustments[execution_row, column] = ADJUST_UNTRADABLE + continue + + open_price = float(opens[execution_row, column]) + if action in (ACTION_FULL_EXIT, ACTION_REDISTRIBUTION_SELL): + quantity = min(target, int(positions[column])) + if quantity < target: + adjustments[execution_row, column] = ADJUST_HOLDING_TRUNCATED + if quantity <= 0: + continue + price = open_price * (1.0 - one_way_slippage) + fee = _commission(price, quantity, commission_multiplier) + cash_now += price * quantity - fee + positions[column] -= quantity + if positions[column] == 0: + common_stop[column] = np.nan + next_add[column] = 0 + if action == ACTION_FULL_EXIT: + unit_counts[column] = 0 + else: + price = open_price * (1.0 + one_way_slippage) + quantity = _affordable_quantity( + cash=cash_now, + target=target, + lot_size=lot_size, + price=price, + commission_multiplier=commission_multiplier, + ) + if quantity < target: + adjustments[execution_row, column] = ADJUST_CASH_TRUNCATED + if quantity <= 0: + continue + fee = _commission(price, quantity, commission_multiplier) + cash_now -= price * quantity + fee + positions[column] += quantity + if action in (ACTION_ENTRY, ACTION_ADDITION): + signal_n = float(plan.signal_n[source_row, column]) + candidate_stop = price - stop_n * signal_n + if action == ACTION_ENTRY or not np.isfinite(common_stop[column]): + common_stop[column] = candidate_stop + next_add[column] = 1 + unit_counts[column] = 1 + else: + common_stop[column] = max(common_stop[column], candidate_stop) + next_add[column] += 1 + unit_counts[column] += 1 + filled[execution_row, column] = quantity + fill_prices[execution_row, column] = price + fees[execution_row, column] = fee + + state_quantities[execution_row] = positions + state_common_stop[execution_row] = common_stop + state_next_add[execution_row] = next_add + state_unit_counts[execution_row] = unit_counts + for column in range(columns): + price = float(close[execution_row, column]) + if np.isfinite(price) and price > 0.0: + last_close[column] = price + held_values = np.where(positions > 0, positions * last_close, 0.0) + if np.any(~np.isfinite(held_values)): + raise ValueError("delayed position has no valid valuation price") + values[execution_row] = cash_now + float(held_values.sum()) + cash_values[execution_row] = cash_now + + order_sizes = np.full((rows, columns), np.nan, dtype=np.float64) + for row, column in zip(*np.nonzero(filled > 0)): + direction = -1.0 if actions[row, column] in ( + ACTION_FULL_EXIT, + ACTION_REDISTRIBUTION_SELL, + ) else 1.0 + order_sizes[row, column] = direction * float(filled[row, column]) + close_frame = pd.DataFrame( + close, + index=pd.DatetimeIndex(np.asarray(inputs.dates, dtype="datetime64[ns]")), + columns=securities, + ) + portfolio = vbt.Portfolio.from_orders( + close_frame, + size=order_sizes, + price=fill_prices, + fixed_fees=fees, + direction="longonly", + init_cash=initial_cash, + cash_sharing=True, + group_by=True, + call_seq=call_seq, + update_value=True, + ffill_val_price=True, + max_orders=rows * columns, + freq="1D", + ) + vectorbt_values = np.asarray(portfolio.value(), dtype=np.float64).reshape(-1) + vectorbt_cash = np.asarray(portfolio.cash(), dtype=np.float64).reshape(-1) + if not np.allclose(vectorbt_values, values, rtol=0.0, atol=0.02): + differences = np.abs(vectorbt_values - values) + row = int(np.nanargmax(differences)) + raise ValueError( + "vectorbt delayed portfolio value does not reconcile: " + f"row={row} expected={values[row]} actual={vectorbt_values[row]} " + f"difference={differences[row]} " + f"expected_orders={int(np.count_nonzero(filled))} " + f"actual_orders={int(portfolio.orders.count())}" + ) + if not np.allclose(vectorbt_cash, cash_values, rtol=0.0, atol=0.02): + raise ValueError("vectorbt delayed portfolio cash does not reconcile") + return DelayedExecutionResult( + portfolio=portfolio, + action_codes=_readonly(actions, "int16"), + reason_codes=_readonly(reasons, "int16"), + requested_quantities=_readonly(requested, "int64"), + planned_quantities=_readonly(planned, "int64"), + filled_quantities=_readonly(filled, "int64"), + fill_prices=_readonly(fill_prices, "float64"), + fees=_readonly(fees, "float64"), + state_quantities=_readonly(state_quantities, "int64"), + state_common_stop=_readonly(state_common_stop, "float64"), + state_next_add_index=_readonly(state_next_add, "int64"), + state_unit_counts=_readonly(state_unit_counts, "int64"), + day_equity=_readonly(values, "float64"), + planned_row_indices=_readonly(planned_rows, "int64"), + execution_adjustment_codes=_readonly(adjustments, "int16"), + frozen_signal_n=_readonly(frozen_n, "float64"), + execution_sequence=tuple(sequences), + horizon_expired_orders=tuple(expired), + ) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py new file mode 100644 index 0000000..64e7996 --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py @@ -0,0 +1,388 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + +import numpy as np +import pandas as pd +import vectorbt as vbt + +from .vectorbt_callbacks import ( + ACTION_NONE, + CallbackInputs, + CallbackParams, + CallbackState, + order_func_nb, + post_order_func_nb, + post_segment_func_nb, + pre_segment_func_nb, + pre_sim_func_nb, +) +from .vectorbt_delayed import freeze_order_plan, run_delayed_execution +from .vectorbt_inputs import SimulationInputs + + +@dataclass(frozen=True) +class VectorbtSimulationResult: + initial_cash: float + asset_group_unit_cap: float + portfolio_unit_cap: float + portfolio: object + action_codes: np.ndarray + reason_codes: np.ndarray + requested_quantities: np.ndarray + planned_quantities: np.ndarray + filled_quantities: np.ndarray + fill_prices: np.ndarray + fees: np.ndarray + state_quantities: np.ndarray + state_common_stop: np.ndarray + state_next_add_index: np.ndarray + state_unit_counts: np.ndarray + candidate_base_quantities: np.ndarray + event_group_scales: np.ndarray + event_portfolio_scales: np.ndarray + event_cash_scales: np.ndarray + day_equity: np.ndarray + planned_row_indices: np.ndarray + execution_adjustment_codes: np.ndarray + frozen_signal_n: np.ndarray + execution_delay_days: int + execution_sequence: tuple[tuple[str, ...], ...] + horizon_expired_orders: tuple[object, ...] + + +def _section(config: Mapping[str, object], name: str) -> Mapping[str, object]: + value = config.get(name) + if not isinstance(value, Mapping): + raise ValueError(f"{name} config must be an object") + return value + + +def _number( + section: Mapping[str, object], + name: str, + default: float | None = None, + *, + positive: bool = True, +) -> float: + value = section.get(name, default) + if value is None: + raise ValueError(f"missing config value: {name}") + try: + result = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be numeric") from exc + if not np.isfinite(result) or (positive and result <= 0.0): + raise ValueError(f"{name} must be finite and positive") + return result + + +_LEGACY_RISK_FIELDS = frozenset( + { + "security_risk_cap", + "security_value_cap", + "asset_group_risk_cap", + "asset_group_value_cap", + "portfolio_risk_cap", + "portfolio_value_cap", + "covariance", + "target_volatility", + "risk_reduction_target_volatility", + "minimum_aligned_samples", + } +) + + +def _reject_legacy_risk_fields(risk: Mapping[str, object]) -> None: + found = sorted(set(risk) & _LEGACY_RISK_FIELDS) + if found: + raise ValueError( + "legacy risk fields are not supported: " + ", ".join(found) + ) + + +def _params(config: Mapping[str, object]) -> tuple[float, CallbackParams]: + research = _section(config, "research") + signal = _section(config, "signal") + risk = _section(config, "risk") + costs_value = config.get("costs", {}) + if not isinstance(costs_value, Mapping): + raise ValueError("costs config must be an object") + initial_cash = _number(research, "initial_cash") + _reject_legacy_risk_fields(risk) + lot_size_value = risk.get("lot_size", 100) + if isinstance(lot_size_value, bool) or int(lot_size_value) != lot_size_value: + raise ValueError("lot_size must be a positive integer") + lot_size = int(lot_size_value) + if lot_size <= 0: + raise ValueError("lot_size must be a positive integer") + slippage = _number( + costs_value, "one_way_slippage", 0.0, positive=False + ) + if slippage < 0.0 or slippage >= 1.0: + raise ValueError("one_way_slippage must be between zero and one") + if "max_units" not in signal: + raise ValueError("missing config value: max_units") + max_units_value = signal["max_units"] + if ( + isinstance(max_units_value, bool) + or not isinstance(max_units_value, int) + or max_units_value != 4 + ): + raise ValueError("max_units must equal four") + max_units = max_units_value + return initial_cash, CallbackParams( + lot_size=lot_size, + unit_risk_per_n=_number(risk, "unit_risk_per_n"), + add_step_n=_number(signal, "add_step_n"), + stop_n=_number(signal, "stop_n"), + max_units=max_units, + asset_group_unit_cap=_number(risk, "asset_group_unit_cap"), + portfolio_unit_cap=_number(risk, "portfolio_unit_cap"), + commission_multiplier=_number( + costs_value, "commission_multiplier", 1.0 + ), + one_way_slippage=slippage, + ) + + +def _mutable_state( + rows: int, columns: int, group_count: int, max_units: int +) -> CallbackState: + if rows <= 0 or columns <= 0 or group_count <= 0 or max_units != 4: + raise ValueError("invalid callback state dimensions") + return CallbackState( + unit_count=np.zeros(columns, dtype=np.int64), + unit_signal_n=np.full( + (columns, max_units), np.nan, dtype=np.float64 + ), + unit_base_quantities=np.zeros( + (columns, max_units), dtype=np.int64 + ), + unit_fill_prices=np.full( + (columns, max_units), np.nan, dtype=np.float64 + ), + initial_fill_price=np.full(columns, np.nan, dtype=np.float64), + initial_signal_n=np.full(columns, np.nan, dtype=np.float64), + common_stop=np.full(columns, np.nan, dtype=np.float64), + next_add_index=np.zeros(columns, dtype=np.int64), + candidate_signal_n=np.full( + (rows, columns), np.nan, dtype=np.float64 + ), + candidate_base_quantity=np.zeros((rows, columns), dtype=np.int64), + action_codes=np.zeros((rows, columns), dtype=np.int16), + reason_codes=np.zeros((rows, columns), dtype=np.int16), + requested_quantities=np.zeros((rows, columns), dtype=np.int64), + planned_quantities=np.zeros((rows, columns), dtype=np.int64), + filled_quantities=np.zeros((rows, columns), dtype=np.int64), + fill_prices=np.full((rows, columns), np.nan, dtype=np.float64), + fees=np.zeros((rows, columns), dtype=np.float64), + state_quantities=np.zeros((rows, columns), dtype=np.int64), + state_common_stop=np.full((rows, columns), np.nan, dtype=np.float64), + state_next_add_index=np.zeros((rows, columns), dtype=np.int64), + state_unit_counts=np.zeros((rows, columns), dtype=np.int64), + event_group_scales=np.ones((rows, columns), dtype=np.float64), + event_portfolio_scales=np.ones(rows, dtype=np.float64), + event_cash_scales=np.ones(rows, dtype=np.float64), + day_equity=np.full(rows, np.nan, dtype=np.float64), + allocation_ready=np.zeros(rows, dtype=np.bool_), + ) + + +def _readonly_copy(values: np.ndarray) -> np.ndarray: + result = np.ascontiguousarray(values.copy()) + result.setflags(write=False) + return result + + +def _delay_days(config: Mapping[str, object]) -> int: + execution = config.get("execution", {}) + if not isinstance(execution, Mapping): + raise ValueError("execution config must be an object") + value = execution.get("additional_delay_days", 0) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("additional_delay_days must be a non-negative integer") + return value + + +def _run_immediate( + inputs: SimulationInputs, + config: Mapping[str, object], +) -> VectorbtSimulationResult: + rows, columns = inputs.close.shape + if rows == 0 or columns == 0: + raise ValueError("simulation inputs must not be empty") + expected_shape = (rows, columns) + for name in ( + "execution_open", + "paused", + "high_limit", + "low_limit", + "signal_close", + "signal_entry_high", + "signal_exit_low", + "signal_n", + ): + if getattr(inputs, name).shape != expected_shape: + raise ValueError(f"invalid simulation input shape: {name}") + if len(inputs.securities) != columns or len(inputs.asset_groups) != columns: + raise ValueError("simulation input identities do not match columns") + + initial_cash, params = _params(config) + group_count = int(np.max(inputs.asset_group_ids)) + 1 + state = _mutable_state(rows, columns, group_count, params.max_units) + callback_inputs = CallbackInputs( + execution_open=inputs.execution_open, + signal_close=inputs.signal_close, + signal_entry_high=inputs.signal_entry_high, + signal_exit_low=inputs.signal_exit_low, + signal_n=inputs.signal_n, + paused=inputs.paused, + high_limit=inputs.high_limit, + low_limit=inputs.low_limit, + asset_group_ids=inputs.asset_group_ids, + ) + close = pd.DataFrame( + inputs.close, + index=pd.DatetimeIndex(inputs.dates.astype("datetime64[ns]")), + columns=inputs.securities, + ) + portfolio = vbt.Portfolio.from_order_func( + close, + order_func_nb, + pre_sim_func_nb=pre_sim_func_nb, + pre_sim_args=(state, callback_inputs, params), + pre_segment_func_nb=pre_segment_func_nb, + post_segment_func_nb=post_segment_func_nb, + post_order_func_nb=post_order_func_nb, + init_cash=initial_cash, + cash_sharing=True, + group_by=True, + call_pre_segment=True, + update_value=True, + ffill_val_price=True, + max_orders=rows * columns, + use_numba=True, + freq="1D", + ) + valid_orders = state.action_codes != ACTION_NONE + row_indices = np.broadcast_to( + np.arange(rows, dtype=np.int64)[:, None], (rows, columns) + ) + return VectorbtSimulationResult( + initial_cash=initial_cash, + asset_group_unit_cap=params.asset_group_unit_cap, + portfolio_unit_cap=params.portfolio_unit_cap, + portfolio=portfolio, + action_codes=_readonly_copy(state.action_codes), + reason_codes=_readonly_copy(state.reason_codes), + requested_quantities=_readonly_copy(state.requested_quantities), + planned_quantities=_readonly_copy(state.planned_quantities), + filled_quantities=_readonly_copy(state.filled_quantities), + fill_prices=_readonly_copy(state.fill_prices), + fees=_readonly_copy(state.fees), + state_quantities=_readonly_copy(state.state_quantities), + state_common_stop=_readonly_copy(state.state_common_stop), + state_next_add_index=_readonly_copy(state.state_next_add_index), + state_unit_counts=_readonly_copy(state.state_unit_counts), + candidate_base_quantities=_readonly_copy( + state.candidate_base_quantity + ), + event_group_scales=_readonly_copy(state.event_group_scales), + event_portfolio_scales=_readonly_copy( + state.event_portfolio_scales + ), + event_cash_scales=_readonly_copy(state.event_cash_scales), + day_equity=_readonly_copy(state.day_equity), + planned_row_indices=_readonly_copy( + np.where(valid_orders, row_indices, -1).astype(np.int64) + ), + execution_adjustment_codes=_readonly_copy( + np.zeros((rows, columns), dtype=np.int16) + ), + frozen_signal_n=_readonly_copy( + np.where(valid_orders, inputs.signal_n, np.nan).astype(np.float64) + ), + execution_delay_days=0, + execution_sequence=tuple( + tuple( + f"immediate-row-{row}:{inputs.securities[column]}" + for column in range(columns) + if state.action_codes[row, column] != ACTION_NONE + ) + for row in range(rows) + ), + horizon_expired_orders=(), + ) + + +def run_vectorbt_simulation( + inputs: SimulationInputs, + config: Mapping[str, object], +) -> VectorbtSimulationResult: + delay_days = _delay_days(config) + immediate = _run_immediate(inputs, config) + if delay_days == 0: + return immediate + initial_cash, params = _params(config) + plan = freeze_order_plan(inputs, immediate) + delayed = run_delayed_execution( + inputs, + plan, + initial_cash=initial_cash, + lot_size=params.lot_size, + stop_n=params.stop_n, + commission_multiplier=params.commission_multiplier, + one_way_slippage=params.one_way_slippage, + delay_days=delay_days, + ) + rows, columns = inputs.close.shape + delayed_candidate_bases = np.zeros((rows, columns), dtype=np.int64) + delayed_group_scales = np.ones((rows, columns), dtype=np.float64) + delayed_portfolio_scales = np.ones(rows, dtype=np.float64) + delayed_cash_scales = np.ones(rows, dtype=np.float64) + for execution_row in range(delay_days, rows): + source_row = execution_row - delay_days + delayed_candidate_bases[execution_row] = ( + immediate.candidate_base_quantities[source_row] + ) + delayed_group_scales[execution_row] = ( + immediate.event_group_scales[source_row] + ) + delayed_portfolio_scales[execution_row] = ( + immediate.event_portfolio_scales[source_row] + ) + delayed_cash_scales[execution_row] = ( + immediate.event_cash_scales[source_row] + ) + return VectorbtSimulationResult( + initial_cash=initial_cash, + asset_group_unit_cap=params.asset_group_unit_cap, + portfolio_unit_cap=params.portfolio_unit_cap, + portfolio=delayed.portfolio, + action_codes=delayed.action_codes, + reason_codes=delayed.reason_codes, + requested_quantities=delayed.requested_quantities, + planned_quantities=delayed.planned_quantities, + filled_quantities=delayed.filled_quantities, + fill_prices=delayed.fill_prices, + fees=delayed.fees, + state_quantities=delayed.state_quantities, + state_common_stop=delayed.state_common_stop, + state_next_add_index=delayed.state_next_add_index, + state_unit_counts=delayed.state_unit_counts, + candidate_base_quantities=_readonly_copy(delayed_candidate_bases), + event_group_scales=_readonly_copy(delayed_group_scales), + event_portfolio_scales=_readonly_copy( + delayed_portfolio_scales + ), + event_cash_scales=_readonly_copy(delayed_cash_scales), + day_equity=delayed.day_equity, + planned_row_indices=delayed.planned_row_indices, + execution_adjustment_codes=delayed.execution_adjustment_codes, + frozen_signal_n=delayed.frozen_signal_n, + execution_delay_days=delay_days, + execution_sequence=delayed.execution_sequence, + horizon_expired_orders=delayed.horizon_expired_orders, + ) diff --git a/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py new file mode 100644 index 0000000..043c021 --- /dev/null +++ b/joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping, Sequence + +import numpy as np +import pandas as pd + +from scripts.research.market_data.economic_returns import ( + CorporateActionApplication, + canonical_corporate_actions_digest, + derive_continuous_prices, +) + +from .indicators import breakout_levels, turtle_n + + +@dataclass(frozen=True) +class SimulationInputs: + dates: np.ndarray + securities: tuple[str, ...] + asset_groups: tuple[str, ...] + asset_group_ids: np.ndarray + raw_open: np.ndarray + raw_high: np.ndarray + raw_low: np.ndarray + raw_close: np.ndarray + raw_pre_close: np.ndarray + continuous_open: np.ndarray + continuous_high: np.ndarray + continuous_low: np.ndarray + continuous_close: np.ndarray + continuous_pre_close: np.ndarray + continuity_factor: np.ndarray + corporate_action_applied: np.ndarray + corporate_actions_digest: str + corporate_action_applications: tuple[CorporateActionApplication, ...] + paused: np.ndarray + high_limit: np.ndarray + low_limit: np.ndarray + signal_source_index: np.ndarray + signal_close: np.ndarray + signal_entry_high: np.ndarray + signal_exit_low: np.ndarray + signal_n: np.ndarray + + @property + def execution_open(self) -> np.ndarray: + return self.continuous_open + + @property + def close(self) -> np.ndarray: + return self.continuous_close + + +def _section(config: Mapping[str, object], name: str) -> Mapping[str, object]: + value = config.get(name) + if not isinstance(value, Mapping): + raise ValueError(f"{name} config must be an object") + return value + + +def _positive_int(value: object, name: str, *, minimum: int = 1) -> int: + if isinstance(value, bool): + raise ValueError(f"{name} must be an integer") + try: + result = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be an integer") from exc + if result < minimum or result != value: + raise ValueError(f"{name} must be at least {minimum}") + return result + + +def _readonly(values: object, dtype: np.dtype[object] | str) -> np.ndarray: + result = np.ascontiguousarray(values, dtype=dtype) + result.setflags(write=False) + return result + + +def _evidence_insufficient(message: str) -> ValueError: + return ValueError(f"evidence_insufficient: {message}") + + +def _normalized_frame( + frame: pd.DataFrame, + *, + security: str, + signal: Mapping[str, object], + actions: Sequence[Mapping[str, object]], +) -> tuple[pd.DataFrame, tuple[CorporateActionApplication, ...]]: + continuous = derive_continuous_prices( + frame, + security=security, + corporate_actions=actions, + ) + result = continuous.frame + result["n"] = turtle_n(result, days=_positive_int(signal.get("n_days"), "n_days")) + levels = breakout_levels( + result, + entry_days=_positive_int(signal.get("entry_days"), "entry_days"), + exit_days=_positive_int(signal.get("exit_days"), "exit_days"), + ) + result["entry_high"] = levels["entry_high"] + result["exit_low"] = levels["exit_low"] + return result.set_index("date", drop=False), continuous.applications + + +def prepare_simulation_inputs( + frames: Mapping[str, pd.DataFrame], + config: Mapping[str, object], + *, + corporate_actions: Sequence[Mapping[str, object]] = (), + corporate_actions_digest: str | None = None, +) -> SimulationInputs: + universe_value = config.get("universe") + if not isinstance(universe_value, list) or not universe_value: + raise ValueError("universe must be a non-empty list") + universe: dict[str, str] = {} + for item in universe_value: + if not isinstance(item, Mapping): + raise ValueError("universe entries must be objects") + security = str(item.get("security", "")) + asset_group = str(item.get("asset_group", "")) + if not security or not asset_group or security in universe: + raise ValueError("universe identities must be non-empty and unique") + universe[security] = asset_group + if set(frames) != set(universe): + raise ValueError("market frames must exactly match the configured universe") + action_securities = { + str(action.get("security", "")) for action in corporate_actions + } + unknown_action_securities = sorted(action_securities - set(universe)) + if unknown_action_securities: + raise _evidence_insufficient( + "corporate actions are outside the configured universe: " + + ", ".join(unknown_action_securities) + ) + computed_action_digest = canonical_corporate_actions_digest(corporate_actions) + if corporate_actions_digest is None: + corporate_actions_digest = computed_action_digest + elif ( + not isinstance(corporate_actions_digest, str) + or len(corporate_actions_digest) != 64 + or any(character not in "0123456789abcdef" for character in corporate_actions_digest) + ): + raise _evidence_insufficient("invalid corporate-actions digest") + elif corporate_actions_digest != computed_action_digest: + raise _evidence_insufficient("corporate-actions digest mismatch") + + securities = tuple(sorted(universe)) + asset_groups = tuple(universe[security] for security in securities) + group_labels = {name: index for index, name in enumerate(sorted(set(asset_groups)))} + asset_group_ids = [group_labels[name] for name in asset_groups] + signal = _section(config, "signal") + normalized: dict[str, pd.DataFrame] = {} + action_applications: list[CorporateActionApplication] = [] + for security in securities: + normalized_frame, applications = _normalized_frame( + frames[security], + security=security, + signal=signal, + actions=corporate_actions, + ) + normalized[security] = normalized_frame + action_applications.extend(applications) + calendar = pd.DatetimeIndex( + sorted({date for frame in normalized.values() for date in frame.index}) + ) + if calendar.empty: + raise ValueError("market calendar must not be empty") + + row_count = len(calendar) + column_count = len(securities) + shape = (row_count, column_count) + raw_open = np.full(shape, np.nan, dtype=np.float64) + raw_high = np.full(shape, np.nan, dtype=np.float64) + raw_low = np.full(shape, np.nan, dtype=np.float64) + raw_close = np.full(shape, np.nan, dtype=np.float64) + raw_pre_close = np.full(shape, np.nan, dtype=np.float64) + continuous_open = np.full(shape, np.nan, dtype=np.float64) + continuous_high = np.full(shape, np.nan, dtype=np.float64) + continuous_low = np.full(shape, np.nan, dtype=np.float64) + continuous_close = np.full(shape, np.nan, dtype=np.float64) + continuous_pre_close = np.full(shape, np.nan, dtype=np.float64) + continuity_factor = np.full(shape, np.nan, dtype=np.float64) + corporate_action_applied = np.zeros(shape, dtype=np.bool_) + paused = np.ones(shape, dtype=np.bool_) + high_limit = np.full(shape, np.nan, dtype=np.float64) + low_limit = np.full(shape, np.nan, dtype=np.float64) + raw_entry_high = np.full(shape, np.nan, dtype=np.float64) + raw_exit_low = np.full(shape, np.nan, dtype=np.float64) + raw_n = np.full(shape, np.nan, dtype=np.float64) + for column, security in enumerate(securities): + aligned = normalized[security].reindex(calendar) + raw_open[:, column] = aligned["raw_open"].to_numpy(dtype=np.float64) + raw_high[:, column] = aligned["raw_high"].to_numpy(dtype=np.float64) + raw_low[:, column] = aligned["raw_low"].to_numpy(dtype=np.float64) + raw_close[:, column] = aligned["raw_close"].to_numpy(dtype=np.float64) + raw_pre_close[:, column] = aligned["raw_pre_close"].to_numpy(dtype=np.float64) + continuous_open[:, column] = aligned["open"].to_numpy(dtype=np.float64) + continuous_high[:, column] = aligned["high"].to_numpy(dtype=np.float64) + continuous_low[:, column] = aligned["low"].to_numpy(dtype=np.float64) + continuous_close[:, column] = aligned["close"].to_numpy(dtype=np.float64) + continuous_pre_close[:, column] = aligned["pre_close"].to_numpy( + dtype=np.float64 + ) + continuity_factor[:, column] = aligned["continuity_factor"].to_numpy( + dtype=np.float64 + ) + corporate_action_applied[:, column] = aligned[ + "corporate_action_applied" + ].fillna(False).to_numpy(dtype=np.bool_) + paused[:, column] = aligned["paused"].fillna(True).to_numpy(dtype=np.bool_) + high_limit[:, column] = aligned["high_limit"].to_numpy(dtype=np.float64) + low_limit[:, column] = aligned["low_limit"].to_numpy(dtype=np.float64) + raw_entry_high[:, column] = aligned["entry_high"].to_numpy(dtype=np.float64) + raw_exit_low[:, column] = aligned["exit_low"].to_numpy(dtype=np.float64) + raw_n[:, column] = aligned["n"].to_numpy(dtype=np.float64) + + shift = 1 + signal_source_index = np.full(row_count, -1, dtype=np.int64) + signal_close = np.full(shape, np.nan, dtype=np.float64) + signal_entry_high = np.full(shape, np.nan, dtype=np.float64) + signal_exit_low = np.full(shape, np.nan, dtype=np.float64) + signal_n = np.full(shape, np.nan, dtype=np.float64) + for execution_row in range(shift, row_count): + source_row = execution_row - shift + signal_source_index[execution_row] = source_row + signal_close[execution_row] = continuous_close[source_row] + signal_entry_high[execution_row] = raw_entry_high[source_row] + signal_exit_low[execution_row] = raw_exit_low[source_row] + signal_n[execution_row] = raw_n[source_row] + + return SimulationInputs( + dates=_readonly(calendar.to_numpy(dtype="datetime64[D]"), "datetime64[D]"), + securities=securities, + asset_groups=asset_groups, + asset_group_ids=_readonly(asset_group_ids, "int64"), + raw_open=_readonly(raw_open, "float64"), + raw_high=_readonly(raw_high, "float64"), + raw_low=_readonly(raw_low, "float64"), + raw_close=_readonly(raw_close, "float64"), + raw_pre_close=_readonly(raw_pre_close, "float64"), + continuous_open=_readonly(continuous_open, "float64"), + continuous_high=_readonly(continuous_high, "float64"), + continuous_low=_readonly(continuous_low, "float64"), + continuous_close=_readonly(continuous_close, "float64"), + continuous_pre_close=_readonly(continuous_pre_close, "float64"), + continuity_factor=_readonly(continuity_factor, "float64"), + corporate_action_applied=_readonly(corporate_action_applied, "bool"), + corporate_actions_digest=corporate_actions_digest, + corporate_action_applications=tuple( + sorted( + action_applications, + key=lambda item: ( + item.effective_date, + item.security, + item.source_event_id, + ), + ) + ), + paused=_readonly(paused, "bool"), + high_limit=_readonly(high_limit, "float64"), + low_limit=_readonly(low_limit, "float64"), + signal_source_index=_readonly(signal_source_index, "int64"), + signal_close=_readonly(signal_close, "float64"), + signal_entry_high=_readonly(signal_entry_high, "float64"), + signal_exit_low=_readonly(signal_exit_low, "float64"), + signal_n=_readonly(signal_n, "float64"), + ) diff --git a/joinquant/strategies/strategy_index.csv b/joinquant/strategies/strategy_index.csv index a8a97b7..6d72f0c 100644 --- a/joinquant/strategies/strategy_index.csv +++ b/joinquant/strategies/strategy_index.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:37b301a54c2bebc809a66cc41e6593eb1f2209ad1fb8f6fa856ba0224bf921a9 +oid sha256:021b707f42792c604d97f4160ed7ecca03a8eba03c8dfd84604d328f5f4047e1 size 820 diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/.comet.yaml b/openspec/changes/build-turtle-etf-local-research-workflow/.comet.yaml index 44ff304..21262d2 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/.comet.yaml +++ b/openspec/changes/build-turtle-etf-local-research-workflow/.comet.yaml @@ -1,6 +1,6 @@ workflow: full language: zh-CN -phase: archive +phase: build context_compression: beta build_mode: executing-plans build_pause: null @@ -10,14 +10,14 @@ review_mode: thorough isolation: branch verify_mode: full auto_transition: true -base_ref: 66cfbce81f95984aeeb7a9705b1d00ee3c8632ce +base_ref: 4400fec8149f02bc7d42f0294be65e9dacc9b639 design_doc: docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md plan: docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md -verify_result: pass +verify_result: pending verification_report: docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md branch_status: handled created_at: 2026-07-13 -verified_at: 2026-07-13 +verified_at: null archived: false direct_override: null handoff_context: openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/brainstorm-summary.md b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/brainstorm-summary.md index cb7cbdc..934f498 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/brainstorm-summary.md +++ b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/brainstorm-summary.md @@ -1,181 +1,74 @@ # Brainstorm Summary - Change: `build-turtle-etf-local-research-workflow` -- Date: `2026-07-14` -- Confirmation: 用户已确认完整技术设计和 Spec Patch（规格补丁）清单 - -## 确认的技术方案 - -- 本变更先实施“本地研究流程”，为后续聚宽正式回测提供策略、数据和研究证据。 -- 通用 Skill（技能）命名为 `run-local-quant-research`，只负责编排；可复用确定性能力沉淀为脚本，不耦合海龟交易系统。 -- 海龟专属资产池、参数、交易规则、策略代码、配置和证据保留在项目目录。 -- 正式回测与模拟交易只在 JoinQuant（聚宽）云端运行；本地结果只用于规则正确性和探索性研究。 -- `joinquant/strategies/strategy-001` 与 `strategy-002` 均已绑定真实聚宽策略对象，不能修改。 -- 项目身份采用方案 B：先创建真实聚宽策略空壳并同步为 `strategy-003`，但不启动正式回测；本地研究归入该策略对象。 -- 当前 GitHub（代码托管平台）仓库是公开仓库。 -- 聚宽官方文档预览无变化，`get_price`（历史行情接口）支持明确字段、停牌和复权参数，`write_file`（研究文件写入）支持把文本或二进制文件保存到投资研究空间。 -- 聚宽官方用户协议限制把平台提供的数据复制、分发或作公共用途，因此完整行情不能提交到当前公开仓库。 -- 行情保存已确认采用最小本地方案：完整行情只保存在仓库已忽略的 `.local/`，不另建远端长期副本、私有数据仓库或公开摘要凭据目录。 -- 价格保存口径已确认采用方案 A：权威 CSV（逗号分隔文件）保存实际未复权价格和复权因子；本地研究不生成也不使用复权价格，后续稳健性分析确有需要时再按该次运行指定的基准临时派生。 -- 行情字段已确认使用 `date`、`security`、`open`、`high`、`low`、`close`、`pre_close`、`volume`、`money`、`factor`、`paused`、`high_limit`、`low_limit`；静态证券信息保存在 `manifest.json`。 -- 用户要求在锁定契约前用聚宽真实研究环境做最小可行性验证，并在验证后删除聚宽端和本地临时产物。 -- 真实 PoC（可行性验证）已通过：聚宽研究环境对 `510300.XSHG` 的 2026-07-01 至 2026-07-10 日线返回 8 行，13 个确认字段全部存在且无空值；`fq=None`、`skip_paused=False` 可执行。 -- 聚宽端 CSV 写入和回读字节一致，SHA256（文件摘要）均为 `7276eb88b6074a368d2baa0bd53bd3add742f2c9c02ffcf08d0eebe3114064b3`；临时 CSV 已在研究环境删除。 -- 本地项目 `.venv` 使用 DuckDB 1.5.4 从同一 CSV 建表成功，CSV 与 DuckDB 各 8 行，规范化内容摘要均为 `bb3600e7b4dcbc3d631d8e29dec5c5343c741f0f5fc9eea3cad6358048c44bec`。 -- 本地 `.local/quant-research/strategy-003/__poc_fields_20260713/` 已删除；聚宽临时 `Untitled.ipynb` 已停止、移入回收站并永久删除，回收站复查为空。 -- 真实环境差异：聚宽 Python 3 研究内核把 `get_price`、`write_file`、`read_file` 注入为内置接口，`from jqdata import get_price` 会失败；聚宽 Pandas（数据处理库）为 0.23.4，`to_csv` 使用 `line_terminator`；`paused` 返回 `float64`，数据桥需要规范化为布尔值。 -- 2026-07-13 再次运行聚宽官方文档 `preview`（预览）和 `verify`（校验）：1,185 个 API（接口）条目无变化，15 个本地快照文件校验通过。 -- 聚宽官方口径不是“ETF 价格一律不得复权”：官方总体建议开启 `use_real_price`（真实价格模式），但明确例外为“含场内基金的策略不建议开启动态复权”，原因是场内基金拆分/合并的除权日披露不标准，聚宽当前采用的折算基准日可能与实际除权日不同。 -- 聚宽策略 API（接口）文档明确定义 `get_price(..., fq=None)` 为不复权并返回实际价格，对股票/基金的价格、成交量和 `factor`（复权因子）字段生效。 -- `strategy-001` 的实际默认是 `use_real_price=False` 与 `fq_mode=None`；其趋势、动量、波动率、协方差、RSRS（阻力支撑相对强度）信号都使用显式 `fq=None` 取得的未复权 `close/high/low`（收盘/最高/最低价）。 -- `strategy-001` 未自行判断涨跌停或指定成交价，只调用 `order_target_value`（按目标市值下单）由聚宽撮合；官方文档说明 `use_real_price=False` 的回测撮合使用基于回测建立日期的前复权价，因此“信号不复权”与“撮合价格模式”必须分层表达。 -- 用户已确认海龟 ETF 沿用 `strategy-001` 价格口径：本地研究和聚宽策略信号显式使用 `fq=None`（不复权），含场内基金的聚宽策略设置 `use_real_price=False`；撮合层固定基准日前复权行为作为平台限制记录。 -- 用户已确认简化复权派生：本地研究快照保存 `factor`（复权因子）仅供数据核验，不生成也不使用复权价序列；后续稳健性分析确有需要时，再按该次运行指定的基准临时派生。 -- 用户已确认完整历史导出采用方案 A：每只 ETF 从自身首个可用完整交易日导出，统一结束于运行时显式指定的 `snapshot_end_date`（快照截止日）；2015-01-01 之前数据只用于指标预热，新 ETF 仍须满足 60 个完整有效样本后才能新增风险。 -- 用户已确认同日订单总顺序采用方案 A：先全仓退出，再强制风险减仓，最后处理有效新建仓与加仓；新建仓和加仓同级，同一 ETF 当日出现退出时取消它的所有买入候选。 -- 仓库没有“最终 11 只 ETF + 55/20 突破 + U0 + 加仓 + 完整风险预算 + 10% 目标波动率”的每日总仓位序列，因此现在无法给出 A1 的正式平均仓位。 -- 已用本地现有 9 只最终池重合 ETF 在 2021-08-01 至 2026-07-10 复算简化 55 日入场/20 日退出路径：共完成 79 次退出，与既有检查点记录一致；平均同时激活 3.41 只 ETF，中位数 3 只，75% 分位 5 只，空仓交易日占 4.1%。这只衡量趋势过滤覆盖，不是资金仓位。 -- 同一 9 只 ETF 路径使用信号日 N 值、次日开盘价和每只最多 30% 的初始 U0 作方向性代理：只持有初始单位时，加入单 ETF/50% 资产组/100% 总仓位上限后，平均仓位 63.7%、中位 67.1%、低于 50% 的交易日占 34.1%、不低于 95% 的交易日占 24.8%。 -- 再叠加 10% 触发/9.5% 目标的“当日即时波动率缩放”代理后，平均仓位约 55.7%、中位 56.2%、低于 50% 的交易日占 42.5%、不低于 95% 的交易日占 12.9%。该数字缺少 `515180.SH`、`516080.SH`、顺势加仓、保护性止损、流动性、费用和风险缩放后不自动买回的路径状态，只能作为设计期方向性区间，不能作为 A1 验收结论。 -- A1 的整体等比缩放在总请求超出预算时不主动降低可用总仓位，只改变候选之间的分配；长期留有现金的根本来源是 55/20 趋势过滤、计划止损风险、资产组上限和 10% 组合波动率上限。 -- 用户已确认首版先使用 A1：所有合格建仓与加仓候选按各自标准请求量的同一完成比例分配共享资金和风险预算，整手余额按小数余额逐手补分并重新检查硬门槛。本地研究必须输出仓位分布和留现原因，但不因结果自动切换到排名分配。 -- 用户已确认整体技术实现采用方案 A：通用运行器 + 项目适配器 + 海龟项目纯计算模块；不建通用量化插件框架，也不把通用数据和证据能力锁进海龟单体。 -- 用户已确认架构与组件边界：`run-local-quant-research` Skill（技能）只编排；仓库共用脚本负责行情中心、契约、运行身份、安全调用项目入口和证据收口；`strategy-003/research` 保存项目适配、聚宽导出、海龟纯计算模块和固定测试夹具；共享行情保存在 `.local/market-data/`，策略运行证据保存在 `.local/quant-research/strategy-003/`。 -- 用户已确认完整数据流：`market-data.csv` 是聚宽原始导出的唯一权威行情文件，DuckDB（嵌入式分析数据库）只由该 CSV 构建；运行身份由快照、配置和代码摘要共同确定，逐日研究按已确认订单顺序和 A1 分配执行并以三个停止状态之一收口。 -- 用户已确认错误处理边界：证据缺失与执行失败严格分开，产物通过全部校验后才固化；停牌和未满 60 个样本按正常研究状态处理，关键风险输入缺失时停止新增风险但保留可执行的退出和强制减仓。 -- 用户新增并确认行情数据中心要求：行情存储和查询与 `strategy-003`、海龟交易及任何固定策略解耦，可供所有策略复用，并能后续追加其他标的；策略研究证据仍归属对应策略。 -- 用户已确认共享行情数据中心采用方案 A：`.local/market-data/` 保存不可变原始导出批次和不可变快照清单，DuckDB（嵌入式分析数据库）只使用可重建的内存查询视图；策略仅引用 `snapshot_id`，不拥有或复制行情。 -- 用户已确认行情数据中心首版范围采用方案 A：通用契约只实现日线行情，实际导入最终 11 只 ETF；通过来源、标的类型、频率和显式字段能力为后续股票、指数、期货等标的留出扩展边界，但本变更不实现分钟线、基本面、财务或因子数据。 -- 用户已确认候选策略采用方案 A：最终候选包包含冻结基线和六个预设单项挑战配置，共用同一份策略代码；本地研究只提供方向性证据与风险提示，不据本地最优结果淘汰候选、修改基线或新增参数。 - -### 已确认约束 - -- 所有本地 Python（编程语言）命令使用项目 `.venv`（虚拟环境）。 -- `market-data.csv` 是精确保留的聚宽原始导出和唯一行情事实源；DuckDB（嵌入式分析数据库）只是可由该 CSV 重建的派生视图。 -- 运行证据不可变；状态仅为 `complete`、`evidence_insufficient`、`failed`。 -- 不复制聚宽认证、浏览器或归档实现；需要时复用 `joinquant-archive-sync`（聚宽归档同步）。 -- 不修改 Vibe-Trading（AI 研究助理）上游源码；已知组合优化器缺陷未修复时跳过该优化器。 - -### 设计确认状态 - -无待确认设计决策。用户已确认完整技术设计和下列 Spec Patch（规格补丁）清单。 - -### 方案取舍 - -- 整体技术实现： - - 方案 A（推荐）：“通用运行器 + 项目适配器 + 海龟纯计算模块”。`run-local-quant-research` Skill（技能）只解析用户意图和调用通用运行入口；共用脚本只负责配置契约、运行身份、快照校验、CSV/DuckDB（逗号分隔文件/嵌入式分析数据库）一致性、项目入口调用和证据收集；`strategy-003` 项目适配器声明资产池、字段、参数和研究任务，海龟信号、状态、风险和 A1 分配保留为项目内可独立测试的纯计算模块。 - - 方案 B：“通用量化插件框架”。共用脚本定义通用信号、仓位、风险、成交和报告插件，海龟通过插件实现。后续策略复用上限更高，但首变更需要先建通用框架，容易过度抽象并扩大验证面。 - - 方案 C：“海龟项目单体入口 + 极薄 Skill”。所有数据、模拟、风险和报告逻辑都位于 `strategy-003` 研究项目，Skill 直接调用。实施最快，但快照校验、数据一致性、运行证据等通用能力会被锁在海龟项目，不满足“共用能力沉淀脚本”的长期目标。 - - 已确认方案 A。 -- 项目身份： - - A：先在独立 `research/turtle-etf-system/` 开展本地研究，真实聚宽策略创建后再迁入并绑定 `strategy-003`。 - - B：先创建真实聚宽策略空壳并同步为 `strategy-003`，本地研究从一开始放在该对象目录下；远端创建仅建立身份，不启动正式回测。 - - 已确认采用 B。 -- 数据保存： - - 已确认在 `.local/market-data/batches//` 保存不可变导出批次，在 `.local/market-data/snapshots/.json` 保存不可变批次选择；策略只引用快照，不复制行情。 - - 每个批次收敛为 `manifest.json`、权威 `market-data.csv` 和 `validation.json`；DuckDB（嵌入式分析数据库）只建内存查询视图，不保存持久数据库副本。 - - 聚宽端文件若为导出传输所必需，只视为临时中转，不构成第二份权威证据。 - - 权威行情采用实际未复权价格加 `factor`（复权因子）；本变更不生成或使用复权序列。 - - 字段和导出入口已通过真实环境验证；导出代码直接调用研究内核注入接口，兼容 Pandas 0.23.4，并把 `paused` 规范化为布尔值。 -- 完整历史导出区间： - - A（推荐）：每只 ETF 从自身首个可用完整交易日导出到运行时显式指定的 `snapshot_end_date`（快照截止日）；研究统计区间仍从 2015-01-01 开始，更早行情只作预热，新 ETF 上市后仍须满足 60 个完整有效样本才能新增风险。 - - B：所有 ETF 统一从 2015-01-01 导出到 `snapshot_end_date`；目录和区间更直观，但 2015 年初的长周期信号缺少足够预热历史。 - - C：从 11 只 ETF 中最晚的可用日期开始统一导出；面板最齐整，但会丢失大量早期历史，无法完整支持既定的 2015–2018 稳健性区间。 -- 同日订单： - - 方案 A（推荐）：全仓退出 → 强制风险减仓 → 所有有效新建仓和加仓按同一优先级申请剩余预算；不因 ETF 在列表中的顺序改变结果。 - - 方案 B：全仓退出 → 强制风险减仓 → 加仓 → 新建仓；优先已经证明的趋势，但更容易集中于现有持仓。 - - 方案 C：全仓退出 → 强制风险减仓 → 新建仓 → 加仓；更倾向分散，但会压缩已有强趋势的加仓。 - - 已确认方案 A。 -- 多 ETF 共享预算： - - 方案 A1（推荐）：约束下按请求比例公平分配。每个候选先按已确认规则生成一个最多 `U0`（标准单位）的请求量，再对所有买入候选使用同一完成比例缩减，直到资金、单 ETF、资产组、组合计划风险和目标波动率全部通过；某候选被自身或所属组上限卡住后，其未用预算可流向仍可增仓的其他候选。所有数量向下取整手；剩余预算按小数余额从大到小逐手补分，每补一手都重新检查全部硬门槛，完全同分时按 ETF 代码升序确定。 - - 语义澄清：“全部建仓”指所有通过个体门槛的突破都进入同一买入候选池，不是保证每只 ETF 都能成交；候选池首先按共享资金和风险门槛整体等比缩放，之后才做整手取整。缩放后不足一个交易整手，或被自身/资产组硬上限拦截的 ETF，当日仍会跳过。 - - 方案 A2：按 `(signal_close - breakout_line) / N`（用 N 值标准化的突破强度）从高到低满足请求；会偏向强趋势，但新增了方案原本没有的排名因子。 - - 方案 A3：按每一单位带来的边际组合波动率从低到高满足请求；资金利用更偏向分散，但会把简单的海龟信号变成组合优化问题。 - - 已确认首版使用 A1，A2/A3 不进入本变更。 -- 全策略共享行情数据中心（已确认）： - - 方案 A（推荐）：`.local/market-data/` 保存不可变聚宽导出批次和不可变快照清单。每个批次保留精确原始 `market-data.csv`、`manifest.json` 和 `validation.json`；`snapshot_id` 只引用一个或多个批次及明确证券、日期、字段和复权口径，不复制行情。DuckDB（嵌入式分析数据库）使用内存查询视图，研究策略只引用 `snapshot_id`，运行证据仍保存在各自策略目录。 - - 方案 B：把全部行情长期写入一个中央 DuckDB。查询和增量写入最直接，但数据修订、字段迁移、并发锁和旧研究复算需要额外数据库版本管理，单库损坏的影响也更大。 - - 方案 C：继续每个策略保存不可变行情快照，只把读取和校验脚本通用化。首版改动最少，但跨策略重复数据和版本分叉仍然存在，不是真正的数据中心。 - - 方案 A 沿用仓库现有“原始事实文件保留，DuckDB 只建可重建查询视图，不保存重复数据库副本”的原则；新增标的只追加批次，旧批次不覆盖。已确认采用方案 A。 -- 本地研究最终候选策略（已确认）： - - 方案 A（推荐）：输出一份可直接交给聚宽回测流程的候选包，包含冻结基线作为主候选，以及最终方案预先声明的 40/60 日入场、1.5N/2.5N 止损、120 日/30 日半衰期 EWMA（指数加权移动平均）协方差六个单项挑战候选。本地报告给出方向性证据和风险提示，但不按本地最优淘汰候选、改写基线或新增参数。 - - 方案 B：根据本地结果从更大参数网格中排名并只输出前若干候选。候选更少，但会引入事后选择和过拟合，并与“不得据本地结果修改冻结基线”的既有边界冲突。 - - 方案 C：只输出冻结基线一个候选。边界最严格，但无法形成少数参数候选，也削弱方向性粗筛对后续聚宽场景编排的交接价值。 - -### 分段设计 - -#### 架构与组件边界（已确认） - -- Skill（技能）只定义流程顺序、输入输出、停止状态和安全边界。 -- 共用脚本位于仓库 `scripts/research/local_quant_research/`，不解释任何海龟字段。 -- `strategy-003/research` 保存项目适配、聚宽研究导出入口、海龟规则与项目报告。 -- 海龟指标、信号、批次状态、风险、A1 分配、成交和报告为可独立测试的项目纯计算模块。 -- 共享行情只保存在 `.local/market-data/`，策略运行证据只保存在 `.local/quant-research/strategy-003/`；正式回测仍只在聚宽云端运行。 - -#### 数据流（已确认） - -1. 先在聚宽建立真实策略空壳并同步为 `strategy-003`，仅建立身份，不启动正式回测。 -2. 项目导出入口在聚宽研究环境对 11 只 ETF 直接调用 `get_price(..., fq=None)`，每只从首个可用完整交易日取到显式 `snapshot_end_date`，按已确认 13 字段和固定排序生成导出批次。 -3. 每个批次进入 `.local/market-data/batches//`，包含 `manifest.json`、精确原始 `market-data.csv` 和 `validation.json`；清单记录来源、证券列表、字段、未复权口径、每只证券实际起止日、行数、导出代码摘要和 CSV 字节 SHA256（文件摘要），远端中转文件在本地校验后删除。 -4. 本地共用入口验证字节摘要、类型、空值、重复键、日期范围和字段完整性，并把 `paused` 统一为布尔查询类型；相同内容去重，冲突重叠拒绝，旧批次不得覆盖。 -5. `.local/market-data/snapshots/.json` 锁定批次集合及证券、日期、字段、来源和价格口径；通用脚本据此建立内存 DuckDB 查询视图并验证规范化内容，研究运行只记录 `snapshot_id` 及摘要。 -6. 共用运行器用“快照摘要 + 项目配置摘要 + 代码摘要”生成不可变 `run_id`，再以参数数组安全调用 `strategy-003` 项目适配器。 -7. 项目引擎逐日执行“收盘信号 → 次日开盘订单/成交 → 批次与共同止损 → 风险减仓 → A1 共享预算分配 → 状态与审计”，不使用复权价序列。 -8. 一次运行输出不可变的输入身份、日度审计、交易/持仓/风险状态、仓位分布、留现原因、评估指标和产物摘要，并以 `complete`、`evidence_insufficient` 或 `failed` 唯一收口。 - -#### 错误处理（已确认） - -- `evidence_insufficient`（证据不足）：真实策略身份、完整快照、清单或 11 只 ETF 的必要字段、范围与来源证据不完整时，在进入研究计算前停止；不得用猜测值、旧快照或部分资产池补齐。 -- `failed`（执行失败）：证据存在但发生摘要不一致、CSV/DuckDB 内容不一致、结构或类型违规、重复键、项目进程异常、硬约束被突破、同输入重复运行结果不一致，或远端临时文件无法确认清理时停止；部分结果不得标记为完成。 -- `complete`（完成）：全部输入门禁、研究流程、输出校验和产物摘要均通过；只允许在快照、配置和代码身份完全一致时复用。 -- 产物先写入暂存位置，全部检查通过后一次性固化；不得覆盖已有 `complete` 运行。失败重试保留失败证据，并产生新的尝试记录。 -- 停牌是正常市场状态，不伪造成交；正常交易日关键行情缺失属于快照证据不足。单只 ETF 未满 60 个有效样本不使全局失败，只禁止该 ETF 新增风险并记录原因。 -- 运行时若持仓 ETF 缺少可用价格或风险输入，停止所有新增风险；其他可交易 ETF 仍允许退出和强制减仓。禁止以零值、陈旧协方差或虚构成交继续计算。 -- Skill（技能）不修改认证、浏览器、环境配置或安装依赖，输出不得包含账号、Cookie（浏览器凭证）或 Token（访问令牌）。 - -#### 测试与验收（已确认） - -- 完整运行必须生成 `research-report.md`、`conclusion.json` 和 `candidate-strategies.json`；任一缺失、结构无效或摘要不匹配时不得输出 `complete`。 -- 共用脚本单元测试只覆盖通用契约、路径边界、运行身份、摘要、数据类型、CSV/DuckDB 同源校验、停止状态和不可变结果，不出现海龟专属参数。 -- 行情中心测试覆盖不可变批次导入、相同内容去重、追加新标的、旧快照复算不变、冲突重叠拒绝、字段能力检查、快照清单摘要，以及从原始 CSV 建立内存 DuckDB 查询视图后结果一致；不得生成长期持久 DuckDB 副本。 -- 海龟纯计算单元测试覆盖 N 值、55/20 日突破且排除当日、U0、0.5N 加仓、共同止损、计划风险、订单优先级、A1 等比分配、整手取整和小数余额补分。 -- 性质与不变量测试覆盖现金不得为负、单 ETF/资产组/组合风险上限不得突破、共同止损不得逆向放宽、候选输入顺序不得改变 A1 结果，以及同输入重复运行产物摘要一致。 -- 项目 E2E（端到端）测试从 `run-local-quant-research` Skill（技能）的用户入口开始，经共用运行器和 `strategy-003` 适配器，使用固定小型行情夹具完成“收盘信号 → 次日开盘处理 → 退出/减仓/买入 → 持仓和风险更新 → 证据收口”的完整业务路径。 -- 通用能力另用一个非海龟最小项目适配器完成 E2E，证明 Skill、行情中心和共用研究脚本均未耦合海龟；同一 `snapshot_id` 经原始 CSV 校验与内存 DuckDB 查询得到的规范化数据必须一致。 -- 异常验收覆盖证据不足、摘要篡改、缺字段、重复键、停牌、关键行情缺失、未满 60 样本、远端临时文件清理失败和项目进程失败，逐项核对停止状态与审计原因。 -- 真实集成验收在聚宽研究环境导出 11 只 ETF 完整历史快照，本地完成校验、内存 DuckDB 视图查询、一次完整研究和临时产物清理；日常自动测试使用仓库内固定小夹具，不依赖网络或聚宽认证。 -- 验收报告必须包含仓位分布、现金占比、留现原因、资产组与组合风险使用率，以及所有输入和输出摘要；本变更不执行聚宽正式回测。 -- `conclusion.json` 必须把建议区分为 `proceed_to_joinquant`（进入聚宽回测）、`revise_and_reassess`（修订后再评估）或 `stop_evidence_insufficient`（证据不足而停止），并列出确定性理由和阻断项；结论必须声明不是正式回测或最终验收结论。 -- `candidate-strategies.json` 必须恰好包含一项冻结基线和六项预设单项挑战配置，全部引用同一代码摘要和本次行情快照；不得按本地收益排名删除候选或产生未预设参数。 -- 最终运行仓库 Build and Verify（构建与验证）要求的检查，并从 Skill 用户入口执行一次完整 E2E，不能用单元测试组合代替。 - -## 关键取舍与风险 - -- 在没有真实聚宽对象时提前建立 `joinquant/strategies/strategy-003`，会破坏仓库“一目录对应一个远端策略详情页”的规则。 -- A 会在后续产生目录迁移、运行身份变化和证据引用重写风险；B 会提前产生一个远端策略对象，并要求明确限制不得启动正式回测或修改现有策略。 -- `.local/` 是单机私有存储，接受其不随 Git（版本管理）同步和需要用户自行备份的取舍。 -- 快照字段或复权口径存在隐式默认值，会导致本地研究无法复算。 -- 把 `use_real_price=False` 误解为“聚宽交易和撮合使用未复权实际价”，会掩盖平台在该模式下使用固定基准日前复权价的行为；需要在设计、证据和回测报告中明确列为平台限制。 -- 同日订单没有唯一顺序与分配规则，会让相同输入产生不同持仓路径。 - -## 测试策略 - -- 通用 Skill 删除海龟项目后仍能通过结构、单元和非海龟 E2E（端到端）测试。 -- 海龟 E2E 覆盖从收盘信号到次日订单、成交回填、持仓、风险和审计证据的完整路径。 -- 权威 CSV 批次、快照清单和内存 DuckDB 查询结果的身份与规范化内容摘要一致。 -- 公开仓库扫描不得发现完整行情文件、账号、Cookie（浏览器凭证）或 Token（访问令牌）。 -- 最小真实验证必须证明 13 个字段可取得、`fq=None` 或等价参数确实返回未复权价格、CSV 可落盘、DuckDB 可重建且内容摘要一致；验证结束后临时产物不存在。 -- 上述最小真实验证已完成；正式实现还需把相同检查扩展到 11 只 ETF 全历史快照和异常夹具。 - -## Spec Patch - -- 项目身份已采用 B，需要把任务 1.1 和相关设计文字明确为：先创建并同步真实 `strategy-003`，仅建立身份，不启动正式回测；本地研究放在其 `research/` 子目录。 -- 数据保存需要补充 `.local` 单一权威目录、公开仓库禁止行情值，以及缺少本地快照时输出 `evidence_insufficient`。 -- 数据契约需要修正为：`market-data.csv` 是精确的聚宽原始导出和唯一行情事实源，DuckDB 只是可重建派生视图；不再描述为“原始快照 + CSV 视图”两层结构。 -- 已确认共享行情数据中心：在通用本地研究规格新增仓库级共享行情能力，并把海龟规格、设计和任务中“海龟项目拥有行情快照与 Data Bridge”的表述改成“strategy-003 只引用共享 `snapshot_id`，不得复制行情”。后续聚宽正式回测仍使用云端行情，稳健性分析仍使用正式回测归档，不得混淆两类快照。 -- 补充共享行情中心的不可变批次、快照引用、内容去重、冲突重叠拒绝、追加新标的、旧快照复算、日线首版范围、显式字段能力和无持久 DuckDB 副本验收场景。 -- 补充已确认价格口径：CSV 保存未复权实际价格及 `factor`，本地研究不生成或使用复权价；聚宽海龟策略信号显式 `fq=None` 且 `use_real_price=False`，撮合层固定基准日前复权行为作为平台限制。 -- 补充每只 ETF 从自身首个完整交易日到显式 `snapshot_end_date` 的导出规则、60 个有效样本冷启动、退出—减仓—A1 买入顺序，以及 A1 等比分配、整手余额和 ETF 代码同分规则。 -- 把本地研究必需输出扩展为研究报告、三态研究建议和候选策略清单；候选清单固定为一项冻结基线加六项预设单项挑战配置，共用代码且不得按本地收益删选或新增参数。 -- 同步 proposal、design、tasks 和最终方案中与上述规格冲突的路径、Data Bridge、输出及策略身份表述；不修改后续流程对“正式回测快照”的定义。 -- 导出契约需要补充聚宽研究内核的内置 API 调用方式、Pandas 0.23.4 兼容写法和 `paused` 类型规范化。 +- Date: `2026-07-15` +- Status: 已按确认方案完成实施与全量验证，等待独立实现审查和人工确认研究结论 + +## 当前有效架构 + +- 整体架构固定为三个独立 Skill（技能）：本地研究流程、JoinQuant（聚宽）回测流程、策略分析。 +- 本变更主要实现本地研究流程、`strategy-003` vectorbt（向量化回测框架）执行路径和统一分析读取能力。 +- 聚宽回测 Skill 与策略分析 Skill 另立变更；聚宽正式复核、模拟交易、规则冻结和实盘不在本变更范围。 +- 本变更在本地研究 Skill 外用通用确定性分析完成一次独立验收，但不创建或修改策略分析 Skill。Vibe-Trading（氛围量化）群体分析存在已知缺陷，禁止作为证据或结论；当前没有安全单体分析入口时只记录证据不足。 + +## 本地研究 Skill 边界 + +- `run-local-quant-research` 每次只接受一个策略项目、一个 `snapshot_id`（快照标识）和一个场景。 +- 每次调用只产出一份本地兼容结果，并以 `next_action=return_to_caller` 停止。 +- Skill 不读取 `analysis-plan.json`，不接收候选数组，不知道七个基础场景，不内部循环，不生成聚合清单，也不执行绩效、归因、稳健性、报告或推荐。 +- 冻结基线和六个挑战由主 agent（代理）从策略自有分析计划读取后分别调用 Skill，合计恰好七次；其他稳健性分析只从基线已有事实确定性计算，不再调用 Skill。 + +## 机器可读分析计划 + +- `joinquant/strategies/strategy-003/research/analysis-plan.json` 是海龟策略自有、版本化的机器可读场景定义。 +- 该计划固定一个冻结基线、六个单项挑战、三个固定时期、三年季度滚动规则、11只 ETF（交易型开放式指数基金）删除集合、6个资产组删除集合、3个成本与延迟场景，以及区块抽样、历史压力、持仓冲击和 CVaR（条件风险价值）的定义、期望数量、固定随机种子和门槛。 +- 通用 `quant_analysis`（量化分析）只按 `analysis-plan.schema.json` 校验和展开通用场景；不得解析 Markdown（文档标记语言）、导入海龟模块或硬编码海龟资产、参数、分组、数量和门槛。 +- 主 agent 每次只把一个基础场景交给本地研究 Skill；计划摘要、七份来源摘要和其他确定性稳健性场景只在独立 `analysis_id` 下聚合。 + +## 行情与基准数据 + +- 共享行情中心使用 `.local/market-data/` 下不可变 Parquet（列式文件）批次和快照引用；DuckDB（嵌入式分析数据库）只使用内存查询。 +- 聚宽 CSV（逗号分隔文件）只作传输暂存，验证转换后删除；不保存持久 DuckDB 副本。 +- 海龟行情固定使用 `fq=None` 的未复权价格并保留 `factor`（复权因子），策略信号不生成或使用复权价。 +- 跨来源比较只使用独立基准集中的沪深300人民币总回报和纳斯达克100人民币总回报。 +- 聚宽 `results.benchmark_returns` 只作平台单基准累计收益参考；本地同名字段保留为全空 `double` 并明确声明 `missing_at_source/independent_benchmark_set`，禁止填零或冒充双基准。 + +## 标准分析数据包 + +- 聚宽现有 `manifest.json` 和六类核心 Parquet 是标准物理基准，现有回测目录、归档流程和文件保持 0 改动。 +- 本地结果使用独立 `schema_version=local-backtest/1`、`object.kind=local_backtest`、`source.kind=local_vectorbt` 和 `authority=local_research`。 +- 每个本地结果位于 `.local/quant-research///backtests//`，包含 `manifest.json`、`code.py`、`params.json`、`params_versions/`、`performance.json` 和 `data/`。 +- 本地物理事实只有 `results`、`balances`、`positions`、`orders`；`risk` 与 `period_risks` 明确声明来源未提供。统一读取器为两种来源建立六类逻辑视图。 +- 通用契约不要求所有策略共享归因字段；`strategy-003` 项目契约强制生成 `attribution_log-.parquet`，固定字段、确定性 `event_id` 唯一主键和 `turtle-etf-attribution/1` 原因码。 + +## vectorbt 唯一执行路径 + +- `strategy-003` 使用 vectorbt 1.1.0 官方 `Portfolio.from_order_func()`，11只 ETF 使用一个 `cash_sharing=True`（共享现金）组合组。 +- 海龟状态、退出、强制减仓、A1（同日共享预算分配）、组合风险、停牌和涨跌停判断由项目专属 Numba（即时编译）回调实现。 +- 官方回调使用 `pre_sim_func_nb`、`pre_segment_func_nb`、`order_func_nb`、`post_order_func_nb`；T日信息显式错位到T+1执行，卖出实际成交后才计算一次 A1，状态只按实际成交更新。 +- 不启用 `flexible=True`（灵活多订单），不调用 vectorbt 私有模拟函数，不另建独立 Numba 回测引擎。 +- 新路径通过后删除旧 `execution.py`、`state.py`、`signals.py`、`risk.py`、`allocation.py`、`reporting.py`、旧专用测试和公开导出；不保留兼容层、双引擎、回退或旧完整对照。 + +## 性能与原子完成门禁 + +- 每个单场景调用在同一暂存区执行冷启动和预热回测；两者均不得超过180秒，规范化结果摘要必须一致。 +- 计时从已准备输入进入 vectorbt 开始，到交易执行、四类共同事实、海龟必需归因日志及其结构/摘要/勾稽校验完成时停止；停止计时后才写入并校验 `performance.json` 与最终清单，二者属于完成门禁但不计入冷/热耗时。 +- 两次执行、摘要一致性和性能门槛通过后，先删除预热副本与可丢弃暂存并确认清理，再把清理结果写入 `performance.json`、生成最终清单并校验全部摘要，最后只原子发布已整理的一份权威结果；发布后不再依赖写入或清理。 +- 任一门禁失败只保留 attempt（尝试）证据，不得先发布 `complete` 目录；七次调用总耗时和 Vibe 分析耗时不能替代单场景门槛。 + +## 独立分析验收 + +- 分析准备先在 `.local/strategy-analysis-preparations//` 固化计划、基准、运行模板和七份单场景配置；七次调用完成后，主 agent 显式登记七组 `scenario_id -> run_id`，禁止扫描历史目录猜测来源。 +- 分析入口必须验证七个 `run_id` 唯一且共享快照、代码身份和执行后端，再由准备身份及全部来源摘要派生不可变 `analysis_id`；同计划下的新一批结果不得覆盖旧分析证据。 +- 主 agent 在 `.local/strategy-analysis//` 保存准备证据、含七个明确来源引用的一份 `source-results.json`、`analysis-scenarios.json`、确定性分析、Parquet（列式文件）证据矩阵、完整报告、Vibe 边界证据和推荐。 +- 七个基础场景逐次调用单场景 Skill；固定时期、滚动窗口、资产删除、成本/延迟、区块抽样、历史压力、持仓冲击和 CVaR 从基线来源事实确定性计算，不额外重跑策略。 +- 确定性分析覆盖收益、回撤、仓位与风险、Alpha/Beta（超额收益/市场暴露）、多维归因、完整稳健性、挑战比较、反对证据和不确定性。 +- Vibe 的研究目标与方法文档只作审计记录，不算实际分析。当前误调用的 `run_swarm`（运行群体分析）已标记无效并从全部结论排除；报告和推荐完全来自确定性分析。 +- 最终输出 `next_action=human_confirmation_required`，人工确认前不得启动聚宽正式回测、替换基线、修改参数、冻结策略或启动模拟交易。 + +## 验证要求 + +- 全部 Python（编程语言）命令使用项目 `.venv`（虚拟环境）。 +- 使用 TDD（测试驱动开发）覆盖 Schema 选择、聚宽零改动读取、本地结果字段、分析计划展开、海龟归因、vectorbt 官方回调、前视偏差、性能原子门禁和临时产物清理。 +- 从 Skill 用户入口只跑通一次单场景完整 E2E（端到端）；主 agent 复数调用和独立分析使用单独集成 E2E，不能把七个场景耦合进 Skill。 +- 不执行旧完整方案性能对照；验收直接使用已确认规则和固定小型合成夹具。 diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/run-state.json b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/run-state.json index 4309e7e..17c220a 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/run-state.json +++ b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/run-state.json @@ -4,8 +4,8 @@ "skillVersion": "1", "skillHash": "e211dc93b4bbc218965022f6d64f54a26e2240bd45b8bf6e1321ceb96775a3a3", "orchestration": "deterministic", - "currentStep": "full.archive.confirm", - "iteration": 9, + "currentStep": "full.build.execute", + "iteration": 12, "pending": null, "pendingRef": ".comet/pending-action.json", "trajectoryRef": ".comet/trajectory.jsonl", diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/state-events.jsonl b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/state-events.jsonl index 73d5621..61b51f0 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/state-events.jsonl +++ b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/state-events.jsonl @@ -2,3 +2,5 @@ {"schemaVersion":1,"timestamp":"2026-07-13T17:17:28.548Z","change":"build-turtle-etf-local-research-workflow","event":"design-complete","source":"comet-guard","from":{"workflow":"full","language":"zh-CN","phase":"design","contextCompression":"beta","buildMode":null,"buildPause":null,"subagentDispatch":null,"tddMode":null,"reviewMode":"standard","isolation":null,"verifyMode":null,"autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":null,"verifyResult":"pending","verificationReport":null,"branchStatus":"pending","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"to":{"workflow":"full","language":"zh-CN","phase":"build","contextCompression":"beta","buildMode":null,"buildPause":null,"subagentDispatch":null,"tddMode":null,"reviewMode":"standard","isolation":null,"verifyMode":null,"autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":null,"verifyResult":"pending","verificationReport":null,"branchStatus":"pending","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"effects":[{"field":"phase","from":"design","to":"build"}]} {"schemaVersion":1,"timestamp":"2026-07-13T22:15:38.464Z","change":"build-turtle-etf-local-research-workflow","event":"build-complete","source":"comet-guard","from":{"workflow":"full","language":"zh-CN","phase":"build","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"pending","verificationReport":null,"branchStatus":"pending","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"to":{"workflow":"full","language":"zh-CN","phase":"verify","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"pending","verificationReport":null,"branchStatus":"pending","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"effects":[{"field":"phase","from":"build","to":"verify"}]} {"schemaVersion":1,"timestamp":"2026-07-13T22:17:05.478Z","change":"build-turtle-etf-local-research-workflow","event":"verify-pass","source":"comet-guard","from":{"workflow":"full","language":"zh-CN","phase":"verify","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"pending","verificationReport":"docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md","branchStatus":"handled","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"to":{"workflow":"full","language":"zh-CN","phase":"archive","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"pass","verificationReport":"docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md","branchStatus":"handled","createdAt":"2026-07-13","verifiedAt":"2026-07-13","archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"effects":[{"field":"verifyResult","from":"pending","to":"pass"},{"field":"phase","from":"verify","to":"archive"},{"field":"verifiedAt","from":null,"to":"2026-07-13"}]} +{"schemaVersion":1,"timestamp":"2026-07-14T07:17:12.761Z","change":"build-turtle-etf-local-research-workflow","event":"archive-reopen","source":"comet-state","from":{"workflow":"full","language":"zh-CN","phase":"archive","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"pass","verificationReport":"docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md","branchStatus":"handled","createdAt":"2026-07-13","verifiedAt":"2026-07-13","archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"to":{"workflow":"full","language":"zh-CN","phase":"verify","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"pending","verificationReport":"docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md","branchStatus":"handled","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"effects":[{"field":"verifyResult","from":"pass","to":"pending"},{"field":"phase","from":"archive","to":"verify"},{"field":"verifiedAt","from":"2026-07-13","to":null}]} +{"schemaVersion":1,"timestamp":"2026-07-14T07:18:40.859Z","change":"build-turtle-etf-local-research-workflow","event":"verify-fail","source":"comet-state","from":{"workflow":"full","language":"zh-CN","phase":"verify","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"pending","verificationReport":"docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md","branchStatus":"handled","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"to":{"workflow":"full","language":"zh-CN","phase":"build","contextCompression":"beta","buildMode":"executing-plans","buildPause":null,"subagentDispatch":null,"tddMode":"tdd","reviewMode":"thorough","isolation":"branch","verifyMode":"full","autoTransition":true,"baseRef":"66cfbce81f95984aeeb7a9705b1d00ee3c8632ce","designDoc":"docs/superpowers/specs/2026-07-14-turtle-etf-local-research-workflow-design.md","plan":"docs/superpowers/plans/2026-07-14-turtle-etf-local-research-workflow.md","verifyResult":"fail","verificationReport":"docs/superpowers/reports/2026-07-14-build-turtle-etf-local-research-workflow-verify.md","branchStatus":"handled","createdAt":"2026-07-13","verifiedAt":null,"archived":false,"directOverride":null,"handoffContext":"openspec/changes/build-turtle-etf-local-research-workflow/.comet/handoff/spec-context.json","handoffHash":"dc23e3d037dbdd7bda188f0a24bdb5d9ef6639ef9ec33a1ce0a5a9fda3f39284","classicProfile":"full","classicMigration":1},"effects":[{"field":"verifyResult","from":"pending","to":"fail"},{"field":"phase","from":"verify","to":"build"}]} diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/trajectory.jsonl b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/trajectory.jsonl index 9569268..ace7381 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/.comet/trajectory.jsonl +++ b/openspec/changes/build-turtle-etf-local-research-workflow/.comet/trajectory.jsonl @@ -9,3 +9,6 @@ {"sequence":9,"timestamp":"2026-07-13T22:14:43.633Z","type":"state_transitioned","runId":"9d21cd10-ebe8-4cf1-910e-acbe1282c8e0","data":{"kind":"classic-config","field":"verify_mode","fromStep":"full.build.configure","toStep":"full.build.complete"}} {"sequence":10,"timestamp":"2026-07-13T22:15:38.463Z","type":"state_transitioned","runId":"9d21cd10-ebe8-4cf1-910e-acbe1282c8e0","data":{"kind":"classic","fromStep":"full.build.complete","toStep":"full.verify.run","event":"build-complete","phase":"build","source":"comet-guard"}} {"sequence":11,"timestamp":"2026-07-13T22:17:05.477Z","type":"state_transitioned","runId":"9d21cd10-ebe8-4cf1-910e-acbe1282c8e0","data":{"kind":"classic","fromStep":"full.verify.run","toStep":"full.archive.confirm","event":"verify-pass","phase":"verify","source":"comet-guard"}} +{"sequence":12,"timestamp":"2026-07-14T07:17:12.759Z","type":"state_transitioned","runId":"9d21cd10-ebe8-4cf1-910e-acbe1282c8e0","data":{"kind":"classic","fromStep":"full.archive.confirm","toStep":"full.verify.run","event":"archive-reopen","source":"comet-state"}} +{"sequence":13,"timestamp":"2026-07-14T07:18:40.857Z","type":"state_transitioned","runId":"9d21cd10-ebe8-4cf1-910e-acbe1282c8e0","data":{"kind":"classic","fromStep":"full.verify.run","toStep":"full.build.fix","event":"verify-fail","source":"comet-state"}} +{"sequence":14,"timestamp":"2026-07-14T17:31:05.664Z","type":"state_transitioned","runId":"9d21cd10-ebe8-4cf1-910e-acbe1282c8e0","data":{"kind":"classic-config","field":"verify_result","fromStep":"full.build.fix","toStep":"full.build.execute"}} diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/design.md b/openspec/changes/build-turtle-etf-local-research-workflow/design.md index 1b795b3..32b0832 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/design.md +++ b/openspec/changes/build-turtle-etf-local-research-workflow/design.md @@ -1,87 +1,116 @@ ## Context -本变更交付四个边界清晰的层次：仓库级 `run-local-quant-research` Skill（技能）负责流程编排；`scripts/research/` 共用脚本负责共享日线行情中心、契约、运行身份和证据；`strategy-003/research` 负责海龟 ETF（交易型开放式指数基金）项目适配、策略、参数和研究计算；`.local/` 保存不进入公开仓库的行情和运行证据。当前仓库已有 `joinquant-archive-sync`（聚宽归档同步）、项目 `.venv`（虚拟环境）、仓库级 Skill 布局和验证约定，应复用这些能力，且不得修改 `strategy-001`、`strategy-002` 或把本地结果冒充 JoinQuant（聚宽）正式回测。 +本变更建立通用单场景本地研究流程、标准分析数据包和 `strategy-003` 海龟 ETF 项目。当前方案已确认：11 只 ETF、6 个资产组、55/20/20、0.5N 加仓、2N 止损、4/6/12 N 风险单位、全量仓位再分配和单场景 180 秒性能门禁。 + +旧本地执行路径的主要瓶颈是组合订单的反复枚举；其分配范围只覆盖当日新增订单，导致资金被信号先后顺序长期占用。新设计不保留旧执行器或兼容开关。 ## Goals / Non-Goals -**Goals:** +**Goals（目标）：** -- 建立配置驱动、与具体策略解耦的本地研究编排入口。 -- 沉淀全策略共享的不可变日线行情批次、快照引用、精确 CSV（逗号分隔文件）与内存 DuckDB（嵌入式分析数据库）查询校验、运行清单和证据索引等确定性脚本。 -- 建立真实 `strategy-003` 身份，完成海龟 ETF 共享行情快照接入、策略规则验证、完整本地主流程、探索性粗筛、研究结论和候选策略交付。 -- 通过 Skill 初始化、结构校验、单元测试、用户入口 E2E（端到端）回归和非海龟前向验证证明通用性。 +- 提供与策略解耦、一次只执行一个场景的本地研究 Skill（技能）； +- 建立共享 Parquet（列式文件）行情和公司行动中心，以内存 DuckDB（嵌入式分析数据库）查询； +- 保持聚宽现有归档 0 改动可读，并让本地结果尽量对齐其目录和四类共同事实； +- 使用 vectorbt（向量化回测框架）官方生命周期执行海龟单场景模拟； +- 用逐单位 N 状态和事件驱动全量仓位再分配消除信号先后资金优先权； +- 输出可供独立策略分析读取的单位、缩放和再分配证据； +- 冷启动和预热单次均小于 180 秒且结果一致。 -**Non-Goals:** +**Non-Goals（非目标）：** -- 不在 Skill（技能）中保存海龟参数、资产池、交易规则、策略代码或研究证据。 -- 不在本地运行或宣称正式回测、模拟交易和最终验收。 -- 不重复实现聚宽认证、远端回测/模拟对象归档或修改 Vibe-Trading（AI 研究助理）上游源码。 -- 不建设行情服务、持久数据库平台、远端数据仓库或通用交易插件；共享行情中心只是 `.local/` 文件式能力。 -- 首版不实现分钟线、基本面、财务或因子数据，也不为以后扩展提前增加来源插件框架。 +- 不修改聚宽正式策略、正式回测、模拟交易或既有归档； +- 不在本地研究 Skill 中运行 7 个场景、稳健性矩阵、Vibe-Trading（AI 研究助理）或报告； +- 不运行旧方案收益或性能对照； +- 不纳入 17 ETF 扩展； +- 不创建策略分析 Skill，不完成聚宽正式复核或规则冻结。 ## Decisions -### 1. 使用“Skill 编排 + 共用脚本 + 项目适配器 + 本地证据”四层结构 +### 1. 三层职责边界 + +整体架构分为本地研究流程、聚宽回测流程和策略分析三个 Skill。当前变更主要实现本地研究流程和标准分析数据包；通用分析代码只用于证明该数据包可真实消费。 + +本地研究 Skill 只验证目标、快照、配置和项目入口，执行一个场景并返回一份结果。海龟规则只存在于 `strategy-003` 项目配置、输入和回调中。 + +### 2. 共享行情与公司行动 + +权威行情保存在 `.local/market-data/` 的不可变 Parquet 批次中。聚宽 CSV 仅作传输暂存，远端回读摘要和本地转换摘要一致后删除。DuckDB 只在内存查询，不保存数据库文件。 + +原始行情为未复权事实。本地使用应用日可见的 `上一交易日原始 close / 当日 pre_close` 更新连续因子，统一派生连续经济 OHLC 和经济单位。公司行动元数据只授权和审计价格基准变化；晚公布数据标为事后核对，不能产生前视回写。该方案是研究级总回报近似，不声称精确复现真实份额、派息日现金或聚宽逐日账户。 + +成交额和流动性不进入海龟策略输入或订单规则。 + +### 3. 标准分析数据包 -`SKILL.md` 只描述执行顺序、输入输出、停止状态和安全边界。`scripts/research/local_quant_research/` 的 Python（编程语言）入口读取 JSON（结构化清单）配置，调用 `scripts/research/market_data/` 验证共享快照，再以参数数组安全调用仓库内项目入口并收口证据。`strategy-003/research` 只实现海龟项目适配和纯计算模块；行情与运行证据分别位于 `.local/market-data/` 和 `.local/quant-research/strategy-003/`。 +聚宽来源继续使用现有 `schema_version=1` 和原目录；读取器按原清单直接建立视图，不生成转换副本。本地来源使用 `local-backtest/1`、`object.kind=local_backtest`、`source.kind=local_vectorbt` 和 `authority=local_research`。 -替代方案是为海龟直接编写专用 Skill，或建立通用策略框架。前者无法复用且形成反向依赖，后者会提前抽象交易领域，因此均不采用。 +本地物理输出 `results`、`balances`、`positions`、`orders` 四类共同执行事实；`risk` 和 `period_risks` 声明为来源缺失，由独立策略分析计算。海龟专用证据保存在版本化归因扩展，不新增第五张标准表。 -### 2. 共享行情中心使用不可变 CSV 批次与快照引用 +### 4. vectorbt 官方执行路径 -`.local/market-data/batches//` 保存精确原始 `market-data.csv`、`manifest.json` 和 `validation.json`；CSV 是唯一行情事实源。`.local/market-data/snapshots/.json` 只引用一个或多个已验证批次及明确证券、日期、字段、来源和价格口径，不复制行情。相同内容按来源身份和字节摘要复用；新标的或新日期追加新批次；同一来源、频率、证券和日期出现不同值时拒绝合并，不覆盖旧批次。 +基线使用 `Portfolio.from_order_func()`、`cash_sharing=True` 和一个组合组。vectorbt 负责时间遍历、订单、费用、现金、持仓和权益；Numba（即时编译）回调负责信号、逻辑单位、风险缩放、可交易性和动作原因。 -DuckDB 只用 `:memory:` 建立可重建查询视图。通用脚本对字段顺序、类型、空值、排序和 `paused` 布尔类型规范化后比较 CSV 与查询结果摘要；不保存持久 `.duckdb` 副本。替代方案是每策略保存快照或以中央持久 DuckDB 为唯一事实源；前者重复且会分叉，后者需要额外时间旅行、迁移和损坏恢复，因此均不采用。 +额外延迟研究先冻结原动作、数量、原因和 N，再用 `Portfolio.from_orders()` 机械延迟执行。基线 `additional_delay_days=0`，仍是收盘检查、次日开盘成交。 -### 3. 海龟交易语义全部留在项目层 +### 5. 逐单位状态 -先创建真实聚宽策略空壳并同步为 `strategy-003`，只建立身份而不启动正式回测。海龟资产池、55 日入场、20 日 N 值、0.5N 加仓、2N 共同止损、20 日退出、风险上限、A1 分配及研究输出由项目配置和模块实现。通用入口只检查配置契约、调用项目入口并记录结果，不解释任何交易字段。海龟项目只引用共享 `snapshot_id`，删除海龟项目后 Skill、共享行情中心和非海龟 E2E（端到端）仍须通过。 +每只 ETF 固定保存最多 4 个单位的 `signal_n`、`base_quantity` 和实际成交价。候选基础数量为： -替代方案是把常见指标或交易规则放入 Skill 脚本。即使名称通用，也会使流程层耦合当前策略，因此不采用。 +```text +floor_to_100_shares(signal_equity × 1% / signal_n) +``` -### 4. 每次运行以不可变清单、唯一状态和完整输出收口 +只有真实的入场或加仓买入成交才建立单位。候选被停牌、涨停、统一缩放或整手取整消除时，不推进状态。 -`run_id` 由共享快照、项目配置和代码摘要共同确定。产物先写暂存位置，全部输入、流程、必需输出和摘要校验通过后才原子固化。只有全部门禁通过才输出 `complete`；执行前身份或输入缺失输出 `evidence_insufficient`；既有证据不一致、执行异常、确定性冲突或临时文件无法确认清理输出 `failed`。同一完整身份可验证后复用，身份变化产生新运行，失败重试保留前次尝试。 +固定加仓档位为首次成交价加 `0.5N`、`1.0N`、`1.5N`，其中 N 为首次信号日 N；每个交易日最多增加一个单位。 -海龟完整运行必须生成 `research-report.md`、`conclusion.json` 和 `candidate-strategies.json`。研究建议与运行状态分离：建议只能是 `proceed_to_joinquant`（进入聚宽回测）、`revise_and_reassess`（修订后再评估）或 `stop_evidence_insufficient`（证据不足而停止）。候选清单固定为一项冻结基线和六项预设单项挑战配置，共用同一代码摘要和快照，不按本地收益删选或新增参数。 +每个成交单位的候选止损为 `fill_price - 2 × frozen_signal_n`，共同止损只取历史最大值。每日 N 和再分配订单都不改变既有止损。 -替代方案是维护可变的 `latest`（最近一次）目录。该方案容易混淆不同代码和数据版本，因此不采用。 +### 6. 4/6/12 缩放 -### 5. 沿用仓库运行环境与现有能力 +```text +group_scale_g = min(1, 6 / group_units_g) +effective_units = sum(unit_count_i × group_scale_group(i)) +portfolio_scale = min(1, 12 / effective_units) +``` -所有本地 Python 命令使用 `.\.venv\Scripts\python.exe`。Skill 使用 `skill-creator`（技能创建指南）的 `init_skill.py` 初始化，以 `quick_validate.py` 校验，并按现有约定在 `.claude/skills/` 建立指向 `.agents/skills/` 的兼容符号链接。优先使用标准库和仓库已具备的 DuckDB、Pandas（数据处理）等依赖；只有现有环境无法完成明确需求时才增加并固定依赖。 +单标的最多 4 个逻辑单位，资产组最多 6 个有效单位，组合最多 12 个有效单位。目标数量为冻结单位基础数量之和依次乘组比例、组合比例和现金比例，再按 100 股向下取整。 -涉及现有聚宽认证和明确远端对象归档时调用 `joinquant-archive-sync`，不复制其浏览器、凭证或归档实现。共享聚宽日线导出器直接使用研究内核注入的 API（接口），兼容 Pandas 0.23.4 的 `line_terminator` 并规范化 `paused`；远端临时文件只有在本地字节摘要验证后才删除，无法确认清理时运行失败。 +### 7. 全量仓位再分配 -替代方案是为新 Skill 建独立虚拟环境或复制归档代码。两者会造成依赖和认证分叉，因此不采用。 +有效入场、有效加仓、止损或趋势退出触发一次组合事件。系统先形成临时单位簿，再计算全部标的的新目标。不能产生至少一个整手净新增买入的候选被移除并重新计算，直到集合稳定。 -### 6. 验证分为结构、确定性、完整入口和通用性四层 +现金比例是在 `[0, 1]` 上求得的组合共同最大可行比例，预计卖出净收入、买入价、滑点和每笔佣金均纳入。禁止余额补仓、最大余数分配和按代码顺序耗尽现金。 -验证依次覆盖 `quick_validate.py`、Skill 布局与符号链接、共享行情中心、通用运行器、海龟纯计算、不变量和从 Skill 用户入口到三类输出的完整海龟 E2E。另以不含海龟词汇和资产的最小非海龟配置做独立前向验证；验证代理只获得 Skill 和原始任务/夹具，不获得预期答案。正式网络数据获取另行留证，不作为日常离线回归前提。 +调用顺序固定为：完整退出、再分配卖出、入场或加仓、再分配买入。再分配买卖只改变实际持仓，不改变单位数、加仓档位或共同止损。 -替代方案是只测试脚本函数。它不能证明 Skill 编排、项目适配和停止状态在真实入口上协同工作，因此不采用。 +### 8. 删除旧交易控制 + +生产配置和执行路径不再包含资金仓位上限、计划风险上限、交易用协方差、最低对齐样本、目标波动率、强制波动率减仓或旧新增订单分配。旧字段被严格拒绝，不用极大值、`null`、隐藏默认值或回退路径模拟删除。 + +协方差、实现波动率、单标的/资产组实际权重和计划损失比例只由独立策略分析从结果事实计算，不反向产生订单。 + +### 9. 分析兼容 + +独立分析继续计算收益、回撤、双基准 Alpha/Beta（阿尔法/贝塔）、仓位、费用和归因。海龟扩展存在时增加最高有效 N 单位、组合单位预算利用率和全量再分配次数；聚宽结果没有该扩展时返回 `None/0`，不要求聚宽改动。 + +Vibe-Trading 仅允许无已知缺陷的单体公开能力；群体分析不得进入结论。 ## Risks / Trade-offs -- [项目命令配置过宽可能执行非预期入口] → 只接受参数数组和仓库内明确路径，不经 Shell 拼接执行。 -- [CSV 与 DuckDB 类型差异造成错误不一致] → 先按契约规范化类型、时区、空值和排序，再计算内容摘要。 -- [权威快照字段或复权口径缺失] → 以 `evidence_insufficient` 停止，不用默认值补齐。 -- [完整行情误入公开仓库] → 行情只写 `.local/market-data/`，最终验证扫描 Git（版本管理）跟踪文件。 -- [批次重叠导致旧研究漂移] → 相同内容去重，冲突重叠拒绝，快照清单和旧批次不可修改。 -- [持久 DuckDB 成为第二事实源] → 只使用内存视图并测试 `.local/market-data/` 不产生长期 `.duckdb` 文件。 -- [通用契约被海龟需求逐步污染] → 通用测试使用非海龟夹具，并增加 Skill 目录不得包含海龟常量的边界检查。 -- [Vibe-Trading 已知前视偏差影响粗筛] → 官方修复前跳过受影响的组合优化器，其他探索结果仍明确标注为非正式。 -- [完整 E2E 无法证明聚宽真实成交] → E2E 只证明本地规则与状态流；正式交易路径留给独立聚宽回测变更。 - -## Migration Plan - -1. 创建并同步真实 `strategy-003` 空壳，只建立身份，不启动正式回测。 -2. 先以 TDD（测试驱动开发）实现共享日线行情中心、不可变批次、快照引用和内存 DuckDB 视图，再接入聚宽导出器。 -3. 使用项目 `.venv` 调用 `init_skill.py` 初始化 `.agents/skills/run-local-quant-research/`，建立 `.claude/skills/` 兼容链接，并实现通用运行器和证据收口。 -4. 在 `strategy-003/research` 新增海龟项目适配、纯计算模块、配置、测试和三类研究输出，不复制行情。 -5. 用固定夹具运行海龟完整本地主流程，再运行非海龟最小 E2E 和独立前向验证；最后用真实 11 只 ETF 快照完成一次本地研究和临时产物清理。 -6. 若实施失败，停用新增 Skill 和共用脚本并保留已验证的不可变证据；不得删除或修改 `strategy-001`、`strategy-002`，也不得把功能分支本地合入主干。 +- 100 股整手会放大小账户的离散误差；通过统一向下取整和归因证据显式保留，不用余额补仓制造顺序偏差。 +- 再分配增加换手和费用；报告必须单独展示再分配次数、成交和费用。 +- 12 个 1N 单位不是 12% 最大损失；2N 止损、跳空和无法成交风险需在报告披露。 +- 连续经济价格是公司行动研究近似；结果不得宣称与聚宽精确账户对账。 +- 首次 JIT（即时编译）有冷启动成本；冷/热分别计时并应用 180 秒硬门禁。 + +## Verification + +- TDD（测试驱动开发）覆盖新配置拒绝旧字段、4/6/12 公式、现金统一缩放、输入顺序不变、后到趋势挤占、逐单位冻结 N、固定加仓档、只上移止损、再分配状态隔离、退出优先和市场不可交易。 +- 标准四表、归因扩展、延迟执行和聚宽无海龟扩展读取通过契约测试。 +- 从公开 CLI（命令行入口）执行完整单场景 E2E（端到端），证明行情、vectorbt、标准结果包、清单、摘要和清理完整闭环。 +- 真实 11 ETF 新基线只运行一次，冷启动和预热均小于 180 秒且摘要一致；不执行旧方案对照或稳健性矩阵。 ## Open Questions -无。项目身份、共享行情位置、13 个字段、未复权口径、导出入口、订单优先级、A1 分配和最终输出均已在深度设计中确认。 +无。方案已确认，最终本地研究报告给出推荐结论后等待人工确认。 diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/proposal.md b/openspec/changes/build-turtle-etf-local-research-workflow/proposal.md index f1c5a36..ff03705 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/proposal.md +++ b/openspec/changes/build-turtle-etf-local-research-workflow/proposal.md @@ -1,21 +1,33 @@ ## Why -海龟 ETF（交易型开放式指数基金）系统已经完成研究方案设计，但仓库还缺少可重复执行的本地研究流程、全策略共享日线行情中心、策略实现、确定性验证和探索性研究证据。若直接把这些步骤写成海龟专用工具或把行情保存在单一策略目录，后续策略研究仍会重复建设并产生不同数据版本，因此需要同时交付与具体策略解耦的流程编排和共享行情能力，以及使用这些能力完成海龟 ETF 本地研究的项目实例。 +仓库需要一个与具体策略解耦的单场景本地研究流程、一个以 JoinQuant（聚宽）现有归档为物理基准的标准分析数据包，以及一个能在三分钟内完成海龟 ETF 基线模拟的项目实现。旧本地执行路径把大量时间消耗在仅针对当日新增订单的组合枚举上，既慢，也让先进入的趋势长期占用资金，后进入的有效趋势无法公平获得风险预算。 + +本变更已确认的解决方案是：保留共享 Parquet（列式文件）行情、内存 DuckDB（嵌入式分析数据库）查询和 vectorbt（向量化回测框架）官方记账能力，把 `strategy-003` 改为逐单位 N 风险和事件驱动的全量仓位再分配。 ## What Changes -- 新增 `run-local-quant-research` Skill（技能），只负责编排研究目标检查、共享快照接入、项目研究入口调用、运行状态记录和证据索引生成，不承载任何具体交易规则。 -- 沉淀可复用的确定性脚本，在 `.local/market-data/` 管理不可变日线导出批次和快照引用，验证快照身份、SHA256（文件摘要）、字段、日期、缺失值及精确原始 CSV（逗号分隔文件）与内存 DuckDB（嵌入式分析数据库）查询结果的一致性；不保存持久 DuckDB 副本。 -- 通过配置、清单和项目入口向通用流程传入策略代码、参数、资产池、数据字段和研究任务；通用 Skill（技能）与脚本不得硬编码或反向依赖海龟 ETF 的入场、加仓、止损、退出、资产分组或验收规则。 -- 先创建真实 JoinQuant（聚宽）策略空壳并同步为 `strategy-003`，只建立身份而不启动正式回测；海龟项目引用共享 `snapshot_id`（快照标识），实现策略及可独立测试的计算模块，完成确定性规则验证、完整本地研究路径验证和方向性粗筛。 -- 海龟 ETF 的策略、冻结参数、研究配置和证据保留在 `strategy-003` 项目产物中，行情只保存在共享中心；完整运行输出本地研究报告、三态研究建议以及“冻结基线 + 六个预设单项挑战”的候选策略清单。本地结果只作为探索和规则正确性证据，不宣称为聚宽正式回测或最终验收结论。 +- 新增通用 `run-local-quant-research` Skill（技能），一次只编排一个项目、一个快照和一个场景，不承载海龟规则、场景矩阵、稳健性、Vibe-Trading（AI 研究助理）或报告算法。 +- 在 `.local/market-data/` 保存不可变 Parquet 行情与公司行动批次；聚宽 CSV（逗号分隔文件）只用于传输，校验后删除；查询只使用内存 DuckDB，不保存持久数据库。 +- 以聚宽现有回测归档为标准物理基准，聚宽结果 0 改动直读；本地结果使用独立 `local-backtest/1` 清单，并尽量镜像聚宽现有目录和四类共同执行事实。 +- 继续使用按应用日可见原始行情派生的连续经济价格和经济单位处理公司行动，明确保留研究级近似边界。 +- 海龟策略层不使用成交额、流动性门槛或单笔成交额占比。 +- `strategy-003` 固定使用 11 只 ETF、6 个资产组、55/20/20、0.5N 加仓、2N 共同止损和收盘检查、次日开盘成交。 +- 每个真实建立的逻辑单位按信号日权益的 1%/N 计算并冻结基础数量和 N；单标的最多 4 单位、资产组最多 6 有效单位、组合最多 12 有效单位。 +- 用全量仓位再分配替代旧的仅新增订单分配：入场、加仓、止损或退出事件触发时，统一重算全部持仓目标；后到趋势可以同比例挤占已有趋势资金。 +- 现金不足时使用统一最大可行比例并按整手向下取整，不做余额补仓或按代码顺序分配。 +- 删除生产路径中的资金仓位上限、计划风险上限、协方差交易门槛、目标波动率控制、强制波动率减仓和旧分配实现，不保留兼容开关或双路径。 +- 本地模拟使用 vectorbt 官方 `Portfolio.from_order_func()` 和 Numba（即时编译）回调；额外延迟研究使用冻结订单的 `Portfolio.from_orders()`，基线不增加额外延迟。 +- 标准四表不变；海龟归因扩展增加单位数、候选基础数量、冻结 N、成交价、共同止损、组/组合/现金比例和再分配状态隔离证据。 +- 独立 `quant_analysis`（量化分析）读取本地或聚宽结果，计算实际权重、计划损失比例、有效 N 单位和再分配次数；没有海龟扩展的聚宽结果仍可读取。 +- 本次只运行一次真实 11 ETF 新基线，不运行旧方案对照、17 ETF 扩展、7 个场景或稳健性矩阵；冷启动和预热单次均不得超过 180 秒。 ## Capabilities ### New Capabilities -- `local-quant-research-workflow`: 与具体策略解耦的本地量化研究编排、共享日线行情中心，以及快照引用、数据桥、运行清单和证据索引所需的确定性通用脚本。 -- `turtle-etf-local-research`: 使用共享行情中心和通用研究流程完成海龟 ETF 数据接入、策略实现、确定性验证、方向性粗筛、研究结论和候选策略产物。 +- `local-quant-research-workflow`：通用单场景研究编排、共享行情中心、运行身份和证据收口。 +- `standard-strategy-analysis-data`：聚宽零改动直读、本地结果适配、统一内存视图和派生查询。 +- `turtle-etf-local-research`：使用共享行情与 vectorbt 完成海龟 ETF 单场景本地研究。 ### Modified Capabilities @@ -23,8 +35,9 @@ ## Impact -- 新增 `.agents/skills/run-local-quant-research/` 下的仓库级 Skill（技能）编排入口、必要参考约定和界面元数据，并增加对应的通用脚本与验证。 -- 新增 `scripts/research/` 下的共享行情中心和通用研究脚本；完整行情仅写入仓库已忽略的 `.local/market-data/`，不得进入公开 Git（版本管理）。 -- 新增 `strategy-003` 海龟项目专属的策略代码、研究配置、测试、研究报告、研究建议和候选策略清单；行情不复制到 Skill 或策略目录。 -- 复用现有 JoinQuant（聚宽）数据获取和归档能力，不重复实现认证或远端对象同步，不保存账号、密码、Token（访问令牌）或 Cookie（浏览器凭证）。 -- 本变更只覆盖本地研究流程；正式交易路径、正式收益、稳定性变体回测和最终验收结论由后续独立变更处理。 +- 新增或更新 `.agents/skills/run-local-quant-research/`、`scripts/research/`、本地清单 Schema（结构约束）、双基准契约和相应测试。 +- `strategy-003` 成为唯一海龟本地研究项目入口；旧自研执行模块、旧风险路径和兼容导出被删除。 +- `.local/` 保存行情、运行和分析证据，不进入 Git（版本管理），历史不可变结果不迁移、不重写、不删除。 +- 本变更不修改聚宽正式策略、正式回测、模拟交易或既有聚宽归档；不创建策略分析 Skill，不执行聚宽正式复核。 + +**状态：已确认。冻结摘要：11 只 ETF、55/20/20、4/6/12、全量仓位再分配、180 秒。** diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/specs/local-quant-research-workflow/spec.md b/openspec/changes/build-turtle-etf-local-research-workflow/specs/local-quant-research-workflow/spec.md index 3a11e26..d912fc2 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/specs/local-quant-research-workflow/spec.md +++ b/openspec/changes/build-turtle-etf-local-research-workflow/specs/local-quant-research-workflow/spec.md @@ -19,15 +19,15 @@ 系统 SHALL（必须）在仓库已忽略的 `.local/market-data/` 提供与任何策略解耦的共享行情中心；完整行情值不得写入公开仓库。首版只实现日线行情，但 SHALL（必须）通过来源、标的类型、频率和显式字段能力允许以后追加其他标的。 #### Scenario: 导入不可变行情批次 -- **WHEN** 导入一个通过字段与来源校验的日线行情批次 -- **THEN** 系统在 `.local/market-data/batches//` 固化 `manifest.json`、精确原始 `market-data.csv` 和 `validation.json`，并记录来源、标的类型、频率、字段、价格口径、每只证券实际起止日、行数、导出代码摘要和 CSV 字节 SHA256（文件摘要） +- **WHEN** 导入一个通过字段与来源校验的日线行情批次及其公司行动事件 +- **THEN** 系统在 `.local/market-data/batches//` 固化 `manifest.json`、权威 `market-data.parquet`（列式行情）、版本化 `corporate-actions.parquet`（公司行动事件）和 `validation.json`，并记录来源、标的类型、频率、字段、价格口径、每只证券实际起止日、公司行动知识截止日、两类行数、导出代码摘要、传输文件字节 SHA256（文件摘要）、规范化内容摘要和 Parquet 文件摘要 #### Scenario: 创建不可变快照引用 - **WHEN** 调用者从一个或多个已验证批次选择明确证券、日期、字段、来源和价格口径 - **THEN** 系统在 `.local/market-data/snapshots/.json` 创建只引用批次、不复制行情的不可变快照，策略运行只保存 `snapshot_id` 及其摘要 #### Scenario: 相同内容去重 -- **WHEN** 新导入内容与既有批次的来源身份和 CSV 字节摘要完全一致 +- **WHEN** 新导入内容与既有批次的来源身份、结构版本和规范化逻辑内容摘要完全一致 - **THEN** 系统复用既有批次，不创建第二份权威行情 #### Scenario: 冲突重叠拒绝 @@ -53,24 +53,40 @@ - **WHEN** 任一身份字段缺失或任一文件摘要不匹配 - **THEN** 系统拒绝执行研究入口；身份或来源本来就缺失时输出 `evidence_insufficient`，既有文件被篡改或内容不一致时输出 `failed` -### Requirement: 权威 CSV 与可重建 DuckDB 视图 -系统 SHALL（必须）把每个批次的精确原始 `market-data.csv` 作为唯一行情事实源；DuckDB（嵌入式分析数据库）只从权威 CSV 建立可重建的内存查询视图，不得保存持久数据库副本或第二份权威行情。 +### Requirement: 权威 Parquet 与可重建 DuckDB 视图 +系统 SHALL（必须）把每个批次的 `market-data.parquet` 作为本地唯一行情事实源；聚宽导出的 CSV（逗号分隔文件）只允许存在于传输和导入暂存阶段。DuckDB（嵌入式分析数据库）只从权威 Parquet 建立可重建的内存查询视图，不得保存持久数据库副本或第二份权威行情。`batch_id` SHALL（必须）绑定规范化逻辑内容与来源契约，Parquet 字节摘要 SHALL（必须）单独用于文件完整性验证，避免编码器版本变化静默改变逻辑身份。 -#### Scenario: CSV 与内存视图一致 -- **WHEN** 已验证快照从权威 CSV 批次建立 DuckDB 内存视图 -- **THEN** 系统规范化字段顺序、类型、空值、排序和 `paused` 布尔类型后，CSV 与查询结果的行数和规范化内容摘要一致 +#### Scenario: Parquet 与内存视图一致 +- **WHEN** 已验证快照从权威 Parquet 批次建立 DuckDB 内存视图 +- **THEN** 系统规范化字段顺序、类型、空值、排序和 `paused` 布尔类型后，清单中的规范化内容摘要与查询结果的行数和规范化内容摘要一致 #### Scenario: 派生视图发生漂移 -- **WHEN** DuckDB 查询结果与权威 CSV 的行集合、字段值、类型或规范化内容摘要不一致 +- **WHEN** DuckDB 查询结果与权威 Parquet 的行集合、字段值、类型或规范化内容摘要不一致 - **THEN** 系统输出 `failed` 并停止项目研究，不把 DuckDB 结果视为替代事实源 -#### Scenario: 未复权价格口径 +#### Scenario: 原始价格与公司行动口径 - **WHEN** 批次通过 `fq=None` 或来源声明的等价方式导入 -- **THEN** CSV 精确保留实际未复权价格和 `factor`（复权因子），本流程不生成或使用复权价格序列 +- **THEN** `market-data.parquet` 按固定结构保存实际未复权价格、`pre_close`（前收盘参考价）和 `factor`（复权因子），同一批次的 `corporate-actions.parquet` 保存版本化公司行动事件；两者及其摘要共同进入批次与快照身份。连续总回报价格只按生效日可见的原始 `close/pre_close` 行情事实在内存派生；公司行动元数据仅用于核对，不回写原始行情，也不固化为第二行情事实 + +#### Scenario: 公司行动只提供可审计事实 +- **WHEN** 项目从共享快照请求拆分或现金分红信息 +- **THEN** 共用行情层只提供证券、事件类型、来源事件标识、公告日、登记日、除权或生效日、支付日、拆分比例、每份现金、状态、知识截止日和来源摘要；它不得导入 vectorbt、生成订单、修改项目持仓或决定现金分红如何进入账户 + +#### Scenario: 公司行动缺失时关闭运行 +- **WHEN** 项目请求区间内存在无法由公司行动事件解释的价格基准变化，或公司行动文件、来源摘要、事件字段和行情勾稽不完整 +- **THEN** 快照校验或项目运行输出 `evidence_insufficient`，不得忽略事件、猜测类型、使用经验阈值或继续生成可被分析的本地结果 + +#### Scenario: 研究级近似不冒充精确账户 +- **WHEN** 项目执行后端不能原生处理派息日现金或拆分后的真实份额状态 +- **THEN** 共用流程允许项目声明版本化的研究级近似口径并继续运行，但本地清单必须明确记录价格基准、数量基准、现金分红处理、不能精确复核的账户字段和公司行动来源摘要；未声明精度边界的结果不得通过输出门禁 + +#### Scenario: 传输文件完成使命后清理 +- **WHEN** 聚宽 CSV 已完成字节摘要核对、结构校验、Parquet 转换和逻辑内容复核 +- **THEN** 系统删除本地暂存 CSV 和聚宽远端临时文件，仅在批次清单保留传输摘要；任一清理步骤无法确认时本次导入输出 `failed` #### Scenario: 禁止持久 DuckDB 副本 - **WHEN** 查询或研究运行结束 -- **THEN** `.local/market-data/` 中不存在作为长期事实源的 `.duckdb` 文件，后续查询可仅凭快照清单和权威 CSV 重建 +- **THEN** `.local/market-data/` 中不存在作为长期事实源的 `.duckdb` 文件，后续查询可仅凭快照清单和权威 Parquet 重建 ### Requirement: 唯一三态收口 每次运行 SHALL（必须）且只能以 `complete`、`evidence_insufficient` 或 `failed` 之一收口；流程状态不得与项目研究建议混为一谈。 @@ -87,17 +103,17 @@ - **WHEN** 输入门禁、项目流程、声明输出、摘要校验和证据固化全部通过 - **THEN** 系统输出 `complete` -#### Scenario: 完成不等于策略通过 -- **WHEN** 流程完整执行但项目建议为 `revise_and_reassess`（修订后再评估） -- **THEN** 运行状态仍可为 `complete`，且不得把该状态解释为正式回测通过或进入实盘 - ### Requirement: 不可变且原子固化的研究证据 -系统 SHALL（必须）以快照摘要、项目配置摘要和代码摘要生成 `run_id`，先在暂存位置生成产物，全部校验通过后一次性固化包含输入、命令、状态、输出路径和输出摘要的不可变证据索引。 +系统 SHALL（必须）以快照摘要、项目配置摘要、规范化单场景配置摘要、代码摘要和执行后端身份生成 `run_id`，先在暂存位置生成产物，全部校验通过后一次性固化包含输入、命令、状态、输出路径和输出摘要的不可变证据索引；不同场景配置不得复用同一 `run_id`。 #### Scenario: 首次成功运行 - **WHEN** 一个新 `run_id` 的全部输入、项目流程和输出校验通过 - **THEN** 系统原子固化运行证据，不留下可被误认成完成的中间目录 +#### Scenario: 性能证据属于原子完成门禁 +- **WHEN** 项目对同一单场景执行冷启动和预热性能复核 +- **THEN** 两次执行、结果摘要一致性比较和各自不超过项目声明上限的判断均在同一暂存区完成；比较通过后先删除预热副本和可丢弃暂存并确认清理，再写入 `performance.json`、生成最终清单并校验全部摘要，最后只原子发布已整理的一份权威结果。发布后不得依赖写入或清理，任一失败只保留失败尝试证据 + #### Scenario: 相同身份重复运行 - **WHEN** 已存在同一 `run_id` 的 `complete` 运行且全部产物重新校验通过 - **THEN** 系统复用既有完整产物，不重写文件或创建第二份权威证据 @@ -114,6 +130,44 @@ - **WHEN** 调用者重试一个 `failed` 或 `evidence_insufficient` 运行 - **THEN** 系统保留原尝试证据并创建新的尝试记录；只有新尝试全部通过才可固化为 `complete` +### Requirement: 项目执行后端身份与通用输出兼容 +通用本地研究流程 SHALL（必须）把交易执行后端视为项目自有实现，不硬编码 vectorbt（向量化回测框架）、海龟回调或任何特定回测引擎。项目 SHALL（必须）在代码身份中声明执行后端名称、版本、依赖摘要和输出适配器版本；无论项目使用何种后端，通用运行器只校验声明的标准分析数据包和证据契约。 + +#### Scenario: 项目声明 vectorbt 执行后端 +- **WHEN** `strategy-003` 使用 vectorbt 官方 `Portfolio.from_order_func()`（自定义订单函数）运行本地交易路径 +- **THEN** 项目代码身份记录 vectorbt、Numba（即时编译）、NumPy（数组计算）、Pandas（数据处理）和输出适配器版本及摘要，通用运行器只按声明校验身份与产物 + +#### Scenario: 非海龟项目使用其他执行方式 +- **WHEN** 非海龟项目适配器不使用 vectorbt 或使用另一种项目内执行方式 +- **THEN** Skill、共享行情中心和通用运行器仍可完成快照校验、项目调用、标准输出校验和证据收口，不要求该项目安装 vectorbt + +#### Scenario: 执行后端依赖缺失 +- **WHEN** 项目声明的 vectorbt、Numba 或兼容依赖在项目 `.venv`（虚拟环境）中缺失或版本不匹配 +- **THEN** 通用运行器在项目执行前输出具体缺项并以 `evidence_insufficient` 收口，不静默安装、升级或回退到另一执行后端 + +#### Scenario: 更换后端不改变通用分析契约 +- **WHEN** 项目从旧逐日实现迁移到 vectorbt 后端 +- **THEN** 项目输出与聚宽现有归档同名同义的 `results`、`balances`、`positions`、`orders` 四类共同执行事实及本地兼容清单，并在清单中把来源未提供的 `risk`、`period_risks` 标记为由独立策略分析计算；既有聚宽归档无需改动，通用绩效、归因、稳健性和报告层无需解释 vectorbt 对象 + +### Requirement: 每次 Skill 调用只交付一个场景结果 +本地研究流程 SHALL（必须）每次只接受一个策略项目、一个快照和一个场景配置，只编排一次项目执行、单份兼容结果校验、运行身份和证据收口，不接收候选数组、不循环多个场景，也不调用或包含策略分析 Skill（技能）。`scripts/research/local_quant_research/` 和策略项目不得导入绩效、归因、稳健性、压力、证据矩阵、报告或推荐算法。 + +#### Scenario: 成功交付单场景结果 +- **WHEN** 调用者提交一个完整场景配置且项目执行与统一契约校验通过 +- **THEN** 本地研究流程在 `.local/quant-research///backtests//` 固化一份按聚宽内部目录结构保存的本地回测结果，记录场景身份、来源权限、代码/参数/行情摘要和结果摘要，并以 `next_action=return_to_caller` 停止 + +#### Scenario: 拒绝批量候选输入 +- **WHEN** 调用者把冻结基线、挑战数组、参数网格或稳健性场景列表作为一次 Skill 输入 +- **THEN** 本地研究流程拒绝批量请求并要求调用者拆成单场景调用；Skill 不生成 `candidate-strategies.json`、`local-research-manifest.json`、排名或聚合结果 + +#### Scenario: 不启动策略分析 +- **WHEN** 单场景兼容结果已经完整 +- **THEN** 本地研究流程不计算绩效、Alpha/Beta（超额收益/市场暴露）、归因、稳健性、压力或推荐，不生成完整策略分析报告，也不调用后续策略分析 Skill + +#### Scenario: 依赖方向检查 +- **WHEN** 扫描本地研究运行器和策略项目的生产导入 +- **THEN** 两者只依赖以聚宽现有结果数据为基准的读取契约，不导入 `quant_analysis` 的指标、归因、稳健性、压力、证据矩阵或报告实现 + ### Requirement: 仓库运行与能力复用边界 系统 MUST（必须）使用项目 `.venv`（虚拟环境）运行本地 Python（编程语言）入口，并复用既有聚宽认证和归档能力，不得保存或打印账号、密码、Token（访问令牌）或 Cookie（浏览器凭证）。 @@ -138,7 +192,7 @@ #### Scenario: 共享行情中心回归 - **WHEN** 运行行情中心自动测试 -- **THEN** 测试覆盖不可变批次导入、相同内容去重、追加新标的、旧快照复算不变、冲突重叠拒绝、字段能力、快照摘要及 CSV 到内存 DuckDB 一致性，并确认未生成持久 DuckDB 文件 +- **THEN** 测试覆盖 CSV 暂存导入、Parquet 不可变批次、逻辑内容去重、追加新标的、旧快照复算不变、冲突重叠拒绝、字段能力、快照摘要、Parquet 到内存 DuckDB 一致性及暂存清理，并确认未生成持久 DuckDB 文件 #### Scenario: 非海龟完整 E2E - **WHEN** 从 Skill 用户入口使用非海龟最小项目适配器和固定日线夹具运行 @@ -146,7 +200,7 @@ #### Scenario: 用户入口完整回归 - **WHEN** 从 Skill 文档公开的用户入口启动离线研究夹具 -- **THEN** 流程实际贯通快照引用、CSV 校验、内存 DuckDB 查询、项目进程、输出验证和三态收口，而不是以若干孤立单元测试代替 +- **THEN** 流程实际贯通 CSV 暂存导入、Parquet 固化、快照引用、内存 DuckDB 查询、项目进程、输出验证和三态收口，而不是以若干孤立单元测试代替 #### Scenario: 公开仓库安全扫描 - **WHEN** 运行仓库安全检查 diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/specs/standard-strategy-analysis-data/spec.md b/openspec/changes/build-turtle-etf-local-research-workflow/specs/standard-strategy-analysis-data/spec.md new file mode 100644 index 0000000..ddc47ce --- /dev/null +++ b/openspec/changes/build-turtle-etf-local-research-workflow/specs/standard-strategy-analysis-data/spec.md @@ -0,0 +1,138 @@ +## ADDED Requirements + +### Requirement: 以聚宽现有回测归档作为标准物理基准 +系统 SHALL（必须）直接接受 `joinquant/strategies//backtests//` 现有目录、原 `manifest.json`、`params.json`、`code.py` 和 `data/` 文件。聚宽回测流程、归档流程和既有回测目录 SHALL NOT（不得）新增、改名、复制、重写或转换任何文件。 + +#### Scenario: 聚宽归档零改动直读 +- **WHEN** 现有聚宽回测 `manifest.json` 的 `schema_version=1`、`object.kind=backtest`、`object.status=done` 且 `gate.status=pass` +- **THEN** 读取器只按现有聚宽 `manifest.schema.json` 验证并直接建立分析视图，不生成 `analysis-data-manifest.json`、适配目录、八表副本或回写字段 + +#### Scenario: 聚宽清单仍是唯一权威入口 +- **WHEN** 读取器处理聚宽回测结果 +- **THEN** 它只按原清单定位文件、摘要、行数、合法空表、数据集状态和来源身份，不扫描“看起来最新”的文件；读取前后聚宽目录摘要必须一致 + +### Requirement: 本地回测使用独立且可执行的清单 Schema +系统 SHALL（必须）提供 `local-backtest-manifest.schema.json`。本地结果从 `.local/quant-research///backtests//` 向内尽量镜像聚宽现有回测目录，但 SHALL（必须）使用 `schema_version=local-backtest/1`、`object.kind=local_backtest`、`source.kind=local_vectorbt` 和 `authority=local_research` 明确来源，不能声称符合聚宽远端归档 Schema。 + +#### Scenario: 本地清单必需证据 +- **WHEN** 本地 vectorbt（向量化回测框架）完成一个场景 +- **THEN** 本地 Schema 顶层 `additionalProperties=false`，必需字段为 `schema_version`、`object`、`source`、`authority`、`run`、`code`、`params`、`datasets`、`performance`、`gate`；其中必须记录 `local_id`、`status`、`run_id`、`scenario_id`、`snapshot_id`、引擎/适配器版本、公司行动核算模式及精度边界、代码路径/字节数/SHA256、当前参数路径/版本路径/字节数/SHA256、数据集状态/文件摘要/行数/时间范围/空表、性能证据路径/字节数/SHA256，以及 `gate.status=pass|fail` 与 `exceptions`；`code.py`、`params.json`、`params_versions/.json`、`performance.json` 与 `data/` 的位置沿用同一回测根目录 + +#### Scenario: 本地公司行动近似口径显式可见 +- **WHEN** 本地 vectorbt 不能原生同步拆分后的真实份额和派息日现金，而改用连续总回报经济价格 +- **THEN** `source.accounting` 必须记录 `corporate_action_mode=point_in_time_total_return_approximation`、`continuity_factor_basis=raw_previous_close_over_current_pre_close`、`corporate_action_metadata_timing=audit_only_may_be_retrospective`、`price_basis=continuous_economic_price`、`quantity_basis=economic_units`、`cash_dividend_mode=implicit_reinvestment_on_ex_date`、`pay_date_cash_supported=false`、`exact_joinquant_reconciliation=false`、公司行动数据摘要和口径版本；逐事件归因还必须记录 `evidence_timing=point_in_time|retrospective_reconciliation`。读取器必须原样暴露这些限制，未知模式或缺少必需声明时拒绝来源 + +#### Scenario: 性能证据可独立复核 +- **WHEN** 本地结果准备通过 `gate.status=pass` +- **THEN** `performance.json` 必须记录环境与依赖摘要、准备后输入摘要、代码/参数/场景摘要、`cold_seconds`、`warm_seconds`、两次规范化结果摘要、摘要一致性结论、性能上限和暂存清理结果；计时统一从准备后输入进入项目执行后端开始，到项目声明的执行事实和必需扩展完成结构/摘要/勾稽校验时停止。停止计时后先删除预热副本和可丢弃暂存并确认清理，再把清理结果写入 `performance.json`、生成最终清单并校验全部摘要；二者属于完成门禁但不计入冷/热耗时。最后只原子发布已整理的权威结果目录，发布后不得依赖写入或清理；任一门禁失败不得发布完成目录 + +#### Scenario: 本地不伪造聚宽专属证据 +- **WHEN** 本地来源没有聚宽详情页、远端原始响应、围栏、官方摘要或官方风险结果 +- **THEN** 本地清单和目录不得包含伪造的聚宽 URL、`research_response`、`research_lineage`、`collection_fence`、`official_summary`、`raw/`、`risk.parquet` 或 `period_risks.parquet` + +#### Scenario: 读取器严格选择 Schema +- **WHEN** 读取器打开来源清单 +- **THEN** 整数 `schema_version=1` 只使用聚宽 Schema，字符串 `schema_version=local-backtest/1` 只使用本地 Schema；未知版本、混合身份、越权字段或校验失败直接拒绝，不得尝试回退另一契约 + +### Requirement: 六类逻辑模型与两种物理形态明确分离 +统一分析模型 SHALL（必须）沿用聚宽现有 `results`、`balances`、`positions`、`orders`、`risk`、`period_risks` 六类名称。聚宽来源物理提供六类数据；本地来源只物理提供四类共同执行事实，并在清单中声明两类官方风险参考缺失。读取器 SHALL（必须）为两种来源建立相同的六类逻辑视图。 + +#### Scenario: 本地只落四类执行事实 +- **WHEN** 本地结果通过门禁 +- **THEN** `results`、`balances`、`positions`、`orders` 必须为 `required=true`、`status=complete`；`risk` 与 `period_risks` 必须为 `required=false`、`status=missing_at_source`、`reason=computed_by_strategy_analysis`，本地研究不得计算 Alpha/Beta（超额收益/市场暴露）、Sharpe（夏普比率）、回撤或分期风险来填满六类物理文件 + +#### Scenario: 收益与权益沿用聚宽字段 +- **WHEN** 分析策略收益、权益、现金和仓位占用 +- **THEN** 使用 `results.time`、`results.returns`、`balances.time`、`total_value`、`net_value`、`cash` 和 `aval_cash`，不得固化第二份 `returns.parquet` 或 `equity.parquet`；本地来源存在研究级公司行动近似时，分析必须把收益和权益解释为连续经济口径，把现金、仓位和订单解释为近似路径，不能与聚宽逐日账户做精确差异归因 + +#### Scenario: 本地 results 保留聚宽单基准字段但不伪造值 +- **WHEN** 本地适配器生成 `data/results.parquet` +- **THEN** 文件固定包含 `time:string`、`returns:double`、`benchmark_returns:double nullable`；`returns` 表示从初始资金起算的累计净收益，`benchmark_returns` 全列为空值且物理类型仍为 `double`，本地清单记录 `source_benchmark_returns.status=missing_at_source`、`reason=independent_benchmark_set` 和空值行数；不得填零或任选一个独立基准冒充聚宽单基准 + +#### Scenario: 累计收益只在查询期转换为单日收益 +- **WHEN** 统一分析需要把来源策略收益与双基准的单日收益对齐 +- **THEN** 读取器先把 `results.time` 规范化为 Asia/Shanghai（亚洲/上海）交易日，再按 `(1 + cumulative_return_t) / (1 + cumulative_return_t-1) - 1` 派生 `daily_returns`；首个样本只在累计收益为零时取单日收益零，否则标记缺少前值并从比较样本排除。分析不得把累计 `returns` 直接与基准单日 `returns` 比较 + +#### Scenario: 交易与持仓沿用聚宽字段 +- **WHEN** 分析持仓、成交、费用、滑点或完整往返交易 +- **THEN** 使用 `positions` 与 `orders` 的现有字段和时间语义；完整往返交易只作为查询期派生视图，不要求来源新增 `trades.parquet` + +#### Scenario: 官方风险只作来源参考 +- **WHEN** 分析聚宽回测风险结果 +- **THEN** 读取器暴露原 `risk` 与 `period_risks` 作为 `source_risk` 官方参考；策略分析仍从共同执行事实复算跨来源可比较指标，不覆盖或改写聚宽官方结果 + +#### Scenario: 合法空持仓和空订单 +- **WHEN** 清单把 `positions` 或 `orders` 标记为 `status=complete`、`rows=0`、`verified_empty=true` 且没有对应 Parquet（列式文件） +- **THEN** 读取器按对应来源 Schema 建立固定字段空内存视图，不补写空文件 + +#### Scenario: 聚宽合法缺失归因例外保持兼容 +- **WHEN** 既有聚宽回测 `gate.status=pass` 且唯一例外为 `attribution_log:missing_at_source` +- **THEN** 读取器必须把该例外视为合法的可选归因缺失并零改动打开来源；其他未知例外仍须拒绝，不能把任意门禁例外降级放行 + +#### Scenario: 兼容已观察物理类型差异 +- **WHEN** 既有聚宽归档只存在列顺序、`cancel_time` 空类型/字符串或风险数值整数/浮点差异 +- **THEN** 读取器只在 DuckDB（嵌入式分析数据库）内存视图按字段名和兼容类型规范化，不重写源 Parquet 或摘要 + +### Requirement: 双基准使用独立且不可变的分析输入契约 +系统 SHALL（必须）在 `.local/market-data/benchmark-sets//` 保存 `manifest.json` 与 `benchmark-returns.parquet`，且只包含 `CSI300_CNY_TOTAL_RETURN`（沪深300人民币总回报）和 `NASDAQ100_CNY_TOTAL_RETURN`（纳斯达克100人民币总回报）两个基准。基准集不属于来源回测目录，不要求聚宽归档改动。 + +#### Scenario: 基准身份与口径完整 +- **WHEN** 创建分析基准集 +- **THEN** 清单逐项记录 `benchmark_id`、人民币币种、总回报定义、美元兑人民币处理公式、实际日期范围、来源标识、底层快照、生成版本、行数和文件 SHA256（文件摘要）；Parquet 固定包含 `time`、`benchmark_id`、`returns`，`time` 是 Asia/Shanghai（亚洲/上海）交易日，`returns` 是小数形式的单日人民币总回报，唯一键为 `(time, benchmark_id)`；纳斯达克100人民币总回报按 `(1 + USD指数总回报) × (1 + USD/CNY变动) - 1` 计算 + +#### Scenario: 基准来源必须真实验证 +- **WHEN** 实施者准备导入或派生两个基准 +- **THEN** 必须先对配置来源做真实最小可行性验证并保存证据；来源身份、总回报、汇率或日期覆盖不能证明时以 `evidence_insufficient` 停止，不得使用 ETF 代理、零收益补齐、前向/后向填充或未声明降级；策略和两个基准只在共同有效交易日计算可比较指标，并在报告披露被排除日期与样本数 + +#### Scenario: 聚宽单基准只作官方参考 +- **WHEN** 聚宽 `results` 包含单列 `benchmark_returns` +- **THEN** 读取器把其累计收益序列暴露为 `source_benchmark_returns`；本地全空同名列则暴露为带 `missing_at_source` 状态的同一参考视图。除非来源清单能证明身份与口径完全一致，否则任何来源的该列都不能代替两条跨来源分析基准 + +### Requirement: 归档证据与分析输入分层 +现有聚宽 `official_summary`、`records`、`attribution_log`、日志、性能剖析和 `raw/` 文件 SHALL（必须）保持原归档职责，不得成为所有策略的强制核心分析表。 + +#### Scenario: 官方摘要不成为第二分析事实源 +- **WHEN** 聚宽归档包含 `data/official-summary.csv` +- **THEN** 它只作为页面口径交叉校验证据；确定性分析使用核心 Parquet 与独立基准集，读取器不复制或用该 CSV 恢复高精度数据 + +#### Scenario: 通用契约不强制统一归因扩展 +- **WHEN** 回测包含 `attribution_log-.parquet` +- **THEN** 策略分析可以读取它作为扩展证据，统一契约不得要求所有策略拥有相同归因字段；具体策略可以在自己的版本化项目契约中把该扩展声明为完成分析所必需 + +#### Scenario: 海龟单位证据为可选扩展 +- **WHEN** 海龟本地结果的归因扩展包含单位数、冻结 N、候选基础数量、实际成交价、共同止损、资产组比例、组合比例、现金比例和全量仓位再分配标记 +- **THEN** 通用策略分析可以派生最高计划损失比例、最高有效 N 风险单位、组合单位预算最高利用率和全量仓位再分配次数，但不得改变 `results`、`balances`、`positions`、`orders` 四类共同事实 +- **AND** 现有聚宽结果缺少该扩展时，上述海龟专用指标返回缺失或零，其他收益、风险、仓位和基准分析继续运行 + +### Requirement: 本地与聚宽共用同一分析读取入口 +系统 SHALL（必须）在 `scripts/research/analysis_data/` 提供清单选择、Schema 校验、六类逻辑视图、内存规范化和派生查询能力。策略分析只使用该入口，不直接消费 vectorbt 对象，也不为聚宽建立转换副本。 + +#### Scenario: 同一算法消费两种来源 +- **WHEN** 分析接收聚宽目录或本地单场景目录 +- **THEN** 它通过同一读取入口计算指标；来源差异只在 Schema 校验、核算精度元数据与缺失参考视图中处理，分析算法不读取 vectorbt 对象或按执行引擎复制算法。报告必须展示来源核算精度，研究级近似不得被描述为聚宽精确账户复核 + +#### Scenario: 主代理聚合多个单场景结果 +- **WHEN** 主 agent（代理）需要比较冻结基线、挑战或稳健性场景 +- **THEN** 它按已校验计划逐次调用单场景本地研究 Skill；调用前以 `preparation_id` 绑定分析计划、基准与每场景配置，调用后以完整来源登记显式绑定所有 `scenario_id -> run_id`，逐场景校验配置、证券集合、精确快照、代码和执行后端，再由准备身份与全部来源摘要派生不可变 `analysis_id`；不得扫描历史目录猜测来源或覆盖旧分析。本地 Skill 不读取分析计划、不接收候选数组、不循环方案、不生成聚合清单 + +#### Scenario: 独立资产扩展使用真实单场景矩阵 +- **WHEN** 资产扩展计划包含原 11 只基线、完整扩展、逐只删除、逐扩展切片删除和五个成本执行压力场景 +- **THEN** 每个计划项必须登记一个独立标准结果包，全部六只候选通过时来源总数为 16 +- **AND** 删除和成本执行结果必须来自真实本地回测，不得用贡献扣除或订单损益一阶调整代替 + +#### Scenario: 不同资产池使用可比较的精确快照 +- **WHEN** 来源登记包含不同证券集合的场景 +- **THEN** 分析读取每个场景自己的配置和完全匹配的快照，拒绝缺失、额外或未知证券 +- **AND** 所有场景共享相同不可变批次、截止日、字段、价格口径、代码和非资产池参数；重叠证券行情及公司行动摘要必须完全一致 + +#### Scenario: Vibe-Trading 分析能力安全降级 +- **WHEN** 主 agent 已收集所需单场景结果、双基准集和确定性分析证据 +- **THEN** 系统在本地 Skill 之外由确定性算法生成绑定来源摘要的完整报告与推荐；Vibe-Trading（氛围量化）只允许调用无已知缺陷的单体公开能力，禁止群体分析。没有安全单体入口时记录 `evidence_insufficient`，不得阻塞或替代确定性报告 + +#### Scenario: 群体分析结果无效 +- **WHEN** 误调用 Vibe `run_swarm`（运行群体分析）或收到任何群体分析结果 +- **THEN** `vibe-evidence.json` 必须把该调用标记为边界违规、`valid_evidence=false` 和 `excluded_from_conclusions=true`，报告、推荐、门槛和证据矩阵不得引用其内容 + +#### Scenario: Vibe 传输数据不形成第二事实源 +- **WHEN** 可用且安全的 Vibe 单体分析入口需要读取统一视图 +- **THEN** 独立分析按明确查询、字段和日期范围物化临时 CSV（逗号分隔文件），记录查询版本和摘要，确认读取后删除；分析身份始终引用原清单、Parquet 与基准集摘要 diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/specs/turtle-etf-local-research/spec.md b/openspec/changes/build-turtle-etf-local-research-workflow/specs/turtle-etf-local-research/spec.md index 0e68dd6..7558c53 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/specs/turtle-etf-local-research/spec.md +++ b/openspec/changes/build-turtle-etf-local-research-workflow/specs/turtle-etf-local-research/spec.md @@ -1,127 +1,197 @@ ## ADDED Requirements -### Requirement: 海龟项目内容与通用 Skill 隔离 -系统 SHALL（必须）先创建真实 JoinQuant（聚宽）策略空壳并同步为 `strategy-003`，仅建立身份且不启动正式回测；海龟 ETF（交易型开放式指数基金）的资产池、参数、交易规则、策略代码、研究配置和证据保存在 `joinquant/strategies/strategy-003/research/` 及对应本地运行证据中，通过通用配置契约接入流程。`strategy-001` 与 `strategy-002` 不得修改。 +**状态：已确认。基线摘要：11 只 ETF、6 个资产组、55/20/20、0.5N、2N、4/6/12、全量仓位再分配、180 秒。** -#### Scenario: 真实身份建立后创建项目目录 -- **WHEN** 聚宽策略空壳、聚宽详情页身份和本地索引已验证为唯一对应的 `strategy-003` -- **THEN** 系统建立 `strategy-003` 研究项目但不启动正式回测；身份缺失、冲突或占用时停止 +### Requirement: 海龟项目使用固定 11 ETF 基线 -#### Scenario: 通用 Skill 调用海龟项目 -- **WHEN** 海龟项目配置传入策略入口、共享 `snapshot_id`（快照标识）和输出路径 -- **THEN** 通用 Skill 只执行契约校验和编排，海龟项目模块独立计算交易规则并生成项目产物 +海龟项目 SHALL（必须）只从项目配置读取证券和分组。当前基线 SHALL 固定为 11 只 ETF 和 6 个资产组，不按动量、信号时间或代码顺序筛选突破标的。 -#### Scenario: 检查反向依赖 -- **WHEN** 检查 Skill 源码、参考和夹具 -- **THEN** 其中不得包含 11 只 ETF、55 日入场、N 值、加仓、止损或海龟验收门槛的硬编码 +#### Scenario: 固定证券与分组 +- **WHEN** 项目加载 `baseline.json` +- **THEN** 证券必须为 `510300.XSHG`、`512100.XSHG`、`512480.XSHG`、`159819.XSHE`、`516160.XSHG`、`513100.XSHG`、`513180.XSHG`、`515180.XSHG`、`516080.XSHG`、`518880.XSHG`、`511010.XSHG` +- **AND** 分组必须与项目配置逐项一致，17 ETF 扩展不得进入当前基线 -#### Scenario: 策略不得拥有行情副本 -- **WHEN** 检查 `strategy-003` 项目与运行证据 -- **THEN** 项目只引用共享 `snapshot_id` 及摘要，不在策略目录复制权威 CSV、快照批次或持久 DuckDB（嵌入式分析数据库） +### Requirement: 行情使用未复权事实和连续经济执行口径 -### Requirement: 聚宽权威行情快照 -海龟项目 SHALL（必须）通过共享行情中心引用由 JoinQuant（聚宽）研究环境取得的方案指定 11 只 ETF 日线行情。每只 ETF 从自身首个可用完整交易日导出到运行时显式指定的 `snapshot_end_date`（快照截止日）；2015-01-01 之前数据只作指标预热。字段固定为 `date`、`security`、`open`、`high`、`low`、`close`、`pre_close`、`volume`、`money`、`factor`、`paused`、`high_limit`、`low_limit`，静态证券信息保存在批次清单。 +项目 SHALL 使用共享 Parquet（列式文件）快照中的原始未复权 OHLC、`pre_close`、`factor`、停牌和涨跌停事实，并在内存中使用应用日可见原始行情派生连续经济价格。成交额和流动性 SHALL NOT（不得）进入海龟策略输入或订单规则。 -#### Scenario: 接受完整权威快照 -- **WHEN** 共享快照覆盖项目清单中的 11 只 ETF、固定 13 个字段、各自实际起止日、未复权口径和完整身份摘要 -- **THEN** 项目接受该 `snapshot_id` 作为本次本地研究的唯一行情引用，不复制行情 +#### Scenario: 未复权行情接入 +- **WHEN** 项目准备一个场景 +- **THEN** `fq=null`、`skip_paused=false`、`use_real_price=false`，行情从共享快照按精确证券集合读取 +- **AND** DuckDB（嵌入式分析数据库）只在内存查询，不生成持久数据库 -#### Scenario: 快照缺失或来源不明确 -- **WHEN** 任一证券或必需字段缺失、价格口径未记录、数据截止日不明确或来源无法追溯 -- **THEN** 项目输出 `evidence_insufficient`，不得用代理数据、旧快照或默认口径补齐 +#### Scenario: 公司行动连续性 +- **WHEN** 有效公司行动授权 `pre_close` 与上一交易日原始 `close` 的价格基准变化 +- **THEN** 项目从实际应用日起按 `raw_previous_close / current_pre_close` 更新连续因子，统一派生连续经济 OHLC 和经济单位 +- **AND** 晚公布元数据只标为事后核对，不得回写未来知识或声称精确复现真实份额、派息日现金及聚宽账户 -#### Scenario: 新 ETF 冷启动 -- **WHEN** 某只 ETF 尚未具有 60 个完整、有效且日期对齐的收益样本 -- **THEN** 项目保留其行情和审计，但禁止该 ETF 新建仓或加仓，不把全局运行标记为失败 +#### Scenario: 无策略层流动性规则 +- **WHEN** 成交额很低、缺失或超过任意假设比例 +- **THEN** 海龟策略不得缩小、拒绝或延迟入场、加仓、退出和止损；不存在单笔成交额 1% 规则 -### Requirement: 聚宽导出与价格口径 -共享聚宽日线导出器 SHALL（必须）在聚宽研究内核直接调用注入的 `get_price`、`write_file` 和 `read_file` 接口，使用 `fq=None` 与 `skip_paused=False`，按固定字段和排序生成精确 `market-data.csv`；实现 SHALL（必须）兼容 Pandas（数据处理库）0.23.4 的 `line_terminator`，并在查询层把 `paused` 规范化为布尔值。 +### Requirement: 信号固定为 55/20/20 海龟规则 -#### Scenario: 未复权信号数据 -- **WHEN** 导出海龟项目日线行情 -- **THEN** CSV 保存实际未复权 OHLCV（开高低收量）及 `factor`（复权因子），本地研究不生成或使用复权价格序列 +项目 SHALL 使用此前 55 日最高价突破入场、此前 20 日最低价退出和 20 日递推 N。所有信号 SHALL 在收盘检查并错位到下一交易日开盘执行。 -#### Scenario: 聚宽策略价格模式 -- **WHEN** 生成供后续聚宽功能校验使用的海龟策略代码 -- **THEN** 信号行情显式使用 `fq=None` 且策略设置 `use_real_price=False`，并把聚宽在该模式下使用固定基准日前复权撮合价记录为平台限制，不宣称撮合使用未复权实际价 +#### Scenario: 收盘突破入场 +- **WHEN** 信号日收盘价严格高于不含当日的此前 55 个交易日盘中最高价 +- **THEN** 项目生成一个入场单位候选；盘中突破但收盘未确认不得入场 -#### Scenario: 导出传输与清理 -- **WHEN** 聚宽端临时文件已传输到本地 -- **THEN** 系统先验证本地字节 SHA256 与远端回读一致，再删除远端临时文件；无法确认删除时本次运行输出 `failed` +#### Scenario: 趋势退出 +- **WHEN** 信号日收盘价严格低于不含当日的此前 20 个交易日盘中最低价 +- **THEN** 项目冻结完整退出动作并在下一交易日开盘尝试卖出全部持仓 -### Requirement: 海龟规则的确定性验证 -海龟项目 MUST（必须）以项目代码和固定夹具验证已确认基线：此前 55 日最高价收盘突破入场、20 日 N 值、0.5% 初始风险单位、固定 0.5N 加仓档位、只上移的 2N 共同止损、此前 20 日最低价收盘跌破退出，以及资金、流动性、单 ETF、资产组、组合风险和目标波动率约束。 +#### Scenario: 无前视成交 +- **WHEN** T 日收盘形成入场、加仓、退出或止损信号 +- **THEN** 订单只能使用 T 日及以前信息，并在 T+1 日开盘执行；基线不增加额外延迟 -#### Scenario: 入场与 N 值计算 -- **WHEN** 固定行情在收盘价突破不含当日的此前 55 日盘中最高价，且具有足够有效 TR（真实波幅）与协方差样本 -- **THEN** 项目按确认公式计算信号日 N、理论单位和次日候选订单；盘中突破但收盘未突破不得产生订单 +### Requirement: 每个逻辑单位冻结自己的 N 风险 -#### Scenario: 加仓与共同止损 -- **WHEN** 已成交仓位在后续收盘跨过一个或多个固定 0.5N 档位 -- **THEN** 项目每天至多申请一次受全部预算约束的加仓，只有实际成交才能推动共同止损，且共同止损不得下移 +项目 SHALL 为每只 ETF 保存最多 4 个逻辑单位。单位候选基础数量 MUST（必须）为 `floor_to_100_shares(signal_equity × 1% / signal_n)`，且只有真实买入成交才建立单位。 -#### Scenario: 退出与故障安全 -- **WHEN** 收盘价跌破不含当日的此前 20 日盘中最低价、触发保护性止损，或数据缺失导致协方差无法可靠更新 -- **THEN** 前两种条件生成全部批次退出；数据故障暂停新增风险但保留能够成交的退出和风险减仓 +#### Scenario: 入场单位建立 +- **WHEN** 入场候选经统一缩放后产生至少一个整手净新增买入并真实成交 +- **THEN** 项目冻结该单位的信号日 N、基础数量和实际成交价，单位数从 0 变为 1 -#### Scenario: 风险或整手预算不足 -- **WHEN** 候选订单超过任一项目风险、资金、流动性或目标波动率上限，或裁剪后不足一个交易整手 -- **THEN** 项目按确认规则缩小或跳过订单，不使用融资、透支或跨 ETF 锁定利润补足预算 +#### Scenario: 候选未成交 +- **WHEN** 停牌、涨停、现金缩放、整手取整或官方拒单使候选没有净新增成交 +- **THEN** 项目不得建立单位、推进加仓档位或更新共同止损，并在后续交易日重新检查 -#### Scenario: 持仓风险输入缺失 -- **WHEN** 任一持仓 ETF 缺少可用价格或无法形成可靠协方差输入 -- **THEN** 项目停止所有新增风险，不以零值、陈旧协方差或虚构成交继续；其他可交易 ETF 仍允许退出和强制减仓 +### Requirement: 加仓使用固定 0.5N 档位 -### Requirement: 完整本地交易主流程 -海龟项目 SHALL（必须）用固定行情和成交夹具完整执行“收盘信号、次日订单、成交回填、持仓与风险状态、审计输出”，并把同日订单优先级和预算分配作为显式项目规则验证。 +项目 SHALL 以首次实际成交价和首次信号日 N 冻结 `0.5N`、`1.0N`、`1.5N` 三个后续档位。每只 ETF 每日最多新增一个单位，最多 4 个单位。 -#### Scenario: 入场主流程贯通 -- **WHEN** 固定行情在交易日收盘产生有效入场信号且次日具备可成交开盘价 -- **THEN** 项目在下一交易日生成订单、回填实际成交、建立批次与共同止损，并在审计记录中关联信号、订单、成交和风险状态 +#### Scenario: 一天跨越多个档位 +- **WHEN** 收盘价一次越过多个尚未完成的固定档位 +- **THEN** 下一交易日最多建立一个新单位；其余档位必须在后续交易日重新满足后逐日处理 -#### Scenario: 多类订单同日出现 -- **WHEN** 同一开盘同时存在退出、风险减仓、建仓或加仓候选 -- **THEN** 项目依次处理全仓退出、强制风险减仓、同级的新建仓与加仓；同一 ETF 当日出现退出时取消其全部买入候选 +#### Scenario: 四单位后停止加仓 +- **WHEN** 单标的已有 4 个逻辑单位 +- **THEN** 项目不得再生成加仓候选，但继续每日检查止损和 20 日趋势退出 -#### Scenario: A1 共享预算分配 -- **WHEN** 多个有效新建仓或加仓候选争用有限资金与风险预算 -- **THEN** 每个候选先生成最多一个 U0（标准单位）的请求量，系统使用同一完成比例缩减全部可行请求，直到资金、单 ETF、资产组、组合计划风险和目标波动率约束全部满足；被自身或所属组上限卡住的未用预算可流向其他仍可增仓候选 +### Requirement: 保护性止损按逐单位冻结 N 只上移 -#### Scenario: A1 整手余额分配 -- **WHEN** 等比缩放后的请求量不是完整交易手数且仍有剩余预算 -- **THEN** 系统先向下取整，再按小数余额从大到小逐手补分；每补一手重新检查全部硬门槛，完全同分时按 ETF 代码升序确定，候选输入顺序不得改变结果 +每个真实成交的入场或加仓单位 SHALL 生成 `actual_fill_price - 2 × frozen_signal_n` 候选止损，共同止损 SHALL 取历史候选止损最大值。 -#### Scenario: 退出主流程贯通 -- **WHEN** 既有持仓产生有效趋势退出或保护性止损条件并在次日成交 -- **THEN** 项目关闭该 ETF 全部批次、更新现金与组合风险状态，并输出可追溯审计记录 +#### Scenario: 新单位提高止损 +- **WHEN** 新单位候选止损高于既有共同止损 +- **THEN** 共同止损上移到新值；若候选更低则保持原值 -### Requirement: 探索性粗筛及结果边界 -海龟项目 SHALL（必须）允许 Vibe-Trading（AI 研究助理）做方向性粗筛、归因和报告，并由确定性项目计算复核事件与风险；所有产物必须关联快照、代码和配置摘要并明确标为非正式结果。 +#### Scenario: 每日 N 不跟踪止损 +- **WHEN** 建仓后每日 N 改变但没有新单位成交 +- **THEN** 共同止损不得重算或下移 -#### Scenario: 生成可复算研究报告 -- **WHEN** 数据桥、策略规则和完整本地主流程均通过验证 -- **THEN** 项目生成 `research-report.md`，包含方法、输入身份、事件与交易结果、仓位分布、现金占比、留现原因、资产组与组合风险使用率、限制和产物摘要，且可按证据索引复算 +#### Scenario: 保护性完整退出 +- **WHEN** 收盘价小于或等于共同止损 +- **THEN** 项目在下一交易日开盘尝试完整退出；只有全部实际持仓退出后才清除单位、档位和止损状态 -#### Scenario: 生成研究建议 -- **WHEN** 完整本地研究运行结束 -- **THEN** 项目生成 `conclusion.json`，研究建议且只能为 `proceed_to_joinquant`（进入聚宽回测）、`revise_and_reassess`（修订后再评估）或 `stop_evidence_insufficient`（证据不足而停止）之一，并列出确定性理由、阻断项及“不是正式回测或最终验收结论”的声明 +### Requirement: 风险预算使用 4/6/12 N 单位 -#### Scenario: 生成固定候选策略包 -- **WHEN** 本地研究报告和研究建议均通过输出校验 -- **THEN** 项目生成 `candidate-strategies.json`，恰好包含一项冻结基线和 40/60 日入场、1.5N/2.5N 止损、120 日/30 日半衰期 EWMA（指数加权移动平均）协方差六项预设单项挑战配置；七项共用同一代码摘要和 `snapshot_id` +项目 SHALL 使用单标的最多 4 个逻辑单位、资产组最多 6 个有效单位、组合最多 12 个有效单位。资产组比例 SHALL 先计算，组合比例 SHALL 再对组缩放后的有效单位计算。 -#### Scenario: 禁止本地结果删选候选 -- **WHEN** 本地粗筛显示某候选收益更高或更低 -- **THEN** 项目不得按本地收益排名删除候选、替换冻结基线或产生未预设参数，只记录方向性证据与风险提示 +#### Scenario: 资产组缩放 +- **WHEN** 某资产组逻辑单位总数超过 6 +- **THEN** 该组全部标的使用同一个 `6 / group_units` 比例，其他未超限组保持 1 -#### Scenario: 已知组合优化器缺陷仍存在 -- **WHEN** 当前 Vibe-Trading 版本尚未包含已知前视偏差修复 -- **THEN** 项目跳过受影响的组合优化器并记录原因，不用其结果筛选参数，也不阻塞其他方向性研究 +#### Scenario: 组合缩放 +- **WHEN** 组缩放后的组合有效单位超过 12 +- **THEN** 全部标的再共同乘 `12 / effective_units` -#### Scenario: 防止本地结果越界 -- **WHEN** 本地粗筛或确定性回归产生收益、交易或风险统计 -- **THEN** 报告必须声明其不是聚宽正式回测、模拟交易或验收结论，并不得据此修改已确认基线参数 +#### Scenario: 输入顺序不影响结果 +- **WHEN** 同一单位簿以不同证券列顺序输入 +- **THEN** 还原证券顺序后的组比例、组合比例、现金比例和目标数量必须一致 -#### Scenario: 三类输出完成门禁 -- **WHEN** `research-report.md`、`conclusion.json` 或 `candidate-strategies.json` 任一缺失、结构无效或摘要不匹配 -- **THEN** 本次运行不得输出 `complete` +### Requirement: 组合事件执行全量仓位再分配 + +有效入场、有效加仓、保护性止损或趋势退出 SHALL 触发一次全组合目标重算。无事件日 SHALL NOT 调仓。 + +#### Scenario: 后到趋势挤占已有趋势 +- **WHEN** 已有趋势占用全部组合单位预算，另一 ETF 后续形成有效突破 +- **THEN** 项目把新候选加入临时单位簿并统一缩放，后到趋势获得目标仓位，已有趋势同比例让出实际持仓；信号先后不形成优先权 + +#### Scenario: 候选稳定重算 +- **WHEN** 某候选在统一缩放和整手取整后不能形成至少一个整手净新增买入 +- **THEN** 项目移除该候选并重新计算全部目标，直到候选集合稳定；被移除候选不建立单位 + +#### Scenario: 统一现金比例 +- **WHEN** 4/6/12 缩放后的目标需要超过可用现金 +- **THEN** 项目将预计卖出净收入、买入价、滑点和每笔佣金纳入计算，在 `[0,1]` 上求全部可调整目标共同的最大可行现金比例，按 100 股向下取整 +- **AND** 不得使用余额补仓、最大余数分配、融资或按代码顺序买到现金耗尽 + +### Requirement: 动作顺序和状态隔离固定 + +同一组合事件 SHALL 按“完整退出、再分配卖出、入场或加仓、再分配买入”执行。再分配动作 SHALL 只改变实际持仓。 + +#### Scenario: 卖出释放现金 +- **WHEN** 同日存在退出、超配卖出和新增买入 +- **THEN** 完整退出与再分配卖出必须先于所有买入，使实际卖出释放的现金可用于后续订单 + +#### Scenario: 再分配不改变海龟状态 +- **WHEN** 再分配买卖真实成交 +- **THEN** 单位数、逐单位冻结 N、基础数量、固定加仓档位和共同止损均不得改变 + +#### Scenario: 同标的退出覆盖买入 +- **WHEN** 同一 ETF 同日同时满足退出和入场或加仓条件 +- **THEN** 只保留完整退出，取消该 ETF 买入候选 + +### Requirement: 市场不可交易时失败安全 + +项目 SHALL 根据执行日开盘价、停牌、涨停和跌停决定订单可交易性，状态 SHALL 只按真实成交更新。 + +#### Scenario: 买入不可交易 +- **WHEN** 执行日停牌、无有效开盘价或开盘价达到涨停 +- **THEN** 买入不得成交，候选单位不得建立 + +#### Scenario: 卖出不可交易 +- **WHEN** 执行日停牌、无有效开盘价或开盘价达到跌停 +- **THEN** 卖出不得成交，持仓和全部单位状态保持不变，后续交易日继续检查 + +#### Scenario: 官方拒单 +- **WHEN** vectorbt 官方订单结果为拒绝或零成交 +- **THEN** 项目记录拒单原因，单位、档位和止损不得推进 + +### Requirement: 使用 vectorbt 官方生命周期作为唯一生产本地执行路径 + +项目 SHALL 使用 `Portfolio.from_order_func()`、一个共享现金组和 Numba 回调执行基线。项目 SHALL NOT 另建订单、现金、持仓和权益引擎，也 SHALL NOT 保留旧执行兼容层。 + +#### Scenario: 官方回调可编译 +- **WHEN** 小型固定夹具运行项目 +- **THEN** 预模拟、分段规划、订单和成交后回调必须以 Numba `nopython`（无 Python 模式）编译，并由 vectorbt 官方模拟函数调用 + +#### Scenario: 额外延迟保持冻结动作 +- **WHEN** 研究配置显式增加一个或多个交易日延迟 +- **THEN** 项目冻结原动作、目标、原因和 N，延迟日只按开盘、费用、现金、持仓和可交易性机械执行；再分配动作仍不得改变海龟单位状态 + +### Requirement: 旧交易控制从生产路径物理删除 + +项目 SHALL NOT 在配置、输入、回调、结果适配或分析订单路径中保留资金仓位上限、计划风险上限、交易用协方差、最低对齐样本、目标波动率、强制波动率减仓或旧新增订单分配。 + +#### Scenario: 旧字段被拒绝 +- **WHEN** 风险配置包含任一旧字段 +- **THEN** 项目必须以明确错误拒绝，不得用极大值、`null`、默认值或回退路径继续 + +### Requirement: 结果包保持标准四表并增加海龟归因 + +项目 SHALL 输出聚宽口径兼容的 `results`、`balances`、`positions`、`orders` 四类共同事实，海龟专用信息 SHALL 只写入归因扩展。 + +#### Scenario: 再分配证据完整 +- **WHEN** 发生入场、加仓或全量仓位再分配 +- **THEN** 归因至少记录单位数、候选基础数量、冻结 N、实际成交价、共同止损、资产组比例、组合比例、现金比例、有效风险单位和 `redistribution_state_changed=false` 证据 + +#### Scenario: 聚宽结果无需海龟扩展 +- **WHEN** 独立策略分析读取没有海龟归因的现有聚宽结果 +- **THEN** 通用收益和实际暴露分析仍可运行，海龟单位指标返回缺失或零，不要求聚宽结果改动 + +### Requirement: 单场景性能与完整入口验收 + +项目 SHALL 从公开本地研究入口完成一个场景，冷启动和预热 SHALL 分别计时并都不超过 180 秒，规范化结果摘要 SHALL 一致。 + +#### Scenario: 真实 11 ETF 新基线 +- **WHEN** 主 agent 使用正式配置调用一次真实 11 ETF 基线 +- **THEN** 运行必须生成一个新的不可变结果包、清理预热副本和暂存、记录 `performance.json`，并返回 `next_action=return_to_caller` +- **AND** 本次不得运行旧方案对照、17 ETF 扩展、7 个场景或稳健性矩阵 + +#### Scenario: 完整本地研究报告 +- **WHEN** 新基线结果完成 +- **THEN** 独立分析从标准结果包计算收益、回撤、双基准 Alpha/Beta（阿尔法/贝塔）、仓位、计划损失、有效 N 单位、再分配和归因，生成明确推荐并等待人工确认 +- **AND** 报告必须声明本次未运行稳健性矩阵，结果不代替聚宽正式裁决 diff --git a/openspec/changes/build-turtle-etf-local-research-workflow/tasks.md b/openspec/changes/build-turtle-etf-local-research-workflow/tasks.md index 8cffc4e..07cae07 100644 --- a/openspec/changes/build-turtle-etf-local-research-workflow/tasks.md +++ b/openspec/changes/build-turtle-etf-local-research-workflow/tasks.md @@ -1,50 +1,56 @@ -## 1. 锁定项目契约 +## 1. 通用本地研究流程与行情中心 -- [x] 1.1 创建真实聚宽策略空壳并同步为 `strategy-003`，验证远端详情页、本地索引和项目目录唯一对应；只建立身份、不启动正式回测，且不修改 `strategy-001`、`strategy-002` -- [x] 1.2 固化已验证的聚宽日线导出契约：13 个字段、`fq=None`、`skip_paused=False`、未复权价格加 `factor`（复权因子）、每只 ETF 自身首个完整交易日至显式 `snapshot_end_date`（快照截止日）、内置 API（接口）、Pandas 0.23.4 写法和 `paused` 布尔规范化 -- [x] 1.3 把“全仓退出 → 强制风险减仓 → A1 建仓/加仓”顺序、同一 ETF 退出取消买入、同一完成比例、整手余额逐手分配和 ETF 代码同分规则写成固定验收夹具 +- [x] 1.1 创建一次只执行一个项目、一个快照和一个场景的 `run-local-quant-research` Skill（技能），拒绝批量候选和场景矩阵。 +- [x] 1.2 建立 `.local/market-data/` 不可变 Parquet（列式文件）批次、快照引用、SHA256（文件摘要）校验和内存 DuckDB（嵌入式分析数据库）查询。 +- [x] 1.3 验证真实聚宽未复权行情与公司行动导出，远端和本地 CSV（逗号分隔文件）暂存均在转换校验后删除。 +- [x] 1.4 实现按应用日可见原始行情派生连续经济价格与经济单位的研究级公司行动近似，并记录时点可知/事后核对边界。 +- [x] 1.5 从海龟策略输入和订单规则中删除全部成交额、流动性门槛与单笔成交额占比。 -## 2. 以 TDD（测试驱动开发）建立通用 Skill 与编排入口 +## 2. 标准分析数据包 -- [x] 2.1 先添加失败的 Skill（技能）布局和公开入口测试，覆盖名称、触发描述、`agents/openai.yaml`、单一 CLI（命令行接口）和 `.claude/skills/` 兼容链接 -- [x] 2.2 使用项目 `.venv`（虚拟环境）调用 `init_skill.py` 初始化 `.agents/skills/run-local-quant-research/`，补齐最小编排说明和兼容链接，使 2.1 测试及 `quick_validate.py` 通过 -- [x] 2.3 先添加失败的 JSON（结构化清单）配置测试，覆盖共享 `snapshot_id`、参数数组、仓库内路径、必需输出门禁和 `complete`、`evidence_insufficient`、`failed` 唯一状态 -- [x] 2.4 实现配置校验与固定阶段编排，使 2.3 测试通过，并证明 Skill 只调用通用脚本和项目入口、不解释策略字段 -- [x] 2.5 先添加失败的 `run_id`、原子固化、不可变清单、重复运行、失败重试、篡改和同输入不同输出测试，再实现证据索引与幂等复用，使全部测试通过 -- [x] 2.6 添加环境与安全边界测试并实现对应门禁，验证只使用项目 `.venv`、不回退系统 Python、不静默安装依赖、不读取或打印凭证 +- [x] 2.1 直接读取聚宽现有 `schema_version=1` 归档，不要求改名、复制、转换或回写。 +- [x] 2.2 建立独立 `local-backtest/1` 清单、本地四类共同执行事实和 `risk`/`period_risks` 来源缺失声明。 +- [x] 2.3 建立统一内存视图、累计收益转单日收益、持仓/订单派生查询和双基准契约。 +- [x] 2.4 保持本地结果目录、`results`、`balances`、`positions`、`orders` 尽量对齐聚宽现有结构，并以海龟归因扩展保存策略专用证据。 -## 3. 以 TDD 实现共享日线行情中心与 Data Bridge（数据桥） +## 3. vectorbt 官方本地执行路径 -- [x] 3.1 先添加失败的不可变批次测试，覆盖 `.local/market-data/batches//`、精确原始 CSV（逗号分隔文件）、清单、验证结果、来源身份、字段、日期和文件 SHA256（文件摘要），再实现批次导入与校验 -- [x] 3.2 先添加失败的内容去重、冲突重叠拒绝、新标的追加和旧批次不变测试，再实现不可变批次选择规则 -- [x] 3.3 先添加失败的 `.local/market-data/snapshots/.json` 测试，覆盖多批次引用、证券/日期/字段/来源/价格口径、旧快照复算和缺失证据，再实现快照创建与验证 -- [x] 3.4 先添加失败的权威 CSV 与内存 DuckDB（嵌入式分析数据库）视图同源测试，覆盖类型、空值、排序、`paused` 布尔类型、行差异和无持久 `.duckdb` 副本，再实现查询与规范化内容摘要 -- [x] 3.5 实现共享聚宽日线导出器和显式导出请求，使任何策略可提供证券与字段需求而不依赖海龟规则；验证远端回读、本地字节摘要和临时文件清理失败门禁 +- [x] 3.1 固定 vectorbt 1.1.0、Numba 0.66.0、NumPy 2.4.6 和 Pandas 3.0.3 的项目运行身份。 +- [x] 3.2 使用 `Portfolio.from_order_func()`、共享现金组和 Numba（即时编译）回调替换旧自研逐日执行路径，不保留双引擎或兼容入口。 +- [x] 3.3 使用冻结订单的 `Portfolio.from_orders()` 支持额外延迟研究，基线仍是收盘检查、次日开盘成交。 +- [x] 3.4 逐项验证订单、费用、现金、持仓、权益和公司行动连续经济口径的跨表勾稽。 -## 4. 以 TDD 实现海龟策略与完整本地主流程 +## 4. 已确认海龟基线 -- [x] 4.1 在 `strategy-003/research` 先添加失败的未复权 55 日入场、TR（真实波幅）、20 日 N 值、0.5% 初始风险单位和次日执行测试，再实现对应纯计算模块 -- [x] 4.2 先添加失败的固定 0.5N 加仓、每日一次、预算裁剪和只上移 2N 共同止损测试，再实现对应状态逻辑 -- [x] 4.3 先添加失败的 20 日退出、保护性止损、整手、流动性、资金、单 ETF、资产组和组合计划风险测试，再实现全部硬门禁 -- [x] 4.4 先添加失败的 60 日对齐样本冷启动、60 日协方差、10% 目标波动率缩减及持仓价格/风险输入缺失场景，再实现“停止新增风险但保留可执行退出和强制减仓”的故障安全逻辑 -- [x] 4.5 先添加失败的完整 E2E（端到端）场景，覆盖收盘信号、次日订单、成交回填、批次/持仓/现金/风险状态和审计输出，再实现项目适配入口使场景通过 -- [x] 4.6 添加多类同日订单、A1 共享预算、整手余数、候选乱序、跳空、涨跌停、不可成交、退出和风险减仓场景，验证已确认顺序与分配得到唯一且可追溯结果 +- [x] 4.1 冻结 11 只 ETF、6 个资产组和 55/20/20 参数。 +- [x] 4.2 实现信号日权益的 1%/N 基础数量、逐单位冻结 N、最多 4 单位、固定 0.5N 加仓档和每天最多一个单位。 +- [x] 4.3 实现每单位成交价减 2N 的候选止损、共同止损只上移、趋势退出或保护性止损完整清仓。 +- [x] 4.4 实现 4/6/12 N 风险单位缩放：单标的 4、资产组 6、组合 12。 +- [x] 4.5 实现事件驱动的全量仓位再分配，后到趋势可同比例挤占已有趋势资金，输入顺序不影响目标。 +- [x] 4.6 实现统一现金比例、整手向下取整和候选稳定重算；不做余额补仓、最大余数或按代码耗尽现金。 +- [x] 4.7 固定“完整退出、再分配卖出、入场或加仓、再分配买入”顺序；再分配成交不改变单位、档位和止损。 +- [x] 4.8 删除资金仓位上限、计划风险上限、交易用协方差、最低对齐样本、目标波动率、强制波动率减仓和旧新增订单分配；旧字段严格拒绝。 -## 5. 完成本地粗筛与研究报告 +## 5. 归因与独立分析 -- [x] 5.1 实现配置驱动的项目研究入口，接入确定性事件与风险计算，并对 Vibe-Trading（AI 研究助理）已知前视偏差组合优化器设置显式跳过门禁 -- [x] 5.2 使用共享行情中心中已验证的 11 只 ETF 快照运行方向性粗筛与确定性复算，输出仓位分布、现金占比、留现原因、资产组和组合风险使用率，并关联快照、代码、配置和输出摘要 -- [x] 5.3 生成并校验 `research-report.md`、`conclusion.json` 和 `candidate-strategies.json`；研究建议使用确认的三种取值，候选清单恰好为一项冻结基线加六项预设单项挑战，共用代码且不按本地收益删选或新增参数 +- [x] 5.1 在海龟归因 `details_json` 增加单位数、候选基础数量、冻结 N、实际成交价、共同止损、组/组合/现金比例、有效风险单位和再分配状态隔离证据。 +- [x] 5.2 保持标准四表字段不变，现有聚宽结果没有海龟扩展时仍可 0 改动读取。 +- [x] 5.3 独立策略分析删除对旧仓位上限和目标波动率参数的依赖，改为实际权重、计划损失比例、有效 N 单位、组合单位预算利用率和再分配次数。 +- [x] 5.4 报告风险部分同步使用新指标；Vibe-Trading（AI 研究助理）群体分析不进入证据。 -## 6. 验证 Skill 结构、流程和通用性 +## 6. TDD 与契约同步 -- [x] 6.1 运行 `quick_validate.py`、Skill 布局、兼容链接和通用单元测试，修复全部失败并保留验证结果 -- [x] 6.2 从 Skill 文档公开的用户入口运行海龟完整离线 E2E，贯通共享批次、快照引用、内存 DuckDB、项目入口、审计、三类输出、三态和不可变证据索引 -- [x] 6.3 在不加载海龟目录、参数、资产或代码的隔离环境运行共享行情中心与通用测试，证明删除海龟项目后共用能力仍可使用 -- [x] 6.4 使用非海龟最小配置和原始日线夹具派发独立前向验证，复核代理能仅依据 Skill 完成共享行情导入、研究流程和可验证证据 +- [x] 6.1 先写失败测试，再实现配置、输入、4/6/12、现金缩放、逐单位状态、后到趋势、市场不可交易、延迟执行、结果归因和分析兼容。 +- [x] 6.2 同步 `baseline.json` 与 7 项分析计划声明，但本次不执行 7 场景或稳健性矩阵。 +- [x] 6.3 删除已失效挑战计划和依赖旧目标波动率的未落地设计，不保留替代兼容文件。 +- [x] 6.4 同步已确认设计、原始最终方案、OpenSpec（开放规格）和执行身份摘要。 -## 7. 接入 Build and Verify（构建与验证）并完成回归 +## 7. 完整验收与真实基线 -- [x] 7.1 更新 `.build-and-verify/config.json`，把 Skill、共享行情脚本、通用运行器、海龟项目和相关测试映射到受影响路径与验证命令，且不得把 `.local` 行情纳入提交输入 -- [x] 7.2 使用真实聚宽研究环境导出最终 11 只 ETF 完整日线，导入共享中心、运行一次完整本地研究并确认远端与本地临时产物已清理 -- [x] 7.3 运行覆盖 Skill 用户入口完整业务流程的端到端回归、仓库完整验证和公开仓库敏感数据扫描，逐项复核两份 capability（能力）规格并记录已验证与无法验证部分 +- [x] 7.1 新增从公开 CLI（命令行入口）进入的海龟完整 E2E（端到端）测试，覆盖共享 Parquet、内存 DuckDB、vectorbt、全量仓位再分配、标准结果包、摘要一致和临时清理。 +- [x] 7.2 运行本地研究与策略分析完整回归、构建验证和 OpenSpec 严格校验。 +- [x] 7.3 从正式配置只运行一次真实 11 ETF 新基线，冷启动和预热均小于 180 秒且摘要一致。 +- [x] 7.4 从新标准结果包运行独立确定性分析，生成收益、回撤、双基准 Alpha/Beta（阿尔法/贝塔）、仓位、风险单位、归因、反对证据和推荐。 +- [x] 7.5 明确声明本次未运行旧方案对照、17 ETF 扩展、7 个场景和稳健性矩阵；最终等待人工确认，不代替聚宽正式裁决。 + +**状态：已确认。当前实施按 11 只 ETF、55/20/20、4/6/12、全量仓位再分配和 180 秒门禁执行。** diff --git a/requirements.txt b/requirements.txt index 006e845..262e998 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,5 @@ -r .agents/skills/joinquant-archive-sync/requirements.txt + +numba==0.66.0 +vectorbt==1.1.0 +jsonschema==4.26.0 diff --git a/scripts/research/analysis_data/__init__.py b/scripts/research/analysis_data/__init__.py new file mode 100644 index 0000000..4c940f1 --- /dev/null +++ b/scripts/research/analysis_data/__init__.py @@ -0,0 +1,22 @@ +"""Read-only JoinQuant and local research result contracts.""" + +from .manifest import ( + CORE_DATASETS, + AnalysisManifestError, + AnalysisSource, + ValidationResult, + open_analysis_source, + validate_analysis_source, +) +from .views import AnalysisDatabase, open_analysis_database + +__all__ = [ + "AnalysisDatabase", + "CORE_DATASETS", + "AnalysisManifestError", + "AnalysisSource", + "ValidationResult", + "open_analysis_database", + "open_analysis_source", + "validate_analysis_source", +] diff --git a/scripts/research/analysis_data/derived.py b/scripts/research/analysis_data/derived.py new file mode 100644 index 0000000..3172d29 --- /dev/null +++ b/scripts/research/analysis_data/derived.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import duckdb + + +class DerivedViewError(ValueError): + """Raised when cumulative source returns cannot be safely normalized.""" + + +def register_return_views(connection: duckdb.DuckDBPyConnection) -> None: + invalid = connection.sql( + "select count(*) from results where returns is null or returns <= -1" + ).fetchone() + if invalid is None or invalid[0] != 0: + raise DerivedViewError("results.returns must be finite cumulative returns above -1") + duplicates = connection.sql( + "select count(*) from (" + "select cast(substr(time, 1, 10) as date) trading_date " + "from results group by trading_date having count(*) <> 1)" + ).fetchone() + if duplicates is None or duplicates[0] != 0: + raise DerivedViewError("results must contain one row per trading date") + + connection.execute( + """ + create view strategy_daily_returns as + with normalized as ( + select + cast(substr(time, 1, 10) as date) as trading_date, + cast(returns as double) as cumulative_returns + from results + ), lagged as ( + select + trading_date, + cumulative_returns, + lag(cumulative_returns) over (order by trading_date) as previous_cumulative + from normalized + ) + select + trading_date, + cumulative_returns, + case + when previous_cumulative is not null + then (1.0 + cumulative_returns) / (1.0 + previous_cumulative) - 1.0 + when abs(cumulative_returns) <= 1e-15 then 0.0 + else cast(null as double) + end as daily_returns, + previous_cumulative is not null or abs(cumulative_returns) <= 1e-15 + as comparable + from lagged + """ + ) + connection.execute( + """ + create view source_benchmark_returns as + select + cast(substr(time, 1, 10) as date) as trading_date, + cast(benchmark_returns as double) as cumulative_returns + from results + """ + ) diff --git a/scripts/research/analysis_data/manifest.py b/scripts/research/analysis_data/manifest.py new file mode 100644 index 0000000..52add6c --- /dev/null +++ b/scripts/research/analysis_data/manifest.py @@ -0,0 +1,575 @@ +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from types import MappingProxyType +from typing import Mapping + +import pyarrow.parquet as pq +from jsonschema import Draft202012Validator, FormatChecker +from jsonschema.exceptions import SchemaError, ValidationError + + +CORE_DATASETS = ( + "results", + "balances", + "positions", + "orders", + "risk", + "period_risks", +) +LOCAL_PHYSICAL_DATASETS = CORE_DATASETS[:4] +_SHA256 = re.compile(r"[0-9a-f]{64}") +_LOCAL_TOP_LEVEL = { + "schema_version", + "object", + "source", + "authority", + "run", + "code", + "params", + "performance", + "datasets", + "source_benchmark_returns", + "gate", + "extensions", +} +_JOINQUANT_ONLY_FIELDS = { + "collection_fence", + "research_response", + "research_lineage", + "official_summary", +} + + +def _load_schema(path: Path) -> Mapping[str, object]: + try: + document = json.loads(path.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(document) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, SchemaError) as exc: + raise RuntimeError(f"analysis schema is invalid: {path.name}") from exc + return document + + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_LOCAL_SCHEMA = _load_schema( + Path(__file__).resolve().parent / "schemas" / "local-backtest-manifest.schema.json" +) +_JOINQUANT_SCHEMA = _load_schema( + _REPO_ROOT + / ".agents" + / "skills" + / "joinquant-archive-sync" + / "references" + / "manifest.schema.json" +) +_FORMAT_CHECKER = FormatChecker() + + +class AnalysisManifestError(ValueError): + """Raised when an analysis source cannot prove its declared identity.""" + + +def _validate_schema( + document: Mapping[str, object], schema: Mapping[str, object], name: str +) -> None: + try: + Draft202012Validator( + schema, format_checker=_FORMAT_CHECKER + ).validate(dict(document)) + except ValidationError as exc: + raise AnalysisManifestError(f"{name} schema validation failed: {exc.message}") from exc + + +@dataclass(frozen=True) +class AnalysisSource: + root: Path + kind: str + schema_version: int | str + manifest: Mapping[str, object] + + def __post_init__(self) -> None: + object.__setattr__(self, "root", Path(self.root).resolve()) + object.__setattr__( + self, "manifest", MappingProxyType(dict(self.manifest)) + ) + + +@dataclass(frozen=True) +class ValidationResult: + status: str + datasets: Mapping[str, str] + + def __post_init__(self) -> None: + object.__setattr__(self, "datasets", MappingProxyType(dict(self.datasets))) + + +def _object(value: object, field: str) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise AnalysisManifestError(f"{field} must be an object") + return value + + +def _exact_keys( + value: Mapping[str, object], + *, + required: set[str], + optional: set[str] | None = None, + field: str, +) -> None: + allowed = required | (optional or set()) + if not required.issubset(value) or set(value) - allowed: + raise AnalysisManifestError(f"{field} structure is invalid") + + +def _non_empty_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value: + raise AnalysisManifestError(f"{field} must be a non-empty string") + return value + + +def _sha(value: object, field: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise AnalysisManifestError(f"{field} must be a lowercase SHA256") + return value + + +def _non_negative_int(value: object, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise AnalysisManifestError(f"{field} must be a non-negative integer") + return value + + +def _date_or_none(value: object, field: str) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise AnalysisManifestError(f"{field} must use YYYY-MM-DD or null") + try: + date.fromisoformat(value) + except ValueError as exc: + raise AnalysisManifestError(f"{field} must use YYYY-MM-DD or null") from exc + return value + + +def _file_ref(value: object, field: str, *, parquet: bool = False) -> Mapping[str, object]: + result = _object(value, field) + required = {"path", "sha256", "bytes"} + if parquet: + required |= {"rows", "format", "compression"} + _exact_keys(result, required=required, field=field) + path = _non_empty_string(result["path"], f"{field}.path") + candidate = Path(path) + if candidate.is_absolute() or ".." in candidate.parts: + raise AnalysisManifestError(f"{field}.path is unsafe") + _sha(result["sha256"], f"{field}.sha256") + _non_negative_int(result["bytes"], f"{field}.bytes") + if parquet: + _non_negative_int(result["rows"], f"{field}.rows") + if result["format"] != "parquet" or result["compression"] != "zstd": + raise AnalysisManifestError(f"{field} must declare zstd Parquet") + return result + + +def _validate_complete_dataset(name: str, value: object) -> None: + dataset = _object(value, f"datasets.{name}") + _exact_keys( + dataset, + required={ + "required", + "status", + "rows", + "verified_empty", + "time_range", + "files", + "evidence", + }, + field=f"datasets.{name}", + ) + if dataset["required"] is not True or dataset["status"] != "complete": + raise AnalysisManifestError(f"datasets.{name} must be complete and required") + rows = _non_negative_int(dataset["rows"], f"datasets.{name}.rows") + if dataset["verified_empty"] is not (rows == 0): + raise AnalysisManifestError(f"datasets.{name}.verified_empty is inconsistent") + time_range = _object(dataset["time_range"], f"datasets.{name}.time_range") + _exact_keys( + time_range, + required={"start", "end"}, + field=f"datasets.{name}.time_range", + ) + start = _date_or_none(time_range["start"], f"datasets.{name}.time_range.start") + end = _date_or_none(time_range["end"], f"datasets.{name}.time_range.end") + if rows == 0: + if start is not None or end is not None: + raise AnalysisManifestError(f"datasets.{name}.time_range must be empty") + elif start is None or end is None or start > end: + raise AnalysisManifestError(f"datasets.{name}.time_range is invalid") + files = dataset["files"] + if not isinstance(files, list) or len(files) != 1: + raise AnalysisManifestError(f"datasets.{name} must have one Parquet file") + file_ref = _file_ref(files[0], f"datasets.{name}.files[0]", parquet=True) + if file_ref["path"] != f"data/{name}.parquet" or file_ref["rows"] != rows: + raise AnalysisManifestError(f"datasets.{name} file identity is invalid") + evidence = _object(dataset["evidence"], f"datasets.{name}.evidence") + _exact_keys( + evidence, + required={"fields", "unique_key"}, + field=f"datasets.{name}.evidence", + ) + for key in ("fields", "unique_key"): + if not isinstance(evidence[key], list) or not evidence[key]: + raise AnalysisManifestError(f"datasets.{name}.evidence.{key} is invalid") + + +def _validate_missing_dataset(name: str, value: object) -> None: + dataset = _object(value, f"datasets.{name}") + _exact_keys( + dataset, + required={ + "required", + "status", + "reason", + "rows", + "verified_empty", + "files", + }, + field=f"datasets.{name}", + ) + if ( + dataset["required"] is not False + or dataset["status"] != "missing_at_source" + or dataset["reason"] != "computed_by_strategy_analysis" + or dataset["rows"] != 0 + or dataset["verified_empty"] is not True + or dataset["files"] != [] + ): + raise AnalysisManifestError(f"datasets.{name} must be missing at source") + + +def validate_local_manifest_document(document: Mapping[str, object]) -> None: + if not isinstance(document, Mapping): + raise AnalysisManifestError("local manifest must be an object") + _validate_schema(document, _LOCAL_SCHEMA, "local manifest") + required = _LOCAL_TOP_LEVEL - {"extensions"} + _exact_keys( + document, + required=required, + optional={"extensions"}, + field="local manifest", + ) + if set(document) & _JOINQUANT_ONLY_FIELDS: + raise AnalysisManifestError("local manifest contains JoinQuant evidence") + if document["schema_version"] != "local-backtest/1": + raise AnalysisManifestError("unsupported local manifest schema") + + object_identity = _object(document["object"], "object") + _exact_keys( + object_identity, + required={"kind", "local_id", "status"}, + field="object", + ) + if object_identity["kind"] != "local_backtest" or object_identity["status"] != "complete": + raise AnalysisManifestError("local object identity is invalid") + _non_empty_string(object_identity["local_id"], "object.local_id") + + source = _object(document["source"], "source") + _exact_keys( + source, + required={"kind", "engine", "accounting"}, + field="source", + ) + if source["kind"] != "local_vectorbt": + raise AnalysisManifestError("local source identity is invalid") + if document["authority"] != "local_research": + raise AnalysisManifestError("local authority is invalid") + run = _object(document["run"], "run") + _exact_keys( + run, + required={"run_id", "scenario_id", "snapshot_id"}, + field="run", + ) + for field in ("run_id", "scenario_id"): + _non_empty_string(run[field], f"run.{field}") + _sha(run["snapshot_id"], "run.snapshot_id") + engine = _object(source["engine"], "source.engine") + _exact_keys( + engine, + required={ + "backend", + "adapter_version", + "vectorbt", + "numba", + "numpy", + "pandas", + }, + field="source.engine", + ) + if engine["backend"] != "vectorbt.Portfolio.from_order_func": + raise AnalysisManifestError("local execution backend is invalid") + for field in ("adapter_version", "vectorbt", "numba", "numpy", "pandas"): + _non_empty_string(engine[field], f"source.engine.{field}") + accounting = _object(source["accounting"], "source.accounting") + accounting_contract = { + "version": "turtle-etf-corporate-actions/1", + "corporate_action_mode": "point_in_time_total_return_approximation", + "continuity_factor_basis": "raw_previous_close_over_current_pre_close", + "corporate_action_metadata_timing": "audit_only_may_be_retrospective", + "price_basis": "continuous_economic_price", + "quantity_basis": "economic_units", + "cash_dividend_mode": "implicit_reinvestment_on_ex_date", + "pay_date_cash_supported": False, + "exact_joinquant_reconciliation": False, + } + _exact_keys( + accounting, + required={*accounting_contract, "corporate_actions_sha256"}, + field="source.accounting", + ) + if any(accounting.get(field) != value for field, value in accounting_contract.items()): + raise AnalysisManifestError("source.accounting precision boundary is invalid") + _sha( + accounting["corporate_actions_sha256"], + "source.accounting.corporate_actions_sha256", + ) + + code = _object(document["code"], "code") + code_ref = _file_ref(code, "code") + if code_ref["path"] != "code.py": + raise AnalysisManifestError("code.path must be code.py") + params = _object(document["params"], "params") + _exact_keys(params, required={"current", "version"}, field="params") + current_params = _file_ref(params["current"], "params.current") + version_params = _file_ref(params["version"], "params.version") + if current_params["path"] != "params.json": + raise AnalysisManifestError("params.current.path must be params.json") + if ( + version_params["path"] != f"params_versions/{version_params['sha256']}.json" + or current_params["sha256"] != version_params["sha256"] + ): + raise AnalysisManifestError("params version identity is invalid") + performance = _file_ref(document["performance"], "performance") + if performance["path"] != "performance.json": + raise AnalysisManifestError("performance.path must be performance.json") + + datasets = _object(document["datasets"], "datasets") + if set(datasets) != set(CORE_DATASETS): + raise AnalysisManifestError("local manifest must declare six core datasets") + for name in LOCAL_PHYSICAL_DATASETS: + _validate_complete_dataset(name, datasets[name]) + for name in ("risk", "period_risks"): + _validate_missing_dataset(name, datasets[name]) + + benchmark = _object(document["source_benchmark_returns"], "source_benchmark_returns") + _exact_keys( + benchmark, + required={"status", "reason", "null_rows"}, + field="source_benchmark_returns", + ) + if ( + benchmark["status"] != "missing_at_source" + or benchmark["reason"] != "independent_benchmark_set" + or benchmark["null_rows"] != datasets["results"]["rows"] + ): + raise AnalysisManifestError("source benchmark return evidence is invalid") + + gate = _object(document["gate"], "gate") + _exact_keys( + gate, + required={"status", "exceptions", "checks"}, + field="gate", + ) + if gate["status"] not in {"pass", "fail"}: + raise AnalysisManifestError("local manifest gate status is invalid") + if not isinstance(gate["exceptions"], list): + raise AnalysisManifestError("local manifest gate exceptions are invalid") + if gate["status"] == "pass" and gate["exceptions"] != []: + raise AnalysisManifestError("passing local manifest has exceptions") + if not isinstance(gate["checks"], list) or not gate["checks"]: + raise AnalysisManifestError("local manifest gate did not pass") + if "extensions" in document and not isinstance(document["extensions"], Mapping): + raise AnalysisManifestError("extensions must be an object") + + +def _resolve_file(root: Path, relative: object, field: str) -> Path: + value = _non_empty_string(relative, field) + candidate = Path(value) + if candidate.is_absolute() or ".." in candidate.parts: + raise AnalysisManifestError(f"{field} is unsafe") + root_resolved = root.resolve() + resolved = (root_resolved / candidate).resolve() + if root_resolved not in resolved.parents: + raise AnalysisManifestError(f"{field} escapes the source") + return resolved + + +def _file_digest(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_declared_file(root: Path, reference: Mapping[str, object], field: str) -> Path: + path = _resolve_file(root, reference["path"], f"{field}.path") + if not path.is_file(): + raise AnalysisManifestError(f"{field} is missing") + if path.stat().st_size != reference["bytes"] or _file_digest(path) != reference["sha256"]: + raise AnalysisManifestError(f"{field} digest or size mismatch") + return path + + +def _validate_local_files(root: Path, document: Mapping[str, object]) -> None: + code = _object(document["code"], "code") + _verify_declared_file(root, code, "code") + params = _object(document["params"], "params") + _verify_declared_file( + root, _object(params["current"], "params.current"), "params.current" + ) + _verify_declared_file( + root, _object(params["version"], "params.version"), "params.version" + ) + _verify_declared_file( + root, _object(document["performance"], "performance"), "performance" + ) + datasets = _object(document["datasets"], "datasets") + for name in LOCAL_PHYSICAL_DATASETS: + entry = _object(datasets[name], f"datasets.{name}") + reference = _object(entry["files"][0], f"datasets.{name}.files[0]") + path = _verify_declared_file(root, reference, f"datasets.{name}.files[0]") + try: + rows = pq.ParquetFile(path).metadata.num_rows + except Exception as exc: + raise AnalysisManifestError(f"datasets.{name} is invalid Parquet") from exc + if rows != entry["rows"] or rows != reference["rows"]: + raise AnalysisManifestError(f"datasets.{name} row count mismatch") + + +def _validate_joinquant_document(document: Mapping[str, object]) -> None: + _validate_schema(document, _JOINQUANT_SCHEMA, "joinquant manifest") + if document.get("schema_version") != 1 or isinstance( + document.get("schema_version"), bool + ): + raise AnalysisManifestError("unsupported joinquant manifest schema") + object_identity = _object(document.get("object"), "joinquant object") + if object_identity.get("kind") != "backtest": + raise AnalysisManifestError("joinquant manifest is not a backtest") + source = _object(document.get("source"), "joinquant source") + if not isinstance(source.get("url"), str) or not isinstance(source.get("aliases"), list): + raise AnalysisManifestError("joinquant source identity is invalid") + gate = _object(document.get("gate"), "joinquant gate") + gate_exceptions = gate.get("exceptions") + allowed_exceptions = {"attribution_log:missing_at_source"} + if ( + gate.get("status") != "pass" + or not isinstance(gate_exceptions, list) + or any(item not in allowed_exceptions for item in gate_exceptions) + ): + raise AnalysisManifestError("joinquant archive gate did not pass") + datasets = _object(document.get("datasets"), "joinquant datasets") + for name in CORE_DATASETS: + entry = _object(datasets.get(name), f"joinquant datasets.{name}") + if entry.get("required") is not True or entry.get("status") != "complete": + raise AnalysisManifestError(f"joinquant datasets.{name} is incomplete") + _non_negative_int(entry.get("rows"), f"joinquant datasets.{name}.rows") + + +def _joinquant_parquet_reference( + entry: Mapping[str, object], name: str +) -> Mapping[str, object] | None: + files = entry.get("files", []) + if not isinstance(files, list): + raise AnalysisManifestError(f"joinquant datasets.{name}.files is invalid") + matches = [ + item + for item in files + if isinstance(item, Mapping) + and item.get("path") == f"data/{name}.parquet" + and item.get("format") == "parquet" + ] + if len(matches) > 1: + raise AnalysisManifestError(f"joinquant datasets.{name} has duplicate Parquet files") + return None if not matches else matches[0] + + +def _validate_joinquant_files(root: Path, document: Mapping[str, object]) -> None: + datasets = _object(document["datasets"], "joinquant datasets") + for name in CORE_DATASETS: + entry = _object(datasets[name], f"joinquant datasets.{name}") + rows = int(entry["rows"]) + reference = _joinquant_parquet_reference(entry, name) + if reference is None: + if rows != 0 or entry.get("verified_empty") is not True: + raise AnalysisManifestError( + f"joinquant datasets.{name} lacks required Parquet evidence" + ) + continue + for field in ("path", "sha256", "bytes", "rows"): + if field not in reference: + raise AnalysisManifestError( + f"joinquant datasets.{name} file evidence is incomplete" + ) + _sha(reference["sha256"], f"joinquant datasets.{name}.sha256") + _non_negative_int(reference["bytes"], f"joinquant datasets.{name}.bytes") + _non_negative_int(reference["rows"], f"joinquant datasets.{name}.rows") + path = _verify_declared_file( + root, reference, f"joinquant datasets.{name}.parquet" + ) + try: + physical_rows = pq.ParquetFile(path).metadata.num_rows + except Exception as exc: + raise AnalysisManifestError( + f"joinquant datasets.{name} is invalid Parquet" + ) from exc + if physical_rows != rows or physical_rows != reference["rows"]: + raise AnalysisManifestError(f"joinquant datasets.{name} row count mismatch") + + +def open_analysis_source(result_dir: Path) -> AnalysisSource: + root = Path(result_dir).resolve() + manifest_path = root / "manifest.json" + try: + document = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AnalysisManifestError("analysis manifest is unreadable") from exc + if not isinstance(document, dict): + raise AnalysisManifestError("analysis manifest must be an object") + version = document.get("schema_version") + if version == 1 and not isinstance(version, bool): + _validate_joinquant_document(document) + _validate_joinquant_files(root, document) + kind = "joinquant_backtest" + elif version == "local-backtest/1": + validate_local_manifest_document(document) + if _object(document["gate"], "gate")["status"] != "pass": + raise AnalysisManifestError("local archive gate did not pass") + _validate_local_files(root, document) + kind = "local_backtest" + else: + raise AnalysisManifestError("unsupported analysis manifest schema") + return AnalysisSource( + root=root, + kind=kind, + schema_version=version, + manifest=document, + ) + + +def validate_analysis_source(source: AnalysisSource) -> ValidationResult: + if source.kind == "joinquant_backtest": + _validate_joinquant_document(source.manifest) + _validate_joinquant_files(source.root, source.manifest) + elif source.kind == "local_backtest": + validate_local_manifest_document(source.manifest) + _validate_local_files(source.root, source.manifest) + else: + raise AnalysisManifestError("unsupported analysis source kind") + datasets = _object(source.manifest["datasets"], "datasets") + return ValidationResult( + status="pass", + datasets={name: str(_object(datasets[name], name)["status"]) for name in CORE_DATASETS}, + ) diff --git a/scripts/research/analysis_data/schemas/local-backtest-manifest.schema.json b/scripts/research/analysis_data/schemas/local-backtest-manifest.schema.json new file mode 100644 index 0000000..80f4f60 --- /dev/null +++ b/scripts/research/analysis_data/schemas/local-backtest-manifest.schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://local.vibe-trading/local-backtest-manifest.schema.json", + "title": "Local vectorbt backtest manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "object", + "source", + "authority", + "run", + "code", + "params", + "performance", + "datasets", + "source_benchmark_returns", + "gate" + ], + "properties": { + "schema_version": {"const": "local-backtest/1"}, + "object": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "local_id", "status"], + "properties": { + "kind": {"const": "local_backtest"}, + "local_id": {"type": "string", "minLength": 1}, + "status": {"const": "complete"} + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "engine", "accounting"], + "properties": { + "kind": {"const": "local_vectorbt"}, + "engine": { + "type": "object", + "additionalProperties": false, + "required": ["backend", "adapter_version", "vectorbt", "numba", "numpy", "pandas"], + "properties": { + "backend": {"const": "vectorbt.Portfolio.from_order_func"}, + "adapter_version": {"type": "string", "minLength": 1}, + "vectorbt": {"type": "string", "minLength": 1}, + "numba": {"type": "string", "minLength": 1}, + "numpy": {"type": "string", "minLength": 1}, + "pandas": {"type": "string", "minLength": 1} + } + }, + "accounting": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "corporate_action_mode", + "continuity_factor_basis", + "corporate_action_metadata_timing", + "price_basis", + "quantity_basis", + "cash_dividend_mode", + "pay_date_cash_supported", + "exact_joinquant_reconciliation", + "corporate_actions_sha256" + ], + "properties": { + "version": {"const": "turtle-etf-corporate-actions/1"}, + "corporate_action_mode": {"const": "point_in_time_total_return_approximation"}, + "continuity_factor_basis": {"const": "raw_previous_close_over_current_pre_close"}, + "corporate_action_metadata_timing": {"const": "audit_only_may_be_retrospective"}, + "price_basis": {"const": "continuous_economic_price"}, + "quantity_basis": {"const": "economic_units"}, + "cash_dividend_mode": {"const": "implicit_reinvestment_on_ex_date"}, + "pay_date_cash_supported": {"const": false}, + "exact_joinquant_reconciliation": {"const": false}, + "corporate_actions_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + } + } + }, + "authority": {"const": "local_research"}, + "run": { + "type": "object", + "additionalProperties": false, + "required": ["run_id", "scenario_id", "snapshot_id"], + "properties": { + "run_id": {"type": "string", "minLength": 1}, + "scenario_id": {"type": "string", "minLength": 1}, + "snapshot_id": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + }, + "code": {"$ref": "#/$defs/file"}, + "params": { + "type": "object", + "additionalProperties": false, + "required": ["current", "version"], + "properties": { + "current": {"$ref": "#/$defs/file"}, + "version": {"$ref": "#/$defs/file"} + } + }, + "performance": {"$ref": "#/$defs/file"}, + "datasets": { + "type": "object", + "additionalProperties": false, + "required": ["results", "balances", "positions", "orders", "risk", "period_risks"], + "properties": { + "results": {"$ref": "#/$defs/completeDataset"}, + "balances": {"$ref": "#/$defs/completeDataset"}, + "positions": {"$ref": "#/$defs/completeDataset"}, + "orders": {"$ref": "#/$defs/completeDataset"}, + "risk": {"$ref": "#/$defs/missingDataset"}, + "period_risks": {"$ref": "#/$defs/missingDataset"} + } + }, + "source_benchmark_returns": { + "type": "object", + "additionalProperties": false, + "required": ["status", "reason", "null_rows"], + "properties": { + "status": {"const": "missing_at_source"}, + "reason": {"const": "independent_benchmark_set"}, + "null_rows": {"type": "integer", "minimum": 0} + } + }, + "gate": { + "type": "object", + "additionalProperties": false, + "required": ["status", "exceptions", "checks"], + "properties": { + "status": {"enum": ["pass", "fail"]}, + "exceptions": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "checks": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}} + } + }, + "extensions": {"type": "object"} + }, + "$defs": { + "file": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256", "bytes"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "bytes": {"type": "integer", "minimum": 0} + } + }, + "parquetFile": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256", "bytes", "rows", "format", "compression"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "bytes": {"type": "integer", "minimum": 0}, + "rows": {"type": "integer", "minimum": 0}, + "format": {"const": "parquet"}, + "compression": {"const": "zstd"} + } + }, + "completeDataset": { + "type": "object", + "additionalProperties": false, + "required": ["required", "status", "rows", "verified_empty", "time_range", "files", "evidence"], + "properties": { + "required": {"const": true}, + "status": {"const": "complete"}, + "rows": {"type": "integer", "minimum": 0}, + "verified_empty": {"type": "boolean"}, + "time_range": { + "type": "object", + "additionalProperties": false, + "required": ["start", "end"], + "properties": { + "start": {"type": ["string", "null"], "format": "date"}, + "end": {"type": ["string", "null"], "format": "date"} + } + }, + "files": {"type": "array", "minItems": 1, "maxItems": 1, "items": {"$ref": "#/$defs/parquetFile"}}, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["fields", "unique_key"], + "properties": { + "fields": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + "unique_key": {"type": "array", "minItems": 1, "items": {"type": "string"}} + } + } + } + }, + "missingDataset": { + "type": "object", + "additionalProperties": false, + "required": ["required", "status", "reason", "rows", "verified_empty", "files"], + "properties": { + "required": {"const": false}, + "status": {"const": "missing_at_source"}, + "reason": {"const": "computed_by_strategy_analysis"}, + "rows": {"const": 0}, + "verified_empty": {"const": true}, + "files": {"const": []} + } + } + } +} diff --git a/scripts/research/analysis_data/views.py b/scripts/research/analysis_data/views.py new file mode 100644 index 0000000..9c21354 --- /dev/null +++ b/scripts/research/analysis_data/views.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from types import TracebackType +from typing import Mapping + +import duckdb +import pyarrow.parquet as pq + +from scripts.research.analysis_data.derived import register_return_views +from scripts.research.analysis_data.manifest import ( + CORE_DATASETS, + AnalysisManifestError, + AnalysisSource, + open_analysis_source, +) + + +_SCHEMAS: Mapping[str, tuple[tuple[str, str], ...]] = { + "results": ( + ("benchmark_returns", "DOUBLE"), + ("returns", "DOUBLE"), + ("time", "VARCHAR"), + ), + "balances": ( + ("total_value", "DOUBLE"), + ("net_value", "DOUBLE"), + ("cash", "DOUBLE"), + ("aval_cash", "DOUBLE"), + ("time", "VARCHAR"), + ), + "positions": ( + ("pindex", "BIGINT"), + ("avg_cost", "DOUBLE"), + ("margin", "DOUBLE"), + ("amount", "DOUBLE"), + ("today_amount", "BIGINT"), + ("hold_cost", "DOUBLE"), + ("side", "VARCHAR"), + ("price", "DOUBLE"), + ("gains", "DOUBLE"), + ("daily_gains", "DOUBLE"), + ("closeable_amount", "BIGINT"), + ("time", "VARCHAR"), + ("security_name", "VARCHAR"), + ("security", "VARCHAR"), + ), + "orders": ( + ("match_time", "VARCHAR"), + ("pindex", "BIGINT"), + ("cancel_time", "VARCHAR"), + ("action", "VARCHAR"), + ("limit_price", "DOUBLE"), + ("comment", "VARCHAR"), + ("entrust_time", "VARCHAR"), + ("finish_time", "VARCHAR"), + ("side", "VARCHAR"), + ("price", "DOUBLE"), + ("commission", "DOUBLE"), + ("gains", "DOUBLE"), + ("type", "VARCHAR"), + ("time", "VARCHAR"), + ("security_name", "VARCHAR"), + ("security", "VARCHAR"), + ("filled", "BIGINT"), + ("amount", "BIGINT"), + ("status", "VARCHAR"), + ), + "risk": ( + ("__version", "BIGINT"), + ("algorithm_return", "DOUBLE"), + ("algorithm_volatility", "DOUBLE"), + ("alpha", "DOUBLE"), + ("annual_algo_return", "DOUBLE"), + ("annual_bm_return", "DOUBLE"), + ("avg_excess_return", "DOUBLE"), + ("avg_position_days", "DOUBLE"), + ("avg_trade_return", "DOUBLE"), + ("benchmark_return", "DOUBLE"), + ("benchmark_volatility", "DOUBLE"), + ("beta", "DOUBLE"), + ("day_win_ratio", "DOUBLE"), + ("excess_return", "DOUBLE"), + ("excess_return_max_drawdown", "DOUBLE"), + ("excess_return_max_drawdown_period", "VARCHAR"), + ("excess_return_sharpe", "DOUBLE"), + ("information", "DOUBLE"), + ("lose_count", "BIGINT"), + ("max_drawdown", "DOUBLE"), + ("max_drawdown_period", "VARCHAR"), + ("max_leverage", "DOUBLE"), + ("period_label", "VARCHAR"), + ("profit_loss_ratio", "DOUBLE"), + ("sharpe", "DOUBLE"), + ("sortino", "DOUBLE"), + ("trading_days", "BIGINT"), + ("treasury_return", "DOUBLE"), + ("turnover_rate", "DOUBLE"), + ("win_count", "BIGINT"), + ("win_ratio", "DOUBLE"), + ), + "period_risks": (("metric", "VARCHAR"), ("payload_json", "VARCHAR")), +} + + +def _quote_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _quote_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _empty_query(schema: tuple[tuple[str, str], ...]) -> str: + columns = ", ".join( + f"cast(null as {kind}) as {_quote_identifier(name)}" for name, kind in schema + ) + return f"select {columns} where false" + + +def _parquet_query(path: Path, schema: tuple[tuple[str, str], ...]) -> str: + source = f"read_parquet({_quote_literal(path.as_posix())})" + columns = ", ".join( + f"cast({_quote_identifier(name)} as {kind}) as {_quote_identifier(name)}" + for name, kind in schema + ) + return f"select {columns} from {source}" + + +def _declared_parquet_path(source: AnalysisSource, name: str) -> Path | None: + entry = source.manifest["datasets"][name] + files = entry["files"] + for reference in files: + if reference.get("path") == f"data/{name}.parquet": + return source.root / str(reference["path"]) + return None + + +def _validate_physical_fields( + source: AnalysisSource, name: str, path: Path +) -> None: + actual = tuple(pq.read_schema(path).names) + expected = tuple(field for field, _ in _SCHEMAS[name]) + fields_match = ( + actual == expected + if source.kind == "local_backtest" + else len(actual) == len(expected) and set(actual) == set(expected) + ) + if not fields_match: + raise AnalysisManifestError( + f"{source.kind} {name} fields do not match the observed contract" + ) + if source.kind == "local_backtest" and name == "results": + schema = pq.read_schema(path) + if ( + str(schema.field("benchmark_returns").type) != "double" + or str(schema.field("returns").type) != "double" + or str(schema.field("time").type) != "string" + ): + raise AnalysisManifestError("local results physical types are invalid") + present = pq.read_table(path, columns=["benchmark_returns"])[ + "benchmark_returns" + ].null_count + expected_nulls = int(source.manifest["datasets"]["results"]["rows"]) + if present != expected_nulls: + raise AnalysisManifestError( + "local results.benchmark_returns must be entirely null" + ) + + +@dataclass +class AnalysisDatabase: + source: AnalysisSource + connection: duckdb.DuckDBPyConnection + + @property + def table_names(self) -> tuple[str, ...]: + return CORE_DATASETS + + def reference_status(self, name: str) -> tuple[str, str | None]: + if name not in CORE_DATASETS: + raise KeyError(name) + entry = self.source.manifest["datasets"][name] + return str(entry["status"]), ( + None if "reason" not in entry else str(entry["reason"]) + ) + + def close(self) -> None: + self.connection.close() + + def __enter__(self) -> AnalysisDatabase: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.close() + + +def open_analysis_database(result_dir: Path) -> AnalysisDatabase: + source = open_analysis_source(result_dir) + connection = duckdb.connect(":memory:") + try: + for name in CORE_DATASETS: + path = _declared_parquet_path(source, name) + schema = _SCHEMAS[name] + if path is None: + query = _empty_query(schema) + else: + _validate_physical_fields(source, name, path) + query = _parquet_query(path, schema) + connection.execute( + f"create view {_quote_identifier(name)} as {query}" + ) + register_return_views(connection) + return AnalysisDatabase(source=source, connection=connection) + except Exception: + connection.close() + raise diff --git a/scripts/research/local_quant_research/adapter_guard.py b/scripts/research/local_quant_research/adapter_guard.py index f62860c..9efe71c 100644 --- a/scripts/research/local_quant_research/adapter_guard.py +++ b/scripts/research/local_quant_research/adapter_guard.py @@ -3,6 +3,7 @@ import argparse import os import runpy +import shutil import sys from pathlib import Path from typing import Sequence @@ -66,6 +67,17 @@ def _open_is_write(args: Sequence[object]) -> bool: return mode_writes or flag_writes +def _is_devnull(value: object) -> bool: + if isinstance(value, int): + return False + try: + return os.path.normcase(os.fsdecode(os.fspath(value))) == os.path.normcase( + os.devnull + ) + except (TypeError, ValueError, OSError): + return False + + def install_access_guard( output_dir: Path, *, @@ -79,7 +91,12 @@ def install_access_guard( venv_root = Path(venv_root).resolve() def audit(event: str, args: tuple[object, ...]) -> None: - if event == "open" and args and _open_is_write(args): + if ( + event == "open" + and args + and _open_is_write(args) + and not _is_devnull(args[0]) + ): _require_staging_path(args[0], output_root) elif event == "open" and args: path = _path_from_event(args[0]) @@ -127,6 +144,14 @@ def main(argv: list[str] | None = None) -> int: or not _inside(entry, execution_root) ): return 2 + runtime_cache = output_dir / ".runtime-cache" + numba_cache = runtime_cache / "numba" + matplotlib_cache = runtime_cache / "matplotlib" + numba_cache.mkdir(parents=True) + matplotlib_cache.mkdir() + os.environ["NUMBA_CACHE_DIR"] = str(numba_cache) + os.environ["MPLCONFIGDIR"] = str(matplotlib_cache) + os.environ["XDG_CACHE_HOME"] = str(runtime_cache) install_access_guard( output_dir, execution_root=execution_root, @@ -136,7 +161,12 @@ def main(argv: list[str] | None = None) -> int: sys.path.insert(0, str(execution_root / "repository")) sys.path.insert(0, str(entry.parent)) sys.argv = [str(entry), *adapter_args] - runpy.run_path(str(entry), run_name="__main__") + try: + runpy.run_path(str(entry), run_name="__main__") + finally: + shutil.rmtree(runtime_cache) + if runtime_cache.exists(): + raise RuntimeError("adapter runtime cache cleanup failed") return 0 diff --git a/scripts/research/local_quant_research/contracts.py b/scripts/research/local_quant_research/contracts.py index b77e780..f2142bd 100644 --- a/scripts/research/local_quant_research/contracts.py +++ b/scripts/research/local_quant_research/contracts.py @@ -12,7 +12,7 @@ @dataclass(frozen=True) class OutputSpec: path: str - format: Literal["json", "csv", "markdown", "text"] + format: Literal["json", "csv", "markdown", "text", "parquet", "directory"] @dataclass(frozen=True) @@ -24,6 +24,7 @@ class RunConfig: command: tuple[str, ...] project_config: Path code_identity: Path + benchmark_input: Path | None declared_inputs: tuple[Path, ...] required_outputs: tuple[OutputSpec, ...] output_root: Path @@ -58,6 +59,7 @@ class RunResult: reused: bool reasons: tuple[str, ...] stages: tuple[StageRecord, ...] + next_action: str | None = None def to_document(self) -> dict[str, object]: return { @@ -69,4 +71,5 @@ def to_document(self) -> dict[str, object]: "reused": self.reused, "reasons": list(self.reasons), "stages": [stage.to_document() for stage in self.stages], + "next_action": self.next_action, } diff --git a/scripts/research/local_quant_research/decision.py b/scripts/research/local_quant_research/decision.py new file mode 100644 index 0000000..d43bab4 --- /dev/null +++ b/scripts/research/local_quant_research/decision.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import uuid +from pathlib import Path +from typing import Mapping + + +_SHA256 = re.compile(r"[0-9a-f]{64}") +_PROJECT_ID = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}") +_DECISIONS = { + "proceed_to_joinquant", + "revise_and_reassess", + "stop_evidence_insufficient", +} +_REQUIRED = { + "decision", + "candidate_focus", + "baseline_action", + "reason", + "confirmed_by", + "confirmed_at", +} + + +class DecisionError(ValueError): + """Raised when a human decision does not match immutable research evidence.""" + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _file_digest(path: Path) -> str: + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError as exc: + raise DecisionError("research evidence is missing") from exc + + +def _recommendation(run_dir: Path) -> tuple[dict[str, object], str]: + path = Path(run_dir) / "recommendation.json" + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DecisionError("recommendation evidence is invalid") from exc + if not isinstance(value, dict): + raise DecisionError("recommendation evidence is invalid") + identity = value.get("identity") + run_id = identity.get("run_id") if isinstance(identity, Mapping) else None + if not isinstance(run_id, str) or _SHA256.fullmatch(run_id) is None: + raise DecisionError("recommendation run identity is invalid") + return value, run_id + + +def _document( + run_dir: Path, + project_id: str, + decision: Mapping[str, object], +) -> dict[str, object]: + if _PROJECT_ID.fullmatch(project_id) is None: + raise DecisionError("project_id is invalid") + if set(decision) != _REQUIRED: + raise DecisionError("human decision fields are incomplete or unknown") + if decision["decision"] not in _DECISIONS: + raise DecisionError("human decision value is invalid") + if not isinstance(decision["candidate_focus"], list) or any( + not isinstance(item, str) or not item for item in decision["candidate_focus"] + ): + raise DecisionError("candidate_focus must be a string array") + for field in ("baseline_action", "reason", "confirmed_by", "confirmed_at"): + if not isinstance(decision[field], str) or not str(decision[field]).strip(): + raise DecisionError(f"{field} must be non-empty") + _, run_id = _recommendation(run_dir) + payload = { + "schema_version": 1, + "project_id": project_id, + "run_id": run_id, + "report_sha256": _file_digest(Path(run_dir) / "local-research-report.md"), + "recommendation_sha256": _file_digest(Path(run_dir) / "recommendation.json"), + **dict(decision), + } + payload["decision_id"] = _digest(payload) + payload["document_sha256"] = _digest(payload) + return payload + + +def record_human_decision( + *, + run_dir: Path, + decision_root: Path, + project_id: str, + decision: Mapping[str, object], +) -> Path: + document = _document(Path(run_dir), project_id, decision) + target = ( + Path(decision_root) + / project_id + / str(document["run_id"]) + / str(document["decision_id"]) + ) + output = target / "human-decision.json" + if output.exists(): + if ( + validate_human_decision( + run_dir=run_dir, + project_id=project_id, + decision_path=output, + ) + != document + ): + raise DecisionError( + "existing human decision conflicts with requested decision" + ) + return output + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.parent / f".{target.name}.tmp-{uuid.uuid4().hex}" + temporary.mkdir() + try: + (temporary / "human-decision.json").write_text( + json.dumps(document, ensure_ascii=False, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + os.replace(temporary, target) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + return output + + +def validate_human_decision( + *, + run_dir: Path, + project_id: str, + decision_path: Path, +) -> dict[str, object]: + try: + document = json.loads(Path(decision_path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DecisionError("human decision is invalid") from exc + if not isinstance(document, dict): + raise DecisionError("human decision is invalid") + semantic = { + key: value for key, value in document.items() if key != "document_sha256" + } + if document.get("document_sha256") != _digest(semantic): + raise DecisionError("human decision document digest mismatch") + _, run_id = _recommendation(Path(run_dir)) + if document.get("run_id") != run_id: + raise DecisionError("human decision run identity mismatch") + if document.get("project_id") != project_id: + raise DecisionError("human decision project identity mismatch") + if document.get("report_sha256") != _file_digest( + Path(run_dir) / "local-research-report.md" + ) or document.get("recommendation_sha256") != _file_digest( + Path(run_dir) / "recommendation.json" + ): + raise DecisionError("human decision evidence digest mismatch") + project_path = Path(decision_path).parents[2] + if project_path.name != project_id: + raise DecisionError("human decision path does not match its project") + expected_parent = project_path / run_id / str(document.get("decision_id")) + if Path(decision_path).parent.resolve() != expected_parent.resolve(): + raise DecisionError("human decision path does not match its identity") + return document diff --git a/scripts/research/local_quant_research/evidence.py b/scripts/research/local_quant_research/evidence.py index 0cbeb6a..1c978b3 100644 --- a/scripts/research/local_quant_research/evidence.py +++ b/scripts/research/local_quant_research/evidence.py @@ -9,6 +9,9 @@ from pathlib import Path from typing import Iterable, Mapping, Sequence +import pyarrow as pa +import pyarrow.parquet as pq + from .contracts import OutputSpec @@ -98,6 +101,13 @@ def _validate_output(path: Path, output_format: str) -> None: ) except (OSError, UnicodeDecodeError, StopIteration, csv.Error) as exc: raise EvidenceError(f"invalid CSV output: {path.name}") from exc + elif output_format == "parquet": + try: + schema = pq.read_schema(path) + except (OSError, pa.ArrowException) as exc: + raise EvidenceError(f"invalid Parquet output: {path.name}") from exc + if not schema.names or len(schema.names) != len(set(schema.names)): + raise EvidenceError(f"Parquet schema is invalid: {path.name}") else: try: text = raw.decode("utf-8") @@ -119,15 +129,44 @@ def collect_output_evidence( path.relative_to(resolved_root) except ValueError as exc: raise EvidenceError("required output escapes the staging directory") from exc - _validate_output(path, spec.format) - evidence.append( - { - "path": spec.path, - "format": spec.format, - "bytes": path.stat().st_size, - "sha256": file_digest(path), - } - ) + if spec.format == "directory": + if not path.is_dir() or path.is_symlink(): + raise EvidenceError(f"required output directory is missing: {path.name}") + files: list[dict[str, object]] = [] + for item in sorted(path.rglob("*")): + if item.is_symlink(): + raise EvidenceError("required output directory contains a symlink") + if not item.is_file(): + continue + relative = item.relative_to(path).as_posix() + files.append( + { + "path": relative, + "bytes": item.stat().st_size, + "sha256": file_digest(item), + } + ) + if not files: + raise EvidenceError("required output directory is empty") + evidence.append( + { + "path": spec.path, + "format": spec.format, + "bytes": sum(int(item["bytes"]) for item in files), + "sha256": canonical_digest(files), + "files": files, + } + ) + else: + _validate_output(path, spec.format) + evidence.append( + { + "path": spec.path, + "format": spec.format, + "bytes": path.stat().st_size, + "sha256": file_digest(path), + } + ) return evidence @@ -252,13 +291,22 @@ def validate_complete_run( raise EvidenceError("completed output digest mismatch") status = _load_json(Path(run_dir) / "project-status.json") - if status != {"schema_version": 1, "status": "complete", "reason_codes": []}: + expected_status = {"schema_version": 1, "status": "complete", "reason_codes": []} + accepted_statuses = [ + expected_status, + {**expected_status, "next_action": "human_confirmation_required"}, + {**expected_status, "next_action": "return_to_caller"}, + ] + if status not in accepted_statuses: raise EvidenceError("completed project status is invalid") - expected_files = { - "run-manifest.json", - "project-status.json", - *(item["path"] for item in outputs), - } + expected_files = {"run-manifest.json", "project-status.json"} + for item in outputs: + if item["format"] == "directory": + expected_files.update( + f"{item['path']}/{nested['path']}" for nested in item["files"] + ) + else: + expected_files.add(item["path"]) actual_files = { path.relative_to(run_dir).as_posix() for path in Path(run_dir).rglob("*") diff --git a/scripts/research/local_quant_research/runner.py b/scripts/research/local_quant_research/runner.py index 256cc6b..8291c0f 100644 --- a/scripts/research/local_quant_research/runner.py +++ b/scripts/research/local_quant_research/runner.py @@ -42,11 +42,13 @@ "output_root", "stop_states", } +_OPTIONAL_CONFIG_FIELDS = {"benchmark_input"} _STOP_STATES = ("complete", "evidence_insufficient", "failed") _PROJECT_ID_PATTERN = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}") _SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") _REASON_PATTERN = re.compile(r"[a-z][a-z0-9_]{0,63}") _SENSITIVE_KEYS = ("password", "token", "cookie", "secret", "credential", "api_key") +_PROJECT_EXECUTION_TIMEOUT_SECONDS = 3_600 _RESERVED_ARGUMENTS = { "--snapshot-manifest", "--market-data-root", @@ -56,6 +58,7 @@ "--snapshot-id", "--code-sha256", "--config-sha256", + "--benchmark-input", } _COMPLETE_STAGE_NAMES = ( "snapshot_validation", @@ -98,6 +101,7 @@ class _FrozenExecutionInputs: market_data: Path project_entry: Path project_config: Path + benchmark_input: Path | None def _inside(path: Path, root: Path) -> bool: @@ -168,7 +172,8 @@ def load_run_config(path: Path, *, repo_root: Path) -> RunConfig: if _contains_sensitive_key(document): raise ConfigurationError("credential_field", "credential fields are forbidden") missing = sorted(_CONFIG_FIELDS - set(document)) - if missing or set(document) != _CONFIG_FIELDS: + unknown = set(document) - (_CONFIG_FIELDS | _OPTIONAL_CONFIG_FIELDS) + if missing or unknown: raise ConfigurationError("invalid_config", "run config fields are incomplete or unknown") if document["schema_version"] != 1: raise ConfigurationError("invalid_config", "schema_version must be 1") @@ -194,6 +199,15 @@ def load_run_config(path: Path, *, repo_root: Path) -> RunConfig: code_identity = _resolve_repo_path( document["code_identity"], repo_root=repo_root, field="code_identity" ) + benchmark_input = ( + _resolve_repo_path( + document["benchmark_input"], + repo_root=repo_root, + field="benchmark_input", + ) + if "benchmark_input" in document + else None + ) command = document["command"] if ( @@ -229,7 +243,12 @@ def load_run_config(path: Path, *, repo_root: Path) -> RunConfig: ) if len(declared_inputs) != len(set(declared_inputs)): raise ConfigurationError("invalid_inputs", "declared_inputs must be unique") - for input_path in (project_config, code_identity, *declared_inputs): + for input_path in ( + project_config, + code_identity, + *declared_inputs, + *((benchmark_input,) if benchmark_input is not None else ()), + ): if not input_path.is_file(): raise ConfigurationError("missing_declared_input", "a declared input is missing") @@ -248,7 +267,14 @@ def load_run_config(path: Path, *, repo_root: Path) -> RunConfig: if not isinstance(item, dict) or set(item) != {"path", "format"}: raise ConfigurationError("invalid_output", "required output structure is invalid") output_format = item["format"] - if output_format not in {"json", "csv", "markdown", "text"}: + if output_format not in { + "json", + "csv", + "markdown", + "text", + "parquet", + "directory", + }: raise ConfigurationError("invalid_output", "required output format is invalid") output_specs.append(OutputSpec(path=_output_path(item["path"]), format=output_format)) if len({spec.path for spec in output_specs}) != len(output_specs): @@ -271,6 +297,7 @@ def load_run_config(path: Path, *, repo_root: Path) -> RunConfig: command=tuple(command), project_config=project_config, code_identity=code_identity, + benchmark_input=benchmark_input, declared_inputs=declared_inputs, required_outputs=tuple(output_specs), output_root=output_root, @@ -341,8 +368,19 @@ def _code_identity(config: RunConfig, *, repo_root: Path) -> tuple[str, dict[str document = json.loads(config.code_identity.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise InputIntegrityError("invalid_code_identity", "code identity is invalid") from exc - if not isinstance(document, dict) or set(document) != {"schema_version", "files"}: + if ( + not isinstance(document, dict) + or not {"schema_version", "files"}.issubset(document) + or set(document) - {"schema_version", "files", "execution"} + ): raise InputIntegrityError("invalid_code_identity", "code identity structure is invalid") + execution = document.get("execution") + if execution is not None and ( + not isinstance(execution, Mapping) + or not execution + or _contains_sensitive_key(execution) + ): + raise InputIntegrityError("invalid_code_identity", "execution identity is invalid") files = document["files"] if document["schema_version"] != 1 or not isinstance(files, list) or not files: raise InputIntegrityError("invalid_code_identity", "code identity files are invalid") @@ -369,7 +407,9 @@ def _code_identity(config: RunConfig, *, repo_root: Path) -> tuple[str, dict[str entry_path = config.project_entry.relative_to(repo_root).as_posix() if entry_path not in {item["path"] for item in normalized}: raise InputIntegrityError("missing_entry_identity", "project entry is absent from code identity") - normalized_document = {"schema_version": 1, "files": normalized} + normalized_document: dict[str, object] = {"schema_version": 1, "files": normalized} + if execution is not None: + normalized_document["execution"] = dict(execution) return canonical_digest(normalized_document), normalized_document @@ -384,11 +424,20 @@ def _input_evidence(config: RunConfig, *, repo_root: Path) -> tuple[str, str, di } for path in config.declared_inputs ] + benchmark_input = ( + { + "path": config.benchmark_input.relative_to(repo_root).as_posix(), + "sha256": file_digest(config.benchmark_input), + } + if config.benchmark_input is not None + else None + ) config_identity = { "run_config": dict(config.document), "project_config_sha256": project_config_digest, "code_identity_sha256": code_identity_digest, "declared_inputs": declared, + "benchmark_input": benchmark_input, } config_digest = canonical_digest(config_identity) evidence = { @@ -398,6 +447,7 @@ def _input_evidence(config: RunConfig, *, repo_root: Path) -> tuple[str, str, di "code_sha256": code_digest, "code_identity": code_document, "declared_inputs": declared, + "benchmark_input": benchmark_input, } return config_digest, code_digest, evidence @@ -460,6 +510,16 @@ def _freeze_execution_inputs( frozen_repo / relative, str(item["sha256"]), ) + frozen_benchmark_input: Path | None = None + benchmark_evidence = inputs.get("benchmark_input") + if isinstance(benchmark_evidence, Mapping): + relative = Path(str(benchmark_evidence["path"])) + frozen_benchmark_input = frozen_repo / relative + _copy_verified_file( + repo_root / relative, + frozen_benchmark_input, + str(benchmark_evidence["sha256"]), + ) snapshot_id = config.snapshot_id _copy_verified_file( @@ -473,7 +533,8 @@ def _freeze_execution_inputs( target_dir = frozen_market / "batches" / batch_id for name, digest_field in ( ("manifest.json", "manifest_sha256"), - ("market-data.csv", "csv_sha256"), + ("market-data.parquet", "parquet_sha256"), + ("corporate-actions.parquet", "corporate_actions_sha256"), ("validation.json", "validation_sha256"), ): _copy_verified_file( @@ -493,6 +554,7 @@ def _freeze_execution_inputs( market_data=frozen_market, project_entry=frozen_repo / config.project_entry.relative_to(repo_root), project_config=frozen_repo / project_config_relative, + benchmark_input=frozen_benchmark_input, ) except (KeyError, TypeError, OSError, MarketDataError, InputIntegrityError) as exc: if execution_root.exists(): @@ -601,11 +663,12 @@ def _project_status(staging: Path) -> tuple[str, tuple[str, ...]]: document = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise EvidenceError("project status is missing or invalid") from exc - if not isinstance(document, dict) or set(document) != { - "schema_version", - "status", - "reason_codes", - }: + required = {"schema_version", "status", "reason_codes"} + if ( + not isinstance(document, dict) + or not required.issubset(document) + or set(document) - required != ({"next_action"} if "next_action" in document else set()) + ): raise EvidenceError("project status structure is invalid") status = document["status"] reasons = document["reason_codes"] @@ -619,9 +682,22 @@ def _project_status(staging: Path) -> tuple[str, tuple[str, ...]]: raise EvidenceError("project status value is invalid") if status == "complete" and reasons: raise EvidenceError("complete project status must not contain reasons") + next_action = document.get("next_action") + if next_action is not None and ( + status != "complete" + or next_action + not in {"human_confirmation_required", "return_to_caller"} + ): + raise EvidenceError("project status next_action is invalid") return status, tuple(reasons) +def _project_next_action(staging: Path) -> str | None: + document = json.loads((Path(staging) / "project-status.json").read_text(encoding="utf-8")) + value = document.get("next_action") + return value if isinstance(value, str) else None + + def _actual_staging_files(staging: Path) -> set[str]: files: set[str] = set() for path in staging.rglob("*"): @@ -766,6 +842,7 @@ def run_project(config_path: Path, *, repo_root: Path) -> RunResult: reused=True, reasons=(), stages=complete_stages, + next_action=_project_next_action(run_dir), ) project_root.mkdir(parents=True, exist_ok=True) @@ -837,6 +914,10 @@ def run_project(config_path: Path, *, repo_root: Path) -> RunResult: "--config-sha256", config_digest, ] + if frozen.benchmark_input is not None: + command.extend( + ["--benchmark-input", str(frozen.benchmark_input.resolve())] + ) completed = None try: completed = subprocess.run( @@ -846,7 +927,7 @@ def run_project(config_path: Path, *, repo_root: Path) -> RunResult: env=_sanitized_environment(frozen.repository), capture_output=True, text=True, - timeout=600, + timeout=_PROJECT_EXECUTION_TIMEOUT_SECONDS, check=False, ) except (OSError, subprocess.SubprocessError): @@ -932,6 +1013,7 @@ def run_project(config_path: Path, *, repo_root: Path) -> RunResult: try: project_status, reason_codes = _project_status(staging) + next_action = _project_next_action(staging) except EvidenceError: return _attempt_result( repo_root=repo_root, @@ -960,8 +1042,21 @@ def run_project(config_path: Path, *, repo_root: Path) -> RunResult: ) try: output_evidence = collect_output_evidence(staging, config.required_outputs) - expected_files = {"project-status.json", *(spec.path for spec in config.required_outputs)} - if _actual_staging_files(staging) != expected_files: + actual_files = _actual_staging_files(staging) + fixed_files = { + "project-status.json", + *(spec.path for spec in config.required_outputs if spec.format != "directory"), + } + directory_prefixes = tuple( + f"{spec.path}/" + for spec in config.required_outputs + if spec.format == "directory" + ) + if not fixed_files.issubset(actual_files) or any( + path not in fixed_files + and not any(path.startswith(prefix) for prefix in directory_prefixes) + for path in actual_files + ): raise EvidenceError("project output file set differs from its declaration") except EvidenceError: return _attempt_result( @@ -1053,4 +1148,5 @@ def run_project(config_path: Path, *, repo_root: Path) -> RunResult: reused=False, reasons=(), stages=tuple(StageRecord(name, "complete") for name in _COMPLETE_STAGE_NAMES), + next_action=next_action, ) diff --git a/scripts/research/market_data/__init__.py b/scripts/research/market_data/__init__.py index 082a781..b273825 100644 --- a/scripts/research/market_data/__init__.py +++ b/scripts/research/market_data/__init__.py @@ -1,12 +1,13 @@ """Immutable local market-data batches and snapshots.""" from .contracts import BatchRecord, SnapshotRecord, SnapshotSelection -from .storage import create_snapshot, import_batch, validate_snapshot +from .storage import audit_store, create_snapshot, import_batch, validate_snapshot __all__ = [ "BatchRecord", "SnapshotRecord", "SnapshotSelection", + "audit_store", "create_snapshot", "import_batch", "validate_snapshot", diff --git a/scripts/research/market_data/benchmark_sets.py b/scripts/research/market_data/benchmark_sets.py new file mode 100644 index 0000000..8fa0431 --- /dev/null +++ b/scripts/research/market_data/benchmark_sets.py @@ -0,0 +1,774 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import shutil +import tempfile +import urllib.parse +import urllib.request +import zipfile +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from html.parser import HTMLParser +from io import BytesIO +from pathlib import Path +from types import MappingProxyType +from typing import Iterable, Mapping, Sequence +from xml.etree import ElementTree + +import pyarrow as pa +import pyarrow.parquet as pq + + +BENCHMARK_IDS = ( + "CSI300_CNY_TOTAL_RETURN", + "NASDAQ100_CNY_TOTAL_RETURN", +) +GENERATOR_VERSION = "official-benchmark-set/1" +_SOURCE_IDENTITIES = { + "csi300_total_return": { + "provider": "China Securities Index Co., Ltd.", + "source_id": "H00300", + "url_prefix": "https://www.csindex.com.cn/", + }, + "nasdaq100_total_return": { + "provider": "Nasdaq, Inc.", + "source_id": "XNDX", + "url_prefix": "https://indexes.nasdaqomx.com/", + }, + "usd_cny": { + "provider": "Board of Governors of the Federal Reserve System", + "source_id": "DEXCHUS", + "url_prefix": "https://www.federalreserve.gov/", + }, +} +_PARQUET_SCHEMA = pa.schema( + [ + pa.field("time", pa.string(), nullable=False), + pa.field("benchmark_id", pa.string(), nullable=False), + pa.field("returns", pa.float64(), nullable=False), + ] +) + + +class BenchmarkSetError(ValueError): + """Raised when an official benchmark set is incomplete or altered.""" + + +@dataclass(frozen=True, order=True) +class BenchmarkLevel: + trading_date: date + value: float + + def __post_init__(self) -> None: + if not isinstance(self.trading_date, date): + raise BenchmarkSetError("benchmark level date is invalid") + if not math.isfinite(float(self.value)) or float(self.value) <= 0: + raise BenchmarkSetError("benchmark level must be finite and positive") + + +@dataclass(frozen=True) +class SourcePayload: + name: str + filename: str + provider: str + source_id: str + url: str + content_type: str + data: bytes + + @property + def sha256(self) -> str: + return hashlib.sha256(self.data).hexdigest() + + +@dataclass(frozen=True) +class BenchmarkSet: + root: Path + benchmark_set_id: str + manifest: Mapping[str, object] + + def __post_init__(self) -> None: + object.__setattr__(self, "root", Path(self.root).resolve()) + object.__setattr__(self, "manifest", MappingProxyType(dict(self.manifest))) + + +def _canonical_bytes(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def _validate_sources(sources: Sequence[SourcePayload]) -> None: + if {item.name for item in sources} != set(_SOURCE_IDENTITIES) or len(sources) != 3: + raise BenchmarkSetError("benchmark set requires exactly three official sources") + if len({item.filename for item in sources}) != len(sources): + raise BenchmarkSetError("benchmark source filenames must be unique") + for source in sources: + expected = _SOURCE_IDENTITIES[source.name] + if ( + source.provider != expected["provider"] + or source.source_id != expected["source_id"] + or not source.url.startswith(expected["url_prefix"]) + or not source.filename + or Path(source.filename).name != source.filename + or not source.data + ): + raise BenchmarkSetError(f"benchmark source identity is invalid: {source.name}") + + +def _validated_levels(levels: Iterable[BenchmarkLevel], name: str) -> list[BenchmarkLevel]: + ordered = sorted(levels) + if len({item.trading_date for item in ordered}) != len(ordered): + raise BenchmarkSetError(f"{name} contains duplicate dates") + if len(ordered) < 2: + raise BenchmarkSetError(f"{name} needs at least two observations") + return ordered + + +def build_benchmark_rows( + *, + csi_levels: Iterable[BenchmarkLevel], + nasdaq_levels: Iterable[BenchmarkLevel], + usd_cny_levels: Iterable[BenchmarkLevel], + start_date: date, + end_date: date, +) -> list[dict[str, object]]: + if start_date > end_date: + raise BenchmarkSetError("benchmark date range is invalid") + csi = _validated_levels(csi_levels, "CSI300 total return") + nasdaq = _validated_levels(nasdaq_levels, "NASDAQ100 total return") + fx = _validated_levels(usd_cny_levels, "USD/CNY") + rows: list[dict[str, object]] = [] + + for previous, current in zip(csi, csi[1:], strict=False): + if start_date <= current.trading_date <= end_date: + rows.append( + { + "time": current.trading_date.isoformat(), + "benchmark_id": BENCHMARK_IDS[0], + "returns": current.value / previous.value - 1.0, + } + ) + + nasdaq_by_date = {item.trading_date: item.value for item in nasdaq} + fx_by_date = {item.trading_date: item.value for item in fx} + common = sorted(set(nasdaq_by_date) & set(fx_by_date)) + combined = [ + BenchmarkLevel(current, nasdaq_by_date[current] * fx_by_date[current]) + for current in common + ] + for previous, current in zip(combined, combined[1:], strict=False): + if start_date <= current.trading_date <= end_date: + rows.append( + { + "time": current.trading_date.isoformat(), + "benchmark_id": BENCHMARK_IDS[1], + "returns": current.value / previous.value - 1.0, + } + ) + rows.sort(key=lambda item: (str(item["time"]), str(item["benchmark_id"]))) + return rows + + +def _validate_rows( + rows: Iterable[Mapping[str, object]], + start_date: date, + end_date: date, +) -> list[dict[str, object]]: + materialized: list[dict[str, object]] = [] + keys: set[tuple[str, str]] = set() + identities: set[str] = set() + for original in rows: + if set(original) != {"time", "benchmark_id", "returns"}: + raise BenchmarkSetError("benchmark row fields are invalid") + try: + trading_date = date.fromisoformat(str(original["time"])) + except ValueError as exc: + raise BenchmarkSetError("benchmark time must use YYYY-MM-DD") from exc + benchmark_id = str(original["benchmark_id"]) + if benchmark_id not in BENCHMARK_IDS: + raise BenchmarkSetError(f"unsupported benchmark identity: {benchmark_id}") + value = float(original["returns"]) + if not math.isfinite(value) or value <= -1: + raise BenchmarkSetError("benchmark return is invalid") + if not start_date <= trading_date <= end_date: + raise BenchmarkSetError("benchmark row is outside the declared range") + key = (trading_date.isoformat(), benchmark_id) + if key in keys: + raise BenchmarkSetError(f"duplicate benchmark row: {key}") + keys.add(key) + identities.add(benchmark_id) + materialized.append( + {"time": key[0], "benchmark_id": benchmark_id, "returns": value} + ) + if identities != set(BENCHMARK_IDS): + raise BenchmarkSetError("benchmark set must contain exactly both required identities") + materialized.sort(key=lambda item: (item["time"], item["benchmark_id"])) + return materialized + + +def _source_manifest(root: Path, source: SourcePayload) -> dict[str, object]: + path = root / "sources" / source.filename + return { + "name": source.name, + "provider": source.provider, + "source_id": source.source_id, + "url": source.url, + "content_type": source.content_type, + "path": path.relative_to(root).as_posix(), + "sha256": _digest(path), + "bytes": path.stat().st_size, + } + + +def write_benchmark_set( + *, + market_data_root: Path, + rows: Iterable[Mapping[str, object]], + sources: Sequence[SourcePayload], + start_date: date, + end_date: date, +) -> BenchmarkSet: + _validate_sources(sources) + materialized = _validate_rows(rows, start_date, end_date) + sets_root = Path(market_data_root).resolve() / "benchmark-sets" + sets_root.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=".benchmark-set-", dir=sets_root)) + try: + source_root = staging / "sources" + source_root.mkdir() + for source in sources: + (source_root / source.filename).write_bytes(source.data) + table = pa.Table.from_pylist(materialized, schema=_PARQUET_SCHEMA) + parquet_path = staging / "benchmark-returns.parquet" + pq.write_table( + table, + parquet_path, + compression="zstd", + use_dictionary=False, + write_statistics=True, + ) + source_entries = [ + _source_manifest(staging, source) + for source in sorted(sources, key=lambda item: item.name) + ] + counts = { + benchmark_id: sum( + row["benchmark_id"] == benchmark_id for row in materialized + ) + for benchmark_id in BENCHMARK_IDS + } + ranges = { + benchmark_id: { + "start": min( + row["time"] + for row in materialized + if row["benchmark_id"] == benchmark_id + ), + "end": max( + row["time"] + for row in materialized + if row["benchmark_id"] == benchmark_id + ), + } + for benchmark_id in BENCHMARK_IDS + } + data_ref = { + "path": "benchmark-returns.parquet", + "sha256": _digest(parquet_path), + "bytes": parquet_path.stat().st_size, + "rows": table.num_rows, + "format": "parquet", + "compression": "zstd", + "fields": ["time", "benchmark_id", "returns"], + "unique_key": ["time", "benchmark_id"], + } + identity = { + "schema_version": "benchmark-set/1", + "generator_version": GENERATOR_VERSION, + "requested_range": { + "start": start_date.isoformat(), + "end": end_date.isoformat(), + }, + "source_sha256": { + item["name"]: item["sha256"] for item in source_entries + }, + "data_sha256": data_ref["sha256"], + } + benchmark_set_id = hashlib.sha256(_canonical_bytes(identity)).hexdigest() + manifest = { + "schema_version": "benchmark-set/1", + "benchmark_set_id": benchmark_set_id, + "status": "complete", + "timezone": "Asia/Shanghai", + "requested_range": identity["requested_range"], + "generator": { + "name": "official benchmark set builder", + "version": GENERATOR_VERSION, + }, + "benchmarks": { + BENCHMARK_IDS[0]: { + "currency": "CNY", + "return_kind": "total_return", + "definition": "CSI 300 Total Return Index close-to-close return", + "source_id": "H00300", + "fx_formula": "not_applicable", + "rows": counts[BENCHMARK_IDS[0]], + "date_range": ranges[BENCHMARK_IDS[0]], + }, + BENCHMARK_IDS[1]: { + "currency": "CNY", + "return_kind": "total_return", + "definition": "NASDAQ-100 Total Return Index converted to CNY", + "source_id": "XNDX", + "fx_source_id": "DEXCHUS", + "fx_formula": "(1 + XNDX_USD_total_return) * (1 + USD_CNY_change) - 1", + "rows": counts[BENCHMARK_IDS[1]], + "date_range": ranges[BENCHMARK_IDS[1]], + }, + }, + "sources": source_entries, + "normalization": { + "csi300": "adjacent official H00300 closing levels", + "nasdaq100": "adjacent official XNDX levels; non-positive market-closure placeholders excluded", + "usd_cny": "Federal Reserve CNY per USD observations; ND excluded", + "alignment": "exact-date inner join only; no zero, forward or backward fill", + }, + "data": data_ref, + "gate": { + "status": "pass", + "exceptions": [], + "checks": [ + "official_source_identity", + "source_snapshots", + "no_proxy", + "no_zero_or_forward_fill", + "parquet_digest", + "unique_key", + ], + }, + } + (staging / "manifest.json").write_bytes(_canonical_bytes(manifest) + b"\n") + target = sets_root / benchmark_set_id + if target.exists(): + existing = open_benchmark_set(target) + shutil.rmtree(staging) + return existing + staging.replace(target) + return open_benchmark_set(target) + finally: + if staging.exists(): + shutil.rmtree(staging) + + +def open_benchmark_set(root: Path) -> BenchmarkSet: + result_root = Path(root).resolve() + try: + manifest = json.loads((result_root / "manifest.json").read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise BenchmarkSetError("benchmark manifest is unreadable") from exc + if ( + not isinstance(manifest, dict) + or manifest.get("schema_version") != "benchmark-set/1" + or manifest.get("status") != "complete" + or manifest.get("benchmark_set_id") != result_root.name + or set(manifest.get("benchmarks", {})) != set(BENCHMARK_IDS) + or manifest.get("gate", {}).get("status") != "pass" + or manifest.get("gate", {}).get("exceptions") != [] + ): + raise BenchmarkSetError("benchmark manifest identity is invalid") + sources = manifest.get("sources") + if not isinstance(sources, list) or len(sources) != 3: + raise BenchmarkSetError("benchmark source evidence is incomplete") + for source in sources: + if not isinstance(source, dict): + raise BenchmarkSetError("benchmark source evidence is invalid") + path = (result_root / str(source.get("path", ""))).resolve() + if result_root not in path.parents or not path.is_file(): + raise BenchmarkSetError("benchmark source snapshot is missing") + if path.stat().st_size != source.get("bytes") or _digest(path) != source.get("sha256"): + raise BenchmarkSetError("benchmark source digest or size mismatch") + data_ref = manifest.get("data") + if not isinstance(data_ref, dict) or data_ref.get("path") != "benchmark-returns.parquet": + raise BenchmarkSetError("benchmark data reference is invalid") + data_path = result_root / "benchmark-returns.parquet" + if ( + not data_path.is_file() + or data_path.stat().st_size != data_ref.get("bytes") + or _digest(data_path) != data_ref.get("sha256") + ): + raise BenchmarkSetError("benchmark data digest or size mismatch") + table = pq.read_table(data_path) + if not table.schema.equals(_PARQUET_SCHEMA) or table.num_rows != data_ref.get("rows"): + raise BenchmarkSetError("benchmark Parquet schema or rows are invalid") + rows = table.to_pylist() + requested = manifest.get("requested_range", {}) + validated = _validate_rows( + rows, + date.fromisoformat(str(requested.get("start"))), + date.fromisoformat(str(requested.get("end"))), + ) + if rows != validated: + raise BenchmarkSetError("benchmark rows are not in canonical order") + for benchmark_id in BENCHMARK_IDS: + declared = manifest["benchmarks"][benchmark_id] + current = [row for row in rows if row["benchmark_id"] == benchmark_id] + if declared.get("rows") != len(current): + raise BenchmarkSetError("benchmark row counts do not match the manifest") + return BenchmarkSet( + root=result_root, + benchmark_set_id=result_root.name, + manifest=manifest, + ) + + +def _http_request(request: urllib.request.Request | str) -> tuple[bytes, str]: + if isinstance(request, str): + request = urllib.request.Request(request) + request.add_header("User-Agent", "Quant-Research-Lab/official-benchmark-set") + with urllib.request.urlopen(request, timeout=60) as response: + if response.status != 200: + raise BenchmarkSetError(f"benchmark source returned HTTP {response.status}") + return response.read(), str(response.headers.get("Content-Type", "")) + + +def fetch_official_sources(start_date: date, end_date: date) -> tuple[SourcePayload, ...]: + buffered_start = start_date - timedelta(days=10) + buffered_end = end_date + timedelta(days=1) + csi_url = ( + "https://www.csindex.com.cn/csindex-home/exportExcel/" + "downloadindex-perf?language=CH" + ) + csi_body = _canonical_bytes( + [ + { + "startDate": buffered_start.strftime("%Y%m%d"), + "endDate": buffered_end.strftime("%Y%m%d"), + "indexCode": "H00300", + } + ] + ) + csi_request = urllib.request.Request( + csi_url, + data=csi_body, + headers={ + "Content-Type": "application/json;charset=UTF-8", + "Origin": "https://www.csindex.com.cn", + "Referer": "https://www.csindex.com.cn/", + }, + ) + csi_data, csi_type = _http_request(csi_request) + + nasdaq_url = "https://indexes.nasdaqomx.com/Index/ExportHistory/XNDX?" + urllib.parse.urlencode( + { + "startDate": buffered_start.isoformat() + "T00:00:00.000Z", + "endDate": buffered_end.isoformat() + "T00:00:00.000Z", + "timeOfDay": "EOD", + } + ) + nasdaq_data, nasdaq_type = _http_request(nasdaq_url) + fed_url = "https://www.federalreserve.gov/releases/h10/Hist/dat00_ch.htm" + fed_data, fed_type = _http_request(fed_url) + return ( + SourcePayload( + name="csi300_total_return", + filename="csi300-total-return.xlsx", + provider=_SOURCE_IDENTITIES["csi300_total_return"]["provider"], + source_id="H00300", + url=csi_url, + content_type=csi_type, + data=csi_data, + ), + SourcePayload( + name="nasdaq100_total_return", + filename="nasdaq100-total-return.xlsx", + provider=_SOURCE_IDENTITIES["nasdaq100_total_return"]["provider"], + source_id="XNDX", + url=nasdaq_url, + content_type=nasdaq_type, + data=nasdaq_data, + ), + SourcePayload( + name="usd_cny", + filename="usd-cny.html", + provider=_SOURCE_IDENTITIES["usd_cny"]["provider"], + source_id="DEXCHUS", + url=fed_url, + content_type=fed_type, + data=fed_data, + ), + ) + + +_XML_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}" + + +def _column_index(reference: str) -> int: + letters = re.match(r"[A-Z]+", reference) + if letters is None: + raise BenchmarkSetError("Excel cell reference is invalid") + result = 0 + for character in letters.group(0): + result = result * 26 + ord(character) - ord("A") + 1 + return result - 1 + + +def _xlsx_rows(data: bytes) -> list[list[object | None]]: + try: + with zipfile.ZipFile(BytesIO(data)) as archive: + shared: list[str] = [] + if "xl/sharedStrings.xml" in archive.namelist(): + root = ElementTree.fromstring(archive.read("xl/sharedStrings.xml")) + for item in root.findall(f"{_XML_NS}si"): + shared.append("".join(node.text or "" for node in item.iter(f"{_XML_NS}t"))) + worksheet = ElementTree.fromstring(archive.read("xl/worksheets/sheet1.xml")) + except (KeyError, zipfile.BadZipFile, ElementTree.ParseError) as exc: + raise BenchmarkSetError("official Excel source is invalid") from exc + result: list[list[object | None]] = [] + for row in worksheet.iter(f"{_XML_NS}row"): + values: list[object | None] = [] + for cell in row.findall(f"{_XML_NS}c"): + index = _column_index(str(cell.attrib.get("r", ""))) + while len(values) <= index: + values.append(None) + kind = cell.attrib.get("t") + if kind == "inlineStr": + value: object | None = "".join( + node.text or "" for node in cell.iter(f"{_XML_NS}t") + ) + else: + node = cell.find(f"{_XML_NS}v") + raw = None if node is None else node.text + if raw is None: + value = None + elif kind == "s": + value = shared[int(raw)] + else: + value = float(raw) + values[index] = value + result.append(values) + return result + + +def parse_csi300_levels(data: bytes) -> tuple[BenchmarkLevel, ...]: + rows = _xlsx_rows(data) + if not rows: + raise BenchmarkSetError("CSI source is empty") + headers = {str(value): index for index, value in enumerate(rows[0])} + required = { + "日期Date", + "指数代码Index Code", + "指数英文全称Index English Name(Full)", + "收盘Close", + } + if not required.issubset(headers): + raise BenchmarkSetError("CSI source columns are incomplete") + result: list[BenchmarkLevel] = [] + for row in rows[1:]: + try: + code = str(row[headers["指数代码Index Code"]]) + name = str(row[headers["指数英文全称Index English Name(Full)"]]) + current = datetime.strptime( + str(row[headers["日期Date"]]), "%Y%m%d" + ).date() + close = float(row[headers["收盘Close"]]) + except (IndexError, TypeError, ValueError) as exc: + raise BenchmarkSetError("CSI source row is invalid") from exc + if code != "H00300" or name != "CSI 300 Total Return Index": + raise BenchmarkSetError("CSI source does not prove H00300 total return") + result.append(BenchmarkLevel(current, close)) + return tuple(_validated_levels(result, "CSI300 total return")) + + +def _excel_date(value: object) -> date: + if isinstance(value, (int, float)): + return date(1899, 12, 30) + timedelta(days=int(value)) + if isinstance(value, str): + for format_string in ("%Y-%m-%d", "%m/%d/%Y"): + try: + return datetime.strptime(value, format_string).date() + except ValueError: + continue + raise BenchmarkSetError("NASDAQ source date is invalid") + + +def parse_nasdaq100_levels(data: bytes) -> tuple[BenchmarkLevel, ...]: + rows = _xlsx_rows(data) + if not rows: + raise BenchmarkSetError("NASDAQ source is empty") + headers = {str(value): index for index, value in enumerate(rows[0])} + if not {"Trade Date", "Index Value"}.issubset(headers): + raise BenchmarkSetError("NASDAQ source columns are incomplete") + result: list[BenchmarkLevel] = [] + for row in rows[1:]: + if not row or all(value is None for value in row): + continue + try: + current = _excel_date(row[headers["Trade Date"]]) + value = float(row[headers["Index Value"]]) + except (IndexError, TypeError, ValueError) as exc: + raise BenchmarkSetError("NASDAQ source row is invalid") from exc + if value <= 0: + continue + result.append(BenchmarkLevel(current, value)) + return tuple(_validated_levels(result, "NASDAQ100 total return")) + + +class _FedTableParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.rows: list[list[str]] = [] + self._row: list[str] | None = None + self._cell: list[str] | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "tr": + self._row = [] + elif tag in {"th", "td"} and self._row is not None: + self._cell = [] + + def handle_data(self, data: str) -> None: + if self._cell is not None: + self._cell.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag in {"th", "td"} and self._cell is not None and self._row is not None: + self._row.append("".join(self._cell).strip()) + self._cell = None + elif tag == "tr" and self._row is not None: + if self._row: + self.rows.append(self._row) + self._row = None + + +def parse_usd_cny_levels(data: bytes) -> tuple[BenchmarkLevel, ...]: + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + raise BenchmarkSetError("Federal Reserve source is not UTF-8") from exc + if "Rates in Chinese yuan per U.S. dollar" not in text: + raise BenchmarkSetError("Federal Reserve source currency identity is invalid") + parser = _FedTableParser() + parser.feed(text) + result: list[BenchmarkLevel] = [] + for row in parser.rows: + if len(row) < 2 or row[0] == "Date" or row[1] == "ND": + continue + try: + current = datetime.strptime(row[0].strip(), "%d-%b-%y").date() + value = float(row[1]) + except ValueError: + continue + result.append(BenchmarkLevel(current, value)) + return tuple(_validated_levels(result, "USD/CNY")) + + +def build_official_benchmark_set( + market_data_root: Path, start_date: date, end_date: date +) -> BenchmarkSet: + sources = fetch_official_sources(start_date, end_date) + by_name = {source.name: source for source in sources} + rows = build_benchmark_rows( + csi_levels=parse_csi300_levels(by_name["csi300_total_return"].data), + nasdaq_levels=parse_nasdaq100_levels(by_name["nasdaq100_total_return"].data), + usd_cny_levels=parse_usd_cny_levels(by_name["usd_cny"].data), + start_date=start_date, + end_date=end_date, + ) + return write_benchmark_set( + market_data_root=market_data_root, + rows=rows, + sources=sources, + start_date=start_date, + end_date=end_date, + ) + + +def probe_official_sources(start_date: date, end_date: date) -> dict[str, object]: + sources = fetch_official_sources(start_date, end_date) + by_name = {source.name: source for source in sources} + parsed = { + "csi300_total_return": parse_csi300_levels( + by_name["csi300_total_return"].data + ), + "nasdaq100_total_return": parse_nasdaq100_levels( + by_name["nasdaq100_total_return"].data + ), + "usd_cny": parse_usd_cny_levels(by_name["usd_cny"].data), + } + rows = build_benchmark_rows( + csi_levels=parsed["csi300_total_return"], + nasdaq_levels=parsed["nasdaq100_total_return"], + usd_cny_levels=parsed["usd_cny"], + start_date=start_date, + end_date=end_date, + ) + counts = { + benchmark_id: sum(row["benchmark_id"] == benchmark_id for row in rows) + for benchmark_id in BENCHMARK_IDS + } + if any(value == 0 for value in counts.values()): + raise BenchmarkSetError("official sources have no comparable rows") + return { + "status": "complete", + "requested_range": { + "start": start_date.isoformat(), + "end": end_date.isoformat(), + }, + "source_rows": {name: len(values) for name, values in parsed.items()}, + "benchmark_rows": counts, + "source_sha256": {source.name: source.sha256 for source in sources}, + } + + +def _main() -> int: + parser = argparse.ArgumentParser(description="Build official total-return benchmarks") + parser.add_argument("command", choices=("probe", "build")) + parser.add_argument("--start-date", required=True, type=date.fromisoformat) + parser.add_argument("--end-date", required=True, type=date.fromisoformat) + parser.add_argument( + "--market-data-root", type=Path, default=Path(".local/market-data") + ) + args = parser.parse_args() + try: + if args.command == "probe": + result: object = probe_official_sources(args.start_date, args.end_date) + else: + created = build_official_benchmark_set( + args.market_data_root, args.start_date, args.end_date + ) + result = { + "status": "complete", + "benchmark_set_id": created.benchmark_set_id, + "root": str(created.root), + } + except BenchmarkSetError as exc: + print( + json.dumps( + {"status": "evidence_insufficient", "reason": str(exc)}, + ensure_ascii=False, + ) + ) + return 2 + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/scripts/research/market_data/cli.py b/scripts/research/market_data/cli.py new file mode 100644 index 0000000..a9cafe4 --- /dev/null +++ b/scripts/research/market_data/cli.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Sequence + +from .storage import audit_store + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="共享行情中心维护工具") + subparsers = parser.add_subparsers(dest="command", required=True) + audit = subparsers.add_parser("audit", help="只读校验全部行情批次和快照") + audit.add_argument("--root", type=Path, required=True, help="行情中心根目录") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + if arguments.command == "audit": + print( + json.dumps( + audit_store(root=arguments.root), + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + raise AssertionError(f"unhandled command: {arguments.command}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/research/market_data/contracts.py b/scripts/research/market_data/contracts.py index ae0d2dd..19d2ff7 100644 --- a/scripts/research/market_data/contracts.py +++ b/scripts/research/market_data/contracts.py @@ -1,9 +1,137 @@ from __future__ import annotations +import hashlib +import json +import math from dataclasses import dataclass +from datetime import date from pathlib import Path from types import MappingProxyType -from typing import Any, Mapping, Sequence +from typing import Any, Iterable, Mapping, Sequence + + +MARKET_DATA_FIELDS = ( + "date", + "security", + "open", + "high", + "low", + "close", + "pre_close", + "volume", + "money", + "factor", + "paused", + "high_limit", + "low_limit", +) +CORPORATE_ACTION_FIELDS = ( + "source_event_id", + "security", + "event_type", + "announcement_date", + "record_date", + "ex_date", + "effective_date", + "pay_date", + "status", + "knowledge_cutoff_date", + "split_ratio", + "cash_per_share", + "source", + "source_record_sha256", +) +_NUMERIC_FIELDS = frozenset(MARKET_DATA_FIELDS) - {"date", "security", "paused"} + + +class MarketDataContractError(ValueError): + """Raised when a daily market-data row violates the shared contract.""" + + +def _normalize_number(value: object, field: str) -> float | None: + if value is None or (isinstance(value, str) and not value.strip()): + return None + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise MarketDataContractError(f"invalid numeric value for {field}") from exc + if not math.isfinite(normalized): + raise MarketDataContractError(f"non-finite numeric value for {field}") + return normalized + + +def _normalize_paused(value: object) -> bool | None: + if value is None or (isinstance(value, str) and not value.strip()): + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"true", "false"}: + return lowered == "true" + normalized = _normalize_number(value, "paused") + if normalized not in {0.0, 1.0}: + raise MarketDataContractError("paused must be 0, 1, false or true") + return normalized == 1.0 + + +def normalize_market_rows( + rows: Iterable[Mapping[str, object]], +) -> list[dict[str, object]]: + normalized_by_key: dict[tuple[str, str], dict[str, object]] = {} + for raw in rows: + if set(raw) != set(MARKET_DATA_FIELDS): + raise MarketDataContractError( + "market-data row does not match the fixed field contract" + ) + row_date = str(raw["date"] or "").strip() + security = str(raw["security"] or "").strip() + try: + date.fromisoformat(row_date) + except ValueError as exc: + raise MarketDataContractError(f"invalid date: {row_date}") from exc + if not security: + raise MarketDataContractError("security must be non-empty") + normalized: dict[str, object] = {"date": row_date, "security": security} + for field in MARKET_DATA_FIELDS[2:]: + if field == "paused": + normalized[field] = _normalize_paused(raw[field]) + elif field in _NUMERIC_FIELDS: + normalized[field] = _normalize_number(raw[field], field) + key = (row_date, security) + existing = normalized_by_key.get(key) + if existing is not None and existing != normalized: + raise MarketDataContractError( + f"conflicting normalized row for {security} {row_date}" + ) + normalized_by_key[key] = normalized + return [normalized_by_key[key] for key in sorted(normalized_by_key)] + + +def normalized_digest(rows: Iterable[Mapping[str, object]]) -> str: + canonical_rows = [dict(row) for row in rows] + canonical_rows.sort(key=lambda row: (str(row["date"]), str(row["security"]))) + payload = json.dumps( + canonical_rows, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def corporate_actions_digest(rows: Iterable[Mapping[str, object]]) -> str: + canonical_rows = [dict(row) for row in rows] + canonical_rows.sort(key=lambda row: str(row["source_event_id"])) + payload = json.dumps( + canonical_rows, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() def _deep_freeze(value: Any) -> Any: diff --git a/scripts/research/market_data/economic_returns.py b/scripts/research/market_data/economic_returns.py new file mode 100644 index 0000000..8a0ea8e --- /dev/null +++ b/scripts/research/market_data/economic_returns.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Mapping, Sequence + +import numpy as np +import pandas as pd + +from .contracts import corporate_actions_digest +from .query import SnapshotView + + +_RECONCILIATION_RTOL = 1e-8 +# 聚宽 ETF 日线价格按 0.001 最小价位导出，而官方每份现金可有四位小数。 +_RECONCILIATION_ATOL = 0.000500001 + + +class EconomicReturnError(ValueError): + """Raised when point-in-time economic returns cannot be reconciled.""" + + +@dataclass(frozen=True) +class CorporateActionApplication: + source_event_id: str + security: str + event_type: str + effective_date: str + application_date: str + announcement_date: str + knowledge_cutoff_date: str + evidence_timing: str + split_ratio: float | None + cash_per_share: float | None + cumulative_factor: float + price_basis_changed: bool + source: str + source_record_sha256: str + + +@dataclass(frozen=True) +class ContinuousPriceResult: + frame: pd.DataFrame + returns: pd.Series + applications: tuple[CorporateActionApplication, ...] + + +def _evidence_insufficient(message: str) -> EconomicReturnError: + return EconomicReturnError(f"evidence_insufficient: {message}") + + +def _numeric(frame: pd.DataFrame, field: str) -> pd.Series: + if field not in frame.columns: + raise ValueError(f"missing market field: {field}") + return pd.to_numeric(frame[field], errors="coerce").astype(float) + + +def _action_date(value: object, field: str) -> pd.Timestamp: + try: + timestamp = pd.Timestamp(value).normalize() + except (TypeError, ValueError) as exc: + raise _evidence_insufficient(f"invalid corporate-action {field}") from exc + if pd.isna(timestamp): + raise _evidence_insufficient(f"missing corporate-action {field}") + return timestamp + + +def _positive_action_number(value: object, field: str) -> float: + try: + number = float(value) + except (TypeError, ValueError) as exc: + raise _evidence_insufficient(f"invalid corporate-action {field}") from exc + if not math.isfinite(number) or number <= 0.0: + raise _evidence_insufficient(f"invalid corporate-action {field}") + return number + + +def _actions_by_date( + actions: Sequence[Mapping[str, object]], + *, + security: str, +) -> dict[pd.Timestamp, list[Mapping[str, object]]]: + result: dict[pd.Timestamp, list[Mapping[str, object]]] = {} + event_ids: set[str] = set() + for action in actions: + if str(action.get("security", "")) != security: + continue + event_id = str(action.get("source_event_id", "")) + if not event_id or event_id in event_ids: + raise _evidence_insufficient( + f"duplicate or missing corporate-action identity for {security}" + ) + event_ids.add(event_id) + effective_date = _action_date(action.get("effective_date"), "effective_date") + result.setdefault(effective_date, []).append(action) + return result + + +def _normalize_frame(frame: pd.DataFrame, *, security: str) -> pd.DataFrame: + required = { + "date", + "open", + "high", + "low", + "close", + "pre_close", + "paused", + "high_limit", + "low_limit", + } + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError(f"missing market fields for {security}: {', '.join(missing)}") + if "security" in frame.columns and set(frame["security"].astype(str)) != {security}: + raise ValueError(f"market frame identity mismatch: {security}") + + result = frame.copy() + result["date"] = pd.to_datetime(result["date"], errors="raise").dt.normalize() + result = result.sort_values("date", kind="stable").reset_index(drop=True) + if result["date"].duplicated().any(): + raise ValueError(f"duplicate market date: {security}") + for field in ( + "open", + "high", + "low", + "close", + "pre_close", + "high_limit", + "low_limit", + ): + result[field] = _numeric(result, field) + result["paused"] = result["paused"].astype(bool) + return result + + +def _derive_forward_only_continuity( + frame: pd.DataFrame, + *, + security: str, + corporate_actions: Sequence[Mapping[str, object]], +) -> tuple[pd.DataFrame, tuple[CorporateActionApplication, ...]]: + result = _normalize_frame(frame, security=security) + price_fields = ( + "open", + "high", + "low", + "close", + "pre_close", + "high_limit", + "low_limit", + ) + for field in price_fields: + result[f"raw_{field}"] = result[field] + factor = np.ones(len(result), dtype=np.float64) + applied = np.zeros(len(result), dtype=np.bool_) + applications: list[CorporateActionApplication] = [] + events_by_date = _actions_by_date(corporate_actions, security=security) + available_dates = set(result["date"]) + in_range_events = { + event_date + for event_date in events_by_date + if result["date"].iloc[0] <= event_date <= result["date"].iloc[-1] + } + if not in_range_events.issubset(available_dates): + raise _evidence_insufficient( + f"corporate-action effective date is not a market row for {security}" + ) + + row_by_date = { + current_date: row for row, current_date in enumerate(result["date"]) + } + scheduled_events: dict[ + pd.Timestamp, list[tuple[pd.Timestamp, Mapping[str, object]]] + ] = {} + for effective_date, events in events_by_date.items(): + if effective_date not in in_range_events: + continue + effective_row = row_by_date[effective_date] + for event in events: + announcement_date = _action_date( + event.get("announcement_date"), "announcement_date" + ) + knowledge_cutoff = _action_date( + event.get("knowledge_cutoff_date"), "knowledge_cutoff_date" + ) + if knowledge_cutoff < announcement_date: + raise _evidence_insufficient( + "corporate action was not known by the snapshot cutoff" + ) + status = str(event.get("status", "")) + if status == "cancelled": + continue + if status != "active": + raise _evidence_insufficient("unknown corporate-action status") + event_type = str(event.get("event_type", "")) + if event_type == "split": + _positive_action_number(event.get("split_ratio"), "split_ratio") + elif event_type == "cash_dividend": + _positive_action_number( + event.get("cash_per_share"), "cash_per_share" + ) + else: + raise _evidence_insufficient("unknown corporate-action type") + + application_row = effective_row + if bool(result["paused"].iloc[application_row]): + application_row += 1 + while application_row < len(result) and bool( + result["paused"].iloc[application_row] + ): + application_row += 1 + if application_row >= len(result): + raise _evidence_insufficient( + f"corporate action has no resumed market row for {security}" + ) + application_date = result["date"].iloc[application_row] + scheduled_events.setdefault(application_date, []).append( + (effective_date, event) + ) + + for row in range(1, len(result)): + factor[row] = factor[row - 1] + current_date = result["date"].iloc[row] + previous_close = float(result["raw_close"].iloc[row - 1]) + current_pre_close = float(result["raw_pre_close"].iloc[row]) + if ( + not math.isfinite(previous_close) + or not math.isfinite(current_pre_close) + or previous_close <= 0.0 + or current_pre_close <= 0.0 + ): + raise _evidence_insufficient( + f"invalid close/pre_close reconciliation input for {security} " + f"{current_date.date()}" + ) + scheduled = scheduled_events.get(current_date, []) + basis_changed = not np.isclose( + previous_close, + current_pre_close, + rtol=_RECONCILIATION_RTOL, + atol=_RECONCILIATION_ATOL, + ) + if scheduled: + if basis_changed: + factor[row] *= previous_close / current_pre_close + applied[row] = True + for source_effective_date, event in scheduled: + event_type = str(event["event_type"]) + applications.append( + CorporateActionApplication( + source_event_id=str(event.get("source_event_id", "")), + security=security, + event_type=event_type, + effective_date=source_effective_date.strftime("%Y-%m-%d"), + application_date=current_date.strftime("%Y-%m-%d"), + announcement_date=_action_date( + event.get("announcement_date"), "announcement_date" + ).strftime("%Y-%m-%d"), + knowledge_cutoff_date=_action_date( + event.get("knowledge_cutoff_date"), + "knowledge_cutoff_date", + ).strftime("%Y-%m-%d"), + evidence_timing=( + "point_in_time" + if _action_date( + event.get("announcement_date"), + "announcement_date", + ) + <= source_effective_date + else "retrospective_reconciliation" + ), + split_ratio=( + float(event["split_ratio"]) + if event_type == "split" + else None + ), + cash_per_share=( + float(event["cash_per_share"]) + if event_type == "cash_dividend" + else None + ), + cumulative_factor=float(factor[row]), + price_basis_changed=bool(basis_changed), + source=str(event.get("source", "")), + source_record_sha256=str( + event.get("source_record_sha256", "") + ), + ) + ) + elif basis_changed: + raise _evidence_insufficient( + f"unexplained price-basis change for {security} {current_date.date()}" + ) + + first_date = result["date"].iloc[0] + if scheduled_events.get(first_date): + raise _evidence_insufficient( + f"corporate action on the first market row cannot be reconciled for {security}" + ) + result["continuity_factor"] = factor + result["corporate_action_applied"] = applied + for field in price_fields: + result[field] = result[f"raw_{field}"] * factor + return result, tuple(applications) + + +def derive_continuous_prices( + frame: pd.DataFrame, + *, + security: str, + corporate_actions: Sequence[Mapping[str, object]], +) -> ContinuousPriceResult: + normalized, applications = _derive_forward_only_continuity( + frame, + security=security, + corporate_actions=corporate_actions, + ) + returns = normalized["close"] / normalized["pre_close"] - 1.0 + return ContinuousPriceResult( + frame=normalized, + returns=returns.astype("float64"), + applications=applications, + ) + + +def snapshot_return_panel( + snapshot: SnapshotView, + securities: Sequence[str] | None = None, +) -> pd.DataFrame: + available = {str(row["security"]) for row in snapshot.rows} + selected = tuple(sorted(securities or available)) + unknown = sorted(set(selected) - available) + if unknown: + raise EconomicReturnError( + "snapshot is missing requested securities: " + ", ".join(unknown) + ) + columns: dict[str, pd.Series] = {} + for security in selected: + frame = pd.DataFrame( + dict(row) for row in snapshot.rows if row["security"] == security + ) + actions = [ + row + for row in snapshot.corporate_actions + if row["security"] == security + ] + result = derive_continuous_prices( + frame, + security=security, + corporate_actions=actions, + ) + columns[security] = pd.Series( + result.returns.to_numpy(), + index=pd.to_datetime(result.frame["date"]).dt.normalize(), + dtype="float64", + ) + return pd.DataFrame(columns).sort_index() + + +def canonical_corporate_actions_digest( + actions: Sequence[Mapping[str, object]], +) -> str: + return corporate_actions_digest(actions) diff --git a/scripts/research/market_data/joinquant_export.py b/scripts/research/market_data/joinquant_export.py index 788b11a..f672c85 100644 --- a/scripts/research/market_data/joinquant_export.py +++ b/scripts/research/market_data/joinquant_export.py @@ -6,9 +6,10 @@ from dataclasses import dataclass from datetime import date from pathlib import Path -from typing import Literal, Sequence +from typing import Callable, Literal, Mapping, Sequence -from .query import MARKET_DATA_FIELDS +from .contracts import MARKET_DATA_FIELDS, BatchRecord +from .storage import MarketDataIntegrityError, import_batch _SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") @@ -58,8 +59,12 @@ def render_export_program(request: ExportRequest) -> str: return textwrap.dedent( f"""\ import hashlib + import json import pandas as pd + if 'finance' not in globals() or 'query' not in globals(): + from jqdata import finance, query + SECURITIES = {securities!r} OUTPUT_FIELDS = {fields!r} PRICE_FIELDS = [ @@ -67,6 +72,65 @@ def render_export_program(request: ExportRequest) -> str: ] SNAPSHOT_END_DATE = {request.snapshot_end_date!r} REMOTE_PATH = 'market-data.csv' + CORPORATE_ACTIONS_PATH = 'corporate-actions.csv' + CORPORATE_ACTION_FIELDS = [ + 'source_event_id', 'security', 'event_type', 'announcement_date', + 'record_date', 'ex_date', 'effective_date', 'pay_date', 'status', + 'knowledge_cutoff_date', 'split_ratio', 'cash_per_share', 'source', + 'source_record_sha256' + ] + + + def _text(value): + if value is None or pd.isna(value): + return '' + if hasattr(value, 'strftime'): + return value.strftime('%Y-%m-%d') + return str(value) + + + def _first_text(*values): + for value in values: + normalized = _text(value) + if normalized: + return normalized + return '' + + + def _status_at_cutoff(process_id, cancellation_date): + cancel_date = _text(cancellation_date) + currently_cancelled = ( + not pd.isna(process_id) and int(process_id) == 405003 + ) + if cancel_date: + return 'cancelled' if cancel_date <= SNAPSHOT_END_DATE else 'active' + if currently_cancelled: + raise ValueError( + 'cancelled FUND_DIVIDEND event has no cancellation date' + ) + return 'active' + + + def _write_verified(path, output): + try: + csv_text = output.to_csv(index=False, line_terminator='\\n') + except TypeError: + csv_text = output.to_csv(index=False, lineterminator='\\n') + payload = csv_text.encode('utf-8') + write_file(path, payload, append=False) + remote_bytes = read_file(path) + if isinstance(remote_bytes, str): + remote_bytes = remote_bytes.encode('utf-8') + local_sha256 = hashlib.sha256(payload).hexdigest() + remote_sha256 = hashlib.sha256(remote_bytes).hexdigest() + if local_sha256 != remote_sha256: + raise ValueError('remote readback SHA256 mismatch') + return {{ + 'remote_path': path, + 'sha256': remote_sha256, + 'bytes': len(remote_bytes), + 'rows': len(output), + }} def export_market_data(): @@ -97,32 +161,101 @@ def export_market_data(): output = output[OUTPUT_FIELDS].sort_values( ['date', 'security'], kind='mergesort' ) - try: - csv_text = output.to_csv(index=False, line_terminator='\\n') - except TypeError: - csv_text = output.to_csv(index=False, lineterminator='\\n') - payload = csv_text.encode('utf-8') - write_file(REMOTE_PATH, payload, append=False) - remote_bytes = read_file(REMOTE_PATH) - if isinstance(remote_bytes, str): - remote_bytes = remote_bytes.encode('utf-8') - local_sha256 = hashlib.sha256(payload).hexdigest() - remote_sha256 = hashlib.sha256(remote_bytes).hexdigest() - if local_sha256 != remote_sha256: - raise ValueError('remote readback SHA256 mismatch') - return {{ - 'remote_path': REMOTE_PATH, - 'sha256': remote_sha256, - 'bytes': len(remote_bytes), - 'rows': len(output), + return _write_verified(REMOTE_PATH, output) + + + def export_corporate_actions(): + security_by_code = {{ + security.split('.')[0]: security for security in SECURITIES }} + output_rows = [] + source_fields = [ + 'id', 'code', 'pub_date', 'event_id', 'event', 'process_id', + 'process', 'proportion', 'split_ratio', 'record_date', 'ex_date', + 'fund_paid_date', 'dividend_cancel_date', 'otc_ex_date', 'pay_date' + ] + for code, security in sorted(security_by_code.items()): + raw = finance.run_query( + query(finance.FUND_DIVIDEND) + .filter(finance.FUND_DIVIDEND.code == code) + .limit(5000) + ) + if len(raw) >= 5000: + raise ValueError('FUND_DIVIDEND query may be truncated') + for _, row in raw.iterrows(): + announcement_date = _text(row.get('pub_date')) + if not announcement_date or announcement_date > SNAPSHOT_END_DATE: + continue + event_code = int(row.get('event_id')) + if event_code == 404001: + event_type = 'cash_dividend' + elif event_code in (404002, 404003, 404004, 404005): + event_type = 'split' + else: + raise ValueError('unsupported FUND_DIVIDEND event type') + ex_date = _first_text( + row.get('ex_date'), row.get('otc_ex_date') + ) + record_date = _text(row.get('record_date')) + effective_date = ex_date or record_date + if not effective_date: + raise ValueError('FUND_DIVIDEND event has no effective date') + cancel_date = _text(row.get('dividend_cancel_date')) + status = _status_at_cutoff( + row.get('process_id'), row.get('dividend_cancel_date') + ) + raw_document = {{ + field: _text(row.get(field)) for field in source_fields + }} + source_record_sha256 = hashlib.sha256( + json.dumps( + raw_document, + ensure_ascii=False, + sort_keys=True, + separators=(',', ':'), + ).encode('utf-8') + ).hexdigest() + output_rows.append({{ + 'source_event_id': 'FUND_DIVIDEND:' + _text(row.get('id')), + 'security': security, + 'event_type': event_type, + 'announcement_date': announcement_date, + 'record_date': record_date, + 'ex_date': ex_date, + 'effective_date': effective_date, + 'pay_date': _first_text( + row.get('pay_date'), row.get('fund_paid_date') + ), + 'status': status, + 'knowledge_cutoff_date': SNAPSHOT_END_DATE, + 'split_ratio': ( + _text(row.get('split_ratio')) if event_type == 'split' else '' + ), + 'cash_per_share': ( + _text(row.get('proportion')) + if event_type == 'cash_dividend' + else '' + ), + 'source': 'joinquant.finance.FUND_DIVIDEND', + 'source_record_sha256': source_record_sha256, + }}) + output = pd.DataFrame(output_rows, columns=CORPORATE_ACTION_FIELDS) + if not output.empty: + output = output.sort_values( + ['source_event_id'], kind='mergesort' + ) + return _write_verified(CORPORATE_ACTIONS_PATH, output) def cleanup_export(delete_file): delete_file(REMOTE_PATH) + delete_file(CORPORATE_ACTIONS_PATH) - export_result = export_market_data() + export_result = {{ + 'market_data': export_market_data(), + 'corporate_actions': export_corporate_actions(), + }} """ ) @@ -155,3 +288,77 @@ def verify_transfer( remote_cleaned=remote_cleaned is True, reasons=tuple(reasons), ) + + +def import_verified_transfer( + *, + local_file: Path, + remote_sha256: str, + corporate_actions_file: Path, + corporate_actions_remote_sha256: str, + cleanup_remote: Callable[[], bool], + manifest: Mapping[str, object], + root: Path, +) -> BatchRecord: + """Publish a verified transfer, then confirm remote and local cleanup.""" + transfer_path = Path(local_file) + actions_transfer_path = Path(corporate_actions_file) + evidence = verify_transfer( + local_file=transfer_path, + remote_sha256=remote_sha256, + remote_cleaned=True, + ) + if evidence.status != "complete": + byte_reasons = tuple( + reason + for reason in evidence.reasons + if reason != "remote cleanup is not confirmed" + ) + raise MarketDataIntegrityError( + "transfer validation failed: " + "; ".join(byte_reasons) + ) + actions_evidence = verify_transfer( + local_file=actions_transfer_path, + remote_sha256=corporate_actions_remote_sha256, + remote_cleaned=True, + ) + if actions_evidence.status != "complete": + byte_reasons = tuple( + reason + for reason in actions_evidence.reasons + if reason != "remote cleanup is not confirmed" + ) + raise MarketDataIntegrityError( + "corporate-actions transfer validation failed: " + + "; ".join(byte_reasons) + ) + + record = import_batch( + csv_path=transfer_path, + corporate_actions_csv_path=actions_transfer_path, + manifest=manifest, + root=Path(root), + ) + try: + remote_cleaned = cleanup_remote() + except Exception as exc: + raise MarketDataIntegrityError("remote cleanup is not confirmed") from exc + if remote_cleaned is not True: + raise MarketDataIntegrityError("remote cleanup is not confirmed") + + try: + try: + transfer_path.unlink(missing_ok=True) + actions_transfer_path.unlink(missing_ok=True) + except OSError as exc: + raise MarketDataIntegrityError( + "local transfer cleanup is not confirmed" + ) from exc + finally: + if transfer_path.exists(): + raise MarketDataIntegrityError("local transfer cleanup is not confirmed") + if actions_transfer_path.exists(): + raise MarketDataIntegrityError( + "local corporate-actions transfer cleanup is not confirmed" + ) + return record diff --git a/scripts/research/market_data/query.py b/scripts/research/market_data/query.py index 89621d1..73727db 100644 --- a/scripts/research/market_data/query.py +++ b/scripts/research/market_data/query.py @@ -1,36 +1,22 @@ from __future__ import annotations -import csv -import hashlib -import json -import math from dataclasses import dataclass -from datetime import date from pathlib import Path from types import MappingProxyType from typing import Iterable, Mapping, Sequence import duckdb +import pyarrow.parquet as pq -from .storage import MarketDataIntegrityError, validate_snapshot - - -MARKET_DATA_FIELDS = ( - "date", - "security", - "open", - "high", - "low", - "close", - "pre_close", - "volume", - "money", - "factor", - "paused", - "high_limit", - "low_limit", +from .contracts import ( + CORPORATE_ACTION_FIELDS, + MARKET_DATA_FIELDS, + MarketDataContractError, + corporate_actions_digest, + normalize_market_rows, + normalized_digest, ) -_NUMERIC_FIELDS = frozenset(MARKET_DATA_FIELDS) - {"date", "security", "paused"} +from .storage import MarketDataIntegrityError, validate_snapshot @dataclass(frozen=True) @@ -39,39 +25,22 @@ class SnapshotView: fields: tuple[str, ...] rows: tuple[Mapping[str, object], ...] digest: str + corporate_action_fields: tuple[str, ...] + corporate_actions: tuple[Mapping[str, object], ...] + corporate_actions_digest: str -def normalized_digest(rows: Iterable[Mapping[str, object]]) -> str: - canonical_rows = [dict(row) for row in rows] - canonical_rows.sort( - key=lambda row: ( - str(row.get("date", "")), - str(row.get("security", "")), - json.dumps( - row, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ), - ) - ) - payload = json.dumps( - canonical_rows, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - return hashlib.sha256(payload).hexdigest() - - -def _read_query_rows(connection, csv_paths: Sequence[Path]) -> list[dict[str, object]]: - relation = connection.read_csv( - [str(path) for path in csv_paths], - header=True, - all_varchar=True, - ) +@dataclass(frozen=True) +class SnapshotOverlapEvidence: + left_snapshot_id: str + right_snapshot_id: str + securities: tuple[str, ...] + market_digest: str + corporate_actions_digest: str + + +def _read_query_rows(connection, parquet_paths: Sequence[Path]) -> list[dict[str, object]]: + relation = connection.read_parquet([str(path) for path in parquet_paths]) columns = tuple(relation.columns) if columns != MARKET_DATA_FIELDS: raise MarketDataIntegrityError( @@ -80,47 +49,46 @@ def _read_query_rows(connection, csv_paths: Sequence[Path]) -> list[dict[str, ob return [dict(zip(columns, values)) for values in relation.fetchall()] -def _read_csv_rows(csv_paths: Sequence[Path]) -> list[dict[str, object]]: +def _read_parquet_rows(parquet_paths: Sequence[Path]) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] - for path in csv_paths: - with path.open(encoding="utf-8-sig", newline="") as handle: - reader = csv.DictReader(handle) - if tuple(reader.fieldnames or ()) != MARKET_DATA_FIELDS: - raise MarketDataIntegrityError( - "CSV field order does not match the fixed market-data contract" - ) - rows.extend(dict(row) for row in reader) + for path in parquet_paths: + table = pq.read_table(path) + if tuple(table.column_names) != MARKET_DATA_FIELDS: + raise MarketDataIntegrityError( + "Parquet field order does not match the fixed market-data contract" + ) + rows.extend(table.to_pylist()) return rows -def _normalize_number(value: object, field: str) -> float | None: - if value is None or (isinstance(value, str) and not value.strip()): - return None - try: - normalized = float(value) - except (TypeError, ValueError) as exc: - raise MarketDataIntegrityError(f"invalid numeric value for {field}") from exc - if not math.isfinite(normalized): - raise MarketDataIntegrityError(f"non-finite numeric value for {field}") - return normalized - - -def _normalize_paused(value: object) -> bool | None: - if value is None or (isinstance(value, str) and not value.strip()): - return None - if isinstance(value, bool): - return value - if isinstance(value, str): - lowered = value.strip().lower() - if lowered in {"true", "false"}: - return lowered == "true" - normalized = _normalize_number(value, "paused") - if normalized not in {0.0, 1.0}: - raise MarketDataIntegrityError("paused must be 0, 1, false or true") - return normalized == 1.0 +def _read_corporate_action_rows( + connection, + parquet_paths: Sequence[Path], +) -> list[dict[str, object]]: + relation = connection.read_parquet([str(path) for path in parquet_paths]) + columns = tuple(relation.columns) + if columns != CORPORATE_ACTION_FIELDS: + raise MarketDataIntegrityError( + "DuckDB corporate-actions field order does not match the contract" + ) + return [dict(zip(columns, values)) for values in relation.fetchall()] -def _normalize_selected_rows( +def _read_corporate_action_parquet_rows( + parquet_paths: Sequence[Path], +) -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + for path in parquet_paths: + table = pq.read_table(path) + if tuple(table.column_names) != CORPORATE_ACTION_FIELDS: + raise MarketDataIntegrityError( + "corporate-actions Parquet field order does not match the contract" + ) + rows.extend(table.to_pylist()) + return rows + + +def _select_corporate_actions( rows: Iterable[Mapping[str, object]], *, securities: Sequence[str], @@ -128,37 +96,41 @@ def _normalize_selected_rows( end_date: str, ) -> list[dict[str, object]]: selected = set(securities) - normalized_by_key: dict[tuple[str, str], dict[str, object]] = {} + by_id: dict[str, dict[str, object]] = {} for raw in rows: - if set(raw) != set(MARKET_DATA_FIELDS): - raise MarketDataIntegrityError( - "market-data row does not match the fixed field contract" - ) - row_date = str(raw["date"] or "").strip() - security = str(raw["security"] or "").strip() - try: - date.fromisoformat(row_date) - except ValueError as exc: - raise MarketDataIntegrityError(f"invalid date: {row_date}") from exc - if security not in selected or not start_date <= row_date <= end_date: + row = dict(raw) + if row.get("security") not in selected: + continue + effective_date = str(row.get("effective_date") or "") + if not start_date <= effective_date <= end_date: continue - normalized: dict[str, object] = { - "date": row_date, - "security": security, - } - for field in MARKET_DATA_FIELDS[2:]: - if field == "paused": - normalized[field] = _normalize_paused(raw[field]) - elif field in _NUMERIC_FIELDS: - normalized[field] = _normalize_number(raw[field], field) - key = (row_date, security) - existing = normalized_by_key.get(key) - if existing is not None and existing != normalized: + event_id = str(row.get("source_event_id") or "") + existing = by_id.get(event_id) + if existing is not None and existing != row: raise MarketDataIntegrityError( - f"conflicting normalized row for {security} {row_date}" + f"snapshot corporate-actions conflict at {event_id}" ) - normalized_by_key[key] = normalized - return [normalized_by_key[key] for key in sorted(normalized_by_key)] + by_id[event_id] = row + return [by_id[key] for key in sorted(by_id)] + + +def _normalize_selected_rows( + rows: Iterable[Mapping[str, object]], + *, + securities: Sequence[str], + start_date: str, + end_date: str, +) -> list[dict[str, object]]: + selected = set(securities) + try: + normalized = normalize_market_rows(rows) + except MarketDataContractError as exc: + raise MarketDataIntegrityError(str(exc)) from exc + return [ + row + for row in normalized + if row["security"] in selected and start_date <= str(row["date"]) <= end_date + ] def open_snapshot(snapshot_id: str, *, root: Path) -> SnapshotView: @@ -171,10 +143,20 @@ def open_snapshot(snapshot_id: str, *, root: Path) -> SnapshotView: ) batch_ids = tuple(str(batch_id) for batch_id in snapshot.document["batch_ids"]) - csv_paths = [Path(root) / "batches" / batch_id / "market-data.csv" for batch_id in batch_ids] + parquet_paths = [ + Path(root) / "batches" / batch_id / "market-data.parquet" + for batch_id in batch_ids + ] + corporate_action_paths = [ + Path(root) / "batches" / batch_id / "corporate-actions.parquet" + for batch_id in batch_ids + ] connection = duckdb.connect(":memory:") try: - query_rows = _read_query_rows(connection, csv_paths) + query_rows = _read_query_rows(connection, parquet_paths) + query_action_rows = _read_corporate_action_rows( + connection, corporate_action_paths + ) finally: connection.close() @@ -184,22 +166,95 @@ def open_snapshot(snapshot_id: str, *, root: Path) -> SnapshotView: "end_date": str(selection["end_date"]), } normalized_query_rows = _normalize_selected_rows(query_rows, **selection_args) - normalized_csv_rows = _normalize_selected_rows( - _read_csv_rows(csv_paths), + normalized_parquet_rows = _normalize_selected_rows( + _read_parquet_rows(parquet_paths), **selection_args, ) query_digest = normalized_digest(normalized_query_rows) - if query_digest != normalized_digest(normalized_csv_rows): + if query_digest != normalized_digest(normalized_parquet_rows): + raise MarketDataIntegrityError( + "DuckDB and authoritative Parquet normalized digest mismatch" + ) + selected_query_actions = _select_corporate_actions( + query_action_rows, + **selection_args, + ) + selected_parquet_actions = _select_corporate_actions( + _read_corporate_action_parquet_rows(corporate_action_paths), + **selection_args, + ) + action_digest = corporate_actions_digest(selected_query_actions) + if action_digest != corporate_actions_digest(selected_parquet_actions): raise MarketDataIntegrityError( - "DuckDB and authoritative CSV normalized digest mismatch" + "DuckDB and authoritative corporate-actions Parquet digest mismatch" ) immutable_rows = tuple( MappingProxyType(dict(row)) for row in normalized_query_rows ) + immutable_actions = tuple( + MappingProxyType(dict(row)) for row in selected_query_actions + ) return SnapshotView( snapshot_id=snapshot.snapshot_id, fields=fields, rows=immutable_rows, digest=query_digest, + corporate_action_fields=CORPORATE_ACTION_FIELDS, + corporate_actions=immutable_actions, + corporate_actions_digest=action_digest, + ) + + +def validate_snapshot_overlap( + left_snapshot_id: str, + right_snapshot_id: str, + *, + root: Path, +) -> SnapshotOverlapEvidence: + left_record = validate_snapshot(left_snapshot_id, root=root) + right_record = validate_snapshot(right_snapshot_id, root=root) + if tuple(left_record.document["batch_ids"]) != tuple( + right_record.document["batch_ids"] + ): + raise MarketDataIntegrityError("snapshot overlap batch identities differ") + left_selection = dict(left_record.document["selection"]) + right_selection = dict(right_record.document["selection"]) + left_selection.pop("securities", None) + right_selection.pop("securities", None) + if left_selection != right_selection: + raise MarketDataIntegrityError("snapshot overlap semantics differ") + + left = open_snapshot(left_snapshot_id, root=root) + right = open_snapshot(right_snapshot_id, root=root) + left_securities = {str(row["security"]) for row in left.rows} + right_securities = {str(row["security"]) for row in right.rows} + overlap = tuple(sorted(left_securities & right_securities)) + if not overlap: + raise MarketDataIntegrityError("snapshot overlap has no common securities") + + left_market = normalized_digest( + row for row in left.rows if row["security"] in overlap + ) + right_market = normalized_digest( + row for row in right.rows if row["security"] in overlap + ) + if left_market != right_market: + raise MarketDataIntegrityError("snapshot overlap market digest differs") + left_actions = corporate_actions_digest( + row for row in left.corporate_actions if row["security"] in overlap + ) + right_actions = corporate_actions_digest( + row for row in right.corporate_actions if row["security"] in overlap + ) + if left_actions != right_actions: + raise MarketDataIntegrityError( + "snapshot overlap corporate-actions digest differs" + ) + return SnapshotOverlapEvidence( + left_snapshot_id=left_snapshot_id, + right_snapshot_id=right_snapshot_id, + securities=overlap, + market_digest=left_market, + corporate_actions_digest=left_actions, ) diff --git a/scripts/research/market_data/storage.py b/scripts/research/market_data/storage.py index 6871a96..1cdea9e 100644 --- a/scripts/research/market_data/storage.py +++ b/scripts/research/market_data/storage.py @@ -14,11 +14,31 @@ from pathlib import Path from typing import Any, Iterable, Mapping, Sequence -from .contracts import BatchRecord, SnapshotRecord, SnapshotSelection - - -_BATCH_FILES = {"manifest.json", "market-data.csv", "validation.json"} -_REQUIRED_MANIFEST_FIELDS = { +import duckdb +import pyarrow as pa +import pyarrow.parquet as pq + +from .contracts import ( + CORPORATE_ACTION_FIELDS, + MARKET_DATA_FIELDS, + BatchRecord, + MarketDataContractError, + SnapshotRecord, + SnapshotSelection, + corporate_actions_digest, + normalize_market_rows, + normalized_digest, +) + + +_BATCH_FILES = { + "manifest.json", + "market-data.parquet", + "corporate-actions.parquet", + "validation.json", +} +_LEGACY_BATCH_FILES = {"manifest.json", "market-data.csv", "validation.json"} +_BASE_MANIFEST_FIELDS = { "schema_version", "source", "asset_type", @@ -27,8 +47,50 @@ "price_semantics", "export_code_sha256", } -_STORED_MANIFEST_FIELDS = _REQUIRED_MANIFEST_FIELDS | {"csv", "securities"} +_REQUIRED_MANIFEST_FIELDS = _BASE_MANIFEST_FIELDS | {"corporate_actions"} +_STORED_MANIFEST_FIELDS = _REQUIRED_MANIFEST_FIELDS | { + "content_sha256", + "transport_csv", + "parquet", + "securities", + "writer", +} +_LEGACY_STORED_MANIFEST_FIELDS = _BASE_MANIFEST_FIELDS | { + "csv", + "securities", +} _SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") +_PARQUET_SCHEMA = pa.schema( + [ + pa.field("date", pa.string(), nullable=False), + pa.field("security", pa.string(), nullable=False), + *[ + pa.field(field, pa.float64(), nullable=True) + for field in MARKET_DATA_FIELDS[2:10] + ], + pa.field("paused", pa.bool_(), nullable=True), + pa.field("high_limit", pa.float64(), nullable=True), + pa.field("low_limit", pa.float64(), nullable=True), + ] +) +_CORPORATE_ACTION_SCHEMA = pa.schema( + [ + pa.field("source_event_id", pa.string(), nullable=False), + pa.field("security", pa.string(), nullable=False), + pa.field("event_type", pa.string(), nullable=False), + pa.field("announcement_date", pa.string(), nullable=False), + pa.field("record_date", pa.string(), nullable=True), + pa.field("ex_date", pa.string(), nullable=True), + pa.field("effective_date", pa.string(), nullable=False), + pa.field("pay_date", pa.string(), nullable=True), + pa.field("status", pa.string(), nullable=False), + pa.field("knowledge_cutoff_date", pa.string(), nullable=False), + pa.field("split_ratio", pa.float64(), nullable=True), + pa.field("cash_per_share", pa.float64(), nullable=True), + pa.field("source", pa.string(), nullable=False), + pa.field("source_record_sha256", pa.string(), nullable=False), + ] +) class MarketDataError(RuntimeError): @@ -134,14 +196,23 @@ def _require_mapping(value: object, field: str) -> dict[str, object]: return dict(value) -def _validate_manifest_input(manifest: Mapping[str, object]) -> dict[str, object]: - missing = sorted(_REQUIRED_MANIFEST_FIELDS - set(manifest)) +def _validate_manifest_input( + manifest: Mapping[str, object], + *, + require_corporate_actions: bool = True, +) -> dict[str, object]: + required = ( + _REQUIRED_MANIFEST_FIELDS + if require_corporate_actions + else _BASE_MANIFEST_FIELDS + ) + missing = sorted(required - set(manifest)) if missing: raise MarketDataIntegrityError( f"manifest is missing required fields: {', '.join(missing)}" ) - if manifest["schema_version"] != 1: - raise MarketDataIntegrityError("manifest schema_version must be 1") + if manifest["schema_version"] not in {1, 2, 3}: + raise MarketDataIntegrityError("manifest schema_version must be 1, 2 or 3") source = _require_mapping(manifest["source"], "source") if source.get("name") != "joinquant": @@ -169,8 +240,49 @@ def _validate_manifest_input(manifest: Mapping[str, object]) -> dict[str, object ): raise MarketDataIntegrityError("export_code_sha256 must be a SHA256 digest") - return { - "schema_version": 1, + corporate_actions: dict[str, object] | None = None + if require_corporate_actions: + corporate_actions = _require_mapping( + manifest["corporate_actions"], "corporate_actions" + ) + action_keys = set(corporate_actions) + required_action_keys = {"source", "knowledge_cutoff_date"} + allowed_action_keys = required_action_keys | { + "status", + "content_sha256", + "transport_csv", + "parquet", + "rows", + } + if not required_action_keys.issubset(action_keys) or not action_keys.issubset( + allowed_action_keys + ): + raise MarketDataIntegrityError( + "corporate_actions manifest structure is invalid" + ) + action_source = _require_mapping( + corporate_actions["source"], "corporate_actions.source" + ) + cutoff = str(corporate_actions["knowledge_cutoff_date"] or "") + try: + date.fromisoformat(cutoff) + except ValueError as exc: + raise MarketDataIntegrityError( + "corporate_actions.knowledge_cutoff_date must use YYYY-MM-DD" + ) from exc + status = corporate_actions.get("status") + if status not in {None, "complete", "verified_empty"}: + raise MarketDataIntegrityError( + "corporate_actions.status must be complete or verified_empty" + ) + corporate_actions = { + "source": action_source, + "knowledge_cutoff_date": cutoff, + **({"status": status} if status is not None else {}), + } + + declared = { + "schema_version": 3 if require_corporate_actions else int(manifest["schema_version"]), "source": source, "asset_type": "etf", "frequency": "1d", @@ -178,6 +290,9 @@ def _validate_manifest_input(manifest: Mapping[str, object]) -> dict[str, object "price_semantics": price_semantics, "export_code_sha256": export_digest.lower(), } + if corporate_actions is not None: + declared["corporate_actions"] = corporate_actions + return declared def _read_csv( @@ -233,7 +348,359 @@ def _read_csv( return rows, securities -def _batch_identity(manifest: Mapping[str, object], csv_sha256: str) -> dict[str, object]: +def _optional_iso_date(value: object, field: str) -> str | None: + text = str(value or "").strip() + if not text: + return None + try: + date.fromisoformat(text) + except ValueError as exc: + raise MarketDataIntegrityError(f"{field} must use YYYY-MM-DD") from exc + return text + + +def _positive_optional_number(value: object, field: str) -> float | None: + text = str(value or "").strip() + if not text: + return None + try: + number = float(text) + except (TypeError, ValueError) as exc: + raise MarketDataIntegrityError(f"{field} must be numeric") from exc + if not number > 0.0: + raise MarketDataIntegrityError(f"{field} must be positive") + return number + + +def _normalize_corporate_action_rows( + rows: Iterable[Mapping[str, object]], + *, + declared: Mapping[str, object], +) -> list[dict[str, object]]: + action_contract = _require_mapping( + declared["corporate_actions"], "corporate_actions" + ) + cutoff = str(action_contract["knowledge_cutoff_date"]) + source_identity = _require_mapping( + action_contract["source"], "corporate_actions.source" + ) + source_name = ".".join( + str(source_identity[key]) + for key in ("name", "dataset") + if source_identity.get(key) + ) + normalized_by_id: dict[str, dict[str, object]] = {} + for raw in rows: + if set(raw) != set(CORPORATE_ACTION_FIELDS): + raise MarketDataIntegrityError( + "corporate-action row does not match the fixed field contract" + ) + event_id = str(raw["source_event_id"] or "").strip() + security = str(raw["security"] or "").strip() + event_type = str(raw["event_type"] or "").strip() + status = str(raw["status"] or "").strip() + source = str(raw["source"] or "").strip() + source_record_sha256 = str(raw["source_record_sha256"] or "").lower() + if not event_id or not security: + raise MarketDataIntegrityError( + "corporate-action source_event_id and security must be non-empty" + ) + if event_type not in {"split", "cash_dividend"}: + raise MarketDataIntegrityError( + "corporate-action event_type must be split or cash_dividend" + ) + if status not in {"active", "cancelled"}: + raise MarketDataIntegrityError( + "corporate-action status must be active or cancelled" + ) + if source != source_name: + raise MarketDataIntegrityError( + "corporate-action source does not match the manifest" + ) + if _SHA256_PATTERN.fullmatch(source_record_sha256) is None: + raise MarketDataIntegrityError( + "corporate-action source_record_sha256 must be a SHA256 digest" + ) + + announcement_date = _optional_iso_date( + raw["announcement_date"], "announcement_date" + ) + effective_date = _optional_iso_date( + raw["effective_date"], "effective_date" + ) + if announcement_date is None or effective_date is None: + raise MarketDataIntegrityError( + "corporate-action announcement_date and effective_date are required" + ) + knowledge_cutoff_date = _optional_iso_date( + raw["knowledge_cutoff_date"], "knowledge_cutoff_date" + ) + if knowledge_cutoff_date != cutoff: + raise MarketDataIntegrityError( + "corporate-action knowledge_cutoff_date does not match the manifest" + ) + if announcement_date > cutoff: + raise MarketDataIntegrityError( + "corporate-action was not known by the knowledge cutoff" + ) + record_date = _optional_iso_date(raw["record_date"], "record_date") + ex_date = _optional_iso_date(raw["ex_date"], "ex_date") + pay_date = _optional_iso_date(raw["pay_date"], "pay_date") + if ex_date is not None and ex_date != effective_date: + raise MarketDataIntegrityError( + "corporate-action ex_date must equal effective_date when provided" + ) + if pay_date is not None and pay_date < effective_date: + raise MarketDataIntegrityError( + "corporate-action pay_date must not precede effective_date" + ) + split_ratio = _positive_optional_number(raw["split_ratio"], "split_ratio") + cash_per_share = _positive_optional_number( + raw["cash_per_share"], "cash_per_share" + ) + if event_type == "split" and ( + split_ratio is None or cash_per_share is not None + ): + raise MarketDataIntegrityError( + "split requires split_ratio and forbids cash_per_share" + ) + if event_type == "cash_dividend" and ( + cash_per_share is None or split_ratio is not None + ): + raise MarketDataIntegrityError( + "cash_dividend requires cash_per_share and forbids split_ratio" + ) + + normalized = { + "source_event_id": event_id, + "security": security, + "event_type": event_type, + "announcement_date": announcement_date, + "record_date": record_date, + "ex_date": ex_date, + "effective_date": effective_date, + "pay_date": pay_date, + "status": status, + "knowledge_cutoff_date": knowledge_cutoff_date, + "split_ratio": split_ratio, + "cash_per_share": cash_per_share, + "source": source, + "source_record_sha256": source_record_sha256, + } + existing = normalized_by_id.get(event_id) + if existing is not None and existing != normalized: + raise MarketDataIntegrityError( + f"conflicting corporate-action event: {event_id}" + ) + normalized_by_id[event_id] = normalized + return [normalized_by_id[key] for key in sorted(normalized_by_id)] + + +def _read_corporate_actions_csv( + path: Path | None, + *, + declared: Mapping[str, object], +) -> tuple[bytes, list[dict[str, object]]]: + action_contract = _require_mapping( + declared["corporate_actions"], "corporate_actions" + ) + if path is None: + if action_contract.get("status") != "verified_empty": + raise MarketDataIntegrityError( + "corporate-actions CSV is required unless verified_empty is declared" + ) + return b"", [] + try: + csv_bytes = Path(path).read_bytes() + with Path(path).open(encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + if reader.fieldnames != list(CORPORATE_ACTION_FIELDS): + raise MarketDataIntegrityError( + "corporate-actions CSV field order does not match the contract" + ) + raw_rows = [dict(row) for row in reader] + except UnicodeDecodeError as exc: + raise MarketDataIntegrityError( + "corporate-actions CSV must use UTF-8 encoding" + ) from exc + if any(None in row or any(value is None for value in row.values()) for row in raw_rows): + raise MarketDataIntegrityError( + "corporate-actions CSV row column count does not match the contract" + ) + rows = _normalize_corporate_action_rows(raw_rows, declared=declared) + declared_status = action_contract.get("status") + if declared_status == "verified_empty" and rows: + raise MarketDataIntegrityError( + "corporate-actions manifest declares verified_empty but rows exist" + ) + if declared_status == "complete" and not rows: + raise MarketDataIntegrityError( + "corporate-actions manifest declares complete but no rows exist" + ) + return csv_bytes, rows + + +def _normalize_rows(rows: Iterable[Mapping[str, object]]) -> list[dict[str, object]]: + try: + return normalize_market_rows(rows) + except MarketDataContractError as exc: + raise MarketDataIntegrityError(str(exc)) from exc + + +def _security_coverage(rows: Iterable[Mapping[str, object]]) -> list[dict[str, object]]: + security_dates: dict[str, list[str]] = defaultdict(list) + for row in rows: + security_dates[str(row["security"])].append(str(row["date"])) + return [ + { + "security": security, + "start_date": min(dates), + "end_date": max(dates), + "rows": len(dates), + } + for security, dates in sorted(security_dates.items()) + ] + + +def _parquet_bytes(rows: Sequence[Mapping[str, object]]) -> bytes: + table = pa.Table.from_pylist([dict(row) for row in rows], schema=_PARQUET_SCHEMA) + sink = pa.BufferOutputStream() + pq.write_table( + table, + sink, + compression="zstd", + use_dictionary=False, + write_statistics=True, + ) + return sink.getvalue().to_pybytes() + + +def _corporate_actions_parquet_bytes( + rows: Sequence[Mapping[str, object]], +) -> bytes: + table = pa.Table.from_pylist( + [dict(row) for row in rows], schema=_CORPORATE_ACTION_SCHEMA + ) + sink = pa.BufferOutputStream() + pq.write_table( + table, + sink, + compression="zstd", + use_dictionary=False, + write_statistics=True, + ) + return sink.getvalue().to_pybytes() + + +def _read_parquet(path: Path) -> list[dict[str, object]]: + try: + table = pq.read_table(path) + except (OSError, pa.ArrowException) as exc: + raise MarketDataIntegrityError(f"invalid Parquet evidence: {path}") from exc + if tuple(table.column_names) != MARKET_DATA_FIELDS: + raise MarketDataIntegrityError( + "Parquet field order does not match the fixed market-data contract" + ) + return _normalize_rows(table.to_pylist()) + + +def _read_corporate_actions_parquet( + path: Path, + *, + declared: Mapping[str, object], +) -> list[dict[str, object]]: + try: + table = pq.read_table(path) + except (OSError, pa.ArrowException) as exc: + raise MarketDataIntegrityError( + f"invalid corporate-actions Parquet evidence: {path}" + ) from exc + if tuple(table.column_names) != CORPORATE_ACTION_FIELDS: + raise MarketDataIntegrityError( + "corporate-actions Parquet field order does not match the contract" + ) + return _normalize_corporate_action_rows(table.to_pylist(), declared=declared) + + +def _duckdb_roundtrip(parquet_bytes: bytes, *, root: Path) -> list[dict[str, object]]: + Path(root).mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=".market-data-import-", suffix=".parquet", dir=root + ) + os.close(descriptor) + temporary = Path(temporary_name) + try: + temporary.write_bytes(parquet_bytes) + connection = duckdb.connect(":memory:") + try: + relation = connection.read_parquet(str(temporary)) + if tuple(relation.columns) != MARKET_DATA_FIELDS: + raise MarketDataIntegrityError( + "DuckDB Parquet field order does not match the contract" + ) + rows = [ + dict(zip(relation.columns, values)) for values in relation.fetchall() + ] + finally: + connection.close() + return _normalize_rows(rows) + finally: + temporary.unlink(missing_ok=True) + + +def _corporate_actions_duckdb_roundtrip( + parquet_bytes: bytes, + *, + declared: Mapping[str, object], + root: Path, +) -> list[dict[str, object]]: + Path(root).mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=".corporate-actions-import-", suffix=".parquet", dir=root + ) + os.close(descriptor) + temporary = Path(temporary_name) + try: + temporary.write_bytes(parquet_bytes) + connection = duckdb.connect(":memory:") + try: + relation = connection.read_parquet(str(temporary)) + if tuple(relation.columns) != CORPORATE_ACTION_FIELDS: + raise MarketDataIntegrityError( + "DuckDB corporate-actions field order does not match the contract" + ) + rows = [ + dict(zip(relation.columns, values)) for values in relation.fetchall() + ] + finally: + connection.close() + return _normalize_corporate_action_rows(rows, declared=declared) + finally: + temporary.unlink(missing_ok=True) + + +def _batch_identity( + manifest: Mapping[str, object], + content_sha256: str, + corporate_actions_content_sha256: str, +) -> dict[str, object]: + return { + "schema_version": 3, + "source": manifest["source"], + "asset_type": manifest["asset_type"], + "frequency": manifest["frequency"], + "fields": manifest["fields"], + "price_semantics": manifest["price_semantics"], + "export_code_sha256": manifest["export_code_sha256"], + "content_sha256": content_sha256, + "corporate_actions": manifest["corporate_actions"], + "corporate_actions_content_sha256": corporate_actions_content_sha256, + } + + +def _legacy_batch_identity( + manifest: Mapping[str, object], csv_sha256: str +) -> dict[str, object]: return { "source": manifest["source"], "asset_type": manifest["asset_type"], @@ -254,6 +721,25 @@ def _dataset_identity(manifest: Mapping[str, object]) -> tuple[bytes, object, ob def _validation_document() -> dict[str, object]: + return { + "schema_version": 3, + "status": "complete", + "checks": { + "field_order": True, + "nonempty": True, + "unique_date_security": True, + "parquet_roundtrip": True, + "normalized_digest": True, + "corporate_actions_field_order": True, + "corporate_actions_primary_key": True, + "corporate_actions_point_in_time": True, + "corporate_actions_parquet_roundtrip": True, + "corporate_actions_normalized_digest": True, + }, + } + + +def _legacy_validation_document() -> dict[str, object]: return { "schema_version": 1, "status": "complete", @@ -280,6 +766,10 @@ def _validate_batch_dir(batch_dir: Path) -> dict[str, Any]: if not batch_dir.is_dir(): raise MarketDataIntegrityError(f"batch does not exist: {batch_dir.name}") names = {path.name for path in batch_dir.iterdir()} + if names == _LEGACY_BATCH_FILES: + raise MarketDataIntegrityError( + f"legacy CSV batch requires migration: {batch_dir.name}" + ) if names != _BATCH_FILES: raise MarketDataIntegrityError( f"batch file set is invalid: {batch_dir.name}" @@ -288,41 +778,114 @@ def _validate_batch_dir(batch_dir: Path) -> dict[str, Any]: if set(manifest) != _STORED_MANIFEST_FIELDS: raise MarketDataIntegrityError("batch manifest structure is invalid") declared = _validate_manifest_input(manifest) - csv_evidence = manifest.get("csv") - if not isinstance(csv_evidence, Mapping) or set(csv_evidence) != { + if manifest.get("schema_version") != 3: + raise MarketDataIntegrityError("batch schema_version must be 3") + transport_evidence = manifest.get("transport_csv") + if not isinstance(transport_evidence, Mapping) or set(transport_evidence) != { "sha256", - "bytes", + "byte_count", "rows", }: - raise MarketDataIntegrityError("batch manifest is missing CSV evidence") - expected_sha = csv_evidence.get("sha256") - csv_path = batch_dir / "market-data.csv" - csv_bytes = csv_path.read_bytes() - actual_sha = _sha256_bytes(csv_bytes) + raise MarketDataIntegrityError( + "batch manifest is missing transport CSV evidence" + ) + parquet_evidence = manifest.get("parquet") + if not isinstance(parquet_evidence, Mapping) or set(parquet_evidence) != { + "sha256", + "byte_count", + "rows", + }: + raise MarketDataIntegrityError("batch manifest is missing Parquet evidence") + parquet_path = batch_dir / "market-data.parquet" + parquet_bytes = parquet_path.read_bytes() + actual_sha = _sha256_bytes(parquet_bytes) + expected_sha = parquet_evidence.get("sha256") if actual_sha != expected_sha: raise MarketDataIntegrityError( - f"CSV SHA256 mismatch for batch {batch_dir.name}" + f"Parquet SHA256 mismatch for batch {batch_dir.name}" ) - if len(csv_bytes) != csv_evidence.get("bytes"): + if len(parquet_bytes) != parquet_evidence.get("byte_count"): + raise MarketDataIntegrityError( + f"Parquet byte count mismatch for batch {batch_dir.name}" + ) + action_evidence = manifest.get("corporate_actions") + if not isinstance(action_evidence, Mapping) or set(action_evidence) != { + "source", + "knowledge_cutoff_date", + "status", + "content_sha256", + "transport_csv", + "parquet", + "rows", + }: + raise MarketDataIntegrityError( + "batch manifest is missing corporate-actions evidence" + ) + action_transport = action_evidence.get("transport_csv") + if not isinstance(action_transport, Mapping) or set(action_transport) != { + "status", + "sha256", + "byte_count", + "rows", + }: + raise MarketDataIntegrityError( + "batch manifest is missing corporate-actions transport evidence" + ) + action_parquet_evidence = action_evidence.get("parquet") + if not isinstance(action_parquet_evidence, Mapping) or set( + action_parquet_evidence + ) != {"sha256", "byte_count", "rows"}: + raise MarketDataIntegrityError( + "batch manifest is missing corporate-actions Parquet evidence" + ) + action_parquet_path = batch_dir / "corporate-actions.parquet" + action_parquet_bytes = action_parquet_path.read_bytes() + if _sha256_bytes(action_parquet_bytes) != action_parquet_evidence.get("sha256"): + raise MarketDataIntegrityError( + f"corporate-actions Parquet SHA256 mismatch for batch {batch_dir.name}" + ) + if len(action_parquet_bytes) != action_parquet_evidence.get("byte_count"): raise MarketDataIntegrityError( - f"CSV byte count mismatch for batch {batch_dir.name}" + f"corporate-actions Parquet byte count mismatch for batch {batch_dir.name}" ) validation = _load_json(batch_dir / "validation.json") if validation != _validation_document(): raise MarketDataIntegrityError( f"batch validation evidence is invalid: {batch_dir.name}" ) - rows, securities = _read_csv(csv_path, declared["fields"]) - if len(rows) != csv_evidence.get("rows"): + rows = _read_parquet(parquet_path) + if len(rows) != parquet_evidence.get("rows"): raise MarketDataIntegrityError( - f"CSV row count mismatch for batch {batch_dir.name}" + f"Parquet row count mismatch for batch {batch_dir.name}" ) + content_sha256 = normalized_digest(rows) + if content_sha256 != manifest.get("content_sha256"): + raise MarketDataIntegrityError( + f"normalized content SHA256 mismatch for batch {batch_dir.name}" + ) + action_rows = _read_corporate_actions_parquet( + action_parquet_path, declared=declared + ) + if len(action_rows) != action_parquet_evidence.get("rows") or len( + action_rows + ) != action_evidence.get("rows"): + raise MarketDataIntegrityError( + f"corporate-actions row count mismatch for batch {batch_dir.name}" + ) + action_content_sha256 = corporate_actions_digest(action_rows) + if action_content_sha256 != action_evidence.get("content_sha256"): + raise MarketDataIntegrityError( + f"corporate-actions content SHA256 mismatch for batch {batch_dir.name}" + ) + securities = _security_coverage(rows) if securities != manifest.get("securities"): raise MarketDataIntegrityError( f"batch security coverage mismatch: {batch_dir.name}" ) expected_batch_id = _sha256_bytes( - _canonical_bytes(_batch_identity(declared, actual_sha)) + _canonical_bytes( + _batch_identity(declared, content_sha256, action_content_sha256) + ) ) if batch_dir.name != expected_batch_id: raise MarketDataIntegrityError( @@ -343,29 +906,88 @@ def _assert_existing_batch_matches( ) -def _existing_rows(batch_dir: Path, manifest: Mapping[str, object]) -> dict[tuple[str, str], dict[str, str]]: - rows, _ = _read_csv(batch_dir / "market-data.csv", manifest["fields"]) - return {(row["security"], row["date"]): row for row in rows} +def _existing_rows( + batch_dir: Path, manifest: Mapping[str, object] +) -> dict[tuple[str, str], dict[str, object]]: + rows = _read_parquet(batch_dir / "market-data.parquet") + return {(str(row["security"]), str(row["date"])): row for row in rows} + + +def _legacy_batch_for_overlap( + batch_dir: Path, +) -> tuple[dict[str, object], dict[tuple[str, str], dict[str, object]]]: + manifest = _load_json(batch_dir / "manifest.json") + if set(manifest) != _LEGACY_STORED_MANIFEST_FIELDS: + raise MarketDataIntegrityError("legacy batch manifest structure is invalid") + if manifest.get("schema_version") != 1: + raise MarketDataIntegrityError("legacy batch schema_version must be 1") + declared = _validate_manifest_input( + manifest, require_corporate_actions=False + ) + csv_evidence = manifest.get("csv") + if not isinstance(csv_evidence, Mapping) or set(csv_evidence) != { + "sha256", + "bytes", + "rows", + }: + raise MarketDataIntegrityError("legacy batch is missing CSV evidence") + csv_path = batch_dir / "market-data.csv" + csv_bytes = csv_path.read_bytes() + csv_sha256 = _sha256_bytes(csv_bytes) + if csv_sha256 != csv_evidence.get("sha256"): + raise MarketDataIntegrityError( + f"legacy CSV SHA256 mismatch for batch {batch_dir.name}" + ) + if len(csv_bytes) != csv_evidence.get("bytes"): + raise MarketDataIntegrityError( + f"legacy CSV byte count mismatch for batch {batch_dir.name}" + ) + if _load_json(batch_dir / "validation.json") != _legacy_validation_document(): + raise MarketDataIntegrityError( + f"legacy batch validation evidence is invalid: {batch_dir.name}" + ) + raw_rows, securities = _read_csv(csv_path, declared["fields"]) + rows = _normalize_rows(raw_rows) + if len(rows) != csv_evidence.get("rows") or securities != manifest.get( + "securities" + ): + raise MarketDataIntegrityError( + f"legacy batch coverage evidence is invalid: {batch_dir.name}" + ) + expected_id = _sha256_bytes( + _canonical_bytes(_legacy_batch_identity(declared, csv_sha256)) + ) + if expected_id != batch_dir.name: + raise MarketDataIntegrityError( + f"legacy batch identity mismatch: {batch_dir.name}" + ) + return declared, { + (str(row["security"]), str(row["date"])): row for row in rows + } def _reject_conflicting_overlap( *, batches_dir: Path, incoming_manifest: Mapping[str, object], - incoming_rows: Iterable[dict[str, str]], + incoming_rows: Iterable[dict[str, object]], ) -> None: incoming_by_key = { - (row["security"], row["date"]): row for row in incoming_rows + (str(row["security"]), str(row["date"])): row for row in incoming_rows } if not batches_dir.exists(): return for batch_dir in sorted(batches_dir.iterdir()): if not batch_dir.is_dir() or batch_dir.name.startswith("."): continue - existing_manifest = _validate_batch_dir(batch_dir) + names = {path.name for path in batch_dir.iterdir()} + if names == _LEGACY_BATCH_FILES: + existing_manifest, existing_by_key = _legacy_batch_for_overlap(batch_dir) + else: + existing_manifest = _validate_batch_dir(batch_dir) + existing_by_key = _existing_rows(batch_dir, existing_manifest) if _dataset_identity(existing_manifest) != _dataset_identity(incoming_manifest): continue - existing_by_key = _existing_rows(batch_dir, existing_manifest) overlap = sorted(incoming_by_key.keys() & existing_by_key.keys()) if not overlap: continue @@ -444,33 +1066,111 @@ def _atomic_file_write(target: Path, content: bytes) -> None: def _import_batch_locked( *, csv_path: Path, + corporate_actions_csv_path: Path | None, manifest: Mapping[str, object], root: Path, ) -> BatchRecord: declared = _validate_manifest_input(manifest) csv_bytes = csv_path.read_bytes() csv_sha256 = _sha256_bytes(csv_bytes) - rows, securities = _read_csv(csv_path, declared["fields"]) + raw_rows, _ = _read_csv(csv_path, declared["fields"]) + rows = _normalize_rows(raw_rows) + securities = _security_coverage(rows) + content_sha256 = normalized_digest(rows) + parquet_bytes = _parquet_bytes(rows) + roundtrip_rows = _duckdb_roundtrip(parquet_bytes, root=Path(root)) + if normalized_digest(roundtrip_rows) != content_sha256: + raise MarketDataIntegrityError( + "DuckDB Parquet roundtrip normalized digest mismatch" + ) + actions_csv_bytes, action_rows = _read_corporate_actions_csv( + corporate_actions_csv_path, + declared=declared, + ) + actions_content_sha256 = corporate_actions_digest(action_rows) + actions_parquet_bytes = _corporate_actions_parquet_bytes(action_rows) + actions_roundtrip_rows = _corporate_actions_duckdb_roundtrip( + actions_parquet_bytes, + declared=declared, + root=Path(root), + ) + if corporate_actions_digest(actions_roundtrip_rows) != actions_content_sha256: + raise MarketDataIntegrityError( + "DuckDB corporate-actions Parquet roundtrip normalized digest mismatch" + ) + declared_actions = _require_mapping( + declared["corporate_actions"], "corporate_actions" + ) + transport_status = ( + "verified_empty" if not action_rows else "complete" + ) stored_manifest = { **declared, - "csv": { + "content_sha256": content_sha256, + "transport_csv": { "sha256": csv_sha256, - "bytes": len(csv_bytes), + "byte_count": len(csv_bytes), + "rows": len(rows), + }, + "parquet": { + "sha256": _sha256_bytes(parquet_bytes), + "byte_count": len(parquet_bytes), "rows": len(rows), }, "securities": securities, + "corporate_actions": { + "source": declared_actions["source"], + "knowledge_cutoff_date": declared_actions["knowledge_cutoff_date"], + "status": transport_status, + "content_sha256": actions_content_sha256, + "transport_csv": { + "status": transport_status, + "sha256": ( + _sha256_bytes(actions_csv_bytes) + if corporate_actions_csv_path is not None + else None + ), + "byte_count": len(actions_csv_bytes), + "rows": len(action_rows), + }, + "parquet": { + "sha256": _sha256_bytes(actions_parquet_bytes), + "byte_count": len(actions_parquet_bytes), + "rows": len(action_rows), + }, + "rows": len(action_rows), + }, + "writer": { + "pyarrow": pa.__version__, + "duckdb": duckdb.__version__, + "compression": "zstd", + }, } validation = _validation_document() - batch_id = _sha256_bytes(_canonical_bytes(_batch_identity(declared, csv_sha256))) + batch_id = _sha256_bytes( + _canonical_bytes( + _batch_identity(declared, content_sha256, actions_content_sha256) + ) + ) batch_dir = Path(root) / "batches" / batch_id files = { "manifest.json": _json_file_bytes(stored_manifest), - "market-data.csv": csv_bytes, + "market-data.parquet": parquet_bytes, + "corporate-actions.parquet": actions_parquet_bytes, "validation.json": _json_file_bytes(validation), } if batch_dir.exists(): - _assert_existing_batch_matches(batch_dir, files) + existing_manifest = _validate_batch_dir(batch_dir) + if existing_manifest.get("content_sha256") != content_sha256: + raise MarketDataIntegrityError( + f"immutable batch collision for {batch_dir.name}" + ) + return BatchRecord( + batch_id=batch_id, + path=batch_dir, + manifest=existing_manifest, + ) else: _reject_conflicting_overlap( batches_dir=Path(root) / "batches", @@ -484,12 +1184,14 @@ def _import_batch_locked( def import_batch( *, csv_path: Path, + corporate_actions_csv_path: Path | None = None, manifest: Mapping[str, object], root: Path, ) -> BatchRecord: with _exclusive_storage_lock(root): return _import_batch_locked( csv_path=csv_path, + corporate_actions_csv_path=corporate_actions_csv_path, manifest=manifest, root=root, ) @@ -514,7 +1216,7 @@ def _selection_rows( if start > end: raise MarketDataIntegrityError("snapshot start_date must not exceed end_date") - selected_rows: dict[str, dict[str, dict[str, str]]] = { + selected_rows: dict[str, dict[str, dict[str, object]]] = { security: {} for security in selected_securities } for batch_id, manifest in manifests: @@ -538,19 +1240,20 @@ def _selection_rows( manifest_fields ): raise MarketDataIntegrityError("snapshot fields are not covered") - rows, _ = _read_csv( - Path(root) / "batches" / batch_id / "market-data.csv", - manifest_fields, + rows = _read_parquet( + Path(root) / "batches" / batch_id / "market-data.parquet" ) for row in rows: - row_date = date.fromisoformat(row["date"]) + row_date_text = str(row["date"]) + row_date = date.fromisoformat(row_date_text) if row["security"] in selected_securities and start <= row_date <= end: - existing = selected_rows[row["security"]].get(row["date"]) + security = str(row["security"]) + existing = selected_rows[security].get(row_date_text) if existing is not None and existing != row: raise MarketDataConflict( f"snapshot batches conflict at {row['security']} {row['date']}" ) - selected_rows[row["security"]][row["date"]] = row + selected_rows[security][row_date_text] = row missing = sorted( security for security, rows in selected_rows.items() if not rows ) @@ -597,7 +1300,16 @@ def _create_snapshot_locked( { "batch_id": batch_id, "manifest_sha256": _sha256_path(batch_dir / "manifest.json"), - "csv_sha256": _sha256_path(batch_dir / "market-data.csv"), + "parquet_sha256": _sha256_path( + batch_dir / "market-data.parquet" + ), + "content_sha256": manifest["content_sha256"], + "corporate_actions_sha256": _sha256_path( + batch_dir / "corporate-actions.parquet" + ), + "corporate_actions_content_sha256": manifest[ + "corporate_actions" + ]["content_sha256"], "validation_sha256": _sha256_path( batch_dir / "validation.json" ), @@ -610,7 +1322,7 @@ def _create_snapshot_locked( root=Path(root), ) payload = { - "schema_version": 1, + "schema_version": 3, "batch_ids": unique_batch_ids, "batches": batch_evidence, "selection": selection.to_document(), @@ -686,8 +1398,29 @@ def validate_snapshot(snapshot_id: str, *, root: Path) -> SnapshotRecord: raise MarketDataIntegrityError( f"manifest SHA256 mismatch for batch {batch_id}" ) - if _sha256_path(batch_dir / "market-data.csv") != evidence.get("csv_sha256"): - raise MarketDataIntegrityError(f"CSV SHA256 mismatch for batch {batch_id}") + if _sha256_path(batch_dir / "market-data.parquet") != evidence.get( + "parquet_sha256" + ): + raise MarketDataIntegrityError( + f"Parquet SHA256 mismatch for batch {batch_id}" + ) + if _sha256_path( + batch_dir / "corporate-actions.parquet" + ) != evidence.get("corporate_actions_sha256"): + raise MarketDataIntegrityError( + f"corporate-actions Parquet SHA256 mismatch for batch {batch_id}" + ) + if manifest.get("content_sha256") != evidence.get("content_sha256"): + raise MarketDataIntegrityError( + f"content SHA256 mismatch for batch {batch_id}" + ) + action_manifest = manifest.get("corporate_actions") + if not isinstance(action_manifest, Mapping) or action_manifest.get( + "content_sha256" + ) != evidence.get("corporate_actions_content_sha256"): + raise MarketDataIntegrityError( + f"corporate-actions content SHA256 mismatch for batch {batch_id}" + ) if _sha256_path(batch_dir / "validation.json") != evidence.get( "validation_sha256" ): @@ -733,3 +1466,184 @@ def validate_snapshot(snapshot_id: str, *, root: Path) -> SnapshotRecord: path=snapshot_path, document=document, ) + + +def _validate_legacy_snapshot_for_audit( + snapshot_id: str, + *, + root: Path, + legacy_batch_ids: set[str], +) -> None: + """Validate an immutable schema-v1 snapshot without making it runnable.""" + + _require_identifier(snapshot_id, "snapshot") + snapshot_path = Path(root) / "snapshots" / f"{snapshot_id}.json" + document = _load_json(snapshot_path) + if document.get("snapshot_id") != snapshot_id: + raise MarketDataIntegrityError("snapshot identity does not match its path") + payload = {key: value for key, value in document.items() if key != "snapshot_id"} + if _sha256_bytes(_canonical_bytes(payload)) != snapshot_id: + raise MarketDataIntegrityError("snapshot identity digest mismatch") + if document.get("schema_version") != 1: + raise MarketDataIntegrityError("legacy snapshot schema_version must be 1") + + batch_ids = document.get("batch_ids") + evidence_rows = document.get("batches") + if not isinstance(batch_ids, list) or not isinstance(evidence_rows, list): + raise MarketDataIntegrityError("snapshot batch evidence is missing") + if ( + batch_ids != sorted(set(batch_ids)) + or not batch_ids + or not set(batch_ids).issubset(legacy_batch_ids) + ): + raise MarketDataIntegrityError("legacy snapshot batch ids are invalid") + evidence_by_id = { + item.get("batch_id"): item + for item in evidence_rows + if isinstance(item, Mapping) + } + if list(evidence_by_id) != batch_ids or len(evidence_rows) != len(batch_ids): + raise MarketDataIntegrityError("snapshot canonical batch evidence is invalid") + + manifests: list[Mapping[str, object]] = [] + rows_by_security: dict[str, dict[str, dict[str, object]]] = {} + for batch_id in batch_ids: + batch_dir = Path(root) / "batches" / batch_id + manifest, rows = _legacy_batch_for_overlap(batch_dir) + evidence = evidence_by_id[batch_id] + expected = { + "batch_id": batch_id, + "manifest_sha256": _sha256_path(batch_dir / "manifest.json"), + "csv_sha256": _sha256_path(batch_dir / "market-data.csv"), + "validation_sha256": _sha256_path(batch_dir / "validation.json"), + "export_code_sha256": manifest["export_code_sha256"], + } + if evidence != expected: + raise MarketDataIntegrityError( + f"legacy snapshot batch evidence mismatch: {batch_id}" + ) + manifests.append(manifest) + for (security, current_date), row in rows.items(): + existing = rows_by_security.setdefault(security, {}).get(current_date) + if existing is not None and existing != row: + raise MarketDataConflict( + f"snapshot batches conflict at {security} {current_date}" + ) + rows_by_security[security][current_date] = row + + selection_document = document.get("selection") + if not isinstance(selection_document, Mapping): + raise MarketDataIntegrityError("snapshot selection is missing") + try: + source_identity = selection_document["source"] + if not isinstance(source_identity, Mapping): + raise TypeError("source must be a mapping") + selection = SnapshotSelection( + source=source_identity, + asset_type=str(selection_document["asset_type"]), + frequency=str(selection_document["frequency"]), + securities=selection_document["securities"], + start_date=str(selection_document["start_date"]), + end_date=str(selection_document["end_date"]), + fields=selection_document["fields"], + price_semantics=selection_document["price_semantics"], + ) + except (KeyError, TypeError) as exc: + raise MarketDataIntegrityError("snapshot selection is incomplete") from exc + + for manifest in manifests: + if ( + manifest["source"] != dict(selection.source) + or manifest["asset_type"] != selection.asset_type + or manifest["frequency"] != selection.frequency + or manifest["price_semantics"] != dict(selection.price_semantics) + or not set(selection.fields).issubset(manifest["fields"]) + ): + raise MarketDataIntegrityError( + "legacy snapshot selection does not match its batch" + ) + + coverage: list[dict[str, object]] = [] + for security in sorted(selection.securities): + dates = sorted( + current_date + for current_date in rows_by_security.get(security, {}) + if selection.start_date <= current_date <= selection.end_date + ) + if not dates or dates[-1] != selection.end_date: + raise MarketDataIntegrityError( + f"legacy snapshot coverage is incomplete for {security}" + ) + coverage.append( + { + "security": security, + "start_date": dates[0], + "end_date": dates[-1], + "rows": len(dates), + } + ) + if document.get("coverage") != coverage: + raise MarketDataIntegrityError("legacy snapshot coverage evidence is invalid") + + +def audit_store(*, root: Path) -> dict[str, object]: + """Read and validate every stored batch and snapshot without mutating the store.""" + + storage_root = Path(root) + legacy_batch_ids: list[str] = [] + parquet_batch_ids: list[str] = [] + batches_dir = storage_root / "batches" + if batches_dir.exists(): + for batch_dir in sorted(batches_dir.iterdir()): + if batch_dir.name.startswith("."): + continue + if not batch_dir.is_dir(): + raise MarketDataIntegrityError( + f"unexpected batch-store entry: {batch_dir.name}" + ) + names = {path.name for path in batch_dir.iterdir()} + if names == _LEGACY_BATCH_FILES: + _legacy_batch_for_overlap(batch_dir) + legacy_batch_ids.append(batch_dir.name) + else: + _validate_batch_dir(batch_dir) + parquet_batch_ids.append(batch_dir.name) + + snapshot_ids: list[str] = [] + legacy_snapshot_ids: list[str] = [] + snapshots_dir = storage_root / "snapshots" + if snapshots_dir.exists(): + for snapshot_path in sorted(snapshots_dir.iterdir()): + if snapshot_path.name.startswith("."): + continue + if not snapshot_path.is_file() or snapshot_path.suffix != ".json": + raise MarketDataIntegrityError( + f"unexpected snapshot-store entry: {snapshot_path.name}" + ) + snapshot_id = snapshot_path.stem + document = _load_json(snapshot_path) + referenced = document.get("batch_ids") + if ( + document.get("schema_version") == 1 + and isinstance(referenced, list) + and referenced + and set(referenced).issubset(set(legacy_batch_ids)) + ): + _validate_legacy_snapshot_for_audit( + snapshot_id, + root=storage_root, + legacy_batch_ids=set(legacy_batch_ids), + ) + legacy_snapshot_ids.append(snapshot_id) + else: + validate_snapshot(snapshot_id, root=storage_root) + snapshot_ids.append(snapshot_id) + + return { + "schema_version": 1, + "status": "complete", + "legacy_batch_ids": legacy_batch_ids, + "parquet_batch_ids": parquet_batch_ids, + "legacy_snapshot_ids": legacy_snapshot_ids, + "snapshot_ids": snapshot_ids, + } diff --git a/scripts/research/quant_analysis/__init__.py b/scripts/research/quant_analysis/__init__.py new file mode 100644 index 0000000..1db8316 --- /dev/null +++ b/scripts/research/quant_analysis/__init__.py @@ -0,0 +1,17 @@ +"""Strategy-agnostic analysis over standard JoinQuant-compatible results.""" + +from .benchmarks import BenchmarkAlignmentError, calculate_benchmark_statistics +from .cvar import calculate_cvar, rolling_compound_returns +from .evidence import ScenarioResult, build_evidence_matrix +from .robustness import block_bootstrap, summarize_bootstrap + +__all__ = [ + "BenchmarkAlignmentError", + "ScenarioResult", + "block_bootstrap", + "build_evidence_matrix", + "calculate_benchmark_statistics", + "calculate_cvar", + "rolling_compound_returns", + "summarize_bootstrap", +] diff --git a/scripts/research/quant_analysis/analysis_plan.py b/scripts/research/quant_analysis/analysis_plan.py new file mode 100644 index 0000000..3fa3a03 --- /dev/null +++ b/scripts/research/quant_analysis/analysis_plan.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator + + +SCHEMA_PATH = Path(__file__).with_name("schemas") / "analysis-plan.schema.json" + + +class AnalysisPlanError(ValueError): + """Raised when an analysis plan cannot be safely and deterministically expanded.""" + + +def _canonical_bytes(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def _sha256(value: object) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _load_json(path: Path, *, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise AnalysisPlanError(f"cannot read {label}: {path}") from exc + if not isinstance(value, dict): + raise AnalysisPlanError(f"{label} must be a JSON object") + return value + + +def _resolve_plan_path(repo_root: Path, path: str | Path) -> Path: + candidate = Path(path) + return candidate.resolve() if candidate.is_absolute() else (repo_root / candidate).resolve() + + +def _resolve_repo_file(repo_root: Path, value: object, *, label: str) -> Path: + if not isinstance(value, str) or not value: + raise AnalysisPlanError(f"{label} must be a repository-relative path") + candidate = Path(value) + if candidate.is_absolute() or ".." in candidate.parts: + raise AnalysisPlanError(f"{label} must stay inside the repository") + resolved = (repo_root / candidate).resolve() + try: + resolved.relative_to(repo_root) + except ValueError as exc: + raise AnalysisPlanError(f"{label} must stay inside the repository") from exc + if not resolved.is_file(): + raise AnalysisPlanError(f"{label} does not exist: {value}") + return resolved + + +def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + merged = copy.deepcopy(base) + for key, value in override.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _deep_merge(merged[key], value) + else: + merged[key] = copy.deepcopy(value) + return merged + + +def _validate_plan(document: dict[str, Any]) -> None: + schema = _load_json(SCHEMA_PATH, label="analysis plan schema") + errors = sorted( + Draft202012Validator(schema).iter_errors(document), + key=lambda error: tuple(str(part) for part in error.absolute_path), + ) + if errors: + paths = [ + ".".join(str(part) for part in error.absolute_path) or "$" + for error in errors + ] + raise AnalysisPlanError(f"analysis plan schema validation failed at {paths}") + + +def expand_analysis_plan(repo_root: str | Path, plan_path: str | Path) -> dict[str, Any]: + root = Path(repo_root).resolve() + plan_file = _resolve_plan_path(root, plan_path) + plan = _load_json(plan_file, label="analysis plan") + _validate_plan(plan) + + baseline_file = _resolve_repo_file( + root, + plan["baseline_config"], + label="baseline_config", + ) + baseline = _load_json(baseline_file, label="baseline config") + if baseline.get("project_id") != plan["strategy_id"]: + raise AnalysisPlanError("baseline project_id must match strategy_id") + + expected_universe = { + item.get("security"): item.get("asset_group") + for item in baseline.get("universe", []) + if isinstance(item, dict) + } + if expected_universe != plan["universe"]: + raise AnalysisPlanError("analysis plan universe must match baseline universe") + + scenarios = plan["scenarios"] + scenario_ids = [scenario["scenario_id"] for scenario in scenarios] + if len(scenario_ids) != len(set(scenario_ids)): + raise AnalysisPlanError("scenario_id values must be unique") + if scenarios[0]["scenario_id"] != "baseline" or scenarios[0]["overrides"]: + raise AnalysisPlanError("baseline must be first and have empty overrides") + if plan["expected"]["scenario_runs"] != len(scenarios): + raise AnalysisPlanError("expected.scenario_runs must match scenarios") + + bootstrap = plan["analyses"]["bootstrap"] + if plan["expected"]["bootstrap_paths"] != bootstrap.get("paths"): + raise AnalysisPlanError("expected.bootstrap_paths must match analyses.bootstrap.paths") + if plan["expected"]["seed"] != bootstrap.get("seed"): + raise AnalysisPlanError("expected.seed must match analyses.bootstrap.seed") + + expanded_scenarios: list[dict[str, Any]] = [] + for scenario in scenarios: + params = _deep_merge(baseline, scenario["overrides"]) + params["scenario_id"] = scenario["scenario_id"] + expanded_scenarios.append( + { + "scenario_id": scenario["scenario_id"], + "dimension": scenario["dimension"], + "overrides": copy.deepcopy(scenario["overrides"]), + "params": params, + "params_sha256": _sha256(params), + } + ) + + return { + "schema_version": "analysis-scenarios/1", + "strategy_id": plan["strategy_id"], + "analysis_plan": plan_file.relative_to(root).as_posix() + if plan_file.is_relative_to(root) + else str(plan_file), + "analysis_plan_sha256": _sha256(plan), + "baseline_config": baseline_file.relative_to(root).as_posix(), + "baseline_config_sha256": _sha256(baseline), + "expected": copy.deepcopy(plan["expected"]), + "universe": copy.deepcopy(plan["universe"]), + "analyses": copy.deepcopy(plan["analyses"]), + "thresholds": copy.deepcopy(plan["thresholds"]), + "scenarios": expanded_scenarios, + } + + +def write_analysis_scenarios(document: dict[str, Any], output_path: str | Path) -> None: + target = Path(output_path) + target.parent.mkdir(parents=True, exist_ok=True) + payload = _canonical_bytes(document) + b"\n" + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{target.name}.", + suffix=".tmp", + dir=target.parent, + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_name, target) + finally: + temporary = Path(temporary_name) + if temporary.exists(): + temporary.unlink() diff --git a/scripts/research/quant_analysis/benchmarks.py b/scripts/research/quant_analysis/benchmarks.py new file mode 100644 index 0000000..e5fb2fd --- /dev/null +++ b/scripts/research/quant_analysis/benchmarks.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import math +import statistics +from typing import Mapping + + +class BenchmarkAlignmentError(ValueError): + """Raised when strategy and benchmark returns do not share exact dates.""" + + +def _product_return(values: list[float]) -> float: + wealth = 1.0 + for value in values: + wealth *= 1.0 + value + return wealth - 1.0 + + +def _ratio(numerator: float, denominator: float) -> float | None: + return None if denominator == 0 else numerator / denominator + + +def _annualized_conditional_return(values: list[float], annualization: int) -> float: + if not values: + return 0.0 + total = _product_return(values) + return (1.0 + total) ** (annualization / len(values)) - 1.0 + + +def calculate_benchmark_statistics( + strategy_returns: Mapping[str, float], + benchmark_returns: Mapping[str, float], + *, + annualization: int = 252, +) -> dict[str, float | None]: + strategy_dates = set(strategy_returns) + benchmark_dates = set(benchmark_returns) + if strategy_dates != benchmark_dates or not strategy_dates: + raise BenchmarkAlignmentError("strategy and benchmark dates must align exactly") + dates = sorted(strategy_dates) + strategy = [float(strategy_returns[current_date]) for current_date in dates] + benchmark = [float(benchmark_returns[current_date]) for current_date in dates] + if any(not math.isfinite(value) for value in (*strategy, *benchmark)): + raise BenchmarkAlignmentError("strategy and benchmark returns must be finite") + mean_strategy = statistics.fmean(strategy) + mean_benchmark = statistics.fmean(benchmark) + benchmark_variance = statistics.fmean( + (value - mean_benchmark) ** 2 for value in benchmark + ) + covariance = statistics.fmean( + (left - mean_strategy) * (right - mean_benchmark) + for left, right in zip(strategy, benchmark) + ) + beta = _ratio(covariance, benchmark_variance) + alpha = ( + None + if beta is None + else (mean_strategy - beta * mean_benchmark) * annualization + ) + strategy_variance = statistics.fmean( + (value - mean_strategy) ** 2 for value in strategy + ) + correlation = _ratio( + covariance, math.sqrt(strategy_variance * benchmark_variance) + ) + active = [left - right for left, right in zip(strategy, benchmark)] + tracking_error = ( + statistics.stdev(active) * math.sqrt(annualization) if len(active) > 1 else 0.0 + ) + information_ratio = _ratio( + statistics.fmean(active) * annualization, tracking_error + ) + up_strategy = [left for left, right in zip(strategy, benchmark) if right > 0] + up_benchmark = [right for right in benchmark if right > 0] + down_strategy = [left for left, right in zip(strategy, benchmark) if right < 0] + down_benchmark = [right for right in benchmark if right < 0] + strategy_total = _product_return(strategy) + benchmark_total = _product_return(benchmark) + return { + "alpha": alpha, + "beta": beta, + "correlation": correlation, + "tracking_error": tracking_error, + "information_ratio": information_ratio, + "up_capture": _ratio( + _annualized_conditional_return(up_strategy, annualization), + _annualized_conditional_return(up_benchmark, annualization), + ), + "down_capture": _ratio( + _annualized_conditional_return(down_strategy, annualization), + _annualized_conditional_return(down_benchmark, annualization), + ), + "strategy_return": strategy_total, + "benchmark_return": benchmark_total, + "active_return": strategy_total - benchmark_total, + "relative_return": (1.0 + strategy_total) / (1.0 + benchmark_total) - 1.0, + } diff --git a/scripts/research/quant_analysis/cvar.py b/scripts/research/quant_analysis/cvar.py new file mode 100644 index 0000000..4dba2b1 --- /dev/null +++ b/scripts/research/quant_analysis/cvar.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import numpy as np + + +def _returns(values: np.ndarray) -> np.ndarray: + normalized = np.asarray(values, dtype=np.float64) + if normalized.ndim != 1 or normalized.size == 0: + raise ValueError("returns must be a non-empty one-dimensional array") + if not np.isfinite(normalized).all(): + raise ValueError("returns must be finite") + if np.any(normalized <= -1.0): + raise ValueError("returns must be greater than -100%") + return normalized + + +def calculate_cvar(returns: np.ndarray, confidence: float) -> float: + values = _returns(returns) + if not 0.0 < confidence < 1.0: + raise ValueError("confidence must be between zero and one") + ordered = np.sort(values) + tail_mass = values.size * (1.0 - confidence) + nearest = round(tail_mass) + if np.isclose(tail_mass, nearest, rtol=0.0, atol=1e-12): + tail_mass = float(nearest) + whole = int(np.floor(tail_mass)) + fraction = tail_mass - whole + total = float(np.sum(ordered[:whole])) + if fraction > 0.0: + total += fraction * float(ordered[whole]) + return max(0.0, -(total / tail_mass)) + + +def rolling_compound_returns(returns: np.ndarray, *, window: int) -> np.ndarray: + values = _returns(returns) + if window <= 0 or values.size < window: + raise ValueError("rolling compound window exceeds the available returns") + windows = np.lib.stride_tricks.sliding_window_view(values, window) + return np.prod(1.0 + windows, axis=1) - 1.0 diff --git a/scripts/research/quant_analysis/evidence.py b/scripts/research/quant_analysis/evidence.py new file mode 100644 index 0000000..7a4f434 --- /dev/null +++ b/scripts/research/quant_analysis/evidence.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Iterable, Literal, Mapping + +import pyarrow as pa +import pyarrow.parquet as pq + + +ScenarioStatus = Literal["pass", "fail", "evidence_insufficient"] +_SHA256 = re.compile(r"[0-9a-f]{64}") +_AUTHORITY = "local_exploratory" +_FORMULA_VERSION = "quant-analysis-v1" +_SCHEMA = pa.schema( + [ + pa.field("scenario_id", pa.string(), nullable=False), + pa.field("dimension", pa.string(), nullable=False), + pa.field("status", pa.string(), nullable=False), + pa.field("authority", pa.string(), nullable=False), + pa.field("formula_version", pa.string(), nullable=False), + pa.field("input_sha256", pa.string(), nullable=False), + pa.field("metrics_json", pa.string(), nullable=False), + pa.field("reasons_json", pa.string(), nullable=False), + ] +) + + +def canonical_bytes(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def evidence_digest(value: object) -> str: + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +@dataclass(frozen=True) +class ScenarioResult: + scenario_id: str + dimension: str + status: ScenarioStatus + metrics: Mapping[str, float | int | None] + input_sha256: str + reasons: tuple[str, ...] = () + authority: str = _AUTHORITY + formula_version: str = _FORMULA_VERSION + + def __post_init__(self) -> None: + if not self.scenario_id or not self.dimension: + raise ValueError("scenario identity must be non-empty") + if self.status not in {"pass", "fail", "evidence_insufficient"}: + raise ValueError("scenario status is invalid") + if _SHA256.fullmatch(self.input_sha256) is None: + raise ValueError("scenario input_sha256 is invalid") + if self.authority != _AUTHORITY or self.formula_version != _FORMULA_VERSION: + raise ValueError("scenario authority or formula version is invalid") + normalized: dict[str, float | int | None] = {} + for key, value in self.metrics.items(): + if not key: + raise ValueError("scenario metric name must be non-empty") + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("scenario metrics must be finite") + if value is not None and not isinstance(value, (int, float)): + raise ValueError("scenario metrics must be numeric or null") + normalized[str(key)] = value + reasons = tuple(dict.fromkeys(str(reason) for reason in self.reasons)) + if self.status == "evidence_insufficient" and not reasons: + raise ValueError("evidence-insufficient scenario requires a reason") + object.__setattr__(self, "metrics", MappingProxyType(normalized)) + object.__setattr__(self, "reasons", reasons) + + def to_document(self) -> dict[str, object]: + return { + "scenario_id": self.scenario_id, + "dimension": self.dimension, + "status": self.status, + "authority": self.authority, + "formula_version": self.formula_version, + "input_sha256": self.input_sha256, + "metrics": dict(self.metrics), + "reasons": list(self.reasons), + } + + +def _rows(results: Iterable[ScenarioResult]) -> list[dict[str, object]]: + ordered = sorted(results, key=lambda row: row.scenario_id) + if len({row.scenario_id for row in ordered}) != len(ordered): + raise ValueError("scenario IDs must be unique") + return [ + { + "scenario_id": row.scenario_id, + "dimension": row.dimension, + "status": row.status, + "authority": row.authority, + "formula_version": row.formula_version, + "input_sha256": row.input_sha256, + "metrics_json": canonical_bytes(dict(row.metrics)).decode("utf-8"), + "reasons_json": canonical_bytes(list(row.reasons)).decode("utf-8"), + } + for row in ordered + ] + + +def _metadata(rows: list[dict[str, object]]) -> dict[bytes, bytes]: + return { + b"schema_version": b"1", + b"table_name": b"local-evidence-matrix", + b"primary_key": b'["scenario_id"]', + b"authority": _AUTHORITY.encode("ascii"), + b"formula_version": _FORMULA_VERSION.encode("ascii"), + b"content_sha256": evidence_digest(rows).encode("ascii"), + } + + +def build_evidence_matrix( + results: Iterable[ScenarioResult], + output: Path, +) -> Path: + rows = _rows(results) + if not rows: + raise ValueError("evidence matrix must contain scenarios") + table = pa.Table.from_pylist(rows, schema=_SCHEMA) + table = table.replace_schema_metadata(_metadata(rows)) + target = Path(output) + target.parent.mkdir(parents=True, exist_ok=True) + pq.write_table( + table, + target, + compression="zstd", + use_dictionary=False, + write_statistics=True, + ) + return target + + +def validate_evidence_matrix(path: Path) -> tuple[ScenarioResult, ...]: + try: + table = pq.read_table(path) + except (OSError, pa.ArrowException) as exc: + raise ValueError("invalid evidence matrix Parquet") from exc + if not table.schema.remove_metadata().equals(_SCHEMA): + raise ValueError("evidence matrix schema mismatch") + rows = table.to_pylist() + if table.schema.metadata != _metadata(rows): + raise ValueError("evidence matrix metadata or digest mismatch") + results: list[ScenarioResult] = [] + for row in rows: + try: + metrics = json.loads(str(row["metrics_json"])) + reasons = json.loads(str(row["reasons_json"])) + except json.JSONDecodeError as exc: + raise ValueError("evidence matrix JSON fields are invalid") from exc + if not isinstance(metrics, dict) or not isinstance(reasons, list): + raise ValueError("evidence matrix JSON fields have invalid types") + results.append( + ScenarioResult( + scenario_id=str(row["scenario_id"]), + dimension=str(row["dimension"]), + status=str(row["status"]), + metrics=metrics, + input_sha256=str(row["input_sha256"]), + reasons=tuple(str(reason) for reason in reasons), + authority=str(row["authority"]), + formula_version=str(row["formula_version"]), + ) + ) + if [row.scenario_id for row in results] != sorted( + row.scenario_id for row in results + ) or len({row.scenario_id for row in results}) != len(results): + raise ValueError("evidence matrix scenario IDs are not sorted and unique") + return tuple(results) diff --git a/scripts/research/quant_analysis/orchestration.py b/scripts/research/quant_analysis/orchestration.py new file mode 100644 index 0000000..bef4b5d --- /dev/null +++ b/scripts/research/quant_analysis/orchestration.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import re +import tempfile +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .analysis_plan import expand_analysis_plan +from .evidence import evidence_digest + + +_SHA256 = re.compile(r"[0-9a-f]{64}") +_FORMULA_VERSION = "strategy-analysis-preparation/1" + + +class AnalysisOrchestrationError(ValueError): + """Raised when independent analysis inputs cannot be prepared safely.""" + + +def _load_json(path: Path, *, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise AnalysisOrchestrationError(f"cannot read {label}: {path}") from exc + if not isinstance(value, dict): + raise AnalysisOrchestrationError(f"{label} must be a JSON object") + return value + + +def _atomic_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = ( + json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_name, path) + finally: + temporary = Path(temporary_name) + if temporary.exists(): + temporary.unlink() + + +def build_scenario_run_documents( + expanded_plan: Mapping[str, object], + run_template: Mapping[str, object], + *, + preparation_id: str, +) -> list[dict[str, object]]: + if _SHA256.fullmatch(preparation_id) is None: + raise AnalysisOrchestrationError("preparation_id must be a SHA256 digest") + scenarios = expanded_plan.get("scenarios") + if not isinstance(scenarios, Sequence) or isinstance(scenarios, (str, bytes)): + raise AnalysisOrchestrationError("expanded plan scenarios are invalid") + if run_template.get("project_id") != expanded_plan.get("strategy_id"): + raise AnalysisOrchestrationError("run template project_id does not match strategy") + + documents: list[dict[str, object]] = [] + for raw in scenarios: + if not isinstance(raw, Mapping): + raise AnalysisOrchestrationError("expanded scenario is invalid") + scenario_id = str(raw.get("scenario_id", "")) + params = raw.get("params") + if not scenario_id or not isinstance(params, Mapping): + raise AnalysisOrchestrationError("expanded scenario identity is invalid") + run_config = copy.deepcopy(dict(run_template)) + run_config["project_config"] = ( + f".local/strategy-analysis-preparations/{preparation_id}/scenario-configs/" + f"{scenario_id}/params.json" + ) + run_config["required_outputs"] = [ + {"path": f"backtests/local-{scenario_id}", "format": "directory"} + ] + documents.append( + { + "scenario_id": scenario_id, + "dimension": str(raw.get("dimension", "")), + "params_sha256": str(raw.get("params_sha256", "")), + "params": copy.deepcopy(dict(params)), + "run_config": run_config, + } + ) + if len({item["scenario_id"] for item in documents}) != len(documents): + raise AnalysisOrchestrationError("scenario_id values must be unique") + return documents + + +def prepare_analysis_workspace( + repo_root: Path, + *, + plan_path: Path, + run_template_path: Path, + benchmark_set_id: str, +) -> dict[str, object]: + root = Path(repo_root).resolve() + if _SHA256.fullmatch(benchmark_set_id) is None: + raise AnalysisOrchestrationError("benchmark_set_id is invalid") + expanded = expand_analysis_plan(root, plan_path) + template_path = ( + run_template_path.resolve() + if run_template_path.is_absolute() + else (root / run_template_path).resolve() + ) + if not template_path.is_relative_to(root): + raise AnalysisOrchestrationError("run template must stay inside the repository") + template = _load_json(template_path, label="run template") + template_sha256 = hashlib.sha256(template_path.read_bytes()).hexdigest() + + benchmark_root = root / ".local" / "market-data" / "benchmark-sets" / benchmark_set_id + benchmark_manifest_path = benchmark_root / "manifest.json" + benchmark_manifest = _load_json(benchmark_manifest_path, label="benchmark set manifest") + if benchmark_manifest.get("benchmark_set_id") != benchmark_set_id: + raise AnalysisOrchestrationError("benchmark set identity mismatch") + data = benchmark_manifest.get("data") + if not isinstance(data, Mapping): + raise AnalysisOrchestrationError("benchmark set data declaration is invalid") + data_relative = Path(str(data.get("path", ""))) + if data_relative.is_absolute() or ".." in data_relative.parts: + raise AnalysisOrchestrationError("benchmark data path is unsafe") + data_path = (benchmark_root / data_relative).resolve() + if not data_path.is_file() or not data_path.is_relative_to(benchmark_root): + raise AnalysisOrchestrationError("benchmark data is missing") + data_sha256 = hashlib.sha256(data_path.read_bytes()).hexdigest() + if data_sha256 != data.get("sha256"): + raise AnalysisOrchestrationError("benchmark data digest mismatch") + + preparation_id = evidence_digest( + { + "formula_version": _FORMULA_VERSION, + "analysis_plan_sha256": expanded["analysis_plan_sha256"], + "baseline_config_sha256": expanded["baseline_config_sha256"], + "run_template_sha256": template_sha256, + "benchmark_set_id": benchmark_set_id, + "benchmark_data_sha256": data_sha256, + } + ) + documents = build_scenario_run_documents( + expanded, + template, + preparation_id=preparation_id, + ) + workspace = ( + root / ".local" / "strategy-analysis-preparations" / preparation_id + ) + _atomic_json(workspace / "analysis-scenarios.json", expanded) + config_paths: list[str] = [] + for document in documents: + scenario_id = str(document["scenario_id"]) + scenario_root = workspace / "scenario-configs" / scenario_id + _atomic_json(scenario_root / "params.json", document["params"]) + _atomic_json(scenario_root / "run.json", document["run_config"]) + config_paths.append( + (scenario_root / "run.json").relative_to(root).as_posix() + ) + preparation = { + "schema_version": "strategy-analysis-preparation/1", + "preparation_id": preparation_id, + "formula_version": _FORMULA_VERSION, + "analysis_scenarios": ( + workspace / "analysis-scenarios.json" + ).relative_to(root).as_posix(), + "run_template": { + "path": template_path.relative_to(root).as_posix(), + "sha256": template_sha256, + }, + "benchmark_set": { + "benchmark_set_id": benchmark_set_id, + "manifest": benchmark_manifest_path.relative_to(root).as_posix(), + "data_sha256": data_sha256, + }, + "scenario_run_configs": config_paths, + "expected_scenario_runs": len(documents), + "next_action": "invoke_each_scenario_once", + } + _atomic_json(workspace / "preparation.json", preparation) + return preparation + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Prepare independent strategy analysis runs") + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--run-template", type=Path, required=True) + parser.add_argument("--benchmark-set-id", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + result = prepare_analysis_workspace( + args.repo_root, + plan_path=args.plan, + run_template_path=args.run_template, + benchmark_set_id=args.benchmark_set_id, + ) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/research/quant_analysis/reporting.py b/scripts/research/quant_analysis/reporting.py new file mode 100644 index 0000000..622394e --- /dev/null +++ b/scripts/research/quant_analysis/reporting.py @@ -0,0 +1,762 @@ +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +import math +import os +from pathlib import Path +import tempfile +from typing import Any, Mapping + + +def enforce_vibe_boundary(evidence: Mapping[str, Any]) -> dict[str, Any]: + """Exclude the known-defective Vibe group-analysis path from conclusions.""" + + corrected = deepcopy(dict(evidence)) + swarm = corrected.get("swarm") + single_agent = corrected.get("single_agent") + valid_single_agent = bool( + isinstance(single_agent, dict) + and single_agent.get("interface") == "vibe-trading-cli-run" + and single_agent.get("status") == "completed" + and isinstance(single_agent.get("run_id"), str) + and bool(single_agent.get("run_id")) + and isinstance(single_agent.get("assessment"), dict) + and single_agent["assessment"].get("status") == "completed" + ) + if isinstance(swarm, dict): + preset = str(swarm.get("preset", "unknown")) + forbidden = f"run_swarm:{preset}" + called = list(corrected.get("forbidden_capabilities_called", [])) + if forbidden not in called: + called.append(forbidden) + corrected["forbidden_capabilities_called"] = called + swarm["valid_evidence"] = False + swarm["excluded_from_conclusions"] = True + corrected["boundary_violation"] = { + "occurred": True, + "capability": forbidden, + "reason": "Vibe 群体分析是已知缺陷路径；误调用结果无效。", + } + if isinstance(single_agent, dict): + single_agent["valid_evidence"] = False + single_agent["excluded_from_conclusions"] = True + elif isinstance(single_agent, dict): + single_agent["valid_evidence"] = valid_single_agent + single_agent["qualitative_only"] = True + + use_result = dict(corrected.get("use_result", {})) + capabilities_loaded = list(corrected.get("capabilities_loaded", [])) + vibe_called = bool( + isinstance(swarm, dict) + or isinstance(single_agent, dict) + or capabilities_loaded + ) + if "vibe_called" in use_result: + vibe_called = bool(use_result["vibe_called"] or vibe_called) + use_result["vibe_called"] = vibe_called + use_result["vibe_conclusion_available"] = bool( + valid_single_agent and not isinstance(swarm, dict) + ) + use_result["loaded_capabilities_are_methodology_only"] = bool( + capabilities_loaded and not valid_single_agent + ) + if valid_single_agent and not isinstance(swarm, dict): + use_result["reason"] = ( + "已通过公开 CLI 完成 Vibe 单体定性复核；结果只作审计，" + "不替代确定性数值裁判。" + ) + elif vibe_called: + use_result["reason"] = ( + "没有完成可用的 Vibe 单体策略结果分析；已加载能力仅是方法文档，" + "不得冒充实际复核。" + ) + else: + use_result["reason"] = "本次未调用 Vibe。" + corrected["use_result"] = use_result + corrected["authority"] = "audit_only" + corrected["next_action"] = "generate_deterministic_local_report" + return corrected + + +def _finite(value: object) -> float | None: + if value is None: + return None + number = float(value) + return number if math.isfinite(number) else None + + +def _pct(value: object, digits: int = 2) -> str: + number = _finite(value) + return "—" if number is None else f"{number * 100:.{digits}f}%" + + +def _num(value: object, digits: int = 3) -> str: + number = _finite(value) + return "—" if number is None else f"{number:.{digits}f}" + + +def _money(value: object) -> str: + number = _finite(value) + return "—" if number is None else f"¥{number:,.2f}" + + +def _mapping(value: object) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _rows(value: object) -> list[Mapping[str, Any]]: + if not isinstance(value, list): + return [] + return [row for row in value if isinstance(row, Mapping)] + + +def _best_iteration_candidate(analysis: Mapping[str, Any]) -> Mapping[str, Any] | None: + challenges = [ + row + for row in _rows(analysis.get("challenge_results")) + if row.get("scenario_id") != "baseline" + ] + measurable = [ + row + for row in challenges + if _finite(_mapping(row.get("metrics")).get("calmar")) is not None + ] + if not measurable: + return None + return max( + measurable, + key=lambda row: float(_mapping(row.get("metrics"))["calmar"]), + ) + + +def build_recommendation(analysis: Mapping[str, Any]) -> dict[str, Any]: + baseline = _mapping(analysis.get("baseline")) + evidence = _mapping(analysis.get("evidence_matrix")) + best = _best_iteration_candidate(analysis) + best_status = None if best is None else str(best.get("status")) + baseline_passed = baseline.get("status") == "pass" + no_failed_evidence = int(evidence.get("fail", 0)) == 0 + no_missing_evidence = int(evidence.get("evidence_insufficient", 0)) == 0 + accepted = bool( + best is not None + and best_status == "pass" + and baseline_passed + and no_failed_evidence + and no_missing_evidence + ) + decision = "proceed_to_joinquant" if accepted else "revise_and_reassess" + candidate_id = None if best is None else str(best.get("scenario_id")) + reasons = [ + "冻结基线未达到全部研究门槛。" if not baseline_passed else "冻结基线通过门槛。", + ( + f"证据矩阵有 {int(evidence.get('fail', 0))} 项失败，" + f"{int(evidence.get('evidence_insufficient', 0))} 项证据不足。" + ), + ] + if candidate_id is not None: + reasons.append( + f"{candidate_id} 是已测试场景中 Calmar（年化收益/最大回撤）最高者，" + "但只作为下一轮迭代候选。" + ) + return { + "schema_version": "strategy-analysis-recommendation/1", + "analysis_id": str(analysis.get("analysis_id", "")), + "strategy_id": str(analysis.get("strategy_id", "")), + "decision": decision, + "recommended_iteration_candidate": candidate_id, + "candidate_accepted": accepted, + "baseline_status": str(baseline.get("status", "unknown")), + "reasons": reasons, + "authority": "local_exploratory", + "not_formal_joinquant_backtest": True, + "vibe_group_analysis_used": False, + "next_action": "human_confirmation_required", + } + + +def _table(headers: list[str], rows: list[list[str]]) -> list[str]: + result = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + result.extend("| " + " | ".join(row) + " |" for row in rows) + return result + + +def _challenge_table(analysis: Mapping[str, Any]) -> list[str]: + rows: list[list[str]] = [] + for item in _rows(analysis.get("challenge_results")): + metrics = _mapping(item.get("metrics")) + rows.append( + [ + str(item.get("scenario_id", "")), + str(item.get("status", "")), + _pct(metrics.get("cumulative_return")), + _pct(metrics.get("cagr")), + _pct(metrics.get("max_drawdown")), + _num(metrics.get("calmar")), + _pct(metrics.get("average_invested_ratio")), + _num(item.get("cold_seconds"), 2), + _num(item.get("warm_seconds"), 2), + ", ".join(str(reason) for reason in item.get("reasons", [])) or "—", + ] + ) + return _table( + [ + "场景", + "状态", + "累计收益", + "年化收益", + "最大回撤", + "Calmar", + "平均仓位", + "冷启动秒", + "预热秒", + "未通过原因", + ], + rows, + ) + + +def _robustness_table(items: object) -> list[str]: + rows: list[list[str]] = [] + for item in _rows(items): + metrics = _mapping(item.get("metrics")) + detail = ( + str(item.get("removed")) + if item.get("removed") is not None + else f"{item.get('start', '')}~{item.get('end', '')}".strip("~") + ) + if not detail: + detail_parts = [] + for key in ( + "worst_account_loss", + "cvar", + "probability_drawdown_over_20pct", + "probability_drawdown_over_30pct", + ): + if key in metrics: + detail_parts.append(f"{key}={_pct(metrics[key])}") + detail = "; ".join(detail_parts) or "—" + rows.append( + [ + str(item.get("scenario_id", "")), + str(item.get("dimension", "")), + str(item.get("status", "")), + _pct(metrics.get("cumulative_return")), + _pct(metrics.get("cagr")), + _pct(metrics.get("max_drawdown")), + _num(metrics.get("calmar")), + detail, + ", ".join(str(reason) for reason in item.get("reasons", [])) or "—", + ] + ) + if not rows: + return ["无场景。"] + return _table( + [ + "场景", + "维度", + "状态", + "累计收益", + "年化收益", + "最大回撤", + "Calmar", + "补充结果", + "原因", + ], + rows, + ) + + +def _attribution_table(items: object) -> list[str]: + rows = [ + [str(item.get("key", "")), _pct(item.get("contribution"), 3)] + for item in _rows(items) + ] + return _table(["项目", "算术收益贡献"], rows) if rows else ["无归因结果。"] + + +def _count_status(items: object) -> str: + rows = _rows(items) + passed = sum(row.get("status") == "pass" for row in rows) + failed = sum(row.get("status") == "fail" for row in rows) + missing = sum(row.get("status") == "evidence_insufficient" for row in rows) + return f"共 {len(rows)} 项：通过 {passed}，失败 {failed}，证据不足 {missing}。" + + +def render_analysis_report( + analysis: Mapping[str, Any], + recommendation: Mapping[str, Any], + vibe_evidence: Mapping[str, Any], +) -> str: + baseline = _mapping(analysis.get("baseline")) + metrics = _mapping(baseline.get("metrics")) + risk = _mapping(baseline.get("risk_control")) + benchmark_block = _mapping(analysis.get("benchmarks")) + benchmark_stats = _mapping(benchmark_block.get("statistics")) + attribution = _mapping(analysis.get("attribution")) + robustness = _mapping(analysis.get("robustness")) + evidence = _mapping(analysis.get("evidence_matrix")) + best = recommendation.get("recommended_iteration_candidate") or "无" + baseline_passed = baseline.get("status") == "pass" + candidate_accepted = bool(recommendation.get("candidate_accepted")) + if baseline_passed and candidate_accepted: + decision_summary = ( + "冻结基线已通过全部研究门槛，证据矩阵也没有失败或证据不足；" + "当前仅可进入人工确认。" + ) + candidate_summary = ( + f"已测试场景中，`{best}` 的 Calmar（年化收益/最大回撤）最高且已通过门槛，" + "但仍不自动替换基线或启动聚宽正式复核。" + ) + elif baseline_passed: + decision_summary = ( + "冻结基线已通过自身研究门槛，但其他挑战或证据仍未全部通过，" + "当前方案不应视为已完成验证。" + ) + candidate_summary = ( + f"已测试场景中，`{best}` 的 Calmar 最高，但只能作为后续人工复核对象。" + ) + else: + decision_summary = ( + "冻结基线未通过全部研究门槛，当前方案不应视为已通过。" + ) + candidate_summary = ( + f"已测试场景中，`{best}` 表现最好，但仍只作为下一轮迭代候选，" + "不自动替换基线。" + ) + + challenge_rows = _rows(analysis.get("challenge_results")) + challenge_pass = sum(row.get("status") == "pass" for row in challenge_rows) + challenge_fail = sum(row.get("status") == "fail" for row in challenge_rows) + challenge_missing = sum( + row.get("status") == "evidence_insufficient" for row in challenge_rows + ) + if challenge_rows and challenge_pass == len(challenge_rows): + challenge_summary = ( + f"{len(challenge_rows)} 个基础场景全部通过各自研究门槛；" + "是否进入下一阶段仍需人工确认。" + ) + elif challenge_rows and challenge_fail == len(challenge_rows): + challenge_summary = ( + f"{len(challenge_rows)} 个基础场景均未通过研究门槛。" + "表现最好的场景仍不能被视为已验证候选。" + ) + else: + challenge_summary = ( + f"基础场景中 {challenge_pass} 项通过、{challenge_fail} 项失败、" + f"{challenge_missing} 项证据不足；不能用单个最佳场景替代整体判断。" + ) + + lines = [ + "# 本地策略完整分析报告", + "", + "## 1. 结论与推荐", + "", + f"推荐结论：`{recommendation.get('decision')}`。{decision_summary}", + "", + f"{candidate_summary} 下一步固定为 `{recommendation.get('next_action')}`，等待人工确认。", + "", + "## 2. 研究身份与边界", + "", + f"- 分析标识：`{analysis.get('analysis_id')}`", + f"- 策略标识：`{analysis.get('strategy_id')}`", + f"- 确定性分析耗时：{_num(analysis.get('analysis_seconds'), 2)} 秒", + "- 权限：本地探索性研究，不是 JoinQuant（聚宽）正式回测、模拟交易或最终验收。", + "- 事实源：标准分析数据包、双基准集和确定性证据矩阵。", + "", + "## 3. 收益与回撤", + "", + *_table( + ["指标", "结果"], + [ + ["累计收益", _pct(metrics.get("cumulative_return"))], + ["年化收益", _pct(metrics.get("cagr"))], + ["最大回撤", _pct(metrics.get("max_drawdown"))], + ["最长回撤期（交易日）", _num(metrics.get("max_drawdown_duration"), 0)], + ["年化波动率", _pct(metrics.get("annualized_volatility"))], + ["Sharpe（夏普比率）", _num(metrics.get("sharpe"))], + ["Sortino（索提诺比率）", _num(metrics.get("sortino"))], + ["Calmar（年化收益/最大回撤）", _num(metrics.get("calmar"))], + ["门槛状态", str(baseline.get("status", "unknown"))], + ["未通过原因", ", ".join(baseline.get("reasons", [])) or "—"], + ], + ), + "", + "## 4. 双基准与 Alpha/Beta（超额收益/市场暴露）", + "", + ] + benchmark_rows: list[list[str]] = [] + for benchmark_id, raw in benchmark_stats.items(): + item = _mapping(raw) + benchmark_rows.append( + [ + str(benchmark_id), + _pct(item.get("strategy_return")), + _pct(item.get("benchmark_return")), + _pct(item.get("active_return")), + _pct(item.get("alpha")), + _num(item.get("beta"), 4), + _num(item.get("correlation"), 4), + _num(item.get("information_ratio"), 4), + _num(item.get("up_capture"), 4), + _num(item.get("down_capture"), 4), + ] + ) + lines.extend( + _table( + [ + "基准", + "共同日策略收益", + "基准收益", + "主动收益", + "Alpha", + "Beta", + "相关性", + "信息比率", + "上涨捕获", + "下跌捕获", + ], + benchmark_rows, + ) + ) + active_returns = [ + value + for raw in benchmark_stats.values() + if (value := _finite(_mapping(raw).get("active_return"))) is not None + ] + if active_returns and all(value > 0 for value in active_returns): + benchmark_summary = ( + "全部基准的主动收益为正；仍需结合 Alpha（超额收益）、Beta（市场暴露）" + "和风险指标判断是否具有可持续竞争力。" + ) + elif active_returns and all(value < 0 for value in active_returns): + benchmark_summary = ( + "全部基准的主动收益为负；低 Beta 或正 Alpha 不能抵消累计收益落后的事实。" + ) + elif active_returns and any(value == 0 for value in active_returns): + benchmark_summary = ( + "至少一个基准的主动收益为零；其余基准应分别结合 Alpha（超额收益）、" + "Beta（市场暴露）和风险指标判断。" + ) + elif active_returns: + benchmark_summary = ( + "不同基准的主动收益有正有负；应分别结合 Alpha（超额收益）、" + "Beta（市场暴露）和风险指标判断。" + ) + else: + benchmark_summary = "缺少可比较的主动收益，基准竞争力证据不足。" + lines.extend( + [ + "", + benchmark_summary, + "", + "## 5. 仓位与风险控制", + "", + *_table( + ["指标", "结果"], + [ + ["平均仓位", _pct(risk.get("average_invested_ratio"))], + ["中位仓位", _pct(risk.get("median_invested_ratio"))], + ["低于半仓的日期占比", _pct(risk.get("below_half_ratio"))], + ["接近满仓的日期占比", _pct(risk.get("near_full_ratio"))], + ["平均现金", _pct(risk.get("average_cash_ratio"))], + ["最高仓位", _pct(risk.get("maximum_invested_ratio"))], + ["最高单标的权重", _pct(risk.get("maximum_security_weight"))], + ["最高资产组权重", _pct(risk.get("maximum_asset_group_weight"))], + ["计划风险覆盖率", _pct(risk.get("planned_risk_coverage"))], + [ + "最高计划损失比例", + _pct(risk.get("maximum_planned_loss_ratio")), + ], + [ + "最高有效 N 风险单位", + _num(risk.get("maximum_effective_risk_units"), 2), + ], + [ + "组合单位预算最高利用率", + _pct(risk.get("maximum_portfolio_unit_utilization")), + ], + ["最高60日已实现波动率", _pct(risk.get("maximum_realized_60d_volatility"))], + ], + ), + "", + "### 交易与成本", + "", + *_table( + ["指标", "结果"], + [ + ["成交订单", _num(risk.get("filled_order_count"), 0)], + ["平仓订单", _num(risk.get("closed_order_count"), 0)], + ["平仓胜率", _pct(risk.get("closed_order_win_rate"))], + ["费用", _money(risk.get("fees"))], + ["保护止损事件", _num(risk.get("protective_stop_events"), 0)], + [ + "全量仓位再分配事件", + _num(risk.get("redistribution_event_count"), 0), + ], + ], + ), + "", + "## 6. 归因分析", + "", + f"口径：{attribution.get('method', '—')}。限制：{attribution.get('limitation', '—')}。勾稽误差：{_num(attribution.get('reconciliation_error'), 8)}。", + "", + "### 证券归因", + "", + *_attribution_table(attribution.get("security")), + "", + "### 资产组归因", + "", + *_attribution_table(attribution.get("asset_group")), + "", + "### 交易原因归因", + "", + *_attribution_table(attribution.get("trading_reason")), + "", + "### 年度归因", + "", + *_attribution_table(attribution.get("period")), + "", + "## 7. 基础场景挑战", + "", + *_challenge_table(analysis), + "", + challenge_summary, + "", + "## 8. 稳健性与压力测试", + "", + ( + f"证据矩阵共 {int(evidence.get('rows', 0))} 项：通过 " + f"{int(evidence.get('pass', 0))}，失败 {int(evidence.get('fail', 0))}，" + f"证据不足 {int(evidence.get('evidence_insufficient', 0))}。" + ), + "", + ] + ) + robustness_sections = ( + ("时期与滚动窗口", "periods"), + ("资产与资产组删除", "asset_deletions"), + ("成本与延迟执行", "cost_execution"), + ("区块抽样", "bootstrap"), + ("历史压力", "historical_stress"), + ("持仓冲击", "position_shocks"), + ("CVaR（条件风险价值）", "cvar"), + ) + for title, key in robustness_sections: + items = robustness.get(key) + lines.extend( + [ + f"### {title}", + "", + _count_status(items), + "", + *_robustness_table(items), + "", + ] + ) + opposing = _rows(analysis.get("opposing_evidence")) + lines.extend(["## 9. 反对证据", ""]) + if opposing: + for item in opposing: + kind = str(item.get("kind", "unknown")) + identity = item.get("scenario_id") or item.get("benchmark_id") or "" + value = ( + _pct(item.get("active_return")) + if item.get("active_return") is not None + else ", ".join(str(reason) for reason in item.get("reasons", [])) + ) + lines.append(f"- {kind}：{identity}；{value or '—'}") + else: + lines.append("- 无。") + violation = _mapping(vibe_evidence.get("boundary_violation")) + swarm = _mapping(vibe_evidence.get("swarm")) + single_agent = _mapping(vibe_evidence.get("single_agent")) + use_result = _mapping(vibe_evidence.get("use_result")) + if violation or swarm: + vibe_lines = [ + "- Vibe 群体分析是已知缺陷路径，本次误调用已作为边界违规记录。", + f"- 运行 `{swarm.get('run_id', '—')}` 的 `valid_evidence` 为 `{swarm.get('valid_evidence', False)}`，并已排除出全部结论。", + f"- 边界原因：{violation.get('reason', '群体分析结果不得使用。')}", + "- 已加载的绩效归因、风险分析和报告能力只是方法文档，不是实际单体分析结果。", + "- 当前公开接口没有可用的单体策略结果分析入口，因此本报告不采用任何 Vibe 结论。", + ] + elif use_result.get("vibe_conclusion_available") and single_agent: + assessment = _mapping(single_agent.get("assessment")) + alignment = str( + assessment.get("recommendation_alignment", "未提供推荐一致性说明。") + ) + capabilities = ", ".join( + str(item) for item in vibe_evidence.get("capabilities_loaded", []) + ) + vibe_lines = [ + "- 本次未调用 Vibe 群体分析，群体结论未进入证据矩阵、报告或推荐。", + f"- Vibe 单体复核运行 `{single_agent.get('run_id', '—')}` 已完成；公开入口为 `{single_agent.get('interface', '—')}`。", + f"- 实际加载能力：{capabilities or '—'}。", + f"- 定性复核：{alignment}", + "- Vibe 结果权限为审计辅助，不替代确定性数值裁判，也不改变任何门槛状态。", + ] + else: + vibe_lines = [ + "- 本次未调用 Vibe 群体分析，Vibe 群体结论未进入证据矩阵、报告或推荐。", + f"- {use_result.get('reason', '本报告只采用确定性本地分析。')}", + ] + uncertainty_lines = [ + "- 本结果是本地探索性模拟，不是聚宽正式回测；平台撮合差异尚未复核。", + "- 收益与权益采用连续经济总回报近似：连续因子只由上一交易日原始收盘价与当日原始前收盘价生成，公司行动元数据只用于审计，且可能包含事后核对记录。", + "- 经济单位不等同于真实 ETF 份额；现金分红按除权日隐含再投资，未模拟支付日现金、税费和零碎份额，因此不能与聚宽逐日账户精确对账。", + "- 固定时期和滚动窗口是基线既有路径切片，不是从空仓和初始资金重新回测。", + "- 资产与资产组删除是收益贡献删除敏感性，不重新分配资金。", + "- 成本与延迟场景采用一阶订单级敏感性估算，不是完整交易路径重跑。", + "- 归因为日度算术贡献，不是几何链式归因；不能直接与复利累计收益逐项相加解释。", + ] + stop_failure = next( + ( + row + for row in _rows(_mapping(analysis.get("robustness")).get("position_shocks")) + if row.get("scenario_id") == "shock-stop-failure" + ), + None, + ) + if stop_failure is not None and stop_failure.get("status") == "evidence_insufficient": + uncertainty_lines.append( + "- `shock-stop-failure` 缺少所需来源输入，保留为证据不足，未用假设值补齐。" + ) + uncertainty_lines.extend( + [ + "- 60日已实现波动率是事后诊断，不等同于下单时协方差预测门禁。", + "- 双基准只在三方共同交易日计算，策略共同日收益与全样本收益不同。", + ] + ) + if recommendation.get("decision") == "proceed_to_joinquant": + confirmation_summary = ( + "建议人工确认是否进入 JoinQuant（聚宽）正式复核；该确认不等于冻结策略、" + "启动模拟交易或接受本地结果为正式结论。" + ) + else: + confirmation_summary = ( + f"建议人工确认“修改后再评估”，并把 `{best}` 作为下一轮研究起点，" + "而不是直接采用、冻结或送入聚宽正式回测。" + ) + lines.extend( + [ + "", + "## 10. Vibe 安全边界", + "", + *vibe_lines, + "", + "## 11. 不确定性", + "", + *uncertainty_lines, + "", + "## 12. 人工确认", + "", + f"当前停止状态：`{recommendation.get('next_action')}`。{confirmation_summary}", + "", + ] + ) + return "\n".join(lines) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_name, path) + finally: + temporary = Path(temporary_name) + if temporary.exists(): + temporary.unlink() + + +def _read_document(path: Path, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read {label}: {path}") from exc + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _write_json(path: Path, value: Mapping[str, Any]) -> None: + content = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + _atomic_write(path, content + "\n") + + +def write_analysis_delivery(workspace: Path) -> dict[str, Any]: + root = Path(workspace).resolve() + analysis_path = root / "deterministic-analysis.json" + vibe_path = root / "vibe-evidence.json" + report_path = root / "local-strategy-analysis-report.md" + recommendation_path = root / "recommendation.json" + + analysis = _read_document(analysis_path, "deterministic analysis") + vibe = enforce_vibe_boundary(_read_document(vibe_path, "Vibe evidence")) + analysis_seconds = _finite(analysis.get("analysis_seconds")) + if analysis_seconds is None or analysis_seconds < 0: + raise ValueError("deterministic analysis must record analysis_seconds") + vibe["analysis_seconds"] = analysis_seconds + recommendation = build_recommendation(analysis) + report = render_analysis_report(analysis, recommendation, vibe) + + _write_json(vibe_path, vibe) + _atomic_write(report_path, report) + recommendation["artifacts"] = { + "deterministic_analysis_sha256": _sha256(analysis_path), + "vibe_evidence_sha256": _sha256(vibe_path), + "report_sha256": _sha256(report_path), + } + _write_json(recommendation_path, recommendation) + return { + "analysis_id": recommendation["analysis_id"], + "decision": recommendation["decision"], + "report": report_path.as_posix(), + "recommendation": recommendation_path.as_posix(), + "vibe_evidence": vibe_path.as_posix(), + "next_action": recommendation["next_action"], + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate a deterministic local strategy analysis delivery" + ) + parser.add_argument("--workspace", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + print( + json.dumps( + write_analysis_delivery(args.workspace), + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/research/quant_analysis/robustness.py b/scripts/research/quant_analysis/robustness.py new file mode 100644 index 0000000..956f41d --- /dev/null +++ b/scripts/research/quant_analysis/robustness.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import math + +import numpy as np + + +def block_bootstrap( + returns: np.ndarray, + block_size: int, + paths: int, + horizon: int, + seed: int, +) -> np.ndarray: + values = np.asarray(returns, dtype=np.float64) + if values.ndim != 1 or values.size == 0 or not np.isfinite(values).all(): + raise ValueError("returns must be a finite one-dimensional array") + if np.any(values <= -1.0): + raise ValueError("returns must be greater than -100%") + if block_size <= 0 or paths <= 0 or horizon <= 0: + raise ValueError("block_size, paths and horizon must be positive") + output = np.empty((paths, horizon), dtype=np.float64) + generator = np.random.default_rng(seed) + blocks = math.ceil(horizon / block_size) + offsets = np.arange(block_size, dtype=np.int64) + batch_size = min(256, paths) + for first in range(0, paths, batch_size): + count = min(batch_size, paths - first) + starts = generator.integers(0, values.size, size=(count, blocks)) + indices = (starts[:, :, None] + offsets) % values.size + output[first : first + count] = values[indices].reshape(count, -1)[:, :horizon] + return output + + +def summarize_bootstrap(paths: np.ndarray) -> dict[str, float]: + values = np.asarray(paths, dtype=np.float64) + if values.ndim != 2 or values.shape[0] == 0 or values.shape[1] == 0: + raise ValueError("bootstrap paths must be a non-empty matrix") + wealth = np.cumprod(1.0 + values, axis=1) + peaks = np.maximum(1.0, np.maximum.accumulate(wealth, axis=1)) + max_drawdown = np.min(wealth / peaks - 1.0, axis=1) + terminal = wealth[:, -1] - 1.0 + return { + "probability_drawdown_over_20pct": float(np.mean(max_drawdown < -0.20)), + "probability_drawdown_over_30pct": float(np.mean(max_drawdown < -0.30)), + "median_terminal_return": float(np.median(terminal)), + } diff --git a/scripts/research/quant_analysis/schemas/analysis-plan.schema.json b/scripts/research/quant_analysis/schemas/analysis-plan.schema.json new file mode 100644 index 0000000..13f4b37 --- /dev/null +++ b/scripts/research/quant_analysis/schemas/analysis-plan.schema.json @@ -0,0 +1,181 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://local.vibe-trading/analysis-plan.schema.json", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "strategy_id", "baseline_config", "scenarios", "universe", "analyses", "expected", "thresholds"], + "properties": { + "schema_version": {"const": "strategy-analysis-plan/1"}, + "strategy_id": {"type": "string", "minLength": 1}, + "baseline_config": {"type": "string", "minLength": 1}, + "scenarios": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["scenario_id", "dimension", "overrides"], + "properties": { + "scenario_id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$"}, + "dimension": {"type": "string", "minLength": 1}, + "overrides": {"type": "object"} + } + } + }, + "universe": { + "type": "object", + "additionalProperties": {"type": "string", "minLength": 1}, + "minProperties": 1 + }, + "analyses": { + "type": "object", + "required": ["fixed_periods", "rolling", "deletions", "cost_execution", "bootstrap", "historical_stress", "position_shocks", "cvar"], + "additionalProperties": false, + "properties": { + "fixed_periods": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "start", "end"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "start": {"type": "string", "format": "date"}, + "end": {"type": "string", "format": "date"} + } + } + }, + "rolling": { + "type": "object", + "additionalProperties": false, + "required": ["window_years", "step_months"], + "properties": { + "window_years": {"type": "integer", "minimum": 1}, + "step_months": {"type": "integer", "minimum": 1} + } + }, + "deletions": { + "type": "object", + "additionalProperties": false, + "required": ["each_security", "each_asset_group"], + "properties": { + "each_security": {"const": true}, + "each_asset_group": {"const": true} + } + }, + "cost_execution": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "commission_multiplier", "slippage", "delay_days"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "commission_multiplier": {"type": "number", "minimum": 0}, + "slippage": {"type": "number", "minimum": 0}, + "delay_days": {"type": "integer", "minimum": 0} + } + } + }, + "bootstrap": { + "type": "object", + "additionalProperties": false, + "required": ["block_sizes", "paths", "horizon_days", "seed", "thresholds"], + "properties": { + "block_sizes": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "integer", "minimum": 1}}, + "paths": {"type": "integer", "minimum": 1}, + "horizon_days": {"type": "integer", "minimum": 1}, + "seed": {"type": "integer"}, + "thresholds": { + "type": "object", + "additionalProperties": false, + "required": ["probability_drawdown_over_20pct_max", "probability_drawdown_over_30pct_max", "median_terminal_return_min_exclusive"], + "properties": { + "probability_drawdown_over_20pct_max": {"type": "number", "minimum": 0, "maximum": 1}, + "probability_drawdown_over_30pct_max": {"type": "number", "minimum": 0, "maximum": 1}, + "median_terminal_return_min_exclusive": {"type": "number"} + } + } + } + }, + "historical_stress": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "start", "end", "max_drawdown_abs_max"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "start": {"type": "string", "format": "date"}, + "end": {"type": "string", "format": "date"}, + "max_drawdown_abs_max": {"type": "number", "exclusiveMinimum": 0} + } + } + }, + "position_shocks": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "maximum_loss_abs_max"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "asset_group_shocks": {"type": "object", "additionalProperties": {"type": "number"}}, + "security_shocks": {"type": "object", "additionalProperties": {"type": "number"}}, + "use_stop_failure_loss": {"type": "boolean"}, + "maximum_loss_abs_max": {"type": "number", "exclusiveMinimum": 0} + }, + "anyOf": [ + {"required": ["asset_group_shocks"]}, + { + "required": ["use_stop_failure_loss"], + "properties": {"use_stop_failure_loss": {"const": true}} + } + ] + } + }, + "cvar": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "horizon_days", "confidence", "maximum_loss_abs_max", "minimum_tail_observations"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "horizon_days": {"type": "integer", "minimum": 1}, + "confidence": {"type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1}, + "maximum_loss_abs_max": {"type": "number", "exclusiveMinimum": 0}, + "minimum_tail_observations": {"type": "number", "minimum": 1} + } + } + } + } + }, + "expected": { + "type": "object", + "additionalProperties": false, + "required": ["scenario_runs", "benchmarks", "bootstrap_paths", "seed"], + "properties": { + "scenario_runs": {"type": "integer", "minimum": 1}, + "benchmarks": {"const": 2}, + "bootstrap_paths": {"type": "integer", "minimum": 1}, + "seed": {"type": "integer"} + } + }, + "thresholds": { + "type": "object", + "additionalProperties": false, + "required": ["cagr_min_exclusive", "max_drawdown_abs_max", "calmar_min"], + "properties": { + "cagr_min_exclusive": {"type": "number"}, + "max_drawdown_abs_max": {"type": "number", "exclusiveMinimum": 0}, + "calmar_min": {"type": "number"} + } + } + } +} diff --git a/scripts/research/quant_analysis/unified_analysis.py b/scripts/research/quant_analysis/unified_analysis.py new file mode 100644 index 0000000..a29e6e1 --- /dev/null +++ b/scripts/research/quant_analysis/unified_analysis.py @@ -0,0 +1,1493 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq + +from scripts.research.analysis_data.manifest import open_analysis_source +from scripts.research.analysis_data.views import open_analysis_database +from scripts.research.market_data.query import open_snapshot + +from .benchmarks import calculate_benchmark_statistics +from .cvar import calculate_cvar, rolling_compound_returns +from .evidence import ScenarioResult, build_evidence_matrix, evidence_digest +from .robustness import block_bootstrap, summarize_bootstrap + + +BENCHMARK_IDS = ( + "CSI300_CNY_TOTAL_RETURN", + "NASDAQ100_CNY_TOTAL_RETURN", +) +_FORMULA_VERSION = "unified-strategy-analysis/1" + + +class UnifiedAnalysisError(ValueError): + """Raised when standard result packages cannot support deterministic analysis.""" + + +def deterministic_next_action() -> str: + return "generate_deterministic_local_report" + + +def _with_analysis_seconds( + summary: Mapping[str, object], + measured_seconds: float, + output_path: Path, +) -> dict[str, object]: + measured = float(measured_seconds) + if not math.isfinite(measured) or measured < 0: + raise UnifiedAnalysisError("analysis_seconds must be finite and non-negative") + seconds = measured + path = Path(output_path) + if path.is_file(): + existing = _load_json(path, label="deterministic analysis") + if existing.get("analysis_id") != summary.get("analysis_id"): + raise UnifiedAnalysisError("existing deterministic analysis identity mismatch") + prior = existing.get("analysis_seconds") + if prior is not None: + prior_seconds = float(prior) + if not math.isfinite(prior_seconds) or prior_seconds < 0: + raise UnifiedAnalysisError( + "existing analysis_seconds must be finite and non-negative" + ) + seconds = prior_seconds + result = dict(summary) + result["analysis_seconds"] = seconds + return result + + +@dataclass(frozen=True) +class ScenarioInput: + scenario_id: str + run_id: str + result_dir: Path + returns: pd.Series + balances: pd.DataFrame + positions: pd.DataFrame + orders: pd.DataFrame + events: pd.DataFrame + params: Mapping[str, object] + performance: Mapping[str, object] + + +def _atomic_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = ( + json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_name, path) + finally: + temporary = Path(temporary_name) + if temporary.exists(): + temporary.unlink() + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _load_json(path: Path, *, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise UnifiedAnalysisError(f"cannot read {label}: {path}") from exc + if not isinstance(value, dict): + raise UnifiedAnalysisError(f"{label} must be a JSON object") + return value + + +def _safe_number(value: object) -> float | int | None: + if value is None: + return None + if isinstance(value, (np.integer, int)): + return int(value) + numeric = float(value) + return numeric if math.isfinite(numeric) else None + + +def _numeric_metrics(values: Mapping[str, object]) -> dict[str, float | int | None]: + return {str(key): _safe_number(value) for key, value in values.items()} + + +def calculate_return_metrics(returns: pd.Series, *, annualization: int = 252) -> dict[str, float | int | None]: + values = np.asarray(returns, dtype=np.float64) + if annualization <= 0 or values.ndim != 1 or values.size == 0: + raise UnifiedAnalysisError("return series must be non-empty") + if not np.isfinite(values).all() or np.any(values <= -1.0): + raise UnifiedAnalysisError("return series contains invalid values") + wealth = np.cumprod(1.0 + values) + cumulative_return = float(wealth[-1] - 1.0) + cagr = float((1.0 + cumulative_return) ** (annualization / values.size) - 1.0) + volatility = float(np.std(values, ddof=1) * math.sqrt(annualization)) if values.size > 1 else 0.0 + mean_annual = float(np.mean(values) * annualization) + downside_values = np.minimum(values, 0.0) + downside = float(math.sqrt(float(np.mean(downside_values**2))) * math.sqrt(annualization)) + peaks = np.maximum(1.0, np.maximum.accumulate(wealth)) + drawdowns = wealth / peaks - 1.0 + maximum_drawdown = float(np.min(drawdowns)) + underwater = drawdowns < 0.0 + maximum_duration = 0 + current_duration = 0 + for value in underwater: + current_duration = current_duration + 1 if value else 0 + maximum_duration = max(maximum_duration, current_duration) + return { + "observations": int(values.size), + "cumulative_return": cumulative_return, + "cagr": cagr, + "annualized_volatility": volatility, + "sharpe": None if volatility == 0.0 else mean_annual / volatility, + "sortino": None if downside == 0.0 else mean_annual / downside, + "max_drawdown": maximum_drawdown, + "max_drawdown_duration": maximum_duration, + "calmar": None if maximum_drawdown == 0.0 else cagr / abs(maximum_drawdown), + "daily_hit_rate": float(np.mean(values[values != 0.0] > 0.0)) if np.any(values != 0.0) else None, + } + + +def evaluate_metrics( + metrics: Mapping[str, object], thresholds: Mapping[str, object] +) -> tuple[str, list[str]]: + checks = ( + ("cagr_min_exclusive", "cagr", lambda value, limit: value > limit), + ( + "max_drawdown_abs_max", + "max_drawdown", + lambda value, limit: abs(value) <= limit, + ), + ("calmar_min", "calmar", lambda value, limit: value >= limit), + ) + reasons: list[str] = [] + for threshold_name, metric_name, predicate in checks: + value = metrics.get(metric_name) + if value is None or not predicate(float(value), float(thresholds[threshold_name])): + reasons.append(threshold_name) + return ("pass" if not reasons else "fail", reasons) + + +def align_three_way_benchmarks( + strategy: pd.Series, + benchmarks: Mapping[str, pd.Series], +) -> tuple[pd.DataFrame, dict[str, object]]: + if tuple(benchmarks) != BENCHMARK_IDS: + raise UnifiedAnalysisError("benchmark set must contain the two canonical benchmarks") + common = set(pd.DatetimeIndex(strategy.index).normalize()) + for benchmark_id in BENCHMARK_IDS: + common &= set(pd.DatetimeIndex(benchmarks[benchmark_id].index).normalize()) + common_index = pd.DatetimeIndex(sorted(common)) + if common_index.empty: + raise UnifiedAnalysisError("strategy and benchmarks have no shared dates") + aligned = pd.DataFrame(index=common_index) + aligned["strategy"] = strategy.groupby(pd.DatetimeIndex(strategy.index).normalize()).last().reindex(common_index) + for benchmark_id in BENCHMARK_IDS: + normalized = benchmarks[benchmark_id].groupby( + pd.DatetimeIndex(benchmarks[benchmark_id].index).normalize() + ).last() + aligned[benchmark_id] = normalized.reindex(common_index) + if aligned.isna().any().any(): + raise UnifiedAnalysisError("benchmark alignment produced missing values") + return aligned, { + "alignment": "three-way exact-date inner join", + "common_samples": len(aligned), + "strategy_samples": len(strategy), + "strategy_excluded_dates": len(strategy) - len(aligned), + "benchmark_samples": { + benchmark_id: len(benchmarks[benchmark_id]) for benchmark_id in BENCHMARK_IDS + }, + "benchmark_excluded_dates": { + benchmark_id: len(benchmarks[benchmark_id]) - len(aligned) + for benchmark_id in BENCHMARK_IDS + }, + } + + +def _find_attribution_file(result_dir: Path, manifest: Mapping[str, object]) -> Path: + extensions = manifest.get("extensions") + if not isinstance(extensions, Mapping): + raise UnifiedAnalysisError("result package has no attribution extension") + matches: list[Mapping[str, object]] = [] + for extension in extensions.values(): + if isinstance(extension, Mapping): + entry = extension.get("attribution_log") + if isinstance(entry, Mapping): + matches.append(entry) + if len(matches) != 1: + raise UnifiedAnalysisError("result package must expose one attribution log") + files = matches[0].get("files") + if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], Mapping): + raise UnifiedAnalysisError("attribution log file declaration is invalid") + relative = Path(str(files[0].get("path", ""))) + path = (result_dir / relative).resolve() + if relative.is_absolute() or ".." in relative.parts or not path.is_relative_to(result_dir): + raise UnifiedAnalysisError("attribution log path is unsafe") + if not path.is_file() or _sha256(path) != files[0].get("sha256"): + raise UnifiedAnalysisError("attribution log digest mismatch") + return path + + +def _load_scenario(result_dir: Path) -> ScenarioInput: + result_dir = Path(result_dir).resolve() + source = open_analysis_source(result_dir) + manifest = dict(source.manifest) + run = manifest.get("run") + if not isinstance(run, Mapping): + raise UnifiedAnalysisError("result run identity is missing") + with open_analysis_database(result_dir) as database: + returns_frame = database.connection.sql( + "select trading_date, daily_returns from strategy_daily_returns " + "where comparable order by trading_date" + ).fetchdf() + balances = database.connection.sql( + "select time, total_value, net_value, cash, aval_cash from balances order by time" + ).fetchdf() + positions = database.connection.sql("select * from positions order by time, security").fetchdf() + orders = database.connection.sql("select * from orders order by time, security").fetchdf() + returns = pd.Series( + returns_frame["daily_returns"].astype(float).to_numpy(), + index=pd.to_datetime(returns_frame["trading_date"]).dt.normalize(), + name="return", + ) + for frame in (balances, positions, orders): + if "time" in frame: + frame["date"] = pd.to_datetime(frame["time"]).dt.normalize() + events = pq.read_table(_find_attribution_file(result_dir, manifest)).to_pandas() + events["event_time"] = pd.to_datetime(events["time"]) + events["date"] = events["event_time"].dt.normalize() + params = _load_json(result_dir / "params.json", label="scenario params") + performance = _load_json(result_dir / "performance.json", label="performance evidence") + return ScenarioInput( + scenario_id=str(run["scenario_id"]), + run_id=str(run["run_id"]), + result_dir=result_dir, + returns=returns, + balances=balances, + positions=positions, + orders=orders, + events=events, + params=params, + performance=performance, + ) + + +def _immutable_json(path: Path, value: object) -> None: + if path.exists(): + existing = _load_json(path, label=path.name) + if existing != value: + raise UnifiedAnalysisError(f"immutable analysis identity collision: {path}") + return + _atomic_json(path, value) + + +def _register_source_results( + repo_root: Path, + preparation_workspace: Path, + source_registry: Mapping[str, str], +) -> dict[str, object]: + preparation_root = Path(preparation_workspace).resolve() + expected_preparation_root = repo_root / ".local" / "strategy-analysis-preparations" + if not preparation_root.is_relative_to(expected_preparation_root): + raise UnifiedAnalysisError( + "preparation workspace is outside .local/strategy-analysis-preparations" + ) + preparation = _load_json( + preparation_root / "preparation.json", label="analysis preparation" + ) + preparation_id = str(preparation.get("preparation_id", "")) + if not preparation_id or preparation_id != preparation_root.name: + raise UnifiedAnalysisError("analysis preparation identity mismatch") + scenarios = _load_json( + preparation_root / "analysis-scenarios.json", label="analysis scenarios" + ) + strategy_id = str(scenarios["strategy_id"]) + run_root = repo_root / ".local" / "quant-research" / strategy_id + planned_scenarios = scenarios.get("scenarios") + if not isinstance(planned_scenarios, Sequence) or isinstance( + planned_scenarios, (str, bytes) + ): + raise UnifiedAnalysisError("analysis scenarios are invalid") + if len(planned_scenarios) < 2: + raise UnifiedAnalysisError( + "source registry requires at least two planned scenarios" + ) + scenario_ids = [str(scenario["scenario_id"]) for scenario in planned_scenarios] + registered_ids = {str(key) for key in source_registry} + if registered_ids != set(scenario_ids): + raise UnifiedAnalysisError( + "source registry must explicitly contain every planned scenario_id exactly once" + ) + run_ids = [str(source_registry[scenario_id]) for scenario_id in scenario_ids] + if any(not run_id or "/" in run_id or "\\" in run_id for run_id in run_ids): + raise UnifiedAnalysisError("source registry contains an unsafe run_id") + if len(run_ids) != len(set(run_ids)): + raise UnifiedAnalysisError("source registry run_id values must be unique") + + sources: list[dict[str, object]] = [] + identities: list[dict[str, object]] = [] + for scenario in planned_scenarios: + scenario_id = str(scenario["scenario_id"]) + run_id = str(source_registry[scenario_id]) + params_path = preparation_root / "scenario-configs" / scenario_id / "params.json" + params_sha256 = _sha256(params_path) + source_root = (run_root / run_id).resolve() + if not source_root.is_relative_to(run_root): + raise UnifiedAnalysisError("source registry run_id escapes the strategy run root") + run_manifest_path = source_root / "run-manifest.json" + run_manifest = _load_json(run_manifest_path, label="run manifest") + inputs = run_manifest.get("inputs") + snapshot = run_manifest.get("snapshot") + if ( + run_manifest.get("status") != "complete" + or run_manifest.get("project_id") != strategy_id + or run_manifest.get("run_id") != run_id + or not isinstance(inputs, Mapping) + or inputs.get("project_config_sha256") != params_sha256 + or not isinstance(snapshot, Mapping) + ): + raise UnifiedAnalysisError( + f"scenario {scenario_id} does not match its explicitly registered run" + ) + code_identity = inputs.get("code_identity") + execution = ( + code_identity.get("execution") + if isinstance(code_identity, Mapping) + else None + ) + identity = { + "snapshot_id": str(snapshot.get("snapshot_id", "")), + "code_identity_sha256": str(inputs.get("code_identity_sha256", "")), + "code_sha256": str(inputs.get("code_sha256", "")), + "execution_backend": dict(execution) if isinstance(execution, Mapping) else {}, + } + execution_dependencies = ( + execution.get("dependencies") if isinstance(execution, Mapping) else None + ) + if ( + not identity["snapshot_id"] + or not identity["code_identity_sha256"] + or not identity["code_sha256"] + or not identity["execution_backend"] + or not isinstance(execution_dependencies, Mapping) + or not execution_dependencies + ): + raise UnifiedAnalysisError(f"scenario {scenario_id} execution identity is incomplete") + identities.append(identity) + + result_dir = source_root / "backtests" / f"local-{scenario_id}" + local_manifest = _load_json( + result_dir / "manifest.json", label="local result manifest" + ) + local_run = local_manifest.get("run") + local_source = local_manifest.get("source") + local_engine = ( + local_source.get("engine") + if isinstance(local_source, Mapping) + else None + ) + if ( + not isinstance(local_run, Mapping) + or local_run.get("scenario_id") != scenario_id + or local_run.get("run_id") != run_id + or local_run.get("snapshot_id") != identity["snapshot_id"] + ): + raise UnifiedAnalysisError( + f"scenario {scenario_id} local result identity mismatch" + ) + if ( + not isinstance(local_engine, Mapping) + or local_engine.get("backend") != execution.get("backend") + or local_engine.get("adapter_version") != execution.get("adapter_version") + or any( + local_engine.get(str(name)) != version + for name, version in execution_dependencies.items() + ) + ): + raise UnifiedAnalysisError( + f"scenario {scenario_id} local result execution backend identity mismatch" + ) + performance_path = result_dir / "performance.json" + performance = _load_json(performance_path, label="performance evidence") + cleanup = performance.get("cleanup") + if ( + performance.get("status") != "pass" + or performance.get("result_match") is not True + or float(performance.get("cold_seconds", math.inf)) > 180.0 + or float(performance.get("warm_seconds", math.inf)) > 180.0 + or not isinstance(cleanup, Mapping) + or not cleanup + or not all(value is True for value in cleanup.values()) + ): + raise UnifiedAnalysisError(f"scenario {scenario_id} failed its performance gate") + sources.append( + { + "scenario_id": scenario_id, + "dimension": str(scenario["dimension"]), + "run_id": run_id, + "run_manifest": run_manifest_path.relative_to(repo_root).as_posix(), + "run_manifest_sha256": _sha256(run_manifest_path), + "result_dir": result_dir.relative_to(repo_root).as_posix(), + "result_manifest_sha256": _sha256(result_dir / "manifest.json"), + "params_sha256": params_sha256, + "output_set_sha256": str(run_manifest["output_set_sha256"]), + "performance": { + "cold_seconds": float(performance["cold_seconds"]), + "warm_seconds": float(performance["warm_seconds"]), + "result_match": True, + "cleanup": dict(cleanup), + }, + } + ) + + identity_names = ( + "snapshot_id", + "code_identity_sha256", + "code_sha256", + "execution_backend", + ) + for identity_name in identity_names: + if any( + identity[identity_name] != identities[0][identity_name] + for identity in identities[1:] + ): + label = ( + "execution backend identity" + if identity_name == "execution_backend" + else identity_name + ) + raise UnifiedAnalysisError(f"registered sources do not share one {label}") + shared_identity = dict(identities[0]) + shared_identity["execution_backend_sha256"] = evidence_digest( + shared_identity["execution_backend"] + ) + source_registry_sha256 = evidence_digest( + [ + { + "scenario_id": source["scenario_id"], + "run_id": source["run_id"], + "run_manifest_sha256": source["run_manifest_sha256"], + "result_manifest_sha256": source["result_manifest_sha256"], + "output_set_sha256": source["output_set_sha256"], + } + for source in sources + ] + ) + analysis_id = evidence_digest( + { + "formula_version": _FORMULA_VERSION, + "preparation_id": preparation_id, + "source_registry_sha256": source_registry_sha256, + "shared_identity": shared_identity, + } + ) + document = { + "schema_version": "strategy-analysis-source-results/1", + "strategy_id": strategy_id, + "analysis_id": analysis_id, + "preparation_id": preparation_id, + "preparation_sha256": _sha256(preparation_root / "preparation.json"), + "analysis_scenarios_sha256": _sha256( + preparation_root / "analysis-scenarios.json" + ), + "source_registry": { + "explicit": True, + "scenario_count": len(scenario_ids), + "run_id_count": len(run_ids), + "sha256": source_registry_sha256, + }, + "shared_identity": shared_identity, + "sources": sources, + "expected": len(planned_scenarios), + "actual": len(sources), + "source_mutation": "forbidden", + } + analysis_root = repo_root / ".local" / "strategy-analysis" / analysis_id + _immutable_json(analysis_root / "preparation.json", preparation) + _immutable_json(analysis_root / "analysis-scenarios.json", scenarios) + _immutable_json(analysis_root / "source-results.json", document) + return document + + +def _valuation_facts(scenario: ScenarioInput) -> pd.DataFrame: + columns = [ + "date", + "security", + "reason_code", + "source_reason", + "security_daily_pnl", + "common_stop_after", + "stop_failure_loss", + ] + if scenario.events.empty: + return pd.DataFrame(columns=columns) + if "event_type" not in scenario.events or "details_json" not in scenario.events: + raise UnifiedAnalysisError("attribution log does not expose valuation facts") + events = scenario.events.loc[ + scenario.events["event_type"] == "valuation" + ].copy() + if events.empty: + return pd.DataFrame(columns=columns) + if "date" not in events: + time_column = "event_time" if "event_time" in events else "time" + events["date"] = pd.to_datetime(events[time_column]).dt.normalize() + records: list[dict[str, object]] = [] + for event in events.to_dict("records"): + try: + details = json.loads(str(event["details_json"])) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise UnifiedAnalysisError("valuation details_json is invalid") from exc + if not isinstance(details, Mapping): + raise UnifiedAnalysisError("valuation details_json must be an object") + try: + security_pnl = float(details["security_daily_pnl"]) + except (KeyError, TypeError, ValueError) as exc: + raise UnifiedAnalysisError( + "valuation event has no security_daily_pnl" + ) from exc + if not np.isfinite(security_pnl): + raise UnifiedAnalysisError("valuation security_daily_pnl is invalid") + records.append( + { + "date": pd.Timestamp(event["date"]).normalize(), + "security": str(event["security"]), + "reason_code": str(event["reason_code"]), + "source_reason": str( + details.get("source_reason", event["reason_code"]) + ), + "security_daily_pnl": security_pnl, + "common_stop_after": _safe_number( + details.get("common_stop_after") + ), + "stop_failure_loss": _safe_number( + details.get("stop_failure_loss") + ), + } + ) + facts = pd.DataFrame.from_records(records, columns=columns) + if facts.duplicated(["date", "security"]).any(): + raise UnifiedAnalysisError( + "valuation facts contain duplicate security dates" + ) + return facts.sort_values(["date", "security"]).reset_index(drop=True) + + +def _position_facts( + scenario: ScenarioInput, universe: Mapping[str, str] +) -> pd.DataFrame: + positions = scenario.positions.copy() + if positions.empty: + return positions + positions["asset_group"] = positions["security"].map(universe) + if positions["asset_group"].isna().any(): + raise UnifiedAnalysisError("position security is absent from analysis universe") + equity = scenario.balances.set_index("date")["total_value"].astype(float) + positions["equity"] = positions["date"].map(equity) + positions["market_value"] = positions["amount"].astype(float) * positions["price"].astype(float) + positions["weight"] = positions["market_value"] / positions["equity"] + valuations = _valuation_facts(scenario).rename( + columns={ + "source_reason": "attribution_reason", + "common_stop_after": "common_stop", + } + ) + if valuations.empty: + positions["attribution_reason"] = "unclassified" + positions["common_stop"] = np.nan + positions["stop_failure_loss"] = np.nan + return positions + annotated = positions.merge( + valuations[ + [ + "date", + "security", + "attribution_reason", + "common_stop", + "stop_failure_loss", + ] + ], + on=["date", "security"], + how="left", + validate="one_to_one", + ) + annotated["attribution_reason"] = annotated["attribution_reason"].fillna( + "unclassified" + ) + return annotated + + +def _security_pnl_facts( + scenario: ScenarioInput, universe: Mapping[str, str] +) -> pd.DataFrame: + facts = _valuation_facts(scenario) + if facts.empty: + return pd.DataFrame( + columns=[ + *facts.columns, + "asset_group", + "previous_equity", + "return_contribution", + "attribution_reason", + ] + ) + facts["asset_group"] = facts["security"].map(universe) + if facts["asset_group"].isna().any(): + raise UnifiedAnalysisError( + "valuation security is absent from analysis universe" + ) + balances = scenario.balances.set_index("date")["total_value"].astype(float) + previous_equity = balances.shift(1) + first_date = balances.index.min() + first_day_pnl = float( + facts.loc[facts["date"] == first_date, "security_daily_pnl"].sum() + ) + previous_equity.loc[first_date] = float(balances.loc[first_date]) - first_day_pnl + facts["previous_equity"] = facts["date"].map(previous_equity) + if facts["previous_equity"].isna().any() or ( + facts["previous_equity"] <= 0.0 + ).any(): + raise UnifiedAnalysisError( + "valuation facts cannot resolve previous equity" + ) + facts["return_contribution"] = ( + facts["security_daily_pnl"].astype(float) + / facts["previous_equity"].astype(float) + ) + facts["attribution_reason"] = facts["source_reason"] + return facts + + +def _risk_metrics(scenario: ScenarioInput, positions: pd.DataFrame) -> dict[str, object]: + balances = scenario.balances.copy() + balances["invested_ratio"] = ( + (balances["total_value"].astype(float) - balances["cash"].astype(float)) + / balances["total_value"].astype(float) + ) + risk = scenario.params.get("risk", {}) + if not isinstance(risk, Mapping): + raise UnifiedAnalysisError("scenario risk config is invalid") + if positions.empty: + max_security_weight = max_group_weight = 0.0 + planned_coverage = 0.0 + max_planned_loss_ratio = None + else: + max_security_weight = float(positions["weight"].max()) + group_weights = positions.groupby(["date", "asset_group"])["weight"].sum() + max_group_weight = float(group_weights.max()) + planned = positions.loc[positions["common_stop"].notna()].copy() + planned_coverage = float(len(planned) / len(positions)) + if planned.empty: + max_planned_loss_ratio = None + else: + planned["planned_loss"] = ( + ( + planned["avg_cost"].astype(float) + - planned["common_stop"].astype(float) + ) + .clip(lower=0.0) + * planned["amount"].astype(float) + ) + daily_risk = planned.groupby("date")["planned_loss"].sum() + equity = balances.set_index("date")["total_value"].astype(float) + loss_ratio = daily_risk / equity.reindex(daily_risk.index) + max_planned_loss_ratio = float(loss_ratio.max()) + rolling_vol = scenario.returns.rolling(60, min_periods=60).std(ddof=1) * math.sqrt(252) + filled_orders = scenario.orders.loc[ + (scenario.orders["status"] == "done") & (scenario.orders["filled"].astype(float) > 0) + ] + closed = filled_orders.loc[filled_orders["action"] == "close"] + average_equity = float(balances["total_value"].mean()) + filled_notional = float( + (filled_orders["filled"].astype(float) * filled_orders["price"].astype(float)).sum() + ) + decision_events = ( + scenario.events.loc[scenario.events["event_type"] == "decision"] + if "event_type" in scenario.events + else scenario.events.iloc[0:0] + ) + event_reason_codes = ( + decision_events["reason_code"] + if "reason_code" in decision_events + else pd.Series(dtype="object") + ) + maximum_effective_risk_units: float | None = None + maximum_portfolio_unit_utilization: float | None = None + if not decision_events.empty and "details_json" in decision_events: + for raw_details in decision_events["details_json"]: + try: + details = json.loads(str(raw_details)) + except (TypeError, ValueError, json.JSONDecodeError): + continue + if not isinstance(details, Mapping): + continue + effective = _safe_number(details.get("effective_risk_units")) + if effective is None: + continue + effective_value = float(effective) + maximum_effective_risk_units = ( + effective_value + if maximum_effective_risk_units is None + else max(maximum_effective_risk_units, effective_value) + ) + cap = _safe_number( + details.get( + "portfolio_unit_cap", risk.get("portfolio_unit_cap") + ) + ) + if cap is not None and float(cap) > 0.0: + utilization = effective_value / float(cap) + maximum_portfolio_unit_utilization = ( + utilization + if maximum_portfolio_unit_utilization is None + else max( + maximum_portfolio_unit_utilization, utilization + ) + ) + return { + "average_invested_ratio": float(balances["invested_ratio"].mean()), + "median_invested_ratio": float(balances["invested_ratio"].median()), + "below_half_ratio": float((balances["invested_ratio"] < 0.5).mean()), + "near_full_ratio": float((balances["invested_ratio"] >= 0.9).mean()), + "average_cash_ratio": float((1.0 - balances["invested_ratio"]).mean()), + "maximum_invested_ratio": float(balances["invested_ratio"].max()), + "maximum_security_weight": max_security_weight, + "maximum_asset_group_weight": max_group_weight, + "planned_risk_coverage": planned_coverage, + "maximum_planned_loss_ratio": max_planned_loss_ratio, + "maximum_effective_risk_units": maximum_effective_risk_units, + "maximum_portfolio_unit_utilization": ( + maximum_portfolio_unit_utilization + ), + "maximum_realized_60d_volatility": _safe_number(rolling_vol.max()), + "filled_order_count": int(len(filled_orders)), + "rejected_order_count": int((scenario.orders["status"] != "done").sum()), + "turnover": None if average_equity == 0.0 else filled_notional / average_equity, + "fees": float(filled_orders["commission"].astype(float).sum()), + "closed_order_count": int(len(closed)), + "closed_order_win_rate": float((closed["gains"].astype(float) > 0.0).mean()) if len(closed) else None, + "closed_order_realized_gains": float(closed["gains"].astype(float).sum()), + "protective_stop_events": int( + (event_reason_codes == "protective_stop").sum() + ), + "redistribution_event_count": int( + (event_reason_codes == "full_position_redistribution").sum() + ), + } + + +def _performance(scenario: ScenarioInput, positions: pd.DataFrame) -> dict[str, object]: + return {**calculate_return_metrics(scenario.returns), **_risk_metrics(scenario, positions)} + + +def _group_contribution(rows: pd.DataFrame, key: str) -> list[dict[str, object]]: + values = rows.groupby(key)["return_contribution"].sum().sort_values(key=lambda item: item.abs(), ascending=False) + return [{"key": str(index), "contribution": float(value)} for index, value in values.items()] + + +def _attribution(scenario: ScenarioInput, pnl_facts: pd.DataFrame) -> dict[str, object]: + daily_asset = pnl_facts.groupby("date")["return_contribution"].sum() if not pnl_facts.empty else pd.Series(dtype=float) + residual = scenario.returns.subtract(daily_asset, fill_value=0.0) + total_target = float(scenario.returns.sum()) + residual_total = float(residual.sum()) + + def with_residual(rows: list[dict[str, object]]) -> list[dict[str, object]]: + completed = [*rows, {"key": "cash_fees_and_unclassified", "contribution": residual_total}] + error = sum(float(row["contribution"]) for row in completed) - total_target + if abs(error) > 1e-10: + raise UnifiedAnalysisError("attribution does not reconcile") + return completed + + security = with_residual(_group_contribution(pnl_facts, "security") if not pnl_facts.empty else []) + groups = with_residual(_group_contribution(pnl_facts, "asset_group") if not pnl_facts.empty else []) + reasons = with_residual(_group_contribution(pnl_facts, "attribution_reason") if not pnl_facts.empty else []) + period_rows = pnl_facts.copy() + if not period_rows.empty: + period_rows["period"] = period_rows["date"].dt.strftime("%Y") + periods = with_residual(_group_contribution(period_rows, "period") if not period_rows.empty else []) + decision_events = ( + scenario.events.loc[scenario.events["event_type"] == "decision"] + if "event_type" in scenario.events + else scenario.events.iloc[0:0] + ) + event_counts = { + str(key): int(value) + for key, value in decision_events.groupby("reason_code") + .size() + .sort_values(ascending=False) + .items() + } + return { + "method": "arithmetic daily PnL contribution divided by prior equity", + "portfolio_arithmetic_return": total_target, + "security": security, + "asset_group": groups, + "trading_reason": reasons, + "period": periods, + "exposure": with_residual( + [{"key": "invested_assets", "contribution": float(daily_asset.sum())}] + ), + "event_counts": event_counts, + "event_count_scope": "decision events only", + "reconciliation_error": 0.0, + "limitation": "arithmetic attribution is not geometric linking", + } + + +def _benchmark_series(path: Path) -> dict[str, pd.Series]: + frame = pq.read_table(path).to_pandas() + frame["date"] = pd.to_datetime(frame["time"]).dt.normalize() + output: dict[str, pd.Series] = {} + for benchmark_id in BENCHMARK_IDS: + rows = frame.loc[frame["benchmark_id"] == benchmark_id].sort_values("date") + if rows.empty: + raise UnifiedAnalysisError(f"benchmark is missing: {benchmark_id}") + output[benchmark_id] = pd.Series( + rows["returns"].astype(float).to_numpy(), index=rows["date"], name=benchmark_id + ) + return output + + +def _period_result( + scenario_id: str, + dimension: str, + returns: pd.Series, + thresholds: Mapping[str, object], +) -> tuple[dict[str, object], ScenarioResult]: + metrics = calculate_return_metrics(returns) + status, reasons = evaluate_metrics(metrics, thresholds) + document = { + "scenario_id": scenario_id, + "dimension": dimension, + "status": status, + "reasons": reasons, + "metrics": metrics, + "start": returns.index.min().date().isoformat(), + "end": returns.index.max().date().isoformat(), + } + evidence = ScenarioResult( + scenario_id=scenario_id, + dimension=dimension, + status=status, + metrics=_numeric_metrics(metrics), + input_sha256=evidence_digest( + {"scenario_id": scenario_id, "returns": returns.astype(float).tolist()} + ), + reasons=tuple(reasons), + ) + return document, evidence + + +def _fixed_and_rolling( + baseline: ScenarioInput, + analyses: Mapping[str, object], + thresholds: Mapping[str, object], +) -> tuple[list[dict[str, object]], list[ScenarioResult]]: + rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + for period in analyses["fixed_periods"]: + selected = baseline.returns.loc[str(period["start"]): str(period["end"])] + if selected.empty: + raise UnifiedAnalysisError(f"fixed period has no observations: {period['id']}") + row, result = _period_result( + f"period-{period['id']}", "fixed_period", selected, thresholds + ) + rows.append(row) + evidence.append(result) + + rolling = analyses["rolling"] + start = baseline.returns.index.min() + last = baseline.returns.index.max() + while start + pd.DateOffset(years=int(rolling["window_years"])) <= last: + stop = start + pd.DateOffset(years=int(rolling["window_years"])) - pd.Timedelta(days=1) + selected = baseline.returns.loc[start:stop] + scenario_id = f"rolling-{int(rolling['window_years'])}y-{start.date().isoformat()}" + row, result = _period_result(scenario_id, "rolling_period", selected, thresholds) + rows.append(row) + evidence.append(result) + start = start + pd.DateOffset(months=int(rolling["step_months"])) + return rows, evidence + + +def _deletion_sensitivity( + baseline: ScenarioInput, + security_pnl: pd.DataFrame, + universe: Mapping[str, str], + thresholds: Mapping[str, object], +) -> tuple[list[dict[str, object]], list[ScenarioResult]]: + rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + definitions = ( + ("asset_delete_security", "security", sorted(universe)), + ("asset_delete_group", "asset_group", sorted(set(universe.values()))), + ) + for dimension, key, values in definitions: + for value in values: + contribution = ( + security_pnl.loc[security_pnl[key] == value] + .groupby("date")["return_contribution"] + .sum() + .reindex(baseline.returns.index, fill_value=0.0) + ) + adjusted = baseline.returns.subtract(contribution) + scenario_id = f"delete-{key}-{str(value).lower().replace('.', '-').replace('_', '-')}" + metrics = calculate_return_metrics(adjusted) + status, reasons = evaluate_metrics(metrics, thresholds) + row = { + "scenario_id": scenario_id, + "dimension": dimension, + "removed": str(value), + "method": "contribution-removal sensitivity; no capital reallocation", + "status": status, + "reasons": reasons, + "metrics": metrics, + } + rows.append(row) + evidence.append( + ScenarioResult( + scenario_id=scenario_id, + dimension=dimension, + status=status, + metrics=_numeric_metrics(metrics), + input_sha256=evidence_digest( + {"scenario_id": scenario_id, "returns": adjusted.tolist()} + ), + reasons=tuple(reasons), + ) + ) + return rows, evidence + + +def _market_open_lookup(repo_root: Path, snapshot_id: str) -> dict[str, pd.Series]: + snapshot = open_snapshot(snapshot_id, root=repo_root / ".local" / "market-data") + frame = pd.DataFrame([dict(row) for row in snapshot.rows]) + frame["date"] = pd.to_datetime(frame["date"]).dt.normalize() + return { + str(security): rows.sort_values("date").set_index("date")["open"].astype(float) + for security, rows in frame.groupby("security") + } + + +def _cost_sensitivity( + repo_root: Path, + baseline: ScenarioInput, + definitions: Sequence[Mapping[str, object]], + thresholds: Mapping[str, object], +) -> tuple[list[dict[str, object]], list[ScenarioResult]]: + filled = baseline.orders.loc[ + (baseline.orders["status"] == "done") & (baseline.orders["filled"].astype(float) > 0) + ].copy() + equity = baseline.balances.set_index("date")["total_value"].astype(float).shift(1) + snapshot_id = _load_json(baseline.result_dir / "manifest.json", label="result manifest")["run"]["snapshot_id"] + opens = _market_open_lookup(repo_root, str(snapshot_id)) + rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + for definition in definitions: + adjustments: dict[pd.Timestamp, float] = {} + delayed = 0 + delay_missing = 0 + for order in filled.to_dict("records"): + date = pd.Timestamp(order["date"]).normalize() + quantity = float(order["filled"]) + price = float(order["price"]) + notional = quantity * price + pnl_adjustment = -( + (float(definition["commission_multiplier"]) - 1.0) * float(order["commission"]) + + float(definition["slippage"]) * notional + ) + delay_days = int(definition["delay_days"]) + if delay_days: + series = opens.get(str(order["security"])) + if series is None or date not in series.index: + delay_missing += 1 + else: + location = series.index.get_loc(date) + target = location + delay_days + if target >= len(series): + delay_missing += 1 + else: + delayed_price = float(series.iloc[target]) + pnl_adjustment += ( + -(delayed_price - price) * quantity + if order["action"] == "open" + else (delayed_price - price) * quantity + ) + delayed += 1 + adjustments[date] = adjustments.get(date, 0.0) + pnl_adjustment + adjusted = baseline.returns.copy() + for date, adjustment in adjustments.items(): + denominator = equity.get(date) + if denominator is not None and pd.notna(denominator) and denominator != 0: + adjusted.loc[date] += adjustment / float(denominator) + metrics = calculate_return_metrics(adjusted) + status, reasons = evaluate_metrics(metrics, thresholds) + if delay_missing: + status = "evidence_insufficient" + reasons = ["missing_delayed_open"] + scenario_id = f"cost-{definition['id']}" + row = { + "scenario_id": scenario_id, + "dimension": "cost_execution", + "method": "first-order order-level cost and delayed-open sensitivity", + "status": status, + "reasons": reasons, + "metrics": metrics, + "delayed_orders": delayed, + "missing_delayed_orders": delay_missing, + } + rows.append(row) + evidence.append( + ScenarioResult( + scenario_id=scenario_id, + dimension="cost_execution", + status=status, + metrics={**_numeric_metrics(metrics), "delayed_orders": delayed, "missing_delayed_orders": delay_missing}, + input_sha256=evidence_digest( + {"definition": dict(definition), "returns": adjusted.tolist()} + ), + reasons=tuple(reasons), + ) + ) + return rows, evidence + + +def _bootstrap( + returns: pd.Series, definition: Mapping[str, object] +) -> tuple[list[dict[str, object]], list[ScenarioResult]]: + rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + thresholds = definition["thresholds"] + for block_size in definition["block_sizes"]: + paths = block_bootstrap( + returns.to_numpy(dtype=np.float64), + block_size=int(block_size), + paths=int(definition["paths"]), + horizon=int(definition["horizon_days"]), + seed=int(definition["seed"]), + ) + summary = summarize_bootstrap(paths) + del paths + reasons: list[str] = [] + if summary["probability_drawdown_over_20pct"] > float(thresholds["probability_drawdown_over_20pct_max"]): + reasons.append("probability_drawdown_over_20pct") + if summary["probability_drawdown_over_30pct"] > float(thresholds["probability_drawdown_over_30pct_max"]): + reasons.append("probability_drawdown_over_30pct") + if summary["median_terminal_return"] <= float(thresholds["median_terminal_return_min_exclusive"]): + reasons.append("median_terminal_return") + scenario_id = f"bootstrap-block-{block_size}" + metrics = { + **summary, + "block_size": int(block_size), + "paths": int(definition["paths"]), + "horizon_days": int(definition["horizon_days"]), + "seed": int(definition["seed"]), + } + status = "fail" if reasons else "pass" + rows.append( + {"scenario_id": scenario_id, "dimension": "block_bootstrap", "status": status, "reasons": reasons, "metrics": metrics} + ) + evidence.append( + ScenarioResult( + scenario_id=scenario_id, + dimension="block_bootstrap", + status=status, + metrics=_numeric_metrics(metrics), + input_sha256=evidence_digest( + {"definition": dict(definition), "block_size": block_size, "returns": returns.tolist()} + ), + reasons=tuple(reasons), + ) + ) + return rows, evidence + + +def _historical_stress( + returns: pd.Series, definitions: Sequence[Mapping[str, object]] +) -> tuple[list[dict[str, object]], list[ScenarioResult]]: + rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + for definition in definitions: + selected = returns.loc[str(definition["start"]): str(definition["end"])] + metrics = calculate_return_metrics(selected) + passed = abs(float(metrics["max_drawdown"])) <= float(definition["max_drawdown_abs_max"]) + status = "pass" if passed else "fail" + reasons = [] if passed else ["max_drawdown_abs_max"] + scenario_id = str(definition["id"]) + rows.append( + {"scenario_id": scenario_id, "dimension": "historical_stress", "status": status, "reasons": reasons, "metrics": metrics} + ) + evidence.append( + ScenarioResult( + scenario_id=scenario_id, + dimension="historical_stress", + status=status, + metrics=_numeric_metrics(metrics), + input_sha256=evidence_digest({"definition": dict(definition), "returns": selected.tolist()}), + reasons=tuple(reasons), + ) + ) + return rows, evidence + + +def _position_shocks( + positions: pd.DataFrame, definitions: Sequence[Mapping[str, object]] +) -> tuple[list[dict[str, object]], list[ScenarioResult]]: + rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + for definition in definitions: + scenario_id = str(definition["id"]) + if definition.get("use_stop_failure_loss") is True: + if ( + "stop_failure_loss" not in positions + or "equity" not in positions + or positions["stop_failure_loss"].isna().any() + ): + status = "evidence_insufficient" + reasons = ["stop_failure_loss_missing_at_source"] + metrics = {"evaluated_dates": 0, "worst_account_loss": None} + else: + daily_loss = positions.groupby("date")["stop_failure_loss"].sum() + daily_equity = positions.groupby("date")["equity"].first() + loss_ratio = daily_loss / daily_equity + worst = float(loss_ratio.max()) if len(loss_ratio) else 0.0 + passed = worst <= float(definition["maximum_loss_abs_max"]) + 1e-12 + status = "pass" if passed else "fail" + reasons = [] if passed else ["maximum_loss_abs_max"] + metrics = { + "evaluated_dates": len(loss_ratio), + "worst_account_loss": worst, + } + else: + losses: list[float] = [] + for _, daily in positions.groupby("date"): + shock_return = 0.0 + for row in daily.to_dict("records"): + shock = definition.get("security_shocks", {}).get( + str(row["security"]), + definition["asset_group_shocks"].get(str(row["asset_group"])), + ) + if shock is None: + raise UnifiedAnalysisError("position shock does not cover every asset group") + shock_return += float(row["weight"]) * float(shock) + losses.append(max(0.0, -shock_return)) + worst = max(losses) if losses else 0.0 + passed = worst <= float(definition["maximum_loss_abs_max"]) + 1e-12 + status = "pass" if passed else "fail" + reasons = [] if passed else ["maximum_loss_abs_max"] + metrics = {"evaluated_dates": len(losses), "worst_account_loss": worst} + rows.append( + {"scenario_id": scenario_id, "dimension": "position_shock", "status": status, "reasons": reasons, "metrics": metrics} + ) + evidence.append( + ScenarioResult( + scenario_id=scenario_id, + dimension="position_shock", + status=status, + metrics=_numeric_metrics(metrics), + input_sha256=evidence_digest({"definition": dict(definition), "positions": len(positions)}), + reasons=tuple(reasons), + ) + ) + return rows, evidence + + +def _cvar( + returns: pd.Series, definitions: Sequence[Mapping[str, object]] +) -> tuple[list[dict[str, object]], list[ScenarioResult]]: + rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + base = returns.to_numpy(dtype=np.float64) + for definition in definitions: + horizon = int(definition["horizon_days"]) + samples = base if horizon == 1 else rolling_compound_returns(base, window=horizon) + tail = float(len(samples) * (1.0 - float(definition["confidence"]))) + sufficient = tail >= float(definition["minimum_tail_observations"]) + value = calculate_cvar(samples, float(definition["confidence"])) if sufficient else None + passed = value is not None and value <= float(definition["maximum_loss_abs_max"]) + status = "evidence_insufficient" if not sufficient else ("pass" if passed else "fail") + reasons = ["insufficient_tail_observations"] if not sufficient else ([] if passed else ["maximum_loss_abs_max"]) + metrics = { + "cvar": value, + "confidence": float(definition["confidence"]), + "horizon_days": horizon, + "samples": len(samples), + "tail_observations": tail, + } + scenario_id = str(definition["id"]) + rows.append( + {"scenario_id": scenario_id, "dimension": "cvar", "status": status, "reasons": reasons, "metrics": metrics} + ) + evidence.append( + ScenarioResult( + scenario_id=scenario_id, + dimension="cvar", + status=status, + metrics=_numeric_metrics(metrics), + input_sha256=evidence_digest({"definition": dict(definition), "returns": base.tolist()}), + reasons=tuple(reasons), + ) + ) + return rows, evidence + + +def run_deterministic_analysis( + repo_root: Path, + preparation_workspace: Path, + source_registry: Mapping[str, str], +) -> dict[str, object]: + started = time.perf_counter() + root = Path(repo_root).resolve() + sources = _register_source_results(root, preparation_workspace, source_registry) + analysis_root = root / ".local" / "strategy-analysis" / str(sources["analysis_id"]) + preparation = _load_json(analysis_root / "preparation.json", label="analysis preparation") + expanded = _load_json(analysis_root / "analysis-scenarios.json", label="analysis scenarios") + if preparation.get("preparation_id") != sources["preparation_id"]: + raise UnifiedAnalysisError("analysis preparation identity mismatch") + source_by_id = {str(item["scenario_id"]): item for item in sources["sources"]} + scenarios = { + scenario_id: _load_scenario(root / str(source["result_dir"])) + for scenario_id, source in source_by_id.items() + } + baseline = scenarios["baseline"] + universe = expanded["universe"] + thresholds = expanded["thresholds"] + positions = _position_facts(baseline, universe) + security_pnl = _security_pnl_facts(baseline, universe) + baseline_metrics = _performance(baseline, positions) + baseline_status, baseline_reasons = evaluate_metrics(baseline_metrics, thresholds) + + challenge_rows: list[dict[str, object]] = [] + evidence: list[ScenarioResult] = [] + for plan_scenario in expanded["scenarios"]: + scenario_id = str(plan_scenario["scenario_id"]) + current = scenarios[scenario_id] + current_positions = _position_facts(current, universe) + metrics = _performance(current, current_positions) + status, reasons = evaluate_metrics(metrics, thresholds) + challenge_rows.append( + { + "scenario_id": scenario_id, + "dimension": str(plan_scenario["dimension"]), + "status": status, + "reasons": reasons, + "metrics": metrics, + "delta_vs_baseline": { + key: ( + None + if metrics.get(key) is None or baseline_metrics.get(key) is None + else float(metrics[key]) - float(baseline_metrics[key]) + ) + for key in ("cagr", "max_drawdown", "calmar", "average_invested_ratio") + }, + "cold_seconds": float(current.performance["cold_seconds"]), + "warm_seconds": float(current.performance["warm_seconds"]), + } + ) + evidence.append( + ScenarioResult( + scenario_id=f"parameter-{scenario_id}", + dimension="baseline" if scenario_id == "baseline" else "parameter", + status=status, + metrics=_numeric_metrics(metrics), + input_sha256=evidence_digest( + {"scenario_id": scenario_id, "run_id": current.run_id, "metrics": _numeric_metrics(metrics)} + ), + reasons=tuple(reasons), + ) + ) + + benchmark_manifest = root / str(preparation["benchmark_set"]["manifest"]) + benchmark_document = _load_json(benchmark_manifest, label="benchmark set manifest") + benchmark_data = benchmark_manifest.parent / str(benchmark_document["data"]["path"]) + benchmarks = _benchmark_series(benchmark_data) + aligned, alignment = align_three_way_benchmarks(baseline.returns, benchmarks) + benchmark_statistics = { + benchmark_id: calculate_benchmark_statistics( + {date.date().isoformat(): float(value) for date, value in aligned["strategy"].items()}, + {date.date().isoformat(): float(value) for date, value in aligned[benchmark_id].items()}, + ) + for benchmark_id in BENCHMARK_IDS + } + + analyses = expanded["analyses"] + period_rows, period_evidence = _fixed_and_rolling(baseline, analyses, thresholds) + deletion_rows, deletion_evidence = _deletion_sensitivity( + baseline, security_pnl, universe, thresholds + ) + cost_rows, cost_evidence = _cost_sensitivity(root, baseline, analyses["cost_execution"], thresholds) + bootstrap_rows, bootstrap_evidence = _bootstrap(baseline.returns, analyses["bootstrap"]) + stress_rows, stress_evidence = _historical_stress(baseline.returns, analyses["historical_stress"]) + shock_rows, shock_evidence = _position_shocks(positions, analyses["position_shocks"]) + cvar_rows, cvar_evidence = _cvar(baseline.returns, analyses["cvar"]) + evidence.extend( + [ + *period_evidence, + *deletion_evidence, + *cost_evidence, + *bootstrap_evidence, + *stress_evidence, + *shock_evidence, + *cvar_evidence, + ] + ) + evidence_path = build_evidence_matrix(evidence, analysis_root / "evidence-matrix.parquet") + + failures = [result.to_document() for result in evidence if result.status == "fail"] + insufficient = [ + result.to_document() for result in evidence if result.status == "evidence_insufficient" + ] + severe_dimensions = {"block_bootstrap", "position_shock", "cvar"} + severe_failures = [row for row in failures if row["dimension"] in severe_dimensions] + advance = baseline_status == "pass" and not severe_failures + recommendation = { + "decision": "recommend_joinquant_confirmation" if advance else "revise_before_joinquant", + "baseline_status": baseline_status, + "baseline_reasons": baseline_reasons, + "severe_failure_count": len(severe_failures), + "evidence_insufficient_count": len(insufficient), + "authority": "local_exploratory", + } + opposing_evidence: list[dict[str, object]] = [] + for benchmark_id, metrics in benchmark_statistics.items(): + if float(metrics["active_return"]) < 0: + opposing_evidence.append( + {"kind": "benchmark_underperformance", "benchmark_id": benchmark_id, "active_return": metrics["active_return"]} + ) + opposing_evidence.extend( + {"kind": "scenario_failure", "scenario_id": row["scenario_id"], "dimension": row["dimension"], "reasons": row["reasons"]} + for row in failures + ) + opposing_evidence.extend( + {"kind": "evidence_insufficient", "scenario_id": row["scenario_id"], "dimension": row["dimension"], "reasons": row["reasons"]} + for row in insufficient + ) + summary = { + "schema_version": "deterministic-strategy-analysis/1", + "formula_version": _FORMULA_VERSION, + "analysis_id": analysis_root.name, + "strategy_id": expanded["strategy_id"], + "authority": "local_exploratory", + "not_formal_joinquant_backtest": True, + "sources": { + "source_results": "source-results.json", + "benchmark_set_id": preparation["benchmark_set"]["benchmark_set_id"], + "analysis_plan_sha256": expanded["analysis_plan_sha256"], + }, + "baseline": { + "status": baseline_status, + "reasons": baseline_reasons, + "metrics": baseline_metrics, + "risk_control": _risk_metrics(baseline, positions), + }, + "benchmarks": {"alignment": alignment, "statistics": benchmark_statistics}, + "attribution": _attribution(baseline, security_pnl), + "challenge_results": challenge_rows, + "robustness": { + "periods": period_rows, + "asset_deletions": deletion_rows, + "cost_execution": cost_rows, + "bootstrap": bootstrap_rows, + "historical_stress": stress_rows, + "position_shocks": shock_rows, + "cvar": cvar_rows, + }, + "evidence_matrix": { + "path": evidence_path.relative_to(analysis_root).as_posix(), + "sha256": _sha256(evidence_path), + "rows": len(evidence), + "pass": sum(result.status == "pass" for result in evidence), + "fail": len(failures), + "evidence_insufficient": len(insufficient), + }, + "opposing_evidence": opposing_evidence, + "pre_vibe_recommendation": recommendation, + "next_action": deterministic_next_action(), + } + analysis_path = analysis_root / "deterministic-analysis.json" + summary = _with_analysis_seconds( + summary, + time.perf_counter() - started, + analysis_path, + ) + _atomic_json(analysis_path, summary) + return summary + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run deterministic analysis over standard result packages") + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--preparation-workspace", type=Path, required=True) + parser.add_argument( + "--source", + action="append", + required=True, + metavar="SCENARIO_ID=RUN_ID", + help="explicitly register one scenario result; repeat for every planned scenario", + ) + return parser + + +def _parse_source_registry(values: Sequence[str]) -> dict[str, str]: + registry: dict[str, str] = {} + for value in values: + scenario_id, separator, run_id = value.partition("=") + if not separator or not scenario_id or not run_id: + raise UnifiedAnalysisError( + "--source must use the SCENARIO_ID=RUN_ID form" + ) + if scenario_id in registry: + raise UnifiedAnalysisError( + f"scenario {scenario_id} is registered more than once" + ) + registry[scenario_id] = run_id + return registry + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + result = run_deterministic_analysis( + args.repo_root, + args.preparation_workspace, + _parse_source_registry(args.source), + ) + print( + json.dumps( + { + "analysis_id": result["analysis_id"], + "status": "complete", + "next_action": result["next_action"], + "baseline": result["baseline"], + "evidence_matrix": result["evidence_matrix"], + "analysis_seconds": result["analysis_seconds"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/local_quant_research/fixtures/plain_project_adapter.py b/tests/local_quant_research/fixtures/plain_project_adapter.py new file mode 100644 index 0000000..550a310 --- /dev/null +++ b/tests/local_quant_research/fixtures/plain_project_adapter.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from scripts.research.market_data.query import open_snapshot + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--snapshot-manifest", required=True) + parser.add_argument("--market-data-root", required=True) + parser.add_argument("--project-config", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--snapshot-id", required=True) + parser.add_argument("--code-sha256", required=True) + parser.add_argument("--config-sha256", required=True) + return parser + + +def main() -> int: + args = _parser().parse_args() + output = Path(args.output_dir) + snapshot = open_snapshot(args.snapshot_id, root=Path(args.market_data_root)) + dates = sorted({str(row["date"]) for row in snapshot.rows}) + if len(dates) != 2: + raise ValueError("plain fixture expects two dates") + (output / "result.json").write_text( + json.dumps( + { + "snapshot_id": snapshot.snapshot_id, + "run_id": args.run_id, + "rows": len(snapshot.rows), + }, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + (output / "project-status.json").write_text( + json.dumps( + { + "schema_version": 1, + "status": "complete", + "reason_codes": [], + "next_action": "return_to_caller", + }, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/local_quant_research/test_analysis_data_views.py b/tests/local_quant_research/test_analysis_data_views.py new file mode 100644 index 0000000..9157964 --- /dev/null +++ b/tests/local_quant_research/test_analysis_data_views.py @@ -0,0 +1,369 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from scripts.research.analysis_data.manifest import AnalysisManifestError +from scripts.research.analysis_data.views import open_analysis_database + + +SHA_FIELDS = ("path", "sha256", "bytes") + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _file_ref(root: Path, path: Path, *, rows: int | None = None) -> dict[str, object]: + result: dict[str, object] = { + "path": path.relative_to(root).as_posix(), + "sha256": _sha(path), + "bytes": path.stat().st_size, + } + if rows is not None: + result.update({"rows": rows, "format": "parquet", "compression": "zstd"}) + return result + + +def _write_local_result(root: Path) -> Path: + data = root / "data" + versions = root / "params_versions" + data.mkdir(parents=True) + versions.mkdir() + code = root / "code.py" + params = root / "params.json" + performance = root / "performance.json" + code.write_text("VALUE = 1\n", encoding="utf-8") + params.write_text('{"scenario_id":"baseline"}\n', encoding="utf-8") + params_sha = _sha(params) + version = versions / f"{params_sha}.json" + version.write_bytes(params.read_bytes()) + performance.write_text('{"status":"pass"}\n', encoding="utf-8") + + tables = { + "results": pa.Table.from_arrays( + [ + pa.array([None, None, None], type=pa.float64()), + pa.array([0.0, 0.1, 0.21], type=pa.float64()), + pa.array( + [ + "2024-01-02 16:00:00", + "2024-01-03 16:00:00", + "2024-01-04 16:00:00", + ], + type=pa.string(), + ), + ], + names=["benchmark_returns", "returns", "time"], + ), + "balances": pa.table( + { + "total_value": pa.array([100.0, 110.0, 121.0], type=pa.float64()), + "net_value": pa.array([100.0, 110.0, 121.0], type=pa.float64()), + "cash": pa.array([100.0, 110.0, 121.0], type=pa.float64()), + "aval_cash": pa.array([100.0, 110.0, 121.0], type=pa.float64()), + "time": pa.array( + [ + "2024-01-02 16:00:00", + "2024-01-03 16:00:00", + "2024-01-04 16:00:00", + ] + ), + } + ), + "positions": pa.Table.from_pylist( + [], + schema=pa.schema( + [ + ("pindex", pa.int64()), + ("avg_cost", pa.float64()), + ("margin", pa.float64()), + ("amount", pa.float64()), + ("today_amount", pa.int64()), + ("hold_cost", pa.float64()), + ("side", pa.string()), + ("price", pa.float64()), + ("gains", pa.float64()), + ("daily_gains", pa.float64()), + ("closeable_amount", pa.int64()), + ("time", pa.string()), + ("security_name", pa.string()), + ("security", pa.string()), + ] + ), + ), + "orders": pa.Table.from_pylist( + [], + schema=pa.schema( + [ + ("match_time", pa.string()), + ("pindex", pa.int64()), + ("cancel_time", pa.string()), + ("action", pa.string()), + ("limit_price", pa.float64()), + ("comment", pa.string()), + ("entrust_time", pa.string()), + ("finish_time", pa.string()), + ("side", pa.string()), + ("price", pa.float64()), + ("commission", pa.float64()), + ("gains", pa.float64()), + ("type", pa.string()), + ("time", pa.string()), + ("security_name", pa.string()), + ("security", pa.string()), + ("filled", pa.int64()), + ("amount", pa.int64()), + ("status", pa.string()), + ] + ), + ), + } + for name, table in tables.items(): + pq.write_table(table, data / f"{name}.parquet", compression="zstd") + + datasets: dict[str, object] = {} + for name, table in tables.items(): + rows = table.num_rows + datasets[name] = { + "required": True, + "status": "complete", + "rows": rows, + "verified_empty": rows == 0, + "time_range": { + "start": None if rows == 0 else "2024-01-02", + "end": None if rows == 0 else "2024-01-04", + }, + "files": [_file_ref(root, data / f"{name}.parquet", rows=rows)], + "evidence": { + "fields": table.schema.names, + "unique_key": ["time"] if name != "orders" else ["time", "security"], + }, + } + for name in ("risk", "period_risks"): + datasets[name] = { + "required": False, + "status": "missing_at_source", + "reason": "computed_by_strategy_analysis", + "rows": 0, + "verified_empty": True, + "files": [], + } + manifest = { + "schema_version": "local-backtest/1", + "object": {"kind": "local_backtest", "local_id": "local-1", "status": "complete"}, + "source": { + "kind": "local_vectorbt", + "engine": { + "backend": "vectorbt.Portfolio.from_order_func", + "adapter_version": "local-vectorbt-adapter/1", + "vectorbt": "1.1.0", + "numba": "0.66.0", + "numpy": "2.4.6", + "pandas": "3.0.3", + }, + "accounting": { + "version": "turtle-etf-corporate-actions/1", + "corporate_action_mode": "point_in_time_total_return_approximation", + "continuity_factor_basis": "raw_previous_close_over_current_pre_close", + "corporate_action_metadata_timing": "audit_only_may_be_retrospective", + "price_basis": "continuous_economic_price", + "quantity_basis": "economic_units", + "cash_dividend_mode": "implicit_reinvestment_on_ex_date", + "pay_date_cash_supported": False, + "exact_joinquant_reconciliation": False, + "corporate_actions_sha256": "a" * 64, + }, + }, + "authority": "local_research", + "run": {"run_id": "run-1", "scenario_id": "baseline", "snapshot_id": "a" * 64}, + "code": _file_ref(root, code), + "params": {"current": _file_ref(root, params), "version": _file_ref(root, version)}, + "performance": _file_ref(root, performance), + "datasets": datasets, + "source_benchmark_returns": { + "status": "missing_at_source", + "reason": "independent_benchmark_set", + "null_rows": 3, + }, + "gate": {"status": "pass", "exceptions": [], "checks": ["digests"]}, + } + (root / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, sort_keys=True), encoding="utf-8" + ) + return root + + +def _tree_sha(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(hashlib.sha256(path.read_bytes()).digest()) + return digest.hexdigest() + + +def test_joinquant_source_builds_six_read_only_logical_views(repo_root: Path) -> None: + root = repo_root / "joinquant/strategies/strategy-001/backtests/111" + before = _tree_sha(root) + + with open_analysis_database(root) as database: + assert database.source.kind == "joinquant_backtest" + assert database.table_names == ( + "results", + "balances", + "positions", + "orders", + "risk", + "period_risks", + ) + assert database.connection.sql("select count(*) from results").fetchone() == (1289,) + assert database.connection.sql("select count(*) from risk").fetchone() == (1,) + columns = database.connection.sql("describe orders").fetchall() + assert dict((name, kind) for name, kind, *_ in columns)["cancel_time"] == "VARCHAR" + + assert _tree_sha(root) == before + + +def test_joinquant_legal_missing_attribution_exception_remains_read_only( + repo_root: Path, +) -> None: + root = repo_root / "joinquant/strategies/strategy-001/backtests/10" + before = _tree_sha(root) + + with open_analysis_database(root) as database: + assert database.source.kind == "joinquant_backtest" + assert database.connection.sql("select count(*) from results").fetchone() == ( + 57, + ) + + assert _tree_sha(root) == before + + +def test_joinquant_unknown_gate_exception_is_not_downgraded( + repo_root: Path, tmp_path: Path +) -> None: + source = repo_root / "joinquant/strategies/strategy-001/backtests/10/manifest.json" + document = json.loads(source.read_text(encoding="utf-8")) + document["gate"]["exceptions"] = ["unknown:exception"] + root = tmp_path / "joinquant-result" + root.mkdir() + (root / "manifest.json").write_text( + json.dumps(document, ensure_ascii=False), encoding="utf-8" + ) + + with pytest.raises(AnalysisManifestError, match="gate did not pass"): + open_analysis_database(root) + + +def test_joinquant_column_order_variants_are_normalized_in_memory( + repo_root: Path, +) -> None: + root = repo_root / "joinquant/strategies/strategy-001/backtests/113" + before = _tree_sha(root) + + with open_analysis_database(root) as database: + columns = database.connection.sql("describe balances").fetchall() + assert [row[0] for row in columns] == [ + "total_value", + "net_value", + "cash", + "aval_cash", + "time", + ] + assert database.connection.sql("select count(*) from balances").fetchone() == ( + 1289, + ) + + assert _tree_sha(root) == before + + +def test_joinquant_legal_empty_tables_become_typed_memory_views(repo_root: Path) -> None: + root = repo_root / "joinquant/strategies/strategy-001/backtests/109" + with open_analysis_database(root) as database: + assert database.connection.sql("select count(*) from positions").fetchone() == (0,) + assert database.connection.sql("select count(*) from orders").fetchone() == (0,) + assert [row[0] for row in database.connection.sql("describe positions").fetchall()] == [ + "pindex", + "avg_cost", + "margin", + "amount", + "today_amount", + "hold_cost", + "side", + "price", + "gains", + "daily_gains", + "closeable_amount", + "time", + "security_name", + "security", + ] + + +def test_local_source_exposes_missing_references_without_fake_files(tmp_path: Path) -> None: + root = _write_local_result(tmp_path / "local") + with open_analysis_database(root) as database: + assert database.source.kind == "local_backtest" + assert database.connection.sql("select count(*) from risk").fetchone() == (0,) + assert database.connection.sql("select count(*) from period_risks").fetchone() == (0,) + schema = database.connection.sql("describe results").fetchall() + assert [(row[0], row[1]) for row in schema] == [ + ("benchmark_returns", "DOUBLE"), + ("returns", "DOUBLE"), + ("time", "VARCHAR"), + ] + assert database.connection.sql( + "select count(benchmark_returns) from results" + ).fetchone() == (0,) + assert database.reference_status("risk") == ( + "missing_at_source", + "computed_by_strategy_analysis", + ) + + +def test_daily_returns_are_derived_from_cumulative_values_at_query_time( + tmp_path: Path, +) -> None: + root = _write_local_result(tmp_path / "local") + with open_analysis_database(root) as database: + rows = database.connection.sql( + "select trading_date, cumulative_returns, daily_returns, comparable " + "from strategy_daily_returns order by trading_date" + ).fetchall() + + assert rows[0][0].isoformat() == "2024-01-02" + assert rows[0][1:] == (0.0, 0.0, True) + assert rows[1][1] == 0.1 + assert abs(rows[1][2] - 0.1) < 1e-12 + assert rows[2][1] == 0.21 + assert abs(rows[2][2] - 0.1) < 1e-12 + + +def test_nonzero_first_cumulative_return_is_not_compared_without_predecessor( + tmp_path: Path, +) -> None: + root = _write_local_result(tmp_path / "local") + results = root / "data/results.parquet" + table = pq.read_table(results) + replacement = table.set_column( + table.schema.get_field_index("returns"), + "returns", + pa.array([0.05, 0.1, 0.21], type=pa.float64()), + ) + pq.write_table(replacement, results, compression="zstd") + manifest_path = root / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["datasets"]["results"]["files"][0] = _file_ref(root, results, rows=3) + manifest_path.write_text(json.dumps(manifest, sort_keys=True), encoding="utf-8") + + with open_analysis_database(root) as database: + first = database.connection.sql( + "select daily_returns, comparable from strategy_daily_returns " + "order by trading_date limit 1" + ).fetchone() + assert first == (None, False) diff --git a/tests/local_quant_research/test_analysis_manifest_schemas.py b/tests/local_quant_research/test_analysis_manifest_schemas.py new file mode 100644 index 0000000..2f3902c --- /dev/null +++ b/tests/local_quant_research/test_analysis_manifest_schemas.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from scripts.research.analysis_data.manifest import ( + AnalysisManifestError, + open_analysis_source, + validate_analysis_source, + validate_local_manifest_document, +) + + +SHA = "a" * 64 + + +def _dataset(name: str, rows: int = 1) -> dict[str, object]: + return { + "required": True, + "status": "complete", + "rows": rows, + "verified_empty": rows == 0, + "time_range": { + "start": None if rows == 0 else "2024-01-02", + "end": None if rows == 0 else "2024-01-02", + }, + "files": [ + { + "path": f"data/{name}.parquet", + "sha256": SHA, + "bytes": 100, + "rows": rows, + "format": "parquet", + "compression": "zstd", + } + ], + "evidence": {"fields": ["time"], "unique_key": ["time"]}, + } + + +def _local_manifest() -> dict[str, object]: + return { + "schema_version": "local-backtest/1", + "object": { + "kind": "local_backtest", + "local_id": "local-bt-001", + "status": "complete", + }, + "source": { + "kind": "local_vectorbt", + "engine": { + "backend": "vectorbt.Portfolio.from_order_func", + "adapter_version": "local-vectorbt-adapter/1", + "vectorbt": "1.1.0", + "numba": "0.66.0", + "numpy": "2.4.6", + "pandas": "3.0.3", + }, + "accounting": { + "version": "turtle-etf-corporate-actions/1", + "corporate_action_mode": "point_in_time_total_return_approximation", + "continuity_factor_basis": "raw_previous_close_over_current_pre_close", + "corporate_action_metadata_timing": "audit_only_may_be_retrospective", + "price_basis": "continuous_economic_price", + "quantity_basis": "economic_units", + "cash_dividend_mode": "implicit_reinvestment_on_ex_date", + "pay_date_cash_supported": False, + "exact_joinquant_reconciliation": False, + "corporate_actions_sha256": SHA, + }, + }, + "authority": "local_research", + "run": { + "run_id": "run-001", + "scenario_id": "baseline", + "snapshot_id": SHA, + }, + "code": { + "path": "code.py", + "sha256": SHA, + "bytes": 100, + }, + "params": { + "current": { + "path": "params.json", + "sha256": SHA, + "bytes": 100, + }, + "version": { + "path": f"params_versions/{SHA}.json", + "sha256": SHA, + "bytes": 100, + }, + }, + "performance": { + "path": "performance.json", + "sha256": SHA, + "bytes": 100, + }, + "datasets": { + **{name: _dataset(name) for name in ("results", "balances", "positions", "orders")}, + "risk": { + "required": False, + "status": "missing_at_source", + "reason": "computed_by_strategy_analysis", + "rows": 0, + "verified_empty": True, + "files": [], + }, + "period_risks": { + "required": False, + "status": "missing_at_source", + "reason": "computed_by_strategy_analysis", + "rows": 0, + "verified_empty": True, + "files": [], + }, + }, + "source_benchmark_returns": { + "status": "missing_at_source", + "reason": "independent_benchmark_set", + "null_rows": 1, + }, + "gate": {"status": "pass", "exceptions": [], "checks": ["file_digests"]}, + } + + +def _tree_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + digest.update(path.relative_to(root).as_posix().encode("utf-8")) + digest.update(hashlib.sha256(path.read_bytes()).digest()) + return digest.hexdigest() + + +def test_local_manifest_schema_is_strict_and_excludes_joinquant_evidence( + repo_root: Path, +) -> None: + schema = json.loads( + ( + repo_root + / "scripts/research/analysis_data/schemas/local-backtest-manifest.schema.json" + ).read_text(encoding="utf-8") + ) + assert schema["properties"]["schema_version"]["const"] == "local-backtest/1" + assert schema["properties"]["object"]["properties"]["kind"]["const"] == "local_backtest" + assert schema["properties"]["source"]["properties"]["kind"]["const"] == "local_vectorbt" + assert schema["properties"]["authority"]["const"] == "local_research" + assert set(schema["required"]) >= { + "authority", + "run", + "params", + "source_benchmark_returns", + } + + valid = _local_manifest() + validate_local_manifest_document(valid) + for forbidden in ( + "collection_fence", + "research_response", + "research_lineage", + "official_summary", + ): + invalid = {**valid, forbidden: {}} + with pytest.raises(AnalysisManifestError): + validate_local_manifest_document(invalid) + invalid = copy.deepcopy(valid) + invalid["source"]["url"] = "https://www.joinquant.com/backtest/fake" + with pytest.raises(AnalysisManifestError): + validate_local_manifest_document(invalid) + + +def test_local_accounting_precision_boundary_is_mandatory_and_closed() -> None: + valid = _local_manifest() + validate_local_manifest_document(valid) + accounting = valid["source"]["accounting"] + for field in tuple(accounting): + invalid = copy.deepcopy(valid) + del invalid["source"]["accounting"][field] + with pytest.raises(AnalysisManifestError): + validate_local_manifest_document(invalid) + + invalid = copy.deepcopy(valid) + invalid["source"]["accounting"]["corporate_action_mode"] = "exact" + with pytest.raises(AnalysisManifestError): + validate_local_manifest_document(invalid) + + invalid = copy.deepcopy(valid) + invalid["source"]["accounting"]["exact_joinquant_reconciliation"] = True + with pytest.raises(AnalysisManifestError): + validate_local_manifest_document(invalid) + + +def test_local_missing_source_references_are_not_fake_physical_tables() -> None: + valid = _local_manifest() + for name in ("risk", "period_risks"): + invalid = copy.deepcopy(valid) + invalid["datasets"][name] = _dataset(name) + with pytest.raises(AnalysisManifestError): + validate_local_manifest_document(invalid) + invalid = copy.deepcopy(valid) + invalid["source_benchmark_returns"]["reason"] = "filled_with_zero" + with pytest.raises(AnalysisManifestError): + validate_local_manifest_document(invalid) + + +def test_reader_selects_only_by_top_level_version_and_never_falls_back( + tmp_path: Path, +) -> None: + root = tmp_path / "backtest" + root.mkdir() + document = _local_manifest() + document["schema_version"] = 1 + (root / "manifest.json").write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(AnalysisManifestError, match="joinquant"): + open_analysis_source(root) + + document["schema_version"] = "unknown/1" + (root / "manifest.json").write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(AnalysisManifestError, match="unsupported"): + open_analysis_source(root) + + +def test_existing_joinquant_backtest_validates_without_modification( + repo_root: Path, +) -> None: + root = repo_root / "joinquant/strategies/strategy-001/backtests/111" + before = _tree_digest(root) + + source = open_analysis_source(root) + result = validate_analysis_source(source) + + assert source.kind == "joinquant_backtest" + assert result.status == "pass" + assert tuple(result.datasets) == ( + "results", + "balances", + "positions", + "orders", + "risk", + "period_risks", + ) + assert _tree_digest(root) == before diff --git a/tests/local_quant_research/test_benchmark_set_contract.py b/tests/local_quant_research/test_benchmark_set_contract.py new file mode 100644 index 0000000..a55ef85 --- /dev/null +++ b/tests/local_quant_research/test_benchmark_set_contract.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import date +from pathlib import Path + +import pyarrow.parquet as pq +import pytest + +from scripts.research.market_data.benchmark_sets import ( + BENCHMARK_IDS, + BenchmarkLevel, + BenchmarkSetError, + SourcePayload, + build_benchmark_rows, + open_benchmark_set, + write_benchmark_set, +) + + +def _payload(name: str, filename: str) -> SourcePayload: + identities = { + "csi300_total_return": ( + "China Securities Index Co., Ltd.", + "H00300", + "https://www.csindex.com.cn/", + ), + "nasdaq100_total_return": ( + "Nasdaq, Inc.", + "XNDX", + "https://indexes.nasdaqomx.com/", + ), + "usd_cny": ( + "Board of Governors of the Federal Reserve System", + "DEXCHUS", + "https://www.federalreserve.gov/", + ), + } + provider, source_id, url = identities[name] + return SourcePayload( + name=name, + filename=filename, + provider=provider, + source_id=source_id, + url=url + filename, + content_type="application/octet-stream", + data=f"source:{name}".encode(), + ) + + +def _sources() -> tuple[SourcePayload, ...]: + return ( + _payload("csi300_total_return", "csi300-total-return.xlsx"), + _payload("nasdaq100_total_return", "nasdaq100-total-return.xlsx"), + _payload("usd_cny", "usd-cny.html"), + ) + + +def test_benchmark_rows_use_total_return_and_currency_formula_without_fill() -> None: + csi = ( + BenchmarkLevel(date(2024, 1, 1), 100.0), + BenchmarkLevel(date(2024, 1, 2), 110.0), + BenchmarkLevel(date(2024, 1, 3), 121.0), + ) + nasdaq = ( + BenchmarkLevel(date(2024, 1, 1), 100.0), + BenchmarkLevel(date(2024, 1, 2), 110.0), + BenchmarkLevel(date(2024, 1, 3), 121.0), + ) + fx = ( + BenchmarkLevel(date(2024, 1, 1), 7.0), + BenchmarkLevel(date(2024, 1, 2), 7.07), + ) + + rows = build_benchmark_rows( + csi_levels=csi, + nasdaq_levels=nasdaq, + usd_cny_levels=fx, + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 3), + ) + + keyed = {(row["time"], row["benchmark_id"]): row["returns"] for row in rows} + assert keyed[("2024-01-02", "CSI300_CNY_TOTAL_RETURN")] == pytest.approx(0.1) + assert keyed[("2024-01-03", "CSI300_CNY_TOTAL_RETURN")] == pytest.approx(0.1) + assert keyed[("2024-01-02", "NASDAQ100_CNY_TOTAL_RETURN")] == pytest.approx(0.111) + assert ("2024-01-03", "NASDAQ100_CNY_TOTAL_RETURN") not in keyed + + +def test_benchmark_set_is_immutable_columnar_and_has_exact_two_identities( + tmp_path: Path, +) -> None: + rows = [ + {"time": "2024-01-02", "benchmark_id": BENCHMARK_IDS[0], "returns": 0.01}, + {"time": "2024-01-02", "benchmark_id": BENCHMARK_IDS[1], "returns": 0.02}, + ] + created = write_benchmark_set( + market_data_root=tmp_path, + rows=rows, + sources=_sources(), + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 2), + ) + opened = open_benchmark_set(created.root) + + assert created.root == opened.root + assert created.root.parent.name == "benchmark-sets" + assert created.root.name == created.benchmark_set_id + assert set(created.manifest["benchmarks"]) == set(BENCHMARK_IDS) + table = pq.read_table(created.root / "benchmark-returns.parquet") + assert table.schema.names == ["time", "benchmark_id", "returns"] + assert str(table.schema.field("returns").type) == "double" + assert table.num_rows == 2 + for source in created.manifest["sources"]: + path = created.root / source["path"] + assert path.is_file() + assert hashlib.sha256(path.read_bytes()).hexdigest() == source["sha256"] + + reused = write_benchmark_set( + market_data_root=tmp_path, + rows=rows, + sources=_sources(), + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 2), + ) + assert reused.benchmark_set_id == created.benchmark_set_id + + +def test_benchmark_set_rejects_proxy_missing_or_unknown_identity(tmp_path: Path) -> None: + valid = [ + {"time": "2024-01-02", "benchmark_id": BENCHMARK_IDS[0], "returns": 0.01}, + {"time": "2024-01-02", "benchmark_id": BENCHMARK_IDS[1], "returns": 0.02}, + ] + for rows in ( + valid[:1], + [*valid, {"time": "2024-01-02", "benchmark_id": "ETF_PROXY", "returns": 0.0}], + ): + with pytest.raises(BenchmarkSetError): + write_benchmark_set( + market_data_root=tmp_path, + rows=rows, + sources=_sources(), + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 2), + ) + + +def test_benchmark_set_detects_manifest_or_parquet_tampering(tmp_path: Path) -> None: + created = write_benchmark_set( + market_data_root=tmp_path, + rows=[ + {"time": "2024-01-02", "benchmark_id": BENCHMARK_IDS[0], "returns": 0.01}, + {"time": "2024-01-02", "benchmark_id": BENCHMARK_IDS[1], "returns": 0.02}, + ], + sources=_sources(), + start_date=date(2024, 1, 2), + end_date=date(2024, 1, 2), + ) + manifest_path = created.root / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["data"]["sha256"] = "0" * 64 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(BenchmarkSetError, match="digest"): + open_benchmark_set(created.root) diff --git a/tests/local_quant_research/test_contract_fixtures.py b/tests/local_quant_research/test_contract_fixtures.py index 3e78e1b..267a52a 100644 --- a/tests/local_quant_research/test_contract_fixtures.py +++ b/tests/local_quant_research/test_contract_fixtures.py @@ -39,19 +39,12 @@ def test_baseline_freezes_universe_rules_and_price_semantics(repo_root: Path) -> "n_days": 20, "add_step_n": 0.5, "stop_n": 2.0, + "max_units": 4, } assert baseline["risk"] == { - "risk_per_unit": 0.005, - "security_risk_cap": 0.0125, - "security_value_cap": 0.30, - "asset_group_risk_cap": 0.025, - "asset_group_value_cap": 0.50, - "portfolio_risk_cap": 0.05, - "portfolio_value_cap": 1.0, - "covariance": {"method": "sample", "window_days": 60}, - "target_volatility": 0.10, - "risk_reduction_target_volatility": 0.095, - "minimum_aligned_samples": 60, + "unit_risk_per_n": 0.01, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, } assert baseline["market_data"] == { "source": "joinquant", @@ -69,7 +62,6 @@ def test_baseline_freezes_universe_rules_and_price_semantics(repo_root: Path) -> "close", "pre_close", "volume", - "money", "factor", "paused", "high_limit", @@ -77,8 +69,14 @@ def test_baseline_freezes_universe_rules_and_price_semantics(repo_root: Path) -> ], } assert baseline["joinquant_export"] == { - "api_source": "research_runtime_injected", - "apis": ["get_price", "write_file", "read_file"], + "api_source": "research_runtime_and_jqdata", + "apis": [ + "get_price", + "query", + "finance.FUND_DIVIDEND", + "write_file", + "read_file", + ], "pandas_version": "0.23.4", "csv_line_terminator_argument": "line_terminator", "paused_source_dtype": "float64", @@ -88,51 +86,79 @@ def test_baseline_freezes_universe_rules_and_price_semantics(repo_root: Path) -> "remote_readback_sha256_required": True, "remote_cleanup_required": True, } - assert baseline["execution"]["order_priority"] == [ - "full_exit", - "mandatory_risk_reduction", - "entry_or_addition", - ] - assert baseline["execution"]["allocation"] == "a1_uniform_completion" - assert baseline["execution"]["acceptance_fixture"] == { - "same_security_exit_cancels_buys": True, - "standard_request_limit": "one_u0", - "uniform_completion_ratio": True, - "capped_budget_redistribution": True, - "lot_rounding": "floor_then_largest_remainder", - "recheck_hard_caps_each_lot": True, - "tie_breaker": "security_code_ascending", - "input_order_invariant": True, + assert baseline["execution"] == { + "additional_delay_days": 0, + "order_priority": [ + "full_exit", + "redistribution_sell", + "entry_or_addition", + "redistribution_buy", + ], + "allocation": "full_position_redistribution", + "acceptance_fixture": { + "same_security_exit_cancels_buys": True, + "candidate_requires_net_buy_lot": True, + "group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + "cash_scaling": "uniform", + "lot_rounding": "floor", + "residual_cash_redistribution": False, + "input_order_invariant": True, + }, } -def test_candidates_are_baseline_plus_six_single_factor_challenges( +def test_analysis_plan_is_baseline_plus_six_single_factor_challenges( repo_root: Path, ) -> None: document = json.loads( - (_research_dir(repo_root) / "candidates.json").read_text(encoding="utf-8") + (_research_dir(repo_root) / "analysis-plan.json").read_text(encoding="utf-8") ) - candidates = document["candidates"] + scenarios = document["scenarios"] - assert document["schema_version"] == 1 - assert document["baseline_config"] == "baseline.json" - assert [item["id"] for item in candidates] == [ + assert document["schema_version"] == "strategy-analysis-plan/1" + assert document["baseline_config"].endswith("/baseline.json") + assert [item["scenario_id"] for item in scenarios] == [ "baseline", "entry-40", "entry-60", - "stop-1.5n", - "stop-2.5n", - "covariance-120d", - "covariance-ewma-30d", + "stop-1-5n", + "stop-2-5n", + "group-unit-cap-5", + "portfolio-unit-cap-10", ] - assert [item["overrides"] for item in candidates] == [ + assert [item["overrides"] for item in scenarios] == [ {}, - {"signal.entry_days": 40}, - {"signal.entry_days": 60}, - {"signal.stop_n": 1.5}, - {"signal.stop_n": 2.5}, - {"risk.covariance": {"method": "sample", "window_days": 120}}, - {"risk.covariance": {"method": "ewma", "half_life_days": 30}}, + {"signal": {"entry_days": 40}}, + {"signal": {"entry_days": 60}}, + {"signal": {"stop_n": 1.5}}, + {"signal": {"stop_n": 2.5}}, + {"risk": {"asset_group_unit_cap": 5.0}}, + {"risk": {"portfolio_unit_cap": 10.0}}, ] - assert all(len(item["overrides"]) <= 1 for item in candidates) - assert all("rank" not in item and "score" not in item for item in candidates) + assert all(len(item["overrides"]) <= 1 for item in scenarios) + assert all("rank" not in item and "score" not in item for item in scenarios) + + +def test_obsolete_challenge_analysis_plan_is_absent(repo_root: Path) -> None: + assert not (_research_dir(repo_root) / "challenge-analysis-plan.json").exists() + + +def test_authoritative_docs_confirm_the_same_new_baseline(repo_root: Path) -> None: + paths = ( + repo_root + / "docs/superpowers/specs/2026-07-16-turtle-full-position-redistribution-design.md", + repo_root / "docs/research/2026-07-13-turtle-etf-system-final-plan.md", + repo_root + / "openspec/changes/build-turtle-etf-local-research-workflow/design.md", + repo_root + / "openspec/changes/build-turtle-etf-local-research-workflow/specs/turtle-etf-local-research/spec.md", + ) + for path in paths: + text = path.read_text(encoding="utf-8") + assert "已确认" in text + assert "11 只 ETF" in text + assert "55/20/20" in text + assert "全量仓位再分配" in text + assert "4/6/12" in text + assert "180 秒" in text diff --git a/tests/local_quant_research/test_evidence.py b/tests/local_quant_research/test_evidence.py index ace5a65..a3a98ae 100644 --- a/tests/local_quant_research/test_evidence.py +++ b/tests/local_quant_research/test_evidence.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib from pathlib import Path import pytest @@ -46,3 +47,33 @@ def test_csv_evidence_rejects_malformed_data_rows( tmp_path, (OutputSpec(path="result.csv", format="csv"),), ) + + +def test_directory_evidence_binds_dynamic_result_files(tmp_path: Path) -> None: + package = tmp_path / "backtests" / "local-baseline" + (package / "data").mkdir(parents=True) + manifest = package / "manifest.json" + attribution = package / "data" / f"attribution_log-{'a' * 64}.parquet" + manifest.write_text('{"status":"complete"}\n', encoding="utf-8") + attribution.write_bytes(b"dynamic") + + evidence = collect_output_evidence( + tmp_path, + (OutputSpec(path="backtests/local-baseline", format="directory"),), + ) + + assert evidence[0]["path"] == "backtests/local-baseline" + assert evidence[0]["format"] == "directory" + assert evidence[0]["files"] == [ + { + "path": "data/attribution_log-" + "a" * 64 + ".parquet", + "bytes": attribution.stat().st_size, + "sha256": hashlib.sha256(attribution.read_bytes()).hexdigest(), + }, + { + "path": "manifest.json", + "bytes": manifest.stat().st_size, + "sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(), + }, + ] + assert len(evidence[0]["sha256"]) == 64 diff --git a/tests/local_quant_research/test_generic_e2e.py b/tests/local_quant_research/test_generic_e2e.py index c24134d..76d89fc 100644 --- a/tests/local_quant_research/test_generic_e2e.py +++ b/tests/local_quant_research/test_generic_e2e.py @@ -23,7 +23,10 @@ def _remove_empty_test_roots(repo_root: Path, market_root: Path) -> None: path.rmdir() except OSError: pass - if not (market_root / "snapshots").exists() and not (market_root / "batches").exists(): + if ( + not (market_root / "snapshots").exists() + and not (market_root / "batches").exists() + ): try: (market_root / ".market-data.lock").unlink(missing_ok=True) market_root.rmdir() @@ -88,9 +91,21 @@ def test_non_strategy_project_completes_through_shared_market_and_runner( "fields": list(MARKET_DATA_FIELDS), "price_semantics": {"fq": None, "skip_paused": False}, "export_code_sha256": export_digest, + "corporate_actions": { + "source": { + "name": "joinquant", + "dataset": "finance.FUND_DIVIDEND", + }, + "knowledge_cutoff_date": "2026-01-06", + "status": "verified_empty", + }, }, root=market_root, ) + source.unlink() + assert not source.exists() + assert (batch.path / "market-data.parquet").is_file() + assert not tuple(batch.path.rglob("*.duckdb")) selection = SnapshotSelection( source={"name": "joinquant", "environment": "research"}, asset_type="etf", @@ -110,40 +125,33 @@ def test_non_strategy_project_completes_through_shared_market_and_runner( batch_ids = list(snapshot_document["batch_ids"]) project_root.mkdir(parents=True) - adapter = project_root / "adapter.py" - adapter.write_text( - "import argparse, json\n" - "from pathlib import Path\n" - "p=argparse.ArgumentParser()\n" - "p.add_argument('--snapshot-manifest', required=True)\n" - "p.add_argument('--market-data-root', required=True)\n" - "p.add_argument('--project-config', required=True)\n" - "p.add_argument('--output-dir', required=True)\n" - "p.add_argument('--run-id', required=True)\n" - "p.add_argument('--snapshot-id', required=True)\n" - "p.add_argument('--code-sha256', required=True)\n" - "p.add_argument('--config-sha256', required=True)\n" - "a=p.parse_args()\n" - "out=Path(a.output_dir)\n" - "snap=json.loads(Path(a.snapshot_manifest).read_text(encoding='utf-8'))\n" - "(out/'result.json').write_text(json.dumps({'snapshot_id':snap['snapshot_id'],'run_id':a.run_id})+'\\n',encoding='utf-8')\n" - "(out/'project-status.json').write_text(json.dumps({'schema_version':1,'status':'complete','reason_codes':[]})+'\\n',encoding='utf-8')\n", - encoding="utf-8", + adapter = ( + repo_root / "tests/local_quant_research/fixtures/plain_project_adapter.py" ) project_config = project_root / "project.json" project_config.write_text('{"schema_version":1}\n', encoding="utf-8") declared = project_root / "input.txt" declared.write_text("plain input\n", encoding="utf-8") code_identity = project_root / "code-identity.json" + shared_sources = [ + repo_root / "scripts/__init__.py", + repo_root / "scripts/research/__init__.py", + *sorted((repo_root / "scripts/research/market_data").glob("*.py")), + ] + identity_sources = sorted( + {adapter, *shared_sources}, + key=lambda path: path.relative_to(repo_root).as_posix(), + ) code_identity.write_text( json.dumps( { "schema_version": 1, "files": [ { - "path": adapter.relative_to(repo_root).as_posix(), - "sha256": _sha256(adapter), + "path": source_path.relative_to(repo_root).as_posix(), + "sha256": _sha256(source_path), } + for source_path in identity_sources ], }, sort_keys=True, @@ -151,6 +159,9 @@ def test_non_strategy_project_completes_through_shared_market_and_runner( + "\n", encoding="utf-8", ) + required_outputs = [ + {"path": "result.json", "format": "json"}, + ] run_config = { "schema_version": 1, "project_id": project_id, @@ -164,12 +175,14 @@ def test_non_strategy_project_completes_through_shared_market_and_runner( "project_config": project_config.relative_to(repo_root).as_posix(), "code_identity": code_identity.relative_to(repo_root).as_posix(), "declared_inputs": [declared.relative_to(repo_root).as_posix()], - "required_outputs": [{"path": "result.json", "format": "json"}], + "required_outputs": required_outputs, "output_root": ".local/quant-research", "stop_states": ["complete", "evidence_insufficient", "failed"], } run_path = project_root / "run.json" - run_path.write_text(json.dumps(run_config, sort_keys=True) + "\n", encoding="utf-8") + run_path.write_text( + json.dumps(run_config, sort_keys=True) + "\n", encoding="utf-8" + ) completed = subprocess.run( [ @@ -184,17 +197,24 @@ def test_non_strategy_project_completes_through_shared_market_and_runner( text=True, shell=False, check=False, - timeout=60, + timeout=120, ) assert completed.returncode == 0, completed.stderr + completed.stdout result = json.loads(completed.stdout) assert result["status"] == "complete" - document = json.loads( - (Path(result["run_path"]) / "result.json").read_text(encoding="utf-8") - ) + run_output = Path(result["run_path"]) + document = json.loads((run_output / "result.json").read_text(encoding="utf-8")) assert document["snapshot_id"] == snapshot.snapshot_id assert document["run_id"] == result["run_id"] + status = json.loads( + (run_output / "project-status.json").read_text(encoding="utf-8") + ) + assert status["next_action"] == "return_to_caller" + assert not (run_output / "recommendation.json").exists() + assert not (run_output / "local-research-report.md").exists() + assert not tuple(run_output.glob("*.parquet")) + assert not tuple(market_root.rglob("*.duckdb")) finally: shutil.rmtree(output_project, ignore_errors=True) shutil.rmtree(project_root, ignore_errors=True) diff --git a/tests/local_quant_research/test_human_decision.py b/tests/local_quant_research/test_human_decision.py new file mode 100644 index 0000000..7ed905a --- /dev/null +++ b/tests/local_quant_research/test_human_decision.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from scripts.research.local_quant_research.decision import ( + DecisionError, + record_human_decision, + validate_human_decision, +) + + +def _run(tmp_path: Path) -> tuple[Path, str]: + run_id = "a" * 64 + run_dir = tmp_path / "run" + run_dir.mkdir() + (run_dir / "local-research-report.md").write_text("report\n", encoding="utf-8") + (run_dir / "recommendation.json").write_text( + json.dumps( + {"identity": {"run_id": run_id}, "recommendation": "revise_and_reassess"} + ), + encoding="utf-8", + ) + return run_dir, run_id + + +def test_human_decision_is_append_only_outside_immutable_run(tmp_path: Path) -> None: + run_dir, run_id = _run(tmp_path) + decision_root = tmp_path / "decisions" + decision = { + "decision": "revise_and_reassess", + "candidate_focus": ["entry-40"], + "baseline_action": "retain_frozen_baseline", + "reason": "等待人工复核稳健性反证", + "confirmed_by": "research-owner", + "confirmed_at": "2026-07-14T12:00:00+08:00", + } + + path = record_human_decision( + run_dir=run_dir, + decision_root=decision_root, + project_id="strategy-003", + decision=decision, + ) + document = validate_human_decision( + run_dir=run_dir, + project_id="strategy-003", + decision_path=path, + ) + + assert path.parent.parent.name == run_id + assert not path.is_relative_to(run_dir) + assert document["decision"] == "revise_and_reassess" + assert ( + document["report_sha256"] + == hashlib.sha256( + (run_dir / "local-research-report.md").read_bytes() + ).hexdigest() + ) + assert ( + record_human_decision( + run_dir=run_dir, + decision_root=decision_root, + project_id="strategy-003", + decision=decision, + ) + == path + ) + + +def test_human_decision_rejects_changed_report_or_recommendation( + tmp_path: Path, +) -> None: + run_dir, _ = _run(tmp_path) + path = record_human_decision( + run_dir=run_dir, + decision_root=tmp_path / "decisions", + project_id="strategy-003", + decision={ + "decision": "stop_evidence_insufficient", + "candidate_focus": [], + "baseline_action": "retain_frozen_baseline", + "reason": "证据不足", + "confirmed_by": "research-owner", + "confirmed_at": "2026-07-14T12:00:00+08:00", + }, + ) + (run_dir / "local-research-report.md").write_text("changed\n", encoding="utf-8") + + with pytest.raises(DecisionError, match="digest"): + validate_human_decision( + run_dir=run_dir, + project_id="strategy-003", + decision_path=path, + ) diff --git a/tests/local_quant_research/test_joinquant_export.py b/tests/local_quant_research/test_joinquant_export.py index a2922a5..4d59ac0 100644 --- a/tests/local_quant_research/test_joinquant_export.py +++ b/tests/local_quant_research/test_joinquant_export.py @@ -5,6 +5,7 @@ from pathlib import Path import pandas as pd +import pytest FIELDS = ( @@ -23,6 +24,71 @@ "low_limit", ) +CORPORATE_ACTION_FIELDS = ( + "source_event_id", + "security", + "event_type", + "announcement_date", + "record_date", + "ex_date", + "effective_date", + "pay_date", + "status", + "knowledge_cutoff_date", + "split_ratio", + "cash_per_share", + "source", + "source_record_sha256", +) + + +def _manifest() -> dict[str, object]: + return { + "schema_version": 1, + "source": {"name": "joinquant", "environment": "research"}, + "asset_type": "etf", + "frequency": "1d", + "fields": list(FIELDS), + "price_semantics": {"fq": None, "skip_paused": False}, + "export_code_sha256": "a" * 64, + "corporate_actions": { + "source": { + "name": "joinquant", + "dataset": "finance.FUND_DIVIDEND", + }, + "knowledge_cutoff_date": "2026-07-15", + "status": "complete", + }, + } + + +def _write_actions(path: Path) -> Path: + path.write_text( + ",".join(CORPORATE_ACTION_FIELDS) + + "\n" + + ",".join( + ( + "FUND_DIVIDEND:101", + "510300.XSHG", + "cash_dividend", + "2026-01-05", + "2026-01-05", + "2026-01-06", + "2026-01-06", + "2026-01-08", + "active", + "2026-07-15", + "", + "0.1", + "joinquant.finance.FUND_DIVIDEND", + "b" * 64, + ) + ) + + "\n", + encoding="utf-8", + ) + return path + def test_rendered_program_uses_verified_joinquant_research_contract() -> None: from scripts.research.market_data.joinquant_export import ( @@ -48,6 +114,11 @@ def test_rendered_program_uses_verified_joinquant_research_contract() -> None: assert "line_terminator='\\n'" in program assert "write_file(" in program assert "read_file(" in program + assert "finance.FUND_DIVIDEND" in program + assert "from jqdata import finance, query" in program + assert "corporate-actions.csv" in program + for field in CORPORATE_ACTION_FIELDS: + assert repr(field) in program assert "hashlib.sha256(remote_bytes).hexdigest()" in program assert "def cleanup_export(delete_file):" in program assert "from jqdata import get_price" not in program @@ -89,11 +160,55 @@ def write_file(path: str, content: bytes, append: bool = False) -> None: def read_file(path: str) -> bytes: return remote_files[path] + class Column: + def __eq__(self, value): + return ("eq", value) + + class FundDividend: + code = Column() + + class Query: + def filter(self, *conditions): + return self + + def limit(self, value: int): + assert value == 5000 + return self + + class Finance: + FUND_DIVIDEND = FundDividend() + + @staticmethod + def run_query(_query): + return pd.DataFrame( + [ + { + "id": 101, + "code": "510300", + "pub_date": date(2026, 1, 5), + "event_id": 404001, + "event": "基金分红", + "process_id": 405003, + "process": "取消分红", + "proportion": 0.1, + "split_ratio": None, + "record_date": date(2026, 1, 5), + "ex_date": pd.NaT, + "fund_paid_date": date(2026, 1, 8), + "dividend_cancel_date": date(2026, 1, 7), + "otc_ex_date": date(2026, 1, 6), + "pay_date": pd.NaT, + } + ] + ) + namespace = { "get_security_info": get_security_info, "get_price": get_price, "write_file": write_file, "read_file": read_file, + "finance": Finance(), + "query": lambda _table: Query(), } program = render_export_program( ExportRequest( @@ -106,9 +221,28 @@ def read_file(path: str) -> bytes: exec(compile(program, "", "exec"), namespace) payload = remote_files["market-data.csv"] + action_payload = remote_files["corporate-actions.csv"] assert payload.decode("utf-8").splitlines()[0] == ",".join(FIELDS) - assert namespace["export_result"]["sha256"] == hashlib.sha256(payload).hexdigest() - assert namespace["export_result"]["rows"] == 2 + assert action_payload.decode("utf-8").splitlines()[0] == ",".join( + CORPORATE_ACTION_FIELDS + ) + assert namespace["export_result"]["market_data"]["sha256"] == hashlib.sha256( + payload + ).hexdigest() + assert namespace["export_result"]["market_data"]["rows"] == 2 + assert namespace["export_result"]["corporate_actions"]["sha256"] == ( + hashlib.sha256(action_payload).hexdigest() + ) + assert namespace["export_result"]["corporate_actions"]["rows"] == 1 + assert b"cash_dividend" in action_payload + action_row = action_payload.decode("utf-8").splitlines()[1].split(",") + assert action_row[5] == "2026-01-06" + assert action_row[7] == "2026-01-08" + assert action_row[8] == "active" + status_at_cutoff = namespace["_status_at_cutoff"] + assert status_at_cutoff(405003, date(2026, 1, 6)) == "cancelled" + with pytest.raises(ValueError, match="cancellation date"): + status_at_cutoff(405003, None) assert calls == [ { "security": "510300.XSHG", @@ -188,3 +322,107 @@ def test_verify_transfer_fails_when_local_file_is_missing(tmp_path: Path) -> Non assert evidence.status == "failed" assert evidence.local_sha256 is None assert "local transfer file is missing" in evidence.reasons + + +def test_import_verified_transfer_publishes_before_deleting_remote_and_local_csv( + repo_root: Path, + tmp_path: Path, +) -> None: + from scripts.research.market_data.joinquant_export import import_verified_transfer + + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + local_file = tmp_path / "transfer" / "market-data.csv" + local_file.parent.mkdir() + local_file.write_bytes(source.read_bytes()) + digest = hashlib.sha256(local_file.read_bytes()).hexdigest() + action_file = _write_actions(local_file.with_name("corporate-actions.csv")) + action_digest = hashlib.sha256(action_file.read_bytes()).hexdigest() + + events: list[str] = [] + + def cleanup_remote() -> bool: + assert (tmp_path / "store" / "batches").is_dir() + assert local_file.exists() + events.append("remote-cleaned") + return True + + record = import_verified_transfer( + local_file=local_file, + remote_sha256=digest, + corporate_actions_file=action_file, + corporate_actions_remote_sha256=action_digest, + cleanup_remote=cleanup_remote, + manifest=_manifest(), + root=tmp_path / "store", + ) + + assert events == ["remote-cleaned"] + assert not local_file.exists() + assert not action_file.exists() + assert (record.path / "market-data.parquet").is_file() + assert not (record.path / "market-data.csv").exists() + + +def test_import_verified_transfer_preserves_both_transports_after_conversion_failure( + tmp_path: Path, +) -> None: + from scripts.research.market_data.joinquant_export import import_verified_transfer + from scripts.research.market_data.storage import MarketDataIntegrityError + + local_file = tmp_path / "market-data.csv" + local_file.write_text("not,the,declared,fields\n", encoding="utf-8") + digest = hashlib.sha256(local_file.read_bytes()).hexdigest() + action_file = _write_actions(tmp_path / "corporate-actions.csv") + action_digest = hashlib.sha256(action_file.read_bytes()).hexdigest() + + remote_cleanup_calls = 0 + + def cleanup_remote() -> bool: + nonlocal remote_cleanup_calls + remote_cleanup_calls += 1 + return True + + with pytest.raises(MarketDataIntegrityError): + import_verified_transfer( + local_file=local_file, + remote_sha256=digest, + corporate_actions_file=action_file, + corporate_actions_remote_sha256=action_digest, + cleanup_remote=cleanup_remote, + manifest=_manifest(), + root=tmp_path / "store", + ) + + assert local_file.exists() + assert action_file.exists() + assert remote_cleanup_calls == 0 + assert not (tmp_path / "store" / "batches").exists() + + +def test_import_verified_transfer_fails_and_preserves_local_when_remote_cleanup_unconfirmed( + repo_root: Path, + tmp_path: Path, +) -> None: + from scripts.research.market_data.joinquant_export import import_verified_transfer + from scripts.research.market_data.storage import MarketDataIntegrityError + + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + local_file = tmp_path / "market-data.csv" + local_file.write_bytes(source.read_bytes()) + digest = hashlib.sha256(local_file.read_bytes()).hexdigest() + action_file = _write_actions(tmp_path / "corporate-actions.csv") + action_digest = hashlib.sha256(action_file.read_bytes()).hexdigest() + + with pytest.raises(MarketDataIntegrityError, match="remote cleanup"): + import_verified_transfer( + local_file=local_file, + remote_sha256=digest, + corporate_actions_file=action_file, + corporate_actions_remote_sha256=action_digest, + cleanup_remote=lambda: False, + manifest=_manifest(), + root=tmp_path / "store", + ) + + assert local_file.exists() + assert action_file.exists() diff --git a/tests/local_quant_research/test_market_data_economic_returns.py b/tests/local_quant_research/test_market_data_economic_returns.py new file mode 100644 index 0000000..994eb3a --- /dev/null +++ b/tests/local_quant_research/test_market_data_economic_returns.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from types import MappingProxyType + +import pandas as pd +import pytest + +from scripts.research.market_data.contracts import ( + CORPORATE_ACTION_FIELDS, + MARKET_DATA_FIELDS, + corporate_actions_digest, + normalized_digest, +) +from scripts.research.market_data.economic_returns import ( + EconomicReturnError, + derive_continuous_prices, + snapshot_return_panel, +) +from scripts.research.market_data.query import SnapshotView + + +def _frame() -> pd.DataFrame: + return pd.DataFrame( + { + "date": pd.date_range("2026-01-05", periods=5, freq="D"), + "security": "ETF-A", + "open": [100.0, 101.0, 50.75, 51.5, 52.0], + "high": [101.0, 103.0, 52.0, 53.0, 54.0], + "low": [99.0, 100.0, 50.0, 51.0, 52.0], + "close": [100.0, 102.0, 51.0, 52.0, 53.0], + "pre_close": [100.0, 100.0, 51.0, 51.0, 52.0], + "volume": [1_000.0] * 5, + "money": [100_000.0] * 5, + "factor": [1.0] * 5, + "paused": [False] * 5, + "high_limit": [110.0, 112.0, 56.0, 57.0, 58.0], + "low_limit": [90.0, 92.0, 46.0, 47.0, 48.0], + } + ) + + +def _split_action() -> dict[str, object]: + return { + "source_event_id": "FUND_DIVIDEND:101", + "security": "ETF-A", + "event_type": "split", + "announcement_date": "2026-01-06", + "record_date": "2026-01-06", + "ex_date": "2026-01-07", + "effective_date": "2026-01-07", + "pay_date": None, + "status": "active", + "knowledge_cutoff_date": "2026-01-10", + "split_ratio": 2.0, + "cash_per_share": None, + "source": "joinquant.finance.FUND_DIVIDEND", + "source_record_sha256": "b" * 64, + } + + +def test_shared_continuous_returns_reconcile_split_without_false_loss() -> None: + result = derive_continuous_prices( + _frame(), + security="ETF-A", + corporate_actions=[_split_action()], + ) + + assert result.returns.to_numpy() == pytest.approx( + result.frame["close"] / result.frame["pre_close"] - 1.0 + ) + assert result.returns.iloc[2] == pytest.approx(0.0) + assert result.applications[0].security == "ETF-A" + + +def test_shared_continuous_returns_reject_unreconciled_basis_change() -> None: + with pytest.raises(EconomicReturnError, match="unexplained price-basis change"): + derive_continuous_prices( + _frame(), + security="ETF-A", + corporate_actions=(), + ) + + +def test_snapshot_return_panel_uses_snapshot_actions_and_sorted_columns() -> None: + frame = _frame() + rows = tuple( + MappingProxyType( + { + field: ( + value.strftime("%Y-%m-%d") + if field == "date" + else value + ) + for field, value in row.items() + if field in MARKET_DATA_FIELDS + } + ) + for row in frame.to_dict(orient="records") + ) + actions = (MappingProxyType(_split_action()),) + snapshot = SnapshotView( + snapshot_id="a" * 64, + fields=MARKET_DATA_FIELDS, + rows=rows, + digest=normalized_digest(rows), + corporate_action_fields=CORPORATE_ACTION_FIELDS, + corporate_actions=actions, + corporate_actions_digest=corporate_actions_digest(actions), + ) + + panel = snapshot_return_panel(snapshot) + + assert panel.columns.tolist() == ["ETF-A"] + assert panel.index.tolist() == list(pd.date_range("2026-01-05", periods=5)) + assert panel.loc[pd.Timestamp("2026-01-07"), "ETF-A"] == pytest.approx(0.0) diff --git a/tests/local_quant_research/test_market_data_query.py b/tests/local_quant_research/test_market_data_query.py index defc6c9..a5e9e7c 100644 --- a/tests/local_quant_research/test_market_data_query.py +++ b/tests/local_quant_research/test_market_data_query.py @@ -1,6 +1,8 @@ from __future__ import annotations +from dataclasses import replace from pathlib import Path +from types import MappingProxyType import pytest @@ -28,6 +30,23 @@ "low_limit", ) +CORPORATE_ACTION_FIELDS = ( + "source_event_id", + "security", + "event_type", + "announcement_date", + "record_date", + "ex_date", + "effective_date", + "pay_date", + "status", + "knowledge_cutoff_date", + "split_ratio", + "cash_per_share", + "source", + "source_record_sha256", +) + def _manifest() -> dict[str, object]: return { @@ -38,6 +57,14 @@ def _manifest() -> dict[str, object]: "fields": list(FIELDS), "price_semantics": {"fq": None, "skip_paused": False}, "export_code_sha256": "a" * 64, + "corporate_actions": { + "source": { + "name": "joinquant", + "dataset": "finance.FUND_DIVIDEND", + }, + "knowledge_cutoff_date": "2026-07-15", + "status": "verified_empty", + }, } @@ -74,16 +101,25 @@ def test_open_snapshot_uses_only_memory_and_returns_normalized_read_only_rows( snapshot = _snapshot(repo_root, tmp_path) real_connect = query.duckdb.connect connections: list[str] = [] + queried_paths: list[tuple[Path, ...]] = [] + real_reader = query._read_query_rows def recording_connect(database: str): connections.append(database) return real_connect(database) + def recording_reader(connection, parquet_paths): + queried_paths.append(tuple(parquet_paths)) + return real_reader(connection, parquet_paths) + monkeypatch.setattr(query.duckdb, "connect", recording_connect) + monkeypatch.setattr(query, "_read_query_rows", recording_reader) view = query.open_snapshot(snapshot.snapshot_id, root=tmp_path) assert connections == [":memory:"] + assert queried_paths + assert all(path.name == "market-data.parquet" for path in queried_paths[0]) assert view.snapshot_id == snapshot.snapshot_id assert view.fields == FIELDS assert [(row["date"], row["security"]) for row in view.rows] == [ @@ -95,11 +131,80 @@ def recording_connect(database: str): assert view.rows[0]["open"] == 10.0 assert view.rows[0]["paused"] is False assert view.digest == query.normalized_digest(view.rows) + assert view.corporate_actions == () + assert view.corporate_actions_digest with pytest.raises(TypeError): view.rows[0]["close"] = 99.0 assert not list(tmp_path.rglob("*.duckdb")) +def test_open_snapshot_returns_corporate_actions_from_the_same_memory_database( + repo_root: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from scripts.research.market_data import query + + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + action_path = tmp_path / "corporate-actions.csv" + action_path.write_text( + ",".join(CORPORATE_ACTION_FIELDS) + + "\n" + + ",".join( + ( + "jq-000001-20260106-cash", + "000001.XSHG", + "cash_dividend", + "2026-01-05", + "2026-01-05", + "2026-01-06", + "2026-01-06", + "2026-01-08", + "active", + "2026-07-15", + "", + "0.1", + "joinquant.finance.FUND_DIVIDEND", + "b" * 64, + ) + ) + + "\n", + encoding="utf-8", + ) + manifest = _manifest() + manifest["corporate_actions"]["status"] = "complete" + batch = import_batch( + csv_path=source, + corporate_actions_csv_path=action_path, + manifest=manifest, + root=tmp_path / "store", + ) + snapshot = create_snapshot( + batch_ids=[batch.batch_id], + selection=_selection("000001.XSHG", "000002.XSHE"), + root=tmp_path / "store", + ) + connections: list[str] = [] + real_connect = query.duckdb.connect + + def recording_connect(database: str): + connections.append(database) + return real_connect(database) + + monkeypatch.setattr(query.duckdb, "connect", recording_connect) + + view = query.open_snapshot(snapshot.snapshot_id, root=tmp_path / "store") + + assert connections == [":memory:"] + assert view.corporate_action_fields == CORPORATE_ACTION_FIELDS + assert len(view.corporate_actions) == 1 + assert view.corporate_actions[0]["source_event_id"] == "jq-000001-20260106-cash" + assert view.corporate_actions[0]["cash_per_share"] == 0.1 + with pytest.raises(TypeError): + view.corporate_actions[0]["cash_per_share"] = 1.0 + assert not list(tmp_path.rglob("*.duckdb")) + + def test_open_snapshot_normalizes_nulls_and_numeric_paused(tmp_path: Path) -> None: from scripts.research.market_data.query import open_snapshot @@ -168,7 +273,7 @@ def test_normalized_digest_is_independent_of_input_order() -> None: assert normalized_digest(rows) == normalized_digest(reversed(rows)) -def test_open_snapshot_rejects_query_and_csv_content_drift( +def test_open_snapshot_rejects_query_and_parquet_content_drift( repo_root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -187,3 +292,90 @@ def changed_reader(*args, **kwargs): with pytest.raises(MarketDataIntegrityError, match="normalized digest"): query.open_snapshot(snapshot.snapshot_id, root=tmp_path) + + +def test_snapshot_overlap_accepts_exact_subset_with_shared_batch( + repo_root: Path, + tmp_path: Path, +) -> None: + from scripts.research.market_data.query import validate_snapshot_overlap + + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + batch = import_batch(csv_path=source, manifest=_manifest(), root=tmp_path) + baseline = create_snapshot( + batch_ids=[batch.batch_id], + selection=_selection("000001.XSHG"), + root=tmp_path, + ) + expanded = create_snapshot( + batch_ids=[batch.batch_id], + selection=_selection("000001.XSHG", "000002.XSHE"), + root=tmp_path, + ) + + evidence = validate_snapshot_overlap( + baseline.snapshot_id, + expanded.snapshot_id, + root=tmp_path, + ) + + assert evidence.securities == ("000001.XSHG",) + assert evidence.market_digest + assert evidence.corporate_actions_digest + + +def test_snapshot_overlap_rejects_corporate_action_drift( + repo_root: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from scripts.research.market_data import query + + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + batch = import_batch(csv_path=source, manifest=_manifest(), root=tmp_path) + baseline = create_snapshot( + batch_ids=[batch.batch_id], + selection=_selection("000001.XSHG"), + root=tmp_path, + ) + expanded = create_snapshot( + batch_ids=[batch.batch_id], + selection=_selection("000001.XSHG", "000002.XSHE"), + root=tmp_path, + ) + left_view = query.open_snapshot(baseline.snapshot_id, root=tmp_path) + right_view = query.open_snapshot(expanded.snapshot_id, root=tmp_path) + changed_action = MappingProxyType( + { + field: value + for field, value in zip( + CORPORATE_ACTION_FIELDS, + ( + "drift", + "000001.XSHG", + "cash_dividend", + "2026-01-05", + "2026-01-05", + "2026-01-06", + "2026-01-06", + "2026-01-08", + "active", + "2026-07-15", + None, + 0.1, + "fixture", + "b" * 64, + ), + ) + } + ) + changed_right = replace(right_view, corporate_actions=(changed_action,)) + views = iter((left_view, changed_right)) + monkeypatch.setattr(query, "open_snapshot", lambda *_args, **_kwargs: next(views)) + + with pytest.raises(MarketDataIntegrityError, match="corporate-actions"): + query.validate_snapshot_overlap( + baseline.snapshot_id, + expanded.snapshot_id, + root=tmp_path, + ) diff --git a/tests/local_quant_research/test_market_data_storage.py b/tests/local_quant_research/test_market_data_storage.py index a6be57c..490e07e 100644 --- a/tests/local_quant_research/test_market_data_storage.py +++ b/tests/local_quant_research/test_market_data_storage.py @@ -6,6 +6,7 @@ import queue from pathlib import Path +import pyarrow.parquet as pq import pytest from scripts.research.market_data.contracts import SnapshotSelection @@ -35,6 +36,23 @@ "low_limit", ) +CORPORATE_ACTION_FIELDS = ( + "source_event_id", + "security", + "event_type", + "announcement_date", + "record_date", + "ex_date", + "effective_date", + "pay_date", + "status", + "knowledge_cutoff_date", + "split_ratio", + "cash_per_share", + "source", + "source_record_sha256", +) + def _import_in_process( csv_path: str, @@ -71,7 +89,7 @@ def _canonical_digest(value: object) -> str: return hashlib.sha256(payload).hexdigest() -def _manifest() -> dict[str, object]: +def _manifest(*, corporate_action_status: str = "verified_empty") -> dict[str, object]: return { "schema_version": 1, "source": {"name": "joinquant", "environment": "research"}, @@ -80,9 +98,57 @@ def _manifest() -> dict[str, object]: "fields": list(FIELDS), "price_semantics": {"fq": None, "skip_paused": False}, "export_code_sha256": "a" * 64, + "corporate_actions": { + "source": { + "name": "joinquant", + "dataset": "finance.FUND_DIVIDEND", + }, + "knowledge_cutoff_date": "2026-07-15", + "status": corporate_action_status, + }, } +def _write_corporate_actions( + path: Path, + *, + split_ratio: str = "2", + status: str = "active", + announcement_date: str = "2026-06-30", +) -> Path: + row = ( + "jq-000001-20260703-split", + "000001.XSHG", + "split", + announcement_date, + "2026-07-02", + "2026-07-03", + "2026-07-03", + "", + status, + "2026-07-15", + split_ratio, + "", + "joinquant.finance.FUND_DIVIDEND", + "b" * 64, + ) + path.write_text( + ",".join(CORPORATE_ACTION_FIELDS) + "\n" + ",".join(row) + "\n", + encoding="utf-8", + newline="", + ) + return path + + +def _write_empty_corporate_actions(path: Path) -> Path: + path.write_text( + ",".join(CORPORATE_ACTION_FIELDS) + "\n", + encoding="utf-8", + newline="", + ) + return path + + def _selection(*securities: str) -> SnapshotSelection: return SnapshotSelection( source={"name": "joinquant", "environment": "research"}, @@ -96,6 +162,106 @@ def _selection(*securities: str) -> SnapshotSelection: ) +def _write_legacy_batch(root: Path, source: Path) -> Path: + csv_bytes = source.read_bytes() + csv_sha256 = hashlib.sha256(csv_bytes).hexdigest() + legacy_base = _manifest() + legacy_base.pop("corporate_actions") + legacy_manifest = { + **legacy_base, + "csv": {"sha256": csv_sha256, "bytes": len(csv_bytes), "rows": 4}, + "securities": [ + { + "security": "000001.XSHG", + "start_date": "2026-01-05", + "end_date": "2026-01-06", + "rows": 2, + }, + { + "security": "000002.XSHE", + "start_date": "2026-01-05", + "end_date": "2026-01-06", + "rows": 2, + }, + ], + } + identity = { + "source": legacy_manifest["source"], + "asset_type": legacy_manifest["asset_type"], + "frequency": legacy_manifest["frequency"], + "fields": legacy_manifest["fields"], + "price_semantics": legacy_manifest["price_semantics"], + "export_code_sha256": legacy_manifest["export_code_sha256"], + "csv_sha256": csv_sha256, + } + batch_dir = root / "batches" / _canonical_digest(identity) + batch_dir.mkdir(parents=True) + (batch_dir / "manifest.json").write_text( + json.dumps( + legacy_manifest, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + "\n", + encoding="utf-8", + ) + (batch_dir / "market-data.csv").write_bytes(csv_bytes) + (batch_dir / "validation.json").write_text( + json.dumps( + { + "schema_version": 1, + "status": "complete", + "checks": { + "field_order": True, + "nonempty": True, + "unique_date_security": True, + }, + }, + sort_keys=True, + separators=(",", ":"), + ) + + "\n", + encoding="utf-8", + ) + return batch_dir + + +def _write_legacy_snapshot(root: Path, batch_dir: Path) -> Path: + manifest_path = batch_dir / "manifest.json" + validation_path = batch_dir / "validation.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + payload = { + "schema_version": 1, + "batch_ids": [batch_dir.name], + "batches": [ + { + "batch_id": batch_dir.name, + "manifest_sha256": _sha256(manifest_path), + "csv_sha256": _sha256(batch_dir / "market-data.csv"), + "validation_sha256": _sha256(validation_path), + "export_code_sha256": manifest["export_code_sha256"], + } + ], + "selection": _selection("000001.XSHG", "000002.XSHE").to_document(), + "coverage": manifest["securities"], + } + snapshot_id = _canonical_digest(payload) + snapshot_path = root / "snapshots" / f"{snapshot_id}.json" + snapshot_path.parent.mkdir(parents=True) + snapshot_path.write_text( + json.dumps( + {**payload, "snapshot_id": snapshot_id}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + "\n", + encoding="utf-8", + ) + return snapshot_path + + def test_import_batch_is_immutable_complete_and_deduplicated( repo_root: Path, tmp_path: Path, @@ -110,16 +276,28 @@ def test_import_batch_is_immutable_complete_and_deduplicated( assert first == second assert {path.name for path in batch_dir.iterdir()} == { "manifest.json", - "market-data.csv", + "market-data.parquet", + "corporate-actions.parquet", "validation.json", } - assert (batch_dir / "market-data.csv").read_bytes() == source.read_bytes() + assert (batch_dir / "market-data.parquet").stat().st_size > 0 assert {path.name: _sha256(path) for path in batch_dir.iterdir()} == before assert [path.name for path in (tmp_path / "batches").iterdir()] == [first.batch_id] stored_manifest = json.loads((batch_dir / "manifest.json").read_text(encoding="utf-8")) - assert stored_manifest["csv"]["sha256"] == _sha256(source) - assert stored_manifest["csv"]["rows"] == 4 + assert stored_manifest["schema_version"] == 3 + assert stored_manifest["transport_csv"]["sha256"] == _sha256(source) + assert stored_manifest["transport_csv"]["rows"] == 4 + assert stored_manifest["parquet"]["sha256"] == _sha256( + batch_dir / "market-data.parquet" + ) + assert stored_manifest["parquet"]["rows"] == 4 + assert len(stored_manifest["content_sha256"]) == 64 + assert stored_manifest["corporate_actions"]["status"] == "verified_empty" + assert stored_manifest["corporate_actions"]["rows"] == 0 + assert stored_manifest["corporate_actions"]["parquet"]["sha256"] == _sha256( + batch_dir / "corporate-actions.parquet" + ) assert stored_manifest["securities"] == [ { "security": "000001.XSHG", @@ -136,16 +314,303 @@ def test_import_batch_is_immutable_complete_and_deduplicated( ] validation = json.loads((batch_dir / "validation.json").read_text(encoding="utf-8")) assert validation == { - "schema_version": 1, + "schema_version": 3, "status": "complete", "checks": { "field_order": True, "nonempty": True, "unique_date_security": True, + "parquet_roundtrip": True, + "normalized_digest": True, + "corporate_actions_field_order": True, + "corporate_actions_primary_key": True, + "corporate_actions_point_in_time": True, + "corporate_actions_parquet_roundtrip": True, + "corporate_actions_normalized_digest": True, }, } +def test_import_batch_publishes_market_data_and_versioned_corporate_actions( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + actions = _write_corporate_actions(tmp_path / "corporate-actions.csv") + + record = import_batch( + csv_path=source, + corporate_actions_csv_path=actions, + manifest=_manifest(corporate_action_status="complete"), + root=tmp_path / "store", + ) + + assert {path.name for path in record.path.iterdir()} == { + "manifest.json", + "market-data.parquet", + "corporate-actions.parquet", + "validation.json", + } + stored = json.loads((record.path / "manifest.json").read_text("utf-8")) + assert stored["schema_version"] == 3 + assert stored["content_sha256"] + assert stored["corporate_actions"]["content_sha256"] + assert stored["corporate_actions"]["rows"] == 1 + assert stored["corporate_actions"]["knowledge_cutoff_date"] == "2026-07-15" + assert stored["corporate_actions"]["parquet"]["sha256"] == _sha256( + record.path / "corporate-actions.parquet" + ) + + +def test_import_batch_requires_an_explicit_valid_empty_corporate_action_set( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + actions = _write_empty_corporate_actions(tmp_path / "corporate-actions.csv") + + record = import_batch( + csv_path=source, + corporate_actions_csv_path=actions, + manifest=_manifest(corporate_action_status="verified_empty"), + root=tmp_path / "store", + ) + + stored = json.loads((record.path / "manifest.json").read_text("utf-8")) + assert stored["corporate_actions"]["rows"] == 0 + assert (record.path / "corporate-actions.parquet").is_file() + + +def test_corporate_actions_participate_in_batch_and_snapshot_identity( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + first_actions = _write_corporate_actions(tmp_path / "first-actions.csv") + second_actions = _write_corporate_actions( + tmp_path / "second-actions.csv", + split_ratio="3", + ) + + first = import_batch( + csv_path=source, + corporate_actions_csv_path=first_actions, + manifest=_manifest(corporate_action_status="complete"), + root=tmp_path / "first-store", + ) + second = import_batch( + csv_path=source, + corporate_actions_csv_path=second_actions, + manifest=_manifest(corporate_action_status="complete"), + root=tmp_path / "second-store", + ) + first_snapshot = create_snapshot( + batch_ids=[first.batch_id], + selection=_selection("000001.XSHG", "000002.XSHE"), + root=tmp_path / "first-store", + ) + second_snapshot = create_snapshot( + batch_ids=[second.batch_id], + selection=_selection("000001.XSHG", "000002.XSHE"), + root=tmp_path / "second-store", + ) + + assert first.batch_id != second.batch_id + assert first_snapshot.snapshot_id != second_snapshot.snapshot_id + assert first_snapshot.document["batches"][0]["corporate_actions_sha256"] + assert first_snapshot.document["batches"][0][ + "corporate_actions_content_sha256" + ] + + +def test_snapshot_validation_rejects_tampered_corporate_actions( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + actions = _write_corporate_actions(tmp_path / "corporate-actions.csv") + batch = import_batch( + csv_path=source, + corporate_actions_csv_path=actions, + manifest=_manifest(corporate_action_status="complete"), + root=tmp_path, + ) + snapshot = create_snapshot( + batch_ids=[batch.batch_id], + selection=_selection("000001.XSHG", "000002.XSHE"), + root=tmp_path, + ) + path = batch.path / "corporate-actions.parquet" + path.write_bytes(path.read_bytes() + b"tampered") + + with pytest.raises(MarketDataIntegrityError, match="corporate-actions.*SHA256"): + validate_snapshot(snapshot.snapshot_id, root=tmp_path) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"status": "unknown"}, "status"), + ({"split_ratio": "0"}, "split_ratio"), + ], +) +def test_import_rejects_invalid_corporate_action_evidence( + repo_root: Path, + tmp_path: Path, + overrides: dict[str, str], + message: str, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + actions = _write_corporate_actions( + tmp_path / "corporate-actions.csv", + **overrides, + ) + + with pytest.raises(MarketDataIntegrityError, match=message): + import_batch( + csv_path=source, + corporate_actions_csv_path=actions, + manifest=_manifest(corporate_action_status="complete"), + root=tmp_path / "store", + ) + + +def test_import_retains_action_metadata_published_after_effective_date( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + actions = _write_corporate_actions( + tmp_path / "corporate-actions.csv", + announcement_date="2026-07-04", + ) + + record = import_batch( + csv_path=source, + corporate_actions_csv_path=actions, + manifest=_manifest(corporate_action_status="complete"), + root=tmp_path / "store", + ) + + stored = pq.read_table(record.path / "corporate-actions.parquet").to_pylist() + assert stored[0]["effective_date"] == "2026-07-03" + assert stored[0]["announcement_date"] == "2026-07-04" + + +def test_batch_identity_uses_logical_content_not_csv_line_endings( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + lf = tmp_path / "lf.csv" + crlf = tmp_path / "crlf.csv" + text = source.read_text(encoding="utf-8").replace("\r\n", "\n") + lf.write_text(text, encoding="utf-8", newline="") + crlf.write_bytes(text.replace("\n", "\r\n").encode("utf-8")) + + first = import_batch(csv_path=lf, manifest=_manifest(), root=tmp_path / "store") + second = import_batch(csv_path=crlf, manifest=_manifest(), root=tmp_path / "store") + + assert first.batch_id == second.batch_id + assert first.manifest["transport_csv"]["sha256"] != _sha256(crlf) + assert len(list((tmp_path / "store" / "batches").iterdir())) == 1 + + +def test_legacy_csv_batch_allows_non_overlapping_parquet_import( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + legacy_dir = _write_legacy_batch(tmp_path, source) + added = tmp_path / "added.csv" + added.write_text( + ",".join(FIELDS) + + "\n2026-01-05,000003.XSHG,30,31,29,30.5,30,100,3050,1,0,33,27\n", + encoding="utf-8", + ) + + record = import_batch(csv_path=added, manifest=_manifest(), root=tmp_path) + + assert (record.path / "market-data.parquet").is_file() + assert (legacy_dir / "market-data.csv").is_file() + + +def test_legacy_csv_batch_still_rejects_conflicting_overlap( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + _write_legacy_batch(tmp_path, source) + conflicting = tmp_path / "conflicting.csv" + conflicting.write_text( + source.read_text(encoding="utf-8").replace( + "2026-01-05,000001.XSHG,10.00,10.20,9.90,10.10,", + "2026-01-05,000001.XSHG,10.00,10.20,9.90,10.15,", + ), + encoding="utf-8", + ) + + with pytest.raises(MarketDataConflict, match="000001.XSHG.*2026-01-05"): + import_batch(csv_path=conflicting, manifest=_manifest(), root=tmp_path) + + +def test_new_snapshot_rejects_legacy_csv_batch_with_migration_message( + repo_root: Path, + tmp_path: Path, +) -> None: + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + legacy_dir = _write_legacy_batch(tmp_path, source) + + with pytest.raises( + MarketDataIntegrityError, + match=f"legacy CSV batch requires migration: {legacy_dir.name}", + ): + create_snapshot( + batch_ids=[legacy_dir.name], + selection=_selection("000001.XSHG", "000002.XSHE"), + root=tmp_path, + ) + + +def test_audit_store_reports_legacy_and_parquet_batches_without_mutation( + repo_root: Path, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + from scripts.research.market_data.cli import main + + source = repo_root / "tests/local_quant_research/fixtures/daily-bars.csv" + legacy_dir = _write_legacy_batch(tmp_path, source) + legacy_snapshot = _write_legacy_snapshot(tmp_path, legacy_dir) + added = tmp_path / "added.csv" + added.write_text( + ",".join(FIELDS) + + "\n2026-01-05,000003.XSHG,30,31,29,30.5,30,100,3050,1,0,33,27\n", + encoding="utf-8", + ) + parquet = import_batch(csv_path=added, manifest=_manifest(), root=tmp_path) + before = { + path.relative_to(tmp_path).as_posix(): _sha256(path) + for path in tmp_path.rglob("*") + if path.is_file() and path.name != ".market-data.lock" + } + + assert main(["audit", "--root", str(tmp_path)]) == 0 + + document = json.loads(capsys.readouterr().out) + assert document["status"] == "complete" + assert document["legacy_batch_ids"] == [legacy_dir.name] + assert document["parquet_batch_ids"] == [parquet.batch_id] + assert document["legacy_snapshot_ids"] == [legacy_snapshot.stem] + assert document["snapshot_ids"] == [] + after = { + path.relative_to(tmp_path).as_posix(): _sha256(path) + for path in tmp_path.rglob("*") + if path.is_file() and path.name != ".market-data.lock" + } + assert after == before + + def test_import_batch_retries_transient_directory_publish_lock( repo_root: Path, tmp_path: Path, @@ -259,7 +724,7 @@ def test_snapshot_only_references_batches_and_remains_stable_after_append( assert not list(tmp_path.rglob("*.tmp*")) -def test_snapshot_validation_rejects_tampered_authoritative_csv( +def test_snapshot_validation_rejects_tampered_authoritative_parquet( repo_root: Path, tmp_path: Path, ) -> None: @@ -270,8 +735,8 @@ def test_snapshot_validation_rejects_tampered_authoritative_csv( selection=_selection("000001.XSHG", "000002.XSHE"), root=tmp_path, ) - csv_path = tmp_path / "batches" / batch.batch_id / "market-data.csv" - csv_path.write_bytes(csv_path.read_bytes() + b"\n") + parquet_path = tmp_path / "batches" / batch.batch_id / "market-data.parquet" + parquet_path.write_bytes(parquet_path.read_bytes() + b"tampered") with pytest.raises(MarketDataIntegrityError, match="SHA256"): validate_snapshot(snapshot.snapshot_id, root=tmp_path) @@ -412,6 +877,7 @@ def test_snapshot_can_reference_multiple_verified_batches( }, ] assert all("validation_sha256" in item for item in snapshot.document["batches"]) + assert all("parquet_sha256" in item for item in snapshot.document["batches"]) assert validate_snapshot(snapshot.snapshot_id, root=tmp_path) == snapshot diff --git a/tests/local_quant_research/test_runner.py b/tests/local_quant_research/test_runner.py index 8e8a1c8..d5ca5d6 100644 --- a/tests/local_quant_research/test_runner.py +++ b/tests/local_quant_research/test_runner.py @@ -8,8 +8,12 @@ from typing import Callable import pytest +import pyarrow as pa +import pyarrow.parquet as pq -from scripts.research.local_quant_research.runner import run_project +from scripts.research.local_quant_research.contracts import OutputSpec +from scripts.research.local_quant_research.evidence import collect_output_evidence +from scripts.research.local_quant_research.runner import _project_status, run_project from scripts.research.local_quant_research.evidence import canonical_digest from scripts.research.market_data.contracts import SnapshotSelection from scripts.research.market_data.storage import create_snapshot, import_batch @@ -32,6 +36,38 @@ ) +def test_complete_project_status_can_declare_human_confirmation_next_action( + tmp_path: Path, +) -> None: + _write_json( + tmp_path / "project-status.json", + { + "schema_version": 1, + "status": "complete", + "reason_codes": [], + "next_action": "human_confirmation_required", + }, + ) + + assert _project_status(tmp_path) == ("complete", ()) + + +def test_complete_project_status_can_return_single_scenario_to_caller( + tmp_path: Path, +) -> None: + _write_json( + tmp_path / "project-status.json", + { + "schema_version": 1, + "status": "complete", + "reason_codes": [], + "next_action": "return_to_caller", + }, + ) + + assert _project_status(tmp_path) == ("complete", ()) + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text( @@ -88,6 +124,14 @@ def _build_repo(tmp_path: Path, source_repo: Path) -> tuple[Path, Path, dict[str "fields": list(FIELDS), "price_semantics": {"fq": None, "skip_paused": False}, "export_code_sha256": "a" * 64, + "corporate_actions": { + "source": { + "name": "joinquant", + "dataset": "finance.FUND_DIVIDEND", + }, + "knowledge_cutoff_date": "2026-01-06", + "status": "verified_empty", + }, } market_root = root / ".local" / "market-data" batch = import_batch(csv_path=fixture, manifest=manifest, root=market_root) @@ -149,6 +193,63 @@ def fake_run(command: list[str], **kwargs): return fake_run +def test_required_output_accepts_valid_parquet_and_rejects_invalid_bytes( + tmp_path: Path, +) -> None: + path = tmp_path / "analysis.parquet" + pq.write_table(pa.table({"date": ["2026-01-05"], "value": [1.0]}), path) + spec = OutputSpec(path=path.name, format="parquet") + + evidence = collect_output_evidence(tmp_path, (spec,)) + + assert evidence[0]["format"] == "parquet" + path.write_bytes(b"not parquet") + with pytest.raises(Exception, match="Parquet"): + collect_output_evidence(tmp_path, (spec,)) + + +def test_optional_benchmark_input_is_frozen_and_passed_to_project( + repo_root: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_root, config_path, config = _build_repo(tmp_path, repo_root) + benchmark = fake_root / ".local/market-data/benchmarks/input.parquet" + benchmark.parent.mkdir(parents=True) + pq.write_table(pa.table({"date": ["2026-01-05"], "return": [0.0]}), benchmark) + config["benchmark_input"] = benchmark.relative_to(fake_root).as_posix() + _write_json(config_path, config) + + def assert_invocation(command: list[str], _: dict[str, object]) -> None: + argument = Path(command[command.index("--benchmark-input") + 1]) + assert argument.is_file() + assert ".inputs" in argument.as_posix() + assert argument.read_bytes() == benchmark.read_bytes() + + monkeypatch.setattr(subprocess, "run", _successful_process(assert_invocation)) + + result = run_project(config_path, repo_root=fake_root) + + assert result.status == "complete" + + +def test_project_execution_timeout_covers_complete_research_workflow( + repo_root: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_root, config_path, _ = _build_repo(tmp_path, repo_root) + + def assert_invocation(_: list[str], kwargs: dict[str, object]) -> None: + assert kwargs["timeout"] >= 3_600 + + monkeypatch.setattr(subprocess, "run", _successful_process(assert_invocation)) + + result = run_project(config_path, repo_root=fake_root) + + assert result.status == "complete" + + @pytest.mark.parametrize( "mutation", [ @@ -230,8 +331,10 @@ def test_missing_snapshot_and_tampered_snapshot_have_distinct_states( missing_root, missing_config, _ = _build_repo(tmp_path / "missing", repo_root) next((missing_root / ".local/market-data/snapshots").glob("*.json")).unlink() tampered_root, tampered_config, _ = _build_repo(tmp_path / "tampered", repo_root) - market_csv = next((tampered_root / ".local/market-data/batches").rglob("market-data.csv")) - market_csv.write_bytes(market_csv.read_bytes() + b"\n") + market_parquet = next( + (tampered_root / ".local/market-data/batches").rglob("market-data.parquet") + ) + market_parquet.write_bytes(market_parquet.read_bytes() + b"tampered") monkeypatch.setattr( subprocess, "run", @@ -618,8 +721,14 @@ def test_adapter_guard_allows_staging_writes_and_blocks_external_writes( adapter.parent.mkdir(parents=True) adapter.write_text( "from pathlib import Path\n" + "import os\n" "import sys\n" + "with open(os.devnull, 'r+b'):\n" + " pass\n" "output = Path(sys.argv[1])\n" + "cache = Path(os.environ['NUMBA_CACHE_DIR'])\n" + "cache.mkdir(parents=True, exist_ok=True)\n" + "(cache / 'compiled.bin').write_bytes(b'cache')\n" "(output / 'inside.txt').write_text('inside', encoding='utf-8')\n" "if len(sys.argv) > 3 and sys.argv[2] == 'write':\n" " Path(sys.argv[3]).write_text('escaped', encoding='utf-8')\n" @@ -673,6 +782,7 @@ def test_adapter_guard_allows_staging_writes_and_blocks_external_writes( assert allowed.returncode == 0, allowed.stderr assert (output_dir / "inside.txt").read_text(encoding="utf-8") == "inside" + assert not (output_dir / ".runtime-cache").exists() assert blocked.returncode != 0 assert not escaped.exists() diff --git a/tests/local_quant_research/test_skill_contract.py b/tests/local_quant_research/test_skill_contract.py index cab4614..2e647e1 100644 --- a/tests/local_quant_research/test_skill_contract.py +++ b/tests/local_quant_research/test_skill_contract.py @@ -24,9 +24,16 @@ def test_local_research_skill_is_thin_and_strategy_agnostic( "---\nname: run-local-quant-research\ndescription: Use when " ) assert text.count(PUBLIC_COMMAND) == 1 - assert all(status in text for status in ("complete", "evidence_insufficient", "failed")) + assert all( + status in text for status in ("complete", "evidence_insufficient", "failed") + ) for required in ( "snapshot_id", + "market-data.parquet", + "DuckDB（嵌入式分析数据库）", + "单场景", + "完整报告", + "return_to_caller", "必需输出", "正式回测", "JoinQuant（聚宽）", @@ -44,16 +51,20 @@ def test_local_research_skill_has_one_fixed_orchestration_order( text = _skill_text(repo_root) stages = [ "校验行情快照", - "校验项目配置", + "校验单场景配置", "运行项目入口", - "校验必需输出", + "校验单场景结果", "固化运行证据", + "返回调用者", ] positions = [text.index(stage) for stage in stages] assert positions == sorted(positions) assert "执行前缺少身份、快照、范围或声明输入" in text assert "既有证据被篡改或摘要不一致" in text + assert "复数场景由主 agent（代理）多次调用" in text + assert "不在 Skill 内聚合" in text + assert "Vibe-Trading（AI 研究助理）" in text def test_local_research_skill_ui_metadata_matches_public_entry( diff --git a/tests/local_quant_research/test_turtle_allocation.py b/tests/local_quant_research/test_turtle_allocation.py deleted file mode 100644 index ff61dd3..0000000 --- a/tests/local_quant_research/test_turtle_allocation.py +++ /dev/null @@ -1,325 +0,0 @@ -from __future__ import annotations - -import itertools -import sys -from decimal import Decimal -from pathlib import Path - - -RESEARCH_ROOT = ( - Path(__file__).resolve().parents[2] - / "joinquant" - / "strategies" - / "strategy-003" - / "research" -) -sys.path.insert(0, str(RESEARCH_ROOT)) - -from turtle_etf.allocation import ( # noqa: E402 - BuyRequest, - PortfolioConstraints, - allocate_a1, -) -from turtle_etf.risk import ( # noqa: E402 - CovarianceEstimate, - PortfolioState, - RiskInputs, - evaluate_risk, -) -from turtle_etf.state import Batch, OrderIntent, TrendState # noqa: E402 - - -def _intent( - security: str, - *, - quantity: int, - group: str = "group-a", - price: str = "10", -) -> OrderIntent: - return OrderIntent( - security=security, - asset_group=group, - action="entry", - quantity=quantity, - expected_price=Decimal(price), - signal_date="2026-01-05", - execution_date="2026-01-06", - signal_n=Decimal("0.05"), - standard_unit=quantity, - common_stop_after=Decimal(price) - Decimal("0.10"), - reason="entry_breakout", - ) - - -def _inputs( - securities: tuple[str, ...], - *, - group_value_cap: str = "0.50", -) -> RiskInputs: - covariance = CovarianceEstimate( - securities=tuple(sorted(securities)), - matrix=tuple( - tuple( - Decimal("0.000001") if left == right else Decimal("0") - for right in range(len(securities)) - ) - for left in range(len(securities)) - ), - aligned_samples=60, - window_days=60, - ) - return RiskInputs( - prices={security: Decimal("10") for security in securities}, - median_turnover_20d={ - security: Decimal("1000000000") for security in securities - }, - covariance=covariance, - security_risk_cap=Decimal("1"), - security_value_cap=Decimal("1"), - asset_group_risk_cap=Decimal("1"), - asset_group_value_cap=Decimal(group_value_cap), - portfolio_risk_cap=Decimal("1"), - portfolio_value_cap=Decimal("1"), - target_volatility=Decimal("1"), - ) - - -def test_a1_uses_common_completion_then_fractional_lot_remainder() -> None: - requests = ( - BuyRequest(_intent("B", quantity=600)), - BuyRequest(_intent("A", quantity=1000)), - ) - constraints = PortfolioConstraints( - state=PortfolioState(Decimal("100000"), Decimal("10010")), - risk_inputs=_inputs(("A", "B")), - ) - - result = allocate_a1(requests, constraints) - - assert dict(result.quantities) == {"A": 600, "B": 400} - assert result.completion_ratios["A"] == Decimal("0.6") - assert result.completion_ratios["B"] == Decimal("0.6666666666666666666666666667") - decision = evaluate_risk(result.allocations, constraints.state, constraints.risk_inputs) - assert decision.reason_codes == () - assert decision.approved == result.allocations - - -def test_a1_releases_group_limited_budget_to_other_candidates() -> None: - requests = tuple( - BuyRequest(_intent(security, quantity=1000, group=group)) - for security, group in (("A", "shared"), ("B", "shared"), ("C", "other")) - ) - constraints = PortfolioConstraints( - state=PortfolioState(Decimal("100000"), Decimal("30000")), - risk_inputs=_inputs(("A", "B", "C"), group_value_cap="0.10"), - ) - - expected = {"A": 500, "B": 500, "C": 1000} - digests: set[str] = set() - for permutation in itertools.permutations(requests): - result = allocate_a1(permutation, constraints) - assert dict(result.quantities) == expected - assert result.remaining_cash == Decimal("9985") - digests.add(result.audit_sha256) - - assert len(digests) == 1 - - -def test_a1_exact_remainder_tie_uses_security_code() -> None: - requests = ( - BuyRequest(_intent("B", quantity=1000)), - BuyRequest(_intent("A", quantity=1000)), - ) - constraints = PortfolioConstraints( - state=PortfolioState(Decimal("100000"), Decimal("1005")), - risk_inputs=_inputs(("A", "B")), - ) - - result = allocate_a1(requests, constraints) - - assert dict(result.quantities) == {"A": 100, "B": 0} - assert result.allocations[0].security == "A" - - -def test_a1_first_lot_uses_largest_fractional_remainder_not_security_code() -> None: - requests = ( - BuyRequest(_intent("A", quantity=100)), - BuyRequest(_intent("B", quantity=200)), - ) - constraints = PortfolioConstraints( - state=PortfolioState(Decimal("100000"), Decimal("1005")), - risk_inputs=_inputs(("A", "B")), - ) - - result = allocate_a1(requests, constraints) - - assert dict(result.quantities) == {"A": 0, "B": 100} - assert result.allocations[0].security == "B" - - -def test_a1_floors_common_scale_before_code_tied_remainder() -> None: - requests = ( - BuyRequest(_intent("A", quantity=100)), - BuyRequest(_intent("B", quantity=300)), - ) - constraints = PortfolioConstraints( - state=PortfolioState(Decimal("100000"), Decimal("2010")), - risk_inputs=_inputs(("A", "B")), - ) - - result = allocate_a1(requests, constraints) - - assert dict(result.quantities) == {"A": 100, "B": 100} - - -def test_a1_infeasible_candidate_does_not_block_other_budget() -> None: - requests = ( - BuyRequest(_intent("A", quantity=1000)), - BuyRequest(_intent("B", quantity=1000)), - ) - inputs = _inputs(("A", "B")) - inputs = RiskInputs( - **{ - **inputs.__dict__, - "median_turnover_20d": { - "A": Decimal("1"), - "B": Decimal("1000000000"), - }, - } - ) - constraints = PortfolioConstraints( - state=PortfolioState(Decimal("100000"), Decimal("10005")), - risk_inputs=inputs, - ) - - result = allocate_a1(requests, constraints) - - assert dict(result.quantities) == {"A": 0, "B": 1000} - assert result.rejected == (requests[0],) - - -def test_a1_can_accept_diversifying_pair_when_each_single_lot_exceeds_target_volatility() -> None: - requests = ( - BuyRequest(_intent("A", quantity=100)), - BuyRequest(_intent("B", quantity=100)), - ) - covariance = CovarianceEstimate( - securities=("A", "B"), - matrix=( - (Decimal("0.01"), Decimal("-0.01")), - (Decimal("-0.01"), Decimal("0.01")), - ), - aligned_samples=60, - window_days=60, - ) - inputs = RiskInputs( - prices={"A": Decimal("10"), "B": Decimal("10")}, - median_turnover_20d={ - "A": Decimal("1000000000"), - "B": Decimal("1000000000"), - }, - covariance=covariance, - security_risk_cap=Decimal("1"), - security_value_cap=Decimal("1"), - asset_group_risk_cap=Decimal("1"), - asset_group_value_cap=Decimal("1"), - portfolio_risk_cap=Decimal("1"), - portfolio_value_cap=Decimal("1"), - target_volatility=Decimal("0.1"), - ) - constraints = PortfolioConstraints( - state=PortfolioState(Decimal("10000"), Decimal("3000")), - risk_inputs=inputs, - ) - - for request in requests: - decision = evaluate_risk( - (request.intent,), - constraints.state, - constraints.risk_inputs, - ) - assert decision.reason_codes == ("target_volatility",) - pair_decision = evaluate_risk( - tuple(request.intent for request in requests), - constraints.state, - constraints.risk_inputs, - ) - assert pair_decision.reason_codes == () - - result = allocate_a1(requests, constraints) - - assert dict(result.quantities) == {"A": 100, "B": 100} - - -def test_a1_releases_hard_block_before_rechecking_joint_stop_risk_reduction() -> None: - positions = tuple( - TrendState( - security=security, - asset_group="shared", - signal_n=Decimal("1"), - standard_unit=100, - initial_fill_price=Decimal("12"), - batches=(Batch("2026-01-02", 100, Decimal("12")),), - common_stop=Decimal("10"), - ) - for security in ("A", "B") - ) - requests = ( - *( - BuyRequest( - OrderIntent( - security=security, - asset_group="shared", - action="addition", - quantity=100, - expected_price=Decimal("12"), - signal_date="2026-01-05", - execution_date="2026-01-06", - signal_n=Decimal("1"), - standard_unit=100, - common_stop_after=Decimal("11.5"), - reason="addition_breakout", - ) - ) - for security in ("A", "B") - ), - BuyRequest(_intent("C", quantity=200, group="other")), - ) - inputs = _inputs(("A", "B", "C"), group_value_cap="1") - inputs = RiskInputs( - **{ - **inputs.__dict__, - "median_turnover_20d": { - "A": Decimal("1000000000"), - "B": Decimal("1000000000"), - "C": Decimal("1"), - }, - "asset_group_risk_cap": Decimal("0.025"), - } - ) - constraints = PortfolioConstraints( - state=PortfolioState( - equity=Decimal("10000"), - cash=Decimal("10000"), - positions=positions, - ), - risk_inputs=inputs, - ) - - for request in requests[:2]: - decision = evaluate_risk( - (request.intent,), - constraints.state, - constraints.risk_inputs, - ) - assert decision.reason_codes == ("group_risk_cap",) - pair_decision = evaluate_risk( - tuple(request.intent for request in requests[:2]), - constraints.state, - constraints.risk_inputs, - ) - assert pair_decision.reason_codes == () - - result = allocate_a1(requests, constraints) - - assert dict(result.quantities) == {"A": 100, "B": 100, "C": 0} diff --git a/tests/local_quant_research/test_turtle_e2e.py b/tests/local_quant_research/test_turtle_e2e.py index 6c799a5..0503c91 100644 --- a/tests/local_quant_research/test_turtle_e2e.py +++ b/tests/local_quant_research/test_turtle_e2e.py @@ -5,648 +5,240 @@ import json import shutil import subprocess -import sys import uuid -from dataclasses import replace -from datetime import date, timedelta -from decimal import Decimal from pathlib import Path -import pytest +import pandas as pd +import pyarrow.parquet as pq +from scripts.research.market_data.contracts import SnapshotSelection +from scripts.research.market_data.query import MARKET_DATA_FIELDS +from scripts.research.market_data.storage import create_snapshot, import_batch -RESEARCH_ROOT = ( - Path(__file__).resolve().parents[2] - / "joinquant" - / "strategies" - / "strategy-003" - / "research" -) -sys.path.insert(0, str(RESEARCH_ROOT)) -from turtle_etf.execution import ( # noqa: E402 - DailyMarket, - MarketQuote, - TradingDay, - process_day, -) -from turtle_etf.cli import run_research # noqa: E402 -from turtle_etf.reporting import ( # noqa: E402 - OutputValidationError, - RunIdentity, - validate_project_outputs, -) -from turtle_etf.risk import ( # noqa: E402 - CovarianceEstimate, - PortfolioState, - RiskInputs, -) -from turtle_etf.state import ( # noqa: E402 - OrderIntent, - apply_entry_fill, -) -from scripts.research.market_data.contracts import SnapshotSelection # noqa: E402 -from scripts.research.market_data.query import MARKET_DATA_FIELDS # noqa: E402 -from scripts.research.market_data.storage import create_snapshot, import_batch # noqa: E402 - - -def _sell( - security: str, - action: str, - quantity: int, - *, - price: str = "10", -) -> OrderIntent: - return OrderIntent( - security=security, - asset_group="group-a", - action=action, - quantity=quantity, - expected_price=Decimal(price), - signal_date="2026-01-05", - execution_date="2026-01-06", - reason="test_sell", - ) - - -def _buy( - security: str, - action: str = "entry", +def _write_market_csv( + path: Path, *, - price: str = "10", - signal_n: str = "1", -) -> OrderIntent: - return OrderIntent( - security=security, - asset_group="group-a", - action=action, - quantity=100, - expected_price=Decimal(price), - signal_date="2026-01-05", - execution_date="2026-01-06", - signal_n=Decimal(signal_n), - standard_unit=100, - common_stop_after=Decimal(price) - Decimal("2") * Decimal(signal_n), - reason="test_buy", - ) - - -def _position(security: str, quantity: int, *, signal_n: str = "1"): - return apply_entry_fill( - security=security, - asset_group="group-a", - execution_date="2026-01-02", - fill_price=Decimal("10"), - quantity=quantity, - signal_n=Decimal(signal_n), - standard_unit=100, - ) - - -def _risk_inputs(securities: tuple[str, ...]) -> RiskInputs: - ordered = tuple(sorted(securities)) - covariance = CovarianceEstimate( - securities=ordered, - matrix=tuple( - tuple( - Decimal("0.000001") if left == right else Decimal("0") - for right in range(len(ordered)) - ) - for left in range(len(ordered)) - ), - aligned_samples=60, - window_days=60, - ) - return RiskInputs( - prices={security: Decimal("10") for security in securities}, - median_turnover_20d={ - security: Decimal("1000000000") for security in securities - }, - covariance=covariance, - security_risk_cap=Decimal("1"), - security_value_cap=Decimal("1"), - asset_group_risk_cap=Decimal("1"), - asset_group_value_cap=Decimal("1"), - portfolio_risk_cap=Decimal("1"), - portfolio_value_cap=Decimal("1"), - target_volatility=Decimal("1"), - ) - - -def test_day_flow_executes_exit_reduction_then_same_level_buys_at_actual_open() -> None: - exit_position = _position("EXIT", 200) - reduce_position = _position("RED", 400) - add_position = replace( - _position("ADD", 100, signal_n="0.5"), - last_add_request_date="2026-01-05", - ) - state = PortfolioState( - equity=Decimal("100000"), - cash=Decimal("93000"), - positions=(add_position, exit_position, reduce_position), - ) - day = TradingDay( - date="2026-01-06", - intents=( - _buy("NEW"), - _buy("EXIT", action="addition"), - _sell("RED", "mandatory_risk_reduction", 100), - _buy("ADD", action="addition", signal_n="0.5"), - _sell("EXIT", "full_exit", 200), - ), - ) - market = DailyMarket( - quotes={ - "EXIT": MarketQuote(open=Decimal("11")), - "RED": MarketQuote(open=Decimal("9")), - "ADD": MarketQuote(open=Decimal("12")), - "NEW": MarketQuote(open=Decimal("8")), - }, - risk_inputs=_risk_inputs(("EXIT", "RED", "ADD", "NEW")), - ) - - first = process_day(day, state, market) - second = process_day(day, state, market) - - filled_actions = [record.action for record in first.audit if record.status == "filled"] - assert filled_actions == [ - "full_exit", - "mandatory_risk_reduction", - "addition", - "entry", - ] - assert any( - record.security == "EXIT" - and record.action == "addition" - and record.status == "cancelled" - for record in first.audit - ) - positions = {position.security: position for position in first.portfolio.positions} - assert set(positions) == {"ADD", "NEW", "RED"} - assert positions["RED"].quantity == 300 - assert positions["ADD"].quantity == 200 - assert positions["ADD"].common_stop == Decimal("11.0") - assert positions["NEW"].common_stop == Decimal("6") - assert first.portfolio.cash == Decimal("94080") - assert first.audit_sha256 == second.audit_sha256 - - -def test_paused_limits_and_missing_open_never_create_fills() -> None: - held = _position("LOW", 100) - state = PortfolioState( - equity=Decimal("100000"), - cash=Decimal("99000"), - positions=(held,), - ) - day = TradingDay( - date="2026-01-06", - intents=( - _buy("PAUSED"), - _buy("HIGH"), - _buy("MISSING"), - _sell("LOW", "full_exit", 100), - ), - ) - market = DailyMarket( - quotes={ - "PAUSED": MarketQuote(open=Decimal("10"), paused=True), - "HIGH": MarketQuote(open=Decimal("10"), high_limit=Decimal("10")), - "MISSING": MarketQuote(open=None), - "LOW": MarketQuote(open=Decimal("8"), low_limit=Decimal("8")), - }, - risk_inputs=_risk_inputs(("PAUSED", "HIGH", "MISSING", "LOW")), - ) - - result = process_day(day, state, market) - - assert all(record.status == "unfilled" for record in result.audit) - assert result.portfolio == state - assert result.allocation.allocations == () - - -def test_gap_open_recomputes_commission_before_cash_gate() -> None: - intent = OrderIntent( - security="GAP", - asset_group="group-a", - action="entry", - quantity=10000, - expected_price=Decimal("1"), - signal_date="2026-01-05", - execution_date="2026-01-06", - signal_n=Decimal("0.1"), - standard_unit=10000, - common_stop_after=Decimal("0.8"), - estimated_fee=Decimal("5"), - reason="entry_breakout", - ) - state = PortfolioState(Decimal("100006"), Decimal("100006")) - market = DailyMarket( - quotes={"GAP": MarketQuote(open=Decimal("10"))}, - risk_inputs=_risk_inputs(("GAP",)), - ) - - result = process_day( - TradingDay(date="2026-01-06", intents=(intent,)), - state, - market, - ) - - assert result.portfolio.cash == Decimal("997.585000") - assert result.portfolio.cash >= 0 - assert result.audit[0].status == "filled" - assert result.audit[0].filled_quantity == 9900 - assert result.allocation.allocations[0].estimated_fee == Decimal("8.415000") - - -def test_gap_exit_recomputes_commission_from_actual_quantity_and_open() -> None: - position = _position("GAP", 10000) - state = PortfolioState( - equity=Decimal("100000"), - cash=Decimal("0"), - positions=(position,), - ) - day = TradingDay( - date="2026-01-06", - intents=(_sell("GAP", "full_exit", 10000, price="1"),), - ) - market = DailyMarket( - quotes={"GAP": MarketQuote(open=Decimal("10"))}, - risk_inputs=_risk_inputs(("GAP",)), - ) - - result = process_day(day, state, market) - - assert result.portfolio.cash == Decimal("99991.500000") - assert result.portfolio.positions == () - assert result.audit[0].filled_quantity == 10000 - - -UNIVERSE = ( - "510300.XSHG", - "512100.XSHG", - "512480.XSHG", - "159819.XSHE", - "516160.XSHG", - "513100.XSHG", - "513180.XSHG", - "515180.XSHG", - "516080.XSHG", - "518880.XSHG", - "511010.XSHG", -) - - -def _remove_empty_test_roots(repo_root: Path, market_root: Path) -> None: - for path in (market_root / "snapshots", market_root / "batches"): - try: - path.rmdir() - except OSError: - pass - if not (market_root / "snapshots").exists() and not (market_root / "batches").exists(): - try: - (market_root / ".market-data.lock").unlink(missing_ok=True) - market_root.rmdir() - except OSError: - pass - for path in ( - repo_root / ".local/e2e-tests", - repo_root / ".local/quant-research", - ): - try: - path.rmdir() - except OSError: - pass - - -def _business_dates(start: date, count: int) -> list[str]: - values: list[str] = [] - current = start - while len(values) < count: - if current.weekday() < 5: - values.append(current.isoformat()) - current += timedelta(days=1) - return values - - -def _research_snapshot( - tmp_path: Path, - *, - market_root: Path | None = None, - export_code_sha256: str = "a" * 64, - start_date: date = date(2025, 1, 2), - cold_security_rows: int = 80, -): - dates = _business_dates(start_date, 90) - csv_path = tmp_path / "daily.csv" - with csv_path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=MARKET_DATA_FIELDS, lineterminator="\n") + securities: tuple[str, ...], + dates: pd.DatetimeIndex, +) -> None: + breakout_rows = { + securities[0]: 55, + securities[1]: 60, + } + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, fieldnames=MARKET_DATA_FIELDS, lineterminator="\n" + ) writer.writeheader() - previous = {security: Decimal("10") for security in UNIVERSE} - for index, row_date in enumerate(dates): - for security in UNIVERSE: - if ( - security == "516080.XSHG" - and index < len(dates) - cold_security_rows - ): - continue - close = Decimal("10") - high = Decimal("11") - low = Decimal("9") - open_price = Decimal("10") - if security == "510300.XSHG" and index == 70: - close = high = Decimal("12") - elif security == "510300.XSHG" and index == 71: - open_price = close = high = Decimal("12") - low = Decimal("11") - elif security == "510300.XSHG" and index == 76: - open_price = close = low = Decimal("8") - high = Decimal("9") + for security in securities: + previous_close = 10.0 + breakout_row = breakout_rows.get(security) + for index, date in enumerate(dates): + close = 11.0 if breakout_row is not None and index >= breakout_row else 10.0 writer.writerow( { - "date": row_date, + "date": date.date().isoformat(), "security": security, - "open": str(open_price), - "high": str(high), - "low": str(low), - "close": str(close), - "pre_close": str(previous[security]), - "volume": "100000000", - "money": "1000000000", + "open": f"{close:.2f}", + "high": f"{close + 0.20:.2f}", + "low": f"{close - 0.20:.2f}", + "close": f"{close:.2f}", + "pre_close": f"{previous_close:.2f}", + "volume": "1000000", + "money": f"{close * 1000000:.2f}", "factor": "1", "paused": "0", - "high_limit": "20", - "low_limit": "1", + "high_limit": f"{close * 1.10:.2f}", + "low_limit": f"{close * 0.90:.2f}", } ) - previous[security] = close - market_root = tmp_path / "market-data" if market_root is None else market_root - batch = import_batch( - csv_path=csv_path, - manifest={ - "schema_version": 1, - "source": {"name": "joinquant", "environment": "research"}, - "asset_type": "etf", - "frequency": "1d", - "fields": list(MARKET_DATA_FIELDS), - "price_semantics": {"fq": None, "skip_paused": False}, - "export_code_sha256": export_code_sha256, - }, - root=market_root, - ) - selection = SnapshotSelection( - source={"name": "joinquant", "environment": "research"}, - asset_type="etf", - frequency="1d", - securities=UNIVERSE, - start_date=dates[0], - end_date=dates[-1], - fields=MARKET_DATA_FIELDS, - price_semantics={"fq": None, "skip_paused": False}, - ) - snapshot = create_snapshot( - batch_ids=(batch.batch_id,), - selection=selection, - root=market_root, - ) - return snapshot, market_root - - -def _run_identity(snapshot_id: str) -> RunIdentity: - return RunIdentity( - run_id="1" * 64, - snapshot_id=snapshot_id, - code_sha256="2" * 64, - config_sha256="3" * 64, - ) + previous_close = close -def test_project_research_writes_reports_conclusion_candidates_and_audits( +def test_turtle_project_completes_full_single_scenario_entrypoint( tmp_path: Path, repo_root: Path, ) -> None: - snapshot, market_root = _research_snapshot(tmp_path) - output_dir = tmp_path / "output" - config_path = ( - repo_root / "joinquant/strategies/strategy-003/research/baseline.json" + token = uuid.uuid4().hex + research_root = repo_root / "joinquant/strategies/strategy-003/research" + baseline = json.loads( + (research_root / "baseline.json").read_text(encoding="utf-8") ) - identity = _run_identity(snapshot.snapshot_id) - - result = run_research( - config_path, - snapshot.path, - output_dir, - market_data_root=market_root, - identity=identity, + securities = tuple( + sorted(str(item["security"]) for item in baseline["universe"]) ) - - assert result.status == "complete" - assert {path.name for path in output_dir.iterdir()} == { - "project-status.json", - "daily-audit.csv", - "trades.csv", - "positions.csv", - "risk.csv", - "research-report.md", - "conclusion.json", - "candidate-strategies.json", - } - conclusion = json.loads((output_dir / "conclusion.json").read_text(encoding="utf-8")) - candidates = json.loads( - (output_dir / "candidate-strategies.json").read_text(encoding="utf-8") - ) - report = (output_dir / "research-report.md").read_text(encoding="utf-8") - assert conclusion["identity"] == identity.to_document() - assert conclusion["metrics"]["filled_trades"] >= 2 - assert conclusion["recommendation"] in { - "proceed_to_joinquant", - "revise_and_reassess", - "stop_evidence_insufficient", - } - assert len(candidates["candidates"]) == 7 - assert {item["code_sha256"] for item in candidates["candidates"]} == { - identity.code_sha256 - } - assert {item["snapshot_id"] for item in candidates["candidates"]} == { - snapshot.snapshot_id - } - assert all("rank" not in item and "score" not in item for item in candidates["candidates"]) - with (output_dir / "trades.csv").open(encoding="utf-8", newline="") as handle: - actions = {row["action"] for row in csv.DictReader(handle)} - assert {"entry", "full_exit"}.issubset(actions) - for required in ( - "方法", - "输入身份", - "事件与交易", - "实际仓位分布", - "现金占比", - "留现原因", - "资产组风险使用率", - "组合风险使用率", - "限制", - "产物摘要", - "不是正式回测或最终验收结论", - "Vibe-Trading（AI 研究助理）组合优化器：已跳过", - ): - assert required in report - assert "63.7%" not in report - assert "55.7%" not in report - validate_project_outputs(output_dir, identity) - - -def test_cold_security_does_not_block_other_securities_or_the_project( - tmp_path: Path, - repo_root: Path, -) -> None: - snapshot, market_root = _research_snapshot(tmp_path, cold_security_rows=10) - output_dir = tmp_path / "cold-output" - identity = _run_identity(snapshot.snapshot_id) - - result = run_research( - repo_root / "joinquant/strategies/strategy-003/research/baseline.json", - snapshot.path, - output_dir, - market_data_root=market_root, - identity=identity, - ) - - assert result.status == "complete" - with (output_dir / "trades.csv").open(encoding="utf-8", newline="") as handle: - trades = list(csv.DictReader(handle)) - assert any(row["security"] != "516080.XSHG" for row in trades) - assert all(row["security"] != "516080.XSHG" for row in trades) - with (output_dir / "risk.csv").open(encoding="utf-8", newline="") as handle: - final_risk = list(csv.DictReader(handle))[-1] - assert "516080.XSHG" in json.loads(final_risk["cold_start_securities"]) - - -@pytest.mark.parametrize("mutation", ["missing", "invalid-json", "identity-mismatch"]) -def test_three_required_outputs_reject_missing_invalid_or_mismatched_evidence( - mutation: str, - tmp_path: Path, - repo_root: Path, -) -> None: - snapshot, market_root = _research_snapshot(tmp_path) - output_dir = tmp_path / "output" - identity = _run_identity(snapshot.snapshot_id) - run_research( - repo_root / "joinquant/strategies/strategy-003/research/baseline.json", - snapshot.path, - output_dir, - market_data_root=market_root, - identity=identity, - ) - if mutation == "missing": - (output_dir / "research-report.md").unlink() - elif mutation == "invalid-json": - (output_dir / "conclusion.json").write_text("[]\n", encoding="utf-8") - else: - path = output_dir / "candidate-strategies.json" - document = json.loads(path.read_text(encoding="utf-8")) - document["identity"]["run_id"] = "9" * 64 - path.write_text(json.dumps(document) + "\n", encoding="utf-8") - - with pytest.raises(OutputValidationError): - validate_project_outputs(output_dir, identity) - - -def test_project_run_config_references_snapshot_and_disables_biased_optimizer( - repo_root: Path, -) -> None: - research_root = repo_root / "joinquant/strategies/strategy-003/research" - run_config = json.loads((research_root / "project-run.json").read_text(encoding="utf-8")) - baseline = json.loads((research_root / "baseline.json").read_text(encoding="utf-8")) - - assert len(run_config["snapshot_id"]) == 64 - assert run_config["project_entry"].endswith("/turtle_etf/cli.py") - assert run_config["project_config"].endswith("/baseline.json") - assert all(not path.lower().endswith(".csv") for path in run_config["declared_inputs"]) - assert baseline["research"]["vibe_optimizer"]["enabled"] is False - assert baseline["research"]["vibe_optimizer"]["reason"] - - -def test_skill_public_command_runs_complete_turtle_workflow( - tmp_path: Path, - repo_root: Path, -) -> None: - token = uuid.uuid4().hex - project_id = f"strategy-003-e2e-{token[:12]}" - temp_project = repo_root / ".local/e2e-tests" / token + dates = pd.bdate_range("2030-01-02", periods=70) market_root = repo_root / ".local/market-data" - output_project = repo_root / ".local/quant-research" / project_id - export_digest = hashlib.sha256(token.encode("ascii")).hexdigest() + project_root = repo_root / ".local/e2e-tests" / token snapshot = None batch_ids: list[str] = [] + run_output: Path | None = None try: - snapshot, _ = _research_snapshot( - tmp_path, - market_root=market_root, - export_code_sha256=export_digest, - start_date=date(2099, 1, 2), + source = tmp_path / "turtle-e2e.csv" + _write_market_csv(source, securities=securities, dates=dates) + batch = import_batch( + csv_path=source, + manifest={ + "schema_version": 1, + "source": {"name": "joinquant", "environment": "research"}, + "asset_type": "etf", + "frequency": "1d", + "fields": list(MARKET_DATA_FIELDS), + "price_semantics": {"fq": None, "skip_paused": False}, + "export_code_sha256": hashlib.sha256( + token.encode("ascii") + ).hexdigest(), + "corporate_actions": { + "source": { + "name": "joinquant", + "dataset": "finance.FUND_DIVIDEND", + }, + "knowledge_cutoff_date": dates[-1].date().isoformat(), + "status": "verified_empty", + }, + }, + root=market_root, ) - snapshot_document = json.loads(snapshot.path.read_text(encoding="utf-8")) - batch_ids = list(snapshot_document["batch_ids"]) - run_config = json.loads( - ( - repo_root - / "joinquant/strategies/strategy-003/research/project-run.json" - ).read_text(encoding="utf-8") + source.unlink() + selection = SnapshotSelection( + source={"name": "joinquant", "environment": "research"}, + asset_type="etf", + frequency="1d", + securities=securities, + start_date=dates[0].date().isoformat(), + end_date=dates[-1].date().isoformat(), + fields=MARKET_DATA_FIELDS, + price_semantics={"fq": None, "skip_paused": False}, ) - run_config.update( - project_id=project_id, - snapshot_id=snapshot.snapshot_id, - snapshot_requirements=snapshot_document["selection"], + snapshot = create_snapshot( + batch_ids=(batch.batch_id,), selection=selection, root=market_root ) - temp_project.mkdir(parents=True) - config_path = temp_project / "run.json" + snapshot_document = json.loads(snapshot.path.read_text(encoding="utf-8")) + batch_ids = list(snapshot_document["batch_ids"]) + + project_root.mkdir(parents=True) + config = json.loads(json.dumps(baseline)) + config["risk"]["portfolio_unit_cap"] = 1.0 + config_path = project_root / "baseline.json" config_path.write_text( - json.dumps(run_config, ensure_ascii=False, sort_keys=True) + "\n", + json.dumps(config, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8", ) - skill = ( - repo_root / ".agents/skills/run-local-quant-research/SKILL.md" - ).read_text(encoding="utf-8") - command_line = next( - line.strip() - for line in skill.splitlines() - if "local_quant_research\\cli.py run --config " in line + run_config = { + "schema_version": 1, + "project_id": "strategy-003", + "snapshot_id": snapshot.snapshot_id, + "snapshot_requirements": snapshot_document["selection"], + "project_entry": ( + "joinquant/strategies/strategy-003/research/" + "turtle_etf/vectorbt_cli.py" + ), + "command": [ + ".venv/Scripts/python.exe", + ( + "joinquant/strategies/strategy-003/research/" + "turtle_etf/vectorbt_cli.py" + ), + ], + "project_config": config_path.relative_to(repo_root).as_posix(), + "code_identity": ( + "joinquant/strategies/strategy-003/research/code-identity.json" + ), + "declared_inputs": [ + "joinquant/strategies/strategy-003/manifest.json" + ], + "required_outputs": [ + {"path": "backtests/local-baseline", "format": "directory"} + ], + "output_root": ".local/quant-research", + "stop_states": ["complete", "evidence_insufficient", "failed"], + } + run_path = project_root / "run.json" + run_path.write_text( + json.dumps(run_config, sort_keys=True) + "\n", encoding="utf-8" ) - relative_config = config_path.relative_to(repo_root).as_posix() - command = command_line.replace("", relative_config).split() completed = subprocess.run( - command, + [ + str(repo_root / ".venv/Scripts/python.exe"), + str(repo_root / "scripts/research/local_quant_research/cli.py"), + "run", + "--config", + str(run_path.relative_to(repo_root)), + ], cwd=repo_root, capture_output=True, text=True, shell=False, check=False, - timeout=120, + timeout=300, ) assert completed.returncode == 0, completed.stderr + completed.stdout - result = json.loads(completed.stdout) - assert result["status"] == "complete" - run_path = Path(result["run_path"]) - assert run_path.parent == output_project + outcome = json.loads(completed.stdout) + assert outcome["status"] == "complete" + run_output = Path(outcome["run_path"]) + status = json.loads( + (run_output / "project-status.json").read_text(encoding="utf-8") + ) + assert status["next_action"] == "return_to_caller" + + result_root = run_output / "backtests/local-baseline" manifest = json.loads( - (run_path / "run-manifest.json").read_text(encoding="utf-8") + (result_root / "manifest.json").read_text(encoding="utf-8") ) - validate_project_outputs( - run_path, - RunIdentity( - run_id=result["run_id"], - snapshot_id=snapshot.snapshot_id, - code_sha256=manifest["inputs"]["code_sha256"], - config_sha256=manifest["inputs"]["config_sha256"], - ), + assert set(manifest["datasets"]) == { + "results", + "balances", + "positions", + "orders", + "risk", + "period_risks", + } + performance = json.loads( + (result_root / "performance.json").read_text(encoding="utf-8") + ) + assert performance["result_match"] is True + assert performance["cold_seconds"] < 180.0 + assert performance["warm_seconds"] < 180.0 + assert performance["cleanup"]["verified"] is True + + attribution_ref = manifest["extensions"]["turtle_etf"][ + "attribution_log" + ]["files"][0]["path"] + attribution = pq.read_table(result_root / attribution_ref).to_pandas() + redistributions = attribution.loc[ + (attribution["event_type"] == "decision") + & ( + attribution["reason_code"] + == "full_position_redistribution" + ) + ] + assert not redistributions.empty + details = [ + json.loads(value) for value in redistributions["details_json"] + ] + assert all( + item["redistribution_state_changed"] is False for item in details ) - assert not list(output_project.glob(".*.tmp")) - assert not list(output_project.glob(".*.inputs")) + assert any(float(item["portfolio_scale"]) < 1.0 for item in details) + assert not tuple(result_root.rglob("*.tmp")) + assert not tuple(market_root.rglob("*.duckdb")) finally: - shutil.rmtree(output_project, ignore_errors=True) - shutil.rmtree(temp_project, ignore_errors=True) + if run_output is not None: + shutil.rmtree(run_output, ignore_errors=True) + shutil.rmtree(project_root, ignore_errors=True) if snapshot is not None: snapshot.path.unlink(missing_ok=True) for batch_id in batch_ids: shutil.rmtree(market_root / "batches" / batch_id, ignore_errors=True) - _remove_empty_test_roots(repo_root, market_root) + for path in ( + repo_root / ".local/e2e-tests", + market_root / "snapshots", + market_root / "batches", + ): + try: + path.rmdir() + except OSError: + pass diff --git a/tests/local_quant_research/test_turtle_indicators.py b/tests/local_quant_research/test_turtle_indicators.py index 5252c17..bbb3fc6 100644 --- a/tests/local_quant_research/test_turtle_indicators.py +++ b/tests/local_quant_research/test_turtle_indicators.py @@ -1,7 +1,6 @@ from __future__ import annotations import sys -from decimal import Decimal from pathlib import Path import pandas as pd @@ -18,13 +17,6 @@ sys.path.insert(0, str(RESEARCH_ROOT)) from turtle_etf.indicators import breakout_levels, true_range, turtle_n # noqa: E402 -from turtle_etf.signals import entry_signal, make_entry_intent, trend_exit_signal # noqa: E402 -from turtle_etf.state import ( # noqa: E402 - apply_addition_fill, - apply_entry_fill, - request_addition, - request_full_exit, -) def test_true_range_uses_all_three_unadjusted_price_components() -> None: @@ -90,162 +82,3 @@ def test_breakout_channels_exclude_the_signal_day_high_and_low() -> None: assert levels.loc[20, "exit_low"] == 80.0 assert pd.isna(levels.loc[54, "entry_high"]) assert pd.isna(levels.loc[19, "exit_low"]) - - -def test_close_breakout_is_strict_and_becomes_only_a_next_open_intent() -> None: - assert entry_signal(Decimal("55"), Decimal("55")) is False - assert entry_signal(Decimal("55.01"), Decimal("55")) is True - assert trend_exit_signal(Decimal("20"), Decimal("20")) is False - assert trend_exit_signal(Decimal("19.99"), Decimal("20")) is True - - intent = make_entry_intent( - security="510300.XSHG", - asset_group="china_sync_equity", - signal_date="2026-01-05", - execution_date="2026-01-06", - expected_price=Decimal("10"), - quantity=1200, - signal_n=Decimal("0.5"), - standard_unit=1250, - ) - - assert intent.action == "entry" - assert intent.signal_date == "2026-01-05" - assert intent.execution_date == "2026-01-06" - assert intent.common_stop_after == Decimal("9.0") - - -def test_addition_uses_fixed_levels_one_request_per_day_and_only_fill_moves_state() -> None: - state = apply_entry_fill( - security="510300.XSHG", - asset_group="china_sync_equity", - execution_date="2026-01-06", - fill_price=Decimal("10"), - quantity=500, - signal_n=Decimal("1"), - standard_unit=500, - ) - assert state.common_stop == Decimal("8") - assert state.next_add_level == Decimal("10.5") - - requested, intent = request_addition( - state, - signal_date="2026-01-07", - execution_date="2026-01-08", - close=Decimal("12"), - expected_price=Decimal("11.2"), - ) - - assert intent is not None - assert intent.quantity == 500 - assert requested.batches == state.batches - assert requested.common_stop == state.common_stop - assert requested.next_add_level == Decimal("10.5") - same_day_state, same_day_intent = request_addition( - requested, - signal_date="2026-01-07", - execution_date="2026-01-08", - close=Decimal("12"), - expected_price=Decimal("11.2"), - ) - assert same_day_state == requested - assert same_day_intent is None - - filled = apply_addition_fill( - requested, - intent, - execution_date="2026-01-08", - fill_price=Decimal("11.2"), - quantity=400, - ) - assert len(filled.batches) == 2 - assert filled.common_stop == Decimal("9.2") - assert filled.next_add_level == Decimal("11.0") - - next_requested, next_intent = request_addition( - filled, - signal_date="2026-01-08", - execution_date="2026-01-09", - close=Decimal("11.3"), - expected_price=Decimal("10.8"), - ) - lowered_fill = apply_addition_fill( - next_requested, - next_intent, - execution_date="2026-01-09", - fill_price=Decimal("10.8"), - quantity=300, - ) - assert lowered_fill.common_stop == Decimal("9.2") - - with pytest.raises(ValueError, match="execution date"): - apply_addition_fill( - requested, - intent, - execution_date="2026-01-09", - fill_price=Decimal("11.2"), - quantity=400, - ) - - -def test_unfilled_addition_remains_eligible_next_day() -> None: - state = apply_entry_fill( - security="510300.XSHG", - asset_group="china_sync_equity", - execution_date="2026-01-06", - fill_price=Decimal("10"), - quantity=500, - signal_n=Decimal("1"), - standard_unit=500, - ) - requested, _ = request_addition( - state, - signal_date="2026-01-07", - execution_date="2026-01-08", - close=Decimal("10.6"), - expected_price=Decimal("10.7"), - ) - - next_day_state, next_day_intent = request_addition( - requested, - signal_date="2026-01-08", - execution_date="2026-01-09", - close=Decimal("10.6"), - expected_price=Decimal("10.7"), - ) - - assert next_day_intent is not None - assert next_day_state.next_add_level == Decimal("10.5") - - -def test_protective_stop_and_twenty_day_exit_both_request_full_exit() -> None: - state = apply_entry_fill( - security="510300.XSHG", - asset_group="china_sync_equity", - execution_date="2026-01-06", - fill_price=Decimal("10"), - quantity=500, - signal_n=Decimal("1"), - standard_unit=500, - ) - protective = request_full_exit( - state, - signal_date="2026-01-07", - execution_date="2026-01-08", - close=Decimal("8"), - exit_level=Decimal("7"), - expected_price=Decimal("7.9"), - ) - trend = request_full_exit( - state, - signal_date="2026-01-07", - execution_date="2026-01-08", - close=Decimal("8.5"), - exit_level=Decimal("9"), - expected_price=Decimal("8.4"), - ) - - assert protective.reason == "protective_stop" - assert trend.reason == "trend_exit" - assert protective.action == trend.action == "full_exit" - assert protective.quantity == trend.quantity == 500 diff --git a/tests/local_quant_research/test_turtle_result_adapter.py b/tests/local_quant_research/test_turtle_result_adapter.py new file mode 100644 index 0000000..c7ee033 --- /dev/null +++ b/tests/local_quant_research/test_turtle_result_adapter.py @@ -0,0 +1,823 @@ +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + + +RESEARCH_ROOT = ( + Path(__file__).resolve().parents[2] + / "joinquant" + / "strategies" + / "strategy-003" + / "research" +) +sys.path.insert(0, str(RESEARCH_ROOT)) + +from scripts.research.analysis_data import open_analysis_source # noqa: E402 +from turtle_etf.result_adapter import ( # noqa: E402 + ATTRIBUTION_FIELDS, + ATTRIBUTION_SCHEMA_VERSION, + ResultContractError, + to_joinquant_facts, + validate_turtle_result, + validate_turtle_attribution, + write_local_result, +) +from turtle_etf.vectorbt_callbacks import ( # noqa: E402 + ACTION_ADDITION, + ACTION_ENTRY, + ACTION_FULL_EXIT, + ACTION_REDISTRIBUTION_SELL, + REASON_ENTRY_BREAKOUT, + REASON_FULL_POSITION_REDISTRIBUTION, + REASON_ORDER_REJECTED, + REASON_PROTECTIVE_STOP, + REASON_TREND_EXIT, +) +from turtle_etf.vectorbt_engine import run_vectorbt_simulation # noqa: E402 +from turtle_etf.vectorbt_inputs import SimulationInputs # noqa: E402 + + +class _Portfolio: + def value(self) -> pd.Series: + return pd.Series([9_990.0, 10_180.0]) + + def cash(self) -> pd.Series: + return pd.Series([6_990.0, 9_130.0]) + + +def _readonly(values: object, dtype: str) -> np.ndarray: + result = np.ascontiguousarray(values, dtype=dtype) + result.setflags(write=False) + return result + + +def _simulation() -> tuple[SimpleNamespace, SimpleNamespace]: + inputs = SimpleNamespace( + dates=np.asarray(["2026-01-05", "2026-01-06"], dtype="datetime64[D]"), + securities=("ETF-A", "ETF-B"), + close=np.asarray([[10.0, 20.0], [11.0, 21.0]], dtype=np.float64), + signal_n=np.asarray([[1.0, 1.0], [1.0, 1.0]], dtype=np.float64), + ) + simulation = SimpleNamespace( + initial_cash=10_000.0, + portfolio=_Portfolio(), + action_codes=np.asarray( + [ + [ACTION_ENTRY, ACTION_ENTRY], + [ACTION_FULL_EXIT, ACTION_REDISTRIBUTION_SELL], + ], + dtype=np.int16, + ), + reason_codes=np.asarray( + [ + [REASON_ENTRY_BREAKOUT, REASON_ENTRY_BREAKOUT], + [ + REASON_PROTECTIVE_STOP, + REASON_FULL_POSITION_REDISTRIBUTION, + ], + ], + dtype=np.int16, + ), + requested_quantities=np.asarray([[100, 100], [100, 50]], dtype=np.int64), + planned_quantities=np.asarray([[100, 100], [100, 50]], dtype=np.int64), + filled_quantities=np.asarray([[100, 100], [100, 50]], dtype=np.int64), + fill_prices=np.asarray([[10.0, 20.0], [11.0, 21.0]], dtype=np.float64), + fees=np.asarray([[5.0, 5.0], [5.0, 5.0]], dtype=np.float64), + state_quantities=np.asarray([[100, 100], [0, 50]], dtype=np.int64), + state_common_stop=np.asarray([[8.0, 18.0], [np.nan, 18.0]], dtype=np.float64), + state_next_add_index=np.asarray([[1, 1], [0, 1]], dtype=np.int64), + state_unit_counts=np.asarray([[1, 1], [0, 1]], dtype=np.int64), + candidate_base_quantities=np.asarray( + [[100, 100], [0, 0]], dtype=np.int64 + ), + event_group_scales=np.asarray( + [[1.0, 1.0], [1.0, 0.75]], dtype=np.float64 + ), + event_portfolio_scales=np.asarray( + [1.0, 12.0 / 13.0], dtype=np.float64 + ), + event_cash_scales=np.asarray([1.0, 0.9], dtype=np.float64), + portfolio_unit_cap=12.0, + ) + return inputs, simulation + + +def test_attribution_exposes_unit_and_redistribution_evidence() -> None: + inputs, simulation = _simulation() + + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + decisions = [ + row + for row in facts.attribution.to_pylist() + if row["event_type"] == "decision" + ] + entry = next( + row + for row in decisions + if row["time"].startswith("2026-01-05") + and row["security"] == "ETF-A" + ) + entry_details = json.loads(entry["details_json"]) + assert entry_details["candidate_base_quantity"] == 100 + assert entry_details["frozen_signal_n"] == 1.0 + assert entry_details["actual_fill_price"] == 10.0 + assert entry_details["unit_count_after"] == 1 + assert entry_details["common_stop_after"] == 8.0 + + redistribution = next( + row + for row in decisions + if row["reason_code"] == "full_position_redistribution" + ) + details = json.loads(redistribution["details_json"]) + assert details["unit_count_after"] == 1 + assert details["group_scale"] == pytest.approx(0.75) + assert details["portfolio_scale"] == pytest.approx(12.0 / 13.0) + assert details["cash_scale"] == pytest.approx(0.9) + assert details["redistribution_state_changed"] is False + + +def _delayed_inputs( + *, + delayed_open: float = 12.0, + rows: int = 3, +) -> SimulationInputs: + dates = np.arange( + np.datetime64("2026-01-05"), + np.datetime64("2026-01-05") + np.timedelta64(rows, "D"), + ) + opens = np.asarray([[10.0], *([[delayed_open]] * (rows - 1))], dtype=np.float64) + signal_close = np.full((rows, 1), np.nan, dtype=np.float64) + signal_entry_high = np.full((rows, 1), np.nan, dtype=np.float64) + signal_n = np.full((rows, 1), 999.0, dtype=np.float64) + signal_close[0, 0] = 11.0 + signal_entry_high[0, 0] = 10.0 + signal_n[0, 0] = 1.5 + return SimulationInputs( + dates=_readonly(dates, "datetime64[D]"), + securities=("ETF-A",), + asset_groups=("group-a",), + asset_group_ids=_readonly([0], "int64"), + raw_open=_readonly(opens, "float64"), + raw_high=_readonly(opens, "float64"), + raw_low=_readonly(opens, "float64"), + raw_close=_readonly(opens, "float64"), + raw_pre_close=_readonly(opens, "float64"), + continuous_open=_readonly(opens, "float64"), + continuous_high=_readonly(opens, "float64"), + continuous_low=_readonly(opens, "float64"), + continuous_close=_readonly(opens, "float64"), + continuous_pre_close=_readonly(opens, "float64"), + continuity_factor=_readonly(np.ones((rows, 1)), "float64"), + corporate_action_applied=_readonly(np.zeros((rows, 1)), "bool"), + corporate_actions_digest="4" * 64, + corporate_action_applications=(), + paused=_readonly(np.zeros((rows, 1)), "bool"), + high_limit=_readonly(np.full((rows, 1), np.nan), "float64"), + low_limit=_readonly(np.full((rows, 1), np.nan), "float64"), + signal_source_index=_readonly(np.arange(rows) - 1, "int64"), + signal_close=_readonly(signal_close, "float64"), + signal_entry_high=_readonly(signal_entry_high, "float64"), + signal_exit_low=_readonly(np.full((rows, 1), np.nan), "float64"), + signal_n=_readonly(signal_n, "float64"), + ) + + +def _delayed_config(*, initial_cash: float = 100_000.0) -> dict[str, object]: + return { + "research": {"initial_cash": initial_cash}, + "signal": {"add_step_n": 0.5, "stop_n": 2.0, "max_units": 4}, + "risk": { + "unit_risk_per_n": 0.025, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + }, + "costs": {"commission_multiplier": 1.0, "one_way_slippage": 0.0}, + "execution": {"additional_delay_days": 1}, + } + + +def test_result_adapter_writes_joinquant_shaped_package(tmp_path: Path) -> None: + inputs, simulation = _simulation() + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + code = tmp_path / "entry.py" + code.write_text("print('local research')\n", encoding="utf-8") + backtest_dir = tmp_path / "run-1" / "backtests" / "local-1" + + package = write_local_result( + backtest_dir, + facts=facts, + run_id="run-1", + local_backtest_id="local-1", + scenario_id="baseline", + snapshot_id="a" * 64, + corporate_actions_sha256="e" * 64, + code_path=code, + params={"scenario_id": "baseline", "research": {"initial_cash": 10_000}}, + performance={"status": "pass", "cold_seconds": 1.2, "warm_seconds": 0.4}, + ) + + assert package.root == backtest_dir.resolve() + expected = { + "manifest.json", + "code.py", + "params.json", + "performance.json", + f"params_versions/{package.params_sha256}.json", + "data/results.parquet", + "data/balances.parquet", + "data/positions.parquet", + "data/orders.parquet", + f"data/attribution_log-{package.attribution_sha256}.parquet", + } + actual = { + path.relative_to(backtest_dir).as_posix() + for path in backtest_dir.rglob("*") + if path.is_file() + } + assert actual == expected + assert not any( + (backtest_dir / relative).exists() + for relative in ( + "data/risk.parquet", + "data/period_risks.parquet", + "data/equity.parquet", + "data/trades.parquet", + "raw", + ) + ) + + source = open_analysis_source(backtest_dir) + assert source.kind == "local_backtest" + manifest = source.manifest + assert manifest["source"]["accounting"] == { + "version": "turtle-etf-corporate-actions/1", + "corporate_action_mode": "point_in_time_total_return_approximation", + "continuity_factor_basis": "raw_previous_close_over_current_pre_close", + "corporate_action_metadata_timing": "audit_only_may_be_retrospective", + "price_basis": "continuous_economic_price", + "quantity_basis": "economic_units", + "cash_dividend_mode": "implicit_reinvestment_on_ex_date", + "pay_date_cash_supported": False, + "exact_joinquant_reconciliation": False, + "corporate_actions_sha256": "e" * 64, + } + assert manifest["run"] == { + "run_id": "run-1", + "scenario_id": "baseline", + "snapshot_id": "a" * 64, + } + attribution = manifest["extensions"]["turtle_etf"]["attribution_log"] + reference = attribution["files"][0] + assert attribution["required"] is True + assert attribution["status"] == "complete" + assert attribution["schema_version"] == ATTRIBUTION_SCHEMA_VERSION + assert reference["path"].endswith(f"{reference['sha256']}.parquet") + assert hashlib.sha256((backtest_dir / reference["path"]).read_bytes()).hexdigest() == reference[ + "sha256" + ] + + +def test_adapter_accepts_real_vectorbt_portfolio() -> None: + inputs = SimulationInputs( + dates=_readonly(["2026-01-05", "2026-01-06"], "datetime64[D]"), + securities=("ETF-A",), + asset_groups=("group-a",), + asset_group_ids=_readonly([0], "int64"), + raw_open=_readonly([[10.0], [10.5]], "float64"), + raw_high=_readonly([[10.0], [10.5]], "float64"), + raw_low=_readonly([[10.0], [10.5]], "float64"), + raw_close=_readonly([[10.0], [10.5]], "float64"), + raw_pre_close=_readonly([[10.0], [10.0]], "float64"), + continuous_open=_readonly([[10.0], [10.5]], "float64"), + continuous_high=_readonly([[10.0], [10.5]], "float64"), + continuous_low=_readonly([[10.0], [10.5]], "float64"), + continuous_close=_readonly([[10.0], [10.5]], "float64"), + continuous_pre_close=_readonly([[10.0], [10.0]], "float64"), + continuity_factor=_readonly([[1.0], [1.0]], "float64"), + corporate_action_applied=_readonly([[False], [False]], "bool"), + corporate_actions_digest="4" * 64, + corporate_action_applications=(), + paused=_readonly([[False], [False]], "bool"), + high_limit=_readonly([[np.nan], [np.nan]], "float64"), + low_limit=_readonly([[np.nan], [np.nan]], "float64"), + signal_source_index=_readonly([-1, 0], "int64"), + signal_close=_readonly([[11.0], [np.nan]], "float64"), + signal_entry_high=_readonly([[10.0], [np.nan]], "float64"), + signal_exit_low=_readonly([[np.nan], [np.nan]], "float64"), + signal_n=_readonly([[1.0], [1.0]], "float64"), + ) + config = { + "research": {"initial_cash": 10_000.0}, + "signal": {"add_step_n": 0.5, "stop_n": 2.0, "max_units": 4}, + "risk": { + "unit_risk_per_n": 0.025, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + }, + "costs": {"commission_multiplier": 1.0, "one_way_slippage": 0.0}, + } + + simulation = run_vectorbt_simulation(inputs, config) + facts = to_joinquant_facts(inputs, simulation, scenario_id="real-vectorbt") + + assert facts.orders.num_rows == 1 + assert facts.positions.num_rows == 2 + assert facts.results["returns"].to_pylist()[0] == pytest.approx(-0.0005) + validate_turtle_attribution(facts) + + +def test_delayed_order_keeps_planned_and_execution_dates_and_frozen_evidence() -> None: + inputs = _delayed_inputs() + simulation = run_vectorbt_simulation(inputs, _delayed_config()) + + facts = to_joinquant_facts(inputs, simulation, scenario_id="delayed") + + order = facts.orders.to_pylist()[0] + assert order["entrust_time"] == "2026-01-05 09:30:00" + assert order["match_time"] == "2026-01-06 09:30:00" + assert order["finish_time"] == order["match_time"] + assert order["time"] == order["match_time"] + assert order["amount"] == int(simulation.planned_quantities[1, 0]) + assert order["filled"] == int(simulation.filled_quantities[1, 0]) + decision = next( + row + for row in facts.attribution.to_pylist() + if row["event_type"] == "decision" + ) + details = json.loads(decision["details_json"]) + assert details["planned_date"] == "2026-01-05" + assert details["execution_date"] == "2026-01-06" + assert details["delay_days"] == 1 + assert details["frozen_reason"] == "entry_breakout" + assert details["frozen_target_amount"] == order["amount"] + assert details["frozen_signal_n"] == 1.5 + assert details["execution_adjustment"] == "none" + + +def test_delayed_partial_fill_preserves_frozen_order_amount() -> None: + inputs = _delayed_inputs(delayed_open=200.0) + simulation = run_vectorbt_simulation( + inputs, _delayed_config(initial_cash=25_000.0) + ) + + facts = to_joinquant_facts(inputs, simulation, scenario_id="delayed-partial") + + order = facts.orders.to_pylist()[0] + assert order["status"] == "done" + assert order["comment"] == "cash_truncated" + assert order["filled"] == 100 + assert order["amount"] > order["filled"] + decision = next( + row + for row in facts.attribution.to_pylist() + if row["event_type"] == "decision" + ) + assert json.loads(decision["details_json"])["execution_adjustment"] == ( + "cash_truncated" + ) + + +def test_delayed_horizon_expiry_is_attribution_only() -> None: + inputs = _delayed_inputs(rows=1) + simulation = run_vectorbt_simulation(inputs, _delayed_config()) + + facts = to_joinquant_facts(inputs, simulation, scenario_id="delayed-expired") + + assert facts.orders.num_rows == 0 + expired = [ + row + for row in facts.attribution.to_pylist() + if json.loads(row["details_json"]).get("execution_adjustment") + == "horizon_expired" + ] + assert len(expired) == 1 + assert expired[0]["time"] == "2026-01-05 09:30:00" + assert expired[0]["requested_amount"] > 0 + assert expired[0]["executed_amount"] == 0 + + +def test_physical_fields_and_cross_table_facts_match_joinquant_contract( + tmp_path: Path, +) -> None: + inputs, simulation = _simulation() + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + + assert facts.results.schema.names == ["benchmark_returns", "returns", "time"] + assert facts.balances.schema.names == [ + "total_value", + "net_value", + "cash", + "aval_cash", + "time", + ] + assert facts.positions.schema.names == [ + "pindex", + "avg_cost", + "margin", + "amount", + "today_amount", + "hold_cost", + "side", + "price", + "gains", + "daily_gains", + "closeable_amount", + "time", + "security_name", + "security", + ] + assert facts.orders.schema.names == [ + "match_time", + "pindex", + "cancel_time", + "action", + "limit_price", + "comment", + "entrust_time", + "finish_time", + "side", + "price", + "commission", + "gains", + "type", + "time", + "security_name", + "security", + "filled", + "amount", + "status", + ] + assert facts.attribution.schema.names == list(ATTRIBUTION_FIELDS) + assert facts.results["benchmark_returns"].null_count == facts.results.num_rows + assert facts.results["returns"].to_pylist() == pytest.approx([-0.001, 0.018]) + assert facts.balances["total_value"].to_pylist() == [9_990.0, 10_180.0] + assert facts.orders.num_rows == 4 + assert sum(facts.orders["filled"].to_pylist()) == 350 + assert facts.attribution.num_rows == 8 + validate_turtle_attribution(facts) + + +def test_corporate_action_application_is_audited_without_fake_order_or_cash() -> None: + inputs, simulation = _simulation() + inputs.corporate_action_applications = ( + SimpleNamespace( + source_event_id="FUND_DIVIDEND:101", + security="ETF-A", + event_type="split", + effective_date="2026-01-05", + application_date="2026-01-06", + announcement_date="2026-01-05", + knowledge_cutoff_date="2026-01-10", + split_ratio=2.0, + cash_per_share=None, + cumulative_factor=2.0, + price_basis_changed=True, + source="joinquant.finance.FUND_DIVIDEND", + source_record_sha256="b" * 64, + ), + ) + + facts = to_joinquant_facts(inputs, simulation, scenario_id="corporate-action") + + rows = [ + row + for row in facts.attribution.to_pylist() + if row["event_type"] == "corporate_action" + ] + assert len(rows) == 1 + assert rows[0]["reason_code"] == "corporate_action_applied" + assert rows[0]["requested_amount"] is None + assert rows[0]["executed_amount"] is None + details = json.loads(rows[0]["details_json"]) + assert details == { + "announcement_date": "2026-01-05", + "cash_per_share": None, + "corporate_action_mode": "point_in_time_total_return_approximation", + "cumulative_factor": 2.0, + "effective_date": "2026-01-05", + "application_date": "2026-01-06", + "event_type": "split", + "evidence_timing": "point_in_time", + "knowledge_cutoff_date": "2026-01-10", + "source": "joinquant.finance.FUND_DIVIDEND", + "source_event_id": "FUND_DIVIDEND:101", + "source_record_sha256": "b" * 64, + "split_ratio": 2.0, + "price_basis_changed": True, + } + assert not any( + order["time"].startswith("2026-01-06") + and order["comment"] == "corporate_action" + for order in facts.orders.to_pylist() + ) + assert facts.attribution.num_rows == 9 + validate_turtle_attribution(facts) + + +def test_attribution_uses_exact_openspec_fields_and_parseable_details() -> None: + inputs, simulation = _simulation() + + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + + assert ATTRIBUTION_FIELDS == ( + "time", + "event_id", + "scope", + "security", + "event_type", + "reason_code", + "requested_amount", + "executed_amount", + "reference_price", + "risk_before", + "risk_after", + "details_json", + ) + assert facts.attribution.schema.names == list(ATTRIBUTION_FIELDS) + rows = facts.attribution.to_pylist() + assert len({row["event_id"] for row in rows}) == len(rows) + for row in rows: + assert row["scope"] == "security" + assert isinstance(json.loads(row["details_json"]), dict) + decision = next( + row + for row in rows + if row["time"].startswith("2026-01-06") + and row["security"] == "ETF-A" + and row["event_type"] == "decision" + ) + assert decision["reason_code"] == "protective_stop" + assert decision["risk_before"] == pytest.approx(200.0) + assert decision["risk_after"] == pytest.approx(0.0) + + +def test_security_daily_pnl_reconciles_entry_partial_exit_and_full_exit() -> None: + inputs, simulation = _simulation() + + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + + held_pnl = { + (row["time"][:10], row["security"]): row["daily_gains"] + for row in facts.positions.to_pylist() + } + assert held_pnl == pytest.approx( + { + ("2026-01-05", "ETF-A"): -5.0, + ("2026-01-05", "ETF-B"): -5.0, + ("2026-01-06", "ETF-B"): 95.0, + } + ) + assert ("2026-01-06", "ETF-A") not in held_pnl + + valuations = [ + row for row in facts.attribution.to_pylist() if row["event_type"] == "valuation" + ] + daily_security_pnl = { + (row["time"][:10], row["security"]): json.loads(row["details_json"])[ + "security_daily_pnl" + ] + for row in valuations + } + assert daily_security_pnl == pytest.approx( + { + ("2026-01-05", "ETF-A"): -5.0, + ("2026-01-05", "ETF-B"): -5.0, + ("2026-01-06", "ETF-A"): 95.0, + ("2026-01-06", "ETF-B"): 95.0, + } + ) + assert sum( + value + for (date, _), value in daily_security_pnl.items() + if date == "2026-01-05" + ) == pytest.approx(-10.0) + assert sum( + value + for (date, _), value in daily_security_pnl.items() + if date == "2026-01-06" + ) == pytest.approx(190.0) + + +def test_security_daily_pnl_prices_additions_and_trend_exit_at_execution() -> None: + inputs = SimpleNamespace( + dates=np.asarray( + ["2026-01-05", "2026-01-06", "2026-01-07", "2026-01-08"], + dtype="datetime64[D]", + ), + securities=("ETF-A",), + close=np.asarray([[10.0], [11.0], [12.0], [11.0]], dtype=np.float64), + signal_n=np.asarray([[1.0], [1.0], [1.0], [1.0]], dtype=np.float64), + ) + simulation = SimpleNamespace( + initial_cash=10_000.0, + portfolio=SimpleNamespace( + value=lambda: pd.Series([10_049.0, 10_173.0, 10_362.0, 10_256.0]), + cash=lambda: pd.Series([9_049.0, 8_523.0, 9_522.0, 10_256.0]), + ), + action_codes=np.asarray( + [ + [ACTION_ENTRY], + [ACTION_ADDITION], + [ACTION_REDISTRIBUTION_SELL], + [ACTION_FULL_EXIT], + ], + dtype=np.int16, + ), + reason_codes=np.asarray( + [ + [REASON_ENTRY_BREAKOUT], + [REASON_ENTRY_BREAKOUT], + [REASON_FULL_POSITION_REDISTRIBUTION], + [REASON_TREND_EXIT], + ], + dtype=np.int16, + ), + requested_quantities=np.asarray([[100], [50], [80], [70]], dtype=np.int64), + planned_quantities=np.asarray([[100], [50], [80], [70]], dtype=np.int64), + filled_quantities=np.asarray([[100], [50], [80], [70]], dtype=np.int64), + fill_prices=np.asarray([[9.5], [10.5], [12.5], [10.5]], dtype=np.float64), + fees=np.asarray([[1.0], [1.0], [1.0], [1.0]], dtype=np.float64), + state_quantities=np.asarray([[100], [150], [70], [0]], dtype=np.int64), + state_common_stop=np.asarray([[8.0], [9.0], [9.0], [np.nan]], dtype=np.float64), + state_next_add_index=np.asarray([[1], [2], [2], [0]], dtype=np.int64), + ) + + facts = to_joinquant_facts(inputs, simulation, scenario_id="path-dependent") + + assert facts.positions["daily_gains"].to_pylist() == pytest.approx( + [49.0, 124.0, 189.0] + ) + valuation_rows = [ + row for row in facts.attribution.to_pylist() if row["event_type"] == "valuation" + ] + valuation_pnl = [ + json.loads(row["details_json"])["security_daily_pnl"] + for row in valuation_rows + ] + assert valuation_pnl == pytest.approx([49.0, 124.0, 189.0, -106.0]) + final_details = json.loads(valuation_rows[-1]["details_json"]) + assert valuation_rows[-1]["reason_code"] == "signal_exit" + assert final_details["source_reason"] == "trend_exit" + assert final_details["position_after"] == 0 + + +def test_adapter_rejects_security_pnl_that_does_not_match_portfolio_change() -> None: + inputs, simulation = _simulation() + simulation.portfolio = SimpleNamespace( + value=lambda: pd.Series([9_990.0, 10_181.0]), + cash=lambda: pd.Series([6_990.0, 9_130.0]), + ) + + with pytest.raises(ResultContractError, match="daily PnL"): + to_joinquant_facts(inputs, simulation, scenario_id="broken-pnl") + + +def test_attribution_rejects_unknown_reason_and_uncovered_order() -> None: + inputs, simulation = _simulation() + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + document = facts.attribution.to_pydict() + document["reason_code"][0] = "unknown_reason" + invalid = facts.with_attribution(document) + with pytest.raises(ResultContractError, match="reason"): + validate_turtle_attribution(invalid) + + document = facts.attribution.slice(1).to_pydict() + uncovered = facts.with_attribution(document) + with pytest.raises(ResultContractError, match="cover"): + validate_turtle_attribution(uncovered) + + +def test_rejected_vectorbt_order_is_preserved_as_canceled_order() -> None: + inputs, simulation = _simulation() + simulation.action_codes[1, 0] = ACTION_ADDITION + simulation.reason_codes[1, 0] = REASON_ORDER_REJECTED + simulation.filled_quantities[1, 0] = 0 + simulation.fill_prices[1, 0] = np.nan + simulation.fees[1, 0] = 0.0 + simulation.state_quantities[1, 0] = 100 + simulation.state_common_stop[1, 0] = 8.0 + simulation.state_next_add_index[1, 0] = 1 + simulation.portfolio = SimpleNamespace( + value=lambda: pd.Series([9_990.0, 10_185.0]), + cash=lambda: pd.Series([6_990.0, 8_035.0]), + ) + + facts = to_joinquant_facts(inputs, simulation, scenario_id="rejected-order") + + canceled = [ + item for item in facts.orders.to_pylist() if item["status"] == "canceled" + ] + assert canceled == [ + { + "match_time": None, + "pindex": 0, + "cancel_time": "2026-01-06 09:30:00", + "action": "open", + "limit_price": 0.0, + "comment": "order_rejected", + "entrust_time": "2026-01-06 09:30:00", + "finish_time": None, + "side": "long", + "price": 0.0, + "commission": 0.0, + "gains": 0.0, + "type": "market", + "time": "2026-01-06 09:30:00", + "security_name": "ETF-A", + "security": "ETF-A", + "filled": 0, + "amount": 100, + "status": "canceled", + } + ] + validate_turtle_attribution(facts) + + +def test_params_current_and_version_are_byte_identical(tmp_path: Path) -> None: + inputs, simulation = _simulation() + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + code = tmp_path / "entry.py" + code.write_text("pass\n", encoding="utf-8") + root = tmp_path / "backtest" + package = write_local_result( + root, + facts=facts, + run_id="run-1", + local_backtest_id="local-1", + scenario_id="baseline", + snapshot_id="b" * 64, + corporate_actions_sha256="e" * 64, + code_path=code, + params={"z": 2, "a": 1}, + performance={"status": "pass"}, + ) + + current = (root / "params.json").read_bytes() + version = (root / "params_versions" / f"{package.params_sha256}.json").read_bytes() + assert current == version + assert hashlib.sha256(current).hexdigest() == package.params_sha256 + assert json.loads(current) == {"a": 1, "z": 2} + + +def test_existing_output_directory_is_never_overwritten(tmp_path: Path) -> None: + inputs, simulation = _simulation() + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + code = tmp_path / "entry.py" + code.write_text("pass\n", encoding="utf-8") + root = tmp_path / "backtest" + root.mkdir() + + with pytest.raises(ResultContractError, match="already exists"): + write_local_result( + root, + facts=facts, + run_id="run-1", + local_backtest_id="local-1", + scenario_id="baseline", + snapshot_id="c" * 64, + corporate_actions_sha256="e" * 64, + code_path=code, + params={"scenario_id": "baseline"}, + performance={"status": "pass"}, + ) + + +def test_project_validator_rejects_missing_attribution_declaration( + tmp_path: Path, +) -> None: + inputs, simulation = _simulation() + facts = to_joinquant_facts(inputs, simulation, scenario_id="baseline") + code = tmp_path / "entry.py" + code.write_text("pass\n", encoding="utf-8") + root = tmp_path / "backtest" + write_local_result( + root, + facts=facts, + run_id="run-1", + local_backtest_id="local-1", + scenario_id="baseline", + snapshot_id="d" * 64, + corporate_actions_sha256="e" * 64, + code_path=code, + params={"scenario_id": "baseline"}, + performance={"status": "pass"}, + ) + manifest_path = root / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["extensions"] = {} + manifest_path.write_text( + json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + + with pytest.raises(ResultContractError, match="attribution declaration"): + validate_turtle_result(root) diff --git a/tests/local_quant_research/test_turtle_risk.py b/tests/local_quant_research/test_turtle_risk.py deleted file mode 100644 index 657abe6..0000000 --- a/tests/local_quant_research/test_turtle_risk.py +++ /dev/null @@ -1,350 +0,0 @@ -from __future__ import annotations - -import sys -from dataclasses import replace -from decimal import Decimal -from pathlib import Path - -import pandas as pd -import pytest - - -RESEARCH_ROOT = ( - Path(__file__).resolve().parents[2] - / "joinquant" - / "strategies" - / "strategy-003" - / "research" -) -sys.path.insert(0, str(RESEARCH_ROOT)) - -from turtle_etf.risk import ( # noqa: E402 - PortfolioState, - RiskInputs, - estimate_covariance, - evaluate_risk, - initial_unit, - portfolio_volatility, - target_volatility_reductions, -) -from turtle_etf.state import OrderIntent, apply_entry_fill # noqa: E402 - - -def _intent( - security: str, - *, - group: str = "group-a", - quantity: int = 100, - price: str = "10", - stop: str = "8", - action: str = "entry", -) -> OrderIntent: - return OrderIntent( - security=security, - asset_group=group, - action=action, - quantity=quantity, - expected_price=Decimal(price), - signal_date="2026-01-05", - execution_date="2026-01-06", - signal_n=Decimal("1") if action in {"entry", "addition"} else None, - standard_unit=max(quantity, 100) if action in {"entry", "addition"} else None, - common_stop_after=Decimal(stop) if action in {"entry", "addition"} else None, - reason="test", - ) - - -def _covariance( - securities: tuple[str, ...], - *, - scale: float = 0.001, - rows: int = 60, -): - frame = pd.DataFrame( - { - security: [ - (1 if index % 2 else -1) * scale * (1 + offset / 10) - for index in range(rows) - ] - for offset, security in enumerate(securities) - } - ) - return estimate_covariance(frame, securities=securities, days=60) - - -def _inputs( - securities: tuple[str, ...], - *, - covariance=None, - turnover: str = "1000000000", -) -> RiskInputs: - return RiskInputs( - prices={security: Decimal("10") for security in securities}, - median_turnover_20d={ - security: Decimal(turnover) for security in securities - }, - covariance=covariance or _covariance(securities), - ) - - -def test_initial_unit_uses_half_percent_two_n_risk_without_lot_rounding() -> None: - assert initial_unit(Decimal("1500000"), Decimal("3")) == 1250 - with pytest.raises(ValueError): - initial_unit(Decimal("1500000"), Decimal("0")) - - -def test_covariance_requires_sixty_complete_aligned_return_rows() -> None: - returns = pd.DataFrame( - { - "A": [0.01, -0.01] * 30 + [0.02], - "B": [0.02, -0.02] * 30 + [None], - } - ) - - estimate = estimate_covariance(returns, securities=("A", "B"), days=60) - insufficient = estimate_covariance( - returns.iloc[:-1], securities=("A", "B"), days=61 - ) - - assert estimate.aligned_samples == 60 - assert estimate.window_days == 60 - assert estimate.securities == ("A", "B") - assert insufficient is None - - -def test_valid_lot_passes_all_risk_gates() -> None: - request = _intent("A") - state = PortfolioState(equity=Decimal("1000000"), cash=Decimal("1000000")) - - decision = evaluate_risk((request,), state, _inputs(("A",))) - - assert decision.allow_new_risk is True - assert decision.approved == (request,) - assert decision.rejected == () - assert decision.reason_codes == () - assert decision.projected_volatility < Decimal("0.10") - - -@pytest.mark.parametrize( - ("intent", "state", "inputs", "reason"), - [ - ( - _intent("A", quantity=50), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - _inputs(("A",)), - "invalid_lot", - ), - ( - _intent("A", quantity=100), - PortfolioState(Decimal("1000000"), Decimal("999")), - _inputs(("A",)), - "insufficient_cash", - ), - ( - _intent("A", quantity=100), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - _inputs(("A",), turnover="99999999"), - "liquidity_floor", - ), - ( - _intent("A", quantity=100100, stop="9.99"), - PortfolioState(Decimal("10000000"), Decimal("10000000")), - _inputs(("A",), turnover="100000000"), - "order_liquidity_cap", - ), - ( - _intent("A", quantity=30100, stop="9.99"), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - _inputs(("A",)), - "security_value_cap", - ), - ( - _intent("A", quantity=13000, stop="9"), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - _inputs(("A",)), - "security_risk_cap", - ), - ], -) -def test_individual_order_cash_lot_liquidity_and_security_gates( - intent: OrderIntent, - state: PortfolioState, - inputs: RiskInputs, - reason: str, -) -> None: - decision = evaluate_risk((intent,), state, inputs) - - assert decision.approved == () - assert reason in decision.reason_codes - - -@pytest.mark.parametrize( - ("requests", "state", "inputs", "reason"), - [ - ( - ( - _intent("A", quantity=26000, stop="9.99"), - _intent("B", quantity=26000, stop="9.99"), - ), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - _inputs(("A", "B")), - "group_value_cap", - ), - ( - ( - _intent("A", quantity=10000, stop="8.8"), - _intent("B", quantity=10000, stop="8.8"), - _intent("C", quantity=1000, stop="8"), - ), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - _inputs(("A", "B", "C")), - "group_risk_cap", - ), - ( - tuple( - _intent(chr(65 + index), group=f"group-{index}", quantity=25100, stop="9.99") - for index in range(4) - ), - PortfolioState(Decimal("1000000"), Decimal("2000000")), - _inputs(("A", "B", "C", "D")), - "portfolio_value_cap", - ), - ( - tuple( - _intent(chr(65 + index), group=f"group-{index}", quantity=10000, stop="8.9") - for index in range(5) - ), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - _inputs(("A", "B", "C", "D", "E")), - "portfolio_risk_cap", - ), - ], -) -def test_asset_group_and_portfolio_hard_caps( - requests: tuple[OrderIntent, ...], - state: PortfolioState, - inputs: RiskInputs, - reason: str, -) -> None: - decision = evaluate_risk(requests, state, inputs) - - assert reason in decision.reason_codes - assert not decision.approved - - -def test_target_volatility_is_a_ten_percent_one_way_cap() -> None: - request = _intent("A", quantity=30000, stop="9.99") - state = PortfolioState(Decimal("1000000"), Decimal("1000000")) - high_volatility = _covariance(("A",), scale=0.03) - - decision = evaluate_risk( - (request,), - state, - _inputs(("A",), covariance=high_volatility, turnover="100000000000"), - ) - - assert decision.projected_volatility > Decimal("0.10") - assert "target_volatility" in decision.reason_codes - assert decision.approved == () - - -def test_existing_high_volatility_positions_scale_toward_nine_point_five_percent() -> None: - position = apply_entry_fill( - security="A", - asset_group="group-a", - execution_date="2026-01-01", - fill_price=Decimal("10"), - quantity=30000, - signal_n=Decimal("0.005"), - standard_unit=30000, - ) - state = PortfolioState(Decimal("1000000"), Decimal("700000"), (position,)) - inputs = _inputs( - ("A",), - covariance=_covariance(("A",), scale=0.03), - turnover="100000000000", - ) - - before = portfolio_volatility(state, inputs) - reductions = target_volatility_reductions( - state, - inputs, - signal_date="2026-01-05", - execution_date="2026-01-06", - ) - - assert before > Decimal("0.10") - assert len(reductions) == 1 - assert reductions[0].action == "mandatory_risk_reduction" - assert reductions[0].quantity % 100 == 0 - remaining_value = Decimal(position.quantity - reductions[0].quantity) * Decimal("10") - remaining_volatility = before * remaining_value / Decimal("300000") - assert remaining_volatility <= Decimal("0.095") - - -def test_cold_security_without_sixty_samples_cannot_add_risk() -> None: - request = _intent("A") - inputs = replace(_inputs(("A",)), covariance=None) - - decision = evaluate_risk( - (request,), - PortfolioState(Decimal("1000000"), Decimal("1000000")), - inputs, - ) - - assert decision.allow_new_risk is True - assert decision.approved == () - assert "covariance_unavailable" in decision.reason_codes - - -def test_missing_held_price_stops_new_risk_but_keeps_exit_and_reduction() -> None: - position = apply_entry_fill( - security="A", - asset_group="group-a", - execution_date="2026-01-01", - fill_price=Decimal("10"), - quantity=1000, - signal_n=Decimal("1"), - standard_unit=1000, - ) - state = PortfolioState( - equity=Decimal("1000000"), - cash=Decimal("990000"), - positions=(position,), - ) - entry = _intent("B", group="group-b") - full_exit = _intent("A", quantity=1000, action="full_exit") - reduction = _intent("A", quantity=100, action="mandatory_risk_reduction") - inputs = _inputs(("A", "B")) - inputs = replace(inputs, prices={"A": None, "B": Decimal("10")}) - - decision = evaluate_risk((entry, full_exit, reduction), state, inputs) - - assert decision.allow_new_risk is False - assert decision.approved == (full_exit, reduction) - assert entry in decision.rejected - assert "held_risk_input_missing" in decision.reason_codes - - -def test_missing_held_covariance_stops_new_risk_without_zero_fill() -> None: - position = apply_entry_fill( - security="A", - asset_group="group-a", - execution_date="2026-01-01", - fill_price=Decimal("10"), - quantity=1000, - signal_n=Decimal("1"), - standard_unit=1000, - ) - state = PortfolioState(Decimal("1000000"), Decimal("990000"), (position,)) - inputs = _inputs(("B",)) - inputs = replace( - inputs, - prices={"A": Decimal("10"), "B": Decimal("10")}, - median_turnover_20d={"A": Decimal("1000000000"), "B": Decimal("1000000000")}, - ) - - decision = evaluate_risk((_intent("B", group="group-b"),), state, inputs) - - assert decision.allow_new_risk is False - assert "held_risk_input_missing" in decision.reason_codes diff --git a/tests/local_quant_research/test_turtle_single_scenario.py b/tests/local_quant_research/test_turtle_single_scenario.py new file mode 100644 index 0000000..9674e1d --- /dev/null +++ b/tests/local_quant_research/test_turtle_single_scenario.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +RESEARCH_ROOT = ( + Path(__file__).resolve().parents[2] + / "joinquant" + / "strategies" + / "strategy-003" + / "research" +) +sys.path.insert(0, str(RESEARCH_ROOT)) + +from turtle_etf import single_scenario # noqa: E402 +from turtle_etf.result_adapter import LocalResultPackage # noqa: E402 + + +def _config() -> dict[str, object]: + return { + "schema_version": 1, + "project_id": "strategy-003", + "scenario_id": "baseline", + "research": {"initial_cash": 1_500_000}, + } + + +def test_one_call_executes_exactly_one_scenario_and_returns_to_caller( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[object] = [] + facts = SimpleNamespace(name="facts") + def fake_benchmark(**kwargs: object) -> SimpleNamespace: + calls.append(kwargs) + return SimpleNamespace(facts=facts, performance={"status": "pass"}) + + monkeypatch.setattr(single_scenario, "benchmark_scenario", fake_benchmark) + + def fake_write(target: Path, **kwargs: object) -> LocalResultPackage: + calls.append((target, kwargs)) + target.mkdir(parents=True) + return LocalResultPackage( + root=target.resolve(), params_sha256="a" * 64, attribution_sha256="b" * 64 + ) + + monkeypatch.setattr(single_scenario, "write_local_result", fake_write) + code = tmp_path / "cli.py" + code.write_text("pass\n", encoding="utf-8") + + outcome = single_scenario.execute_prepared_scenario( + prepared_inputs=SimpleNamespace( + name="prepared", corporate_actions_digest="f" * 64 + ), + config=_config(), + output_dir=tmp_path / "output", + run_id="run-1", + snapshot_id="c" * 64, + code_sha256="e" * 64, + config_sha256="d" * 64, + code_path=code, + ) + + benchmark_calls = [call for call in calls if isinstance(call, dict)] + assert len(benchmark_calls) == 1 + assert benchmark_calls[0]["prepared_inputs"].name == "prepared" + assert benchmark_calls[0]["scenario_id"] == "baseline" + write_calls = [call for call in calls if isinstance(call, tuple)] + assert write_calls[0][1]["corporate_actions_sha256"] == "f" * 64 + assert outcome.scenario_id == "baseline" + assert outcome.local_backtest_id == "local-baseline" + assert outcome.next_action == "return_to_caller" + assert outcome.result_path == ( + tmp_path / "output" / "backtests" / "local-baseline" + ).resolve() + assert len(list((tmp_path / "output" / "backtests").iterdir())) == 1 + + +@pytest.mark.parametrize("forbidden", ["candidates", "scenarios", "analysis_plan"]) +def test_single_scenario_rejects_batch_or_analysis_inputs( + forbidden: str, +) -> None: + config = _config() + config[forbidden] = [] + + with pytest.raises(single_scenario.SingleScenarioError, match="single scenario"): + single_scenario.validate_single_scenario_config(config) + + +def test_project_status_is_minimal_and_returns_to_caller(tmp_path: Path) -> None: + single_scenario.write_project_status( + tmp_path, + status="complete", + reason_codes=(), + next_action="return_to_caller", + ) + + assert json.loads((tmp_path / "project-status.json").read_text(encoding="utf-8")) == { + "schema_version": 1, + "status": "complete", + "reason_codes": [], + "next_action": "return_to_caller", + } + + +def test_repository_entry_and_run_config_expose_only_one_result( + repo_root: Path, +) -> None: + research_root = repo_root / "joinquant/strategies/strategy-003/research" + baseline = json.loads((research_root / "baseline.json").read_text(encoding="utf-8")) + run_config = json.loads( + (research_root / "project-run.json").read_text(encoding="utf-8") + ) + entry = ( + research_root / "turtle_etf/vectorbt_cli.py" + ).read_text(encoding="utf-8") + + assert baseline["scenario_id"] == "baseline" + assert run_config["project_entry"].endswith("/vectorbt_cli.py") + assert run_config["required_outputs"] == [ + {"path": "backtests/local-baseline", "format": "directory"} + ] + assert "benchmark_input" not in run_config + assert "candidates.json" not in run_config["declared_inputs"] + assert "corporate_actions=snapshot.corporate_actions" in entry + assert "corporate_actions_digest=snapshot.corporate_actions_digest" in entry + for forbidden in ( + "run_candidate_set", + "quant_analysis", + "analysis-plan.json", + "candidate-strategies.json", + "Vibe-Trading", + ): + assert forbidden not in entry diff --git a/tests/local_quant_research/test_turtle_vectorbt_callbacks.py b/tests/local_quant_research/test_turtle_vectorbt_callbacks.py new file mode 100644 index 0000000..0638006 --- /dev/null +++ b/tests/local_quant_research/test_turtle_vectorbt_callbacks.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +from vectorbt.portfolio.enums import OrderResult, OrderSide, OrderStatus + + +RESEARCH_ROOT = ( + Path(__file__).resolve().parents[2] + / "joinquant" + / "strategies" + / "strategy-003" + / "research" +) +sys.path.insert(0, str(RESEARCH_ROOT)) + +from turtle_etf import vectorbt_callbacks as callbacks # noqa: E402 +from turtle_etf.vectorbt_engine import ( # noqa: E402 + _mutable_state, + _params, + run_vectorbt_simulation, +) +from turtle_etf.vectorbt_inputs import SimulationInputs # noqa: E402 + + +def _ro(values: object, dtype: str) -> np.ndarray: + result = np.ascontiguousarray(values, dtype=dtype) + result.setflags(write=False) + return result + + +def _inputs( + *, + opens: list[list[float]], + signal_close: list[list[float]], + entry_high: list[list[float]], + exit_low: list[list[float]] | None = None, + signal_n: list[list[float]] | None = None, + paused: list[list[bool]] | None = None, + high_limit: list[list[float]] | None = None, + low_limit: list[list[float]] | None = None, + group_ids: list[int] | None = None, +) -> SimulationInputs: + open_array = np.asarray(opens, dtype=np.float64) + rows, columns = open_array.shape + securities = tuple(f"ETF-{chr(65 + column)}" for column in range(columns)) + close = np.where(np.isfinite(open_array), open_array, 10.0) + ids = np.arange(columns) if group_ids is None else np.asarray(group_ids) + return SimulationInputs( + dates=_ro( + np.arange( + np.datetime64("2026-01-05"), + np.datetime64("2026-01-05") + np.timedelta64(rows, "D"), + ), + "datetime64[D]", + ), + securities=securities, + asset_groups=tuple(f"group-{group}" for group in ids), + asset_group_ids=_ro(ids, "int64"), + raw_open=_ro(open_array, "float64"), + raw_high=_ro(open_array, "float64"), + raw_low=_ro(open_array, "float64"), + raw_close=_ro(close, "float64"), + raw_pre_close=_ro(close, "float64"), + continuous_open=_ro(open_array, "float64"), + continuous_high=_ro(open_array, "float64"), + continuous_low=_ro(open_array, "float64"), + continuous_close=_ro(close, "float64"), + continuous_pre_close=_ro(close, "float64"), + continuity_factor=_ro(np.ones((rows, columns)), "float64"), + corporate_action_applied=_ro( + np.zeros((rows, columns), dtype=np.bool_), "bool" + ), + corporate_actions_digest=( + "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e4d8e6f7a0f8f0d7c6b3f3b" + ), + corporate_action_applications=(), + paused=_ro( + np.zeros((rows, columns), dtype=np.bool_) if paused is None else paused, + "bool", + ), + high_limit=_ro( + np.full((rows, columns), np.nan) if high_limit is None else high_limit, + "float64", + ), + low_limit=_ro( + np.full((rows, columns), np.nan) if low_limit is None else low_limit, + "float64", + ), + signal_source_index=_ro(np.arange(rows) - 1, "int64"), + signal_close=_ro(signal_close, "float64"), + signal_entry_high=_ro(entry_high, "float64"), + signal_exit_low=_ro( + np.full((rows, columns), np.nan) if exit_low is None else exit_low, + "float64", + ), + signal_n=_ro( + np.ones((rows, columns)) if signal_n is None else signal_n, + "float64", + ), + ) + + +def _config( + initial_cash: float = 100_000.0, + *, + unit_risk: float = 0.01, + group_cap: float = 6.0, + portfolio_cap: float = 12.0, +) -> dict[str, object]: + return { + "research": {"initial_cash": initial_cash}, + "signal": {"add_step_n": 0.5, "stop_n": 2.0, "max_units": 4}, + "risk": { + "unit_risk_per_n": unit_risk, + "asset_group_unit_cap": group_cap, + "portfolio_unit_cap": portfolio_cap, + }, + "costs": {"commission_multiplier": 1.0, "one_way_slippage": 0.0}, + } + + +def test_group_and_portfolio_unit_scales_follow_confirmed_formula() -> None: + group_scales, portfolio_scale = callbacks._risk_scales_nb.py_func( + np.asarray([4, 4, 4], dtype=np.int64), + np.asarray([0, 0, 1], dtype=np.int64), + 2, + 6.0, + 12.0, + ) + assert group_scales.tolist() == pytest.approx([0.75, 1.0]) + assert portfolio_scale == pytest.approx(1.0) + + group_scales, portfolio_scale = callbacks._risk_scales_nb.py_func( + np.asarray([4, 4, 4, 4], dtype=np.int64), + np.asarray([0, 1, 2, 3], dtype=np.int64), + 4, + 6.0, + 12.0, + ) + assert group_scales.tolist() == pytest.approx([1.0, 1.0, 1.0, 1.0]) + assert portfolio_scale == pytest.approx(0.75) + + +def test_target_rounding_is_uniform_and_input_order_invariant() -> None: + bases = np.asarray([[1000, 0, 0, 0], [2000, 0, 0, 0]], dtype=np.int64) + counts = np.asarray([1, 1], dtype=np.int64) + groups = np.asarray([0, 1], dtype=np.int64) + scales = np.asarray([1.0, 1.0]) + locked = np.asarray([-1, -1], dtype=np.int64) + + targets = callbacks._targets_for_scale_nb.py_func( + bases, counts, groups, scales, 1.0, 0.55, locked, 100 + ) + permuted = callbacks._targets_for_scale_nb.py_func( + bases[::-1], counts[::-1], groups[::-1], scales, 1.0, 0.55, locked, 100 + )[::-1] + + assert targets.tolist() == [500, 1100] + assert targets.tolist() == permuted.tolist() + assert np.all(targets % 100 == 0) + + +def test_late_breakout_displaces_earlier_position_without_changing_its_unit() -> None: + inputs = _inputs( + opens=[[10.0, 10.0], [10.0, 10.0], [10.0, 10.0]], + signal_close=[[11.0, np.nan], [10.0, 11.0], [10.0, 10.0]], + entry_high=[[10.0, np.nan], [20.0, 10.0], [20.0, 20.0]], + ) + result = run_vectorbt_simulation( + inputs, _config(initial_cash=100_005.0, portfolio_cap=1.0) + ) + + assert result.state_quantities[0].tolist() == [1000, 0] + assert result.action_codes[1].tolist() == [ + callbacks.ACTION_REDISTRIBUTION_SELL, + callbacks.ACTION_ENTRY, + ] + assert result.state_quantities[1].tolist() == [500, 500] + assert result.state_unit_counts[1].tolist() == [1, 1] + assert result.event_portfolio_scales[1] == pytest.approx(0.5) + assert result.state_common_stop[1, 0] == result.state_common_stop[0, 0] + assert result.action_codes[2].tolist() == [callbacks.ACTION_NONE] * 2 + assert result.filled_quantities[2].tolist() == [0, 0] + + +def test_same_group_units_scale_uniformly() -> None: + inputs = _inputs( + opens=[[10.0, 10.0], [10.0, 10.0]], + signal_close=[[11.0, np.nan], [10.0, 11.0]], + entry_high=[[10.0, np.nan], [20.0, 10.0]], + group_ids=[0, 0], + ) + result = run_vectorbt_simulation( + inputs, _config(initial_cash=100_005.0, group_cap=1.0) + ) + + assert result.state_quantities[1].tolist() == [500, 500] + assert result.event_group_scales[1].tolist() == pytest.approx([0.5, 0.5]) + assert result.event_portfolio_scales[1] == pytest.approx(1.0) + + +def test_each_filled_unit_freezes_its_own_n_and_only_raises_common_stop() -> None: + inputs = _inputs( + opens=[[10.0], [13.0], [13.0]], + signal_close=[[11.0], [10.6], [10.0]], + entry_high=[[10.0], [20.0], [20.0]], + signal_n=[[1.0], [2.0], [999.0]], + ) + result = run_vectorbt_simulation(inputs, _config()) + + assert result.state_unit_counts[:, 0].tolist() == [1, 2, 2] + assert result.state_common_stop[0, 0] == pytest.approx(8.0) + assert result.state_common_stop[1, 0] == pytest.approx(9.0) + assert result.state_common_stop[2, 0] == pytest.approx(9.0) + assert result.state_next_add_index[:, 0].tolist() == [1, 2, 2] + + +def test_additions_use_fixed_initial_levels_one_per_day_and_stop_at_four() -> None: + inputs = _inputs( + opens=[[10.0], [11.0], [11.5], [12.0], [12.5]], + signal_close=[[11.0], [12.0], [12.0], [12.0], [12.0]], + entry_high=[[10.0], [20.0], [20.0], [20.0], [20.0]], + signal_n=[[1.0], [8.0], [8.0], [8.0], [8.0]], + ) + result = run_vectorbt_simulation(inputs, _config()) + + assert result.action_codes[:, 0].tolist() == [ + callbacks.ACTION_ENTRY, + callbacks.ACTION_ADDITION, + callbacks.ACTION_ADDITION, + callbacks.ACTION_ADDITION, + callbacks.ACTION_NONE, + ] + assert result.state_unit_counts[:, 0].tolist() == [1, 2, 3, 4, 4] + + +def test_untradeable_candidate_does_not_advance_unit_stop_or_add_level() -> None: + inputs = _inputs( + opens=[[10.0], [10.0], [11.0]], + signal_close=[[11.0], [11.0], [11.0]], + entry_high=[[10.0], [10.0], [10.0]], + paused=[[True], [False], [False]], + high_limit=[[np.nan], [np.nan], [11.0]], + ) + result = run_vectorbt_simulation(inputs, _config()) + + assert result.reason_codes[0, 0] == callbacks.REASON_PAUSED + assert result.state_unit_counts[0, 0] == 0 + assert result.state_unit_counts[1, 0] == 1 + assert result.reason_codes[2, 0] == callbacks.REASON_HIGH_LIMIT + assert result.state_unit_counts[2, 0] == 1 + assert result.state_next_add_index[2, 0] == 1 + assert result.state_common_stop[2, 0] == result.state_common_stop[1, 0] + + +def test_exit_sells_before_same_day_entry_uses_released_cash() -> None: + inputs = _inputs( + opens=[[10.0, 20.0], [12.0, 20.0], [12.0, 20.0]], + signal_close=[[11.0, np.nan], [5.0, 21.0], [np.nan, np.nan]], + entry_high=[[10.0, np.nan], [np.nan, 20.0], [np.nan, np.nan]], + exit_low=[[np.nan, np.nan], [6.0, np.nan], [np.nan, np.nan]], + ) + result = run_vectorbt_simulation(inputs, _config(initial_cash=10_005.0)) + day_two = result.portfolio.orders.records_readable.loc[ + lambda frame: frame["Timestamp"] == np.datetime64("2026-01-06") + ] + + assert day_two["Side"].tolist() == ["Sell", "Buy"] + assert result.state_quantities[1, 0] == 0 + assert result.state_unit_counts[1, 0] == 0 + assert result.state_quantities[1, 1] > 0 + + +def test_low_limit_blocks_full_exit_and_preserves_unit_state() -> None: + inputs = _inputs( + opens=[[10.0], [8.0]], + signal_close=[[11.0], [5.0]], + entry_high=[[10.0], [20.0]], + exit_low=[[np.nan], [6.0]], + low_limit=[[np.nan], [8.0]], + ) + result = run_vectorbt_simulation(inputs, _config()) + + assert result.reason_codes[1, 0] == callbacks.REASON_LOW_LIMIT + assert result.filled_quantities[1, 0] == 0 + assert result.state_quantities[1, 0] == result.state_quantities[0, 0] + assert result.state_unit_counts[1, 0] == 1 + assert result.state_common_stop[1, 0] == result.state_common_stop[0, 0] + + +def test_rejected_official_order_does_not_establish_candidate() -> None: + inputs = _inputs( + opens=[[10.0]], signal_close=[[11.0]], entry_high=[[10.0]] + ) + _, params = _params(_config()) + state = _mutable_state(1, 1, 1, 4) + state.action_codes[0, 0] = callbacks.ACTION_ENTRY + state.candidate_signal_n[0, 0] = 1.0 + state.candidate_base_quantity[0, 0] = 1000 + callback_inputs = callbacks.CallbackInputs( + inputs.execution_open, + inputs.signal_close, + inputs.signal_entry_high, + inputs.signal_exit_low, + inputs.signal_n, + inputs.paused, + inputs.high_limit, + inputs.low_limit, + inputs.asset_group_ids, + ) + context = SimpleNamespace( + i=0, + col=0, + call_idx=0, + group_len=1, + from_col=0, + to_col=1, + position_now=0.0, + last_position=np.zeros(1, dtype=np.float64), + order_result=OrderResult( + size=100.0, + price=10.0, + fees=5.0, + side=OrderSide.Buy, + status=OrderStatus.Rejected, + status_info=0, + ), + ) + + callbacks.post_order_func_nb.py_func(context, state, callback_inputs, params) + + assert state.reason_codes[0, 0] == callbacks.REASON_ORDER_REJECTED + assert state.unit_count[0] == 0 + assert state.state_quantities[0, 0] == 0 diff --git a/tests/local_quant_research/test_turtle_vectorbt_delayed.py b/tests/local_quant_research/test_turtle_vectorbt_delayed.py new file mode 100644 index 0000000..638d7b3 --- /dev/null +++ b/tests/local_quant_research/test_turtle_vectorbt_delayed.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np + + +RESEARCH_ROOT = ( + Path(__file__).resolve().parents[2] + / "joinquant" + / "strategies" + / "strategy-003" + / "research" +) +sys.path.insert(0, str(RESEARCH_ROOT)) + +from turtle_etf.vectorbt_callbacks import ( # noqa: E402 + ACTION_ENTRY, + ACTION_FULL_EXIT, + ACTION_REDISTRIBUTION_BUY, + ACTION_REDISTRIBUTION_SELL, + REASON_ENTRY_BREAKOUT, + REASON_FULL_POSITION_REDISTRIBUTION, + REASON_PROTECTIVE_STOP, +) +from turtle_etf.vectorbt_delayed import ( # noqa: E402 + ADJUST_CASH_TRUNCATED, + ADJUST_HOLDING_TRUNCATED, + ADJUST_NONE, + freeze_order_plan, + run_delayed_execution, +) + + +def _matrix(values: object, dtype: str) -> np.ndarray: + return np.asarray(values, dtype=dtype) + + +def _inputs(opens: list[list[float]]) -> SimpleNamespace: + open_values = _matrix(opens, "float64") + rows, columns = open_values.shape + return SimpleNamespace( + dates=np.arange( + np.datetime64("2026-01-05"), + np.datetime64("2026-01-05") + np.timedelta64(rows, "D"), + ).astype("datetime64[D]"), + securities=tuple(f"ETF-{chr(65 + column)}" for column in range(columns)), + execution_open=open_values, + close=np.where(np.isfinite(open_values), open_values, 10.0), + paused=np.zeros((rows, columns), dtype=np.bool_), + high_limit=np.full((rows, columns), np.nan), + low_limit=np.full((rows, columns), np.nan), + signal_n=np.ones((rows, columns), dtype=np.float64), + ) + + +def _immediate( + *, + rows: int, + columns: int, + orders: list[tuple[int, int, int, int, int]], +) -> SimpleNamespace: + action = np.zeros((rows, columns), dtype=np.int16) + reason = np.zeros((rows, columns), dtype=np.int16) + requested = np.zeros((rows, columns), dtype=np.int64) + planned = np.zeros((rows, columns), dtype=np.int64) + filled = np.zeros((rows, columns), dtype=np.int64) + for row, column, action_code, reason_code, quantity in orders: + action[row, column] = action_code + reason[row, column] = reason_code + requested[row, column] = quantity + planned[row, column] = quantity + filled[row, column] = quantity + return SimpleNamespace( + action_codes=action, + reason_codes=reason, + requested_quantities=requested, + planned_quantities=planned, + filled_quantities=filled, + ) + + +def _run( + inputs: SimpleNamespace, + immediate: SimpleNamespace, + *, + initial_cash: float = 100_000.0, +): + plan = freeze_order_plan(inputs, immediate) + return plan, run_delayed_execution( + inputs, + plan, + initial_cash=initial_cash, + lot_size=100, + stop_n=2.0, + commission_multiplier=1.0, + one_way_slippage=0.0, + delay_days=1, + ) + + +def test_delayed_execution_freezes_original_action_target_reason_and_signal_n() -> None: + inputs = _inputs([[10.0], [11.0], [12.0], [13.0]]) + inputs.signal_n[1, 0] = 1.5 + inputs.signal_n[2, 0] = 999.0 + immediate = _immediate( + rows=4, + columns=1, + orders=[(1, 0, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 200)], + ) + + plan, delayed = _run(inputs, immediate) + + assert plan.signal_n[1, 0] == 1.5 + assert delayed.action_codes[2, 0] == ACTION_ENTRY + assert delayed.reason_codes[2, 0] == REASON_ENTRY_BREAKOUT + assert delayed.planned_quantities[2, 0] == 200 + assert delayed.planned_row_indices[2, 0] == 1 + assert delayed.frozen_signal_n[2, 0] == 1.5 + assert delayed.state_common_stop[2, 0] == 12.0 - 2.0 * 1.5 + assert delayed.execution_adjustment_codes[2, 0] == ADJUST_NONE + + +def test_delayed_buy_only_uses_lot_cash_truncation() -> None: + inputs = _inputs([[10.0], [10.0], [200.0], [200.0]]) + immediate = _immediate( + rows=4, + columns=1, + orders=[(1, 0, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 200)], + ) + + _, delayed = _run(inputs, immediate, initial_cash=25_000.0) + + assert delayed.execution_adjustment_codes[2, 0] == ADJUST_CASH_TRUNCATED + assert delayed.filled_quantities[2, 0] == 100 + assert delayed.filled_quantities[2, 0] % 100 == 0 + assert delayed.filled_quantities[2, 0] < delayed.planned_quantities[2, 0] + + +def test_delayed_sell_is_mechanically_truncated_to_actual_holding() -> None: + inputs = _inputs([[10.0], [10.0], [10.0], [10.0], [10.0]]) + immediate = _immediate( + rows=5, + columns=1, + orders=[ + (0, 0, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 100), + (2, 0, ACTION_FULL_EXIT, REASON_PROTECTIVE_STOP, 200), + ], + ) + + _, delayed = _run(inputs, immediate) + + assert delayed.filled_quantities[1, 0] == 100 + assert delayed.planned_quantities[3, 0] == 200 + assert delayed.filled_quantities[3, 0] == 100 + assert delayed.execution_adjustment_codes[3, 0] == ADJUST_HOLDING_TRUNCATED + assert delayed.state_quantities[3, 0] == 0 + + +def test_delayed_queue_is_executed_in_original_priority_then_security_order() -> None: + inputs = _inputs( + [ + [10.0, 10.0], + [10.0, 10.0], + [10.0, 10.0], + [10.0, 10.0], + ] + ) + immediate = _immediate( + rows=4, + columns=2, + orders=[ + (1, 1, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 100), + (1, 0, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 100), + ], + ) + + _, delayed = _run(inputs, immediate) + + assert delayed.execution_sequence[2] == ( + "queued-from-row-1:ETF-A", + "queued-from-row-1:ETF-B", + ) + + +def test_vectorbt_ledger_uses_priority_sequence_not_inverse_column_ranks() -> None: + inputs = _inputs( + [ + [10.0, 10.0, 100.0], + [10.0, 10.0, 100.0], + [10.0, 10.0, 100.0], + [10.0, 10.0, 100.0], + ] + ) + immediate = _immediate( + rows=4, + columns=3, + orders=[ + (0, 2, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 100), + (1, 2, ACTION_FULL_EXIT, REASON_PROTECTIVE_STOP, 100), + (1, 0, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 100), + (1, 1, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 100), + ], + ) + + _, delayed = _run(inputs, immediate, initial_cash=10_005.0) + + assert delayed.execution_sequence[2] == ( + "queued-from-row-1:ETF-C", + "queued-from-row-1:ETF-A", + "queued-from-row-1:ETF-B", + ) + assert delayed.filled_quantities[2].tolist() == [100, 100, 100] + assert delayed.portfolio.orders.count() == 4 + + +def test_delayed_redistribution_keeps_units_and_stops_and_uses_priority() -> None: + inputs = _inputs( + [ + [10.0, 10.0, 10.0], + [10.0, 10.0, 10.0], + [11.0, 11.0, 11.0], + [11.0, 11.0, 11.0], + ] + ) + immediate = _immediate( + rows=4, + columns=3, + orders=[ + (0, 0, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 200), + (0, 2, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 200), + ( + 1, + 0, + ACTION_REDISTRIBUTION_SELL, + REASON_FULL_POSITION_REDISTRIBUTION, + 100, + ), + (1, 1, ACTION_ENTRY, REASON_ENTRY_BREAKOUT, 100), + ( + 1, + 2, + ACTION_REDISTRIBUTION_BUY, + REASON_FULL_POSITION_REDISTRIBUTION, + 100, + ), + ], + ) + + _, delayed = _run(inputs, immediate) + + assert delayed.execution_sequence[2] == ( + "queued-from-row-1:ETF-A", + "queued-from-row-1:ETF-B", + "queued-from-row-1:ETF-C", + ) + assert delayed.state_quantities[2].tolist() == [100, 100, 300] + assert delayed.state_unit_counts[2].tolist() == [1, 1, 1] + assert delayed.state_common_stop[2].tolist() == [8.0, 9.0, 8.0] + assert delayed.state_next_add_index[2].tolist() == [1, 1, 1] diff --git a/tests/local_quant_research/test_turtle_vectorbt_engine.py b/tests/local_quant_research/test_turtle_vectorbt_engine.py new file mode 100644 index 0000000..042c889 --- /dev/null +++ b/tests/local_quant_research/test_turtle_vectorbt_engine.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import hashlib +import importlib.metadata +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +RESEARCH_ROOT = ( + Path(__file__).resolve().parents[2] + / "joinquant" + / "strategies" + / "strategy-003" + / "research" +) +sys.path.insert(0, str(RESEARCH_ROOT)) + +from turtle_etf.vectorbt_callbacks import CallbackInputs, CallbackParams # noqa: E402 +from turtle_etf.vectorbt_engine import _params # noqa: E402 + + +def test_vectorbt_runtime_is_pinned_and_available(repo_root: Path) -> None: + requirements = { + line.strip() + for line in (repo_root / "requirements.txt").read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + shared_requirements = { + line.strip() + for line in ( + repo_root / ".agents/skills/joinquant-archive-sync/requirements.txt" + ).read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + assert "vectorbt==1.1.0" in requirements + assert "numpy==2.4.6" in shared_requirements + assert "pandas==3.0.3" in shared_requirements + assert not any(line.startswith(("numpy==", "pandas==")) for line in requirements) + assert importlib.util.find_spec("vectorbt") is not None + assert importlib.metadata.version("vectorbt") == "1.1.0" + + +def test_vectorbt_execution_identity_and_license_are_auditable(repo_root: Path) -> None: + research_root = repo_root / "joinquant/strategies/strategy-003/research" + callback_path = research_root / "turtle_etf/vectorbt_callbacks.py" + identity = json.loads( + (research_root / "code-identity.json").read_text(encoding="utf-8") + ) + execution = identity["execution"] + + assert execution == { + "backend": "vectorbt.Portfolio.from_order_func", + "delayed_backend": "vectorbt.Portfolio.from_orders", + "adapter_version": "local-vectorbt-adapter/2", + "dependencies": { + "vectorbt": "1.1.0", + "numba": "0.66.0", + "numpy": "2.4.6", + "pandas": "3.0.3", + }, + "callbacks_sha256": hashlib.sha256(callback_path.read_bytes()).hexdigest(), + "accounting": { + "version": "turtle-etf-corporate-actions/1", + "corporate_action_mode": "point_in_time_total_return_approximation", + "continuity_factor_basis": "raw_previous_close_over_current_pre_close", + "corporate_action_metadata_timing": "audit_only_may_be_retrospective", + "price_basis": "continuous_economic_price", + "quantity_basis": "economic_units", + "cash_dividend_mode": "implicit_reinvestment_on_ex_date", + "pay_date_cash_supported": False, + "exact_joinquant_reconciliation": False, + }, + "license": { + "expression": "Apache-2.0 WITH Commons-Clause", + "usage": "internal_research_only", + "resale_prohibited": True, + }, + } + identity_paths = {item["path"] for item in identity["files"]} + assert { + "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_inputs.py", + "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_callbacks.py", + "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_delayed.py", + "joinquant/strategies/strategy-003/research/turtle_etf/vectorbt_engine.py", + "scripts/research/market_data/economic_returns.py", + }.issubset(identity_paths) + + distribution = importlib.metadata.distribution("vectorbt") + license_file = next( + file for file in distribution.files or () if str(file).endswith("LICENSE.md") + ) + license_text = distribution.locate_file(license_file).read_text(encoding="utf-8") + assert "Apache License" in license_text + assert "Commons Clause" in license_text + + +def test_callback_contract_contains_only_strategy_inputs_and_risk_parameters() -> None: + config = { + "research": {"initial_cash": 1_000_000.0}, + "signal": {"add_step_n": 0.5, "stop_n": 2.0, "max_units": 4}, + "risk": { + "unit_risk_per_n": 0.005, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + }, + } + + _, params = _params(config) + + assert CallbackInputs._fields == ( + "execution_open", + "signal_close", + "signal_entry_high", + "signal_exit_low", + "signal_n", + "paused", + "high_limit", + "low_limit", + "asset_group_ids", + ) + assert CallbackParams._fields == ( + "lot_size", + "unit_risk_per_n", + "add_step_n", + "stop_n", + "max_units", + "asset_group_unit_cap", + "portfolio_unit_cap", + "commission_multiplier", + "one_way_slippage", + ) + assert params.max_units == 4 + assert params.asset_group_unit_cap == 6.0 + assert params.portfolio_unit_cap == 12.0 + + +def _new_config() -> dict[str, object]: + return { + "research": {"initial_cash": 1_000_000.0}, + "signal": {"add_step_n": 0.5, "stop_n": 2.0, "max_units": 4}, + "risk": { + "unit_risk_per_n": 0.01, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + }, + } + + +def test_max_units_requires_exactly_four() -> None: + config = _new_config() + + _, params = _params(config) + assert params.unit_risk_per_n == 0.01 + assert params.max_units == 4 + + for invalid in (None, True, False, 0, -1, 1.5, 3, 5): + config["signal"]["max_units"] = invalid + with pytest.raises(ValueError, match="max_units must equal four"): + _params(config) + + del config["signal"]["max_units"] + with pytest.raises(ValueError, match="missing config value: max_units"): + _params(config) + + +@pytest.mark.parametrize( + "legacy_field", + ( + "security_risk_cap", + "security_value_cap", + "asset_group_risk_cap", + "asset_group_value_cap", + "portfolio_risk_cap", + "portfolio_value_cap", + "covariance", + "target_volatility", + "risk_reduction_target_volatility", + "minimum_aligned_samples", + ), +) +def test_legacy_risk_fields_are_rejected(legacy_field: str) -> None: + config = _new_config() + config["risk"][legacy_field] = ( + {"method": "sample", "window_days": 60} + if legacy_field == "covariance" + else 1.0 + ) + + with pytest.raises(ValueError, match="legacy risk fields are not supported"): + _params(config) diff --git a/tests/local_quant_research/test_turtle_vectorbt_inputs.py b/tests/local_quant_research/test_turtle_vectorbt_inputs.py new file mode 100644 index 0000000..aeab905 --- /dev/null +++ b/tests/local_quant_research/test_turtle_vectorbt_inputs.py @@ -0,0 +1,481 @@ +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + + +RESEARCH_ROOT = ( + Path(__file__).resolve().parents[2] + / "joinquant" + / "strategies" + / "strategy-003" + / "research" +) +sys.path.insert(0, str(RESEARCH_ROOT)) + +from turtle_etf.vectorbt_engine import run_vectorbt_simulation # noqa: E402 +from turtle_etf.vectorbt_inputs import prepare_simulation_inputs # noqa: E402 + + +def _frame( + security: str, + close: list[float], + *, + money: float | None = 250_000_000.0, +) -> pd.DataFrame: + dates = pd.date_range("2026-01-05", periods=len(close), freq="D") + high = np.asarray(close, dtype=np.float64) + 1.0 + low = np.asarray(close, dtype=np.float64) - 1.0 + data: dict[str, object] = { + "date": dates.strftime("%Y-%m-%d"), + "security": security, + "open": np.asarray(close, dtype=np.float64) + 0.25, + "high": high, + "low": low, + "close": close, + "pre_close": np.r_[close[0], close[:-1]], + "paused": False, + "high_limit": high + 1.0, + "low_limit": low - 1.0, + } + if money is not None: + data["money"] = np.full(len(close), money, dtype=np.float64) + return pd.DataFrame(data) + + +def _config() -> dict[str, object]: + return { + "universe": [ + {"security": "ETF-B", "asset_group": "group-b"}, + {"security": "ETF-A", "asset_group": "group-a"}, + ], + "signal": {"entry_days": 2, "exit_days": 2, "n_days": 2}, + "risk": {}, + "execution": {"additional_delay_days": 0}, + } + + +def _simulation_config() -> dict[str, object]: + config = _config() + config["research"] = {"initial_cash": 1_000_000.0} + config["signal"] = { + **config["signal"], + "add_step_n": 0.5, + "stop_n": 2.0, + "max_units": 4, + } + config["risk"] = { + "lot_size": 100, + "unit_risk_per_n": 0.005, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + } + config["costs"] = { + "commission_multiplier": 1.0, + "one_way_slippage": 0.0, + } + return config + + +def _single_config() -> dict[str, object]: + config = _config() + config["universe"] = [{"security": "ETF-A", "asset_group": "group-a"}] + return config + + +def _corporate_action( + *, + event_type: str = "split", + announcement_date: str = "2026-01-06", + effective_date: str = "2026-01-07", + status: str = "active", + split_ratio: float | None = 2.0, + cash_per_share: float | None = None, +) -> dict[str, object]: + return { + "source_event_id": "FUND_DIVIDEND:101", + "security": "ETF-A", + "event_type": event_type, + "announcement_date": announcement_date, + "record_date": "2026-01-06", + "ex_date": effective_date, + "effective_date": effective_date, + "pay_date": "2026-01-09" if event_type == "cash_dividend" else None, + "status": status, + "knowledge_cutoff_date": "2026-01-10", + "split_ratio": split_ratio, + "cash_per_share": cash_per_share, + "source": "joinquant.finance.FUND_DIVIDEND", + "source_record_sha256": "b" * 64, + } + + +def _corporate_action_frame() -> pd.DataFrame: + return pd.DataFrame( + { + "date": pd.date_range("2026-01-05", periods=5, freq="D").strftime( + "%Y-%m-%d" + ), + "security": "ETF-A", + "open": [100.0, 101.0, 50.75, 51.5, 52.0], + "high": [101.0, 103.0, 52.0, 53.0, 54.0], + "low": [99.0, 100.0, 50.0, 51.0, 52.0], + "close": [100.0, 102.0, 51.0, 52.0, 53.0], + "pre_close": [100.0, 100.0, 51.0, 51.0, 52.0], + "paused": False, + "high_limit": [110.0, 112.0, 56.0, 57.0, 58.0], + "low_limit": [90.0, 92.0, 46.0, 47.0, 48.0], + } + ) + + +def _actions_digest(actions: list[dict[str, object]]) -> str: + payload = json.dumps( + actions, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def test_simulation_inputs_are_aligned_stable_and_read_only() -> None: + frames = { + "ETF-B": _frame("ETF-B", [20.0, 21.0, 22.0, 23.0, 24.0, 25.0]), + "ETF-A": _frame("ETF-A", [9.0, 10.0, 13.0, 12.0, 14.0, 15.0]), + } + + inputs = prepare_simulation_inputs(frames, _config()) + + assert inputs.securities == ("ETF-A", "ETF-B") + assert inputs.asset_groups == ("group-a", "group-b") + assert inputs.dates.dtype == np.dtype("datetime64[D]") + assert inputs.asset_group_ids.dtype == np.dtype("int64") + assert inputs.paused.dtype == np.dtype("bool") + assert inputs.close.dtype == np.dtype("float64") + assert inputs.close.flags.c_contiguous + for value in vars(inputs).values(): + if isinstance(value, np.ndarray): + assert not value.flags.writeable + with pytest.raises(ValueError): + inputs.close[0, 0] = 999.0 + + +def test_additional_execution_delay_does_not_move_signal_inputs() -> None: + frames = { + "ETF-B": _frame("ETF-B", [20.0, 21.0, 22.0, 23.0, 24.0, 25.0]), + "ETF-A": _frame("ETF-A", [9.0, 10.0, 13.0, 12.0, 14.0, 15.0]), + } + immediate_config = _config() + delayed_config = _config() + delayed_config["execution"] = {"additional_delay_days": 1} + + immediate = prepare_simulation_inputs(frames, immediate_config) + delayed = prepare_simulation_inputs(frames, delayed_config) + + assert np.array_equal(delayed.signal_source_index, immediate.signal_source_index) + assert np.array_equal(delayed.signal_close, immediate.signal_close, equal_nan=True) + assert np.array_equal( + delayed.signal_entry_high, immediate.signal_entry_high, equal_nan=True + ) + assert np.array_equal( + delayed.signal_exit_low, immediate.signal_exit_low, equal_nan=True + ) + assert np.array_equal(delayed.signal_n, immediate.signal_n, equal_nan=True) + + +def test_signal_inputs_are_shifted_to_execution_row_without_future_data() -> None: + frames = { + "ETF-B": _frame("ETF-B", [20.0, 21.0, 22.0, 23.0, 24.0, 25.0]), + "ETF-A": _frame("ETF-A", [9.0, 10.0, 13.0, 12.0, 14.0, 15.0]), + } + original = prepare_simulation_inputs(frames, _config()) + changed_frames = {key: value.copy() for key, value in frames.items()} + changed_frames["ETF-A"].loc[3, "close"] = 999.0 + changed_frames["ETF-A"].loc[4, "pre_close"] = 999.0 + changed = prepare_simulation_inputs(changed_frames, _config()) + + execution_row = 3 + asset = original.securities.index("ETF-A") + assert original.signal_source_index[execution_row] == 2 + assert original.signal_close[execution_row, asset] == 13.0 + assert original.signal_entry_high[execution_row, asset] == 11.0 + assert original.signal_close[execution_row, asset] == changed.signal_close[execution_row, asset] + assert original.signal_entry_high[execution_row, asset] == changed.signal_entry_high[execution_row, asset] + assert original.close[execution_row, asset] != changed.close[execution_row, asset] + + +def test_market_money_is_outside_the_turtle_order_contract() -> None: + close_a = [10.0] * 20 + [13.0, 13.0, 13.0] + close_b = [20.0] * len(close_a) + + def simulate(money: float | None): + frames = { + "ETF-A": _frame("ETF-A", close_a, money=money), + "ETF-B": _frame("ETF-B", close_b, money=money), + } + return run_vectorbt_simulation( + prepare_simulation_inputs(frames, _simulation_config()), + _simulation_config(), + ) + + low = simulate(1.0) + high = simulate(1_000_000_000_000.0) + assert np.array_equal(low.action_codes, high.action_codes) + assert np.array_equal(low.filled_quantities, high.filled_quantities) + assert low.portfolio.orders.count() == high.portfolio.orders.count() + + missing = simulate(None) + assert np.array_equal(low.action_codes, missing.action_codes) + assert np.array_equal(low.filled_quantities, missing.filled_quantities) + assert low.portfolio.orders.count() == missing.portfolio.orders.count() + assert int(low.filled_quantities.sum()) > 0 + + +def test_simulation_inputs_expose_only_strategy_and_risk_arrays() -> None: + frames = { + "ETF-B": _frame("ETF-B", [20.0, 21.0, 22.0, 23.0, 24.0, 25.0]), + "ETF-A": _frame("ETF-A", [9.0, 10.0, 13.0, 12.0, 14.0, 15.0]), + } + + inputs = prepare_simulation_inputs(frames, _config()) + + assert tuple(vars(inputs)) == ( + "dates", + "securities", + "asset_groups", + "asset_group_ids", + "raw_open", + "raw_high", + "raw_low", + "raw_close", + "raw_pre_close", + "continuous_open", + "continuous_high", + "continuous_low", + "continuous_close", + "continuous_pre_close", + "continuity_factor", + "corporate_action_applied", + "corporate_actions_digest", + "corporate_action_applications", + "paused", + "high_limit", + "low_limit", + "signal_source_index", + "signal_close", + "signal_entry_high", + "signal_exit_low", + "signal_n", + ) + + +def test_split_uses_a_forward_only_continuity_factor_and_economic_units() -> None: + actions = [_corporate_action()] + digest = _actions_digest(actions) + inputs = prepare_simulation_inputs( + {"ETF-A": _corporate_action_frame()}, + _single_config(), + corporate_actions=actions, + corporate_actions_digest=digest, + ) + + assert np.array_equal(inputs.raw_close[:, 0], [100.0, 102.0, 51.0, 52.0, 53.0]) + assert np.array_equal(inputs.continuity_factor[:, 0], [1.0, 1.0, 2.0, 2.0, 2.0]) + assert np.array_equal(inputs.continuous_close[:, 0], [100.0, 102.0, 102.0, 104.0, 106.0]) + assert np.array_equal(inputs.continuous_pre_close[:, 0], [100.0, 100.0, 102.0, 102.0, 104.0]) + assert np.array_equal(inputs.corporate_action_applied[:, 0], [False, False, True, False, False]) + assert inputs.corporate_actions_digest == digest + assert len(inputs.corporate_action_applications) == 1 + application = inputs.corporate_action_applications[0] + assert application.source_event_id == "FUND_DIVIDEND:101" + assert application.event_type == "split" + assert application.security == "ETF-A" + assert application.effective_date == "2026-01-07" + assert application.application_date == "2026-01-07" + assert application.split_ratio == 2.0 + assert application.cash_per_share is None + assert application.cumulative_factor == 2.0 + assert application.price_basis_changed is True + assert application.evidence_timing == "point_in_time" + assert inputs.execution_open is inputs.continuous_open + assert inputs.close is inputs.continuous_close + + +def test_cash_dividend_is_implicit_total_return_without_pay_date_cash() -> None: + frame = _corporate_action_frame() + frame.loc[2:, ["open", "high", "low", "close", "pre_close", "high_limit", "low_limit"]] = [ + [9.55, 9.7, 9.4, 9.6, 9.5, 10.45, 8.55], + [9.7, 9.9, 9.6, 9.8, 9.6, 10.56, 8.64], + [9.9, 10.1, 9.8, 10.0, 9.8, 10.78, 8.82], + ] + frame.loc[0:1, ["open", "high", "low", "close", "pre_close", "high_limit", "low_limit"]] = [ + [9.8, 10.1, 9.7, 9.9, 9.9, 10.89, 8.91], + [9.9, 10.2, 9.8, 10.0, 9.9, 10.89, 8.91], + ] + action = _corporate_action( + event_type="cash_dividend", + split_ratio=None, + cash_per_share=0.5, + ) + + inputs = prepare_simulation_inputs( + {"ETF-A": frame}, + _single_config(), + corporate_actions=[action], + ) + + factor = 10.0 / 9.5 + assert inputs.continuity_factor[2, 0] == pytest.approx(factor) + assert inputs.continuous_pre_close[2, 0] == pytest.approx(10.0) + assert inputs.continuous_close[2, 0] == pytest.approx(9.6 * factor) + assert inputs.corporate_action_applied[2, 0] + assert not inputs.corporate_action_applied[4, 0] + + +def test_cash_dividend_factor_uses_raw_market_prices_not_cash_metadata() -> None: + frame = _corporate_action_frame() + frame.loc[1, "close"] = 140.847 + frame.loc[2, "pre_close"] = 140.193 + frame.loc[2:, ["open", "high", "low", "close", "high_limit", "low_limit"]] = [ + [140.2, 140.3, 140.1, 140.285, 154.212, 126.174], + [140.3, 140.4, 140.2, 140.35, 154.385, 126.225], + [140.4, 140.5, 140.3, 140.45, 154.495, 126.315], + ] + frame.loc[3, "pre_close"] = 140.285 + frame.loc[4, "pre_close"] = 140.35 + + inputs = prepare_simulation_inputs( + {"ETF-A": frame}, + _single_config(), + corporate_actions=[ + _corporate_action( + event_type="cash_dividend", + split_ratio=None, + cash_per_share=0.6542, + ) + ], + ) + + assert inputs.continuity_factor[2, 0] == pytest.approx(140.847 / 140.193) + + +def test_split_on_paused_effective_date_applies_on_first_resumed_market_row() -> None: + frame = _corporate_action_frame() + frame.loc[2, ["open", "high", "low", "close", "pre_close", "high_limit", "low_limit"]] = [ + 102.0, + 102.0, + 102.0, + 102.0, + 102.0, + 102.0, + 102.0, + ] + frame.loc[2, "paused"] = True + frame.loc[3, ["open", "high", "low", "close", "pre_close", "high_limit", "low_limit"]] = [ + 51.25, + 52.0, + 50.5, + 51.5, + 51.0, + 56.0, + 46.0, + ] + frame.loc[4, "pre_close"] = 51.5 + + inputs = prepare_simulation_inputs( + {"ETF-A": frame}, + _single_config(), + corporate_actions=[_corporate_action()], + ) + + application = inputs.corporate_action_applications[0] + assert application.effective_date == "2026-01-07" + assert application.application_date == "2026-01-08" + assert application.price_basis_changed is True + assert inputs.continuity_factor[2, 0] == pytest.approx(1.0) + assert inputs.continuity_factor[3, 0] == pytest.approx(2.0) + + +def test_active_event_without_price_basis_change_is_audited_without_factor() -> None: + frame = _frame("ETF-A", [100.0, 101.0, 102.0, 103.0, 104.0]) + inputs = prepare_simulation_inputs( + {"ETF-A": frame}, + _single_config(), + corporate_actions=[_corporate_action(split_ratio=1.46301)], + ) + + application = inputs.corporate_action_applications[0] + assert application.application_date == "2026-01-07" + assert application.price_basis_changed is False + assert application.cumulative_factor == pytest.approx(1.0) + assert np.array_equal(inputs.continuity_factor[:, 0], np.ones(5)) + + +@pytest.mark.parametrize( + ("action", "message"), + [ + (_corporate_action(status="cancelled"), "evidence_insufficient"), + ], +) +def test_invalid_or_unreconciled_corporate_action_closes_the_run( + action: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + prepare_simulation_inputs( + {"ETF-A": _corporate_action_frame()}, + _single_config(), + corporate_actions=[action], + ) + + +def test_official_split_ratio_is_audit_metadata_not_factor_input() -> None: + inputs = prepare_simulation_inputs( + {"ETF-A": _corporate_action_frame()}, + _single_config(), + corporate_actions=[_corporate_action(split_ratio=3.0)], + ) + + application = inputs.corporate_action_applications[0] + assert application.split_ratio == 3.0 + assert application.cumulative_factor == pytest.approx(2.0) + + +def test_late_action_metadata_is_only_retrospective_reconciliation() -> None: + inputs = prepare_simulation_inputs( + {"ETF-A": _corporate_action_frame()}, + _single_config(), + corporate_actions=[_corporate_action(announcement_date="2026-01-08")], + ) + + application = inputs.corporate_action_applications[0] + assert application.evidence_timing == "retrospective_reconciliation" + assert inputs.continuity_factor[2, 0] == pytest.approx(2.0) + + +def test_unexplained_price_basis_change_closes_the_run() -> None: + with pytest.raises(ValueError, match="evidence_insufficient"): + prepare_simulation_inputs( + {"ETF-A": _corporate_action_frame()}, + _single_config(), + corporate_actions=[], + ) + + +def test_corporate_action_digest_mismatch_closes_the_run() -> None: + with pytest.raises(ValueError, match="evidence_insufficient"): + prepare_simulation_inputs( + {"ETF-A": _corporate_action_frame()}, + _single_config(), + corporate_actions=[_corporate_action()], + corporate_actions_digest="0" * 64, + ) + diff --git a/tests/local_quant_research/test_turtle_vectorbt_performance.py b/tests/local_quant_research/test_turtle_vectorbt_performance.py new file mode 100644 index 0000000..b07602b --- /dev/null +++ b/tests/local_quant_research/test_turtle_vectorbt_performance.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +RESEARCH_ROOT = ( + Path(__file__).resolve().parents[2] + / "joinquant" + / "strategies" + / "strategy-003" + / "research" +) +sys.path.insert(0, str(RESEARCH_ROOT)) + +from turtle_etf import vectorbt_benchmark # noqa: E402 +from turtle_etf.vectorbt_inputs import CorporateActionApplication # noqa: E402 + + +def test_benchmark_runs_cold_and_warm_once_then_cleans_temp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + simulations = iter(("cold-simulation", "warm-simulation")) + facts = SimpleNamespace(name="same-facts") + run_calls: list[str] = [] + + def run(inputs: object, config: object) -> str: + value = next(simulations) + run_calls.append(value) + return value + + monkeypatch.setattr(vectorbt_benchmark, "run_vectorbt_simulation", run) + monkeypatch.setattr( + vectorbt_benchmark, + "to_joinquant_facts", + lambda inputs, simulation, scenario_id: facts, + ) + + def materialize(path: Path, value: object) -> str: + path.mkdir(parents=True) + (path / "evidence.bin").write_bytes(b"same") + return "e" * 64 + + monkeypatch.setattr( + vectorbt_benchmark, "materialize_execution_facts", materialize + ) + ticks = iter((10.0, 11.5, 20.0, 20.4)) + monkeypatch.setattr(vectorbt_benchmark.time, "perf_counter", lambda: next(ticks)) + work = tmp_path / "benchmark-work" + + result = vectorbt_benchmark.benchmark_scenario( + prepared_inputs=SimpleNamespace( + identity="prepared", + corporate_action_applications=( + CorporateActionApplication( + source_event_id="FUND_DIVIDEND:101", + security="ETF-A", + event_type="split", + effective_date="2026-01-05", + application_date="2026-01-06", + announcement_date="2026-01-05", + knowledge_cutoff_date="2026-01-10", + evidence_timing="point_in_time", + split_ratio=2.0, + cash_per_share=None, + cumulative_factor=2.0, + price_basis_changed=True, + source="joinquant.finance.FUND_DIVIDEND", + source_record_sha256="b" * 64, + ), + ), + ), + config={"scenario_id": "baseline", "research": {"initial_cash": 1}}, + scenario_id="baseline", + work_dir=work, + code_sha256="a" * 64, + config_sha256="b" * 64, + ) + + assert run_calls == ["cold-simulation", "warm-simulation"] + assert result.facts is facts + assert result.performance["cold_seconds"] == 1.5 + assert result.performance["warm_seconds"] == pytest.approx(0.4) + assert result.performance["cold_result_sha256"] == "e" * 64 + assert result.performance["warm_result_sha256"] == "e" * 64 + assert result.performance["result_match"] is True + assert result.performance["limit_seconds"] == 180.0 + assert result.performance["cleanup"] == { + "cold_temporary_result_removed": True, + "warm_temporary_result_removed": True, + "work_directory_removed": True, + "verified": True, + } + assert not work.exists() + + +def test_benchmark_rejects_nondeterminism_and_cleans_temp( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + vectorbt_benchmark, + "run_vectorbt_simulation", + lambda inputs, config: object(), + ) + monkeypatch.setattr( + vectorbt_benchmark, + "to_joinquant_facts", + lambda inputs, simulation, scenario_id: object(), + ) + digests = iter(("c" * 64, "d" * 64)) + + def materialize(path: Path, facts: object) -> str: + path.mkdir(parents=True) + return next(digests) + + monkeypatch.setattr( + vectorbt_benchmark, "materialize_execution_facts", materialize + ) + work = tmp_path / "benchmark-work" + + with pytest.raises(vectorbt_benchmark.PerformanceGateError, match="deterministic"): + vectorbt_benchmark.benchmark_scenario( + prepared_inputs=SimpleNamespace(identity="prepared"), + config={"scenario_id": "baseline"}, + scenario_id="baseline", + work_dir=work, + code_sha256="a" * 64, + config_sha256="b" * 64, + ) + + assert not work.exists() + + +def test_benchmark_rejects_either_run_over_180_seconds( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + vectorbt_benchmark, "run_vectorbt_simulation", lambda inputs, config: object() + ) + monkeypatch.setattr( + vectorbt_benchmark, + "to_joinquant_facts", + lambda inputs, simulation, scenario_id: object(), + ) + + def materialize(path: Path, facts: object) -> str: + path.mkdir(parents=True) + return "e" * 64 + + monkeypatch.setattr( + vectorbt_benchmark, "materialize_execution_facts", materialize + ) + ticks = iter((0.0, 180.1, 200.0, 200.2)) + monkeypatch.setattr(vectorbt_benchmark.time, "perf_counter", lambda: next(ticks)) + + with pytest.raises(vectorbt_benchmark.PerformanceGateError, match="180"): + vectorbt_benchmark.benchmark_scenario( + prepared_inputs=SimpleNamespace(identity="prepared"), + config={"scenario_id": "baseline"}, + scenario_id="baseline", + work_dir=tmp_path / "benchmark-work", + code_sha256="a" * 64, + config_sha256="b" * 64, + ) diff --git a/tests/quant_analysis/test_analysis_plan.py b/tests/quant_analysis/test_analysis_plan.py new file mode 100644 index 0000000..615c585 --- /dev/null +++ b/tests/quant_analysis/test_analysis_plan.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from scripts.research.quant_analysis.analysis_plan import ( + AnalysisPlanError, + expand_analysis_plan, +) +from scripts.research.quant_analysis.orchestration import ( + build_scenario_run_documents, +) + + +PLAN_PATH = Path("joinquant/strategies/strategy-003/research/analysis-plan.json") + + +def _load_plan(repo_root: Path) -> dict[str, object]: + return json.loads((repo_root / PLAN_PATH).read_text(encoding="utf-8")) + + +def test_expands_baseline_and_six_challenges_deterministically(repo_root: Path) -> None: + first = expand_analysis_plan(repo_root, PLAN_PATH) + second = expand_analysis_plan(repo_root, PLAN_PATH) + + assert first == second + assert first["schema_version"] == "analysis-scenarios/1" + assert first["strategy_id"] == "strategy-003" + assert first["expected"] == { + "scenario_runs": 7, + "benchmarks": 2, + "bootstrap_paths": 10000, + "seed": 20260714, + } + assert [item["scenario_id"] for item in first["scenarios"]] == [ + "baseline", + "entry-40", + "entry-60", + "stop-1-5n", + "stop-2-5n", + "group-unit-cap-5", + "portfolio-unit-cap-10", + ] + assert first["scenarios"][0]["params"]["scenario_id"] == "baseline" + assert first["scenarios"][1]["params"]["signal"]["entry_days"] == 40 + assert first["scenarios"][5]["params"]["risk"][ + "asset_group_unit_cap" + ] == 5.0 + assert first["scenarios"][6]["params"]["risk"][ + "portfolio_unit_cap" + ] == 10.0 + assert len({item["params_sha256"] for item in first["scenarios"]}) == 7 + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda plan: plan["scenarios"].append(copy.deepcopy(plan["scenarios"][0])), "unique"), + (lambda plan: plan["expected"].update({"scenario_runs": 8}), "scenario_runs"), + (lambda plan: plan["expected"].update({"bootstrap_paths": 9999}), "bootstrap_paths"), + (lambda plan: plan["expected"].update({"seed": 1}), "seed"), + (lambda plan: plan["scenarios"][0].update({"overrides": {"signal": {"entry_days": 1}}}), "baseline"), + ], +) +def test_rejects_inconsistent_plan( + repo_root: Path, + tmp_path: Path, + mutation, + message: str, +) -> None: + plan = _load_plan(repo_root) + mutation(plan) + path = tmp_path / "analysis-plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + + with pytest.raises(AnalysisPlanError, match=message): + expand_analysis_plan(repo_root, path) + + +def test_rejects_baseline_path_outside_repository(repo_root: Path, tmp_path: Path) -> None: + plan = _load_plan(repo_root) + plan["baseline_config"] = str(tmp_path / "outside.json") + path = tmp_path / "analysis-plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + + with pytest.raises(AnalysisPlanError, match="baseline_config"): + expand_analysis_plan(repo_root, path) + + +def test_rejects_false_stop_failure_flag_without_market_shocks( + repo_root: Path, + tmp_path: Path, +) -> None: + plan = _load_plan(repo_root) + plan["analyses"]["position_shocks"][0] = { + "id": "invalid-stop-failure", + "use_stop_failure_loss": False, + "maximum_loss_abs_max": 0.15, + } + path = tmp_path / "analysis-plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + + with pytest.raises(AnalysisPlanError, match="schema validation"): + expand_analysis_plan(repo_root, path) + + +def test_builds_seven_independent_public_runner_configs(repo_root: Path) -> None: + expanded = expand_analysis_plan(repo_root, PLAN_PATH) + template = json.loads( + ( + repo_root + / "joinquant/strategies/strategy-003/research/project-run.json" + ).read_text(encoding="utf-8") + ) + + documents = build_scenario_run_documents( + expanded, + template, + preparation_id="a" * 64, + ) + + assert len(documents) == 7 + assert [item["scenario_id"] for item in documents] == [ + item["scenario_id"] for item in expanded["scenarios"] + ] + for item in documents: + scenario_id = item["scenario_id"] + assert item["params"]["scenario_id"] == scenario_id + assert item["run_config"]["project_config"] == ( + f".local/strategy-analysis-preparations/{'a' * 64}/scenario-configs/" + f"{scenario_id}/params.json" + ) + assert item["run_config"]["required_outputs"] == [ + {"path": f"backtests/local-{scenario_id}", "format": "directory"} + ] + encoded = json.dumps(item["run_config"], sort_keys=True) + assert "analysis-plan" not in encoded + assert "candidates" not in encoded diff --git a/tests/quant_analysis/test_reporting.py b/tests/quant_analysis/test_reporting.py new file mode 100644 index 0000000..78f0804 --- /dev/null +++ b/tests/quant_analysis/test_reporting.py @@ -0,0 +1,453 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from scripts.research.quant_analysis.reporting import ( + build_recommendation, + enforce_vibe_boundary, + render_analysis_report, + write_analysis_delivery, +) + + +def test_vibe_swarm_is_recorded_as_invalid_and_excluded() -> None: + evidence = { + "capabilities_loaded": ["performance-attribution", "risk-analysis"], + "forbidden_capabilities_called": [], + "swarm": { + "run_id": "swarm-1", + "preset": "portfolio_review_board", + "status": "running_without_progress", + "final_report": None, + }, + "use_result": {}, + } + + corrected = enforce_vibe_boundary(evidence) + + assert corrected["forbidden_capabilities_called"] == [ + "run_swarm:portfolio_review_board" + ] + assert corrected["swarm"]["valid_evidence"] is False + assert corrected["swarm"]["excluded_from_conclusions"] is True + assert corrected["boundary_violation"]["occurred"] is True + assert corrected["use_result"]["vibe_conclusion_available"] is False + assert corrected["use_result"]["loaded_capabilities_are_methodology_only"] is True + assert corrected["next_action"] == "generate_deterministic_local_report" + + +def test_vibe_not_called_is_recorded_without_a_boundary_violation() -> None: + corrected = enforce_vibe_boundary( + { + "capabilities_loaded": [], + "forbidden_capabilities_called": [], + "use_result": {"vibe_called": False}, + } + ) + + assert corrected["forbidden_capabilities_called"] == [] + assert "boundary_violation" not in corrected + assert corrected["use_result"]["vibe_called"] is False + assert corrected["use_result"]["vibe_conclusion_available"] is False + assert corrected["use_result"]["loaded_capabilities_are_methodology_only"] is False + assert corrected["use_result"]["reason"] == "本次未调用 Vibe。" + + +def test_vibe_public_single_agent_is_valid_qualitative_audit_evidence() -> None: + corrected = enforce_vibe_boundary( + { + "capabilities_loaded": [ + "performance-attribution", + "risk-analysis", + "report-generate", + ], + "forbidden_capabilities_called": [], + "single_agent": { + "interface": "vibe-trading-cli-run", + "run_id": "vibe-run-1", + "status": "completed", + "assessment": { + "status": "completed", + "recommendation_alignment": "建议修改后再评估。", + }, + }, + } + ) + + assert corrected["single_agent"]["valid_evidence"] is True + assert corrected["single_agent"]["qualitative_only"] is True + assert corrected["use_result"]["vibe_called"] is True + assert corrected["use_result"]["vibe_conclusion_available"] is True + assert corrected["use_result"]["loaded_capabilities_are_methodology_only"] is False + assert corrected["authority"] == "audit_only" + assert "boundary_violation" not in corrected + + +def test_report_includes_real_vibe_single_agent_review_without_numerical_authority() -> None: + vibe = enforce_vibe_boundary( + { + "capabilities_loaded": ["performance-attribution", "risk-analysis"], + "forbidden_capabilities_called": [], + "single_agent": { + "interface": "vibe-trading-cli-run", + "run_id": "vibe-run-1", + "status": "completed", + "assessment": { + "status": "completed", + "recommendation_alignment": "修改后再评估与证据一致。", + }, + }, + } + ) + + report = render_analysis_report(_analysis(), build_recommendation(_analysis()), vibe) + + assert "Vibe 单体复核" in report + assert "vibe-run-1" in report + assert "修改后再评估与证据一致" in report + assert "不替代确定性数值裁判" in report + + +def _analysis() -> dict[str, object]: + return { + "analysis_id": "analysis-1", + "strategy_id": "strategy-003", + "authority": "local_exploratory", + "not_formal_joinquant_backtest": True, + "analysis_seconds": 1.5, + "baseline": { + "status": "fail", + "reasons": ["calmar_min"], + "metrics": { + "cumulative_return": 0.293995, + "cagr": 0.019105, + "max_drawdown": -0.100971, + "calmar": 0.18921, + "sharpe": 0.3901, + "sortino": 0.5111, + "annualized_volatility": 0.05201, + "max_drawdown_duration": 946, + }, + "risk_control": { + "average_invested_ratio": 0.263, + "median_invested_ratio": 0.303, + "below_half_ratio": 0.855, + "near_full_ratio": 0.0017, + "average_cash_ratio": 0.737, + "maximum_invested_ratio": 1.0, + "maximum_security_weight": 0.4, + "maximum_asset_group_weight": 0.5, + "planned_risk_coverage": 1.0, + "maximum_planned_loss_ratio": 0.0325, + "maximum_effective_risk_units": 12.0, + "maximum_portfolio_unit_utilization": 1.0, + "maximum_realized_60d_volatility": 0.1835, + "filled_order_count": 438, + "closed_order_count": 201, + "closed_order_win_rate": 0.5423, + "fees": 7454.77, + "protective_stop_events": 65, + "redistribution_event_count": 3245, + }, + }, + "benchmarks": { + "alignment": {"common_samples": 3276}, + "statistics": { + "CSI300_CNY_TOTAL_RETURN": { + "strategy_return": 0.3309, + "benchmark_return": 0.8766, + "active_return": -0.5457, + "alpha": 0.0175, + "beta": 0.0789, + "correlation": 0.3527, + "information_ratio": -0.2389, + "up_capture": 0.0262, + "down_capture": 0.1808, + } + }, + }, + "attribution": { + "method": "arithmetic", + "limitation": "not geometric linking", + "reconciliation_error": 0.0, + "security": [{"key": "513100.XSHG", "contribution": 0.11}], + "asset_group": [{"key": "equity", "contribution": 0.14}], + "trading_reason": [{"key": "entry_breakout", "contribution": 0.15}], + "period": [{"key": "2024", "contribution": 0.06}], + "event_counts": {"entry_breakout": 99}, + }, + "challenge_results": [ + { + "scenario_id": "baseline", + "status": "fail", + "metrics": { + "cumulative_return": 0.294, + "cagr": 0.0191, + "max_drawdown": -0.101, + "calmar": 0.189, + "average_invested_ratio": 0.263, + }, + "reasons": ["calmar_min"], + "cold_seconds": 28.4, + "warm_seconds": 3.7, + }, + { + "scenario_id": "covariance-ewma-30d", + "status": "fail", + "metrics": { + "cumulative_return": 0.3338, + "cagr": 0.0214, + "max_drawdown": -0.0956, + "calmar": 0.224, + "average_invested_ratio": 0.2517, + }, + "reasons": ["calmar_min"], + "cold_seconds": 27.9, + "warm_seconds": 3.5, + }, + ], + "robustness": { + "periods": [ + { + "scenario_id": "period-a", + "dimension": "fixed_period", + "status": "fail", + "metrics": {"cagr": 0.01, "max_drawdown": -0.05, "calmar": 0.2}, + "reasons": ["calmar_min"], + } + ], + "asset_deletions": [], + "cost_execution": [], + "bootstrap": [], + "historical_stress": [], + "position_shocks": [ + { + "scenario_id": "shock-stop-failure", + "dimension": "position_shock", + "status": "evidence_insufficient", + "metrics": {}, + "reasons": ["missing_source_input"], + } + ], + "cvar": [], + }, + "evidence_matrix": { + "rows": 3, + "pass": 0, + "fail": 2, + "evidence_insufficient": 1, + }, + "opposing_evidence": [ + { + "kind": "benchmark_underperformance", + "benchmark_id": "CSI300_CNY_TOTAL_RETURN", + "active_return": -0.5457, + } + ], + } + + +def test_recommendation_keeps_best_failed_challenge_as_iteration_only() -> None: + recommendation = build_recommendation(_analysis()) + + assert recommendation["decision"] == "revise_and_reassess" + assert recommendation["recommended_iteration_candidate"] == "covariance-ewma-30d" + assert recommendation["candidate_accepted"] is False + assert recommendation["next_action"] == "human_confirmation_required" + + +def test_report_is_complete_and_explicitly_excludes_vibe_group_analysis() -> None: + vibe = enforce_vibe_boundary( + { + "capabilities_loaded": ["report-generate"], + "swarm": {"preset": "portfolio_review_board", "run_id": "swarm-1"}, + } + ) + recommendation = build_recommendation(_analysis()) + + report = render_analysis_report(_analysis(), recommendation, vibe) + + for heading in ( + "结论与推荐", + "收益与回撤", + "双基准与 Alpha/Beta", + "仓位与风险控制", + "归因分析", + "基础场景挑战", + "稳健性与压力测试", + "反对证据", + "Vibe 安全边界", + "不确定性", + "人工确认", + ): + assert heading in report + assert "covariance-ewma-30d" in report + assert "群体分析" in report + assert "排除" in report + assert "既有路径切片" in report + assert "不重新分配资金" in report + assert "连续经济总回报近似" in report + assert "公司行动元数据只用于审计" in report + assert "经济单位不等同于真实 ETF 份额" in report + assert "不能与聚宽逐日账户精确对账" in report + assert "最高计划损失比例" in report + assert "最高有效 N 风险单位" in report + assert "组合单位预算最高利用率" in report + assert "全量仓位再分配事件" in report + assert "高于入场上限" not in report + assert "高于目标" not in report + assert "human_confirmation_required" in report + + +def test_report_states_vibe_was_not_called_and_uses_actual_stop_shock_status() -> None: + analysis = _analysis() + analysis["robustness"]["position_shocks"][0] = { + "scenario_id": "shock-stop-failure", + "dimension": "position_shock", + "status": "pass", + "metrics": {"maximum_loss": -0.08}, + "reasons": [], + } + vibe = enforce_vibe_boundary( + { + "capabilities_loaded": [], + "forbidden_capabilities_called": [], + "use_result": {"vibe_called": False}, + } + ) + + report = render_analysis_report(analysis, build_recommendation(analysis), vibe) + + assert "本次未调用 Vibe 群体分析" in report + assert "本次误调用" not in report + assert "`shock-stop-failure` 缺少所需来源输入" not in report + + +def test_report_adapts_to_all_passed_results_and_positive_benchmarks() -> None: + analysis = _analysis() + analysis["baseline"]["status"] = "pass" + analysis["baseline"]["reasons"] = [] + for challenge in analysis["challenge_results"]: + challenge["status"] = "pass" + challenge["reasons"] = [] + challenge["metrics"]["calmar"] = 0.8 + analysis["evidence_matrix"] = { + "rows": 3, + "pass": 3, + "fail": 0, + "evidence_insufficient": 0, + } + analysis["benchmarks"]["statistics"]["CSI300_CNY_TOTAL_RETURN"][ + "active_return" + ] = 0.05 + analysis["opposing_evidence"] = [] + recommendation = build_recommendation(analysis) + + report = render_analysis_report(analysis, recommendation, {}) + + assert recommendation["decision"] == "proceed_to_joinquant" + assert "冻结基线已通过全部研究门槛" in report + assert "基础场景全部通过" in report + assert "全部基准的主动收益为正" in report + assert "冻结基线未通过" not in report + assert "基础场景均未通过" not in report + assert "均明显落后" not in report + + +def test_report_adapts_to_mixed_challenge_and_benchmark_results() -> None: + analysis = _analysis() + analysis["baseline"]["status"] = "pass" + analysis["baseline"]["reasons"] = [] + analysis["challenge_results"][0]["status"] = "pass" + analysis["challenge_results"][0]["reasons"] = [] + analysis["benchmarks"]["statistics"]["SECOND_BENCHMARK"] = { + **analysis["benchmarks"]["statistics"]["CSI300_CNY_TOTAL_RETURN"], + "active_return": 0.04, + } + + report = render_analysis_report( + analysis, + build_recommendation(analysis), + {}, + ) + + assert "基础场景中 1 项通过、1 项失败、0 项证据不足" in report + assert "不同基准的主动收益有正有负" in report + assert "基础场景均未通过" not in report + assert "均明显落后" not in report + + +def test_report_describes_zero_active_return_without_calling_it_mixed() -> None: + analysis = _analysis() + analysis["benchmarks"]["statistics"]["CSI300_CNY_TOTAL_RETURN"][ + "active_return" + ] = 0.0 + + report = render_analysis_report(analysis, build_recommendation(analysis), {}) + + assert "至少一个基准的主动收益为零" in report + assert "不同基准的主动收益有正有负" not in report + + +def test_write_delivery_records_analysis_seconds_in_report_and_vibe_evidence( + tmp_path: Path, +) -> None: + analysis = _analysis() + analysis["analysis_seconds"] = 7.25 + (tmp_path / "deterministic-analysis.json").write_text( + json.dumps(analysis, ensure_ascii=False), encoding="utf-8" + ) + (tmp_path / "vibe-evidence.json").write_text( + json.dumps( + { + "capabilities_loaded": [], + "forbidden_capabilities_called": [], + "use_result": {"vibe_called": False}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + write_analysis_delivery(tmp_path) + + vibe = json.loads((tmp_path / "vibe-evidence.json").read_text(encoding="utf-8")) + report = (tmp_path / "local-strategy-analysis-report.md").read_text( + encoding="utf-8" + ) + assert vibe["analysis_seconds"] == 7.25 + assert "确定性分析耗时：7.25 秒" in report + + +def test_write_delivery_persists_report_recommendation_and_corrected_vibe( + tmp_path: Path, +) -> None: + (tmp_path / "deterministic-analysis.json").write_text( + json.dumps(_analysis(), ensure_ascii=False), encoding="utf-8" + ) + (tmp_path / "vibe-evidence.json").write_text( + json.dumps( + { + "capabilities_loaded": ["report-generate"], + "swarm": { + "preset": "portfolio_review_board", + "run_id": "swarm-1", + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + result = write_analysis_delivery(tmp_path) + + assert result["next_action"] == "human_confirmation_required" + assert (tmp_path / "local-strategy-analysis-report.md").is_file() + recommendation = json.loads( + (tmp_path / "recommendation.json").read_text(encoding="utf-8") + ) + vibe = json.loads((tmp_path / "vibe-evidence.json").read_text(encoding="utf-8")) + assert recommendation["decision"] == "revise_and_reassess" + assert recommendation["artifacts"]["report_sha256"] + assert vibe["swarm"]["excluded_from_conclusions"] is True diff --git a/tests/quant_analysis/test_statistics.py b/tests/quant_analysis/test_statistics.py new file mode 100644 index 0000000..9fcdc52 --- /dev/null +++ b/tests/quant_analysis/test_statistics.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from scripts.research.quant_analysis.benchmarks import ( + BenchmarkAlignmentError, + calculate_benchmark_statistics, +) +from scripts.research.quant_analysis.cvar import ( + calculate_cvar, + rolling_compound_returns, +) +from scripts.research.quant_analysis.evidence import ( + ScenarioResult, + build_evidence_matrix, + validate_evidence_matrix, +) +from scripts.research.quant_analysis.robustness import ( + block_bootstrap, + summarize_bootstrap, +) + + +def test_benchmark_statistics_match_alpha_beta_and_capture() -> None: + benchmark = { + "2026-01-05": -0.01, + "2026-01-06": 0.0, + "2026-01-07": 0.01, + } + strategy = {date: 0.001 + 2.0 * value for date, value in benchmark.items()} + + statistics = calculate_benchmark_statistics(strategy, benchmark, annualization=252) + + assert statistics["beta"] == pytest.approx(2.0) + assert statistics["alpha"] == pytest.approx(0.252) + assert statistics["correlation"] == pytest.approx(1.0) + assert "information_ratio" in statistics + + +def test_benchmark_statistics_reject_date_misalignment() -> None: + with pytest.raises(BenchmarkAlignmentError, match="dates"): + calculate_benchmark_statistics( + {"2026-01-05": 0.01}, + {"2026-01-06": 0.01}, + ) + + +def test_bootstrap_is_seeded_and_includes_loss_from_initial_capital() -> None: + sample = np.array([0.01, -0.02, 0.03, -0.25, 0.02], dtype=np.float64) + + first = block_bootstrap(sample, block_size=3, paths=128, horizon=37, seed=20260714) + second = block_bootstrap(sample, block_size=3, paths=128, horizon=37, seed=20260714) + + np.testing.assert_array_equal(first, second) + assert first.shape == (128, 37) + assert np.isin(first, sample).all() + summary = summarize_bootstrap(np.array([[-0.25, 0.0]], dtype=np.float64)) + assert summary["probability_drawdown_over_20pct"] == 1.0 + + +def test_cvar_uses_exact_tail_mass_and_compounds_windows() -> None: + returns = np.array([-0.10, -0.10, -0.05, 0.0, 0.01], dtype=np.float64) + + assert calculate_cvar(returns, 0.8) == pytest.approx(0.10) + compounded = rolling_compound_returns( + np.array([0.1, -0.1, 0.1, -0.1, 0.1, 0.0], dtype=np.float64), + window=5, + ) + assert compounded[0] == pytest.approx(1.1 * 0.9 * 1.1 * 0.9 * 1.1 - 1.0) + assert compounded.shape == (2,) + + +def test_evidence_matrix_is_deterministic_and_local_only(tmp_path: Path) -> None: + results = ( + ScenarioResult( + scenario_id="scenario-a", + dimension="parameter", + status="pass", + metrics={"cagr": 0.1}, + input_sha256="a" * 64, + ), + ScenarioResult( + scenario_id="scenario-b", + dimension="history", + status="evidence_insufficient", + metrics={}, + input_sha256="b" * 64, + reasons=("missing_window",), + ), + ) + path = tmp_path / "evidence-matrix.parquet" + + build_evidence_matrix(results, path) + rows = validate_evidence_matrix(path) + before = path.read_bytes() + build_evidence_matrix(results, path) + + assert [row.scenario_id for row in rows] == ["scenario-a", "scenario-b"] + assert all(row.authority == "local_exploratory" for row in rows) + assert path.read_bytes() == before diff --git a/tests/quant_analysis/test_unified_analysis.py b/tests/quant_analysis/test_unified_analysis.py new file mode 100644 index 0000000..0d6fc09 --- /dev/null +++ b/tests/quant_analysis/test_unified_analysis.py @@ -0,0 +1,694 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pandas as pd +import pytest + +from scripts.research.quant_analysis import unified_analysis as analysis +from scripts.research.quant_analysis.unified_analysis import ( + ScenarioInput, + UnifiedAnalysisError, + _deletion_sensitivity, + _position_facts, + _position_shocks, + _register_source_results, + _risk_metrics, + align_three_way_benchmarks, + calculate_return_metrics, + deterministic_next_action, + evaluate_metrics, +) + + +SCENARIO_IDS = ( + "baseline", + "entry-40", + "entry-60", + "stop-1-5n", + "stop-2-5n", + "covariance-120d", + "covariance-ewma-30d", +) + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, sort_keys=True), encoding="utf-8") + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _source_fixture( + root: Path, + workspace: Path, + *, + identity_override: dict[str, object] | None = None, + run_prefix: str = "run", + shared_identity: dict[str, object] | None = None, +) -> dict[str, str]: + _write_json( + workspace / "analysis-scenarios.json", + { + "strategy_id": "strategy-003", + "scenarios": [ + {"scenario_id": scenario_id, "dimension": "baseline" if index == 0 else "parameter"} + for index, scenario_id in enumerate(SCENARIO_IDS) + ], + }, + ) + registry: dict[str, str] = {} + for index, scenario_id in enumerate(SCENARIO_IDS): + params_path = workspace / "scenario-configs" / scenario_id / "params.json" + _write_json(params_path, {"scenario_id": scenario_id}) + run_id = f"{run_prefix}-{index}" + registry[scenario_id] = run_id + run_root = root / ".local" / "quant-research" / "strategy-003" / run_id + result_dir = run_root / "backtests" / f"local-{scenario_id}" + performance = { + "status": "pass", + "result_match": True, + "cold_seconds": 10.0, + "warm_seconds": 1.0, + "cleanup": {"verified": True}, + } + _write_json(result_dir / "performance.json", performance) + identity = { + "snapshot_id": "snapshot-1", + "code_identity_sha256": "code-identity-1", + "code_sha256": "code-1", + "execution": { + "adapter_version": "adapter-1", + "backend": "vectorbt.Portfolio.from_order_func", + "callbacks_sha256": "callbacks-1", + }, + } + if shared_identity: + identity.update(shared_identity) + if index == len(SCENARIO_IDS) - 1 and identity_override: + identity.update(identity_override) + _write_json( + result_dir / "manifest.json", + { + "run": { + "run_id": run_id, + "scenario_id": scenario_id, + "snapshot_id": identity["snapshot_id"], + }, + "source": { + "engine": { + "adapter_version": identity["execution"]["adapter_version"], + "backend": identity["execution"]["backend"], + "numba": "0.66.0", + "vectorbt": "1.1.0", + } + }, + }, + ) + run_manifest = { + "schema_version": 1, + "project_id": "strategy-003", + "run_id": run_id, + "status": "complete", + "snapshot": {"snapshot_id": identity["snapshot_id"]}, + "inputs": { + "project_config_sha256": _sha256(params_path), + "code_identity_sha256": identity["code_identity_sha256"], + "code_sha256": identity["code_sha256"], + "code_identity": { + "execution": { + **identity["execution"], + "dependencies": {"numba": "0.66.0", "vectorbt": "1.1.0"}, + } + }, + }, + "output_set_sha256": f"outputs-{index}", + } + _write_json(run_root / "run-manifest.json", run_manifest) + return registry + + +def test_calculate_return_metrics_compounds_and_measures_drawdown() -> None: + series = pd.Series( + [0.10, -0.20, 0.05], + index=pd.to_datetime(["2024-01-02", "2024-01-03", "2024-01-04"]), + ) + + metrics = calculate_return_metrics(series) + + assert abs(metrics["cumulative_return"] - (1.10 * 0.80 * 1.05 - 1.0)) < 1e-12 + assert abs(metrics["max_drawdown"] + 0.20) < 1e-12 + assert metrics["observations"] == 3 + assert metrics["cagr"] is not None + assert metrics["calmar"] is not None + + +def test_aligns_strategy_and_both_benchmarks_on_one_shared_calendar() -> None: + strategy = pd.Series( + [0.01, 0.02, 0.03], + index=pd.to_datetime(["2024-01-02", "2024-01-03", "2024-01-04"]), + ) + benchmarks = { + "CSI300_CNY_TOTAL_RETURN": pd.Series( + [0.001, 0.002], + index=pd.to_datetime(["2024-01-02", "2024-01-03"]), + ), + "NASDAQ100_CNY_TOTAL_RETURN": pd.Series( + [0.004, 0.005], + index=pd.to_datetime(["2024-01-03", "2024-01-04"]), + ), + } + + aligned, evidence = align_three_way_benchmarks(strategy, benchmarks) + + assert list(aligned.index) == [pd.Timestamp("2024-01-03")] + assert list(aligned.columns) == [ + "strategy", + "CSI300_CNY_TOTAL_RETURN", + "NASDAQ100_CNY_TOTAL_RETURN", + ] + assert evidence["common_samples"] == 1 + assert evidence["strategy_excluded_dates"] == 2 + + +def test_evaluates_all_declared_strategy_thresholds() -> None: + thresholds = { + "cagr_min_exclusive": 0.0, + "max_drawdown_abs_max": 0.2, + "calmar_min": 0.5, + } + + assert evaluate_metrics( + {"cagr": 0.10, "max_drawdown": -0.10, "calmar": 1.0}, + thresholds, + ) == ("pass", []) + assert evaluate_metrics( + {"cagr": -0.01, "max_drawdown": -0.25, "calmar": -0.04}, + thresholds, + ) == ( + "fail", + ["cagr_min_exclusive", "max_drawdown_abs_max", "calmar_min"], + ) + + +def test_position_facts_use_same_day_state_before_end_of_day_valuation() -> None: + scenario = ScenarioInput( + scenario_id="baseline", + run_id="run-1", + result_dir=Path("."), + returns=pd.Series([0.0], index=pd.to_datetime(["2024-01-02"])), + balances=pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-02"]), + "total_value": [1000.0], + "cash": [500.0], + } + ), + positions=pd.DataFrame( + { + "date": pd.to_datetime(["2024-01-02"]), + "security": ["510300.XSHG"], + "amount": [50.0], + "price": [10.0], + "daily_gains": [0.0], + } + ), + orders=pd.DataFrame(), + events=pd.DataFrame( + { + "time": ["2024-01-02 16:00:00"], + "event_id": ["event-1"], + "scope": ["security"], + "security": ["510300.XSHG"], + "event_type": ["valuation"], + "reason_code": ["signal_entry"], + "requested_amount": [None], + "executed_amount": [None], + "reference_price": [10.0], + "risk_before": [0.0], + "risk_after": [50.0], + "details_json": [ + '{"average_cost_after":10.0,"close":10.0,' + '"common_stop_after":9.0,"position_after":50,' + '"security_daily_pnl":0.0,"source_reason":"entry_breakout",' + '"stop_failure_loss":150.0}' + ], + } + ), + params={}, + performance={}, + ) + + facts = _position_facts(scenario, {"510300.XSHG": "equity"}) + + assert facts.loc[0, "common_stop"] == 9.0 + assert facts.loc[0, "attribution_reason"] == "entry_breakout" + + +def test_security_pnl_facts_include_full_exit_without_fake_source_position() -> None: + dates = pd.to_datetime(["2024-01-02", "2024-01-03"]) + scenario = ScenarioInput( + scenario_id="baseline", + run_id="run-1", + result_dir=Path("."), + returns=pd.Series([99.0 / 999.0], index=dates[1:]), + balances=pd.DataFrame( + { + "date": dates, + "total_value": [999.0, 1098.0], + "cash": [499.0, 1098.0], + } + ), + positions=pd.DataFrame( + { + "date": [dates[0]], + "security": ["ETF-A"], + "amount": [50.0], + "price": [10.0], + "avg_cost": [10.0], + "daily_gains": [-1.0], + } + ), + orders=pd.DataFrame(), + events=pd.DataFrame( + { + "time": ["2024-01-02 16:00:00", "2024-01-03 16:00:00"], + "event_id": ["valuation-1", "valuation-2"], + "scope": ["security", "security"], + "security": ["ETF-A", "ETF-A"], + "event_type": ["valuation", "valuation"], + "reason_code": ["signal_entry", "protective_stop"], + "requested_amount": [None, None], + "executed_amount": [None, None], + "reference_price": [10.0, 12.0], + "risk_before": [0.0, 50.0], + "risk_after": [50.0, 0.0], + "details_json": [ + '{"security_daily_pnl":-1.0,"source_reason":"entry_breakout"}', + '{"security_daily_pnl":99.0,"source_reason":"protective_stop"}', + ], + } + ), + params={}, + performance={}, + ) + + pnl = analysis._security_pnl_facts(scenario, {"ETF-A": "equity"}) + + assert scenario.positions["date"].tolist() == [dates[0]] + assert pnl["date"].tolist() == list(dates) + assert pnl["security_daily_pnl"].tolist() == pytest.approx([-1.0, 99.0]) + assert pnl["return_contribution"].tolist() == pytest.approx( + [-0.001, 99.0 / 999.0] + ) + assert pnl["attribution_reason"].tolist() == [ + "entry_breakout", + "protective_stop", + ] + + +def test_stop_failure_shock_uses_reproducible_loss_from_valuation_details() -> None: + date = pd.Timestamp("2024-01-02") + positions = pd.DataFrame( + { + "date": [date, date], + "security": ["ETF-A", "ETF-B"], + "asset_group": ["equity", "bond"], + "weight": [0.4, 0.2], + "equity": [1000.0, 1000.0], + "stop_failure_loss": [100.0, 50.0], + } + ) + definitions = [ + { + "id": "shock-stop-failure", + "use_stop_failure_loss": True, + "maximum_loss_abs_max": 0.20, + } + ] + + rows, _ = _position_shocks(positions, definitions) + + assert rows[0]["status"] == "pass" + assert rows[0]["reasons"] == [] + assert rows[0]["metrics"]["evaluated_dates"] == 1 + assert rows[0]["metrics"]["worst_account_loss"] == pytest.approx(0.15) + + +def test_deterministic_analysis_continues_to_local_report_not_vibe() -> None: + assert deterministic_next_action() == "generate_deterministic_local_report" + + +def test_analysis_seconds_are_recorded_once_and_preserved_on_rerun( + tmp_path: Path, +) -> None: + output = tmp_path / "deterministic-analysis.json" + summary = {"analysis_id": "analysis-1"} + + first = analysis._with_analysis_seconds(summary, 7.25, output) + _write_json(output, first) + second = analysis._with_analysis_seconds(summary, 9.5, output) + + assert first["analysis_seconds"] == 7.25 + assert second["analysis_seconds"] == 7.25 + + +def test_risk_diagnostics_use_actual_exposure_and_optional_turtle_units() -> None: + date = pd.Timestamp("2024-01-02") + scenario = ScenarioInput( + scenario_id="baseline", + run_id="run-1", + result_dir=Path("."), + returns=pd.Series([0.0] * 60, index=pd.date_range(date, periods=60)), + balances=pd.DataFrame( + {"date": [date], "total_value": [1000.0], "cash": [500.0]} + ), + positions=pd.DataFrame(), + orders=pd.DataFrame( + { + "status": ["done"], + "filled": [10.0], + "action": ["open"], + "price": [10.0], + "commission": [1.0], + "gains": [0.0], + } + ), + events=pd.DataFrame( + { + "event_type": ["decision", "valuation", "decision", "valuation"], + "reason_code": [ + "full_position_redistribution", + "full_position_redistribution", + "protective_stop", + "protective_stop", + ], + "details_json": [ + json.dumps( + { + "effective_risk_units": 12.0, + "portfolio_unit_cap": 12.0, + } + ), + "{}", + "{}", + "{}", + ], + } + ), + params={ + "risk": { + "unit_risk_per_n": 0.01, + "asset_group_unit_cap": 6.0, + "portfolio_unit_cap": 12.0, + } + }, + performance={}, + ) + positions = pd.DataFrame( + { + "date": [date], + "asset_group": ["equity"], + "weight": [0.4], + "common_stop": [9.0], + "avg_cost": [10.0], + "amount": [10.0], + } + ) + + metrics = _risk_metrics(scenario, positions) + + assert metrics["maximum_security_weight"] == pytest.approx(0.4) + assert metrics["maximum_asset_group_weight"] == pytest.approx(0.4) + assert metrics["maximum_planned_loss_ratio"] == pytest.approx(0.01) + assert metrics["maximum_effective_risk_units"] == pytest.approx(12.0) + assert metrics["maximum_portfolio_unit_utilization"] == pytest.approx(1.0) + assert metrics["redistribution_event_count"] == 1 + assert metrics["protective_stop_events"] == 1 + assert "mark_to_market_security_weight_above_entry_cap_rows" not in metrics + assert "realized_60d_volatility_above_target_days" not in metrics + + +def test_risk_diagnostics_accept_joinquant_results_without_turtle_evidence() -> None: + date = pd.Timestamp("2024-01-02") + scenario = ScenarioInput( + scenario_id="joinquant", + run_id="run-1", + result_dir=Path("."), + returns=pd.Series([0.0], index=[date]), + balances=pd.DataFrame( + {"date": [date], "total_value": [1000.0], "cash": [1000.0]} + ), + positions=pd.DataFrame(), + orders=pd.DataFrame( + columns=["status", "filled", "action", "price", "commission", "gains"] + ), + events=pd.DataFrame(), + params={}, + performance={}, + ) + + metrics = _risk_metrics(scenario, pd.DataFrame()) + + assert metrics["maximum_effective_risk_units"] is None + assert metrics["maximum_portfolio_unit_utilization"] is None + assert metrics["redistribution_event_count"] == 0 + + +def test_deletion_sensitivity_keeps_unheld_securities_and_groups_in_matrix() -> None: + dates = pd.to_datetime(["2024-01-02", "2024-01-03"]) + scenario = ScenarioInput( + scenario_id="baseline", + run_id="run-1", + result_dir=Path("."), + returns=pd.Series([0.01, -0.01], index=dates), + balances=pd.DataFrame(), + positions=pd.DataFrame(), + orders=pd.DataFrame(), + events=pd.DataFrame(), + params={}, + performance={}, + ) + positions = pd.DataFrame( + { + "date": dates, + "security": ["ETF-A", "ETF-A"], + "asset_group": ["equity", "equity"], + "return_contribution": [0.01, -0.01], + } + ) + thresholds = { + "cagr_min_exclusive": 0.0, + "max_drawdown_abs_max": 0.2, + "calmar_min": 0.5, + } + + rows, _ = _deletion_sensitivity( + scenario, + positions, + {"ETF-A": "equity", "ETF-B": "bond"}, + thresholds, + ) + + assert {row["removed"] for row in rows} == {"ETF-A", "ETF-B", "equity", "bond"} + + +def test_source_registration_uses_only_explicit_scenario_run_mapping(tmp_path: Path) -> None: + root = tmp_path + workspace = root / ".local" / "strategy-analysis-preparations" / "preparation-1" + _write_json(workspace / "preparation.json", {"preparation_id": "preparation-1"}) + registry = _source_fixture(root, workspace) + duplicate = root / ".local" / "quant-research" / "strategy-003" / "old-run" + _write_json( + duplicate / "run-manifest.json", + { + "run_id": "old-run", + "status": "complete", + "inputs": { + "project_config_sha256": _sha256( + workspace / "scenario-configs" / "baseline" / "params.json" + ) + }, + }, + ) + + document = _register_source_results(root, workspace, registry) + + assert [source["run_id"] for source in document["sources"]] == list(registry.values()) + assert document["source_registry"] == { + "explicit": True, + "scenario_count": 7, + "run_id_count": 7, + "sha256": document["source_registry"]["sha256"], + } + assert document["shared_identity"] == { + "snapshot_id": "snapshot-1", + "code_identity_sha256": "code-identity-1", + "code_sha256": "code-1", + "execution_backend": { + "adapter_version": "adapter-1", + "backend": "vectorbt.Portfolio.from_order_func", + "callbacks_sha256": "callbacks-1", + "dependencies": {"numba": "0.66.0", "vectorbt": "1.1.0"}, + }, + "execution_backend_sha256": document["shared_identity"][ + "execution_backend_sha256" + ], + } + assert "old-run" not in json.dumps(document) + final_workspace = root / ".local" / "strategy-analysis" / document["analysis_id"] + assert json.loads( + (final_workspace / "source-results.json").read_text(encoding="utf-8") + ) == document + + +def test_source_registration_rejects_duplicate_run_ids(tmp_path: Path) -> None: + root = tmp_path + workspace = root / ".local" / "strategy-analysis-preparations" / "preparation-1" + _write_json(workspace / "preparation.json", {"preparation_id": "preparation-1"}) + registry = _source_fixture(root, workspace) + registry["entry-40"] = registry["baseline"] + + with pytest.raises(UnifiedAnalysisError, match="run_id values must be unique"): + _register_source_results(root, workspace, registry) + + +def test_source_registration_supports_baseline_plus_one_challenge( + tmp_path: Path, +) -> None: + root = tmp_path + workspace = root / ".local" / "strategy-analysis-preparations" / "preparation-1" + _write_json(workspace / "preparation.json", {"preparation_id": "preparation-1"}) + registry = _source_fixture(root, workspace) + scenarios = json.loads( + (workspace / "analysis-scenarios.json").read_text(encoding="utf-8") + ) + scenarios["scenarios"] = scenarios["scenarios"][:2] + registry = { + scenario["scenario_id"]: registry[scenario["scenario_id"]] + for scenario in scenarios["scenarios"] + } + _write_json(workspace / "analysis-scenarios.json", scenarios) + + document = _register_source_results(root, workspace, registry) + + assert [source["scenario_id"] for source in document["sources"]] == [ + "baseline", + "entry-40", + ] + assert document["source_registry"]["scenario_count"] == 2 + + +def test_source_registration_requires_at_least_two_scenarios(tmp_path: Path) -> None: + root = tmp_path + workspace = root / ".local" / "strategy-analysis-preparations" / "preparation-1" + _write_json(workspace / "preparation.json", {"preparation_id": "preparation-1"}) + registry = _source_fixture(root, workspace) + scenarios = json.loads( + (workspace / "analysis-scenarios.json").read_text(encoding="utf-8") + ) + scenarios["scenarios"] = scenarios["scenarios"][:1] + registry = {"baseline": registry["baseline"]} + _write_json(workspace / "analysis-scenarios.json", scenarios) + + with pytest.raises(UnifiedAnalysisError, match="at least two planned scenarios"): + _register_source_results(root, workspace, registry) + + +def test_source_registration_rejects_result_manifest_with_another_backend( + tmp_path: Path, +) -> None: + root = tmp_path + workspace = root / ".local" / "strategy-analysis-preparations" / "preparation-1" + _write_json(workspace / "preparation.json", {"preparation_id": "preparation-1"}) + registry = _source_fixture(root, workspace) + result_manifest = ( + root + / ".local" + / "quant-research" + / "strategy-003" + / registry["baseline"] + / "backtests" + / "local-baseline" + / "manifest.json" + ) + document = json.loads(result_manifest.read_text(encoding="utf-8")) + document["source"]["engine"]["backend"] = "forged.Backend" + _write_json(result_manifest, document) + + with pytest.raises( + UnifiedAnalysisError, match="local result execution backend identity" + ): + _register_source_results(root, workspace, registry) + + +@pytest.mark.parametrize( + ("identity_override", "message"), + [ + ({"snapshot_id": "snapshot-2"}, "snapshot_id"), + ({"code_identity_sha256": "code-identity-2"}, "code_identity_sha256"), + ({"code_sha256": "code-2"}, "code_sha256"), + ( + { + "execution": { + "adapter_version": "adapter-2", + "backend": "vectorbt.Portfolio.from_order_func", + "callbacks_sha256": "callbacks-1", + } + }, + "execution backend identity", + ), + ], +) +def test_source_registration_requires_one_shared_execution_identity( + tmp_path: Path, + identity_override: dict[str, object], + message: str, +) -> None: + root = tmp_path + workspace = root / ".local" / "strategy-analysis-preparations" / "preparation-1" + _write_json(workspace / "preparation.json", {"preparation_id": "preparation-1"}) + registry = _source_fixture(root, workspace, identity_override=identity_override) + + with pytest.raises(UnifiedAnalysisError, match=message): + _register_source_results(root, workspace, registry) + + +def test_analysis_identity_changes_when_registered_source_code_changes( + tmp_path: Path, +) -> None: + root = tmp_path + preparation = ( + root / ".local" / "strategy-analysis-preparations" / "preparation-1" + ) + _write_json(preparation / "preparation.json", {"preparation_id": "preparation-1"}) + first_registry = _source_fixture(root, preparation, run_prefix="first") + first = _register_source_results(root, preparation, first_registry) + first_path = root / ".local" / "strategy-analysis" / first["analysis_id"] + first_bytes = (first_path / "source-results.json").read_bytes() + + second_registry = _source_fixture( + root, + preparation, + run_prefix="second", + shared_identity={ + "code_identity_sha256": "code-identity-2", + "code_sha256": "code-2", + "execution": { + "adapter_version": "adapter-2", + "backend": "vectorbt.Portfolio.from_order_func", + "callbacks_sha256": "callbacks-2", + }, + }, + ) + second = _register_source_results(root, preparation, second_registry) + + assert first["analysis_id"] != second["analysis_id"] + assert (first_path / "source-results.json").read_bytes() == first_bytes + assert ( + root / ".local" / "strategy-analysis" / second["analysis_id"] / "source-results.json" + ).is_file() diff --git a/tests/test_skill_layout.py b/tests/test_skill_layout.py index d4c0947..a6d527b 100644 --- a/tests/test_skill_layout.py +++ b/tests/test_skill_layout.py @@ -130,24 +130,25 @@ def test_build_and_verify_covers_local_quant_research_without_local_data( "pytest", "tests\\local_quant_research", "-k", - "not test_skill_public_command_runs_complete_turtle_workflow and not test_non_strategy_project_completes_through_shared_market_and_runner", + "not test_non_strategy_project_completes_through_shared_market_and_runner", ] assert e2e["command"] == [ ".\\.venv\\Scripts\\python.exe", "-m", "pytest", - "tests\\local_quant_research\\test_turtle_e2e.py", "tests\\local_quant_research\\test_generic_e2e.py", - "-k", - "skill_public_command_runs_complete_turtle_workflow or non_strategy_project_completes_through_shared_market_and_runner", + "tests\\local_quant_research\\test_turtle_e2e.py", ] required_paths = { ".agents/skills/run-local-quant-research/**", ".claude/skills/run-local-quant-research", "scripts/research/market_data/**", "scripts/research/local_quant_research/**", + "scripts/research/analysis_data/**", + "scripts/research/quant_analysis/**", "joinquant/strategies/strategy-003/research/**", "tests/local_quant_research/**", + "tests/quant_analysis/**", } assert required_paths.issubset(unit["paths"]) assert required_paths.issubset(e2e["paths"]) @@ -163,3 +164,11 @@ def test_build_and_verify_covers_local_quant_research_without_local_data( assert e2e["checkParallel"] is False assert all(not item.startswith(".local/") for item in unit["inputs"]) assert all(not item.startswith(".local/") for item in e2e["inputs"]) + + +def test_full_verify_checkout_downloads_git_lfs_objects(repo_root: Path) -> None: + workflow = ( + repo_root / ".github" / "workflows" / "full-verify.yml" + ).read_text(encoding="utf-8") + + assert "uses: actions/checkout@v4\n with:\n lfs: true" in workflow