From 488922c15f231bbc0ad3b5720a0b6aa5da271c54 Mon Sep 17 00:00:00 2001 From: Rene Zander Date: Fri, 9 Oct 2026 05:26:55 +0000 Subject: [PATCH 1/6] lint: add segment-overlap and segment-offscreen checks segment-overlap (error) reports overlapping segments on every non-text track; --fix ends the earlier segment at the next start for overlaps of at most one frame, keeping source duration proportional to speed. Wider overlaps stay report-only. segment-offscreen (warning) reports visual segments that are off the canvas, at zero scale or at zero opacity, skipping a reason whose property is keyframed. --- src/lint.ts | 197 ++++++++++++++++++++++++++- test/lint-overlap-offscreen.test.mjs | 181 ++++++++++++++++++++++++ 2 files changed, 371 insertions(+), 7 deletions(-) create mode 100644 test/lint-overlap-offscreen.test.mjs diff --git a/src/lint.ts b/src/lint.ts index ec7d1f9..2109330 100644 --- a/src/lint.ts +++ b/src/lint.ts @@ -40,7 +40,8 @@ export interface LintIssue { // Codes that lintDraft can mechanically repair via fixDraft. Membership here // is necessary but not sufficient for fixable:true — line-too-long, -// caption-gap-too-small, main-track-gap, and media-outside-draft are +// caption-gap-too-small, main-track-gap, segment-overlap (only overlaps of at +// most one frame), and media-outside-draft are // additionally stamped per instance, so an issue is only marked fixable when // fixDraft can actually clear that exact instance. dangling-companion-ref is // always safely fixable: the repair drops a ref that points at nothing — @@ -62,6 +63,7 @@ const FIXABLE_CODES = new Set([ "media-unlinked", "text-range-doubled", "segment-off-frame-grid", + "segment-overlap", ]); // Floor for any duration --fix writes: 100ms = three frames at the 30fps @@ -182,7 +184,7 @@ export function lintDraft(draft: Draft, opts: LintOptions = DEFAULT_LINT_OPTIONS }; if (opts.frameGrid) { - const fps = typeof draft.fps === "number" && Number.isFinite(draft.fps) && draft.fps > 0 ? draft.fps : 30; + const fps = draftFps(draft); for (const track of draft.tracks) { for (const segment of track.segments) { const start = segment.target_timerange.start; @@ -372,6 +374,128 @@ export function lintDraft(draft: Draft, opts: LintOptions = DEFAULT_LINT_OPTIONS } } + // Overlaps on every other track type. Text tracks keep caption-overlap + // above; on a video, audio, sticker, effect or filter track two segments + // sharing the same microseconds is a draft the app either refuses or + // resolves by its own rule (one clip hides the other, or the later one is + // pushed to a new track on the first edit). Consecutive pairs by start, the + // caption-overlap walk. The classic cause is independent rounding — start + // and duration rounded separately leave the earlier clip a microsecond past + // the next one's start — so an overlap of at most one frame is fixable + // (--fix pulls the earlier end back to the next start); anything wider is a + // real edit collision and stays report-only, because which clip should give + // way is an authoring decision. + const overlapFrameUs = Math.ceil(1_000_000 / draftFps(draft)); + for (const track of draft.tracks) { + if (track.type === "text") continue; + const segs = [...track.segments].sort((a, b) => a.target_timerange.start - b.target_timerange.start); + for (let i = 0; i < segs.length - 1; i++) { + const s = segs[i]; + const next = segs[i + 1]; + const overlap = s.target_timerange.start + s.target_timerange.duration - next.target_timerange.start; + if (overlap <= 0) continue; + const repairable = overlap <= overlapFrameUs && s.target_timerange.duration - overlap > 0; + issues.push({ + severity: "error", + code: "segment-overlap", + message: + `Segments ${shortId(s.id)} and ${shortId(next.id)} overlap by ${overlap}us on ${track.type} track "${track.name}"` + + (repairable + ? " — a sub-frame rounding overlap; --fix ends the earlier segment where the next begins" + : ` — wider than one frame (${overlapFrameUs}us at ${draftFps(draft)}fps), so --fix leaves it: trim or move one of them`), + fixable: FIXABLE_CODES.has("segment-overlap") && repairable, + location: { track: track.name, segment_id: s.id }, + }); + } + } + + // Visual segments nobody can see: a clip transform that parks the whole + // bounding box off the canvas, a zero scale on either axis, or zero + // opacity. Usually the leftover of a botched batch edit (a percent written + // where a fraction was meant, x in pixels instead of canvas units), and + // invisible in every timeline view except the preview itself. + // + // Transform convention (render.ts's overlay math, the same one the app + // uses): clip.transform x/y are normalized to the canvas with the origin at + // its centre, one unit = half the canvas — x = +/-1 puts the clip's centre + // on the right/left edge, y = +/-1 on the top/bottom edge (y points up). + // The material is first fitted inside the canvas (contain, aspect kept), + // then multiplied by clip.scale. A box of width w (pixels) is therefore + // fully off-canvas once |x| * W/2 >= W/2 + w/2, i.e. |x| >= 1 + w/W; same + // for y. Rotation widens the box to its rotated bounding box. Without a + // known material size (stickers, text, a video material with no + // width/height) the material is assumed to fill the canvas at scale 1 — + // the generous reading, so an unknown size never produces a false alarm. + // + // `visible` is not judged: the CLI only ever writes `visible: true` and + // nothing in the code base establishes how the app treats false. + // + // Keyframes: a reason is skipped when the segment animates the property + // behind it (position for off-canvas, scale for zero scale, alpha for zero + // opacity) — the static value is then only the pre-animation value, and a + // fade-in from alpha 0 is the most common keyframe there is. Warning, no + // --fix: where the clip was meant to be is not recoverable. + const canvasW = draft.canvas_config?.width; + const canvasH = draft.canvas_config?.height; + const hasCanvas = typeof canvasW === "number" && canvasW > 0 && typeof canvasH === "number" && canvasH > 0; + for (const track of draft.tracks) { + if (track.type !== "video" && track.type !== "sticker" && track.type !== "text") continue; + for (const s of track.segments) { + const clip = s.clip; + if (!clip || typeof clip !== "object") continue; + const animated = keyframedProperties(s); + const reasons: string[] = []; + const sx = clip.scale?.x; + const sy = clip.scale?.y; + const scaleAnimated = + animated.has("KFTypeScaleX") || animated.has("KFTypeScaleY") || animated.has("UNIFORM_SCALE"); + if (!scaleAnimated && (sx === 0 || sy === 0)) reasons.push("zero scale"); + if (!animated.has("KFTypeAlpha") && clip.alpha === 0) reasons.push("zero opacity"); + const x = clip.transform?.x; + const y = clip.transform?.y; + const positionAnimated = animated.has("KFTypePositionX") || animated.has("KFTypePositionY"); + if (!positionAnimated && typeof x === "number" && typeof y === "number" && !reasons.includes("zero scale")) { + // Fitted material size as a fraction of the canvas (1 = fills that axis). + let fitW = 1; + let fitH = 1; + if (track.type === "video" && hasCanvas) { + const mat = findMaterial(draft.materials?.videos ?? [], s.material_id) as + | { width?: unknown; height?: unknown } + | undefined; + const mw = mat?.width; + const mh = mat?.height; + if (typeof mw === "number" && mw > 0 && typeof mh === "number" && mh > 0) { + const fit = Math.min(canvasW / mw, canvasH / mh); + fitW = (mw * fit) / canvasW; + fitH = (mh * fit) / canvasH; + } + } + // Work in pixels so a rotation mixes the axes at the right aspect; an + // unknown canvas uses a square unit canvas (only the ratio matters). + const W = hasCanvas ? canvasW : 1; + const H = hasCanvas ? canvasH : 1; + const w = fitW * W * Math.abs(typeof sx === "number" ? sx : 1); + const h = fitH * H * Math.abs(typeof sy === "number" ? sy : 1); + const rad = ((typeof clip.rotation === "number" ? clip.rotation : 0) * Math.PI) / 180; + const cos = Math.abs(Math.cos(rad)); + const sin = Math.abs(Math.sin(rad)); + const boxW = w * cos + h * sin; + const boxH = w * sin + h * cos; + if (Math.abs(x) >= 1 + boxW / W || Math.abs(y) >= 1 + boxH / H) reasons.push("outside canvas"); + } + if (reasons.length === 0) continue; + issues.push({ + severity: "warning", + code: "segment-offscreen", + message: + `Segment ${shortId(s.id)} on ${track.type} track "${track.name}" cannot be seen: ${reasons.join(", ")}` + + ` (scale ${sx ?? "?"}x${sy ?? "?"}, alpha ${clip.alpha ?? "?"}, position ${x ?? "?"},${y ?? "?"} in canvas half-widths)`, + fixable: false, + location: { track: track.name, segment_id: s.id }, + }); + } + } + // Source range against the material it reads from. CapCut treats a // `source_timerange` that reaches past the material's duration as out of // range and clamps the in-point to zero, so every such clip plays from the @@ -1059,7 +1183,7 @@ export function fixDraft(draft: Draft, opts: LintOptions = DEFAULT_LINT_OPTIONS) // Pass -2: snap the two boundaries, then derive duration. Rounding start // and duration independently is the exact 1us-overlap failure this fixes. if (opts.frameGrid) { - const fps = typeof draft.fps === "number" && Number.isFinite(draft.fps) && draft.fps > 0 ? draft.fps : 30; + const fps = draftFps(draft); const timelineEnd = (): number => { let maxEnd = 0; for (const track of draft.tracks) { @@ -1078,10 +1202,7 @@ export function fixDraft(draft: Draft, opts: LintOptions = DEFAULT_LINT_OPTIONS) const newEnd = quantizeToFrame(oldStart + oldDuration, fps); if (newEnd <= newStart) continue; segment.target_timerange.start = newStart; - segment.target_timerange.duration = newEnd - newStart; - if (segment.source_timerange?.duration === oldDuration && (segment.speed ?? 1) === 1) { - segment.source_timerange.duration = newEnd - newStart; - } + setTargetDuration(segment, newEnd - newStart); } } if (draft.duration === oldMaxEnd) { @@ -1122,6 +1243,25 @@ export function fixDraft(draft: Draft, opts: LintOptions = DEFAULT_LINT_OPTIONS) } } + // Pass 0b: end the earlier of two overlapping non-text segments where the + // next one begins — only for overlaps of at most one frame, exactly the + // instances lintDraft stamped fixable. After pass 0 on purpose: shortening + // a non-main-track clip could flip canCloseMainTrackGap from unsafe to + // safe, and pass 0 must act on the state the stamp was read from. Pass 0 + // moves both members of an overlapping pair by the same shift, so the + // overlap widths read here are the ones lint reported. + const overlapFrameUs = Math.ceil(1_000_000 / draftFps(draft)); + for (const track of draft.tracks) { + if (track.type === "text") continue; + const segs = [...track.segments].sort((a, b) => a.target_timerange.start - b.target_timerange.start); + for (let i = 0; i < segs.length - 1; i++) { + const s = segs[i]; + const overlap = s.target_timerange.start + s.target_timerange.duration - segs[i + 1].target_timerange.start; + if (overlap <= 0 || overlap > overlapFrameUs || s.target_timerange.duration - overlap <= 0) continue; + setTargetDuration(s, s.target_timerange.duration - overlap); + } + } + // Pass 1: cap over-long cues. Shrinking these first can also close overlaps. for (const track of getTracksByType(draft, "text")) { for (const s of track.segments) { @@ -1451,6 +1591,49 @@ function undoubleContent(content: string): string | null { return JSON.stringify(parsed); } +// The draft's frame rate, 30 when absent or malformed (the template default). +function draftFps(draft: Draft): number { + return typeof draft.fps === "number" && Number.isFinite(draft.fps) && draft.fps > 0 ? draft.fps : 30; +} + +// Set a segment's target duration and keep its source span consistent with +// it: source.duration = target.duration * speed, the invariant `capcut speed` +// maintains. The source span is only rewritten when it satisfied that +// invariant before (exactly at speed 1, within the speed-timerange-mismatch +// 1% tolerance otherwise) — a span that already disagreed is +// speed-timerange-mismatch's report, not something a timing repair should +// silently reinterpret. +function setTargetDuration(segment: Segment, newDuration: number): void { + const oldDuration = segment.target_timerange.duration; + segment.target_timerange.duration = newDuration; + const src = segment.source_timerange; + if (!src || typeof src.duration !== "number") return; + const speed = + typeof segment.speed === "number" && Number.isFinite(segment.speed) && segment.speed > 0 ? segment.speed : 1; + if (speed === 1) { + if (src.duration === oldDuration) src.duration = newDuration; + return; + } + if (oldDuration > 0 && Math.abs(src.duration / oldDuration - speed) / speed <= 0.01) { + src.duration = Math.round(newDuration * speed); + } +} + +// Keyframed property types on a segment (common_keyframes[].property_type, +// e.g. KFTypePositionX), for checks that judge a static clip value only when +// nothing animates it. +function keyframedProperties(segment: Segment): Set { + const out = new Set(); + const lists = (segment as { common_keyframes?: unknown }).common_keyframes; + if (!Array.isArray(lists)) return out; + for (const list of lists as Array<{ property_type?: unknown; keyframe_list?: unknown }>) { + if (typeof list?.property_type !== "string") continue; + if (Array.isArray(list.keyframe_list) && list.keyframe_list.length === 0) continue; + out.add(list.property_type); + } + return out; +} + // True when closing the main-track gap that opens at `gapStartUs` is // mechanically safe: no OTHER track has a segment still playing at or // starting after that point, so the later main-track segments can move left diff --git a/test/lint-overlap-offscreen.test.mjs b/test/lint-overlap-offscreen.test.mjs new file mode 100644 index 0000000..e3cb111 --- /dev/null +++ b/test/lint-overlap-offscreen.test.mjs @@ -0,0 +1,181 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { DEFAULT_LINT_OPTIONS, fixDraft, lintDraft } from "../dist/lint.js"; + +const opts = { ...DEFAULT_LINT_OPTIONS, checkLocalPaths: false, probeMedia: false }; + +function seg(id, materialId, start, duration, extra = {}) { + return { + id, + material_id: materialId, + target_timerange: { start, duration }, + source_timerange: { start: 0, duration }, + speed: 1, + volume: 1, + visible: true, + clip: { alpha: 1, rotation: 0, scale: { x: 1, y: 1 }, transform: { x: 0, y: 0 } }, + extra_material_refs: [], + render_index: 0, + ...extra, + }; +} + +function draftWith(tracks, videos = [{ id: "v", type: "video", path: "", duration: 60_000_000 }]) { + return { + fps: 30, + duration: 10_000_000, + canvas_config: { width: 1920, height: 1080, ratio: "16:9" }, + materials: { + videos, + audios: [{ id: "a", type: "extract_music", path: "", duration: 60_000_000 }], + texts: [], + speeds: [], + }, + tracks, + }; +} + +function track(type, name, segments) { + return { id: name, type, name, attribute: 0, segments }; +} + +const codes = (issues, code) => issues.filter((i) => i.code === code); + +describe("lint segment-overlap", () => { + it("fixes a 1us rounding overlap between back-to-back clips", () => { + const draft = draftWith([ + track("video", "main", [seg("clip-a", "v", 0, 2_000_001), seg("clip-b", "v", 2_000_000, 2_000_000)]), + ]); + const [issue] = codes(lintDraft(draft, opts), "segment-overlap"); + assert.equal(issue.severity, "error"); + assert.equal(issue.fixable, true); + assert.equal(issue.location.segment_id, "clip-a"); + assert.match(issue.message, /clip-b/); + assert.match(issue.message, /1us on video track "main"/); + + const result = fixDraft(draft, opts); + assert.ok(result.fixed.some((i) => i.code === "segment-overlap")); + assert.equal(codes(result.remaining, "segment-overlap").length, 0); + const a = draft.tracks[0].segments[0]; + assert.equal(a.target_timerange.duration, 2_000_000); + assert.equal(a.source_timerange.duration, 2_000_000); + assert.equal(draft.tracks[0].segments[1].target_timerange.start, 2_000_000); + }); + + it("keeps the source span proportional to speed when shortening", () => { + const fast = seg("fast", "v", 0, 1_000_010, { speed: 2, source_timerange: { start: 0, duration: 2_000_020 } }); + const draft = draftWith([track("video", "main", [fast, seg("next", "v", 1_000_000, 1_000_000)])]); + fixDraft(draft, opts); + assert.equal(fast.target_timerange.duration, 1_000_000); + assert.equal(fast.source_timerange.duration, 2_000_000); + }); + + it("reports a 2s overlap on an audio track without fixing it", () => { + const draft = draftWith([ + track("audio", "music", [seg("bed-a", "a", 0, 5_000_000), seg("bed-b", "a", 3_000_000, 4_000_000)]), + ]); + const [issue] = codes(lintDraft(draft, opts), "segment-overlap"); + assert.equal(issue.fixable, false); + assert.match(issue.message, /2000000us on audio track "music"/); + assert.match(issue.message, /--fix leaves it/); + + const result = fixDraft(draft, opts); + assert.equal(codes(result.remaining, "segment-overlap").length, 1); + assert.equal(draft.tracks[0].segments[0].target_timerange.duration, 5_000_000); + }); + + it("leaves text tracks to caption-overlap", () => { + const draft = draftWith([ + track("text", "subs", [seg("cap-a", "t", 0, 2_000_001), seg("cap-b", "t", 2_000_000, 1_000_000)]), + ]); + const issues = lintDraft(draft, opts); + assert.equal(codes(issues, "segment-overlap").length, 0); + assert.equal(codes(issues, "caption-overlap").length, 1); + }); + + it("stays quiet on touching and gapped segments", () => { + const draft = draftWith([ + track("sticker", "stickers", [seg("s1", "x", 0, 1_000_000), seg("s2", "x", 1_000_000, 1_000_000)]), + track("effect", "fx", [seg("e1", "x", 0, 1_000_000), seg("e2", "x", 1_500_000, 1_000_000)]), + ]); + assert.equal(codes(lintDraft(draft, opts), "segment-overlap").length, 0); + }); +}); + +describe("lint segment-offscreen", () => { + const clip = (over) => ({ alpha: 1, rotation: 0, scale: { x: 1, y: 1 }, transform: { x: 0, y: 0 }, ...over }); + + it("keeps an on-screen segment clean, including one partly off the edge", () => { + const draft = draftWith([ + track("video", "main", [seg("centre", "v", 0, 1_000_000)]), + track("video", "pip", [ + seg("corner", "v", 0, 1_000_000, { clip: clip({ scale: { x: 0.3, y: 0.3 }, transform: { x: 1.2, y: 0.9 } }) }), + ]), + track("text", "subs", [seg("caption", "t", 0, 1_000_000, { clip: clip({ transform: { x: 0, y: -0.8 } }) })]), + ]); + assert.equal(codes(lintDraft(draft, opts), "segment-offscreen").length, 0); + }); + + it("flags a clip whose box sits entirely outside the canvas", () => { + // 0.3 scale on a 1920-wide canvas: box half-width is 0.3 canvas + // half-widths, so the clip leaves the canvas once |x| >= 1.3. + const draft = draftWith([ + track("video", "pip", [ + seg("gone", "v", 0, 1_000_000, { clip: clip({ scale: { x: 0.3, y: 0.3 }, transform: { x: 1.35, y: 0 } }) }), + ]), + ]); + const [issue] = codes(lintDraft(draft, opts), "segment-offscreen"); + assert.equal(issue.severity, "warning"); + assert.equal(issue.fixable, false); + assert.equal(issue.location.segment_id, "gone"); + assert.match(issue.message, /outside canvas/); + }); + + it("uses the material size: a portrait clip on a landscape canvas leaves sooner", () => { + // 1080x1920 fitted into 1920x1080 is 607.5px wide (0.316 of the canvas), + // so at x = 1.5 it is fully off; a canvas-filling assumption would not be. + const videos = [{ id: "portrait", type: "video", path: "", duration: 60_000_000, width: 1080, height: 1920 }]; + const draft = draftWith( + [track("video", "pip", [seg("tall", "portrait", 0, 1_000_000, { clip: clip({ transform: { x: 1.5, y: 0 } }) })])], + videos, + ); + assert.match(codes(lintDraft(draft, opts), "segment-offscreen")[0].message, /outside canvas/); + // Same position with unknown size: assumed to fill the canvas, still visible. + const unknown = draftWith([ + track("video", "pip", [seg("tall", "v", 0, 1_000_000, { clip: clip({ transform: { x: 1.5, y: 0 } }) })]), + ]); + assert.equal(codes(lintDraft(unknown, opts), "segment-offscreen").length, 0); + }); + + it("flags zero scale and zero opacity", () => { + const draft = draftWith([ + track("sticker", "stickers", [ + seg("flat", "x", 0, 1_000_000, { clip: clip({ scale: { x: 1, y: 0 } }) }), + seg("clear", "x", 1_000_000, 1_000_000, { clip: clip({ alpha: 0 }) }), + ]), + ]); + const issues = codes(lintDraft(draft, opts), "segment-offscreen"); + assert.equal(issues.length, 2); + assert.match(issues.find((i) => i.location.segment_id === "flat").message, /zero scale/); + assert.match(issues.find((i) => i.location.segment_id === "clear").message, /zero opacity/); + }); + + it("skips a reason whose property is keyframed", () => { + const keyframes = (type) => [{ id: "k", property_type: type, keyframe_list: [{ id: "k1", time_offset: 0 }] }]; + const draft = draftWith([ + track("video", "pip", [ + seg("fade-in", "v", 0, 1_000_000, { clip: clip({ alpha: 0 }), common_keyframes: keyframes("KFTypeAlpha") }), + seg("slide-in", "v", 1_000_000, 1_000_000, { + clip: clip({ transform: { x: 3, y: 0 } }), + common_keyframes: keyframes("KFTypePositionX"), + }), + ]), + ]); + assert.equal(codes(lintDraft(draft, opts), "segment-offscreen").length, 0); + }); + + it("ignores audio tracks", () => { + const draft = draftWith([track("audio", "music", [seg("bed", "a", 0, 1_000_000, { clip: clip({ alpha: 0 }) })])]); + assert.equal(codes(lintDraft(draft, opts), "segment-offscreen").length, 0); + }); +}); From dba6556c9dcc9d33add6f695cfeb647ea6ce7df2 Mon Sep 17 00:00:00 2001 From: Rene Zander Date: Fri, 9 Oct 2026 05:26:27 +0000 Subject: [PATCH 2/6] render: report a fidelity census and verify the output duration Every render plan (dry-run included) now carries a fidelity object listing what the proxy leaves out for the draft at hand, per category with counts and segment ids, plus faithful and expected_duration_us. --strict refuses with refused [render-unfaithful] before anything is written. A real render probes the written file with ffprobe and reports verification (duration, drift, one-frame tolerance); --verify exits non-zero on drift or when the file cannot be probed, keeping the file. --- docs/command-reference.json | 27 +++ src/command-specs.ts | 16 ++ src/index.ts | 24 +- src/render.ts | 273 ++++++++++++++++++++- test/render-fidelity.test.mjs | 432 ++++++++++++++++++++++++++++++++++ 5 files changed, 770 insertions(+), 2 deletions(-) create mode 100644 test/render-fidelity.test.mjs diff --git a/docs/command-reference.json b/docs/command-reference.json index 60a64aa..3617f30 100644 --- a/docs/command-reference.json +++ b/docs/command-reference.json @@ -6584,6 +6584,33 @@ "required": false, "description": "Stream ffmpeg's progress to stderr instead of buffering it." }, + { + "name": "strict", + "flags": [ + "--strict" + ], + "type": "boolean", + "required": false, + "description": "Refuse (refused [render-unfaithful], nothing rendered) when the fidelity census finds anything the proxy drops: transitions, effects, filters, masks, keyframes, uncomposited tracks, stickers, text, animations, blend modes, chroma, matting, gaps or missing media." + }, + { + "name": "verify", + "flags": [ + "--verify" + ], + "type": "boolean", + "required": false, + "description": "Probe the written file with ffprobe and exit non-zero when its duration drifts from the draft's by more than one frame, or when it cannot be probed. The file is kept." + }, + { + "name": "ffprobe_cmd", + "flags": [ + "--ffprobe-cmd" + ], + "type": "path", + "required": false, + "description": "ffprobe binary for output verification." + }, { "name": "active_timeline", "flags": [ diff --git a/src/command-specs.ts b/src/command-specs.ts index 888d8ca..8b23ec8 100644 --- a/src/command-specs.ts +++ b/src/command-specs.ts @@ -861,6 +861,19 @@ const optionsByCommand: Record = { ), option("all_video_tracks", ["--all-video-tracks"], "boolean", "Composite every video track."), option("progress", ["--progress"], "boolean", "Stream ffmpeg's progress to stderr instead of buffering it."), + option( + "strict", + ["--strict"], + "boolean", + "Refuse (refused [render-unfaithful], nothing rendered) when the fidelity census finds anything the proxy drops: transitions, effects, filters, masks, keyframes, uncomposited tracks, stickers, text, animations, blend modes, chroma, matting, gaps or missing media.", + ), + option( + "verify", + ["--verify"], + "boolean", + "Probe the written file with ffprobe and exit non-zero when its duration drifts from the draft's by more than one frame, or when it cannot be probed. The file is kept.", + ), + option("ffprobe_cmd", ["--ffprobe-cmd"], "path", "ffprobe binary for output verification."), ], "detect-scenes": [ option("threshold", ["--threshold"], "number", "Scene-change score a cut must exceed (0..1).", { default: 0.4 }), @@ -956,6 +969,7 @@ optionsByCommand["image-anim"] = optionsByCommand["text-anim"]; // --script -> caption (v0.22 transcript-guided alignment) // --window, --similarity, --min-words -> detect-retakes (v0.22); --json also scopes there // --soft-captions -> render (v0.22 mov_text subtitle stream) +// --strict, --verify -> render (v0.29 fidelity census gate + output duration check) // --like, --from-store -> migrate (v0.23 schema-marker restamp from a donor project) // --word-reveal, --min-script-match, --audio-stream -> caption (v0.26 caption controls) // --from -> shift-all; --ripple -> remove (v0.26 boundary-safe ripple editing) @@ -1017,12 +1031,14 @@ export const RELEASE_SCOPED_FLAGS: ReadonlySet = new Set([ "--script", "--similarity", "--soft-captions", + "--strict", "--sync", "--text", "--text-file", "--threshold", "--threshold-db", "--tts-cmd", + "--verify", "--window", ]); diff --git a/src/index.ts b/src/index.ts index d4b17de..db0c9b5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1078,6 +1078,9 @@ interface Flags { videoBitrate?: string; burnCaptions?: boolean; allVideoTracks?: boolean; + // render: refuse an unfaithful proxy / fail on output duration drift + strict?: boolean; + verify?: boolean; progress?: boolean; maxCps?: number; safeArea?: number; @@ -1263,6 +1266,10 @@ function parseFlags(args: string[]): { positional: string[]; flags: Flags } { flags.minWords = parseInt(args[++i], 10); } else if (a === "--soft-captions") { flags.softCaptions = true; + } else if (a === "--strict") { + flags.strict = true; + } else if (a === "--verify") { + flags.verify = true; } else if (a === "--center-x" && i + 1 < args.length) { flags.centerX = parseFloat(args[++i]); } else if (a === "--center-y" && i + 1 < args.length) { @@ -5591,7 +5598,7 @@ async function cmdCompileData(specPath: string, flags: Flags): Promise { // it never mutates the draft. With --dry-run it returns the ffmpeg plan without // executing, so the filter graph is inspectable (and the path is ffmpeg-free). async function cmdRender(draft: Draft, filePath: string, flags: Flags): Promise { - const { buildRenderPlan, renderDraft } = await import("./render.js"); + const { assertFaithful, buildRenderPlan, renderDraft } = await import("./render.js"); if (flags.crf !== undefined && flags.videoBitrate !== undefined) { die("--crf and --video-bitrate are mutually exclusive."); } @@ -5614,6 +5621,8 @@ async function cmdRender(draft: Draft, filePath: string, flags: Flags): Promise< allVideoTracks: flags.allVideoTracks, dryRun: isDryRun(), progress: flags.progress, + strict: flags.strict, + ffprobeCmd: flags.ffprobeCmd, }; if (opts.dryRun) { // Build-only: surface the plan; no ffmpeg needed. @@ -5621,12 +5630,25 @@ async function cmdRender(draft: Draft, filePath: string, flags: Flags): Promise< ...opts, out: opts.out ?? path.join(draftProjectDir(filePath), "preview.mp4"), }); + if (opts.strict) assertFaithful(plan.fidelity); out({ ok: true, executed: false, ...plan }, flags); return; } const result = renderDraft(draft, filePath, opts); out(result, flags); if (!flags.quiet) process.stderr.write(`Rendered: ${result.output}\n`); + // --verify: the file stays on disk either way; the result above says why. + const check = result.verification; + if (flags.verify && check) { + if (!check.verified) die(`render --verify: output could not be verified: ${check.reason}`); + if (!check.within_tolerance) { + die( + `render --verify: output duration ${check.duration_us}us drifts ${check.drift_us}us from the draft's ` + + `${check.expected_duration_us}us, beyond the one-frame tolerance of ${check.tolerance_us}us. ` + + `The file was kept at ${result.output}; the result's \`fidelity\` lists what the proxy dropped (main_track_gaps shortens it).`, + ); + } + } } async function cmdDetectScenes(positional: string[], flags: Flags): Promise { diff --git a/src/render.ts b/src/render.ts index 692a8fe..b0015e0 100644 --- a/src/render.ts +++ b/src/render.ts @@ -3,6 +3,7 @@ import { existsSync, unlinkSync, writeFileSync } from "node:fs"; import { dirname, extname, join } from "node:path"; import type { Draft, Segment } from "./draft.js"; import { extractText } from "./draft.js"; +import { ffprobeAvailable, probeMedia } from "./probe.js"; import { renderSrt } from "./srt.js"; import { draftProjectDir } from "./store.js"; @@ -16,6 +17,9 @@ import { draftProjectDir } from "./store.js"; * with `--burn-captions`. The result is a watchable preview MP4 — NOT CapCut's * final render (no multi-track video compositing, no effects/transitions). It * exists to verify "did my edit land where I meant it" without launching CapCut. + * Every plan carries a `fidelity` census of what this proxy leaves out for the + * draft at hand (`--strict` refuses when it is not faithful), and a real render + * probes the written file's duration against the draft's (`--verify` gates it). * * Architecture mirrors `caption` (shell-out to an external binary, here ffmpeg) * and `export --batch` (a deterministic, unit-tested command builder, with the @@ -39,6 +43,8 @@ export interface RenderOptions { allVideoTracks?: boolean; // composite overlay video tracks dryRun?: boolean; // build the plan, do not execute ffmpeg progress?: boolean; // stream ffmpeg's own stderr instead of buffering it + strict?: boolean; // refuse (nothing rendered) when the fidelity census is not faithful + ffprobeCmd?: string; // ffprobe binary for output verification (default "ffprobe") } /** @@ -161,11 +167,27 @@ export interface RenderPlan { /** Long graphs are supplied through a file so Windows command-line limits cannot truncate them. */ filterScript?: { path: string; content: string }; quality: { mode: "crf"; crf: number } | { mode: "bitrate"; bitrate: string }; + /** What the draft carries that this proxy does not reproduce (see fidelityCensus). */ + fidelity: FidelityCensus; } export interface RenderResult extends RenderPlan { ok: boolean; executed: boolean; + /** Present after a real render: the written file's duration against the draft's. */ + verification?: OutputVerification; +} + +/** + * The --strict gate, shared by renderDraft and the CLI's ffmpeg-free dry-run + * path. Throws before anything is written. + */ +export function assertFaithful(census: FidelityCensus): void { + if (census.faithful) return; + throw new Error( + `refused [render-unfaithful]: the proxy would drop ${describeDropped(census)} that the app shows. ` + + "Nothing was rendered. Drop --strict to render the approximation anyway; the result's `fidelity` lists the affected segment ids.", + ); } /** @@ -386,6 +408,247 @@ function textSegments(draft: Draft): Array<{ seg: Segment; text: string; color: return out.sort((a, b) => a.seg.target_timerange.start - b.seg.target_timerange.start); } +/** + * Fidelity census: what the draft carries that this proxy does not put on the + * picture. The header comment above states the limits (main track only unless + * --all-video-tracks, no effects/transitions); the census turns those limits + * into counts for THIS draft, so a reader of `render`'s JSON knows whether the + * preview can be trusted as "what the app shows" or only as a timing check. + * + * Pure, deterministic and one pass over tracks + materials (no ffmpeg, no + * clock). It is computed from the options the plan was actually built with, + * so renderDraft's drawtext/overlay fallbacks show up as dropped text/overlays. + */ +export const FIDELITY_CATEGORIES = [ + "missing_media", + "main_track_gaps", + "overlay_tracks", + "text", + "stickers", + "transitions", + "effects", + "filters", + "masks", + "keyframes", + "text_animations", + "video_animations", + "mix_modes", + "chroma", + "matting", +] as const; + +export type FidelityCategory = (typeof FIDELITY_CATEGORIES)[number]; + +/** Affected segment ids are listed up to this many per category; `count` is always the full total. */ +export const FIDELITY_ID_CAP = 20; + +const FIDELITY_HINTS: Record = { + missing_media: "segment media is missing or has no path; `capcut relink` repairs moved media", + main_track_gaps: "the proxy concatenates main-track segments, so timeline gaps close up and later clips play early", + overlay_tracks: "extra video tracks are not composited; pass --all-video-tracks", + text: "text is not drawn on the picture; pass --burn-captions (--soft-captions only adds a subtitle stream)", + stickers: "sticker tracks are not rendered", + transitions: "transitions are not rendered; segments cut hard", + effects: "video effects are not rendered", + filters: "filters (colour looks) are not rendered", + masks: "masks are not applied", + keyframes: "keyframed animation is not applied; segments hold their base transform/volume", + text_animations: "text intro/outro animations are not rendered", + video_animations: "video/photo in/out/combo animations are not rendered", + mix_modes: "blend modes are not applied; overlays composite as Normal", + chroma: "chroma key is not applied", + matting: "smart portrait matting is not applied", +}; + +// The three on-disk mask array spellings (decorators.ts MASK_FIELDS; inlined +// so render does not load the decorator module). +const MASK_MATERIAL_FIELDS = ["common_masks", "common_mask", "masks"] as const; + +export interface FidelityEntry { + count: number; + segment_ids: string[]; + hint: string; +} + +export interface FidelityCensus { + /** True when no category below has anything dropped. */ + faithful: boolean; + /** The draft duration the proxy targets (draft.duration, else the last segment end). */ + expected_duration_us: number; + /** Only categories with at least one affected segment, in FIDELITY_CATEGORIES order. */ + dropped: Partial>; + /** Every category the census checks, so an absent key reads as "checked, none". */ + checked: FidelityCategory[]; +} + +export function expectedDurationUs(draft: Draft): number { + if (Number.isFinite(draft.duration) && draft.duration > 0) return draft.duration; + let end = 0; + for (const track of draft.tracks) { + for (const seg of track.segments) { + end = Math.max(end, seg.target_timerange.start + seg.target_timerange.duration); + } + } + return end; +} + +function hasKeyframes(seg: Segment): boolean { + const lists = seg.common_keyframes; + if (!Array.isArray(lists)) return false; + return lists.some((list) => { + const frames = (list as { keyframe_list?: unknown } | null)?.keyframe_list; + return !Array.isArray(frames) || frames.length > 0; + }); +} + +export function fidelityCensus( + draft: Draft, + opts: Pick, + skipped: Array<{ segmentId: string; reason: string }> = [], +): FidelityCensus { + const hits = new Map(); + const add = (category: FidelityCategory, segId: string) => { + const list = hits.get(category) ?? []; + if (!list.includes(segId)) list.push(segId); + hits.set(category, list); + }; + const materials = draft.materials as Record; + const byId = (field: string): Map> => { + const arr = materials[field]; + const map = new Map>(); + if (Array.isArray(arr)) { + for (const m of arr as Array>) if (m && typeof m.id === "string") map.set(m.id, m); + } + return map; + }; + const transitions = byId("transitions"); + const chromas = byId("chromas"); + const animations = byId("material_animations"); + const effectMaterials = new Map([...byId("effects"), ...byId("video_effects")]); + const maskIds = new Set(MASK_MATERIAL_FIELDS.flatMap((field) => [...byId(field).keys()])); + const videoMaterials = new Map((draft.materials.videos ?? []).map((v) => [v.id, v as Record])); + + const segmentIds = new Set(draft.tracks.flatMap((t) => t.segments.map((s) => s.id))); + for (const { segmentId } of skipped) if (segmentIds.has(segmentId)) add("missing_media", segmentId); + + // Gaps on the main track: concat ignores target starts, so any segment that + // begins after the previous one ended (or after 0) is shifted earlier. + let cursor = 0; + for (const seg of mainVideoSegments(draft)) { + if (seg.target_timerange.start > cursor) add("main_track_gaps", seg.id); + cursor = Math.max(cursor, seg.target_timerange.start + seg.target_timerange.duration); + } + + const mainTrack = draft.tracks.find((t) => t.type === "video"); + for (const track of draft.tracks) { + for (const seg of track.segments) { + if (track.type === "video" && track !== mainTrack && !opts.allVideoTracks) add("overlay_tracks", seg.id); + if (track.type === "sticker") add("stickers", seg.id); + if (track.type === "effect") add("effects", seg.id); + if (track.type === "filter") add("filters", seg.id); + if (hasKeyframes(seg)) add("keyframes", seg.id); + for (const ref of seg.extra_material_refs ?? []) { + if (transitions.has(ref)) add("transitions", seg.id); + if (maskIds.has(ref)) add("masks", seg.id); + if (chromas.has(ref)) add("chroma", seg.id); + const effect = effectMaterials.get(ref); + if (effect) add(effect.type === "filter" ? "filters" : "effects", seg.id); + const anim = animations.get(ref); + if (anim && Array.isArray(anim.animations) && anim.animations.length > 0) { + if (track.type === "text") add("text_animations", seg.id); + else if (track.type === "video") add("video_animations", seg.id); + } + } + if (track.type === "video") { + const mat = videoMaterials.get(seg.material_id); + const mix = mat?.mix_mode; + if (typeof mix === "string" && mix !== "" && mix !== "Normal") add("mix_modes", seg.id); + const flag = (mat?.matting as { flag?: unknown } | undefined)?.flag; + if (typeof flag === "number" && flag !== 0) add("matting", seg.id); + } + } + } + if (!opts.burnCaptions) for (const { seg } of textSegments(draft)) add("text", seg.id); + + const dropped: Partial> = {}; + for (const category of FIDELITY_CATEGORIES) { + const ids = hits.get(category); + if (!ids || ids.length === 0) continue; + dropped[category] = { + count: ids.length, + segment_ids: ids.slice(0, FIDELITY_ID_CAP), + hint: FIDELITY_HINTS[category], + }; + } + return { + faithful: Object.keys(dropped).length === 0, + expected_duration_us: expectedDurationUs(draft), + dropped, + checked: [...FIDELITY_CATEGORIES], + }; +} + +/** One-line summary of a census for refusal messages: "transitions 2, masks 1". */ +export function describeDropped(census: FidelityCensus): string { + return Object.entries(census.dropped) + .map(([category, entry]) => `${category} ${entry?.count}`) + .join(", "); +} + +/** + * Output verification: the written file's duration against the draft's, with + * a one-frame tolerance at the render fps. Pure so the arithmetic is testable + * without ffprobe; probeRenderOutput does the probing. + */ +export interface OutputVerification { + verified: boolean; + /** Present when verified is false. */ + reason?: string; + duration_us?: number; + expected_duration_us: number; + drift_us?: number; + tolerance_us: number; + within_tolerance?: boolean; +} + +export function frameToleranceUs(fps: number): number { + return Math.round(US / (fps > 0 ? fps : 30)); +} + +export function checkOutputDuration(durationUs: number, expectedUs: number, fps: number): OutputVerification { + const tolerance = frameToleranceUs(fps); + const drift = durationUs - expectedUs; + return { + verified: true, + duration_us: durationUs, + expected_duration_us: expectedUs, + drift_us: drift, + tolerance_us: tolerance, + within_tolerance: Math.abs(drift) <= tolerance, + }; +} + +export function probeRenderOutput( + output: string, + expectedUs: number, + fps: number, + ffprobeCmd = "ffprobe", +): OutputVerification { + const unverified = (reason: string): OutputVerification => ({ + verified: false, + reason, + expected_duration_us: expectedUs, + tolerance_us: frameToleranceUs(fps), + }); + if (!existsSync(output)) return unverified(`output file not found: ${output}`); + if (!ffprobeAvailable(ffprobeCmd)) { + return unverified(`ffprobe is unavailable at '${ffprobeCmd}'; install ffmpeg or pass --ffprobe-cmd `); + } + const probe = probeMedia(output, ffprobeCmd, false); + if (!probe || probe.durationUs === null) return unverified(`ffprobe could not read a duration from ${output}`); + return checkOutputDuration(probe.durationUs, expectedUs, fps); +} + // atempo only accepts 0.5..2.0 per filter instance; we keep proxy audio simple // and only retime when a single atempo can express it. function atempoFor(speed: number): string | null { @@ -698,6 +961,7 @@ export function buildRenderPlan(draft: Draft, opts: RenderOptions): RenderPlan { ...(softCaptions ? { softCaptions } : {}), ...(filterScript ? { filterScript } : {}), quality, + fidelity: fidelityCensus(draft, opts, skipped), }; } @@ -765,6 +1029,7 @@ export function renderDraft(draft: Draft, filePath: string, opts: RenderOptions) } const basePlan = buildRenderPlan(draft, effective); const plan = { ...basePlan, capabilities, skipped: [...basePlan.skipped, ...fallbackSkipped] }; + if (opts.strict) assertFaithful(plan.fidelity); // Speed/volume/fade filters are checked against the plan actually built: // a draft that never sets them must keep rendering on a build without them, @@ -865,5 +1130,11 @@ export function renderDraft(draft: Draft, filePath: string, opts: RenderOptions) "Re-run with --dry-run to inspect the filter graph without executing.", ); } - return { ...plan, ok: true, executed: true }; + const verification = probeRenderOutput( + plan.output, + plan.fidelity.expected_duration_us, + plan.fps, + opts.ffprobeCmd ?? "ffprobe", + ); + return { ...plan, ok: true, executed: true, verification }; } diff --git a/test/render-fidelity.test.mjs b/test/render-fidelity.test.mjs new file mode 100644 index 0000000..bf958e2 --- /dev/null +++ b/test/render-fidelity.test.mjs @@ -0,0 +1,432 @@ +import assert from "node:assert/strict"; +import { chmodSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, describe, it } from "node:test"; +import { + buildRenderPlan, + checkOutputDuration, + FIDELITY_CATEGORIES, + FIDELITY_ID_CAP, + fidelityCensus, + frameToleranceUs, + probeRenderOutput, + renderDraft, +} from "../dist/render.js"; +import { spawnCli } from "./helpers/spawn-cli.mjs"; + +const US = 1_000_000; +const isWindows = process.platform === "win32"; // fake ffmpeg/ffprobe are /bin/sh scripts + +function setup() { + const dir = mkdtempSync(join(tmpdir(), "capcut-render-fidelity-")); + return { dir, cleanup: () => rmSync(dir, { recursive: true, force: true }) }; +} + +const seg = (id, matId, start, dur, extra = {}) => ({ + id, + material_id: matId, + target_timerange: { start, duration: dur }, + source_timerange: { start: 0, duration: dur }, + speed: 1, + volume: 1, + visible: true, + clip: null, + extra_material_refs: [], + render_index: 0, + ...extra, +}); + +// Two contiguous main-track clips, one audio bed, one caption: everything the +// proxy reproduces once --burn-captions is on. +function plainDraft(dir) { + const v1 = join(dir, "clip1.mp4"); + const v2 = join(dir, "clip2.mp4"); + const a1 = join(dir, "music.mp3"); + for (const p of [v1, v2, a1]) writeFileSync(p, ""); + return { + id: "d", + name: "t", + duration: 4 * US, + fps: 30, + canvas_config: { width: 720, height: 1280, ratio: "9:16" }, + tracks: [ + { + id: "tv", + type: "video", + name: "video", + attribute: 0, + segments: [seg("v1", "mv1", 0, 2 * US), seg("v2", "mv2", 2 * US, 2 * US)], + }, + { id: "ta", type: "audio", name: "audio", attribute: 0, segments: [seg("a1", "ma1", 0, 4 * US)] }, + { id: "tt", type: "text", name: "captions", attribute: 0, segments: [seg("t1", "mt1", 0, 2 * US)] }, + ], + materials: { + videos: [ + { id: "mv1", path: v1, type: "video", material_name: "clip1.mp4", duration: 2 * US }, + { id: "mv2", path: v2, type: "video", material_name: "clip2.mp4", duration: 2 * US }, + ], + audios: [{ id: "ma1", path: a1, name: "music", type: "extract_music", duration: 4 * US }], + texts: [{ id: "mt1", type: "text", content: JSON.stringify({ text: "Hook line" }) }], + speeds: [], + material_animations: [], + audio_fades: [], + transitions: [], + }, + }; +} + +// Every category the census checks, one segment each where possible. +function richDraft(dir) { + const d = plainDraft(dir); + const ov = join(dir, "overlay.mp4"); + writeFileSync(ov, ""); + const main = d.tracks[0].segments; + main[0].extra_material_refs = ["tr1", "mask1", "chroma1", "anim-video"]; + main[0].common_keyframes = [{ property_type: "KFTypePositionX", keyframe_list: [{ time_offset: 0 }] }]; + // A gap before the second clip: concat would pull it to 2s. + main[1].target_timerange = { start: 3 * US, duration: 1 * US }; + main[1].source_timerange = { start: 0, duration: 1 * US }; + // An empty keyframe list is not animation. + main[1].common_keyframes = [{ property_type: "KFTypeScaleX", keyframe_list: [] }]; + d.tracks[2].segments[0].extra_material_refs = ["anim-text"]; + d.tracks.push( + { id: "to", type: "video", name: "overlay", attribute: 0, segments: [seg("o1", "mov", 0, 1 * US)] }, + { id: "ts", type: "sticker", name: "sticker", attribute: 0, segments: [seg("s1", "st1", 0, 1 * US)] }, + { id: "te", type: "effect", name: "effect", attribute: 0, segments: [seg("e1", "fx1", 0, 1 * US)] }, + { id: "tf", type: "filter", name: "filter", attribute: 0, segments: [seg("f1", "flt1", 0, 1 * US)] }, + ); + d.materials.videos[1].mix_mode = "Screen"; + d.materials.videos[1].matting = { flag: 2 }; + d.materials.videos.push({ id: "mov", path: ov, type: "video", material_name: "overlay.mp4", duration: US }); + d.materials.transitions = [{ id: "tr1", name: "Dissolve" }]; + d.materials.common_mask = [{ id: "mask1", type: "mask" }]; + d.materials.chromas = [{ id: "chroma1", type: "chromas" }]; + d.materials.video_effects = [ + { id: "fx1", type: "video_effect" }, + { id: "flt1", type: "filter" }, + ]; + d.materials.material_animations = [ + { id: "anim-video", type: "sticker_animation", animations: [{ type: "in" }] }, + { id: "anim-text", type: "sticker_animation", animations: [{ type: "out" }] }, + ]; + return d; +} + +describe("render fidelity census (pure)", () => { + it("a plain draft with burned captions is faithful and targets the draft duration", () => { + const s = setup(); + after(s.cleanup); + const plan = buildRenderPlan(plainDraft(s.dir), { out: join(s.dir, "p.mp4"), burnCaptions: true }); + assert.equal(plan.fidelity.faithful, true); + assert.deepEqual(plan.fidelity.dropped, {}); + assert.equal(plan.fidelity.expected_duration_us, 4 * US); + assert.deepEqual(plan.fidelity.checked, [...FIDELITY_CATEGORIES]); + }); + + it("text that is not burned counts as dropped", () => { + const s = setup(); + after(s.cleanup); + const plan = buildRenderPlan(plainDraft(s.dir), { out: join(s.dir, "p.mp4"), softCaptions: true }); + assert.equal(plan.fidelity.faithful, false); + assert.deepEqual(plan.fidelity.dropped.text.segment_ids, ["t1"]); + assert.match(plan.fidelity.dropped.text.hint, /--burn-captions/); + }); + + it("names every category the draft carries, with the affected segment ids", () => { + const s = setup(); + after(s.cleanup); + const census = buildRenderPlan(richDraft(s.dir), { out: join(s.dir, "p.mp4"), burnCaptions: true }).fidelity; + const ids = Object.fromEntries(Object.entries(census.dropped).map(([k, v]) => [k, v.segment_ids])); + assert.deepEqual(ids, { + main_track_gaps: ["v2"], + overlay_tracks: ["o1"], + stickers: ["s1"], + transitions: ["v1"], + effects: ["e1"], + filters: ["f1"], + masks: ["v1"], + keyframes: ["v1"], + text_animations: ["t1"], + video_animations: ["v1"], + mix_modes: ["v2"], + chroma: ["v1"], + matting: ["v2"], + }); + assert.equal(census.faithful, false); + for (const entry of Object.values(census.dropped)) assert.equal(entry.count, entry.segment_ids.length); + }); + + it("--all-video-tracks composites overlays, so they leave the census", () => { + const s = setup(); + after(s.cleanup); + const census = fidelityCensus(richDraft(s.dir), { allVideoTracks: true, burnCaptions: true }); + assert.equal(census.dropped.overlay_tracks, undefined); + }); + + it("caps the id list but keeps the full count", () => { + const s = setup(); + after(s.cleanup); + const d = plainDraft(s.dir); + const n = FIDELITY_ID_CAP + 5; + d.tracks.push({ + id: "ts", + type: "sticker", + name: "sticker", + attribute: 0, + segments: Array.from({ length: n }, (_, i) => seg(`s${i}`, "st", i * 1000, 1000)), + }); + const census = fidelityCensus(d, { burnCaptions: true }); + assert.equal(census.dropped.stickers.count, n); + assert.equal(census.dropped.stickers.segment_ids.length, FIDELITY_ID_CAP); + }); + + it("main-track segments whose media is missing count as missing_media", () => { + const s = setup(); + after(s.cleanup); + const d = plainDraft(s.dir); + d.materials.videos[1].path = join(s.dir, "gone.mp4"); + const plan = buildRenderPlan(d, { out: join(s.dir, "p.mp4"), burnCaptions: true }); + assert.deepEqual(plan.fidelity.dropped.missing_media.segment_ids, ["v2"]); + }); + + it("falls back to the last segment end when draft.duration is unset", () => { + const s = setup(); + after(s.cleanup); + const d = plainDraft(s.dir); + d.duration = 0; + assert.equal(fidelityCensus(d, {}).expected_duration_us, 4 * US); + }); +}); + +describe("render output verification (pure)", () => { + it("tolerance is one frame at the render fps; drift within it passes", () => { + assert.equal(frameToleranceUs(30), 33_333); + assert.equal(frameToleranceUs(25), 40_000); + const ok = checkOutputDuration(4 * US + 33_333, 4 * US, 30); + assert.deepEqual(ok, { + verified: true, + duration_us: 4 * US + 33_333, + expected_duration_us: 4 * US, + drift_us: 33_333, + tolerance_us: 33_333, + within_tolerance: true, + }); + const short = checkOutputDuration(3 * US, 4 * US, 30); + assert.equal(short.drift_us, -US); + assert.equal(short.within_tolerance, false); + }); + + it("an unprobeable output reports verified:false with a reason instead of failing", () => { + const s = setup(); + after(s.cleanup); + const missing = probeRenderOutput(join(s.dir, "nope.mp4"), 4 * US, 30); + assert.equal(missing.verified, false); + assert.match(missing.reason, /not found/); + const out = join(s.dir, "p.mp4"); + writeFileSync(out, ""); + const noProbe = probeRenderOutput(out, 4 * US, 30, join(s.dir, "no-ffprobe")); + assert.equal(noProbe.verified, false); + assert.match(noProbe.reason, /ffprobe is unavailable/); + }); +}); + +// A fake ffmpeg with every probed filter and libx264 that "renders" by +// touching its last argument (the output path). +function fakeFfmpeg(dir) { + const path = join(dir, "fake-ffmpeg"); + const filters = [ + "fps", + "scale", + "pad", + "setsar", + "format", + "trim", + "setpts", + "concat", + "atrim", + "asetpts", + "adelay", + "anull", + "amix", + "atempo", + "volume", + "afade", + "drawtext", + "overlay", + ]; + writeFileSync( + path, + [ + "#!/bin/sh", + 'case "$*" in', + " *-filters*)", + " cat <<'EOF'", + ...filters.map((name) => ` ... ${name.padEnd(18)} V->V fake.`), + "EOF", + " exit 0 ;;", + " *-encoders*)", + " cat <<'EOF'", + " V..... libx264 H.264", + " S..... mov_text 3GPP Timed Text subtitle", + "EOF", + " exit 0 ;;", + "esac", + 'for last; do :; done; : > "$last"', + "exit 0", + "", + ].join("\n"), + ); + chmodSync(path, 0o755); + return path; +} + +// A fake ffprobe reporting the container duration given in FAKE_FFPROBE_DURATION (seconds). +function fakeFfprobe(dir) { + const path = join(dir, "fake-ffprobe"); + writeFileSync( + path, + [ + "#!/bin/sh", + 'if [ "$1" = "-version" ]; then echo "ffprobe version fake"; exit 0; fi', + 'printf \'{"streams":[{"codec_type":"video","width":360,"height":640}],"format":{"duration":"%s"}}\' "$FAKE_FFPROBE_DURATION"', + "", + ].join("\n"), + ); + chmodSync(path, 0o755); + return path; +} + +function writeDraft(dir, draft) { + const path = join(dir, "draft_content.json"); + writeFileSync(path, JSON.stringify(draft)); + return path; +} + +describe("render --strict", () => { + it("--dry-run reports the census; --strict refuses an unfaithful plan", () => { + const s = setup(); + after(s.cleanup); + const draftPath = writeDraft(s.dir, richDraft(s.dir)); + const plain = spawnCli(["render", draftPath, "--out", join(s.dir, "p.mp4"), "--dry-run"]); + assert.equal(plain.status, 0, plain.stderr); + assert.equal(plain.json.fidelity.faithful, false); + assert.equal(plain.json.fidelity.dropped.transitions.count, 1); + + const strict = spawnCli(["render", draftPath, "--out", join(s.dir, "p.mp4"), "--dry-run", "--strict"]); + assert.notEqual(strict.status, 0); + assert.match(strict.stderr, /refused \[render-unfaithful\]/); + assert.match(strict.stderr, /transitions 1/); + }); + + it("--strict passes a faithful plan through", () => { + const s = setup(); + after(s.cleanup); + const draftPath = writeDraft(s.dir, plainDraft(s.dir)); + const r = spawnCli([ + "render", + draftPath, + "--out", + join(s.dir, "p.mp4"), + "--dry-run", + "--strict", + "--burn-captions", + ]); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.json.fidelity.faithful, true); + }); + + it("refuses before ffmpeg runs, so nothing is written", { skip: isWindows }, () => { + const s = setup(); + after(s.cleanup); + const out = join(s.dir, "p.mp4"); + assert.throws( + () => + renderDraft(richDraft(s.dir), join(s.dir, "draft_content.json"), { + out, + ffmpegCmd: fakeFfmpeg(s.dir), + strict: true, + }), + /refused \[render-unfaithful\]/, + ); + assert.equal(existsSync(out), false); + }); +}); + +describe("render output verification (fake ffmpeg + ffprobe)", { skip: isWindows }, () => { + function run(dir, durationSeconds, extra = []) { + const draftPath = writeDraft(dir, plainDraft(dir)); + const out = join(dir, "p.mp4"); + const r = spawnCli( + [ + "render", + draftPath, + "--out", + out, + "--ffmpeg-cmd", + fakeFfmpeg(dir), + "--ffprobe-cmd", + fakeFfprobe(dir), + "--burn-captions", + ...extra, + ], + { env: { FAKE_FFPROBE_DURATION: String(durationSeconds) } }, + ); + return { r, out }; + } + + it("reports the probed duration and drift within one frame", () => { + const s = setup(); + after(s.cleanup); + const { r } = run(s.dir, "4.02", ["--verify"]); + assert.equal(r.status, 0, r.stderr); + assert.deepEqual(r.json.verification, { + verified: true, + duration_us: 4_020_000, + expected_duration_us: 4 * US, + drift_us: 20_000, + tolerance_us: 33_333, + within_tolerance: true, + }); + assert.equal(typeof r.json.output, "string", "the existing output path field is unchanged"); + }); + + it("drift beyond tolerance is reported; only --verify turns it into a failure, and the file stays", () => { + const s = setup(); + after(s.cleanup); + const lax = run(s.dir, "3.5"); + assert.equal(lax.r.status, 0, lax.r.stderr); + assert.equal(lax.r.json.verification.within_tolerance, false); + assert.equal(lax.r.json.verification.drift_us, -500_000); + + const strict = run(s.dir, "3.5", ["--verify"]); + assert.notEqual(strict.r.status, 0); + assert.equal(strict.r.json.verification.within_tolerance, false); + assert.match(strict.r.stderr, /drifts -500000us/); + assert.ok(existsSync(strict.out), "the rendered file is kept"); + }); + + it("missing ffprobe: verified:false normally, a clear failure with --verify", () => { + const s = setup(); + after(s.cleanup); + const draftPath = writeDraft(s.dir, plainDraft(s.dir)); + const args = [ + "render", + draftPath, + "--out", + join(s.dir, "p.mp4"), + "--ffmpeg-cmd", + fakeFfmpeg(s.dir), + "--ffprobe-cmd", + join(s.dir, "no-such-ffprobe"), + ]; + const lax = spawnCli(args); + assert.equal(lax.status, 0, lax.stderr); + assert.equal(lax.json.verification.verified, false); + assert.match(lax.json.verification.reason, /ffprobe is unavailable/); + + const strict = spawnCli([...args, "--verify"]); + assert.notEqual(strict.status, 0); + assert.match(strict.stderr, /could not be verified: ffprobe is unavailable/); + }); +}); From 2be65b69b13e996bcc8b1ecd229688480f034545 Mon Sep 17 00:00:00 2001 From: Rene Zander Date: Fri, 9 Oct 2026 05:26:16 +0000 Subject: [PATCH 3/6] Add a pronunciation lexicon to tts (--lexicon) Rules rewrite only the text handed to the TTS engine: leftmost-longest matching at word boundaries (CJK rules match anywhere), optional case-insensitive rules, and refusals for malformed (lexicon-invalid) or ambiguous equal-length rules (lexicon-ambiguous) before any engine runs. The result reports the applied rules with UTF-16 offsets and the spoken text. --- docs/command-reference.json | 16 ++- docs/command-reference.md | 2 +- src/command-specs.ts | 13 +- src/index.ts | 37 +++++- src/lexicon.ts | 156 ++++++++++++++++++++++++ test/tts-lexicon.test.mjs | 229 ++++++++++++++++++++++++++++++++++++ 6 files changed, 449 insertions(+), 4 deletions(-) create mode 100644 src/lexicon.ts create mode 100644 test/tts-lexicon.test.mjs diff --git a/docs/command-reference.json b/docs/command-reference.json index 3617f30..ffe3ab0 100644 --- a/docs/command-reference.json +++ b/docs/command-reference.json @@ -1405,7 +1405,7 @@ { "name": "tts", "summary": "Synthesize a voiceover from text via a local TTS command (--tts-cmd) and add it as an audio segment.", - "usage": "capcut tts [start] [duration] (--text | --text-file ) --tts-cmd