diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a037b0f9..ab17e7be 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -38,13 +38,6 @@ on: paths: - 'bench/**' -# Keyed on the PR or ref rather than shared with CI, so asking for benchmarks -# never cancels a correctness run. A separate workflow file already gets a -# distinct `github.workflow`, so this only needs to disambiguate within itself. -concurrency: - group: ${{ github.workflow }}-${{ github.event.issue.number || github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: benchmark: # The COMMENT author, not the PR author, is the trust boundary on the @@ -60,6 +53,12 @@ jobs: github.event.comment.author_association == 'COLLABORATOR' ) ) + # Apply concurrency after the job's comment filter. Workflow-level concurrency + # would let an ordinary PR comment cancel a running benchmark even when its + # own benchmark job is skipped. + concurrency: + group: ${{ github.workflow }}-job-${{ github.event.issue.number || github.event.pull_request.number || github.ref }} + cancel-in-progress: true runs-on: ubuntu-latest timeout-minutes: 90 permissions: @@ -135,6 +134,8 @@ jobs: key: lake-build-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('lean-toolchain') }}-${{ hashFiles('lake-manifest.json') }}-${{ github.sha }} restore-keys: | lake-build-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('lean-toolchain') }}-${{ hashFiles('lake-manifest.json') }} + - name: Drop host-tuned native objects + run: ./scripts/drop-native-objects.sh - name: Set up Lean environment uses: leanprover/lean-action@v1 with: @@ -226,7 +227,12 @@ jobs: # Same marker-upsert shape as the build-timing comment in # `lean_action_ci.yml`, so repeated runs replace rather than pile up. - name: Upsert benchmark PR comment - if: always() && steps.target.outputs.pr != '' + # Fork pull_request tokens are read-only; keep their report in the + # Actions summary and artifact instead of failing on a comment write. + if: >- + always() && steps.target.outputs.pr != '' && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) uses: actions/github-script@v7 with: script: | diff --git a/.github/workflows/lean_action_ci.yml b/.github/workflows/lean_action_ci.yml index a3dcb23b..1ecfa12e 100644 --- a/.github/workflows/lean_action_ci.yml +++ b/.github/workflows/lean_action_ci.yml @@ -143,6 +143,8 @@ jobs: key: lake-build-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('lean-toolchain') }}-${{ hashFiles('lake-manifest.json') }}-${{ github.sha }} restore-keys: | lake-build-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('lean-toolchain') }}-${{ hashFiles('lake-manifest.json') }} + - name: Drop host-tuned native objects + run: ./scripts/drop-native-objects.sh - name: Set up Lean environment uses: leanprover/lean-action@v1 with: @@ -185,7 +187,19 @@ jobs: continue-on-error: true run: lake build --wfail - name: Build evaluation benchmark executable - run: lake build CompPolyBench + run: lake build CompPolyBench CompPolyFieldFixtures + - name: Install field benchmark Rust toolchain + working-directory: bench/rust + run: rustup show active-toolchain + - name: Test and build field Rust benchmarks + working-directory: bench/rust + env: + RUSTFLAGS: "-C target-cpu=native" + run: | + cargo test --release --locked -j 1 + cargo build --release --locked -j 1 + - name: Check Lean and Rust workload agreement + run: python3 scripts/bench-fields.py --suite all --validate-only - name: Validate benchmark implementations # Correctness only: every implementation in a group runs over the same # inputs and must agree on a digest, so a wrong-but-fast implementation diff --git a/.gitignore b/.gitignore index 5b1b1f7d..b33f6084 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /.lake .DS_Store +__pycache__/ diff --git a/CompPoly.lean b/CompPoly.lean index abf469b0..71ba8a9e 100644 --- a/CompPoly.lean +++ b/CompPoly.lean @@ -62,12 +62,16 @@ public import CompPoly.Bivariate.GuruswamiSudan.Root.ShiftedSubstitution.Lemmas public import CompPoly.Bivariate.GuruswamiSudan.Util public import CompPoly.Bivariate.Kronecker public import CompPoly.Bivariate.ToPoly +public import CompPoly.Data.Array.Involution public import CompPoly.Data.Array.Lemmas +public import CompPoly.Data.ByteArray.Pack +public import CompPoly.Data.ByteArray.Packed public import CompPoly.Data.Bytes.Bias public import CompPoly.Data.Bytes.CanonicalNat public import CompPoly.Data.Bytes.Codec public import CompPoly.Data.Bytes.Delimited public import CompPoly.Data.Bytes.LittleEndian +public import CompPoly.Data.Bytes.UInt32 public import CompPoly.Data.Bytes.Vector public import CompPoly.Data.Classes.CanonicalNat public import CompPoly.Data.Classes.DCast @@ -78,6 +82,7 @@ public import CompPoly.Data.ExtTreeMap.DTreeMap public import CompPoly.Data.ExtTreeMap.ExtDTreeMap public import CompPoly.Data.ExtTreeMap.ExtTreeMap public import CompPoly.Data.Fin.BigOperators +public import CompPoly.Data.List.BatchInv public import CompPoly.Data.List.Lemmas public import CompPoly.Data.MvPolynomial.Notation public import CompPoly.Data.Nat.Bitwise @@ -90,6 +95,7 @@ public import CompPoly.Data.RingTheory.AlgebraTower public import CompPoly.Data.RingTheory.AlgebraTower.Basis public import CompPoly.Data.RingTheory.AlgebraTower.Coordinates public import CompPoly.Data.RingTheory.CanonicalEuclideanDomain +public import CompPoly.Data.Tasks public import CompPoly.Data.Vector.Basic public import CompPoly.Fields.BLS12_377 public import CompPoly.Fields.BLS12_377.Basic @@ -104,6 +110,8 @@ public import CompPoly.Fields.BabyBear public import CompPoly.Fields.BabyBear.Basic public import CompPoly.Fields.BabyBear.Ext4 public import CompPoly.Fields.BabyBear.Fast +public import CompPoly.Fields.BabyBear.FastExt4 +public import CompPoly.Fields.BabyBear.InterpolateCoset public import CompPoly.Fields.Basic public import CompPoly.Fields.Binary.AdditiveNTT.AdditiveNTT public import CompPoly.Fields.Binary.AdditiveNTT.Algorithm @@ -169,6 +177,7 @@ public import CompPoly.Fields.Extension.Bytes public import CompPoly.Fields.Extension.Cardinality public import CompPoly.Fields.Extension.Defs public import CompPoly.Fields.Extension.Field +public import CompPoly.Fields.Extension.Map public import CompPoly.Fields.Goldilocks public import CompPoly.Fields.Goldilocks.Basic public import CompPoly.Fields.Goldilocks.Bytes @@ -188,15 +197,28 @@ public import CompPoly.Fields.KoalaBear.Ext6.GaloisField public import CompPoly.Fields.KoalaBear.Ext6.SexticCertData public import CompPoly.Fields.KoalaBear.Ext6.SexticIrreducible public import CompPoly.Fields.KoalaBear.Fast +public import CompPoly.Fields.KoalaBear.FastExt4 +public import CompPoly.Fields.KoalaBear.InterpolateCoset public import CompPoly.Fields.Mersenne31 public import CompPoly.Fields.Mersenne31.Basic public import CompPoly.Fields.Mersenne31.Bytes public import CompPoly.Fields.Mersenne31.Circle public import CompPoly.Fields.Mersenne31.Fast public import CompPoly.Fields.Montgomery.Basic +public import CompPoly.Fields.Montgomery.InterpolateCoset +public import CompPoly.Fields.Montgomery.InterpolateCosetPacked +public import CompPoly.Fields.Montgomery.InterpolateCosetPackedCorrectness public import CompPoly.Fields.Montgomery.Native32 public import CompPoly.Fields.Montgomery.Native32Bytes +public import CompPoly.Fields.Montgomery.Native32Ext4 public import CompPoly.Fields.Montgomery.Native32Field +public import CompPoly.Fields.Montgomery.Native64x4 +public import CompPoly.Fields.Montgomery.Native64x4Bytes +public import CompPoly.Fields.Montgomery.Native64x4Defs +public import CompPoly.Fields.Montgomery.Native64x4Field +public import CompPoly.Fields.Montgomery.Native64x4Inv +public import CompPoly.Fields.Montgomery.Native64x4InvDefs +public import CompPoly.Fields.Montgomery.Native64x4Mul public import CompPoly.Fields.Montgomery.Native64x8 public import CompPoly.Fields.Montgomery.Native64x8Bytes public import CompPoly.Fields.Montgomery.Native64x8Defs @@ -210,6 +232,8 @@ public import CompPoly.Fields.Pasta.Basic public import CompPoly.Fields.Pasta.Fast public import CompPoly.Fields.PrattCertificate public import CompPoly.Fields.Secp256k1 +public import CompPoly.Fields.Secp256k1.Basic +public import CompPoly.Fields.Secp256k1.Fast public import CompPoly.LinearAlgebra.Dense public import CompPoly.LinearAlgebra.Dense.Basic public import CompPoly.LinearAlgebra.Dense.Kernel @@ -317,6 +341,7 @@ public import CompPoly.Univariate.Modular public import CompPoly.Univariate.NTT.BabyBear public import CompPoly.Univariate.NTT.Barycentric public import CompPoly.Univariate.NTT.Coset +public import CompPoly.Univariate.NTT.CosetBarycentric public import CompPoly.Univariate.NTT.Domain public import CompPoly.Univariate.NTT.Evaluation public import CompPoly.Univariate.NTT.FastMul @@ -327,6 +352,10 @@ public import CompPoly.Univariate.NTT.Inverse public import CompPoly.Univariate.NTT.Kernel public import CompPoly.Univariate.NTT.KoalaBear public import CompPoly.Univariate.NTT.Transform +public import CompPoly.Univariate.NTTFast.Butterfly +public import CompPoly.Univariate.NTTFast.ButterflyDIT +public import CompPoly.Univariate.NTTFast.Columns +public import CompPoly.Univariate.NTTFast.ColumnsCorrectness public import CompPoly.Univariate.NTTFast.Correctness public import CompPoly.Univariate.NTTFast.Correctness.Basic public import CompPoly.Univariate.NTTFast.Correctness.DIF @@ -339,7 +368,51 @@ public import CompPoly.Univariate.NTTFast.Evaluation public import CompPoly.Univariate.NTTFast.FastMul public import CompPoly.Univariate.NTTFast.FastMulLow public import CompPoly.Univariate.NTTFast.Interpolation +public import CompPoly.Univariate.NTTFast.Natural +public import CompPoly.Univariate.NTTFast.NaturalParallel +public import CompPoly.Univariate.NTTFast.Packed.Arithmetic +public import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas +public import CompPoly.Univariate.NTTFast.Packed.Arrays +public import CompPoly.Univariate.NTTFast.Packed.BatchCorrectness +public import CompPoly.Univariate.NTTFast.Packed.BitReversal +public import CompPoly.Univariate.NTTFast.Packed.ButterflyLemmas +public import CompPoly.Univariate.NTTFast.Packed.Columns +public import CompPoly.Univariate.NTTFast.Packed.ColumnsCorrectness +public import CompPoly.Univariate.NTTFast.Packed.CosetLde +public import CompPoly.Univariate.NTTFast.Packed.CosetLdeCorrectness +public import CompPoly.Univariate.NTTFast.Packed.CosetLdeTransposes +public import CompPoly.Univariate.NTTFast.Packed.Decoder +public import CompPoly.Univariate.NTTFast.Packed.DifSpec +public import CompPoly.Univariate.NTTFast.Packed.Expressions +public import CompPoly.Univariate.NTTFast.Packed.FieldStorage +public import CompPoly.Univariate.NTTFast.Packed.Folds +public import CompPoly.Univariate.NTTFast.Packed.Instances +public import CompPoly.Univariate.NTTFast.Packed.KernelCorrectness +public import CompPoly.Univariate.NTTFast.Packed.KernelModels +public import CompPoly.Univariate.NTTFast.Packed.KernelRefinement +public import CompPoly.Univariate.NTTFast.Packed.KernelSpecialization +public import CompPoly.Univariate.NTTFast.Packed.LeafLayers +public import CompPoly.Univariate.NTTFast.Packed.LeafStages +public import CompPoly.Univariate.NTTFast.Packed.LocalLayers +public import CompPoly.Univariate.NTTFast.Packed.LoopRefinement +public import CompPoly.Univariate.NTTFast.Packed.Native +public import CompPoly.Univariate.NTTFast.Packed.NativeLast +public import CompPoly.Univariate.NTTFast.Packed.NativeLastCorrectness +public import CompPoly.Univariate.NTTFast.Packed.NativeOrder +public import CompPoly.Univariate.NTTFast.Packed.NativeOrderCorrectness +public import CompPoly.Univariate.NTTFast.Packed.PassRefinement +public import CompPoly.Univariate.NTTFast.Packed.Plan +public import CompPoly.Univariate.NTTFast.Packed.QuadCells +public import CompPoly.Univariate.NTTFast.Packed.Radix2Refinement +public import CompPoly.Univariate.NTTFast.Packed.Radix4Lemmas +public import CompPoly.Univariate.NTTFast.Packed.Rows +public import CompPoly.Univariate.NTTFast.Packed.StorageLemmas +public import CompPoly.Univariate.NTTFast.Packed.Twiddles +public import CompPoly.Univariate.NTTFast.Packed.Unrolled +public import CompPoly.Univariate.NTTFast.Parallel +public import CompPoly.Univariate.NTTFast.Permutation public import CompPoly.Univariate.NTTFast.Plan +public import CompPoly.Univariate.NTTFast.Reverse32 public import CompPoly.Univariate.Quotient.Core public import CompPoly.Univariate.Quotient.Equiv public import CompPoly.Univariate.Raw diff --git a/CompPoly/Data/Array/Involution.lean b/CompPoly/Data/Array/Involution.lean new file mode 100644 index 00000000..d7c90e83 --- /dev/null +++ b/CompPoly/Data/Array/Involution.lean @@ -0,0 +1,140 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import Mathlib.Init +public import Init.Data.Range.Lemmas + +/-! # Array permutations from index involutions + +Swap each nontrivial pair once instead of allocating a second array for a gather. +-/ + +@[expose] public section +namespace Array +variable {α : Type*} + +@[inline] private def involutionStep (f : Nat → Nat) (a : Array α) (i : Nat) : Array α := + if i < f i then a.swapIfInBounds i (f i) else a + +@[simp] private theorem size_involutionStep (f : Nat → Nat) (a : Array α) (i : Nat) : + (involutionStep f a i).size = a.size := by + simp only [involutionStep]; split <;> simp only [Array.size_swapIfInBounds] + +private theorem size_fold (f : Nat → Nat) (is : List Nat) (a : Array α) : + (is.foldl (involutionStep f) a).size = a.size := by + induction is generalizing a with + | nil => rfl + | cons i is ih => + simpa only [List.foldl_cons, size_involutionStep] using ih (involutionStep f a i) + +private theorem getD_swap (a : Array α) (i j k : Nat) (d : α) + (hi : i < a.size) (hj : j < a.size) (hk : k < a.size) : + (a.swapIfInBounds i j).getD k d = + if k = i then a.getD j d else if k = j then a.getD i d else a.getD k d := by + simp only [Array.swapIfInBounds, hi, hj, ↓reduceDIte, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (show k < (a.swap i j hi hj).size by simpa using hk), + Array.getElem?_eq_getElem hi, Array.getElem?_eq_getElem hj, + Array.getElem?_eq_getElem hk, Option.getD_some, Array.getElem_swap] + +private theorem prefix_invariant (f : Nat → Nat) (a : Array α) (d : α) + (hb : ∀ i < a.size, f i < a.size) + (hf : ∀ i < a.size, f (f i) = i) (k : Nat) (hk : k ≤ a.size) : + ∀ j < a.size, ((List.range k).foldl (involutionStep f) a).getD j d = + if min j (f j) < k then a.getD (f j) d else a.getD j d := by + induction k with + | zero => intro j hj; simp only [List.range_zero, List.foldl_nil, Nat.not_lt_zero, ↓reduceIte] + | succ k ih => + have hk' : k < a.size := by omega + have ih := ih (by omega) + intro j hj + simp only [List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil] + by_cases hswap : k < f k + · simp only [involutionStep, hswap, ↓reduceIte] + rw [getD_swap _ _ _ _ _ (by simpa only [size_fold] using hk') + (by simpa only [size_fold] using hb k hk') (by simpa only [size_fold] using hj)] + by_cases hji : j = k + · subst j + rw [ite_eq_left rfl, ih (f k) (hb k hk'), hf k hk'] + have hmin1 : ¬ min (f k) k < k := by omega + have hmin2 : min k (f k) < k + 1 := by omega + simp only [hmin1, hmin2, ↓reduceIte] + · rw [ite_eq_right hji] + by_cases hjf : j = f k + · subst j + rw [ite_eq_left rfl, ih k hk', hf k hk'] + have hmin1 : ¬ min k (f k) < k := by omega + have hmin2 : min (f k) k < k + 1 := by omega + simp only [hmin1, hmin2, ↓reduceIte] + · rw [ite_eq_right hjf, ih j hj] + have hfj : f j ≠ k := by + intro he + have hh := hf j hj + simp only [he] at hh + exact hjf hh.symm + have he : (min j (f j) < k) = (min j (f j) < k + 1) := by + apply propext; omega + simp only [he] + · simp only [involutionStep, hswap, ↓reduceIte] + rw [ih j hj] + by_cases hfix : f j = j + · simp only [hfix, ite_self] + · have hmin : min j (f j) ≠ k := by + by_cases hji : j = k + · subst j; omega + · by_cases hfj : f j = k + · have hh := hf j hj + rw [hfj] at hh + omega + · omega + have he : (min j (f j) < k) = (min j (f j) < k + 1) := by + apply propext; omega + simp only [he] + +/-- Apply an index involution by swapping each pair once, reusing a uniquely owned array. -/ +@[inline] def permuteInvolution (order : Array Nat) (a : Array α) : Array α := Id.run do + let mut out := a + for i in [:order.size] do + let j := order[i]! + if i < j then out := out.swapIfInBounds i j + return out + +private theorem permuteInvolution_eq_fold (order : Array Nat) (a : Array α) : + permuteInvolution order a = + (List.range order.size).foldl (involutionStep (fun i ↦ order[i]!)) a := by + simp [permuteInvolution, ← apply_ite, Std.Legacy.Range.forIn_eq_forIn_range', + List.forIn_pure_yield_eq_foldl, + Std.Legacy.Range.size, List.range_eq_range'] + rfl + +/-- Swapping entries preserves the array length, including for invalid index tables. -/ +@[simp] theorem size_permuteInvolution (order : Array Nat) (a : Array α) : + (permuteInvolution order a).size = a.size := by + rw [permuteInvolution_eq_fold, size_fold] + +/-- A bounded involutive index table gives the same result as gathering a fresh array. -/ +theorem permuteInvolution_eq_map (order : Array Nat) (a : Array α) (d : α) + (hs : order.size = a.size) + (hb : ∀ i < order.size, order[i]! < order.size) + (hf : ∀ i < order.size, order[order[i]!]! = i) : + permuteInvolution order a = order.map (fun i ↦ a.getD i d) := by + apply Array.ext + · rw [size_permuteInvolution, Array.size_map, hs] + · intro j hj hk + have hja : j < a.size := by simpa only [size_permuteInvolution] using hj + have hjo : j < order.size := by omega + have hfb : ∀ i < a.size, order[i]! < a.size := by simpa only [hs] using hb + have hff : ∀ i < a.size, order[order[i]!]! = i := by simpa only [hs] using hf + have hm := prefix_invariant (fun i ↦ order[i]!) a d hfb hff a.size (Nat.le_refl _) j hja + have hmin : min j order[j]! < a.size := Nat.lt_of_le_of_lt (Nat.min_le_left _ _) hja + simp only [hmin, ↓reduceIte] at hm + have he : (permuteInvolution order a).getD j d = a.getD order[j]! d := by + rw [permuteInvolution_eq_fold, hs] + exact hm + simpa only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hj, + Option.getD_some, getElem!_pos order j hjo, Array.getElem_map] using he + +end Array diff --git a/CompPoly/Data/ByteArray/Pack.lean b/CompPoly/Data/ByteArray/Pack.lean new file mode 100644 index 00000000..20f45161 --- /dev/null +++ b/CompPoly/Data/ByteArray/Pack.lean @@ -0,0 +1,158 @@ +/- +Copyright (c) 2026 Lean FRO, LLC. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Kim Morrison +-/ +module + +public import Init.Data.ByteArray.Basic +import Init.Data.ByteArray.Lemmas + +/-! +# Little-endian `UInt32` load/store on `ByteArray` (backport) + +The little-endian `UInt32` accessors of Lean core's `Init.Data.ByteArray.Pack`, from +[lean4#14053](https://github.com/leanprover/lean4/pull/14053), with the same names, signatures +and Lean definitions. They read or write a `UInt32` at a byte offset of a `ByteArray` with a +single native load or store. + +* `getUInt32LE!` / `setUInt32LE!`: `Nat` offset, no proof. All-or-nothing on bounds: a read + returns `0` and a write leaves the array unchanged unless the whole four-byte window fits. + These are the proof-level specification; the proof-carrying variants below are definitionally + equal to them. +* `getUInt32LE` / `setUInt32LE`: `Nat` offset with an in-bounds proof. +* `ugetUInt32LE` / `usetUInt32LE`: `USize` offset with an in-bounds proof, for hot loops. + +Until a toolchain ships them, their C is inlined here. It differs from the PR's byte-shift +loops in two ways that matter for kernels issuing many adjacent loads and stores, such as the +packed NTT: + +* Loads and stores use a fixed four-byte `memcpy`, byte-swapped on big-endian hosts. The + byte-shift form let clang's vectorizer turn adjacent loads into shuffle sequences. +* After a store, the C states that the array is still unshared (`__builtin_assume`), so the + compiler keeps one sharing check for a run of stores instead of one per store. + +Delete this module when the toolchain provides `Init.Data.ByteArray.Pack`. +-/ + +@[expose] public section + +namespace ByteArray + +/-- Reads the little-endian `UInt32` at byte offset `off`, or `0` if `off + 4 > a.size`. -/ +@[extern c inline "({ b_lean_obj_arg a = #1, i = #2; size_t sz = lean_sarray_size(a), k; + uint32_t r = 0; if (lean_is_scalar(i) && sz >= 4 && (k = lean_unbox(i)) <= sz - 4) { + __builtin_memcpy(&r, lean_sarray_cptr(a) + k, 4); + if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) r = __builtin_bswap32(r); } r; })"] +def getUInt32LE! (a : @& ByteArray) (off : @& Nat) : UInt32 := + if off + 4 ≤ a.size then + (a.get! off).toUInt32 ||| + (a.get! (off+1)).toUInt32 <<< 0x8 ||| + (a.get! (off+2)).toUInt32 <<< 0x10 ||| + (a.get! (off+3)).toUInt32 <<< 0x18 + else 0 + +/-- Writes `v` as a little-endian `UInt32` at byte offset `off`; unchanged if `off + 4 > a.size`. -/ +@[extern c inline "({ lean_object *r = #1; b_lean_obj_arg i = #2; uint32_t v = #3; + if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) v = __builtin_bswap32(v); + size_t sz = lean_sarray_size(r), k; if (lean_is_scalar(i) && sz >= 4 && + (k = lean_unbox(i)) <= sz - 4) { r = lean_sarray_ensure_exclusive(r); + __builtin_memcpy(lean_sarray_cptr(r) + k, &v, 4); __builtin_assume(r->m_rc == 1); } r; })"] +def setUInt32LE! (a : ByteArray) (off : @& Nat) (v : UInt32) : ByteArray := + if off + 4 ≤ a.size then + (((a.set! off v.toUInt8).set! (off+1) (v >>> 0x8).toUInt8).set! (off+2) + (v >>> 0x10).toUInt8).set! (off+3) (v >>> 0x18).toUInt8 + else a + +/-- Reads the little-endian `UInt32` at byte offset `i`. Requires the window to be in bounds. -/ +@[extern c inline "({ uint32_t r; __builtin_memcpy(&r, lean_sarray_cptr(#1) + lean_unbox(#2), 4); + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ? __builtin_bswap32(r) : r; })"] +def getUInt32LE : (a : @& ByteArray) → (i : @& Nat) → (h : i + 4 ≤ a.size := by get_elem_tactic) → + UInt32 + | a, i, _ => a.getUInt32LE! i + +/-- Reads the little-endian `UInt32` at byte offset `i` (`USize`). Requires the window to be in +bounds. -/ +@[extern c inline "({ uint32_t r; __builtin_memcpy(&r, lean_sarray_cptr(#1) + #2, 4); + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ? __builtin_bswap32(r) : r; })"] +def ugetUInt32LE : (a : @& ByteArray) → (i : USize) → + (h : i.toNat + 4 ≤ a.size := by get_elem_tactic) → UInt32 + | a, i, _ => a.getUInt32LE! i.toNat + +/-- Writes `v` as a little-endian `UInt32` at byte offset `i`. Requires the window to be in +bounds. -/ +@[extern c inline "({ lean_object *r = #1; uint32_t v = #3; + if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) v = __builtin_bswap32(v); + r = lean_sarray_ensure_exclusive(r); + __builtin_memcpy(lean_sarray_cptr(r) + lean_unbox(#2), &v, 4); + __builtin_assume(r->m_rc == 1); r; })"] +def setUInt32LE : (a : ByteArray) → (i : @& Nat) → (v : UInt32) → + (h : i + 4 ≤ a.size := by get_elem_tactic) → ByteArray + | a, i, v, _ => a.setUInt32LE! i v + +/-- Writes `v` as a little-endian `UInt32` at byte offset `i` (`USize`). Requires the window to +be in bounds. -/ +@[extern c inline "({ lean_object *r = #1; uint32_t v = #3; + if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) v = __builtin_bswap32(v); + r = lean_sarray_ensure_exclusive(r); + __builtin_memcpy(lean_sarray_cptr(r) + #2, &v, 4); __builtin_assume(r->m_rc == 1); r; })"] +def usetUInt32LE : (a : ByteArray) → (i : USize) → (v : UInt32) → + (h : i.toNat + 4 ≤ a.size := by get_elem_tactic) → ByteArray + | a, i, v, _ => a.setUInt32LE! i.toNat v + +/-! ### Lemmas -/ + +theorem get!_eq_getElem! (a : ByteArray) (i : Nat) : a.get! i = a[i]! := by + by_cases h : i < a.size + · rw [getElem!_pos a i h] + simp only [get!, Array.getElem!_eq_getD, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (show i < a.data.size from h), Option.getD_some] + rfl + · rw [getElem!_neg a i h] + simp only [get!, Array.getElem!_eq_getD, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_none (show a.data.size ≤ i by simp only [size] at h; omega), + Option.getD_none] + +@[simp] theorem size_setUInt32LE! (a : ByteArray) (off : Nat) (v : UInt32) : + (a.setUInt32LE! off v).size = a.size := by unfold setUInt32LE!; split <;> simp + +@[simp] theorem size_usetUInt32LE (a : ByteArray) (i : USize) (v : UInt32) (h) : + (a.usetUInt32LE i v h).size = a.size := size_setUInt32LE! a i.toNat v + +theorem getUInt32LE!_setUInt32LE!_self (a : ByteArray) (off : Nat) (v : UInt32) + (h : off + 4 ≤ a.size) : (a.setUInt32LE! off v).getUInt32LE! off = v := by + unfold getUInt32LE! setUInt32LE! + rw [ite_eq_left h] + simp only [size_set!, get!_eq_getElem!] + rw [ite_eq_left h] + simp (disch := (first | omega | (simp only [size_set!]; omega))) only + [getElem!_set!_self, getElem!_set!_ne] + apply UInt32.toBitVec_inj.mp + apply BitVec.eq_of_getLsbD_eq + intro i hi + simp only [UInt32.toBitVec_or, UInt32.toBitVec_shiftLeft, UInt32.toBitVec_shiftRight, + UInt32.toBitVec_toUInt8, UInt8.toBitVec_toUInt32, BitVec.shiftLeft_eq', BitVec.ushiftRight_eq', + BitVec.getLsbD_or, BitVec.getLsbD_shiftLeft, BitVec.getLsbD_ushiftRight, + BitVec.getLsbD_setWidth, BitVec.toNat_umod] + by_cases h0 : i < 8 <;> by_cases h1 : i < 16 <;> by_cases h2 : i < 24 <;> + first | omega | (cases hb : v.toBitVec[i] <;> simp_all <;> omega) + +theorem getElem!_setUInt32LE!_of_outside (a : ByteArray) (o j : Nat) (v : UInt32) + (h : j < o ∨ o + 4 ≤ j) : (a.setUInt32LE! o v)[j]! = a[j]! := by + unfold setUInt32LE! + split + · simp (disch := (first | omega | (simp only [size_set!]; omega))) only [getElem!_set!_ne] + · rfl + +theorem getUInt32LE!_setUInt32LE!_of_disjoint (a : ByteArray) (o₁ o₂ : Nat) (v : UInt32) + (h₂ : o₂ + 4 ≤ a.size) (hd : o₂ + 4 ≤ o₁ ∨ o₁ + 4 ≤ o₂) : + (a.setUInt32LE! o₁ v).getUInt32LE! o₂ = a.getUInt32LE! o₂ := by + unfold getUInt32LE! + simp only [size_setUInt32LE!, get!_eq_getElem!] + rw [ite_eq_left h₂, ite_eq_left h₂, + getElem!_setUInt32LE!_of_outside _ _ _ _ (by omega), + getElem!_setUInt32LE!_of_outside _ _ _ _ (by omega), + getElem!_setUInt32LE!_of_outside _ _ _ _ (by omega), + getElem!_setUInt32LE!_of_outside _ _ _ _ (by omega)] + +end ByteArray diff --git a/CompPoly/Data/ByteArray/Packed.lean b/CompPoly/Data/ByteArray/Packed.lean new file mode 100644 index 00000000..248ef9ed --- /dev/null +++ b/CompPoly/Data/ByteArray/Packed.lean @@ -0,0 +1,711 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Data.ByteArray.Pack +public import CompPoly.Data.Bytes.LittleEndian + +/-! +# Packed arrays of fixed-width elements + +`PackedRepr R` stores every `R` in `width` bytes. `toBytes` and `ofBytes` are the specification; +`read` and `write` are the fast accesses at a byte offset, proved equal to it. The class stores +the carrier as is: a Montgomery field stores its residue, unlike its canonical `ByteCodec` +serialization. + +A packed array is a `ByteArray` read through `PackedRepr.decode`; `PackedRepr.encode` packs an +`Array R`. Unlike an array of boxed values, a packed array is one scalar array: no per-element +indirection, and the runtime marks it as shared at once when a task returns it. + +The byte-range lemmas (`encode_extract`, `replaceAt_encode`) describe the batch stores of fast +kernels; `read_encode` and `write_encode` describe single elements. +-/ + +@[expose] public section + +universe u + +namespace ByteArray + +/-- Bytes `off, …, off + w - 1` of `b`; zero past the end. -/ +def bytesAt (b : ByteArray) (off w : Nat) : Vector UInt8 w := + Vector.ofFn fun j ↦ b.get! (off + j.val) + +/-- `b` with the bytes from `off` replaced by `values`; bytes past the end are dropped. -/ +def replaceAt (b : ByteArray) (off : Nat) (values : ByteArray) : ByteArray := + ⟨b.data.extract 0 off ++ values.data ++ b.data.extract (off + values.size) b.size⟩ + +theorem size_replaceAt (b values : ByteArray) (off : Nat) (h : off + values.size ≤ b.size) : + (b.replaceAt off values).size = b.size := by + simp only [replaceAt, ByteArray.size, Array.size_append, Array.size_extract] + simp only [ByteArray.size] at h + omega + +theorem replaceAt_empty (b : ByteArray) (off : Nat) (h : off ≤ b.size) : + b.replaceAt off ByteArray.empty = b := by + apply ByteArray.ext + apply Array.ext + · simp only [replaceAt, ByteArray.size, Array.size_append, Array.size_extract] + simp only [ByteArray.size] at h + change min off b.data.size - 0 + 0 + (min b.data.size b.data.size - (off + 0)) = _ + omega + · intro j h1 h2 + have he : ByteArray.empty.data.size = 0 := rfl + simp only [ByteArray.size] at h + simp only [replaceAt, ByteArray.size, Array.getElem_append, Array.getElem_extract, + Array.size_append, Array.size_extract] + split_ifs <;> first | omega | (congr 1; omega) + +/-- Replacing two adjacent byte ranges is replacing their concatenation. -/ +theorem replaceAt_replaceAt (b p q : ByteArray) (off : Nat) + (h : off + p.size + q.size ≤ b.size) : + (b.replaceAt off p).replaceAt (off + p.size) q = b.replaceAt off (p ++ q) := by + apply ByteArray.ext + apply Array.ext + · simp only [replaceAt, ByteArray.size, ByteArray.data_append, Array.size_append, + Array.size_extract] + simp only [ByteArray.size] at h + omega + · intro j h1 h2 + simp only [ByteArray.size] at h + simp only [replaceAt, ByteArray.size, ByteArray.data_append, Array.getElem_append, + Array.getElem_extract, Array.size_append, Array.size_extract] + split_ifs <;> first | omega | (congr 1; omega) + +/-- Replacing a byte range with its own bytes changes nothing. -/ +theorem replaceAt_self (b : ByteArray) (off w : Nat) (h : off + w ≤ b.size) : + b.replaceAt off ⟨(b.bytesAt off w).toArray⟩ = b := by + apply ByteArray.ext + apply Array.ext + · simp only [replaceAt, ByteArray.size, Array.size_append, Array.size_extract, + Vector.size_toArray] + simp only [ByteArray.size] at h + omega + · intro j h1 h2 + simp only [ByteArray.size] at h + simp only [replaceAt, ByteArray.size, Array.getElem_append, Array.getElem_extract, + Array.size_append, Array.size_extract, Vector.size_toArray] + split_ifs + · simp only [Nat.zero_add] + · simp only [bytesAt, Vector.getElem_toArray, Vector.getElem_ofFn, ByteArray.get!, + Array.getElem!_eq_getD, Array.getD_eq_getD_getElem?] + rw [Array.getElem?_eq_getElem (by omega), Option.getD_some] + congr 1 + omega + · congr 1 + omega + +/-- A 4 KiB zero block, the source of zero fills. -/ +def zeroBlock : ByteArray := ⟨Array.replicate 4096 0⟩ + +/-- Append zero blocks to `b` until it holds `target` bytes, at most `fuel` times. The source is +a separate object, so the destination stays exclusive and is written in place. -/ +def fillZeros (target : Nat) : Nat → ByteArray → ByteArray + | 0, b => b + | fuel + 1, b => + let n := b.size + if n < target then + fillZeros target fuel (zeroBlock.copySlice 0 b n (min 4096 (target - n)) false) + else b + +/-- `n` zero bytes, filled in place. -/ +def zeros (n : Nat) : ByteArray := + fillZeros n (n / 4096 + 1) (ByteArray.emptyWithCapacity n) + +theorem copySlice_replicate (k m len : Nat) (hl : len ≤ k) : + (⟨Array.replicate k 0⟩ : ByteArray).copySlice 0 ⟨Array.replicate m 0⟩ m len false = + ⟨Array.replicate (m + len) 0⟩ := by + rw [ByteArray.copySlice_eq_append] + apply ByteArray.ext + apply Array.ext + · simp only [ByteArray.data_append, ByteArray.data_extract, Array.size_append, + Array.size_extract, Array.size_replicate] + omega + · intro j h1 h2 + simp only [ByteArray.data_append, ByteArray.data_extract, Array.getElem_append, + Array.getElem_extract, Array.getElem_replicate] + split_ifs <;> rfl + +theorem fillZeros_replicate (target : Nat) : + ∀ (fuel m : Nat), m ≤ target → target ≤ m + 4096 * fuel → + fillZeros target fuel ⟨Array.replicate m 0⟩ = ⟨Array.replicate target 0⟩ := by + intro fuel + induction fuel with + | zero => + intro m hle hge + rw [show m = target by omega] + rfl + | succ fuel ih => + intro m hle hge + have hm : (⟨Array.replicate m 0⟩ : ByteArray).size = m := Array.size_replicate + unfold fillZeros + by_cases hlt : m < target + · simp only [hm, hlt, ↓reduceIte] + rw [show zeroBlock = ⟨Array.replicate 4096 0⟩ from rfl, + copySlice_replicate 4096 m _ (by omega)] + exact ih _ (by omega) (by omega) + · simp only [hm, hlt, ↓reduceIte] + rw [show m = target by omega] + +theorem zeros_eq (n : Nat) : zeros n = ⟨Array.replicate n 0⟩ := by + unfold zeros + rw [show ByteArray.emptyWithCapacity n = ⟨Array.replicate 0 0⟩ from rfl] + exact fillZeros_replicate _ _ _ (by omega) (by omega) + +@[simp] theorem size_zeros (n : Nat) : (zeros n).size = n := by + rw [zeros_eq] + exact Array.size_replicate + +end ByteArray + +namespace CompPoly + +/-- A fixed-width byte representation of `R`, with fast accesses at byte offsets. -/ +class PackedRepr (R : Type u) where + /-- Bytes per element. -/ + width : Nat + width_pos : 0 < width + /-- The stored bytes of an element. -/ + toBytes : R → Vector UInt8 width + /-- The element of stored bytes; every byte string decodes. -/ + ofBytes : Vector UInt8 width → R + ofBytes_toBytes (x : R) : ofBytes (toBytes x) = x + /-- Read the element at byte offset `i`. -/ + read : (b : ByteArray) → (i : USize) → i.toNat + width ≤ b.size → b.size < USize.size → R + /-- Overwrite the element at byte offset `i`. -/ + write : (b : ByteArray) → (i : USize) → R → i.toNat + width ≤ b.size → b.size < USize.size → + ByteArray + read_eq (b : ByteArray) (i : USize) (h : i.toNat + width ≤ b.size) (hu : b.size < USize.size) : + read b i h hu = ofBytes (b.bytesAt i.toNat width) + write_eq (b : ByteArray) (i : USize) (x : R) (h : i.toNat + width ≤ b.size) + (hu : b.size < USize.size) : write b i x h hu = b.replaceAt i.toNat ⟨(toBytes x).toArray⟩ + +attribute [simp] PackedRepr.ofBytes_toBytes + +namespace PackedRepr + +variable {R : Type u} [PackedRepr R] + +/-- Pack elements into consecutive `width`-byte slots. -/ +def encode (a : Array R) : ByteArray := + ⟨(a.toList.flatMap fun x ↦ (toBytes x).toList).toArray⟩ + +/-- The elements of the complete slots of a byte array. -/ +def decode (b : ByteArray) : Array R := + Array.ofFn (n := b.size / width R) fun i ↦ ofBytes (b.bytesAt (width R * i.val) (width R)) + +private theorem length_flatMap (l : List R) : + (l.flatMap fun x ↦ (toBytes x).toList).length = width R * l.length := by + induction l with + | nil => simp only [List.flatMap_nil, List.length_nil, Nat.mul_zero] + | cons x xs ih => + simp only [List.flatMap_cons, List.length_append, Vector.length_toList, ih, + List.length_cons] + rw [Nat.mul_succ] + omega + +@[simp] theorem size_encode (a : Array R) : (encode a).size = width R * a.size := by + simp only [encode, ByteArray.size, List.size_toArray, length_flatMap, Array.length_toList] + +@[simp] theorem encode_empty : encode (#[] : Array R) = ByteArray.empty := rfl + +/-- Packing preserves concatenation. -/ +theorem encode_append (a b : Array R) : encode (a ++ b) = encode a ++ encode b := by + apply ByteArray.ext + simp only [encode, ByteArray.data_append, Array.toList_append, List.flatMap_append, + List.append_toArray] + +private theorem drop_flatMap (l : List R) (i : Nat) : + (l.flatMap fun x ↦ (toBytes x).toList).drop (width R * i) = + (l.drop i).flatMap fun x ↦ (toBytes x).toList := by + induction i generalizing l with + | zero => simp only [Nat.mul_zero, List.drop_zero] + | succ i ih => + cases l with + | nil => simp only [List.flatMap_nil, List.drop_nil] + | cons x xs => + rw [List.flatMap_cons, List.drop_append, + List.drop_of_length_le (show (toBytes x).toList.length ≤ width R * (i + 1) by + rw [Vector.length_toList, Nat.mul_succ]; omega)] + simp only [List.nil_append, Vector.length_toList] + rw [show width R * (i + 1) - width R = width R * i by rw [Nat.mul_succ]; omega, ih, + List.drop_succ_cons] + +private theorem take_flatMap (l : List R) (i : Nat) : + (l.flatMap fun x ↦ (toBytes x).toList).take (width R * i) = + (l.take i).flatMap fun x ↦ (toBytes x).toList := by + induction i generalizing l with + | zero => simp only [Nat.mul_zero, List.take_zero, List.flatMap_nil] + | succ i ih => + cases l with + | nil => simp only [List.flatMap_nil, List.take_nil] + | cons x xs => + rw [List.flatMap_cons, + show width R * (i + 1) = (toBytes x).toList.length + width R * i by + rw [Vector.length_toList, Nat.mul_succ]; omega, + List.take_length_add_append, ih, List.take_succ_cons, List.flatMap_cons] + +/-- Byte extraction on slot boundaries is element extraction. -/ +theorem encode_extract (a : Array R) (first last : Nat) : + (encode a).data.extract (width R * first) (width R * last) = + (encode (a.extract first last)).data := by + apply Array.toList_inj.mp + simp only [encode, Array.toList_extract, List.toList_toArray, List.extract, + ← Nat.mul_sub_left_distrib, drop_flatMap, take_flatMap] + +/-- Appending a slot range of a packed array copies its elements. -/ +theorem copySlice_encode (a o : Array R) (i c : Nat) : + (encode a).copySlice (width R * i) (encode o) (encode o).size (width R * c) = + encode (o ++ a.extract i (i + c)) := by + have hm : (encode a).extract (width R * i) (width R * i + width R * c) = + encode (a.extract i (i + c)) := by + apply ByteArray.ext + rw [ByteArray.data_extract, ← Nat.mul_add, encode_extract] + rw [ByteArray.copySlice_eq_append, ByteArray.extract_zero_size, hm, + (ByteArray.extract_eq_empty_iff).mpr (by simp only [ByteArray.size]; omega), + ByteArray.append_empty, encode_append] + +/-- Replacing packed slots is the corresponding element splice. -/ +theorem replaceAt_encode (a v : Array R) (i : Nat) : + (encode a).replaceAt (width R * i) (encode v) = + encode (a.extract 0 i ++ v ++ a.extract (i + v.size) a.size) := by + simp only [ByteArray.replaceAt, size_encode, encode_append] + apply ByteArray.ext + simp only [ByteArray.data_append] + have hp := encode_extract a 0 i + simp only [Nat.mul_zero] at hp + rw [hp, show width R * i + width R * v.size = width R * (i + v.size) by + rw [Nat.mul_add], encode_extract] + +/-- The bytes of a slot are its element's bytes. -/ +theorem bytesAt_encode (a : Array R) (i : Nat) (hi : i < a.size) : + (encode a).bytesAt (width R * i) (width R) = toBytes a[i] := by + apply Vector.ext + intro j hj + have hd := congrArg (·[j]?) (drop_flatMap a.toList i) + simp only [List.getElem?_drop] at hd + rw [List.drop_eq_getElem_cons (by simpa only [Array.length_toList] using hi), + List.flatMap_cons, List.getElem?_append_left (by simpa only [Vector.length_toList])] at hd + simp only [ByteArray.bytesAt, Vector.getElem_ofFn, ByteArray.get!, encode, + Array.getElem!_eq_getD, Array.getD_eq_getD_getElem?, List.getElem?_toArray, hd, + Vector.getElem?_toList, Array.getElem_toList, Vector.getElem?_eq_getElem hj, + Option.getD_some] + +/-- Reading slot `i` of packed elements returns element `i`. -/ +theorem read_encode (a : Array R) (off : USize) (i : Nat) (ho : off.toNat = width R * i) + (hi : i < a.size) (h hu) : read (encode a) off h hu = a[i] := by + rw [read_eq, ho, bytesAt_encode a i hi, ofBytes_toBytes] + +omit [PackedRepr R] in +theorem set_eq_splice (a : Array R) (i : Nat) (x : R) (hi : i < a.size) : + a.set i x hi = a.extract 0 i ++ #[x] ++ a.extract (i + 1) a.size := by + apply Array.ext + · simp only [Array.size_set, Array.size_append, Array.size_extract, List.size_toArray, + List.length_cons, List.length_nil] + omega + · intro j h1 h2 + simp only [Array.size_set] at h1 + rw [Array.getElem_set] + simp only [Array.getElem_append, Array.getElem_extract, Array.size_extract, + List.size_toArray, List.length_cons, List.length_nil, Array.size_append] + split_ifs <;> first | omega | (congr 1; omega) | + (simp only [show j - (min i a.size - 0) = 0 by omega]; rfl) + +/-- Writing slot `i` of packed elements sets element `i`. -/ +theorem write_encode (a : Array R) (off : USize) (i : Nat) (x : R) + (ho : off.toNat = width R * i) (hi : i < a.size) (h hu) : + write (encode a) off x h hu = encode (a.set i x hi) := by + rw [write_eq, ho, show (⟨(toBytes x).toArray⟩ : ByteArray) = encode #[x] by + apply ByteArray.ext + simp only [encode, List.flatMap_cons, List.flatMap_nil, List.append_nil, Vector.toList, + Array.toArray_toList], + replaceAt_encode, set_eq_splice a i x hi] + rfl + +/-- Decoding inverts packing. -/ +@[simp] theorem decode_encode (a : Array R) : decode (encode a) = a := by + have hw := width_pos (R := R) + apply Array.ext + · simp only [decode, Array.size_ofFn, size_encode, Nat.mul_div_cancel_left _ hw] + · intro i h1 h2 + simp only [decode, Array.getElem_ofFn] + rw [bytesAt_encode a i h2, ofBytes_toBytes] + +theorem size_decode (b : ByteArray) : (decode (R := R) b).size = b.size / width R := by + simp only [decode, Array.size_ofFn] + +theorem getElem_decode (b : ByteArray) (i : Nat) (hi : i < (decode (R := R) b).size) : + (decode (R := R) b)[i] = ofBytes (b.bytesAt (width R * i) (width R)) := by + simp only [decode, Array.getElem_ofFn] + +/-! ### Elements by index -/ + +/-- Element `j` of a byte array: the decoding of slot `j`, zero-padded past the end. -/ +def get (b : ByteArray) (j : Nat) : R := ofBytes (b.bytesAt (width R * j) (width R)) + +theorem get_encode (a : Array R) (j : Nat) (hj : j < a.size) : get (encode a) j = a[j] := by + rw [get, bytesAt_encode a j hj, ofBytes_toBytes] + +theorem getElem_decode_eq_get (b : ByteArray) (j : Nat) (hj : j < (decode (R := R) b).size) : + (decode (R := R) b)[j] = get b j := + getElem_decode b j hj + +/-- Slot `k` of a buffer of `n` bytes with at least `k + 1` slots, at a machine offset. -/ +theorem slot_toNat (k : USize) (n : Nat) (hk : k.toNat < n / width R) (hu : n < USize.size) : + (k * (width R).toUSize).toNat = width R * k.toNat ∧ width R * k.toNat + width R ≤ n := by + have hw := width_pos (R := R) + have hle := (Nat.le_div_iff_mul_le hw).mp (Nat.succ_le_of_lt hk) + rw [Nat.succ_mul, Nat.mul_comm] at hle + have hwu : width R < USize.size := by omega + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + refine ⟨?_, hle⟩ + rw [USize.toNat_mul, USize.toNat_ofNat_of_lt' hwu, hsize, Nat.mod_eq_of_lt (by + rw [Nat.mul_comm]; omega), Nat.mul_comm] + +/-- Read element `k` of a buffer with at least `k + 1` slots. -/ +@[inline] def getAt (b : ByteArray) (k : USize) (hk : k.toNat < b.size / width R) + (hu : b.size < USize.size) : R := + read b (k * (width R).toUSize) (by have := slot_toNat k b.size hk hu; rw [this.1]; omega) hu + +theorem getAt_eq (b : ByteArray) (k : USize) (hk) (hu) : + getAt (R := R) b k hk hu = get b k.toNat := by + rw [getAt, read_eq, (slot_toNat k b.size hk hu).1, get] + +/-- Overwrite element `k` of a buffer with at least `k + 1` slots. -/ +@[inline] def setAt (b : ByteArray) (k : USize) (x : R) (hk : k.toNat < b.size / width R) + (hu : b.size < USize.size) : ByteArray := + write b (k * (width R).toUSize) x (by have := slot_toNat k b.size hk hu; rw [this.1]; omega) + hu + +@[simp] theorem size_setAt (b : ByteArray) (k : USize) (x : R) (hk) (hu) : + (setAt b k x hk hu).size = b.size := by + have := slot_toNat k b.size hk hu + have hsz : (⟨(toBytes x).toArray⟩ : ByteArray).size = width R := Vector.size_toArray _ + rw [setAt, write_eq, ByteArray.size_replaceAt _ _ _ (by rw [this.1, hsz]; exact this.2)] + +theorem get!_replaceAt (b v : ByteArray) (off i : Nat) (h : off + v.size ≤ b.size) : + (b.replaceAt off v).get! i = if off ≤ i ∧ i < off + v.size then v.get! (i - off) + else b.get! i := by + simp only [ByteArray.get!, Array.getElem!_eq_getD, Array.getD_eq_getD_getElem?, ByteArray.size] + simp only [ByteArray.size] at h + by_cases hi : i < b.data.size + · have hr : i < (b.replaceAt off v).data.size := by + simp only [ByteArray.replaceAt, Array.size_append, Array.size_extract, ByteArray.size] + omega + rw [Array.getElem?_eq_getElem hr, Option.getD_some] + simp only [ByteArray.replaceAt, Array.getElem_append, Array.getElem_extract, + Array.size_append, Array.size_extract, ByteArray.size] + split_ifs <;> first + | omega + | (rw [Array.getElem?_eq_getElem (by omega), Option.getD_some]; congr 1; omega) + · have hr : ¬ i < (b.replaceAt off v).data.size := by + simp only [ByteArray.replaceAt, Array.size_append, Array.size_extract, ByteArray.size] + omega + rw [Array.getElem?_eq_none (show (b.replaceAt off v).data.size ≤ i by omega)] + split_ifs + · omega + · rw [Array.getElem?_eq_none (show b.data.size ≤ i by omega)] + +theorem get_setAt (b : ByteArray) (k : USize) (x : R) (hk) (hu) (j : Nat) : + get (setAt b k x hk hu) j = if j = k.toNat then x else get b j := by + have hs := slot_toNat k b.size hk hu + have hw := width_pos (R := R) + have hs2 : width R * k.toNat + width R ≤ b.data.size := hs.2 + rw [setAt, write_eq, hs.1] + unfold get + split + · rename_i hj + subst hj + conv_rhs => rw [← ofBytes_toBytes (R := R) x] + congr 1 + apply Vector.ext + intro t ht + simp only [ByteArray.bytesAt, Vector.getElem_ofFn] + rw [get!_replaceAt _ _ _ _ (by simp only [ByteArray.size, Vector.size_toArray]; omega)] + simp only [ByteArray.size, Vector.size_toArray] + rw [ite_eq_left_of_eq_true _ _ (eq_true (by omega)), + show width R * k.toNat + t - width R * k.toNat = t by omega] + simp only [ByteArray.get!, Array.getElem!_eq_getD, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (show t < (toBytes x).toArray.size by + simp only [Vector.size_toArray]; exact ht), Option.getD_some, Vector.getElem_toArray] + · rename_i hj + congr 1 + apply Vector.ext + intro t ht + simp only [ByteArray.bytesAt, Vector.getElem_ofFn] + rw [get!_replaceAt _ _ _ _ (by simp only [ByteArray.size, Vector.size_toArray]; omega)] + simp only [ByteArray.size, Vector.size_toArray] + refine ite_eq_right (fun hc ↦ ?_) + obtain ⟨h1, h2⟩ := hc + rcases Nat.lt_or_gt_of_ne hj with hlt | hgt + · have : width R * (j + 1) ≤ width R * k.toNat := Nat.mul_le_mul_left _ hlt + rw [Nat.mul_succ] at this + omega + · have : width R * (k.toNat + 1) ≤ width R * j := Nat.mul_le_mul_left _ hgt + rw [Nat.mul_succ] at this + omega + +/-- Overwrite element `k` of a buffer of `n` bytes when it is in range; otherwise leave the +buffer unchanged. The machine-size bound is proved once for `n`, not checked per store. -/ +@[inline] def setIf (n : Nat) (hn : n < USize.size) (b : ByteArray) (k : USize) (x : R) : + ByteArray := + if h : b.size = n ∧ k.toNat < b.size / width R then setAt b k x h.2 (h.1 ▸ hn) else b + +theorem setAt_encode (a : Array R) (k : USize) (x : R) (hk) (hu) : + setAt (encode a) k x hk hu = + encode (a.set k.toNat x (by rwa [size_encode, Nat.mul_div_cancel_left _ width_pos] at hk)) := + write_encode a _ k.toNat x (slot_toNat k _ hk hu).1 _ _ _ + +/-- A guarded store into packed elements is the guarded array store. -/ +theorem setIf_encode (a : Array R) (k : USize) (x : R) (hn : width R * a.size < USize.size) : + setIf (width R * a.size) hn (encode a) k x = encode (a.setIfInBounds k.toNat x) := by + unfold setIf + have hdiv : (encode a).size / width R = a.size := by + rw [size_encode, Nat.mul_div_cancel_left _ width_pos] + split + · rename_i h + rw [setAt_encode] + congr 1 + simp only [Array.setIfInBounds, show k.toNat < a.size by rw [← hdiv]; exact h.2, + ↓reduceDIte] + · rename_i h + have hk : ¬ k.toNat < a.size := fun hk ↦ h ⟨size_encode a, by rw [hdiv]; exact hk⟩ + simp only [Array.setIfInBounds, hk, ↓reduceDIte] + +theorem setIf_encode_of_eq (a : Array R) (k : USize) (x : R) (m : Nat) (hm : m < USize.size) + (h : width R * a.size = m) : + setIf m hm (encode a) k x = encode (a.setIfInBounds k.toNat x) := by + subst h + exact setIf_encode a k x hm + +/-- A zero-filled buffer of `n` slots. -/ +def alloc (R : Type u) [PackedRepr R] (n : Nat) : ByteArray := ByteArray.zeros (width R * n) + +@[simp] theorem size_alloc (n : Nat) : (alloc R n).size = width R * n := ByteArray.size_zeros _ + +@[simp] theorem size_alloc_div (n : Nat) : (alloc R n).size / width R = n := by + rw [size_alloc, Nat.mul_div_cancel_left _ width_pos] + +/-- `n` copies of `block` appended to `acc`, by binary doubling. -/ +def replicateGo (n : Nat) (block acc : ByteArray) : ByteArray := + if n = 0 then acc + else replicateGo (n / 2) (block ++ block) (if n % 2 = 1 then acc ++ block else acc) +termination_by n +decreasing_by omega + +/-- `n` packed copies of `x`, filled by doubling copies of its bytes. -/ +def replicate (n : Nat) (x : R) : ByteArray := + replicateGo n ⟨(toBytes x).toArray⟩ (ByteArray.emptyWithCapacity (width R * n)) + +theorem replicateGo_encode (x : R) : ∀ (n m a : Nat), + replicateGo n (encode (Array.replicate m x)) (encode (Array.replicate a x)) = + encode (Array.replicate (a + n * m) x) := by + intro n + induction n using Nat.strongRecOn with + | _ n ih => + intro m a + unfold replicateGo + split + · subst n; simp only [Nat.zero_mul, Nat.add_zero] + · rename_i hn + have hb : encode (Array.replicate m x) ++ encode (Array.replicate m x) = + encode (Array.replicate (m + m) x) := by + rw [← encode_append, Array.replicate_append_replicate] + have hacc : encode (Array.replicate a x) ++ encode (Array.replicate m x) = + encode (Array.replicate (a + m) x) := by + rw [← encode_append, Array.replicate_append_replicate] + rw [hb] + have hk : n = 2 * (n / 2) + n % 2 := (Nat.div_add_mod n 2).symm + generalize n / 2 = k at hk ih ⊢ + split + · rename_i ho + rw [hacc, ih k (by omega)] + congr 2 + rw [hk, ho] + simp only [Nat.add_mul, Nat.mul_add, Nat.mul_assoc, Nat.one_mul] + omega + · rename_i ho + rw [ih k (by omega)] + congr 2 + rw [hk, show n % 2 = 0 by omega] + simp only [Nat.add_mul, Nat.mul_add, Nat.mul_assoc, Nat.zero_mul] + omega + +theorem replicate_eq (n : Nat) (x : R) : replicate n x = encode (Array.replicate n x) := by + have h1 : (⟨(toBytes x).toArray⟩ : ByteArray) = encode (Array.replicate 1 x) := by + apply ByteArray.ext + simp only [encode, Array.replicate_one, List.flatMap_cons, List.flatMap_nil, + List.append_nil, Vector.toList, Array.toArray_toList] + have h0 : ByteArray.emptyWithCapacity (width R * n) = encode (Array.replicate 0 x) := rfl + rw [replicate, h1, h0, replicateGo_encode, Nat.zero_add, Nat.mul_one] + +/-! ### Four-byte words -/ + +/-- The little-endian bytes of a word. -/ +def wordBytes (v : UInt32) : Vector UInt8 4 := Bytes.toVecLE 4 v.toNat + +/-- The word of four little-endian bytes. -/ +def ofWordBytes (v : Vector UInt8 4) : UInt32 := UInt32.ofNat (Bytes.ofVecLE v) + +theorem ofWordBytes_wordBytes (v : UInt32) : ofWordBytes (wordBytes v) = v := by + apply UInt32.toNat_inj.mp + simp only [ofWordBytes, wordBytes, Bytes.ofVecLE_toVecLE_of_lt (show v.toNat < 256 ^ 4 from + v.toNat_lt), UInt32.toNat_ofNat', Nat.mod_eq_of_lt v.toNat_lt] + +theorem wordBytes_ofWordBytes (v : Vector UInt8 4) : wordBytes (ofWordBytes v) = v := by + have hl := Bytes.ofVecLE_lt v + simp only [wordBytes, ofWordBytes, UInt32.toNat_ofNat', + Nat.mod_eq_of_lt (show Bytes.ofVecLE v < 2 ^ 32 by simpa using hl), Bytes.toVecLE_ofVecLE] + +theorem toUInt8_shiftRight (v : UInt32) (s : UInt32) (k : Nat) (hs : s.toNat = 8 * k) + (hk : k < 4) : (v >>> s).toUInt8 = UInt8.ofNat (v.toNat / 2 ^ (8 * k) % 256) := by + apply UInt8.toNat_inj.mp + rw [UInt8.toNat_ofNat', UInt32.toNat_toUInt8, UInt32.toNat_shiftRight, Nat.shiftRight_eq_div_pow, + hs, Nat.mod_eq_of_lt (show 8 * k < 32 by omega), Nat.mod_mod] + +/-- A little-endian word store replaces four bytes with the word's bytes. -/ +theorem setUInt32LE!_eq_replaceAt (b : ByteArray) (off : Nat) (v : UInt32) + (h : off + 4 ≤ b.size) : + b.setUInt32LE! off v = b.replaceAt off ⟨(wordBytes v).toArray⟩ := by + have hp : (wordBytes v).toArray = #[UInt8.ofNat (v.toNat % 256), + UInt8.ofNat (v.toNat / 256 % 256), UInt8.ofNat (v.toNat / 256 / 256 % 256), + UInt8.ofNat (v.toNat / 256 / 256 / 256 % 256)] := by + apply Array.toList_inj.mp + simp only [wordBytes, Vector.toList_toArray, Bytes.toList_toVecLE, Bytes.toListLE_succ, + Bytes.toListLE_zero] + have hv0 : v.toUInt8 = UInt8.ofNat (v.toNat % 256) := by + have := toUInt8_shiftRight v 0 0 rfl (by decide) + rwa [UInt32.shiftRight_zero, Nat.mul_zero, Nat.pow_zero, Nat.div_one] at this + have hv1 : (v >>> 8).toUInt8 = UInt8.ofNat (v.toNat / 256 % 256) := + toUInt8_shiftRight v 8 1 rfl (by decide) + have hv2 : (v >>> 16).toUInt8 = UInt8.ofNat (v.toNat / 256 / 256 % 256) := by + rw [Nat.div_div_eq_div_mul]; exact toUInt8_shiftRight v 16 2 rfl (by decide) + have hv3 : (v >>> 24).toUInt8 = UInt8.ofNat (v.toNat / 256 / 256 / 256 % 256) := by + rw [Nat.div_div_eq_div_mul, Nat.div_div_eq_div_mul] + exact toUInt8_shiftRight v 24 3 rfl (by decide) + rw [hp] + unfold ByteArray.setUInt32LE! + rw [ite_eq_left h] + apply ByteArray.ext + apply Array.ext + · simp only [ByteArray.replaceAt, ByteArray.size, ByteArray.set!, + Array.set!_eq_setIfInBounds, Array.size_setIfInBounds, Array.size_append, + Array.size_extract, List.size_toArray, List.length_cons, List.length_nil] + simp only [ByteArray.size] at h + omega + · intro j h1' h2' + simp only [ByteArray.size] at h + have hj : j < b.data.size := by + simpa only [ByteArray.set!, Array.set!_eq_setIfInBounds, Array.size_setIfInBounds] using h1' + simp only [ByteArray.set!, Array.set!_eq_setIfInBounds] + rw [Array.getElem_setIfInBounds (by simp only [Array.size_setIfInBounds]; exact hj), + Array.getElem_setIfInBounds (by simp only [Array.size_setIfInBounds]; exact hj), + Array.getElem_setIfInBounds (by simp only [Array.size_setIfInBounds]; exact hj), + Array.getElem_setIfInBounds hj] + simp only [ByteArray.replaceAt, ByteArray.size, Array.getElem_append, Array.getElem_extract, + Array.size_append, Array.size_extract, List.size_toArray, List.length_cons, + List.length_nil, hv0, hv1, hv2, hv3] + split_ifs <;> first + | omega | (congr 1; omega) + | (simp only [show j - (min off b.data.size - 0) = 0 by omega]; rfl) + | (simp only [show j - (min off b.data.size - 0) = 1 by omega]; rfl) + | (simp only [show j - (min off b.data.size - 0) = 2 by omega]; rfl) + | (simp only [show j - (min off b.data.size - 0) = 3 by omega]; rfl) + +/-- A little-endian word load reads the word of four bytes. -/ +theorem getUInt32LE!_eq_ofWordBytes (b : ByteArray) (off : Nat) (h : off + 4 ≤ b.size) : + b.getUInt32LE! off = ofWordBytes (b.bytesAt off 4) := by + have hs := setUInt32LE!_eq_replaceAt b off (ofWordBytes (b.bytesAt off 4)) h + rw [wordBytes_ofWordBytes, ByteArray.replaceAt_self b off 4 h] at hs + have := ByteArray.getUInt32LE!_setUInt32LE!_self b off (ofWordBytes (b.bytesAt off 4)) h + rwa [hs] at this + +/-- Writing back the word just read changes nothing. -/ +theorem setUInt32LE!_getUInt32LE! (b : ByteArray) (off : Nat) (h : off + 4 ≤ b.size) : + b.setUInt32LE! off (b.getUInt32LE! off) = b := by + rw [getUInt32LE!_eq_ofWordBytes b off h, setUInt32LE!_eq_replaceAt b off _ h, + wordBytes_ofWordBytes, ByteArray.replaceAt_self b off 4 h] + +/-- A representation by one 32-bit word: `toWord` stores, `ofWord` decodes every word. The +accesses are single word loads and stores. -/ +@[inline, reducible] def ofWord32 {R : Type u} (toWord : R → UInt32) (ofWord : UInt32 → R) + (ofWord_toWord : ∀ x, ofWord (toWord x) = x) : PackedRepr R where + width := 4 + width_pos := by decide + toBytes x := wordBytes (toWord x) + ofBytes v := ofWord (ofWordBytes v) + ofBytes_toBytes x := by simp only [ofWordBytes_wordBytes, ofWord_toWord] + read b i h _ := ofWord (b.ugetUInt32LE i h) + write b i x h _ := b.usetUInt32LE i (toWord x) h + read_eq b i h _ := by + simp only [ByteArray.ugetUInt32LE, getUInt32LE!_eq_ofWordBytes b i.toNat h] + write_eq b i x h _ := by + simp only [ByteArray.usetUInt32LE, setUInt32LE!_eq_replaceAt b i.toNat _ h] + +@[inline] instance : PackedRepr UInt32 := ofWord32 id id fun _ ↦ rfl + +@[simp] theorem width_uint32 : width UInt32 = 4 := rfl + +@[simp] theorem size_encode_uint32 (W : Array UInt32) : (encode W).size = 4 * W.size := + size_encode W + +/-- A little-endian word store replaces four bytes with the packed word. -/ +theorem setUInt32LE!_eq_replaceAt_encode (b : ByteArray) (off : Nat) (v : UInt32) + (h : off + 4 ≤ b.size) : b.setUInt32LE! off v = b.replaceAt off (encode #[v]) := by + rw [setUInt32LE!_eq_replaceAt b off v h] + rfl + +/-- Replacing packed word slots is the corresponding word splice. -/ +theorem replaceAt_encode_uint32 (W V : Array UInt32) (i : Nat) : + (encode W).replaceAt (4 * i) (encode V) = + encode (W.extract 0 i ++ V ++ W.extract (i + V.size) W.size) := + replaceAt_encode W V i + +/-- Packing elements of a one-word representation is packing their words. -/ +theorem encode_ofWord32 {R : Type u} (toWord : R → UInt32) (ofWord : UInt32 → R) + (h : ∀ x, ofWord (toWord x) = x) (a : Array R) : + @encode R (ofWord32 toWord ofWord h) a = encode (a.map toWord) := by + apply ByteArray.ext + simp only [encode, Array.toList_map, List.flatMap_map] + rfl + +/-- A word load from packed words reads one word. -/ +theorem getUInt32LE!_encode (W : Array UInt32) (k : Nat) : + (encode W).getUInt32LE! (4 * k) = W.getD k 0 := by + by_cases hk : k < W.size + · rw [getUInt32LE!_eq_ofWordBytes _ _ (by rw [size_encode]; simp; omega)] + have := bytesAt_encode W k hk + simp only [width_uint32] at this + rw [this, Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hk, Option.getD_some] + exact ofWordBytes_wordBytes _ + · unfold ByteArray.getUInt32LE! + rw [ite_eq_right_iff.mpr (fun h ↦ absurd h (by rw [size_encode, width_uint32]; omega)), + Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by omega), Option.getD_none] + +/-- A word store into packed words sets one word. -/ +theorem setUInt32LE!_encode (W : Array UInt32) (k : Nat) (v : UInt32) : + (encode W).setUInt32LE! (4 * k) v = encode (W.set! k v) := by + by_cases hk : k < W.size + · rw [setUInt32LE!_eq_replaceAt _ _ _ (by rw [size_encode, width_uint32]; omega)] + have := replaceAt_encode W #[v] k + simp only [width_uint32] at this + rw [show (⟨(wordBytes v).toArray⟩ : ByteArray) = encode #[v] by + apply ByteArray.ext + simp only [encode, List.flatMap_cons, List.flatMap_nil, List.append_nil] + rfl, this] + simp only [Array.set!_eq_setIfInBounds, Array.setIfInBounds, hk, ↓reduceDIte, + set_eq_splice W k v hk] + rfl + · have hs : W.set! k v = W := by + simp only [Array.set!_eq_setIfInBounds, Array.setIfInBounds, show ¬k < W.size from hk, + ↓reduceDIte] + rw [hs] + unfold ByteArray.setUInt32LE! + rw [ite_eq_right_iff.mpr (fun h ↦ absurd h (by rw [size_encode, width_uint32]; omega))] + +end PackedRepr + +end CompPoly diff --git a/CompPoly/Data/Bytes/UInt32.lean b/CompPoly/Data/Bytes/UInt32.lean new file mode 100644 index 00000000..5b0706ea --- /dev/null +++ b/CompPoly/Data/Bytes/UInt32.lean @@ -0,0 +1,40 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Data.Bytes.CanonicalNat +public import CompPoly.Data.Bytes.Vector + +/-! # Four-byte little-endian word storage + +This codec stores the word itself. A Montgomery field coordinate is a word in this format; +it is distinct from the canonical field-element serialization. +-/ + +@[expose] public section +namespace CompPoly + +instance : CanonicalNat UInt32 where + bound := 2 ^ 32 + toNat := UInt32.toNat + ofNat := UInt32.ofNat + toNat_lt := UInt32.toNat_lt + ofNat_toNat x := UInt32.ofNat_toNat + toNat_ofNat_of_lt h := by + simp only [UInt32.toNat_ofNat', Nat.mod_eq_of_lt h] + ofNat_mod n := by + apply UInt32.toNat_inj.mp + simp only [UInt32.toNat_ofNat', Nat.mod_mod] + +instance : ByteCodec UInt32 := ByteCodec.ofCanonicalNat _ + +@[simp] theorem ByteCodec.width_uint32 : ByteCodec.width UInt32 = 4 := by decide + +/-- The word codec exposes precisely the four little-endian bytes of its bit pattern. -/ +theorem ByteCodec.toBytes_uint32 (x : UInt32) : + (ByteCodec.toBytes x).toList = Bytes.toListLE 4 x.toNat := rfl + +end CompPoly diff --git a/CompPoly/Data/List/BatchInv.lean b/CompPoly/Data/List/BatchInv.lean new file mode 100644 index 00000000..6c128e86 --- /dev/null +++ b/CompPoly/Data/List/BatchInv.lean @@ -0,0 +1,170 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import Mathlib.Algebra.BigOperators.Group.List.Basic +public import Mathlib.Algebra.BigOperators.Ring.List +public import Mathlib.Algebra.GroupWithZero.Basic + +/-! +# Batch inversion + +Montgomery's trick inverts `n` elements with one inversion and `3 (n - 1)` multiplications: invert +the product of all elements, then walk back through the prefix products, peeling one factor at a +time. + +The algorithm only needs `*`, `⁻¹` and `1`, so it runs on carriers without a `Field` instance; +`batchInv_map` transfers it along any map preserving those operations, and `batchInv_eq` proves it +in a commutative group with zero, for inputs without zeros. +-/ + +@[expose] public section + +namespace CompPoly.List + +variable {K L : Type*} + +section Ops + +variable [Mul K] + +/-- The products `q, q x₀, q x₀ x₁, …` of all proper prefixes, each times `q`. -/ +def prefixProducts (q : K) : List K → List K + | [] => [] + | x :: l => q :: prefixProducts (q * x) l + +/-- `q x₀ x₁ ⋯`. -/ +def productFrom (q : K) : List K → K + | [] => q + | x :: l => productFrom (q * x) l + +/-- Walk the reversed elements and prefix products, peeling one factor off the running inverse at +each step and prepending that element's inverse to `out`. -/ +def batchInvGo : List K → List K → K → List K → List K + | x :: xs, p :: ps, acc, out => batchInvGo xs ps (acc * x) ((acc * p) :: out) + | _, _, _, out => out + +/-- The inverses of all elements, with one inversion. -/ +def batchInv [Inv K] [One K] (xs : List K) : List K := + batchInvGo xs.reverse (prefixProducts 1 xs).reverse (productFrom 1 xs)⁻¹ [] + +theorem length_prefixProducts (q : K) (l : List K) : + (prefixProducts q l).length = l.length := by + induction l generalizing q with + | nil => rfl + | cons x l ih => simp only [prefixProducts, List.length_cons, ih] + +theorem length_batchInvGo (xs ps : List K) (hl : xs.length = ps.length) (acc : K) + (out : List K) : (batchInvGo xs ps acc out).length = xs.length + out.length := by + induction xs generalizing ps acc out with + | nil => cases ps <;> simp [batchInvGo] + | cons x xs ih => + cases ps with + | nil => simp at hl + | cons p ps => + rw [batchInvGo, ih ps (by simpa using hl)] + simp only [List.length_cons]; omega + +@[simp] theorem length_batchInv [Inv K] [One K] (xs : List K) : + (batchInv xs).length = xs.length := by + rw [batchInv, length_batchInvGo _ _ (by simp [length_prefixProducts]), List.length_reverse, + List.length_nil, Nat.add_zero] + +end Ops + +/-! ### Transfer along structure-preserving maps -/ + +section Map + +variable [Mul K] [Mul L] (f : K → L) + +theorem prefixProducts_map (hmul : ∀ a b, f (a * b) = f a * f b) (q : K) (l : List K) : + prefixProducts (f q) (l.map f) = (prefixProducts q l).map f := by + induction l generalizing q with + | nil => rfl + | cons x l ih => simp only [List.map_cons, prefixProducts, ← hmul, ih] + +theorem productFrom_map (hmul : ∀ a b, f (a * b) = f a * f b) (q : K) (l : List K) : + productFrom (f q) (l.map f) = f (productFrom q l) := by + induction l generalizing q with + | nil => rfl + | cons x l ih => simp only [List.map_cons, productFrom, ← hmul, ih] + +theorem batchInvGo_map (hmul : ∀ a b, f (a * b) = f a * f b) (xs ps : List K) (acc : K) + (out : List K) : + batchInvGo (xs.map f) (ps.map f) (f acc) (out.map f) = (batchInvGo xs ps acc out).map f := by + induction xs generalizing ps acc out with + | nil => cases ps <;> rfl + | cons x xs ih => + cases ps with + | nil => rfl + | cons p ps => + simp only [List.map_cons, batchInvGo, ← hmul] + rw [← ih, List.map_cons, hmul] + +/-- Batch inversion commutes with maps preserving `*`, `⁻¹` and `1`. -/ +theorem batchInv_map [Inv K] [One K] [Inv L] [One L] (hmul : ∀ a b, f (a * b) = f a * f b) + (hinv : ∀ a, f a⁻¹ = (f a)⁻¹) + (hone : f 1 = 1) (xs : List K) : + batchInv (xs.map f) = (batchInv xs).map f := by + rw [batchInv, batchInv, ← List.map_reverse, ← hone, prefixProducts_map f hmul, + productFrom_map f hmul, ← hinv, ← List.map_reverse, ← batchInvGo_map f hmul, + List.map_nil] + +end Map + +/-! ### Correctness -/ + +section Field + +variable [CommGroupWithZero K] + +theorem productFrom_eq (q : K) (l : List K) : productFrom q l = q * l.prod := by + induction l generalizing q with + | nil => simp only [productFrom, List.prod_nil, mul_one] + | cons x l ih => rw [productFrom, ih, List.prod_cons, mul_assoc] + +theorem prefixProducts_append_singleton (q : K) (l : List K) (x : K) : + prefixProducts q (l ++ [x]) = prefixProducts q l ++ [q * l.prod] := by + induction l generalizing q with + | nil => simp only [List.nil_append, prefixProducts, List.prod_nil, mul_one] + | cons y l ih => rw [List.cons_append, prefixProducts, prefixProducts, ih, List.prod_cons, + mul_assoc, List.cons_append] + +theorem batchInvGo_spec_reverse (r : List K) (hl : ∀ x ∈ r, x ≠ 0) (q : K) (hq : q ≠ 0) + (out : List K) : + batchInvGo r.reverse.reverse (prefixProducts q r.reverse).reverse (q * r.reverse.prod)⁻¹ out = + r.reverse.map (·⁻¹) ++ out := by + induction r generalizing out with + | nil => simp only [List.reverse_nil, prefixProducts, List.map_nil, List.nil_append]; rfl + | cons x r ih => + have hx : x ≠ 0 := hl x (by simp) + have hl' : ∀ y ∈ r, y ≠ 0 := fun y hy ↦ hl y (by simp [hy]) + have hp : q * r.reverse.prod ≠ 0 := + mul_ne_zero hq (List.prod_ne_zero (fun h ↦ hl' 0 (List.mem_reverse.mp h) rfl)) + rw [List.reverse_cons, prefixProducts_append_singleton, List.reverse_append, + List.reverse_append] + simp only [List.reverse_cons, List.reverse_nil, List.nil_append, List.singleton_append, + batchInvGo, List.prod_append, List.prod_singleton] + have e1 : (q * (r.reverse.prod * x))⁻¹ * (q * r.reverse.prod) = x⁻¹ := by + rw [← mul_assoc q, mul_inv_rev, mul_assoc, inv_mul_cancel₀ hp, mul_one] + have e2 : (q * (r.reverse.prod * x))⁻¹ * x = (q * r.reverse.prod)⁻¹ := by + rw [← mul_assoc q, mul_inv_rev, mul_comm x⁻¹, mul_assoc, inv_mul_cancel₀ hx, mul_one] + rw [e1, e2, + ih hl', List.map_append, List.map_singleton, List.append_assoc, List.singleton_append] + +/-- Batch inversion inverts every element of a list without zeros. -/ +theorem batchInv_eq (xs : List K) (hxs : ∀ x ∈ xs, x ≠ 0) : batchInv xs = xs.map (·⁻¹) := by + have h := batchInvGo_spec_reverse xs.reverse (fun x hx ↦ hxs x (List.mem_reverse.mp hx)) 1 + one_ne_zero [] + simp only [List.reverse_reverse] at h + rw [one_mul, List.append_nil] at h + rw [batchInv, productFrom_eq, one_mul] + exact h + +end Field + +end CompPoly.List diff --git a/CompPoly/Data/Tasks.lean b/CompPoly/Data/Tasks.lean new file mode 100644 index 00000000..5f88315f --- /dev/null +++ b/CompPoly/Data/Tasks.lean @@ -0,0 +1,78 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +/-! # Task graphs + +`spawnAll` runs independent tasks, `spawnAfter` runs tasks once a set of tasks is done, and +`join` collects results without blocking a worker. The lemmas state the results as plain +computations, so proofs about a task graph reduce to proofs about the functions it runs. +-/ + +@[expose] public section +namespace CompPoly.Tasks + +universe u v +variable {α : Type u} {β : Type v} + +/-- Wait for the tasks `ts`, after the results `as` collected so far, latest first. A list collects +the results, so no result array is copied while a finished task still holds it. -/ +def joinGo : List (Task α) → List α → Task (Array α) + | [], as => .pure as.reverse.toArray + | t :: ts, as => t.bind (sync := true) fun a ↦ joinGo ts (a :: as) + +/-- Wait for every task without blocking a worker; results keep the task order. -/ +def join (ts : Array (Task α)) : Task (Array α) := joinGo ts.toList [] + +/-- `f i` for every `i < n`, as tasks. -/ +@[inline] def spawnAll (n : Nat) (f : Nat → α) : Array (Task α) := + (Array.range n).map fun i ↦ Task.spawn fun _ ↦ f i + +/-- `f xs i` for every `i < n`, as tasks that start once every task of `deps` is done; `xs` are +their results. -/ +@[inline] def spawnAfter (deps : Array (Task α)) (n : Nat) (f : Array α → Nat → β) : + Array (Task β) := + let all := join deps + (Array.range n).map fun i ↦ all.bind fun xs ↦ Task.spawn fun _ ↦ f xs i + +/-- A dependent task's result is its continuation's result on the dependency's result. -/ +@[simp] theorem get_bind (t : Task α) (f : α → Task β) (prio : Task.Priority) (sync : Bool) : + (t.bind f prio sync).get = (f t.get).get := rfl + +/-- A spawned task's result is its function's result. -/ +@[simp] theorem get_spawn (fn : Unit → α) (prio : Task.Priority) : + (Task.spawn fn prio).get = fn () := rfl + +/-- `joinGo`'s results: those collected so far, then the tasks'. -/ +theorem get_joinGo (ts : List (Task α)) (as : List α) : + (joinGo ts as).get = (as.reverse ++ ts.map Task.get).toArray := by + induction ts generalizing as with + | nil => rw [joinGo, List.map_nil, List.append_nil] + | cons t ts ih => + change (joinGo ts (t.get :: as)).get = _ + rw [ih, List.reverse_cons, List.map_cons, List.append_assoc, List.singleton_append] + +/-- Joining tasks keeps their results in order. -/ +theorem get_join (ts : Array (Task α)) : (join ts).get = ts.map Task.get := by + rw [join, get_joinGo, List.reverse_nil, List.nil_append, ← Array.toList_map, + Array.toArray_toList] + +/-- The results of independent tasks. -/ +@[simp] theorem map_get_spawnAll (n : Nat) (f : Nat → α) : + (spawnAll n f).map Task.get = (Array.range n).map f := by + simp only [spawnAll, Array.map_map] + rfl + +/-- The results of dependent tasks, given the results of their dependencies. -/ +@[simp] theorem map_get_spawnAfter (deps : Array (Task α)) (n : Nat) (f : Array α → Nat → β) : + (spawnAfter deps n f).map Task.get = (Array.range n).map (f (deps.map Task.get)) := by + simp only [spawnAfter, Array.map_map] + congr 1 + funext i + change f (join deps).get i = _ + rw [get_join] + +end CompPoly.Tasks diff --git a/CompPoly/Fields/BLS12_377/Fast.lean b/CompPoly/Fields/BLS12_377/Fast.lean index 5aace507..2d0d92d2 100644 --- a/CompPoly/Fields/BLS12_377/Fast.lean +++ b/CompPoly/Fields/BLS12_377/Fast.lean @@ -6,13 +6,13 @@ Authors: Georgios Raikos module public import CompPoly.Fields.BLS12_377.Basic -public import CompPoly.Fields.Montgomery.Native64x8Inv +public import CompPoly.Fields.Montgomery.Native64x4Inv /-! # Fast BLS12-377 Scalar Field -A native eight-limb Montgomery implementation of BLS12-377 scalar arithmetic -(`CompPoly.Fields.Montgomery.Native64x8Field`). This module supplies the BLS12-377 +A native four-limb Montgomery implementation of BLS12-377 scalar arithmetic +(`CompPoly.Fields.Montgomery.Native64x4Field`). This module supplies the BLS12-377 constants. -/ @@ -20,7 +20,7 @@ constants. namespace BLS12_377.Fast -open Montgomery.Native64x8 (Mont64x8Field FastField GcdData) +open Montgomery.Native64x4 (Mont64x4Field FastField GcdData) set_option exponentiation.threshold 1100 @@ -29,26 +29,21 @@ set_option exponentiation.threshold 1100 /-- Divstep schedule of the binary-GCD inverse candidate. -/ instance instGcdData : GcdData BLS12_377.scalarFieldSize where finalRounds := 39 - initU := - ⟨0x94e01332, 0xa5707af0, 0x6f1abbfb, 0xf9af464a, 0x7f75c179, 0x65dbc6f7, 0xc9364c65, - 0x10e76cd3⟩ + initU := ⟨0x50d904ef75fb587f, 0x25a2b48c71615c44, 0x8203611dc5f581c3, 0xd6a3a20c872ad35⟩ -/-- The per-field data realizing BLS12-377's scalar field as a fast eight-limb -(32-bit-limb) Montgomery field. -/ -instance instMont64x8Field : Mont64x8Field BLS12_377.scalarFieldSize where +/-- The per-field data realizing BLS12-377's scalar field as a fast four-limb Montgomery +field. -/ +instance instMont64x4Field : Mont64x4Field BLS12_377.scalarFieldSize where prime := BLS12_377.ScalarField_is_prime modulusLimbs := - ⟨0x1, 0xa118000, 0xd0000001, 0x59aa76fe, 0x5c37b001, 0x60b44d1e, 0x9a2ca556, - 0x12ab655e⟩ + ⟨0xa11800000000001, 0x59aa76fed0000001, 0x60b44d1e5c37b001, 0x12ab655e9a2ca556⟩ rModModulus := - ⟨0xfffffff3, 0x7d1c7fff, 0x6ffffff2, 0x7257f50f, 0x512c0fee, 0x16d81575, 0x2bbb9a9d, - 0xd4bda32⟩ + ⟨0x7d1c7ffffffffff3, 0x7257f50f6ffffff2, 0x16d81575512c0fee, 0xd4bda322bbb9a9d⟩ r2ModModulus := - ⟨0xb861857b, 0x25d577ba, 0x8860591f, 0xcc2c27b5, 0xe5dc8593, 0xa7cc008f, 0xeff1c939, - 0x11fdae7⟩ - montgomeryNegInv := 0xffffffff + ⟨0x25d577bab861857b, 0xcc2c27b58860591f, 0xa7cc008fe5dc8593, 0x11fdae7eff1c939⟩ + montgomeryNegInv := 0xa117fffffffffff -/-- The eight-limb BLS12-377 scalar field carrier, stored as a Montgomery residue. -/ +/-- The four-limb BLS12-377 scalar field carrier, stored as a Montgomery residue. -/ abbrev ScalarField : Type := FastField BLS12_377.scalarFieldSize /-- Convert from the canonical `BLS12_377.ScalarField` field into fast Montgomery form. -/ @@ -56,7 +51,7 @@ abbrev ScalarField : Type := FastField BLS12_377.scalarFieldSize def ofField (x : BLS12_377.ScalarField) : ScalarField := FastField.ofField x -/-- Ring equivalence between the eight-limb representation and the canonical +/-- Ring equivalence between the four-limb representation and the canonical `BLS12_377.ScalarField`. -/ def ringEquiv : ScalarField ≃+* BLS12_377.ScalarField := FastField.ringEquiv BLS12_377.scalarFieldSize diff --git a/CompPoly/Fields/BLS12_381/Fast.lean b/CompPoly/Fields/BLS12_381/Fast.lean index 5fe69e4b..5e4dd63a 100644 --- a/CompPoly/Fields/BLS12_381/Fast.lean +++ b/CompPoly/Fields/BLS12_381/Fast.lean @@ -6,13 +6,13 @@ Authors: Georgios Raikos module public import CompPoly.Fields.BLS12_381.Basic -public import CompPoly.Fields.Montgomery.Native64x8Inv +public import CompPoly.Fields.Montgomery.Native64x4Inv /-! # Fast BLS12-381 Scalar Field -A native eight-limb Montgomery implementation of BLS12-381 scalar arithmetic -(`CompPoly.Fields.Montgomery.Native64x8Field`). This module supplies the BLS12-381 +A native four-limb Montgomery implementation of BLS12-381 scalar arithmetic +(`CompPoly.Fields.Montgomery.Native64x4Field`). This module supplies the BLS12-381 constants. -/ @@ -20,7 +20,7 @@ constants. namespace BLS12_381.Fast -open Montgomery.Native64x8 (Mont64x8Field FastField GcdData) +open Montgomery.Native64x4 (Mont64x4Field FastField GcdData) set_option exponentiation.threshold 1100 @@ -29,26 +29,21 @@ set_option exponentiation.threshold 1100 /-- Divstep schedule of the binary-GCD inverse candidate. -/ instance instGcdData : GcdData BLS12_381.scalarFieldSize where finalRounds := 43 - initU := - ⟨0xfe9ec31c, 0x408b03b2, 0x22e5e5f4, 0x49ada1d5, 0x00f31bd7, 0x0a662f58, 0x6fc84e4d, - 0x1f758dcc⟩ + initU := ⟨0x92413fd93c25fbb9, 0x69ef6a08aca8f1e3, 0xbdff4014f383b8a5, 0x2d9fa11995e2391a⟩ -/-- The per-field data realizing BLS12-381's scalar field as a fast eight-limb -(32-bit-limb) Montgomery field. -/ -instance instMont64x8Field : Mont64x8Field BLS12_381.scalarFieldSize where +/-- The per-field data realizing BLS12-381's scalar field as a fast four-limb Montgomery +field. -/ +instance instMont64x4Field : Mont64x4Field BLS12_381.scalarFieldSize where prime := BLS12_381.ScalarField_is_prime modulusLimbs := - ⟨0x1, 0xffffffff, 0xfffe5bfe, 0x53bda402, 0x9a1d805, 0x3339d808, 0x299d7d48, - 0x73eda753⟩ + ⟨0xffffffff00000001, 0x53bda402fffe5bfe, 0x3339d80809a1d805, 0x73eda753299d7d48⟩ rModModulus := - ⟨0xfffffffe, 0x1, 0x34802, 0x5884b7fa, 0xecbc4ff5, 0x998c4fef, 0xacc5056f, - 0x1824b159⟩ + ⟨0x1fffffffe, 0x5884b7fa00034802, 0x998c4fefecbc4ff5, 0x1824b159acc5056f⟩ r2ModModulus := - ⟨0xf3f29c6d, 0xc999e990, 0x87925c23, 0x2b6cedcb, 0x7254398f, 0x5d31496, 0x9f59ff11, - 0x748d9d9⟩ - montgomeryNegInv := 0xffffffff + ⟨0xc999e990f3f29c6d, 0x2b6cedcb87925c23, 0x5d314967254398f, 0x748d9d99f59ff11⟩ + montgomeryNegInv := 0xfffffffeffffffff -/-- The eight-limb BLS12-381 scalar field carrier, stored as a Montgomery residue. -/ +/-- The four-limb BLS12-381 scalar field carrier, stored as a Montgomery residue. -/ abbrev ScalarField : Type := FastField BLS12_381.scalarFieldSize /-- Convert from the canonical `BLS12_381.ScalarField` field into fast Montgomery form. -/ @@ -56,7 +51,7 @@ abbrev ScalarField : Type := FastField BLS12_381.scalarFieldSize def ofField (x : BLS12_381.ScalarField) : ScalarField := FastField.ofField x -/-- Ring equivalence between the eight-limb representation and the canonical +/-- Ring equivalence between the four-limb representation and the canonical `BLS12_381.ScalarField`. -/ def ringEquiv : ScalarField ≃+* BLS12_381.ScalarField := FastField.ringEquiv BLS12_381.scalarFieldSize diff --git a/CompPoly/Fields/BN254/Fast.lean b/CompPoly/Fields/BN254/Fast.lean index a74ce592..88a8a5bf 100644 --- a/CompPoly/Fields/BN254/Fast.lean +++ b/CompPoly/Fields/BN254/Fast.lean @@ -6,20 +6,20 @@ Authors: Georgios Raikos module public import CompPoly.Fields.BN254.Basic -public import CompPoly.Fields.Montgomery.Native64x8Inv +public import CompPoly.Fields.Montgomery.Native64x4Inv /-! # Fast BN254 Scalar Field -A native eight-limb Montgomery implementation of BN254 scalar arithmetic -(`CompPoly.Fields.Montgomery.Native64x8Field`). This module supplies the BN254 constants. +A native four-limb Montgomery implementation of BN254 scalar arithmetic +(`CompPoly.Fields.Montgomery.Native64x4Field`). This module supplies the BN254 constants. -/ @[expose] public section namespace BN254.Fast -open Montgomery.Native64x8 (Mont64x8Field FastField GcdData) +open Montgomery.Native64x4 (Mont64x4Field FastField GcdData) set_option exponentiation.threshold 1100 @@ -28,26 +28,21 @@ set_option exponentiation.threshold 1100 /-- Divstep schedule of the binary-GCD inverse candidate. -/ instance instGcdData : GcdData BN254.scalarFieldSize where finalRounds := 41 - initU := - ⟨0x1cafd3b2, 0xb542296a, 0x9c820da9, 0xec8811f1, 0x6bd1d274, 0x29bed602, 0xa8ab8d54, - 0x0fb316d8⟩ + initU := ⟨0x31db48599a71d3a, 0x21a9c717fffa68b1, 0xe09ca8b1e4e66050, 0x12e69c542f33d99a⟩ -/-- The per-field data realizing BN254's scalar field as a fast eight-limb (32-bit-limb) -Montgomery field. -/ -instance instMont64x8Field : Mont64x8Field BN254.scalarFieldSize where +/-- The per-field data realizing BN254's scalar field as a fast four-limb Montgomery +field. -/ +instance instMont64x4Field : Mont64x4Field BN254.scalarFieldSize where prime := BN254.ScalarField_is_prime modulusLimbs := - ⟨0xf0000001, 0x43e1f593, 0x79b97091, 0x2833e848, 0x8181585d, 0xb85045b6, 0xe131a029, - 0x30644e72⟩ + ⟨0x43e1f593f0000001, 0x2833e84879b97091, 0xb85045b68181585d, 0x30644e72e131a029⟩ rModModulus := - ⟨0x4ffffffb, 0xac96341c, 0x9f60cd29, 0x36fc7695, 0x7879462e, 0x666ea36f, 0x9a07df2f, - 0xe0a77c1⟩ + ⟨0xac96341c4ffffffb, 0x36fc76959f60cd29, 0x666ea36f7879462e, 0xe0a77c19a07df2f⟩ r2ModModulus := - ⟨0xae216da7, 0x1bb8e645, 0xe35c59e3, 0x53fe3ab1, 0x53bb8085, 0x8c49833d, 0x7f4e44a5, - 0x216d0b1⟩ - montgomeryNegInv := 0xefffffff + ⟨0x1bb8e645ae216da7, 0x53fe3ab1e35c59e3, 0x8c49833d53bb8085, 0x216d0b17f4e44a5⟩ + montgomeryNegInv := 0xc2e1f593efffffff -/-- The eight-limb BN254 scalar field carrier, stored as a Montgomery residue. -/ +/-- The four-limb BN254 scalar field carrier, stored as a Montgomery residue. -/ abbrev ScalarField : Type := FastField BN254.scalarFieldSize /-- Convert from the canonical `BN254.ScalarField` field into fast Montgomery form. -/ @@ -55,7 +50,7 @@ abbrev ScalarField : Type := FastField BN254.scalarFieldSize def ofField (x : BN254.ScalarField) : ScalarField := FastField.ofField x -/-- Ring equivalence between the eight-limb representation and the canonical +/-- Ring equivalence between the four-limb representation and the canonical `BN254.ScalarField`. -/ def ringEquiv : ScalarField ≃+* BN254.ScalarField := FastField.ringEquiv BN254.scalarFieldSize diff --git a/CompPoly/Fields/BabyBear/FastExt4.lean b/CompPoly/Fields/BabyBear/FastExt4.lean new file mode 100644 index 00000000..040f85cf --- /dev/null +++ b/CompPoly/Fields/BabyBear/FastExt4.lean @@ -0,0 +1,43 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Extension.Map +public import CompPoly.Fields.BabyBear.Ext4 +public import CompPoly.Fields.BabyBear.Fast +public import CompPoly.Fields.Montgomery.Native32Ext4 + +/-! +# The degree-4 extension of BabyBear over the fast carrier + +`BabyBear.Fast.Ext4` is `Montgomery.Native32.Ext4` for BabyBear and the binomial `X^4 - 11`, the +same field as Plonky3's `BinomialExtensionField`. Irreducibility is the spec +certificate `BabyBear.ext4Params_poly_irreducible`, transferred from `BabyBear.Field` to the fast +carrier along `ringEquiv`. +-/ + +@[expose] public section + +namespace BabyBear.Fast + +open CompPoly.Extension Montgomery.Native32 Polynomial + +/-- `X^4 - 11` is irreducible over the fast carrier. -/ +theorem irreducible_X_pow_four_sub_eleven : Irreducible (X ^ 4 - C 11 : Field[X]) := by + have h := irreducible_map_ringEquiv ringEquiv.symm ext4Params_poly_irreducible + rwa [BinomialParams.poly, ext4Params_d, ext4Params_W, Polynomial.map_sub, Polynomial.map_pow, + map_X, map_C, map_ofNat] at h + +/-- BabyBear's quartic extension is `X^4 - 11`. Inlined, so that specialized code multiplies by +`W` directly. -/ +@[inline] instance instBinomial4 : Binomial4 BabyBear.fieldSize where + W := 11 + irreducible := irreducible_X_pow_four_sub_eleven + +/-- The degree-4 extension of BabyBear with fast coefficients. -/ +abbrev Ext4 : Type := Montgomery.Native32.Ext4 BabyBear.fieldSize + +end BabyBear.Fast diff --git a/CompPoly/Fields/BabyBear/InterpolateCoset.lean b/CompPoly/Fields/BabyBear/InterpolateCoset.lean new file mode 100644 index 00000000..a1a1b7d6 --- /dev/null +++ b/CompPoly/Fields/BabyBear/InterpolateCoset.lean @@ -0,0 +1,33 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.BabyBear.FastExt4 +public import CompPoly.Fields.Montgomery.InterpolateCosetPacked + +/-! +# Compiled packed coset interpolation over BabyBear + +The `PackedInterpolation` instance for BabyBear's quartic extension. Its `run` applies the generic +`InterpolatePacked.run` at BabyBear's modulus, so the compiler specializes the loops here, in the +CPU-targeted `CompPolyInterpolateNative` library. +-/ + +@[expose] public section + +namespace BabyBear.Fast + +open Montgomery.Native32 + +set_option maxHeartbeats 1000000 in +/-- Packed coset interpolation specialized to BabyBear. -/ +def interpolateCosetRun (logN : ℕ) (ω s : Field) (width : ℕ) (evals : ByteArray) (z : Ext4) + (logTasks : ℕ) : Array Ext4 := + InterpolatePacked.run logN ω s width evals z logTasks + +instance : PackedInterpolation BabyBear.fieldSize := ⟨interpolateCosetRun, rfl⟩ + +end BabyBear.Fast diff --git a/CompPoly/Fields/Extension.lean b/CompPoly/Fields/Extension.lean index b2ad6908..e980d504 100644 --- a/CompPoly/Fields/Extension.lean +++ b/CompPoly/Fields/Extension.lean @@ -11,6 +11,7 @@ public import CompPoly.Fields.Extension.Bridge public import CompPoly.Fields.Extension.Cardinality public import CompPoly.Fields.Extension.Defs public import CompPoly.Fields.Extension.Field +public import CompPoly.Fields.Extension.Map /-! # Computable field extensions @@ -29,6 +30,8 @@ individual modules for details: multiply-by-`X` law `toQuot_shiftReduce`, and the `CommRing` structure. * `CompPoly/Fields/Extension/Cardinality.lean` — optional finiteness and cardinality certificates. * `CompPoly/Fields/Extension/Field.lean` — bijectivity and the certified `Field` structure. +* `CompPoly/Fields/Extension/Map.lean` — base change of the parameters along a ring homomorphism, + and transfer of irreducibility along a ring isomorphism. -/ @[expose] public section diff --git a/CompPoly/Fields/Extension/Map.lean b/CompPoly/Fields/Extension/Map.lean new file mode 100644 index 00000000..9a132f1c --- /dev/null +++ b/CompPoly/Fields/Extension/Map.lean @@ -0,0 +1,58 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Extension.Defs + +/-! +# Base change of extension parameters + +`ExtensionParams.map e P` maps the coefficients of `P`'s modulus along a ring homomorphism +`e : F →+* G`, and `ExtensionParams.map_poly` shows that its polynomial is `P.poly.map e`. Along a +ring isomorphism irreducibility transfers (`irreducible_map_ringEquiv`), so an irreducibility +certificate proved over one presentation of a field, such as `ZMod p`, serves every isomorphic +one, such as a Montgomery carrier. +-/ + +@[expose] public section + +namespace CompPoly.Extension + +open Polynomial + +variable {F G : Type*} [Field F] [Field G] + +/-- Irreducibility transfers along a ring isomorphism of the coefficients. -/ +theorem irreducible_map_ringEquiv (e : F ≃+* G) {f : F[X]} (h : Irreducible f) : + Irreducible (f.map (e : F →+* G)) := + h.map (mapEquiv e) + +namespace ExtensionParams + +/-- The parameters with the modulus' coefficients mapped along `e`. -/ +def map (e : F →+* G) (P : ExtensionParams F) : ExtensionParams G where + d := P.d + two_le := P.two_le + lower := P.lower.map e + q := P.q + +@[simp] theorem map_d (e : F →+* G) (P : ExtensionParams F) : (P.map e).d = P.d := rfl +@[simp] theorem map_q (e : F →+* G) (P : ExtensionParams F) : (P.map e).q = P.q := rfl + +theorem map_poly (e : F →+* G) (P : ExtensionParams F) : (P.map e).poly = P.poly.map e := by + simp only [poly, Polynomial.map_add, Polynomial.map_pow, map_X, Polynomial.map_sum, + Polynomial.map_mul, map_C] + congr 1 + exact Finset.sum_congr rfl fun i _ ↦ by + simp only [lowerCoeff, map, Vector.getElem_map] + +theorem irreducible_map_poly (e : F ≃+* G) (P : ExtensionParams F) (h : Irreducible P.poly) : + Irreducible (P.map (e : F →+* G)).poly := by + rw [map_poly]; exact irreducible_map_ringEquiv e h + +end ExtensionParams + +end CompPoly.Extension diff --git a/CompPoly/Fields/Goldilocks/Fast.lean b/CompPoly/Fields/Goldilocks/Fast.lean index 572b5261..f27a512d 100644 --- a/CompPoly/Fields/Goldilocks/Fast.lean +++ b/CompPoly/Fields/Goldilocks/Fast.lean @@ -18,8 +18,8 @@ kernels from `FastDefs` with the bounds proved in `FastReduction`. Every operati identified with its counterpart in the canonical `ZMod` model, and the field instances are transferred across `toField`. -Reduction rests on `2^64 ≡ 2^32 - 1 (mod p)`, so a 128-bit product folds back into one -word with shifts, one multiply by `2^32 - 1`, and carry corrections. +Reduction rests on `2^64 ≡ 2^32 - 1 (mod p)`, which folds a 128-bit product back into one +word with shifts, additions and carry corrections. -/ @[expose] public section @@ -122,11 +122,7 @@ def toField (x : Field) : Goldilocks.Field := /-- Fast modular addition in canonical form. -/ @[inline] def add (x y : Field) : Field := - let lo := x.val + y.val - let carry := decide (lo < x.val) - ⟨reduceAddWithCarryRaw lo carry, - reduceAddWithCarryRaw_lt lo carry - (addWithCarry_bound x.val y.val x.property y.property)⟩ + ⟨addRaw x.val y.val, addRaw_lt x.property y.property⟩ /-- Fast modular negation in canonical form. -/ @[inline] @@ -148,41 +144,25 @@ def mul (x y : Field) : Field := def square (x : Field) : Field := mul x x -/-- Repeated squaring: `squareN x n` computes `x^(2^n)`. -/ +/-- Repeated squaring, `squareN x n = x^(2^n)`, as a tail-recursive loop. -/ @[inline] def squareN (x : Field) : Nat → Field | 0 => x - | n + 1 => square (squareN x n) + | n + 1 => squareN (square x) n -/-- Exponentiation over the fast representation using binary exponentiation. -/ +/-- Exponentiation through the lazy ladder `powLazy`, canonicalized once. -/ @[inline] def pow (x : Field) (n : Nat) : Field := - @npowBinRec Field ⟨one⟩ ⟨mul⟩ n x + ⟨reduceUInt64Raw (powLazy 1 x.val n), reduceUInt64Raw_lt _⟩ /-- Fermat exponent used for inversion in the Goldilocks prime field. -/ @[inline] private def invExponent : Nat := Goldilocks.fieldSize - 2 -/-- Fast modular inversion using an addition chain for `p - 2`. - -For Goldilocks, `p - 2 = 0xFFFFFFFEFFFFFFFF`. The chain builds -`x^(2^31 - 1)`, derives `x^(2^32 - 2)` and `x^(2^32 - 1)`, then combines them as - -`(2^32 - 2) * 2^32 + (2^32 - 1) = p - 2`. --/ +/-- Fast modular inversion through the lazy Fermat chain `invLazy`, canonicalized once. -/ +@[inline] def inv (x : Field) : Field := - let t2 := mul (square x) x - let t4 := mul (squareN t2 2) t2 - let t8 := mul (squareN t4 4) t4 - let t16 := mul (squareN t8 8) t8 - let t31 := - mul (squareN t16 15) - (mul (squareN t8 7) - (mul (squareN t4 3) - (mul (square t2) x))) - let t32m2 := square t31 - let t32m1 := mul t32m2 x - mul (squareN t32m2 32) t32m1 + ⟨reduceUInt64Raw (invLazy x.val), reduceUInt64Raw_lt _⟩ /-- Division through inversion and fast multiplication. -/ @[inline] @@ -421,25 +401,9 @@ theorem toField_one : toField (1 : Field) = 1 := by /-- Fast addition agrees with canonical-field addition. -/ @[simp] theorem toField_add (x y : Field) : toField (x + y) = toField x + toField y := by - change - (((add x y).val.toNat : Goldilocks.Field) = - (x.val.toNat : Goldilocks.Field) + (y.val.toNat : Goldilocks.Field)) - unfold add - rw [reduceAddWithCarryRaw_cast _ _ - (addWithCarry_bound x.val y.val x.property y.property)] - let lo := x.val + y.val - let carry := decide (lo < x.val) - have hvalue := addWithCarry_value x.val y.val - change lo.toNat + (if carry then UInt64.size else 0) = x.val.toNat + y.val.toNat at hvalue - change - ((lo.toNat : Goldilocks.Field) + - (if carry then (UInt64.size : Goldilocks.Field) else 0) = - (x.val.toNat : Goldilocks.Field) + (y.val.toNat : Goldilocks.Field)) - by_cases hcarry : carry = true - · simp only [hcarry, ite_true] at hvalue ⊢ - rw [← Nat.cast_add, hvalue, Nat.cast_add] - · simp only [hcarry, Bool.false_eq_true, ite_false, add_zero] at hvalue ⊢ - rw [hvalue, Nat.cast_add] + change toField (add x y) = toField x + toField y + simp only [add, toField, toNat] + exact addRaw_cast x.property y.property /-- Fast negation agrees with canonical-field negation. -/ @[simp] @@ -483,46 +447,26 @@ theorem toField_squareN (x : Field) (n : Nat) : simp | succ n ih => unfold squareN - rw [toField_square, ih] - rw [← pow_add] + rw [ih, toField_square, ← pow_two, ← pow_mul] congr 1 rw [Nat.pow_succ] omega -/-- Fast multiplication is associative, proved by transporting to the canonical field. -/ -private theorem mul_assoc_field (x y z : Field) : (x * y) * z = x * (y * z) := by - apply toField_injective - rw [toField_mul, toField_mul, toField_mul, toField_mul] - ring - -/-- Binary exponentiation satisfies the expected successor equation. -/ -private theorem pow_succ (x : Field) (n : Nat) : pow x (n + 1) = pow x n * x := by - unfold pow - let _ : Semigroup Field := { - mul := (· * ·) - mul_assoc := mul_assoc_field - } - exact npowBinRec_succ n x - /-- Fast natural-power computation agrees with powers in the canonical field. -/ @[simp] theorem toField_pow (x : Field) (n : Nat) : toField (pow x n) = toField x ^ n := by - induction n with - | zero => - unfold pow - rw [npowBinRec_zero] - rw [toField_one] - simp - | succ n ih => - rw [pow_succ, toField_mul, ih, _root_.pow_succ] + change ((reduceUInt64Raw (powLazy 1 x.val n)).toNat : Goldilocks.Field) = + (x.val.toNat : Goldilocks.Field) ^ n + rw [reduceUInt64Raw_cast, powLazy_cast] + simp /-- The optimized inversion chain computes the Fermat inverse exponent. -/ private theorem toField_inv_chain (x : Field) : toField (inv x) = toField x ^ invExponent := by - unfold inv - simp only [toField_mul_def, toField_square, toField_squareN] - ring_nf - simp [invExponent, Goldilocks.fieldSize] + change ((reduceUInt64Raw (invLazy x.val)).toNat : Goldilocks.Field) = + (x.val.toNat : Goldilocks.Field) ^ invExponent + rw [reduceUInt64Raw_cast, invLazy_cast] + rfl /-- Fast inversion agrees with canonical inversion before notation is unfolded. -/ private theorem toField_inv_raw (x : Field) : toField (inv x) = (toField x)⁻¹ := by diff --git a/CompPoly/Fields/Goldilocks/FastDefs.lean b/CompPoly/Fields/Goldilocks/FastDefs.lean index ebdb0b20..11007fd3 100644 --- a/CompPoly/Fields/Goldilocks/FastDefs.lean +++ b/CompPoly/Fields/Goldilocks/FastDefs.lean @@ -8,13 +8,9 @@ module /-! # Fast Goldilocks: runtime definitions (zero-import) -The runtime definitions of the native-word Goldilocks arithmetic, split out of -`CompPoly.Fields.Goldilocks.Fast` verbatim. All correctness statements about them -live in that sibling module, which imports this one. - -This module deliberately has **zero imports**: downstream consumers put it into -`precompileModules` native-compilation lanes, and `precompileModules` compiles the -entire import closure, so the runtime definitions must not pull in mathlib. +Runtime word kernels of the native Goldilocks field, proved correct in +`CompPoly.Fields.Goldilocks.FastReduction`. This module has zero imports so that +`precompileModules` lanes, which compile the whole import closure, can take it without Mathlib. -/ @[expose] public section @@ -33,11 +29,12 @@ def negModulus : UInt64 := 0xffffffff /-! ## Raw word kernels -Every kernel takes canonical `UInt64` inputs and returns a canonical representative -below the modulus. Correctness lives in `CompPoly.Fields.Goldilocks.Fast`. -/ +Kernels take and return canonical words below the modulus; the `Lazy` ones take and +return any congruent word, so a chain canonicalizes once at the end. -/ -/-- Full 64-by-64 product as `(lo, hi)` words, computed from 32-bit limbs. -/ +/-- Full 64-by-64 product as `(lo, hi)` words from 32-bit limbs, in the shape clang fuses +into one widening multiply. -/ @[inline] def wideMul (x y : UInt64) : UInt64 × UInt64 := let xLo := x &&& negModulus @@ -48,50 +45,59 @@ def wideMul (x y : UInt64) : UInt64 × UInt64 := let p01 := xLo * yHi let p10 := xHi * yLo let p11 := xHi * yHi - let carry := (p00 >>> 32) + (p01 &&& negModulus) + (p10 &&& negModulus) - let hi := p11 + (p01 >>> 32) + (p10 >>> 32) + (carry >>> 32) + let t := (p00 >>> 32) + p01 + let u := (t &&& negModulus) + p10 + let hi := p11 + (t >>> 32) + (u >>> 32) (x * y, hi) -/-- Raw one-word reduction for a `UInt64` value. - -Since every `UInt64` is below `2^64 = p + 2^32 - 1`, one subtraction by `p` -is enough to canonicalize a native word. --/ +/-- Canonicalize a word by one conditional subtraction, since every `UInt64` is below `2 * p`. -/ @[inline] def reduceUInt64Raw (x : UInt64) : UInt64 := if x < modulus then x else x - modulus +/-- Borrow arm of the 128-bit fold, out of line so the common path takes a predicted branch. -/ +@[noinline] +def foldUInt128Borrow (lo hiHi hiLo : UInt64) : UInt64 := + let t0 := lo - hiHi - negModulus + let t1 := hiLo * negModulus + let t2 := t0 + t1 + if t2 < t0 then t2 + negModulus else t2 + +/-- Fold `lo + hi * 2^64` into one congruent, not necessarily canonical, word using +`2^64 ≡ 2^32 - 1`; the middle term is `(hi <<< 32) - hiLo` rather than a multiply. -/ +@[inline] +def foldUInt128Lazy (lo hi : UInt64) : UInt64 := + let hiHi := hi >>> 32 + let hiLo := hi &&& negModulus + if lo < hiHi then foldUInt128Borrow lo hiHi hiLo + else + let t0 := lo - hiHi + let t1 := (hi <<< 32) - hiLo + let t2 := t0 + t1 + if t2 < t0 then t2 + negModulus else t2 + /-- Raw reduction of a 128-bit integer represented by low and high words modulo Goldilocks. -/ @[inline] def reduceUInt128Raw (lo hi : UInt64) : UInt64 := - let hi_hi := hi >>> 32 - let hi_lo := hi &&& negModulus - - let borrow := lo < hi_hi - let t0 := lo - hi_hi - let t0 := if borrow then t0 - negModulus else t0 - - let t1 := hi_lo * negModulus + reduceUInt64Raw (foldUInt128Lazy lo hi) - let t2 := t0 + t1 - let overflow := t2 < t0 - let t2 := if overflow then t2 + negModulus else t2 - - reduceUInt64Raw t2 +/-- Product of two arbitrary words as one congruent, not necessarily canonical, word. -/ +@[inline] +def mulLazy (x y : UInt64) : UInt64 := + let product := wideMul x y + foldUInt128Lazy product.1 product.2 /-- Raw reduction of a 64-by-64 product modulo Goldilocks. -/ @[inline] def reduceMulRaw (x y : UInt64) : UInt64 := - let product := wideMul x y - reduceUInt128Raw product.1 product.2 + reduceUInt64Raw (mulLazy x y) -/-- Raw one-step reduction for a 65-bit addition represented by low word and carry. -/ +/-- Canonical sum as `x - (p - y)`, which borrows exactly when `x + y < p`. -/ @[inline] -def reduceAddWithCarryRaw (lo : UInt64) (carry : Bool) : UInt64 := - if carry then - lo + negModulus - else - reduceUInt64Raw lo +def addRaw (x y : UInt64) : UInt64 := + let u := modulus - y + let s := x - u + if x < u then s - negModulus else s /-- Raw modular negation in canonical form. -/ @[inline] @@ -103,4 +109,37 @@ def negRaw (x : UInt64) : UInt64 := def subRaw (x y : UInt64) : UInt64 := if y ≤ x then x - y else x - y - negModulus +/-! ## Lazy exponentiation -/ + + +/-- Square-and-multiply on unreduced words: `powLazy acc x n` computes `acc * x^n`. -/ +def powLazy (acc x : UInt64) (n : Nat) : UInt64 := + if h : n = 0 then acc + else + let acc := if n % 2 = 1 then mulLazy acc x else acc + powLazy acc (mulLazy x x) (n / 2) +termination_by n +decreasing_by omega + +/-- `x^(2^n)` on unreduced words. -/ +def squareNLazy (x : UInt64) : Nat → UInt64 + | 0 => x + | n + 1 => squareNLazy (mulLazy x x) n + +/-- Fermat chain for `x^(p - 2)` on unreduced words, with +`p - 2 = (2^32 - 2) * 2^32 + (2^32 - 1)`. -/ +def invLazy (x : UInt64) : UInt64 := + let t2 := mulLazy (mulLazy x x) x + let t4 := mulLazy (squareNLazy t2 2) t2 + let t8 := mulLazy (squareNLazy t4 4) t4 + let t16 := mulLazy (squareNLazy t8 8) t8 + let t31 := + mulLazy (squareNLazy t16 15) + (mulLazy (squareNLazy t8 7) + (mulLazy (squareNLazy t4 3) + (mulLazy (mulLazy t2 t2) x))) + let t32m2 := mulLazy t31 t31 + let t32m1 := mulLazy t32m2 x + mulLazy (squareNLazy t32m2 32) t32m1 + end Goldilocks.Fast diff --git a/CompPoly/Fields/Goldilocks/FastReduction.lean b/CompPoly/Fields/Goldilocks/FastReduction.lean index 3c9f703e..6ad080ef 100644 --- a/CompPoly/Fields/Goldilocks/FastReduction.lean +++ b/CompPoly/Fields/Goldilocks/FastReduction.lean @@ -268,213 +268,38 @@ theorem wideMul_low_toNat (lo : UInt64) (x y : UInt64) (hlo : lo = x * y) : lo.toNat = x.toNat * y.toNat % UInt64.size := by rw [hlo, UInt64.toNat_mul] -/-- Pure Nat carry formula for the high word of a 32-bit-limb 64-by-64 product. -/ -theorem wideMul_high_nat - (p00 p01 p10 p11 : Nat) - (_hp00 : p00 < 2 ^ 64) - (_hp01 : p01 < 2 ^ 64) - (_hp10 : p10 < 2 ^ 64) - (_hp11 : p11 < 2 ^ 64) : - let B := 2 ^ 32 - let carry := p00 / B + p01 % B + p10 % B - p11 + p01 / B + p10 / B + carry / B = - (p00 + B * (p01 + p10) + B ^ 2 * p11) / B ^ 2 := by - dsimp - let carry := p00 / 4294967296 + p01 % 4294967296 + p10 % 4294967296 - let q := p11 + p01 / 4294967296 + p10 / 4294967296 + carry / 4294967296 - have hN : - p00 + 4294967296 * (p01 + p10) + 18446744073709551616 * p11 = - p00 % 4294967296 + 4294967296 * (carry % 4294967296) + - 18446744073709551616 * q := by - have hp00d : p00 % 4294967296 + 4294967296 * (p00 / 4294967296) = p00 := - Nat.mod_add_div p00 4294967296 - have hp01d : p01 % 4294967296 + 4294967296 * (p01 / 4294967296) = p01 := - Nat.mod_add_div p01 4294967296 - have hp10d : p10 % 4294967296 + 4294967296 * (p10 / 4294967296) = p10 := - Nat.mod_add_div p10 4294967296 - have hcd : carry % 4294967296 + 4294967296 * (carry / 4294967296) = carry := - Nat.mod_add_div carry 4294967296 - subst q - subst carry - omega - rw [hN] - change q = - (p00 % 4294967296 + 4294967296 * (carry % 4294967296) + - 18446744073709551616 * q) / - 18446744073709551616 - rw [Nat.mul_comm 18446744073709551616 q] - rw [Nat.add_mul_div_right _ _ (show 0 < 18446744073709551616 by decide)] - rw [Nat.div_eq_of_lt] - · rw [Nat.zero_add] - · subst carry - have hmod0 : p00 % 4294967296 < 4294967296 := Nat.mod_lt _ (by decide) - have hmod1 : - (p00 / 4294967296 + p01 % 4294967296 + p10 % 4294967296) % - 4294967296 < - 4294967296 := Nat.mod_lt _ (by decide) - omega +/-- Carry formula for the high word of a 32-bit-limb product. -/ +private theorem wideMul_hi_nat (p00 p01 p10 p11 : Nat) : + p11 + (p00 / 2 ^ 32 + p01) / 2 ^ 32 + ((p00 / 2 ^ 32 + p01) % 2 ^ 32 + p10) / 2 ^ 32 = + (p00 + 2 ^ 32 * (p01 + p10) + 2 ^ 64 * p11) / 2 ^ 64 := by + omega -/-- High word returned by the 32-bit-limb `UInt64` multiplication algorithm. -/ -theorem wideMul_high_toNat - (x y hi : UInt64) - (hhi : - hi = - let xLo := x &&& negModulus - let xHi := x >>> 32 - let yLo := y &&& negModulus - let yHi := y >>> 32 - let p00 := xLo * yLo - let p01 := xLo * yHi - let p10 := xHi * yLo - let p11 := xHi * yHi - let carry := (p00 >>> 32) + (p01 &&& negModulus) + (p10 &&& negModulus) - p11 + (p01 >>> 32) + (p10 >>> 32) + (carry >>> 32)) : - hi.toNat = x.toNat * y.toNat / UInt64.size := by - let xLo := x &&& negModulus - let xHi := x >>> 32 - let yLo := y &&& negModulus - let yHi := y >>> 32 - let p00 := xLo * yLo - let p01 := xLo * yHi - let p10 := xHi * yLo - let p11 := xHi * yHi - let carry := (p00 >>> 32) + (p01 &&& negModulus) + (p10 &&& negModulus) - have hxLo_lt : xLo.toNat < 2 ^ 32 := by - subst xLo - exact uint64_low32_lt x - have hxHi_lt : xHi.toNat < 2 ^ 32 := by - subst xHi - exact uint64_high32_lt x - have hyLo_lt : yLo.toNat < 2 ^ 32 := by - subst yLo - exact uint64_low32_lt y - have hyHi_lt : yHi.toNat < 2 ^ 32 := by - subst yHi - exact uint64_high32_lt y - have hp00_nat : p00.toNat = xLo.toNat * yLo.toNat := by - subst p00 - exact mul32_toNat xLo yLo hxLo_lt hyLo_lt - have hp01_nat : p01.toNat = xLo.toNat * yHi.toNat := by - subst p01 - exact mul32_toNat xLo yHi hxLo_lt hyHi_lt - have hp10_nat : p10.toNat = xHi.toNat * yLo.toNat := by - subst p10 - exact mul32_toNat xHi yLo hxHi_lt hyLo_lt - have hp11_nat : p11.toNat = xHi.toNat * yHi.toNat := by - subst p11 - exact mul32_toNat xHi yHi hxHi_lt hyHi_lt - have hp00_lt : p00.toNat < 2 ^ 64 := by - rw [hp00_nat] - nlinarith [hxLo_lt, hyLo_lt] - have hp01_lt : p01.toNat < 2 ^ 64 := by - rw [hp01_nat] - nlinarith [hxLo_lt, hyHi_lt] - have hp10_lt : p10.toNat < 2 ^ 64 := by - rw [hp10_nat] - nlinarith [hxHi_lt, hyLo_lt] - have hp11_lt : p11.toNat < 2 ^ 64 := by - rw [hp11_nat] - nlinarith [hxHi_lt, hyHi_lt] - have hcarry_nat : - carry.toNat = p00.toNat / 2 ^ 32 + p01.toNat % 2 ^ 32 + p10.toNat % 2 ^ 32 := by - subst carry - rw [UInt64.toNat_add, UInt64.toNat_add] - rw [shiftRight32_toNat, and_negModulus_toNat, and_negModulus_toNat] - have hp00_hi_lt : p00.toNat / 2 ^ 32 < 2 ^ 32 := by - rw [Nat.div_lt_iff_lt_mul (by decide : 0 < 2 ^ 32)] - simpa [pow_add] using hp00_lt - have hp01_lo_lt : p01.toNat % 2 ^ 32 < 2 ^ 32 := Nat.mod_lt _ (by decide) - have hp10_lo_lt : p10.toNat % 2 ^ 32 < 2 ^ 32 := Nat.mod_lt _ (by decide) - have hsum01 : - p00.toNat / 2 ^ 32 + p01.toNat % 2 ^ 32 < 2 ^ 64 := by - omega - have hsum012 : - p00.toNat / 2 ^ 32 + p01.toNat % 2 ^ 32 + p10.toNat % 2 ^ 32 < - 2 ^ 64 := by - omega - rw [Nat.mod_eq_of_lt hsum01, Nat.mod_eq_of_lt hsum012] - have hwide := - wideMul_high_nat p00.toNat p01.toNat p10.toNat p11.toNat hp00_lt hp01_lt hp10_lt hp11_lt - have hwide' : - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 + - (p00.toNat / 2 ^ 32 + p01.toNat % 2 ^ 32 + p10.toNat % 2 ^ 32) / 2 ^ 32 = - (p00.toNat + 2 ^ 32 * (p01.toNat + p10.toNat) + (2 ^ 32) ^ 2 * p11.toNat) / - (2 ^ 32) ^ 2 := by - simpa using hwide - have hquot_lt : - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 + carry.toNat / 2 ^ 32 < - UInt64.size := by - rw [hcarry_nat] - rw [hwide'] - have hprod_bound : x.toNat * y.toNat < UInt64.size * UInt64.size := by - exact - mul_lt_mul'' (UInt64.toNat_lt_size x) (UInt64.toNat_lt_size y) (Nat.zero_le _) - (Nat.zero_le _) - have hsplit : - p00.toNat + 2 ^ 32 * (p01.toNat + p10.toNat) + (2 ^ 32) ^ 2 * p11.toNat = - x.toNat * y.toNat := by - rw [hp00_nat, hp01_nat, hp10_nat, hp11_nat] - subst p00 - subst p01 - subst p10 - subst p11 - subst xLo - subst xHi - subst yLo - subst yHi - simpa [pow_add, pow_mul] using (product_split32 x y).symm - rw [hsplit] - change x.toNat * y.toNat / UInt64.size < UInt64.size - rw [Nat.div_lt_iff_lt_mul (by decide : 0 < UInt64.size)] - exact hprod_bound - have hhi_nat : - hi.toNat = p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 + carry.toNat / 2 ^ 32 := by - rw [hhi] - dsimp only - change (p11 + (p01 >>> 32) + (p10 >>> 32) + (carry >>> 32)).toNat = - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 + carry.toNat / 2 ^ 32 - rw [UInt64.toNat_add, UInt64.toNat_add, UInt64.toNat_add] - rw [shiftRight32_toNat, shiftRight32_toNat, shiftRight32_toNat] - have hsum01 : p11.toNat + p01.toNat / 2 ^ 32 < UInt64.size := by - have hle : p11.toNat + p01.toNat / 2 ^ 32 ≤ - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 + carry.toNat / 2 ^ 32 := by - omega - exact lt_of_le_of_lt hle hquot_lt - have hsum012 : - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 < UInt64.size := by - have hle : p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 ≤ - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 + carry.toNat / 2 ^ 32 := by - omega - exact lt_of_le_of_lt hle hquot_lt - have hquot_lt_pow : - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 + carry.toNat / 2 ^ 32 < - 2 ^ 64 := by - simpa [UInt64.size] using hquot_lt - have hsum01_pow : p11.toNat + p01.toNat / 2 ^ 32 < 2 ^ 64 := by - simpa [UInt64.size] using hsum01 - have hsum012_pow : - p11.toNat + p01.toNat / 2 ^ 32 + p10.toNat / 2 ^ 32 < 2 ^ 64 := by - simpa [UInt64.size] using hsum012 - rw [Nat.mod_eq_of_lt hsum01_pow, Nat.mod_eq_of_lt hsum012_pow, - Nat.mod_eq_of_lt hquot_lt_pow] - rw [hhi_nat] - rw [hcarry_nat] - rw [hwide'] - have hsplit : - p00.toNat + 2 ^ 32 * (p01.toNat + p10.toNat) + (2 ^ 32) ^ 2 * p11.toNat = - x.toNat * y.toNat := by - rw [hp00_nat, hp01_nat, hp10_nat, hp11_nat] - subst p00 - subst p01 - subst p10 - subst p11 - subst xLo - subst xHi - subst yLo - subst yHi - simpa [pow_add, pow_mul] using (product_split32 x y).symm - rw [hsplit] - rfl +/-- The high word of `wideMul` is the high half of the product. -/ +theorem wideMul_snd_toNat (x y : UInt64) : + (wideMul x y).2.toNat = x.toNat * y.toNat / 2 ^ 64 := by + have hxLo := uint64_low32_lt x + have hxHi := uint64_high32_lt x + have hyLo := uint64_low32_lt y + have hyHi := uint64_high32_lt y + have hp00 := mul32_toNat _ _ hxLo hyLo + have hp01 := mul32_toNat _ _ hxLo hyHi + have hp10 := mul32_toNat _ _ hxHi hyLo + have hp11 := mul32_toNat _ _ hxHi hyHi + have hwide := wideMul_hi_nat ((x &&& negModulus).toNat * (y &&& negModulus).toNat) + ((x &&& negModulus).toNat * (y >>> 32).toNat) ((x >>> 32).toNat * (y &&& negModulus).toNat) + ((x >>> 32).toNat * (y >>> 32).toNat) + have h00 : (x &&& negModulus).toNat * (y &&& negModulus).toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := + Nat.mul_le_mul (by omega) (by omega) + have h01 : (x &&& negModulus).toNat * (y >>> 32).toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := + Nat.mul_le_mul (by omega) (by omega) + have h10 : (x >>> 32).toNat * (y &&& negModulus).toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := + Nat.mul_le_mul (by omega) (by omega) + have h11 : (x >>> 32).toNat * (y >>> 32).toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := + Nat.mul_le_mul (by omega) (by omega) + rw [product_split32 x y] + simp only [wideMul, UInt64.toNat_add, shiftRight32_toNat, and_negModulus_toNat, hp00, hp01, + hp10, hp11] at hwide h00 h01 h10 h11 hxLo hxHi hyLo hyHi ⊢ + omega /-- Combined semantic correctness of a 64-by-64 product represented by low and high words. -/ theorem wideMul_cast @@ -527,216 +352,118 @@ theorem reduceUInt64Raw_cast (x : UInt64) : exact hmod_le_x)] simp +/-- Tail of the 128-bit fold: add the middle term and correct one overflow. -/ +private theorem foldTail_cast (lo hiHi hiLo t0 : UInt64) (hhiLo : hiLo.toNat < 2 ^ 32) + (ht0 : (t0.toNat : Goldilocks.Field) = + (lo.toNat : Goldilocks.Field) - (hiHi.toNat : Goldilocks.Field)) : + (((if t0 + hiLo * negModulus < t0 then t0 + hiLo * negModulus + negModulus + else t0 + hiLo * negModulus)).toNat : Goldilocks.Field) = + (lo.toNat : Goldilocks.Field) - (hiHi.toNat : Goldilocks.Field) + + (hiLo.toNat : Goldilocks.Field) * (negModulus.toNat : Goldilocks.Field) := by + have ht1 : ((hiLo * negModulus).toNat : Goldilocks.Field) = + (hiLo.toNat : Goldilocks.Field) * (negModulus.toNat : Goldilocks.Field) := by + rw [mul_negModulus_toNat_of_lt hiLo hhiLo, Nat.cast_mul] + have hsum : t0.toNat + (hiLo * negModulus).toNat < 2 * UInt64.size - negModulus.toNat := by + have ht0 := UInt64.toNat_lt_size t0 + have ht1 := mul_negModulus_toNat_le hiLo hhiLo + have htwice : 2 * negModulus.toNat ≤ UInt64.size := by decide + omega + rw [addOverflowBounded_cast t0 (hiLo * negModulus) hsum, ht0, ht1] + +/-- The middle limb times `2^32 - 1`, formed as a shift and a subtraction. -/ +theorem shiftLeft32_sub_low (hi : UInt64) : + (hi <<< 32) - (hi &&& negModulus) = (hi &&& negModulus) * negModulus := by + apply UInt64.toNat_inj.mp + have hlo := uint64_low32_lt hi + have hhi := UInt64.toNat_lt_size hi + have hshl : (hi <<< 32).toNat = (hi &&& negModulus).toNat * 2 ^ 32 := by + rw [UInt64.toNat_shiftLeft, and_negModulus_toNat] + have h : (32 : UInt64).toNat % 64 = 32 := by decide + rw [h, Nat.shiftLeft_eq] + simp only [UInt64.size] at hhi + omega + have hle : (hi &&& negModulus) ≤ hi <<< 32 := by + rw [UInt64.le_iff_toNat_le, hshl] + omega + rw [UInt64.toNat_sub_of_le _ _ hle, hshl, mul_negModulus_toNat_of_lt _ hlo, negModulus_toNat] + omega + +/-- The lazy fold represents `lo + hi * 2^64`. -/ +theorem foldUInt128Lazy_cast (lo hi : UInt64) : + ((foldUInt128Lazy lo hi).toNat : Goldilocks.Field) = + (lo.toNat : Goldilocks.Field) + + (hi.toNat : Goldilocks.Field) * (UInt64.size : Goldilocks.Field) := by + have hhiLo := uint64_low32_lt hi + have hsub := subBorrow_cast lo (hi >>> 32) (uint64_high32_lt hi) + rw [hi_split_cast hi] + simp only [foldUInt128Lazy, foldUInt128Borrow] + by_cases hborrow : lo < hi >>> 32 + · rw [ite_eq_left hborrow] at hsub ⊢ + rw [foldTail_cast lo (hi >>> 32) (hi &&& negModulus) _ hhiLo hsub] + ring + · rw [ite_eq_right hborrow] at hsub ⊢ + rw [shiftLeft32_sub_low hi, foldTail_cast lo (hi >>> 32) (hi &&& negModulus) _ hhiLo hsub] + ring + /-- The raw 128-bit reducer returns a canonical representative below the modulus. -/ theorem reduceUInt128Raw_lt (lo hi : UInt64) : - (reduceUInt128Raw lo hi).toNat < Goldilocks.fieldSize := by - unfold reduceUInt128Raw - apply reduceUInt64Raw_lt + (reduceUInt128Raw lo hi).toNat < Goldilocks.fieldSize := + reduceUInt64Raw_lt _ /-- Semantic correctness of raw 128-bit Goldilocks reduction. -/ theorem reduceUInt128Raw_cast (lo hi : UInt64) : ((reduceUInt128Raw lo hi).toNat : Goldilocks.Field) = (lo.toNat : Goldilocks.Field) + (hi.toNat : Goldilocks.Field) * (UInt64.size : Goldilocks.Field) := by - let hi_hi := hi >>> 32 - let hi_lo := hi &&& negModulus - let t0 := if lo < hi_hi then lo - hi_hi - negModulus else lo - hi_hi - let t1 := hi_lo * negModulus - let t2 := if t0 + t1 < t0 then t0 + t1 + negModulus else t0 + t1 - change ((reduceUInt64Raw t2).toNat : Goldilocks.Field) = - (lo.toNat : Goldilocks.Field) + - (hi.toNat : Goldilocks.Field) * (UInt64.size : Goldilocks.Field) - have hred := reduceUInt64Raw_cast t2 - change ((reduceUInt64Raw t2).toNat : Goldilocks.Field) = - (t2.toNat : Goldilocks.Field) at hred - rw [hred] - have hhi_hi_lt : hi_hi.toNat < 2 ^ 32 := by - rw [show hi_hi = hi >>> 32 by rfl, shiftRight32_toNat] - have hhi := UInt64.toNat_lt_size hi - change hi.toNat < 2 ^ 64 at hhi - omega - have hhi_lo_lt : hi_lo.toNat < 2 ^ 32 := by - rw [show hi_lo = hi &&& negModulus by rfl, and_negModulus_toNat] - exact Nat.mod_lt _ (by decide) - have ht0_cast : - (t0.toNat : Goldilocks.Field) = - (lo.toNat : Goldilocks.Field) - (hi_hi.toNat : Goldilocks.Field) := by - rw [show t0 = if lo < hi_hi then lo - hi_hi - negModulus else lo - hi_hi by rfl] - exact subBorrow_cast lo hi_hi hhi_hi_lt - have ht1_cast : - (t1.toNat : Goldilocks.Field) = - (hi_lo.toNat : Goldilocks.Field) * - (negModulus.toNat : Goldilocks.Field) := by - rw [show t1 = hi_lo * negModulus by rfl] - rw [mul_negModulus_toNat_of_lt hi_lo hhi_lo_lt] - rw [Nat.cast_mul] - have ht2_bound : t0.toNat + t1.toNat < 2 * UInt64.size - negModulus.toNat := by - have ht0_lt := UInt64.toNat_lt_size t0 - have ht1_le : t1.toNat ≤ UInt64.size - 2 * negModulus.toNat := by - simpa [t1] using mul_negModulus_toNat_le hi_lo hhi_lo_lt - have htwice_neg_le_size : 2 * negModulus.toNat ≤ UInt64.size := by - decide - omega - have ht2_cast : - (t2.toNat : Goldilocks.Field) = - (t0.toNat : Goldilocks.Field) + (t1.toNat : Goldilocks.Field) := by - rw [show t2 = if t0 + t1 < t0 then t0 + t1 + negModulus else t0 + t1 by rfl] - exact addOverflowBounded_cast t0 t1 ht2_bound - rw [ht2_cast, ht0_cast, ht1_cast] - rw [hi_split_cast hi] - ring + unfold reduceUInt128Raw + rw [reduceUInt64Raw_cast, foldUInt128Lazy_cast] + +/-- The lazy product of any two words represents their product. -/ +@[simp] +theorem mulLazy_cast (x y : UInt64) : + ((mulLazy x y).toNat : Goldilocks.Field) = + (x.toNat : Goldilocks.Field) * (y.toNat : Goldilocks.Field) := by + unfold mulLazy + rw [foldUInt128Lazy_cast] + exact wideMul_cast x y _ _ rfl (wideMul_snd_toNat x y) /-- Product reduction returns a canonical representative below the modulus. -/ theorem reduceMulRaw_lt (x y : UInt64) : - (reduceMulRaw x y).toNat < Goldilocks.fieldSize := by - unfold reduceMulRaw - apply reduceUInt128Raw_lt + (reduceMulRaw x y).toNat < Goldilocks.fieldSize := + reduceUInt64Raw_lt _ /-- Semantic correctness of native 64-by-64 product reduction. -/ theorem reduceMulRaw_cast (x y : UInt64) : ((reduceMulRaw x y).toNat : Goldilocks.Field) = (x.toNat : Goldilocks.Field) * (y.toNat : Goldilocks.Field) := by unfold reduceMulRaw - rw [reduceUInt128Raw_cast] - exact - wideMul_cast x y (wideMul x y).1 (wideMul x y).2 - (by unfold wideMul; rfl) - (by - unfold wideMul - exact wideMul_high_toNat x y _ rfl) - -/-- Addition reduction returns a canonical representative. -/ -theorem reduceAddWithCarryRaw_lt (lo : UInt64) (carry : Bool) - (h : - lo.toNat + (if carry then UInt64.size else 0) < 2 * Goldilocks.fieldSize) : - (reduceAddWithCarryRaw lo carry).toNat < Goldilocks.fieldSize := by - unfold reduceAddWithCarryRaw - cases carry - · simp only [Bool.false_eq_true, ite_false] - exact reduceUInt64Raw_lt lo - · simp only [↓reduceIte] at h ⊢ - rw [UInt64.toNat_add] - have hsum_lt_field : lo.toNat + negModulus.toNat < Goldilocks.fieldSize := by - rw [negModulus_toNat, Goldilocks.fieldSize] - rw [Goldilocks.fieldSize, UInt64.size] at h - omega - have hsum_lt_size : lo.toNat + negModulus.toNat < UInt64.size := - Nat.lt_trans hsum_lt_field fieldSize_lt_uint64Size - rw [Nat.mod_eq_of_lt hsum_lt_size] - exact hsum_lt_field - -/-- Semantic correctness of addition reduction with carry. -/ -theorem reduceAddWithCarryRaw_cast (lo : UInt64) (carry : Bool) - (h : - lo.toNat + (if carry then UInt64.size else 0) < 2 * Goldilocks.fieldSize) : - ((reduceAddWithCarryRaw lo carry).toNat : Goldilocks.Field) = - (lo.toNat : Goldilocks.Field) + - (if carry then (UInt64.size : Goldilocks.Field) else 0) := by - unfold reduceAddWithCarryRaw - cases carry - · simp only [Bool.false_eq_true, ite_false, add_zero] - exact reduceUInt64Raw_cast lo - · simp only [↓reduceIte] - change lo.toNat + UInt64.size < 2 * Goldilocks.fieldSize at h - rw [UInt64.toNat_add] - have hsum_lt_field : lo.toNat + negModulus.toNat < Goldilocks.fieldSize := by - rw [negModulus_toNat, Goldilocks.fieldSize] - rw [Goldilocks.fieldSize, UInt64.size] at h - omega - have hsum_lt_size : lo.toNat + negModulus.toNat < UInt64.size := - Nat.lt_trans hsum_lt_field fieldSize_lt_uint64Size - rw [Nat.mod_eq_of_lt hsum_lt_size] - rw [Nat.cast_add] - rw [uint64_cast_eq_negModulus] + rw [reduceUInt64Raw_cast, mulLazy_cast] -/-- The wrapped word and carry produced by adding two canonical representatives is bounded. -/ -theorem addWithCarry_bound (x y : UInt64) - (hx : x.toNat < Goldilocks.fieldSize) +/-- Raw addition of canonical words computes the reduced sum. -/ +theorem addRaw_toNat {x y : UInt64} (hx : x.toNat < Goldilocks.fieldSize) (hy : y.toNat < Goldilocks.fieldSize) : - let lo := x + y - let carry := decide (lo < x) - lo.toNat + (if carry then UInt64.size else 0) < 2 * Goldilocks.fieldSize := by - intro lo carry - by_cases hcarry : carry - · simp only [hcarry] - have hlo_lt_x : lo.toNat < x.toNat := by - simpa [carry, UInt64.lt_iff_toNat_lt] using hcarry - have hsum_ge_size : UInt64.size ≤ x.toNat + y.toNat := by - by_contra hnot - have hsum_lt_size : x.toNat + y.toNat < UInt64.size := - Nat.lt_of_not_ge hnot - have hlo_eq : lo.toNat = x.toNat + y.toNat := by - rw [show lo = x + y by rfl, UInt64.toNat_add] - exact Nat.mod_eq_of_lt hsum_lt_size - omega - have hsum_lt_2size : x.toNat + y.toNat < 2 * UInt64.size := by - nlinarith [UInt64.toNat_lt_size x, UInt64.toNat_lt_size y] - have hlo_eq : lo.toNat = x.toNat + y.toNat - UInt64.size := by - rw [show lo = x + y by rfl, UInt64.toNat_add] - rw [Nat.mod_eq_sub_mod (show x.toNat + y.toNat ≥ UInt64.size by - exact hsum_ge_size)] - rw [Nat.mod_eq_of_lt] - omega - rw [hlo_eq] - change x.toNat + y.toNat - UInt64.size + UInt64.size < - 2 * Goldilocks.fieldSize - have hsum_lt_field : x.toNat + y.toNat < 2 * Goldilocks.fieldSize := by - omega + (addRaw x y).toNat = (x.toNat + y.toNat) % Goldilocks.fieldSize := by + have hx64 := UInt64.toNat_lt_size x + have hy64 := UInt64.toNat_lt_size y + simp only [addRaw] + split_ifs with h <;> + simp only [UInt64.lt_iff_toNat_lt, UInt64.toNat_sub, negModulus_toNat, modulus_toNat, + UInt64.size, Goldilocks.fieldSize] at * <;> omega - · simp only [hcarry] - exact uint64_toNat_lt_two_fieldSize lo - -/-- The wrapped word and carry produced by native addition reconstruct the exact Nat sum. -/ -theorem addWithCarry_value (x y : UInt64) : - let lo := x + y - let carry := decide (lo < x) - lo.toNat + (if carry then UInt64.size else 0) = x.toNat + y.toNat := by - intro lo carry - by_cases hcarry : carry - · simp only [hcarry] - have hsum_ge_size : UInt64.size ≤ x.toNat + y.toNat := by - by_contra hnot - have hsum_lt_size : x.toNat + y.toNat < UInt64.size := - Nat.lt_of_not_ge hnot - have hlo_eq : lo.toNat = x.toNat + y.toNat := by - rw [show lo = x + y by rfl, UInt64.toNat_add] - exact Nat.mod_eq_of_lt hsum_lt_size - have hlo_lt_x : lo.toNat < x.toNat := by - simpa [carry, UInt64.lt_iff_toNat_lt] using hcarry - omega - have hsum_lt_2size : x.toNat + y.toNat < 2 * UInt64.size := by - nlinarith [UInt64.toNat_lt_size x, UInt64.toNat_lt_size y] - have hlo_eq : lo.toNat = x.toNat + y.toNat - UInt64.size := by - rw [show lo = x + y by rfl, UInt64.toNat_add] - rw [Nat.mod_eq_sub_mod (show x.toNat + y.toNat ≥ UInt64.size by - exact hsum_ge_size)] - rw [Nat.mod_eq_of_lt] - omega - rw [hlo_eq] - change x.toNat + y.toNat - UInt64.size + UInt64.size = - x.toNat + y.toNat - omega - · simp only [hcarry] - have hnot_lo_lt_x : ¬lo.toNat < x.toNat := by - intro hlo_lt_x - apply hcarry - simpa [carry, UInt64.lt_iff_toNat_lt] using hlo_lt_x - have hsum_lt_size : x.toNat + y.toNat < UInt64.size := by - by_contra hnot - have hsum_ge_size : UInt64.size ≤ x.toNat + y.toNat := - Nat.le_of_not_gt hnot - have hsum_lt_2size : x.toNat + y.toNat < 2 * UInt64.size := by - nlinarith [UInt64.toNat_lt_size x, UInt64.toNat_lt_size y] - have hlo_eq : lo.toNat = x.toNat + y.toNat - UInt64.size := by - rw [show lo = x + y by rfl, UInt64.toNat_add] - rw [Nat.mod_eq_sub_mod (show x.toNat + y.toNat ≥ UInt64.size by - exact hsum_ge_size)] - rw [Nat.mod_eq_of_lt] - omega - have hy_lt_size : y.toNat < UInt64.size := UInt64.toNat_lt_size y - omega - rw [show lo = x + y by rfl, UInt64.toNat_add] - exact Nat.mod_eq_of_lt hsum_lt_size + +/-- Raw addition of canonical words returns a canonical representative. -/ +theorem addRaw_lt {x y : UInt64} (hx : x.toNat < Goldilocks.fieldSize) + (hy : y.toNat < Goldilocks.fieldSize) : + (addRaw x y).toNat < Goldilocks.fieldSize := by + rw [addRaw_toNat hx hy] + exact Nat.mod_lt _ fieldSize_pos + +/-- Raw addition agrees with canonical-field addition on canonical words. -/ +theorem addRaw_cast {x y : UInt64} (hx : x.toNat < Goldilocks.fieldSize) + (hy : y.toNat < Goldilocks.fieldSize) : + ((addRaw x y).toNat : Goldilocks.Field) = + (x.toNat : Goldilocks.Field) + (y.toNat : Goldilocks.Field) := by + rw [addRaw_toNat hx hy, ZMod.natCast_mod, Nat.cast_add] /-- Raw negation returns a canonical representative when given one. -/ theorem negRaw_lt (x : UInt64) (hx : x.toNat < Goldilocks.fieldSize) : @@ -879,5 +606,52 @@ theorem subRaw_cast (x y : UInt64) rw [uint64_cast_eq_negModulus] ring +/-! ## Lazy exponentiation -/ + + +/-- The lazy ladder computes `acc * x^n`. -/ +theorem powLazy_cast (acc x : UInt64) (n : Nat) : + ((powLazy acc x n).toNat : Goldilocks.Field) = + (acc.toNat : Goldilocks.Field) * (x.toNat : Goldilocks.Field) ^ n := by + induction n using Nat.strong_induction_on generalizing acc x with + | _ n ih => + rw [powLazy] + by_cases hn : n = 0 + · simp [hn] + · rw [dite_eq_right hn] + have hlt : n / 2 < n := Nat.div_lt_self (Nat.pos_of_ne_zero hn) (by decide) + rw [ih (n / 2) hlt, mulLazy_cast, ← pow_two, ← pow_mul] + have hsplit : n = 2 * (n / 2) + n % 2 := (Nat.div_add_mod n 2).symm + by_cases hodd : n % 2 = 1 + · rw [ite_eq_left hodd, mulLazy_cast] + conv_rhs => rw [hsplit, hodd, pow_succ] + ring + · rw [ite_eq_right hodd] + have hev : n % 2 = 0 := by omega + conv_rhs => rw [hsplit, hev, Nat.add_zero] + +/-- Lazy repeated squaring computes `x^(2^n)`. -/ +@[simp] +theorem squareNLazy_cast (x : UInt64) (n : Nat) : + ((squareNLazy x n).toNat : Goldilocks.Field) = (x.toNat : Goldilocks.Field) ^ (2 ^ n) := by + induction n generalizing x with + | zero => + unfold squareNLazy + simp + | succ n ih => + unfold squareNLazy + rw [ih, mulLazy_cast, ← pow_two, ← pow_mul] + congr 1 + rw [Nat.pow_succ] + omega + +/-- The lazy chain computes the Fermat exponent `p - 2`. -/ +theorem invLazy_cast (x : UInt64) : + ((invLazy x).toNat : Goldilocks.Field) = + (x.toNat : Goldilocks.Field) ^ (Goldilocks.fieldSize - 2) := by + unfold invLazy + simp only [mulLazy_cast, squareNLazy_cast] + ring_nf + end Fast end Goldilocks diff --git a/CompPoly/Fields/KoalaBear/FastExt4.lean b/CompPoly/Fields/KoalaBear/FastExt4.lean new file mode 100644 index 00000000..449ceca8 --- /dev/null +++ b/CompPoly/Fields/KoalaBear/FastExt4.lean @@ -0,0 +1,45 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Extension.Map +public import CompPoly.Fields.KoalaBear.Ext4 +public import CompPoly.Fields.KoalaBear.Fast +public import CompPoly.Fields.Montgomery.Native32Ext4 + +/-! +# The degree-4 extension of KoalaBear over the fast carrier + +`KoalaBear.Fast.Ext4` is `Montgomery.Native32.Ext4` for KoalaBear and the binomial `X^4 - 3`, the +same field as Plonky3's `BinomialExtensionField`. Multiplication by `W = 3` is +`x + x + x`. Irreducibility is the spec certificate `KoalaBear.ext4Params_poly_irreducible`, +transferred from `KoalaBear.Field` to the fast carrier along `ringEquiv`. +-/ + +@[expose] public section + +namespace KoalaBear.Fast + +open CompPoly.Extension Montgomery.Native32 Polynomial + +/-- `X^4 - 3` is irreducible over the fast carrier. -/ +theorem irreducible_X_pow_four_sub_three : Irreducible (X ^ 4 - C 3 : Field[X]) := by + have h := irreducible_map_ringEquiv ringEquiv.symm ext4Params_poly_irreducible + rwa [BinomialParams.poly, ext4Params_d, ext4Params_W, Polynomial.map_sub, Polynomial.map_pow, + map_X, map_C, map_ofNat] at h + +/-- KoalaBear's quartic extension is `X^4 - 3`. Inlined, so that specialized code multiplies by +`W` directly. -/ +@[inline] instance instBinomial4 : Binomial4 KoalaBear.fieldSize where + W := 3 + mulW x := x + x + x + mulW_eq x := by ring + irreducible := irreducible_X_pow_four_sub_three + +/-- The degree-4 extension of KoalaBear with fast coefficients. -/ +abbrev Ext4 : Type := Montgomery.Native32.Ext4 KoalaBear.fieldSize + +end KoalaBear.Fast diff --git a/CompPoly/Fields/KoalaBear/InterpolateCoset.lean b/CompPoly/Fields/KoalaBear/InterpolateCoset.lean new file mode 100644 index 00000000..6ac3d55f --- /dev/null +++ b/CompPoly/Fields/KoalaBear/InterpolateCoset.lean @@ -0,0 +1,33 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.KoalaBear.FastExt4 +public import CompPoly.Fields.Montgomery.InterpolateCosetPacked + +/-! +# Compiled packed coset interpolation over KoalaBear + +The `PackedInterpolation` instance for KoalaBear's quartic extension. Its `run` applies the generic +`InterpolatePacked.run` at KoalaBear's modulus, so the compiler specializes the loops here, in the +CPU-targeted `CompPolyInterpolateNative` library. +-/ + +@[expose] public section + +namespace KoalaBear.Fast + +open Montgomery.Native32 + +set_option maxHeartbeats 1000000 in +/-- Packed coset interpolation specialized to KoalaBear. -/ +def interpolateCosetRun (logN : ℕ) (ω s : Field) (width : ℕ) (evals : ByteArray) (z : Ext4) + (logTasks : ℕ) : Array Ext4 := + InterpolatePacked.run logN ω s width evals z logTasks + +instance : PackedInterpolation KoalaBear.fieldSize := ⟨interpolateCosetRun, rfl⟩ + +end KoalaBear.Fast diff --git a/CompPoly/Fields/Montgomery/InterpolateCoset.lean b/CompPoly/Fields/Montgomery/InterpolateCoset.lean new file mode 100644 index 00000000..096f47b4 --- /dev/null +++ b/CompPoly/Fields/Montgomery/InterpolateCoset.lean @@ -0,0 +1,230 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Montgomery.Native32Ext4 +public import CompPoly.Data.List.BatchInv +public import CompPoly.Univariate.NTT.CosetBarycentric + +/-! +# Evaluating polynomials from coset values at an extension point + +`Montgomery.Native32.interpolateCoset` is Plonky3's `interpolate_coset` for a 32-bit Montgomery +field `FastField p` and its quartic binomial extension `Ext4 p`. Given a row-major `2^logN × width` +matrix whose column `j` holds the values of a polynomial `pⱼ` of degree below `2^logN` on the coset +`s · ⟨ω⟩`, it returns `pⱼ(z)` for every column, for a point `z` of the extension outside the coset: + +`pⱼ(z) = (zⁿ - sⁿ) / (n sⁿ) · ∑ᵢ xᵢ / (z - xᵢ) · pⱼ(xᵢ)`, with `xᵢ = s ωⁱ`. + +The weights `xᵢ / (z - xᵢ)` are shared by all columns and need one batch inversion. +`toSpec_interpolateCoset` proves the result equal to `aeval z pⱼ`. +-/ + +@[expose] public section + +namespace Montgomery.Native32.Ext4 + +open CompPoly CompPoly.Extension + +variable {p : ℕ} [Mont32Field p] [Binomial4 p] + +/-- `z ^ 2 ^ k`, by squaring. -/ +def powTwo (z : Ext4 p) : ℕ → Ext4 p + | 0 => z + | k + 1 => powTwo (z * z) k + +theorem toSpec_powTwo (z : Ext4 p) (k : ℕ) : toSpec (powTwo z k) = toSpec z ^ 2 ^ k := by + induction k generalizing z with + | zero => rw [powTwo, pow_zero, pow_one] + | succ k ih => + rw [powTwo, ih, toSpec_mul] + apply Ext.toQuot_injective + rw [Ext.toQuot_pow, Ext.toQuot_pow, Ext.toQuot_mul, ← pow_two, ← pow_mul, Nat.pow_succ'] + +theorem toSpec_foldl_add (n : ℕ) (g : Fin n → Ext4 p) (a : Ext4 p) : + toSpec (Fin.foldl n (fun acc i ↦ acc + g i) a) = toSpec a + ∑ i, toSpec (g i) := by + induction n generalizing a with + | zero => simp only [Fin.foldl_zero, Finset.univ_eq_empty, Finset.sum_empty, add_zero] + | succ n ih => + rw [Fin.foldl_succ, ih, Fin.sum_univ_succ, toSpec_add, add_assoc] + +end Montgomery.Native32.Ext4 + +namespace Montgomery.Native32 + +open CompPoly CompPoly.Extension Polynomial + +variable {p : ℕ} [Mont32Field p] [Binomial4 p] + +/-- The nodes `s, s ω, s ω², …` of a coset. -/ +@[specialize] +def cosetNodes (ω s : FastField p) (n : ℕ) : List (FastField p) := List.prefixProducts s + (List.replicate n ω) + +/-- The weights `xᵢ / (z - xᵢ)` of the coset nodes, with one batch inversion. -/ +@[specialize] +def cosetWeights (ω s : FastField p) (n : ℕ) (z : Ext4 p) : List (Ext4 p) := + List.zipWith Ext4.smul (cosetNodes ω s n) + (List.batchInv ((cosetNodes ω s n).map fun x ↦ z - Ext4.ofBase x)) + +/-- The scale `(zⁿ - sⁿ) / (n sⁿ)` for `n = 2^logN`. -/ +@[specialize] +def cosetFactor (logN : ℕ) (s : FastField p) (z : Ext4 p) : Ext4 p := + let sn := Montgomery.Native32.pow s (2 ^ logN) + (Ext4.powTwo z logN - Ext4.ofBase sn) * (Ext4.ofBase (((2 ^ logN : ℕ) : FastField p) * sn))⁻¹ + +/-- Evaluate each column of a row-major `2^logN × width` matrix of values on the coset `s · ⟨ω⟩` +at `z`. -/ +@[specialize] +def interpolateCoset (logN : ℕ) (ω s : FastField p) (width : ℕ) (evals : Array (FastField p)) + (z : Ext4 p) : + Array (Ext4 p) := + let n := 2 ^ logN + let weights := (cosetWeights ω s n z).toArray + let factor := cosetFactor logN s z + Array.ofFn (n := width) fun j ↦ + factor * Fin.foldl n + (fun acc i ↦ acc + Ext4.smul (evals.getD (i * width + j) 0) (weights.getD i 0)) 0 + +/-! ### Correctness -/ + +section Nodes + +variable {M : Type*} [CommMonoid M] + +theorem getElem_prefixProducts_replicate (q w : M) (n i : ℕ) + (hi : i < (List.prefixProducts q (List.replicate n w)).length) : + (List.prefixProducts q (List.replicate n w))[i] = q * w ^ i := by + induction n generalizing q i with + | zero => simp [List.prefixProducts] at hi + | succ n ih => + cases i with + | zero => simp only [List.replicate_succ, List.prefixProducts, List.getElem_cons_zero, + pow_zero, mul_one] + | succ i => + simp only [List.replicate_succ, List.prefixProducts, List.getElem_cons_succ] + rw [ih, pow_succ', mul_assoc] + +end Nodes + +omit [Binomial4 p] in +theorem length_cosetNodes (ω s : FastField p) (n : ℕ) : (cosetNodes ω s n).length = n := by + rw [cosetNodes, List.length_prefixProducts, List.length_replicate] + +omit [Binomial4 p] in +theorem getElem_cosetNodes (ω s : FastField p) (n i : ℕ) (hi : i < (cosetNodes ω s n).length) : + (cosetNodes ω s n)[i] = s * ω ^ i := + getElem_prefixProducts_replicate s ω n i hi + +theorem length_cosetWeights (ω s : FastField p) (n : ℕ) (z : Ext4 p) : + (cosetWeights ω s n z).length = n := by + rw [cosetWeights, List.length_zipWith, List.length_batchInv, List.length_map, length_cosetNodes, + min_self] + +theorem toSpec_getElem_cosetWeights (ω s : FastField p) (n : ℕ) (z : Ext4 p) + (hz : ∀ i < n, Ext4.toSpec z ≠ Ext.ofBase (s * ω ^ i)) (i : ℕ) + (hi : i < (cosetWeights ω s n z).length) : + Ext4.toSpec (cosetWeights ω s n z)[i] = + Ext.ofBase (s * ω ^ i) * (Ext4.toSpec z - Ext.ofBase (s * ω ^ i))⁻¹ := by + have hin : i < n := by rwa [length_cosetWeights] at hi + have hnode : i < (cosetNodes ω s n).length := by rw [length_cosetNodes]; exact hin + set diffs := (cosetNodes ω s n).map fun x ↦ z - Ext4.ofBase x + have hdiffs : ∀ k (hk : k < diffs.length), + Ext4.toSpec diffs[k] = Ext4.toSpec z - Ext.ofBase (s * ω ^ k) := by + intro k hk + simp only [diffs, List.getElem_map, Ext4.toSpec_sub, Ext4.toSpec_ofBase] + rw [getElem_cosetNodes] + have hne : ∀ x ∈ diffs.map Ext4.toSpec, x ≠ 0 := by + intro x hx + obtain ⟨k, hk, rfl⟩ := List.mem_iff_getElem.mp hx + rw [List.getElem_map, hdiffs, sub_ne_zero] + apply hz + simpa [diffs, length_cosetNodes] using hk + have hb := List.batchInv_map Ext4.toSpec Ext4.toSpec_mul Ext4.toSpec_inv Ext4.toSpec_one diffs + rw [List.batchInv_eq _ hne] at hb + have hdi : i < diffs.length := by simpa [diffs] using hnode + have hbi : i < (List.batchInv diffs).length := by rwa [List.length_batchInv] + have e := congrArg (·[i]?) hb + simp only [List.getElem?_map, List.getElem?_eq_getElem hdi, List.getElem?_eq_getElem hbi, + Option.map_some, Option.some.injEq] at e + simp only [cosetWeights, List.getElem_zipWith, Ext4.toSpec_smul, Algebra.smul_def, + Ext.algebraMap_eq_ofBase, getElem_cosetNodes] + rw [← e, hdiffs] + +theorem size_interpolateCoset (logN : ℕ) (ω s : FastField p) (width : ℕ) (evals : Array + (FastField p)) + (z : Ext4 p) : (interpolateCoset logN ω s width evals z).size = width := by + simp only [interpolateCoset, Array.size_ofFn] + +/-- Each output of `interpolateCoset` is the scale times a sum over the rows, for `z` off the +nodes. -/ +theorem toSpec_getElem_interpolateCoset (logN : ℕ) (ω s : FastField p) (width : ℕ) + (evals : Array (FastField p)) (z : Ext4 p) + (hz : ∀ i < 2 ^ logN, Ext4.toSpec z ≠ Ext.ofBase (s * ω ^ i)) (j : ℕ) (hj : j < width) : + Ext4.toSpec ((interpolateCoset logN ω s width evals z)[j]'(by + rw [size_interpolateCoset]; exact hj)) = + Ext4.toSpec (cosetFactor logN s z) * ∑ i ∈ Finset.range (2 ^ logN), + evals.getD (i * width + j) 0 • + (Ext.ofBase (s * ω ^ i) * (Ext4.toSpec z - Ext.ofBase (s * ω ^ i))⁻¹) := by + simp only [interpolateCoset] + rw [Array.getElem_ofFn, Ext4.toSpec_mul, Ext4.toSpec_foldl_add, Ext4.toSpec_zero, zero_add, + Fin.sum_univ_eq_sum_range (fun i ↦ Ext4.toSpec (Ext4.smul (evals.getD (i * width + j) 0) + ((cosetWeights ω s (2 ^ logN) z).toArray.getD i 0)))] + refine congrArg (_ * ·) (Finset.sum_congr rfl fun i hi ↦ ?_) + have hi' : i < (cosetWeights ω s (2 ^ logN) z).length := by + rw [length_cosetWeights]; exact Finset.mem_range.mp hi + have hw : (cosetWeights ω s (2 ^ logN) z).toArray.getD i 0 = + (cosetWeights ω s (2 ^ logN) z)[i] := by + rw [Array.getD_eq_getD_getElem?, List.getElem?_toArray, List.getElem?_eq_getElem hi', + Option.getD_some] + rw [hw, Ext4.toSpec_smul, toSpec_getElem_cosetWeights ω s _ z hz i hi'] + +/-- A point whose `n`-th power differs from `sⁿ` is off every node of the coset `s · ⟨ω⟩`. -/ +theorem toSpec_ne_cosetNode (D : CPolynomial.NTT.Domain (FastField p)) (s : FastField p) + (z : Ext4 p) + (hz : Ext4.toSpec z ^ D.n ≠ algebraMap (FastField p) (Ext4.Spec p) s ^ D.n) : + ∀ i < D.n, Ext4.toSpec z ≠ Ext.ofBase (s * D.omega ^ i) := by + intro i hi h + apply hz + have := D.cosetNode_pow_n (E := Ext4.Spec p) s ⟨i, hi⟩ + rw [h, ← Ext.algebraMap_eq_ofBase, ← this] + simp only [CPolynomial.NTT.Domain.cosetNode, CPolynomial.NTT.Domain.node] + +/-- **Correctness of `interpolateCoset`.** If column `j` holds the values of `pⱼ`, of degree below +`n`, on the coset `s · ⟨ω⟩`, and `z` lies outside the coset, then output `j` is `pⱼ(z)`. -/ +theorem toSpec_interpolateCoset (D : CPolynomial.NTT.Domain (FastField p)) (s : FastField p) + (hs : s ≠ 0) + (width : ℕ) (evals : Array (FastField p)) (z : Ext4 p) + (hz : Ext4.toSpec z ^ D.n ≠ algebraMap (FastField p) (Ext4.Spec p) s ^ D.n) + (f : Fin width → (FastField p)[X]) (hf : ∀ j, (f j).degree < D.n) + (hevals : ∀ (i : D.Idx) (j : Fin width), + evals.getD (i * width + j) 0 = (f j).eval (s * D.node i)) + (j : Fin width) : + Ext4.toSpec ((interpolateCoset D.logN D.omega s width evals z)[j.val]'(by + rw [size_interpolateCoset]; exact j.isLt)) = aeval (Ext4.toSpec z) (f j) := by + have hz' := toSpec_ne_cosetNode D s z hz + rw [CPolynomial.NTT.Domain.aeval_eq_cosetBarycentric D hs (f j) (hf j) _ hz] + simp only [interpolateCoset, cosetFactor] + rw [Array.getElem_ofFn, Ext4.toSpec_mul, Ext4.toSpec_foldl_add, Ext4.toSpec_zero, zero_add, + Ext4.toSpec_mul, Ext4.toSpec_sub, Ext4.toSpec_powTwo, Ext4.toSpec_ofBase, Ext4.toSpec_inv, + Ext4.toSpec_ofBase] + have hsn : Montgomery.Native32.pow s (2 ^ D.logN) = s ^ D.n := pow_eq_npow s _ + simp only [hsn] + simp only [← Ext.algebraMap_eq_ofBase, map_mul, map_pow, map_natCast] + refine congrArg₂ (· * ·) rfl (Finset.sum_congr rfl fun i _ ↦ ?_) + have hi : (i : ℕ) < (cosetWeights D.omega s (2 ^ D.logN) z).length := by + rw [length_cosetWeights]; exact i.isLt + have hw : (cosetWeights D.omega s (2 ^ D.logN) z).toArray.getD i 0 = + (cosetWeights D.omega s (2 ^ D.logN) z)[(i : ℕ)] := by + rw [Array.getD_eq_getD_getElem?, List.getElem?_toArray, List.getElem?_eq_getElem hi, + Option.getD_some] + rw [hw, Ext4.toSpec_smul, Algebra.smul_def, + toSpec_getElem_cosetWeights D.omega s (2 ^ D.logN) z hz' i hi, hevals i j] + simp only [Ext.algebraMap_eq_ofBase, CPolynomial.NTT.Domain.cosetNode, + CPolynomial.NTT.Domain.node] + ring + +end Montgomery.Native32 diff --git a/CompPoly/Fields/Montgomery/InterpolateCosetPacked.lean b/CompPoly/Fields/Montgomery/InterpolateCosetPacked.lean new file mode 100644 index 00000000..8f3c57a2 --- /dev/null +++ b/CompPoly/Fields/Montgomery/InterpolateCosetPacked.lean @@ -0,0 +1,432 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Montgomery.InterpolateCoset +public import CompPoly.Univariate.NTTFast.Packed.StorageLemmas + +/-! +# Coset interpolation over packed words + +`Montgomery.Native32.interpolateCosetPacked` is `interpolateCoset` on a row-major matrix of +Montgomery words in a `ByteArray`, Plonky3's layout, for any `Mont32Field p` with a quartic +binomial extension. The implementation `InterpolatePacked.run` is generic and `@[specialize]`d; +`interpolateCosetPacked` calls it through the class `PackedInterpolation p`, whose instances +compile it once for a concrete field (`KoalaBear/InterpolateCoset.lean`, +`BabyBear/InterpolateCoset.lean`). + +* **Base-field weights.** For `t = z₀ - x`, the inverse of `z - x` is `(b₀ + t b₁ + t² b₂ + t³) / N` + with extension constants `bₖ` that depend on `z` only and the norm `N = N(t)` of `z - x`. So the + weight `x / (z - x)` is `∑ₖ vₖ bₖ` for the base-field values `vₖ = x tᵏ / N`, and every column + sum is `∑ₖ Sₖ bₖ` for the base-field sums `Sₖ = ∑ᵢ eᵢ vᵢₖ`. +* **Blocks.** Rows are processed in blocks: one pass stores each row's node, `t²`, norm and + prefix product of norms, one base-field inversion inverts the block's product, and a backward + pass overwrites each row with its four weights. The column sums then read the block back. +* **Lazy sums.** A column sum accumulates unreduced 64-bit products, with one conditional + subtraction per product and one Montgomery reduction per block; four columns share each row's + weights. +* **Tasks.** Row ranges run on parallel tasks, the caller taking the lowest. + +`interpolateCosetPacked_eq` proves the result equal to `interpolateCoset` on the decoded words for +every point off the coset. +-/ + +@[expose] public section + +namespace Montgomery.Native32.Ext4 + +open Binomial4 (mulW) + +variable {p : ℕ} [Mont32Field p] [Binomial4 p] + +/-- `W z₂² - 2 W z₁ z₃`: `g₀(z - x) = t² + k₀` for `t = z₀ - x`. -/ +@[inline] def normK0 (z : Ext4 p) : FastField p := + mulW (z.c2 * z.c2) - mulW (z.c1 * z.c3 + z.c1 * z.c3) + +/-- `-z₁² - W z₃²`: `g₁(z - x) = 2 z₂ t + k₁` for `t = z₀ - x`. -/ +@[inline] def normK1 (z : Ext4 p) : FastField p := -(z.c1 * z.c1) - mulW (z.c3 * z.c3) + +/-- The norm `g₀² - W g₁²` of `z - x` from `t = z₀ - x`, `t2 = t²` and the constants `k₀`, `k₁` +and `2 z₂`. -/ +@[inline] def normAt (k0 k1 z2x2 t t2 : FastField p) : FastField p := + let g0 := t2 + k0 + let g1 := z2x2 * t + k1 + g0 * g0 - mulW (g1 * g1) + +/-- `2 W a`. -/ +@[inline] def twoW (a : FastField p) : FastField p := mulW (a + a) + +/-- `b₀` in `σ(z - x) (g₀ - g₁ X²) = b₀ + t b₁ + t² b₂ + t³`. -/ +def adj0 (z : Ext4 p) : Ext4 p := + ⟨-mulW (z.c2 * normK1 z), mulW (z.c3 * normK1 z) - z.c1 * normK0 z, z.c2 * normK0 z, + z.c1 * normK1 z - z.c3 * normK0 z⟩ + +/-- `b₁` in `σ(z - x) (g₀ - g₁ X²) = b₀ + t b₁ + t² b₂ + t³`. -/ +def adj1 (z : Ext4 p) : Ext4 p := + ⟨normK0 z - twoW (z.c2 * z.c2), twoW (z.c2 * z.c3), -normK1 z, z.c1 * z.c2 + z.c1 * z.c2⟩ + +/-- `b₂` in `σ(z - x) (g₀ - g₁ X²) = b₀ + t b₁ + t² b₂ + t³`. -/ +def adj2 (z : Ext4 p) : Ext4 p := ⟨0, -z.c1, -z.c2, -z.c3⟩ + +end Montgomery.Native32.Ext4 + +namespace Montgomery.Native32 + +open CompPoly +open CompPoly.CPolynomial.NTTFast.Packed + +variable {p : ℕ} [Mont32Field p] + +/-- Four base-field sums: a column sum's coordinates over `b₀, b₁, b₂, 1`. -/ +structure Quad (p : ℕ) [Mont32Field p] where + s0 : FastField p + s1 : FastField p + s2 : FastField p + s3 : FastField p + +namespace Quad + +instance : Zero (Quad p) := ⟨⟨0, 0, 0, 0⟩⟩ +instance : Inhabited (Quad p) := ⟨0⟩ +instance : Add (Quad p) := ⟨fun a b ↦ ⟨a.s0 + b.s0, a.s1 + b.s1, a.s2 + b.s2, a.s3 + b.s3⟩⟩ + +/-- `s₀ b₀ + s₁ b₁ + s₂ b₂ + s₃` for the point `z`. -/ +def combine [Binomial4 p] (z : Ext4 p) (q : Quad p) : Ext4 p := + Ext4.smul q.s0 (Ext4.adj0 z) + Ext4.smul q.s1 (Ext4.adj1 z) + Ext4.smul q.s2 (Ext4.adj2 z) + + Ext4.ofBase q.s3 + +end Quad + +/-- The words of a byte array, as residues. -/ +def decodeWords (p : ℕ) [Mont32Field p] (b : ByteArray) : Array (FastField p) := + Array.ofFn (n := b.size / 4) fun k ↦ ofWordMod (b.getUInt32LE! (4 * k.val)) + +namespace InterpolatePacked + +/-- A lazy column-sum accumulator. -/ +abbrev Acc (p : ℕ) [Mont32Field p] : Type := LazyAcc p + +/-- Rows per block. -/ +def blockRows : ℕ := 512 + +/-- A stored residue; a word from the modulus up reads as zero. -/ +@[inline] def wordField (p : ℕ) [P : Mont32Field p] (x : UInt32) : FastField p := + if h : x < P.modulus32 then + ⟨x, by rw [UInt32.lt_iff_toNat_lt, Mont32Field.modulus32_toNat] at h; exact h⟩ + else 0 + +theorem wordField_val (x : FastField p) : wordField p x.val = x := by + have hx : x.val < Mont32Field.modulus32 p := by + rw [UInt32.lt_iff_toNat_lt, Mont32Field.modulus32_toNat]; exact x.property + simp only [wordField, hx, ↓reduceDIte]; rfl + +/-- Word `i + o` of a buffer of residues. -/ +@[inline] def readField (p : ℕ) [Mont32Field p] (b : @& ByteArray) (i o : USize) + (h : 4 * (i.toNat + o.toNat) + 3 < b.size ∧ b.size < USize.size) : FastField p := + wordField p (Native.readUOffset b i o h) + +/-- `n` rows from row `i` of a block, from node `x`: store `x`, `t²`, the norm `N` of `z - x` and +the product `acc` of the norms before it at words `4 i, …, 4 i + 3`. Returns the buffer, the +product of all norms and the next node. -/ +@[specialize] +def forward [Binomial4 p] + (ω z0 k0 k1 z2x2 : FastField p) : ℕ → USize → FastField p → FastField p → ByteArray → + ByteArray × FastField p × FastField p + | 0, _, x, acc, buf => (buf, acc, x) + | n + 1, i, x, acc, buf => + let t := z0 - x + let t2 := t * t + let nm := Ext4.normAt k0 k1 z2x2 t t2 + forward ω z0 k0 k1 z2x2 n (i + 1) (x * ω) (acc * nm) + (Native.storeWords buf (16 * i) 4 x.val t2.val nm.val acc.val 0 0 0 0 0 0 0 0 0 0 0 0) + +theorem size_forward [Binomial4 p] (ω z0 k0 k1 z2x2 : FastField p) (n : ℕ) (i : USize) + (x acc : FastField p) + (buf : ByteArray) : (forward ω z0 k0 k1 z2x2 n i x acc buf).1.size = buf.size := by + induction n generalizing i x acc buf with + | zero => rfl + | succ n ih => rw [forward, ih, Native.size_storeWords] + +theorem usize_four_mul (j : USize) (n : ℕ) (h : 16 * j.toNat ≤ n) (hn : n < USize.size) : + (4 * j).toNat = 4 * j.toNat := by + have hsize : (2 : ℕ) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_mul, Native.usize_numeral 4 (by decide), hsize] + exact Nat.mod_eq_of_lt (by omega) + +theorem usize_add_lt (a b : USize) (h : a.toNat + b.toNat < USize.size) : + (a + b).toNat = a.toNat + b.toNat := by + have hsize : (2 : ℕ) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_add, hsize, Nat.mod_eq_of_lt h] + +/-- `n` rows back from row `i` of a block: `inv` inverts the product of the norms before row `i`. +Row `i - 1`'s norm inverse is `inv` times its prefix product; its words become its weights +`u, u t, u t², u t³` for `u = x / N`, and `inv` times its norm inverts the earlier prefix. -/ +@[specialize] +def backward (z0 : FastField p) : (n : ℕ) → (i : USize) → FastField p → (buf : ByteArray) → + 16 * i.toNat ≤ buf.size ∧ buf.size < USize.size ∧ n ≤ i.toNat → ByteArray + | 0, _, _, buf, _ => buf + | n + 1, i, inv, buf, h => + have hj : (i - 1).toNat = i.toNat - 1 := by + rw [USize.toNat_sub_of_le _ _ (by rw [USize.le_iff_toNat_le]; simp; omega)]; simp + have h4 := usize_four_mul (i - 1) buf.size (by omega) h.2.1 + let x := readField p buf (4 * (i - 1)) 0 (by + rw [h4, Native.usize_numeral 0 (by decide)]; exact ⟨by omega, h.2.1⟩) + let t2 := readField p buf (4 * (i - 1)) 1 (by + rw [h4, Native.usize_numeral 1 (by decide)]; exact ⟨by omega, h.2.1⟩) + let nm := readField p buf (4 * (i - 1)) 2 (by + rw [h4, Native.usize_numeral 2 (by decide)]; exact ⟨by omega, h.2.1⟩) + let pre := readField p buf (4 * (i - 1)) 3 (by + rw [h4, Native.usize_numeral 3 (by decide)]; exact ⟨by omega, h.2.1⟩) + let u := x * (inv * pre) + let v1 := u * (z0 - x) + backward z0 n (i - 1) (inv * nm) + (Native.storeWords buf (16 * (i - 1)) 4 u.val v1.val (u * t2).val (v1 * t2).val + 0 0 0 0 0 0 0 0 0 0 0 0) + (by rw [Native.size_storeWords]; omega) + +theorem size_backward (z0 : FastField p) (n : ℕ) (i : USize) (inv : FastField p) (buf : ByteArray) + (h) : + (backward z0 n i inv buf h).size = buf.size := by + induction n generalizing i inv buf with + | zero => rfl + | succ n ih => rw [backward, ih, Native.size_storeWords] + +/-- One column of `n` block rows: `e` steps through the column's words by `width`, `w` through +the rows' weights. -/ +@[specialize] +def dot1 (p : ℕ) [Mont32Field p] (evals buf : @& ByteArray) (width : USize) : (n : ℕ) → + (e w : USize) → + (a0 a1 a2 a3 : Acc p) → + (∀ r < n, 4 * (e.toNat + r * width.toNat) + 3 < evals.size) ∧ evals.size < USize.size ∧ + 4 * (w.toNat + 4 * n) ≤ buf.size ∧ buf.size < USize.size → Quad p + | 0, _, _, a0, a1, a2, a3, _ => ⟨a0.result, a1.result, a2.result, a3.result⟩ + | n + 1, e, w, a0, a1, a2, a3, h => + have h0 := Native.usize_numeral 0 (by decide) + have h1 := Native.usize_numeral 1 (by decide) + have h2 := Native.usize_numeral 2 (by decide) + have h3 := Native.usize_numeral 3 (by decide) + have he := h.1 0 (by omega) + let v := Native.readUOffset evals e 0 (by rw [h0]; simp at he; exact ⟨by omega, h.2.1⟩) + let w0 := readField p buf w 0 (by rw [h0]; exact ⟨by omega, h.2.2.2⟩) + let w1 := readField p buf w 1 (by rw [h1]; exact ⟨by omega, h.2.2.2⟩) + let w2 := readField p buf w 2 (by rw [h2]; exact ⟨by omega, h.2.2.2⟩) + let w3 := readField p buf w 3 (by rw [h3]; exact ⟨by omega, h.2.2.2⟩) + dot1 p evals buf width n (e + width) (w + 4) (a0.add v w0) (a1.add v w1) (a2.add v w2) + (a3.add v w3) (by + refine ⟨fun r hr ↦ ?_, h.2.1, ?_, h.2.2.2⟩ + · have := h.1 (r + 1) (by omega) + have h1' := h.1 1 (by omega) + rw [Nat.one_mul] at h1' + rw [usize_add_lt _ _ (by have := h.2.1; omega)] + rw [Nat.add_mul, Nat.one_mul] at this + omega + · have hb := h.2.2.2 + have h4 : (4 : USize).toNat = 4 := Native.usize_numeral 4 (by decide) + rw [usize_add_lt _ _ (by rw [h4]; omega), h4] + omega) + +/-- Four adjacent columns of `n` block rows, sharing each row's weights. -/ +@[specialize] +def dot4 (p : ℕ) [Mont32Field p] (evals buf : @& ByteArray) (width : USize) : (n : ℕ) → + (e w : USize) → + (a0 a1 a2 a3 b0 b1 b2 b3 c0 c1 c2 c3 d0 d1 d2 d3 : Acc p) → + (∀ r < n, 4 * (e.toNat + r * width.toNat + 3) + 3 < evals.size) ∧ + evals.size < USize.size ∧ 4 * (w.toNat + 4 * n) ≤ buf.size ∧ buf.size < USize.size → + Quad p × Quad p × Quad p × Quad p + | 0, _, _, a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3, _ => + (⟨a0.result, a1.result, a2.result, a3.result⟩, ⟨b0.result, b1.result, b2.result, b3.result⟩, + ⟨c0.result, c1.result, c2.result, c3.result⟩, ⟨d0.result, d1.result, d2.result, d3.result⟩) + | n + 1, e, w, a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3, h => + have h0 := Native.usize_numeral 0 (by decide) + have h1 := Native.usize_numeral 1 (by decide) + have h2 := Native.usize_numeral 2 (by decide) + have h3 := Native.usize_numeral 3 (by decide) + have he := h.1 0 (by omega) + let v0 := Native.readUOffset evals e 0 (by rw [h0]; simp at he; exact ⟨by omega, h.2.1⟩) + let v1 := Native.readUOffset evals e 1 (by rw [h1]; simp at he; exact ⟨by omega, h.2.1⟩) + let v2 := Native.readUOffset evals e 2 (by rw [h2]; simp at he; exact ⟨by omega, h.2.1⟩) + let v3 := Native.readUOffset evals e 3 (by rw [h3]; simp at he; exact ⟨by omega, h.2.1⟩) + let w0 := readField p buf w 0 (by rw [h0]; exact ⟨by omega, h.2.2.2⟩) + let w1 := readField p buf w 1 (by rw [h1]; exact ⟨by omega, h.2.2.2⟩) + let w2 := readField p buf w 2 (by rw [h2]; exact ⟨by omega, h.2.2.2⟩) + let w3 := readField p buf w 3 (by rw [h3]; exact ⟨by omega, h.2.2.2⟩) + dot4 p evals buf width n (e + width) (w + 4) + (a0.add v0 w0) (a1.add v0 w1) (a2.add v0 w2) (a3.add v0 w3) + (b0.add v1 w0) (b1.add v1 w1) (b2.add v1 w2) (b3.add v1 w3) + (c0.add v2 w0) (c1.add v2 w1) (c2.add v2 w2) (c3.add v2 w3) + (d0.add v3 w0) (d1.add v3 w1) (d2.add v3 w2) (d3.add v3 w3) (by + refine ⟨fun r hr ↦ ?_, h.2.1, ?_, h.2.2.2⟩ + · have := h.1 (r + 1) (by omega) + have h1' := h.1 1 (by omega) + rw [Nat.one_mul] at h1' + rw [usize_add_lt _ _ (by have := h.2.1; omega)] + rw [Nat.add_mul, Nat.one_mul] at this + omega + · have hb := h.2.2.2 + have h4 : (4 : USize).toNat = 4 := Native.usize_numeral 4 (by decide) + rw [usize_add_lt _ _ (by rw [h4]; omega), h4] + omega) + +theorem dot_bound (row b width c r k : ℕ) (hr : r < b) (hc : c + k + 1 ≤ width) : + row * width + c + r * width + k < (row + b) * width := by + have := Nat.mul_le_mul_right width (show r + 1 ≤ b from hr) + rw [Nat.add_mul, Nat.one_mul] at this + rw [Nat.add_mul] + omega + +theorem toUSize_toNat_of_lt (a n : ℕ) (h : a < n) (hn : n < USize.size) : a.toUSize.toNat = a := + USize.toNat_ofNat_of_lt' (by omega) + +/-- The block's evaluation words and weights fit: rows `row, …, row + b - 1` of a +`width`-column matrix in `evals`, and `b` rows of weights in `buf`. -/ +def BlockFits (evals buf : ByteArray) (width row b : ℕ) : Prop := + 4 * ((row + b) * width) ≤ evals.size ∧ evals.size < USize.size ∧ 16 * b ≤ buf.size ∧ + buf.size < USize.size + +theorem blockFits_dot {evals buf : ByteArray} {width row b : ℕ} + (h : BlockFits evals buf width row b) (c k : ℕ) (hc : c + k + 1 ≤ width) : + (∀ r < b, 4 * ((row * width + c).toUSize.toNat + r * width.toUSize.toNat + k) + 3 < + evals.size) ∧ evals.size < USize.size ∧ 4 * ((0 : USize).toNat + 4 * b) ≤ buf.size ∧ + buf.size < USize.size := by + obtain ⟨he, hes, hb, hbs⟩ := h + refine ⟨fun r hr ↦ ?_, hes, by simp only [USize.toNat_zero]; omega, hbs⟩ + have hd := dot_bound row b width c r k hr hc + have hw := dot_bound row b width c 0 k (by omega) hc + rw [Nat.zero_mul, Nat.add_zero] at hw + have hw' : width ≤ (row + b) * width := Nat.le_mul_of_pos_left _ (by omega) + rw [toUSize_toNat_of_lt _ ((row + b) * width) (by omega) (by omega), + toUSize_toNat_of_lt _ ((row + b) * width + 1) (by omega) (by omega)] + omega + +/-- Add the block's sums of the four columns from `c` into `sums`. -/ +@[specialize] +def addColumns4 (p : ℕ) [Mont32Field p] (evals buf : @& ByteArray) (width row b : ℕ) + (hfit : BlockFits evals buf width row b) (c : ℕ) (hc : c + 4 ≤ width) (sums : Array (Quad p)) : + Array (Quad p) := + let a : Acc p := LazyAcc.zero + let q := dot4 p evals buf width.toUSize b (row * width + c).toUSize 0 a a a a a a a a a a a a + a a a a (blockFits_dot hfit c 3 (by omega)) + (((sums.modify c (· + q.1)).modify (c + 1) (· + q.2.1)).modify (c + 2) (· + q.2.2.1)).modify + (c + 3) (· + q.2.2.2) + +/-- Add the block's sums of `g` groups of four columns from column `c` into `sums`. -/ +@[specialize] +def columns4 (p : ℕ) [Mont32Field p] (evals buf : @& ByteArray) (width row b : ℕ) + (hfit : BlockFits evals buf width row b) : + (g c : ℕ) → c + 4 * g ≤ width → Array (Quad p) → Array (Quad p) + | 0, _, _, sums => sums + | g + 1, c, hc, sums => + columns4 p evals buf width row b hfit g (c + 4) (by omega) + (addColumns4 p evals buf width row b hfit c (by omega) sums) + +/-- Add the block's sum of column `c` into `sums`. -/ +@[specialize] +def addColumn (p : ℕ) [Mont32Field p] (evals buf : @& ByteArray) (width row b : ℕ) + (hfit : BlockFits evals buf width row b) (c : ℕ) (hc : c + 1 ≤ width) (sums : Array (Quad p)) : + Array (Quad p) := + let a : Acc p := LazyAcc.zero + sums.modify c (· + dot1 p evals buf width.toUSize b (row * width + c).toUSize 0 a a a a (by + have := blockFits_dot hfit c 0 (by omega) + simp only [Nat.add_zero] at this + exact this)) + +/-- Add the block's sums of `k` single columns from column `c` into `sums`. -/ +@[specialize] +def columns1 (p : ℕ) [Mont32Field p] (evals buf : @& ByteArray) (width row b : ℕ) + (hfit : BlockFits evals buf width row b) : + (k c : ℕ) → c + k ≤ width → Array (Quad p) → Array (Quad p) + | 0, _, _, sums => sums + | k + 1, c, hc, sums => + columns1 p evals buf width row b hfit k (c + 1) (by omega) + (addColumn p evals buf width row b hfit c (by omega) sums) + +/-- Blocks of rows from `row` up to `hi`, `k` of them, from node `x`. -/ +@[specialize] +def blocks [Binomial4 p] (evals : @& ByteArray) (ω z0 k0 k1 z2x2 : FastField p) (width hi : ℕ) : + (k row : ℕ) → FastField p → (buf : ByteArray) → Array (Quad p) → + row ≤ hi ∧ 4 * (hi * width) ≤ evals.size ∧ evals.size < USize.size ∧ + buf.size = 16 * blockRows → Array (Quad p) + | 0, _, _, _, sums, _ => sums + | k + 1, row, x, buf, sums, h => + let b := min blockRows (hi - row) + have hb : b ≤ blockRows := Nat.min_le_left _ _ + have hu := USize.le_size + have hbt : b.toUSize.toNat = b := toUSize_toNat_of_lt b (blockRows + 1) (by + omega) (by simp only [blockRows]; omega) + let f := forward ω z0 k0 k1 z2x2 b 0 x 1 buf + let buf := backward z0 b b.toUSize f.2.1⁻¹ f.1 (by + rw [size_forward, h.2.2.2, hbt]; simp only [blockRows] at hb ⊢ + exact ⟨by omega, by omega, le_refl _⟩) + have hbuf : buf.size = 16 * blockRows := by + simp only [buf, size_backward, f, size_forward, h.2.2.2] + have hfit : BlockFits evals buf width row b := by + have hr : row + b ≤ hi := by have := Nat.min_le_right blockRows (hi - row); omega + have := Nat.mul_le_mul_right width hr + refine ⟨by omega, h.2.2.1, by omega, ?_⟩ + rw [hbuf]; simp only [blockRows]; omega + let sums := columns4 p evals buf width row b hfit (width / 4) 0 (by omega) sums + let sums := columns1 p evals buf width row b hfit (width % 4) (4 * (width / 4)) (by omega) sums + blocks evals ω z0 k0 k1 z2x2 width hi k (row + b) f.2.2 buf sums (by + have := Nat.min_le_right blockRows (hi - row) + exact ⟨by omega, h.2.1, h.2.2.1, hbuf⟩) + +/-- The column sums of rows `lo, …, hi - 1`. -/ +@[specialize] +def leaf [Binomial4 p] (evals : @& ByteArray) (ω s : FastField p) (z : Ext4 p) (width lo hi : ℕ) + (h : lo ≤ hi ∧ 4 * (hi * width) ≤ evals.size ∧ evals.size < USize.size) : Array (Quad p) := + blocks evals ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) width hi + ((hi - lo + blockRows - 1) / blockRows) lo (s * Montgomery.Native32.pow ω lo) + (ByteArray.mk (Array.replicate (16 * blockRows) 0)) (Array.replicate width 0) + ⟨h.1, h.2.1, h.2.2, by simp [ByteArray.size]⟩ + +/-- Add the lower half's sums, computed by the caller, to the upper half's task. -/ +@[noinline, specialize] def join (upper : Task (Array (Quad p))) (low : Unit → Array + (Quad p)) : Array (Quad p) := + let low := low () + Array.zipWith (· + ·) low upper.get + +/-- The column sums of rows `lo, …, hi - 1` over `2 ^ depth` tasks; the caller takes the lowest +range. -/ +@[specialize] +def tree [Binomial4 p] (evals : ByteArray) (ω s : FastField p) (z : Ext4 p) (width : ℕ) (lo hi : ℕ) + (h : lo ≤ hi ∧ 4 * (hi * width) ≤ evals.size ∧ evals.size < USize.size) : ℕ → Array (Quad p) + | 0 => leaf evals ω s z width lo hi h + | depth + 1 => + let mid := lo + (hi - lo) / 2 + have hm : mid ≤ hi := by omega + have := Nat.mul_le_mul_right width hm + join (Task.spawn fun _ ↦ tree evals ω s z width mid hi ⟨by omega, h.2⟩ depth) + (fun _ ↦ tree evals ω s z width lo mid ⟨by omega, by omega, h.2.2⟩ depth) + +/-- `interpolateCoset` on a row-major `2^logN × width` matrix of Montgomery words, over up to +`2 ^ logTasks` tasks. -/ +@[specialize] +def run [Binomial4 p] (logN : ℕ) (ω s : FastField p) (width : ℕ) (evals : ByteArray) + (z : Ext4 p) (logTasks : ℕ) : Array (Ext4 p) := + if h : evals.size = 4 * (2 ^ logN * width) ∧ evals.size < USize.size then + let sums := tree evals ω s z width 0 (2 ^ logN) ⟨Nat.zero_le _, h.1.ge, h.2⟩ + (min logTasks (logN - 9)) + let factor := cosetFactor logN s z + sums.map fun q ↦ factor * q.combine z + else interpolateCoset logN ω s width (decodeWords p evals) z + +end InterpolatePacked + +/-- `InterpolatePacked.run` as compiled for one field: an instance sets `run` to it at the +field's concrete modulus, where the compiler specializes the loops. -/ +class PackedInterpolation (p : ℕ) [Mont32Field p] [Binomial4 p] where + /-- The compiled interpolation. -/ + run : ℕ → FastField p → FastField p → ℕ → ByteArray → Ext4 p → ℕ → Array (Ext4 p) + run_eq : run = InterpolatePacked.run := by rfl + +/-- `interpolateCoset` on a row-major `2^logN × width` matrix of Montgomery words, over up to +`2 ^ logTasks` tasks, with the field's compiled loops. -/ +@[inline] +def interpolateCosetPacked [Binomial4 p] [PackedInterpolation p] (logN : ℕ) (ω s : FastField p) + (width : ℕ) (evals : ByteArray) (z : Ext4 p) (logTasks : ℕ := 3) : Array (Ext4 p) := + PackedInterpolation.run logN ω s width evals z logTasks + +end Montgomery.Native32 diff --git a/CompPoly/Fields/Montgomery/InterpolateCosetPackedCorrectness.lean b/CompPoly/Fields/Montgomery/InterpolateCosetPackedCorrectness.lean new file mode 100644 index 00000000..04322ece --- /dev/null +++ b/CompPoly/Fields/Montgomery/InterpolateCosetPackedCorrectness.lean @@ -0,0 +1,925 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Montgomery.InterpolateCosetPacked + +/-! +# Correctness of coset interpolation over packed words + +`interpolateCosetPacked_eq` proves `interpolateCosetPacked` equal to `interpolateCoset` on the +decoded words, for every point off the coset. +-/ + +@[expose] public section + +namespace Montgomery.Native32.Ext4 + +open Binomial4 (mulW_eq) + +variable {p : ℕ} [Mont32Field p] + +@[simp] theorem add_c0 (a b : Ext4 p) : (a + b).c0 = a.c0 + b.c0 := rfl +@[simp] theorem add_c1 (a b : Ext4 p) : (a + b).c1 = a.c1 + b.c1 := rfl +@[simp] theorem add_c2 (a b : Ext4 p) : (a + b).c2 = a.c2 + b.c2 := rfl +@[simp] theorem add_c3 (a b : Ext4 p) : (a + b).c3 = a.c3 + b.c3 := rfl +@[simp] theorem sub_c0 (a b : Ext4 p) : (a - b).c0 = a.c0 - b.c0 := rfl +@[simp] theorem sub_c1 (a b : Ext4 p) : (a - b).c1 = a.c1 - b.c1 := rfl +@[simp] theorem sub_c2 (a b : Ext4 p) : (a - b).c2 = a.c2 - b.c2 := rfl +@[simp] theorem sub_c3 (a b : Ext4 p) : (a - b).c3 = a.c3 - b.c3 := rfl +@[simp] theorem ofBase_c0 (x : FastField p) : (ofBase x).c0 = x := rfl +@[simp] theorem ofBase_c1 (x : FastField p) : (ofBase x).c1 = 0 := rfl +@[simp] theorem ofBase_c2 (x : FastField p) : (ofBase x).c2 = 0 := rfl +@[simp] theorem ofBase_c3 (x : FastField p) : (ofBase x).c3 = 0 := rfl +@[simp] theorem smul_c0 (x : FastField p) (a : Ext4 p) : (smul x a).c0 = x * a.c0 := rfl +@[simp] theorem smul_c1 (x : FastField p) (a : Ext4 p) : (smul x a).c1 = x * a.c1 := rfl +@[simp] theorem smul_c2 (x : FastField p) (a : Ext4 p) : (smul x a).c2 = x * a.c2 := rfl +@[simp] theorem smul_c3 (x : FastField p) (a : Ext4 p) : (smul x a).c3 = x * a.c3 := rfl + +variable [Binomial4 p] + +/-- The norm of `z - x` in terms of `t = z₀ - x`. -/ +theorem norm_sub_ofBase (z : Ext4 p) (x : FastField p) : + norm (z - ofBase x) = + normAt (normK0 z) (normK1 z) (z.c2 + z.c2) (z.c0 - x) ((z.c0 - x) * (z.c0 - x)) := by + simp only [norm, g0, g1, normAt, normK0, normK1, mulW_eq, sub_c0, sub_c1, sub_c2, sub_c3, + ofBase_c0, ofBase_c1, ofBase_c2, ofBase_c3] + ring + +/-- **The weight of a node.** `x / (z - x)` is `u b₀ + u t b₁ + u t² b₂ + u t³` for `t = z₀ - x` +and `u = x / N`, `N` the norm of `z - x`. -/ +theorem smul_inv_sub_ofBase (z : Ext4 p) (x : FastField p) : + smul x (z - ofBase x)⁻¹ = + let t := z.c0 - x + let u := x * (normAt (normK0 z) (normK1 z) (z.c2 + z.c2) t (t * t))⁻¹ + smul u (adj0 z) + smul (u * t) (adj1 z) + smul (u * (t * t)) (adj2 z) + + ofBase (u * t * (t * t)) := by + rw [show (z - ofBase x)⁻¹ = inv (z - ofBase x) from rfl] + unfold inv + rw [norm_sub_ofBase] + apply ext' <;> + simp only [adj0, adj1, adj2, g0, g1, twoW, normK0, normK1, mulW_eq, add_c0, add_c1, + add_c2, add_c3, sub_c0, sub_c1, sub_c2, sub_c3, ofBase_c0, ofBase_c1, ofBase_c2, ofBase_c3, + smul_c0, smul_c1, smul_c2, smul_c3] <;> + ring + +end Montgomery.Native32.Ext4 + +namespace Montgomery.Native32 + +open CompPoly CompPoly.Extension Polynomial +open CompPoly.CPolynomial.NTTFast.Packed + +variable {p : ℕ} [Mont32Field p] + +namespace Quad + +@[simp] theorem add_s0 (a b : Quad p) : (a + b).s0 = a.s0 + b.s0 := rfl +@[simp] theorem add_s1 (a b : Quad p) : (a + b).s1 = a.s1 + b.s1 := rfl +@[simp] theorem add_s2 (a b : Quad p) : (a + b).s2 = a.s2 + b.s2 := rfl +@[simp] theorem add_s3 (a b : Quad p) : (a + b).s3 = a.s3 + b.s3 := rfl +@[simp] theorem zero_s0 : (0 : Quad p).s0 = 0 := rfl +@[simp] theorem zero_s1 : (0 : Quad p).s1 = 0 := rfl +@[simp] theorem zero_s2 : (0 : Quad p).s2 = 0 := rfl +@[simp] theorem zero_s3 : (0 : Quad p).s3 = 0 := rfl + +/-- A column sum in the spec field: `s₀ b₀ + s₁ b₁ + s₂ b₂ + s₃`. -/ +theorem toSpec_combine [Binomial4 p] (z : Ext4 p) (q : Quad p) : + Ext4.toSpec (q.combine z) = + q.s0 • Ext4.toSpec (Ext4.adj0 z) + q.s1 • Ext4.toSpec (Ext4.adj1 z) + + q.s2 • Ext4.toSpec (Ext4.adj2 z) + q.s3 • 1 := by + simp only [combine, Ext4.toSpec_add, Ext4.toSpec_smul, Ext4.toSpec_ofBase, + ← Ext.algebraMap_eq_ofBase, Algebra.algebraMap_eq_smul_one] + +theorem toSpec_combine_add [Binomial4 p] (z : Ext4 p) (a b : Quad p) : + Ext4.toSpec ((a + b).combine z) = Ext4.toSpec (a.combine z) + Ext4.toSpec (b.combine z) := by + simp only [toSpec_combine, add_s0, add_s1, add_s2, add_s3, add_smul] + abel + +@[simp] theorem toSpec_combine_zero [Binomial4 p] (z : Ext4 p) : + Ext4.toSpec ((0 : Quad p).combine z) = 0 := by + simp only [toSpec_combine, zero_s0, zero_s1, zero_s2, zero_s3, zero_smul, add_zero] + +end Quad + +namespace InterpolatePacked + +/-! ### Words in byte arrays -/ + +/-- An unchecked read is the word at its index. -/ +theorem readUOffset_eq (b : ByteArray) (i o : USize) (h) : + Native.readUOffset b i o h = b.getUInt32LE! (4 * (i.toNat + o.toNat)) := by + have hb : 4 * (i.toNat + o.toNat) + 3 < USize.size := h.1.trans h.2 + rw [Native.readUOffset, Native.ugetUInt32LE_eq, Native.wordOffset_toNat i o hb] + +/-- Word `4 r + k` of a word array: coordinate `k` of row `r`. -/ +def row (W : Array UInt32) (r k : ℕ) : UInt32 := W.getD (4 * r + k) 0 + +/-- Overwrite row `j` of a word array. -/ +def setRow (W : Array UInt32) (j : ℕ) (v0 v1 v2 v3 : UInt32) : Array UInt32 := + W.extract 0 (4 * j) ++ + #[v0, v1, v2, v3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0].extract 0 (4 : UInt8).toNat ++ + W.extract (4 * j + (4 : UInt8).toNat) W.size + +theorem size_setRow (W : Array UInt32) (j : ℕ) (v0 v1 v2 v3 : UInt32) (hj : 4 * j + 4 ≤ W.size) : + (setRow W j v0 v1 v2 v3).size = W.size := by + simp only [setRow, Array.size_append, Array.size_extract, List.size_toArray, List.length_cons, + List.length_nil] + change min (4 * j) W.size - 0 + (min 4 16 - 0) + (min W.size W.size - (4 * j + 4)) = W.size + omega + +theorem getD_setRow (W : Array UInt32) (j : ℕ) (v0 v1 v2 v3 : UInt32) (hj : 4 * j + 4 ≤ W.size) + (m : ℕ) : + (setRow W j v0 v1 v2 v3).getD m 0 = + if m < 4 * j then W.getD m 0 + else if m < 4 * j + 4 then #[v0, v1, v2, v3].getD (m - 4 * j) 0 else W.getD m 0 := by + simp only [setRow, Array.getD_eq_getD_getElem?] + change (W.extract 0 (4 * j) ++ #[v0, v1, v2, v3] ++ W.extract (4 * j + 4) W.size)[m]?.getD 0 = _ + have hA : (W.extract 0 (4 * j)).size = 4 * j := by rw [Array.size_extract]; omega + have hB : (W.extract 0 (4 * j) ++ #[v0, v1, v2, v3]).size = 4 * j + 4 := by + rw [Array.size_append, hA]; rfl + split_ifs with h1 h2 + · rw [Array.getElem?_append_left (by rw [hB]; omega), + Array.getElem?_append_left (by rw [hA]; omega), Array.getElem?_extract] + simp only [Nat.zero_add, show m < min (4 * j) W.size - 0 by omega, ↓reduceIte] + · rw [Array.getElem?_append_left (by rw [hB]; omega), + Array.getElem?_append_right (by rw [hA]; omega), hA] + · rw [Array.getElem?_append_right (by rw [hB]; omega), hB, Array.getElem?_extract] + by_cases hm : m < W.size + · simp only [show m - (4 * j + 4) < min W.size W.size - (4 * j + 4) by omega, ↓reduceIte] + congr 2; omega + · simp only [show ¬m - (4 * j + 4) < min W.size W.size - (4 * j + 4) by omega, ↓reduceIte, + Array.getElem?_eq_none (show W.size ≤ m by omega)] + +theorem row_setRow_ne (W : Array UInt32) (j : ℕ) (v0 v1 v2 v3 : UInt32) (hj : 4 * j + 4 ≤ W.size) + (r k : ℕ) (hk : k < 4) (hr : r ≠ j) : row (setRow W j v0 v1 v2 v3) r k = row W r k := by + simp only [row, getD_setRow W j v0 v1 v2 v3 hj] + split_ifs <;> first | rfl | omega + +theorem row_setRow (W : Array UInt32) (j : ℕ) (v0 v1 v2 v3 : UInt32) (hj : 4 * j + 4 ≤ W.size) : + row (setRow W j v0 v1 v2 v3) j 0 = v0 ∧ row (setRow W j v0 v1 v2 v3) j 1 = v1 ∧ + row (setRow W j v0 v1 v2 v3) j 2 = v2 ∧ row (setRow W j v0 v1 v2 v3) j 3 = v3 := by + simp only [row, getD_setRow W j v0 v1 v2 v3 hj, Nat.add_sub_cancel_left, + show ¬4 * j + 0 < 4 * j by omega, show 4 * j + 0 < 4 * j + 4 by omega, + show ¬4 * j + 1 < 4 * j by omega, show 4 * j + 1 < 4 * j + 4 by omega, + show ¬4 * j + 2 < 4 * j by omega, show 4 * j + 2 < 4 * j + 4 by omega, + show ¬4 * j + 3 < 4 * j by omega, show 4 * j + 3 < 4 * j + 4 by omega, ↓reduceIte] + exact ⟨rfl, rfl, rfl, rfl⟩ + +/-- Storing four words at row `j` of a packed array. -/ +theorem storeRow (W : Array UInt32) (j : USize) (hj : 4 * j.toNat + 4 ≤ W.size) + (hs : 4 * W.size < USize.size) (v0 v1 v2 v3 : UInt32) : + Native.storeWords (PackedRepr.encode W) (16 * j) 4 v0 v1 v2 v3 0 0 0 0 0 0 0 0 0 0 0 0 = + PackedRepr.encode (setRow W j.toNat v0 v1 v2 v3) := by + have hsize : (2 : ℕ) ^ System.Platform.numBits = USize.size := rfl + rw [Native.storeWords_replace_encodeWords W (16 * j) 4 (4 * j.toNat) (by decide) (by + rw [USize.toNat_mul, Native.usize_numeral 16 (by decide), hsize, + Nat.mod_eq_of_lt (by omega)] + omega) (by change 4 * j.toNat + 4 ≤ W.size; omega) + (by rw [PackedRepr.size_encode_uint32]; exact hs)] + rfl + +/-! ### The forward pass -/ + +/-- The nodes `x, x ω, x ω², …` as the passes compute them. -/ +def nodeAt (ω x : FastField p) : ℕ → FastField p + | 0 => x + | r + 1 => nodeAt ω (x * ω) r + +/-- The norm of `z - x`, with `z`'s constants. -/ +def rowNorm [Binomial4 p] (z0 k0 k1 z2x2 x : FastField p) : FastField p := + Ext4.normAt k0 k1 z2x2 (z0 - x) ((z0 - x) * (z0 - x)) + +/-- `acc` times the norms of the first `r` nodes. -/ +def normPrefix [Binomial4 p] (ω z0 k0 k1 z2x2 : FastField p) (x acc : FastField p) : ℕ → FastField p + | 0 => acc + | r + 1 => normPrefix ω z0 k0 k1 z2x2 (x * ω) (acc * rowNorm z0 k0 k1 z2x2 x) r + +theorem usize_add_one_of (i : USize) (n : ℕ) (h : i.toNat + 1 ≤ n) (hn : n < USize.size) : + (i + 1).toNat = i.toNat + 1 := by + rw [usize_add_lt i 1 (by rw [USize.toNat_one]; omega), USize.toNat_one] + +/-- The forward pass stores each row's node, `t²`, norm and norm prefix, and returns the product +of all norms and the next node. -/ +theorem forward_spec [Binomial4 p] (ω z0 k0 k1 z2x2 : FastField p) (n : ℕ) (i : USize) + (x acc : FastField p) + (W : Array UInt32) (hW : 4 * (i.toNat + n) ≤ W.size) (hs : 4 * W.size < USize.size) : + ∃ W', forward ω z0 k0 k1 z2x2 n i x acc (PackedRepr.encode W) = + (PackedRepr.encode W', normPrefix ω z0 k0 k1 z2x2 x acc n, nodeAt ω x n) ∧ + W'.size = W.size ∧ + (∀ r k, k < 4 → (r < i.toNat ∨ i.toNat + n ≤ r) → row W' r k = row W r k) ∧ + ∀ r < n, row W' (i.toNat + r) 0 = (nodeAt ω x r).val ∧ + row W' (i.toNat + r) 1 = ((z0 - nodeAt ω x r) * (z0 - nodeAt ω x r)).val ∧ + row W' (i.toNat + r) 2 = (rowNorm z0 k0 k1 z2x2 (nodeAt ω x r)).val ∧ + row W' (i.toNat + r) 3 = (normPrefix ω z0 k0 k1 z2x2 x acc r).val := by + induction n generalizing i x acc W with + | zero => + exact ⟨W, rfl, rfl, fun _ _ _ _ ↦ rfl, fun r hr ↦ absurd hr (Nat.not_lt_zero r)⟩ + | succ n ih => + have hi1 := usize_add_one_of i W.size (by omega) (by omega) + have hj : 4 * i.toNat + 4 ≤ W.size := by omega + rw [forward, storeRow W i hj hs] + obtain ⟨W', heq, hsize, hout, hrows⟩ := ih (i + 1) (x * ω) + (acc * rowNorm z0 k0 k1 z2x2 x) (setRow W i.toNat x.val ((z0 - x) * (z0 - x)).val + (rowNorm z0 k0 k1 z2x2 x).val acc.val) + (by rw [size_setRow W _ _ _ _ _ hj, hi1]; omega) + (by rw [size_setRow W _ _ _ _ _ hj]; exact hs) + have hW1 := row_setRow W i.toNat x.val ((z0 - x) * (z0 - x)).val + (rowNorm z0 k0 k1 z2x2 x).val acc.val hj + refine ⟨W', heq, by rw [hsize, size_setRow W _ _ _ _ _ hj], fun r k hk hr ↦ ?_, + fun r hr ↦ ?_⟩ + · rw [hout r k hk (by omega), row_setRow_ne W _ _ _ _ _ hj r k hk (by omega)] + · rcases r with _ | r + · have hl : i.toNat < (i + 1).toNat ∨ (i + 1).toNat + n ≤ i.toNat := Or.inl (by omega) + rw [Nat.add_zero, hout _ 0 (by omega) hl, hout _ 1 (by omega) hl, hout _ 2 (by omega) hl, + hout _ 3 (by omega) hl] + exact hW1 + · have := hrows r (by omega) + rw [hi1, show i.toNat + 1 + r = i.toNat + (r + 1) by omega] at this + exact this + +/-! ### The backward pass -/ + +theorem readField_pack (W : Array UInt32) (i o : USize) (h) : + readField p (PackedRepr.encode W) i o h = wordField p (W.getD (i.toNat + o.toNat) 0) := by + rw [readField, readUOffset_eq, PackedRepr.getUInt32LE!_encode] + +/-- Peeling one norm off the inverse of a prefix product. -/ +theorem inv_mul_prefix (a m : FastField p) (ha : a ≠ 0) (hm : m ≠ 0) : + (a * m)⁻¹ * a = m⁻¹ ∧ (a * m)⁻¹ * m = a⁻¹ := by + constructor <;> field_simp + +/-- The backward pass turns rows `i - n, …, i - 1` of node, `t²`, norm and prefix product into +the weights `u, u t, u t², u t³` for `u = x / N`, given that `inv` inverts the prefix product +before row `i`. -/ +theorem backward_spec (z0 : FastField p) (X T2 Nn Pp : ℕ → FastField p) (n : ℕ) (i : USize) + (inv : FastField p) + (W : Array UInt32) (h) (hW : 4 * i.toNat ≤ W.size) + (hrows : ∀ r < i.toNat, row W r 0 = (X r).val ∧ row W r 1 = (T2 r).val ∧ + row W r 2 = (Nn r).val ∧ row W r 3 = (Pp r).val) + (hP : ∀ r < i.toNat, Pp (r + 1) = Pp r * Nn r) (hN : ∀ r < i.toNat, Nn r ≠ 0) + (hPne : ∀ r < i.toNat, Pp r ≠ 0) (hinv : inv = (Pp i.toNat)⁻¹) : + ∃ W', backward z0 n i inv (PackedRepr.encode W) h = PackedRepr.encode W' ∧ W'.size = W.size ∧ + (∀ r k, k < 4 → (r + n < i.toNat ∨ i.toNat ≤ r) → row W' r k = row W r k) ∧ + ∀ r < i.toNat, i.toNat ≤ r + n → + row W' r 0 = (X r * (Nn r)⁻¹).val ∧ + row W' r 1 = (X r * (Nn r)⁻¹ * (z0 - X r)).val ∧ + row W' r 2 = (X r * (Nn r)⁻¹ * T2 r).val ∧ + row W' r 3 = (X r * (Nn r)⁻¹ * (z0 - X r) * T2 r).val := by + induction n generalizing i inv W with + | zero => + exact ⟨W, rfl, rfl, fun _ _ _ _ ↦ rfl, fun r hr hr' ↦ absurd hr' (by omega)⟩ + | succ n ih => + have hsize : (PackedRepr.encode W).size = 4 * W.size := PackedRepr.size_encode_uint32 W + have hi : 1 ≤ i.toNat := by have := h.2.2; omega + have hj : (i - 1).toNat = i.toNat - 1 := by + rw [USize.toNat_sub_of_le _ _ (by rw [USize.le_iff_toNat_le]; simp; omega)]; simp + have hus : 4 * W.size < USize.size := by rw [← hsize]; exact h.2.1 + have h4 : (4 * (i - 1)).toNat = 4 * (i.toNat - 1) := by + rw [usize_four_mul (i - 1) (PackedRepr.encode W).size (by rw [hsize, hj]; omega) h.2.1, hj] + obtain ⟨hx, ht2, hn, hpre⟩ := hrows (i.toNat - 1) (by omega) + have hrow : ∀ k, (4 * (i - 1)).toNat + (OfNat.ofNat k : USize).toNat = 4 * (i.toNat - 1) + k + ∨ k ≥ 4 := by + intro k + by_cases hk : k < 4 + · left; rw [h4, Native.usize_numeral k (by omega)] + · right; omega + have hread : ∀ (k : ℕ) (hk : k < 4) (v : FastField p), row W (i.toNat - 1) k = v.val → + ∀ hb, readField p (PackedRepr.encode W) (4 * (i - 1)) (OfNat.ofNat k) hb = v := by + intro k hk v hv hb + rw [readField_pack, h4, Native.usize_numeral k (by omega)] + change wordField p (row W (i.toNat - 1) k) = v + rw [hv, wordField_val] + have hnp := inv_mul_prefix (Pp (i.toNat - 1)) (Nn (i.toNat - 1)) (hPne _ (by omega)) + (hN _ (by omega)) + have hinv' : inv = (Pp (i.toNat - 1) * Nn (i.toNat - 1))⁻¹ := by + rw [hinv, ← hP _ (by omega), Nat.sub_add_cancel hi] + rw [backward] + simp only [hread 0 (by decide) _ hx, hread 1 (by decide) _ ht2, hread 2 (by decide) _ hn, + hread 3 (by decide) _ hpre, hinv', hnp.1, hnp.2] + have hj' : 4 * (i - 1).toNat + 4 ≤ W.size := by rw [hj]; omega + simp only [storeRow W (i - 1) hj' hus, hj] + have hj'' : 4 * (i.toNat - 1) + 4 ≤ W.size := by rw [← hj]; exact hj' + have hW1 := row_setRow W (i.toNat - 1) (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹).val + (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹ * (z0 - X (i.toNat - 1))).val + (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹ * T2 (i.toNat - 1)).val + (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹ * (z0 - X (i.toNat - 1)) * T2 (i.toNat - 1)).val hj'' + have hsz1 := size_setRow W (i.toNat - 1) (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹).val + (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹ * (z0 - X (i.toNat - 1))).val + (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹ * T2 (i.toNat - 1)).val + (X (i.toNat - 1) * (Nn (i.toNat - 1))⁻¹ * (z0 - X (i.toNat - 1)) * T2 (i.toNat - 1)).val hj'' + obtain ⟨W', heq, hsz, hout, hrows'⟩ := ih (i - 1) (Pp (i.toNat - 1))⁻¹ _ + (by rw [PackedRepr.size_encode_uint32, hsz1, hj] + exact ⟨by omega, by omega, by have := h.2.2; omega⟩) + (by rw [hsz1, hj]; omega) + (fun r hr ↦ by + rw [hj] at hr + rw [row_setRow_ne W _ _ _ _ _ hj'' r 0 (by omega) (by omega), + row_setRow_ne W _ _ _ _ _ hj'' r 1 (by omega) (by omega), + row_setRow_ne W _ _ _ _ _ hj'' r 2 (by omega) (by omega), + row_setRow_ne W _ _ _ _ _ hj'' r 3 (by omega) (by omega)] + exact hrows r (by omega)) + (fun r hr ↦ hP r (by rw [hj] at hr; omega)) (fun r hr ↦ hN r (by rw [hj] at hr; omega)) + (fun r hr ↦ hPne r (by rw [hj] at hr; omega)) (by rw [hj]) + refine ⟨W', heq, by rw [hsz, hsz1], fun r k hk hr ↦ ?_, fun r hr hr' ↦ ?_⟩ + · rw [hout r k hk (by rw [hj]; omega), row_setRow_ne W _ _ _ _ _ hj'' r k hk (by omega)] + · by_cases hr1 : r = i.toNat - 1 + · subst hr1 + have hl : (i - 1).toNat ≤ i.toNat - 1 := by rw [hj] + rw [hout _ 0 (by omega) (Or.inr hl), hout _ 1 (by omega) (Or.inr hl), + hout _ 2 (by omega) (Or.inr hl), hout _ 3 (by omega) (Or.inr hl)] + exact hW1 + · exact hrows' r (by rw [hj]; omega) (by rw [hj]; omega) + +/-! ### Column sums -/ + +/-- Term `r` of a column sum: the residue of evaluation word `e + r · width` times coordinate `k` +of row `r`'s weights. -/ +def dotTerm (p : ℕ) [Mont32Field p] (evals buf : ByteArray) (e width w r k : ℕ) : FastField p := + (ofWordMod (evals.getUInt32LE! (4 * (e + r * width))) : FastField p) * + wordField p (buf.getUInt32LE! (4 * (w + 4 * r + k))) + +theorem dot1_spec (evals buf : ByteArray) (width : USize) (n : ℕ) (e w : USize) + (a0 a1 a2 a3 : Acc p) (h) : + FastField.toField (dot1 p evals buf width n e w a0 a1 a2 a3 h).s0 = + a0.value + ∑ r ∈ Finset.range n, + FastField.toField (dotTerm p evals buf e.toNat width.toNat w.toNat r 0) ∧ + FastField.toField (dot1 p evals buf width n e w a0 a1 a2 a3 h).s1 = + a1.value + ∑ r ∈ Finset.range n, + FastField.toField (dotTerm p evals buf e.toNat width.toNat w.toNat r 1) ∧ + FastField.toField (dot1 p evals buf width n e w a0 a1 a2 a3 h).s2 = + a2.value + ∑ r ∈ Finset.range n, + FastField.toField (dotTerm p evals buf e.toNat width.toNat w.toNat r 2) ∧ + FastField.toField (dot1 p evals buf width n e w a0 a1 a2 a3 h).s3 = + a3.value + ∑ r ∈ Finset.range n, + FastField.toField (dotTerm p evals buf e.toNat width.toNat w.toNat r 3) := by + induction n generalizing e w a0 a1 a2 a3 with + | zero => + simp only [dot1, LazyAcc.toField_result, Finset.range_zero, Finset.sum_empty, add_zero, + and_self] + | succ n ih => + have hw4 : (w + 4).toNat = w.toNat + 4 := by + have := h.2.2.1; have := h.2.2.2 + rw [usize_add_lt _ _ (by rw [Native.usize_numeral 4 (by decide)]; omega), + Native.usize_numeral 4 (by decide)] + have hterm : ∀ k, ∀ r < n, dotTerm p evals buf (e + width).toNat width.toNat (w + 4).toNat r k = + dotTerm p evals buf e.toNat width.toNat w.toNat (r + 1) k := by + intro k r hr + have h1 := h.1 1 (by omega) + have := h.2.1 + rw [Nat.one_mul] at h1 + simp only [dotTerm, usize_add_lt e width (by omega), hw4] + congr 4 <;> ring + rw [dot1] + simp only [readUOffset_eq, readField, Native.usize_numeral 0 (by decide), + Native.usize_numeral 1 (by decide), Native.usize_numeral 2 (by decide), + Native.usize_numeral 3 (by decide)] + obtain ⟨i0, i1, i2, i3⟩ := ih (e + width) (w + 4) _ _ _ _ _ + rw [i0, i1, i2, i3] + simp only [LazyAcc.value_add, Finset.sum_range_succ'] + refine ⟨?_, ?_, ?_, ?_⟩ <;> + · rw [Finset.sum_congr rfl fun r hr ↦ + congrArg FastField.toField (hterm _ r (Finset.mem_range.mp hr))] + simp only [dotTerm, toField_mul, Nat.zero_mul, Nat.add_zero, Nat.mul_zero] + ring + +/-- `dot1`'s range condition. -/ +abbrev Dot1Fits (evals buf : ByteArray) (width : USize) (n : ℕ) (e w : USize) : Prop := + (∀ r < n, 4 * (e.toNat + r * width.toNat) + 3 < evals.size) ∧ evals.size < USize.size ∧ + 4 * (w.toNat + 4 * n) ≤ buf.size ∧ buf.size < USize.size + +theorem dot4_fits {evals buf : ByteArray} {width : USize} {n : ℕ} {e w : USize} + (h : (∀ r < n, 4 * (e.toNat + r * width.toNat + 3) + 3 < evals.size) ∧ + evals.size < USize.size ∧ 4 * (w.toNat + 4 * n) ≤ buf.size ∧ buf.size < USize.size) + (c : ℕ) (hc : c < 4) : Dot1Fits evals buf width n (e + OfNat.ofNat c) w := by + refine ⟨fun r hr ↦ ?_, h.2⟩ + have := h.1 r hr + have := h.2.1 + rw [usize_add_lt _ _ (by rw [Native.usize_numeral c (by omega)]; nlinarith), + Native.usize_numeral c (by omega)] + omega + +theorem dot4_fits0 {evals buf : ByteArray} {width : USize} {n : ℕ} {e w : USize} + (h : (∀ r < n, 4 * (e.toNat + r * width.toNat + 3) + 3 < evals.size) ∧ + evals.size < USize.size ∧ 4 * (w.toNat + 4 * n) ≤ buf.size ∧ buf.size < USize.size) : + Dot1Fits evals buf width n e w := + ⟨fun r hr ↦ by have := h.1 r hr; omega, h.2⟩ + +/-- Four adjacent columns at once are four single columns. -/ +theorem dot4_eq (evals buf : ByteArray) (width : USize) (n : ℕ) (e w : USize) + (a0 a1 a2 a3 b0 b1 b2 b3 c0 c1 c2 c3 d0 d1 d2 d3 : Acc p) (h) : + dot4 p evals buf width n e w a0 a1 a2 a3 b0 b1 b2 b3 c0 c1 c2 c3 d0 d1 d2 d3 h = + (dot1 p evals buf width n e w a0 a1 a2 a3 (dot4_fits0 h), + dot1 p evals buf width n (e + 1) w b0 b1 b2 b3 (dot4_fits h 1 (by decide)), + dot1 p evals buf width n (e + 2) w c0 c1 c2 c3 (dot4_fits h 2 (by decide)), + dot1 p evals buf width n (e + 3) w d0 d1 d2 d3 (dot4_fits h 3 (by decide))) := by + induction n generalizing e w a0 a1 a2 a3 b0 b1 b2 b3 c0 c1 c2 c3 d0 d1 d2 d3 with + | zero => rfl + | succ n ih => + have he := h.1 0 (by omega) + have hes := h.2.1 + simp only [Nat.zero_mul, Nat.add_zero] at he + have hc : ∀ c < 4, (e + OfNat.ofNat c).toNat = e.toNat + c := fun c hc ↦ by + rw [usize_add_lt _ _ (by rw [Native.usize_numeral c (by omega)]; omega), + Native.usize_numeral c (by omega)] + simp only [dot4, dot1] + simp only [readUOffset_eq, Native.usize_numeral 0 (by decide), + Native.usize_numeral 1 (by decide), Native.usize_numeral 2 (by decide), + Native.usize_numeral 3 (by decide), hc 1 (by decide), hc 2 (by decide), hc 3 (by decide), + Nat.add_zero] + rw [ih] + simp only [USize.add_assoc, USize.add_comm width] + +/-- Rows `0, …, b - 1` of `buf` hold weights whose column sums are `wt r`. -/ +def WeightsAt [Binomial4 p] (buf : ByteArray) (z : Ext4 p) (wt : ℕ → Ext4.Spec p) (b : ℕ) : Prop := + ∀ r < b, Ext4.toSpec (Quad.combine z ⟨wordField p (buf.getUInt32LE! (4 * (4 * r))), + wordField p (buf.getUInt32LE! (4 * (4 * r + 1))), + wordField p (buf.getUInt32LE! (4 * (4 * r + 2))), + wordField p (buf.getUInt32LE! (4 * (4 * r + 3)))⟩) = wt r + +/-- The block's sum of column `c`. -/ +def colBlock [Binomial4 p] (evals : ByteArray) (width row b c : ℕ) (wt : ℕ → Ext4.Spec p) : + Ext4.Spec p := + ∑ r ∈ Finset.range b, + (ofWordMod (evals.getUInt32LE! (4 * ((row + r) * width + c))) : FastField p) • wt r + +/-- A column's sums give the column's block sum. -/ +theorem toSpec_combine_dot1 [Binomial4 p] (evals buf : ByteArray) (z : Ext4 p) + (wt : ℕ → Ext4.Spec p) + (width row b c : ℕ) (hw : WeightsAt buf z wt b) + (hi : b = 0 ∨ width.toUSize.toNat = width ∧ (row * width + c).toUSize.toNat = row * width + c) + (h) : + Ext4.toSpec ((dot1 p evals buf width.toUSize b (row * width + c).toUSize 0 LazyAcc.zero + LazyAcc.zero LazyAcc.zero LazyAcc.zero h).combine z) = colBlock evals width row b c wt := by + obtain ⟨d0, d1, d2, d3⟩ := dot1_spec (p := p) evals buf width.toUSize b + (row * width + c).toUSize 0 + LazyAcc.zero LazyAcc.zero LazyAcc.zero LazyAcc.zero h + have e : ∀ (k : ℕ) (x : FastField p), FastField.toField x = LazyAcc.value LazyAcc.zero + + ∑ r ∈ Finset.range b, FastField.toField + (dotTerm p evals buf (row * width + c).toUSize.toNat width.toUSize.toNat + (0 : USize).toNat r k) → + x = ∑ r ∈ Finset.range b, dotTerm p evals buf (row * width + c).toUSize.toNat + width.toUSize.toNat (0 : USize).toNat r k := fun k x hx ↦ + toField_injective (by rw [hx, LazyAcc.value_zero, zero_add, toField_sum]) + rw [Quad.toSpec_combine, e 0 _ d0, e 1 _ d1, e 2 _ d2, e 3 _ d3] + rcases hi with rfl | ⟨hwt, he⟩ + · simp only [Finset.range_zero, Finset.sum_empty, zero_smul, add_zero, colBlock] + simp only [colBlock, dotTerm, he, hwt, USize.toNat_zero, Nat.zero_add] + refine Eq.trans ?_ (Finset.sum_congr rfl fun r hr ↦ by rw [← hw r (Finset.mem_range.mp hr)]) + simp only [Quad.toSpec_combine, Finset.sum_smul, smul_add, smul_smul, ← Finset.sum_add_distrib, + Nat.add_zero] + refine Finset.sum_congr rfl fun r _ ↦ ?_ + rw [show row * width + c + r * width = (row + r) * width + c by ring] + +theorem blockFits_index {evals buf : ByteArray} {width row b : ℕ} + (h : BlockFits evals buf width row b) (c : ℕ) (hc : c < width) : + b = 0 ∨ width.toUSize.toNat = width ∧ (row * width + c).toUSize.toNat = row * width + c := by + rcases Nat.eq_zero_or_pos b with hb | hb + · exact Or.inl hb + · right + have hd := dot_bound row b width c 0 0 hb (by omega) + have := h.1; have := h.2.1 + simp only [Nat.zero_mul, Nat.add_zero] at hd + have hw : width ≤ (row + b) * width := Nat.le_mul_of_pos_left _ (by omega) + exact ⟨toUSize_toNat_of_lt _ ((row + b) * width + 1) (by omega) (by omega), + toUSize_toNat_of_lt _ ((row + b) * width) hd (by omega)⟩ + +/-- The spec value of column `c` of a sums array. -/ +def sumAt [Binomial4 p] (z : Ext4 p) (sums : Array (Quad p)) (c : ℕ) : Ext4.Spec p := + Ext4.toSpec ((sums.getD c 0).combine z) + +theorem sumAt_modify [Binomial4 p] (z : Ext4 p) (sums : Array (Quad p)) (c : ℕ) + (hc : c < sums.size) (q : Quad p) + (c' : ℕ) : + sumAt z (sums.modify c (· + q)) c' = + sumAt z sums c' + if c = c' then Ext4.toSpec (q.combine z) else 0 := by + simp only [sumAt, Array.getD_eq_getD_getElem?, Array.getElem?_modify] + split + · subst c' + rw [Array.getElem?_eq_getElem hc, Option.map_some, Option.getD_some, Option.getD_some, + Quad.toSpec_combine_add] + · rw [add_zero] + +theorem toUSize_add_numeral (a j : ℕ) : a.toUSize + (OfNat.ofNat j : USize) = (a + j).toUSize := + (USize.ofNat_add a j).symm + +theorem addColumn_spec [Binomial4 p] (evals buf : ByteArray) (z : Ext4 p) (wt : ℕ → Ext4.Spec p) + (width row b : ℕ) (hfit : BlockFits evals buf width row b) (hw : WeightsAt buf z wt b) + (c : ℕ) (hc : c + 1 ≤ width) (sums : Array (Quad p)) : + (addColumn p evals buf width row b hfit c hc sums).size = sums.size ∧ + ∀ c', c < sums.size → sumAt z (addColumn p evals buf width row b hfit c hc sums) c' = + sumAt z sums c' + if c = c' then colBlock evals width row b c wt else 0 := by + refine ⟨Array.size_modify, fun c' hcs ↦ ?_⟩ + rw [addColumn, sumAt_modify z sums c hcs, + toSpec_combine_dot1 evals buf z wt width row b c hw (blockFits_index hfit c (by omega))] + +theorem addColumns4_spec [Binomial4 p] (evals buf : ByteArray) (z : Ext4 p) (wt : ℕ → Ext4.Spec p) + (width row b : ℕ) (hfit : BlockFits evals buf width row b) (hw : WeightsAt buf z wt b) + (c : ℕ) (hc : c + 4 ≤ width) (sums : Array (Quad p)) (hcs : c + 4 ≤ sums.size) : + (addColumns4 p evals buf width row b hfit c hc sums).size = sums.size ∧ + ∀ c', sumAt z (addColumns4 p evals buf width row b hfit c hc sums) c' = + sumAt z sums c' + if c ≤ c' ∧ c' < c + 4 then colBlock evals width row b c' wt else 0 := by + refine ⟨by simp only [addColumns4, Array.size_modify], fun c' ↦ ?_⟩ + have hcol : ∀ j, j < 4 → ∀ hj, Ext4.toSpec ((dot1 p evals buf width.toUSize b + ((row * width + c).toUSize + OfNat.ofNat j) 0 LazyAcc.zero LazyAcc.zero LazyAcc.zero + LazyAcc.zero hj).combine z) = colBlock evals width row b (c + j) wt := by + intro j hj hj' + simp only [toUSize_add_numeral, Nat.add_assoc] at hj' ⊢ + exact toSpec_combine_dot1 evals buf z wt width row b (c + j) hw + (blockFits_index hfit (c + j) (by omega)) hj' + have hcol0 : ∀ hj, Ext4.toSpec ((dot1 p evals buf width.toUSize b (row * width + c).toUSize 0 + LazyAcc.zero LazyAcc.zero LazyAcc.zero LazyAcc.zero hj).combine z) = + colBlock evals width row b c wt := fun hj ↦ + toSpec_combine_dot1 evals buf z wt width row b c hw (blockFits_index hfit c (by omega)) hj + simp only [addColumns4, dot4_eq] + rw [sumAt_modify z _ _ (by simp only [Array.size_modify]; omega), + sumAt_modify z _ _ (by simp only [Array.size_modify]; omega), + sumAt_modify z _ _ (by simp only [Array.size_modify]; omega), + sumAt_modify z _ _ (by omega), hcol0, hcol 1 (by decide), hcol 2 (by decide), + hcol 3 (by decide)] + by_cases h0 : c = c' + · subst h0; simp [show c ≤ c ∧ c < c + 4 by omega] + by_cases h1 : c + 1 = c' + · subst h1; simp [show c ≤ c + 1 ∧ c + 1 < c + 4 by omega] + by_cases h2 : c + 2 = c' + · subst h2; simp [show c ≤ c + 2 ∧ c + 2 < c + 4 by omega] + by_cases h3 : c + 3 = c' + · subst h3; simp [show c ≤ c + 3 ∧ c + 3 < c + 4 by omega] + simp only [h0, h1, h2, h3, ↓reduceIte, add_zero, show ¬(c ≤ c' ∧ c' < c + 4) by omega] + +theorem columns1_spec [Binomial4 p] (evals buf : ByteArray) (z : Ext4 p) (wt : ℕ → Ext4.Spec p) + (width row b : ℕ) (hfit : BlockFits evals buf width row b) (hw : WeightsAt buf z wt b) + (k c : ℕ) (hc : c + k ≤ width) (sums : Array (Quad p)) (hs : sums.size = width) : + (columns1 p evals buf width row b hfit k c hc sums).size = width ∧ + ∀ c', sumAt z (columns1 p evals buf width row b hfit k c hc sums) c' = + sumAt z sums c' + if c ≤ c' ∧ c' < c + k then colBlock evals width row b c' wt else 0 := by + induction k generalizing c sums with + | zero => + refine ⟨hs, fun c' ↦ ?_⟩ + simp only [columns1] + split_ifs + · omega + · rw [add_zero] + | succ k ih => + obtain ⟨hs1, hv1⟩ := addColumn_spec evals buf z wt width row b hfit hw c (by omega) sums + obtain ⟨hsz, hval⟩ := ih (c + 1) (by omega) + (addColumn p evals buf width row b hfit c (by omega) sums) (by rw [hs1, hs]) + refine ⟨hsz, fun c' ↦ ?_⟩ + rw [columns1, hval, hv1 c' (by omega)] + by_cases h1 : c = c' + · subst h1 + simp only [↓reduceIte, show ¬(c + 1 ≤ c ∧ c < c + 1 + k) by omega, + show c ≤ c ∧ c < c + (k + 1) by omega, add_zero, and_self] + · simp only [h1, ↓reduceIte, add_zero] + congr 1 + split_ifs <;> first | rfl | omega + +theorem columns4_spec [Binomial4 p] (evals buf : ByteArray) (z : Ext4 p) (wt : ℕ → Ext4.Spec p) + (width row b : ℕ) (hfit : BlockFits evals buf width row b) (hw : WeightsAt buf z wt b) + (g c : ℕ) (hc : c + 4 * g ≤ width) (sums : Array (Quad p)) (hs : sums.size = width) : + (columns4 p evals buf width row b hfit g c hc sums).size = width ∧ + ∀ c', sumAt z (columns4 p evals buf width row b hfit g c hc sums) c' = + sumAt z sums c' + + if c ≤ c' ∧ c' < c + 4 * g then colBlock evals width row b c' wt else 0 := by + induction g generalizing c sums with + | zero => + refine ⟨hs, fun c' ↦ ?_⟩ + simp only [columns4] + split_ifs + · omega + · rw [add_zero] + | succ g ih => + obtain ⟨hs1, hv1⟩ := addColumns4_spec evals buf z wt width row b hfit hw c (by omega) sums + (by omega) + obtain ⟨hsz, hval⟩ := ih (c + 4) (by omega) + (addColumns4 p evals buf width row b hfit c (by omega) sums) (by rw [hs1, hs]) + refine ⟨hsz, fun c' ↦ ?_⟩ + rw [columns4, hval, hv1 c'] + rw [add_assoc] + congr 1 + split_ifs <;> first | (exfalso; omega) | simp only [add_zero, zero_add] + +/-! ### Blocks -/ + +theorem nodeAt_eq (ω x : FastField p) (r : ℕ) : nodeAt ω x r = x * ω ^ r := by + induction r generalizing x with + | zero => rw [nodeAt, pow_zero, mul_one] + | succ r ih => rw [nodeAt, ih, pow_succ', mul_assoc] + +theorem nodeAt_succ (ω x : FastField p) (r : ℕ) : nodeAt ω x (r + 1) = nodeAt ω x r * ω := by + induction r generalizing x with + | zero => rfl + | succ r ih => rw [nodeAt, ih, ← nodeAt] + +theorem normPrefix_succ [Binomial4 p] (ω z0 k0 k1 z2x2 x acc : FastField p) (r : ℕ) : + normPrefix ω z0 k0 k1 z2x2 x acc (r + 1) = + normPrefix ω z0 k0 k1 z2x2 x acc r * rowNorm z0 k0 k1 z2x2 (nodeAt ω x r) := by + induction r generalizing x acc with + | zero => rfl + | succ r ih => rw [normPrefix, ih, ← normPrefix, nodeAt] + +/-- Off the coset, the norm of `z - x` is a unit. -/ +theorem rowNorm_ne_zero [Binomial4 p] (z : Ext4 p) (x : FastField p) + (hz : Ext4.toSpec z ≠ Ext.ofBase x) : + rowNorm z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) x ≠ 0 := by + rw [rowNorm, ← Ext4.norm_sub_ofBase] + exact Ext4.norm_ne_zero _ (by rw [Ext4.toSpec_sub, Ext4.toSpec_ofBase, sub_ne_zero]; exact hz) + +/-- The weight `x / (z - x)` in the spec field. -/ +def specWeight [Binomial4 p] (z : Ext4 p) (x : FastField p) : Ext4.Spec p := + Ext.ofBase x * (Ext4.toSpec z - Ext.ofBase x)⁻¹ + +theorem toSpec_weight [Binomial4 p] (z : Ext4 p) (x : FastField p) : + Ext4.toSpec (Ext4.smul x (z - Ext4.ofBase x)⁻¹) = specWeight z x := by + rw [Ext4.toSpec_smul, Ext4.toSpec_inv, Ext4.toSpec_sub, Ext4.toSpec_ofBase, Algebra.smul_def, + Ext.algebraMap_eq_ofBase, specWeight] + +/-- Term `R` of column `c`: the residue of its word times the weight of node `s ω^R`. -/ +def rowTerm [Binomial4 p] (evals : ByteArray) (width : ℕ) (ω s : FastField p) (z : Ext4 p) + (R c : ℕ) : + Ext4.Spec p := + (ofWordMod (evals.getUInt32LE! (4 * (R * width + c))) : FastField p) • specWeight z (s * ω ^ R) + +/-- The weights a block's two passes leave in its buffer. -/ +theorem weightsAt_passes [Binomial4 p] (z : Ext4 p) (ω x : FastField p) (b : ℕ) (W2 : Array UInt32) + (hrows : ∀ r < b, + row W2 r 0 = (nodeAt ω x r * (rowNorm z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) + (nodeAt ω x r))⁻¹).val ∧ + row W2 r 1 = (nodeAt ω x r * (rowNorm z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) + (nodeAt ω x r))⁻¹ * (z.c0 - nodeAt ω x r)).val ∧ + row W2 r 2 = (nodeAt ω x r * (rowNorm z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) + (nodeAt ω x r))⁻¹ * ((z.c0 - nodeAt ω x r) * (z.c0 - nodeAt ω x r))).val ∧ + row W2 r 3 = (nodeAt ω x r * (rowNorm z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) + (nodeAt ω x r))⁻¹ * (z.c0 - nodeAt ω x r) * + ((z.c0 - nodeAt ω x r) * (z.c0 - nodeAt ω x r))).val) : + WeightsAt (PackedRepr.encode W2) z + (fun r ↦ specWeight z (nodeAt ω x r)) b := by + intro r hr + obtain ⟨h0, h1, h2, h3⟩ := hrows r hr + simp only [PackedRepr.getUInt32LE!_encode] + rw [show W2.getD (4 * r) 0 = row W2 r 0 from rfl, + show W2.getD (4 * r + 1) 0 = row W2 r 1 from rfl, + show W2.getD (4 * r + 2) 0 = row W2 r 2 from rfl, + show W2.getD (4 * r + 3) 0 = row W2 r 3 from rfl, h0, h1, h2, h3, wordField_val, wordField_val, + wordField_val, wordField_val, ← toSpec_weight, Ext4.smul_inv_sub_ofBase] + rfl + +theorem blocks_spec [Binomial4 p] (evals : ByteArray) (ω s : FastField p) (z : Ext4 p) + (width hi k row : ℕ) (x : FastField p) + (W : Array UInt32) (sums : Array (Quad p)) (h) (hW : W.size = 4 * blockRows) + (hx : x = s * ω ^ row) + (hs : sums.size = width) + (hz : ∀ R, row ≤ R → R < hi → + Ext4.toSpec z ≠ Ext.ofBase (s * ω ^ R)) : + (blocks evals ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) width hi k row x + (PackedRepr.encode W) sums h).size = width ∧ + ∀ c < width, sumAt z (blocks evals ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) + width hi k row x (PackedRepr.encode W) sums h) c = + sumAt z sums c + ∑ R ∈ Finset.Ico row (min hi (row + k * blockRows)), + rowTerm evals width ω s z R c := by + induction k generalizing row x W sums with + | zero => + refine ⟨hs, fun c _ ↦ ?_⟩ + rw [blocks, Finset.Ico_eq_empty (by omega), Finset.sum_empty, add_zero] + | succ k ih => + obtain ⟨b, hb⟩ : ∃ b, b = min blockRows (hi - row) := ⟨_, rfl⟩ + have hbB : b ≤ blockRows := hb ▸ Nat.min_le_left _ _ + have hrb : row + b ≤ hi := by have := Nat.min_le_right blockRows (hi - row); have := h.1; omega + have hus := USize.le_size + have hbt : b.toUSize.toNat = b := toUSize_toNat_of_lt b (blockRows + 1) (by + omega) (by simp only [blockRows]; omega) + have hWs : 4 * W.size < USize.size := by rw [hW]; simp only [blockRows]; omega + obtain ⟨W1, eq1, hs1, -, rows1⟩ := forward_spec ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) + (z.c2 + z.c2) b 0 x 1 W (by rw [USize.toNat_zero, hW]; omega) hWs + have hX : ∀ r, nodeAt ω x r = s * ω ^ (row + r) := by + intro r; rw [nodeAt_eq, hx, pow_add, mul_assoc] + have hN : ∀ r < b, rowNorm z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) + (nodeAt ω x r) ≠ 0 := fun r hr ↦ + rowNorm_ne_zero z _ (by rw [hX]; exact hz _ (by omega) (by omega)) + have hPne : ∀ r ≤ b, + normPrefix ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) x 1 r ≠ 0 := by + intro r hr + induction r with + | zero => rw [normPrefix]; exact one_ne_zero + | succ r ihr => + rw [normPrefix_succ] + exact mul_ne_zero (ihr (by omega)) (hN r (by omega)) + obtain ⟨W2, eq2, hs2, -, rows2⟩ := backward_spec z.c0 (nodeAt ω x) + (fun r ↦ (z.c0 - nodeAt ω x r) * (z.c0 - nodeAt ω x r)) + (fun r ↦ rowNorm z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) (nodeAt ω x r)) + (normPrefix ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) x 1) b b.toUSize + (normPrefix ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) x 1 b)⁻¹ W1 + (by rw [PackedRepr.size_encode_uint32, hs1, hW, hbt]; simp only [blockRows] at hbB ⊢ + exact ⟨by omega, by omega, le_refl _⟩) + (by rw [hs1, hW, hbt]; simp only [blockRows] at hbB ⊢; omega) + (fun r hr ↦ by + rw [hbt] at hr + have := rows1 r hr + simp only [USize.toNat_zero, Nat.zero_add] at this + exact this) + (fun r _ ↦ normPrefix_succ _ _ _ _ _ _ _ r) (fun r hr ↦ hN r (by omega)) + (fun r hr ↦ hPne r (by omega)) (by rw [hbt]) + have hw := weightsAt_passes z ω x b W2 (fun r hr ↦ by + have := rows2 r (by omega) (by omega) + exact this) + have hfit : BlockFits evals (PackedRepr.encode W2) width row b := by + have := Nat.mul_le_mul_right width hrb + refine ⟨by omega, h.2.2.1, ?_, ?_⟩ <;> rw [PackedRepr.size_encode_uint32, hs2, hs1, hW] <;> + simp only [blockRows] at hbB ⊢ <;> omega + have hcol : ∀ c, colBlock evals width row b c + (fun r ↦ specWeight z (nodeAt ω x r)) = + ∑ R ∈ Finset.Ico row (row + b), rowTerm evals width ω s z R c := by + intro c + rw [Finset.sum_Ico_eq_sum_range, Nat.add_sub_cancel_left, colBlock] + refine Finset.sum_congr rfl fun r _ ↦ ?_ + rw [rowTerm, hX] + have key : ∀ (S : Array (Quad p)) (hS : S.size = width) + (_ : ∀ c < width, sumAt z S c = sumAt z sums c + colBlock evals width row b c + (fun r ↦ specWeight z (nodeAt ω x r))) (h''), + (blocks evals ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) width hi k (row + b) + (nodeAt ω x b) (PackedRepr.encode W2) S h'').size = width ∧ + ∀ c < width, sumAt z (blocks evals ω z.c0 (Ext4.normK0 z) (Ext4.normK1 z) (z.c2 + z.c2) + width hi k (row + b) (nodeAt ω x b) (PackedRepr.encode W2) S h'') c = + sumAt z sums c + ∑ R ∈ Finset.Ico row (min hi (row + (k + 1) * blockRows)), + rowTerm evals width ω s z R c := by + intro S hS hv h'' + obtain ⟨hsz, hval⟩ := ih (row + b) (nodeAt ω x b) W2 S h'' (by rw [hs2, hs1, hW]) + (by rw [hX]) hS (fun R h1 h2 ↦ hz R (by omega) h2) + refine ⟨hsz, fun c hc ↦ ?_⟩ + rw [hval c hc, hv c hc, add_assoc, hcol, + Finset.sum_Ico_consecutive _ (by omega) (by omega)] + congr 3 + simp only [blockRows] at hb ⊢ + omega + rw [blocks] + simp only [← hb, eq1, eq2] + have h4 := columns4_spec evals _ z _ width row b hfit hw (width / 4) 0 (by omega) sums hs + have h1 := columns1_spec evals _ z _ width row b hfit hw (width % 4) (4 * (width / 4)) + (by omega) _ h4.1 + refine key _ h1.1 (fun c hc ↦ ?_) _ + rw [h1.2 c, h4.2 c] + split_ifs <;> first | omega | simp only [add_zero] + +/-! ### Leaves and tasks -/ + +theorem sumAt_replicate_zero [Binomial4 p] (z : Ext4 p) (width c : ℕ) : + sumAt z (Array.replicate width 0) c = 0 := by + simp only [sumAt, Array.getD_eq_getD_getElem?, Array.getElem?_replicate] + split <;> simp only [Option.getD_some, Option.getD_none, Quad.toSpec_combine_zero] + +theorem leaf_spec [Binomial4 p] (evals : ByteArray) (ω s : FastField p) (z : Ext4 p) + (width lo hi : ℕ) (h) + (hz : ∀ R, lo ≤ R → R < hi → + Ext4.toSpec z ≠ Ext.ofBase (s * ω ^ R)) : + (leaf evals ω s z width lo hi h).size = width ∧ + ∀ c < width, sumAt z (leaf evals ω s z width lo hi h) c = + ∑ R ∈ Finset.Ico lo hi, rowTerm evals width ω s z R c := by + have hpack : ByteArray.mk (Array.replicate (16 * blockRows) 0) = + PackedRepr.encode (Array.replicate (4 * blockRows) (0 : UInt32)) := by + rw [← Native.zeroBuffer_eq_encodeWords, ← Nat.mul_assoc] + have hmin : min hi (lo + (hi - lo + blockRows - 1) / blockRows * blockRows) = hi := by + simp only [blockRows]; have := h.1; omega + unfold leaf + simp only [hpack] + obtain ⟨hsz, hval⟩ := blocks_spec evals ω s z width hi ((hi - lo + blockRows - 1) / blockRows) lo + (s * Montgomery.Native32.pow ω lo) (Array.replicate (4 * blockRows) 0) + (Array.replicate width 0) + ⟨h.1, h.2.1, h.2.2, by rw [PackedRepr.size_encode_uint32, Array.size_replicate]; ring⟩ + (Array.size_replicate) (congrArg (s * ·) (pow_eq_npow ω lo)) Array.size_replicate hz + refine ⟨hsz, fun c hc ↦ ?_⟩ + rw [hval c hc, sumAt_replicate_zero, zero_add, hmin] + +theorem sumAt_join [Binomial4 p] (z : Ext4 p) (a b : Array (Quad p)) (c : ℕ) (ha : c < a.size) + (hb : c < b.size) : + sumAt z (Array.zipWith (· + ·) a b) c = sumAt z a c + sumAt z b c := by + simp only [sumAt, Array.getD_eq_getD_getElem?, Array.getElem?_zipWith, + Array.getElem?_eq_getElem ha, Array.getElem?_eq_getElem hb, + Option.getD_some, Quad.toSpec_combine_add] + +theorem tree_spec [Binomial4 p] (evals : ByteArray) (ω s : FastField p) (z : Ext4 p) + (width depth lo hi : ℕ) (h) + (hz : ∀ R, lo ≤ R → R < hi → + Ext4.toSpec z ≠ Ext.ofBase (s * ω ^ R)) : + (tree evals ω s z width lo hi h depth).size = width ∧ + ∀ c < width, sumAt z (tree evals ω s z width lo hi h depth) c = + ∑ R ∈ Finset.Ico lo hi, rowTerm evals width ω s z R c := by + induction depth generalizing lo hi with + | zero => exact leaf_spec evals ω s z width lo hi h hz + | succ depth ih => + have hm : lo ≤ lo + (hi - lo) / 2 ∧ lo + (hi - lo) / 2 ≤ hi := by have := h.1; omega + obtain ⟨hs1, hv1⟩ := ih lo (lo + (hi - lo) / 2) ⟨hm.1, by + have := Nat.mul_le_mul_right width hm.2; omega, h.2.2⟩ (fun R h1 h2 ↦ hz R h1 (by omega)) + obtain ⟨hs2, hv2⟩ := ih (lo + (hi - lo) / 2) hi ⟨hm.2, h.2⟩ (fun R h1 h2 ↦ hz R (by omega) h2) + simp only [tree, join, Task.spawn] + refine ⟨by rw [Array.size_zipWith, hs1, hs2, Nat.min_self], fun c hc ↦ ?_⟩ + rw [sumAt_join z _ _ c (by rw [hs1]; exact hc) (by rw [hs2]; exact hc), hv1 c hc, hv2 c hc, + Finset.sum_Ico_consecutive _ hm.1 hm.2] + +end InterpolatePacked + +open InterpolatePacked + +theorem getD_decodeWords (b : ByteArray) (k : ℕ) (hk : k < b.size / 4) : + (decodeWords p b).getD k 0 = ofWordMod (b.getUInt32LE! (4 * k)) := by + rw [decodeWords, Array.getD_eq_getD_getElem?, Array.getElem?_ofFn] + simp only [hk, ↓reduceDIte, Option.getD_some] + +/-- **Correctness of `interpolateCosetPacked`.** For a point off the coset, it computes +`interpolateCoset` on the matrix's words, read as residues. -/ +theorem interpolateCosetPacked_eq [Binomial4 p] [PackedInterpolation p] (logN : ℕ) + (ω s : FastField p) (width : ℕ) (evals : ByteArray) + (z : Ext4 p) (logTasks : ℕ) + (hz : ∀ i < 2 ^ logN, Ext4.toSpec z ≠ Ext.ofBase (s * ω ^ i)) : + interpolateCosetPacked logN ω s width evals z logTasks = + interpolateCoset logN ω s width (decodeWords p evals) z := by + rw [interpolateCosetPacked, PackedInterpolation.run_eq] + unfold InterpolatePacked.run + split + · rename_i h + obtain ⟨hsz, hval⟩ := tree_spec evals ω s z width (min logTasks (logN - 9)) 0 (2 ^ logN) + ⟨Nat.zero_le _, h.1.ge, h.2⟩ (fun R _ hR ↦ hz R hR) + apply Array.ext + · rw [Array.size_map, hsz, size_interpolateCoset] + · intro j hj1 hj2 + have hj : j < width := by rw [Array.size_map, hsz] at hj1; exact hj1 + apply Ext4.toSpec_injective + rw [Array.getElem_map, Ext4.toSpec_mul, + toSpec_getElem_interpolateCoset logN ω s width _ z hz j hj] + congr 1 + have hs : Ext4.toSpec (Quad.combine z (tree evals ω s z width 0 (2 ^ logN) + ⟨Nat.zero_le _, h.1.ge, h.2⟩ (min logTasks (logN - 9)))[j]) = + sumAt z (tree evals ω s z width 0 (2 ^ logN) ⟨Nat.zero_le _, h.1.ge, h.2⟩ + (min logTasks (logN - 9))) j := by + rw [sumAt, Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem, Option.getD_some] + rw [hs, hval j hj, Finset.range_eq_Ico] + refine Finset.sum_congr rfl fun R hR ↦ ?_ + have hR' := (Finset.mem_Ico.mp hR).2 + have hk : R * width + j < evals.size / 4 := by + rw [h.1, Nat.mul_div_cancel_left _ (by decide)] + have := Nat.mul_le_mul_right width (show R + 1 ≤ 2 ^ logN from hR') + rw [Nat.add_mul, Nat.one_mul] at this + omega + rw [rowTerm, getD_decodeWords evals _ hk] + rfl + · rfl + +theorem size_columns1 (evals buf : ByteArray) (width row b : ℕ) (hfit) (k c : ℕ) (hc) + (sums : Array (Quad p)) : + (columns1 p evals buf width row b hfit k c hc sums).size = sums.size := by + induction k generalizing c sums with + | zero => rfl + | succ k ih => rw [columns1, ih, addColumn, Array.size_modify] + +theorem size_columns4 (evals buf : ByteArray) (width row b : ℕ) (hfit) (g c : ℕ) (hc) + (sums : Array (Quad p)) : + (columns4 p evals buf width row b hfit g c hc sums).size = sums.size := by + induction g generalizing c sums with + | zero => rfl + | succ g ih => simp only [columns4, ih, addColumns4, Array.size_modify] + +theorem size_blocks [Binomial4 p] (evals : ByteArray) (ω z0 k0 k1 z2x2 : FastField p) + (width hi k row : ℕ) + (x : FastField p) (buf : ByteArray) (sums : Array (Quad p)) (h) : + (blocks evals ω z0 k0 k1 z2x2 width hi k row x buf sums h).size = sums.size := by + induction k generalizing row x buf sums with + | zero => rfl + | succ k ih => simp only [blocks, ih, size_columns1, size_columns4] + +theorem size_tree [Binomial4 p] (evals : ByteArray) (ω s : FastField p) (z : Ext4 p) + (width lo hi : ℕ) (h) + (depth : ℕ) : (tree evals ω s z width lo hi h depth).size = width := by + induction depth generalizing lo hi with + | zero => simp only [tree, leaf, size_blocks, Array.size_replicate] + | succ depth ih => simp only [tree, join, Task.spawn, Array.size_zipWith, ih, Nat.min_self] + +theorem size_interpolateCosetPacked [Binomial4 p] [PackedInterpolation p] (logN : ℕ) + (ω s : FastField p) (width : ℕ) (evals : ByteArray) + (z : Ext4 p) (logTasks : ℕ) : + (interpolateCosetPacked logN ω s width evals z logTasks).size = width := by + rw [interpolateCosetPacked, PackedInterpolation.run_eq] + unfold InterpolatePacked.run + split + · rw [Array.size_map, size_tree] + · exact size_interpolateCoset _ _ _ _ _ _ + +/-- If column `j` of the words holds the values of `pⱼ`, of degree below `n`, on the coset +`s · ⟨ω⟩`, and `z` lies outside the coset, then output `j` is `pⱼ(z)`. -/ +theorem toSpec_interpolateCosetPacked [Binomial4 p] [PackedInterpolation p] + (D : CPolynomial.NTT.Domain (FastField p)) + (s : FastField p) (hs : s ≠ 0) (width : ℕ) (evals : ByteArray) (z : Ext4 p) (logTasks : ℕ) + (hz : Ext4.toSpec z ^ D.n ≠ algebraMap (FastField p) (Ext4.Spec p) s ^ D.n) + (f : Fin width → (FastField p)[X]) (hf : ∀ j, (f j).degree < D.n) + (hevals : ∀ (i : D.Idx) (j : Fin width), + (decodeWords p evals).getD (i * width + j) 0 = (f j).eval (s * D.node i)) + (j : Fin width) : + Ext4.toSpec ((interpolateCosetPacked D.logN D.omega s width evals z logTasks)[j.val]'(by + rw [size_interpolateCosetPacked]; exact j.isLt)) = + Polynomial.aeval (Ext4.toSpec z) (f j) := by + simp only [interpolateCosetPacked_eq D.logN D.omega s width evals z logTasks + (toSpec_ne_cosetNode D s z hz)] + exact toSpec_interpolateCoset D s hs width _ z hz f hf hevals j + +end Montgomery.Native32 diff --git a/CompPoly/Fields/Montgomery/Native32.lean b/CompPoly/Fields/Montgomery/Native32.lean index 0a4ebed7..96f1d966 100644 --- a/CompPoly/Fields/Montgomery/Native32.lean +++ b/CompPoly/Fields/Montgomery/Native32.lean @@ -1,7 +1,7 @@ /- Copyright (c) 2026 CompPoly Contributors. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Valerii Huhnin, Georgios Raikos +Authors: Valerii Huhnin, Georgios Raikos, Gregor Mitscha-Baude -/ module @@ -30,6 +30,7 @@ def reduceQuotient (negInv : UInt32) (p x : UInt64) : UInt32 := ((x + (x.toUInt32 * negInv).toUInt64 * p) >>> 32).toUInt32 /-- Conditional subtraction of the modulus in 32-bit Montgomery reduction. -/ +@[inline] def conditionalSubtract (p32 : UInt32) (u : UInt32) : UInt32 := if u < p32 then u else u - p32 diff --git a/CompPoly/Fields/Montgomery/Native32Bytes.lean b/CompPoly/Fields/Montgomery/Native32Bytes.lean index df46d2ac..4a7f350d 100644 --- a/CompPoly/Fields/Montgomery/Native32Bytes.lean +++ b/CompPoly/Fields/Montgomery/Native32Bytes.lean @@ -7,6 +7,7 @@ module public import CompPoly.Fields.Montgomery.Native32Field public import CompPoly.Data.Bytes.CanonicalNat +public import CompPoly.Data.ByteArray.Packed /-! # Serialization of the 32-bit Montgomery carrier @@ -44,6 +45,18 @@ instance : CanonicalNat (FastField modulus) := instance : ByteCodec (FastField modulus) := ByteCodec.ofCanonicalNat _ +/-- The stored Montgomery residue as a raw word; words at or above the modulus decode to +zero. -/ +@[inline] instance : PackedRepr (FastField modulus) := + PackedRepr.ofWord32 (fun x ↦ x.1) + (fun v ↦ if h : v < P.modulus32 then ⟨v, by simpa [UInt32.lt_iff_toNat_lt] using h⟩ + else ⟨0, by simp⟩) + (fun x ↦ by + have hx : x.1 < P.modulus32 := by + rw [UInt32.lt_iff_toNat_lt, P.modulus32_toNat]; exact x.2 + simp only [hx, ↓reduceDIte] + rfl) + @[simp] theorem canonicalNat_bound : CanonicalNat.bound (FastField modulus) = modulus := rfl @[simp] theorem canonicalNat_toNat (x : FastField modulus) : CanonicalNat.toNat x = toNat x := rfl diff --git a/CompPoly/Fields/Montgomery/Native32Ext4.lean b/CompPoly/Fields/Montgomery/Native32Ext4.lean new file mode 100644 index 00000000..f774eff4 --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native32Ext4.lean @@ -0,0 +1,417 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Extension +public import CompPoly.Fields.Extension.Bytes +public import CompPoly.Fields.Montgomery.Native32Bytes + +/-! +# Fast quartic binomial extensions of 32-bit Montgomery fields + +For a prime `p` with a `Mont32Field p` instance, a `Binomial4 p` instance chooses `W` with +`X^4 - W` irreducible over `FastField p`. `Ext4 p` is then `FastField p [X] / (X^4 - W)` with four +Montgomery coefficients, Plonky3's `BinomialExtensionField`. + +* Multiplication reduces each output coefficient once: it is a sum of four products, with `W` + folded into the left factors, and `mulAdd4` reduces such a sum with a single Montgomery step. + `Binomial4.mulW` multiplies by `W`, so a field can supply a cheaper form than a product, such + as `x + x + x` for `W = 3`. +* Inversion uses the norm to the base field. Writing `a = α + X β` with `α, β ∈ F(X²)`, + `a · σ(a) = g₀ + g₁ X²` for the automorphism `σ : X ↦ -X`, and `N = g₀² - W g₁²`, so + `a⁻¹ = σ(a) (g₀ - g₁ X²) N⁻¹` costs one base-field inversion. `N ≠ 0` for `a ≠ 0` because `W` is + not a square, which follows from the irreducibility of `X^4 - W`. + +The spec field is `Ext4.Spec p`, the framework's `Ext` over the fast base field itself for the +binomial `X^4 - W` (`Ext4.params p`). `toSpec` maps into it, `toSpec_add`, `toSpec_mul`, +`toSpec_inv`, … show that every operation agrees with it, and `specEquiv` packages it as a ring +isomorphism. The byte encoding is the four coefficient encodings in ascending order, as for the +spec field and Plonky3 (`toBytes_toSpec`). +-/ + +@[expose] public section + +namespace Montgomery.Native32 + +open CompPoly CompPoly.Extension Polynomial + +/-- An irreducible binomial `X^4 - W` over `FastField p`, with a fast multiplication by `W`. -/ +class Binomial4 (p : ℕ) [Mont32Field p] where + /-- The extension adjoins a fourth root of `W`. -/ + W : FastField p + /-- Multiplication by `W`, by default the product. -/ + mulW : FastField p → FastField p := (W * ·) + mulW_eq : ∀ x, mulW x = W * x := by intro; rfl + /-- `X^4 - W` is irreducible, so the quotient is a field. -/ + irreducible : Irreducible (X ^ 4 - C W : (FastField p)[X]) + +/-- `c0 + c1 X + c2 X^2 + c3 X^3` in `FastField p [X] / (X^4 - W)`. -/ +structure Ext4 (p : ℕ) [Mont32Field p] where + c0 : FastField p + c1 : FastField p + c2 : FastField p + c3 : FastField p +deriving DecidableEq + +namespace Ext4 + +variable {p : ℕ} [Mont32Field p] + +instance : Zero (Ext4 p) := ⟨⟨0, 0, 0, 0⟩⟩ +instance : One (Ext4 p) := ⟨⟨1, 0, 0, 0⟩⟩ +instance : Inhabited (Ext4 p) := ⟨0⟩ +instance : Add (Ext4 p) := ⟨fun a b ↦ ⟨a.c0 + b.c0, a.c1 + b.c1, a.c2 + b.c2, a.c3 + b.c3⟩⟩ +instance : Neg (Ext4 p) := ⟨fun a ↦ ⟨-a.c0, -a.c1, -a.c2, -a.c3⟩⟩ +instance : Sub (Ext4 p) := ⟨fun a b ↦ ⟨a.c0 - b.c0, a.c1 - b.c1, a.c2 - b.c2, a.c3 - b.c3⟩⟩ + +/-- A base-field element. -/ +@[inline] def ofBase (x : FastField p) : Ext4 p := ⟨x, 0, 0, 0⟩ + +/-- Multiplication by a base-field element. -/ +@[inline] def smul (x : FastField p) (a : Ext4 p) : Ext4 p := + ⟨x * a.c0, x * a.c1, x * a.c2, x * a.c3⟩ + +theorem ext' {a b : Ext4 p} (h0 : a.c0 = b.c0) (h1 : a.c1 = b.c1) (h2 : a.c2 = b.c2) + (h3 : a.c3 = b.c3) : a = b := by + cases a; cases b; simp_all + +variable [B : Binomial4 p] + +open Binomial4 (W mulW mulW_eq) + +/-- Product modulo `X^4 - W`, one Montgomery reduction per output coefficient. -/ +@[inline] def mul (a b : Ext4 p) : Ext4 p := + let w1 := mulW a.c1 + let w2 := mulW a.c2 + let w3 := mulW a.c3 + ⟨mulAdd4 a.c0 b.c0 w1 b.c3 w2 b.c2 w3 b.c1, + mulAdd4 a.c0 b.c1 a.c1 b.c0 w2 b.c3 w3 b.c2, + mulAdd4 a.c0 b.c2 a.c1 b.c1 a.c2 b.c0 w3 b.c3, + mulAdd4 a.c0 b.c3 a.c1 b.c2 a.c2 b.c1 a.c3 b.c0⟩ + +instance : Mul (Ext4 p) := ⟨mul⟩ + +/-- The constant coordinate of `a · σ(a)` for `σ : X ↦ -X`. -/ +@[inline] def g0 (a : Ext4 p) : FastField p := + a.c0 * a.c0 + mulW (a.c2 * a.c2) - mulW (a.c1 * a.c3 + a.c1 * a.c3) + +/-- The `X²` coordinate of `a · σ(a)`. -/ +@[inline] def g1 (a : Ext4 p) : FastField p := + a.c0 * a.c2 + a.c0 * a.c2 - a.c1 * a.c1 - mulW (a.c3 * a.c3) + +/-- The norm `g₀² - W g₁²` of `a` to the base field. -/ +@[inline] def norm (a : Ext4 p) : FastField p := g0 a * g0 a - mulW (g1 a * g1 a) + +/-- Inverse through the norm to the base field: `a · σ(a) = g₀ + g₁ X²` for `σ : X ↦ -X`, and +`a⁻¹ = σ(a) (g₀ - g₁ X²) / (g₀² - W g₁²)`. The zero element maps to zero. -/ +@[inline] def inv (a : Ext4 p) : Ext4 p := + let g0 := g0 a + let g1 := g1 a + let ni := (norm a)⁻¹ + ⟨(a.c0 * g0 - mulW (a.c2 * g1)) * ni, (mulW (a.c3 * g1) - a.c1 * g0) * ni, + (a.c2 * g0 - a.c0 * g1) * ni, (a.c1 * g1 - a.c3 * g0) * ni⟩ + +instance : Inv (Ext4 p) := ⟨inv⟩ +instance : Div (Ext4 p) := ⟨fun a b ↦ a * b⁻¹⟩ + +/-! ### The spec field -/ + +variable (p) in +/-- The binomial `X^4 - W` as extension parameters over the fast base field. -/ +def params : BinomialParams (FastField p) where + d := 4 + W := W + two_le := by norm_num + q := p + + +variable (p) in +/-- The spec field's parameters, the binomial as a monic modulus. -/ +abbrev P : ExtensionParams (FastField p) := (params p).toExtensionParams + +variable (p) in +/-- The spec field `FastField p [X] / (X^4 - W)`. -/ +abbrev Spec : Type := Ext (P p) + +theorem d_eq : (P p).d = 4 := rfl + +theorem params_poly : (params p).poly = X ^ 4 - C W := rfl + +instance : Fact (Nat.card (FastField p) = (params p).q) := ⟨natCard_fastField⟩ + +instance : Fact (Irreducible (params p).poly) := ⟨B.irreducible⟩ + +/-- The irreducibility fact in the form the framework's `Field` instance consumes. -/ +instance : Fact (Irreducible (params p).toExtensionParams.poly) := + ⟨(params p).toExtensionParams_poly ▸ B.irreducible⟩ + +@[simp] theorem card_spec : Fintype.card (Spec p) = p ^ 4 := Ext.card_ext + +/-- The `k`-th coordinate. -/ +def coord (a : Ext4 p) : ℕ → FastField p + | 0 => a.c0 + | 1 => a.c1 + | 2 => a.c2 + | _ => a.c3 + +/-- The same element of the spec field. -/ +def toSpec (a : Ext4 p) : Spec p := Ext.ofFn (P := P p) fun i ↦ coord a i + +/-- The fast element with the spec field element's coordinates. -/ +def ofSpec (x : Spec p) : Ext4 p := + ⟨Ext.coeffNat x 0, Ext.coeffNat x 1, Ext.coeffNat x 2, Ext.coeffNat x 3⟩ + +theorem coeffNat_toSpec (a : Ext4 p) {k : ℕ} (hk : k < 4) : + Ext.coeffNat (toSpec a) k = coord a k := by + rw [Ext.coeffNat_of_lt _ (by rw [d_eq]; exact hk)] + simp only [toSpec, Ext.coeff_ofFn] + +/-- Two spec elements agree when their first four coordinates do. -/ +theorem spec_ext {x y : Spec p} (h : ∀ k < 4, Ext.coeffNat x k = Ext.coeffNat y k) : + x = y := by + ext ⟨k, hk⟩ + have := h k hk + rwa [Ext.coeffNat_of_lt _ hk, Ext.coeffNat_of_lt _ hk] at this + +theorem toSpec_ofSpec (x : Spec p) : toSpec (ofSpec x) = x := by + apply spec_ext + intro k hk + rw [coeffNat_toSpec _ hk] + interval_cases k <;> rfl + +theorem ofSpec_toSpec (a : Ext4 p) : ofSpec (toSpec a) = a := by + apply ext' + · rw [ofSpec, coeffNat_toSpec a (by decide : 0 < 4)]; rfl + · rw [ofSpec, coeffNat_toSpec a (by decide : 1 < 4)]; rfl + · rw [ofSpec, coeffNat_toSpec a (by decide : 2 < 4)]; rfl + · rw [ofSpec, coeffNat_toSpec a (by decide : 3 < 4)]; rfl + +theorem toSpec_injective : Function.Injective (toSpec (p := p)) := + Function.LeftInverse.injective ofSpec_toSpec + +/-- The coordinates' image in the quotient. -/ +local notation "ι" => algebraMap (FastField p) (AdjoinRoot (ExtensionParams.poly (P p))) + +/-- The coordinates of `toSpec a` in the adjoined-root quotient. -/ +theorem toQuot_toSpec (a : Ext4 p) : + Ext.toQuot (toSpec a) = + ι a.c0 + ι a.c1 * Ext.rt (P p) + ι a.c2 * Ext.rt (P p) ^ 2 + ι a.c3 * Ext.rt (P p) ^ 3 := by + rw [Ext.toQuot_rangeForm, d_eq] + simp only [Finset.sum_range_succ, Finset.sum_range_zero, pow_zero, pow_one, mul_one, zero_add] + rw [coeffNat_toSpec a (by decide : 0 < 4), coeffNat_toSpec a (by decide : 1 < 4), + coeffNat_toSpec a (by decide : 2 < 4), coeffNat_toSpec a (by decide : 3 < 4)] + rfl + +/-- The defining relation of the root in the quotient. -/ +theorem rt_pow_four : Ext.rt (P p) ^ 4 = ι W := Ext.rt_pow_d_binomial (params p) + +@[simp] theorem toSpec_zero : toSpec (0 : Ext4 p) = 0 := by + apply spec_ext; intro k hk + have hk' : k < (P p).d := by rw [d_eq]; exact hk + rw [coeffNat_toSpec _ hk, Ext.coeffNat_of_lt _ hk', Ext.coeff_zero] + interval_cases k <;> rfl + +@[simp] theorem toSpec_one : toSpec (1 : Ext4 p) = 1 := by + apply spec_ext; intro k hk + have hk' : k < (P p).d := by rw [d_eq]; exact hk + rw [coeffNat_toSpec _ hk, Ext.coeffNat_of_lt _ hk', Ext.coeff_one] + interval_cases k <;> rfl + +@[simp] theorem toSpec_add (a b : Ext4 p) : toSpec (a + b) = toSpec a + toSpec b := by + apply spec_ext; intro k hk + have hk' : k < (P p).d := by rw [d_eq]; exact hk + rw [coeffNat_toSpec _ hk, Ext.coeffNat_of_lt _ hk', Ext.coeff_add, + ← Ext.coeffNat_of_lt _ hk', ← Ext.coeffNat_of_lt _ hk', + coeffNat_toSpec _ hk, coeffNat_toSpec _ hk] + interval_cases k <;> rfl + +@[simp] theorem toSpec_neg (a : Ext4 p) : toSpec (-a) = -toSpec a := by + apply spec_ext; intro k hk + have hk' : k < (P p).d := by rw [d_eq]; exact hk + rw [coeffNat_toSpec _ hk, Ext.coeffNat_of_lt _ hk', Ext.coeff_neg, + ← Ext.coeffNat_of_lt _ hk', coeffNat_toSpec _ hk] + interval_cases k <;> rfl + +@[simp] theorem toSpec_sub (a b : Ext4 p) : toSpec (a - b) = toSpec a - toSpec b := by + apply spec_ext; intro k hk + have hk' : k < (P p).d := by rw [d_eq]; exact hk + rw [coeffNat_toSpec _ hk, Ext.coeffNat_of_lt _ hk', Ext.coeff_sub, + ← Ext.coeffNat_of_lt _ hk', ← Ext.coeffNat_of_lt _ hk', + coeffNat_toSpec _ hk, coeffNat_toSpec _ hk] + interval_cases k <;> rfl + +@[simp] theorem toSpec_mul (a b : Ext4 p) : toSpec (a * b) = toSpec a * toSpec b := by + apply Ext.toQuot_injective + rw [Ext.toQuot_mul, toQuot_toSpec, toQuot_toSpec, toQuot_toSpec] + simp only [show a * b = mul a b from rfl, mul, mulAdd4_eq, mulW_eq, map_add, map_mul] + linear_combination -(ι a.c1 * ι b.c3 + ι a.c2 * ι b.c2 + ι a.c3 * ι b.c1 + + (ι a.c2 * ι b.c3 + ι a.c3 * ι b.c2) * Ext.rt (P p) + ι a.c3 * ι b.c3 * Ext.rt (P p) ^ 2) * + rt_pow_four (p := p) + +/-! ### Inversion -/ + +/-- `W` is not a square: otherwise `X^4 - W = (X^2 - x) (X^2 + x)` would factor. -/ +theorem W_not_square (x : FastField p) : x ^ 2 ≠ W := by + intro hx + have h : (X ^ 4 - C W : (FastField p)[X]) = (X ^ 2 - C x) * (X ^ 2 + C x) := by + rw [← hx, C_pow]; ring + rcases B.irreducible.isUnit_or_isUnit h with hu | hu + · have := natDegree_eq_zero_of_isUnit hu + rw [natDegree_X_pow_sub_C] at this; omega + · have := natDegree_eq_zero_of_isUnit hu + rw [natDegree_X_pow_add_C] at this; omega + +/-- The automorphism `X ↦ -X` of the quotient field. -/ +noncomputable def sigma : AdjoinRoot (P p).poly →ₐ[FastField p] AdjoinRoot (P p).poly := + AdjoinRoot.liftAlgHom (P p).poly (Algebra.ofId _ _) (-Ext.rt (P p)) (by + conv_lhs => arg 3; rw [(params p).toExtensionParams_poly, params_poly] + rw [eval₂_sub, eval₂_X_pow, eval₂_C, neg_pow, + show ((-1 : AdjoinRoot (P p).poly) ^ 4) = 1 by norm_num, one_mul, rt_pow_four] + exact sub_self _) + +theorem sigma_rt : sigma (Ext.rt (P p)) = -Ext.rt (P p) := AdjoinRoot.liftAlgHom_root _ _ _ _ + +theorem g0_eq (a : Ext4 p) : g0 a = a.c0 ^ 2 + W * a.c2 ^ 2 - 2 * W * a.c1 * a.c3 := by + simp only [g0, mulW_eq]; ring + +theorem g1_eq (a : Ext4 p) : g1 a = 2 * a.c0 * a.c2 - a.c1 ^ 2 - W * a.c3 ^ 2 := by + simp only [g1, mulW_eq]; ring + +theorem norm_eq (a : Ext4 p) : norm a = g0 a ^ 2 - W * g1 a ^ 2 := by + simp only [norm, mulW_eq]; ring + +/-- `a · σ(a)` lies in `F(X²)`. -/ +theorem toQuot_mul_sigma (a : Ext4 p) : + Ext.toQuot (toSpec a) * sigma (Ext.toQuot (toSpec a)) = + ι (g0 a) + ι (g1 a) * Ext.rt (P p) ^ 2 := by + rw [toQuot_toSpec, g0_eq, g1_eq] + simp only [map_add, map_mul, map_pow, map_sub, AlgHom.commutes, sigma_rt, map_ofNat] + linear_combination (ι a.c2 ^ 2 - 2 * ι a.c1 * ι a.c3 - ι a.c3 ^ 2 * Ext.rt (P p) ^ 2) * + rt_pow_four (p := p) + +/-- The norm of a nonzero element is nonzero. -/ +theorem norm_ne_zero (a : Ext4 p) (ha : toSpec a ≠ 0) : norm a ≠ 0 := by + have hA : Ext.toQuot (toSpec a) ≠ 0 := by + rwa [Ne, ← Ext.toQuot_zero, Ext.toQuot_inj] + have hσ : sigma (Ext.toQuot (toSpec a)) ≠ 0 := by + rw [Ne, ← map_zero sigma] + exact fun h ↦ + hA ((sigma (p := p) : AdjoinRoot (P p).poly →+* AdjoinRoot (P p).poly).injective h) + have hγ : ι (g0 a) + ι (g1 a) * Ext.rt (P p) ^ 2 ≠ 0 := by + rw [← toQuot_mul_sigma]; exact mul_ne_zero hA hσ + rw [norm_eq] + intro hN + by_cases hg1 : g1 a = 0 + · have hg0 : g0 a = 0 := by + rw [hg1] at hN; simpa using hN + exact hγ (by rw [hg0, hg1, map_zero, zero_mul, add_zero]) + · apply W_not_square (g0 a * (g1 a)⁻¹) + field_simp + linear_combination hN + +/-- `a` times its inverse is `N · N⁻¹` for the norm `N`. -/ +theorem toQuot_mul_inv (a : Ext4 p) : + Ext.toQuot (toSpec a) * Ext.toQuot (toSpec a⁻¹) = ι (norm a) * ι (norm a)⁻¹ := by + rw [toQuot_toSpec, toQuot_toSpec] + simp only [show a⁻¹ = inv a from rfl, inv, mulW_eq, map_mul, map_sub] + rw [norm_eq] + set x0 := ι a.c0 + set x1 := ι a.c1 + set x2 := ι a.c2 + set x3 := ι a.c3 + set w := ι W + set y0 := ι (g0 a) + set y1 := ι (g1 a) + have h0 : y0 = x0 ^ 2 + w * x2 ^ 2 - 2 * w * x1 * x3 := by + simp only [y0, x0, x1, x2, x3, w, g0_eq, map_add, map_sub, map_mul, map_pow, map_ofNat] + have h1 : y1 = 2 * x0 * x2 - x1 ^ 2 - w * x3 ^ 2 := by + simp only [y1, x0, x1, x2, x3, w, g1_eq, map_sub, map_mul, map_pow, map_ofNat] + set n := ι (g0 a ^ 2 - W * g1 a ^ 2)⁻¹ + have hn : ι (g0 a ^ 2 - W * g1 a ^ 2) = y0 ^ 2 - w * y1 ^ 2 := by + simp only [y0, y1, w, map_sub, map_mul, map_pow] + rw [hn, h0, h1] + set r := Ext.rt (P p) + linear_combination n * (-(2 * x0 * x2 - x1 ^ 2 - w * x3 ^ 2) ^ 2 + + (x2 ^ 2 - 2 * x1 * x3 - x3 ^ 2 * r ^ 2) * + ((x0 ^ 2 + w * x2 ^ 2 - 2 * w * x1 * x3) - (2 * x0 * x2 - x1 ^ 2 - w * x3 ^ 2) * r ^ 2) - + (x0 + x1 * r + x2 * r ^ 2 + x3 * r ^ 3) * + (-x2 * (2 * x0 * x2 - x1 ^ 2 - w * x3 ^ 2) + x3 * (2 * x0 * x2 - x1 ^ 2 - w * x3 ^ 2) * r)) * + rt_pow_four (p := p) + +theorem inv_zero' : (0 : Ext4 p)⁻¹ = 0 := by + have hN : norm (0 : Ext4 p) = 0 := by + rw [norm_eq, g0_eq, g1_eq] + simp only [show (0 : Ext4 p).c0 = 0 from rfl, show (0 : Ext4 p).c1 = 0 from rfl, + show (0 : Ext4 p).c2 = 0 from rfl, show (0 : Ext4 p).c3 = 0 from rfl] + ring + simp only [show (0 : Ext4 p)⁻¹ = inv 0 from rfl, inv, hN, inv_zero, mul_zero] + rfl + +theorem toSpec_inv (a : Ext4 p) : toSpec a⁻¹ = (toSpec a)⁻¹ := by + by_cases ha : toSpec a = 0 + · obtain rfl : a = 0 := toSpec_injective (by rw [ha, toSpec_zero]) + rw [toSpec_zero, _root_.inv_zero, inv_zero', toSpec_zero] + · have hN := norm_ne_zero a ha + apply eq_inv_of_mul_eq_one_right + apply Ext.toQuot_injective + rw [Ext.toQuot_mul, toQuot_mul_inv, Ext.toQuot_one, ← map_mul, mul_inv_cancel₀ hN, map_one] + +@[simp] theorem toSpec_div (a b : Ext4 p) : toSpec (a / b) = toSpec a / toSpec b := by + rw [div_eq_mul_inv, ← toSpec_inv, ← toSpec_mul]; rfl + +@[simp] theorem toSpec_ofBase (x : FastField p) : toSpec (ofBase x) = Ext.ofBase x := by + apply spec_ext; intro k hk + have hk' : k < (P p).d := by rw [d_eq]; exact hk + rw [coeffNat_toSpec _ hk, Ext.coeffNat_of_lt _ hk', Ext.coeff_ofBase] + interval_cases k <;> rfl + +@[simp] theorem toSpec_smul (x : FastField p) (a : Ext4 p) : toSpec (smul x a) = x • toSpec a := by + apply spec_ext; intro k hk + have hk' : k < (P p).d := by rw [d_eq]; exact hk + rw [coeffNat_toSpec _ hk, Ext.coeffNat_of_lt _ hk', Ext.coeff_smul, + ← Ext.coeffNat_of_lt _ hk', coeffNat_toSpec _ hk] + interval_cases k <;> rfl + +variable (p) in +/-- The fast and spec quartic extensions are isomorphic rings. -/ +def specEquiv : Ext4 p ≃+* Spec p where + toFun := toSpec + invFun := ofSpec + left_inv := ofSpec_toSpec + right_inv := toSpec_ofSpec + map_mul' := toSpec_mul + map_add' := toSpec_add + +@[simp] theorem specEquiv_apply (a : Ext4 p) : specEquiv p a = toSpec a := rfl + +/-! ### Encoding -/ + +omit B in +/-- The four coefficient encodings in ascending order. -/ +instance : ByteCodec (Ext4 p) where + width := 4 * ByteCodec.width (FastField p) + toBytes a := ByteCodec.toBytes (#v[a.c0, a.c1, a.c2, a.c3] : Vector (FastField p) 4) + ofBytes? b := + (ByteCodec.ofBytes? b : Option (Vector (FastField p) 4)).map fun v ↦ ⟨v[0], v[1], v[2], v[3]⟩ + ofBytes?_toBytes a := by simp only [ByteCodec.ofBytes?_toBytes, Option.map_some]; rfl + +/-- The fast and spec quartic extensions encode the same element identically. -/ +theorem toBytes_toSpec (a : Ext4 p) : ByteCodec.toBytes (toSpec a) = ByteCodec.toBytes a := by + have hc : (toSpec a).coeffs.toList = [a.c0, a.c1, a.c2, a.c3] := by + apply List.ext_getElem (by rw [Vector.length_toList]; rfl) + intro k hk _ + have hk4 : k < 4 := by rw [Vector.length_toList] at hk; exact hk + have e : (toSpec a).coeffs.toList[k]'hk = Ext.coeffNat (toSpec a) k := by + rw [Vector.getElem_toList, Ext.coeffNat_of_lt _ (by rw [d_eq]; exact hk4)]; rfl + rw [e, coeffNat_toSpec _ hk4] + interval_cases k <;> rfl + apply Vector.toList_inj.mp + rw [Ext.toBytes_eq, ByteCodec.toList_toBytes_vector, hc] + rfl + +end Ext4 + +end Montgomery.Native32 diff --git a/CompPoly/Fields/Montgomery/Native32Field.lean b/CompPoly/Fields/Montgomery/Native32Field.lean index 8389d537..a5fde810 100644 --- a/CompPoly/Fields/Montgomery/Native32Field.lean +++ b/CompPoly/Fields/Montgomery/Native32Field.lean @@ -492,6 +492,207 @@ theorem toField_mul (x y : FastField modulus) : toField (x * y) = toField x * to toField_eq_val_toNat_cast_mul_inv (x := y), mul_def, mul_val_toNat_cast] ring +/-- The unreduced 64-bit product of two Montgomery words, below `modulus ^ 2`. -/ +@[inline] +def mulWide (x y : FastField modulus) : UInt64 := x.val.toUInt64 * y.val.toUInt64 + +theorem mulWide_toNat (x y : FastField modulus) : + (mulWide x y).toNat = x.val.toNat * y.val.toNat := by + simp only [mulWide, UInt64.toNat_mul, UInt32.toNat_toUInt64] + exact Nat.mod_eq_of_lt (by nlinarith [x.property, y.property, P.modulus_sq_lt_two_pow_64]) + +theorem modulus64_shift_toNat : (P.modulus64 <<< 32).toNat = modulus * 2 ^ 32 := by + rw [UInt64.toNat_shiftLeft, Mont32Field.modulus64_toNat, + show (32 : UInt64).toNat % 64 = 32 from rfl, Nat.shiftLeft_eq] + have := P.modulus_lt_two_pow_31 + exact Nat.mod_eq_of_lt (by omega) + +/-- Montgomery reduction for inputs below `2 * modulus * 2^32`, such as a sum of four products: +one conditional subtraction of `modulus * 2^32` brings them into `reduce`'s range. -/ +@[inline] +def reduceWide (x : UInt64) (h : x.toNat < 2 * (modulus * 2 ^ 32)) : FastField modulus := + if hx : x < P.modulus64 <<< 32 then + reduce x (by rw [UInt64.lt_iff_toNat_lt, modulus64_shift_toNat] at hx; exact hx) + else + reduce (x - P.modulus64 <<< 32) (by + rw [UInt64.lt_iff_toNat_lt, modulus64_shift_toNat] at hx + rw [UInt64.toNat_sub_of_le _ _ (by + rw [UInt64.le_iff_toNat_le, modulus64_shift_toNat]; omega), modulus64_shift_toNat] + omega) + +private theorem reduceWide_val_toNat_cast {x : UInt64} (h : x.toNat < 2 * (modulus * 2 ^ 32)) : + ((reduceWide x h).val.toNat : ZMod modulus) = + (x.toNat : ZMod modulus) * ((2 ^ 32 : ℕ) : ZMod modulus)⁻¹ := by + unfold reduceWide + split + · exact reduce_val_toNat_cast _ + · rename_i hx + rw [UInt64.lt_iff_toNat_lt, modulus64_shift_toNat] at hx + rw [reduce_val_toNat_cast, UInt64.toNat_sub_of_le _ _ (by + rw [UInt64.le_iff_toNat_le, modulus64_shift_toNat]; omega), modulus64_shift_toNat, + Nat.cast_sub (by omega), Nat.cast_mul, ZMod.natCast_self, zero_mul, sub_zero] + +theorem val_mul_val_lt (x y : FastField modulus) : + x.val.toNat * y.val.toNat < modulus * modulus := + Nat.mul_lt_mul'' x.property y.property + +/-- A sum of four products with a single Montgomery reduction. -/ +@[inline] +def mulAdd4 (a b c d e f g k : FastField modulus) : FastField modulus := + reduceWide (mulWide a b + mulWide c d + mulWide e f + mulWide g k) (by + have := P.modulus_lt_two_pow_31 + have h1 := mulWide_toNat a b; have h2 := mulWide_toNat c d + have h3 := mulWide_toNat e f; have h4 := mulWide_toNat g k + have b1 := val_mul_val_lt a b + have b2 := val_mul_val_lt c d + have b3 := val_mul_val_lt e f + have b4 := val_mul_val_lt g k + have hm : modulus * modulus * 4 ≤ 2 * (modulus * 2 ^ 32) := by nlinarith + simp only [UInt64.toNat_add, h1, h2, h3, h4] + rw [Nat.mod_eq_of_lt (by omega), Nat.mod_eq_of_lt (by omega), Nat.mod_eq_of_lt (by omega)] + omega) + +@[simp] +theorem toField_mulAdd4 (a b c d e f g k : FastField modulus) : + toField (mulAdd4 a b c d e f g k) = + toField a * toField b + toField c * toField d + toField e * toField f + + toField g * toField k := by + have := P.modulus_lt_two_pow_31 + have b1 := val_mul_val_lt a b + have b2 := val_mul_val_lt c d + have b3 := val_mul_val_lt e f + have b4 := val_mul_val_lt g k + have hm : modulus * modulus * 4 < 2 ^ 64 := by nlinarith + rw [toField_eq_val_toNat_cast_mul_inv, mulAdd4, reduceWide_val_toNat_cast] + simp only [UInt64.toNat_add, mulWide_toNat] + rw [Nat.mod_eq_of_lt (by omega), Nat.mod_eq_of_lt (by omega), Nat.mod_eq_of_lt (by omega)] + rw [Nat.cast_add, Nat.cast_add, Nat.cast_add, Nat.cast_mul, Nat.cast_mul, Nat.cast_mul, + Nat.cast_mul, val_toNat_cast_eq_toField_mul (x := a), val_toNat_cast_eq_toField_mul (x := b), + val_toNat_cast_eq_toField_mul (x := c), val_toNat_cast_eq_toField_mul (x := d), + val_toNat_cast_eq_toField_mul (x := e), val_toNat_cast_eq_toField_mul (x := f), + val_toNat_cast_eq_toField_mul (x := g), val_toNat_cast_eq_toField_mul (x := k)] + have hR := P.two_pow_32_ne_zero + generalize ((2 ^ 32 : ℕ) : ZMod modulus) = R at hR ⊢ + field_simp + +/-- `mulAdd4` is the sum of the four products. -/ +theorem mulAdd4_eq (a b c d e f g k : FastField modulus) : + mulAdd4 a b c d e f g k = a * b + c * d + e * f + g * k := + toField_injective (by simp only [toField_mulAdd4, toField_add, toField_mul]) + +/-- Add a product below `modulus * 2^32` to an accumulator below `modulus * 2^32`, keeping it +there with one conditional subtraction of `modulus * 2^32`: a dot product reduces only once. -/ +@[inline] +def accumulate (modulus : ℕ) [P : Mont32Field modulus] (acc prod : UInt64) : UInt64 := + let s := acc + prod + if s < P.modulus64 <<< 32 then s else s - P.modulus64 <<< 32 + +private theorem accumulate_spec {acc prod : UInt64} (ha : acc.toNat < modulus * 2 ^ 32) + (hp : prod.toNat < modulus * 2 ^ 32) : + (accumulate modulus acc prod).toNat < modulus * 2 ^ 32 ∧ + ((accumulate modulus acc prod).toNat : ZMod modulus) = acc.toNat + prod.toNat := by + have := P.modulus_lt_two_pow_31 + have hs : (acc + prod).toNat = acc.toNat + prod.toNat := by + rw [UInt64.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + unfold accumulate + dsimp only + split + · rename_i h + rw [UInt64.lt_iff_toNat_lt, modulus64_shift_toNat, hs] at h + exact ⟨by rw [hs]; exact h, by rw [hs, Nat.cast_add]⟩ + · rename_i h + rw [UInt64.lt_iff_toNat_lt, modulus64_shift_toNat, hs] at h + rw [UInt64.toNat_sub_of_le _ _ (by + rw [UInt64.le_iff_toNat_le, modulus64_shift_toNat, hs]; omega), + modulus64_shift_toNat, hs] + refine ⟨by omega, ?_⟩ + rw [Nat.cast_sub (by omega), Nat.cast_mul, ZMod.natCast_self, zero_mul, sub_zero, + Nat.cast_add] + +theorem accumulate_lt {acc prod : UInt64} (ha : acc.toNat < modulus * 2 ^ 32) + (hp : prod.toNat < modulus * 2 ^ 32) : (accumulate modulus acc prod).toNat < modulus * 2 ^ 32 := + (accumulate_spec ha hp).1 + +theorem accumulate_cast {acc prod : UInt64} (ha : acc.toNat < modulus * 2 ^ 32) + (hp : prod.toNat < modulus * 2 ^ 32) : + ((accumulate modulus acc prod).toNat : ZMod modulus) = acc.toNat + prod.toNat := + (accumulate_spec ha hp).2 + +/-- The product of any word and a residue fits an accumulator. -/ +theorem word_mul_val_lt (e : UInt32) (x : FastField modulus) : + (e.toUInt64 * x.val.toUInt64).toNat < modulus * 2 ^ 32 := by + have he := e.toNat_lt + have hx := x.property + have := P.modulus_lt_two_pow_31 + rw [UInt64.toNat_mul, UInt32.toNat_toUInt64, UInt32.toNat_toUInt64, + Nat.mod_eq_of_lt (by nlinarith)] + rw [Nat.mul_comm] + exact Nat.mul_lt_mul_of_lt_of_le hx (by omega) (by omega) + +/-- The residue a raw 32-bit Montgomery word stands for; words from the modulus up wrap. -/ +@[inline] +def ofWordMod (x : UInt32) : FastField modulus := + ⟨x % P.modulus32, by + rw [UInt32.toNat_mod, Mont32Field.modulus32_toNat]; exact Nat.mod_lt _ P.modulus_pos⟩ + +theorem toField_ofWordMod (x : UInt32) : + toField (ofWordMod x : FastField modulus) = + (x.toNat : ZMod modulus) * ((2 ^ 32 : ℕ) : ZMod modulus)⁻¹ := by + rw [toField_eq_val_toNat_cast_mul_inv] + simp only [ofWordMod, UInt32.toNat_mod, Mont32Field.modulus32_toNat, ZMod.natCast_mod] + +/-- A residue's word is its value times `2^32`. -/ +theorem val_toNat_cast (x : FastField modulus) : + (x.val.toNat : ZMod modulus) = toField x * ((2 ^ 32 : ℕ) : ZMod modulus) := + val_toNat_cast_eq_toField_mul + +/-- Reducing a sum of products of words gives the sum of products of their residues. -/ +theorem toField_reduce (x : UInt64) (h : x.toNat < modulus * 2 ^ 32) : + toField (reduce x h) = + (x.toNat : ZMod modulus) * ((2 ^ 32 : ℕ) : ZMod modulus)⁻¹ * + ((2 ^ 32 : ℕ) : ZMod modulus)⁻¹ := by + rw [toField_eq_val_toNat_cast_mul_inv, reduce_val_toNat_cast] + +/-- A lazy dot-product accumulator: a 64-bit word below `modulus * 2^32`. -/ +abbrev LazyAcc (modulus : ℕ) [Mont32Field modulus] : Type := + { a : UInt64 // a.toNat < modulus * 2 ^ 32 } + +namespace LazyAcc + +/-- The empty sum. -/ +@[inline] def zero : LazyAcc modulus := ⟨0, by simp⟩ + +/-- Accumulate the product of a raw word and a residue. -/ +@[inline] def add (a : LazyAcc modulus) (e : UInt32) (x : FastField modulus) : LazyAcc modulus := + ⟨accumulate modulus a.val (e.toUInt64 * x.val.toUInt64), + accumulate_lt a.property (word_mul_val_lt e x)⟩ + +/-- The residue of the accumulated sum, with one Montgomery reduction. -/ +@[inline] def result (a : LazyAcc modulus) : FastField modulus := reduce a.val a.property + +/-- The sum accumulated so far. -/ +def value (a : LazyAcc modulus) : ZMod modulus := + (a.val.toNat : ZMod modulus) * ((2 ^ 32 : ℕ) : ZMod modulus)⁻¹ * ((2 ^ 32 : ℕ) : ZMod modulus)⁻¹ + +@[simp] theorem value_zero : value (zero : LazyAcc modulus) = 0 := by + simp only [value, zero, UInt64.toNat_zero, Nat.cast_zero, zero_mul] + +@[simp] theorem value_add (a : LazyAcc modulus) (e : UInt32) (x : FastField modulus) : + value (a.add e x) = value a + toField (ofWordMod e : FastField modulus) * toField x := by + have hR := P.two_pow_32_ne_zero + simp only [value, add] + rw [accumulate_cast a.property (word_mul_val_lt e x), toField_ofWordMod, + toField_eq_val_toNat_cast_mul_inv (x := x), UInt64.toNat_mul, UInt32.toNat_toUInt64, + UInt32.toNat_toUInt64, Nat.mod_eq_of_lt (by + have := e.toNat_lt; have := x.property; have := P.modulus_lt_two_pow_31; nlinarith)] + push_cast + ring + +@[simp] theorem toField_result (a : LazyAcc modulus) : toField a.result = value a := + toField_reduce a.val a.property + +end LazyAcc + private theorem mul_assoc (x y z : FastField modulus) : (x * y) * z = x * (y * z) := by apply toField_injective rw [toField_mul, toField_mul, toField_mul, toField_mul] @@ -574,6 +775,9 @@ theorem toField_zsmul (n : Int) (x : FastField modulus) : toField (n • x) = n change toField ((n : FastField modulus) * x) = n • toField x rw [toField_mul, toField_intCast, zsmul_eq_mul] +/-- `pow` is the `Pow` instance's power. -/ +theorem pow_eq_npow (x : FastField modulus) (n : ℕ) : pow x n = x ^ n := rfl + /-- Natural powers through the `Pow` instance are preserved by `toField`. -/ @[simp] theorem toField_npow (x : FastField modulus) (n : ℕ) : toField (x ^ n) = toField x ^ n := by @@ -651,6 +855,23 @@ instance instNonBinaryField : NonBinaryField (FastField modulus) where _ = toField (0 : FastField modulus) := congrArg toField h _ = 0 := toField_zero +/-- `toField` commutes with finite sums. -/ +theorem toField_sum {ι : Type*} (s : Finset ι) (f : ι → FastField modulus) : + toField (∑ i ∈ s, f i) = ∑ i ∈ s, toField (f i) := + map_sum (ringEquiv modulus) f s + +/-- The fast carrier is finite, enumerated through the canonical field. -/ +instance instFintype : Fintype (FastField modulus) := + Fintype.ofEquiv _ (ringEquiv modulus).symm.toEquiv + +/-- The fast carrier has `modulus` elements. -/ +theorem card_fastField : Fintype.card (FastField modulus) = modulus := by + rw [Fintype.card_congr (ringEquiv modulus).toEquiv, ZMod.card] + +/-- The fast carrier has `modulus` elements, as a `Nat.card`. -/ +theorem natCard_fastField : Nat.card (FastField modulus) = modulus := by + rw [Nat.card_eq_fintype_card, card_fastField] + end end Native32 diff --git a/CompPoly/Fields/Montgomery/Native64x4.lean b/CompPoly/Fields/Montgomery/Native64x4.lean new file mode 100644 index 00000000..30ccdaba --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native64x4.lean @@ -0,0 +1,507 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos, Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Montgomery.Basic +public import CompPoly.Fields.Montgomery.Native64x4Defs + +/-! +# Native Montgomery arithmetic over four 64-bit limbs + +Raw word operations for prime moduli below `2 ^ 256`, represented as four 64-bit limbs +(`Limbs4`); the definitions live in the zero-import module `Montgomery/Native64x4Defs`. Each +word helper's specification exhibits the pair it returns, so a chain of `let (s, c) := ...` +bindings unfolds to a linear system over naturals that `omega` closes. + +## Main results + +* `condSub_toNat`, `condSubWide_toNat`, `add_toNat`, `sub_toNat`, `neg_toNat` — correctness + of the raw operations +* `addLimbs_spec`, `subLimbs_spec` — the underlying carry/borrow chains +* `mac_spec`, `mulHi_spec` — the widening multiply-accumulate +-/ + +@[expose] public section + +namespace Montgomery +namespace Native64x4 + +/-! ## Word-level specifications -/ + +/-- One wrapped addition with its overflow flag recomposes the exact sum. -/ +private theorem addWord_value (a b : UInt64) : + (a + b).toNat + 2 ^ 64 * (if a + b < a then (1 : UInt64) else 0).toNat = + a.toNat + b.toNat := by + have ha := UInt64.toNat_lt a + have hb := UInt64.toNat_lt b + have hmod := Nat.mod_add_div (a.toNat + b.toNat) (2 ^ 64) + by_cases h : a + b < a + · have h' := h + rw [UInt64.lt_iff_toNat_lt, UInt64.toNat_add] at h' + rw [ite_eq_left h, UInt64.toNat_add, UInt64.toNat_one] + omega + · have h' := h + rw [UInt64.lt_iff_toNat_lt, UInt64.toNat_add] at h' + rw [ite_eq_right h, UInt64.toNat_add, UInt64.toNat_zero] + omega + +/-- One wrapped subtraction with its borrow flag recomposes the exact difference. -/ +private theorem subWord_value (a b : UInt64) : + (a - b).toNat + b.toNat = a.toNat + 2 ^ 64 * (if a < b then (1 : UInt64) else 0).toNat := by + have ha := UInt64.toNat_lt a + have hb := UInt64.toNat_lt b + have hmod := Nat.mod_add_div (2 ^ 64 - b.toNat + a.toNat) (2 ^ 64) + by_cases h : a < b + · have h' := h + rw [UInt64.lt_iff_toNat_lt] at h' + rw [ite_eq_left h, UInt64.toNat_sub, UInt64.toNat_one] + omega + · have h' := h + rw [UInt64.lt_iff_toNat_lt] at h' + rw [ite_eq_right h, UInt64.toNat_sub, UInt64.toNat_zero] + omega + +private theorem flag_le_one (p : Prop) [Decidable p] : + (if p then (1 : UInt64) else 0).toNat ≤ 1 := by + split <;> simp + +private theorem flag_or_value (p q : Prop) [Decidable p] [Decidable q] + (h : (if p then (1 : UInt64) else 0).toNat + + (if q then (1 : UInt64) else 0).toNat ≤ 1) : + ((if p then (1 : UInt64) else 0) ||| (if q then (1 : UInt64) else 0)).toNat = + (if p then (1 : UInt64) else 0).toNat + + (if q then (1 : UInt64) else 0).toNat := by + by_cases hp : p <;> by_cases hq : q <;> + simp only [hp, hq, ite_true, ite_false, UInt64.toNat_one, UInt64.toNat_zero] at * + · omega + all_goals rfl + +/-- Add-with-carry, existential form: the exact sum identity with a one-bit carry. -/ +theorem adc_spec (x y c : UInt64) (hc : c.toNat ≤ 1) : + ∃ s co : UInt64, adc x y c = (s, co) ∧ + s.toNat + 2 ^ 64 * co.toNat = x.toNat + y.toNat + c.toNat ∧ co.toNat ≤ 1 := by + have h1 := addWord_value x y + have h2 := addWord_value (x + y) c + have hf1 := flag_le_one (x + y < x) + have hf2 := flag_le_one (x + y + c < x + y) + have hx := UInt64.toNat_lt x + have hy := UInt64.toNat_lt y + have hs := UInt64.toNat_lt (x + y + c) + have hsum : ((if x + y < x then (1 : UInt64) else 0) ||| + (if x + y + c < x + y then (1 : UInt64) else 0)).toNat = + (if x + y < x then (1 : UInt64) else 0).toNat + + (if x + y + c < x + y then (1 : UInt64) else 0).toNat := by + exact flag_or_value _ _ (by omega) + refine ⟨_, _, rfl, ?_, ?_⟩ <;> rw [hsum] <;> omega + +/-- Subtract-with-borrow, existential form: the exact difference identity with a one-bit +borrow. -/ +theorem sbb_spec (x y b : UInt64) (hb : b.toNat ≤ 1) : + ∃ d bo : UInt64, sbb x y b = (d, bo) ∧ + d.toNat + y.toNat + b.toNat = x.toNat + 2 ^ 64 * bo.toNat ∧ bo.toNat ≤ 1 := by + have h1 := subWord_value x y + have h2 := subWord_value (x - y) b + have hf1 := flag_le_one (x < y) + have hf2 := flag_le_one (x - y < b) + have hx := UInt64.toNat_lt x + have hy := UInt64.toNat_lt y + have hd := UInt64.toNat_lt (x - y - b) + have hsum : ((if x < y then (1 : UInt64) else 0) ||| + (if x - y < b then (1 : UInt64) else 0)).toNat = + (if x < y then (1 : UInt64) else 0).toNat + + (if x - y < b then (1 : UInt64) else 0).toNat := by + exact flag_or_value _ _ (by omega) + refine ⟨_, _, rfl, ?_, ?_⟩ <;> rw [hsum] <;> omega + +private theorem low32_value (x : UInt64) : (x &&& 0xffffffff).toNat = x.toNat % 2 ^ 32 := by + rw [UInt64.toNat_and, show ((0xffffffff : UInt64).toNat) = 2 ^ 32 - 1 from by decide, + Nat.and_two_pow_sub_one_eq_mod] + +private theorem high32_value (x : UInt64) : (x >>> 32).toNat = x.toNat / 2 ^ 32 := by + rw [UInt64.toNat_shiftRight, show ((32 : UInt64).toNat % 64) = 32 from by decide, + Nat.shiftRight_eq_div_pow] + +/-- The low and high words recompose the exact `64 × 64` product. -/ +theorem mulHi_spec (a b : UInt64) : + (a * b).toNat + 2 ^ 64 * (mulHi a b).toNat = a.toNat * b.toNat := by + let mask : UInt64 := 0xffffffff + let a0 := a &&& mask + let a1 := a >>> 32 + let b0 := b &&& mask + let b1 := b >>> 32 + let w0 := a0 * b0 + let t := a1 * b0 + (w0 >>> 32) + let w1 := (t &&& mask) + a0 * b1 + change (a * b).toNat + 2 ^ 64 * (a1 * b1 + (t >>> 32) + (w1 >>> 32)).toNat = _ + have lo32_lt : ∀ x : UInt64, (x &&& mask).toNat < 2 ^ 32 := fun x => by + rw [low32_value]; exact Nat.mod_lt _ (by decide) + have hi32_lt : ∀ x : UInt64, (x >>> 32).toNat < 2 ^ 32 := fun x => by + rw [high32_value]; have := UInt64.toNat_lt x; omega + have split : ∀ x : UInt64, x.toNat = (x &&& mask).toNat + 2 ^ 32 * (x >>> 32).toNat := + fun x => by rw [low32_value, high32_value]; exact (Nat.mod_add_div x.toNat (2 ^ 32)).symm + -- Every fact below is phrased in the `let` names, so `omega` sees one atom per word. + have ha : a.toNat = a0.toNat + 2 ^ 32 * a1.toNat := split a + have hb : b.toNat = b0.toNat + 2 ^ 32 * b1.toNat := split b + have ha0 : a0.toNat < 2 ^ 32 := lo32_lt a + have ha1 : a1.toNat < 2 ^ 32 := hi32_lt a + have hb0 : b0.toNat < 2 ^ 32 := lo32_lt b + have hb1 : b1.toNat < 2 ^ 32 := hi32_lt b + have tb : ∀ u v : ℕ, u < 2 ^ 32 → v < 2 ^ 32 → u * v ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := + fun u v hu hv => Nat.mul_le_mul (by omega) (by omega) + have p00 : a0.toNat * b0.toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := tb _ _ ha0 hb0 + have p10 : a1.toNat * b0.toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := tb _ _ ha1 hb0 + have p01 : a0.toNat * b1.toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := tb _ _ ha0 hb1 + have p11 : a1.toNat * b1.toNat ≤ (2 ^ 32 - 1) * (2 ^ 32 - 1) := tb _ _ ha1 hb1 + have gw0 : (w0 >>> 32).toNat < 2 ^ 32 := hi32_lt w0 + have gt : (t >>> 32).toNat < 2 ^ 32 := hi32_lt t + have gw1 : (w1 >>> 32).toNat < 2 ^ 32 := hi32_lt w1 + have lt0 : (t &&& mask).toNat < 2 ^ 32 := lo32_lt t + have lw0 : (w0 &&& mask).toNat < 2 ^ 32 := lo32_lt w0 + have lw1 : (w1 &&& mask).toNat < 2 ^ 32 := lo32_lt w1 + have e00 : a0.toNat * b0.toNat < 2 ^ 64 := by omega + have hw0 : w0.toNat = a0.toNat * b0.toNat := by + show (a0 * b0).toNat = _ + rw [UInt64.toNat_mul, Nat.mod_eq_of_lt e00] + have e10 : a1.toNat * b0.toNat < 2 ^ 64 := by omega + have e10' : a1.toNat * b0.toNat + (w0 >>> 32).toNat < 2 ^ 64 := by omega + have ht : t.toNat = a1.toNat * b0.toNat + (w0 >>> 32).toNat := by + show (a1 * b0 + (w0 >>> 32)).toNat = _ + rw [UInt64.toNat_add, UInt64.toNat_mul, Nat.mod_eq_of_lt e10, Nat.mod_eq_of_lt e10'] + have e01 : a0.toNat * b1.toNat < 2 ^ 64 := by omega + have e01' : (t &&& mask).toNat + a0.toNat * b1.toNat < 2 ^ 64 := by omega + have hw1 : w1.toNat = (t &&& mask).toNat + a0.toNat * b1.toNat := by + show ((t &&& mask) + a0 * b1).toNat = _ + rw [UInt64.toNat_add, UInt64.toNat_mul, Nat.mod_eq_of_lt e01, Nat.mod_eq_of_lt e01'] + have e11 : a1.toNat * b1.toNat < 2 ^ 64 := by omega + have e11' : a1.toNat * b1.toNat + (t >>> 32).toNat < 2 ^ 64 := by omega + have e11'' : a1.toNat * b1.toNat + (t >>> 32).toNat + (w1 >>> 32).toNat < 2 ^ 64 := by omega + have hhi : (a1 * b1 + (t >>> 32) + (w1 >>> 32)).toNat = + a1.toNat * b1.toNat + (t >>> 32).toNat + (w1 >>> 32).toNat := by + rw [UInt64.toNat_add, UInt64.toNat_add, UInt64.toNat_mul, Nat.mod_eq_of_lt e11, + Nat.mod_eq_of_lt e11', Nat.mod_eq_of_lt e11''] + have hprod : a.toNat * b.toNat = + a0.toNat * b0.toNat + 2 ^ 32 * (a1.toNat * b0.toNat) + 2 ^ 32 * (a0.toNat * b1.toNat) + + 2 ^ 64 * (a1.toNat * b1.toNat) := by + rw [ha, hb] + ring + have sw0 : w0.toNat = (w0 &&& mask).toNat + 2 ^ 32 * (w0 >>> 32).toNat := split w0 + have st : t.toNat = (t &&& mask).toNat + 2 ^ 32 * (t >>> 32).toNat := split t + have sw1 : w1.toNat = (w1 &&& mask).toNat + 2 ^ 32 * (w1 >>> 32).toNat := split w1 + rw [UInt64.toNat_mul, hprod, hhi] + omega + +/-- Multiply-accumulate, existential form: `t + a * b + c` never overflows two words. -/ +theorem mac_spec (t a b c : UInt64) : + ∃ lo hi : UInt64, mac t a b c = (lo, hi) ∧ + lo.toNat + 2 ^ 64 * hi.toNat = t.toNat + a.toNat * b.toNat + c.toNat := by + refine ⟨_, _, rfl, ?_⟩ + have hp := mulHi_spec a b + have h1 := addWord_value t (a * b) + have h2 := addWord_value (t + a * b) c + have hf1 := flag_le_one (t + a * b < t) + have hf2 := flag_le_one (t + a * b + c < t + a * b) + have ht := UInt64.toNat_lt t + have ha := UInt64.toNat_lt a + have hb := UInt64.toNat_lt b + have hc := UInt64.toNat_lt c + have hs := UInt64.toNat_lt (t + a * b + c) + have hab : a.toNat * b.toNat ≤ (2 ^ 64 - 1) * (2 ^ 64 - 1) := + Nat.mul_le_mul (by omega) (by omega) + generalize a.toNat * b.toNat = P at * + have hsum : (mulHi a b + (if t + a * b < t then (1 : UInt64) else 0) + + (if t + a * b + c < t + a * b then (1 : UInt64) else 0)).toNat = + (mulHi a b).toNat + (if t + a * b < t then (1 : UInt64) else 0).toNat + + (if t + a * b + c < t + a * b then (1 : UInt64) else 0).toNat := by + rw [UInt64.toNat_add, UInt64.toNat_add, Nat.mod_eq_of_lt (by omega), + Nat.mod_eq_of_lt (by omega)] + rw [hsum] + omega + +/-- The Montgomery multiplier agrees with its natural-number specification. -/ +theorem montM_toNat (s negInv : UInt64) : + (montM s negInv).toNat = s.toNat * negInv.toNat % 2 ^ 64 := by + simp only [montM, UInt64.toNat_mul] + +/-! ## Four-limb values -/ + +namespace Limbs4 + +theorem zero_toNat : zero.toNat = 0 := by simp only [toNat, zero, UInt64.toNat_zero] + +theorem one_toNat : one.toNat = 1 := by decide + +theorem ofNat_toNat (n : ℕ) : (ofNat n).toNat = n % 2 ^ 256 := by + simp only [toNat, ofNat, UInt64.toNat_ofNat', Nat.shiftRight_eq_div_pow] + omega + +/-- A four-limb value is below `2 ^ 256`. -/ +theorem toNat_lt (x : Limbs4) : x.toNat < 2 ^ 256 := by + have h0 := UInt64.toNat_lt x.l0 + have h1 := UInt64.toNat_lt x.l1 + have h2 := UInt64.toNat_lt x.l2 + have h3 := UInt64.toNat_lt x.l3 + simp only [toNat] + omega + +/-- The low limb of a value is its residue modulo `2 ^ 64`. -/ +theorem toNat_mod (x : Limbs4) : x.toNat % 2 ^ 64 = x.l0.toNat := by + have h0 := UInt64.toNat_lt x.l0 + simp only [toNat] + omega + +/-- Limb vectors are determined by their value. -/ +theorem ext_of_toNat {x y : Limbs4} (h : x.toNat = y.toNat) : x = y := by + have hx0 := UInt64.toNat_lt x.l0 + have hx1 := UInt64.toNat_lt x.l1 + have hx2 := UInt64.toNat_lt x.l2 + have hx3 := UInt64.toNat_lt x.l3 + have hy0 := UInt64.toNat_lt y.l0 + have hy1 := UInt64.toNat_lt y.l1 + have hy2 := UInt64.toNat_lt y.l2 + have hy3 := UInt64.toNat_lt y.l3 + simp only [toNat] at h + have e0 : x.l0.toNat = y.l0.toNat := by omega + have e1 : x.l1.toNat = y.l1.toNat := by omega + have e2 : x.l2.toNat = y.l2.toNat := by omega + have e3 : x.l3.toNat = y.l3.toNat := by omega + obtain ⟨x0, x1, x2, x3⟩ := x + obtain ⟨y0, y1, y2, y3⟩ := y + simp only [mk.injEq] + exact ⟨UInt64.toNat_inj.mp e0, UInt64.toNat_inj.mp e1, UInt64.toNat_inj.mp e2, + UInt64.toNat_inj.mp e3⟩ + +end Limbs4 + +/-! ### Chain lemmas -/ + +/-- Weighted telescoping of a four-step carry chain. -/ +theorem carry_chain_sum {x0 x1 x2 x3 y0 y1 y2 y3 s0 s1 s2 s3 cin c0 c1 c2 c3 : ℕ} + (e0 : s0 + 2 ^ 64 * c0 = x0 + y0 + cin) + (e1 : s1 + 2 ^ 64 * c1 = x1 + y1 + c0) + (e2 : s2 + 2 ^ 64 * c2 = x2 + y2 + c1) + (e3 : s3 + 2 ^ 64 * c3 = x3 + y3 + c2) : + s0 + 2 ^ 64 * s1 + 2 ^ 128 * s2 + 2 ^ 192 * s3 + 2 ^ 256 * c3 = + (x0 + 2 ^ 64 * x1 + 2 ^ 128 * x2 + 2 ^ 192 * x3) + + (y0 + 2 ^ 64 * y1 + 2 ^ 128 * y2 + 2 ^ 192 * y3) + cin := by + omega + +/-- Weighted telescoping of a four-step borrow chain. -/ +theorem borrow_chain_sum {x0 x1 x2 x3 y0 y1 y2 y3 m0 m1 m2 m3 bin b0 b1 b2 b3 : ℕ} + (e0 : m0 + y0 + bin = x0 + 2 ^ 64 * b0) + (e1 : m1 + y1 + b0 = x1 + 2 ^ 64 * b1) + (e2 : m2 + y2 + b1 = x2 + 2 ^ 64 * b2) + (e3 : m3 + y3 + b2 = x3 + 2 ^ 64 * b3) : + (m0 + 2 ^ 64 * m1 + 2 ^ 128 * m2 + 2 ^ 192 * m3) + + (y0 + 2 ^ 64 * y1 + 2 ^ 128 * y2 + 2 ^ 192 * y3) + bin = + x0 + 2 ^ 64 * x1 + 2 ^ 128 * x2 + 2 ^ 192 * x3 + 2 ^ 256 * b3 := by + omega + +/-! ### Branch lemmas -/ + +private theorem cond_of_borrow_zero {M Q T : ℕ} (h : M + Q = T) (hM : M < 2 ^ 256) : + M = if T < Q then T else T - Q := by + rw [ite_eq_right (by omega)] + omega + +private theorem cond_of_borrow_one {M Q T : ℕ} (h : M + Q = T + 2 ^ 256) (hM : M < 2 ^ 256) : + T = if T < Q then T else T - Q := by + rw [ite_eq_left (by omega)] + +private theorem add_of_carry {S A B Q D bo : ℕ} (hchain : S + 2 ^ 256 * 1 = A + B) + (hsub : D + Q = S + 2 ^ 256 * bo) (hbo : bo ≤ 1) (hD : D < 2 ^ 256) (hQ : Q < 2 ^ 256) + (hA : A < Q) (hB : B < Q) : D = (A + B) % Q := by + rw [show A + B = D + Q by omega, Nat.add_mod_right, Nat.mod_eq_of_lt (by omega)] + +private theorem wide_of_head {T L t4 Q D bo : ℕ} (hdec : T = L + 2 ^ 256 * t4) + (hsub : D + Q = L + 2 ^ 256 * bo) (hbo : bo ≤ 1) (hD : D < 2 ^ 256) + (hQ : Q < 2 ^ 256) (hT : T < 2 * Q) (h : ¬t4 = 0) : + D = if T < Q then T else T - Q := by + rw [ite_eq_right (by omega)] + omega + +private theorem sub_of_borrow_zero {A B Q D : ℕ} (h : D + B = A + 2 ^ 256 * 0) + (hB : B < Q) (hA : A < Q) : D = (A + (Q - B)) % Q := by + conv_rhs => rw [show A + (Q - B) = D + Q by omega] + rw [Nat.add_mod_right, Nat.mod_eq_of_lt (by omega)] + +private theorem sub_of_borrow_one {A B Q D E c : ℕ} (hD : D + B = A + 2 ^ 256 * 1) + (hE : E + 2 ^ 256 * c = D + Q) (hDlt : D < 2 ^ 256) (hElt : E < 2 ^ 256) + (hB : B < Q) (hA : A < Q) (hQ : Q < 2 ^ 256) : E = (A + (Q - B)) % Q := by + rw [show A + (Q - B) = E by omega, Nat.mod_eq_of_lt (by omega)] + +/-! ## Limbwise addition and subtraction -/ + +/-- The limbwise sum and the top carry recompose the sum of the inputs. -/ +theorem addLimbs_spec (a b : Limbs4) : + ∃ s0 s1 s2 s3 c : UInt64, addLimbs a b = (s0, s1, s2, s3, c) ∧ c.toNat ≤ 1 ∧ + (⟨s0, s1, s2, s3⟩ : Limbs4).toNat + 2 ^ 256 * c.toNat = a.toNat + b.toNat := by + obtain ⟨s0, c0, e0, h0, g0⟩ := adc_spec a.l0 b.l0 0 (by decide) + obtain ⟨s1, c1, e1, h1, g1⟩ := adc_spec a.l1 b.l1 c0 g0 + obtain ⟨s2, c2, e2, h2, g2⟩ := adc_spec a.l2 b.l2 c1 g1 + obtain ⟨s3, c3, e3, h3, g3⟩ := adc_spec a.l3 b.l3 c2 g2 + refine ⟨s0, s1, s2, s3, c3, ?_, g3, ?_⟩ + · simp only [addLimbs, e0, e1, e2, e3] + · rw [UInt64.toNat_zero] at h0 + simp only [Limbs4.toNat] + have h := carry_chain_sum h0 h1 h2 h3 + simp only [Nat.add_zero] at h + exact h + +/-- The limbwise difference and the final borrow recompose the difference of the inputs. -/ +theorem subLimbs_spec (a b : Limbs4) : + ∃ d0 d1 d2 d3 bo : UInt64, subLimbs a b = (d0, d1, d2, d3, bo) ∧ bo.toNat ≤ 1 ∧ + (⟨d0, d1, d2, d3⟩ : Limbs4).toNat + b.toNat = a.toNat + 2 ^ 256 * bo.toNat := by + obtain ⟨d0, b0, e0, h0, g0⟩ := sbb_spec a.l0 b.l0 0 (by decide) + obtain ⟨d1, b1, e1, h1, g1⟩ := sbb_spec a.l1 b.l1 b0 g0 + obtain ⟨d2, b2, e2, h2, g2⟩ := sbb_spec a.l2 b.l2 b1 g1 + obtain ⟨d3, b3, e3, h3, g3⟩ := sbb_spec a.l3 b.l3 b2 g2 + refine ⟨d0, d1, d2, d3, b3, ?_, g3, ?_⟩ + · simp only [subLimbs, e0, e1, e2, e3] + · rw [UInt64.toNat_zero] at h0 + simp only [Limbs4.toNat] + have h := borrow_chain_sum h0 h1 h2 h3 + simp only [Nat.add_zero] at h + exact h + +/-! ## Conditional subtraction -/ + +/-- `condSub` subtracts the modulus exactly when the input is at least the modulus. -/ +theorem condSub_toNat (q t : Limbs4) : + (condSub q t).toNat = if t.toNat < q.toNat then t.toNat else t.toNat - q.toNat := by + obtain ⟨d0, d1, d2, d3, bo, e, hbo, hchain⟩ := subLimbs_spec t q + have hD := Limbs4.toNat_lt ⟨d0, d1, d2, d3⟩ + simp only [condSub, e, beq_iff_eq, ← UInt64.toNat_inj, UInt64.toNat_zero] + split + case isTrue h => + rw [h, Nat.mul_zero, Nat.add_zero] at hchain + exact cond_of_borrow_zero hchain hD + case isFalse h => + rw [show bo.toNat = 1 by omega, Nat.mul_one] at hchain + exact cond_of_borrow_one hchain hD + +/-- `condSub` returns a canonical representative for inputs below `2 * q`. -/ +theorem condSub_lt (q t : Limbs4) (h : t.toNat < 2 * q.toNat) : + (condSub q t).toNat < q.toNat := by + rw [condSub_toNat q t] + split <;> omega + +/-- `condSubWide` subtracts the modulus exactly when an accumulator below `2 * q` is at least +the modulus. -/ +theorem condSubWide_toNat (q : Limbs4) (t : State5) (h : t.toNat < 2 * q.toNat) : + (condSubWide q t).toNat = if t.toNat < q.toNat then t.toNat else t.toNat - q.toNat := by + obtain ⟨d0, d1, d2, d3, bo, e, hbo, hsub⟩ := subLimbs_spec t.toLimbs4 q + have hD := Limbs4.toNat_lt ⟨d0, d1, d2, d3⟩ + have hQ := Limbs4.toNat_lt q + have hdec : t.toNat = t.toLimbs4.toNat + 2 ^ 256 * t.t4.toNat := rfl + have hr : (⟨t.t0, t.t1, t.t2, t.t3⟩ : Limbs4) = t.toLimbs4 := rfl + simp only [condSubWide, e, bne_iff_ne, ne_eq, beq_iff_eq, ← UInt64.toNat_inj, UInt64.toNat_zero] + split + case isTrue hc => exact wide_of_head hdec hsub hbo hD hQ h hc + case isFalse hc => + rw [show t.t4.toNat = 0 by omega, Nat.mul_zero, Nat.add_zero] at hdec + rw [hdec] + split + case isTrue h0 => + rw [h0, Nat.mul_zero, Nat.add_zero] at hsub + exact cond_of_borrow_zero hsub hD + case isFalse h0 => + rw [show bo.toNat = 1 by omega, Nat.mul_one] at hsub + rw [hr] + exact cond_of_borrow_one hsub hD + +/-- `condSubWide` returns a canonical representative for accumulators below `2 * q`. -/ +theorem condSubWide_lt (q : Limbs4) (t : State5) (h : t.toNat < 2 * q.toNat) : + (condSubWide q t).toNat < q.toNat := by + rw [condSubWide_toNat q t h] + split <;> omega + +/-! ## Field operations -/ + +/-- Modular addition is correct for canonical inputs of any modulus. -/ +theorem add_toNat (q a b : Limbs4) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (add q a b).toNat = (a.toNat + b.toNat) % q.toNat := by + obtain ⟨s0, s1, s2, s3, c, ea, hc, ha⟩ := addLimbs_spec a b + obtain ⟨d0, d1, d2, d3, bo, es, hbo, hs⟩ := subLimbs_spec ⟨s0, s1, s2, s3⟩ q + have hS := Limbs4.toNat_lt ⟨s0, s1, s2, s3⟩ + have hD := Limbs4.toNat_lt ⟨d0, d1, d2, d3⟩ + have hQ := Limbs4.toNat_lt q + have hcZero (hq : q.l3 < 0x8000000000000000) : c = 0 := by + have h0 := UInt64.toNat_lt q.l0 + have h1 := UInt64.toNat_lt q.l1 + have h2 := UInt64.toNat_lt q.l2 + rw [UInt64.lt_iff_toNat_lt] at hq + change q.l3.toNat < 2 ^ 63 at hq + have hsmall : q.toNat < 2 ^ 255 := by + simp only [Limbs4.toNat] + omega + apply UInt64.toNat_inj.mp + simp only [UInt64.toNat_zero] + omega + have he : add q a b = + if c != 0 then ⟨d0, d1, d2, d3⟩ + else if bo == 0 then ⟨d0, d1, d2, d3⟩ else ⟨s0, s1, s2, s3⟩ := by + simp only [add, ea, es] + split + · rename_i hq + rw [hcZero hq] + rfl + · rfl + rw [he] + simp only [bne_iff_ne, ne_eq, beq_iff_eq, ← UInt64.toNat_inj, UInt64.toNat_zero] + split + case isTrue h => + rw [show c.toNat = 1 by omega] at ha + exact add_of_carry ha hs hbo hD hQ haq hbq + case isFalse h => + rw [show c.toNat = 0 by omega, Nat.mul_zero, Nat.add_zero] at ha + rw [← ha] + split + case isTrue h0 => + rw [h0, Nat.mul_zero, Nat.add_zero] at hs + rw [← hs, Nat.add_mod_right, Nat.mod_eq_of_lt (by omega)] + case isFalse h0 => + rw [show bo.toNat = 1 by omega, Nat.mul_one] at hs + exact (Nat.mod_eq_of_lt (by omega)).symm + +theorem add_lt (q a b : Limbs4) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (add q a b).toNat < q.toNat := by + rw [add_toNat q a b haq hbq] + exact Nat.mod_lt _ (by omega) + +/-- Modular subtraction is correct for canonical inputs of any modulus. -/ +theorem sub_toNat (q a b : Limbs4) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (sub q a b).toNat = (a.toNat + (q.toNat - b.toNat)) % q.toNat := by + obtain ⟨d0, d1, d2, d3, bo, e, hbo, hchain⟩ := subLimbs_spec a b + have hD := Limbs4.toNat_lt ⟨d0, d1, d2, d3⟩ + simp only [sub, e, beq_iff_eq, ← UInt64.toNat_inj, UInt64.toNat_zero] + split + case isTrue h => + rw [h] at hchain + exact sub_of_borrow_zero hchain hbq haq + case isFalse h => + rw [show bo.toNat = 1 by omega] at hchain + obtain ⟨r0, r1, r2, r3, c, er, _, hr⟩ := addLimbs_spec ⟨d0, d1, d2, d3⟩ q + simp only [er] + exact sub_of_borrow_one hchain hr hD (Limbs4.toNat_lt ⟨r0, r1, r2, r3⟩) hbq haq + (Limbs4.toNat_lt q) + +theorem sub_lt (q a b : Limbs4) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (sub q a b).toNat < q.toNat := by + rw [sub_toNat q a b haq hbq] + exact Nat.mod_lt _ (by omega) + +/-- Modular negation is correct for canonical inputs of any modulus. -/ +theorem neg_toNat (q a : Limbs4) (haq : a.toNat < q.toNat) : + (neg q a).toNat = (q.toNat - a.toNat) % q.toNat := by + simp only [neg] + rw [sub_toNat q Limbs4.zero a (by rw [Limbs4.zero_toNat]; omega) haq, Limbs4.zero_toNat, + Nat.zero_add] + +theorem neg_lt (q a : Limbs4) (haq : a.toNat < q.toNat) : (neg q a).toNat < q.toNat := + sub_lt q Limbs4.zero a (by rw [Limbs4.zero_toNat]; omega) haq + +end Native64x4 +end Montgomery diff --git a/CompPoly/Fields/Montgomery/Native64x4Bytes.lean b/CompPoly/Fields/Montgomery/Native64x4Bytes.lean new file mode 100644 index 00000000..61ebc965 --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native64x4Bytes.lean @@ -0,0 +1,69 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Derek Sorensen, Georgios Raikos +-/ +module + +public import CompPoly.Fields.Montgomery.Native64x4Field +public import CompPoly.Data.Bytes.CanonicalNat + +/-! +# Serialization of the four-limb Montgomery carrier + +`Native64x4.FastField modulus` stores a Montgomery residue in four 64-bit limbs. Its canonical +natural is the value after one Montgomery reduction, `FastField.toNat`, so the bytes agree with +the `ZMod modulus` codec (`toBytes_ofField`, `toNat_ofField`). Decoding converts through +`ofField`. +-/ + +@[expose] public section + +namespace Montgomery.Native64x4 + +open CompPoly + +variable {modulus : ℕ} [P : Mont64x4Field modulus] + +namespace FastField + +/-- The canonical natural of a fast element is the residue of its canonical field value. -/ +theorem toNat_eq_val_toField (x : FastField modulus) : toNat x = (toField x).val := by + rw [toField, ZMod.val_natCast, Nat.mod_eq_of_lt (toNat_lt x)] + +/-- Carrier agreement: the canonical natural of `ofField n` is the residue `n.val`. -/ +theorem toNat_ofField (n : ZMod modulus) : toNat (ofField n) = n.val := by + rw [toNat_eq_val_toField, toField_ofField] + +theorem ofField_cast_toNat (x : FastField modulus) : ofField (toNat x : ZMod modulus) = x := + ofField_toField x + +instance : CanonicalNat (FastField modulus) := + CanonicalNat.ofToField toNat ofField toNat_lt ofField_cast_toNat toNat_ofField + +instance : ByteCodec (FastField modulus) := ByteCodec.ofCanonicalNat _ + +@[simp] theorem canonicalNat_bound : CanonicalNat.bound (FastField modulus) = modulus := rfl + +@[simp] theorem canonicalNat_toNat (x : FastField modulus) : CanonicalNat.toNat x = toNat x := rfl + +@[simp] theorem canonicalNat_ofNat (n : ℕ) : + CanonicalNat.ofNat (F := FastField modulus) n = ofField (n : ZMod modulus) := rfl + +@[simp] theorem byteCodec_width : ByteCodec.width (FastField modulus) = Bytes.bytesFor modulus := + rfl + +/-- The fast carrier and `ZMod modulus` encode the same element identically. -/ +theorem toBytes_ofField (n : ZMod modulus) : + ByteCodec.toBytes (ofField n) = ByteCodec.toBytes n := by + show Bytes.toVecLE _ (toNat (ofField n)) = Bytes.toVecLE _ n.val + rw [toNat_ofField] + +/-- Encoding then converting to the spec field is encoding in the spec field. -/ +theorem toBytes_eq_toBytes_toField (x : FastField modulus) : + ByteCodec.toBytes x = ByteCodec.toBytes (toField x) := by + rw [← toBytes_ofField, ofField_toField] + +end FastField + +end Montgomery.Native64x4 diff --git a/CompPoly/Fields/Montgomery/Native64x4Defs.lean b/CompPoly/Fields/Montgomery/Native64x4Defs.lean new file mode 100644 index 00000000..e5f21d53 --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native64x4Defs.lean @@ -0,0 +1,344 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos, Gregor Mitscha-Baude +-/ +module + +/-! +# Four-limb Montgomery arithmetic: runtime definitions (zero-import) + +The runtime definitions of the four-limb Montgomery arithmetic over 64-bit limbs; all +correctness statements live in `CompPoly.Fields.Montgomery.Native64x4`, which imports this +one. Products widen through `mulHi` (four 32-bit partial products) and every word helper +returns a pair the caller destructures, so nothing is boxed. + +This module deliberately has **zero imports**, for `precompileModules` consumers. +-/ + +@[expose] public section + +namespace Montgomery +namespace Native64x4 + +/-! ## Word helpers -/ + +/-- High word of the 64-bit product `a * b`, from four 32-bit partial products. -/ +@[inline] def mulHi (a b : UInt64) : UInt64 := + let mask : UInt64 := 0xffffffff + let a0 := a &&& mask + let a1 := a >>> 32 + let b0 := b &&& mask + let b1 := b >>> 32 + let w0 := a0 * b0 + let t := a1 * b0 + (w0 >>> 32) + let w1 := (t &&& mask) + a0 * b1 + a1 * b1 + (t >>> 32) + (w1 >>> 32) + +/-- Add-with-carry: the low word and the carry-out of `x + y + c`, for `c ≤ 1`. +The two overflow flags cannot both be set; OR exposes the one-bit carry to code generation. -/ +@[inline] def adc (x y c : UInt64) : UInt64 × UInt64 := + let s := x + y + let s' := s + c + (s', (if s < x then 1 else 0) ||| (if s' < s then 1 else 0)) + +/-- Subtract-with-borrow: the low word and the borrow-out of `x - y - b`, for `b ≤ 1`. +The two borrow flags cannot both be set; OR exposes the one-bit borrow to code generation. -/ +@[inline] def sbb (x y b : UInt64) : UInt64 × UInt64 := + let d := x - y + (d - b, (if x < y then 1 else 0) ||| (if d < b then 1 else 0)) + +/-- Multiply-accumulate: the low and high words of `t + a * b + c`. -/ +@[inline] def mac (t a b c : UInt64) : UInt64 × UInt64 := + let s := t + a * b + let s' := s + c + (s', mulHi a b + (if s < t then 1 else 0) + (if s' < s then 1 else 0)) + +/-- The Montgomery multiplier of a limb: `(s * negInv) mod 2 ^ 64`. -/ +@[inline] def montM (s negInv : UInt64) : UInt64 := s * negInv + +/-! ## Four-limb values -/ + +/-- A 256-bit value as four little-endian 64-bit limbs. -/ +structure Limbs4 where + /-- Limb of weight `2 ^ 0`. -/ + l0 : UInt64 + /-- Limb of weight `2 ^ 64`. -/ + l1 : UInt64 + /-- Limb of weight `2 ^ 128`. -/ + l2 : UInt64 + /-- Limb of weight `2 ^ 192`. -/ + l3 : UInt64 +deriving DecidableEq, Repr, Inhabited + +namespace Limbs4 + +/-- The zero value. -/ +def zero : Limbs4 := ⟨0, 0, 0, 0⟩ + +/-- The value one. -/ +def one : Limbs4 := ⟨1, 0, 0, 0⟩ + +/-- Split a natural number into four 64-bit limbs, discarding bits above `2 ^ 256`. -/ +@[inline] def ofNat (n : Nat) : Limbs4 := + ⟨UInt64.ofNat n, UInt64.ofNat (n >>> 64), UInt64.ofNat (n >>> 128), UInt64.ofNat (n >>> 192)⟩ + +/-- The natural number represented by the limbs: `∑ lᵢ * 2 ^ (64 * i)`. -/ +def toNat (x : Limbs4) : Nat := + x.l0.toNat + 2 ^ 64 * x.l1.toNat + 2 ^ 128 * x.l2.toNat + 2 ^ 192 * x.l3.toNat + +end Limbs4 + +/-! ## Limbwise addition and subtraction + +Chains return flat words and a `Limbs4` is built only inside a branch: one bound before the +branch is allocated whether or not that side runs. -/ + +/-- Limbwise add-with-carry: the four sum limbs and the carry out of the top limb. -/ +@[inline] def addLimbs (a b : Limbs4) : UInt64 × UInt64 × UInt64 × UInt64 × UInt64 := + let (s0, c) := adc a.l0 b.l0 0 + let (s1, c) := adc a.l1 b.l1 c + let (s2, c) := adc a.l2 b.l2 c + let (s3, c) := adc a.l3 b.l3 c + (s0, s1, s2, s3, c) + +/-- Limbwise subtract-with-borrow: the four difference limbs and the borrow out of the top +limb. -/ +@[inline] def subLimbs (a b : Limbs4) : UInt64 × UInt64 × UInt64 × UInt64 × UInt64 := + let (d0, bo) := sbb a.l0 b.l0 0 + let (d1, bo) := sbb a.l1 b.l1 bo + let (d2, bo) := sbb a.l2 b.l2 bo + let (d3, bo) := sbb a.l3 b.l3 bo + (d0, d1, d2, d3, bo) + +/-! ## Conditional subtraction and field operations -/ + +/-- Subtract the modulus once if the value is at least the modulus. -/ +@[inline] def condSub (q t : Limbs4) : Limbs4 := + let (d0, d1, d2, d3, bo) := subLimbs t q + if bo == 0 then ⟨d0, d1, d2, d3⟩ else t + +/-- Modular addition; a carry out of the top limb forces the (wrapping, exact) subtraction +of the modulus. For canonical inputs and a modulus below `2 ^ 255`, the top carry is +impossible, so the spare-bit branch omits its test. -/ +@[inline] def add (q a b : Limbs4) : Limbs4 := + let (s0, s1, s2, s3, c) := addLimbs a b + let (d0, d1, d2, d3, bo) := subLimbs ⟨s0, s1, s2, s3⟩ q + if q.l3 < 0x8000000000000000 then + if bo == 0 then ⟨d0, d1, d2, d3⟩ else ⟨s0, s1, s2, s3⟩ + else + if c != 0 then ⟨d0, d1, d2, d3⟩ else if bo == 0 then ⟨d0, d1, d2, d3⟩ else ⟨s0, s1, s2, s3⟩ + +/-- Modular subtraction: on a borrow, the modulus is added back. -/ +@[inline] def sub (q a b : Limbs4) : Limbs4 := + let (d0, d1, d2, d3, bo) := subLimbs a b + if bo == 0 then ⟨d0, d1, d2, d3⟩ + else + let (r0, r1, r2, r3, _) := addLimbs ⟨d0, d1, d2, d3⟩ q + ⟨r0, r1, r2, r3⟩ + +/-- Modular negation. -/ +@[inline] def neg (q a : Limbs4) : Limbs4 := sub q Limbs4.zero a + +/-! ## CIOS multiplication -/ + +/-- The CIOS accumulator between rounds: four limbs plus one head limb. -/ +structure State5 where + /-- Limb of weight `2 ^ 0`. -/ + t0 : UInt64 + /-- Limb of weight `2 ^ 64`. -/ + t1 : UInt64 + /-- Limb of weight `2 ^ 128`. -/ + t2 : UInt64 + /-- Limb of weight `2 ^ 192`. -/ + t3 : UInt64 + /-- Head limb of weight `2 ^ 256`. -/ + t4 : UInt64 +deriving DecidableEq, Repr, Inhabited + +/-- The CIOS accumulator after the multiply half of a round: five limbs plus a carry limb. -/ +structure State6 where + /-- Limb of weight `2 ^ 0`. -/ + t0 : UInt64 + /-- Limb of weight `2 ^ 64`. -/ + t1 : UInt64 + /-- Limb of weight `2 ^ 128`. -/ + t2 : UInt64 + /-- Limb of weight `2 ^ 192`. -/ + t3 : UInt64 + /-- Limb of weight `2 ^ 256`. -/ + t4 : UInt64 + /-- Carry limb of weight `2 ^ 320`. -/ + t5 : UInt64 +deriving DecidableEq, Repr, Inhabited + +namespace State5 + +/-- The zero accumulator. -/ +@[inline] def zero : State5 := ⟨0, 0, 0, 0, 0⟩ + +/-- The four low limbs of the accumulator. -/ +@[inline] def toLimbs4 (t : State5) : Limbs4 := ⟨t.t0, t.t1, t.t2, t.t3⟩ + +/-- The natural number represented by the accumulator. -/ +def toNat (t : State5) : Nat := t.toLimbs4.toNat + 2 ^ 256 * t.t4.toNat + +end State5 + +namespace State6 + +/-- The natural number represented by the accumulator. -/ +def toNat (t : State6) : Nat := + t.t0.toNat + 2 ^ 64 * t.t1.toNat + 2 ^ 128 * t.t2.toNat + 2 ^ 192 * t.t3.toNat + + 2 ^ 256 * t.t4.toNat + 2 ^ 320 * t.t5.toNat + +end State6 + +/-- The multiply half of a CIOS round, `t + a * bi`, keeping the carry out of the head limb. -/ +@[inline] def mulAccum (a : Limbs4) (bi : UInt64) (t : State5) : State6 := + let (s0, k) := mac t.t0 a.l0 bi 0 + let (s1, k) := mac t.t1 a.l1 bi k + let (s2, k) := mac t.t2 a.l2 bi k + let (s3, k) := mac t.t3 a.l3 bi k + let (s4, s5) := adc t.t4 k 0 + ⟨s0, s1, s2, s3, s4, s5⟩ + +/-- The reduce half of a CIOS round: add `montM s.t0 negInv * q`, which cancels the low limb, +and drop that limb; `negInv` must be `-q⁻¹ mod 2 ^ 64`. -/ +@[inline] def mulReduce (q : Limbs4) (negInv : UInt64) (s : State6) : State5 := + let m := montM s.t0 negInv + let (_, u) := mac s.t0 m q.l0 0 + let (t0, u) := mac s.t1 m q.l1 u + let (t1, u) := mac s.t2 m q.l2 u + let (t2, u) := mac s.t3 m q.l3 u + let (t3, u) := adc s.t4 u 0 + ⟨t0, t1, t2, t3, s.t5 + u⟩ + +/-- One CIOS outer round: accumulate `a * bi` into the accumulator, then reduce away one +limb. -/ +@[inline] def mulRound (q : Limbs4) (negInv : UInt64) (a : Limbs4) (bi : UInt64) + (t : State5) : State5 := + mulReduce q negInv (mulAccum a bi t) + +/-- `condSub` for an accumulator below `2 * q`; a set head limb forces the (wrapping, exact) +subtraction of the modulus, and never happens for a modulus below `2 ^ 255`. -/ +@[inline] def condSubWide (q : Limbs4) (t : State5) : Limbs4 := + let (d0, d1, d2, d3, bo) := subLimbs t.toLimbs4 q + if t.t4 != 0 then ⟨d0, d1, d2, d3⟩ else if bo == 0 then ⟨d0, d1, d2, d3⟩ + else ⟨t.t0, t.t1, t.t2, t.t3⟩ + +/-- Four CIOS Montgomery rounds, retaining the carry limb and omitting normalization. -/ +@[inline] def mulUnreduced (q : Limbs4) (negInv : UInt64) (a b : Limbs4) : State5 := + let t := mulRound q negInv a b.l0 State5.zero + let t := mulRound q negInv a b.l1 t + let t := mulRound q negInv a b.l2 t + let t := mulRound q negInv a b.l3 t + t + +/-- CIOS Montgomery multiplication followed by one conditional subtraction. -/ +@[inline] def mul (q : Limbs4) (negInv : UInt64) (a b : Limbs4) : Limbs4 := + condSubWide q (mulUnreduced q negInv a b) + +/-- Accumulate a precomputed low/high product, preserving the full carry word. -/ +@[inline] def macWords (t lo hi c : UInt64) : UInt64 × UInt64 := + let s := t + lo + let s' := s + c + (s', hi + (if s < t then 1 else 0) + (if s' < s then 1 else 0)) + +/-- Montgomery square sharing the ten distinct limb products across CIOS rounds. -/ +@[inline] def squareCached (q : Limbs4) (negInv : UInt64) (a : Limbs4) : Limbs4 := + let lo00 := a.l0 * a.l0 + let hi00 := mulHi a.l0 a.l0 + let lo01 := a.l0 * a.l1 + let hi01 := mulHi a.l0 a.l1 + let lo02 := a.l0 * a.l2 + let hi02 := mulHi a.l0 a.l2 + let lo03 := a.l0 * a.l3 + let hi03 := mulHi a.l0 a.l3 + let lo11 := a.l1 * a.l1 + let hi11 := mulHi a.l1 a.l1 + let lo12 := a.l1 * a.l2 + let hi12 := mulHi a.l1 a.l2 + let lo13 := a.l1 * a.l3 + let hi13 := mulHi a.l1 a.l3 + let lo22 := a.l2 * a.l2 + let hi22 := mulHi a.l2 a.l2 + let lo23 := a.l2 * a.l3 + let hi23 := mulHi a.l2 a.l3 + let lo33 := a.l3 * a.l3 + let hi33 := mulHi a.l3 a.l3 + let t := State5.zero + let (s0, c) := macWords t.t0 lo00 hi00 0 + let (s1, c) := macWords t.t1 lo01 hi01 c + let (s2, c) := macWords t.t2 lo02 hi02 c + let (s3, c) := macWords t.t3 lo03 hi03 c + let (s4, s5) := adc t.t4 c 0 + let t := mulReduce q negInv ⟨s0, s1, s2, s3, s4, s5⟩ + let (s0, c) := macWords t.t0 lo01 hi01 0 + let (s1, c) := macWords t.t1 lo11 hi11 c + let (s2, c) := macWords t.t2 lo12 hi12 c + let (s3, c) := macWords t.t3 lo13 hi13 c + let (s4, s5) := adc t.t4 c 0 + let t := mulReduce q negInv ⟨s0, s1, s2, s3, s4, s5⟩ + let (s0, c) := macWords t.t0 lo02 hi02 0 + let (s1, c) := macWords t.t1 lo12 hi12 c + let (s2, c) := macWords t.t2 lo22 hi22 c + let (s3, c) := macWords t.t3 lo23 hi23 c + let (s4, s5) := adc t.t4 c 0 + let t := mulReduce q negInv ⟨s0, s1, s2, s3, s4, s5⟩ + let (s0, c) := macWords t.t0 lo03 hi03 0 + let (s1, c) := macWords t.t1 lo13 hi13 c + let (s2, c) := macWords t.t2 lo23 hi23 c + let (s3, c) := macWords t.t3 lo33 hi33 c + let (s4, s5) := adc t.t4 c 0 + let t := mulReduce q negInv ⟨s0, s1, s2, s3, s4, s5⟩ + condSubWide q t + +/-- Montgomery squaring. -/ +@[inline] def square (q : Limbs4) (negInv : UInt64) (a : Limbs4) : Limbs4 := + mul q negInv a a + +/-! ## Scalar inputs for inlined hot loops -/ + +namespace Scalar + +/-- Scalar-limb add. Inline the call and immediately destructure the result to eliminate +intermediate limb objects. Inputs and output use Montgomery residues with the same +preconditions as `Native64x4.add`. -/ +@[inline] def add (q0 q1 q2 q3 : UInt64) + (a0 a1 a2 a3 b0 b1 b2 b3 : UInt64) : + UInt64 × UInt64 × UInt64 × UInt64 := + let r := Native64x4.add ⟨q0, q1, q2, q3⟩ ⟨a0, a1, a2, a3⟩ ⟨b0, b1, b2, b3⟩ + (r.l0, r.l1, r.l2, r.l3) + +/-- Scalar-limb sub. Inline the call and immediately destructure the result to eliminate +intermediate limb objects. Inputs and output use Montgomery residues with the same +preconditions as `Native64x4.sub`. -/ +@[inline] def sub (q0 q1 q2 q3 : UInt64) + (a0 a1 a2 a3 b0 b1 b2 b3 : UInt64) : + UInt64 × UInt64 × UInt64 × UInt64 := + let r := Native64x4.sub ⟨q0, q1, q2, q3⟩ ⟨a0, a1, a2, a3⟩ ⟨b0, b1, b2, b3⟩ + (r.l0, r.l1, r.l2, r.l3) + +/-- Scalar-limb mul. Inline the call and immediately destructure the result to eliminate +intermediate limb objects. Inputs and output use Montgomery residues with the same +preconditions as `Native64x4.mul`. -/ +@[inline] def mul (q0 q1 q2 q3 : UInt64) (negInv : UInt64) + (a0 a1 a2 a3 b0 b1 b2 b3 : UInt64) : + UInt64 × UInt64 × UInt64 × UInt64 := + let r := Native64x4.mul ⟨q0, q1, q2, q3⟩ negInv ⟨a0, a1, a2, a3⟩ ⟨b0, b1, b2, b3⟩ + (r.l0, r.l1, r.l2, r.l3) + +/-- Scalar-limb square. Inline the call and immediately destructure the result to eliminate +intermediate limb objects. Inputs and output use Montgomery residues with the same +preconditions as `Native64x4.square`. -/ +@[inline] def square (q0 q1 q2 q3 : UInt64) (negInv : UInt64) + (a0 a1 a2 a3 : UInt64) : + UInt64 × UInt64 × UInt64 × UInt64 := + let r := Native64x4.square ⟨q0, q1, q2, q3⟩ negInv ⟨a0, a1, a2, a3⟩ + (r.l0, r.l1, r.l2, r.l3) + +end Scalar + +end Native64x4 +end Montgomery diff --git a/CompPoly/Fields/Montgomery/Native64x4Field.lean b/CompPoly/Fields/Montgomery/Native64x4Field.lean new file mode 100644 index 00000000..4f88b9a6 --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native64x4Field.lean @@ -0,0 +1,613 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos, Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Basic +public import CompPoly.Fields.Montgomery.Native64x4Mul +public import Mathlib.Algebra.Field.TransferInstance +public import Mathlib.FieldTheory.Finite.Basic + +/-! +# Fast four-limb Montgomery fields + +The carrier `{ x : Limbs4 // x.toNat < modulus }` stores the Montgomery residue +`x * 2 ^ 256 mod modulus` for any prime modulus below `2 ^ 256`; `toField` divides by `2 ^ 256` +and lands in `ZMod modulus`, and all arithmetic is transported along that ring isomorphism. + +## Main results + +* `FastField` — the carrier, `{ x : Limbs4 // x.toNat < modulus }` +* `toField_add`, `toField_mul`, … — `@[simp]` equivalences with the canonical field +* `Mont64x4Field` — the per-field constants class +* `ringEquiv`, `instField` — the ring isomorphism and the transferred `Field` instance +-/ + +@[expose] public section + +namespace Montgomery +namespace Native64x4 + +/-! ## Per-field constants -/ + +/-- Per-field data for a fast four-limb Montgomery field with radix `R = 2 ^ 256`. All side +conditions are concrete numeral facts, discharged by `decide` at instantiation. -/ +class Mont64x4Field (modulus : ℕ) where + /-- `modulus` is prime. -/ + prime : modulus.Prime + /-- The modulus in four 64-bit limbs. -/ + modulusLimbs : Limbs4 + /-- `2 ^ 256 mod modulus`, the Montgomery representation of one. -/ + rModModulus : Limbs4 + /-- `(2 ^ 256) ^ 2 mod modulus`, used to enter Montgomery form. -/ + r2ModModulus : Limbs4 + /-- `-modulus⁻¹ mod 2 ^ 64`, used by Montgomery reduction. -/ + montgomeryNegInv : UInt64 + modulusLimbs_toNat : modulusLimbs.toNat = modulus := by decide + two_lt_modulus : 2 < modulus := by decide + rModModulus_toNat : rModModulus.toNat = 2 ^ 256 % modulus := by decide + r2ModModulus_toNat : r2ModModulus.toNat = (2 ^ 256) ^ 2 % modulus := by decide + montgomeryNegInv_mul_modulus_mod_two_pow_64 : + montgomeryNegInv.toNat * modulus % 2 ^ 64 = 2 ^ 64 - 1 := by decide + +/-! ## Modulus facts -/ + +namespace Mont64x4Field + +variable {modulus : ℕ} [P : Mont64x4Field modulus] + +instance : Fact (Nat.Prime modulus) := ⟨P.prime⟩ + +theorem modulus_pos : 0 < modulus := Nat.zero_lt_of_lt P.two_lt_modulus + +theorem modulus_lt : modulus < 2 ^ 256 := by + have h := Limbs4.toNat_lt P.modulusLimbs + rwa [P.modulusLimbs_toNat] at h + +theorem q_toNat : P.modulusLimbs.toNat = modulus := P.modulusLimbs_toNat + +theorem negInv_mul_q : P.montgomeryNegInv.toNat * P.modulusLimbs.toNat % 2 ^ 64 = 2 ^ 64 - 1 := by + rw [q_toNat] + exact P.montgomeryNegInv_mul_modulus_mod_two_pow_64 + +instance : NeZero modulus := ⟨modulus_pos.ne'⟩ + +theorem two_ne_zero : (2 : ZMod modulus) ≠ 0 := by + intro h + have hdvd : modulus ∣ 2 := (ZMod.natCast_eq_zero_iff 2 modulus).mp h + exact (Nat.not_le_of_gt P.two_lt_modulus) (Nat.le_of_dvd (by decide) hdvd) + +theorem r_ne_zero : ((2 ^ 256 : ℕ) : ZMod modulus) ≠ 0 := by + rw [Nat.cast_pow, Nat.cast_ofNat] + exact pow_ne_zero _ two_ne_zero + +end Mont64x4Field + +/-! ## The carrier -/ + +/-- The fast carrier: four 64-bit limbs holding a Montgomery residue below `modulus`, erasing +to `Limbs4` at runtime. -/ +@[implicit_reducible] +def FastField (modulus : ℕ) [Mont64x4Field modulus] : Type := + { x : Limbs4 // x.toNat < modulus } + +section + +variable {modulus : ℕ} [P : Mont64x4Field modulus] + +instance : DecidableEq (FastField modulus) := + inferInstanceAs (DecidableEq { x : Limbs4 // x.toNat < modulus }) + +namespace FastField + +theorem val_lt (x : FastField modulus) : x.val.toNat < P.modulusLimbs.toNat := by + rw [Mont64x4Field.q_toNat] + exact x.property + +/-! ### Arithmetic -/ + +/-- The zero element. -/ +def zero (modulus : ℕ) [P : Mont64x4Field modulus] : FastField modulus := + ⟨Limbs4.zero, by + rw [Limbs4.zero_toNat] + exact Mont64x4Field.modulus_pos⟩ + +/-- The one element, the Montgomery residue `2 ^ 256 mod modulus`. -/ +def one (modulus : ℕ) [P : Mont64x4Field modulus] : FastField modulus := + ⟨P.rModModulus, by + rw [P.rModModulus_toNat] + exact Nat.mod_lt _ Mont64x4Field.modulus_pos⟩ + +/-- Fast modular addition in Montgomery form. Borrow the second operand so chained additions +can reuse the first operand's storage. Keep the function specialized rather than inline so +this ownership boundary survives at call sites. -/ +@[specialize P] def add (x : FastField modulus) (y : @& FastField modulus) : FastField modulus := + ⟨Native64x4.add P.modulusLimbs x.val y.val, by + have h := add_lt _ _ _ x.val_lt y.val_lt + rwa [Mont64x4Field.q_toNat] at h⟩ + +/-- Fast modular subtraction in Montgomery form. -/ +@[inline] def sub (x y : FastField modulus) : FastField modulus := + ⟨Native64x4.sub P.modulusLimbs x.val y.val, by + have h := sub_lt _ _ _ x.val_lt y.val_lt + rwa [Mont64x4Field.q_toNat] at h⟩ + +/-- Fast modular negation in Montgomery form. -/ +@[inline] def neg (x : FastField modulus) : FastField modulus := + ⟨Native64x4.neg P.modulusLimbs x.val, by + have h := neg_lt _ _ x.val_lt + rwa [Mont64x4Field.q_toNat] at h⟩ + +/-- Keep modulus words dynamic so clang recognizes widening products. The scalar API +continues to inline `Native64x4.mul` directly for callers that retain unboxed state. -/ +@[noinline] def mulKernel (q : Limbs4) (negInv : UInt64) (a b : Limbs4) : Limbs4 := + Native64x4.mul q negInv a b + +/-- The same code-generation boundary for the dedicated symmetric square. -/ +@[noinline] def squareKernel (q : Limbs4) (negInv : UInt64) (a : Limbs4) : Limbs4 := + Native64x4.squareCached q negInv a + +/-- Fast Montgomery multiplication. -/ +@[inline] def mul (x y : FastField modulus) : FastField modulus := + ⟨mulKernel P.modulusLimbs P.montgomeryNegInv x.val y.val, by + have h := (mul_spec P.modulusLimbs P.montgomeryNegInv x.val y.val + Mont64x4Field.negInv_mul_q x.val_lt).1 + rwa [Mont64x4Field.q_toNat] at h⟩ + +/-- Fast squaring. -/ +@[inline] def square (x : FastField modulus) : FastField modulus := mul x x + +/-- Exponentiation by repeated squaring. -/ +@[specialize] def pow (x : FastField modulus) (n : ℕ) : FastField modulus := + @npowBinRec (FastField modulus) ⟨one modulus⟩ ⟨mul⟩ n x + +/-- Dedicated square with shared symmetric limb products. -/ +@[specialize P] def squareCached (x : FastField modulus) : FastField modulus := + ⟨squareKernel P.modulusLimbs P.montgomeryNegInv x.val, by + unfold squareKernel + rw [squareCached_eq_mul] + exact (mul x x).property⟩ + +private theorem squareCached_eq (x : FastField modulus) : squareCached x = mul x x := by + apply Subtype.ext + exact squareCached_eq_mul P.modulusLimbs P.montgomeryNegInv x.val + +/-- Binary powering without the final unused square. -/ +@[specialize P] def powTrimmedGo (n : ℕ) (a x : FastField modulus) : FastField modulus := + if _h : n ≤ 1 then if n == 0 then a else mul a x + else powTrimmedGo (n / 2) (if n % 2 == 1 then mul a x else a) (squareCached x) +termination_by n + +/-- Start with the first selected power instead of multiplying by one. -/ +@[specialize P] def powTrimmedStart (n : ℕ) (x : FastField modulus) : FastField modulus := + if _h : n ≤ 1 then if n == 0 then one modulus else x + else if n % 2 == 1 then powTrimmedGo (n / 2) x (squareCached x) + else powTrimmedStart (n / 2) (squareCached x) +termination_by n + +/-- Exponentiation skipping the initial identity product and final unused square. -/ +@[specialize P] def powTrimmed (x : FastField modulus) (n : ℕ) : FastField modulus := + powTrimmedStart n x + +/-- Inversion by Fermat's little theorem, `x⁻¹ = x ^ (modulus - 2)`. -/ +@[inline] def inv (x : FastField modulus) : FastField modulus := pow x (modulus - 2) + +/-- Division through inversion. -/ +@[inline] def div (x y : FastField modulus) : FastField modulus := mul x (inv y) + +/-! ### Conversions -/ + +/-- The canonical limb representative of a fast element: one Montgomery reduction. -/ +@[inline] def toLimbs4 (x : FastField modulus) : Limbs4 := + Native64x4.mul P.modulusLimbs P.montgomeryNegInv x.val Limbs4.one + +/-- The canonical natural representative of a fast element. -/ +@[inline] def toNat (x : FastField modulus) : ℕ := (toLimbs4 x).toNat + +/-- The canonical `ZMod` value of a fast element. -/ +@[inline] def toField (x : FastField modulus) : ZMod modulus := (toNat x : ZMod modulus) + +private theorem ofNat_lt_of_lt {n : ℕ} (h : n < modulus) : + (Limbs4.ofNat n).toNat < P.modulusLimbs.toNat := by + rw [Limbs4.ofNat_toNat, Mont64x4Field.q_toNat, + Nat.mod_eq_of_lt (h.trans Mont64x4Field.modulus_lt)] + exact h + +/-- Build a fast element from a canonical natural representative. -/ +@[inline] def ofCanonicalNat (n : ℕ) (h : n < modulus) : FastField modulus := + ⟨Native64x4.mul P.modulusLimbs P.montgomeryNegInv (Limbs4.ofNat n) P.r2ModModulus, by + have hlt := (mul_spec P.modulusLimbs P.montgomeryNegInv (Limbs4.ofNat n) P.r2ModModulus + Mont64x4Field.negInv_mul_q (ofNat_lt_of_lt h)).1 + rwa [Mont64x4Field.q_toNat] at hlt⟩ + +/-- Convert a natural number into fast Montgomery form. -/ +@[inline] def ofNat (modulus : ℕ) [P : Mont64x4Field modulus] (n : ℕ) : FastField modulus := + ofCanonicalNat (n % modulus) (Nat.mod_lt _ Mont64x4Field.modulus_pos) + +/-- Convert from the canonical field into fast Montgomery form. -/ +@[inline] def ofField (x : ZMod modulus) : FastField modulus := + ofCanonicalNat x.val (ZMod.val_lt x) + +/-- Convert an integer into fast Montgomery form. -/ +@[inline] def ofInt (modulus : ℕ) [P : Mont64x4Field modulus] (n : Int) : + FastField modulus := + ofField (n : ZMod modulus) + +instance : Zero (FastField modulus) := ⟨FastField.zero modulus⟩ +instance : One (FastField modulus) := ⟨FastField.one modulus⟩ +instance : Add (FastField modulus) := ⟨FastField.add⟩ +instance : Neg (FastField modulus) := ⟨FastField.neg⟩ +instance : Sub (FastField modulus) := ⟨FastField.sub⟩ +instance : Mul (FastField modulus) := ⟨FastField.mul⟩ +instance : Inv (FastField modulus) := ⟨FastField.inv⟩ +instance : Div (FastField modulus) := ⟨FastField.div⟩ +instance : NatCast (FastField modulus) := ⟨FastField.ofNat modulus⟩ +instance : IntCast (FastField modulus) := ⟨FastField.ofInt modulus⟩ +instance : Pow (FastField modulus) ℕ := ⟨FastField.pow⟩ + +instance : SMul ℕ (FastField modulus) where + smul n x := FastField.ofNat modulus n * x + +instance : SMul Int (FastField modulus) where + smul n x := FastField.ofInt modulus n * x + +instance : Pow (FastField modulus) Int where + pow x n := + match n with + | Int.ofNat k => FastField.pow x k + | Int.negSucc k => FastField.pow (FastField.inv x) (k + 1) + +theorem zero_def : (0 : FastField modulus) = FastField.zero modulus := rfl +theorem one_def : (1 : FastField modulus) = FastField.one modulus := rfl +theorem add_def (x y : FastField modulus) : x + y = FastField.add x y := rfl +theorem neg_def (x : FastField modulus) : -x = FastField.neg x := rfl +theorem sub_def (x y : FastField modulus) : x - y = FastField.sub x y := rfl +theorem mul_def (x y : FastField modulus) : x * y = FastField.mul x y := rfl +theorem inv_def (x : FastField modulus) : x⁻¹ = FastField.inv x := rfl +theorem div_def (x y : FastField modulus) : x / y = x * y⁻¹ := rfl +theorem square_def (x : FastField modulus) : FastField.square x = x * x := rfl + +/-- The limbs of a product. -/ +theorem val_mul (x y : FastField modulus) : + (x * y).val = Native64x4.mul P.modulusLimbs P.montgomeryNegInv x.val y.val := + Subtype.coe_mk _ _ + +/-! ## Correctness -/ + +section Bridge + +instance : NNRatCast (FastField modulus) where + nnratCast q := FastField.ofField (q : ZMod modulus) + +instance : RatCast (FastField modulus) where + ratCast q := FastField.ofField (q : ZMod modulus) + +instance : SMul ℚ≥0 (FastField modulus) where + smul q x := FastField.ofField (q • FastField.toField x) + +instance : SMul ℚ (FastField modulus) where + smul q x := FastField.ofField (q • FastField.toField x) + +/-- The raw Montgomery product divides by `2 ^ 256` in the canonical field. -/ +private theorem mul_cast (x y : Limbs4) (hxq : x.toNat < modulus) : + ((Native64x4.mul P.modulusLimbs P.montgomeryNegInv x y).toNat : ZMod modulus) = + (x.toNat : ZMod modulus) * (y.toNat : ZMod modulus) * + ((2 ^ 256 : ℕ) : ZMod modulus)⁻¹ := by + have hmod := (mul_spec P.modulusLimbs P.montgomeryNegInv x y Mont64x4Field.negInv_mul_q + (by rw [Mont64x4Field.q_toNat]; exact hxq)).2 + rw [Mont64x4Field.q_toNat] at hmod + have hcast := (ZMod.natCast_eq_natCast_iff _ _ _).2 hmod + rw [Nat.cast_mul, Nat.cast_mul] at hcast + rw [← hcast, mul_comm ((2 ^ 256 : ℕ) : ZMod modulus), mul_assoc, + mul_inv_cancel₀ Mont64x4Field.r_ne_zero, mul_one] + +/-- The canonical value of a fast element is its residue divided by `2 ^ 256`. -/ +private theorem toField_eq (x : FastField modulus) : + toField x = (x.val.toNat : ZMod modulus) * ((2 ^ 256 : ℕ) : ZMod modulus)⁻¹ := by + rw [toField, toNat, toLimbs4, mul_cast x.val Limbs4.one x.property, Limbs4.one_toNat] + simp + +private theorem val_cast (x : FastField modulus) : + (x.val.toNat : ZMod modulus) = toField x * ((2 ^ 256 : ℕ) : ZMod modulus) := by + rw [toField_eq, mul_assoc, inv_mul_cancel₀ Mont64x4Field.r_ne_zero, mul_one] + +theorem toNat_lt (x : FastField modulus) : toNat x < modulus := by + have := (mul_spec P.modulusLimbs P.montgomeryNegInv x.val Limbs4.one + Mont64x4Field.negInv_mul_q x.val_lt).1 + rwa [Mont64x4Field.q_toNat] at this + +private theorem ofCanonicalNat_val_cast {n : ℕ} (h : n < modulus) : + ((ofCanonicalNat n h).val.toNat : ZMod modulus) = + (n : ZMod modulus) * ((2 ^ 256 : ℕ) : ZMod modulus) := by + have hR2 : ((((2 ^ 256) ^ 2 : ℕ) % modulus : ℕ) : ZMod modulus) + = ((2 ^ 256 : ℕ) : ZMod modulus) * ((2 ^ 256 : ℕ) : ZMod modulus) := by + rw [ZMod.natCast_mod, pow_two, Nat.cast_mul] + rw [ofCanonicalNat, + mul_cast _ _ + (by rw [Limbs4.ofNat_toNat, Nat.mod_eq_of_lt (h.trans Mont64x4Field.modulus_lt)]; exact h), + Limbs4.ofNat_toNat, Nat.mod_eq_of_lt (h.trans Mont64x4Field.modulus_lt), + P.r2ModModulus_toNat, hR2, mul_assoc, mul_assoc, + mul_inv_cancel₀ Mont64x4Field.r_ne_zero, mul_one] + +@[simp] +theorem toField_ofCanonicalNat {n : ℕ} (h : n < modulus) : + toField (ofCanonicalNat n h) = (n : ZMod modulus) := by + rw [toField_eq, ofCanonicalNat_val_cast h, mul_assoc, + mul_inv_cancel₀ Mont64x4Field.r_ne_zero, mul_one] + +@[simp] +theorem toNat_ofCanonicalNat {n : ℕ} (h : n < modulus) : toNat (ofCanonicalNat n h) = n := + Montgomery.natCast_inj_of_lt (toField_ofCanonicalNat h) (toNat_lt _) h + +/-- Converting from the canonical field to fast form and back is the identity. -/ +@[simp] +theorem toField_ofField (x : ZMod modulus) : toField (ofField x) = x := by + simp [ofField, toField_ofCanonicalNat] + +/-- Converting from fast form to the canonical field and back is the identity. -/ +@[simp] +theorem ofField_toField (x : FastField modulus) : ofField (toField x) = x := by + apply Subtype.ext + have hval : ((ofField (toField x)).val.toNat : ZMod modulus) = (x.val.toNat : ZMod modulus) := by + rw [val_cast, val_cast, toField_ofField] + have hnat : (ofField (toField x)).val.toNat = x.val.toNat := + Montgomery.natCast_inj_of_lt hval (ofField (toField x)).property x.property + exact Limbs4.ext_of_toNat hnat + +/-- The canonical-field interpretation distinguishes fast values. -/ +theorem toField_injective : Function.Injective (toField (modulus := modulus)) := + Function.LeftInverse.injective ofField_toField + +/-! ### Field operations -/ + +@[simp] +theorem toField_zero : toField (0 : FastField modulus) = 0 := by + rw [toField_eq, zero_def, zero] + simp [Limbs4.zero_toNat] + +@[simp] +theorem toField_one : toField (1 : FastField modulus) = 1 := by + rw [toField_eq, one_def, one] + simp only [P.rModModulus_toNat, ZMod.natCast_mod] + exact mul_inv_cancel₀ Mont64x4Field.r_ne_zero + +@[simp] +theorem toField_add (x y : FastField modulus) : toField (x + y) = toField x + toField y := by + rw [toField_eq, toField_eq x, toField_eq y, add_def, add] + have h := add_toNat P.modulusLimbs x.val y.val x.val_lt y.val_lt + rw [h, Mont64x4Field.q_toNat, ZMod.natCast_mod, Nat.cast_add] + ring + +@[simp] +theorem toField_sub (x y : FastField modulus) : toField (x - y) = toField x - toField y := by + rw [toField_eq, toField_eq x, toField_eq y, sub_def, sub] + have h := sub_toNat P.modulusLimbs x.val y.val x.val_lt y.val_lt + rw [h, Mont64x4Field.q_toNat, ZMod.natCast_mod, Nat.cast_add, + Nat.cast_sub (le_of_lt y.property), ZMod.natCast_self] + ring + +@[simp] +theorem toField_neg (x : FastField modulus) : toField (-x) = -toField x := by + rw [toField_eq, toField_eq x, neg_def, neg] + have h := neg_toNat P.modulusLimbs x.val x.val_lt + rw [h, Mont64x4Field.q_toNat, ZMod.natCast_mod, Nat.cast_sub (le_of_lt x.property), + ZMod.natCast_self] + ring + +@[simp] +theorem toField_mul (x y : FastField modulus) : toField (x * y) = toField x * toField y := by + rw [toField_eq, toField_eq x, toField_eq y, mul_def] + simp only [mul, mulKernel] + rw [mul_cast x.val y.val x.property] + ring + +private theorem mul_assoc' (x y z : FastField modulus) : x * y * z = x * (y * z) := by + apply toField_injective + rw [toField_mul, toField_mul, toField_mul, toField_mul] + ring + +private theorem pow_succ_field (x : FastField modulus) (n : ℕ) : pow x (n + 1) = pow x n * x := by + unfold pow + let : Semigroup (FastField modulus) := { mul, mul_assoc := mul_assoc' } + exact npowBinRec_succ n x + +@[simp] +theorem toField_square (x : FastField modulus) : toField (square x) = toField x * toField x := by + simp only [square_def, toField_mul] + +@[simp] +theorem toField_pow (x : FastField modulus) (n : ℕ) : toField (pow x n) = toField x ^ n := by + induction n with + | zero => + unfold pow + rw [npowBinRec_zero, toField_one] + simp + | succ n ih => rw [pow_succ_field, toField_mul, ih, _root_.pow_succ] + +/-- Fermat-style inversion in `ZMod modulus`. -/ +private theorem inv_eq_pow {a : ZMod modulus} (ha : a ≠ 0) : a⁻¹ = a ^ (modulus - 2) := by + have hcard : Fintype.card (ZMod modulus) = modulus := ZMod.card modulus + have h1 : a ^ (modulus - 1) = 1 := by + have h := FiniteField.pow_card_sub_one_eq_one a ha + rw [hcard] at h + exact h + have hmul : a * a ^ (modulus - 2) = 1 := by + rw [← pow_succ'] + show a ^ (modulus - 2 + 1) = 1 + have hsucc : modulus - 2 + 1 = modulus - 1 := by + have := P.two_lt_modulus + omega + rw [hsucc] + exact h1 + exact (eq_inv_of_mul_eq_one_left (by rwa [mul_comm])).symm + +@[simp] +theorem toField_inv (x : FastField modulus) : toField x⁻¹ = (toField x)⁻¹ := by + simp only [inv_def, inv, toField_pow] + by_cases hx : toField x = 0 + · rw [hx, inv_zero, zero_pow] + have := P.two_lt_modulus + omega + · rw [inv_eq_pow hx] + +@[simp] +theorem toField_div (x y : FastField modulus) : toField (x / y) = toField x / toField y := by + simp only [div_def, toField_mul, toField_inv] + rfl + +@[simp] +theorem toField_natCast (n : ℕ) : toField (n : FastField modulus) = (n : ZMod modulus) := by + change toField (ofNat modulus n) = (n : ZMod modulus) + rw [ofNat, toField_ofCanonicalNat, ZMod.natCast_eq_natCast_iff] + exact Nat.mod_modEq _ _ + +@[simp] +theorem toField_intCast (n : Int) : toField (n : FastField modulus) = (n : ZMod modulus) := by + change toField (ofField n) = (n : ZMod modulus) + rw [toField_ofField] + +@[simp] +theorem toField_nsmul (n : ℕ) (x : FastField modulus) : toField (n • x) = n • toField x := by + change toField ((n : FastField modulus) * x) = n • toField x + rw [toField_mul, toField_natCast, nsmul_eq_mul] + +@[simp] +theorem toField_zsmul (n : Int) (x : FastField modulus) : toField (n • x) = n • toField x := by + change toField ((n : FastField modulus) * x) = n • toField x + rw [toField_mul, toField_intCast, zsmul_eq_mul] + +@[simp] +theorem toField_npow (x : FastField modulus) (n : ℕ) : toField (x ^ n) = toField x ^ n := by + change toField (pow x n) = toField x ^ n + rw [toField_pow] + +@[simp] +theorem toField_zpow (x : FastField modulus) (n : Int) : toField (x ^ n) = toField x ^ n := by + cases n with + | ofNat n => + change toField (pow x n) = toField x ^ (Int.ofNat n) + rw [toField_pow] + exact (zpow_natCast (toField x) n).symm + | negSucc n => + change toField (pow (inv x) (n + 1)) = toField x ^ (Int.negSucc n) + have hinv : toField (inv x) = (toField x)⁻¹ := by + change toField x⁻¹ = (toField x)⁻¹ + rw [toField_inv] + rw [toField_pow, hinv, zpow_negSucc, inv_pow] + +@[simp] +theorem toField_nnratCast (q : ℚ≥0) : toField (q : FastField modulus) = (q : ZMod modulus) := by + change toField (ofField (q : ZMod modulus)) = (q : ZMod modulus) + rw [toField_ofField] + +@[simp] +theorem toField_ratCast (q : ℚ) : toField (q : FastField modulus) = (q : ZMod modulus) := by + change toField (ofField (q : ZMod modulus)) = (q : ZMod modulus) + rw [toField_ofField] + +@[simp] +theorem toField_nnqsmul (q : ℚ≥0) (x : FastField modulus) : toField (q • x) = q • toField x := by + change toField (ofField (q • toField x)) = q • toField x + rw [toField_ofField] + +@[simp] +theorem toField_qsmul (q : ℚ) (x : FastField modulus) : toField (q • x) = q • toField x := by + change toField (ofField (q • toField x)) = q • toField x + rw [toField_ofField] + +/-! ### Algebraic structure -/ + +/-- Ring equivalence between the fast Montgomery representation and the canonical field. -/ +def ringEquiv (modulus : ℕ) [P : Mont64x4Field modulus] : + FastField modulus ≃+* ZMod modulus where + toFun := toField + invFun := ofField + left_inv := ofField_toField + right_inv := toField_ofField + map_add' := toField_add + map_mul' := toField_mul + +@[simp] +theorem ringEquiv_apply {x : FastField modulus} : ringEquiv modulus x = toField x := rfl + +@[simp] +theorem ringEquiv_symm_apply {x : ZMod modulus} : (ringEquiv modulus).symm x = ofField x := rfl + +/-- Field instance transferred from the canonical field through `toField`. -/ +instance instField : _root_.Field (FastField modulus) := by + apply toField_injective.field toField <;> simp + +private theorem toField_powTrimmedGo (n : ℕ) (a x : FastField modulus) : + toField (powTrimmedGo n a x) = toField a * toField x ^ n := by + induction n using Nat.strong_induction_on generalizing a x with + | h n ih => + rw [powTrimmedGo] + split + · next hn => + interval_cases n <;> simp only [beq_iff_eq, ite_true, ite_false, pow_zero, + mul_one, Nat.one_ne_zero, pow_one, ← mul_def, toField_mul] + · next hn => + rw [ih (n / 2) (by omega), squareCached_eq] + simp only [← mul_def, toField_mul] + by_cases hb : n % 2 = 1 + · simp only [hb, beq_iff_eq, ite_true, toField_mul] + have he : n = 1 + 2 * (n / 2) := by omega + conv_rhs => rw [he] + simp only [pow_add, pow_mul, pow_one, pow_two] + ring + · have hb0 : n % 2 = 0 := by omega + simp only [hb0, beq_iff_eq, Nat.zero_ne_one, ite_false] + have he : n = 2 * (n / 2) := by omega + conv_rhs => rw [he] + simp only [pow_mul, pow_two] + +private theorem toField_powTrimmedStart (n : ℕ) (x : FastField modulus) : + toField (powTrimmedStart n x) = toField x ^ n := by + induction n using Nat.strong_induction_on generalizing x with + | h n ih => + rw [powTrimmedStart] + split + · next hn => + interval_cases n <;> simp only [beq_iff_eq, ite_true, ite_false, pow_zero, + ← one_def, toField_one, Nat.one_ne_zero, pow_one] + · next hn => + by_cases hb : n % 2 = 1 + · simp only [hb, beq_iff_eq, ite_true, toField_powTrimmedGo, + squareCached_eq, ← mul_def, toField_mul] + have he : n = 1 + 2 * (n / 2) := by omega + conv_rhs => rw [he] + simp only [pow_add, pow_mul, pow_one, pow_two] + · have hb0 : n % 2 = 0 := by omega + simp only [hb0, beq_iff_eq, Nat.zero_ne_one, ite_false] + rw [ih (n / 2) (by omega), squareCached_eq] + simp only [← mul_def, toField_mul] + have he : n = 2 * (n / 2) := by omega + conv_rhs => rw [he] + simp only [pow_mul, pow_two] + +/-- The optimized exponentiation implements the original field power. -/ +@[csimp] theorem pow_eq_powTrimmed : @pow = @powTrimmed := by + funext modulus P x n + apply toField_injective + rw [toField_pow, powTrimmed, toField_powTrimmedStart] + +/-- A fast four-limb field is non-binary. -/ +instance instNonBinaryField : NonBinaryField (FastField modulus) where + char_neq_2 := by + intro h + apply Mont64x4Field.two_ne_zero (modulus := modulus) + calc + _ = toField ((2 : ℕ) : FastField modulus) := (toField_natCast 2).symm + _ = toField (0 : FastField modulus) := congrArg toField h + _ = 0 := toField_zero + +end Bridge + +end FastField + +end + +end Native64x4 +end Montgomery diff --git a/CompPoly/Fields/Montgomery/Native64x4Inv.lean b/CompPoly/Fields/Montgomery/Native64x4Inv.lean new file mode 100644 index 00000000..1271bd9c --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native64x4Inv.lean @@ -0,0 +1,375 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos +-/ +module + +public import CompPoly.Fields.Montgomery.Native64x4Field +public import CompPoly.Fields.Montgomery.Native64x4InvDefs +public import CompPoly.Fields.Montgomery.Native64x8Inv +public import Mathlib.Tactic.Ring + +/-! +# Fast inversion for four-limb Montgomery fields + +Correctness of the checked inversion of `Montgomery/Native64x4InvDefs`: `invGcdRaw` +computes the field inverse and `FastField.invGcd` is its proof-carrying wrapper. The +divstep coefficient bound is shared with the eight-limb stack; the mac-width safety of the +candidate is proved here for the five-word linear combinations. +-/ + +@[expose] public section + +namespace Montgomery.Native64x4 + +open Montgomery.Native64x8 (gcdInner gcdInner_natAbs_le_31) + +/-! ## Order from the borrow chain -/ + +/-- The final borrow decides the numeric order. -/ +theorem subLimbs_borrow_eq_one_iff {a b : Limbs4} {d0 d1 d2 d3 bo : UInt64} + (e : subLimbs a b = (d0, d1, d2, d3, bo)) : bo = 1 ↔ a.toNat < b.toNat := by + obtain ⟨d0', d1', d2', d3', bo', e', hbo, hchain⟩ := subLimbs_spec a b + rw [e] at e' + simp only [Prod.mk.injEq] at e' + obtain ⟨rfl, rfl, rfl, rfl, rfl⟩ := e' + have hd := Limbs4.toNat_lt ⟨d0, d1, d2, d3⟩ + rw [← UInt64.toNat_inj, show (1 : UInt64).toNat = 1 from rfl] + omega + +/-! ## Five-word specifications -/ + +/-- `mulWord` is the exact five-word product. -/ +theorem mulWord_spec (x : Limbs4) (k : UInt64) : + ∃ w0 w1 w2 w3 w4 : UInt64, mulWord x k = (w0, w1, w2, w3, w4) ∧ + w0.toNat + 2 ^ 64 * w1.toNat + 2 ^ 128 * w2.toNat + 2 ^ 192 * w3.toNat + + 2 ^ 256 * w4.toNat = x.toNat * k.toNat := by + obtain ⟨w0, c0, e0, h0⟩ := mac_spec 0 x.l0 k 0 + obtain ⟨w1, c1, e1, h1⟩ := mac_spec 0 x.l1 k c0 + obtain ⟨w2, c2, e2, h2⟩ := mac_spec 0 x.l2 k c1 + obtain ⟨w3, w4, e3, h3⟩ := mac_spec 0 x.l3 k c2 + refine ⟨w0, w1, w2, w3, w4, by simp only [mulWord, e0, e1, e2, e3], ?_⟩ + rw [UInt64.toNat_zero] at h0 h1 h2 h3 + rw [Limbs4.toNat, sum4_mul] + omega + +/-- `add5` is the exact five-word sum with a one-bit carry. -/ +theorem add5_spec (a0 a1 a2 a3 a4 b0 b1 b2 b3 b4 : UInt64) : + ∃ s0 s1 s2 s3 s4 c : UInt64, + add5 a0 a1 a2 a3 a4 b0 b1 b2 b3 b4 = (s0, s1, s2, s3, s4, c) ∧ c.toNat ≤ 1 ∧ + s0.toNat + 2 ^ 64 * s1.toNat + 2 ^ 128 * s2.toNat + 2 ^ 192 * s3.toNat + + 2 ^ 256 * s4.toNat + 2 ^ 320 * c.toNat = + (a0.toNat + 2 ^ 64 * a1.toNat + 2 ^ 128 * a2.toNat + 2 ^ 192 * a3.toNat + + 2 ^ 256 * a4.toNat) + + (b0.toNat + 2 ^ 64 * b1.toNat + 2 ^ 128 * b2.toNat + 2 ^ 192 * b3.toNat + + 2 ^ 256 * b4.toNat) := by + obtain ⟨s0, c0, e0, h0, g0⟩ := adc_spec a0 b0 0 (by decide) + obtain ⟨s1, c1, e1, h1, g1⟩ := adc_spec a1 b1 c0 g0 + obtain ⟨s2, c2, e2, h2, g2⟩ := adc_spec a2 b2 c1 g1 + obtain ⟨s3, c3, e3, h3, g3⟩ := adc_spec a3 b3 c2 g2 + obtain ⟨s4, c4, e4, h4, g4⟩ := adc_spec a4 b4 c3 g3 + refine ⟨s0, s1, s2, s3, s4, c4, by simp only [add5, e0, e1, e2, e3, e4], g4, ?_⟩ + rw [UInt64.toNat_zero] at h0 + omega + +/-! ## Mac-width safety of the candidate -/ + +section MacSafety + +/-- The Montgomery linear combination stays canonical for magnitudes below the modulus. -/ +theorem lincombTail_lt {q uS vS : Limbs4} {negInv F G : UInt64} (hq0 : 0 < q.toNat) + (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) + (huS : uS.toNat ≤ q.toNat) (hvS : vS.toNat ≤ q.toNat) + (hFG : F.toNat + G.toNat ≤ 2 ^ 31) : + (lincombTail q negInv uS vS F G).toNat < q.toNat := by + obtain ⟨a0, a1, a2, a3, a4, ea, ha⟩ := mulWord_spec uS F + obtain ⟨b0, b1, b2, b3, b4, eb, hb⟩ := mulWord_spec vS G + obtain ⟨s0, s1, s2, s3, s4, c, es, hc, hs⟩ := add5_spec a0 a1 a2 a3 a4 b0 b1 b2 b3 b4 + obtain ⟨n0, n1, n2, n3, n4, en, hn⟩ := mulWord_spec q (montM s0 negInv) + obtain ⟨t0, t1, t2, t3, t4, t5, et, ht5, ht⟩ := add5_spec s0 s1 s2 s3 s4 n0 n1 n2 n3 n4 + simp only [lincombTail, ea, eb, es, en, et] + -- The combination is at most `q * 2 ^ 31`. + have hAB : uS.toNat * F.toNat + vS.toNat * G.toNat ≤ q.toNat * 2 ^ 31 := + calc uS.toNat * F.toNat + vS.toNat * G.toNat + ≤ q.toNat * F.toNat + q.toNat * G.toNat := + Nat.add_le_add (Nat.mul_le_mul huS (Nat.le_refl _)) + (Nat.mul_le_mul hvS (Nat.le_refl _)) + _ = q.toNat * (F.toNat + G.toNat) := (Nat.mul_add _ _ _).symm + _ ≤ q.toNat * 2 ^ 31 := Nat.mul_le_mul (Nat.le_refl _) hFG + -- The Montgomery multiple of the modulus cancels the low word. + rw [montM_toNat, Nat.mul_comm q.toNat] at hn + have hdvd := Montgomery.dvd_add (2 ^ 64) q.toNat negInv.toNat (by norm_num) hnq s0.toNat + rw [Nat.mod_eq_of_lt (UInt64.toNat_lt s0)] at hdvd + obtain ⟨k, hk⟩ := hdvd + have hN : s0.toNat * negInv.toNat % 2 ^ 64 * q.toNat < 2 ^ 64 * q.toNat := + Nat.mul_lt_mul_of_pos_right (Nat.mod_lt _ (by decide)) hq0 + have hQ := Limbs4.toNat_lt q + have w0 := UInt64.toNat_lt s0 + have w1 := UInt64.toNat_lt n0 + have w2 := UInt64.toNat_lt t0 + have w3 := UInt64.toNat_lt a4 + have w4 := UInt64.toNat_lt b4 + have w5 := UInt64.toNat_lt n4 + have w6 := UInt64.toNat_lt s4 + apply condSubWide_lt + rw [State5.toNat_eq] + dsimp only + omega + +/-- The Montgomery lincomb stays canonical for canonical inputs. -/ +theorem gcdLinearCombMontyRed_lt {q : Limbs4} {negInv : UInt64} {u v : Limbs4} {f g : Int} + (hq0 : 0 < q.toNat) (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) + (huq : u.toNat < q.toNat) (hvq : v.toNat < q.toNat) + (hfg : f.natAbs + g.natAbs ≤ 2 ^ 31) : + (gcdLinearCombMontyRed q negInv u v f g).toNat < q.toNat := by + have hofF : (UInt64.ofNat f.natAbs).toNat = f.natAbs := by + rw [UInt64.toNat_ofNat'] + omega + have hofG : (UInt64.ofNat g.natAbs).toNat = g.natAbs := by + rw [UInt64.toNat_ofNat'] + omega + have hneg : ∀ {c : Limbs4}, c.toNat < q.toNat → (negMod q c).toNat ≤ q.toNat := by + intro c hc + obtain ⟨d0, d1, d2, d3, bo, e, hbo, hchain⟩ := subLimbs_spec q c + simp only [negMod, e] + have hD := Limbs4.toNat_lt ⟨d0, d1, d2, d3⟩ + omega + unfold gcdLinearCombMontyRed + apply lincombTail_lt hq0 hnq + · split + · exact hneg huq + · exact Nat.le_of_lt huq + · split + · exact hneg hvq + · exact Nat.le_of_lt hvq + · rw [hofF, hofG] + exact hfg + +set_option maxRecDepth 4000 in +/-- The main loop keeps the Montgomery pair canonical. -/ +theorem gcdMainLoop_bounded {q : Limbs4} {negInv : UInt64} {rounds : ℕ} + {a u b v A U B V : Limbs4} + (hq0 : 0 < q.toNat) (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) + (huq : u.toNat < q.toNat) (hvq : v.toNat < q.toNat) + (heq : gcdMainLoop q negInv rounds a u b v = (A, U, B, V)) : + U.toNat < q.toNat ∧ V.toNat < q.toNat := by + induction rounds generalizing a u b v with + | zero => + rw [show gcdMainLoop q negInv 0 a u b v = (a, u, b, v) from rfl] at heq + simp only [Prod.mk.injEq] at heq + obtain ⟨rfl, rfl, rfl, rfl⟩ := heq + exact ⟨huq, hvq⟩ + | succ k ih => + simp only [gcdMainLoop] at heq + rcases hnb : gcdNumBits a b with ⟨limbIdx, bits⟩ + rw [hnb] at heq + dsimp only at heq + rcases hI : gcdInner 31 (gcdApprox a limbIdx bits) (gcdApprox b limbIdx bits) 1 0 0 1 + with ⟨a1, b1, f0, g0, f1, g1⟩ + rw [hI] at heq + dsimp only at heq + rcases hdA : gcdLinearCombDiv a b f0 g0 with ⟨newA, signA⟩ + rw [hdA] at heq + dsimp only at heq + rcases hdB : gcdLinearCombDiv a b f1 g1 with ⟨newB, signB⟩ + rw [hdB] at heq + dsimp only at heq + obtain ⟨hrow0, hrow1⟩ := gcdInner_natAbs_le_31 (Nat.le_refl 31) hI + have hc0 : (if signA < 0 then -f0 else f0).natAbs + + (if signA < 0 then -g0 else g0).natAbs ≤ 2 ^ 31 := by + split <;> simpa [Int.natAbs_neg] using hrow0 + have hc1 : (if signB < 0 then -f1 else f1).natAbs + + (if signB < 0 then -g1 else g1).natAbs ≤ 2 ^ 31 := by + split <;> simpa [Int.natAbs_neg] using hrow1 + have hU' := gcdLinearCombMontyRed_lt (f := if signA < 0 then -f0 else f0) + (g := if signA < 0 then -g0 else g0) hq0 hnq huq hvq hc0 + have hV' := gcdLinearCombMontyRed_lt (f := if signB < 0 then -f1 else f1) + (g := if signB < 0 then -g1 else g1) hq0 hnq huq hvq hc1 + exact ih hU' hV' heq + +-- Tuple witnesses without evaluating `p`; a bare `rfl` witness would whnf-expand it. +private theorem exists_eq_tuple4 {α β γ δ : Type} (p : α × β × γ × δ) : + ∃ a b c d, p = (a, b, c, d) := + ⟨p.1, p.2.1, p.2.2.1, p.2.2.2, by simp only [Prod.mk.eta]⟩ + +private theorem exists_eq_tuple6 {α β γ δ ε ζ : Type} (p : α × β × γ × δ × ε × ζ) : + ∃ a b c d e f, p = (a, b, c, d, e, f) := + ⟨p.1, p.2.1, p.2.2.1, p.2.2.2.1, p.2.2.2.2.1, p.2.2.2.2.2, by simp only [Prod.mk.eta]⟩ + +/-- The final chunks stay canonical for canonical inputs. -/ +theorem gcdFinalChunks_lt {q : Limbs4} {negInv : UInt64} {finalRounds : ℕ} + {a u b v : Limbs4} (hfr : finalRounds ≤ 62) + (hq0 : 0 < q.toNat) (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) + (huq : u.toNat < q.toNat) (hvq : v.toNat < q.toNat) : + (gcdFinalChunks q negInv finalRounds a u b v).toNat < q.toNat := by + obtain ⟨aw1, bw1, f0, g0, f1, g1, hI1⟩ := + exists_eq_tuple6 (gcdInner ((finalRounds + 1) / 2) a.l0 b.l0 1 0 0 1) + obtain ⟨hrow0, hrow1⟩ := gcdInner_natAbs_le_31 (by omega) hI1 + have hu1 := gcdLinearCombMontyRed_lt (f := f0) (g := g0) hq0 hnq huq hvq hrow0 + have hv1 := gcdLinearCombMontyRed_lt (f := f1) (g := g1) hq0 hnq huq hvq hrow1 + obtain ⟨aw2, bw2, F0, G0, F1, G1, hI2⟩ := + exists_eq_tuple6 (gcdInner (finalRounds - (finalRounds + 1) / 2) aw1 bw1 1 0 0 1) + obtain ⟨-, hrowF⟩ := gcdInner_natAbs_le_31 (by omega) hI2 + have hfinal := gcdLinearCombMontyRed_lt (f := F1) (g := G1) hq0 hnq hu1 hv1 hrowF + have heq : gcdFinalChunks q negInv finalRounds a u b v + = gcdLinearCombMontyRed q negInv + (gcdLinearCombMontyRed q negInv u v f0 g0) + (gcdLinearCombMontyRed q negInv u v f1 g1) F1 G1 := by + rw [gcdFinalChunks.eq_def] + dsimp only + rw [hI1] + dsimp only + rw [hI2] + rw [heq] + exact hfinal + +set_option maxRecDepth 4000 in +/-- The candidate is canonical. -/ +theorem gcdInvCandidate_lt {modulus : ℕ} [P : GcdData modulus] {q x : Limbs4} + {negInv : UInt64} (hqm : q.toNat = modulus) (hq0 : 0 < q.toNat) + (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) : + (gcdInvCandidate modulus q negInv x).toNat < q.toNat := by + have hu0 : P.initU.toNat < q.toNat := by + rw [P.initU_toNat, hqm] + exact Nat.mod_lt _ (by omega) + have hv0 : Limbs4.zero.toNat < q.toNat := by + rw [Limbs4.zero_toNat] + omega + obtain ⟨a, u, b, v, hML⟩ := + exists_eq_tuple4 (gcdMainLoop q negInv 15 x P.initU q Limbs4.zero) + obtain ⟨hUq, hVq⟩ := gcdMainLoop_bounded hq0 hnq hu0 hv0 hML + -- `unfold` only; generating `gcdInvCandidate.eq_def` times out on the 15-round loop. + have hcand : gcdInvCandidate modulus q negInv x + = gcdFinalChunks q negInv P.finalRounds a u b v := by + unfold gcdInvCandidate + rw [hML] + rw [hcand] + exact gcdFinalChunks_lt P.finalRounds_le hq0 hnq hUq hVq + +variable {modulus : ℕ} [P : Mont64x4Field modulus] + +/-- With the class data, the candidate is canonical for any input. -/ +theorem gcdInvCandidate_lt_modulus [GcdData modulus] (x : Limbs4) : + (gcdInvCandidate modulus P.modulusLimbs P.montgomeryNegInv x).toNat < modulus := by + have h := gcdInvCandidate_lt (q := P.modulusLimbs) (negInv := P.montgomeryNegInv) (x := x) + Mont64x4Field.q_toNat (by rw [Mont64x4Field.q_toNat]; exact Mont64x4Field.modulus_pos) + Mont64x4Field.negInv_mul_q + rwa [Mont64x4Field.q_toNat] at h + +end MacSafety + +/-! ## The Fermat fallback and the checked raw inversion -/ + +section + +variable {modulus : ℕ} [P : Mont64x4Field modulus] + +private theorem toField_hpow (x : FastField modulus) (n : ℕ) : + (x ^ n).toField = x.toField ^ n := + FastField.toField_pow x n + +/-- `montPow` computes `acc · xⁿ`. -/ +theorem montPow_eq_mul_pow (acc x : FastField modulus) (n : ℕ) : + montPow P.modulusLimbs P.montgomeryNegInv acc.val x.val n = (acc * x ^ n).val := by + induction n using Nat.strong_induction_on generalizing acc x with + | _ n ih => + rw [montPow.eq_def] + split + next h => + subst h + refine congrArg Subtype.val (FastField.toField_injective ?_) + rw [FastField.toField_mul, toField_hpow, pow_zero, mul_one] + next h => + have hlt : n / 2 < n := by omega + by_cases hodd : n % 2 == 1 + · rw [ite_eq_left hodd, ← FastField.val_mul, ← FastField.val_mul, ih _ hlt] + simp only [beq_iff_eq] at hodd + refine congrArg Subtype.val (FastField.toField_injective ?_) + simp only [FastField.toField_mul, toField_hpow] + conv_rhs => rw [show n = 2 * (n / 2) + 1 by omega] + ring + · rw [ite_eq_right hodd, ← FastField.val_mul, ih _ hlt] + simp only [beq_iff_eq] at hodd + refine congrArg Subtype.val (FastField.toField_injective ?_) + simp only [FastField.toField_mul, toField_hpow] + conv_rhs => rw [show n = 2 * (n / 2) by omega] + ring + +/-- The `montPow` fallback computes the field inverse. -/ +theorem montPow_eq_inv (x : FastField modulus) : + montPow P.modulusLimbs P.montgomeryNegInv P.rModModulus x.val (modulus - 2) + = (x⁻¹).val := by + rw [show P.rModModulus = (1 : FastField modulus).val from rfl, montPow_eq_mul_pow] + have h1 : (1 : FastField modulus) * x ^ (modulus - 2) = x ^ (modulus - 2) := + FastField.toField_injective + (by rw [FastField.toField_mul, FastField.toField_one, one_mul]) + -- The `rfl` holds since `x⁻¹` unfolds to `pow x (modulus - 2)`. + exact congrArg Subtype.val (h1.trans rfl) + +/-- Accept a raw candidate `z` for `x⁻¹` if it verifies, else fall back to `x⁻¹`. -/ +private def invWithCandidate (x : FastField modulus) (z : Limbs4) : FastField modulus := + if h : (subLimbs z P.modulusLimbs).2.2.2.2 = 1 ∧ + Native64x4.mul P.modulusLimbs P.montgomeryNegInv z x.val = P.rModModulus then + ⟨z, by + obtain ⟨d0, d1, d2, d3, bo, e, -, -⟩ := subLimbs_spec z P.modulusLimbs + have hb := h.1 + rw [e] at hb + have hlt := (subLimbs_borrow_eq_one_iff e).mp hb + rwa [Mont64x4Field.q_toNat] at hlt⟩ + else x⁻¹ + +private theorem invWithCandidate_eq_inv (x : FastField modulus) (z : Limbs4) : + invWithCandidate x z = x⁻¹ := by + unfold invWithCandidate + split + case isTrue h => + refine eq_inv_of_mul_eq_one_left (Subtype.ext ?_) + show Native64x4.mul P.modulusLimbs P.montgomeryNegInv z x.val = P.rModModulus + exact h.2 + case isFalse _ => rfl + +-- Stated for a generic candidate: `rfl` against the GCD candidate would unfold it. +private theorem invGcdRaw_eq_invWithCandidate (x : FastField modulus) (z : Limbs4) : + (if (subLimbs z P.modulusLimbs).2.2.2.2 = 1 ∧ + Native64x4.mul P.modulusLimbs P.montgomeryNegInv z x.val = P.rModModulus then z + else montPow P.modulusLimbs P.montgomeryNegInv P.rModModulus x.val (modulus - 2)) + = (invWithCandidate x z).val := by + unfold invWithCandidate + split + next h => rfl + next h => exact montPow_eq_inv x + +/-- `invGcdRaw` computes the field inverse. -/ +theorem invGcdRaw_eq_inv [GcdData modulus] (x : FastField modulus) : + invGcdRaw modulus P.modulusLimbs P.montgomeryNegInv P.rModModulus x.val + = (x⁻¹).val := + (invGcdRaw_eq_invWithCandidate x _).trans + (congrArg Subtype.val (invWithCandidate_eq_inv x _)) + +end + +/-! ## The proof-carrying wrapper -/ + +namespace FastField + +variable {modulus : ℕ} [P : Mont64x4Field modulus] + +/-- The proof-carrying wrapper of the checked inversion `invGcdRaw`. -/ +@[inline] def invGcd [GcdData modulus] (x : FastField modulus) : FastField modulus := + ⟨invGcdRaw modulus P.modulusLimbs P.montgomeryNegInv P.rModModulus x.val, by + rw [invGcdRaw_eq_inv x] + exact (x⁻¹).property⟩ + +/-- `invGcd` agrees with the `Field` inverse. -/ +@[simp] +theorem invGcd_eq_inv [GcdData modulus] (x : FastField modulus) : + invGcd x = x⁻¹ := by + have hval : (invGcd x).val + = invGcdRaw modulus P.modulusLimbs P.montgomeryNegInv P.rModModulus x.val := by + simp only [invGcd] + exact Subtype.ext (hval.trans (invGcdRaw_eq_inv x)) + +end FastField + +end Montgomery.Native64x4 diff --git a/CompPoly/Fields/Montgomery/Native64x4InvDefs.lean b/CompPoly/Fields/Montgomery/Native64x4InvDefs.lean new file mode 100644 index 00000000..bed835ca --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native64x4InvDefs.lean @@ -0,0 +1,219 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos, Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Montgomery.Native64x4Defs +public import CompPoly.Fields.Montgomery.Native64x8InvDefs + +/-! +# Four-limb inversion: runtime definitions (Mathlib-free) + +The Pornin binary-GCD inverse candidate over `Limbs4` and its checked wrapper +(`invGcdRaw`), Mathlib-free for `precompileModules` consumers. The word-sized divstep loop +and bit-length helper are shared with the eight-limb stack. The proof side is +`Montgomery/Native64x4Inv`. +-/ + +@[expose] public section + +namespace Montgomery.Native64x4 + +open Montgomery.Native64x8 (gcdInner gcdInnerFast gcdBitLen) + +/-! ## Per-field schedule -/ + +/-- Per-field schedule of the binary-GCD inverse candidate; every obligation defaults to +`decide`. -/ +class GcdData (modulus : Nat) where + /-- Divsteps in the final word-sized phase: `2·bits(p) - 2 - 15·31`. -/ + finalRounds : Nat + /-- Initial `u`: the power of two that keeps the candidate in Montgomery form. -/ + initU : Limbs4 + initU_toNat : initU.toNat = 2 ^ (1135 - finalRounds) % modulus := by decide + /-- Final-phase chunks stay at mac width. -/ + finalRounds_le : finalRounds ≤ 62 := by decide + +/-! ## Five-word arithmetic -/ + +/-- `x * k` as five words. -/ +@[inline] def mulWord (x : Limbs4) (k : UInt64) : UInt64 × UInt64 × UInt64 × UInt64 × UInt64 := + let (w0, c) := mac 0 x.l0 k 0 + let (w1, c) := mac 0 x.l1 k c + let (w2, c) := mac 0 x.l2 k c + let (w3, w4) := mac 0 x.l3 k c + (w0, w1, w2, w3, w4) + +/-- Five-word add: the sum words and the carry out. -/ +@[inline] def add5 (a0 a1 a2 a3 a4 b0 b1 b2 b3 b4 : UInt64) : + UInt64 × UInt64 × UInt64 × UInt64 × UInt64 × UInt64 := + let (s0, c) := adc a0 b0 0 + let (s1, c) := adc a1 b1 c + let (s2, c) := adc a2 b2 c + let (s3, c) := adc a3 b3 c + let (s4, c) := adc a4 b4 c + (s0, s1, s2, s3, s4, c) + +/-- Five-word subtract: the difference words and the borrow out. -/ +@[inline] def sub5 (a0 a1 a2 a3 a4 b0 b1 b2 b3 b4 : UInt64) : + UInt64 × UInt64 × UInt64 × UInt64 × UInt64 × UInt64 := + let (d0, b) := sbb a0 b0 0 + let (d1, b) := sbb a1 b1 b + let (d2, b) := sbb a2 b2 b + let (d3, b) := sbb a3 b3 b + let (d4, b) := sbb a4 b4 b + (d0, d1, d2, d3, d4, b) + +/-! ## Linear combinations -/ + +/-- `(f·a + g·b) / 2^31` as magnitude and sign. Requires `|f| + |g| ≤ 2^31`. -/ +@[inline] def gcdLinearCombDiv (a b : Limbs4) (f g : Int) : Limbs4 × Int := + let (fa0, fa1, fa2, fa3, fa4) := mulWord a (UInt64.ofNat f.natAbs) + let (gb0, gb1, gb2, gb3, gb4) := mulWord b (UInt64.ofNat g.natAbs) + let fNeg := f < 0 + let gNeg := g < 0 + let (p0, p1, p2, p3, p4) := + if fNeg then ((0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64)) + else (fa0, fa1, fa2, fa3, fa4) + let (q0, q1, q2, q3, q4) := + if gNeg then ((0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64)) + else (gb0, gb1, gb2, gb3, gb4) + let (n0, n1, n2, n3, n4) := + if fNeg then (fa0, fa1, fa2, fa3, fa4) + else ((0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64)) + let (m0, m1, m2, m3, m4) := + if gNeg then (gb0, gb1, gb2, gb3, gb4) + else ((0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64), (0 : UInt64)) + let (s0, s1, s2, s3, s4, _) := add5 p0 p1 p2 p3 p4 q0 q1 q2 q3 q4 + let (t0, t1, t2, t3, t4, _) := add5 n0 n1 n2 n3 n4 m0 m1 m2 m3 m4 + let (d0, d1, d2, d3, d4, borrow) := sub5 s0 s1 s2 s3 s4 t0 t1 t2 t3 t4 + if borrow == 0 then + (⟨(d0 >>> 31) ||| (d1 <<< 33), (d1 >>> 31) ||| (d2 <<< 33), (d2 >>> 31) ||| (d3 <<< 33), + (d3 >>> 31) ||| (d4 <<< 33)⟩, (0 : Int)) + else + let (e0, e1, e2, e3, e4, _) := sub5 t0 t1 t2 t3 t4 s0 s1 s2 s3 s4 + (⟨(e0 >>> 31) ||| (e1 <<< 33), (e1 >>> 31) ||| (e2 <<< 33), (e2 >>> 31) ||| (e3 <<< 33), + (e3 >>> 31) ||| (e4 <<< 33)⟩, (-1 : Int)) + +/-- `n - u` as limbs, for `u < n`. -/ +@[inline] def negMod (q u : Limbs4) : Limbs4 := + let (d0, d1, d2, d3, _) := subLimbs q u + ⟨d0, d1, d2, d3⟩ + +/-- The Montgomery linear combination on magnitudes: `(F·uS + G·vS) · 2^-64 mod q`, reduced +once through `condSubWide`. Requires `F + G ≤ 2^31` and `uS, vS ≤ q`. -/ +@[inline] def lincombTail (q : Limbs4) (negInv : UInt64) (uS vS : Limbs4) (F G : UInt64) : + Limbs4 := + let (a0, a1, a2, a3, a4) := mulWord uS F + let (b0, b1, b2, b3, b4) := mulWord vS G + let (s0, s1, s2, s3, s4, _) := add5 a0 a1 a2 a3 a4 b0 b1 b2 b3 b4 + let m := montM s0 negInv + let (n0, n1, n2, n3, n4) := mulWord q m + let (_, t1, t2, t3, t4, t5) := add5 s0 s1 s2 s3 s4 n0 n1 n2 n3 n4 + condSubWide q ⟨t1, t2, t3, t4, t5⟩ + +/-- `(f·u + g·v) · 2^-64 mod q`, canonical. Requires `|f| + |g| ≤ 2^31` and `u, v < q`. -/ +@[inline] def gcdLinearCombMontyRed (q : Limbs4) (negInv : UInt64) (u v : Limbs4) + (f g : Int) : Limbs4 := + lincombTail q negInv (if f < 0 then negMod q u else u) (if g < 0 then negMod q v else v) + (UInt64.ofNat f.natAbs) (UInt64.ofNat g.natAbs) + +/-! ## Approximation: one word at the shared bit length -/ + +/-- Read limb `i` (out-of-range indices read the top limb). -/ +@[inline] def gcdLimb (v : Limbs4) : Nat → UInt64 + | 0 => v.l0 + | 1 => v.l1 + | 2 => v.l2 + | _ => v.l3 + +/-- Highest nonzero limb index among limbs 1-3 of `a ||| b` and its bit length; `(0, 0)` +when all are zero. -/ +@[inline] def gcdNumBits (a b : Limbs4) : Nat × Nat := + let v3 := gcdBitLen (a.l3 ||| b.l3) + if v3 != 0 then (3, v3) else + let v2 := gcdBitLen (a.l2 ||| b.l2) + if v2 != 0 then (2, v2) else + let v1 := gcdBitLen (a.l1 ||| b.l1) + if v1 != 0 then (1, v1) else (0, 0) + +/-- One-word approximation: the top 33 bits at the shared bit length above the bottom 31; +exact once both values fit one word. -/ +@[inline] def gcdApprox (val : Limbs4) (limbIdx bits : Nat) : UInt64 := + if limbIdx == 0 then val.l0 + else + let hi := gcdLimb val limbIdx + let lo := gcdLimb val (limbIdx - 1) + let top := + if bits ≥ 33 then hi >>> (UInt64.ofNat (bits - 33)) + else ((hi <<< (UInt64.ofNat (33 - bits))) ||| (lo >>> (UInt64.ofNat (bits + 31)))) &&& + 0x1FFFFFFFF + (top <<< 31) ||| (val.l0 &&& 0x7FFFFFFF) + +/-! ## Main loop and candidate -/ + +/-- The outer rounds: 31 divsteps on one-word approximations, then the transition matrix +applied to both tracks. -/ +@[specialize] def gcdMainLoop (q : Limbs4) (negInv : UInt64) (rounds : Nat) (a u b v : Limbs4) + (inner : Nat → UInt64 → UInt64 → Int → Int → Int → Int → + UInt64 × UInt64 × Int × Int × Int × Int := gcdInner) : + Limbs4 × Limbs4 × Limbs4 × Limbs4 := + match rounds with + | 0 => (a, u, b, v) + | n + 1 => + let (limbIdx, bits) := gcdNumBits a b + let aT := gcdApprox a limbIdx bits + let bT := gcdApprox b limbIdx bits + let (_, _, f0, g0, f1, g1) := inner 31 aT bT 1 0 0 1 + let (newA, signA) := gcdLinearCombDiv a b f0 g0 + let f0 := if signA < 0 then -f0 else f0 + let g0 := if signA < 0 then -g0 else g0 + let (newB, signB) := gcdLinearCombDiv a b f1 g1 + let f1 := if signB < 0 then -f1 else f1 + let g1 := if signB < 0 then -g1 else g1 + let newU := gcdLinearCombMontyRed q negInv u v f0 g0 + let newV := gcdLinearCombMontyRed q negInv u v f1 g1 + gcdMainLoop q negInv n newA newU newB newV inner + +/-- The final divsteps as two mac-width chunks, folding the Montgomery pair. -/ +@[specialize] def gcdFinalChunks (q : Limbs4) (negInv : UInt64) (finalRounds : Nat) + (a u b v : Limbs4) + (inner : Nat → UInt64 → UInt64 → Int → Int → Int → Int → + UInt64 × UInt64 × Int × Int × Int × Int := gcdInner) : Limbs4 := + let c1 := (finalRounds + 1) / 2 + let (aw1, bw1, f0, g0, f1, g1) := inner c1 a.l0 b.l0 1 0 0 1 + let u1 := gcdLinearCombMontyRed q negInv u v f0 g0 + let v1 := gcdLinearCombMontyRed q negInv u v f1 g1 + let (_, _, _, _, fF, gF) := inner (finalRounds - c1) aw1 bw1 1 0 0 1 + gcdLinearCombMontyRed q negInv u1 v1 fF gF + +/-- Pornin binary-GCD candidate for the Montgomery inverse, canonical nonzero `x·R mod p` +to `x⁻¹·R mod p`; proof-free, callers verify. -/ +@[specialize] def gcdInvCandidate (modulus : Nat) [P : GcdData modulus] (q : Limbs4) + (negInv : UInt64) (x : Limbs4) + (inner : Nat → UInt64 → UInt64 → Int → Int → Int → Int → + UInt64 × UInt64 × Int × Int × Int × Int := gcdInner) : Limbs4 := + let (a, u, b, v) := gcdMainLoop q negInv 15 x P.initU q Limbs4.zero inner + gcdFinalChunks q negInv P.finalRounds a u b v inner + +/-! ## Checked inversion over raw limbs -/ + +/-- `acc · x^n` in Montgomery form by binary powering. -/ +def montPow (q : Limbs4) (negInv : UInt64) (acc x : Limbs4) (n : Nat) : Limbs4 := + if h : n = 0 then acc + else + montPow q negInv (if n % 2 == 1 then mul q negInv acc x else acc) + (mul q negInv x x) (n / 2) +termination_by n +decreasing_by omega + +/-- The GCD candidate, accepted only if it verifies (`z · x = 1`); else Fermat `montPow`. -/ +def invGcdRaw (modulus : Nat) [GcdData modulus] (q : Limbs4) (negInv : UInt64) + (rMod : Limbs4) (x : Limbs4) : Limbs4 := + let cand := gcdInvCandidate modulus q negInv x gcdInnerFast + if (subLimbs cand q).2.2.2.2 = 1 ∧ mul q negInv cand x = rMod then cand + else montPow q negInv rMod x (modulus - 2) + +end Montgomery.Native64x4 diff --git a/CompPoly/Fields/Montgomery/Native64x4Mul.lean b/CompPoly/Fields/Montgomery/Native64x4Mul.lean new file mode 100644 index 00000000..724aafd5 --- /dev/null +++ b/CompPoly/Fields/Montgomery/Native64x4Mul.lean @@ -0,0 +1,229 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos, Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Montgomery.Native64x4 +public import Mathlib.Tactic.Linarith + +/-! +# Correctness of four-limb CIOS Montgomery multiplication + +A round accumulates `a * bi` exactly (`mulAccum_spec`) and cancels the low limb against a +multiple of the modulus (`mulReduce_spec`); folding four rounds gives +`2 ^ 256 * ⟦t₄⟧ = ⟦a⟧ * ⟦b⟧ + M * q`, the Montgomery product up to the final conditional +subtraction. + +## Main results + +* `mulRound_spec` — the round invariant with the `2 * q` bound +* `mul_spec` — `mul` is canonical and satisfies `2 ^ 256 * ⟦mul a b⟧ ≡ ⟦a⟧ * ⟦b⟧ [MOD q]` +-/ + +@[expose] public section + +namespace Montgomery +namespace Native64x4 + +/-! ### Arithmetic helpers -/ + +/-- Scalar multiplication distributes over a limb recomposition. -/ +theorem sum4_mul (a0 a1 a2 a3 b : ℕ) : + (a0 + 2 ^ 64 * a1 + 2 ^ 128 * a2 + 2 ^ 192 * a3) * b = + a0 * b + 2 ^ 64 * (a1 * b) + 2 ^ 128 * (a2 * b) + 2 ^ 192 * (a3 * b) := by + ring + +/-- Scalar multiplication distributes over a limb recomposition, from the left. -/ +theorem mul_sum4 (b a0 a1 a2 a3 : ℕ) : + b * (a0 + 2 ^ 64 * a1 + 2 ^ 128 * a2 + 2 ^ 192 * a3) = + b * a0 + 2 ^ 64 * (b * a1) + 2 ^ 128 * (b * a2) + 2 ^ 192 * (b * a3) := by + ring + +/-- The Montgomery multiplier makes the low limb of the reduction vanish. -/ +private theorem montM_low_zero {s negInv Q q0 w u : ℕ} (hs : s < 2 ^ 64) + (hq0 : Q % 2 ^ 64 = q0) (hnq : negInv * Q % 2 ^ 64 = 2 ^ 64 - 1) + (h : w + 2 ^ 64 * u = s + s * negInv % 2 ^ 64 * q0) (hw : w < 2 ^ 64) : w = 0 := by + have hdvd : 2 ^ 64 ∣ s + s % 2 ^ 64 * negInv % 2 ^ 64 * Q := + Montgomery.dvd_add (2 ^ 64) Q negInv (by norm_num) hnq s + rw [Nat.mod_eq_of_lt hs] at hdvd + have hq0' : q0 % 2 ^ 64 = Q % 2 ^ 64 := by omega + have hcong : (s + s * negInv % 2 ^ 64 * q0) % 2 ^ 64 + = (s + s * negInv % 2 ^ 64 * Q) % 2 ^ 64 := + Nat.ModEq.add_left s (Nat.ModEq.mul_left _ hq0') + obtain ⟨c, hc⟩ := hdvd + omega + +/-- The accumulator stays below `2 * q` from one round to the next. -/ +private theorem round_bound {T A bi m Q Tp : ℕ} (h : 2 ^ 64 * Tp = T + A * bi + m * Q) + (hT : T < 2 * Q) (hA : A < Q) (hbi : bi < 2 ^ 64) (hm : m < 2 ^ 64) : Tp < 2 * Q := by + nlinarith [h, hT, hA, hbi, hm] + +/-! ### The accumulator -/ + +theorem State5.zero_toNat : State5.zero.toNat = 0 := by + simp only [State5.toNat, State5.toLimbs4, State5.zero, Limbs4.toNat, UInt64.toNat_zero] + +/-- The value of an accumulator in terms of its limbs. -/ +theorem State5.toNat_eq (t : State5) : + t.toNat = t.t0.toNat + 2 ^ 64 * t.t1.toNat + 2 ^ 128 * t.t2.toNat + 2 ^ 192 * t.t3.toNat + + 2 ^ 256 * t.t4.toNat := by + simp only [State5.toNat, State5.toLimbs4, Limbs4.toNat] + +/-! ### The accumulation step -/ + +/-- `mulAccum` adds `a * bi` to the accumulator exactly, with a one-bit carry limb. -/ +theorem mulAccum_spec (a : Limbs4) (bi : UInt64) (t : State5) : + (mulAccum a bi t).t5.toNat ≤ 1 ∧ + (mulAccum a bi t).toNat = t.toNat + a.toNat * bi.toNat := by + obtain ⟨s0, k0, e0, h0⟩ := mac_spec t.t0 a.l0 bi 0 + obtain ⟨s1, k1, e1, h1⟩ := mac_spec t.t1 a.l1 bi k0 + obtain ⟨s2, k2, e2, h2⟩ := mac_spec t.t2 a.l2 bi k1 + obtain ⟨s3, k3, e3, h3⟩ := mac_spec t.t3 a.l3 bi k2 + obtain ⟨s4, s5, e4, h4, g4⟩ := adc_spec t.t4 k3 0 (by decide) + rw [UInt64.toNat_zero] at h0 h4 + have hchain := carry_chain_sum h0 h1 h2 h3 + rw [← sum4_mul, ← Limbs4.toNat, Nat.add_zero] at hchain + simp only [mulAccum, e0, e1, e2, e3, e4] + refine ⟨g4, ?_⟩ + rw [State5.toNat_eq] + simp only [State6.toNat, Limbs4.toNat] at hchain ⊢ + omega + +/-! ### The reduction step -/ + +/-- `mulReduce` adds the multiple of the modulus that cancels the low limb, so dropping that +limb divides by `2 ^ 64` exactly. -/ +theorem mulReduce_spec (q : Limbs4) (negInv : UInt64) (s : State6) (hs5 : s.t5.toNat ≤ 1) + (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) : + 2 ^ 64 * (mulReduce q negInv s).toNat = + s.toNat + (montM s.t0 negInv).toNat * q.toNat := by + have hmv : (montM s.t0 negInv).toNat = s.t0.toNat * negInv.toNat % 2 ^ 64 := + montM_toNat _ _ + obtain ⟨w0, u0, f0, g0⟩ := mac_spec s.t0 (montM s.t0 negInv) q.l0 0 + obtain ⟨w1, u1, f1, g1⟩ := mac_spec s.t1 (montM s.t0 negInv) q.l1 u0 + obtain ⟨w2, u2, f2, g2⟩ := mac_spec s.t2 (montM s.t0 negInv) q.l2 u1 + obtain ⟨w3, u3, f3, g3⟩ := mac_spec s.t3 (montM s.t0 negInv) q.l3 u2 + obtain ⟨w4, u4, f4, g4, k4⟩ := adc_spec s.t4 u3 0 (by decide) + rw [UInt64.toNat_zero] at g0 g4 + have hw0 : w0.toNat = 0 := by + have hw := UInt64.toNat_lt w0 + rw [hmv] at g0 + exact montM_low_zero (UInt64.toNat_lt s.t0) (Limbs4.toNat_mod q) hnq + (by rw [Nat.add_zero] at g0; exact g0) hw + have hchain := carry_chain_sum g0 g1 g2 g3 + rw [← mul_sum4, ← Limbs4.toNat, Nat.add_zero] at hchain + have hhead : (s.t5 + u4).toNat = s.t5.toNat + u4.toNat := by + rw [UInt64.toNat_add, Nat.mod_eq_of_lt (by omega)] + simp only [mulReduce, f0, f1, f2, f3, f4] + rw [State5.toNat_eq] + simp only [State6.toNat, hhead] + omega + +/-! ### The round invariant -/ + +/-- The CIOS round invariant: one round accumulates `a * bi` and cancels the low limb against +a multiple `m` of the modulus; the accumulator stays below `2 * q`. -/ +theorem mulRound_spec (q a : Limbs4) (negInv bi : UInt64) (t : State5) + (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) + (haq : a.toNat < q.toNat) (htq : t.toNat < 2 * q.toNat) : + (mulRound q negInv a bi t).toNat < 2 * q.toNat ∧ + ∃ m : ℕ, m < 2 ^ 64 ∧ + 2 ^ 64 * (mulRound q negInv a bi t).toNat = + t.toNat + a.toNat * bi.toNat + m * q.toNat := by + obtain ⟨ha5, hav⟩ := mulAccum_spec a bi t + have hrv := mulReduce_spec q negInv (mulAccum a bi t) ha5 hnq + rw [hav] at hrv + have hinv : 2 ^ 64 * (mulRound q negInv a bi t).toNat = + t.toNat + a.toNat * bi.toNat + (montM (mulAccum a bi t).t0 negInv).toNat * q.toNat := by + rw [mulRound] + exact hrv + exact ⟨round_bound hinv htq haq (UInt64.toNat_lt bi) (UInt64.toNat_lt _), _, + UInt64.toNat_lt _, hinv⟩ + +/-! ### The four-round fold -/ + +/-- Folding the four round invariants into a single Montgomery identity. -/ +private theorem fold4 {T1 T2 T3 T4 A Q B0 B1 B2 B3 m0 m1 m2 m3 : ℕ} + (h0 : 2 ^ 64 * T1 = 0 + A * B0 + m0 * Q) + (h1 : 2 ^ 64 * T2 = T1 + A * B1 + m1 * Q) + (h2 : 2 ^ 64 * T3 = T2 + A * B2 + m2 * Q) + (h3 : 2 ^ 64 * T4 = T3 + A * B3 + m3 * Q) : + 2 ^ 256 * T4 = + (A * B0 + 2 ^ 64 * (A * B1) + 2 ^ 128 * (A * B2) + 2 ^ 192 * (A * B3)) + + (m0 * Q + 2 ^ 64 * (m1 * Q) + 2 ^ 128 * (m2 * Q) + 2 ^ 192 * (m3 * Q)) := by + omega + +/-- The unnormalized product is below `2 * q` and agrees with Montgomery multiplication. +Only the first operand must be canonical; the second may be any four-limb value. -/ +theorem mulUnreduced_spec (q : Limbs4) (negInv : UInt64) (a b : Limbs4) + (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) (haq : a.toNat < q.toNat) : + (mulUnreduced q negInv a b).toNat < 2 * q.toNat ∧ + 2 ^ 256 * (mulUnreduced q negInv a b).toNat ≡ a.toNat * b.toNat [MOD q.toNat] := by + obtain ⟨L1, m0, hm0, r0⟩ := + mulRound_spec q a negInv b.l0 State5.zero hnq haq (by rw [State5.zero_toNat]; omega) + obtain ⟨L2, m1, hm1, r1⟩ := mulRound_spec q a negInv b.l1 _ hnq haq L1 + obtain ⟨L3, m2, hm2, r2⟩ := mulRound_spec q a negInv b.l2 _ hnq haq L2 + obtain ⟨L4, m3, hm3, r3⟩ := mulRound_spec q a negInv b.l3 _ hnq haq L3 + rw [State5.zero_toNat] at r0 + have hfold := fold4 r0 r1 r2 r3 + rw [← mul_sum4, ← sum4_mul, ← Limbs4.toNat] at hfold + refine ⟨L4, ?_⟩ + change (2 ^ 256 * (mulUnreduced q negInv a b).toNat) % q.toNat = _ + simp only [mulUnreduced] + rw [hfold, Nat.add_mul_mod_self_right] + +/-- The final conditional subtraction preserves the unnormalized product's congruence. -/ +private theorem mul_finish (q : Limbs4) (t : State5) {A : ℕ} + (hlt : t.toNat < 2 * q.toNat) (hmod : 2 ^ 256 * t.toNat ≡ A [MOD q.toNat]) : + (condSubWide q t).toNat < q.toNat ∧ + 2 ^ 256 * (condSubWide q t).toNat ≡ A [MOD q.toNat] := by + refine ⟨condSubWide_lt q _ hlt, ?_⟩ + rw [condSubWide_toNat q _ hlt] + split + · exact hmod + · refine (Nat.ModEq.mul_left _ ?_).trans hmod + exact (Nat.modEq_iff_dvd' (by omega)).2 ⟨1, by omega⟩ + +/-- Montgomery multiplication is canonical and computes `a * b * (2 ^ 256)⁻¹ mod q`. -/ +theorem mul_spec (q : Limbs4) (negInv : UInt64) (a b : Limbs4) + (hnq : negInv.toNat * q.toNat % 2 ^ 64 = 2 ^ 64 - 1) (haq : a.toNat < q.toNat) : + (mul q negInv a b).toNat < q.toNat ∧ + 2 ^ 256 * (mul q negInv a b).toNat ≡ a.toNat * b.toNat [MOD q.toNat] := by + obtain ⟨hbound, hmod⟩ := mulUnreduced_spec q negInv a b hnq haq + exact mul_finish q _ hbound hmod + +private theorem mulHi_comm (a b : UInt64) : mulHi a b = mulHi b a := by + have h1 := mulHi_spec a b + have h2 := mulHi_spec b a + rw [UInt64.mul_comm b a, Nat.mul_comm b.toNat a.toNat] at h2 + apply UInt64.toNat_inj.mp + omega + +private theorem macWords_mul (t a b c : UInt64) : + macWords t (a * b) (mulHi a b) c = mac t a b c := rfl + +private theorem mac_comm (t a b c : UInt64) : mac t a b c = mac t b a c := by + simp only [mac, UInt64.mul_comm a b, mulHi_comm a b] + +/-- Sharing symmetric products preserves the CIOS square exactly. -/ +theorem squareCached_eq_mul (q : Limbs4) (negInv : UInt64) (a : Limbs4) : + squareCached q negInv a = mul q negInv a a := by + have h01 : ∀ t c, mac t a.l1 a.l0 c = mac t a.l0 a.l1 c := + fun t c ↦ mac_comm t a.l1 a.l0 c + have h02 : ∀ t c, mac t a.l2 a.l0 c = mac t a.l0 a.l2 c := + fun t c ↦ mac_comm t a.l2 a.l0 c + have h03 : ∀ t c, mac t a.l3 a.l0 c = mac t a.l0 a.l3 c := + fun t c ↦ mac_comm t a.l3 a.l0 c + have h12 : ∀ t c, mac t a.l2 a.l1 c = mac t a.l1 a.l2 c := + fun t c ↦ mac_comm t a.l2 a.l1 c + have h13 : ∀ t c, mac t a.l3 a.l1 c = mac t a.l1 a.l3 c := + fun t c ↦ mac_comm t a.l3 a.l1 c + have h23 : ∀ t c, mac t a.l3 a.l2 c = mac t a.l2 a.l3 c := + fun t c ↦ mac_comm t a.l3 a.l2 c + simp only [squareCached, mul, mulUnreduced, mulRound, mulAccum, macWords_mul, + h01, h02, h03, h12, h13, h23] + +end Native64x4 +end Montgomery diff --git a/CompPoly/Fields/Montgomery/Native64x8.lean b/CompPoly/Fields/Montgomery/Native64x8.lean index 13afc545..0c75fe31 100644 --- a/CompPoly/Fields/Montgomery/Native64x8.lean +++ b/CompPoly/Fields/Montgomery/Native64x8.lean @@ -7,11 +7,12 @@ module public import CompPoly.Fields.Montgomery.Basic public import CompPoly.Fields.Montgomery.Native64x8Defs +public import Mathlib.Tactic.SplitIfs /-! # Native Montgomery arithmetic over eight 32-bit limbs -Raw word operations for prime moduli below `2 ^ 255`, represented as eight 32-bit limbs +Raw word operations for prime moduli below `2 ^ 256`, represented as eight 32-bit limbs carried in the low halves of `UInt64` words (`Limbs8`). This is the multi-limb sibling of `Montgomery/Native32.lean`: every operation is a straight-line chain of `@[inline]` word helpers, and each helper comes with an existential `toNat` specification that names its @@ -30,8 +31,9 @@ invariant is proved in the sibling module that builds the field carrier. ## Main results -* `condSub_toNat`, `add_toNat`, `sub_toNat`, `neg_toNat` — correctness of the raw operations -* `addLimbs_toNat`, `subLimbs_spec` — the underlying carry/borrow chains +* `condSub_toNat`, `condSubWide_toNat`, `add_toNat`, `sub_toNat`, `neg_toNat` — correctness + of the raw operations +* `addLimbs_spec`, `subLimbs_spec` — the underlying carry/borrow chains -/ @[expose] public section @@ -252,9 +254,16 @@ private theorem cond_eq_mod {T Q : ℕ} (h : T < 2 * Q) : conv_rhs => rw [show T = T - Q + Q by omega] rw [Nat.add_mod_right, Nat.mod_eq_of_lt (by omega)] -private theorem carry_top_zero {S A B Q c : ℕ} (h : S + 2 ^ 256 * c = A + B) - (hS : S < 2 ^ 256) (hA : A < Q) (hB : B < Q) (hQ : 2 * Q < 2 ^ 256) : - S = A + B ∧ S < 2 * Q := by +private theorem add_of_carry {S A B Q D bo : ℕ} (hchain : S + 2 ^ 256 * 1 = A + B) + (hsub : D + Q = S + 2 ^ 256 * bo) (hbo : bo ≤ 1) (hD : D < 2 ^ 256) (hQ : Q < 2 ^ 256) + (hA : A < Q) (hB : B < Q) : D = (A + B) % Q := by + rw [show A + B = D + Q by omega, Nat.add_mod_right, Nat.mod_eq_of_lt (by omega)] + +private theorem wide_of_head {T L t8 Q D bo : ℕ} (hdec : T = L + 2 ^ 256 * t8) + (hsub : D + Q = L + 2 ^ 256 * bo) (hbo : bo ≤ 1) (hD : D < 2 ^ 256) + (hQ : Q < 2 ^ 256) (hT : T < 2 * Q) (h : ¬t8 = 0) : + D = if T < Q then T else T - Q := by + rw [ite_eq_right (by omega)] omega private theorem sub_of_borrow_zero {A B Q D : ℕ} (h : D + B = A + 2 ^ 256 * 0) @@ -264,7 +273,7 @@ private theorem sub_of_borrow_zero {A B Q D : ℕ} (h : D + B = A + 2 ^ 256 * 0) private theorem sub_of_borrow_one {A B Q D E c : ℕ} (hD : D + B = A + 2 ^ 256 * 1) (hE : E + 2 ^ 256 * c = D + Q) (hDlt : D < 2 ^ 256) (hElt : E < 2 ^ 256) - (hB : B < Q) (hA : A < Q) (hQ : 2 * Q < 2 ^ 256) : E = (A + (Q - B)) % Q := by + (hB : B < Q) (hA : A < Q) (hQ : Q < 2 ^ 256) : E = (A + (Q - B)) % Q := by rw [show A + (Q - B) = E by omega, Nat.mod_eq_of_lt (by omega)] /-! ## Limbwise addition and subtraction -/ @@ -274,9 +283,10 @@ theorem addLimbs_bounded (a b : Limbs8) : (addLimbs a b).Bounded := by exact ⟨adcLo_lt _ _ _, adcLo_lt _ _ _, adcLo_lt _ _ _, adcLo_lt _ _ _, adcLo_lt _ _ _, adcLo_lt _ _ _, adcLo_lt _ _ _, adcLo_lt _ _ _⟩ -/-- The limbwise sum and the discarded top carry recompose the sum of the inputs. -/ -theorem addLimbs_toNat (a b : Limbs8) (ha : a.Bounded) (hb : b.Bounded) : - ∃ c : ℕ, c ≤ 1 ∧ (addLimbs a b).toNat + 2 ^ 256 * c = a.toNat + b.toNat := by +/-- The limbwise sum and the top carry recompose the sum of the inputs. -/ +theorem addLimbs_spec (a b : Limbs8) (ha : a.Bounded) (hb : b.Bounded) : + (addCarry a b).toNat ≤ 1 ∧ + (addLimbs a b).toNat + 2 ^ 256 * (addCarry a b).toNat = a.toNat + b.toNat := by obtain ⟨ha0, ha1, ha2, ha3, ha4, ha5, ha6, ha7⟩ := ha obtain ⟨hb0, hb1, hb2, hb3, hb4, hb5, hb6, hb7⟩ := hb obtain ⟨s0, c0, ds0, dc0, e0, gc0, hs0⟩ := @@ -296,11 +306,19 @@ theorem addLimbs_toNat (a b : Limbs8) (ha : a.Bounded) (hb : b.Bounded) : obtain ⟨s7, c7, ds7, dc7, e7, gc7, hs7⟩ := adc_spec a.l7 b.l7 _ ha7 hb7 (dc6 ▸ gc6) simp only [dc0, dc1, dc2, dc3, dc4, dc5, dc6, toNat_zero] at e0 e1 e2 e3 e4 e5 e6 e7 - refine ⟨c7, gc7, ?_⟩ - simp only [addLimbs, Limbs8.toNat, ds0, ds1, ds2, ds3, ds4, ds5, ds6, ds7] - have h := carry_chain_sum e0 e1 e2 e3 e4 e5 e6 e7 - simp only [Nat.add_zero] at h - exact h + refine ⟨?_, ?_⟩ + · simp only [addCarry, dc7] + exact gc7 + · simp only [addLimbs, addCarry, Limbs8.toNat, ds0, ds1, ds2, ds3, ds4, ds5, ds6, ds7, + dc7] + have h := carry_chain_sum e0 e1 e2 e3 e4 e5 e6 e7 + simp only [Nat.add_zero] at h + exact h + +/-- The limbwise sum and some top carry recompose the sum of the inputs. -/ +theorem addLimbs_toNat (a b : Limbs8) (ha : a.Bounded) (hb : b.Bounded) : + ∃ c : ℕ, c ≤ 1 ∧ (addLimbs a b).toNat + 2 ^ 256 * c = a.toNat + b.toNat := + ⟨_, (addLimbs_spec a b ha hb).1, (addLimbs_spec a b ha hb).2⟩ theorem subLimbs_bounded (a b : Limbs8) : (subLimbs a b).Bounded := by simp only [subLimbs, Limbs8.Bounded] @@ -341,25 +359,29 @@ theorem subLimbs_spec (a b : Limbs8) (ha : a.Bounded) (hb : b.Bounded) : /-! ## Conditional subtraction -/ +/-- Lexicographic comparison agrees with the represented natural number on bounded limbs. -/ +theorem limbsLt_eq_true_iff (a b : Limbs8) (ha : a.Bounded) (hb : b.Bounded) : + limbsLt a b = true ↔ a.toNat < b.toNat := by + simp only [limbsLt, Limbs8.Bounded, Limbs8.toNat, beq_iff_eq, + Bool.ite_eq_true_distrib, decide_eq_true_eq, UInt64.lt_iff_toNat_lt, + ← UInt64.toNat_inj] at * + split_ifs <;> omega + theorem condSub_bounded (q t : Limbs8) (ht : t.Bounded) : (condSub q t).Bounded := by simp only [condSub] split - · exact subLimbs_bounded t q · exact ht + · exact subLimbs_bounded t q /-- `condSub` subtracts the modulus exactly when the input is at least the modulus. -/ theorem condSub_toNat (q t : Limbs8) (hq : q.Bounded) (ht : t.Bounded) : (condSub q t).toNat = if t.toNat < q.toNat then t.toNat else t.toNat - q.toNat := by obtain ⟨hbo, hchain⟩ := subLimbs_spec t q ht hq have hD := Limbs8.toNat_lt (subLimbs_bounded t q) - simp only [condSub, beq_iff_eq, ← UInt64.toNat_inj, toNat_zero] + simp only [condSub, limbsLt_eq_true_iff t q ht hq] split - case isTrue h => - rw [h, Nat.mul_zero, Nat.add_zero] at hchain - exact cond_of_borrow_zero hchain hD - case isFalse h => - rw [show (subBorrow t q).toNat = 1 by omega, Nat.mul_one] at hchain - exact cond_of_borrow_one hchain hD + · rfl + · omega /-- `condSub` returns a canonical representative for inputs below `2 * q`. -/ theorem condSub_lt (q t : Limbs8) (hq : q.Bounded) (ht : t.Bounded) @@ -367,10 +389,41 @@ theorem condSub_lt (q t : Limbs8) (hq : q.Bounded) (ht : t.Bounded) rw [condSub_toNat q t hq ht] split <;> omega +theorem condSubWide_bounded (q : Limbs8) (t : State9) (ht : t.Bounded) : + (condSubWide q t).Bounded := by + simp only [condSubWide] + split + · exact subLimbs_bounded t.toLimbs8 q + · exact condSub_bounded q _ ht.1 + +/-- `condSubWide` subtracts the modulus exactly when an accumulator below `2 * q` is at least +the modulus. -/ +theorem condSubWide_toNat (q : Limbs8) (t : State9) (hq : q.Bounded) (ht : t.Bounded) + (h : t.toNat < 2 * q.toNat) : + (condSubWide q t).toNat = if t.toNat < q.toNat then t.toNat else t.toNat - q.toNat := by + have hdec : t.toNat = t.toLimbs8.toNat + 2 ^ 256 * t.t8.toNat := rfl + simp only [condSubWide, bne_iff_ne, ne_eq, ← UInt64.toNat_inj, toNat_zero] + split + case isTrue hc => + obtain ⟨hbo, hsub⟩ := subLimbs_spec t.toLimbs8 q ht.1 hq + exact wide_of_head hdec hsub hbo (Limbs8.toNat_lt (subLimbs_bounded t.toLimbs8 q)) + (Limbs8.toNat_lt hq) h hc + case isFalse hc => + rw [condSub_toNat q _ hq ht.1, show t.toLimbs8.toNat = t.toNat by omega] + +/-- `condSubWide` returns a canonical representative for accumulators below `2 * q`. -/ +theorem condSubWide_lt (q : Limbs8) (t : State9) (hq : q.Bounded) (ht : t.Bounded) + (h : t.toNat < 2 * q.toNat) : (condSubWide q t).toNat < q.toNat := by + rw [condSubWide_toNat q t hq ht h] + split <;> omega + /-! ## Field operations -/ -theorem add_bounded (q a b : Limbs8) : (add q a b).Bounded := - condSub_bounded _ _ (addLimbs_bounded a b) +theorem add_bounded (q a b : Limbs8) : (add q a b).Bounded := by + simp only [add] + split + · exact subLimbs_bounded _ _ + · exact condSub_bounded _ _ (addLimbs_bounded a b) theorem sub_bounded (q a b : Limbs8) : (sub q a b).Bounded := by simp only [sub] @@ -380,27 +433,32 @@ theorem sub_bounded (q a b : Limbs8) : (sub q a b).Bounded := by theorem neg_bounded (q a : Limbs8) : (neg q a).Bounded := sub_bounded _ _ _ -/-- Modular addition is correct for canonical inputs of a modulus below `2 ^ 255`. -/ +/-- Modular addition is correct for canonical inputs of any bounded modulus. -/ theorem add_toNat (q a b : Limbs8) (hq : q.Bounded) (ha : a.Bounded) (hb : b.Bounded) - (hq2 : 2 * q.toNat < 2 ^ 256) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : (add q a b).toNat = (a.toNat + b.toNat) % q.toNat := by - obtain ⟨c, hc, hchain⟩ := addLimbs_toNat a b ha hb - obtain ⟨hsum, hlt⟩ := - carry_top_zero hchain (Limbs8.toNat_lt (addLimbs_bounded a b)) haq hbq hq2 - simp only [add] - rw [condSub_toNat q _ hq (addLimbs_bounded a b), hsum] - rw [hsum] at hlt - exact cond_eq_mod hlt + obtain ⟨hc, hchain⟩ := addLimbs_spec a b ha hb + simp only [add, bne_iff_ne, ne_eq, ← UInt64.toNat_inj, toNat_zero] + split + case isTrue h => + obtain ⟨hbo, hsub⟩ := subLimbs_spec (addLimbs a b) q (addLimbs_bounded a b) hq + rw [show (addCarry a b).toNat = 1 by omega] at hchain + exact add_of_carry hchain hsub hbo (Limbs8.toNat_lt (subLimbs_bounded (addLimbs a b) q)) + (Limbs8.toNat_lt hq) haq hbq + case isFalse h => + rw [condSub_toNat q _ hq (addLimbs_bounded a b), + show (addLimbs a b).toNat = a.toNat + b.toNat by omega] + exact cond_eq_mod (by omega) theorem add_lt (q a b : Limbs8) (hq : q.Bounded) (ha : a.Bounded) (hb : b.Bounded) - (hq2 : 2 * q.toNat < 2 ^ 256) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : (add q a b).toNat < q.toNat := by - rw [add_toNat q a b hq ha hb hq2 haq hbq] + rw [add_toNat q a b hq ha hb haq hbq] exact Nat.mod_lt _ (by omega) -/-- Modular subtraction is correct for canonical inputs of a modulus below `2 ^ 255`. -/ +/-- Modular subtraction is correct for canonical inputs of any bounded modulus. -/ theorem sub_toNat (q a b : Limbs8) (hq : q.Bounded) (ha : a.Bounded) (hb : b.Bounded) - (hq2 : 2 * q.toNat < 2 ^ 256) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : (sub q a b).toNat = (a.toNat + (q.toNat - b.toNat)) % q.toNat := by obtain ⟨hbo, hchain⟩ := subLimbs_spec a b ha hb have hD := Limbs8.toNat_lt (subLimbs_bounded a b) @@ -413,25 +471,25 @@ theorem sub_toNat (q a b : Limbs8) (hq : q.Bounded) (ha : a.Bounded) (hb : b.Bou rw [show (subBorrow a b).toNat = 1 by omega] at hchain obtain ⟨c, _, hchainE⟩ := addLimbs_toNat _ q (subLimbs_bounded a b) hq exact sub_of_borrow_one hchain hchainE hD - (Limbs8.toNat_lt (addLimbs_bounded _ _)) hbq haq hq2 + (Limbs8.toNat_lt (addLimbs_bounded _ _)) hbq haq (Limbs8.toNat_lt hq) theorem sub_lt (q a b : Limbs8) (hq : q.Bounded) (ha : a.Bounded) (hb : b.Bounded) - (hq2 : 2 * q.toNat < 2 ^ 256) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : + (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) : (sub q a b).toNat < q.toNat := by - rw [sub_toNat q a b hq ha hb hq2 haq hbq] + rw [sub_toNat q a b hq ha hb haq hbq] exact Nat.mod_lt _ (by omega) -/-- Modular negation is correct for canonical inputs of a modulus below `2 ^ 255`. -/ +/-- Modular negation is correct for canonical inputs of any bounded modulus. -/ theorem neg_toNat (q a : Limbs8) (hq : q.Bounded) (ha : a.Bounded) - (hq2 : 2 * q.toNat < 2 ^ 256) (haq : a.toNat < q.toNat) : + (haq : a.toNat < q.toNat) : (neg q a).toNat = (q.toNat - a.toNat) % q.toNat := by simp only [neg] - rw [sub_toNat q Limbs8.zero a hq Limbs8.zero_bounded ha hq2 + rw [sub_toNat q Limbs8.zero a hq Limbs8.zero_bounded ha (by rw [Limbs8.zero_toNat]; omega) haq, Limbs8.zero_toNat, Nat.zero_add] theorem neg_lt (q a : Limbs8) (hq : q.Bounded) (ha : a.Bounded) - (hq2 : 2 * q.toNat < 2 ^ 256) (haq : a.toNat < q.toNat) : (neg q a).toNat < q.toNat := - sub_lt q Limbs8.zero a hq Limbs8.zero_bounded ha hq2 + (haq : a.toNat < q.toNat) : (neg q a).toNat < q.toNat := + sub_lt q Limbs8.zero a hq Limbs8.zero_bounded ha (by rw [Limbs8.zero_toNat]; omega) haq end Native64x8 diff --git a/CompPoly/Fields/Montgomery/Native64x8Defs.lean b/CompPoly/Fields/Montgomery/Native64x8Defs.lean index 08a09fa4..76b10bbf 100644 --- a/CompPoly/Fields/Montgomery/Native64x8Defs.lean +++ b/CompPoly/Fields/Montgomery/Native64x8Defs.lean @@ -115,6 +115,17 @@ end Limbs8 ⟨adcLo a.l0 b.l0 0, adcLo a.l1 b.l1 c0, adcLo a.l2 b.l2 c1, adcLo a.l3 b.l3 c2, adcLo a.l4 b.l4 c3, adcLo a.l5 b.l5 c4, adcLo a.l6 b.l6 c5, adcLo a.l7 b.l7 c6⟩ +/-- Carry out of the top limb of `addLimbs`. -/ +@[inline] def addCarry (a b : Limbs8) : UInt64 := + let c0 := adcCo a.l0 b.l0 0 + let c1 := adcCo a.l1 b.l1 c0 + let c2 := adcCo a.l2 b.l2 c1 + let c3 := adcCo a.l3 b.l3 c2 + let c4 := adcCo a.l4 b.l4 c3 + let c5 := adcCo a.l5 b.l5 c4 + let c6 := adcCo a.l6 b.l6 c5 + adcCo a.l7 b.l7 c6 + /-- Limbwise subtract-with-borrow. -/ @[inline] def subLimbs (a b : Limbs8) : Limbs8 := let b0 := sbbBo a.l0 b.l0 0 @@ -140,13 +151,33 @@ end Limbs8 /-! ## Conditional subtraction and field operations -/ -/-- Subtract the modulus once if the value is at least the modulus. The borrow chain -decides the branch, so no comparison is needed. -/ +/-- Compare bounded limbs from the most significant end; stop at the first difference. -/ +@[inline] def limbsLt (a b : Limbs8) : Bool := + if a.l7 == b.l7 then + if a.l6 == b.l6 then + if a.l5 == b.l5 then + if a.l4 == b.l4 then + if a.l3 == b.l3 then + if a.l2 == b.l2 then + if a.l1 == b.l1 then decide (a.l0 < b.l0) + else decide (a.l1 < b.l1) + else decide (a.l2 < b.l2) + else decide (a.l3 < b.l3) + else decide (a.l4 < b.l4) + else decide (a.l5 < b.l5) + else decide (a.l6 < b.l6) + else decide (a.l7 < b.l7) + + +/-- Subtract the modulus only when the input is at least the modulus. -/ @[inline] def condSub (q t : Limbs8) : Limbs8 := - if subBorrow t q == 0 then subLimbs t q else t + if limbsLt t q then t else subLimbs t q -/-- Modular addition. -/ -@[inline] def add (q a b : Limbs8) : Limbs8 := condSub q (addLimbs a b) +/-- Modular addition; a carry out of the top limb forces the (wrapping, exact) subtraction +of the modulus, and never happens for a modulus below `2 ^ 255`. -/ +@[inline] def add (q a b : Limbs8) : Limbs8 := + let s := addLimbs a b + if addCarry a b != 0 then subLimbs s q else condSub q s /-- Modular subtraction: on a borrow, the modulus is added back. -/ @[inline] def sub (q a b : Limbs8) : Limbs8 := @@ -233,6 +264,12 @@ limb. -/ (t : State9) : State9 := mulReduce q negInv (mulAccum a bi t) +/-- `condSub` for an accumulator below `2 * q`; a set head limb forces the (wrapping, exact) +subtraction of the modulus, and never happens for a modulus below `2 ^ 255`. -/ +@[inline] def condSubWide (q : Limbs8) (t : State9) : Limbs8 := + let r := t.toLimbs8 + if t.t8 != 0 then subLimbs r q else condSub q r + /-- CIOS Montgomery multiplication: eight rounds followed by one conditional subtraction. -/ @[inline] def mul (q : Limbs8) (negInv : UInt64) (a b : Limbs8) : Limbs8 := @@ -244,7 +281,7 @@ subtraction. -/ let t := mulRound q negInv a b.l5 t let t := mulRound q negInv a b.l6 t let t := mulRound q negInv a b.l7 t - condSub q t.toLimbs8 + condSubWide q t /-- Montgomery squaring. -/ @[inline] def square (q : Limbs8) (negInv : UInt64) (a : Limbs8) : Limbs8 := diff --git a/CompPoly/Fields/Montgomery/Native64x8Field.lean b/CompPoly/Fields/Montgomery/Native64x8Field.lean index 5afe1b21..6a8d7f12 100644 --- a/CompPoly/Fields/Montgomery/Native64x8Field.lean +++ b/CompPoly/Fields/Montgomery/Native64x8Field.lean @@ -15,7 +15,7 @@ public import Mathlib.FieldTheory.Finite.Basic The bounded carrier, conversions, arithmetic, and field instances built on the raw eight-limb Montgomery operations of `Montgomery/Native64x8`. This is the multi-limb analogue of -`Montgomery/Native32Field`, for prime moduli below `2 ^ 255`. +`Montgomery/Native32Field`, for any prime modulus below `2 ^ 256`. A carrier element stores the Montgomery residue `x * 2 ^ 256 mod q` as eight 32-bit limbs; `toField` divides that residue by `2 ^ 256` again and lands in `ZMod modulus`. All arithmetic @@ -52,7 +52,6 @@ class Mont64x8Field (modulus : ℕ) where modulusLimbs_bounded : modulusLimbs.Bounded := by decide modulusLimbs_toNat : modulusLimbs.toNat = modulus := by decide two_lt_modulus : 2 < modulus := by decide - two_mul_modulus_lt : 2 * modulus < 2 ^ 256 := by decide rModModulus_bounded : rModModulus.Bounded := by decide rModModulus_toNat : rModModulus.toNat = 2 ^ 256 % modulus := by decide r2ModModulus_bounded : r2ModModulus.Bounded := by decide @@ -72,15 +71,11 @@ instance : Fact (Nat.Prime modulus) := ⟨P.prime⟩ theorem modulus_pos : 0 < modulus := Nat.zero_lt_of_lt P.two_lt_modulus theorem modulus_lt : modulus < 2 ^ 256 := by - have := P.two_mul_modulus_lt - omega + have h := Limbs8.toNat_lt P.modulusLimbs_bounded + rwa [P.modulusLimbs_toNat] at h theorem q_toNat : P.modulusLimbs.toNat = modulus := P.modulusLimbs_toNat -theorem two_mul_q_lt : 2 * P.modulusLimbs.toNat < 2 ^ 256 := by - rw [q_toNat] - exact P.two_mul_modulus_lt - theorem negInv_mul_q : P.montgomeryNegInv.toNat * P.modulusLimbs.toNat % 2 ^ 32 = 2 ^ 32 - 1 := by rw [q_toNat] exact P.montgomeryNegInv_mul_modulus_mod_two_pow_32 @@ -140,7 +135,7 @@ def one (modulus : ℕ) [P : Mont64x8Field modulus] : FastField modulus := ⟨Native64x8.add P.modulusLimbs x.val y.val, add_bounded _ _ _, by have h := add_lt _ _ _ P.modulusLimbs_bounded x.val_bounded y.val_bounded - Mont64x8Field.two_mul_q_lt x.val_lt y.val_lt + x.val_lt y.val_lt rwa [Mont64x8Field.q_toNat] at h⟩ /-- Fast modular subtraction in Montgomery form. -/ @@ -148,27 +143,24 @@ def one (modulus : ℕ) [P : Mont64x8Field modulus] : FastField modulus := ⟨Native64x8.sub P.modulusLimbs x.val y.val, sub_bounded _ _ _, by have h := sub_lt _ _ _ P.modulusLimbs_bounded x.val_bounded y.val_bounded - Mont64x8Field.two_mul_q_lt x.val_lt y.val_lt + x.val_lt y.val_lt rwa [Mont64x8Field.q_toNat] at h⟩ /-- Fast modular negation in Montgomery form. -/ @[inline] def neg (x : FastField modulus) : FastField modulus := ⟨Native64x8.neg P.modulusLimbs x.val, neg_bounded _ _, by - have h := neg_lt _ _ P.modulusLimbs_bounded x.val_bounded Mont64x8Field.two_mul_q_lt - x.val_lt + have h := neg_lt _ _ P.modulusLimbs_bounded x.val_bounded x.val_lt rwa [Mont64x8Field.q_toNat] at h⟩ /-- Fast Montgomery multiplication. -/ @[inline] def mul (x y : FastField modulus) : FastField modulus := ⟨Native64x8.mul P.modulusLimbs P.montgomeryNegInv x.val y.val, (mul_spec _ _ _ _ P.modulusLimbs_bounded x.val_bounded y.val_bounded - P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q x.val_lt - Mont64x8Field.two_mul_q_lt).1, + P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q x.val_lt).1, by have h := (mul_spec _ _ _ _ P.modulusLimbs_bounded x.val_bounded y.val_bounded - P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q x.val_lt - Mont64x8Field.two_mul_q_lt).2.1 + P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q x.val_lt).2.1 rwa [Mont64x8Field.q_toNat] at h⟩ /-- Fast squaring. -/ @@ -176,7 +168,9 @@ def one (modulus : ℕ) [P : Mont64x8Field modulus] : FastField modulus := /-- Exponentiation by repeated squaring. -/ @[specialize] def pow (x : FastField modulus) (n : ℕ) : FastField modulus := - @npowBinRec (FastField modulus) ⟨one modulus⟩ ⟨mul⟩ n x + n.binaryRec (fun (y _ : FastField modulus) ↦ y) + (fun bit _ rec y x ↦ rec (cond bit (mul y x) y) (square x)) + (one modulus) x /-- Inversion by Fermat's little theorem, `x⁻¹ = x ^ (modulus - 2)`. -/ @[inline] def inv (x : FastField modulus) : FastField modulus := pow x (modulus - 2) @@ -204,16 +198,14 @@ def one (modulus : ℕ) [P : Mont64x8Field modulus] : FastField modulus := (by rw [Limbs8.ofNat_toNat, Mont64x8Field.q_toNat, Nat.mod_eq_of_lt (h.trans Mont64x8Field.modulus_lt)] - exact h) - Mont64x8Field.two_mul_q_lt).1, + exact h)).1, by have hlt := (mul_spec _ _ _ _ P.modulusLimbs_bounded (Limbs8.ofNat_bounded n) P.r2ModModulus_bounded P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q (by rw [Limbs8.ofNat_toNat, Mont64x8Field.q_toNat, Nat.mod_eq_of_lt (h.trans Mont64x8Field.modulus_lt)] - exact h) - Mont64x8Field.two_mul_q_lt).2.1 + exact h)).2.1 rwa [Mont64x8Field.q_toNat] at hlt⟩ /-- Convert a natural number into fast Montgomery form. -/ @@ -286,8 +278,7 @@ private theorem mul_cast (x y : Limbs8) (hx : x.Bounded) (hy : y.Bounded) (x.toNat : ZMod modulus) * (y.toNat : ZMod modulus) * ((2 ^ 256 : ℕ) : ZMod modulus)⁻¹ := by have hmod := (mul_spec _ _ _ _ P.modulusLimbs_bounded hx hy P.montgomeryNegInv_lt - Mont64x8Field.negInv_mul_q (by rw [Mont64x8Field.q_toNat]; exact hxq) - Mont64x8Field.two_mul_q_lt).2.2 + Mont64x8Field.negInv_mul_q (by rw [Mont64x8Field.q_toNat]; exact hxq)).2.2 rw [Mont64x8Field.q_toNat] at hmod have hcast := (ZMod.natCast_eq_natCast_iff _ _ _).2 hmod rw [Nat.cast_mul, Nat.cast_mul] at hcast @@ -307,7 +298,7 @@ private theorem val_cast (x : FastField modulus) : theorem toNat_lt (x : FastField modulus) : toNat x < modulus := by have := (mul_spec _ _ _ _ P.modulusLimbs_bounded x.val_bounded Limbs8.one_bounded - P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q x.val_lt Mont64x8Field.two_mul_q_lt).2.1 + P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q x.val_lt).2.1 rwa [Mont64x8Field.q_toNat] at this private theorem ofCanonicalNat_val_cast {n : ℕ} (h : n < modulus) : @@ -371,7 +362,7 @@ theorem toField_one : toField (1 : FastField modulus) = 1 := by theorem toField_add (x y : FastField modulus) : toField (x + y) = toField x + toField y := by rw [toField_eq, toField_eq x, toField_eq y, add_def, add] have h := add_toNat P.modulusLimbs x.val y.val P.modulusLimbs_bounded x.val_bounded - y.val_bounded Mont64x8Field.two_mul_q_lt x.val_lt y.val_lt + y.val_bounded x.val_lt y.val_lt rw [h, Mont64x8Field.q_toNat, ZMod.natCast_mod, Nat.cast_add] ring @@ -379,7 +370,7 @@ theorem toField_add (x y : FastField modulus) : toField (x + y) = toField x + to theorem toField_sub (x y : FastField modulus) : toField (x - y) = toField x - toField y := by rw [toField_eq, toField_eq x, toField_eq y, sub_def, sub] have h := sub_toNat P.modulusLimbs x.val y.val P.modulusLimbs_bounded x.val_bounded - y.val_bounded Mont64x8Field.two_mul_q_lt x.val_lt y.val_lt + y.val_bounded x.val_lt y.val_lt rw [h, Mont64x8Field.q_toNat, ZMod.natCast_mod, Nat.cast_add, Nat.cast_sub (le_of_lt y.property.2), ZMod.natCast_self] ring @@ -387,8 +378,7 @@ theorem toField_sub (x y : FastField modulus) : toField (x - y) = toField x - to @[simp] theorem toField_neg (x : FastField modulus) : toField (-x) = -toField x := by rw [toField_eq, toField_eq x, neg_def, neg] - have h := neg_toNat P.modulusLimbs x.val P.modulusLimbs_bounded x.val_bounded - Mont64x8Field.two_mul_q_lt x.val_lt + have h := neg_toNat P.modulusLimbs x.val P.modulusLimbs_bounded x.val_bounded x.val_lt rw [h, Mont64x8Field.q_toNat, ZMod.natCast_mod, Nat.cast_sub (le_of_lt x.property.2), ZMod.natCast_self] ring @@ -417,9 +407,8 @@ theorem toField_square (x : FastField modulus) : toField (square x) = toField x theorem toField_pow (x : FastField modulus) (n : ℕ) : toField (pow x n) = toField x ^ n := by induction n with | zero => - unfold pow - rw [npowBinRec_zero, toField_one] - simp + change toField (one modulus) = toField x ^ 0 + rw [← one_def, toField_one, pow_zero] | succ n ih => rw [pow_succ_field, toField_mul, ih, _root_.pow_succ] /-- Fermat-style inversion in `ZMod modulus`. -/ diff --git a/CompPoly/Fields/Montgomery/Native64x8Inv.lean b/CompPoly/Fields/Montgomery/Native64x8Inv.lean index fd45d0b9..ec58f13c 100644 --- a/CompPoly/Fields/Montgomery/Native64x8Inv.lean +++ b/CompPoly/Fields/Montgomery/Native64x8Inv.lean @@ -112,15 +112,15 @@ theorem gcdLinearCombDiv_bounded (a b : Limbs8) (f g : Int) : and_mask_toNat_lt _, and_mask_toNat_lt _, and_mask_toNat_lt _, and_mask_toNat_lt _⟩ private theorem lincomb_stack_lt {q aS bS : Limbs8} {negInv F G : UInt64} - (hq : q.Bounded) (hq0 : 0 < q.toNat) (hq2 : 2 * q.toNat < 2 ^ 256) + (hq : q.Bounded) (hq0 : 0 < q.toNat) (hn : negInv.toNat < 2 ^ 32) (hnq : negInv.toNat * q.toNat % 2 ^ 32 = 2 ^ 32 - 1) (haS : aS.Bounded) (hbS : bS.Bounded) (haSq : aS.toNat ≤ q.toNat) (hbSq : bS.toNat ≤ q.toNat) (hFG : F.toNat + G.toNat ≤ 2 ^ 31) : - (condSub q (mulReduce q negInv - (mulAccum bS G (mulAccum aS F State9.zero))).toLimbs8).Bounded ∧ - (condSub q (mulReduce q negInv - (mulAccum bS G (mulAccum aS F State9.zero))).toLimbs8).toNat < q.toNat := by + (condSubWide q (mulReduce q negInv + (mulAccum bS G (mulAccum aS F State9.zero)))).Bounded ∧ + (condSubWide q (mulReduce q negInv + (mulAccum bS G (mulAccum aS F State9.zero)))).toNat < q.toNat := by obtain ⟨h1b, -, h1v⟩ := mulAccum_spec aS F State9.zero haS State9.zero_bounded (by omega) set s1 := mulAccum aS F State9.zero @@ -145,12 +145,11 @@ private theorem lincomb_stack_lt {q aS bS : Limbs8} {negInv F G : UInt64} have hmq : (montM s2.t0 negInv).toNat * q.toNat ≤ (2 ^ 32 - 1) * q.toNat := by have := montM_lt s2.t0 negInv exact Nat.mul_le_mul (by omega) (Nat.le_refl _) - have hdec2 : red.toNat = red.toLimbs8.toNat + 2 ^ 256 * red.t8.toNat := rfl - exact ⟨condSub_bounded q _ h3b.1, condSub_lt q _ hq h3b.1 (by omega)⟩ + exact ⟨condSubWide_bounded q _ h3b, condSubWide_lt q _ hq h3b (by omega)⟩ /-- The Montgomery lincomb stays canonical for canonical inputs. -/ theorem gcdLinearCombMontyRed_lt {q : Limbs8} {negInv : UInt64} {a b : Limbs8} {f g : Int} - (hq : q.Bounded) (hq0 : 0 < q.toNat) (hq2 : 2 * q.toNat < 2 ^ 256) + (hq : q.Bounded) (hq0 : 0 < q.toNat) (hn : negInv.toNat < 2 ^ 32) (hnq : negInv.toNat * q.toNat % 2 ^ 32 = 2 ^ 32 - 1) (ha : a.Bounded) (hb : b.Bounded) (haq : a.toNat < q.toNat) (hbq : b.toNat < q.toNat) (hfg : f.natAbs + g.natAbs ≤ 2 ^ 31) : @@ -170,7 +169,7 @@ theorem gcdLinearCombMontyRed_lt {q : Limbs8} {negInv : UInt64} {a b : Limbs8} { have hqlt := Limbs8.toNat_lt hq omega simp only [gcdLinearCombMontyRed] - refine lincomb_stack_lt hq hq0 hq2 hn hnq ?_ ?_ ?_ ?_ ?_ + refine lincomb_stack_lt hq hq0 hn hnq ?_ ?_ ?_ ?_ ?_ · split · exact subLimbs_bounded q a · exact ha @@ -190,7 +189,7 @@ set_option maxRecDepth 4000 in /-- The main loop keeps both tracks at mac width and the Montgomery pair canonical. -/ theorem gcdMainLoop_bounded {q : Limbs8} {negInv : UInt64} {rounds : ℕ} {a u b v A U B V : Limbs8} - (hq : q.Bounded) (hq0 : 0 < q.toNat) (hq2 : 2 * q.toNat < 2 ^ 256) + (hq : q.Bounded) (hq0 : 0 < q.toNat) (hn : negInv.toNat < 2 ^ 32) (hnq : negInv.toNat * q.toNat % 2 ^ 32 = 2 ^ 32 - 1) (ha : a.Bounded) (hb : b.Bounded) (hu : u.Bounded) (huq : u.toNat < q.toNat) (hv : v.Bounded) (hvq : v.toNat < q.toNat) @@ -234,9 +233,9 @@ theorem gcdMainLoop_bounded {q : Limbs8} {negInv : UInt64} {rounds : ℕ} + (if signB < 0 then -g1 else g1).natAbs ≤ 2 ^ 31 := by split <;> simpa [Int.natAbs_neg] using hrow1 have hU' := gcdLinearCombMontyRed_lt (f := if signA < 0 then -f0 else f0) - (g := if signA < 0 then -g0 else g0) hq hq0 hq2 hn hnq hu hv huq hvq hc0 + (g := if signA < 0 then -g0 else g0) hq hq0 hn hnq hu hv huq hvq hc0 have hV' := gcdLinearCombMontyRed_lt (f := if signB < 0 then -f1 else f1) - (g := if signB < 0 then -g1 else g1) hq hq0 hq2 hn hnq hu hv huq hvq hc1 + (g := if signB < 0 then -g1 else g1) hq hq0 hn hnq hu hv huq hvq hc1 exact ih hA' hB' hU'.1 hU'.2 hV'.1 hV'.2 heq -- Tuple witnesses without evaluating `p`; a bare `rfl` witness would whnf-expand it. @@ -251,7 +250,7 @@ private theorem exists_eq_tuple6 {α β γ δ ε ζ : Type} (p : α × β × γ /-- The final chunks stay canonical for canonical inputs. -/ theorem gcdFinalChunks_lt {q : Limbs8} {negInv : UInt64} {finalRounds : ℕ} {a u b v : Limbs8} (hfr : finalRounds ≤ 62) - (hq : q.Bounded) (hq0 : 0 < q.toNat) (hq2 : 2 * q.toNat < 2 ^ 256) + (hq : q.Bounded) (hq0 : 0 < q.toNat) (hn : negInv.toNat < 2 ^ 32) (hnq : negInv.toNat * q.toNat % 2 ^ 32 = 2 ^ 32 - 1) (hu : u.Bounded) (huq : u.toNat < q.toNat) (hv : v.Bounded) (hvq : v.toNat < q.toNat) : @@ -261,14 +260,14 @@ theorem gcdFinalChunks_lt {q : Limbs8} {negInv : UInt64} {finalRounds : ℕ} exists_eq_tuple6 (gcdInner ((finalRounds + 1) / 2) ((a.l1 <<< 32) ||| a.l0) ((b.l1 <<< 32) ||| b.l0) 1 0 0 1) obtain ⟨hrow0, hrow1⟩ := gcdInner_natAbs_le_31 (by omega) hI1 - have hu1 := gcdLinearCombMontyRed_lt (f := f0) (g := g0) hq hq0 hq2 hn hnq hu hv huq + have hu1 := gcdLinearCombMontyRed_lt (f := f0) (g := g0) hq hq0 hn hnq hu hv huq hvq hrow0 - have hv1 := gcdLinearCombMontyRed_lt (f := f1) (g := g1) hq hq0 hq2 hn hnq hu hv huq + have hv1 := gcdLinearCombMontyRed_lt (f := f1) (g := g1) hq hq0 hn hnq hu hv huq hvq hrow1 obtain ⟨aw2, bw2, F0, G0, F1, G1, hI2⟩ := exists_eq_tuple6 (gcdInner (finalRounds - (finalRounds + 1) / 2) aw1 bw1 1 0 0 1) obtain ⟨-, hrowF⟩ := gcdInner_natAbs_le_31 (by omega) hI2 - have hfinal := gcdLinearCombMontyRed_lt (f := F1) (g := G1) hq hq0 hq2 hn hnq + have hfinal := gcdLinearCombMontyRed_lt (f := F1) (g := G1) hq hq0 hn hnq hu1.1 hv1.1 hu1.2 hv1.2 hrowF have heq : gcdFinalChunks q negInv finalRounds a u b v = gcdLinearCombMontyRed q negInv @@ -287,7 +286,7 @@ set_option maxRecDepth 4000 in theorem gcdInvCandidate_lt {modulus : ℕ} [P : GcdData modulus] {q x : Limbs8} {negInv : UInt64} (hq : q.Bounded) (hqm : q.toNat = modulus) (hq0 : 0 < q.toNat) - (hq2 : 2 * q.toNat < 2 ^ 256) (hn : negInv.toNat < 2 ^ 32) + (hn : negInv.toNat < 2 ^ 32) (hnq : negInv.toNat * q.toNat % 2 ^ 32 = 2 ^ 32 - 1) (hx : x.Bounded) : (gcdInvCandidate modulus q negInv x).Bounded ∧ (gcdInvCandidate modulus q negInv x).toNat < q.toNat := by @@ -299,14 +298,14 @@ theorem gcdInvCandidate_lt {modulus : ℕ} [P : GcdData modulus] {q x : Limbs8} omega obtain ⟨a, u, b, v, hML⟩ := exists_eq_tuple4 (gcdMainLoop q negInv 15 x P.initU q Limbs8.zero) - obtain ⟨-, -, hU, hUq, hV, hVq⟩ := gcdMainLoop_bounded hq hq0 hq2 hn hnq hx hq + obtain ⟨-, -, hU, hUq, hV, hVq⟩ := gcdMainLoop_bounded hq hq0 hn hnq hx hq P.initU_bounded hu0 Limbs8.zero_bounded hv0 hML -- Unfold syntactically; whnf of the 15-round main loop times out. have hcand : gcdInvCandidate modulus q negInv x = gcdFinalChunks q negInv P.finalRounds a u b v := by rw [gcdInvCandidate.eq_def, hML] rw [hcand] - exact gcdFinalChunks_lt P.finalRounds_le hq hq0 hq2 hn hnq hU hUq hV hVq + exact gcdFinalChunks_lt P.finalRounds_le hq hq0 hn hnq hU hUq hV hVq variable {modulus : ℕ} [P : Mont64x8Field modulus] @@ -316,7 +315,7 @@ theorem gcdInvCandidate_lt_modulus [GcdData modulus] {x : Limbs8} (hx : x.Bounde (gcdInvCandidate modulus P.modulusLimbs P.montgomeryNegInv x).toNat < modulus := by have h := gcdInvCandidate_lt P.modulusLimbs_bounded Mont64x8Field.q_toNat (by rw [Mont64x8Field.q_toNat]; exact Mont64x8Field.modulus_pos) - Mont64x8Field.two_mul_q_lt P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q hx + P.montgomeryNegInv_lt Mont64x8Field.negInv_mul_q hx rwa [Mont64x8Field.q_toNat] at h end MacSafety @@ -402,7 +401,7 @@ theorem invGcdRaw_eq_inv [GcdData modulus] (x : FastField modulus) : = (x⁻¹).val := by have hval : invGcdRaw modulus P.modulusLimbs P.montgomeryNegInv P.rModModulus x.val = (invWithCandidate x - (gcdInvCandidate modulus P.modulusLimbs P.montgomeryNegInv x.val)).val := by + (gcdInvCandidate modulus P.modulusLimbs P.montgomeryNegInv x.val gcdInnerFast)).val := by simp only [invGcdRaw, invWithCandidate] split next h => rfl diff --git a/CompPoly/Fields/Montgomery/Native64x8InvDefs.lean b/CompPoly/Fields/Montgomery/Native64x8InvDefs.lean index 9fed126f..0d81eac1 100644 --- a/CompPoly/Fields/Montgomery/Native64x8InvDefs.lean +++ b/CompPoly/Fields/Montgomery/Native64x8InvDefs.lean @@ -1,7 +1,7 @@ /- Copyright (c) 2026 CompPoly Contributors. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Georgios Raikos +Authors: Georgios Raikos, Gregor Mitscha-Baude -/ module @@ -63,6 +63,25 @@ def gcdInner (rounds : Nat) (a b : UInt64) (f0 g0 f1 g1 : Int) : (a - b, b, f0 - f1, g0 - g1, f1, g1) gcdInner n (a >>> 1) b f0 g0 (f1 * 2) (g1 * 2) +/-- Native signed-word divsteps. The candidate is verified before it becomes a field inverse. -/ +def gcdInnerWord (rounds : Nat) (a b : UInt64) (f0 g0 f1 g1 : Int64) : + UInt64 × UInt64 × Int × Int × Int × Int := + match rounds with + | 0 => (a, b, f0.toInt, g0.toInt, f1.toInt, g1.toInt) + | n + 1 => + if a &&& 1 == 0 then + gcdInnerWord n (a >>> 1) b f0 g0 (f1 * 2) (g1 * 2) + else if a < b then + gcdInnerWord n ((b - a) >>> 1) a (f1 - f0) (g1 - g0) (f0 * 2) (g0 * 2) + else + gcdInnerWord n ((a - b) >>> 1) b (f0 - f1) (g0 - g1) (f1 * 2) (g1 * 2) + +/-- Word coefficients are widened back to integers only after the divstep loop. -/ +@[inline] def gcdInnerFast (rounds : Nat) (a b : UInt64) (f0 g0 f1 g1 : Int) : + UInt64 × UInt64 × Int × Int × Int × Int := + gcdInnerWord rounds a b (Int64.ofInt f0) (Int64.ofInt g0) + (Int64.ofInt f1) (Int64.ofInt g1) + /-! ## Approximation: 64-bit windows over 32-bit limbs -/ /-- Read limb `i` (out-of-range indices read the top limb). -/ @@ -165,13 +184,15 @@ exact once both values fit one 64-bit word. -/ let bS := if g < 0 then subLimbs q b else b let s := mulAccum bS (UInt64.ofNat g.natAbs) (mulAccum aS (UInt64.ofNat f.natAbs) State9.zero) - condSub q (mulReduce q negInv s).toLimbs8 + condSubWide q (mulReduce q negInv s) /-! ## Main loop and candidate -/ /-- The outer rounds: 31 divsteps on one-word approximations, then the transition matrix applied to both tracks. -/ -def gcdMainLoop (q : Limbs8) (negInv : UInt64) (rounds : Nat) (a u b v : Limbs8) : +@[specialize] def gcdMainLoop (q : Limbs8) (negInv : UInt64) (rounds : Nat) (a u b v : Limbs8) + (inner : Nat → UInt64 → UInt64 → Int → Int → Int → Int → + UInt64 × UInt64 × Int × Int × Int × Int := gcdInner) : Limbs8 × Limbs8 × Limbs8 × Limbs8 := match rounds with | 0 => (a, u, b, v) @@ -179,7 +200,7 @@ def gcdMainLoop (q : Limbs8) (negInv : UInt64) (rounds : Nat) (a u b v : Limbs8) let (limbIdx, bits) := gcdNumBits a b let aT := gcdApprox a limbIdx bits let bT := gcdApprox b limbIdx bits - let (_, _, f0, g0, f1, g1) := gcdInner 31 aT bT 1 0 0 1 + let (_, _, f0, g0, f1, g1) := inner 31 aT bT 1 0 0 1 let (newA, signA) := gcdLinearCombDiv a b f0 g0 let f0 := if signA < 0 then -f0 else f0 let g0 := if signA < 0 then -g0 else g0 @@ -188,26 +209,30 @@ def gcdMainLoop (q : Limbs8) (negInv : UInt64) (rounds : Nat) (a u b v : Limbs8) let g1 := if signB < 0 then -g1 else g1 let newU := gcdLinearCombMontyRed q negInv u v f0 g0 let newV := gcdLinearCombMontyRed q negInv u v f1 g1 - gcdMainLoop q negInv n newA newU newB newV + gcdMainLoop q negInv n newA newU newB newV inner /-- The final divsteps as two mac-width chunks, folding the Montgomery pair. -/ -def gcdFinalChunks (q : Limbs8) (negInv : UInt64) (finalRounds : Nat) - (a u b v : Limbs8) : Limbs8 := +@[specialize] def gcdFinalChunks (q : Limbs8) (negInv : UInt64) (finalRounds : Nat) + (a u b v : Limbs8) + (inner : Nat → UInt64 → UInt64 → Int → Int → Int → Int → + UInt64 × UInt64 × Int × Int × Int × Int := gcdInner) : Limbs8 := let aw := (a.l1 <<< 32) ||| a.l0 let bw := (b.l1 <<< 32) ||| b.l0 let c1 := (finalRounds + 1) / 2 - let (aw1, bw1, f0, g0, f1, g1) := gcdInner c1 aw bw 1 0 0 1 + let (aw1, bw1, f0, g0, f1, g1) := inner c1 aw bw 1 0 0 1 let u1 := gcdLinearCombMontyRed q negInv u v f0 g0 let v1 := gcdLinearCombMontyRed q negInv u v f1 g1 - let (_, _, _, _, fF, gF) := gcdInner (finalRounds - c1) aw1 bw1 1 0 0 1 + let (_, _, _, _, fF, gF) := inner (finalRounds - c1) aw1 bw1 1 0 0 1 gcdLinearCombMontyRed q negInv u1 v1 fF gF /-- Pornin binary-GCD candidate for the Montgomery inverse, canonical nonzero `x·R mod p` to `x⁻¹·R mod p`; proof-free, callers verify. -/ -def gcdInvCandidate (modulus : Nat) [P : GcdData modulus] (q : Limbs8) - (negInv : UInt64) (x : Limbs8) : Limbs8 := - let (a, u, b, v) := gcdMainLoop q negInv 15 x P.initU q Limbs8.zero - gcdFinalChunks q negInv P.finalRounds a u b v +@[specialize] def gcdInvCandidate (modulus : Nat) [P : GcdData modulus] (q : Limbs8) + (negInv : UInt64) (x : Limbs8) + (inner : Nat → UInt64 → UInt64 → Int → Int → Int → Int → + UInt64 × UInt64 × Int × Int × Int × Int := gcdInner) : Limbs8 := + let (a, u, b, v) := gcdMainLoop q negInv 15 x P.initU q Limbs8.zero inner + gcdFinalChunks q negInv P.finalRounds a u b v inner /-! ## Checked inversion over raw limbs -/ @@ -223,7 +248,7 @@ decreasing_by omega /-- The GCD candidate, accepted only if it verifies (`z · x = 1`); else Fermat `montPow`. -/ def invGcdRaw (modulus : Nat) [GcdData modulus] (q : Limbs8) (negInv : UInt64) (rMod : Limbs8) (x : Limbs8) : Limbs8 := - let cand := gcdInvCandidate modulus q negInv x + let cand := gcdInvCandidate modulus q negInv x gcdInnerFast if cand.Bounded ∧ subBorrow cand q = 1 ∧ mul q negInv cand x = rMod then cand else montPow q negInv rMod x (modulus - 2) diff --git a/CompPoly/Fields/Montgomery/Native64x8Mul.lean b/CompPoly/Fields/Montgomery/Native64x8Mul.lean index aa0ba6d0..a5dfe23d 100644 --- a/CompPoly/Fields/Montgomery/Native64x8Mul.lean +++ b/CompPoly/Fields/Montgomery/Native64x8Mul.lean @@ -255,19 +255,15 @@ private theorem fold8 {T1 T2 T3 T4 T5 T6 T7 T8 A Q : ℕ} /-- The final conditional subtraction of `mul`, given the folded Montgomery identity. -/ private theorem mul_finish (q : Limbs8) (t : State9) {A M : ℕ} (hq : q.Bounded) - (hb : t.Bounded) (hlt : t.toNat < 2 * q.toNat) (hq2 : 2 * q.toNat < 2 ^ 256) + (hb : t.Bounded) (hlt : t.toNat < 2 * q.toNat) (hfold : 2 ^ 256 * t.toNat = A + M * q.toNat) : - (condSub q t.toLimbs8).Bounded ∧ (condSub q t.toLimbs8).toNat < q.toNat ∧ - 2 ^ 256 * (condSub q t.toLimbs8).toNat ≡ A [MOD q.toNat] := by - have hlimbs : t.toLimbs8.toNat = t.toNat := by - have h1 := Limbs8.toNat_lt hb.1 - have h2 : t.toNat = t.toLimbs8.toNat + 2 ^ 256 * t.t8.toNat := rfl - omega + (condSubWide q t).Bounded ∧ (condSubWide q t).toNat < q.toNat ∧ + 2 ^ 256 * (condSubWide q t).toNat ≡ A [MOD q.toNat] := by have hmod : 2 ^ 256 * t.toNat ≡ A [MOD q.toNat] := by unfold Nat.ModEq rw [hfold, Nat.add_mul_mod_self_right] - refine ⟨condSub_bounded _ _ hb.1, condSub_lt q _ hq hb.1 (by omega), ?_⟩ - rw [condSub_toNat q _ hq hb.1, hlimbs] + refine ⟨condSubWide_bounded _ _ hb, condSubWide_lt q _ hq hb hlt, ?_⟩ + rw [condSubWide_toNat q _ hq hb hlt] split · exact hmod · refine (Nat.ModEq.mul_left _ ?_).trans hmod @@ -277,7 +273,7 @@ private theorem mul_finish (q : Limbs8) (t : State9) {A M : ℕ} (hq : q.Bounded theorem mul_spec (q : Limbs8) (negInv : UInt64) (a b : Limbs8) (hq : q.Bounded) (ha : a.Bounded) (hb : b.Bounded) (hn : negInv.toNat < 2 ^ 32) (hnq : negInv.toNat * q.toNat % 2 ^ 32 = 2 ^ 32 - 1) - (haq : a.toNat < q.toNat) (hq2 : 2 * q.toNat < 2 ^ 256) : + (haq : a.toNat < q.toNat) : (mul q negInv a b).Bounded ∧ (mul q negInv a b).toNat < q.toNat ∧ 2 ^ 256 * (mul q negInv a b).toNat ≡ a.toNat * b.toNat [MOD q.toNat] := by obtain ⟨hb0, hb1, hb2, hb3, hb4, hb5, hb6, hb7⟩ := hb @@ -309,7 +305,7 @@ theorem mul_spec (q : Limbs8) (negInv : UInt64) (a b : Limbs8) have hfold := fold8 r0 r1 r2 r3 r4 r5 r6 r7 rw [← mul_sum8, ← sum8_mul, ← Limbs8.toNat] at hfold simp only [mul] - exact mul_finish q _ hq B8 L8 hq2 hfold + exact mul_finish q _ hq B8 L8 hfold end Native64x8 end Montgomery diff --git a/CompPoly/Fields/Pasta/Fast.lean b/CompPoly/Fields/Pasta/Fast.lean index 72585d2a..4ed68204 100644 --- a/CompPoly/Fields/Pasta/Fast.lean +++ b/CompPoly/Fields/Pasta/Fast.lean @@ -6,37 +6,34 @@ Authors: Gregor Mitscha-Baude module public import CompPoly.Fields.Pasta.Basic -public import CompPoly.Fields.Montgomery.Native64x8Field +public import CompPoly.Fields.Montgomery.Native64x4Field /-! # Fast Pasta base fields Native-word Montgomery implementations of the Pallas and Vesta base field arithmetic. The -shared algorithms and proofs live in `CompPoly.Fields.Montgomery.Native64x8Field`; this module +shared algorithms and proofs live in `CompPoly.Fields.Montgomery.Native64x4Field`; this module supplies the two sets of constants and the concrete API. - -Both Pasta primes are `1 mod 2 ^ 32`, so both have `montgomeryNegInv = 2 ^ 32 - 1`. -/ @[expose] public section namespace Pallas.Fast -open Montgomery.Native64x8 (Mont64x8Field FastField) +open Montgomery.Native64x4 (Mont64x4Field FastField) /-! ## Parameters and carrier -/ -/-- The per-field data realizing the Pallas base field as a fast eight-limb Montgomery +/-- The per-field data realizing the Pallas base field as a fast four-limb Montgomery field. -/ -instance instMont64x8Field : Mont64x8Field Pallas.baseFieldSize where +instance instMont64x4Field : Mont64x4Field Pallas.baseFieldSize where prime := Pallas.baseFieldSize_is_prime - modulusLimbs := ⟨0x1, 0x992d30ed, 0x94cf91b, 0x224698fc, 0x0, 0x0, 0x0, 0x40000000⟩ + modulusLimbs := ⟨0x992d30ed00000001, 0x224698fc094cf91b, 0x0, 0x4000000000000000⟩ rModModulus := - ⟨0xfffffffd, 0x34786d38, 0xe41914ad, 0x992c350b, 0xffffffff, 0xffffffff, 0xffffffff, - 0x3fffffff⟩ + ⟨0x34786d38fffffffd, 0x992c350be41914ad, 0xffffffffffffffff, 0x3fffffffffffffff⟩ r2ModModulus := - ⟨0xf, 0x8c78ecb3, 0x8b0de0e7, 0xd7d30dbd, 0xc3c95d18, 0x7797a99b, 0x7b9cb714, 0x96d41af⟩ - montgomeryNegInv := 0xffffffff + ⟨0x8c78ecb30000000f, 0xd7d30dbd8b0de0e7, 0x7797a99bc3c95d18, 0x96d41af7b9cb714⟩ + montgomeryNegInv := 0x992d30ecffffffff /-- The fast native-word Pallas base field carrier, stored as a Montgomery residue. -/ abbrev Field : Type := FastField Pallas.baseFieldSize @@ -46,33 +43,32 @@ abbrev Field : Type := FastField Pallas.baseFieldSize /-- Convert from the canonical `ZMod` Pallas base field into fast Montgomery form. -/ @[inline] def ofField (x : Pallas.BaseField) : Field := - Montgomery.Native64x8.FastField.ofField x + Montgomery.Native64x4.FastField.ofField x /-! ## Canonical bridge -/ /-- Ring equivalence between the fast Montgomery representation and `Pallas.BaseField`. -/ def ringEquiv : Field ≃+* Pallas.BaseField := - Montgomery.Native64x8.FastField.ringEquiv Pallas.baseFieldSize + Montgomery.Native64x4.FastField.ringEquiv Pallas.baseFieldSize end Pallas.Fast namespace Vesta.Fast -open Montgomery.Native64x8 (Mont64x8Field FastField) +open Montgomery.Native64x4 (Mont64x4Field FastField) /-! ## Parameters and carrier -/ -/-- The per-field data realizing the Vesta base field as a fast eight-limb Montgomery +/-- The per-field data realizing the Vesta base field as a fast four-limb Montgomery field. -/ -instance instMont64x8Field : Mont64x8Field Vesta.baseFieldSize where +instance instMont64x4Field : Mont64x4Field Vesta.baseFieldSize where prime := Vesta.baseFieldSize_is_prime - modulusLimbs := ⟨0x1, 0x8c46eb21, 0x994a8dd, 0x224698fc, 0x0, 0x0, 0x0, 0x40000000⟩ + modulusLimbs := ⟨0x8c46eb2100000001, 0x224698fc0994a8dd, 0x0, 0x4000000000000000⟩ rModModulus := - ⟨0xfffffffd, 0x5b2b3e9c, 0xe3420567, 0x992c350b, 0xffffffff, 0xffffffff, 0xffffffff, - 0x3fffffff⟩ + ⟨0x5b2b3e9cfffffffd, 0x992c350be3420567, 0xffffffffffffffff, 0x3fffffffffffffff⟩ r2ModModulus := - ⟨0xf, 0xfc9678ff, 0x891a16e3, 0x67bb433d, 0x4ccf590, 0x7fae2310, 0x7ccfdaa9, 0x96d41af⟩ - montgomeryNegInv := 0xffffffff + ⟨0xfc9678ff0000000f, 0x67bb433d891a16e3, 0x7fae231004ccf590, 0x96d41af7ccfdaa9⟩ + montgomeryNegInv := 0x8c46eb20ffffffff /-- The fast native-word Vesta base field carrier, stored as a Montgomery residue. -/ abbrev Field : Type := FastField Vesta.baseFieldSize @@ -82,12 +78,12 @@ abbrev Field : Type := FastField Vesta.baseFieldSize /-- Convert from the canonical `ZMod` Vesta base field into fast Montgomery form. -/ @[inline] def ofField (x : Vesta.BaseField) : Field := - Montgomery.Native64x8.FastField.ofField x + Montgomery.Native64x4.FastField.ofField x /-! ## Canonical bridge -/ /-- Ring equivalence between the fast Montgomery representation and `Vesta.BaseField`. -/ def ringEquiv : Field ≃+* Vesta.BaseField := - Montgomery.Native64x8.FastField.ringEquiv Vesta.baseFieldSize + Montgomery.Native64x4.FastField.ringEquiv Vesta.baseFieldSize end Vesta.Fast diff --git a/CompPoly/Fields/README.md b/CompPoly/Fields/README.md index 195b348c..745200d6 100644 --- a/CompPoly/Fields/README.md +++ b/CompPoly/Fields/README.md @@ -11,15 +11,15 @@ This directory contains formally verified field infrastructure used in zero-know | **BabyBear.lean** | Facade for BabyBear modules, re-exporting the canonical field and fast native-word implementation. | | **BabyBear/Basic.lean** | \(2^{31} - 2^{27} + 1\) — Risc Zero. | | **BabyBear/Fast.lean** | BabyBear-namespaced API over the shared fast-field implementation (`Montgomery/Native32Field.lean`): thin wrappers forwarding the native `UInt32` Montgomery-residue operations and their `BabyBear.Field` equivalence (`@[simp]`) lemmas. | -| **BLS12_377.lean** | Facade for the BLS12-377 modules, re-exporting the canonical field and the fast eight-limb implementation. | +| **BLS12_377.lean** | Facade for the BLS12-377 modules, re-exporting the canonical field and the fast four-limb implementation. | | **BLS12_377/Basic.lean** | Scalar field of BLS12-377 (253-bit, 2-adicity 47) — Zexe. | -| **BLS12_377/Fast.lean** | Eight-limb Montgomery instantiation of the BLS12-377 scalar field (`Mont64x8Field` and `GcdData` constants, `ScalarField`, `ringEquiv`). | -| **BLS12_381.lean** | Facade for the BLS12-381 modules, re-exporting the canonical field and the fast eight-limb implementation. | +| **BLS12_377/Fast.lean** | Four-limb Montgomery instantiation of the BLS12-377 scalar field (`Mont64x4Field` and `GcdData` constants, `ScalarField`, `ringEquiv`). | +| **BLS12_381.lean** | Facade for the BLS12-381 modules, re-exporting the canonical field and the fast four-limb implementation. | | **BLS12_381/Basic.lean** | Scalar field of BLS12-381 (255-bit, 2-adicity 32). | -| **BLS12_381/Fast.lean** | Eight-limb Montgomery instantiation of the BLS12-381 scalar field (`Mont64x8Field` and `GcdData` constants, `ScalarField`, `ringEquiv`). | -| **BN254.lean** | Facade for the BN254 modules, re-exporting the canonical field and the fast eight-limb implementation. | +| **BLS12_381/Fast.lean** | Four-limb Montgomery instantiation of the BLS12-381 scalar field (`Mont64x4Field` and `GcdData` constants, `ScalarField`, `ringEquiv`). | +| **BN254.lean** | Facade for the BN254 modules, re-exporting the canonical field and the fast four-limb implementation. | | **BN254/Basic.lean** | Scalar field of BN254 (254-bit, 2-adicity 28). | -| **BN254/Fast.lean** | Eight-limb Montgomery instantiation of the BN254 scalar field (`Mont64x8Field` and `GcdData` constants, `ScalarField`, `ringEquiv`). | +| **BN254/Fast.lean** | Four-limb Montgomery instantiation of the BN254 scalar field (`Mont64x4Field` and `GcdData` constants, `ScalarField`, `ringEquiv`). | | **Extension.lean** | Facade for the field-extension stack (arbitrary monic modulus; binomial as a special case). | | **Extension/Binomial.lean** | Irreducibility of `X^d - W` over a finite field: Rabin's test collapsed to two base-field exponentiations (`irreducible_X_pow_four_sub_C_iff`). | | **Extension/Arithmetic.lean** | `ExtensionParams` (degree, lower coefficients of the monic modulus, base cardinality), `BinomialParams` and its `toExtensionParams`, and the presentation-indexed carrier `Ext P` with length-`d` coefficient vectors and ring operations — including the `red` reduction table and the `@[csimp]`-registered `mulTbl`. | @@ -27,8 +27,13 @@ This directory contains formally verified field infrastructure used in zero-know | **Extension/Cardinality.lean** | Proof-only finiteness, optional enumeration, and certified cardinality of the coefficient carrier. | | **Extension/Bridge.lean** | `toQuot : Ext P → AdjoinRoot P.poly`, its ring-hom and injectivity proofs, and `CommRing (Ext P)`. | | **Extension/Field.lean** | Bijectivity (`ringEquivQuot`), cardinality, Fermat inversion, and `Field (Ext P)`. | +| **Extension/Map.lean** | Base change `ExtensionParams.map` along a ring homomorphism, and transfer of irreducibility along a ring isomorphism (`irreducible_map_ringEquiv`). | | **BabyBear/Ext4.lean** | \(\mathrm{BabyBear}[X]/(X^4 - 11)\). | +| **BabyBear/FastExt4.lean** | The `Binomial4` instance `X^4 - 11` over the fast BabyBear carrier and `BabyBear.Fast.Ext4`; irreducibility transferred from `BabyBear/Ext4.lean`. | +| **BabyBear/InterpolateCoset.lean** | The compiled `PackedInterpolation` instance for BabyBear, in `CompPolyInterpolateNative`. | | **KoalaBear/Ext4.lean** | \(\mathrm{KoalaBear}[X]/(X^4 - 3)\). | +| **KoalaBear/FastExt4.lean** | The `Binomial4` instance `X^4 - 3` over the fast KoalaBear carrier, with `mulW x = x + x + x`, and `KoalaBear.Fast.Ext4`; irreducibility transferred from `KoalaBear/Ext4.lean`. | +| **KoalaBear/InterpolateCoset.lean** | The compiled `PackedInterpolation` instance for KoalaBear, in `CompPolyInterpolateNative`. | | **KoalaBear/Ext5.lean** | \(\mathrm{KoalaBear}[X]/(X^5 + X^2 - 1)\) — first non-binomial modulus; no degree-5 binomial exists since \(\gcd(5, p-1) = 1\). | | **KoalaBear/Ext5/QuinticCertData.lean** | Generated Rabin certificate data for the quintic. Regenerate with `scripts/gen_rabin_certificate.py`; do not hand-edit. | | **KoalaBear/Ext5/QuinticIrreducible.lean** | Kernel-checked irreducibility of \(X^5 + X^2 - 1\) via Rabin's test at prime degree. | @@ -38,8 +43,9 @@ This directory contains formally verified field infrastructure used in zero-know | **KoalaBear/Ext6/GaloisField.lean** | Opt-in bridge identifying `Ext6` with Mathlib's abstract `GaloisField KoalaBear.fieldSize 6` (ArkLib's `KoalaSextic` parameter point). Separate module so the GaloisField import is not forced on `Ext6` users. | | **Goldilocks.lean** | Facade for the \(2^{64} - 2^{32} + 1\) Plonky2/3 field, re-exporting the canonical `ZMod` model and fast native-word implementation. | | **Goldilocks/Basic.lean** | Canonical \(2^{64} - 2^{32} + 1\) field model and primality proof. | -| **Goldilocks/Fast.lean** | Verified `UInt64` implementation of Goldilocks arithmetic. A single-word 64-bit prime fits neither Montgomery carrier — `Mont32Field` requires modulus < 2^31 and `Mont64x8Field` is an eight-limb layout — so this is a bespoke implementation resting on \(2^{64} \equiv 2^{32} - 1\). | -| **Goldilocks/FastDefs.lean** | Zero-import runtime word kernels behind `Goldilocks/Fast.lean`, kept importless so `precompileModules` lanes can compile them. | +| **Goldilocks/Fast.lean** | Verified `UInt64` Goldilocks field over the `FastDefs` kernels, bespoke since a one-word 64-bit prime fits neither Montgomery carrier. | +| **Goldilocks/FastDefs.lean** | Zero-import runtime word kernels for Goldilocks, so `precompileModules` lanes can compile them. | +| **Goldilocks/FastReduction.lean** | Bounds and canonical-field cast lemmas for the `FastDefs` kernels. | | **Hachi.lean** | \(2^{32} - 99\) — 32-bit prime field. **Name provisional.** Included as a 32-bit example rather than a production target: it exercises a base field with no Montgomery fast path (`Mont32Field` requires modulus < 2^31) and two-adicity 2, so no radix-2 NTT domain exists for it. | | **Hachi/Ext4.lean** | \(\mathrm{Hachi}[X]/(X^4 - 2)\). | | **KoalaBear.lean** | Facade for KoalaBear modules, re-exporting the canonical field and fast native-word implementation. | @@ -52,17 +58,31 @@ This directory contains formally verified field infrastructure used in zero-know | **Montgomery/Basic.lean** | Radix-generic Montgomery reduction, field-agnostic number theory shared by the fast prime fields. | | **Montgomery/Native32.lean** | Raw `UInt32`/`UInt64` Montgomery reduction over explicit word constants, including bounds and correctness. | | **Montgomery/Native32Field.lean** | Per-field parameters, the shared `FastField` carrier, arithmetic, instances, and canonical-field bridge. | -| **Montgomery/Native64x8Defs.lean** | Zero-import runtime definitions of the eight-limb Montgomery arithmetic, for `precompileModules` consumers. | +| **Montgomery/Native32Ext4.lean** | The fast quartic binomial extension `Ext4 p` over any `Mont32Field p` with a `Binomial4 p` instance: one-reduction multiplication, norm-based inversion, `specEquiv` with the spec field `Ext4.Spec p` over `FastField p`, and the byte codec. | +| **Montgomery/InterpolateCoset.lean** | Plonky3's `interpolate_coset` over `FastField p` and `Ext4 p`: barycentric evaluation from coset values at an extension point, proved equal to `aeval`. | +| **Montgomery/InterpolateCosetPacked.lean** | The packed-word implementation: base-field weights in blocks, lazy column sums, parallel row ranges; dispatched through the class `PackedInterpolation p`. | +| **Montgomery/InterpolateCosetPackedCorrectness.lean** | `interpolateCosetPacked_eq`: the packed implementation equals the reference off the coset. | +| **Montgomery/Native64x4Defs.lean** | Zero-import runtime definitions of the four-limb (64-bit-radix) Montgomery arithmetic, for `precompileModules` consumers. | +| **Montgomery/Native64x4.lean** | Word-level specifications and add/sub/negate correctness for the four-limb arithmetic. | +| **Montgomery/Native64x4Mul.lean** | Correctness of the four-limb CIOS Montgomery multiplication. | +| **Montgomery/Native64x4Field.lean** | The `Mont64x4Field` class, `FastField` carrier, arithmetic, instances, and canonical-field bridge for any prime modulus below `2^256`. | +| **Montgomery/Native64x4Bytes.lean** | Byte codec of the four-limb carrier, agreeing with the `ZMod` codec. | +| **Montgomery/Native64x4InvDefs.lean** | Mathlib-free binary-GCD inversion runtime ([eprint 2020/972](https://eprint.iacr.org/2020/972)) over four limbs: the `GcdData` schedule, the candidate, and the checked `invGcdRaw`. | +| **Montgomery/Native64x4Inv.lean** | Correctness of the checked four-limb inversion (`invGcdRaw`, wrapper `FastField.invGcd`) and the candidate's mac-width safety. | +| **Montgomery/Native64x8Defs.lean** | Zero-import runtime definitions of the eight-limb (32-bit-radix) Montgomery arithmetic; the carrier of `ScalarFft.lean`. | | **Montgomery/Native64x8.lean** | Word-level specifications and add/sub/negate correctness for the eight-limb arithmetic. | | **Montgomery/Native64x8Mul.lean** | Correctness of the eight-limb CIOS Montgomery multiplication. | -| **Montgomery/Native64x8Field.lean** | The `Mont64x8Field` class, `FastField` carrier, arithmetic, instances, and canonical-field bridge for moduli below `2^255`. | -| **Montgomery/Native64x8InvDefs.lean** | Mathlib-free binary-GCD inversion runtime ([eprint 2020/972](https://eprint.iacr.org/2020/972)): the `GcdData` schedule, the candidate, and the checked `invGcdRaw`. | -| **Montgomery/Native64x8Inv.lean** | Correctness of the checked inversion (`invGcdRaw`, wrapper `FastField.invGcd`), the divstep coefficient bounds, and the candidate's mac-width safety. | +| **Montgomery/Native64x8Field.lean** | The `Mont64x8Field` class, `FastField` carrier, arithmetic, instances, and canonical-field bridge over eight limbs; no concrete field instantiates it since the four-limb carrier replaced it. | +| **Montgomery/Native64x8Bytes.lean** | Byte codec of the eight-limb carrier. | +| **Montgomery/Native64x8InvDefs.lean** | Mathlib-free binary-GCD inversion runtime over eight limbs, whose word-sized divstep loop `gcdInner` is shared with the four-limb inversion. | +| **Montgomery/Native64x8Inv.lean** | Correctness of the checked eight-limb inversion and the divstep coefficient bounds, which the four-limb proofs reuse. | | **Montgomery/ScalarFft.lean** | Zero-import in-place radix-2 DIT FFT over eight-limb Montgomery residues and precomputed twiddles. | | **Pasta.lean** | Facade for the Pasta modules, re-exporting the canonical Pallas/Vesta base fields and their fast native-word implementations. | | **Pasta/Basic.lean** | The two 255-bit Pasta base primes (Pallas base = Vesta scalar and vice versa), with Pratt primality certificates. | -| **Pasta/Fast.lean** | Pallas- and Vesta-namespaced API over the shared eight-limb fast-field implementation, with per-field constants and canonical-field bridges. | -| **Secp256k1.lean** | Base and scalar fields for the Secp256k1 curve (used in Bitcoin/Ethereum). | +| **Pasta/Fast.lean** | Pallas- and Vesta-namespaced API over the shared four-limb fast-field implementation, with per-field constants and canonical-field bridges. | +| **Secp256k1.lean** | Facade for the Secp256k1 modules, re-exporting the canonical base and scalar fields and their fast four-limb implementations. | +| **Secp256k1/Basic.lean** | Base and scalar fields for the Secp256k1 curve (used in Bitcoin/Ethereum), with Pratt primality certificates. | +| **Secp256k1/Fast.lean** | Four-limb Montgomery instantiation of both Secp256k1 fields (`Mont64x4Field` and `GcdData` constants, `ScalarField`, `BaseField`, ring equivalences). | ## Binary-field modules diff --git a/CompPoly/Fields/Secp256k1.lean b/CompPoly/Fields/Secp256k1.lean index c9443815..fe51bff0 100644 --- a/CompPoly/Fields/Secp256k1.lean +++ b/CompPoly/Fields/Secp256k1.lean @@ -1,155 +1,17 @@ /- Copyright (c) 2024 ArkLib Contributors. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Quang Dao +Authors: Quang Dao, Georgios Raikos -/ module -public import CompPoly.Fields.PrattCertificate -public import Mathlib.Algebra.Field.ZMod +public import CompPoly.Fields.Secp256k1.Basic +public import CompPoly.Fields.Secp256k1.Fast /-! - # The Secp256k1 base and scalar prime fields - - We define the two primes underlying the Secp256k1 elliptic curve. - - The base prime is the prime on which the elliptic curve is defined. - - The scalar prime is the prime that divides the order of the curve. - - ## References - - `p` is the base prime, and `n` is the scalar prime in [Section 2.4.1](http://www.secg.org/sec2-v2.pdf). - --/ - -@[expose] public section - -namespace Secp256k1 - --- Base field - -@[reducible] -def BASE_FIELD_CARD : Nat := 0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f - -/- Alternative representation -/ -example : - BASE_FIELD_CARD = 2 ^ 256 - 2 ^ 32 - 2 ^ 9 - 2 ^ 8 - 2 ^ 7 - 2 ^ 6 - 2 ^ 4 - 1 := by - unfold BASE_FIELD_CARD; norm_num - -abbrev BaseField := ZMod BASE_FIELD_CARD - - -/- Pratt certificate for BASE_FIELD_CARD - -(3 (2 3 7 13441 205115282021455665897114700593932402728804164701536103180137503955397371) - (1 1 1 1 1) - (() () () () - (10 (2 3 5 29 31 7723 132896956044521568488119 255515944373312847190720520512484175977) - (1 1 1 2 1 1 1 1) - (() () () () () () - (6 (2 3 22149492674086928081353) - (1 1 1) - (() () - (5 (2 3 5323 173378833005251801) - (3 1 1 1) - (() () () - (6 (2 5 2621 24809 13331831) - (3 2 1 1 1) - (() () () () - (13 (2 5 971 1373) - (1 1 1 1) - (() () () ())) - )) - )) - )) - (3 (2 7 11 1627 2657 4423 41201 96557 7240687 107590001) - (3 2 1 1 1 1 1 1 1 1) - (() () () () () () () () () - (3 (2 5 7 29 53) - (4 4 1 1 1) - (() () () () ())) - )) - )) - ))) +# Secp256k1 Fields +Facade module for the secp256k1 base and scalar fields. It re-exports the canonical `ZMod` +models from `CompPoly.Fields.Secp256k1.Basic` and the native-word Montgomery implementations +from `CompPoly.Fields.Secp256k1.Fast`. -/ - -theorem BaseField_is_prime : Nat.Prime BASE_FIELD_CARD := by - unfold BASE_FIELD_CARD - refine PrattCertificate'.out (p := BASE_FIELD_CARD) ⟨3, (by reduce_mod_char), ?_⟩ - refine .split [2, 3, 7, 13441, - 205115282021455665897114700593932402728804164701536103180137503955397371] - (fun r hr => ?_) (by norm_num) - simp at hr - rcases hr with hr | hr | hr | hr | hr - all_goals rw [hr] - · exact .prime 2 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 3 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 7 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 13441 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · refine .prime 205115282021455665897114700593932402728804164701536103180137503955397371 1 _ ?_ - (by reduce_mod_char; decide) (by norm_num) - · refine PrattCertificate'.out ⟨10, (by reduce_mod_char), ?_⟩ - refine .split [2, 3, 5, 29 ^ 2, 31, 7723, 132896956044521568488119, - 255515944373312847190720520512484175977] (fun r hr => ?_) (by norm_num) - simp at hr - rcases hr with hr | hr | hr | hr | hr | hr | hr | hr - all_goals rw [hr] - · exact .prime 2 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 3 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 5 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 29 2 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 31 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 7723 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 132896956044521568488119 1 _ (by pratt) - (by reduce_mod_char; decide) (by norm_num) - · exact .prime 255515944373312847190720520512484175977 1 _ (by pratt) - (by reduce_mod_char; decide) (by norm_num) - -instance : Fact (Nat.Prime BASE_FIELD_CARD) := ⟨BaseField_is_prime⟩ - -instance : Field BaseField := ZMod.instField BASE_FIELD_CARD - - - --- Scalar field - -@[reducible] -def SCALAR_FIELD_CARD : Nat := 0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141 - -abbrev ScalarField := ZMod SCALAR_FIELD_CARD - -theorem ScalarField_is_prime : Nat.Prime SCALAR_FIELD_CARD := by - unfold SCALAR_FIELD_CARD - refine PrattCertificate'.out (p := SCALAR_FIELD_CARD) ⟨7, (by reduce_mod_char), ?_⟩ - refine .split [2 ^ 6, 3, 149, 631, 107361793816595537, 174723607534414371449, - 341948486974166000522343609283189] (fun r hr => ?_) (by norm_num) - simp at hr - rcases hr with hr | hr | hr | hr | hr | hr | hr - all_goals rw [hr] - · exact .prime 2 6 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 3 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 149 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 631 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 107361793816595537 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 174723607534414371449 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · refine .prime 341948486974166000522343609283189 1 _ ?_ (by reduce_mod_char; decide) - (by norm_num) - · refine PrattCertificate'.out ⟨2, (by reduce_mod_char), ?_⟩ - refine .split [2 ^ 2, 3 ^ 3, 109, 29047611873442575647497758179] (fun r hr => ?_) - (by norm_num) - simp at hr - rcases hr with hr | hr | hr | hr - all_goals rw [hr] - · exact .prime 2 2 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 3 3 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 109 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) - · exact .prime 29047611873442575647497758179 1 _ (by pratt) (by reduce_mod_char; decide) - (by norm_num) - -instance : Fact (Nat.Prime SCALAR_FIELD_CARD) := ⟨ScalarField_is_prime⟩ - -instance : Field ScalarField := ZMod.instField SCALAR_FIELD_CARD - -end Secp256k1 diff --git a/CompPoly/Fields/Secp256k1/Basic.lean b/CompPoly/Fields/Secp256k1/Basic.lean new file mode 100644 index 00000000..b636ac6c --- /dev/null +++ b/CompPoly/Fields/Secp256k1/Basic.lean @@ -0,0 +1,155 @@ +/- +Copyright (c) 2024 ArkLib Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Quang Dao +-/ +module + +public import CompPoly.Fields.PrattCertificate +public import Mathlib.Algebra.Field.ZMod + +/-! + # The Secp256k1 base and scalar prime fields + + We define the two primes underlying the Secp256k1 elliptic curve. + + The base prime is the prime on which the elliptic curve is defined. + + The scalar prime is the prime that divides the order of the curve. + + ## References + + `p` is the base prime, and `n` is the scalar prime in [Section 2.4.1](http://www.secg.org/sec2-v2.pdf). + +-/ + +@[expose] public section + +namespace Secp256k1 + +-- Base field + +@[reducible] +def baseFieldSize : Nat := 0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f + +/- Alternative representation -/ +example : + baseFieldSize = 2 ^ 256 - 2 ^ 32 - 2 ^ 9 - 2 ^ 8 - 2 ^ 7 - 2 ^ 6 - 2 ^ 4 - 1 := by + unfold baseFieldSize; norm_num + +abbrev BaseField := ZMod baseFieldSize + + +/- Pratt certificate for baseFieldSize + +(3 (2 3 7 13441 205115282021455665897114700593932402728804164701536103180137503955397371) + (1 1 1 1 1) + (() () () () + (10 (2 3 5 29 31 7723 132896956044521568488119 255515944373312847190720520512484175977) + (1 1 1 2 1 1 1 1) + (() () () () () () + (6 (2 3 22149492674086928081353) + (1 1 1) + (() () + (5 (2 3 5323 173378833005251801) + (3 1 1 1) + (() () () + (6 (2 5 2621 24809 13331831) + (3 2 1 1 1) + (() () () () + (13 (2 5 971 1373) + (1 1 1 1) + (() () () ())) + )) + )) + )) + (3 (2 7 11 1627 2657 4423 41201 96557 7240687 107590001) + (3 2 1 1 1 1 1 1 1 1) + (() () () () () () () () () + (3 (2 5 7 29 53) + (4 4 1 1 1) + (() () () () ())) + )) + )) + ))) + +-/ + +theorem BaseField_is_prime : Nat.Prime baseFieldSize := by + unfold baseFieldSize + refine PrattCertificate'.out (p := baseFieldSize) ⟨3, (by reduce_mod_char), ?_⟩ + refine .split [2, 3, 7, 13441, + 205115282021455665897114700593932402728804164701536103180137503955397371] + (fun r hr => ?_) (by norm_num) + simp at hr + rcases hr with hr | hr | hr | hr | hr + all_goals rw [hr] + · exact .prime 2 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 3 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 7 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 13441 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · refine .prime 205115282021455665897114700593932402728804164701536103180137503955397371 1 _ ?_ + (by reduce_mod_char; decide) (by norm_num) + · refine PrattCertificate'.out ⟨10, (by reduce_mod_char), ?_⟩ + refine .split [2, 3, 5, 29 ^ 2, 31, 7723, 132896956044521568488119, + 255515944373312847190720520512484175977] (fun r hr => ?_) (by norm_num) + simp at hr + rcases hr with hr | hr | hr | hr | hr | hr | hr | hr + all_goals rw [hr] + · exact .prime 2 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 3 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 5 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 29 2 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 31 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 7723 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 132896956044521568488119 1 _ (by pratt) + (by reduce_mod_char; decide) (by norm_num) + · exact .prime 255515944373312847190720520512484175977 1 _ (by pratt) + (by reduce_mod_char; decide) (by norm_num) + +instance : Fact (Nat.Prime baseFieldSize) := ⟨BaseField_is_prime⟩ + +instance : Field BaseField := ZMod.instField baseFieldSize + + + +-- Scalar field + +@[reducible] +def scalarFieldSize : Nat := 0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141 + +abbrev ScalarField := ZMod scalarFieldSize + +theorem ScalarField_is_prime : Nat.Prime scalarFieldSize := by + unfold scalarFieldSize + refine PrattCertificate'.out (p := scalarFieldSize) ⟨7, (by reduce_mod_char), ?_⟩ + refine .split [2 ^ 6, 3, 149, 631, 107361793816595537, 174723607534414371449, + 341948486974166000522343609283189] (fun r hr => ?_) (by norm_num) + simp at hr + rcases hr with hr | hr | hr | hr | hr | hr | hr + all_goals rw [hr] + · exact .prime 2 6 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 3 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 149 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 631 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 107361793816595537 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 174723607534414371449 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · refine .prime 341948486974166000522343609283189 1 _ ?_ (by reduce_mod_char; decide) + (by norm_num) + · refine PrattCertificate'.out ⟨2, (by reduce_mod_char), ?_⟩ + refine .split [2 ^ 2, 3 ^ 3, 109, 29047611873442575647497758179] (fun r hr => ?_) + (by norm_num) + simp at hr + rcases hr with hr | hr | hr | hr + all_goals rw [hr] + · exact .prime 2 2 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 3 3 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 109 1 _ (by pratt) (by reduce_mod_char; decide) (by norm_num) + · exact .prime 29047611873442575647497758179 1 _ (by pratt) (by reduce_mod_char; decide) + (by norm_num) + +instance : Fact (Nat.Prime scalarFieldSize) := ⟨ScalarField_is_prime⟩ + +instance : Field ScalarField := ZMod.instField scalarFieldSize + +end Secp256k1 diff --git a/CompPoly/Fields/Secp256k1/Fast.lean b/CompPoly/Fields/Secp256k1/Fast.lean new file mode 100644 index 00000000..ced59eb2 --- /dev/null +++ b/CompPoly/Fields/Secp256k1/Fast.lean @@ -0,0 +1,87 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos +-/ +module + +public import CompPoly.Fields.Secp256k1.Basic +public import CompPoly.Fields.Montgomery.Native64x4Inv + +/-! +# Fast secp256k1 Fields + +Native four-limb Montgomery implementations of the secp256k1 scalar and base fields +(`CompPoly.Fields.Montgomery.Native64x4Field`). This module supplies the constants. +-/ + +@[expose] public section + +namespace Secp256k1.Fast + +open Montgomery.Native64x4 (Mont64x4Field FastField GcdData) + +set_option exponentiation.threshold 1100 + +/-! ## Scalar field -/ + +/-- Divstep schedule of the binary-GCD inverse candidate over the scalar field. -/ +instance instScalarGcdData : GcdData Secp256k1.scalarFieldSize where + finalRounds := 45 + initU := ⟨0x5fbba3af31b96a9b, 0x4f4f0e1fb393fef9, 0x362b8d1280b4ca5e, 0x215bb648c78ffa6d⟩ + +/-- The per-field data realizing the secp256k1 scalar field as a fast four-limb Montgomery +field. -/ +instance instScalarMont64x4Field : Mont64x4Field Secp256k1.scalarFieldSize where + prime := Secp256k1.ScalarField_is_prime + modulusLimbs := + ⟨0xbfd25e8cd0364141, 0xbaaedce6af48a03b, 0xfffffffffffffffe, 0xffffffffffffffff⟩ + rModModulus := ⟨0x402da1732fc9bebf, 0x4551231950b75fc4, 0x1, 0x0⟩ + r2ModModulus := + ⟨0x896cf21467d7d140, 0x741496c20e7cf878, 0xe697f5e45bcd07c6, 0x9d671cd581c69bc5⟩ + montgomeryNegInv := 0x4b0dff665588b13f + +/-- The four-limb secp256k1 scalar field carrier, stored as a Montgomery residue. -/ +abbrev ScalarField : Type := FastField Secp256k1.scalarFieldSize + +/-- Convert from the canonical `Secp256k1.ScalarField` into fast Montgomery form. -/ +@[inline] +def ofScalarField (x : Secp256k1.ScalarField) : ScalarField := + FastField.ofField x + +/-- Ring equivalence between the four-limb representation and the canonical +`Secp256k1.ScalarField`. -/ +def scalarRingEquiv : ScalarField ≃+* Secp256k1.ScalarField := + FastField.ringEquiv Secp256k1.scalarFieldSize + +/-! ## Base field -/ + +/-- Divstep schedule of the binary-GCD inverse candidate over the base field. -/ +instance instBaseGcdData : GcdData Secp256k1.baseFieldSize where + finalRounds := 45 + initU := ⟨0x0, 0x795f6a608d461504, 0x3d10015d8f1b, 0x4⟩ + +/-- The per-field data realizing the secp256k1 base field as a fast four-limb Montgomery +field. -/ +instance instBaseMont64x4Field : Mont64x4Field Secp256k1.baseFieldSize where + prime := Secp256k1.BaseField_is_prime + modulusLimbs := + ⟨0xfffffffefffffc2f, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff⟩ + rModModulus := ⟨0x1000003d1, 0x0, 0x0, 0x0⟩ + r2ModModulus := ⟨0x7a2000e90a1, 0x1, 0x0, 0x0⟩ + montgomeryNegInv := 0xd838091dd2253531 + +/-- The four-limb secp256k1 base field carrier, stored as a Montgomery residue. -/ +abbrev BaseField : Type := FastField Secp256k1.baseFieldSize + +/-- Convert from the canonical `Secp256k1.BaseField` into fast Montgomery form. -/ +@[inline] +def ofBaseField (x : Secp256k1.BaseField) : BaseField := + FastField.ofField x + +/-- Ring equivalence between the four-limb representation and the canonical +`Secp256k1.BaseField`. -/ +def baseRingEquiv : BaseField ≃+* Secp256k1.BaseField := + FastField.ringEquiv Secp256k1.baseFieldSize + +end Secp256k1.Fast diff --git a/CompPoly/Univariate/NTT/CosetBarycentric.lean b/CompPoly/Univariate/NTT/CosetBarycentric.lean new file mode 100644 index 00000000..a0985fd9 --- /dev/null +++ b/CompPoly/Univariate/NTT/CosetBarycentric.lean @@ -0,0 +1,115 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTT.Barycentric + +/-! +# Barycentric evaluation from a coset into an extension field + +A polynomial `p` over `F` of degree below `n` is determined by its values on the coset `s · H` of +the order-`n` subgroup `H = ⟨ω⟩`. At any point `z` of an extension field `E` outside the coset, + +`p(z) = (zⁿ - sⁿ) / (n sⁿ) · ∑ᵢ (s ωⁱ) / (z - s ωⁱ) · p(s ωⁱ)`. + +This is the formula of Plonky3's `interpolate_coset`. The nodal polynomial of the coset is +`Xⁿ - sⁿ`, so the barycentric weight of the node `x = s ωⁱ` is `x / (n sⁿ)`. + +## Main results + +* `NTT.Domain.cosetNodal_eq`: the nodal polynomial of the coset, over `E`, is `Xⁿ - sⁿ`. +* `NTT.Domain.cosetNodalWeight_eq`: the barycentric weight of `s ωⁱ` is `s ωⁱ / (n sⁿ)`. +* `NTT.Domain.aeval_eq_cosetBarycentric`: the evaluation formula. +-/ + +@[expose] public section + +open Polynomial + +namespace CompPoly +namespace CPolynomial +namespace NTT +namespace Domain + +variable {F E : Type*} [Field F] [Field E] [Algebra F E] + +/-- The coset node `s ωⁱ`, mapped into `E`. -/ +def cosetNode (D : Domain F) (s : F) (i : D.Idx) : E := algebraMap F E (s * D.node i) + +theorem cosetNode_pow_n (D : Domain F) (s : F) (i : D.Idx) : + (D.cosetNode s i : E) ^ D.n = algebraMap F E s ^ D.n := by + rw [cosetNode, ← map_pow, mul_pow, node_pow_n, mul_one, map_pow] + +theorem cosetNode_injective (D : Domain F) {s : F} (hs : s ≠ 0) : + Function.Injective (D.cosetNode s : D.Idx → E) := fun _ _ h ↦ + D.node_injective (mul_left_cancel₀ hs ((algebraMap F E).injective h)) + +/-- The nodal polynomial of a coset of the order-`n` subgroup is `Xⁿ - sⁿ`. -/ +theorem cosetNodal_eq (D : Domain F) {s : F} (hs : s ≠ 0) : + Lagrange.nodal Finset.univ (D.cosetNode s : D.Idx → E) = + Polynomial.X ^ D.n - Polynomial.C (algebraMap F E s ^ D.n) := by + have h : (Polynomial.C (algebraMap F E s ^ D.n)).degree < + ((Polynomial.X : E[X]) ^ D.n).degree := by + rw [Polynomial.degree_X_pow] + exact Polynomial.degree_C_le.trans_lt (by exact_mod_cast D.n_pos) + apply Polynomial.eq_of_degree_le_of_eval_index_eq (v := D.cosetNode s) Finset.univ + · exact (D.cosetNode_injective hs).injOn + · exact Lagrange.degree_nodal.le + · rw [Lagrange.degree_nodal, Polynomial.degree_sub_eq_left_of_degree_lt h] + simp + · rw [Lagrange.nodal_monic, Polynomial.leadingCoeff_sub_of_degree_lt h, + Polynomial.monic_X_pow] + · intro i hi + rw [Lagrange.eval_nodal_at_node hi, Polynomial.eval_sub, Polynomial.eval_pow, + Polynomial.eval_X, Polynomial.eval_C, cosetNode_pow_n, sub_self] + +/-- `n` is invertible in `E`. -/ +theorem natCast_n_ne_zero (D : Domain F) : ((D.n : ℕ) : E) ≠ 0 := by + rw [← map_natCast (algebraMap F E), ne_eq, map_eq_zero_iff _ (algebraMap F E).injective] + exact D.natCast_ne_zero + +/-- The barycentric weight of the coset node `x = s ωⁱ` is `x / (n sⁿ)`. -/ +theorem cosetNodalWeight_eq (D : Domain F) {s : F} (hs : s ≠ 0) (i : D.Idx) : + Lagrange.nodalWeight Finset.univ (D.cosetNode s : D.Idx → E) i = + D.cosetNode s i * ((D.n : E) * algebraMap F E s ^ D.n)⁻¹ := by + rw [Lagrange.nodalWeight_eq_eval_derivative_nodal (Finset.mem_univ i), cosetNodal_eq D hs] + simp only [Polynomial.derivative_sub, Polynomial.derivative_X_pow, Polynomial.derivative_C, + sub_zero, Polynomial.eval_mul, Polynomial.eval_C, Polynomial.eval_pow, Polynomial.eval_X] + have hx : (D.cosetNode s i : E) ≠ 0 := by + rw [cosetNode, ne_eq, map_eq_zero_iff _ (algebraMap F E).injective] + exact mul_ne_zero hs (pow_ne_zero _ (D.primitive.ne_zero D.n_ne_zero)) + have hpow : (D.cosetNode s i : E) * D.cosetNode s i ^ (D.n - 1) = algebraMap F E s ^ D.n := by + rw [← pow_succ', Nat.sub_add_cancel D.n_pos, cosetNode_pow_n] + have hn := D.natCast_n_ne_zero (E := E) + rw [← hpow] + field_simp + +/-- **Barycentric evaluation on a coset.** A polynomial of degree below `n` evaluates at a point +`z` of the extension field outside the coset as `(zⁿ - sⁿ) / (n sⁿ) · ∑ᵢ xᵢ / (z - xᵢ) · p(xᵢ)`, +where `xᵢ = s ωⁱ`. -/ +theorem aeval_eq_cosetBarycentric (D : Domain F) {s : F} (hs : s ≠ 0) (p : F[X]) + (hp : p.degree < D.n) (z : E) (hz : z ^ D.n ≠ algebraMap F E s ^ D.n) : + Polynomial.aeval z p = + (z ^ D.n - algebraMap F E s ^ D.n) * ((D.n : E) * algebraMap F E s ^ D.n)⁻¹ * + ∑ i : D.Idx, D.cosetNode s i * (z - D.cosetNode s i)⁻¹ * + algebraMap F E (p.eval (s * D.node i)) := by + have hinj : Set.InjOn (D.cosetNode s : D.Idx → E) ((Finset.univ : Finset D.Idx) : Set D.Idx) := + (D.cosetNode_injective hs).injOn + have hdeg : (p.map (algebraMap F E)).degree < (Finset.univ : Finset D.Idx).card := by + rw [Polynomial.degree_map, Finset.card_univ, Fintype.card_fin]; exact hp + rw [Polynomial.aeval_def, ← Polynomial.eval_map, Lagrange.eq_interpolate hinj hdeg, + Lagrange.eval_interpolate_not_at_node _ (fun i _ h ↦ hz (by rw [h, cosetNode_pow_n])), + cosetNodal_eq D hs, Finset.mul_sum, Finset.mul_sum] + refine Finset.sum_congr rfl fun i _ ↦ ?_ + rw [Polynomial.eval_sub, Polynomial.eval_pow, Polynomial.eval_X, Polynomial.eval_C, + cosetNodalWeight_eq D hs, Polynomial.eval_map, ← Polynomial.aeval_def, cosetNode, + Polynomial.aeval_algebraMap_apply_eq_algebraMap_eval] + ring + +end Domain +end NTT +end CPolynomial +end CompPoly diff --git a/CompPoly/Univariate/NTTFast/Butterfly.lean b/CompPoly/Univariate/NTTFast/Butterfly.lean new file mode 100644 index 00000000..95bb0f49 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Butterfly.lean @@ -0,0 +1,459 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Plan +/-! # Bounds-proved radix-four DIF butterflies + +Check array ranges once per stage and use machine-word indices inside the loop. +The refinement theorems preserve the existing total functions, including invalid ranges. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Plan + +variable {R : Type*} + +/-- DIF inner loop with array bounds checked once by its caller. -/ +def boundedDIF [Add R] [Sub R] [Mul R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) (a : Array R) + (ht : 2 * limit ≤ th.size ∧ limit ≤ tl.size) + (ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size) : Array R := + if hj : j < limit then + let w0 := th[j]'(by omega) + let w1 := th[j + limit]'(by omega) + let wl := tl[j]'(by omega) + let x0 := a[i0]'(by omega) + let x1 := a[i1]'(by omega) + let x2 := a[i2]'(by omega) + let x3 := a[i3]'(by omega) + let a0 := x0 + x2 + let a2 := w0 * (x0 - x2) + let a1 := x1 + x3 + let a3 := w1 * (x1 - x3) + let b := a.set i0 (a0 + a1) (by omega) + have hb : b.size = a.size := Array.size_set .. + let c := b.set i1 (wl * (a0 - a1)) (by omega) + have hc : c.size = a.size := (Array.size_set ..).trans hb + let d := c.set i2 (a2 + a3) (by omega) + have hd : d.size = a.size := (Array.size_set ..).trans hc + let e := d.set i3 (wl * (a2 - a3)) (by omega) + have he : e.size = a.size := (Array.size_set ..).trans hd + boundedDIF th tl limit (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) e ht + (by omega) + else a +termination_by limit - j +decreasing_by omega + +/-- Bounds proofs change execution costs, not butterfly results. -/ +theorem boundedDIF_eq [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) (a : Array R) + (ht : 2 * limit ≤ th.size ∧ limit ≤ tl.size) + (ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size) : + boundedDIF th tl limit j i0 i1 i2 i3 a ht ha = + butterflyDIFRadix4Inner th tl limit j i0 i1 i2 i3 a := by + rw [boundedDIF, butterflyDIFRadix4Inner] + split + · rename_i hj + rw [boundedDIF_eq] + have h0 : i0 < a.size := by omega + have h1 : i1 < a.size := by omega + have h2 : i2 < a.size := by omega + have h3 : i3 < a.size := by omega + have hw0 : j < th.size := by omega + have hw1 : j + limit < th.size := by omega + have hwl : j < tl.size := by omega + simp only [Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem h0, Array.getElem?_eq_getElem h1, + Array.getElem?_eq_getElem h2, Array.getElem?_eq_getElem h3, + Array.getElem?_eq_getElem hw0, Array.getElem?_eq_getElem hw1, + Array.getElem?_eq_getElem hwl, Option.getD_some, + Array.set!_eq_setIfInBounds, Array.setIfInBounds, Array.size_set, + h0, h1, h2, h3, ↓reduceDIte] + · rfl +termination_by limit - j +decreasing_by omega + +theorem usize_add_one (i : USize) (h : i.toNat + 1 < USize.size) : + (i + 1).toNat = i.toNat + 1 := by + simp only [USize.toNat_add, USize.toNat_one, Nat.mod_eq_of_lt h] + +/-- Machine-word indices for the bounds-proved butterfly loop. -/ +def boundedUDIF [Add R] [Sub R] [Mul R] (th tl : Array R) + (limit j i0 i1 i2 i3 : USize) (a : Array R) + (ht : 2 * limit.toNat ≤ th.size ∧ limit.toNat ≤ tl.size) + (ha : i0.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i1.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i2.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i3.toNat + (limit.toNat - j.toNat) ≤ a.size) + (hu : a.size < USize.size ∧ th.size < USize.size) : Array R := + if hj : j < limit then + have hjn : j.toNat < limit.toNat := hj + have hsum : (j + limit).toNat = j.toNat + limit.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show j.toNat + limit.toNat < USize.size by omega) + let w0 := th.uget j (by omega) + let w1 := th.uget (j + limit) (by omega) + let wl := tl.uget j (by omega) + let x0 := a.uget i0 (by omega) + let x1 := a.uget i1 (by omega) + let x2 := a.uget i2 (by omega) + let x3 := a.uget i3 (by omega) + let a0 := x0 + x2 + let a2 := w0 * (x0 - x2) + let a1 := x1 + x3 + let a3 := w1 * (x1 - x3) + let b := a.uset i0 (a0 + a1) (by omega) + have hb : b.size = a.size := Array.size_uset .. + let c := b.uset i1 (wl * (a0 - a1)) (by omega) + have hc : c.size = a.size := (Array.size_uset ..).trans hb + let d := c.uset i2 (a2 + a3) (by omega) + have hd : d.size = a.size := (Array.size_uset ..).trans hc + let e := d.uset i3 (wl * (a2 - a3)) (by omega) + have he : e.size = a.size := (Array.size_uset ..).trans hd + have hj' := usize_add_one j (by omega) + have h0' := usize_add_one i0 (by omega) + have h1' := usize_add_one i1 (by omega) + have h2' := usize_add_one i2 (by omega) + have h3' := usize_add_one i3 (by omega) + boundedUDIF th tl limit (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) e ht + (by omega) (by omega) + else a +termination_by limit.toNat - j.toNat +decreasing_by omega + +/-- Machine indices refine the natural-index implementation. -/ +theorem boundedUDIF_eq [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : USize) (a : Array R) + (ht : 2 * limit.toNat ≤ th.size ∧ limit.toNat ≤ tl.size) + (ha : i0.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i1.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i2.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i3.toNat + (limit.toNat - j.toNat) ≤ a.size) + (hu : a.size < USize.size ∧ th.size < USize.size) : + boundedUDIF th tl limit j i0 i1 i2 i3 a ht ha hu = + boundedDIF th tl limit.toNat j.toNat i0.toNat i1.toNat i2.toNat i3.toNat a ht ha := by + rw [boundedUDIF, boundedDIF] + split + · rename_i hj + have hjn : j.toNat < limit.toNat := hj + have hsum : (j + limit).toNat = j.toNat + limit.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show j.toNat + limit.toNat < USize.size by omega) + have hj' := usize_add_one j (by omega) + have h0' := usize_add_one i0 (by omega) + have h1' := usize_add_one i1 (by omega) + have h2' := usize_add_one i2 (by omega) + have h3' := usize_add_one i3 (by omega) + rw [boundedUDIF_eq] + simp only [hjn, ↓reduceDIte, hj', h0', h1', h2', h3', + Array.uget, Array.uset_eq_set, hsum] + · rename_i hj + have hjn : ¬ j.toNat < limit.toNat := hj + simp only [hjn, ↓reduceDIte] +termination_by limit.toNat - j.toNat +decreasing_by omega + +/-- Use the checked loop for valid ranges, preserving the general API otherwise. -/ +@[inline] def checkedDIF [Field R] (th tl : Array R) + (limit j i0 i1 i2 i3 : Nat) (a : Array R) : Array R := + if ht : 2 * limit ≤ th.size ∧ limit ≤ tl.size then + if ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size then + if hu : a.size < USize.size ∧ th.size < USize.size then + if hj : j < limit then + boundedUDIF th tl (USize.ofNatLT limit (by omega)) (USize.ofNatLT j (by omega)) + (USize.ofNatLT i0 (by omega)) (USize.ofNatLT i1 (by omega)) + (USize.ofNatLT i2 (by omega)) (USize.ofNatLT i3 (by omega)) a ht ha hu + else a + else boundedDIF th tl limit j i0 i1 i2 i3 a ht ha + else butterflyDIFRadix4Inner th tl limit j i0 i1 i2 i3 a + else butterflyDIFRadix4Inner th tl limit j i0 i1 i2 i3 a + +/-- The range-checked entry point preserves the existing total function. -/ +@[simp] theorem checkedDIF_eq [Field R] (th tl : Array R) + (limit j i0 i1 i2 i3 : Nat) (a : Array R) : + checkedDIF th tl limit j i0 i1 i2 i3 a = + butterflyDIFRadix4Inner th tl limit j i0 i1 i2 i3 a := by + unfold checkedDIF + split + · split + · split + · split + · exact (boundedUDIF_eq ..).trans (boundedDIF_eq ..) + · rw [butterflyDIFRadix4Inner]; simp_all only [↓reduceIte] + · exact boundedDIF_eq .. + · rfl + · rfl + +/-- Process independent radix-four blocks through the checked inner loop. -/ +def checkedBlocks [Field R] (th tl : Array R) (blockSize quarter blocks block : Nat) + (a : Array R) : Array R := + if block < blocks then + let base := block * blockSize + let a := checkedDIF th tl quarter 0 base (base + quarter) + (base + 2 * quarter) (base + 3 * quarter) a + checkedBlocks th tl blockSize quarter blocks (block + 1) a + else a +termination_by blocks - block +decreasing_by omega + +/-- Block traversal retains the original butterfly schedule. -/ +@[simp] theorem checkedBlocks_eq [Field R] (th tl : Array R) (blockSize quarter blocks block : Nat) + (a : Array R) : checkedBlocks th tl blockSize quarter blocks block a = + butterflyDIFRadix4Blocks th tl blockSize quarter blocks block a := by + rw [checkedBlocks, butterflyDIFRadix4Blocks] + split + · rw [checkedBlocks_eq, checkedDIF_eq] + · rfl +termination_by blocks - block +decreasing_by omega + +/-- The scalar butterfly loop changes coefficients only. -/ +@[simp] theorem size_boundedDIF [Add R] [Sub R] [Mul R] + (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) (a : Array R) (ht) (ha) : + (boundedDIF th tl limit j i0 i1 i2 i3 a ht ha).size = a.size := by + rw [boundedDIF] + split + · rw [size_boundedDIF] + simp only [Array.size_set] + · rfl +termination_by limit - j +decreasing_by omega + +/-- Machine-index butterflies preserve the array length. -/ +@[simp] theorem size_boundedUDIF [Add R] [Sub R] [Mul R] (th tl : Array R) + (limit j i0 i1 i2 i3 : USize) (a : Array R) (ht) (ha) (hu) : + (boundedUDIF th tl limit j i0 i1 i2 i3 a ht ha hu).size = a.size := by + rw [boundedUDIF] + split + · rw [size_boundedUDIF] + simp only [Array.size_uset] + · rfl +termination_by limit.toNat - j.toNat +decreasing_by + have _hj : j.toNat < limit.toNat := ‹j < limit› + have _hn := usize_add_one j (by omega) + omega + +/-- Machine-index block traversal, with the complete stage range checked once. -/ +def stageUBlocksDIF [Add R] [Sub R] [Mul R] (th tl : Array R) (q base : USize) (a : Array R) + (ht : 2 * q.toNat ≤ th.size ∧ q.toNat ≤ tl.size) + (hu : a.size < USize.size ∧ th.size < USize.size) + (remaining : Nat) (ha : base.toNat + remaining * (4 * q.toNat) ≤ a.size) : Array R := + match remaining with + | 0 => a + | remaining + 1 => + have hstep : base.toNat + 4 * q.toNat ≤ a.size := by + have hm : 4 * q.toNat ≤ (remaining + 1) * (4 * q.toNat) := + Nat.le_mul_of_pos_left _ (by omega) + omega + have h1 : (base + q).toNat = base.toNat + q.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show base.toNat + q.toNat < USize.size by omega) + have h2 : (base + q + q).toNat = base.toNat + 2 * q.toNat := by + rw [USize.toNat_add, h1] + exact (Nat.mod_eq_of_lt (show base.toNat + q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h3 : (base + q + q + q).toNat = base.toNat + 3 * q.toNat := by + rw [USize.toNat_add, h2] + exact (Nat.mod_eq_of_lt (show base.toNat + 2 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h4 : (base + q + q + q + q).toNat = base.toNat + 4 * q.toNat := by + rw [USize.toNat_add, h3] + exact (Nat.mod_eq_of_lt (show base.toNat + 3 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + let b := boundedUDIF th tl q 0 base (base + q) (base + q + q) + (base + q + q + q) a ht (by simp only [USize.toNat_zero]; omega) hu + have hs : b.size = a.size := size_boundedUDIF .. + stageUBlocksDIF th tl q (base + q + q + q + q) b ht (by omega) remaining + (by simp only [h4, hs]; rw [Nat.succ_mul] at ha; omega) +termination_by remaining + +/-- Hoisting range checks preserves the complete block traversal. -/ +theorem stageUBlocksDIF_eq [Field R] (th tl : Array R) (q base : USize) (a : Array R) + (ht) (hu) (remaining : Nat) (ha) (block : Nat) + (hb : base.toNat = block * (4 * q.toNat)) : + stageUBlocksDIF th tl q base a ht hu remaining ha = + butterflyDIFRadix4Blocks th tl (4 * q.toNat) q.toNat (block + remaining) block a := by + induction remaining generalizing base a block with + | zero => simp only [stageUBlocksDIF, butterflyDIFRadix4Blocks, Nat.add_zero, Nat.lt_irrefl, + ↓reduceIte] + | succ remaining ih => + have hstep : base.toNat + 4 * q.toNat ≤ a.size := by + have hm : 4 * q.toNat ≤ (remaining + 1) * (4 * q.toNat) := + Nat.le_mul_of_pos_left _ (by omega) + omega + have h1 : (base + q).toNat = base.toNat + q.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show base.toNat + q.toNat < USize.size by omega) + have h2 : (base + q + q).toNat = base.toNat + 2 * q.toNat := by + rw [USize.toNat_add, h1] + exact (Nat.mod_eq_of_lt (show base.toNat + q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h3 : (base + q + q + q).toNat = base.toNat + 3 * q.toNat := by + rw [USize.toNat_add, h2] + exact (Nat.mod_eq_of_lt (show base.toNat + 2 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h4 : (base + q + q + q + q).toNat = base.toNat + 4 * q.toNat := by + rw [USize.toNat_add, h3] + exact (Nat.mod_eq_of_lt (show base.toNat + 3 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + rw [stageUBlocksDIF, butterflyDIFRadix4Blocks] + simp only [show block < block + (remaining + 1) by omega, ↓reduceIte] + rw [ih (base + q + q + q + q) _ _ _ (block + 1) (by rw [h4, hb, Nat.add_mul, Nat.one_mul])] + simp only [boundedUDIF_eq, boundedDIF_eq, USize.toNat_zero, h1, h2, h3, hb, + show block + 1 + remaining = block + (remaining + 1) by omega] + +/-- Radix-four blocks with the three unit-twiddle products removed. -/ +def unitUBlocksDIF [Add R] [Sub R] [Mul R] (w : R) (base : USize) (a : Array R) + (hu : a.size < USize.size) (remaining : Nat) + (ha : base.toNat + 4 * remaining ≤ a.size) : Array R := + match remaining with + | 0 => a + | remaining + 1 => + have hstep : base.toNat + 4 ≤ a.size := by omega + have h1 : (base + 1).toNat = base.toNat + 1 := usize_add_one base (by omega) + have h2 : (base + 1 + 1).toNat = base.toNat + 2 := by + rw [usize_add_one (base + 1) (by omega), h1] + have h3 : (base + 1 + 1 + 1).toNat = base.toNat + 3 := by + rw [usize_add_one (base + 1 + 1) (by omega), h2] + have h4 : (base + 1 + 1 + 1 + 1).toNat = base.toNat + 4 := by + rw [usize_add_one (base + 1 + 1 + 1) (by omega), h3] + let x0 := a.uget base (by omega) + let x1 := a.uget (base + 1) (by omega) + let x2 := a.uget (base + 1 + 1) (by omega) + let x3 := a.uget (base + 1 + 1 + 1) (by omega) + let a0 := x0 + x2 + let a2 := x0 - x2 + let a1 := x1 + x3 + let a3 := w * (x1 - x3) + let y0 := a0 + a1 + let y1 := a0 - a1 + let y2 := a2 + a3 + let y3 := a2 - a3 + let b := a.uset base y0 (by omega) + have hb : b.size = a.size := Array.size_uset .. + let c := b.uset (base + 1) y1 (by omega) + have hc : c.size = a.size := (Array.size_uset ..).trans hb + let d := c.uset (base + 1 + 1) y2 (by omega) + have hd : d.size = a.size := (Array.size_uset ..).trans hc + let e := d.uset (base + 1 + 1 + 1) y3 (by omega) + have he : e.size = a.size := (Array.size_uset ..).trans hd + unitUBlocksDIF w (base + 1 + 1 + 1 + 1) e (by omega) remaining (by omega) +termination_by remaining + +/-- Unit-twiddle blocks compute the same checked butterflies. -/ +theorem unitUBlocksDIF_eq [Field R] (th tl : Array R) (base : USize) (a : Array R) + (ht : 2 ≤ th.size ∧ 1 ≤ tl.size) (hu : a.size < USize.size ∧ th.size < USize.size) + (remaining : Nat) (ha : base.toNat + 4 * remaining ≤ a.size) + (hw : (th.getD 0 0) = 1 ∧ (tl.getD 0 0) = 1) : + unitUBlocksDIF (th.getD 1 0) base a hu.1 remaining ha = + stageUBlocksDIF th tl 1 base a (by simpa only [USize.toNat_one] using ht) hu remaining + (by simpa only [USize.toNat_one, Nat.mul_comm] using ha) := by + induction remaining generalizing base a with + | zero => simp only [unitUBlocksDIF, stageUBlocksDIF] + | succ remaining ih => + have hstep : base.toNat + 4 ≤ a.size := by omega + have h1 : (base + 1).toNat = base.toNat + 1 := usize_add_one base (by omega) + have h2 : (base + 1 + 1).toNat = base.toNat + 2 := by + rw [usize_add_one (base + 1) (by omega), h1] + have h3 : (base + 1 + 1 + 1).toNat = base.toNat + 3 := by + rw [usize_add_one (base + 1 + 1) (by omega), h2] + have h4 : (base + 1 + 1 + 1 + 1).toNat = base.toNat + 4 := by + rw [usize_add_one (base + 1 + 1 + 1) (by omega), h3] + rw [unitUBlocksDIF, stageUBlocksDIF] + rw [ih _ _ (by simpa only [Array.size_uset] using hu)] + simp only [boundedUDIF, USize.lt_iff_toNat_lt, USize.toNat_zero, USize.toNat_one, + Nat.zero_lt_one, Nat.lt_irrefl, USize.zero_add, ↓reduceDIte, + Array.uget, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (show 0 < th.size by omega), + Array.getElem?_eq_getElem (show 1 < th.size by omega), + Array.getElem?_eq_getElem (show 0 < tl.size by omega), Option.getD_some] at hw ⊢ + simp only [hw.1, hw.2, one_mul] + +/-- Check a complete radix-four stage before its machine-index traversal. -/ +def stageBlocksDIF [Field R] [DecidableEq R] (th tl : Array R) + (blockSize quarter blocks block : Nat) (a : Array R) : Array R := + if hh : blockSize = 4 * quarter ∧ 0 < quarter ∧ block ≤ blocks ∧ + blocks * blockSize ≤ a.size ∧ 2 * quarter ≤ th.size ∧ quarter ≤ tl.size ∧ + a.size < USize.size ∧ th.size < USize.size then + if hz : quarter = 1 ∧ (th.getD 0 0) = 1 ∧ (tl.getD 0 0) = 1 then + unitUBlocksDIF (th.getD 1 0) (USize.ofNatLT (block * blockSize) (by + have := Nat.mul_le_mul_right blockSize hh.2.2.1; omega)) a hh.2.2.2.2.2.2.1 + (blocks - block) (by + simp only [USize.toNat_ofNatLT] + have he : block * blockSize + (blocks - block) * blockSize = blocks * blockSize := by + rw [← Nat.add_mul, Nat.add_sub_of_le hh.2.2.1] + have hb : blockSize = 4 := by simp only [hh.1, hz.1, Nat.mul_one] + rw [Nat.mul_comm 4, ← hb, he]; exact hh.2.2.2.1) + else + stageUBlocksDIF th tl (USize.ofNatLT quarter (by omega)) + (USize.ofNatLT (block * blockSize) (by + have := Nat.mul_le_mul_right blockSize hh.2.2.1 + omega)) a ⟨hh.2.2.2.2.1, hh.2.2.2.2.2.1⟩ + ⟨hh.2.2.2.2.2.2.1, hh.2.2.2.2.2.2.2⟩ (blocks - block) (by + simp only [USize.toNat_ofNatLT] + have he : block * blockSize + (blocks - block) * blockSize = blocks * blockSize := by + rw [← Nat.add_mul, Nat.add_sub_of_le hh.2.2.1] + rw [← hh.1, he]; exact hh.2.2.2.1) + else checkedBlocks th tl blockSize quarter blocks block a + +@[simp] theorem stageBlocksDIF_eq [Field R] [DecidableEq R] (th tl : Array R) + (blockSize quarter blocks block : Nat) (a : Array R) : + stageBlocksDIF th tl blockSize quarter blocks block a = + checkedBlocks th tl blockSize quarter blocks block a := by + unfold stageBlocksDIF + split + · rename_i hh + split + · rename_i hz + rw [unitUBlocksDIF_eq (th := th) (tl := tl) + (ht := by simpa only [hz.1, Nat.mul_one] using + And.intro hh.2.2.2.2.1 hh.2.2.2.2.2.1) + (hu := ⟨hh.2.2.2.2.2.2.1, hh.2.2.2.2.2.2.2⟩) (hw := hz.2)] + rw [stageUBlocksDIF_eq (block := block) (hb := by + simp only [USize.toNat_ofNatLT, USize.toNat_one]; rw [hh.1, hz.1])] + simp only [USize.toNat_one, Nat.mul_one, + show blockSize = 4 by simp only [hh.1, hz.1, Nat.mul_one], hz.1, + Nat.add_sub_of_le hh.2.2.1, checkedBlocks_eq] + · rw [stageUBlocksDIF_eq (block := block) (hb := by + simp only [USize.toNat_ofNatLT]; rw [hh.1])] + simp only [USize.toNat_ofNatLT, ← hh.1, + Nat.add_sub_of_le hh.2.2.1, checkedBlocks_eq] + · rfl + +/-- Planned DIF transform with bounds outside the hot butterfly loops. -/ +def checkedStages [Field R] [DecidableEq R] (D : NTT.Domain R) (tw : Array (Array R)) + (a : Array R) : Array R := Id.run do + let mut a := a + for pass in [0:D.logN / 2] do + let high := D.logN - 1 - 2 * pass + let low := high - 1 + let blockSize := 2 ^ (low + 2) + let quarter := 2 ^ low + a := stageBlocksDIF (tw.getD high #[]) (tw.getD low #[]) + blockSize quarter (D.n / blockSize) 0 a + if D.logN % 2 = 1 then + a := butterflyStageDIFWithTwiddles D 0 (tw.getD 0 #[]) a + return a + +/-- The checked transform agrees with the verified planned transform. -/ +theorem checkedStages_eq [Field R] [DecidableEq R] (D : NTT.Domain R) + (tw : Array (Array R)) (a : Array R) : + checkedStages D tw a = runStagesDIFRadix4WithTwiddles D tw a := by + simp only [checkedStages, runStagesDIFRadix4WithTwiddles, + butterflyRadix4StageDIFWithTwiddles, stageBlocksDIF_eq, checkedBlocks_eq] + +/-- Forward variant using the bounds-proved DIF loops. -/ +@[inline] def checkedForward [Field R] [DecidableEq R] (P : Plan R) (a : Array R) : Array R := + checkedStages P.domain P.twiddles (NTT.loadNaturalArray P.domain a) + +/-- The bounds-proved variant agrees with the existing plan entry point. -/ +theorem forwardImpl_eq_checked [Field R] [DecidableEq R] (P : Plan R) (a : Array R) : + forwardImpl P a = checkedForward P a := by + simp only [forwardImpl, checkedForward, checkedStages_eq] + +end CompPoly.CPolynomial.NTTFast.Plan diff --git a/CompPoly/Univariate/NTTFast/ButterflyDIT.lean b/CompPoly/Univariate/NTTFast/ButterflyDIT.lean new file mode 100644 index 00000000..bdd659b5 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/ButterflyDIT.lean @@ -0,0 +1,464 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Butterfly +/-! # Bounds-proved radix-four DIT butterflies + +Check array ranges once per stage and use machine-word indices inside the loop. +The refinement theorems preserve the existing total functions, including invalid ranges. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Plan + +variable {R : Type*} + +/-- DIT inner loop with array bounds checked once by its caller. -/ +def boundedDIT [Add R] [Sub R] [Mul R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) (a : Array R) + (ht : 2 * limit ≤ th.size ∧ limit ≤ tl.size) + (ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size) : Array R := + if hj : j < limit then + let w0 := th[j]'(by omega) + let w1 := th[j + limit]'(by omega) + let wl := tl[j]'(by omega) + let x0 := a[i0]'(by omega) + let x1 := a[i1]'(by omega) + let x2 := a[i2]'(by omega) + let x3 := a[i3]'(by omega) + let t1 := wl * x1 + let t3 := wl * x3 + let a0 := x0 + t1 + let a1 := x0 - t1 + let a2 := x2 + t3 + let a3 := x2 - t3 + let u2 := w0 * a2 + let u3 := w1 * a3 + let b := a.set i0 (a0 + u2) (by omega) + have hb : b.size = a.size := Array.size_set .. + let c := b.set i1 (a1 + u3) (by omega) + have hc : c.size = a.size := (Array.size_set ..).trans hb + let d := c.set i2 (a0 - u2) (by omega) + have hd : d.size = a.size := (Array.size_set ..).trans hc + let e := d.set i3 (a1 - u3) (by omega) + have he : e.size = a.size := (Array.size_set ..).trans hd + boundedDIT th tl limit (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) e ht + (by omega) + else a +termination_by limit - j +decreasing_by omega + +/-- Bounds proofs change execution costs, not butterfly results. -/ +theorem boundedDIT_eq [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) (a : Array R) + (ht : 2 * limit ≤ th.size ∧ limit ≤ tl.size) + (ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size) : + boundedDIT th tl limit j i0 i1 i2 i3 a ht ha = + butterflyDITRadix4Inner tl th limit j i0 i1 i2 i3 a := by + rw [boundedDIT, butterflyDITRadix4Inner] + split + · rename_i hj + rw [boundedDIT_eq] + have h0 : i0 < a.size := by omega + have h1 : i1 < a.size := by omega + have h2 : i2 < a.size := by omega + have h3 : i3 < a.size := by omega + have hw0 : j < th.size := by omega + have hw1 : j + limit < th.size := by omega + have hwl : j < tl.size := by omega + simp only [Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem h0, Array.getElem?_eq_getElem h1, + Array.getElem?_eq_getElem h2, Array.getElem?_eq_getElem h3, + Array.getElem?_eq_getElem hw0, Array.getElem?_eq_getElem hw1, + Array.getElem?_eq_getElem hwl, Option.getD_some, + Array.set!_eq_setIfInBounds, Array.setIfInBounds, Array.size_set, + h0, h1, h2, h3, ↓reduceDIte] + · rfl +termination_by limit - j +decreasing_by omega + +/-- Machine-word indices for the bounds-proved butterfly loop. -/ +def boundedUDIT [Add R] [Sub R] [Mul R] (th tl : Array R) + (limit j i0 i1 i2 i3 : USize) (a : Array R) + (ht : 2 * limit.toNat ≤ th.size ∧ limit.toNat ≤ tl.size) + (ha : i0.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i1.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i2.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i3.toNat + (limit.toNat - j.toNat) ≤ a.size) + (hu : a.size < USize.size ∧ th.size < USize.size) : Array R := + if hj : j < limit then + have hjn : j.toNat < limit.toNat := hj + have hsum : (j + limit).toNat = j.toNat + limit.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show j.toNat + limit.toNat < USize.size by omega) + let w0 := th.uget j (by omega) + let w1 := th.uget (j + limit) (by omega) + let wl := tl.uget j (by omega) + let x0 := a.uget i0 (by omega) + let x1 := a.uget i1 (by omega) + let x2 := a.uget i2 (by omega) + let x3 := a.uget i3 (by omega) + let t1 := wl * x1 + let t3 := wl * x3 + let a0 := x0 + t1 + let a1 := x0 - t1 + let a2 := x2 + t3 + let a3 := x2 - t3 + let u2 := w0 * a2 + let u3 := w1 * a3 + let b := a.uset i0 (a0 + u2) (by omega) + have hb : b.size = a.size := Array.size_uset .. + let c := b.uset i1 (a1 + u3) (by omega) + have hc : c.size = a.size := (Array.size_uset ..).trans hb + let d := c.uset i2 (a0 - u2) (by omega) + have hd : d.size = a.size := (Array.size_uset ..).trans hc + let e := d.uset i3 (a1 - u3) (by omega) + have he : e.size = a.size := (Array.size_uset ..).trans hd + have hj' := usize_add_one j (by omega) + have h0' := usize_add_one i0 (by omega) + have h1' := usize_add_one i1 (by omega) + have h2' := usize_add_one i2 (by omega) + have h3' := usize_add_one i3 (by omega) + boundedUDIT th tl limit (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) e ht + (by omega) (by omega) + else a +termination_by limit.toNat - j.toNat +decreasing_by omega + +/-- Machine indices refine the natural-index implementation. -/ +theorem boundedUDIT_eq [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : USize) (a : Array R) + (ht : 2 * limit.toNat ≤ th.size ∧ limit.toNat ≤ tl.size) + (ha : i0.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i1.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i2.toNat + (limit.toNat - j.toNat) ≤ a.size ∧ + i3.toNat + (limit.toNat - j.toNat) ≤ a.size) + (hu : a.size < USize.size ∧ th.size < USize.size) : + boundedUDIT th tl limit j i0 i1 i2 i3 a ht ha hu = + boundedDIT th tl limit.toNat j.toNat i0.toNat i1.toNat i2.toNat i3.toNat a ht ha := by + rw [boundedUDIT, boundedDIT] + split + · rename_i hj + have hjn : j.toNat < limit.toNat := hj + have hsum : (j + limit).toNat = j.toNat + limit.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show j.toNat + limit.toNat < USize.size by omega) + have hj' := usize_add_one j (by omega) + have h0' := usize_add_one i0 (by omega) + have h1' := usize_add_one i1 (by omega) + have h2' := usize_add_one i2 (by omega) + have h3' := usize_add_one i3 (by omega) + rw [boundedUDIT_eq] + simp only [hjn, ↓reduceDIte, hj', h0', h1', h2', h3', + Array.uget, Array.uset_eq_set, hsum] + · rename_i hj + have hjn : ¬ j.toNat < limit.toNat := hj + simp only [hjn, ↓reduceDIte] +termination_by limit.toNat - j.toNat +decreasing_by omega + +/-- Use the checked loop for valid ranges, preserving the general API otherwise. -/ +@[inline] def checkedDIT [Field R] (th tl : Array R) + (limit j i0 i1 i2 i3 : Nat) (a : Array R) : Array R := + if ht : 2 * limit ≤ th.size ∧ limit ≤ tl.size then + if ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size then + if hu : a.size < USize.size ∧ th.size < USize.size then + if hj : j < limit then + boundedUDIT th tl (USize.ofNatLT limit (by omega)) (USize.ofNatLT j (by omega)) + (USize.ofNatLT i0 (by omega)) (USize.ofNatLT i1 (by omega)) + (USize.ofNatLT i2 (by omega)) (USize.ofNatLT i3 (by omega)) a ht ha hu + else a + else boundedDIT th tl limit j i0 i1 i2 i3 a ht ha + else butterflyDITRadix4Inner tl th limit j i0 i1 i2 i3 a + else butterflyDITRadix4Inner tl th limit j i0 i1 i2 i3 a + +/-- The range-checked entry point preserves the existing total function. -/ +@[simp] theorem checkedDIT_eq [Field R] (th tl : Array R) + (limit j i0 i1 i2 i3 : Nat) (a : Array R) : + checkedDIT th tl limit j i0 i1 i2 i3 a = + butterflyDITRadix4Inner tl th limit j i0 i1 i2 i3 a := by + unfold checkedDIT + split + · split + · split + · split + · exact (boundedUDIT_eq ..).trans (boundedDIT_eq ..) + · rw [butterflyDITRadix4Inner]; simp_all only [↓reduceIte] + · exact boundedDIT_eq .. + · rfl + · rfl + +/-- Process independent radix-four blocks through the checked inner loop. -/ +def checkedDITBlocks [Field R] (th tl : Array R) (blockSize quarter blocks block : Nat) + (a : Array R) : Array R := + if block < blocks then + let base := block * blockSize + let a := checkedDIT th tl quarter 0 base (base + quarter) + (base + 2 * quarter) (base + 3 * quarter) a + checkedDITBlocks th tl blockSize quarter blocks (block + 1) a + else a +termination_by blocks - block +decreasing_by omega + +/-- Block traversal retains the original butterfly schedule. -/ +@[simp] theorem checkedDITBlocks_eq [Field R] (th tl : Array R) + (blockSize quarter blocks block : Nat) + (a : Array R) : checkedDITBlocks th tl blockSize quarter blocks block a = + butterflyDITRadix4Blocks tl th blockSize quarter blocks block a := by + rw [checkedDITBlocks, butterflyDITRadix4Blocks] + split + · rw [checkedDITBlocks_eq, checkedDIT_eq] + · rfl +termination_by blocks - block +decreasing_by omega + +/-- The scalar butterfly loop changes coefficients only. -/ +@[simp] theorem size_boundedDIT [Add R] [Sub R] [Mul R] + (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) (a : Array R) (ht) (ha) : + (boundedDIT th tl limit j i0 i1 i2 i3 a ht ha).size = a.size := by + rw [boundedDIT] + split + · rw [size_boundedDIT] + simp only [Array.size_set] + · rfl +termination_by limit - j +decreasing_by omega + +/-- Machine-index butterflies preserve the array length. -/ +@[simp] theorem size_boundedUDIT [Add R] [Sub R] [Mul R] (th tl : Array R) + (limit j i0 i1 i2 i3 : USize) (a : Array R) (ht) (ha) (hu) : + (boundedUDIT th tl limit j i0 i1 i2 i3 a ht ha hu).size = a.size := by + rw [boundedUDIT] + split + · rw [size_boundedUDIT] + simp only [Array.size_uset] + · rfl +termination_by limit.toNat - j.toNat +decreasing_by + have _hj : j.toNat < limit.toNat := ‹j < limit› + have _hn := usize_add_one j (by omega) + omega + +/-- Machine-index block traversal, with the complete stage range checked once. -/ +def stageUBlocksDIT [Add R] [Sub R] [Mul R] (th tl : Array R) (q base : USize) (a : Array R) + (ht : 2 * q.toNat ≤ th.size ∧ q.toNat ≤ tl.size) + (hu : a.size < USize.size ∧ th.size < USize.size) + (remaining : Nat) (ha : base.toNat + remaining * (4 * q.toNat) ≤ a.size) : Array R := + match remaining with + | 0 => a + | remaining + 1 => + have hstep : base.toNat + 4 * q.toNat ≤ a.size := by + have hm : 4 * q.toNat ≤ (remaining + 1) * (4 * q.toNat) := + Nat.le_mul_of_pos_left _ (by omega) + omega + have h1 : (base + q).toNat = base.toNat + q.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show base.toNat + q.toNat < USize.size by omega) + have h2 : (base + q + q).toNat = base.toNat + 2 * q.toNat := by + rw [USize.toNat_add, h1] + exact (Nat.mod_eq_of_lt (show base.toNat + q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h3 : (base + q + q + q).toNat = base.toNat + 3 * q.toNat := by + rw [USize.toNat_add, h2] + exact (Nat.mod_eq_of_lt (show base.toNat + 2 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h4 : (base + q + q + q + q).toNat = base.toNat + 4 * q.toNat := by + rw [USize.toNat_add, h3] + exact (Nat.mod_eq_of_lt (show base.toNat + 3 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + let b := boundedUDIT th tl q 0 base (base + q) (base + q + q) + (base + q + q + q) a ht (by simp only [USize.toNat_zero]; omega) hu + have hs : b.size = a.size := size_boundedUDIT .. + stageUBlocksDIT th tl q (base + q + q + q + q) b ht (by omega) remaining + (by simp only [h4, hs]; rw [Nat.succ_mul] at ha; omega) +termination_by remaining + +/-- Hoisting range checks preserves the complete block traversal. -/ +theorem stageUBlocksDIT_eq [Field R] (th tl : Array R) (q base : USize) (a : Array R) + (ht) (hu) (remaining : Nat) (ha) (block : Nat) + (hb : base.toNat = block * (4 * q.toNat)) : + stageUBlocksDIT th tl q base a ht hu remaining ha = + butterflyDITRadix4Blocks tl th (4 * q.toNat) q.toNat (block + remaining) block a := by + induction remaining generalizing base a block with + | zero => simp only [stageUBlocksDIT, butterflyDITRadix4Blocks, Nat.add_zero, Nat.lt_irrefl, + ↓reduceIte] + | succ remaining ih => + have hstep : base.toNat + 4 * q.toNat ≤ a.size := by + have hm : 4 * q.toNat ≤ (remaining + 1) * (4 * q.toNat) := + Nat.le_mul_of_pos_left _ (by omega) + omega + have h1 : (base + q).toNat = base.toNat + q.toNat := by + rw [USize.toNat_add] + exact Nat.mod_eq_of_lt (show base.toNat + q.toNat < USize.size by omega) + have h2 : (base + q + q).toNat = base.toNat + 2 * q.toNat := by + rw [USize.toNat_add, h1] + exact (Nat.mod_eq_of_lt (show base.toNat + q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h3 : (base + q + q + q).toNat = base.toNat + 3 * q.toNat := by + rw [USize.toNat_add, h2] + exact (Nat.mod_eq_of_lt (show base.toNat + 2 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + have h4 : (base + q + q + q + q).toNat = base.toNat + 4 * q.toNat := by + rw [USize.toNat_add, h3] + exact (Nat.mod_eq_of_lt (show base.toNat + 3 * q.toNat + q.toNat < USize.size by omega)).trans + (by omega) + rw [stageUBlocksDIT, butterflyDITRadix4Blocks] + simp only [show block < block + (remaining + 1) by omega, ↓reduceIte] + rw [ih (base + q + q + q + q) _ _ _ (block + 1) (by rw [h4, hb, Nat.add_mul, Nat.one_mul])] + simp only [boundedUDIT_eq, boundedDIT_eq, USize.toNat_zero, h1, h2, h3, hb, + show block + 1 + remaining = block + (remaining + 1) by omega] + +/-- Radix-four blocks with the three unit-twiddle products removed. -/ +def unitUBlocksDIT [Add R] [Sub R] [Mul R] (w : R) (base : USize) (a : Array R) + (hu : a.size < USize.size) (remaining : Nat) + (ha : base.toNat + 4 * remaining ≤ a.size) : Array R := + match remaining with + | 0 => a + | remaining + 1 => + have hstep : base.toNat + 4 ≤ a.size := by omega + have h1 : (base + 1).toNat = base.toNat + 1 := usize_add_one base (by omega) + have h2 : (base + 1 + 1).toNat = base.toNat + 2 := by + rw [usize_add_one (base + 1) (by omega), h1] + have h3 : (base + 1 + 1 + 1).toNat = base.toNat + 3 := by + rw [usize_add_one (base + 1 + 1) (by omega), h2] + have h4 : (base + 1 + 1 + 1 + 1).toNat = base.toNat + 4 := by + rw [usize_add_one (base + 1 + 1 + 1) (by omega), h3] + let x0 := a.uget base (by omega) + let x1 := a.uget (base + 1) (by omega) + let x2 := a.uget (base + 1 + 1) (by omega) + let x3 := a.uget (base + 1 + 1 + 1) (by omega) + let a0 := x0 + x1 + let a1 := x0 - x1 + let a2 := x2 + x3 + let a3 := w * (x2 - x3) + let y0 := a0 + a2 + let y1 := a1 + a3 + let y2 := a0 - a2 + let y3 := a1 - a3 + let b := a.uset base y0 (by omega) + have hb : b.size = a.size := Array.size_uset .. + let c := b.uset (base + 1) y1 (by omega) + have hc : c.size = a.size := (Array.size_uset ..).trans hb + let d := c.uset (base + 1 + 1) y2 (by omega) + have hd : d.size = a.size := (Array.size_uset ..).trans hc + let e := d.uset (base + 1 + 1 + 1) y3 (by omega) + have he : e.size = a.size := (Array.size_uset ..).trans hd + unitUBlocksDIT w (base + 1 + 1 + 1 + 1) e (by omega) remaining (by omega) +termination_by remaining + +/-- Unit-twiddle blocks compute the same checked butterflies. -/ +theorem unitUBlocksDIT_eq [Field R] (th tl : Array R) (base : USize) (a : Array R) + (ht : 2 ≤ th.size ∧ 1 ≤ tl.size) (hu : a.size < USize.size ∧ th.size < USize.size) + (remaining : Nat) (ha : base.toNat + 4 * remaining ≤ a.size) + (hw : (th.getD 0 0) = 1 ∧ (tl.getD 0 0) = 1) : + unitUBlocksDIT (th.getD 1 0) base a hu.1 remaining ha = + stageUBlocksDIT th tl 1 base a (by simpa only [USize.toNat_one] using ht) hu remaining + (by simpa only [USize.toNat_one, Nat.mul_comm] using ha) := by + induction remaining generalizing base a with + | zero => simp only [unitUBlocksDIT, stageUBlocksDIT] + | succ remaining ih => + have hstep : base.toNat + 4 ≤ a.size := by omega + have h1 : (base + 1).toNat = base.toNat + 1 := usize_add_one base (by omega) + have h2 : (base + 1 + 1).toNat = base.toNat + 2 := by + rw [usize_add_one (base + 1) (by omega), h1] + have h3 : (base + 1 + 1 + 1).toNat = base.toNat + 3 := by + rw [usize_add_one (base + 1 + 1) (by omega), h2] + have h4 : (base + 1 + 1 + 1 + 1).toNat = base.toNat + 4 := by + rw [usize_add_one (base + 1 + 1 + 1) (by omega), h3] + rw [unitUBlocksDIT, stageUBlocksDIT] + rw [ih _ _ (by simpa only [Array.size_uset] using hu)] + simp only [boundedUDIT, USize.lt_iff_toNat_lt, USize.toNat_zero, USize.toNat_one, + Nat.zero_lt_one, Nat.lt_irrefl, USize.zero_add, ↓reduceDIte, + Array.uget, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (show 0 < th.size by omega), + Array.getElem?_eq_getElem (show 1 < th.size by omega), + Array.getElem?_eq_getElem (show 0 < tl.size by omega), Option.getD_some] at hw ⊢ + simp only [hw.1, hw.2, one_mul] + +/-- Check a complete radix-four stage before its machine-index traversal. -/ +def stageBlocksDIT [Field R] [DecidableEq R] (th tl : Array R) + (blockSize quarter blocks block : Nat) (a : Array R) : Array R := + if hh : blockSize = 4 * quarter ∧ 0 < quarter ∧ block ≤ blocks ∧ + blocks * blockSize ≤ a.size ∧ 2 * quarter ≤ th.size ∧ quarter ≤ tl.size ∧ + a.size < USize.size ∧ th.size < USize.size then + if hz : quarter = 1 ∧ (th.getD 0 0) = 1 ∧ (tl.getD 0 0) = 1 then + unitUBlocksDIT (th.getD 1 0) (USize.ofNatLT (block * blockSize) (by + have := Nat.mul_le_mul_right blockSize hh.2.2.1; omega)) a hh.2.2.2.2.2.2.1 + (blocks - block) (by + simp only [USize.toNat_ofNatLT] + have he : block * blockSize + (blocks - block) * blockSize = blocks * blockSize := by + rw [← Nat.add_mul, Nat.add_sub_of_le hh.2.2.1] + have hb : blockSize = 4 := by simp only [hh.1, hz.1, Nat.mul_one] + rw [Nat.mul_comm 4, ← hb, he]; exact hh.2.2.2.1) + else + stageUBlocksDIT th tl (USize.ofNatLT quarter (by omega)) + (USize.ofNatLT (block * blockSize) (by + have := Nat.mul_le_mul_right blockSize hh.2.2.1 + omega)) a ⟨hh.2.2.2.2.1, hh.2.2.2.2.2.1⟩ + ⟨hh.2.2.2.2.2.2.1, hh.2.2.2.2.2.2.2⟩ (blocks - block) (by + simp only [USize.toNat_ofNatLT] + have he : block * blockSize + (blocks - block) * blockSize = blocks * blockSize := by + rw [← Nat.add_mul, Nat.add_sub_of_le hh.2.2.1] + rw [← hh.1, he]; exact hh.2.2.2.1) + else checkedDITBlocks th tl blockSize quarter blocks block a + +@[simp] theorem stageBlocksDIT_eq [Field R] [DecidableEq R] (th tl : Array R) + (blockSize quarter blocks block : Nat) (a : Array R) : + stageBlocksDIT th tl blockSize quarter blocks block a = + checkedDITBlocks th tl blockSize quarter blocks block a := by + unfold stageBlocksDIT + split + · rename_i hh + split + · rename_i hz + rw [unitUBlocksDIT_eq (th := th) (tl := tl) + (ht := by simpa only [hz.1, Nat.mul_one] using + And.intro hh.2.2.2.2.1 hh.2.2.2.2.2.1) + (hu := ⟨hh.2.2.2.2.2.2.1, hh.2.2.2.2.2.2.2⟩) (hw := hz.2)] + rw [stageUBlocksDIT_eq (block := block) (hb := by + simp only [USize.toNat_ofNatLT, USize.toNat_one]; rw [hh.1, hz.1])] + simp only [USize.toNat_one, Nat.mul_one, + show blockSize = 4 by simp only [hh.1, hz.1, Nat.mul_one], hz.1, + Nat.add_sub_of_le hh.2.2.1, checkedDITBlocks_eq] + · rw [stageUBlocksDIT_eq (block := block) (hb := by + simp only [USize.toNat_ofNatLT]; rw [hh.1])] + simp only [USize.toNat_ofNatLT, ← hh.1, + Nat.add_sub_of_le hh.2.2.1, checkedDITBlocks_eq] + · rfl + +/-- Planned DIT transform with bounds outside the hot butterfly loops. -/ +def checkedDITStages [Field R] [DecidableEq R] (D : NTT.Domain R) (tw : Array (Array R)) + (a : Array R) : Array R := Id.run do + let mut a := a + for pass in [0:D.logN / 2] do + let low := 2 * pass + let high := low + 1 + let blockSize := 2 ^ (low + 2) + let quarter := 2 ^ low + a := stageBlocksDIT (tw.getD high #[]) (tw.getD low #[]) + blockSize quarter (D.n / blockSize) 0 a + if D.logN % 2 = 1 then + a := butterflyStageWithTwiddles D (D.logN - 1) (tw.getD (D.logN - 1) #[]) a + return a + +/-- The checked transform agrees with the verified planned transform. -/ +theorem checkedDITStages_eq [Field R] [DecidableEq R] (D : NTT.Domain R) + (tw : Array (Array R)) (a : Array R) : + checkedDITStages D tw a = runStagesRadix4WithTwiddles D tw a := by + simp only [checkedDITStages, runStagesRadix4WithTwiddles, + butterflyRadix4StageWithTwiddles, stageBlocksDIT_eq, checkedDITBlocks_eq] + +/-- Inverse variant using the bounds-proved DIT loops. -/ +@[inline] def checkedInverse [Field R] [DecidableEq R] (P : Plan R) (a : Array R) : Array R := + P.normalize (checkedDITStages P.inverseDomain P.inverseTwiddles a) + +/-- The bounds-proved variant agrees with the existing plan entry point. -/ +theorem inverseImpl_eq_checked [Field R] [DecidableEq R] (P : Plan R) (a : Array R) : + inverseImpl P a = checkedInverse P a := by + simp only [inverseImpl, checkedInverse, checkedDITStages_eq] + +end CompPoly.CPolynomial.NTTFast.Plan diff --git a/CompPoly/Univariate/NTTFast/Columns.lean b/CompPoly/Univariate/NTTFast/Columns.lean new file mode 100644 index 00000000..04c5765b --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Columns.lean @@ -0,0 +1,426 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Parallel +public import CompPoly.Univariate.NTTFast.Reverse32 +public import CompPoly.Data.ByteArray.Packed +public import CompPoly.Data.Tasks + +/-! # Column-parallel natural-order transform on field arrays + +A transform of `2 ^ logN` coefficients is viewed as sixteen rows of `M = 2 ^ (logN - 4)` +columns. Its top four layers act independently on every column, so `S` column tasks each +copy their column range of the sixteen rows into a local buffer and run the two top +radix-four passes there, with the matching twiddle columns. Leaf task `l` collects row `l` +from every column task, runs the remaining passes and reverses its own bit order, with +optional scaling. One sequential pass interleaves the sixteen leaves into natural order. + +Every coefficient crosses task boundaries twice. The runtime marks an array of boxed values for +sharing element by element, under the task manager's lock, when a task returns it; so tasks +return their results as packed elements in a `ByteArray` (`PackedRepr`), which is marked at +once. Leaf `l` stores its words after `16 l` padding words, so that the final pass, which reads +all sixteen leaves at the same position, does not map them to the same cache sets. No externs +are used. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Columns + +variable {α R : Type*} + +/-- Copy entries `k, …, cnt - 1` of a range of `src` into a range of `dst`. -/ +def copyGo (src : @& Array α) (si di cnt k : USize) (dst : Array α) + (_hk : k ≤ cnt) (hs : si.toNat + cnt.toNat ≤ src.size) (hd : di.toNat + cnt.toNat ≤ dst.size) + (hu : src.size < USize.size ∧ dst.size < USize.size) : Array α := + if hlt : k < cnt then + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hkn : k.toNat < cnt.toNat := hlt + have hcs := cnt.toNat_lt_size + have hds := di.toNat_lt_size + have ha : (si + k).toNat = si.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hb : (di + k).toNat = di.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hk1 : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by omega) + copyGo src si di cnt (k + 1) (dst.uset (di + k) (src.uget (si + k) (by omega)) (by omega)) + (by rw [USize.le_iff_toNat_le, hk1]; omega) hs + (by simp only [Array.size_uset]; exact hd) (by simp only [Array.size_uset]; exact hu) + else dst +termination_by cnt.toNat - k.toNat +decreasing_by + have : k.toNat < cnt.toNat := hlt + have : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by + have := cnt.toNat_lt_size; omega) + omega + +/-- Copy `cnt` entries of `src` from `si` into `dst` from `di`; out-of-range copies do nothing. -/ +def copyRange (src : @& Array α) (si : Nat) (dst : Array α) (di cnt : Nat) : Array α := + if h : si + cnt ≤ src.size ∧ di + cnt ≤ dst.size ∧ src.size < USize.size ∧ + dst.size < USize.size then + copyGo src (USize.ofNatLT si (by omega)) (USize.ofNatLT di (by omega)) + (USize.ofNatLT cnt (by omega)) 0 dst (by simp only [USize.zero_le]) + (by simp only [USize.toNat_ofNatLT]; omega) (by simp only [USize.toNat_ofNatLT]; omega) + ⟨h.2.2.1, h.2.2.2⟩ + else dst + +/-- The number of column tasks: sixteen, but at least 256 columns each. Narrower columns keep a +task's rows and gathered twiddles in L2 at every worker count. -/ +def columnCount (logN : Nat) : Nat := 2 ^ min 4 (logN - 12) + +/-- Rows `0, …, r - 1` of a matrix with rows of length `M`, restricted to the columns +`j0, …, j0 + cs - 1`, as a row-major `r × cs` array. -/ +def gatherRows [Zero α] (src : @& Array α) (M j0 cs r : Nat) : Array α := + (List.range r).foldl (fun acc k ↦ copyRange src (k * M + j0) acc (k * cs) cs) + (Array.replicate (r * cs) 0) + +/-- The top two radix-four passes of a `16 × M` transform on columns `j0, …, j0 + cs - 1`. -/ +@[specialize] def columnPasses [Field R] [DecidableEq R] (a : Array R) + (tw : @& Array (Array R)) (logN M cs j0 : Nat) : Array R := + let a := Plan.stageBlocksDIF (gatherRows (tw.getD (logN - 1) #[]) M j0 cs 8) + (gatherRows (tw.getD (logN - 2) #[]) M j0 cs 4) (16 * cs) (4 * cs) 1 0 a + Plan.stageBlocksDIF (gatherRows (tw.getD (logN - 3) #[]) M j0 cs 2) + (gatherRows (tw.getD (logN - 4) #[]) M j0 cs 1) (4 * cs) cs 4 0 a + +/-- The top two radix-four passes of a `16 × M` transform on columns `j0, …, j0 + cs - 1`. -/ +@[specialize] def columnTask [Field R] [DecidableEq R] (x : @& Array R) + (tw : @& Array (Array R)) (logN M cs j0 : Nat) : Array R := + columnPasses (gatherRows x M j0 cs 16) tw logN M cs j0 + +/-- Store entries `k, …, cnt - 1` of `src` as elements `off + k, …` of `dst`. -/ +def encodeGo [PackedRepr α] (src : @& Array α) (off cnt k : USize) (dst : ByteArray) + (_hk : k ≤ cnt) (hs : cnt.toNat ≤ src.size) + (hd : off.toNat + cnt.toNat ≤ dst.size / PackedRepr.width α) + (hu : dst.size < USize.size) : ByteArray := + if hlt : k < cnt then + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hdl := Nat.div_le_self dst.size (PackedRepr.width α) + have hkn : k.toNat < cnt.toNat := hlt + have ha : (off + k).toNat = off.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hk1 : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by + have := cnt.toNat_lt_size; omega) + encodeGo src off cnt (k + 1) + (PackedRepr.setAt dst (off + k) (src.uget k (by omega)) (by omega) hu) + (by rw [USize.le_iff_toNat_le, hk1]; omega) hs + (by simp only [PackedRepr.size_setAt]; exact hd) + (by simp only [PackedRepr.size_setAt]; exact hu) + else dst +termination_by cnt.toNat - k.toNat +decreasing_by + have : k.toNat < cnt.toNat := hlt + have : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by + have := cnt.toNat_lt_size; omega) + omega + +/-- The entries of `a` as packed elements, after `off` padding slots. -/ +def encode [PackedRepr α] (a : @& Array α) (off : Nat) : ByteArray := + let dst := PackedRepr.alloc α (off + a.size) + if h : dst.size < USize.size then + have hw := PackedRepr.width_pos (R := α) + have hs : off + a.size < USize.size := by + simp only [dst, PackedRepr.size_alloc] at h + exact Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ hw) h + encodeGo a (USize.ofNatLT off (by omega)) (USize.ofNatLT a.size (by omega)) 0 dst + (by simp only [USize.zero_le]) (by simp only [USize.toNat_ofNatLT]; omega) + (by simp only [USize.toNat_ofNatLT, dst, PackedRepr.size_alloc_div]; omega) h + else dst + +/-- Decode elements `si + k, …, si + cnt - 1` of `src` into entries `di + k, …` of `dst`. -/ +def decodeGo [PackedRepr α] (src : @& ByteArray) (si di cnt k : USize) (dst : Array α) + (_hk : k ≤ cnt) (hs : si.toNat + cnt.toNat ≤ src.size / PackedRepr.width α) + (hd : di.toNat + cnt.toNat ≤ dst.size) + (hu : src.size < USize.size ∧ dst.size < USize.size) : Array α := + if hlt : k < cnt then + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hsl := Nat.div_le_self src.size (PackedRepr.width α) + have hkn : k.toNat < cnt.toNat := hlt + have ha : (si + k).toNat = si.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hb : (di + k).toNat = di.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hk1 : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by + have := cnt.toNat_lt_size; omega) + decodeGo src si di cnt (k + 1) + (dst.uset (di + k) + (PackedRepr.getAt src (si + k) (by omega) hu.1) + (by omega)) + (by rw [USize.le_iff_toNat_le, hk1]; omega) hs + (by simp only [Array.size_uset]; exact hd) (by simp only [Array.size_uset]; exact hu) + else dst +termination_by cnt.toNat - k.toNat +decreasing_by + have : k.toNat < cnt.toNat := hlt + have : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by + have := cnt.toNat_lt_size; omega) + omega + +/-- Decode `cnt` elements of `src` from `si` into `dst` from `di`; out-of-range copies do +nothing. -/ +def decodeRange [PackedRepr α] (src : @& ByteArray) (si : Nat) (dst : Array α) (di cnt : Nat) : + Array α := + if h : si + cnt ≤ src.size / PackedRepr.width α ∧ di + cnt ≤ dst.size ∧ src.size < USize.size ∧ + dst.size < USize.size then + have := Nat.div_le_self src.size (PackedRepr.width α) + decodeGo src (USize.ofNatLT si (by omega)) (USize.ofNatLT di (by omega)) + (USize.ofNatLT cnt (by omega)) 0 dst (by simp only [USize.zero_le]) + (by simp only [USize.toNat_ofNatLT]; omega) (by simp only [USize.toNat_ofNatLT]; omega) + ⟨h.2.2.1, h.2.2.2⟩ + else dst + +/-- Row `l` of every column result, in column order. -/ +def gatherLeaf [Zero α] [PackedRepr α] (cols : @& Array ByteArray) (cs l : Nat) : Array α := + (List.range cols.size).foldl + (fun acc s ↦ decodeRange (cols.getD s ByteArray.empty) (l * cs) acc (s * cs) cs) + (Array.replicate (cols.size * cs) 0) + +theorem toNat_usize_le (a : Array α) : a.usize.toNat ≤ a.size := by + simp only [Array.usize, Nat.toUSize, USize.toNat_ofNat'] + exact Nat.mod_le _ _ + +/-- Store `v` at `i`; out-of-range stores do nothing. -/ +@[inline] def setW (a : Array α) (i : USize) (v : α) : Array α := + if h : i < a.usize then + a.uset i v (Nat.lt_of_lt_of_le (USize.lt_iff_toNat_lt.mp h) (toNat_usize_le a)) + else a + +/-- Store entries `q, …, M - 1` of the bit-reversed `a`, optionally scaled by `f`, as elements +`off + q, …` of `out`. -/ +def reverseEncodeGo [Mul α] [PackedRepr α] (a : @& Array α) (shift : UInt32) (scale : Bool) + (f : α) (off M q : USize) (out : ByteArray) + (hd : off.toNat + M.toNat ≤ out.size / PackedRepr.width α) + (hu : out.size < USize.size) : ByteArray := + if hq : q < M then + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hol := Nat.div_le_self out.size (PackedRepr.width α) + have hqM : q.toNat < M.toNat := hq + have ho : (off + q).toNat = off.toNat + q.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + let j := (reverse32 q.toUInt32 >>> shift).toUSize + if h : j.toNat < a.size then + let v := a.uget j h + reverseEncodeGo a shift scale f off M (q + 1) + (PackedRepr.setAt out (off + q) (if scale then f * v else v) (by omega) hu) + (by simp only [PackedRepr.size_setAt]; exact hd) + (by simp only [PackedRepr.size_setAt]; exact hu) + else out + else out +termination_by M.toNat - q.toNat +decreasing_by + have : q.toNat < M.toNat := hq + have : (q + 1).toNat = q.toNat + 1 := Plan.usize_add_one q (by + have := M.toNat_lt_size; omega) + omega + +/-- A `2 ^ m`-entry array in bit-reversed order, optionally scaled by `f`, as elements after +`off` padding slots. -/ +def reverseEncode [Mul α] [PackedRepr α] (m : Nat) (scale : Bool) (f : α) (a : @& Array α) + (off : Nat) : ByteArray := + let out := PackedRepr.alloc α (off + 2 ^ m) + if h : out.size < USize.size then + have hw := PackedRepr.width_pos (R := α) + have hs : off + 2 ^ m < USize.size := by + simp only [out, PackedRepr.size_alloc] at h + exact Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ hw) h + have := Nat.two_pow_pos m + reverseEncodeGo a (32 - m).toUInt32 scale f (USize.ofNatLT off (by omega)) + (USize.ofNatLT (2 ^ m) (by omega)) 0 out + (by simp only [USize.toNat_ofNatLT, out, PackedRepr.size_alloc_div]; omega) h + else out + +/-- The final radix-two layer of an odd-size transform: butterflies on adjacent pairs. -/ +@[inline] def lastPair [Field R] (tw0 : Array R) (a : Array R) : Array R := + Plan.butterflyDIFBlocks tw0 2 1 (a.size / 2) 0 a + +/-- The leaf's remaining layers: radix-four passes, then the radix-two layer of an odd size. -/ +@[specialize] def leafPasses [Field R] [DecidableEq R] (tw : @& Array (Array R)) + (rest : List Nat) (odd : Bool) (a : Array R) : Array R := + let a := Parallel.runPasses false tw rest a + if odd then lastPair (tw.getD 0 #[]) a else a + +/-- Leaf `l`: collect its row, run the remaining layers and reverse its bit order, and store +it after `16 l` padding words. -/ +@[specialize] def leafTask [Field R] [DecidableEq R] [PackedRepr R] (cols : @& Array ByteArray) + (tw : @& Array (Array R)) (rest : List Nat) (odd : Bool) (m cs l : Nat) (scale : Bool) + (f : R) : ByteArray := + reverseEncode m scale f (leafPasses tw rest odd (gatherLeaf cols cs l)) (16 * l) + +/-- Element `o + q` of a stored leaf is in range. -/ +theorem leafIndex (M q o : USize) (hq : q < M) (s n : Nat) + (h : o.toNat + M.toNat ≤ n) (hs : s < USize.size) (hn : n ≤ s) : + (o + q).toNat < n := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hqM : q.toNat < M.toNat := hq + rw [USize.toNat_add, hsize, Nat.mod_eq_of_lt (by omega)] + omega + +/-- Store natural-order entries `16 * q + c` from the stored leaves, for `q` from `q` to +`M - 1`. Entry `16 * q + c` is entry `q` of leaf `bitrev₄ c`, stored as word `16 l + q`. -/ +@[specialize] def assembleGo [PackedRepr α] {β : Type*} (set : β → USize → α → β) + (r : @& Array ByteArray) + (M : USize) (hr : r.size = 16) + (hb : ∀ l (h : l < 16), 16 * l + M.toNat ≤ (r[l]'(by omega)).size / PackedRepr.width α ∧ + (r[l]'(by omega)).size < USize.size) + (q : USize) (out : β) : β := + if hq : q < M then + let i := 16 * q + let out := set out i (PackedRepr.getAt (r[0]'(by omega)) + (0 + q) (leafIndex M q 0 hq _ _ (by simpa using (hb 0 (by decide)).1) + (hb 0 (by decide)).2 (Nat.div_le_self _ _)) (hb 0 (by decide)).2) + let out := set out (i + 1) (PackedRepr.getAt (r[8]'(by omega)) + (128 + q) (leafIndex M q 128 hq _ _ (by simpa using (hb 8 (by decide)).1) + (hb 8 (by decide)).2 (Nat.div_le_self _ _)) (hb 8 (by decide)).2) + let out := set out (i + 2) (PackedRepr.getAt (r[4]'(by omega)) + (64 + q) (leafIndex M q 64 hq _ _ (by simpa using (hb 4 (by decide)).1) + (hb 4 (by decide)).2 (Nat.div_le_self _ _)) (hb 4 (by decide)).2) + let out := set out (i + 3) (PackedRepr.getAt (r[12]'(by omega)) + (192 + q) (leafIndex M q 192 hq _ _ (by simpa using (hb 12 (by decide)).1) + (hb 12 (by decide)).2 (Nat.div_le_self _ _)) (hb 12 (by decide)).2) + let out := set out (i + 4) (PackedRepr.getAt (r[2]'(by omega)) + (32 + q) (leafIndex M q 32 hq _ _ (by simpa using (hb 2 (by decide)).1) + (hb 2 (by decide)).2 (Nat.div_le_self _ _)) (hb 2 (by decide)).2) + let out := set out (i + 5) (PackedRepr.getAt (r[10]'(by omega)) + (160 + q) (leafIndex M q 160 hq _ _ (by simpa using (hb 10 (by decide)).1) + (hb 10 (by decide)).2 (Nat.div_le_self _ _)) (hb 10 (by decide)).2) + let out := set out (i + 6) (PackedRepr.getAt (r[6]'(by omega)) + (96 + q) (leafIndex M q 96 hq _ _ (by simpa using (hb 6 (by decide)).1) + (hb 6 (by decide)).2 (Nat.div_le_self _ _)) (hb 6 (by decide)).2) + let out := set out (i + 7) (PackedRepr.getAt (r[14]'(by omega)) + (224 + q) (leafIndex M q 224 hq _ _ (by simpa using (hb 14 (by decide)).1) + (hb 14 (by decide)).2 (Nat.div_le_self _ _)) (hb 14 (by decide)).2) + let out := set out (i + 8) (PackedRepr.getAt (r[1]'(by omega)) + (16 + q) (leafIndex M q 16 hq _ _ (by simpa using (hb 1 (by decide)).1) + (hb 1 (by decide)).2 (Nat.div_le_self _ _)) (hb 1 (by decide)).2) + let out := set out (i + 9) (PackedRepr.getAt (r[9]'(by omega)) + (144 + q) (leafIndex M q 144 hq _ _ (by simpa using (hb 9 (by decide)).1) + (hb 9 (by decide)).2 (Nat.div_le_self _ _)) (hb 9 (by decide)).2) + let out := set out (i + 10) (PackedRepr.getAt (r[5]'(by omega)) + (80 + q) (leafIndex M q 80 hq _ _ (by simpa using (hb 5 (by decide)).1) + (hb 5 (by decide)).2 (Nat.div_le_self _ _)) (hb 5 (by decide)).2) + let out := set out (i + 11) (PackedRepr.getAt (r[13]'(by omega)) + (208 + q) (leafIndex M q 208 hq _ _ (by simpa using (hb 13 (by decide)).1) + (hb 13 (by decide)).2 (Nat.div_le_self _ _)) (hb 13 (by decide)).2) + let out := set out (i + 12) (PackedRepr.getAt (r[3]'(by omega)) + (48 + q) (leafIndex M q 48 hq _ _ (by simpa using (hb 3 (by decide)).1) + (hb 3 (by decide)).2 (Nat.div_le_self _ _)) (hb 3 (by decide)).2) + let out := set out (i + 13) (PackedRepr.getAt (r[11]'(by omega)) + (176 + q) (leafIndex M q 176 hq _ _ (by simpa using (hb 11 (by decide)).1) + (hb 11 (by decide)).2 (Nat.div_le_self _ _)) (hb 11 (by decide)).2) + let out := set out (i + 14) (PackedRepr.getAt (r[7]'(by omega)) + (112 + q) (leafIndex M q 112 hq _ _ (by simpa using (hb 7 (by decide)).1) + (hb 7 (by decide)).2 (Nat.div_le_self _ _)) (hb 7 (by decide)).2) + let out := set out (i + 15) (PackedRepr.getAt (r[15]'(by omega)) + (240 + q) (leafIndex M q 240 hq _ _ (by simpa using (hb 15 (by decide)).1) + (hb 15 (by decide)).2 (Nat.div_le_self _ _)) (hb 15 (by decide)).2) + assembleGo set r M hr hb (q + 1) out + else out +termination_by M.toNat - q.toNat +decreasing_by + have : q.toNat < M.toNat := hq + have : (q + 1).toNat = q.toNat + 1 := Plan.usize_add_one q (by + have := M.toNat_lt_size; omega) + omega + +/-- Interleave sixteen stored `M`-entry leaves into natural order, writing into `out`. -/ +@[specialize] def assemble [PackedRepr α] {β : Type*} (set : β → USize → α → β) + (r : @& Array ByteArray) + (M : Nat) (out : β) : β := + if hM : M < USize.size then + if hr : r.size = 16 then + if hb : (List.range 16).all (fun l ↦ decide (16 * l + M ≤ + (r.getD l ByteArray.empty).size / PackedRepr.width α ∧ + (r.getD l ByteArray.empty).size < USize.size)) then + assembleGo set r (USize.ofNatLT M hM) hr (fun l h ↦ by + have := of_decide_eq_true (List.all_eq_true.mp hb l (List.mem_range.mpr h)) + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem (show l < r.size by + omega), Option.getD_some] at this + simpa only [USize.toNat_ofNatLT] using this) 0 out + else out + else out + else out + +/-- Rows `q0, …, q1 - 1` of an `M`-row interleave, stored with `set` at the global indices +`16 q0, …, 16 q1 - 1`. The leaf check is the one of `assemble`. -/ +@[specialize] def assembleRows [PackedRepr α] {β : Type*} (set : β → USize → α → β) + (r : @& Array ByteArray) (M q0 q1 : Nat) (out : β) : β := + if h : q0 ≤ q1 ∧ q1 ≤ M ∧ M < USize.size ∧ r.size = 16 then + if hb : (List.range 16).all (fun l ↦ decide (16 * l + M ≤ + (r.getD l ByteArray.empty).size / PackedRepr.width α ∧ + (r.getD l ByteArray.empty).size < USize.size)) then + assembleGo set r (USize.ofNatLT q1 (by omega)) h.2.2.2 (fun l hl ↦ by + have := of_decide_eq_true (List.all_eq_true.mp hb l (List.mem_range.mpr hl)) + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem (show l < r.size by + omega), Option.getD_some] at this + simp only [USize.toNat_ofNatLT] + exact ⟨by omega, this.2⟩) (USize.ofNatLT q0 (by omega)) out + else out + else out + +/-- Rows `q0, …, q0 + K - 1` of an `M`-row interleave as `16 K` packed elements. -/ +@[specialize] def assembleChunk [Zero α] [PackedRepr α] (r : @& Array ByteArray) (M q0 K : Nat) : + ByteArray := + let out := PackedRepr.replicate (16 * K) (0 : α) + if hm : PackedRepr.width α * (16 * K) < USize.size then + assembleRows (α := α) (fun b i v ↦ PackedRepr.setIf (R := α) _ hm b (i - (16 * q0).toUSize) v) + r M q0 (q0 + K) out + else out + +/-- The complete column-parallel transform: natural-order input, natural-order output, +optionally scaled by `f`; `rest` are the passes of a `2 ^ (logN - 4)`-entry leaf. -/ +@[specialize] def transform [Field R] [DecidableEq R] [PackedRepr R] (tw : Array (Array R)) + (logN S : Nat) (rest : List Nat) (scale : Bool) (f : R) (x : Array R) : Array R := + let M := 2 ^ (logN - 4) + let cs := M / S + let cols := Tasks.spawnAll S fun s ↦ encode (columnTask x tw logN M cs (s * cs)) 0 + let leaves := Tasks.spawnAfter cols 16 fun cols l ↦ + leafTask cols tw rest (logN % 2 == 1) (logN - 4) cs l scale f + -- allocated while the tasks run + let out := Array.replicate (16 * M) 0 + assemble setW (leaves.map Task.get) M out + +/-- Rows `0, …, r - 1` of a packed matrix with rows of length `M`, restricted to the columns +`j0, …, j0 + cs - 1`, decoded as a row-major `r × cs` array. -/ +def gatherRowsPacked [Zero α] [PackedRepr α] (src : @& ByteArray) (M j0 cs r : Nat) : + Array α := + (List.range r).foldl (fun acc k ↦ decodeRange src (k * M + j0) acc (k * cs) cs) + (Array.replicate (r * cs) 0) + +/-- `transform` on packed elements: column tasks decode their own columns, and the final +interleave stores packed elements. -/ +@[specialize] def transformPacked [Field R] [DecidableEq R] [PackedRepr R] + (tw : Array (Array R)) (logN S : Nat) (rest : List Nat) (scale : Bool) (f : R) + (x : ByteArray) : ByteArray := + let M := 2 ^ (logN - 4) + let cs := M / S + let cols := Tasks.spawnAll S fun s ↦ + encode (columnPasses (gatherRowsPacked x M (s * cs) cs 16) tw logN M cs (s * cs)) 0 + let leaves := Tasks.spawnAfter cols 16 fun cols l ↦ + leafTask cols tw rest (logN % 2 == 1) (logN - 4) cs l scale f + -- sixteen interleave tasks, each storing a contiguous range of rows; one copy joins them + let K := M / 16 + let chunks := Tasks.spawnAfter leaves 16 fun r c ↦ assembleChunk (α := R) r M (c * K) K + (chunks.map Task.get).foldl (· ++ ·) (ByteArray.emptyWithCapacity (PackedRepr.width R * (16 * M))) + +/-- The elements of the complete slots of `b`. -/ +def decodeAll [Zero α] [PackedRepr α] (b : @& ByteArray) : Array α := + if b.size < USize.size then + decodeRange b 0 (Array.replicate (b.size / PackedRepr.width α) 0) 0 + (b.size / PackedRepr.width α) + else PackedRepr.decode b + +/-- Store entries `i, …, i + n - 1` of `a` into a buffer of `m` bytes. -/ +def encodeAllGo [Zero α] [PackedRepr α] (a : @& Array α) (m : Nat) (hm : m < USize.size) : + Nat → Nat → ByteArray → ByteArray + | 0, _, b => b + | n + 1, i, b => encodeAllGo a m hm n (i + 1) (PackedRepr.setIf m hm b i.toUSize (a.getD i 0)) + +/-- `a` as packed elements. -/ +def encodeAll [Zero α] [PackedRepr α] (a : @& Array α) : ByteArray := + if hm : PackedRepr.width α * a.size < USize.size then + encodeAllGo a _ hm a.size 0 (PackedRepr.replicate a.size (0 : α)) + else PackedRepr.encode a + +end CompPoly.CPolynomial.NTTFast.Columns diff --git a/CompPoly/Univariate/NTTFast/ColumnsCorrectness.lean b/CompPoly/Univariate/NTTFast/ColumnsCorrectness.lean new file mode 100644 index 00000000..8452d252 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/ColumnsCorrectness.lean @@ -0,0 +1,1474 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Plan +import all CompPoly.Univariate.NTTFast.Columns +public import CompPoly.Univariate.NTTFast.Columns +public import CompPoly.Univariate.NTTFast.Packed.Radix4Lemmas +import Mathlib.Tactic.Ring +import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas +import CompPoly.Univariate.NTTFast.Packed.BitReversal +import all CompPoly.Univariate.NTTFast.Correctness.Basic + +/-! # Correctness of the column-parallel transform + +The copies are characterized by their entries. Each radix-four pass has a coordinate formula +in row/column form, so the top two passes commute with gathering a column range: the column +tasks compute exactly the gathered columns of the global passes. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Columns + +variable {α R : Type*} + +/-- An unchecked read is the entry at its index. -/ +theorem uget_eq_getD (a : Array α) (j : USize) (h : j.toNat < a.size) (d : α) (n : Nat) + (hn : j.toNat = n) : a.uget j h = a.getD n d := by + subst hn + simp only [Array.uget, Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem h, + Option.getD_some] + +/-- An unchecked write changes exactly its index. -/ +theorem getD_uset (a : Array α) (j : USize) (v : α) (h : j.toNat < a.size) (i : Nat) (d : α) : + (a.uset j v h).getD i d = if j.toNat = i then v else a.getD i d := by + simp only [Array.uset, Array.getD_eq_getD_getElem?, Array.getElem?_set] + split <;> rfl + +/-- The copy loop keeps the size and writes the remaining range. -/ +theorem copyGo_spec (src : Array α) (si di cnt : USize) (d : α) : + ∀ (m : Nat) (k : USize) (dst : Array α) (hk hs hd hu), cnt.toNat - k.toNat = m → + (copyGo src si di cnt k dst hk hs hd hu).size = dst.size ∧ + ∀ i, (copyGo src si di cnt k dst hk hs hd hu).getD i d = + if di.toNat + k.toNat ≤ i ∧ i < di.toNat + cnt.toNat then + src.getD (si.toNat + (i - di.toNat)) d + else dst.getD i d := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + intro m + induction m with + | zero => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + rw [copyGo] + have hnot : ¬k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + simp only [hnot, ↓reduceDIte, true_and] + intro i + rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + | succ m ih => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + have hlt : k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + have hcs := cnt.toNat_lt_size + have hds := di.toNat_lt_size + have ha : (si + k).toNat = si.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hb : (di + k).toNat = di.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hk1 : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by omega) + rw [copyGo] + simp only [hlt, ↓reduceDIte] + obtain ⟨h1, h2⟩ := ih (k + 1) _ _ _ _ _ (by rw [hk1]; omega) + refine ⟨by rw [h1, Array.size_uset], fun i ↦ ?_⟩ + rw [h2 i, hk1, getD_uset, hb] + split_ifs <;> first + | rfl + | omega + | (rw [uget_eq_getD _ _ _ d _ ha]; congr 1; omega) + +/-- A guarded range copy writes exactly its range. -/ +theorem copyRange_getD (src : Array α) (si : Nat) (dst : Array α) (di cnt : Nat) (d : α) + (h : si + cnt ≤ src.size ∧ di + cnt ≤ dst.size ∧ src.size < USize.size ∧ + dst.size < USize.size) (i : Nat) : + (copyRange src si dst di cnt).getD i d = + if di ≤ i ∧ i < di + cnt then src.getD (si + (i - di)) d else dst.getD i d := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + unfold copyRange + rw [dite_eq_left_of_eq_true (eq_true h)] + obtain ⟨_, h2⟩ := copyGo_spec src (USize.ofNatLT si (by omega)) (USize.ofNatLT di (by omega)) + (USize.ofNatLT cnt (by omega)) d _ 0 dst (by simp only [USize.zero_le]) + (by simp only [USize.toNat_ofNatLT]; omega) (by simp only [USize.toNat_ofNatLT]; omega) + ⟨h.2.2.1, h.2.2.2⟩ rfl + rw [h2 i] + simp only [USize.toNat_ofNatLT, USize.toNat_zero, Nat.add_zero] + +/-- The copy loop keeps the destination size. -/ +theorem size_copyGo (src : Array α) (si di cnt : USize) : + ∀ (m : Nat) (k : USize) (dst : Array α) (hk hs hd hu), cnt.toNat - k.toNat = m → + (copyGo src si di cnt k dst hk hs hd hu).size = dst.size := by + intro m + induction m with + | zero => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + rw [copyGo] + have hnot : ¬k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + simp only [hnot, ↓reduceDIte] + | succ m ih => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + have hlt : k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + have hk1 : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by + have := cnt.toNat_lt_size; omega) + rw [copyGo] + simp only [hlt, ↓reduceDIte] + rw [ih (k + 1) _ _ _ _ _ (by rw [hk1]; omega), Array.size_uset] + +/-- A range copy keeps the destination size. -/ +@[simp] theorem size_copyRange (src : Array α) (si : Nat) (dst : Array α) (di cnt : Nat) : + (copyRange src si dst di cnt).size = dst.size := by + unfold copyRange + split + · exact size_copyGo src _ _ _ _ 0 dst _ _ _ _ rfl + · rfl + +/-- Filling row `k` of a fresh `r × cs` array by `op k`, for every `k < r`. -/ +theorem foldl_rows_spec [Zero α] (op : Nat → Array α → Array α) (val : Nat → Nat → α) + (cs r : Nat) (hop : ∀ k < r, ∀ acc : Array α, acc.size = r * cs → + (op k acc).size = r * cs ∧ ∀ i, (op k acc).getD i 0 = + if k * cs ≤ i ∧ i < k * cs + cs then val k (i - k * cs) else acc.getD i 0) : + ((List.range r).foldl (fun acc k ↦ op k acc) (Array.replicate (r * cs) 0)).size = r * cs ∧ + ∀ k < r, ∀ c < cs, + ((List.range r).foldl (fun acc k ↦ op k acc) (Array.replicate (r * cs) 0)).getD + (k * cs + c) 0 = val k c := by + suffices h : ∀ t ≤ r, + ((List.range t).foldl (fun acc k ↦ op k acc) (Array.replicate (r * cs) 0)).size = r * cs ∧ + ∀ k < t, ∀ c < cs, + ((List.range t).foldl (fun acc k ↦ op k acc) (Array.replicate (r * cs) 0)).getD + (k * cs + c) 0 = val k c by + exact h r (Nat.le_refl r) + intro t + induction t with + | zero => intro _; simp only [List.range_zero, List.foldl_nil, Array.size_replicate, + Nat.not_lt_zero, false_imp_iff, implies_true, and_self] + | succ t ih => + intro ht + obtain ⟨hsz, hval⟩ := ih (by omega) + rw [List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil] + obtain ⟨h1, h2⟩ := hop t (by omega) _ hsz + refine ⟨h1, fun k hk c hc ↦ ?_⟩ + rw [h2] + by_cases hkt : k = t + · subst k + rw [ite_eq_left_of_eq_true _ _ (eq_true (by omega)), show t * cs + c - t * cs = c by omega] + · have hlt : k * cs + c < t * cs := by + have := Nat.mul_le_mul_right cs (show k + 1 ≤ t by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega + rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + exact hval k (by omega) c hc + +/-- Gathered rows have the requested shape and entries. -/ +theorem gatherRows_spec [Zero α] (src : Array α) (M j0 cs r : Nat) + (hsrc : ∀ k < r, k * M + j0 + cs ≤ src.size) + (hu : src.size < USize.size ∧ r * cs < USize.size) : + (gatherRows src M j0 cs r).size = r * cs ∧ + ∀ k < r, ∀ c < cs, (gatherRows src M j0 cs r).getD (k * cs + c) 0 = + src.getD (k * M + (j0 + c)) 0 := by + refine foldl_rows_spec (fun k acc ↦ copyRange src (k * M + j0) acc (k * cs) cs) + (fun k c ↦ src.getD (k * M + (j0 + c)) 0) cs r (fun k hk acc hacc ↦ ?_) + have hcopy : k * cs + cs ≤ r * cs := by + have := Nat.mul_le_mul_right cs (show k + 1 ≤ r by omega) + rw [Nat.add_mul, Nat.one_mul] at this + exact this + refine ⟨by rw [size_copyRange, hacc], fun i ↦ ?_⟩ + rw [copyRange_getD _ _ _ _ _ _ ⟨hsrc k hk, by omega, hu.1, by omega⟩, Nat.add_assoc] + +/-- The word loop keeps the size and decodes the remaining range. -/ +theorem decodeGo_spec [Zero α] [PackedRepr α] (src : ByteArray) (si di cnt : USize) : + ∀ (m : Nat) (k : USize) (dst : Array α) (hk hs hd hu), cnt.toNat - k.toNat = m → + (decodeGo src si di cnt k dst hk hs hd hu).size = dst.size ∧ + ∀ i, (decodeGo src si di cnt k dst hk hs hd hu).getD i 0 = + if di.toNat + k.toNat ≤ i ∧ i < di.toNat + cnt.toNat then + (PackedRepr.get src (si.toNat + (i - di.toNat)) : α) + else dst.getD i 0 := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hsl := Nat.div_le_self src.size (PackedRepr.width α) + intro m + induction m with + | zero => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + rw [decodeGo] + have hnot : ¬k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + simp only [hnot, ↓reduceDIte, true_and] + intro i + rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + | succ m ih => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + have hlt : k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + have hcs := cnt.toNat_lt_size + have hds := di.toNat_lt_size + have ha : (si + k).toNat = si.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hb : (di + k).toNat = di.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hk1 : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by omega) + rw [decodeGo] + simp only [hlt, ↓reduceDIte] + obtain ⟨h1, h2⟩ := ih (k + 1) _ _ _ _ _ (by rw [hk1]; omega) + refine ⟨by rw [h1, Array.size_uset], fun i ↦ ?_⟩ + rw [h2 i, hk1, getD_uset, hb, PackedRepr.getAt_eq, ha] + split_ifs <;> first + | rfl + | omega + | (congr 2; omega) + +/-- A guarded range decode writes exactly its range. -/ +theorem decodeRange_getD [Zero α] [PackedRepr α] (src : ByteArray) (si : Nat) (dst : Array α) + (di cnt : Nat) + (h : si + cnt ≤ src.size / PackedRepr.width α ∧ di + cnt ≤ dst.size ∧ src.size < USize.size ∧ + dst.size < USize.size) (i : Nat) : + (decodeRange src si dst di cnt).getD i 0 = + if di ≤ i ∧ i < di + cnt then (PackedRepr.get src (si + (i - di)) : α) + else dst.getD i 0 := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hsl := Nat.div_le_self src.size (PackedRepr.width α) + unfold decodeRange + rw [dite_eq_left_of_eq_true (eq_true h)] + obtain ⟨_, h2⟩ := decodeGo_spec src (USize.ofNatLT si (by omega)) + (USize.ofNatLT di (by omega)) (USize.ofNatLT cnt (by omega)) _ 0 dst + (by simp only [USize.zero_le]) (by simp only [USize.toNat_ofNatLT]; omega) + (by simp only [USize.toNat_ofNatLT]; omega) ⟨h.2.2.1, h.2.2.2⟩ rfl + rw [h2 i] + simp only [USize.toNat_ofNatLT, USize.toNat_zero, Nat.add_zero] + +/-- A range decode keeps the destination size. -/ +@[simp] theorem size_decodeRange [Zero α] [PackedRepr α] (src : ByteArray) (si : Nat) + (dst : Array α) (di cnt : Nat) : (decodeRange src si dst di cnt).size = dst.size := by + unfold decodeRange + split + · exact (decodeGo_spec src _ _ _ _ 0 dst _ _ _ _ rfl).1 + · rfl + +/-- Row `l` of every stored column result, decoded. -/ +theorem gatherLeaf_spec [Zero α] [PackedRepr α] (cols : Array ByteArray) (cs l : Nat) + (hsrc : ∀ s < cols.size, + l * cs + cs ≤ (cols.getD s ByteArray.empty).size / PackedRepr.width α) + (hu : ∀ s < cols.size, (cols.getD s ByteArray.empty).size < USize.size) + (hr : cols.size * cs < USize.size) : + (gatherLeaf cols cs l : Array α).size = cols.size * cs ∧ + ∀ s < cols.size, ∀ c < cs, (gatherLeaf cols cs l : Array α).getD (s * cs + c) 0 = + (PackedRepr.get (cols.getD s ByteArray.empty) (l * cs + c) : α) := by + refine foldl_rows_spec (fun s acc ↦ decodeRange (cols.getD s ByteArray.empty) (l * cs) acc + (s * cs) cs) (fun s c ↦ (PackedRepr.get (cols.getD s ByteArray.empty) (l * cs + c) : α)) + cs cols.size (fun s hs acc hacc ↦ ?_) + have hcopy : s * cs + cs ≤ cols.size * cs := by + have := Nat.mul_le_mul_right cs (show s + 1 ≤ cols.size by omega) + rw [Nat.add_mul, Nat.one_mul] at this + exact this + refine ⟨by rw [size_decodeRange, hacc], fun i ↦ ?_⟩ + rw [decodeRange_getD _ _ _ _ _ ⟨by have := hsrc s hs; omega, by omega, hu s hs, by omega⟩ i] + +/-- The encoding loop keeps the size and stores the remaining range. -/ +theorem encodeGo_spec [Zero α] [PackedRepr α] (src : Array α) (off cnt : USize) : + ∀ (m : Nat) (k : USize) (dst : ByteArray) (hk hs hd hu), cnt.toNat - k.toNat = m → + (encodeGo src off cnt k dst hk hs hd hu).size = dst.size ∧ + ∀ j, (PackedRepr.get (encodeGo src off cnt k dst hk hs hd hu) j : α) = + if off.toNat + k.toNat ≤ j ∧ j < off.toNat + cnt.toNat then + src.getD (j - off.toNat) 0 + else PackedRepr.get dst j := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + intro m + induction m with + | zero => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + rw [encodeGo] + have hnot : ¬k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + simp only [hnot, ↓reduceDIte, true_and] + intro j + rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + | succ m ih => + intro k dst hk hs hd hu hm + have hk' := USize.le_iff_toNat_le.mp hk + have hlt : k < cnt := by rw [USize.lt_iff_toNat_lt]; omega + have hcs := cnt.toNat_lt_size + have hdl := Nat.div_le_self dst.size (PackedRepr.width α) + have ha : (off + k).toNat = off.toNat + k.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hk1 : (k + 1).toNat = k.toNat + 1 := Plan.usize_add_one k (by omega) + rw [encodeGo] + simp only [hlt, ↓reduceDIte] + obtain ⟨h1, h2⟩ := ih (k + 1) _ _ _ _ _ (by rw [hk1]; omega) + refine ⟨by rw [h1, PackedRepr.size_setAt], fun j ↦ ?_⟩ + rw [h2 j, hk1, PackedRepr.get_setAt, ha] + split_ifs <;> first + | rfl + | omega + | (rw [uget_eq_getD _ _ _ 0 _ rfl]; congr 2; omega) + +/-- An encoded array: `off` padding slots, then the entries. -/ +theorem encode_spec [Zero α] [PackedRepr α] (a : Array α) (off : Nat) + (hu : PackedRepr.width α * (off + a.size) < USize.size) : + (encode a off).size = PackedRepr.width α * (off + a.size) ∧ + ∀ i < a.size, (PackedRepr.get (encode a off) (off + i) : α) = a.getD i 0 := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hs : off + a.size < USize.size := + Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ PackedRepr.width_pos) hu + unfold encode + rw [dite_eq_left_of_eq_true (eq_true (by rwa [PackedRepr.size_alloc]))] + obtain ⟨h1, h2⟩ := encodeGo_spec a (USize.ofNatLT off (by omega)) + (USize.ofNatLT a.size (by omega)) + _ 0 (PackedRepr.alloc α (off + a.size)) (by simp only [USize.zero_le]) + (by simp only [USize.toNat_ofNatLT]; omega) + (by simp only [USize.toNat_ofNatLT, PackedRepr.size_alloc_div]; omega) + (by rwa [PackedRepr.size_alloc]) rfl + refine ⟨by rw [h1, PackedRepr.size_alloc], fun i hi ↦ ?_⟩ + rw [h2] + simp only [USize.toNat_ofNatLT, USize.toNat_zero, Nat.add_zero] + rw [ite_eq_left_of_eq_true _ _ (eq_true (by omega)), Nat.add_sub_cancel_left] + +/-- A guarded store changes exactly its index. -/ +theorem getD_setW (a : Array α) (i : USize) (v : α) (k : Nat) (d : α) (hu : a.size < USize.size) : + (setW a i v).getD k d = if i.toNat = k ∧ i.toNat < a.size then v else a.getD k d := by + have hus : a.usize.toNat = a.size := by + simp only [Array.usize, Nat.toUSize, USize.toNat_ofNat'] + exact Nat.mod_eq_of_lt hu + unfold setW + split + · rename_i h + have h' : i.toNat < a.size := by rw [← hus]; exact h + rw [getD_uset] + by_cases hk : i.toNat = k + · simp only [hk, true_and, ↓reduceIte, show k < a.size by omega] + · simp only [hk, false_and, ↓reduceIte] + · rename_i h + have h' : ¬i.toNat < a.size := by rw [← hus]; exact h + simp only [h', and_false, ↓reduceIte] + +@[simp] theorem size_setW (a : Array α) (i : USize) (v : α) : (setW a i v).size = a.size := by + unfold setW + split + · exact Array.size_uset .. + · rfl + +/-- Component `t` of a radix-four cell's outputs. -/ +def qsel (t : Nat) (o : R × R × R × R) : R := + match t with + | 0 => o.1 + | 1 => o.2.1 + | 2 => o.2.2.1 + | _ => o.2.2.2 + +open Packed in +/-- Output `t` at lane `o` of one block of radix-four cells. -/ +theorem inner_getD [Field R] (th tl : Array R) (q base : Nat) (a : Array R) + (hs : base + 4 * q ≤ a.size) (t o : Nat) (ht : t < 4) (ho : o < q) : + (Plan.butterflyDIFRadix4Inner th tl q 0 base (base + q) (base + 2 * q) (base + 3 * q) + a).getD (base + t * q + o) 0 = + qsel t (quadOutputs (th.getD o 0) (th.getD (o + q) 0) (tl.getD o 0) (a.getD (base + o) 0) + (a.getD (base + q + o) 0) (a.getD (base + 2 * q + o) 0) (a.getD (base + 3 * q + o) 0)) := by + rw [getD_butterflyDIFRadix4Inner _ _ _ _ _ _ _ _ _ (by omega) (by omega) (by omega) + (by omega)] + simp only [quadValue, quadAt, Nat.zero_add] + interval_cases t + · rw [ite_eq_left (by omega), show base + 0 * q + o - base = o by omega] + simp only [qsel, Nat.add_assoc] + · rw [ite_eq_right (by omega), ite_eq_left (by omega), + show base + 1 * q + o - (base + q) = o by omega] + simp only [qsel, Nat.add_assoc] + · rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_left (by omega), + show base + 2 * q + o - (base + 2 * q) = o by omega] + simp only [qsel, Nat.add_assoc] + · rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_right (by omega), + ite_eq_left (by omega), show base + 3 * q + o - (base + 3 * q) = o by omega] + simp only [qsel, Nat.add_assoc] + +open Packed in +/-- One block of radix-four cells leaves every entry outside the block unchanged. -/ +theorem inner_frame [Field R] (th tl : Array R) (q base : Nat) (a : Array R) + (hs : base + 4 * q ≤ a.size) (k : Nat) (hk : k < base ∨ base + 4 * q ≤ k) : + (Plan.butterflyDIFRadix4Inner th tl q 0 base (base + q) (base + 2 * q) (base + 3 * q) + a).getD k 0 = a.getD k 0 := by + rw [getD_butterflyDIFRadix4Inner _ _ _ _ _ _ _ _ _ (by omega) (by omega) (by omega) + (by omega)] + simp only [quadValue] + rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_right (by omega), + ite_eq_right (by omega)] + +open Packed in +/-- Coordinates of a run of radix-four blocks of `4 * q` entries: output `t` at lane `o` of +block `b ≥ block` is the cell of block `b`'s quarters; earlier blocks are unchanged. -/ +theorem blocks_getD [Field R] (th tl : Array R) (q blocks : Nat) : + ∀ (block : Nat) (a : Array R), a.size = blocks * (4 * q) → + (∀ b, block ≤ b → b < blocks → ∀ t < 4, ∀ o < q, + (Plan.butterflyDIFRadix4Blocks th tl (4 * q) q blocks block a).getD + (b * (4 * q) + t * q + o) 0 = + qsel t (quadOutputs (th.getD o 0) (th.getD (o + q) 0) (tl.getD o 0) + (a.getD (b * (4 * q) + o) 0) (a.getD (b * (4 * q) + q + o) 0) + (a.getD (b * (4 * q) + 2 * q + o) 0) (a.getD (b * (4 * q) + 3 * q + o) 0))) ∧ + ∀ k < block * (4 * q), + (Plan.butterflyDIFRadix4Blocks th tl (4 * q) q blocks block a).getD k 0 = a.getD k 0 := by + intro block + induction h : blocks - block generalizing block with + | zero => + intro a _ + rw [Plan.butterflyDIFRadix4Blocks, ite_eq_right (by omega)] + exact ⟨fun b hb hb' ↦ absurd hb' (by omega), fun _ _ ↦ rfl⟩ + | succ n ih => + intro a ha + have hlt : block < blocks := by omega + rw [Plan.butterflyDIFRadix4Blocks, ite_eq_left hlt] + have hb1 : (block + 1) * (4 * q) ≤ blocks * (4 * q) := Nat.mul_le_mul_right _ hlt + have hb2 : (block + 1) * (4 * q) = block * (4 * q) + 4 * q := Nat.succ_mul _ _ + have hsz : block * (4 * q) + 4 * q ≤ a.size := by omega + have hs' : (Plan.butterflyDIFRadix4Inner th tl q 0 (block * (4 * q)) + (block * (4 * q) + q) (block * (4 * q) + 2 * q) (block * (4 * q) + 3 * q) a).size = + blocks * (4 * q) := by rw [Plan.size_butterflyDIFRadix4Inner, ha] + obtain ⟨ihA, ihB⟩ := ih (block + 1) (by omega) _ hs' + have hframe := inner_frame th tl q (block * (4 * q)) a hsz + refine ⟨fun b hb hb' t ht o ho ↦ ?_, fun k hk ↦ ?_⟩ + · rcases Nat.eq_or_lt_of_le hb with rfl | hgt + · rw [ihB _ (by + have : t * q + o < 4 * q := by + have := Nat.mul_le_mul_right q (show t + 1 ≤ 4 by omega) + rw [Nat.succ_mul] at this; omega + omega)] + exact inner_getD th tl q _ a hsz t o ht ho + · have hge : (block + 1) * (4 * q) ≤ b * (4 * q) := Nat.mul_le_mul_right _ hgt + rw [ihA b hgt hb' t ht o ho] + rw [hframe _ (by omega), hframe _ (by omega), hframe _ (by omega), hframe _ (by omega)] + · rw [ihB k (by omega), hframe k (by omega)] + +open Packed in +/-- The first top pass in row/column coordinates: sixteen rows of length `L`, entry +`(4 u + v, col)` is output `u` of the cell on rows `v, v + 4, v + 8, v + 12`. -/ +theorem passA_getD [Field R] [DecidableEq R] (th tl Z : Array R) (L : Nat) + (hZ : Z.size = 16 * L) (u v col : Nat) (hu : u < 4) (hv : v < 4) (hc : col < L) : + (Plan.stageBlocksDIF th tl (4 * (4 * L)) (4 * L) 1 0 Z).getD ((4 * u + v) * L + col) 0 = + qsel u (quadOutputs (th.getD (v * L + col) 0) (th.getD ((v + 4) * L + col) 0) + (tl.getD (v * L + col) 0) (Z.getD (v * L + col) 0) (Z.getD ((v + 4) * L + col) 0) + (Z.getD ((v + 8) * L + col) 0) (Z.getD ((v + 12) * L + col) 0)) := by + rw [Plan.stageBlocksDIF_eq, Plan.checkedBlocks_eq] + have hv' : v * L + col < 4 * L := by + have := Nat.mul_le_mul_right L (show v + 1 ≤ 4 by omega) + rw [Nat.add_mul, Nat.one_mul] at this; omega + rw [show (4 * u + v) * L + col = 0 * (4 * (4 * L)) + u * (4 * L) + (v * L + col) by ring, + (blocks_getD th tl (4 * L) 1 0 Z (by omega)).1 0 le_rfl one_pos u hu _ hv'] + rw [show v * L + col + 4 * L = (v + 4) * L + col by ring, + show 0 * (4 * (4 * L)) + (v * L + col) = v * L + col by ring, + show 0 * (4 * (4 * L)) + 4 * L + (v * L + col) = (v + 4) * L + col by ring, + show 0 * (4 * (4 * L)) + 2 * (4 * L) + (v * L + col) = (v + 8) * L + col by ring, + show 0 * (4 * (4 * L)) + 3 * (4 * L) + (v * L + col) = (v + 12) * L + col by ring] + +open Packed in +/-- The second top pass in row/column coordinates: entry `(4 u + v, col)` is output `v` of +the cell on rows `4 u, …, 4 u + 3`. -/ +theorem passB_getD [Field R] [DecidableEq R] (th tl Z : Array R) (L : Nat) + (hZ : Z.size = 16 * L) (u v col : Nat) (hu : u < 4) (hv : v < 4) (hc : col < L) : + (Plan.stageBlocksDIF th tl (4 * L) L 4 0 Z).getD ((4 * u + v) * L + col) 0 = + qsel v (quadOutputs (th.getD (0 * L + col) 0) (th.getD (1 * L + col) 0) + (tl.getD (0 * L + col) 0) + (Z.getD (4 * u * L + col) 0) (Z.getD ((4 * u + 1) * L + col) 0) + (Z.getD ((4 * u + 2) * L + col) 0) (Z.getD ((4 * u + 3) * L + col) 0)) := by + rw [Plan.stageBlocksDIF_eq, Plan.checkedBlocks_eq] + rw [show (4 * u + v) * L + col = u * (4 * L) + v * L + col by ring, + (blocks_getD th tl L 4 0 Z (by omega)).1 u (Nat.zero_le _) hu v hv col hc] + simp only [Nat.zero_mul, Nat.zero_add, Nat.one_mul] + rw [show col + L = L + col by ring, + show u * (4 * L) + col = 4 * u * L + col by ring, + show u * (4 * L) + L + col = (4 * u + 1) * L + col by ring, + show u * (4 * L) + 2 * L + col = (4 * u + 2) * L + col by ring, + show u * (4 * L) + 3 * L + col = (4 * u + 3) * L + col by ring] + +/-- `l` holds rows `0, …, rows - 1` of the `M`-column matrix `g`, restricted to the columns +`j0, …, j0 + cs - 1`. -/ +def ColAgree [Zero α] (l g : Array α) (rows cs M j0 : Nat) : Prop := + ∀ r < rows, ∀ c < cs, l.getD (r * cs + c) 0 = g.getD (r * M + (j0 + c)) 0 + +/-- The first top pass commutes with restricting to a column range. -/ +theorem passA_col [Field R] [DecidableEq R] (thl tll Zl thg tlg Zg : Array R) (cs M j0 : Nat) + (hl : Zl.size = 16 * cs) (hg : Zg.size = 16 * M) (hj : j0 + cs ≤ M) + (hZ : ColAgree Zl Zg 16 cs M j0) (hth : ColAgree thl thg 8 cs M j0) + (htl : ColAgree tll tlg 4 cs M j0) : + ColAgree (Plan.stageBlocksDIF thl tll (4 * (4 * cs)) (4 * cs) 1 0 Zl) + (Plan.stageBlocksDIF thg tlg (4 * (4 * M)) (4 * M) 1 0 Zg) 16 cs M j0 := by + intro r hr c hc + obtain ⟨u, v, rfl, hu, hv⟩ : ∃ u v, r = 4 * u + v ∧ u < 4 ∧ v < 4 := + ⟨r / 4, r % 4, by omega, by omega, by omega⟩ + rw [passA_getD _ _ _ _ hl u v c hu hv hc, passA_getD _ _ _ _ hg u v (j0 + c) hu hv (by omega), + hZ v (by omega) c hc, hZ (v + 4) (by omega) c hc, hZ (v + 8) (by omega) c hc, + hZ (v + 12) (by omega) c hc, hth v (by omega) c hc, hth (v + 4) (by omega) c hc, + htl v (by omega) c hc] + +/-- The second top pass commutes with restricting to a column range. -/ +theorem passB_col [Field R] [DecidableEq R] (thl tll Zl thg tlg Zg : Array R) (cs M j0 : Nat) + (hl : Zl.size = 16 * cs) (hg : Zg.size = 16 * M) (hj : j0 + cs ≤ M) + (hZ : ColAgree Zl Zg 16 cs M j0) (hth : ColAgree thl thg 2 cs M j0) + (htl : ColAgree tll tlg 1 cs M j0) : + ColAgree (Plan.stageBlocksDIF thl tll (4 * cs) cs 4 0 Zl) + (Plan.stageBlocksDIF thg tlg (4 * M) M 4 0 Zg) 16 cs M j0 := by + intro r hr c hc + obtain ⟨u, v, rfl, hu, hv⟩ : ∃ u v, r = 4 * u + v ∧ u < 4 ∧ v < 4 := + ⟨r / 4, r % 4, by omega, by omega, by omega⟩ + rw [passB_getD _ _ _ _ hl u v c hu hv hc, passB_getD _ _ _ _ hg u v (j0 + c) hu hv (by omega), + hZ (4 * u) (by omega) c hc, hZ (4 * u + 1) (by omega) c hc, hZ (4 * u + 2) (by omega) c hc, + hZ (4 * u + 3) (by omega) c hc, hth 0 (by omega) c hc, hth 1 (by omega) c hc, + htl 0 (by omega) c hc] + +theorem size_stageBlocksDIF [Field R] [DecidableEq R] (th tl : Array R) + (blockSize quarter blocks block : Nat) (a : Array R) : + (Plan.stageBlocksDIF th tl blockSize quarter blocks block a).size = a.size := by + rw [Plan.stageBlocksDIF_eq, Plan.checkedBlocks_eq, Plan.size_butterflyDIFRadix4Blocks] + +/-- The top two passes of the whole transform, as a function of the input. -/ +def topPasses [Field R] [DecidableEq R] (tw : Array (Array R)) (logN : Nat) (x : Array R) : + Array R := + Parallel.step false tw (logN - 4) (Parallel.step false tw (logN - 2) x) + +/-- A column task computes its column range of the top two passes. -/ +theorem columnTask_col [Field R] [DecidableEq R] (x : Array R) (tw : Array (Array R)) + (logN cs j0 : Nat) (h4 : 4 ≤ logN) (hx : x.size = 2 ^ logN) + (htw : ∀ s < logN, (tw.getD s #[]).size = 2 ^ s) (hu : 2 ^ logN < USize.size) + (hj : j0 + cs ≤ 2 ^ (logN - 4)) : + (columnTask x tw logN (2 ^ (logN - 4)) cs j0).size = 16 * cs ∧ + ColAgree (columnTask x tw logN (2 ^ (logN - 4)) cs j0) (topPasses tw logN x) 16 cs + (2 ^ (logN - 4)) j0 := by + obtain ⟨m, rfl⟩ : ∃ m, logN = m + 4 := ⟨logN - 4, by omega⟩ + simp only [show m + 4 - 4 = m by omega] at hj ⊢ + have hp (k : Nat) : 2 ^ (m + k) = 2 ^ k * 2 ^ m := by rw [Nat.pow_add, Nat.mul_comm] + have hM := Nat.two_pow_pos m + have hx' : x.size = 16 * 2 ^ m := by rw [hx, hp 4]; rfl + have hcs : 16 * cs ≤ 16 * 2 ^ m := by omega + have hrow (s k : Nat) (hs : s < m + 4) (hk : 2 ^ s = k * 2 ^ m) : + ∀ r < k, r * 2 ^ m + j0 + cs ≤ (tw.getD s #[]).size := by + intro r hr + rw [htw s hs, hk] + have := Nat.mul_le_mul_right (2 ^ m) (show r + 1 ≤ k by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega + have hts (s : Nat) (hs : s < m + 4) : (tw.getD s #[]).size < USize.size := by + rw [htw s hs] + exact lt_of_le_of_lt (Nat.pow_le_pow_right (by omega) (by omega)) hu + have hu16 (k : Nat) (hk : k ≤ 16) : k * cs < USize.size := by + have : k * cs ≤ 16 * cs := Nat.mul_le_mul_right _ hk + rw [hx] at hx' + omega + have hg (s k : Nat) (hs : s < m + 4) (hk : 2 ^ s = k * 2 ^ m) (hk16 : k ≤ 16) := + gatherRows_spec (tw.getD s #[]) (2 ^ m) j0 cs k (hrow s k hs hk) ⟨hts s hs, hu16 k hk16⟩ + obtain ⟨hs1, hv1⟩ := hg (m + 4 - 1) 8 (by omega) + (by rw [show m + 4 - 1 = m + 3 by omega, hp]; rfl) (by omega) + obtain ⟨hs2, hv2⟩ := hg (m + 4 - 2) 4 (by omega) + (by rw [show m + 4 - 2 = m + 2 by omega, hp]; rfl) (by omega) + obtain ⟨hs3, hv3⟩ := hg (m + 4 - 3) 2 (by omega) + (by rw [show m + 4 - 3 = m + 1 by omega, hp]; rfl) (by omega) + obtain ⟨hs4, hv4⟩ := hg (m + 4 - 4) 1 (by omega) (by rw [show m + 4 - 4 = m by omega]; omega) + (by omega) + obtain ⟨hsx, hvx⟩ := gatherRows_spec x (2 ^ m) j0 cs 16 (fun r hr ↦ by + have := Nat.mul_le_mul_right (2 ^ m) (show r + 1 ≤ 16 by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega) ⟨by omega, hu16 16 le_rfl⟩ + have hA := passA_col _ _ _ _ _ _ cs (2 ^ m) j0 hsx hx' hj hvx hv1 hv2 + have hB := passB_col _ _ _ _ _ _ cs (2 ^ m) j0 (by rw [size_stageBlocksDIF, hsx]) + (by rw [size_stageBlocksDIF, hx']) hj hA hv3 hv4 + refine ⟨by simp only [columnTask, columnPasses, size_stageBlocksDIF, hsx], ?_⟩ + have e1 : m + 4 - 2 = m + 2 := by omega + have e2 : m + 4 - 4 = m := by omega + have hq1 : 2 ^ (m + 2) = 4 * 2 ^ m := by rw [hp]; rfl + have hd1 : x.size / (4 * 2 ^ (m + 2)) = 1 := by + rw [hx', hq1, show 4 * (4 * 2 ^ m) = 16 * 2 ^ m by ring]; exact Nat.div_self (by omega) + have hd2 : x.size / (4 * 2 ^ m) = 4 := by + rw [hx', show 16 * 2 ^ m = 4 * (4 * 2 ^ m) by ring] + exact Nat.mul_div_cancel _ (by omega) + unfold topPasses + rw [e1, e2] + unfold columnTask columnPasses Parallel.step + simp only [Bool.false_eq_true, ↓reduceIte, size_stageBlocksDIF] + rw [hd2, hd1, show m + 2 + 1 = m + 4 - 1 by omega, show m + 1 = m + 4 - 3 by omega, + show m + 4 - 2 = m + 2 by omega, show m + 4 - 4 = m by omega, hq1] + rw [show 16 * cs = 4 * (4 * cs) by ring] + simp only [show m + 4 - 2 = m + 2 by omega, show m + 4 - 4 = m by omega] at hB + exact hB + +/-- The optional scaling of a reversed leaf. -/ +def scaleBy [Mul α] (scale : Bool) (f v : α) : α := if scale then f * v else v + +/-- Reversal steps store bit-reversed, optionally scaled words from position `q` on. -/ +theorem reverseEncodeGo_spec [Zero α] [Mul α] [PackedRepr α] (a : Array α) (m : Nat) + (scale : Bool) (f : α) (hm : 0 < m) (h32 : m ≤ 32) (ha : a.size = 2 ^ m) (off M : USize) + (hM : M.toNat = 2 ^ m) : + ∀ (n : Nat) (q : USize) (out : ByteArray) (hd hu), M.toNat - q.toNat = n → + (reverseEncodeGo a (32 - m).toUInt32 scale f off M q out hd hu).size = out.size ∧ + ∀ k, (PackedRepr.get (reverseEncodeGo a (32 - m).toUInt32 scale f off M q out hd hu) k : + α) = + if off.toNat + q.toNat ≤ k ∧ k < off.toNat + 2 ^ m then + scaleBy scale f (a.getD (NTT.Transform.bitRevNat m (k - off.toNat)) 0) + else PackedRepr.get out k := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + intro n + induction n with + | zero => + intro q out hd hu hn + rw [reverseEncodeGo, dite_eq_right_iff.mpr (fun h ↦ absurd h (by + rw [USize.lt_iff_toNat_lt]; omega))] + exact ⟨rfl, fun k ↦ by rw [ite_eq_right (by omega)]⟩ + | succ n ih => + intro q out hd hu hn + have hq : q < M := by rw [USize.lt_iff_toNat_lt]; omega + have hMs := M.toNat_lt_size + have hq1 : (q + 1).toNat = q.toNat + 1 := Plan.usize_add_one q (by omega) + have hol := Nat.div_le_self out.size (PackedRepr.width α) + have ho : (off + q).toNat = off.toNat + q.toNat := by + rw [USize.toNat_add]; exact Nat.mod_eq_of_lt (by omega) + have hr : (reverse32 q.toUInt32 >>> (32 - m).toUInt32).toUSize.toNat = + NTT.Transform.bitRevNat m q.toNat := by + rw [UInt32.toNat_toUSize, show q.toUInt32 = q.toNat.toUInt32 from UInt32.toFin_inj.mp rfl] + exact reverse32_shift_eq_bitRevNat m q.toNat hm h32 + have hb : (reverse32 q.toUInt32 >>> (32 - m).toUInt32).toUSize.toNat < a.size := by + rw [hr, ha]; exact NTT.Transform.bitRevNat_lt _ _ + rw [reverseEncodeGo, dite_eq_left_of_eq_true (eq_true hq)] + dsimp only + rw [dite_eq_left_of_eq_true (eq_true hb)] + obtain ⟨hs', hv'⟩ := ih (q + 1) _ _ _ (by rw [hq1]; omega) + refine ⟨by rw [hs', PackedRepr.size_setAt], fun k ↦ ?_⟩ + rw [hv' k, hq1, PackedRepr.get_setAt, ho] + by_cases hk : k = off.toNat + q.toNat + · subst k + simp only [show ¬(off.toNat + (q.toNat + 1) ≤ off.toNat + q.toNat) by omega, false_and, + ↓reduceIte, Nat.le_refl, true_and, show off.toNat + q.toNat < off.toNat + 2 ^ m by omega, + Nat.add_sub_cancel_left, scaleBy] + rw [uget_eq_getD _ _ _ 0 _ hr] + · by_cases hk' : off.toNat + (q.toNat + 1) ≤ k ∧ k < off.toNat + 2 ^ m + · rw [ite_eq_left hk', ite_eq_left (by omega)] + · rw [ite_eq_right hk', ite_eq_right hk, ite_eq_right (by omega)] + +/-- A reversed leaf: element `off + k` is entry `bitrev k`, optionally scaled. -/ +theorem reverseEncode_spec [Zero α] [Mul α] [PackedRepr α] (a : Array α) (m : Nat) + (scale : Bool) (f : α) (off : Nat) (hm : 0 < m) (h32 : m ≤ 32) (ha : a.size = 2 ^ m) + (hu : PackedRepr.width α * (off + 2 ^ m) < USize.size) : + (reverseEncode m scale f a off).size = PackedRepr.width α * (off + 2 ^ m) ∧ ∀ k < 2 ^ m, + (PackedRepr.get (reverseEncode m scale f a off) (off + k) : α) = + scaleBy scale f (a.getD (NTT.Transform.bitRevNat m k) 0) := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have h2 := Nat.two_pow_pos m + have hs0 : off + 2 ^ m < USize.size := + Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ PackedRepr.width_pos) hu + unfold reverseEncode + rw [dite_eq_left_of_eq_true (eq_true (by rwa [PackedRepr.size_alloc]))] + obtain ⟨hs, hv⟩ := reverseEncodeGo_spec a m scale f hm h32 ha (USize.ofNatLT off (by omega)) + (USize.ofNatLT (2 ^ m) (by omega)) (by simp only [USize.toNat_ofNatLT]) _ 0 + (PackedRepr.alloc α (off + 2 ^ m)) + (by simp only [USize.toNat_ofNatLT, PackedRepr.size_alloc_div]; omega) + (by rwa [PackedRepr.size_alloc]) rfl + refine ⟨by rw [hs, PackedRepr.size_alloc], fun k hk ↦ ?_⟩ + rw [hv] + simp only [USize.toNat_ofNatLT, USize.toNat_zero, Nat.add_zero] + rw [ite_eq_left_of_eq_true _ _ (eq_true (by omega)), Nat.add_sub_cancel_left] + +theorem getD_set! (a : Array α) (i : Nat) (v : α) (k : Nat) (d : α) : + (a.set! i v).getD k d = if i = k ∧ i < a.size then v else a.getD k d := by + simp only [Array.set!, Array.getD_eq_getD_getElem?, Array.getElem?_setIfInBounds] + by_cases h : i = k + · subst h + by_cases hi : i < a.size + · simp only [hi, and_self, ↓reduceIte, Option.getD_some] + · simp only [hi, and_false, ↓reduceIte, Option.getD_none, + Array.getElem?_eq_none (Nat.le_of_not_lt hi)] + · simp only [h, false_and, ↓reduceIte] + +/-- Machine indices `16 q + c` and `o + q` of the assembly do not overflow. -/ +theorem toNat_rowIndex (q c : USize) (n : Nat) (hc : c.toNat < 16) (hq : 16 * q.toNat + 16 ≤ n) + (hn : n < USize.size) : (16 * q + c).toNat = 16 * q.toNat + c.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + simp only [USize.toNat_add, USize.toNat_mul, USize.reduceToNat, Nat.mod_add_mod, hsize] + exact Nat.mod_eq_of_lt (by omega) + +theorem getElem_eq_getD (r : Array β) (l : Nat) (h : l < r.size) (d : β) : r[l] = r.getD l d := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem h, Option.getD_some] + +set_option maxHeartbeats 1000000 in +/-- One assembly step stores the sixteen entries of row `q`. -/ +theorem assembleGo_spec [Zero α] [PackedRepr α] (r : Array ByteArray) (M : USize) + (hr : r.size = 16) + (hb : ∀ l (h : l < 16), 16 * l + M.toNat ≤ (r[l]'(by omega)).size / PackedRepr.width α ∧ + (r[l]'(by omega)).size < USize.size) (hM : 16 * M.toNat < USize.size) : + ∀ (n : Nat) (q : USize) (out : Array α), M.toNat - q.toNat = n → out.size = 16 * M.toNat → + (assembleGo setW r M hr hb q out).size = 16 * M.toNat ∧ + ∀ k, (assembleGo setW r M hr hb q out).getD k 0 = + if 16 * q.toNat ≤ k ∧ k < 16 * M.toNat then + (PackedRepr.get (r.getD (NTT.Transform.bitRevNat 4 (k % 16)) + ByteArray.empty) (16 * NTT.Transform.bitRevNat 4 (k % 16) + k / 16) : α) + else out.getD k 0 := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + intro n + induction n with + | zero => + intro q out hn hout + rw [assembleGo] + have hnot : ¬q < M := by rw [USize.lt_iff_toNat_lt]; omega + simp only [hnot, ↓reduceDIte] + exact ⟨hout, fun k ↦ by rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega))]⟩ + | succ n ih => + intro q out hn hout + have hq : q < M := by rw [USize.lt_iff_toNat_lt]; omega + have hq1 : (q + 1).toNat = q.toNat + 1 := Plan.usize_add_one q (by omega) + rw [assembleGo] + simp only [hq, ↓reduceDIte] + refine ⟨(ih (q + 1) _ (by rw [hq1]; omega) ?_).1, fun k ↦ ?_⟩ + · simp only [size_setW, hout] + rw [(ih (q + 1) _ (by rw [hq1]; omega) (by simp only [size_setW, hout])).2 k, hq1] + by_cases hk : 16 * (q.toNat + 1) ≤ k ∧ k < 16 * M.toNat + · rw [ite_eq_left_of_eq_true _ _ (eq_true hk), ite_eq_left_of_eq_true _ _ (eq_true (by omega))] + rw [ite_eq_right_of_eq_false _ _ (eq_false hk)] + have hrow (c : USize) (hc : c.toNat < 16) : (16 * q + c).toNat = 16 * q.toNat + c.toNat := + toNat_rowIndex q c (16 * M.toNat) hc (by omega) hM + have hrow0 : (16 * q).toNat = 16 * q.toNat := by + simpa only [USize.add_zero, USize.reduceToNat, Nat.add_zero] using + hrow 0 (by simp only [USize.reduceToNat]; omega) + simp only [PackedRepr.getAt_eq] + simp only [getD_setW, size_setW, hout, hM] + have hqM : q.toNat < M.toNat := hq + have hbig : 4294967296 ≤ USize.size := by + cases System.Platform.numBits_eq <;> simp_all [USize.size] + have hoq (o : USize) (ho : o.toNat ≤ 240) : (o + q).toNat = o.toNat + q.toNat := by + rw [USize.toNat_add, hsize, Nat.mod_eq_of_lt (by omega)] + simp only [hrow 1 (by simp only [USize.reduceToNat]; omega), + hrow 2 (by simp only [USize.reduceToNat]; omega), + hrow 3 (by simp only [USize.reduceToNat]; omega), + hrow 4 (by simp only [USize.reduceToNat]; omega), + hrow 5 (by simp only [USize.reduceToNat]; omega), + hrow 6 (by simp only [USize.reduceToNat]; omega), + hrow 7 (by simp only [USize.reduceToNat]; omega), + hrow 8 (by simp only [USize.reduceToNat]; omega), + hrow 9 (by simp only [USize.reduceToNat]; omega), + hrow 10 (by simp only [USize.reduceToNat]; omega), + hrow 11 (by simp only [USize.reduceToNat]; omega), + hrow 12 (by simp only [USize.reduceToNat]; omega), + hrow 13 (by simp only [USize.reduceToNat]; omega), + hrow 14 (by simp only [USize.reduceToNat]; omega), + hrow 15 (by simp only [USize.reduceToNat]; omega)] + simp only [hoq 0 (by simp only [USize.reduceToNat]; omega), + hoq 16 (by simp only [USize.reduceToNat]; omega), + hoq 32 (by simp only [USize.reduceToNat]; omega), + hoq 48 (by simp only [USize.reduceToNat]; omega), + hoq 64 (by simp only [USize.reduceToNat]; omega), + hoq 80 (by simp only [USize.reduceToNat]; omega), + hoq 96 (by simp only [USize.reduceToNat]; omega), + hoq 112 (by simp only [USize.reduceToNat]; omega), + hoq 128 (by simp only [USize.reduceToNat]; omega), + hoq 144 (by simp only [USize.reduceToNat]; omega), + hoq 160 (by simp only [USize.reduceToNat]; omega), + hoq 176 (by simp only [USize.reduceToNat]; omega), + hoq 192 (by simp only [USize.reduceToNat]; omega), + hoq 208 (by simp only [USize.reduceToNat]; omega), + hoq 224 (by simp only [USize.reduceToNat]; omega), + hoq 240 (by simp only [USize.reduceToNat]; omega)] + simp only [USize.reduceToNat, Nat.zero_add, hrow0, + getElem_eq_getD _ _ _ ByteArray.empty] + by_cases hr : 16 * q.toNat ≤ k ∧ k < 16 * q.toNat + 16 + · obtain ⟨c, rfl, hc⟩ : ∃ c, k = 16 * q.toNat + c ∧ c < 16 := + ⟨k - 16 * q.toNat, by omega, by omega⟩ + rw [ite_eq_left_of_eq_true _ _ + (eq_true (show 16 * q.toNat ≤ 16 * q.toNat + c ∧ 16 * q.toNat + c < 16 * M.toNat by + omega)), + show (16 * q.toNat + c) % 16 = c by omega, show (16 * q.toNat + c) / 16 = q.toNat by omega] + have hlt : ∀ i, i < 16 → (16 * q.toNat + i < 16 * M.toNat) = True := + fun i hi ↦ eq_true (by omega) + have hlt0 : (16 * q.toNat < 16 * M.toNat) = True := eq_true (by omega) + interval_cases c <;> simp (disch := decide) only [Nat.add_zero, hlt, hlt0, and_true, + Nat.reduceEqDiff, ↓reduceIte, Nat.add_eq_left, Nat.add_left_cancel_iff] + all_goals first | rfl | simp only [show NTT.Transform.bitRevNat 4 0 = 0 by decide, + Nat.mul_zero, Nat.zero_add] + · have hne : ∀ i, i < 16 → (16 * q.toNat + i = k) = False := fun i hi ↦ eq_false (by omega) + have hne0 : (16 * q.toNat = k) = False := eq_false (by omega) + simp (disch := decide) only [hne, hne0, false_and, ↓reduceIte] + rw [ite_eq_right_of_eq_false _ _ + (eq_false (show ¬(16 * q.toNat ≤ k ∧ k < 16 * M.toNat) by omega))] + +theorem getD_extract [Zero α] (a : Array α) (i j k : Nat) (hk : k < j - i) (hj : j ≤ a.size) : + (a.extract i j).getD k 0 = a.getD (i + k) 0 := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_extract, Nat.min_eq_left hj, + show k < j - i from hk, ↓reduceIte] + +/-- Later passes run independently on each aligned block of `M` entries. -/ +theorem runPasses_extract [Field R] [DecidableEq R] (tw : Array (Array R)) (rest : List Nat) + (Y : Array R) (M k l : Nat) (hY : Y.size = k * M) (hl : l < k) + (hrest : ∀ low ∈ rest, 4 * 2 ^ low ∣ M) : + Parallel.runPasses false tw rest (Y.extract (l * M) ((l + 1) * M)) = + (Parallel.runPasses false tw rest Y).extract (l * M) ((l + 1) * M) := by + have h1 : (l + 1) * M ≤ k * M := Nat.mul_le_mul_right _ hl + have h2 : l * M ≤ (l + 1) * M := Nat.mul_le_mul_right _ (Nat.le_succ l) + have hsplit : Y = Y.extract 0 (l * M) ++ + (Y.extract (l * M) ((l + 1) * M) ++ Y.extract ((l + 1) * M) Y.size) := by + rw [Array.extract_append_extract, Nat.min_eq_left (by omega), Nat.max_eq_right (by omega), + Array.extract_append_extract, Nat.min_eq_left (by omega), Nat.max_eq_right (by omega)] + exact (Array.extract_eq_self_of_le (Nat.le_refl _)).symm + have hd (i j : Nat) (hij : i ≤ j) (hj : j ≤ Y.size) (hi : M ∣ i) (hj' : M ∣ j) : + ∀ low ∈ rest, 4 * 2 ^ low ∣ (Y.extract i j).size := by + intro low hlow + rw [Array.size_extract_of_le hj] + exact Nat.dvd_trans (hrest low hlow) (Nat.dvd_sub hj' hi) + conv_rhs => rw [hsplit] + rw [Parallel.runPasses_append, Parallel.runPasses_append] + · rw [Array.extract_append, Array.extract_append] + simp only [Parallel.size_runPasses, Array.size_extract_of_le (show l * M ≤ Y.size by omega), + Array.size_extract_of_le (show (l + 1) * M ≤ Y.size by omega), Nat.sub_zero, Nat.sub_self] + rw [show (l + 1) * M - l * M = M by rw [Nat.succ_mul]; omega] + have hA : (Parallel.runPasses false tw rest (Y.extract 0 (l * M))).extract (l * M) + ((l + 1) * M) = #[] := Array.extract_eq_empty_of_le (by + simp only [Parallel.size_runPasses, Array.size_extract]; omega) + have hB : (Parallel.runPasses false tw rest (Y.extract (l * M) ((l + 1) * M))).extract 0 M = + Parallel.runPasses false tw rest (Y.extract (l * M) ((l + 1) * M)) := + Array.extract_eq_self_of_le (by + simp only [Parallel.size_runPasses, Array.size_extract]; rw [Nat.succ_mul]; omega) + rw [hA, hB, Nat.zero_sub, Array.extract_eq_empty_of_le + (as := Parallel.runPasses false tw rest (Y.extract ((l + 1) * M) Y.size)) (i := 0) (j := 0) + (Nat.min_le_left _ _), Array.empty_append, Array.append_empty] + · intro low hlow + exact ⟨hd _ _ h2 (by omega) (Nat.dvd_mul_left _ _) (Nat.dvd_mul_left _ _) low hlow, + hd _ _ (by omega) (le_refl _) (Nat.dvd_mul_left _ _) (by rw [hY]; exact Nat.dvd_mul_left _ _) + low hlow⟩ + · intro low hlow + refine ⟨hd _ _ (Nat.zero_le _) (by omega) (Nat.dvd_zero _) (Nat.dvd_mul_left _ _) low hlow, ?_⟩ + rw [Array.size_append] + exact Nat.dvd_add (hd _ _ h2 (by omega) (Nat.dvd_mul_left _ _) (Nat.dvd_mul_left _ _) low hlow) + (hd _ _ (by omega) (le_refl _) (Nat.dvd_mul_left _ _) (by rw [hY]; exact Nat.dvd_mul_left _ _) + low hlow) + +theorem getD_of_size_le [Zero α] (a : Array α) (k : Nat) (hk : a.size ≤ k) : a.getD k 0 = 0 := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_none hk, Option.getD_none] + +theorem getD_map_range (F : Nat → α) (n k : Nat) (hk : k < n) (d : α) : + ((Array.range n).map F).getD k d = F k := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_map, Array.getElem?_range, hk, + ↓reduceIte, Option.map_some, Option.getD_some] + + +/-! ### The radix-two layer of odd sizes -/ + +/-- The value of entry `k` after the pair butterflies of `a`. -/ +def pairValue [Field R] (tw0 : Array R) (a : Array R) (k : Nat) : R := + if k % 2 = 0 then a.getD k 0 + a.getD (k + 1) 0 + else tw0.getD 0 0 * (a.getD (k - 1) 0 - a.getD k 0) + +theorem butterflyDIFBlocks_pairs_getD [Field R] (tw0 : Array R) (a : Array R) : + ∀ (n block : Nat) (acc : Array R), n = (a.size / 2) - block → + acc.size = a.size → (∀ k, 2 * block ≤ k → acc.getD k 0 = a.getD k 0) → + ∀ k, (Plan.butterflyDIFBlocks tw0 2 1 (a.size / 2) block acc).getD k 0 = + if 2 * block ≤ k ∧ k < 2 * (a.size / 2) then pairValue tw0 a k else acc.getD k 0 := by + intro n + induction n with + | zero => + intro block acc hn _ _ k + rw [Plan.butterflyDIFBlocks] + simp only [show ¬block < a.size / 2 by omega, ↓reduceIte] + rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + | succ n ih => + intro block acc hn hs ha k + rw [Plan.butterflyDIFBlocks] + simp only [show block < a.size / 2 by omega, ↓reduceIte] + have hstep : Plan.butterflyDIFInner tw0 1 0 (block * 2) (block * 2 + 1) acc = + (acc.set! (block * 2) (acc.getD (block * 2) 0 + acc.getD (block * 2 + 1) 0)).set! + (block * 2 + 1) (tw0.getD 0 0 * (acc.getD (block * 2) 0 - + acc.getD (block * 2 + 1) 0)) := by + rw [Plan.butterflyDIFInner] + simp only [Nat.zero_lt_one, ↓reduceIte] + rw [Plan.butterflyDIFInner] + simp only [Nat.lt_irrefl, ↓reduceIte] + rw [hstep] + have hsz : ((acc.set! (block * 2) (acc.getD (block * 2) 0 + acc.getD (block * 2 + 1) 0)).set! + (block * 2 + 1) (tw0.getD 0 0 * (acc.getD (block * 2) 0 - + acc.getD (block * 2 + 1) 0))).size = a.size := by + simp only [Array.set!_eq_setIfInBounds, Array.size_setIfInBounds, hs] + rw [ih (block + 1) _ (by omega) hsz (fun j hj ↦ by + rw [getD_set!, getD_set!] + simp only [Array.set!_eq_setIfInBounds, Array.size_setIfInBounds] + rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + exact ha j (by omega)) k] + have h0 := ha (block * 2) (by omega) + have h1 := ha (block * 2 + 1) (by omega) + by_cases hk : 2 * (block + 1) ≤ k ∧ k < 2 * (a.size / 2) + · rw [ite_eq_left_of_eq_true _ _ (eq_true hk), ite_eq_left_of_eq_true _ _ (eq_true (by omega))] + rw [ite_eq_right_of_eq_false _ _ (eq_false hk), getD_set!, getD_set!] + simp only [Array.set!_eq_setIfInBounds, Array.size_setIfInBounds] + by_cases hk1 : k = block * 2 + 1 + · subst hk1 + rw [ite_eq_left_of_eq_true _ _ (eq_true ⟨rfl, by omega⟩), + ite_eq_left_of_eq_true _ _ (eq_true ⟨by omega, by omega⟩)] + simp only [pairValue, show (block * 2 + 1) % 2 = 1 by omega, Nat.one_ne_zero, ↓reduceIte, + Nat.add_sub_cancel, h0, h1] + rw [ite_eq_right_of_eq_false _ _ (eq_false (fun h ↦ hk1 h.1.symm))] + by_cases hk0 : k = block * 2 + · subst hk0 + rw [ite_eq_left_of_eq_true _ _ (eq_true ⟨rfl, by omega⟩), + ite_eq_left_of_eq_true _ _ (eq_true ⟨by omega, by omega⟩)] + simp only [pairValue, show block * 2 % 2 = 0 by omega, ↓reduceIte, h0, h1] + rw [ite_eq_right_of_eq_false _ _ (eq_false (fun h ↦ hk0 h.1.symm)), + ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + +theorem lastPair_getD [Field R] (tw0 a : Array R) (k : Nat) : + (Columns.lastPair tw0 a).getD k 0 = + if k < 2 * (a.size / 2) then pairValue tw0 a k else a.getD k 0 := by + rw [Columns.lastPair, butterflyDIFBlocks_pairs_getD tw0 a _ 0 a rfl rfl (fun _ _ ↦ rfl) k] + simp only [Nat.mul_zero, Nat.zero_le, true_and] + +@[simp] theorem size_lastPair [Field R] (tw0 a : Array R) : + (Columns.lastPair tw0 a).size = a.size := by + simp only [Columns.lastPair, Plan.size_butterflyDIFBlocks] + +/-- Pair butterflies on an even-aligned slice are the slice of the pair butterflies. -/ +theorem lastPair_extract [Field R] (tw0 Y : Array R) (i j : Nat) (hi : i % 2 = 0) + (hj : j % 2 = 0) (hij : i ≤ j) (hjY : j ≤ Y.size) : + Columns.lastPair tw0 (Y.extract i j) = (Columns.lastPair tw0 Y).extract i j := by + apply Packed.array_eq_of_getD _ _ 0 (by simp only [size_lastPair, Array.size_extract]) + intro k + by_cases hk : k < j - i + · rw [getD_extract _ _ _ _ hk (by rw [size_lastPair]; exact hjY), lastPair_getD, + lastPair_getD] + have hs : (Y.extract i j).size = j - i := by rw [Array.size_extract]; omega + rw [hs, ite_eq_left_of_eq_true _ _ (eq_true (by omega)), + ite_eq_left_of_eq_true _ _ (eq_true (by omega))] + unfold pairValue + rw [show (i + k) % 2 = k % 2 by omega] + split + · rw [getD_extract _ _ _ _ hk hjY, getD_extract _ _ _ _ (by omega) hjY] + rw [show i + (k + 1) = i + k + 1 by omega] + · rw [getD_extract _ _ _ _ hk hjY, getD_extract _ _ _ _ (by omega) hjY] + rw [show i + (k - 1) = i + k - 1 by omega] + · rw [getD_of_size_le _ _ (by simp only [size_lastPair, Array.size_extract]; omega), + getD_of_size_le _ _ (by simp only [Array.size_extract]; omega)] + +@[simp] theorem size_leafPasses [Field R] [DecidableEq R] (tw : Array (Array R)) + (rest : List Nat) (odd : Bool) (a : Array R) : + (leafPasses tw rest odd a).size = a.size := by + unfold leafPasses + split <;> simp only [size_lastPair, Parallel.size_runPasses] + +/-- The leaf layers of block `l` are block `l` of the leaf layers of the whole array. -/ +theorem leafPasses_extract [Field R] [DecidableEq R] (tw : Array (Array R)) (rest : List Nat) + (odd : Bool) (Y : Array R) (M k l : Nat) (hY : Y.size = k * M) (hl : l < k) + (hM : M % 2 = 0) (hrest : ∀ low ∈ rest, 4 * 2 ^ low ∣ M) : + leafPasses tw rest odd (Y.extract (l * M) ((l + 1) * M)) = + (leafPasses tw rest odd Y).extract (l * M) ((l + 1) * M) := by + unfold leafPasses + rw [runPasses_extract tw rest _ _ k l hY hl hrest] + split + · have h1 : (l + 1) * M ≤ k * M := Nat.mul_le_mul_right _ hl + exact lastPair_extract _ _ _ _ (by rw [Nat.mul_mod, hM, Nat.mul_zero, Nat.zero_mod]) + (by rw [Nat.mul_mod, hM, Nat.mul_zero, Nat.zero_mod]) (Nat.mul_le_mul_right _ (by omega)) + (by rw [Parallel.size_runPasses, hY]; exact h1) + · rfl + +/-- The column-parallel transform: the top two passes, the remaining passes on every block, and +the bit-reversal permutation, optionally scaled. -/ +theorem transform_eq [Field R] [DecidableEq R] [PackedRepr R] (tw : Array (Array R)) (logN S : Nat) + (rest : List Nat) (scale : Bool) (f : R) (x : Array R) (h4 : 4 < logN) (h36 : logN ≤ 36) + (hx : x.size = 2 ^ logN) (htw : ∀ s < logN, (tw.getD s #[]).size = 2 ^ s) + (hu : PackedRepr.width R * (2 ^ logN + 256) ≤ USize.size) (hS : 0 < S) + (hdiv : S ∣ 2 ^ (logN - 4)) + (hrest : ∀ low ∈ rest, 4 * 2 ^ low ∣ 2 ^ (logN - 4)) : + transform tw logN S rest scale f x = Array.ofFn (n := 2 ^ logN) fun i ↦ + scaleBy scale f ((leafPasses tw rest (logN % 2 == 1) (topPasses tw logN x)).getD + (NTT.Transform.bitRevNat logN i) 0) := by + obtain ⟨m, rfl⟩ : ∃ m, logN = m + 4 := ⟨logN - 4, by omega⟩ + have hm4 : m + 4 - 4 = m := by omega + rw [hm4] at hdiv hrest + have hp : 2 ^ (m + 4) = 16 * 2 ^ m := by rw [Nat.pow_add, Nat.mul_comm] + have hM := Nat.two_pow_pos m + obtain ⟨cs, hcs⟩ := hdiv + have hcs0 : 0 < cs := Nat.pos_of_ne_zero (by rintro rfl; omega) + have hcsM : 2 ^ m / S = cs := by rw [hcs, Nat.mul_div_cancel_left _ hS] + have hw := PackedRepr.width_pos (R := R) + have hu1 : 2 ^ (m + 4) + 256 ≤ USize.size := le_trans (Nat.le_mul_of_pos_left _ hw) hu + have hmono (t : Nat) (ht : t ≤ 2 ^ (m + 4) + 255) : PackedRepr.width R * t < USize.size := + Nat.lt_of_lt_of_le (Nat.mul_lt_mul_of_pos_left (by omega) hw) hu + have hu' : 2 ^ (m + 4) < USize.size := by omega + have hY : (topPasses tw (m + 4) x).size = 16 * 2 ^ m := by + rw [topPasses, Parallel.size_step, Parallel.size_step, hx, hp] + let cols := (Array.range S).map fun s ↦ encode (columnTask x tw (m + 4) (2 ^ m) cs (s * cs)) 0 + have hcolsz : cols.size = S := by simp only [cols, Array.size_map, Array.size_range] + have hcol (s : Nat) (hs : s < S) : + (columnTask x tw (m + 4) (2 ^ m) cs (s * cs)).size = 16 * cs ∧ + ColAgree (columnTask x tw (m + 4) (2 ^ m) cs (s * cs)) (topPasses tw (m + 4) x) 16 cs + (2 ^ m) (s * cs) := by + have := columnTask_col x tw (m + 4) cs (s * cs) (by omega) hx htw hu' (by + rw [hm4, hcs] + have := Nat.mul_le_mul_right cs (show s + 1 ≤ S by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega) + rwa [hm4] at this + have hcolsg (s : Nat) (hs : s < S) : + cols.getD s ByteArray.empty = encode (columnTask x tw (m + 4) (2 ^ m) cs (s * cs)) 0 := + getD_map_range _ S s hs _ + have hcs16 : 16 * cs ≤ 16 * 2 ^ m := by + have : cs ≤ 2 ^ m := by rw [hcs]; exact Nat.le_mul_of_pos_left _ hS + omega + have henc (s : Nat) (hs : s < S) := + encode_spec (columnTask x tw (m + 4) (2 ^ m) cs (s * cs)) 0 + (hmono _ (by rw [(hcol s hs).1]; omega)) + have hleafIn (l : Nat) (hl : l < 16) : + (gatherLeaf cols cs l : Array R) = + (topPasses tw (m + 4) x).extract (l * 2 ^ m) ((l + 1) * 2 ^ m) := by + have hlM : (l + 1) * 2 ^ m ≤ 16 * 2 ^ m := Nat.mul_le_mul_right _ (by omega) + have hlc : l * cs + cs ≤ 16 * cs := by + have := Nat.mul_le_mul_right cs (show l + 1 ≤ 16 by omega) + rw [Nat.add_mul, Nat.one_mul] at this; exact this + obtain ⟨hgs, hgv⟩ := gatherLeaf_spec (α := R) cols cs l + (fun s hs ↦ by + rw [hcolsz] at hs + rw [hcolsg s hs, (henc s hs).1, (hcol s hs).1, Nat.mul_div_cancel_left _ hw]; omega) + (fun s hs ↦ by + rw [hcolsz] at hs + rw [hcolsg s hs, (henc s hs).1, (hcol s hs).1] + exact hmono _ (by omega)) + (by rw [hcolsz, ← hcs]; rw [hp] at hu1; omega) + apply Packed.array_eq_of_getD _ _ 0 (by + rw [hgs, hcolsz, Array.size_extract_of_le (by omega), ← hcs, Nat.succ_mul]; omega) + intro k + by_cases hk : k < 2 ^ m + · have hkd : k = k / cs * cs + k % cs := (Nat.div_add_mod' k cs).symm + have hks : k / cs < S := by + rw [Nat.div_lt_iff_lt_mul hcs0, ← hcs]; exact hk + rw [getD_extract _ _ _ _ (by rw [Nat.succ_mul]; omega) (by omega), hkd, + hgv _ (by rw [hcolsz]; exact hks) _ (Nat.mod_lt _ hcs0), hcolsg _ hks] + have hlk : l * cs + k % cs < (columnTask x tw (m + 4) (2 ^ m) cs (k / cs * cs)).size := by + rw [(hcol _ hks).1]; have := Nat.mod_lt k hcs0; omega + have he := (henc _ hks).2 _ hlk + rw [Nat.zero_add] at he + rw [he] + exact (hcol _ hks).2 l hl (k % cs) (Nat.mod_lt _ hcs0) + · rw [getD_of_size_le _ _ (by rw [hgs, hcolsz, ← hcs]; omega), + getD_of_size_le _ _ (by rw [Array.size_extract_of_le (by omega), Nat.succ_mul]; omega)] + have hZ : (Parallel.runPasses false tw rest (topPasses tw (m + 4) x)).size = 16 * 2 ^ m := by + rw [Parallel.size_runPasses, hY] + let odd := (m + 4) % 2 == 1 + let L := fun l ↦ leafPasses tw rest odd (gatherLeaf cols cs l : Array R) + have hm2 : 2 ^ m % 2 = 0 := by + rw [show m = (m - 1) + 1 by omega, Nat.pow_succ, Nat.mul_mod_left] + have hleaf (l : Nat) (hl : l < 16) : + (L l).size = 2 ^ m ∧ ∀ i < 2 ^ m, (L l).getD i 0 = + (leafPasses tw rest odd (topPasses tw (m + 4) x)).getD (l * 2 ^ m + i) 0 := by + have hlM : (l + 1) * 2 ^ m ≤ 16 * 2 ^ m := Nat.mul_le_mul_right _ (by omega) + refine ⟨?_, fun i hi ↦ ?_⟩ + · simp only [L] + rw [size_leafPasses, hleafIn l hl, Array.size_extract_of_le (by omega), + Nat.succ_mul]; omega + · simp only [L] + rw [hleafIn l hl, leafPasses_extract tw rest _ _ _ 16 l hY hl hm2 hrest, + getD_extract _ _ _ _ (by rw [Nat.succ_mul]; omega) (by + rw [size_leafPasses, hY]; omega)] + have hT : (Tasks.spawnAfter (Tasks.spawnAll S fun s ↦ encode (columnTask x tw (m + 4) + (2 ^ (m + 4 - 4)) (2 ^ (m + 4 - 4) / S) (s * (2 ^ (m + 4 - 4) / S))) 0) 16 fun cols l ↦ + leafTask cols tw rest ((m + 4) % 2 == 1) (m + 4 - 4) (2 ^ (m + 4 - 4) / S) l scale + f).map Task.get = + (Array.range 16).map fun l ↦ leafTask cols tw rest odd m cs l scale f := by + rw [Tasks.map_get_spawnAfter, Tasks.map_get_spawnAll] + simp only [hm4, hcsM] + rfl + have hL (l : Nat) (hl : l < 16) := reverseEncode_spec (L l) m scale f (16 * l) (by omega) + (by omega) (hleaf l hl).1 (hmono _ (by omega)) + unfold transform + dsimp only + rw [hT, hm4] + let r := (Array.range 16).map fun l ↦ leafTask cols tw rest odd m cs l scale f + have hr16 : r.size = 16 := by simp only [r, Array.size_map, Array.size_range] + have hrg (l : Nat) (hl : l < 16) : + r.getD l ByteArray.empty = reverseEncode m scale f (L l) (16 * l) := + getD_map_range _ 16 l hl _ + have hMu : 2 ^ m < USize.size := by omega + have hall : (List.range 16).all (fun l ↦ decide (16 * l + 2 ^ m ≤ + (r.getD l ByteArray.empty).size / PackedRepr.width R ∧ + (r.getD l ByteArray.empty).size < USize.size)) = true := + List.all_eq_true.mpr fun l hl ↦ by + have hl' := List.mem_range.mp hl + rw [hrg l hl', (hL l hl').1] + exact decide_eq_true ⟨by rw [Nat.mul_div_cancel_left _ hw], hmono _ (by omega)⟩ + change assemble setW r (2 ^ m) (Array.replicate (16 * 2 ^ m) 0) = _ + unfold assemble + rw [dite_eq_left_of_eq_true (eq_true hMu), dite_eq_left_of_eq_true (eq_true hr16), + dite_eq_left_of_eq_true (eq_true hall)] + obtain ⟨has, hav⟩ := assembleGo_spec (α := R) r (USize.ofNatLT (2 ^ m) hMu) hr16 _ + (by simp only [USize.toNat_ofNatLT]; omega) _ 0 (Array.replicate (16 * 2 ^ m) 0) rfl + (by simp only [Array.size_replicate, USize.toNat_ofNatLT]) + simp only [USize.toNat_ofNatLT, USize.toNat_zero, Nat.mul_zero] at has hav + apply Packed.array_eq_of_getD _ _ 0 (by rw [has, Array.size_ofFn, hp]) + intro k + by_cases hk : k < 16 * 2 ^ m + · have hb := NTT.Transform.bitRevNat_lt 4 (k % 16) + rw [hav k, ite_eq_left_of_eq_true _ _ (eq_true ⟨Nat.zero_le _, hk⟩), + Packed.getD_ofFn_bounded _ _ (by rw [hp]; exact hk), hrg _ hb, + (hL _ hb).2 _ (by omega), + (hleaf _ hb).2 _ (NTT.Transform.bitRevNat_lt _ _)] + have hc := Packed.bitRevNat_concat m 4 (k / 16) (k % 16) (Nat.mod_lt _ (by decide)) + rw [show 2 ^ 4 * (k / 16) + k % 16 = k by omega] at hc + rw [hc, Nat.mul_comm (2 ^ m)] + · rw [getD_of_size_le _ _ (by rw [has]; omega), + getD_of_size_le _ _ (by rw [Array.size_ofFn, hp]; omega)] + +/-! ### Packed input and output -/ + +theorem setW_eq (a : Array α) (i : USize) (v : α) (hu : a.size < USize.size) : + setW a i v = a.setIfInBounds i.toNat v := by + have hus : a.usize.toNat = a.size := by + simp only [Array.usize, Nat.toUSize, USize.toNat_ofNat'] + exact Nat.mod_eq_of_lt hu + unfold setW + split + · rename_i h + have hi : i.toNat < a.size := by rw [← hus]; exact USize.lt_iff_toNat_lt.mp h + simp only [Array.uset, Array.setIfInBounds, hi, ↓reduceDIte] + · rename_i h + have hi : ¬ i.toNat < a.size := fun h' ↦ h (USize.lt_iff_toNat_lt.mpr (by rw [hus]; exact h')) + simp only [Array.setIfInBounds, hi, ↓reduceDIte] + +theorem setIf_encode_setW [PackedRepr α] (a : Array α) (i : USize) (v : α) + (hu : PackedRepr.width α * a.size < USize.size) : + PackedRepr.setIf _ hu (PackedRepr.encode a) i v = PackedRepr.encode (setW a i v) := by + rw [PackedRepr.setIf_encode _ _ _ hu, setW_eq _ _ _ + (Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ PackedRepr.width_pos) hu)] + +theorem setIf_encode_setW_of_eq [PackedRepr α] (a : Array α) (i : USize) (v : α) (m : Nat) + (hm : m < USize.size) (hu : PackedRepr.width α * a.size = m) : + PackedRepr.setIf m hm (PackedRepr.encode a) i v = PackedRepr.encode (setW a i v) := by + subst hu + exact setIf_encode_setW a i v hm + +/-- Two interleave runs whose stores preserve a relation on every index they store at keep +the relation. -/ +theorem assembleGo_rel [PackedRepr α] {β₁ β₂ : Type*} (set₁ : β₁ → USize → α → β₁) + (set₂ : β₂ → USize → α → β₂) (Rel : β₁ → β₂ → Prop) (r : Array ByteArray) (M : USize) + (hr : r.size = 16) (hb) (lo : Nat) (hM : 16 * M.toNat < USize.size) + (hstep : ∀ b₁ b₂ (i : USize) v, lo ≤ i.toNat → i.toNat < 16 * M.toNat → Rel b₁ b₂ → + Rel (set₁ b₁ i v) (set₂ b₂ i v)) : + ∀ (n : Nat) (q : USize) b₁ b₂, M.toNat - q.toNat = n → lo ≤ 16 * q.toNat → Rel b₁ b₂ → + Rel (assembleGo set₁ r M hr hb q b₁) (assembleGo set₂ r M hr hb q b₂) := by + intro n + induction n with + | zero => + intro q b₁ b₂ hn _ hR + have hnot : ¬q < M := by rw [USize.lt_iff_toNat_lt]; omega + rw [assembleGo, assembleGo] + simp only [hnot, ↓reduceDIte] + exact hR + | succ n ih => + intro q b₁ b₂ hn hlo hR + have hq : q < M := by rw [USize.lt_iff_toNat_lt]; omega + have hqM : q.toNat < M.toNat := hq + have hq1 : (q + 1).toNat = q.toNat + 1 := Plan.usize_add_one q (by omega) + have hrow (c : USize) (hc : c.toNat < 16) : (16 * q + c).toNat = 16 * q.toNat + c.toNat := + toNat_rowIndex q c (16 * M.toNat) hc (by omega) hM + have hrow0 : (16 * q).toNat = 16 * q.toNat := by + simpa only [USize.add_zero, USize.reduceToNat, Nat.add_zero] using + hrow 0 (by simp only [USize.reduceToNat]; omega) + rw [assembleGo, assembleGo] + simp only [hq, ↓reduceDIte] + apply ih (q + 1) _ _ (by rw [hq1]; omega) (by rw [hq1]; omega) + repeat' (refine hstep _ _ _ _ ?_ ?_ ?_) + all_goals first + | exact hR + | (rw [hrow0]; omega) + | (rw [hrow _ (by simp only [USize.reduceToNat]; omega)]; simp only [USize.reduceToNat] + omega) + +theorem setW_extract (F : Array α) (i d : USize) (e : Nat) (v : α) (hF : F.size = e) + (he : e < USize.size) (hd : d.toNat ≤ i.toNat) (hi : i.toNat < e) : + (setW F i v).extract d.toNat e = setW (F.extract d.toNat e) (i - d) v := by + have hs : (F.extract d.toNat e).size < USize.size := by + rw [Array.size_extract]; omega + rw [setW_eq _ _ _ (by omega), setW_eq _ _ _ hs, USize.toNat_sub_of_le _ _ + (USize.le_iff_toNat_le.mpr hd)] + apply Array.ext + · simp only [Array.size_extract, Array.size_setIfInBounds] + · intro j h1 h2 + simp only [Array.size_extract, Array.size_setIfInBounds] at h1 h2 + simp only [Array.getElem_extract, Array.getElem_setIfInBounds (show d.toNat + j < F.size by + omega), Array.getElem_setIfInBounds (show j < (F.extract d.toNat e).size by + rw [Array.size_extract]; omega)] + split <;> split <;> first | rfl | omega + +/-- Decoding a range of packed elements is copying the range of the elements. -/ +theorem decodeRange_encode [Zero α] [PackedRepr α] (xs : Array α) (si : Nat) (dst : Array α) + (di cnt : Nat) (hx : PackedRepr.width α * xs.size < USize.size) : + decodeRange (PackedRepr.encode xs) si dst di cnt = copyRange xs si dst di cnt := by + have hdiv : (PackedRepr.encode xs).size / PackedRepr.width α = xs.size := by + rw [PackedRepr.size_encode, Nat.mul_div_cancel_left _ PackedRepr.width_pos] + have hxs : xs.size < USize.size := + Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ PackedRepr.width_pos) hx + by_cases h : si + cnt ≤ xs.size ∧ di + cnt ≤ dst.size ∧ dst.size < USize.size + · apply Packed.array_eq_of_getD _ _ 0 (by rw [size_decodeRange, size_copyRange]) + intro i + rw [decodeRange_getD _ _ _ _ _ ⟨by rw [hdiv]; exact h.1, h.2.1, + by rw [PackedRepr.size_encode]; exact hx, h.2.2⟩, + copyRange_getD _ _ _ _ _ _ ⟨h.1, h.2.1, hxs, h.2.2⟩] + split + · rw [PackedRepr.get_encode _ _ (by omega), Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (by omega), Option.getD_some] + · rfl + · have h1 : decodeRange (PackedRepr.encode xs) si dst di cnt = dst := by + unfold decodeRange + rw [dite_eq_right_iff.mpr (fun hc ↦ absurd ⟨by rw [← hdiv]; exact hc.1, hc.2.1, + hc.2.2.2⟩ h)] + have h2 : copyRange xs si dst di cnt = dst := by + unfold copyRange + rw [dite_eq_right_iff.mpr (fun hc ↦ absurd ⟨hc.1, hc.2.1, hc.2.2.2⟩ h)] + rw [h1, h2] + +theorem gatherRowsPacked_encode [Zero α] [PackedRepr α] (xs : Array α) (M j0 cs r : Nat) + (hx : PackedRepr.width α * xs.size < USize.size) : + gatherRowsPacked (PackedRepr.encode xs) M j0 cs r = gatherRows xs M j0 cs r := by + simp only [gatherRowsPacked, gatherRows, decodeRange_encode _ _ _ _ _ hx] + +/-- `setW` at index `i - d`. -/ +def setWOff (d : USize) (a : Array α) (i : USize) (v : α) : Array α := setW a (i - d) v + +@[simp] theorem size_assembleGo_setW [PackedRepr α] (r : Array ByteArray) (M : USize) (hr hb) : + ∀ (n : Nat) (q : USize) (out : Array α), M.toNat - q.toNat = n → + (assembleGo setW r M hr hb q out).size = out.size := by + intro n + induction n with + | zero => + intro q out hn + have hnot : ¬q < M := by rw [USize.lt_iff_toNat_lt]; omega + rw [assembleGo] + simp only [hnot, ↓reduceDIte] + | succ n ih => + intro q out hn + have hq : q < M := by rw [USize.lt_iff_toNat_lt]; omega + have hq1 : (q + 1).toNat = q.toNat + 1 := Plan.usize_add_one q (by + have := M.toNat_lt_size; omega) + rw [assembleGo] + simp only [hq, ↓reduceDIte] + rw [ih (q + 1) _ (by rw [hq1]; omega)] + simp only [size_setW] + +theorem size_assemble_setW [PackedRepr α] (r : Array ByteArray) (M : Nat) (out : Array α) : + (assemble setW r M out).size = out.size := by + unfold assemble + split_ifs + · exact size_assembleGo_setW _ _ _ _ _ 0 _ rfl + all_goals rfl + +/-- A chunk run with offset packed stores is the packed slice of the global array run. -/ +theorem assembleGo_chunk [Zero α] [PackedRepr α] (r : Array ByteArray) (M : USize) (hr hb) + (q : USize) (q0 K : Nat) (hq0 : q.toNat = q0) (hK : M.toNat = q0 + K) + (hm : PackedRepr.width α * (16 * K) < USize.size) (hMU : 16 * M.toNat < USize.size) : + assembleGo (fun b i v ↦ PackedRepr.setIf (R := α) _ hm b (i - (16 * q0).toUSize) v) + r M hr hb q (PackedRepr.encode (Array.replicate (16 * K) (0 : α))) = + PackedRepr.encode ((assembleGo setW r M hr hb q + (Array.replicate (16 * M.toNat) (0 : α))).extract (16 * q0) (16 * M.toNat)) := by + have hd : ((16 * q0).toUSize : USize).toNat = 16 * q0 := + USize.toNat_ofNat_of_lt' (by omega) + have e1 := assembleGo_rel (fun b i v ↦ setWOff (16 * q0).toUSize b i v) + (fun b i v ↦ PackedRepr.setIf (R := α) _ hm b (i - (16 * q0).toUSize) v) + (fun (b₁ : Array α) b₂ ↦ b₂ = PackedRepr.encode b₁ ∧ + PackedRepr.width α * b₁.size = PackedRepr.width α * (16 * K)) + r M hr hb 0 hMU + (fun b₁ b₂ i v _ _ hR ↦ by + obtain ⟨rfl, hs⟩ := hR + refine ⟨?_, by simp only [setWOff, size_setW]; exact hs⟩ + exact setIf_encode_setW_of_eq _ _ _ _ _ hs) + _ q (Array.replicate (16 * K) 0) _ rfl (Nat.zero_le _) + ⟨rfl, by rw [Array.size_replicate]⟩ + rw [e1.1] + congr 1 + have e2 := assembleGo_rel setW (fun b i v ↦ setWOff (16 * q0).toUSize b i v) + (fun (F : Array α) S ↦ S = F.extract (16 * q0) (16 * M.toNat) ∧ F.size = 16 * M.toNat) + r M hr hb (16 * q0) hMU + (fun F S i v hlo hhi hR ↦ by + obtain ⟨rfl, hs⟩ := hR + refine ⟨?_, by simp only [size_setW]; exact hs⟩ + have := setW_extract F i (16 * q0).toUSize (16 * M.toNat) v hs hMU + (by rw [hd]; exact hlo) hhi + rw [hd] at this + exact this.symm) + _ q (Array.replicate (16 * M.toNat) 0) (Array.replicate (16 * K) 0) rfl (by rw [hq0]) + ⟨by rw [Array.extract_replicate, show min (16 * M.toNat) (16 * M.toNat) - 16 * q0 = 16 * K + by omega], by rw [Array.size_replicate]⟩ + exact e2.1 + +/-- One interleave chunk is the packed slice of the whole interleave. -/ +theorem assembleChunk_eq [Zero α] [PackedRepr α] (r : Array ByteArray) (M q0 K : Nat) + (hq : q0 + K ≤ M) (hm : PackedRepr.width α * (16 * M) < USize.size) : + assembleChunk (α := α) r M q0 K = PackedRepr.encode + ((assemble setW r M (Array.replicate (16 * M) (0 : α))).extract (16 * q0) + (16 * (q0 + K))) := by + have hw := PackedRepr.width_pos (R := α) + have hmK : PackedRepr.width α * (16 * K) < USize.size := + Nat.lt_of_le_of_lt (Nat.mul_le_mul_left _ (by omega)) hm + have hMU : 16 * M < USize.size := Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ hw) hm + have hrep : ((Array.replicate (16 * M) (0 : α)).extract (16 * q0) (16 * (q0 + K))) = + Array.replicate (16 * K) 0 := by + rw [Array.extract_replicate]; congr 1; omega + unfold assembleChunk + simp only [dite_eq_left_of_eq_true (eq_true hmK), PackedRepr.replicate_eq] + unfold assembleRows assemble + by_cases hg : M < USize.size ∧ r.size = 16 + · obtain ⟨hM, hr⟩ := hg + have hg' : q0 ≤ q0 + K ∧ q0 + K ≤ M ∧ M < USize.size ∧ r.size = 16 := + ⟨by omega, hq, hM, hr⟩ + rw [dite_eq_left_of_eq_true (eq_true hg'), dite_eq_left_of_eq_true (eq_true hM), + dite_eq_left_of_eq_true (eq_true hr)] + split + · rw [assembleGo_chunk r _ hr _ _ q0 K rfl rfl hmK (by simp only [USize.toNat_ofNatLT]; omega)] + congr 1 + simp only [USize.toNat_ofNatLT] + -- both runs store the same entries on the slice + apply Packed.array_eq_of_getD _ _ 0 (by + simp only [Array.size_extract, size_assembleGo_setW _ _ _ _ _ _ _ rfl, + Array.size_replicate]; omega) + intro j + by_cases hj : j < 16 * K + · rw [getD_extract _ _ _ _ (by omega) (by + rw [size_assembleGo_setW _ _ _ _ _ _ _ rfl, Array.size_replicate]), + getD_extract _ _ _ _ (by omega) (by + rw [size_assembleGo_setW _ _ _ _ _ _ _ rfl, Array.size_replicate]; omega), + (assembleGo_spec (α := α) r _ hr _ ?h1 _ _ _ rfl ?h2).2, + (assembleGo_spec (α := α) r _ hr _ ?h3 _ _ _ rfl ?h4).2] + · simp only [USize.toNat_zero, USize.toNat_ofNatLT, Nat.mul_zero] + rw [ite_eq_left_of_eq_true _ _ (eq_true ⟨by omega, by omega⟩), + ite_eq_left_of_eq_true _ _ (eq_true ⟨by omega, by omega⟩)] + all_goals simp only [USize.toNat_ofNatLT, Array.size_replicate] + all_goals omega + · rw [getD_of_size_le _ _ (by + simp only [Array.size_extract, size_assembleGo_setW _ _ _ _ _ _ _ rfl, + Array.size_replicate]; omega), + getD_of_size_le _ _ (by + simp only [Array.size_extract, size_assembleGo_setW _ _ _ _ _ _ _ rfl, + Array.size_replicate]; omega)] + · rw [hrep] + · rw [dite_eq_right_of_eq_false (eq_false (fun h ↦ hg ⟨h.2.2.1, h.2.2.2⟩))] + by_cases hM : M < USize.size + · rw [dite_eq_left_of_eq_true (eq_true hM), + dite_eq_right_of_eq_false (eq_false (fun h ↦ hg ⟨hM, h⟩)), hrep] + · rw [dite_eq_right_of_eq_false (eq_false hM), hrep] + +theorem foldl_chunks [PackedRepr α] (A : Array α) (K : Nat) (init : ByteArray) + (f : Nat → ByteArray) : ∀ n, (∀ c < n, f c = + PackedRepr.encode (A.extract (16 * (c * K)) (16 * (c * K + K)))) → + (List.range n).foldl (fun acc c ↦ acc ++ f c) init = + init ++ PackedRepr.encode (A.extract 0 (16 * (n * K))) := by + intro n + induction n with + | zero => + intro _ + simp only [List.range_zero, List.foldl_nil, Nat.zero_mul, Nat.mul_zero] + rw [Array.extract_eq_empty_of_le (by omega), PackedRepr.encode_empty, + ByteArray.append_empty] + | succ n ih => + intro hf + rw [List.range_succ, List.foldl_append, ih (fun c hc ↦ hf c (by omega)), List.foldl_cons, + List.foldl_nil, hf n (by omega), ByteArray.append_assoc, ← PackedRepr.encode_append, + Array.extract_append_extract, Nat.min_eq_left (Nat.zero_le _), + Nat.max_eq_right (by omega), show 16 * (n * K + K) = 16 * ((n + 1) * K) by ring] + + +/-- Joining the sixteen interleave chunks packs the whole interleave. -/ +theorem foldl_assembleChunk [Zero α] [PackedRepr α] (LS : Array ByteArray) (K : Nat) + (ho : PackedRepr.width α * (16 * (16 * K)) < USize.size) : + ((Array.range 16).map fun c ↦ assembleChunk (α := α) LS (16 * K) (c * (16 * K / 16)) + (16 * K / 16)).foldl (· ++ ·) (ByteArray.emptyWithCapacity + (PackedRepr.width α * (16 * (16 * K)))) = + PackedRepr.encode (assemble setW LS (16 * K) (Array.replicate (16 * (16 * K)) (0 : α))) := by + rw [show 16 * K / 16 = K by omega, Array.foldl_map, ← Array.foldl_toList, Array.toList_range, + foldl_chunks (assemble setW LS (16 * K) (Array.replicate (16 * (16 * K)) (0 : α))) + K _ _ 16 (fun c hc ↦ assembleChunk_eq _ _ _ _ (by + have := Nat.mul_le_mul_right K (show c + 1 ≤ 16 by omega) + rw [Nat.succ_mul] at this; exact this) ho)] + rw [show ByteArray.emptyWithCapacity (PackedRepr.width α * (16 * (16 * K))) = + ByteArray.empty from rfl, ByteArray.empty_append] + congr 1 + exact Array.extract_eq_self_of_le (by rw [size_assemble_setW, Array.size_replicate]) + +/-- The packed transform of packed elements is the packed array transform. -/ +theorem transformPacked_encode [Field R] [DecidableEq R] [PackedRepr R] (tw : Array (Array R)) + (logN S : Nat) (rest : List Nat) (scale : Bool) (f : R) (xs : Array R) + (hx : PackedRepr.width R * xs.size < USize.size) + (ho : PackedRepr.width R * (16 * 2 ^ (logN - 4)) < USize.size) + (h16 : 16 ∣ 2 ^ (logN - 4)) : + transformPacked tw logN S rest scale f (PackedRepr.encode xs) = + PackedRepr.encode (transform tw logN S rest scale f xs) := by + unfold transformPacked transform + simp only [gatherRowsPacked_encode _ _ _ _ _ hx, columnTask, Tasks.map_get_spawnAfter, + Tasks.map_get_spawnAll] + generalize 2 ^ (logN - 4) = M at ho h16 ⊢ + obtain ⟨K, rfl⟩ := h16 + exact foldl_assembleChunk _ K ho + +theorem decodeRange_all [Zero α] [PackedRepr α] (xs : Array α) + (hx : PackedRepr.width α * xs.size < USize.size) : + decodeRange (PackedRepr.encode xs) 0 + (Array.replicate ((PackedRepr.encode xs).size / PackedRepr.width α) 0) 0 + ((PackedRepr.encode xs).size / PackedRepr.width α) = xs := by + have hdiv : (PackedRepr.encode xs).size / PackedRepr.width α = xs.size := by + rw [PackedRepr.size_encode, Nat.mul_div_cancel_left _ PackedRepr.width_pos] + have hxs : xs.size < USize.size := + Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ PackedRepr.width_pos) hx + rw [hdiv, decodeRange_encode _ _ _ _ _ hx] + apply Packed.array_eq_of_getD _ _ 0 (by rw [size_copyRange, Array.size_replicate]) + intro i + rw [copyRange_getD xs 0 (Array.replicate xs.size 0) 0 xs.size 0 ⟨by omega, + by rw [Array.size_replicate]; omega, hxs, by rw [Array.size_replicate]; exact hxs⟩] + split + · simp only [Nat.zero_add, Nat.sub_zero] + · rename_i h + rw [getD_of_size_le _ _ (by rw [Array.size_replicate]; omega), + getD_of_size_le _ _ (by omega)] + +/-- Decoding all slots of packed elements. -/ +theorem decodeAll_encode [Zero α] [PackedRepr α] (xs : Array α) : + decodeAll (PackedRepr.encode xs) = xs := by + unfold decodeAll + split + · rename_i hx + rw [PackedRepr.size_encode] at hx + exact decodeRange_all xs hx + · exact PackedRepr.decode_encode xs + +theorem encodeAllGo_encode [Zero α] [PackedRepr α] (a : Array α) + (hu : PackedRepr.width α * a.size < USize.size) : + ∀ (n i : Nat) (B : Array α), B.size = a.size → i + n = a.size → + encodeAllGo a _ hu n i (PackedRepr.encode B) = + PackedRepr.encode (B.extract 0 i ++ a.extract i a.size) := by + intro n + induction n with + | zero => + intro i B hB hi + simp only [encodeAllGo, Nat.add_zero] at hi ⊢ + have e1 : a.extract a.size a.size = #[] := Array.extract_eq_empty_of_le (by omega) + have e2 : B.extract 0 a.size = B := by rw [← hB, Array.extract_size] + rw [hi, e1, e2, Array.append_empty] + | succ n ih => + intro i B hB hi + have hiu : i < USize.size := Nat.lt_of_lt_of_le (by omega) + (Nat.le_of_lt (Nat.lt_of_le_of_lt (Nat.le_mul_of_pos_left _ PackedRepr.width_pos) hu)) + rw [encodeAllGo, PackedRepr.setIf_encode_of_eq _ _ _ _ _ (by rw [hB]), + USize.toNat_ofNat_of_lt' hiu, + ih (i + 1) _ (by simp only [Array.size_setIfInBounds, hB]) (by omega)] + congr 1 + apply Array.ext + · simp only [Array.size_append, Array.size_extract, Array.size_setIfInBounds] + omega + · intro j h1 h2 + simp only [Array.getElem_append, Array.getElem_extract, Array.size_extract, + Array.size_setIfInBounds, Nat.zero_add, Nat.sub_zero] + split_ifs with ha hb hb + · rw [Array.getElem_setIfInBounds (by omega)] + split + · omega + · rfl + · rw [Array.getElem_setIfInBounds (by omega)] + split + · rw [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem (by omega), + Option.getD_some] + simp only [show i + (j - min i B.size) = i by omega] + · omega + · omega + · simp only [show i + 1 + (j - min (i + 1) B.size) = i + (j - min i B.size) by omega] + +/-- The fast encoder packs every element. -/ +theorem encodeAll_eq [Zero α] [PackedRepr α] (a : Array α) : + encodeAll a = PackedRepr.encode a := by + unfold encodeAll + split + · rename_i hu + rw [PackedRepr.replicate_eq, encodeAllGo_encode a hu _ 0 _ Array.size_replicate (by omega)] + simp only [Array.extract_zero, Array.empty_append, Array.extract_size] + · rfl + +end CompPoly.CPolynomial.NTTFast.Columns diff --git a/CompPoly/Univariate/NTTFast/Correctness/Radix4DIF.lean b/CompPoly/Univariate/NTTFast/Correctness/Radix4DIF.lean index a90fcd6f..9894fcce 100644 --- a/CompPoly/Univariate/NTTFast/Correctness/Radix4DIF.lean +++ b/CompPoly/Univariate/NTTFast/Correctness/Radix4DIF.lean @@ -719,6 +719,73 @@ private theorem runStagesDIFRadix4WithTwiddles_eq_difMathStageSpec · have hcompleted : 2 * (D.logN / 2) = D.logN := by omega simp [hodd, hcompleted] +/-- Radix-four passes from stage `top` down are the radix-two stages from `top` down. -/ +private theorem foldl_radix4_eq_stages (D : NTT.Domain R) (tw : Array (Array R)) (top : Nat) + (htop : top < D.logN) : + ∀ (P : Nat) (a : Array R), 2 * P ≤ top + 1 → a.size = D.n → + (List.range P).foldl (fun acc pass ↦ + butterflyRadix4StageDIFWithTwiddles D (top - 2 * pass - 1) (tw.getD (top - 2 * pass) #[]) + (tw.getD (top - 2 * pass - 1) #[]) acc) a = + (List.range (2 * P)).foldl (fun acc i ↦ + butterflyStageDIFWithTwiddles D (top - i) (tw.getD (top - i) #[]) acc) a := by + intro P + induction P with + | zero => intro a _ _; rfl + | succ P ih => + intro a hP ha + rw [List.range_succ, List.foldl_append, ih a (by omega) ha, + show 2 * (P + 1) = 2 * P + 1 + 1 by omega, List.range_succ, List.range_succ, + List.foldl_append, List.foldl_append] + simp only [List.foldl_cons, List.foldl_nil] + have hs : ((List.range (2 * P)).foldl (fun acc i ↦ + butterflyStageDIFWithTwiddles D (top - i) (tw.getD (top - i) #[]) acc) a).size = D.n := by + rw [← ha] + clear ih + induction (List.range (2 * P)) generalizing a with + | nil => rfl + | cons i l ihl => + rw [List.foldl_cons, ihl _ (by rw [size_butterflyStageDIFWithTwiddles]; exact ha), + size_butterflyStageDIFWithTwiddles] + rw [butterflyRadix4StageDIFWithTwiddles_eq_two_stages D _ _ _ _ hs (by omega), + show top - 2 * P - 1 + 1 = top - 2 * P by omega, show top - (2 * P + 1) = top - 2 * P - 1 by + omega] + +/-- An odd transform may run its radix-two stage first: the top stage, then radix-four passes +on the remaining stages, is the mixed radix-four loop. -/ +theorem runStagesDIFRadix4WithTwiddles_eq_top (D : NTT.Domain R) (tw : Array (Array R)) + (a : Array R) (ha : a.size = D.n) (hodd : D.logN % 2 = 1) : + (List.range (D.logN / 2)).foldl (fun acc pass ↦ + butterflyRadix4StageDIFWithTwiddles D (D.logN - 2 - 2 * pass - 1) + (tw.getD (D.logN - 2 - 2 * pass) #[]) (tw.getD (D.logN - 2 - 2 * pass - 1) #[]) acc) + (butterflyStageDIFWithTwiddles D (D.logN - 1) (tw.getD (D.logN - 1) #[]) a) = + runStagesDIFRadix4WithTwiddles D tw a := by + have hP : 2 * (D.logN / 2) + 1 = D.logN := by omega + have hchain (a : Array R) (ha : a.size = D.n) := foldl_radix4_eq_stages D tw (D.logN - 1) + (by omega) (D.logN / 2) a (by omega) ha + simp only [runStagesDIFRadix4WithTwiddles, Std.Legacy.Range.forIn_eq_forIn_range', + Std.Legacy.Range.size, Nat.sub_zero, Nat.add_sub_cancel, Nat.div_one, + ← List.range_eq_range', List.forIn_pure_yield_eq_foldl, pure_bind, Id.run_pure, + hodd, ↓reduceIte] + rw [show D.logN - 2 = D.logN - 1 - 1 by omega, + foldl_radix4_eq_stages D tw (D.logN - 1 - 1) (by omega) (D.logN / 2) _ (by omega) + (by rw [size_butterflyStageDIFWithTwiddles, ha])] + rw [hchain a ha] + let st : Nat → Array R → Array R := fun s acc ↦ + butterflyStageDIFWithTwiddles D s (tw.getD s #[]) acc + change (List.range (2 * (D.logN / 2))).foldl (fun acc i ↦ st (D.logN - 1 - 1 - i) acc) + (st (D.logN - 1) a) = + st 0 ((List.range (2 * (D.logN / 2))).foldl (fun acc i ↦ st (D.logN - 1 - i) acc) a) + have hl : (List.range (2 * (D.logN / 2))).foldl (fun acc i ↦ st (D.logN - 1 - 1 - i) acc) + (st (D.logN - 1) a) = + (List.range (2 * (D.logN / 2) + 1)).foldl (fun acc i ↦ st (D.logN - 1 - i) acc) a := by + rw [List.range_succ_eq_map, List.foldl_cons, List.foldl_map] + congr 1 + funext acc i + rw [show D.logN - 1 - (i + 1) = D.logN - 1 - 1 - i by omega] + rw [hl, List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil, + show D.logN - 1 - 2 * (D.logN / 2) = 0 by omega] + + /-- The mixed radix-4 DIF stage loop computes the bit-reversed forward NTT output. -/ theorem runStagesDIFRadix4WithTwiddles_correct (D : NTT.Domain R) (a : Array R) : runStagesDIFRadix4WithTwiddles D (twiddleTable D) (NTT.loadNaturalArray D a) = diff --git a/CompPoly/Univariate/NTTFast/Natural.lean b/CompPoly/Univariate/NTTFast/Natural.lean new file mode 100644 index 00000000..f64f7723 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Natural.lean @@ -0,0 +1,246 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all Init.Data.Array.Basic +public import CompPoly.Univariate.NTTFast.Correctness +public import CompPoly.Univariate.NTTFast.ButterflyDIT +public import CompPoly.Univariate.NTTFast.Permutation + +/-! # Reusable natural-order NTT plans + +Cache the permutation along with the twiddles: computing a reversed index for every +coefficient on every transform otherwise adds a second logarithmic-time traversal. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast + +/-- Specializable array builder: callbacks stay inside the worker's scalar loop. -/ +@[specialize] def tabulateGo {n : Nat} (f : Fin n → α) (acc : Array α) : + (i : Nat) → i ≤ n → Array α + | i + 1, h => + tabulateGo f (acc.push (f ⟨n - i - 1, by omega⟩)) i (by omega) + | 0, _ => acc + +/-- Construct an array without a generic callback in its element loop. -/ +@[inline, specialize] def tabulate {n : Nat} (f : Fin n → α) : Array α := + tabulateGo f (Array.emptyWithCapacity n) n (Nat.le_refl n) + +private theorem tabulateGo_eq {n : Nat} (f : Fin n → α) (i : Nat) (h : i ≤ n) + (acc : Array α) : tabulateGo f acc i h = Array.ofFn.go f acc i h := by + induction i generalizing acc with + | zero => rfl + | succ i ih => exact ih _ _ + +/-- The specialized builder has exactly the standard array semantics. -/ +theorem tabulate_eq {n : Nat} (f : Fin n → α) : tabulate f = Array.ofFn f := + tabulateGo_eq f n (Nat.le_refl n) _ + +/-- Map a same-carrier array with one write per coefficient and machine-word indices. -/ +@[specialize] def mapUGo (f : α → α) (limit i : USize) (a : Array α) + (hs : a.size = limit.toNat) : Array α := + if hi : i < limit then + have hin : i.toNat < a.size := by have h : i.toNat < limit.toNat := hi; omega + let b := a.uset i (f (a.uget i hin)) hin + have hb : b.size = limit.toNat := (Array.size_uset ..).trans hs + have hn : (i + 1).toNat = i.toNat + 1 := + Plan.usize_add_one i (by + have := limit.toNat_lt_size + have h : i.toNat < limit.toNat := hi + omega) + mapUGo f limit (i + 1) b hb + else a +termination_by limit.toNat - i.toNat +decreasing_by omega + +/-- Mapping changes coefficients but preserves the array length. -/ +@[simp] theorem size_mapUGo (f : α → α) (limit i : USize) (a : Array α) (hs) : + (mapUGo f limit i a hs).size = a.size := by + rw [mapUGo] + split + · rw [size_mapUGo] + exact Array.size_uset .. + · rfl +termination_by limit.toNat - i.toNat +decreasing_by + have _h : i.toNat < limit.toNat := ‹i < limit› + have _hn := Plan.usize_add_one i (by have := limit.toNat_lt_size; omega) + omega + +private theorem getD_uset (a : Array α) (i : USize) (v d : α) (hi) (k : Nat) : + (a.uset i v hi).getD k d = if i.toNat = k then v else a.getD k d := by + simp only [Array.uset_eq_set, Array.getD_eq_getD_getElem?, Array.getElem?_set] + split <;> rfl + +/-- The loop maps the untouched suffix and leaves the preceding coefficients alone. -/ +theorem getD_mapUGo (f : α → α) (limit i : USize) (a : Array α) (hs) + (k : Nat) (hk : k < a.size) (d : α) : + (mapUGo f limit i a hs).getD k d = + if i.toNat ≤ k then f (a.getD k d) else a.getD k d := by + rw [mapUGo] + split + · rename_i hi + have hin : i.toNat < limit.toNat := hi + have hia : i.toNat < a.size := by omega + have hn := Plan.usize_add_one i (by have := limit.toNat_lt_size; omega) + let b := a.uset i (f (a.uget i hia)) hia + have hbs : b.size = limit.toNat := (Array.size_uset ..).trans hs + have hkb : k < b.size := by simpa only [b, Array.size_uset] using hk + rw [getD_mapUGo f limit (i + 1) b hbs k hkb d, getD_uset] + by_cases he : i.toNat = k + · subst k + simp only [hn, Nat.not_succ_le_self, Nat.le_refl, ↓reduceIte, Array.uget] + rfl + · have hp : ((i + 1).toNat ≤ k) = (i.toNat ≤ k) := by apply propext; rw [hn]; omega + simp only [he, hp, ↓reduceIte] + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hk, Option.getD_some] + rfl + · rename_i hi + have hin : ¬ i.toNat < limit.toNat := hi + have hp : ¬ i.toNat ≤ k := by omega + simp only [hp, ↓reduceIte] +termination_by limit.toNat - i.toNat +decreasing_by + have _h : i.toNat < limit.toNat := ‹i < limit› + have _hn := Plan.usize_add_one i (by have := limit.toNat_lt_size; omega) + omega + +/-- The machine-index map has the ordinary array-map semantics from index zero. -/ +theorem mapUGo_zero (f : α → α) (limit : USize) (a : Array α) (hs) : + mapUGo f limit 0 a hs = a.map f := by + apply Array.ext + · simp only [size_mapUGo, Array.size_map] + · intro k hk hj + have hka : k < a.size := by simpa only [size_mapUGo] using hk + have h := getD_mapUGo f limit 0 a hs k hka a[k] + simpa only [USize.toNat_zero, Nat.zero_le, ↓reduceIte, + Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hk, + Array.getElem?_eq_getElem hka, Option.getD_some, Array.getElem_map] using h + +/-- A natural-order transform plan, including its cached output permutation. -/ +structure NaturalPlan (R : Type*) [Field R] where + plan : Plan R + order : Array Nat + order_eq : order = Array.ofFn (fun i : plan.domain.Idx ↦ + NTT.Transform.bitRevNat plan.domain.logN i.val) + wellFormed : Plan.WellFormed plan + +namespace NaturalPlan +variable {R : Type*} [Field R] + +/-- Prepare the twiddles and permutation once, outside repeated transforms. -/ +def ofDomain (D : NTT.Domain R) : NaturalPlan R := + ⟨Plan.ofDomain D, Array.ofFn (fun i : D.Idx ↦ NTT.Transform.bitRevNat D.logN i.val), rfl, + Plan.ofDomain_wellFormed D⟩ + +/-- The cached index table has one entry per domain element. -/ +@[simp] theorem order_size (P : NaturalPlan R) : P.order.size = P.plan.domain.n := by + simp only [P.order_eq, Array.size_ofFn] + +/-- Each cached entry is the corresponding reversed index. -/ +theorem order_get (P : NaturalPlan R) (i : Nat) (hi : i < P.plan.domain.n) : + P.order[i]! = NTT.Transform.bitRevNat P.plan.domain.logN i := by + have hio : i < P.order.size := by simpa only [order_size] using hi + rw [getElem!_pos P.order i hio] + simp only [P.order_eq, Array.getElem_ofFn] + +/-- Reuse a correctly sized array by swapping each reversed pair once. -/ +@[inline] def permute (P : NaturalPlan R) (a : Array R) : Array R := + if a.size = P.order.size then Array.permuteInvolution P.order a + else P.order.map (fun i ↦ a.getD i 0) + +/-- Pair swaps preserve the original gather, including padding and truncation cases. -/ +theorem permute_eq_map (P : NaturalPlan R) (a : Array R) : + P.permute a = P.order.map (fun i ↦ a.getD i 0) := by + unfold permute + split + · rename_i hs + apply Array.permuteInvolution_eq_map P.order a 0 hs.symm + · intro i hi + have hi' : i < P.plan.domain.n := by simpa only [order_size] using hi + rw [P.order_get i hi', order_size] + exact NTT.Transform.bitRevNat_lt _ _ + · intro i hi + have hi' : i < P.plan.domain.n := by simpa only [order_size] using hi + rw [P.order_get i hi'] + rw [P.order_get _ (NTT.Transform.bitRevNat_lt _ _)] + exact NTT.Transform.bitRevNat_involutive _ _ hi' + · rfl + +/-- Cached indexing computes the same permutation as the mathematical definition. -/ +theorem permute_eq (P : NaturalPlan R) (a : Array R) : + P.permute a = NTT.Transform.bitRevPermute P.plan.domain a := by + simp only [permute_eq_map, P.order_eq, Array.map_ofFn, NTT.Transform.bitRevPermute, + Function.comp_def] + +/-- Reuse an input of the right size; otherwise pad or truncate it as before. -/ +@[inline] def load (P : NaturalPlan R) (a : Array R) : Array R := + if a.size = P.plan.domain.n then a + else tabulate (fun i : P.plan.domain.Idx ↦ a.getD i.val 0) + +/-- Reusing a correctly sized array preserves the total loading semantics. -/ +theorem load_eq (P : NaturalPlan R) (a : Array R) : + P.load a = NTT.loadNaturalArray P.plan.domain a := by + unfold load + split + · rename_i h + apply Array.ext + · simp only [NTT.size_loadNaturalArray, h] + · intro i hi hj + simp only [NTT.getElem_loadNaturalArray, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem hi, Option.getD_some] + · exact tabulate_eq _ + +/-- Specialize normalization so each scalar multiplication stays inside its loop. -/ +@[inline] def normalize (P : NaturalPlan R) (a : Array R) : Array R := + if hs : a.size = P.plan.domain.n then + if hu : a.size < USize.size then + mapUGo (fun x ↦ P.plan.nInv * x) (USize.ofNatLT a.size hu) 0 a + (by simp only [USize.toNat_ofNatLT]) + else tabulate (fun i : P.plan.domain.Idx ↦ P.plan.nInv * a.getD i.val 0) + else tabulate (fun i : P.plan.domain.Idx ↦ P.plan.nInv * a.getD i.val 0) + +/-- Specialized normalization agrees with the existing plan. -/ +theorem normalize_eq (P : NaturalPlan R) (a : Array R) : + P.normalize a = P.plan.normalize a := by + unfold normalize + split + · rename_i hs + split + · rw [mapUGo_zero] + apply Array.ext + · simp only [Array.size_map, Plan.normalize, Array.size_ofFn, hs] + · intro i hi hj + simp only [Plan.normalize, Array.getElem_map, Array.getElem_ofFn, + Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (by simpa only [Array.size_map] using hi), + Option.getD_some] + · exact tabulate_eq _ + · exact tabulate_eq _ + +/-- Forward NTT with natural-order input and output. -/ +@[inline] def forward [DecidableEq R] (P : NaturalPlan R) (a : Array R) : Array R := + P.permute (Plan.checkedStages P.plan.domain P.plan.twiddles (P.load a)) + +/-- Inverse NTT with natural-order input and output. -/ +@[inline] def inverse [DecidableEq R] (P : NaturalPlan R) (a : Array R) : Array R := + P.normalize (Plan.checkedDITStages P.plan.inverseDomain P.plan.inverseTwiddles (P.permute a)) + +/-- The cached forward path equals the existing natural-order pipeline. -/ +theorem forward_eq [DecidableEq R] (P : NaturalPlan R) (a : Array R) : + P.forward a = NTT.Transform.bitRevPermute P.plan.domain (P.plan.forwardImpl a) := by + rw [forward, Plan.checkedStages_eq, permute_eq, load_eq] + rfl + +/-- The cached inverse path equals the existing natural-order pipeline. -/ +theorem inverse_eq [DecidableEq R] (P : NaturalPlan R) (a : Array R) : + P.inverse a = P.plan.inverseImpl (NTT.Transform.bitRevPermute P.plan.domain a) := by + rw [inverse, normalize_eq, Plan.checkedDITStages_eq, permute_eq] + rfl + +end NaturalPlan +end CompPoly.CPolynomial.NTTFast diff --git a/CompPoly/Univariate/NTTFast/NaturalParallel.lean b/CompPoly/Univariate/NTTFast/NaturalParallel.lean new file mode 100644 index 00000000..b8517d76 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/NaturalParallel.lean @@ -0,0 +1,245 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Natural +public import CompPoly.Univariate.NTTFast.Columns +public import CompPoly.Univariate.NTTFast.ColumnsCorrectness +import CompPoly.Univariate.NTTFast.Correctness.Pipeline + +/-! # Parallel natural-order transforms on field arrays + +Even sizes with enough columns use the column-parallel transform of `Columns`; the other +large sizes run the passes on independent array segments. Both paths are proved equal to the +scalar natural-order transforms. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.NaturalPlan +variable {R : Type*} + +/-- The column-parallel transform applies to an exactly sized input of at least `2 ^ 15` +elements; smaller transforms run as one leaf, without tasks or copies. -/ +def ColumnsShape (R : Type*) [PackedRepr R] (n logN : Nat) : Prop := + n = 2 ^ logN ∧ 15 ≤ logN ∧ logN ≤ 36 ∧ PackedRepr.width R * (2 ^ logN + 256) ≤ USize.size + +instance (R : Type*) [PackedRepr R] (n logN : Nat) : Decidable (ColumnsShape R n logN) := by + unfold ColumnsShape; exact inferInstance + +/-- A domain's twiddle table holds `2 ^ s` entries at stage `s`. -/ +theorem size_twiddleTable_getD [Field R] (D : NTT.Domain R) (s : Nat) (hs : s < D.logN) : + ((Plan.twiddleTable D).getD s #[]).size = 2 ^ s := by + rw [Plan.twiddleTable_getD_eq_twiddlePowers D s hs, Plan.twiddlePowers_size] + +/-- The first two forward passes are the column transform's top passes. -/ +theorem runPasses_topPasses [Field R] [DecidableEq R] (D : NTT.Domain R) (tw : Array (Array R)) + (a : Array R) (h4 : 4 ≤ D.logN) : + Parallel.runPasses false tw ((Parallel.forwardPasses D).drop 2) + (Columns.topPasses tw D.logN a) = + Parallel.runPasses false tw (Parallel.forwardPasses D) a := by + have htake : (Parallel.forwardPasses D).take 2 = [D.logN - 2, D.logN - 4] := by + rw [Parallel.forwardPasses, ← List.map_take, List.take_range, Nat.min_eq_left (by omega)] + simp only [List.range_succ, List.range_zero, List.nil_append, List.cons_append, + List.map_cons, List.map_nil, List.cons.injEq, and_true] + omega + conv_rhs => rw [← List.take_append_drop 2 (Parallel.forwardPasses D)] + rw [htake] + rfl + +/-- The pair butterflies of an odd transform are its last radix-two stage. -/ +theorem lastPair_eq [Field R] (D : NTT.Domain R) (tw0 a : Array R) (ha : a.size = D.n) : + Columns.lastPair tw0 a = Plan.butterflyStageDIFWithTwiddles D 0 tw0 a := by + simp only [Columns.lastPair, Plan.butterflyStageDIFWithTwiddles, ha, Nat.zero_add, + Nat.pow_one, Nat.pow_zero] + +/-- The leaf layers after the top passes complete the scalar forward stages. -/ +theorem leafPasses_topPasses [Field R] [DecidableEq R] (D : NTT.Domain R) (tw : Array (Array R)) + (a : Array R) (ha : a.size = D.n) (h4 : 4 ≤ D.logN) : + Columns.leafPasses tw ((Parallel.forwardPasses D).drop 2) (D.logN % 2 == 1) + (Columns.topPasses tw D.logN a) = Plan.checkedStages D tw a := by + rw [← Parallel.forwardStages_eq D tw a ha] + unfold Columns.leafPasses + rw [runPasses_topPasses D tw a h4] + by_cases ho : D.logN % 2 = 1 + · simp only [ho, beq_self_eq_true, ↓reduceIte] + rw [lastPair_eq D _ _ (by rw [Parallel.size_runPasses, ha])] + · simp only [ho, beq_iff_eq, ↓reduceIte] + +/-- Every leaf pass after the top two divides a `2 ^ (logN - 4)`-entry leaf into whole blocks. -/ +theorem dvd_of_mem_forwardPasses_drop [Field R] (D : NTT.Domain R) (low : Nat) + (hlow : low ∈ (Parallel.forwardPasses D).drop 2) : 4 * 2 ^ low ∣ 2 ^ (D.logN - 4) := by + simp only [Parallel.forwardPasses, List.mem_drop_iff_getElem, List.length_map, + List.length_range, List.getElem_map, List.getElem_range] at hlow + obtain ⟨i, hi, rfl⟩ := hlow + rw [show 4 * 2 ^ (D.logN - 1 - 2 * (2 + i) - 1) = 2 ^ (D.logN - 1 - 2 * (2 + i) - 1 + 2) by + rw [Nat.pow_add]; ring] + exact Nat.pow_dvd_pow 2 (by omega) + +/-- The column transform computes the scalar forward stages of a domain, in natural order. -/ +theorem transform_eq_checkedStages [Field R] [DecidableEq R] [PackedRepr R] (D : NTT.Domain R) + (tw : Array (Array R)) (S : Nat) (scale : Bool) (f : R) (a : Array R) + (htw : ∀ s < D.logN, (tw.getD s #[]).size = 2 ^ s) (hS : 0 < S) + (hdiv : S ∣ 2 ^ (D.logN - 4)) (hshape : ColumnsShape R a.size D.logN) : + Columns.transform tw D.logN S ((Parallel.forwardPasses D).drop 2) scale f a = + Array.ofFn (n := 2 ^ D.logN) fun i ↦ Columns.scaleBy scale f + ((Plan.checkedStages D tw a).getD (NTT.Transform.bitRevNat D.logN i) 0) := by + obtain ⟨hn, h15, h36, hu⟩ := hshape + rw [Columns.transform_eq tw D.logN S _ scale f a (by omega) h36 hn htw hu hS hdiv + (dvd_of_mem_forwardPasses_drop D), leafPasses_topPasses D tw a hn (by omega)] + +/-- Forward NTT: if `parallel`, from `2 ^ 15` elements, `columnCount` column tasks run the top two +radix-four passes and sixteen leaf tasks the remaining layers; otherwise, and for smaller +transforms, the sequential transform runs as one leaf. -/ +@[inline] def forwardParallel [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : Array R) (parallel : Bool := true) : Array R := + if parallel ∧ ColumnsShape R a.size P.plan.domain.logN then + Columns.transform P.plan.twiddles P.plan.domain.logN + (Columns.columnCount P.plan.domain.logN) + ((Parallel.forwardPasses P.plan.domain).drop 2) false 1 a + else P.forward a + +/-- Parallel inverse NTT with the same shapes; the column transform runs with the inverse +twiddles and scales the leaves. -/ +@[inline] def inverseParallel [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : Array R) (parallel : Bool := true) : Array R := + if parallel ∧ P.plan.inverseDomain.n = P.plan.domain.n ∧ + ColumnsShape R a.size P.plan.domain.logN then + Columns.transform P.plan.inverseTwiddles P.plan.domain.logN + (Columns.columnCount P.plan.domain.logN) + ((Parallel.forwardPasses P.plan.inverseDomain).drop 2) true P.plan.nInv a + else P.inverse a + +/-- The complete parallel forward transform refines the proved scalar transform. -/ +theorem forwardParallel_eq [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : Array R) (parallel : Bool) : P.forwardParallel a parallel = P.forward a := by + unfold forwardParallel + split + · rename_i h + obtain ⟨_, hc⟩ := h + have htw : ∀ s < P.plan.domain.logN, (P.plan.twiddles.getD s #[]).size = 2 ^ s := by + rw [P.wellFormed.2.2.1]; exact size_twiddleTable_getD _ + have hn := hc.1 + rw [transform_eq_checkedStages _ _ (Columns.columnCount P.plan.domain.logN) false 1 a + htw (by unfold Columns.columnCount; exact Nat.two_pow_pos _) + (by unfold Columns.columnCount; exact Nat.pow_dvd_pow 2 (by have := hc.2.1; omega)) hc, + forward, permute_eq, load, + ite_eq_left_of_eq_true _ _ (eq_true (show a.size = P.plan.domain.n from hn))] + rfl + · rfl + +/-- The complete parallel inverse transform refines the proved scalar transform. -/ +theorem inverseParallel_eq [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : Array R) (parallel : Bool) : P.inverseParallel a parallel = P.inverse a := by + unfold inverseParallel + split + · rename_i h + obtain ⟨_, hn, hc⟩ := h + obtain ⟨hs, h15, h36, hu⟩ := hc + obtain ⟨hinv, hnInv, _, hitw⟩ := P.wellFormed + have hl : P.plan.domain.inverse.logN = P.plan.domain.logN := rfl + have htw : ∀ s < P.plan.domain.inverse.logN, + (P.plan.inverseTwiddles.getD s #[]).size = 2 ^ s := by + rw [hitw]; exact size_twiddleTable_getD _ + have hload : NTT.loadNaturalArray P.plan.domain.inverse a = a := by + apply Array.ext + · rw [NTT.size_loadNaturalArray, hs]; rfl + · intro i hi _ + rw [NTT.getElem_loadNaturalArray, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (by assumption), Option.getD_some] + have key := transform_eq_checkedStages P.plan.domain.inverse P.plan.inverseTwiddles + (Columns.columnCount P.plan.domain.logN) true P.plan.nInv a htw + (by unfold Columns.columnCount; exact Nat.two_pow_pos _) + (by unfold Columns.columnCount; rw [hl]; exact Nat.pow_dvd_pow 2 (by omega)) + ⟨by rw [hl]; exact hs, by omega, by omega, by rw [hl]; exact hu⟩ + rw [hinv] + refine key.trans ?_ + rw [inverse_eq, Plan.inverseImpl_correct _ P.wellFormed, Plan.checkedStages_eq, hitw] + conv_lhs => rw [← hload] + rw [Plan.runStagesDIFRadix4WithTwiddles_correct] + apply Array.ext + · simp only [Array.size_ofFn, NTT.Inverse.size_inverseSpec]; rfl + · intro i hi _ + simp only [Array.getElem_ofFn, Columns.scaleBy, ↓reduceIte, NTT.Inverse.inverseSpec, + NTT.Transform.bitRevPermute, hnInv] + rw [Array.getD_eq_getD_getElem?, Array.getElem?_ofFn] + simp only [NTT.Domain.n, NTT.Transform.bitRevNat_lt, ↓reduceDIte, Option.getD_some] + simp only [Array.size_ofFn] at hi + have hi' : i < 2 ^ P.plan.domain.inverse.logN := by rw [hl]; exact hi + rw [NTT.Transform.bitRevNat_involutive _ _ hi', NTT.Forward.forwardSpec, + Array.getD_eq_getD_getElem?, Array.getElem?_ofFn] + simp only [NTT.Domain.n, hi', ↓reduceDIte, Option.getD_some] + rfl + · rfl + +/-- Parallel forward NTT of packed elements (`PackedRepr.encode`), with the shapes of +`forwardParallel`: the column transform runs directly on the packed buffer, and one leaf +unpacks, transforms and packs. -/ +@[inline] def forwardPacked [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : ByteArray) (parallel : Bool := true) : ByteArray := + if parallel ∧ + ColumnsShape R (a.size / PackedRepr.width R) P.plan.domain.logN then + Columns.transformPacked P.plan.twiddles P.plan.domain.logN + (Columns.columnCount P.plan.domain.logN) + ((Parallel.forwardPasses P.plan.domain).drop 2) false 1 a + else Columns.encodeAll (P.forward (Columns.decodeAll a)) + +/-- Parallel inverse NTT of packed elements, with the shapes of `inverseParallel`. -/ +@[inline] def inversePacked [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : ByteArray) (parallel : Bool := true) : ByteArray := + if parallel ∧ P.plan.inverseDomain.n = P.plan.domain.n ∧ + ColumnsShape R (a.size / PackedRepr.width R) P.plan.domain.logN then + Columns.transformPacked P.plan.inverseTwiddles P.plan.domain.logN + (Columns.columnCount P.plan.domain.logN) + ((Parallel.forwardPasses P.plan.inverseDomain).drop 2) true P.plan.nInv a + else Columns.encodeAll (P.inverse (Columns.decodeAll a)) + +private theorem shape_bounds [PackedRepr R] (n logN : Nat) (h : ColumnsShape R n logN) : + PackedRepr.width R * n < USize.size ∧ + PackedRepr.width R * (16 * 2 ^ (logN - 4)) < USize.size ∧ 16 ∣ 2 ^ (logN - 4) := by + obtain ⟨hn, h15, _, hu⟩ := h + have hw := PackedRepr.width_pos (R := R) + have hp : 16 * 2 ^ (logN - 4) = 2 ^ logN := by + rw [show logN = 4 + (logN - 4) by omega, Nat.pow_add, Nat.add_sub_cancel_left] + have hlt : PackedRepr.width R * 2 ^ logN < PackedRepr.width R * (2 ^ logN + 256) := + Nat.mul_lt_mul_of_pos_left (by omega) hw + refine ⟨by rw [hn]; omega, by rw [hp]; omega, ?_⟩ + exact Nat.pow_dvd_pow 2 (show 4 ≤ logN - 4 by omega) + +/-- The packed parallel forward transform packs the proved scalar transform. -/ +theorem forwardPacked_eq [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : Array R) (parallel : Bool) : + P.forwardPacked (PackedRepr.encode a) parallel = PackedRepr.encode (P.forward a) := by + have hdiv : (PackedRepr.encode a).size / PackedRepr.width R = a.size := by + rw [PackedRepr.size_encode, Nat.mul_div_cancel_left _ PackedRepr.width_pos] + unfold forwardPacked + rw [hdiv] + split + · rename_i h + obtain ⟨hw, hc⟩ := h + have hb := shape_bounds a.size _ hc + rw [Columns.transformPacked_encode _ _ _ _ _ _ _ hb.1 hb.2.1 hb.2.2, ← forwardParallel_eq P a] + unfold forwardParallel + rw [ite_eq_left_of_eq_true _ _ (eq_true ⟨hw, hc⟩)] + · rw [Columns.decodeAll_encode, Columns.encodeAll_eq] + +/-- The packed parallel inverse transform packs the proved scalar inverse. -/ +theorem inversePacked_eq [Field R] [DecidableEq R] [PackedRepr R] (P : NaturalPlan R) + (a : Array R) (parallel : Bool) : + P.inversePacked (PackedRepr.encode a) parallel = PackedRepr.encode (P.inverse a) := by + have hdiv : (PackedRepr.encode a).size / PackedRepr.width R = a.size := by + rw [PackedRepr.size_encode, Nat.mul_div_cancel_left _ PackedRepr.width_pos] + unfold inversePacked + rw [hdiv] + split + · rename_i h + obtain ⟨hw, hn, hc⟩ := h + have hb := shape_bounds a.size _ hc + rw [Columns.transformPacked_encode _ _ _ _ _ _ _ hb.1 hb.2.1 hb.2.2, ← inverseParallel_eq P a] + unfold inverseParallel + rw [ite_eq_left_of_eq_true _ _ (eq_true ⟨hw, hn, hc⟩)] + · rw [Columns.decodeAll_encode, Columns.encodeAll_eq] + +end CompPoly.CPolynomial.NTTFast.NaturalPlan diff --git a/CompPoly/Univariate/NTTFast/Packed/Arithmetic.lean b/CompPoly/Univariate/NTTFast/Packed/Arithmetic.lean new file mode 100644 index 00000000..871ddc2f --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Arithmetic.lean @@ -0,0 +1,213 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.Montgomery.Native32Field + +/-! # Raw Montgomery words for the packed FFT + +The packed kernels operate on the Montgomery residues of a `Mont32Field p` as raw words, with +the field's constants taken from the instance. These lemmas connect their word arithmetic +directly to the verified field operations of `FastField p`. Kernels using them are +`@[specialize]`d, so a concrete instance turns the constants into immediates. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +open Montgomery.Native32 + +variable (p : ℕ) [P : Mont32Field p] + +/-- Reduce `u < 2 p` below `p` as the unsigned minimum of `u` and `u - p`, which vectorizes to a +single minimum instruction. -/ +@[inline] def reduceOnce (u : UInt32) : UInt32 := + let v := u - P.modulus32 + if v < u then v else u + +/-- Add canonical Montgomery residues without converting their representation. -/ +@[inline] def add (x y : UInt32) : UInt32 := + reduceOnce p (x + y) + +/-- Subtract canonical Montgomery residues: the unsigned minimum of `x - y` and `x - y + p`. -/ +@[inline] def sub (x y : UInt32) : UInt32 := + let d := x - y + let e := d + P.modulus32 + if e < d then e else d + +/-- Multiply canonical Montgomery residues: the high word of the product minus the high word of +its Montgomery quotient `q = x y p⁻¹ mod 2 ^ 32` times `p`, reduced as in `sub`. The quotient is a +product of words, so the compiler computes it with one vector multiplication per lane group, and +the difference needs no 64-bit comparison. -/ +@[inline] def mul (x y : UInt32) : UInt32 := + let q := x * y * (0 - P.montgomeryNegInv) + sub p ((x.toUInt64 * y.toUInt64) >>> 32).toUInt32 ((q.toUInt64 * P.modulus64) >>> 32).toUInt32 + +/-- Decode a Montgomery coordinate, rejecting a noncanonical word. -/ +@[inline] def ofWord (x : UInt32) : FastField p := + if h : x.toNat < p then ⟨x, h⟩ else 0 + +variable {p} + +@[simp] theorem ofWord_val (x : FastField p) : ofWord p x.val = x := by + apply Subtype.ext + simp only [ofWord, x.property, ↓reduceDIte] + +/-- The minimum form of the conditional subtraction. -/ +theorem reduceOnce_eq (u : UInt32) : + reduceOnce p u = if u < P.modulus32 then u else u - P.modulus32 := by + have := P.modulus_lt_two_pow_31 + have := P.modulus32_toNat + simp only [reduceOnce] + split <;> split <;> simp_all only [UInt32.lt_iff_toNat_lt, UInt32.toNat_sub, ← UInt32.toNat_inj, + Nat.not_lt] <;> omega + +/-- The raw addition is exactly the carrier's verified addition. -/ +theorem add_val (x y : FastField p) : add p x.val y.val = (x + y).val := by + rw [add, reduceOnce_eq] + rfl + +/-- The minimum form of the subtraction of canonical residues. -/ +theorem sub_eq (x y : UInt32) (hx : x.toNat < p) (hy : y.toNat < p) : + sub p x y = if y ≤ x then x - y else x + P.modulus32 - y := by + have := P.modulus_lt_two_pow_31 + have := P.modulus32_toNat + simp only [sub] + split <;> split <;> simp_all only [UInt32.lt_iff_toNat_lt, UInt32.le_iff_toNat_le, + UInt32.toNat_sub, UInt32.toNat_add, ← UInt32.toNat_inj, Nat.not_lt, Nat.not_le] <;> omega + +/-- The raw subtraction is exactly the carrier's verified subtraction. -/ +theorem sub_val (x y : FastField p) : sub p x.val y.val = (x - y).val := by + rw [sub_eq _ _ x.property y.property] + simp only [Montgomery.Native32.sub_def, Montgomery.Native32.sub] + split <;> rfl + +/-- The Montgomery quotients `q₁ = L n` and `q₂ = L (-n)` of `L`, with `n m ≡ -1` modulo +`2 ^ 32`: `L + q₁ m ≡ 0` and `q₁ + q₂ ≡ 0`. -/ +theorem mont_quotients (L n m : Nat) (hn : n < 2 ^ 32) (hnp : n * m % 2 ^ 32 = 2 ^ 32 - 1) : + (L + L * n % 2 ^ 32 * m) % 2 ^ 32 = 0 ∧ + (L * n % 2 ^ 32 + L * ((2 ^ 32 - n) % 2 ^ 32) % 2 ^ 32) % 2 ^ 32 = 0 := by + constructor + · have h1 : L * n % 2 ^ 32 * m ≡ L * (2 ^ 32 - 1) [MOD 2 ^ 32] := by + rw [show L * (2 ^ 32 - 1) = L * ((n * m) % 2 ^ 32) by rw [hnp]] + refine Nat.ModEq.trans (Nat.ModEq.mul_right _ (Nat.mod_modEq _ _)) ?_ + rw [Nat.mul_assoc] + exact Nat.ModEq.mul_left _ (Nat.mod_modEq _ _).symm + have h2 := Nat.ModEq.add_left L h1 + have h3 : L + L * (2 ^ 32 - 1) = L * 2 ^ 32 := by + have h4 : L ≤ L * 2 ^ 32 := Nat.le_mul_of_pos_right L (by decide) + rw [Nat.mul_sub, Nat.mul_one] + omega + rw [h3] at h2 + rw [h2, Nat.mul_mod_left] + · have h1 : L * n % 2 ^ 32 + L * ((2 ^ 32 - n) % 2 ^ 32) % 2 ^ 32 ≡ + L * n + L * (2 ^ 32 - n) [MOD 2 ^ 32] := + Nat.ModEq.add (Nat.mod_modEq _ _) + (Nat.ModEq.trans (Nat.mod_modEq _ _) (Nat.ModEq.mul_left _ (Nat.mod_modEq _ _))) + rw [← Nat.mul_add, Nat.add_sub_cancel' hn.le] at h1 + rw [h1, Nat.mul_mod_left] + +/-- On canonical residues, `mul` is the Montgomery reduction `(z + (z n mod 2 ^ 32) p) / 2 ^ 32` +of the product `z`, reduced once. -/ +theorem mul_eq_reduceOnce (x y : UInt32) (hx : x.toNat < p) (hy : y.toNat < p) : + mul p x y = reduceOnce p (((x.toUInt64 * y.toUInt64 + ((x.toUInt64 * y.toUInt64).toUInt32 * + P.montgomeryNegInv).toUInt64 * P.modulus64) >>> 32).toUInt32) := by + have hp31 := P.modulus_lt_two_pow_31 + have hp32 := P.modulus32_toNat + have hp64 := P.modulus64_toNat + have hp2 := P.two_lt_modulus + have hnp := P.montgomeryNegInv_mul_modulus_mod_two_pow_32 + have hn := P.montgomeryNegInv.toNat_lt + set n := P.montgomeryNegInv.toNat with hn_def + have hz : x.toNat * y.toNat < p * 2 ^ 32 := + lt_of_lt_of_le (Nat.mul_lt_mul_of_lt_of_le hx hy.le (by omega)) + (Nat.mul_le_mul_left _ (by omega)) + set z := x.toNat * y.toNat with hz_def + have hz64 : (x.toUInt64 * y.toUInt64).toNat = z := by + rw [UInt64.toNat_mul, UInt32.toNat_toUInt64, UInt32.toNat_toUInt64] + exact Nat.mod_eq_of_lt (by omega) + set L := z % 2 ^ 32 with hL + set q1 := L * n % 2 ^ 32 with hq1 + set q2 := L * ((2 ^ 32 - n) % 2 ^ 32) % 2 ^ 32 with hq2 + obtain ⟨c1, c2⟩ := mont_quotients L n p hn hnp + rw [← hq1, ← hq2] at c2 + rw [← hq1] at c1 + have hq1R : q1 < 2 ^ 32 := Nat.mod_lt _ (by decide) + have hq2R : q2 < 2 ^ 32 := Nat.mod_lt _ (by decide) + have hq1p : q1 * p < 2 ^ 32 * p := Nat.mul_lt_mul_of_pos_right hq1R (by omega) + have e32 : (32 : UInt64).toNat % 64 = 32 := rfl + have hz' : x.toNat * y.toNat % 2 ^ 64 = z := Nat.mod_eq_of_lt (by omega) + -- The reduction. + have hA : (reduceOnce p (((x.toUInt64 * y.toUInt64 + ((x.toUInt64 * y.toUInt64).toUInt32 * + P.montgomeryNegInv).toUInt64 * P.modulus64) >>> 32).toUInt32)).toNat = + if (z + q1 * p) / 2 ^ 32 < p then (z + q1 * p) / 2 ^ 32 else (z + q1 * p) / 2 ^ 32 - p := by + have hS : (((x.toUInt64 * y.toUInt64 + ((x.toUInt64 * y.toUInt64).toUInt32 * + P.montgomeryNegInv).toUInt64 * P.modulus64) >>> 32).toUInt32).toNat = + (z + q1 * p) / 2 ^ 32 := by + simp only [UInt64.toNat_toUInt32, UInt64.toNat_shiftRight, UInt64.toNat_add, + UInt64.toNat_mul, UInt32.toNat_toUInt64, UInt32.toNat_mul, e32, hz', hp64, ← hL] + rw [Nat.mod_eq_of_lt (show q1 * p < 2 ^ 64 by omega), + Nat.mod_eq_of_lt (show z + q1 * p < 2 ^ 64 by omega), Nat.shiftRight_eq_div_pow] + exact Nat.mod_eq_of_lt (by omega) + rw [reduceOnce_eq] + split + · rename_i h + rw [UInt32.lt_iff_toNat_lt, hS, hp32] at h + rw [hS, ite_eq_left h] + · rename_i h + rw [UInt32.lt_iff_toNat_lt, hS, hp32] at h + rw [UInt32.toNat_sub_of_le _ _ (UInt32.le_iff_toNat_le.mpr (by rw [hS, hp32]; omega)), hS, + hp32, ite_eq_right h] + -- The difference of high words. + have hB : (mul p x y).toNat = if (q2 * p) / 2 ^ 32 ≤ z / 2 ^ 32 then + z / 2 ^ 32 - (q2 * p) / 2 ^ 32 else z / 2 ^ 32 + p - (q2 * p) / 2 ^ 32 := by + have hq2p : q2 * p < 2 ^ 32 * p := Nat.mul_lt_mul_of_pos_right hq2R (by omega) + have hH : ((x.toUInt64 * y.toUInt64) >>> 32).toUInt32.toNat = z / 2 ^ 32 := by + rw [UInt64.toNat_toUInt32, UInt64.toNat_shiftRight, hz64, e32, Nat.shiftRight_eq_div_pow] + exact Nat.mod_eq_of_lt (by omega) + have hH2 : (((x * y * (0 - P.montgomeryNegInv)).toUInt64 * P.modulus64) >>> 32).toUInt32.toNat = + q2 * p / 2 ^ 32 := by + simp only [UInt64.toNat_toUInt32, UInt64.toNat_shiftRight, UInt64.toNat_mul, + UInt32.toNat_toUInt64, UInt32.toNat_mul, UInt32.toNat_sub, UInt32.toNat_zero, Nat.add_zero, + e32, hp64, ← hn_def] + rw [← hz_def, ← hL, ← hq2, Nat.mod_eq_of_lt (show q2 * p < 2 ^ 64 by omega), + Nat.shiftRight_eq_div_pow] + exact Nat.mod_eq_of_lt (by omega) + unfold mul + dsimp only + rw [sub_eq _ _ (by rw [hH]; omega) (by rw [hH2]; omega)] + split + · rename_i h + rw [UInt32.le_iff_toNat_le, hH, hH2] at h + rw [UInt32.toNat_sub_of_le _ _ (UInt32.le_iff_toNat_le.mpr (by rw [hH, hH2]; omega)), hH, + hH2, ite_eq_left h] + · rename_i h + rw [UInt32.le_iff_toNat_le, hH, hH2] at h + have hs : (((x.toUInt64 * y.toUInt64) >>> 32).toUInt32 + P.modulus32).toNat = + z / 2 ^ 32 + p := by + rw [UInt32.toNat_add, hH, hp32]; exact Nat.mod_eq_of_lt (by omega) + rw [UInt32.toNat_sub_of_le _ _ (UInt32.le_iff_toNat_le.mpr (by rw [hs, hH2]; omega)), hs, + hH2, ite_eq_right h] + apply UInt32.toNat_inj.mp + rw [hA, hB] + have hzL : z = z / 2 ^ 32 * 2 ^ 32 + L := by omega + rcases Nat.eq_zero_or_pos q1 with h0 | h0 + · have hq20 : q2 = 0 := by omega + have hL0 : L = 0 := by rw [h0, Nat.zero_mul, Nat.add_zero] at c1; omega + rw [h0, hq20, Nat.zero_mul, Nat.add_zero, Nat.zero_div] + split <;> split <;> omega + · have hq2' : q2 = 2 ^ 32 - q1 := by omega + have hQ2 : q2 * p = 2 ^ 32 * p - q1 * p := by rw [hq2', Nat.sub_mul] + have hp1 : p ≤ q1 * p := Nat.le_mul_of_pos_left p h0 + rw [hQ2] + split <;> split <;> omega + +/-- The raw multiplication is exactly the carrier's verified multiplication. -/ +theorem mul_val (x y : FastField p) : mul p x.val y.val = (x * y).val := by + rw [mul_eq_reduceOnce _ _ x.property y.property, reduceOnce_eq] + rfl + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/ArrayLemmas.lean b/CompPoly/Univariate/NTTFast/Packed/ArrayLemmas.lean new file mode 100644 index 00000000..86d07fc9 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/ArrayLemmas.lean @@ -0,0 +1,56 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.Arrays + +/-! # Pointwise equations for packed-kernel array models -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- Reading an in-bounds tabulated coordinate returns its defining value. -/ +theorem getD_ofFn_bounded (f : Fin n → α) (i : Nat) (hi : i < n) (d : α) : + (Array.ofFn f).getD i d = f ⟨i, hi⟩ := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_ofFn, hi, + ↓reduceDIte, Option.getD_some] + +/-- A bounded update changes one coordinate, including when the queried coordinate is absent. -/ +theorem getD_setIfInBounds (a : Array α) (i k : Nat) (x d : α) (hi : i < a.size) : + (a.setIfInBounds i x).getD k d = if k = i then x else a.getD k d := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_setIfInBounds] + by_cases h : k = i + · subst k + simp only [hi, ↓reduceIte, Option.getD_some] + · have hn : i ≠ k := Ne.symm h + simp only [h, hn, ↓reduceIte] + +/-- Reading within an extracted segment reads the corresponding original coordinate. -/ +theorem getD_extract (a : Array α) (first count k : Nat) (d : α) + (hs : first + count ≤ a.size) (hk : k < count) : + (a.extract first (first + count)).getD k d = a.getD (first + k) d := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_extract, + Nat.min_eq_left hs, Nat.add_sub_cancel_left, hk, ↓reduceIte] + +/-- Equal-length arrays are equal when all coordinates with the same default agree. -/ +theorem array_eq_of_getD (a b : Array α) (d : α) (hs : a.size = b.size) + (h : ∀ k, a.getD k d = b.getD k d) : a = b := by + apply Array.ext hs + intro k hk hj + have he := h k + simpa only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hk, + Array.getElem?_eq_getElem hj, Option.getD_some] using he + +/-- Reading a literal list-backed array uses ordinary list indexing. -/ +theorem getD_toArray (xs : List α) (k : Nat) (d : α) : + xs.toArray.getD k d = xs[k]?.getD d := by + rw [Array.getD_eq_getD_getElem?, List.getElem?_toArray] + +/-- A sixteen-entry literal has sixteen coordinates, independently of its values. -/ +@[simp] theorem size_literal16 (x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 : α) : + (#[x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15]).size = 16 := rfl + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Arrays.lean b/CompPoly/Univariate/NTTFast/Packed/Arrays.lean new file mode 100644 index 00000000..884895f8 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Arrays.lean @@ -0,0 +1,112 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Natural + +/-! # Array slices used in packed-kernel refinements -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- Replace consecutive entries by a complete array segment. -/ +def splice (a : Array α) (index : Nat) (values : Array α) : Array α := + a.extract 0 index ++ values ++ a.extract (index + values.size) a.size + +/-- An in-bounds splice preserves the array size. -/ +@[simp] theorem size_splice (a : Array α) (i : Nat) (values : Array α) + (h : i + values.size ≤ a.size) : (splice a i values).size = a.size := by + have hi : i ≤ a.size := by omega + simp only [splice, Array.size_append, Array.size_extract, Nat.min_self, + Nat.min_eq_left hi, Nat.sub_zero] + omega + +/-- Every in-bounds sixteen-entry batch preserves the working array's size. -/ +@[simp] theorem size_splice16 (a : Array α) (i : Nat) + (x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 : α) (hi : i + 16 ≤ a.size) : + (splice a i #[x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15]).size = + a.size := by + apply size_splice + change i + 16 ≤ a.size + exact hi + +/-- Dropping a prefix of a tail drops the combined prefix. -/ +theorem extract_extract_tail (Z : Array α) (a b : Nat) : + (Z.extract a Z.size).extract b (Z.extract a Z.size).size = Z.extract (a + b) Z.size := by + rw [Array.extract_extract, Array.size_extract] + congr 1 + omega + +/-- Splicing a batch at the end of a written prefix moves it into the prefix. -/ +theorem splice_cursor (l Z V : Array α) (hZ : V.size ≤ Z.size) : + splice (l ++ Z) l.size V = l ++ V ++ Z.extract V.size Z.size := by + simp only [splice, Array.size_append] + rw [Array.extract_append, Array.extract_append] + simp only [Nat.sub_self, Array.extract_size, Array.extract_zero, Array.append_empty] + rw [Array.extract_eq_empty_of_le (by omega), Array.empty_append, + show l.size + V.size - l.size = V.size by omega, + show l.size + Z.size - l.size = Z.size by omega] + +private theorem getD_append (a b : Array α) (i : Nat) (d : α) : + (a ++ b).getD i d = if i < a.size then a.getD i d else b.getD (i - a.size) d := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_append] + split <;> rfl + +private theorem getD_extract (a : Array α) (first last i : Nat) (d : α) : + (a.extract first last).getD i d = + if i < min last a.size - first then a.getD (first + i) d else d := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_extract] + split <;> rfl + +/-- A splice changes exactly the entries in its range. -/ +theorem getD_splice (a : Array α) (i : Nat) (values : Array α) + (h : i + values.size ≤ a.size) (k : Nat) (d : α) : + (splice a i values).getD k d = + if k < i then a.getD k d else + if k < i + values.size then values.getD (k - i) d else a.getD k d := by + have hi : i ≤ a.size := by omega + have hp : (a.extract 0 i).size = i := by + simp only [Array.size_extract, Nat.min_eq_left hi, Nat.sub_zero] + simp only [splice, getD_append, Array.size_append, hp, getD_extract, + Nat.min_eq_left hi, Nat.min_self, Nat.sub_zero, Nat.zero_add] + by_cases hki : k < i + · have hkiv : k < i + values.size := by omega + simp only [hki, hkiv, ↓reduceIte] + · by_cases hkv : k < i + values.size + · simp only [hki, hkv, ↓reduceIte] + · have hsum : i + values.size + (k - (i + values.size)) = k := by omega + by_cases hka : k < a.size + · have ht : k - (i + values.size) < a.size - (i + values.size) := by omega + simp only [hki, hkv, ht, hsum, ↓reduceIte] + · have ht : ¬k - (i + values.size) < a.size - (i + values.size) := by omega + simp only [hki, hkv, ht, ↓reduceIte, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_none (Nat.le_of_not_lt hka), Option.getD_none] + +/-- A single-entry splice is the usual bounded array update. -/ +theorem splice_singleton (a : Array α) (i : Nat) (x : α) (hi : i < a.size) : + splice a i #[x] = a.setIfInBounds i x := by + have h : i + (#[x]).size ≤ a.size := by change i + 1 ≤ a.size; omega + apply Array.ext + · simp only [size_splice a i #[x] h, Array.size_setIfInBounds] + · intro k hk hj + have hs : k < a.size := by simpa only [size_splice a i #[x] h] using hk + have he := getD_splice a i #[x] h k x + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hk, Option.getD_some] at he + rw [he] + simp only [Array.getElem_setIfInBounds hs, + Array.getElem?_eq_getElem hs, Option.getD_some] + by_cases hki : k < i + · have hne : i ≠ k := by omega + simp only [hki, hne, ↓reduceIte] + · by_cases hke : k = i + · subst k + simp only [Nat.lt_irrefl, Array.size_singleton, Nat.lt_succ_self, ↓reduceIte, Nat.sub_self] + rfl + · have hk1 : ¬k < i + (#[x]).size := by change ¬k < i + 1; omega + have hne : i ≠ k := Ne.symm hke + simp only [hki, hk1, hne, ↓reduceIte] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/BatchCorrectness.lean b/CompPoly/Univariate/NTTFast/Packed/BatchCorrectness.lean new file mode 100644 index 00000000..ca3b53bd --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/BatchCorrectness.lean @@ -0,0 +1,101 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.Expressions +public import CompPoly.Univariate.NTTFast.Packed.QuadCells +import Mathlib.Tactic.IntervalCases + +/-! # Mathematical refinement of sixteen-lane butterfly batches -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed.Expressions + +set_option maxRecDepth 16384 in +set_option maxHeartbeats 1200000 in +/-- The batched field kernel executes sixteen independent radix-four cells. -/ +theorem getD_step16Field [Field R] (th tl : Array R) (q : Nat) + (j j1 i0 i1 i2 i3 : USize) (a : Array R) + (hj1 : j1.toNat = j.toNat + q) (hs : i3.toNat + 16 ≤ a.size) + (h01 : i0.toNat + 16 ≤ i1.toNat) (h12 : i1.toNat + 16 ≤ i2.toNat) + (h23 : i2.toNat + 16 ≤ i3.toNat) (k : Nat) : + (step16Field th tl j j1 i0 i1 i2 i3 a).getD k 0 = + quadValue th tl q j.toNat i0.toNat i1.toNat i2.toNat i3.toNat 16 a k := by + have hi0 : i0.toNat + 16 ≤ a.size := by omega + have hi1 : i1.toNat + 16 ≤ a.size := by omega + have hi2 : i2.toNat + 16 ≤ a.size := by omega + unfold step16Field readField fieldAdd fieldSub fieldMul quadValue quadAt quadOutputs + simp only [Native.usize_numeral 0 (by decide), Native.usize_numeral 1 (by decide), + Native.usize_numeral 2 (by decide), Native.usize_numeral 3 (by decide), Native.usize_numeral + 4 (by decide), Native.usize_numeral 5 (by decide), Native.usize_numeral 6 (by decide), + Native.usize_numeral 7 (by decide), Native.usize_numeral 8 (by decide), Native.usize_numeral + 9 (by decide), Native.usize_numeral 10 (by decide), Native.usize_numeral 11 (by decide), + Native.usize_numeral 12 (by decide), Native.usize_numeral 13 (by decide), + Native.usize_numeral 14 (by decide), Native.usize_numeral 15 (by decide), hj1] + simp (config := { maxDischargeDepth := 8, maxSteps := 1000000 }) (disch := (first + | (simp -failIfUnchanged only + [splice, Array.size_append, Array.size_extract, size_literal16] <;> omega) + | fail)) only + [getD_splice, size_literal16] + by_cases hk0 : i0.toNat ≤ k ∧ k < i0.toNat + 16 + · obtain ⟨offset, ho, he⟩ : ∃ offset, offset < 16 ∧ k = i0.toNat + offset := by + exact ⟨k - i0.toNat, by omega, by omega⟩ + subst k + interval_cases offset <;> + simp (disch := omega) only [Nat.add_assoc, Nat.sub_self, + Nat.add_sub_cancel_left, Nat.add_zero, ite_eq_left, ite_eq_right, + getD_toArray, List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some] + · by_cases hk1 : i1.toNat ≤ k ∧ k < i1.toNat + 16 + · obtain ⟨offset, ho, he⟩ : ∃ offset, offset < 16 ∧ k = i1.toNat + offset := by + exact ⟨k - i1.toNat, by omega, by omega⟩ + subst k + interval_cases offset <;> + simp (disch := omega) only [Nat.add_assoc, Nat.sub_self, + Nat.add_sub_cancel_left, Nat.add_zero, ite_eq_left, ite_eq_right, + getD_toArray, List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some] + · by_cases hk2 : i2.toNat ≤ k ∧ k < i2.toNat + 16 + · obtain ⟨offset, ho, he⟩ : ∃ offset, offset < 16 ∧ k = i2.toNat + offset := by + exact ⟨k - i2.toNat, by omega, by omega⟩ + subst k + interval_cases offset <;> + simp (disch := omega) only [Nat.add_assoc, Nat.sub_self, + Nat.add_sub_cancel_left, Nat.add_zero, ite_eq_left, ite_eq_right, + getD_toArray, List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some] <;> simp + only [Nat.add_comm] + · by_cases hk3 : i3.toNat ≤ k ∧ k < i3.toNat + 16 + · obtain ⟨offset, ho, he⟩ : ∃ offset, offset < 16 ∧ k = i3.toNat + offset := by + exact ⟨k - i3.toNat, by omega, by omega⟩ + subst k + interval_cases offset <;> + simp (disch := omega) only [Nat.add_assoc, Nat.sub_self, + Nat.add_sub_cancel_left, Nat.add_zero, ite_eq_left, ite_eq_right, + getD_toArray, List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some] <;> simp + only [Nat.add_comm] + · simp (disch := omega) only [ite_eq_left, ite_eq_right, ite_self] + +/-- The batched expression graph equals the explicit-length scalar batch. -/ +theorem step16Field_eq_quadCells [Field R] (th tl : Array R) (q : Nat) + (j j1 i0 i1 i2 i3 : USize) (a : Array R) + (hj1 : j1.toNat = j.toNat + q) (hs : i3.toNat + 16 ≤ a.size) + (h01 : i0.toNat + 16 ≤ i1.toNat) (h12 : i1.toNat + 16 ≤ i2.toNat) + (h23 : i2.toNat + 16 ≤ i3.toNat) : + step16Field th tl j j1 i0 i1 i2 i3 a = + quadCells th tl q j.toNat i0.toNat i1.toNat i2.toNat i3.toNat a 16 := by + apply array_eq_of_getD _ _ (0 : R) + · have hi0 : i0.toNat + 16 ≤ a.size := by omega + have hi1 : i1.toNat + 16 ≤ a.size := by omega + have hi2 : i2.toNat + 16 ≤ a.size := by omega + unfold step16Field + simp (config := { maxDischargeDepth := 8, maxSteps := 1000000 }) (disch := (first + | (simp -failIfUnchanged only + [splice, Array.size_append, Array.size_extract, size_literal16] <;> omega) + | fail)) only + [size_splice16, size_quadCells] + · intro k + rw [getD_step16Field th tl q j j1 i0 i1 i2 i3 a hj1 hs h01 h12 h23, + getD_quadCells th tl q j.toNat i0.toNat i1.toNat i2.toNat i3.toNat a 16 hs h01 h12 h23] + +end CompPoly.CPolynomial.NTTFast.Packed.Expressions diff --git a/CompPoly/Univariate/NTTFast/Packed/BitReversal.lean b/CompPoly/Univariate/NTTFast/Packed/BitReversal.lean new file mode 100644 index 00000000..f71953f4 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/BitReversal.lean @@ -0,0 +1,40 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Reverse32 +import Mathlib.Tactic.Ring + +/-! # Bit-reversal identities for interleaving reversed blocks -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- Reversing concatenated low and high bit groups reverses each group and swaps their positions. -/ +theorem bitRevNat_concat (high low x y : Nat) (hy : y < 2 ^ low) : + NTT.Transform.bitRevNat (high + low) (2 ^ low * x + y) = + 2 ^ high * NTT.Transform.bitRevNat low y + NTT.Transform.bitRevNat high x := by + apply Nat.eq_of_testBit_eq + intro k + rw [bitRevNat_testBit, + Nat.testBit_two_pow_mul_add _ (NTT.Transform.bitRevNat_lt high x)] + by_cases hk : k < high + · have hsum : k < high + low := by omega + have hindex : ¬high + low - 1 - k < low := by omega + have he : high + low - 1 - k - low = high - 1 - k := by omega + rw [ite_eq_left hsum, ite_eq_left hk, + Nat.testBit_two_pow_mul_add x hy, ite_eq_right hindex, + bitRevNat_testBit, ite_eq_left hk, he] + · by_cases hsum : k < high + low + · have hindex : high + low - 1 - k < low := by omega + have hbit : k - high < low := by omega + have he : high + low - 1 - k = low - 1 - (k - high) := by omega + rw [ite_eq_left hsum, ite_eq_right hk, Nat.testBit_two_pow_mul_add x hy, + ite_eq_left hindex, bitRevNat_testBit, ite_eq_left hbit, he] + · have hbit : ¬k - high < low := by omega + rw [ite_eq_right hsum, ite_eq_right hk, bitRevNat_testBit, ite_eq_right hbit] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/ButterflyLemmas.lean b/CompPoly/Univariate/NTTFast/Packed/ButterflyLemmas.lean new file mode 100644 index 00000000..f2931b0b --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/ButterflyLemmas.lean @@ -0,0 +1,101 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Plan +public import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas + +/-! # Coordinate semantics of disjoint DIF butterflies -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- Coordinate formula for a consecutive range of independent DIF butterflies. -/ +def difValue [Field R] (tw : Array R) (j i0 i1 count : Nat) (a : Array R) (k : Nat) : R := + if i0 ≤ k ∧ k < i0 + count then a.getD k 0 + a.getD (i1 + (k - i0)) 0 else + if i1 ≤ k ∧ k < i1 + count then + tw.getD (j + (k - i1)) 0 * (a.getD (i0 + (k - i1)) 0 - a.getD k 0) + else a.getD k 0 + +private theorem difValue_step [Field R] (tw : Array R) (j i0 i1 count : Nat) + (a : Array R) (hs : i1 + (count + 1) ≤ a.size) + (hsep : i0 + (count + 1) ≤ i1) (k : Nat) : + difValue tw (j + 1) (i0 + 1) (i1 + 1) count + ((a.setIfInBounds i0 (a.getD i0 0 + a.getD i1 0)).setIfInBounds i1 + (tw.getD j 0 * (a.getD i0 0 - a.getD i1 0))) k = + difValue tw j i0 i1 (count + 1) a k := by + have hi0 : i0 < a.size := by omega + have hi1 : i1 < a.size := by omega + have hsz : i1 < (a.setIfInBounds i0 (a.getD i0 0 + a.getD i1 0)).size := by + simpa only [Array.size_setIfInBounds] using hi1 + unfold difValue + simp only [getD_setIfInBounds _ i1 _ _ _ hsz, getD_setIfInBounds a i0 _ _ _ hi0] + by_cases hk0 : k = i0 + · subst k + simp (disch := (simp only [Array.size_setIfInBounds] at *; omega)) only + [Nat.sub_self, Nat.add_zero, ite_eq_left, ite_eq_right, ↓reduceIte] + · by_cases hk1 : k = i1 + · subst k + simp (disch := (simp only [Array.size_setIfInBounds] at *; omega)) only + [Nat.sub_self, Nat.add_zero, ite_eq_left, ite_eq_right, ↓reduceIte] + · by_cases hl : i0 ≤ k ∧ k < i0 + (count + 1) + · have hks : i1 + 1 + (k - (i0 + 1)) = i1 + (k - i0) := by omega + simp (disch := omega) only [hks, ite_eq_left, ite_eq_right] + · by_cases hr : i1 ≤ k ∧ k < i1 + (count + 1) + · have hks : i0 + 1 + (k - (i1 + 1)) = i0 + (k - i1) := by omega + have hws : j + 1 + (k - (i1 + 1)) = j + (k - i1) := by omega + simp (disch := omega) only [hks, hws, ite_eq_left, ite_eq_right] + · simp (disch := omega) only [ite_eq_left, ite_eq_right] + +/-- An ordinary DIF inner loop has the pointwise formula for disjoint butterfly pairs. -/ +theorem getD_butterflyDIFInner [Field R] (tw : Array R) (limit j i0 i1 : Nat) (a : Array R) + (hs : i1 + (limit - j) ≤ a.size) (hsep : i0 + (limit - j) ≤ i1) (k : Nat) : + (Plan.butterflyDIFInner tw limit j i0 i1 a).getD k 0 = + difValue tw j i0 i1 (limit - j) a k := by + rw [Plan.butterflyDIFInner] + split + · rename_i hj + have he : limit - j = (limit - (j + 1)) + 1 := by omega + rw [getD_butterflyDIFInner tw limit (j + 1) (i0 + 1) (i1 + 1) _ + (by simp only [Array.size_set!]; omega) (by omega), he] + exact difValue_step tw j i0 i1 (limit - (j + 1)) a (by omega) (by omega) k + · have he : limit - j = 0 := by omega + simp (disch := omega) only [he, difValue, Nat.add_zero, ite_eq_right] +termination_by limit - j +decreasing_by omega + +/-- A completed lower-half butterfly coordinate is the sum of its original pair. -/ +theorem getD_butterflyDIFInner_left [Field R] (tw : Array R) (limit j i0 i1 : Nat) + (a : Array R) (hs : i1 + (limit - j) ≤ a.size) + (hsep : i0 + (limit - j) ≤ i1) (k : Nat) + (hk : i0 ≤ k ∧ k < i0 + (limit - j)) : + (Plan.butterflyDIFInner tw limit j i0 i1 a).getD k 0 = + a.getD k 0 + a.getD (i1 + (k - i0)) 0 := by + rw [getD_butterflyDIFInner tw limit j i0 i1 a hs hsep] + simp only [difValue, hk, and_self, ↓reduceIte] + +/-- A completed upper-half coordinate is the twiddle-scaled difference of its pair. -/ +theorem getD_butterflyDIFInner_right [Field R] (tw : Array R) (limit j i0 i1 : Nat) + (a : Array R) (hs : i1 + (limit - j) ≤ a.size) + (hsep : i0 + (limit - j) ≤ i1) (k : Nat) + (hk : i1 ≤ k ∧ k < i1 + (limit - j)) : + (Plan.butterflyDIFInner tw limit j i0 i1 a).getD k 0 = + tw.getD (j + (k - i1)) 0 * (a.getD (i0 + (k - i1)) 0 - a.getD k 0) := by + have hl : ¬(i0 ≤ k ∧ k < i0 + (limit - j)) := by omega + rw [getD_butterflyDIFInner tw limit j i0 i1 a hs hsep] + simp only [difValue, hk, hl, and_self, ↓reduceIte] + +/-- All coordinates outside the completed pair ranges are unchanged. -/ +theorem getD_butterflyDIFInner_outside [Field R] (tw : Array R) (limit j i0 i1 : Nat) + (a : Array R) (hs : i1 + (limit - j) ≤ a.size) + (hsep : i0 + (limit - j) ≤ i1) (k : Nat) + (hk0 : ¬(i0 ≤ k ∧ k < i0 + (limit - j))) + (hk1 : ¬(i1 ≤ k ∧ k < i1 + (limit - j))) : + (Plan.butterflyDIFInner tw limit j i0 i1 a).getD k 0 = a.getD k 0 := by + rw [getD_butterflyDIFInner tw limit j i0 i1 a hs hsep] + simp only [difValue, hk0, hk1, ↓reduceIte] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Columns.lean b/CompPoly/Univariate/NTTFast/Packed/Columns.lean new file mode 100644 index 00000000..3d961208 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Columns.lean @@ -0,0 +1,69 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.NativeLast +public import CompPoly.Univariate.NTTFast.Columns + +/-! # The packed column-parallel transform + +The packed plan runs the same pipeline as `Columns.transformPacked`, on packed words throughout: +column tasks run the top two radix-four passes of the transform, viewed as a `16 × M` matrix, on +their columns; sixteen leaf tasks run the remaining layers of one row each with `leafNatural`, +which also reverses and scales it; and `naturalLeaves` interleaves the leaves into natural order. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed.Native +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +/-- Words `j0, …, j0 + cs - 1` of rows `0, …, r - 1` of a row-major matrix with rows of `M` +words, as an `r × cs` matrix. -/ +def gatherWords (src : @& ByteArray) (M j0 cs r : Nat) : ByteArray := + (List.range r).foldl (fun out k ↦ src.copySlice (4 * (k * M + j0)) out out.size (4 * cs)) + (ByteArray.emptyWithCapacity (4 * (r * cs))) + +/-- The top two radix-four passes of a `16 × M` transform on columns `j0, …, j0 + cs - 1`. -/ +@[specialize] +def columnWords (p : ℕ) [Mont32Field p] (tw : @& Array ByteArray) (logN M cs j0 : Nat) + (x : @& ByteArray) : ByteArray := + let a := radix4Blocks p (gatherWords (tw.getD (logN - 1) .empty) M j0 cs 8) + (gatherWords (tw.getD (logN - 2) .empty) M j0 cs 4) (4 * cs) 1 (gatherWords x M j0 cs 16) + radix4Blocks p (gatherWords (tw.getD (logN - 3) .empty) M j0 cs 2) + (gatherWords (tw.getD (logN - 4) .empty) M j0 cs 1) cs 4 a + +/-- Row `l` of the column results, in column order. -/ +def leafWords (cols : @& Array ByteArray) (cs l : Nat) : ByteArray := + cols.foldl (fun out col ↦ col.copySlice (4 * (l * cs)) out out.size (4 * cs)) + (ByteArray.emptyWithCapacity (4 * (cols.size * cs))) + +/-- The column-parallel transform of `2 ^ logN` packed words: `S` column tasks run the top two +radix-four passes, sixteen leaf tasks the remaining layers in natural order, and sixteen tasks +interleave the leaves. -/ +@[specialize] +def runColumns (p : ℕ) [Mont32Field p] (tw : Array ByteArray) (logN S : Nat) (nInv : UInt32) + (inverse : Bool) + (a : ByteArray) : ByteArray := + let M := 2 ^ (logN - 4) + let cs := M / S + let cols := Tasks.spawnAll S fun s ↦ columnWords p tw logN M cs (s * cs) a + let leaves := Tasks.spawnAfter cols 16 fun cols l ↦ + leafNatural p tw (logN - 4) nInv inverse (leafWords cols cs l) + naturalLeaves (leaves.map Task.get) logN + +/-- The complete transform of `2 ^ logN` packed words in natural order: if `parallel`, the column +transform from `2 ^ 15` words, otherwise one leaf. -/ +@[specialize] +def run (p : ℕ) [Mont32Field p] (tw : Array ByteArray) (logN : Nat) (parallel : Bool) + (nInv : UInt32) (inverse : Bool) + (a : ByteArray) : ByteArray := + if parallel ∧ 15 ≤ logN then + runColumns p tw logN (Columns.columnCount logN) nInv inverse a + else leafNatural p tw logN nInv inverse a + +end CompPoly.CPolynomial.NTTFast.Packed.Native diff --git a/CompPoly/Univariate/NTTFast/Packed/ColumnsCorrectness.lean b/CompPoly/Univariate/NTTFast/Packed/ColumnsCorrectness.lean new file mode 100644 index 00000000..742dedb5 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/ColumnsCorrectness.lean @@ -0,0 +1,324 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Packed.Columns +public import CompPoly.Univariate.NTTFast.Packed.Columns +public import CompPoly.Univariate.NTTFast.Packed.NativeLastCorrectness +public import CompPoly.Univariate.NTTFast.Packed.DifSpec +public import CompPoly.Univariate.NTTFast.NaturalParallel +import all CompPoly.Univariate.NTTFast.Columns +import Mathlib.Tactic.Ring + +/-! # Correctness of the packed column-parallel transform + +Every stage of `runColumns` packs a stage of the field-level column transform: column tasks +pack `Columns.columnTask`, a leaf's words pack its block of the top two passes, and a leaf task +packs the natural-order read of that block of the DIF transform. The interleave then packs the +natural-order DFT. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +local notation "F" => FastField p + +/-- Appending a range of a packed field array copies its coordinates. -/ +theorem copySlice_encode_fields (a o : Array F) (i c : Nat) : + (encode a).copySlice (4 * i) (encode o) (encode o).size (4 * c) = + encode (o ++ a.extract i (i + c)) := + PackedRepr.copySlice_encode a o i c + +/-- A fold of slot-range copies packs the fold of the element ranges. -/ +theorem foldl_copySlice_encode {ι : Type*} (f : ι → Array F) (g : ι → Nat) (cs : Nat) + (L : List ι) (o : Array F) : + L.foldl (fun out k ↦ (encode (f k)).copySlice (4 * g k) out out.size (4 * cs)) (encode o) = + encode (L.foldl (fun acc k ↦ acc ++ (f k).extract (g k) (g k + cs)) o) := by + induction L generalizing o with + | nil => rfl + | cons k L ih => + simp only [List.foldl_cons] + rw [copySlice_encode_fields] + exact ih _ + +/-- Appended ranges of `cs` entries form a row-major matrix. -/ +theorem appendRows_spec (f : Nat → Array F) (g : Nat → Nat) (cs r : Nat) + (h : ∀ k < r, g k + cs ≤ (f k).size) : + ((List.range r).foldl (fun acc k ↦ acc ++ (f k).extract (g k) (g k + cs)) #[]).size = + r * cs ∧ + ∀ k < r, ∀ c < cs, + ((List.range r).foldl (fun acc k ↦ acc ++ (f k).extract (g k) (g k + cs)) #[]).getD + (k * cs + c) 0 = (f k).getD (g k + c) 0 := by + induction r with + | zero => simp only [List.range_zero, List.foldl_nil, Array.size_empty, Nat.zero_mul, + Nat.not_lt_zero, false_imp_iff, implies_true, and_self] + | succ r ih => + obtain ⟨hs, hv⟩ := ih fun k hk ↦ h k (by omega) + rw [List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil] + have he : (Array.extract (f r) (g r) (g r + cs)).size = cs := by + have := h r (by omega) + rw [Array.size_extract]; omega + refine ⟨by rw [Array.size_append, hs, he, Nat.succ_mul], fun k hk c hc ↦ ?_⟩ + simp only [Array.getD_eq_getD_getElem?] + by_cases hkr : k < r + · have hlt : k * cs + c < r * cs := by + have := Nat.mul_le_mul_right cs (show k + 1 ≤ r by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega + rw [Array.getElem?_append_left (by rw [hs]; exact hlt)] + have := hv k hkr c hc + simpa only [Array.getD_eq_getD_getElem?] using this + · have hkr' : k = r := by omega + subst hkr' + rw [Array.getElem?_append_right (by rw [hs]; omega), hs, + show k * cs + c - k * cs = c by omega] + have := getD_extract (f k) (g k) cs c 0 (h k hk) hc + simpa only [Array.getD_eq_getD_getElem?] using this + +namespace Native + +/-- Gathered words pack the gathered rows. -/ +theorem gatherWords_encode (src : Array F) (M j0 cs r : Nat) + (hsrc : ∀ k < r, k * M + j0 + cs ≤ src.size) + (hu : src.size < USize.size ∧ r * cs < USize.size) : + gatherWords (encode src) M j0 cs r = encode (Columns.gatherRows src M j0 cs r) := by + unfold gatherWords + rw [show ByteArray.emptyWithCapacity (4 * (r * cs)) = encode (#[] : Array F) from rfl, + foldl_copySlice_encode (fun _ ↦ src) (fun k ↦ k * M + j0) cs _ _] + obtain ⟨has, hav⟩ := appendRows_spec (fun _ ↦ src) (fun k ↦ k * M + j0) cs r hsrc + obtain ⟨hgs, hgv⟩ := Columns.gatherRows_spec src M j0 cs r hsrc hu + congr 1 + apply array_eq_of_getD _ _ 0 (by rw [has, hgs]) + intro i + by_cases hi : i < r * cs + · have hcs : 0 < cs := Nat.pos_of_ne_zero (by rintro rfl; simp at hi) + have hk : i / cs < r := by rw [Nat.div_lt_iff_lt_mul hcs]; exact hi + rw [← Nat.div_add_mod' i cs, hav _ hk _ (Nat.mod_lt _ hcs), hgv _ hk _ (Nat.mod_lt _ hcs), + Nat.add_assoc] + · rw [Columns.getD_of_size_le _ _ (by omega), Columns.getD_of_size_le _ _ (by omega)] + +/-- A packed column task packs the field-level column task. -/ +theorem columnWords_encode (x : Array F) (tw : Array (Array F)) (logN cs j0 : Nat) + (h4 : 4 ≤ logN) (hx : x.size = 2 ^ logN) (htw : ∀ s < logN, (tw.getD s #[]).size = 2 ^ s) + (hu : 4 * 2 ^ logN < USize.size) (hj : j0 + cs ≤ 2 ^ (logN - 4)) : + columnWords p (tw.map encode) logN (2 ^ (logN - 4)) cs j0 (encode x) = + encode (Columns.columnTask x tw logN (2 ^ (logN - 4)) cs j0) := by + obtain ⟨m, rfl⟩ : ∃ m, logN = m + 4 := ⟨logN - 4, by omega⟩ + simp only [show m + 4 - 4 = m by omega] at hj ⊢ + have hp (k : Nat) : 2 ^ (m + k) = 2 ^ k * 2 ^ m := by rw [Nat.pow_add, Nat.mul_comm] + have hM := Nat.two_pow_pos m + have hx' : x.size = 16 * 2 ^ m := by rw [hx, hp 4]; rfl + have hu' : 4 * (16 * 2 ^ m) < USize.size := by rw [hp 4] at hu; exact hu + have hrow (s k : Nat) (hs : s < m + 4) (hk : 2 ^ s = k * 2 ^ m) : + ∀ r < k, r * 2 ^ m + j0 + cs ≤ (tw.getD s #[]).size := by + intro r hr + rw [htw s hs, hk] + have := Nat.mul_le_mul_right (2 ^ m) (show r + 1 ≤ k by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega + have hts (s k : Nat) (hs : s < m + 4) (hk : 2 ^ s = k * 2 ^ m) (hk16 : k ≤ 16) : + (tw.getD s #[]).size = k * 2 ^ m ∧ (tw.getD s #[]).size < USize.size := by + rw [htw s hs, hk] + exact ⟨rfl, by have := Nat.mul_le_mul_right (2 ^ m) hk16; omega⟩ + have hcs16 (k : Nat) (hk : k ≤ 16) : 4 * (k * cs) < USize.size := by + have := Nat.mul_le_mul_right cs hk + have : 16 * cs ≤ 16 * 2 ^ m := by omega + omega + have hg (s k : Nat) (hs : s < m + 4) (hk : 2 ^ s = k * 2 ^ m) (hk16 : k ≤ 16) := + gatherWords_encode (tw.getD s #[]) (2 ^ m) j0 cs k (hrow s k hs hk) + ⟨(hts s k hs hk hk16).2, by have := hcs16 k hk16; omega⟩ + have hgs (s k : Nat) (hs : s < m + 4) (hk : 2 ^ s = k * 2 ^ m) (hk16 : k ≤ 16) := + (Columns.gatherRows_spec (tw.getD s #[]) (2 ^ m) j0 cs k (hrow s k hs hk) + ⟨(hts s k hs hk hk16).2, by have := hcs16 k hk16; omega⟩).1 + have e1 : 2 ^ (m + 4 - 1) = 8 * 2 ^ m := by rw [show m + 4 - 1 = m + 3 by omega, hp]; rfl + have e2 : 2 ^ (m + 4 - 2) = 4 * 2 ^ m := by rw [show m + 4 - 2 = m + 2 by omega, hp]; rfl + have e3 : 2 ^ (m + 4 - 3) = 2 * 2 ^ m := by rw [show m + 4 - 3 = m + 1 by omega, hp]; rfl + have e4 : 2 ^ (m + 4 - 4) = 1 * 2 ^ m := by rw [show m + 4 - 4 = m by omega, Nat.one_mul] + have hxr : ∀ r < 16, r * 2 ^ m + j0 + cs ≤ x.size := fun r hr ↦ by + have := Nat.mul_le_mul_right (2 ^ m) (show r + 1 ≤ 16 by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega + have hxu : x.size < USize.size ∧ 16 * cs < USize.size := ⟨by omega, by have := hcs16 16; omega⟩ + obtain ⟨hxs, -⟩ := Columns.gatherRows_spec x (2 ^ m) j0 cs 16 hxr hxu + unfold columnWords Columns.columnTask Columns.columnPasses + simp only [getD_map_encode] + rw [hg _ 8 (by omega) e1 (by omega), hg _ 4 (by omega) e2 (by omega), + hg _ 2 (by omega) e3 (by omega), hg _ 1 (by omega) e4 (by omega), + gatherWords_encode x (2 ^ m) j0 cs 16 hxr hxu] + have hfs (a : Array F) (ha : a.size = 16 * cs) : (encode a).size < USize.size := by + rw [size_encode_fields, ha]; exact hcs16 16 (by omega) + have hrs (s k : Nat) (hs : s < m + 4) (hk : 2 ^ s = k * 2 ^ m) (hk16 : k ≤ 16) : + (encode (Columns.gatherRows (tw.getD s #[]) (2 ^ m) j0 cs k)).size < USize.size := by + rw [size_encode_fields, hgs s k hs hk hk16]; exact hcs16 k hk16 + rw [Native.radix4Blocks_encode _ _ _ _ _ (hfs _ hxs) (hrs _ 8 (by omega) e1 (by omega)) + (hrs _ 4 (by omega) e2 (by omega)) (by rw [hgs _ 8 (by omega) e1 (by omega)]; omega) + (hgs _ 4 (by omega) e2 (by omega)).ge (by rw [hxs]; omega), + Native.radix4Blocks_encode _ _ _ _ _ (hfs _ (by rw [Plan.size_butterflyDIFRadix4Blocks, hxs])) + (hrs _ 2 (by omega) e3 (by omega)) (hrs _ 1 (by omega) e4 (by omega)) + (hgs _ 2 (by omega) e3 (by omega)).ge + (by rw [hgs _ 1 (by omega) e4 (by omega)]; omega) + (by rw [Plan.size_butterflyDIFRadix4Blocks, hxs]; omega)] + simp only [Plan.stageBlocksDIF_eq, Plan.checkedBlocks_eq, show 4 * (4 * cs) = 16 * cs by ring] + +/-- A leaf's words pack the concatenated row ranges of the column results. -/ +theorem leafWords_encode (C : Nat → Array F) (S cs l : Nat) : + leafWords ((Array.range S).map fun s ↦ encode (C s)) cs l = + encode ((List.range S).foldl (fun acc s ↦ acc ++ (C s).extract (l * cs) (l * cs + cs)) + #[]) := by + unfold leafWords + rw [Array.foldl_map, ← Array.foldl_toList, Array.toList_range, Array.size_map, Array.size_range, + show ByteArray.emptyWithCapacity (4 * (S * cs)) = encode (#[] : Array F) from rfl] + exact foldl_copySlice_encode C (fun _ ↦ l * cs) cs _ _ + +/-- A leaf's DIF stages are the column transform's leaf layers. -/ +theorem leafStages_eq_leafPasses (D : NTT.Domain F) (h4 : 4 ≤ D.logN) (tw : Array (Array F)) + (X : Array F) (hX : X.size = 2 ^ (D.logN - 4)) : + Plan.runStagesDIFRadix4WithTwiddles (powDomain D 4 h4) tw X = + Columns.leafPasses tw ((Parallel.forwardPasses D).drop 2) (D.logN % 2 == 1) X := by + let D4 := powDomain D 4 h4 + have hp : Parallel.forwardPasses D4 = (Parallel.forwardPasses D).drop 2 := by + unfold Parallel.forwardPasses + apply List.ext_getElem (by + simp only [D4, powDomain, List.length_map, List.length_range, List.length_drop]; omega) + intro i _ _ + simp only [D4, powDomain, List.getElem_map, List.getElem_range, List.getElem_drop] + omega + have hX4 : X.size = D4.n := hX + rw [← Plan.checkedStages_eq, ← Parallel.forwardStages_eq D4 tw X hX4] + unfold Columns.leafPasses + rw [← hp] + have hl : D4.logN = D.logN - 4 := rfl + by_cases ho : D.logN % 2 = 1 + · simp only [hl, show (D.logN - 4) % 2 = 1 by omega, ho, beq_self_eq_true, ↓reduceIte] + rw [NaturalPlan.lastPair_eq D4 _ _ (by rw [Parallel.size_runPasses, hX4])] + · simp only [hl, show ¬(D.logN - 4) % 2 = 1 by omega, ho, beq_iff_eq, ↓reduceIte] + +/-- The packed column transform packs the natural-order read of the DIF transform. -/ +theorem runColumns_encode (D : NTT.Domain F) (S : Nat) (factor : F) (a : Array F) + (inverse : Bool) (hs : a.size = D.n) (h15 : 15 ≤ D.logN) (h32 : D.logN ≤ 32) + (hu : 4 * D.n < USize.size) (hS : 0 < S) (hdiv : S ∣ 2 ^ (D.logN - 4)) : + runColumns p ((Plan.twiddleTable D).map encode) D.logN S factor.val inverse (encode a) = + encode (decodedFields D.logN (difSpec D a) factor inverse) := by + let tw := Plan.twiddleTable D + obtain ⟨htw, hone⟩ := twiddleTable_invariants D + have hn : a.size = 2 ^ D.logN := hs + have hu' : 4 * 2 ^ D.logN < USize.size := hu + let M := 2 ^ (D.logN - 4) + have hM : 2 ^ D.logN = 16 * M := by + rw [show D.logN = D.logN - 4 + 4 by omega, Nat.pow_add, Nat.mul_comm] + have hM2 : M % 2 = 0 := by + simp only [M]; rw [show D.logN - 4 = D.logN - 5 + 1 by omega, Nat.pow_succ, Nat.mul_mod_left] + obtain ⟨cs, hcs⟩ := hdiv + have hcs' : M = S * cs := hcs + have hcsM : 2 ^ (D.logN - 4) / S = cs := by rw [hcs, Nat.mul_div_cancel_left _ hS] + let T := Columns.topPasses tw D.logN a + have hT : T.size = 16 * M := by + simp only [T, Columns.topPasses, Parallel.size_step, hn, hM] + let C := fun s ↦ Columns.columnTask a tw D.logN M cs (s * cs) + have hj (s : Nat) (hsS : s < S) : s * cs + cs ≤ 2 ^ (D.logN - 4) := by + rw [hcs] + have := Nat.mul_le_mul_right cs (show s + 1 ≤ S by omega) + rw [Nat.add_mul, Nat.one_mul] at this + exact this + have hcol (s : Nat) (hsS : s < S) := + Columns.columnTask_col a tw D.logN cs (s * cs) (by omega) hn htw (by omega) (hj s hsS) + let X := fun l ↦ T.extract (l * M) (l * M + M) + have hleafIn (l : Nat) (hl : l < 16) : + leafWords ((Array.range S).map fun s ↦ encode (C s)) cs l = encode (X l) := by + have hlc : l * cs + cs ≤ 16 * cs := by + have := Nat.mul_le_mul_right cs (show l + 1 ≤ 16 by omega) + rw [Nat.add_mul, Nat.one_mul] at this; exact this + have hC : ∀ s < S, l * cs + cs ≤ (C s).size := fun s hsS ↦ by + rw [(hcol s hsS).1]; exact hlc + have hlM : l * M + M ≤ T.size := by + rw [hT] + have := Nat.mul_le_mul_right M (show l + 1 ≤ 16 by omega) + rw [Nat.add_mul, Nat.one_mul] at this; exact this + obtain ⟨hbs, hbv⟩ := appendRows_spec C (fun _ ↦ l * cs) cs S hC + rw [leafWords_encode C S cs l] + congr 1 + apply array_eq_of_getD _ _ 0 (by + rw [hbs, Array.size_extract, Nat.min_eq_left hlM, ← hcs']; omega) + intro k + by_cases hk : k < M + · have hcs0 : 0 < cs := Nat.pos_of_ne_zero (by rintro rfl; simp only [M] at hk; omega) + have hks : k / cs < S := by rw [Nat.div_lt_iff_lt_mul hcs0, ← hcs]; exact hk + rw [← Nat.div_add_mod' k cs, hbv _ hks _ (Nat.mod_lt _ hcs0), + (hcol _ hks).2 l hl _ (Nat.mod_lt _ hcs0), getD_extract T (l * M) M _ 0 hlM (by + rw [Nat.div_add_mod' k cs]; exact hk)] + · rw [Columns.getD_of_size_le _ _ (by rw [hbs, ← hcs]; omega), + Columns.getD_of_size_le _ _ (by rw [Array.size_extract]; omega)] + have hX (l : Nat) (hl : l < 16) : (X l).size = 2 ^ (D.logN - 4) := by + simp only [X, Array.size_extract, hT] + have := Nat.mul_le_mul_right M (show l + 1 ≤ 16 by omega) + rw [Nat.add_mul, Nat.one_mul] at this + omega + have hleaf (l : Nat) (hl : l < 16) : + leafNatural p (tw.map encode) (D.logN - 4) factor.val inverse (encode (X l)) = + encode (decodedFields (D.logN - 4) ((difSpec D a).extract (l * M) ((l + 1) * M)) + factor inverse) := by + have htw4 : TwiddleSizes (powDomain D 4 (by omega)).logN tw := fun s hs ↦ htw s (by + simp only [powDomain] at hs; omega) + have hone4 : TwiddleOnes (powDomain D 4 (by omega)).logN tw := fun s hs ↦ hone s (by + simp only [powDomain] at hs; omega) + have hp : 2 ^ (D.logN - 4) ≤ 2 ^ D.logN := Nat.pow_le_pow_right (by omega) (by omega) + erw [Native.leafNatural_encode (powDomain D 4 (by omega)) tw (X l) factor inverse htw4 hone4 + (hX l hl) (by simp only [powDomain, NTT.Domain.n]; omega) + (by simp only [powDomain]; omega)] + rw [leafStages_eq_leafPasses D (by omega) tw (X l) (hX l hl)] + simp only [X] + rw [show l * M + M = (l + 1) * M by ring, + Columns.leafPasses_extract tw _ _ T M 16 l hT hl hM2 + (NaturalPlan.dvd_of_mem_forwardPasses_drop D), + NaturalPlan.leafPasses_topPasses D tw a hs (by omega), Plan.checkedStages_eq, + planStages_eq_difSpec D a hs] + rfl + have hcols : ((Array.range S).map fun s ↦ + columnWords p (tw.map encode) D.logN M cs (s * cs) (encode a)) = + (Array.range S).map fun s ↦ encode (C s) := by + apply Array.ext (by simp only [Array.size_map, Array.size_range]) + intro s h1 _ + simp only [Array.size_map, Array.size_range] at h1 + simp only [Array.getElem_map, Array.getElem_range] + exact columnWords_encode a tw D.logN cs (s * cs) (by omega) hn htw hu' (hj s h1) + have hleaves : (Tasks.spawnAfter (Tasks.spawnAll S fun s ↦ + columnWords p (tw.map encode) D.logN M cs (s * cs) (encode a)) 16 fun cols l ↦ + leafNatural p (tw.map encode) (D.logN - 4) factor.val inverse (leafWords cols cs l)).map + Task.get = + (Array.range 16).map fun l ↦ encode (decodedFields (D.logN - 4) + ((difSpec D a).extract (l * M) ((l + 1) * M)) factor inverse) := by + rw [Tasks.map_get_spawnAfter, Tasks.map_get_spawnAll, hcols] + apply Array.ext (by simp only [Array.size_map, Array.size_range]) + intro l h1 _ + simp only [Array.size_map, Array.size_range] at h1 + rw [Array.getElem_map, Array.getElem_map, Array.getElem_range, hleafIn l h1, hleaf l h1] + dsimp only [runColumns] + rw [hcsM, hleaves, naturalLeaves_encode (difSpec D a) D.logN factor inverse (by omega) h32 + (by rw [size_difSpec]; rfl) hu'] + +/-- The packed transform computes the DFT, scaled for an inverse transform. -/ +theorem run_dft (D : NTT.Domain F) (parallel : Bool) (factor : F) (a : Array F) + (inverse : Bool) (hs : a.size = D.n) (h32 : D.logN ≤ 32) (hu : 4 * D.n < USize.size) : + run p ((Plan.twiddleTable D).map encode) D.logN parallel factor.val inverse (encode a) = + encode (if inverse then (NTT.Forward.forwardSpec D a).map (fun x ↦ factor * x) + else NTT.Forward.forwardSpec D a) := by + rw [← decodedFields_difSpec D a factor inverse] + unfold run + split + · rename_i h + exact runColumns_encode D _ factor a inverse hs h.2 h32 hu (Nat.two_pow_pos _) + (Nat.pow_dvd_pow 2 (by omega)) + · obtain ⟨htw, hone⟩ := twiddleTable_invariants D + rw [Native.leafNatural_encode D _ a factor inverse htw hone hs hu h32, + planStages_eq_difSpec D a hs] + +end Native +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/CosetLde.lean b/CompPoly/Univariate/NTTFast/Packed/CosetLde.lean new file mode 100644 index 00000000..2f85471a --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/CosetLde.lean @@ -0,0 +1,409 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.NativeLast +public import CompPoly.Univariate.NTTFast.Packed.DifSpec +public import CompPoly.Univariate.NTTFast.Coset +public import CompPoly.Data.Tasks +import CompPoly.Univariate.NTTFast.Packed.Unrolled + +/-! # Batched coset low-degree extension over packed words + +The low-degree extension of a row-major `h × w` matrix, `h = 2 ^ logH`, by `2 ^ bits`: each +column holds the values of a polynomial `p` of degree below `h` on the subgroup of order `h`, +and output row `r` holds the values of every column's `p` at `shift · ω ^ bitrev r`, where `ω` +generates the subgroup of order `H = h · 2 ^ bits`. That is Plonky3's `coset_lde_batch`, in the +bit-reversed row order its two-adic FRI commitment commits. + +In that order the output splits into `2 ^ bits` blocks of `h` rows: block `t` holds the coset +`shift · ω ^ c · ⟨ω ^ (2 ^ bits)⟩`, `c = bitrev t`, in DIF order. So each column is one inverse +transform followed by `2 ^ bits` forward transforms of size `h`, each of the coefficients +scaled by the powers of `shift · ω ^ c` (and by `1 / h`), with no padding and no reversal. + +The input comes in chunks of consecutive rows of equal heights, so that a caller producing its +rows in parallel need not join them. Tasks transpose row ranges of the chunks into columns, +extend each column, and transpose row ranges of the extended columns back into rows. The result +stays in those row chunks, which a commitment hashes in parallel; `joinRows` copies them into one +buffer. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +/-- The scaled coset powers `(1 / h) · (shift · ω ^ c) ^ j`, `j < h`, of coset `c` of the +subgroup of order `h = 2 ^ (D.logN - bits)` in `D`'s. -/ +def cosetTable (D : NTT.Domain (FastField p)) (bits : Nat) (shift : FastField p) (c : Nat) : Array + (FastField p) := + (NTTFast.powers (shift * D.omega ^ c) (2 ^ (D.logN - bits))).map + (((2 ^ (D.logN - bits) : Nat) : FastField p)⁻¹ * ·) + +/-- A reusable plan for low-degree extensions of `2 ^ (domain.logN - bits)`-row matrices onto +the coset `shift · ⟨domain.omega⟩`, with certified cached twiddles and coset tables. -/ +structure LdePlan (p : ℕ) [Mont32Field p] where + domain : NTT.Domain (FastField p) + bits : Nat + bits_le : bits ≤ domain.logN + shift : FastField p + log_bound : domain.logN ≤ 32 + byte_bound : 4 * domain.n < USize.size + forwardTwiddles : Array ByteArray + inverseTwiddles : Array ByteArray + cosetTables : Array ByteArray + forward_eq : forwardTwiddles = + (NTTFast.Plan.twiddleTable (powDomain domain bits bits_le)).map encode + inverse_eq : inverseTwiddles = + (NTTFast.Plan.twiddleTable (powDomain domain bits bits_le).inverse).map encode + tables_eq : cosetTables = + (Array.range (2 ^ bits)).map fun c ↦ encode (cosetTable domain bits shift c) + +namespace LdePlan + +/-- Encode the twiddles of the subgroup of order `2 ^ (D.logN - bits)` and the coset tables +once. -/ +def ofDomain (D : NTT.Domain (FastField p)) (bits : Nat) (hb : bits ≤ D.logN) (shift : FastField p) + (h32 : D.logN ≤ 32) (hu : 4 * D.n < USize.size) : LdePlan p where + domain := D + bits := bits + bits_le := hb + shift := shift + log_bound := h32 + byte_bound := hu + forwardTwiddles := (NTTFast.Plan.twiddleTable (powDomain D bits hb)).map Native.encodeFields + inverseTwiddles := + (NTTFast.Plan.twiddleTable (powDomain D bits hb).inverse).map Native.encodeFields + cosetTables := (Array.range (2 ^ bits)).map fun c ↦ + Native.encodeFields (cosetTable D bits shift c) + forward_eq := by + apply congrArg (Array.map · _) + funext a + exact Native.encodeFields_eq a + inverse_eq := by + apply congrArg (Array.map · _) + funext a + exact Native.encodeFields_eq a + tables_eq := by + apply congrArg (Array.map · _) + funext c + exact Native.encodeFields_eq _ + +/-- The rows of the input matrix. -/ +abbrev logH (P : LdePlan p) : Nat := P.domain.logN - P.bits + +end LdePlan + +namespace Native + +/-- The products of words `i, …, i + 15` of `t` and `a`, stored from word `i` of `out`. -/ +@[noinline, specialize] +def mul16 (p : ℕ) [Mont32Field p] (t a : @& ByteArray) (i : USize) (out : ByteArray) + (h : 4 * (i.toNat + 15) + 3 < t.size ∧ 4 * (i.toNat + 15) + 3 < a.size ∧ + t.size < USize.size ∧ a.size < USize.size) : ByteArray := + mul16_words% + +/-- `count` sixteen-word products from word `i`, after one range check. -/ +@[specialize] +def mul16Loop (p : ℕ) [Mont32Field p] (t a : @& ByteArray) : + (count : Nat) → (i : USize) → (out : ByteArray) → + 4 * (i.toNat + 16 * count) ≤ t.size ∧ 4 * (i.toNat + 16 * count) ≤ a.size ∧ + t.size < USize.size ∧ a.size < USize.size → ByteArray + | 0, _, out, _ => out + | count + 1, i, out, h => + have hi : (i + 16).toNat = i.toNat + 16 := usize_add16 i t.size (by omega) h.2.2.1 + mul16Loop p t a count (i + 16) (mul16 p t a i out (by omega)) (by rw [hi]; omega) + +/-- The products of words `0, …, n - 1` of `t` and `a`: whole sixteen-word blocks after one +range check, then word by word. -/ +@[specialize] +def mulWords (p : ℕ) [Mont32Field p] (t a : @& ByteArray) (n : Nat) : ByteArray := Id.run do + let mut out := zeroWords n + if h : 4 * (16 * (n / 16)) ≤ t.size ∧ 4 * (16 * (n / 16)) ≤ a.size ∧ + t.size < USize.size ∧ a.size < USize.size then + out := mul16Loop p t a (n / 16) 0 out (by rw [USize.toNat_zero]; omega) + for k in [16 * (n / 16):n] do + out := write out k (mul p (read t k) (read a k)) + return out + +/-- The transpose of the `16 × 16` tile whose line `t` is words `q + t * qs, …, q + t * qs + 15` +of `s`, line `u` stored from word `p + u * ps` of `out`. -/ +@[noinline] def tileStrided (s : @& ByteArray) (q qs : USize) (out : ByteArray) (p ps : USize) : + ByteArray := + tile16_strided% + +/-- Rows `r0, …, r0 + rows - 1` of columns `c0, …, c0 + n - 1` of a row-major matrix of width +`w`, transposed: column `c0 + j` at words `j * rows, …`. Whole `16 × 16` tiles move as tiles, a +block of up to 1024 rows at a time and down the block before across, so that each column receives +four kilobytes per block and the pages both sides touch stay few; the rest moves word by word. -/ +def transposeRows (m : @& ByteArray) (w c0 n r0 rows : Nat) : ByteArray := Id.run do + let mut out := zeroWords (n * rows) + for rb in [:(rows / 16 + 63) / 64] do + for ct in [:n / 16] do + for rt in [64 * rb:min (64 * rb + 64) (rows / 16)] do + out := tileStrided m ((r0 + 16 * rt) * w + c0 + 16 * ct).toUSize w.toUSize out + (16 * ct * rows + 16 * rt).toUSize rows.toUSize + for j in [:n] do + let r1 := if j < 16 * (n / 16) then 16 * (rows / 16) else 0 + for r in [r1:rows] do + out := write out (j * rows + r) (read m ((r0 + r) * w + c0 + j)) + return out + +/-- Column `j` from the transposed row ranges `ps`, each `rows` rows long. -/ +def gatherColumn (ps : @& Array ByteArray) (rows j : Nat) : ByteArray := + ps.foldl (fun out p ↦ p.copySlice (4 * (j * rows)) out out.size (4 * rows)) + (ByteArray.emptyWithCapacity (4 * (ps.size * rows))) + +/-- Words `q, …, q + 7` fit in `b`, checked on machine words. -/ +abbrev Line8Fits (b : ByteArray) (q : USize) : Prop := 8 ≤ b.usize / 4 ∧ q ≤ b.usize / 4 - 8 + +theorem line8Fits_toNat {b : ByteArray} {q : USize} (h : Line8Fits b q) (k : Nat) (hk : k < 8) : + ((q + OfNat.ofNat k) * 4).toNat = 4 * (q.toNat + k) ∧ 4 * (q.toNat + k) + 4 ≤ b.size := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hb := toNat_usize_le b + have hu := b.usize.toNat_lt_size + have h8 : (8 : USize).toNat = 8 := usize_numeral 8 (by decide) + have h4 : (4 : USize).toNat = 4 := usize_numeral 4 (by decide) + obtain ⟨h1, h2⟩ := h + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, h8] at h1 + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ (by + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, h8]; exact h1), USize.toNat_div, h4, h8] + at h2 + have hk' : (OfNat.ofNat k : USize).toNat = k := usize_numeral k (by omega) + have hadd : (q + OfNat.ofNat k).toNat = q.toNat + k := by + rw [USize.toNat_add, hk', hsize, Nat.mod_eq_of_lt (by omega)] + refine ⟨?_, by omega⟩ + rw [USize.toNat_mul, hadd, h4, hsize, Nat.mod_eq_of_lt (by omega)] + omega + +/-- A word offset that fits a buffer of `n < USize.size` bytes does not wrap. -/ +theorem wordOffset_toNat' (i : USize) (h : 4 * i.toNat < USize.size) : + (i * 4).toNat = 4 * i.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_mul, usize_numeral 4 (by decide), hsize, Nat.mod_eq_of_lt (by omega)] + omega + +/-- Lines `p + u * ps`, `u < 8`, of eight words fit in `b`, checked on machine words. -/ +abbrev Tile8Fits (b : ByteArray) (p ps : USize) : Prop := + ps ≤ b.usize / 28 ∧ 8 + 7 * ps ≤ b.usize / 4 ∧ p ≤ b.usize / 4 - (8 + 7 * ps) + +/-- The check of `Tile8Fits` in natural numbers. -/ +theorem tile8Fits_iff (b : ByteArray) (p ps : USize) : + Tile8Fits b p ps ↔ 28 * ps.toNat ≤ b.usize.toNat ∧ + 4 * (p.toNat + 7 * ps.toNat + 8) ≤ b.usize.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hU := b.usize.toNat_lt_size + have hle := USize.le_size + have h4 : (4 : USize).toNat = 4 := usize_numeral 4 (by decide) + have h7 : (7 : USize).toNat = 7 := usize_numeral 7 (by decide) + have h8 : (8 : USize).toNat = 8 := usize_numeral 8 (by decide) + have h28 : (28 : USize).toNat = 28 := usize_numeral 28 (by decide) + unfold Tile8Fits + rw [USize.le_iff_toNat_le, USize.toNat_div, h28] + constructor + · rintro ⟨h1, h2, h3⟩ + have e8 : (8 + 7 * ps).toNat = 8 + 7 * ps.toNat := by + rw [USize.toNat_add, USize.toNat_mul, h7, h8, hsize, + Nat.mod_eq_of_lt (show 7 * ps.toNat < USize.size by omega), Nat.mod_eq_of_lt (by omega)] + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, e8] at h2 + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ (by + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, e8]; exact h2), USize.toNat_div, h4, e8] + at h3 + omega + · rintro ⟨h1, h2⟩ + have e8 : (8 + 7 * ps).toNat = 8 + 7 * ps.toNat := by + rw [USize.toNat_add, USize.toNat_mul, h7, h8, hsize, + Nat.mod_eq_of_lt (show 7 * ps.toNat < USize.size by omega), Nat.mod_eq_of_lt (by omega)] + have h2' : 8 + 7 * ps ≤ b.usize / 4 := by + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, e8]; omega + refine ⟨by omega, h2', ?_⟩ + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ h2', USize.toNat_div, h4, e8] + omega + +theorem tile8Fits_toNat {b : ByteArray} {p ps : USize} (h : Tile8Fits b p ps) (u : Nat) + (hu : u < 8) : ((p + OfNat.ofNat u * ps) * 4).toNat + 4 * 8 ≤ b.size ∧ + ((p + OfNat.ofNat u * ps) * 4).toNat + 4 * 8 < USize.size := by + have hb := toNat_usize_le b + have hU := b.usize.toNat_lt_size + have hn := (tile8Fits_iff b p ps).mp h + have hm : u * ps.toNat ≤ 7 * ps.toNat := Nat.mul_le_mul_right _ (by omega) + have eu : (OfNat.ofNat u * ps : USize).toNat = u * ps.toNat := + usize_mul_toNat u ps (by omega) (by omega) + rw [wordOffset_toNat' _ (by rw [usize_add_toNat _ _ (by omega), eu]; omega), + usize_add_toNat _ _ (by omega), eu] + omega + +/-- A tile that fits has room for the touched last word. -/ +theorem tile8Fits_usize {b : ByteArray} {p ps : USize} (h : Tile8Fits b p ps) : 4 ≤ b.usize := by + have hn := (tile8Fits_iff b p ps).mp h + rw [USize.le_iff_toNat_le, usize_numeral 4 (by decide)] + omega + +/-- The transpose of the `8 × 8` tile whose line `t` is words `q, …, q + 7` of `s{t}`, line `u` +stored from word `p + u * ps` of `out`, if every line fits. Eight lines of eight words fit the +vector registers. -/ +@[noinline] def tile8 (s0 s1 s2 s3 s4 s5 s6 s7 : @& ByteArray) (q : USize) (out : ByteArray) + (p ps : USize) : ByteArray := + tile8_sources% + +/-- `count` transposed `8 × 8` tiles of the columns `s0, …, s7`: the tile at column word `q + 8 * k` +goes to the rows of width `w` from word `p + 8 * k * w` of `out`. The columns are bound once, so +tiles do not touch their reference counts. -/ +def rowsTiles (s0 s1 s2 s3 s4 s5 s6 s7 : @& ByteArray) (w : USize) : + Nat → USize → USize → ByteArray → ByteArray + | 0, _, _, out => out + | count + 1, q, p, out => + rowsTiles s0 s1 s2 s3 s4 s5 s6 s7 w count (q + 8) (p + 8 * w) + (tile8 s0 s1 s2 s3 s4 s5 s6 s7 q out p w) + +/-- Rows `r0, …, r0 + rows - 1` of the row-major matrix with columns `cs`. Whole `8 × 8` tiles +move as tiles, a block of up to 512 rows at a time so that each block is completed in cache; the +rest moves word by word. -/ +def rowsOfColumns (cs : @& Array ByteArray) (r0 rows : Nat) : ByteArray := Id.run do + let w := cs.size + let block := min rows 512 + let mut out := zeroWords (rows * w) + for rb in [:rows / block] do + for ct in [:w / 8] do + let s (t : Nat) : ByteArray := cs.getD (8 * ct + t) .empty + out := rowsTiles (s 0) (s 1) (s 2) (s 3) (s 4) (s 5) (s 6) (s 7) w.toUSize (block / 8) + (r0 + rb * block).toUSize (rb * block * w + 8 * ct).toUSize out + for r in [:rows] do + let j1 := if r < 8 * (rows / 8) then 8 * (w / 8) else 0 + for j in [j1:w] do + out := write out (r * w + j) (read (cs.getD j .empty) (r0 + r)) + return out + +/-- The coefficients of a column, scaled by `h`: its unnormalized inverse transform. -/ +@[specialize] +def columnCoefficients (P : LdePlan p) (col : ByteArray) : ByteArray := + leafNatural p P.inverseTwiddles P.logH 0 false col + +/-- The first radix-two DIF layer's butterflies `j, …, j + 15` of `c ⊙ T`, `c` and `T` read at +words `j` and `j + h`, stored at the same words of `out`. -/ +@[noinline, specialize] +def pair16Scaled (p : ℕ) [Mont32Field p] (w T c : @& ByteArray) (j i0 i1 : USize) + (out : ByteArray) + (h : 4 * (i0.toNat + 15) + 3 < c.size ∧ 4 * (i1.toNat + 15) + 3 < c.size ∧ + 4 * (i0.toNat + 15) + 3 < T.size ∧ 4 * (i1.toNat + 15) + 3 < T.size ∧ + 4 * (j.toNat + 15) + 3 < w.size ∧ c.size < USize.size ∧ T.size < USize.size ∧ + w.size < USize.size) : ByteArray := + pair16Scaled_words% + +/-- `n` batches of `pair16Scaled` from butterfly `j` of half `h`, after one range check. -/ +@[specialize] +def pair16ScaledLoop (p : ℕ) [Mont32Field p] (w T c : @& ByteArray) (h : USize) : + (n : Nat) → (j : USize) → (out : ByteArray) → + 4 * (h.toNat + j.toNat + 16 * n) ≤ c.size ∧ 4 * (h.toNat + j.toNat + 16 * n) ≤ T.size ∧ + 4 * (j.toNat + 16 * n) ≤ w.size ∧ c.size < USize.size ∧ T.size < USize.size ∧ + w.size < USize.size → ByteArray + | 0, _, out, _ => out + | n + 1, j, out, hb => + have hj : (j + 16).toNat = j.toNat + 16 := usize_add16 j c.size (by omega) hb.2.2.2.1 + have hh : (j + h).toNat = j.toNat + h.toNat := usize_add_toNat _ _ (by omega) + pair16ScaledLoop p w T c h n (j + 16) + (pair16Scaled p w T c j j (j + h) out (by rw [hh]; omega)) (by rw [hj]; omega) + +/-- The first radix-two DIF layer of the `2 h`-word `c ⊙ T`, out of place: whole batches of +sixteen butterflies after one range check, then butterfly by butterfly. -/ +@[specialize] +def pairsScaled (p : ℕ) [Mont32Field p] (w T c : @& ByteArray) (h : Nat) : ByteArray := Id.run do + let mut out := zeroWords (2 * h) + if hb : 4 * (h + 16 * (h / 16)) ≤ c.size ∧ 4 * (h + 16 * (h / 16)) ≤ T.size ∧ + 4 * (16 * (h / 16)) ≤ w.size ∧ c.size < USize.size ∧ T.size < USize.size ∧ + w.size < USize.size then + out := pair16ScaledLoop p w T c (USize.ofNatLT h (by omega)) (h / 16) 0 out (by + rw [USize.toNat_ofNatLT, USize.toNat_zero]; omega) + for k in [16 * (h / 16):h] do + let x := mul p (read T k) (read c k) + let y := mul p (read T (h + k)) (read c (h + k)) + out := write (write out k (add p x y)) (h + k) (mul p (read w k) (sub p x y)) + return out + +/-- `leafReversed` of an odd leaf after its radix-two layer. -/ +@[specialize] +def leafReversedOdd (p : ℕ) [Mont32Field p] (tw : @& Array ByteArray) (m : Nat) + (a : ByteArray) : ByteArray := + let a := (List.range (if 4 ≤ m then m / 2 - 2 else m / 2)).foldl (fun a pass ↦ + let high := m - 1 - m % 2 - 2 * pass + let q := 2 ^ (high - 1) + radix4Blocks p (tw.getD high .empty) (tw.getD (high - 1) .empty) q (2 ^ m / (4 * q)) a) a + if 4 ≤ m then + leaf16Blocks p (tw.getD 3 .empty) (tw.getD 2 .empty) (tw.getD 1 .empty) (2 ^ m / 16) a + else a + +/-- Block `t` of a column's extension: the DIF-order values on coset `bitrev t` of the +coefficients `c`, scaled by `h`. For an odd leaf the coset table multiplies the coefficients in its +radix-two layer. For an even leaf the products come first: a first radix-four pass reading both +the coefficients and the table would stream twelve lines a power of two apart, more than a cache +set holds. -/ +@[specialize] +def cosetBlock (P : LdePlan p) (c : ByteArray) (t : Nat) : ByteArray := + let T := P.cosetTables.getD (NTT.Transform.bitRevNat P.bits t) .empty + if P.logH % 2 = 1 then + leafReversedOdd p P.forwardTwiddles P.logH + (pairsScaled p (P.forwardTwiddles.getD (P.logH - 1) .empty) T c (2 ^ (P.logH - 1))) + else leafReversed p P.forwardTwiddles P.logH (mulWords p T c (2 ^ P.logH)) + +/-- The number of row ranges the input transpose splits into. -/ +def rangeCount (logRows : Nat) : Nat := 2 ^ min 4 logRows + +/-- The number of row ranges the input transpose splits each of `chunks` input chunks into. -/ +def rangesPerChunk (logRows chunks : Nat) : Nat := max 1 (rangeCount logRows / chunks) + +/-- The number of output chunks per block of `2 ^ logRows` rows. -/ +def chunkCount (logRows : Nat) : Nat := 2 ^ min 3 logRows + +/-- The extension of the `w`-column matrix whose rows are the chunks `ms` of equal heights, in +order, as row chunks in commitment order. Tasks transpose row ranges of the chunks into columns; +each column's coefficients are one task, and each of its coset blocks another; each output chunk +transposes rows of one block as soon as that block's columns are done. -/ +@[specialize] +def cosetLde (P : LdePlan p) (w : Nat) (ms : Array ByteArray) : Array ByteArray := + let q := rangesPerChunk P.logH ms.size + let rows := 2 ^ P.logH / (ms.size * q) + let pieces := Tasks.join (Tasks.spawnAll (ms.size * q) fun k ↦ + transposeRows (ms.getD (k / q) .empty) w 0 w (k % q * rows) rows) + let coeffs := (Array.range w).map fun j ↦ pieces.bind fun ps ↦ Task.spawn fun _ ↦ + columnCoefficients P (gatherColumn ps rows j) + let C := chunkCount P.logH + let rows' := 2 ^ P.logH / C + let chunks := (Array.range (2 ^ P.bits)).flatMap fun t ↦ + let block := Tasks.join (coeffs.map fun c ↦ c.bind fun c ↦ Task.spawn fun _ ↦ cosetBlock P c t) + (Array.range C).map fun k ↦ block.bind fun cs ↦ Task.spawn fun _ ↦ + rowsOfColumns cs (k * rows') rows' + chunks.map Task.get + +/-- The rows of the row-major matrix `M` of width `w` and `2 ^ logH` rows in `2 ^ k` chunks of +`2 ^ (logH - k)` consecutive rows, packed: `cosetLde`'s input. -/ +def rowChunks (M : Array (FastField p)) (w logH k : Nat) : Array ByteArray := + (Array.range (2 ^ k)).map fun c ↦ + encode (M.extract (c * (2 ^ (logH - k) * w)) (c * (2 ^ (logH - k) * w) + 2 ^ (logH - k) * w)) + +/-- Row chunks joined into one row-major buffer. -/ +def joinRows (chunks : Array ByteArray) : ByteArray := + chunks.foldl (· ++ ·) (ByteArray.emptyWithCapacity (chunks.foldl (· + ·.size) 0)) + +end Native + +/-- `Native.cosetLde` as compiled for one modulus: an instance sets `run` to it at its concrete +`p`, where the compiler specializes the kernels. -/ +class PackedCosetLde (p : ℕ) [Mont32Field p] where + /-- The compiled extension. -/ + run : LdePlan p → Nat → Array ByteArray → Array ByteArray + run_eq : run = Native.cosetLde := by rfl + +/-- The extension of the `w`-column packed row-major matrix whose rows are the chunks `ms` of equal +heights, as row chunks in commitment order, with the field's compiled kernels. -/ +@[inline] def LdePlan.cosetLde [PackedCosetLde p] (P : LdePlan p) (w : Nat) + (ms : Array ByteArray) : Array ByteArray := + PackedCosetLde.run P w ms + + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/CosetLdeCorrectness.lean b/CompPoly/Univariate/NTTFast/Packed/CosetLdeCorrectness.lean new file mode 100644 index 00000000..061a32cb --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/CosetLdeCorrectness.lean @@ -0,0 +1,1101 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.CosetLdeTransposes +public import CompPoly.Univariate.NTTFast.Packed.NativeLastCorrectness +public import CompPoly.Univariate.NTTFast.Packed.LeafStages +import all CompPoly.Univariate.NTTFast.Packed.CosetLde +import all CompPoly.Univariate.NTTFast.Packed.NativeLast +import all CompPoly.Univariate.NTTFast.Packed.Native +import all CompPoly.Univariate.NTTFast.Packed.KernelModels +import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas +import CompPoly.Univariate.NTT.Interpolation +import all CompPoly.Univariate.ToPoly.Core +import Mathlib.Tactic.Ring + +/-! # Correctness of the batched coset low-degree extension + +Each column's coefficients are the unnormalized inverse transform of the column, and block `t` +of its extension is the DIF transform of the coefficients scaled by coset table `bitrev t`. In +the bit-reversed row order the blocks concatenate to the values of the column's interpolant on +the coset `shift · ⟨ω⟩`; the transposes move columns and rows between them. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode size_encode_uint32) + +/-! ### Kernels -/ + +/-- A loop whose body guards a single update is a fold of the guarded update. -/ +theorem forIn_dite_yield {α β : Type} (L : List α) (init : β) (c : α → β → Prop) + [∀ x b, Decidable (c x b)] (f : ∀ x b, c x b → β) : + (forIn (m := Id) L init fun x b ↦ + if h : c x b then pure (ForInStep.yield (f x b h)) else pure (ForInStep.yield b)) = + pure (L.foldl (fun b x ↦ if h : c x b then f x b h else b) init) := by + induction L generalizing init with + | nil => rfl + | cons x L ih => + rw [List.forIn_cons, List.foldl_cons] + split <;> exact ih _ + +/-- The in-place final-layer word kernel agrees with its field expression graph. -/ +theorem Native.leaf16_encode (t3 t2 t1 : Array (FastField p)) (i : USize) (a : Array + (FastField p)) (h) : + Native.leaf16 p (encode t3) (encode t2) (encode t1) i (encode a) h = + encode (leaf16Field t3 t2 t1 i a) := by + have hi : i.toNat + 16 ≤ a.size := by + have hh := h.1 + rw [size_encode_fields] at hh + omega + unfold Native.leaf16 + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val] + rw [Native.write16U_encode _ _ h.2.1 hi] + rfl + +/-- The in-place final-layer kernel runs the four last DIF layers of its block. -/ +theorem Native.leaf16_correct (t3 t2 t1 : Array (FastField p)) (i : USize) (a : Array + (FastField p)) (h) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) : + Native.leaf16 p (encode t3) (encode t2) (encode t1) i (encode a) h = + encode (Expressions.leaf16Stages t3 t2 t1 a i.toNat) := by + have hi : i.toNat + 16 ≤ a.size := by + have hi := h.1 + rw [size_encode_fields] at hi + omega + rw [Native.leaf16_encode, leaf16Field_eq_expression, + Expressions.leaf16Field_eq_stages t3 t2 t1 a i hi h3 h2 h1] + +/-- An eight-word store into encoded field values splices the values in. -/ +theorem Native.storeWords8_encode (a : Array (FastField p)) (off : USize) (i : Nat) + (ho : off.toNat = 4 * i) (hs : (encode a).size < USize.size) (hi : i + 8 ≤ a.size) + (x0 x1 x2 x3 x4 x5 x6 x7 : FastField p) : + Native.storeWords (encode a) off 8 x0.val x1.val x2.val x3.val x4.val x5.val x6.val x7.val + 0 0 0 0 0 0 0 0 = encode (splice a i #[x0, x1, x2, x3, x4, x5, x6, x7]) := by + rw [encode_eq_map_val] at hs ⊢ + rw [Native.storeWords8_encodeWords _ off i ho (by rwa [size_encode_uint32] at hs) + (by rw [Array.size_map]; exact hi), encode_eq_map_val] + congr 1 + simp only [splice, Array.map_append, map_val_extract, Array.size_map, List.map_toArray, + List.map_cons, List.map_nil, List.size_toArray, List.length_cons, List.length_nil] + +/-- An eight-word store from word `i + c` of encoded field values splices the values in. -/ +theorem Native.storeWords8_encode_at (a : Array (FastField p)) (i c : USize) + (hs : 4 * a.size < USize.size) (hi : i.toNat + c.toNat + 8 ≤ a.size) + (x0 x1 x2 x3 x4 x5 x6 x7 : FastField p) : + Native.storeWords (encode a) ((i + c) * 4) 8 x0.val x1.val x2.val x3.val x4.val + x5.val x6.val x7.val 0 0 0 0 0 0 0 0 = + encode (splice8 a (i.toNat + c.toNat) x0 x1 x2 x3 x4 x5 x6 x7) := by + rw [splice8_eq_splice _ _ _ _ _ _ _ _ _ _ hi] + exact Native.storeWords8_encode a _ _ (Native.wordOffset_toNat _ _ (by omega)) + (by rw [size_encode_fields]; exact hs) hi _ _ _ _ _ _ _ _ + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The first stage kernel runs the first layer's model on its four blocks in turn. -/ +theorem Native.leafStage8_encode (t a : Array (FastField p)) (i : USize) (h) : + (Native.leafStage8 p (encode t) i (encode a) h).1 = + encode (Expressions.leafStage8Field t (i.toNat + 48) + (Expressions.leafStage8Field t (i.toNat + 32) + (Expressions.leafStage8Field t (i.toNat + 16) + (Expressions.leafStage8Field t i.toNat a)))) := by + obtain ⟨h1, h2, -, -⟩ := h + have hb : 4 * (i.toNat + 64) ≤ 4 * a.size := by rwa [size_encode_fields] at h1 + have hs : 4 * a.size < USize.size := by rwa [size_encode_fields] at h2 + unfold Native.leafStage8 + simp only [Native.store8_val _ _ _ _ _ _ _ _ _ _ _ h2, Native.touchLast, Native.touch_eq] + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val] + simp (disch := ((try simp only [size_splice8, USize.reduceToNat, USize.toNat_zero]); omega)) + only [Native.storeWords8_encode_at] + apply congrArg encode + unfold Expressions.leafStage8Field + simp (disch := omega) only [getD_splice8_out] + simp only [Expressions.fieldAdd, Expressions.fieldSub, Expressions.fieldMul, USize.reduceToNat, + Nat.add_assoc, Nat.reduceAdd, Nat.add_zero, Nat.zero_add] + + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The second stage kernel runs the second layer's model on its four blocks in turn. -/ +theorem Native.leafStage4_encode (t a : Array (FastField p)) (i : USize) (h) : + (Native.leafStage4 p (encode t) i (encode a) h).1 = + encode (Expressions.leafStage4Field t (i.toNat + 48) + (Expressions.leafStage4Field t (i.toNat + 32) + (Expressions.leafStage4Field t (i.toNat + 16) + (Expressions.leafStage4Field t i.toNat a)))) := by + obtain ⟨h1, h2, -, -⟩ := h + have hb : 4 * (i.toNat + 64) ≤ 4 * a.size := by rwa [size_encode_fields] at h1 + have hs : 4 * a.size < USize.size := by rwa [size_encode_fields] at h2 + unfold Native.leafStage4 + simp only [Native.store8_val _ _ _ _ _ _ _ _ _ _ _ h2, Native.touchLast, Native.touch_eq] + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val] + simp (disch := ((try simp only [size_splice8, USize.reduceToNat, USize.toNat_zero]); omega)) + only [Native.storeWords8_encode_at] + apply congrArg encode + unfold Expressions.leafStage4Field + simp (disch := omega) only [getD_splice8_out] + simp only [Expressions.fieldAdd, Expressions.fieldSub, Expressions.fieldMul, USize.reduceToNat, + Nat.add_assoc, Nat.reduceAdd, Nat.add_zero, Nat.zero_add] + + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The third stage kernel runs the third layer's model on its four blocks in turn. -/ +theorem Native.leafStage2_encode (t a : Array (FastField p)) (i : USize) (h) : + (Native.leafStage2 p (encode t) i (encode a) h).1 = + encode (Expressions.leafStage2Field t (i.toNat + 48) + (Expressions.leafStage2Field t (i.toNat + 32) + (Expressions.leafStage2Field t (i.toNat + 16) + (Expressions.leafStage2Field t i.toNat a)))) := by + obtain ⟨h1, h2, -, -⟩ := h + have hb : 4 * (i.toNat + 64) ≤ 4 * a.size := by rwa [size_encode_fields] at h1 + have hs : 4 * a.size < USize.size := by rwa [size_encode_fields] at h2 + unfold Native.leafStage2 + simp only [Native.store8_val _ _ _ _ _ _ _ _ _ _ _ h2, Native.touchLast, Native.touch_eq] + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val] + simp (disch := ((try simp only [size_splice8, USize.reduceToNat, USize.toNat_zero]); omega)) + only [Native.storeWords8_encode_at] + apply congrArg encode + unfold Expressions.leafStage2Field + simp (disch := omega) only [getD_splice8_out] + simp only [Expressions.fieldAdd, Expressions.fieldSub, Expressions.fieldMul, USize.reduceToNat, + Nat.add_assoc, Nat.reduceAdd, Nat.add_zero, Nat.zero_add] + + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The last stage kernel runs the last layer's model on its four blocks in turn. -/ +theorem Native.leafStage1_encode (t : ByteArray) (a : Array (FastField p)) (i : USize) (h) : + (Native.leafStage1 p t i (encode a) h).1 = + encode (Expressions.leafStage1Field (i.toNat + 48) + (Expressions.leafStage1Field (i.toNat + 32) + (Expressions.leafStage1Field (i.toNat + 16) + (Expressions.leafStage1Field i.toNat a)))) := by + obtain ⟨h1, h2, -, -⟩ := h + have hb : 4 * (i.toNat + 64) ≤ 4 * a.size := by rwa [size_encode_fields] at h1 + have hs : 4 * a.size < USize.size := by rwa [size_encode_fields] at h2 + unfold Native.leafStage1 + simp only [Native.store8_val _ _ _ _ _ _ _ _ _ _ _ h2, Native.touchLast, Native.touch_eq] + simp only [Native.readUOffset_encode, add_val, sub_val] + simp (disch := ((try simp only [size_splice8, USize.reduceToNat, USize.toNat_zero]); omega)) + only [Native.storeWords8_encode_at] + apply congrArg encode + unfold Expressions.leafStage1Field + simp (disch := omega) only [getD_splice8_out] + simp only [Expressions.fieldAdd, Expressions.fieldSub, USize.reduceToNat, + Nat.add_assoc, Nat.reduceAdd, Nat.add_zero] + + +/-- The stage loop depends only on the buffer's value. -/ +theorem Native.leafStageLoop_congr {K : Nat} + {k : (t : ByteArray) → (i : USize) → (b : ByteArray) → Native.LeafStageFits b t i K → + Native.Buf b.size} + {t : ByteArray} {count : Nat} {i : USize} {b b' : ByteArray} (h) (e : b = b') : + Native.leafStageLoop K k t count i b h = Native.leafStageLoop K k t count i b' (e ▸ h) := by + subst e + rfl + +/-- A stage loop whose kernel runs `F` on its four blocks in turn runs `F` on blocks +`4 g, …, 4 (g + count) - 1`. -/ +theorem Native.leafStageLoop_encode {K : Nat} + (k : (t : ByteArray) → (i : USize) → (b : ByteArray) → Native.LeafStageFits b t i K → + Native.Buf b.size) + (t : ByteArray) (F : Array (FastField p) → Nat → Array (FastField p)) + (hk : ∀ i (b : Array (FastField p)) h, (k t i (encode b) h).1 = + encode (F (F (F (F b i.toNat) (i.toNat + 16)) (i.toNat + 32)) (i.toNat + 48))) : + ∀ (count g : Nat) (i : USize) (b : Array (FastField p)) (h), i.toNat = 64 * g → + Native.leafStageLoop K k t count i (encode b) h = + encode ((List.range' (4 * g) (4 * count)).foldl (fun b blk ↦ F b (blk * 16)) b) := by + intro count + induction count with + | zero => intro _ _ _ _ _; rfl + | succ count ih => + intro g i b h hi + have hb := h.1 + have hs := h.2.1 + rw [size_encode_fields] at hb hs + have h64 : (i + 64).toNat = i.toNat + 64 := by + rw [Native.usize_add_toNat _ _ (by rw [Native.usize_numeral 64 (by decide)]; omega), + Native.usize_numeral 64 (by decide)] + rw [Native.leafStageLoop, Native.leafStageLoop_congr _ (hk i b _), + ih (g + 1) _ _ _ (by rw [h64, hi]; ring), show 4 * (count + 1) = 4 + 4 * count by ring, + ← List.range'_append, List.foldl_append, show 4 * (g + 1) = 4 * g + 4 by ring] + simp only [List.range'_succ, List.range'_zero, List.foldl_cons, List.foldl_nil, hi] + rw [show 4 * g * 16 = 64 * g by ring, show (4 * g + 1) * 16 = 64 * g + 16 by ring, + show (4 * g + 1 + 1) * 16 = 64 * g + 32 by ring, + show (4 * g + 1 + 1 + 1) * 16 = 64 * g + 48 by ring, Nat.one_mul] + +/-- A fold of block operations agreeing with `G` on arrays of the original size is the fold of +`G`. -/ +theorem foldl_blocks_congr (F G : Array (FastField p) → Nat → Array (FastField p)) (n : Nat) + (a : Array (FastField p)) (hn : n * 16 ≤ a.size) + (hFG : ∀ b base, base + 16 ≤ b.size → F b base = G b base) + (hG : ∀ b base, base + 16 ≤ b.size → (G b base).size = b.size) : + (List.range n).foldl (fun b blk ↦ F b (blk * 16)) a = + (List.range n).foldl (fun b blk ↦ G b (blk * 16)) a := by + suffices ∀ (xs : List Nat) (b : Array (FastField p)), b.size = a.size → (∀ x ∈ xs, x < n) → + xs.foldl (fun b blk ↦ F b (blk * 16)) b = xs.foldl (fun b blk ↦ G b (blk * 16)) b from + this _ a rfl (fun x hx ↦ List.mem_range.mp hx) + intro xs + induction xs with + | nil => intro _ _ _; rfl + | cons x xs ih => + intro b hb hx + have hx' := hx x (List.mem_cons_self ..) + have hfit : x * 16 + 16 ≤ b.size := by rw [hb]; nlinarith + rw [List.foldl_cons, List.foldl_cons, hFG b _ hfit] + exact ih _ (by rw [hG b _ hfit, hb]) (fun y hy ↦ hx y (List.mem_cons_of_mem x hy)) + +/-- The first stage's pass over `4 count` blocks is the first final layer on each block. -/ +theorem Native.leafStage8Loop_encode (t a : Array (FastField p)) (count : Nat) (h) : + Native.leafStageLoop 8 (Native.leafStage8 p) (encode t) count 0 (encode a) h = + encode ((List.range (4 * count)).foldl (fun b blk ↦ layer16_8 t b (blk * 16)) a) := by + have hb := h.1 + rw [size_encode_fields, USize.toNat_zero] at hb + rw [Native.leafStageLoop_encode _ _ (fun b base ↦ Expressions.leafStage8Field t base b) + (fun i b h ↦ Native.leafStage8_encode t b i h) count 0 0 a h rfl, Nat.mul_zero, + ← List.range_eq_range', + foldl_blocks_congr _ _ _ a (by omega) + (fun b base hb ↦ Expressions.leafStage8Field_eq t b base hb) + (fun b base _ ↦ size_layer16_8 t b base)] + +/-- The second stage's pass over `4 count` blocks is the second final layer on each block. -/ +theorem Native.leafStage4Loop_encode (t a : Array (FastField p)) (count : Nat) (h) : + Native.leafStageLoop 4 (Native.leafStage4 p) (encode t) count 0 (encode a) h = + encode ((List.range (4 * count)).foldl (fun b blk ↦ layer16_4 t b (blk * 16)) a) := by + have hb := h.1 + rw [size_encode_fields, USize.toNat_zero] at hb + rw [Native.leafStageLoop_encode _ _ (fun b base ↦ Expressions.leafStage4Field t base b) + (fun i b h ↦ Native.leafStage4_encode t b i h) count 0 0 a h rfl, Nat.mul_zero, + ← List.range_eq_range', + foldl_blocks_congr _ _ _ a (by omega) + (fun b base hb ↦ Expressions.leafStage4Field_eq t b base hb) + (fun b base _ ↦ size_layer16_4 t b base)] + +/-- The third stage's pass over `4 count` blocks is the third final layer on each block. -/ +theorem Native.leafStage2Loop_encode (t a : Array (FastField p)) (count : Nat) (h) : + Native.leafStageLoop 2 (Native.leafStage2 p) (encode t) count 0 (encode a) h = + encode ((List.range (4 * count)).foldl (fun b blk ↦ layer16_2 t b (blk * 16)) a) := by + have hb := h.1 + rw [size_encode_fields, USize.toNat_zero] at hb + rw [Native.leafStageLoop_encode _ _ (fun b base ↦ Expressions.leafStage2Field t base b) + (fun i b h ↦ Native.leafStage2_encode t b i h) count 0 0 a h rfl, Nat.mul_zero, + ← List.range_eq_range', + foldl_blocks_congr _ _ _ a (by omega) + (fun b base hb ↦ Expressions.leafStage2Field_eq t b base hb) + (fun b base _ ↦ size_layer16_2 t b base)] + +/-- The last stage's pass over `4 count` blocks is the last final layer on each block. -/ +theorem Native.leafStage1Loop_encode (t : ByteArray) (a : Array (FastField p)) (count : Nat) (h) : + Native.leafStageLoop 0 (Native.leafStage1 p) t count 0 (encode a) h = + encode ((List.range (4 * count)).foldl (fun b blk ↦ layer16_1 #[1] b (blk * 16)) a) := by + have hb := h.1 + rw [size_encode_fields, USize.toNat_zero] at hb + rw [Native.leafStageLoop_encode _ _ (fun b base ↦ Expressions.leafStage1Field base b) + (fun i b h ↦ Native.leafStage1_encode t b i h) count 0 0 a h rfl, Nat.mul_zero, + ← List.range_eq_range', + foldl_blocks_congr _ _ _ a (by omega) + (fun b base hb ↦ Expressions.leafStage1Field_eq b base hb) + (fun b base _ ↦ size_layer16_1 #[1] b base)] + +/-- The final layers of `blocks` sixteen-word blocks. -/ +theorem Native.leaf16Blocks_encode (t3 t2 t1 : Array (FastField p)) (blocks : Nat) (a : Array + (FastField p)) + (ha : 4 * a.size < USize.size) (hs : blocks * 16 ≤ a.size) + (ht3 : t3.size = 8) (ht2 : t2.size = 4) (ht1 : t1.size = 2) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) : + Native.leaf16Blocks p (encode t3) (encode t2) (encode t1) blocks (encode a) = + encode ((List.range blocks).foldl (fun b block ↦ + Expressions.leaf16Stages t3 t2 t1 b (block * 16)) a) := by + let st (b : Array (FastField p)) (block : Nat) : Array (FastField p) := + Expressions.leaf16Stages t3 t2 t1 b (block * 16) + have hsize (b : Array (FastField p)) : (encode b).size = 4 * b.size := size_encode_fields b + unfold Native.leaf16Blocks + simp only [Std.Legacy.Range.forIn_eq_forIn_range', Std.Legacy.Range.size, + Nat.add_sub_cancel, Nat.div_one, bind_pure, forIn_dite_yield] + rw [dite_eq_left_of_eq_true (eq_true (by + simp only [hsize, ht3, ht2, ht1] + have := USize.le_size + omega))] + simp only [Native.leafStage8Loop_encode, Native.leafStage4Loop_encode, + Native.leafStage2Loop_encode, Native.leafStage1Loop_encode, layer16_8_eq_localLayer, + layer16_4_eq_localLayer, layer16_2_eq_localLayer, layer16_1_eq_localLayer] + rw [fold_localLayer _ 8 1 _ (by decide), fold_localLayer _ 4 2 _ (by decide), + fold_localLayer _ 2 4 _ (by decide), fold_localLayer _ 1 8 _ (by decide)] + simp only [Nat.reduceMul, Nat.one_mul] + rw [← fold_leaf16Stages] + rw [foldl_encode _ (fun b ↦ b.size = a.size) _ st (fun block hblock b hb ↦ by + have hblock := List.mem_range'_1.mp hblock + have hc : 4 * (16 * block + 15) + 3 < (encode b).size ∧ (encode b).size < USize.size ∧ + 31 < (encode t3).size ∧ (encode t3).size < USize.size ∧ + 15 < (encode t2).size ∧ (encode t2).size < USize.size ∧ + 7 < (encode t1).size ∧ (encode t1).size < USize.size := by + rw [hsize, hsize, hsize, hsize, ht3, ht2, ht1, hb] + have := USize.le_size + omega + rw [dite_eq_left_of_eq_true (eq_true hc), Native.leaf16_correct _ _ _ _ _ _ h3 h2 h1, + USize.toNat_ofNatLT, Nat.mul_comm]) + (fun _ _ b hb ↦ by simp only [st, Expressions.size_leaf16Stages, hb]) _ + (foldl_invariant _ (fun b ↦ b.size = a.size) _ + (fun _ _ b hb ↦ by simp only [Expressions.size_leaf16Stages, hb]) a rfl)] + have hn : blocks = 4 * (blocks / 4) + (blocks - 4 * (blocks / 4)) := by omega + conv => rhs; rw [hn, List.range_eq_range', ← List.range'_append, List.foldl_append, + ← List.range_eq_range', Nat.zero_add, Nat.one_mul] + rfl + +/-- A DIF-order leaf is the mixed radix-four DIF stages. -/ +theorem Native.leafReversed_encode (D : NTT.Domain (FastField p)) (tw : Array (Array + (FastField p))) (X : Array (FastField p)) + (htw : TwiddleSizes D.logN tw) (hone : TwiddleOnes D.logN tw) (hs : X.size = D.n) + (hu : 4 * D.n < USize.size) : + Native.leafReversed p (tw.map encode) D.logN (encode X) = + encode (Plan.runStagesDIFRadix4WithTwiddles D tw X) := by + have hn : X.size = 2 ^ D.logN := hs + have hu' : 4 * 2 ^ D.logN < USize.size := hu + rw [← fieldLayers_eq D tw X hs] + unfold Native.leafReversed + split + · rename_i hm + have hzero : tw.getD 0 #[] = #[1] := + twiddle_zero_eq_one _ (by simpa using htw 0 (by omega)) (hone 0 (by omega)) + let U := fieldLayers tw D.logN (D.logN / 2 - 2) X + have hUs : U.size = 2 ^ D.logN := by simp only [U, size_fieldLayers, hn] + rw [Native.leafLayers_encode tw _ _ X (by omega) htw hn hu', getD_map_encode, + getD_map_encode, getD_map_encode, + Native.leaf16Blocks_encode _ _ _ _ U (by rw [hUs]; exact hu') + (by rw [hUs]; exact Nat.div_mul_le_self _ _) (by simpa using htw 3 (by omega)) + (by simpa using htw 2 (by omega)) (by simpa using htw 1 (by omega)) (hone 3 (by omega)) + (hone 2 (by omega)) (hone 1 (by omega)), + fold_leaf16Stages_eq_radix4 _ _ _ _ U (by rw [hUs]; exact Nat.div_mul_le_self _ _)] + congr 1 + have hrange : List.range (D.logN / 2) = + List.range (D.logN / 2 - 2) ++ [D.logN / 2 - 2, D.logN / 2 - 1] := by + obtain ⟨P, hP⟩ : ∃ P, D.logN / 2 = P + 2 := ⟨D.logN / 2 - 2, by omega⟩ + rw [hP, List.range_succ, List.range_succ, List.append_assoc] + rfl + have h3 : D.logN - 1 - D.logN % 2 - 2 * (D.logN / 2 - 2) = 3 := by omega + have h1 : D.logN - 1 - D.logN % 2 - 2 * (D.logN / 2 - 1) = 1 := by omega + have hd : 16 ∣ 2 ^ D.logN := by simpa using Nat.pow_dvd_pow 2 (show 4 ≤ D.logN by omega) + unfold fieldLayers + rw [hrange, List.foldl_append, List.foldl_cons, List.foldl_cons, List.foldl_nil] + simp only [h3, h1, Nat.reduceSub, Nat.reducePow, hzero] + rw [show 2 ^ D.logN / 4 = 4 * (2 ^ D.logN / 16) by omega] + rfl + · exact Native.leafLayers_encode tw _ _ X (le_refl _) htw hn hu' + +/-- The sixteen-word product kernel fills its words with the products. -/ +theorem Native.fill_mul16 (t a : Array (FastField p)) (i : USize) (N : Nat) + (hN : 4 * N < USize.size) + (hi : i.toNat + 16 ≤ N) (h) : + Native.Fill (fun k ↦ (t.getD k 0 * a.getD k 0).val) (fun k ↦ i.toNat ≤ k ∧ k < i.toNat + 16) + N (fun b ↦ Native.mul16 p (encode t) (encode a) i b h) := by + refine (Native.fill_write16U _ N i (fun k ↦ (t.getD (i.toNat + k) 0 * a.getD (i.toNat + k) 0).val) + hN hi fun _ _ ↦ rfl).congr fun b ↦ ?_ + unfold Native.mul16 + simp only [Native.readUOffset_encode, mul_val, USize.reduceToNat] + +/-- The checked product loop fills its `16 * count` words with the products. -/ +theorem Native.fill_mul16Loop (t a : Array (FastField p)) (N : Nat) (hN : 4 * N < USize.size) : + ∀ (count : Nat) (i : USize) (h), i.toNat + 16 * count ≤ N → + Native.Fill (fun k ↦ (t.getD k 0 * a.getD k 0).val) + (fun k ↦ i.toNat ≤ k ∧ k < i.toNat + 16 * count) N + (fun b ↦ Native.mul16Loop p (encode t) (encode a) count i b h) := by + intro count + induction count with + | zero => + intro i h _ + exact (Native.Fill.id _ N).mono fun k hk ↦ by omega + | succ count ih => + intro i h hN' + have ht := h.2.2.1 + have hi : (i + 16).toNat = i.toNat + 16 := Native.usize_add16 i _ (by omega) ht + have h0 : 4 * (i.toNat + 15) + 3 < (encode t).size ∧ 4 * (i.toNat + 15) + 3 < (encode a).size ∧ + (encode t).size < USize.size ∧ (encode a).size < USize.size := by omega + have h1 : 4 * ((i + 16).toNat + 16 * count) ≤ (encode t).size ∧ + 4 * ((i + 16).toNat + 16 * count) ≤ (encode a).size ∧ + (encode t).size < USize.size ∧ (encode a).size < USize.size := by rw [hi]; omega + refine (((Native.fill_mul16 t a i N hN (by omega) h0).comp + (ih (i + 16) h1 (by rw [hi]; omega))).congr fun b ↦ ?_).mono fun k hk ↦ ?_ + · rfl + · rw [hi] at * + omega + +/-- Products of the first `n` coordinates. -/ +theorem Native.mulWords_encode (t a : Array (FastField p)) (n : Nat) (ht : n ≤ t.size) + (ha : n ≤ a.size) + (hts : 4 * t.size < USize.size) (has : 4 * a.size < USize.size) : + Native.mulWords p (encode t) (encode a) n = + encode (Array.ofFn (n := n) fun i ↦ t.getD i.val 0 * a.getD i.val 0) := by + let T (k : Nat) : UInt32 := (t.getD k 0 * a.getD k 0).val + have hN : 4 * n < USize.size := by omega + have hedge (k : Nat) (hk : k ∈ List.range' (16 * (n / 16)) (n - 16 * (n / 16))) : + Native.Fill T (· = k) n (fun b ↦ + Native.write b k (mul p (Native.read (encode t) k) (Native.read (encode a) k))) := by + have hk := List.mem_range'_1.mp hk + refine Native.fill_write T n k _ hN (by omega) ?_ + rw [Native.read_encode _ _ (by rw [size_encode_fields]; exact hts) (by omega), + Native.read_encode _ _ (by rw [size_encode_fields]; exact has) (by omega), mul_val] + have hg : 4 * (16 * (n / 16)) ≤ (encode t).size ∧ 4 * (16 * (n / 16)) ≤ (encode a).size ∧ + (encode t).size < USize.size ∧ (encode a).size < USize.size := by + simp only [size_encode_fields] + omega + have hall := (Native.fill_mul16Loop t a n hN (n / 16) 0 + (by rw [USize.toNat_zero, Nat.zero_add]; exact hg) (by rw [USize.toNat_zero]; omega)).comp + (Native.Fill.foldl T _ n _ _ hedge) + unfold Native.mulWords + simp only [Std.Legacy.Range.forIn_eq_forIn_range', Std.Legacy.Range.size, + Nat.add_sub_cancel, Nat.div_one, List.forIn_pure_yield_eq_foldl, bind_pure, + Native.zeroWords_eq] + rw [dite_eq_left_of_eq_true (eq_true hg)] + refine (hall.eq_ofFn (fun k hk ↦ ?_) _ Array.size_replicate).trans ?_ + · by_cases hk' : k < 16 * (n / 16) + · exact Or.inl ⟨by rw [USize.toNat_zero]; omega, by rw [USize.toNat_zero]; omega⟩ + · exact Or.inr ⟨k, List.mem_range'_1.mpr (by omega), rfl⟩ + · rw [encode_eq_map_val, Array.map_ofFn] + rfl + +/-- The scaled out-of-place radix-two kernel stores its sixteen sums and sixteen differences. -/ +theorem Native.pair16Scaled_eq (w T c : Array (FastField p)) (j i0 i1 : USize) (out : ByteArray) + (h) : + Native.pair16Scaled p (encode w) (encode T) (encode c) j i0 i1 out h = + let x (k : Nat) := T.getD (i0.toNat + k) 0 * c.getD (i0.toNat + k) 0 + let y (k : Nat) := T.getD (i1.toNat + k) 0 * c.getD (i1.toNat + k) 0 + let s (k : Nat) := (x k + y k).val + let d (k : Nat) := (w.getD (j.toNat + k) 0 * (x k - y k)).val + Native.write16U (Native.write16U out i0 (s 0) (s 1) (s 2) (s 3) (s 4) (s 5) (s 6) (s 7) + (s 8) (s 9) (s 10) (s 11) (s 12) (s 13) (s 14) (s 15)) i1 (d 0) (d 1) (d 2) (d 3) (d 4) + (d 5) (d 6) (d 7) (d 8) (d 9) (d 10) (d 11) (d 12) (d 13) (d 14) (d 15) := by + unfold Native.pair16Scaled + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val, USize.reduceToNat] + +/-- The coordinates of the first radix-two DIF layer of `X` with half `h`. -/ +theorem getD_firstLayer (w X : Array (FastField p)) (h : Nat) (hX : X.size = 2 * h) (k : Nat) : + (Plan.butterflyDIFInner w h 0 0 h X).getD k 0 = + if k < h then X.getD k 0 + X.getD (h + k) 0 + else if k < 2 * h then w.getD (k - h) 0 * (X.getD (k - h) 0 - X.getD k 0) + else 0 := by + split_ifs with h1 h2 + · rw [getD_butterflyDIFInner_left w h 0 0 h X (by omega) (by omega) k (by omega), + Nat.sub_zero] + · rw [getD_butterflyDIFInner_right w h 0 0 h X (by omega) (by omega) k (by omega)] + simp only [Nat.zero_add] + · rw [Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by + rw [Plan.size_butterflyDIFInner]; omega), Option.getD_none] + +/-- The products `T ⊙ c` of the first `n` coordinates. -/ +abbrev scaledCoeffs (T c : Array (FastField p)) (n : Nat) : Array (FastField p) := + Array.ofFn (n := n) fun i ↦ T.getD i.val 0 * c.getD i.val 0 + +theorem getD_scaledCoeffs (T c : Array (FastField p)) (n i : Nat) (hi : i < n) : + (scaledCoeffs T c n).getD i 0 = T.getD i 0 * c.getD i 0 := + getD_ofFn_bounded _ i hi 0 + +/-- The scaled radix-two kernel at butterfly `j` fills its words of the first layer of `T ⊙ c`. -/ +theorem Native.fill_pair16Scaled (w T c : Array (FastField p)) (h N : Nat) (hU j : USize) + (hhU : hU.toNat = h) (hN : 4 * N < USize.size) (hj : j.toNat + 16 ≤ h) (hNh : 2 * h ≤ N) + (hb) : + Native.Fill (fun k ↦ ((Plan.butterflyDIFInner w h 0 0 h (scaledCoeffs T c (2 * h))).getD k + 0).val) + (fun k ↦ (j.toNat ≤ k ∧ k < j.toNat + 16) ∨ (j.toNat + h ≤ k ∧ k < j.toNat + h + 16)) N + (fun b ↦ Native.pair16Scaled p (encode w) (encode T) (encode c) j j (j + hU) b hb) := by + have hjh : (j + hU).toNat = j.toNat + h := by + rw [Native.usize_add_toNat _ _ (by rw [hhU]; omega), hhU] + have hX : (scaledCoeffs T c (2 * h)).size = 2 * h := Array.size_ofFn + let x (k : Nat) := T.getD (j.toNat + k) 0 * c.getD (j.toNat + k) 0 + let y (k : Nat) := T.getD ((j + hU).toNat + k) 0 * c.getD ((j + hU).toNat + k) 0 + refine ((Native.fill_write16U _ N j (fun k ↦ (x k + y k).val) hN (by omega) + fun t ht ↦ ?_).comp (Native.fill_write16U _ N (j + hU) + (fun k ↦ (w.getD (j.toNat + k) 0 * (x k - y k)).val) hN (by omega) + fun t ht ↦ ?_)).congr (fun b ↦ ?_) |>.mono fun k hk ↦ ?_ + · simp only [x, y] + rw [getD_firstLayer _ _ _ hX, ite_eq_left (by omega), getD_scaledCoeffs _ _ _ _ (by omega), + getD_scaledCoeffs _ _ _ _ (by omega), hjh, show h + (j.toNat + t) = j.toNat + h + t by omega] + · simp only [x, y] + rw [hjh, getD_firstLayer _ _ _ hX, ite_eq_right (by omega), ite_eq_left (by omega), + getD_scaledCoeffs _ _ _ _ (by omega), getD_scaledCoeffs _ _ _ _ (by omega), + show j.toNat + h + t - h = j.toNat + t by omega] + · rw [Native.pair16Scaled_eq]; rfl + · rw [hjh]; omega + +/-- The checked scaled radix-two loop fills its butterflies' words of the first layer of +`T ⊙ c`. -/ +theorem Native.fill_pair16ScaledLoop (w T c : Array (FastField p)) (h N : Nat) (hU : USize) + (hhU : hU.toNat = h) (hN : 4 * N < USize.size) (hNh : 2 * h ≤ N) : + ∀ (n : Nat) (j : USize) (hb), j.toNat + 16 * n ≤ h → + Native.Fill (fun k ↦ ((Plan.butterflyDIFInner w h 0 0 h (scaledCoeffs T c (2 * h))).getD k + 0).val) + (fun k ↦ (j.toNat ≤ k ∧ k < j.toNat + 16 * n) ∨ + (j.toNat + h ≤ k ∧ k < j.toNat + h + 16 * n)) N + (fun b ↦ Native.pair16ScaledLoop p (encode w) (encode T) (encode c) hU n j b hb) := by + intro n + induction n with + | zero => + intro j hb _ + exact (Native.Fill.id _ N).mono fun k hk ↦ by omega + | succ n ih => + intro j hb hjn + have hj : (j + 16).toNat = j.toNat + 16 := Native.usize_add16 j _ (by omega) hb.2.2.2.1 + have hh : (j + hU).toNat = j.toNat + hU.toNat := Native.usize_add_toNat _ _ (by omega) + have H0 : 4 * (j.toNat + 15) + 3 < (encode c).size ∧ + 4 * ((j + hU).toNat + 15) + 3 < (encode c).size ∧ + 4 * (j.toNat + 15) + 3 < (encode T).size ∧ + 4 * ((j + hU).toNat + 15) + 3 < (encode T).size ∧ + 4 * (j.toNat + 15) + 3 < (encode w).size ∧ (encode c).size < USize.size ∧ + (encode T).size < USize.size ∧ (encode w).size < USize.size := by rw [hh]; omega + have H1 : 4 * (hU.toNat + (j + 16).toNat + 16 * n) ≤ (encode c).size ∧ + 4 * (hU.toNat + (j + 16).toNat + 16 * n) ≤ (encode T).size ∧ + 4 * ((j + 16).toNat + 16 * n) ≤ (encode w).size ∧ (encode c).size < USize.size ∧ + (encode T).size < USize.size ∧ (encode w).size < USize.size := by rw [hj]; omega + refine (((Native.fill_pair16Scaled w T c h N hU j hhU hN (by omega) hNh H0).comp + (ih (j + 16) H1 (by rw [hj]; omega))).congr fun b ↦ ?_).mono fun k hk ↦ ?_ + · rfl + · rw [hj] at * + omega + +/-- The scaled out-of-place radix-two pass is the first DIF layer of `T ⊙ c`. -/ +theorem Native.pairsScaled_encode (w T c : Array (FastField p)) (h : Nat) (hw : h ≤ w.size) + (hT : 2 * h ≤ T.size) (hc : 2 * h ≤ c.size) (hws : 4 * w.size < USize.size) + (hTs : 4 * T.size < USize.size) (hcs : 4 * c.size < USize.size) : + Native.pairsScaled p (encode w) (encode T) (encode c) h = + encode (Plan.butterflyDIFInner w h 0 0 h (scaledCoeffs T c (2 * h))) := by + let Y := Plan.butterflyDIFInner w h 0 0 h (scaledCoeffs T c (2 * h)) + let Tg (k : Nat) : UInt32 := (Y.getD k 0).val + have hX : (scaledCoeffs T c (2 * h)).size = 2 * h := Array.size_ofFn + have hY : Y.size = 2 * h := by simp only [Y, Plan.size_butterflyDIFInner, hX] + have hN : 4 * (2 * h) < USize.size := by omega + have hrd (a : Array (FastField p)) (ha : 4 * a.size < USize.size) (i : Nat) (hi : i < a.size) : + Native.read (encode a) i = (a.getD i 0).val := + Native.read_encode a i (by rw [size_encode_fields]; exact ha) hi + have hedge (k : Nat) (hk : k ∈ List.range' (16 * (h / 16)) (h - 16 * (h / 16))) : + Native.Fill Tg (fun i ↦ i = k ∨ i = h + k) (2 * h) (fun b ↦ + let x := mul p (Native.read (encode T) k) (Native.read (encode c) k) + let y := mul p (Native.read (encode T) (h + k)) (Native.read (encode c) (h + k)) + Native.write (Native.write b k (add p x y)) (h + k) + (mul p (Native.read (encode w) k) (sub p x y))) := by + have hk := List.mem_range'_1.mp hk + refine ((Native.fill_write Tg (2 * h) k _ hN (by omega) ?_).comp + (Native.fill_write Tg (2 * h) (h + k) _ hN (by omega) ?_)).congr fun b ↦ rfl + · rw [hrd T hTs k (by omega), hrd c hcs k (by omega), hrd T hTs (h + k) (by omega), + hrd c hcs (h + k) (by omega), mul_val, mul_val, add_val] + simp only [Tg, Y] + rw [getD_firstLayer _ _ _ hX, ite_eq_left (by omega), getD_scaledCoeffs _ _ _ _ (by omega), + getD_scaledCoeffs _ _ _ _ (by omega)] + · rw [hrd T hTs k (by omega), hrd c hcs k (by omega), hrd T hTs (h + k) (by omega), + hrd c hcs (h + k) (by omega), hrd w hws k (by omega), mul_val, mul_val, sub_val, mul_val] + simp only [Tg, Y] + rw [getD_firstLayer _ _ _ hX, ite_eq_right (by omega), ite_eq_left (by omega), + getD_scaledCoeffs _ _ _ _ (by omega), getD_scaledCoeffs _ _ _ _ (by omega), + show h + k - h = k by omega] + have hg : 4 * (h + 16 * (h / 16)) ≤ (encode c).size ∧ + 4 * (h + 16 * (h / 16)) ≤ (encode T).size ∧ 4 * (16 * (h / 16)) ≤ (encode w).size ∧ + (encode c).size < USize.size ∧ (encode T).size < USize.size ∧ + (encode w).size < USize.size := by + simp only [size_encode_fields] + omega + have hU : (USize.ofNatLT h (by omega)).toNat = h := USize.toNat_ofNatLT + have hall := (Native.fill_pair16ScaledLoop w T c h (2 * h) _ hU hN (le_refl _) (h / 16) 0 + (by rw [hU, USize.toNat_zero]; simpa only [size_encode_fields] using + (show 4 * (h + 0 + 16 * (h / 16)) ≤ 4 * c.size ∧ 4 * (h + 0 + 16 * (h / 16)) ≤ 4 * T.size ∧ + 4 * (0 + 16 * (h / 16)) ≤ 4 * w.size ∧ 4 * c.size < USize.size ∧ + 4 * T.size < USize.size ∧ 4 * w.size < USize.size by omega)) + (by rw [USize.toNat_zero]; omega)).comp (Native.Fill.foldl Tg _ (2 * h) _ _ hedge) + unfold Native.pairsScaled + simp only [Std.Legacy.Range.forIn_eq_forIn_range', Std.Legacy.Range.size, + Nat.add_sub_cancel, Nat.div_one, List.forIn_pure_yield_eq_foldl, bind_pure, + Native.zeroWords_eq] + rw [dite_eq_left_of_eq_true (eq_true hg)] + refine (hall.eq_ofFn (fun k hk ↦ ?_) _ Array.size_replicate).trans ?_ + · by_cases hk' : k < 16 * (h / 16) + · exact Or.inl (Or.inl ⟨by rw [USize.toNat_zero]; omega, by rw [USize.toNat_zero]; omega⟩) + · by_cases hk2 : h ≤ k ∧ k < h + 16 * (h / 16) + · exact Or.inl (Or.inr ⟨by rw [USize.toNat_zero]; omega, by rw [USize.toNat_zero]; omega⟩) + · by_cases hk3 : k < h + · exact Or.inr ⟨k, List.mem_range'_1.mpr (by omega), Or.inl rfl⟩ + · exact Or.inr ⟨k - h, List.mem_range'_1.mpr (by omega), Or.inr (by omega)⟩ + · rw [encode_eq_map_val Y] + congr 1 + apply Array.ext (by rw [Array.size_ofFn, Array.size_map, hY]) + intro k h1 h2 + rw [Array.getElem_ofFn, Array.getElem_map] + have hk : k < Y.size := by rw [Array.size_map] at h2; exact h2 + simp only [Y, Array.getD_eq_getD_getElem?] at hk ⊢ + rw [Array.getElem?_eq_getElem hk, Option.getD_some] + +/-- An odd leaf in DIF order is its radix-two layer, then `leafReversedOdd`. -/ +theorem Native.leafReversed_odd (tw : Array ByteArray) (m : Nat) (hm : m % 2 = 1) + (a : ByteArray) : + Native.leafReversed p tw m a = Native.leafReversedOdd p tw m + (Native.pairs p (tw.getD (m - 1) .empty) (2 ^ (m - 1)) 0 0 (2 ^ (m - 1)) a) := by + unfold Native.leafReversed Native.leafReversedOdd Native.leafLayers + simp only [hm, ↓reduceIte] + split <;> rfl + +/-! ### Column transforms -/ + +/-- The subgroup of order `2 ^ logH`, the domain of the input columns. -/ +abbrev LdePlan.low (P : LdePlan p) : NTT.Domain (FastField p) := powDomain P.domain P.bits P.bits_le + +/-- Packed columns fit machine indices. -/ +theorem LdePlan.byte_bound_low (P : LdePlan p) : 4 * P.low.n < USize.size := by + have := P.byte_bound + have : P.low.n ≤ P.domain.n := Nat.pow_le_pow_right (by decide) (Nat.sub_le _ _) + omega + +/-- Packed columns fit machine indices, stated for the inverse domain. -/ +theorem LdePlan.byte_bound_inverse (P : LdePlan p) : 4 * P.low.inverse.n < USize.size := by + have := P.byte_bound_low + have e1 : P.low.inverse.n = 2 ^ P.logH := rfl + have e2 : P.low.n = 2 ^ P.logH := rfl + omega + +/-- A column's coefficients, scaled by `h`, are its unnormalized inverse transform. -/ +theorem Native.columnCoefficients_encode (P : LdePlan p) (col : Array (FastField p)) + (hs : col.size = 2 ^ P.logH) : + Native.columnCoefficients P (encode col) = + encode (NTT.Forward.forwardSpec P.low.inverse col) := by + obtain ⟨htw, hone⟩ := twiddleTable_invariants P.low.inverse + have h := Native.leafNatural_encode P.low.inverse (Plan.twiddleTable P.low.inverse) col 0 false + htw hone hs P.byte_bound_inverse + (by have := P.log_bound; change P.domain.logN - P.bits ≤ 32; omega) + rw [planStages_eq_difSpec _ _ hs, decodedFields_difSpec] at h + unfold Native.columnCoefficients + rw [P.inverse_eq] + exact h + +/-- The coset tables have one entry per row. -/ +@[simp] theorem size_cosetTable (D : NTT.Domain (FastField p)) (bits : Nat) (shift : FastField p) + (c : Nat) : + (cosetTable D bits shift c).size = 2 ^ (D.logN - bits) := by + simp only [cosetTable, Array.size_map, NTTFast.powers_eq_ofFn, Array.size_ofFn] + +/-- Block `t` of a column's extension is the DIF transform of its coefficients, scaled by coset +table `bitrev t`. -/ +theorem Native.cosetBlock_encode (P : LdePlan p) (c : Array (FastField p)) + (hc : c.size = 2 ^ P.logH) (t : Nat) : + Native.cosetBlock P (encode c) t = encode (difSpec P.low (Array.ofFn (n := 2 ^ P.logH) fun i ↦ + (cosetTable P.domain P.bits P.shift (NTT.Transform.bitRevNat P.bits t)).getD i.val 0 * + c.getD i.val 0)) := by + obtain ⟨htw, hone⟩ := twiddleTable_invariants P.low + have hu := P.byte_bound_low + have hn : P.low.n = 2 ^ P.logH := rfl + have hX : (Array.ofFn (n := 2 ^ P.logH) fun i ↦ + (cosetTable P.domain P.bits P.shift (NTT.Transform.bitRevNat P.bits t)).getD i.val 0 * + c.getD i.val 0).size = P.low.n := Array.size_ofFn + have hkey : Native.cosetBlock P (encode c) t = Native.leafReversed p P.forwardTwiddles P.logH + (encode (Array.ofFn (n := 2 ^ P.logH) fun i ↦ + (cosetTable P.domain P.bits P.shift (NTT.Transform.bitRevNat P.bits t)).getD i.val 0 * + c.getD i.val 0)) := by + unfold Native.cosetBlock + dsimp only + rw [P.tables_eq, getD_map_range _ _ _ (NTT.Transform.bitRevNat_lt _ _)] + split + · rename_i hm + have h2 : 2 ^ P.logH = 2 * 2 ^ (P.logH - 1) := by + rw [← Nat.pow_succ']; congr 1; omega + have hu' : 4 * 2 ^ P.logH < USize.size := hu + have hw : ((Plan.twiddleTable (powDomain P.domain P.bits P.bits_le)).getD (P.logH - 1) + #[]).size = 2 ^ (P.logH - 1) := + htw (P.logH - 1) (by change P.logH - 1 < P.logH; omega) + have ht : (cosetTable P.domain P.bits P.shift (NTT.Transform.bitRevNat P.bits t)).size = + 2 ^ P.logH := size_cosetTable _ _ _ _ + have hX' : (Array.ofFn (n := 2 ^ P.logH) fun i ↦ + (cosetTable P.domain P.bits P.shift (NTT.Transform.bitRevNat P.bits t)).getD i.val 0 * + c.getD i.val 0).size = 2 ^ P.logH := Array.size_ofFn + rw [Native.leafReversed_odd _ _ hm, P.forward_eq, getD_map_encode, + Native.pairsScaled_encode _ _ _ _ (by omega) (by omega) (by omega) (by omega) + (by omega) (by omega), + Native.pairs_encode _ _ _ _ _ _ (by rw [size_encode_fields]; omega) + (by rw [size_encode_fields]; omega) (by omega) (Nat.zero_le _) (by omega) (by omega)] + congr 3 + rw [h2] + · rw [Native.mulWords_encode _ _ _ (by rw [size_cosetTable]) (by rw [hc]) (by + rw [size_cosetTable]; exact hu) (by rw [hc]; exact hu)] + rw [hkey, P.forward_eq] + exact (Native.leafReversed_encode P.low _ _ htw hone hX hu).trans + (congrArg encode (planStages_eq_difSpec _ _ hX)) + +/-! ### Values on the coset -/ + +/-- Coset table `c` holds `(1 / h) · (shift · ω ^ c) ^ l`. -/ +theorem getD_cosetTable (D : NTT.Domain (FastField p)) (bits : Nat) (shift : FastField p) + (c l : Nat) + (hl : l < 2 ^ (D.logN - bits)) : + (cosetTable D bits shift c).getD l 0 = + ((2 ^ (D.logN - bits) : Nat) : FastField p)⁻¹ * (shift * D.omega ^ c) ^ l := by + rw [Array.getD_eq_getD_getElem?, cosetTable, Array.getElem?_map, NTTFast.powers_eq_ofFn, + Array.getElem?_ofFn, dite_eq_left_of_eq_true (eq_true hl)] + rfl + +/-- Entry `i` of block `t` of a column's extension is the column's interpolant on the subgroup +of order `h` at `shift · ω ^ bitrev (h t + i)`. -/ +theorem getD_block (P : LdePlan p) (col : Vector (FastField p) (2 ^ P.logH)) (t i : Nat) + (hi : i < 2 ^ P.logH) : + (difSpec P.low (Array.ofFn (n := 2 ^ P.logH) fun l ↦ + (cosetTable P.domain P.bits P.shift (NTT.Transform.bitRevNat P.bits t)).getD l.val 0 * + (NTT.Forward.forwardSpec P.low.inverse col.toArray).getD l.val 0)).getD i 0 = + (CPolynomial.CLagrange.interpolatePow (P.domain.omega ^ 2 ^ P.bits) col).eval + (P.shift * P.domain.omega ^ + NTT.Transform.bitRevNat P.domain.logN (2 ^ P.logH * t + i)) := by + let h := 2 ^ P.logH + let c := NTT.Transform.bitRevNat P.bits t + let b := NTT.Transform.bitRevNat P.logH i + let ω := P.domain.omega + let coef := NTT.Forward.forwardSpec P.low.inverse col.toArray + have hlog : P.domain.logN = P.bits + P.logH := by + have := P.bits_le; simp only [LdePlan.logH]; omega + have hrev : NTT.Transform.bitRevNat P.domain.logN (2 ^ P.logH * t + i) = 2 ^ P.bits * b + c := by + rw [hlog]; exact bitRevNat_concat P.bits P.logH t i hi + -- The interpolant's coefficients are the normalized inverse transform. + have hq (x : FastField p) : + (CPolynomial.CLagrange.interpolatePow (P.domain.omega ^ 2 ^ P.bits) col).eval x = + CPolynomial.Raw.eval x (NTT.Inverse.inverseSpec P.low col.toArray) := by + have he := NTT.Inverse.inverseSpec_interpolatePow_eq P.low col.toArray + have hl : NTT.loadNaturalVector P.low col.toArray = col := + NTT.loadNaturalVector_toArray P.low col + rw [hl] at he + have he' : CPolynomial.Raw.trim (NTT.Inverse.inverseSpec P.low col.toArray) = + (CPolynomial.CLagrange.interpolatePow (P.domain.omega ^ 2 ^ P.bits) col).val := he + rw [CPolynomial.eval_toPoly, ← CPolynomial.Raw.eval_toPoly_eq_eval, + ← CPolynomial.Raw.toPoly_trim, he'] + rfl + have hcoef (l : Nat) (hl : l < h) : + (NTT.Inverse.inverseSpec P.low col.toArray).getD l 0 = + (h : FastField p)⁻¹ * coef.getD l 0 := by + rw [← NTT.Inverse.normalize_forwardSpec_inverse_eq_inverseSpec, NTT.Inverse.normalize, + getD_ofFn_bounded _ _ (show l < P.low.n from hl)] + rfl + rw [hq, NTT.Coset.eval_eq_sum_fin _ P.low.n + (NTT.Coset.natDegree_lt_of_size_le _ (Nat.two_pow_pos _) + (by rw [NTT.Inverse.size_inverseSpec])), + getD_difSpec _ _ _ hi, dftValue, ← Fin.sum_univ_eq_sum_range] + apply Finset.sum_congr rfl + intro l _ + have hl : l.val < 2 ^ P.logH := l.isLt + rw [getD_ofFn_bounded _ _ hl, getD_cosetTable _ _ _ _ _ hl, hcoef l hl, hrev] + change (h : FastField p)⁻¹ * (P.shift * ω ^ c) ^ l.val * coef.getD l 0 * + (ω ^ 2 ^ P.bits) ^ (b * l.val) = + (h : FastField p)⁻¹ * coef.getD l 0 * (P.shift * ω ^ (2 ^ P.bits * b + c)) ^ l.val + rw [pow_add, pow_mul, pow_mul] + ring + +/-! ### Specification -/ + +/-- Column `j` of a row-major matrix of width `w` with `h` rows. -/ +def column (M : Array (FastField p)) (w h j : Nat) : Vector (FastField p) h := + Vector.ofFn fun i ↦ M.getD (i.val * w + j) 0 + +/-- The low-degree extension of a row-major matrix `M` of width `w` in commitment order: entry +`(r, j)` is column `j`'s interpolant on the subgroup of order `h = 2 ^ logH` at +`shift · ω ^ bitrev r`, where `ω` generates the subgroup of order `2 ^ logN`. -/ +def ldeSpec (P : LdePlan p) (w : Nat) (M : Array (FastField p)) : Array (FastField p) := + Array.ofFn (n := 2 ^ P.domain.logN * w) fun k ↦ + (CPolynomial.CLagrange.interpolatePow (P.domain.omega ^ 2 ^ P.bits) + (column M w (2 ^ P.logH) (k.val % w))).eval + (P.shift * P.domain.omega ^ NTT.Transform.bitRevNat P.domain.logN (k.val / w)) + +/-! ### Assembly -/ + +/-- A grid of tasks in row order. -/ +theorem flatMap_range_map {α : Type*} (a b : Nat) (f : Nat → Nat → α) : + (Array.range a).flatMap (fun t ↦ (Array.range b).map (f t)) = + (Array.range (a * b)).map fun c ↦ f (c / b) (c % b) := by + apply Array.toList_inj.mp + simp only [Array.toList_flatMap, Array.toList_range, Array.toList_map] + induction a with + | zero => simp + | succ a ih => + rw [List.range_succ, List.flatMap_append, ih, Nat.succ_mul, List.range_add, List.map_append, + List.map_map] + simp only [List.flatMap_cons, List.flatMap_nil, List.append_nil] + congr 1 + apply List.map_congr_left + intro x hx + have hx := List.mem_range.mp hx + simp only [Function.comp_apply] + rw [(Native.divMod_add a x b hx).1, (Native.divMod_add a x b hx).2] + +/-- Joining consecutive ranges of an array packs the array. -/ +theorem joinRows_extract (S : Array (FastField p)) (K L : Nat) (hS : S.size = K * L) : + Native.joinRows ((Array.range K).map fun c ↦ encode (S.extract (c * L) (c * L + L))) = + encode S := by + unfold Native.joinRows + rw [show ByteArray.emptyWithCapacity _ = encode (#[] : Array (FastField p)) from rfl, + Array.foldl_map, + ← Array.foldl_toList, Array.toList_range] + have h (K : Nat) (hK : K * L ≤ S.size) : (List.range K).foldl + (fun acc c ↦ acc ++ encode (S.extract (c * L) (c * L + L))) (encode (#[] : Array + (FastField p))) = + encode (S.extract 0 (K * L)) := by + induction K with + | zero => simp + | succ K ih => + rw [List.range_succ, List.foldl_append, ih (by rw [Nat.succ_mul] at hK; omega), + List.foldl_cons, List.foldl_nil, ← PackedRepr.encode_append, Array.extract_append_extract, + Nat.min_eq_left (Nat.zero_le _), Nat.max_eq_right (by omega), Nat.succ_mul] + rw [h K (by omega), ← hS, Array.extract_size] + +/-- A power of two split into `2 ^ min k a` equal parts. -/ +theorem pow_div_mul_pow_min (a k : Nat) : 2 ^ a / 2 ^ min k a * 2 ^ min k a = 2 ^ a := + Nat.div_mul_cancel (Nat.pow_dvd_pow 2 (Nat.min_le_right k a)) + +/-- Column `c0 + j` from the transposed row ranges of columns `c0, …, c0 + n - 1` of a packed +matrix. -/ +theorem Native.gatherColumn_eq_column (M : Array (FastField p)) (w c0 n h R rows j : Nat) + (hn : c0 + n ≤ w) (hj : j < n) (hR : R * rows = h) (hM : M.size = h * w) + (hs : 4 * M.size < USize.size) : + Native.gatherColumn ((Array.range R).map fun k ↦ + Native.transposeRows (encode M) w c0 n (k * rows) rows) rows j = + encode (column M w h (c0 + j)).toArray := by + rw [encode_eq_map_val M, Native.gatherColumn_transposeRows _ w c0 n R rows j hn hj + (by rw [hR, Array.size_map, hM]) (by rw [Array.size_map]; exact hs), encode_eq_map_val] + congr 1 + apply Array.ext (by simp only [Array.size_ofFn, Array.size_map, column, Vector.toArray_ofFn, hR]) + intro i h1 h2 + simp only [Array.getElem_ofFn, Array.getElem_map, column, Vector.toArray_ofFn, getD_map_val, + Nat.add_assoc] + +/-- Entry `i` of a slice is entry `a + i` of the array, inside the slice. -/ +theorem getD_extract_map_val (M : Array (FastField p)) (a b i : Nat) (hb : b ≤ M.size) + (hi : a + i < b) : + ((M.extract a b).map Subtype.val).getD i 0 = (M.map Subtype.val).getD (a + i) 0 := by + rw [getD_map_val, getD_map_val] + have h1 : i < (M.extract a b).size := by rw [Array.size_extract]; omega + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem h1, + Array.getElem?_eq_getElem (show a + i < M.size by omega), Option.getD_some, + Array.getElem_extract] + +/-- The split of `2 ^ k` input chunks into `2 ^ k · q` row ranges of `rows` rows: `2 ^ r` ranges +for `r = max k (min 4 logH)`. -/ +theorem rangesPerChunk_spec (logH k : Nat) (hk : k ≤ logH) : + 0 < Native.rangesPerChunk logH (2 ^ k) ∧ + 2 ^ k * Native.rangesPerChunk logH (2 ^ k) * + (2 ^ logH / (2 ^ k * Native.rangesPerChunk logH (2 ^ k))) = 2 ^ logH ∧ + Native.rangesPerChunk logH (2 ^ k) * + (2 ^ logH / (2 ^ k * Native.rangesPerChunk logH (2 ^ k))) = 2 ^ (logH - k) := by + unfold Native.rangesPerChunk Native.rangeCount + by_cases h : k ≤ min 4 logH + · rw [Nat.pow_div h (by decide), Nat.max_eq_right (Nat.one_le_two_pow), ← Nat.pow_add, + Nat.add_sub_cancel' h, Nat.pow_div (Nat.min_le_right _ _) (by decide), ← Nat.pow_add, + ← Nat.pow_add] + refine ⟨Nat.two_pow_pos _, by congr 1; omega, by congr 1; omega⟩ + · rw [Nat.div_eq_of_lt (Nat.pow_lt_pow_right (by decide) (by omega)), Nat.max_eq_left + (Nat.zero_le _), Nat.mul_one, Nat.one_mul, Nat.pow_div hk (by decide), ← Nat.pow_add] + exact ⟨Nat.one_pos, by congr 1; omega, rfl⟩ + +/-- A row range of a chunk transposes as the same rows of the whole matrix. -/ +theorem Native.transposeRows_rowChunks (M : Array (FastField p)) (w c0 n logH k q rows : Nat) + (hn : c0 + n ≤ w) (hM : M.size = 2 ^ logH * w) (hk : k ≤ logH) (hq0 : 0 < q) + (hq : q * rows = 2 ^ (logH - k)) (hs : 4 * M.size < USize.size) (j : Nat) + (hj : j < 2 ^ k * q) : + Native.transposeRows ((Native.rowChunks M w logH k).getD (j / q) .empty) w c0 n + (j % q * rows) rows = Native.transposeRows (encode M) w c0 n (j * rows) rows := by + have hc : j / q < 2 ^ k := (Nat.div_lt_iff_lt_mul hq0).mpr hj + have hkL : 2 ^ k * 2 ^ (logH - k) = 2 ^ logH := by rw [← Nat.pow_add]; congr 1; omega + have hjq := Nat.div_add_mod j q + have hsq : j % q < q := Nat.mod_lt _ hq0 + -- Row `j * rows` of the matrix is row `j % q * rows` of chunk `j / q`. + have e : j * rows = j / q * 2 ^ (logH - k) + j % q * rows := by + rw [← hq] + conv_lhs => rw [← hjq] + ring + have hsr : j % q * rows + rows ≤ 2 ^ (logH - k) := by + have := Nat.mul_le_mul_right rows (show j % q + 1 ≤ q by omega) + rw [Nat.succ_mul, hq] at this; exact this + have hcL : (j / q + 1) * 2 ^ (logH - k) ≤ 2 ^ logH := by + rw [← hkL]; exact Nat.mul_le_mul_right _ hc + have hcLw : j / q * (2 ^ (logH - k) * w) + 2 ^ (logH - k) * w ≤ M.size := by + rw [hM] + have := Nat.mul_le_mul_right w hcL + rw [Nat.succ_mul] at this + calc _ = (j / q * 2 ^ (logH - k) + 2 ^ (logH - k)) * w := by ring + _ ≤ _ := this + have hrowM : (j * rows + rows) * w ≤ M.size := by + rw [hM, e] + have := Nat.mul_le_mul_right w (show j / q * 2 ^ (logH - k) + j % q * rows + rows ≤ + 2 ^ logH by rw [Nat.succ_mul] at hcL; omega) + exact this + have hrow : (j % q * rows + rows) * w ≤ 2 ^ (logH - k) * w := Nat.mul_le_mul_right w hsr + simp only [Native.rowChunks] + rw [Array.getD_eq_getD_getElem?, Array.getElem?_map, Array.getElem?_range] + simp only [hc, ↓reduceIte, Option.map_some, Option.getD_some] + rw [encode_eq_map_val, encode_eq_map_val M, + Native.transposeRows_encodeWords _ w c0 n _ rows hn (by + rw [Array.size_map, Array.size_extract]; omega) + (by rw [Array.size_map, Array.size_extract]; omega), + Native.transposeRows_encodeWords _ w c0 n _ rows hn (by rw [Array.size_map]; exact hrowM) + (by rw [Array.size_map]; exact hs)] + congr 1 + apply Array.ext (by simp only [Array.size_ofFn]) + intro i h1 _ + simp only [Array.size_ofFn] at h1 + simp only [Array.getElem_ofFn] + have hrows : 0 < rows := Nat.pos_of_ne_zero (by rintro rfl; simp at h1) + have hw : i / rows < n := (Nat.div_lt_iff_lt_mul hrows).mpr h1 + have hr := Nat.mod_lt i hrows + have hin : (j % q * rows + i % rows) * w + c0 + i / rows < 2 ^ (logH - k) * w := by + have := Nat.mul_le_mul_right w (show j % q * rows + i % rows + 1 ≤ 2 ^ (logH - k) by + omega) + rw [Nat.succ_mul] at this; omega + rw [getD_extract_map_val M _ _ _ hcLw (by omega), e] + congr 1 + ring + +/-- The extension of a packed row-major matrix `M` of width `w`, given in `2 ^ k` chunks of rows, +is the packed specification, in row chunks: chunk `c` holds rows `c · L, …` with +`L = 2 ^ logH / chunkCount logH`. -/ +theorem Native.cosetLde_eq (P : LdePlan p) (w : Nat) (M : Array (FastField p)) (k : Nat) + (hk : k ≤ P.logH) (hM : M.size = 2 ^ P.logH * w) + (hu : 4 * (2 ^ P.domain.logN * w) < USize.size) : + Native.cosetLde P w (Native.rowChunks M w P.logH k) = + (Array.range (2 ^ P.bits * Native.chunkCount P.logH)).map fun c ↦ + encode ((ldeSpec P w M).extract (c * (2 ^ P.logH / Native.chunkCount P.logH * w)) + (c * (2 ^ P.logH / Native.chunkCount P.logH * w) + + 2 ^ P.logH / Native.chunkCount P.logH * w)) := by + have hlog : P.domain.logN = P.bits + P.logH := by + have := P.bits_le; simp only [LdePlan.logH]; omega + have hHN : 2 ^ P.logH * w ≤ 2 ^ P.domain.logN * w := + Nat.mul_le_mul_right w (Nat.pow_le_pow_right (by decide) (Nat.sub_le _ _)) + have hHu : 4 * 2 ^ P.logH < USize.size := by + have := P.byte_bound_low; exact this + have hMs : 4 * M.size < USize.size := by omega + let q := Native.rangesPerChunk P.logH (2 ^ k) + let R := 2 ^ k * q + let rows := 2 ^ P.logH / R + obtain ⟨hq0, hR, hq⟩ := rangesPerChunk_spec P.logH k hk + have hpieces : (Array.range R).map (fun j ↦ Native.transposeRows + ((Native.rowChunks M w P.logH k).getD (j / q) .empty) w 0 w (j % q * rows) rows) = + (Array.range R).map fun j ↦ Native.transposeRows (encode M) w 0 w (j * rows) rows := by + apply Array.ext (by simp only [Array.size_map]) + intro j h1 _ + simp only [Array.size_map, Array.size_range] at h1 + simp only [Array.getElem_map, Array.getElem_range] + exact Native.transposeRows_rowChunks M w 0 w P.logH k q rows (by omega) hM hk hq0 hq hMs j h1 + let C := Native.chunkCount P.logH + let rows' := 2 ^ P.logH / C + have hC : rows' * C = 2 ^ P.logH := pow_div_mul_pow_min P.logH 3 + have hC0 : 0 < C := Nat.two_pow_pos _ + have hrows' : rows' < 512 ∨ 512 ∣ rows' := by + have he : rows' = 2 ^ (P.logH - min 3 P.logH) := by + simp only [rows', C, Native.chunkCount]; rw [Nat.pow_div (Nat.min_le_right _ _) (by decide)] + rw [he] + by_cases h9 : 9 ≤ P.logH - min 3 P.logH + · exact Or.inr (by simpa using Nat.pow_dvd_pow 2 h9) + · exact Or.inl (by + have := Nat.pow_lt_pow_right (show 1 < 2 by decide) + (show P.logH - min 3 P.logH < 9 by omega) + simpa using this) + -- Block `t` of column `j`, as field coordinates. + let B (t j : Nat) : Array (FastField p) := difSpec P.low (Array.ofFn (n := 2 ^ P.logH) fun l ↦ + (cosetTable P.domain P.bits P.shift (NTT.Transform.bitRevNat P.bits t)).getD l.val 0 * + (NTT.Forward.forwardSpec P.low.inverse (column M w (2 ^ P.logH) j).toArray).getD l.val 0) + have hB (t j : Nat) : (B t j).size = 2 ^ P.logH := size_difSpec _ _ + have hblock (t : Nat) : (Array.range w).map (fun j ↦ Native.cosetBlock P + (Native.columnCoefficients P (Native.gatherColumn ((Array.range R).map fun k ↦ + Native.transposeRows (encode M) w 0 w (k * rows) rows) rows j)) t) = + ((Array.range w).map fun j ↦ (B t j).map Subtype.val).map encode := by + rw [Array.map_map] + apply Array.ext (by simp only [Array.size_map]) + intro j h1 _ + simp only [Array.size_map, Array.size_range] at h1 + simp only [Array.getElem_map, Array.getElem_range, Function.comp_apply] + rw [Native.gatherColumn_eq_column M w 0 w _ R rows j (by omega) h1 hR hM hMs, Nat.zero_add, + Native.columnCoefficients_encode P _ (by simp only [Vector.size_toArray]), + Native.cosetBlock_encode P _ (by rw [NTT.Forward.forwardSpec, Array.size_ofFn]; rfl), + encode_eq_map_val] + unfold Native.cosetLde + simp only [Array.map_flatMap, Array.map_map, Function.comp_def, Tasks.get_bind, Tasks.get_spawn, + Tasks.get_join, Tasks.map_get_spawnAll, Native.rowChunks, Array.size_map, Array.size_range] + rw [← Native.rowChunks, hpieces, flatMap_range_map] + apply Array.ext (by simp only [Array.size_map]) + intro c h1 _ + simp only [Array.size_map, Array.size_range] at h1 + simp only [Array.getElem_map, Array.getElem_range] + have hcb : c + 1 ≤ 2 ^ P.bits * C := h1 + have hk : c % C < C := Nat.mod_lt _ hC0 + have hrk : (c % C) * rows' + rows' ≤ 2 ^ P.logH := by + have := Nat.mul_le_mul_right rows' (show c % C + 1 ≤ C by omega) + rw [Nat.succ_mul, Nat.mul_comm C] at this; omega + have hrw : 4 * (rows' * w) < USize.size := by + have := Nat.mul_le_mul_right w (show rows' ≤ 2 ^ P.logH by + have := Nat.mul_le_mul_left rows' (show 1 ≤ C by omega); rw [hC] at this; omega) + omega + rw [hblock (c / C), Native.rowsOfColumns_encodeWords _ _ _ (fun j hj ↦ by + simp only [Array.size_map, Array.size_range] at hj + rw [getD_map_range _ _ _ hj, Array.size_map, hB] + exact ⟨hrk, hHu⟩) (by simpa only [Array.size_map, Array.size_range] using hrw) hrows', + encode_eq_map_val] + congr 1 + have hW : (Array.map (fun j ↦ Array.map Subtype.val (B (c / C) j)) (Array.range w)).size = w := by + simp only [Array.size_map, Array.size_range] + have hcC : c * rows' = 2 ^ P.logH * (c / C) + c % C * rows' := by + have := Nat.div_add_mod c C + rw [← hC] + nth_rewrite 1 [← this] + ring + have hL : c * (rows' * w) + rows' * w ≤ 2 ^ P.domain.logN * w := by + have h1 : (c + 1) * rows' ≤ 2 ^ P.bits * C * rows' := Nat.mul_le_mul_right _ hcb + have h2 : 2 ^ P.bits * C * rows' = 2 ^ P.domain.logN := by + rw [Nat.mul_assoc, Nat.mul_comm C, hC, hlog, Nat.pow_add] + have h3 := Nat.mul_le_mul_right w (h1.trans h2.le) + have e : (c + 1) * rows' * w = c * (rows' * w) + rows' * w := by ring + omega + have hL' : c * (2 ^ P.logH / Native.chunkCount P.logH * w) + + 2 ^ P.logH / Native.chunkCount P.logH * w ≤ 2 ^ P.domain.logN * w := hL + apply Array.ext (by + simp only [Array.size_ofFn, Array.size_map, Array.size_extract, ldeSpec, hW] + have : rows' * w = 2 ^ P.logH / Native.chunkCount P.logH * w := rfl + omega) + intro idx h1 _ + simp only [Array.size_ofFn, hW] at h1 + have hw0 : 0 < w := Nat.pos_of_ne_zero (by rintro rfl; simp at h1) + have hq : idx / w < rows' := (Nat.div_lt_iff_lt_mul hw0).mpr h1 + have he : c * (rows' * w) + idx = w * (c * rows') + idx := by ring + have hdiv : (c * (rows' * w) + idx) / w = c * rows' + idx / w := by + rw [he, Nat.mul_add_div hw0] + have hmod : (c * (rows' * w) + idx) % w = idx % w := by + rw [he, Nat.mul_add_mod] + have hi : c % C * rows' + idx / w < 2 ^ P.logH := by omega + rw [Array.getElem_ofFn, Array.getElem_map, Array.getElem_extract] + simp only [hW] + rw [getD_map_range _ _ _ (Nat.mod_lt _ hw0), getD_map_val, getD_block P _ _ _ hi] + simp only [ldeSpec, Array.getElem_ofFn] + rw [hdiv, hmod, hcC, Nat.add_assoc] + +/-- **The batched coset LDE is correct.** Joined, the row chunks of the extension of a packed +row-major matrix `M` of width `w` and `2 ^ logH` rows, given in `2 ^ k` chunks of rows, are the +packed specification: row `r` holds +every column's interpolant on the subgroup of order `2 ^ logH`, evaluated at +`shift · ω ^ bitrev r`. -/ +theorem Native.joinRows_cosetLde (P : LdePlan p) (w : Nat) (M : Array (FastField p)) (k : Nat) + (hk : k ≤ P.logH) (hM : M.size = 2 ^ P.logH * w) + (hu : 4 * (2 ^ P.domain.logN * w) < USize.size) : + Native.joinRows (Native.cosetLde P w (Native.rowChunks M w P.logH k)) = + encode (ldeSpec P w M) := by + rw [Native.cosetLde_eq P w M k hk hM hu] + apply joinRows_extract + have hlog : P.domain.logN = P.bits + P.logH := by + have := P.bits_le; simp only [LdePlan.logH]; omega + have hC := pow_div_mul_pow_min P.logH 3 + simp only [ldeSpec, Array.size_ofFn, hlog, Nat.pow_add] + unfold Native.chunkCount + generalize 2 ^ P.logH / 2 ^ min 3 P.logH = q at hC ⊢ + rw [← hC] + ring + +/-- `Native.joinRows_cosetLde` for the compiled extension. -/ +theorem LdePlan.joinRows_cosetLde [PackedCosetLde p] (P : LdePlan p) (w : Nat) + (M : Array (FastField p)) (k : Nat) (hk : k ≤ P.logH) (hM : M.size = 2 ^ P.logH * w) + (hu : 4 * (2 ^ P.domain.logN * w) < USize.size) : + Native.joinRows (P.cosetLde w (Native.rowChunks M w P.logH k)) = encode (ldeSpec P w M) := by + rw [LdePlan.cosetLde, PackedCosetLde.run_eq] + exact Native.joinRows_cosetLde P w M k hk hM hu + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/CosetLdeTransposes.lean b/CompPoly/Univariate/NTTFast/Packed/CosetLdeTransposes.lean new file mode 100644 index 00000000..d4021763 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/CosetLdeTransposes.lean @@ -0,0 +1,690 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.CosetLde +public import CompPoly.Univariate.NTTFast.Packed.NativeOrderCorrectness +import all CompPoly.Univariate.NTTFast.Packed.CosetLde +import all CompPoly.Univariate.NTTFast.Packed.Native +import all CompPoly.Univariate.NTTFast.Packed.NativeOrder +import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas + +/-! # Correctness of the coset LDE transposes + +The transposes write every output word once or more, and every write stores the word's final +value. `Fill T S N g` says that `g` overwrites words of an `N`-word buffer only with their +targets `T`, among them every word in `S`. Fills compose along folds with the union of their +sets, so a transpose whose kernels and edge loops together cover every word computes the buffer +of targets, whatever the order and overlap of its writes. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed.Native +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode size_encode_uint32) + +/-! ### Fills -/ + +/-- `o'` is `o` with words overwritten by their targets `T`, among them every word in `S`. -/ +def Fills (T : Nat → UInt32) (S : Nat → Prop) (o o' : Array UInt32) : Prop := + o'.size = o.size ∧ (∀ k, o.getD k 0 = T k → o'.getD k 0 = T k) ∧ + ∀ k, S k → o'.getD k 0 = T k + +/-- `g` maps every packed `N`-word buffer to one that fills `S` with targets `T`. -/ +def Fill (T : Nat → UInt32) (S : Nat → Prop) (N : Nat) (g : ByteArray → ByteArray) : Prop := + ∀ o : Array UInt32, o.size = N → ∃ o', g (encode o) = encode o' ∧ Fills T S o o' + +/-- The identity fills nothing. -/ +theorem Fill.id (T : Nat → UInt32) (N : Nat) : Fill T (fun _ ↦ False) N id := + fun o _ ↦ ⟨o, rfl, rfl, fun _ h ↦ h, fun _ h ↦ h.elim⟩ + +/-- One fill after another fills both sets. -/ +theorem Fill.comp {T : Nat → UInt32} {S S' : Nat → Prop} {N : Nat} {g g' : ByteArray → ByteArray} + (h : Fill T S N g) (h' : Fill T S' N g') : Fill T (fun k ↦ S k ∨ S' k) N (g' ∘ g) := by + intro o ho + obtain ⟨o1, e1, s1, k1, f1⟩ := h o ho + obtain ⟨o2, e2, s2, k2, f2⟩ := h' o1 (by rw [s1, ho]) + refine ⟨o2, by rw [Function.comp_apply, e1, e2], by rw [s2, s1], fun k hk ↦ k2 k (k1 k hk), ?_⟩ + rintro k (hk | hk) + · exact k2 k (f1 k hk) + · exact f2 k hk + +/-- A fill fills every subset of its set. -/ +theorem Fill.mono {T : Nat → UInt32} {S S' : Nat → Prop} {N : Nat} {g : ByteArray → ByteArray} + (h : Fill T S N g) (hS : ∀ k, S' k → S k) : Fill T S' N g := by + intro o ho + obtain ⟨o', e, s, k, f⟩ := h o ho + exact ⟨o', e, s, k, fun j hj ↦ f j (hS j hj)⟩ + +/-- Fills transfer along pointwise equal functions. -/ +theorem Fill.congr {T : Nat → UInt32} {S : Nat → Prop} {N : Nat} {g g' : ByteArray → ByteArray} + (h : Fill T S N g) (he : ∀ b, g b = g' b) : Fill T S N g' := by + intro o ho + obtain ⟨o', e, f⟩ := h o ho + exact ⟨o', by rw [← he, e], f⟩ + +/-- Fills transfer along functions equal on packed `N`-word buffers. -/ +theorem Fill.congr_encode {T : Nat → UInt32} {S : Nat → Prop} {N : Nat} + {g g' : ByteArray → ByteArray} (h : Fill T S N g) + (he : ∀ o : Array UInt32, o.size = N → g (encode o) = g' (encode o)) : Fill T S N g' := by + intro o ho + obtain ⟨o', e, f⟩ := h o ho + exact ⟨o', by rw [← he o ho, e], f⟩ + +/-- A fold of fills fills the union of their sets. -/ +theorem Fill.foldl {ι : Type*} (T : Nat → UInt32) (S : ι → Nat → Prop) (N : Nat) (L : List ι) + (f : ByteArray → ι → ByteArray) (hf : ∀ x ∈ L, Fill T (S x) N (f · x)) : + Fill T (fun k ↦ ∃ x ∈ L, S x k) N (fun b ↦ L.foldl f b) := by + induction L with + | nil => exact (Fill.id T N).mono fun k ⟨_, h, _⟩ ↦ absurd h List.not_mem_nil + | cons x L ih => + have h := (hf x List.mem_cons_self).comp (ih fun y hy ↦ hf y (List.mem_cons_of_mem x hy)) + refine (h.congr fun b ↦ rfl).mono ?_ + rintro k ⟨y, hy, hk⟩ + rcases List.mem_cons.mp hy with rfl | hy + · exact Or.inl hk + · exact Or.inr ⟨y, hy, hk⟩ + +/-- A fill that covers every word computes the buffer of targets. -/ +theorem Fill.eq_ofFn {T : Nat → UInt32} {S : Nat → Prop} {N : Nat} {g : ByteArray → ByteArray} + (h : Fill T S N g) (hS : ∀ k < N, S k) (o : Array UInt32) (ho : o.size = N) : + g (encode o) = encode (Array.ofFn (n := N) fun k ↦ T k.val) := by + obtain ⟨o', e, s, -, f⟩ := h o ho + rw [e] + congr 1 + apply array_eq_of_getD _ _ 0 (by rw [s, ho, Array.size_ofFn]) + intro k + by_cases hk : k < N + · rw [f k (hS k hk), getD_ofFn_bounded _ _ hk] + · rw [Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by rw [s, ho]; omega), + Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by rw [Array.size_ofFn]; omega)] + +/-- A splice of target values fills its range. -/ +theorem fills_splice (T : Nat → UInt32) (o V : Array UInt32) (i : Nat) + (hi : i + V.size ≤ o.size) (hV : ∀ t < V.size, V.getD t 0 = T (i + t)) : + Fills T (fun k ↦ i ≤ k ∧ k < i + V.size) o (splice o i V) := by + refine ⟨size_splice o i V hi, fun k hk ↦ ?_, fun k ⟨h1, h2⟩ ↦ ?_⟩ + · rw [getD_splice o i V hi] + split + · exact hk + · split + · rw [hV _ (by omega), show i + (k - i) = k by omega] + · exact hk + · rw [getD_splice o i V hi, ite_eq_right_of_eq_false _ _ (eq_false (by omega)), + ite_eq_left_of_eq_true _ _ (eq_true h2), hV _ (by omega), + show i + (k - i) = k by omega] + +/-! ### Word stores -/ + +/-- A sixteen-word store splices the words in. -/ +theorem write16U_encodeWords (o : Array UInt32) (i : USize) (hs : 4 * o.size < USize.size) + (hi : i.toNat + 16 ≤ o.size) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) : + write16U (encode o) i v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 = + encode (splice o i.toNat + #[v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15]) := by + have ho : (i * 4).toNat = 4 * i.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_mul, usize_numeral 4 (by decide), hsize, Nat.mod_eq_of_lt (by omega)] + omega + unfold write16U + rw [storeWords_replace_encodeWords o (i * 4) 16 i.toNat (by decide) ho + (by rw [show (16 : UInt8).toNat = 16 from rfl]; omega) (by rw [size_encode_uint32]; exact hs)] + rfl + +/-- An eight-word store splices the words in. -/ +theorem storeWords8_encodeWords (o : Array UInt32) (off : USize) (i : Nat) + (ho : off.toNat = 4 * i) (hs : 4 * o.size < USize.size) (hi : i + 8 ≤ o.size) + (v0 v1 v2 v3 v4 v5 v6 v7 : UInt32) : + storeWords (encode o) off 8 v0 v1 v2 v3 v4 v5 v6 v7 0 0 0 0 0 0 0 0 = + encode (splice o i #[v0, v1, v2, v3, v4, v5, v6, v7]) := by + rw [storeWords_replace_encodeWords o off 8 i (by decide) ho + (by rw [show (8 : UInt8).toNat = 8 from rfl]; omega) (by rw [size_encode_uint32]; exact hs)] + rfl + +/-- A single-word store sets the word. -/ +theorem write_encodeWords (o : Array UInt32) (i : Nat) (v : UInt32) + (hs : 4 * o.size < USize.size) (hi : i < o.size) : + write (encode o) i v = encode (splice o i #[v]) := by + have hin : i < USize.size := by omega + have ho : (i.toUSize * 4).toNat = 4 * i := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_mul, USize.toNat_ofNat_of_lt' hin, usize_numeral 4 (by decide), hsize, + Nat.mod_eq_of_lt (by omega)] + omega + unfold write + rw [storeWords_replace_encodeWords o (i.toUSize * 4) 1 i (by decide) ho + (by rw [show (1 : UInt8).toNat = 1 from rfl]; omega) (by rw [size_encode_uint32]; exact hs)] + rfl + +/-- A single-word store of the target fills the word. -/ +theorem fill_write (T : Nat → UInt32) (N i : Nat) (v : UInt32) (hs : 4 * N < USize.size) + (hi : i < N) (hv : v = T i) : Fill T (· = i) N (fun b ↦ write b i v) := by + intro o ho + have hf := fills_splice T o #[v] i (by change i + 1 ≤ o.size; omega) fun t ht ↦ by + change t < 1 at ht + rw [show t = 0 by omega, hv] + rfl + exact ⟨_, write_encodeWords o i v (by omega) (by omega), hf.1, hf.2.1, + fun k hk ↦ hf.2.2 k ⟨by omega, by change k < i + 1; omega⟩⟩ + +/-- A sixteen-word store of targets fills its range. -/ +theorem fill_write16U (T : Nat → UInt32) (N : Nat) (i : USize) (g : Nat → UInt32) + (hs : 4 * N < USize.size) (hi : i.toNat + 16 ≤ N) (hg : ∀ t < 16, g t = T (i.toNat + t)) : + Fill T (fun k ↦ i.toNat ≤ k ∧ k < i.toNat + 16) N (fun b ↦ write16U b i (g 0) (g 1) (g 2) + (g 3) (g 4) (g 5) (g 6) (g 7) (g 8) (g 9) (g 10) (g 11) (g 12) (g 13) (g 14) (g 15)) := by + intro o ho + have hf := fills_splice T o (lit16 g) i.toNat (by rw [size_lit16]; omega) fun t ht ↦ by + rw [getD_lit16 g t (by simpa using ht), hg t (by simpa using ht)] + exact ⟨_, write16U_encodeWords o i (by omega) (by omega) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _, hf⟩ + +/-- An eight-word store of targets fills its range. -/ +theorem fill_storeWords8 (T : Nat → UInt32) (N : Nat) (off : USize) (i : Nat) (g : Nat → UInt32) + (ho : off.toNat = 4 * i) (hs : 4 * N < USize.size) (hi : i + 8 ≤ N) + (hg : ∀ t < 8, g t = T (i + t)) : + Fill T (fun k ↦ i ≤ k ∧ k < i + 8) N (fun b ↦ storeWords b off 8 (g 0) (g 1) (g 2) (g 3) + (g 4) (g 5) (g 6) (g 7) 0 0 0 0 0 0 0 0) := by + intro o ho' + have hf := fills_splice T o #[g 0, g 1, g 2, g 3, g 4, g 5, g 6, g 7] i + (by change i + 8 ≤ o.size; omega) fun t ht ↦ by + change t < 8 at ht + rw [← hg t ht] + interval_cases t <;> rfl + exact ⟨_, storeWords8_encodeWords o off i ho (by omega) (by omega) _ _ _ _ _ _ _ _, hf⟩ + +/-! ### Tile kernels -/ + +/-- A machine offset `p + u * ps` that cannot wrap. -/ +theorem usize_add_mul_numeral (p ps : USize) (u : Nat) (hu : u < 4294967296) + (h : p.toNat + u * ps.toNat < USize.size) : + (p + OfNat.ofNat u * ps).toNat = p.toNat + u * ps.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_add, USize.toNat_mul, usize_numeral u hu, hsize, + Nat.mod_eq_of_lt (show u * ps.toNat < USize.size by omega), Nat.mod_eq_of_lt h] + +/-- The strided tile kernel stores its sixteen transposed lines one after another. -/ +theorem tileStrided_eq_foldl (src : Array UInt32) (q qs : USize) (out : ByteArray) (p ps : USize) + (hl : ∀ t < 16, (q + OfNat.ofNat t * qs).toNat + 16 ≤ src.size) + (hs : 4 * src.size < USize.size) : + tileStrided (encode src) q qs out p ps = (List.range 16).foldl (fun b u ↦ + let g (t : Nat) : UInt32 := src.getD ((q + OfNat.ofNat t * qs).toNat + u) 0 + write16U b (p + OfNat.ofNat u * ps) (g 0) (g 1) (g 2) (g 3) (g 4) (g 5) (g 6) (g 7) (g 8) + (g 9) (g 10) (g 11) (g 12) (g 13) (g 14) (g 15)) out := by + have hf (t : Nat) (ht : t < 16) : LineFits (encode src) (q + OfNat.ofNat t * qs) := + lineFits_encodeWords _ _ (hl t ht) (by rw [size_encode_uint32]; exact hs) + unfold tileStrided + rw [dite_eq_left_of_eq_true (eq_true ⟨hf 0 (by decide), hf 1 (by decide), hf 2 (by decide), + hf 3 (by decide), hf 4 (by decide), hf 5 (by decide), hf 6 (by decide), hf 7 (by decide), + hf 8 (by decide), hf 9 (by decide), hf 10 (by decide), hf 11 (by decide), hf 12 (by decide), + hf 13 (by decide), hf 14 (by decide), hf 15 (by decide)⟩)] + simp only [line_encodeWords] + rfl + +/-- The strided tile kernel fills the transposed tile with its targets. -/ +theorem fill_tileStrided (T : Nat → UInt32) (N : Nat) (src : Array UInt32) (q qs p ps : USize) + (hs : 4 * src.size < USize.size) (hN : 4 * N < USize.size) + (hq : q.toNat + 15 * qs.toNat + 16 ≤ src.size) (hp : p.toNat + 15 * ps.toNat + 16 ≤ N) + (hT : ∀ u < 16, ∀ t < 16, + src.getD (q.toNat + t * qs.toNat + u) 0 = T (p.toNat + u * ps.toNat + t)) : + Fill T (fun k ↦ ∃ u < 16, ∃ t < 16, k = p.toNat + u * ps.toNat + t) N + (fun b ↦ tileStrided (encode src) q qs b p ps) := by + have hqt (t : Nat) (ht : t < 16) : (q + OfNat.ofNat t * qs).toNat = q.toNat + t * qs.toNat := by + have := Nat.mul_le_mul_right qs.toNat (show t ≤ 15 by omega) + exact usize_add_mul_numeral q qs t (by omega) (by omega) + have hpu (u : Nat) (hu : u < 16) : (p + OfNat.ofNat u * ps).toNat = p.toNat + u * ps.toNat := by + have := Nat.mul_le_mul_right ps.toNat (show u ≤ 15 by omega) + exact usize_add_mul_numeral p ps u (by omega) (by omega) + let f (b : ByteArray) (u : Nat) : ByteArray := + let g (t : Nat) : UInt32 := src.getD ((q + OfNat.ofNat t * qs).toNat + u) 0 + write16U b (p + OfNat.ofNat u * ps) (g 0) (g 1) (g 2) (g 3) (g 4) (g 5) (g 6) (g 7) (g 8) + (g 9) (g 10) (g 11) (g 12) (g 13) (g 14) (g 15) + have hstep (u : Nat) (hu : u ∈ List.range 16) : + Fill T (fun k ↦ (p + OfNat.ofNat u * ps).toNat ≤ k ∧ k < (p + OfNat.ofNat u * ps).toNat + 16) + N (f · u) := by + have hu := List.mem_range.mp hu + have := Nat.mul_le_mul_right ps.toNat (show u ≤ 15 by omega) + refine fill_write16U T N (p + OfNat.ofNat u * ps) + (fun t ↦ src.getD ((q + OfNat.ofNat t * qs).toNat + u) 0) hN + (by rw [hpu u hu]; omega) fun t ht ↦ ?_ + rw [hqt t ht, hpu u hu, hT u hu t ht] + have he (b : ByteArray) : (List.range 16).foldl f b = tileStrided (encode src) q qs b p ps := + (tileStrided_eq_foldl src q qs b p ps (fun t ht ↦ by + have := Nat.mul_le_mul_right qs.toNat (show t ≤ 15 by omega) + rw [hqt t ht] + omega) hs).symm + refine ((Fill.foldl T _ N (List.range 16) f hstep).congr he).mono ?_ + rintro k ⟨u, hu, t, ht, rfl⟩ + exact ⟨u, List.mem_range.mpr hu, by rw [hpu u hu]; omega, by rw [hpu u hu]; omega⟩ + +/-! ### Input transpose -/ + +/-- Quotient and remainder of `a * n + b`, `b < n`. -/ +theorem divMod_add (a b n : Nat) (hb : b < n) : (a * n + b) / n = a ∧ (a * n + b) % n = b := by + constructor + · rw [Nat.add_comm, Nat.add_mul_div_right _ _ (by omega), Nat.div_eq_of_lt hb, Nat.zero_add] + · rw [Nat.add_comm, Nat.add_mul_mod_self_right, Nat.mod_eq_of_lt hb] + +/-- Rows `r0, …, r0 + rows - 1` of columns `c0, …, c0 + n - 1` of a row-major matrix of width +`w`, transposed: word `j * rows + r` is entry `(r0 + r, c0 + j)`. -/ +theorem transposeRows_encodeWords (M : Array UInt32) (w c0 n r0 rows : Nat) (hn : c0 + n ≤ w) + (hM : (r0 + rows) * w ≤ M.size) (hs : 4 * M.size < USize.size) : + transposeRows (encode M) w c0 n r0 rows = encode (Array.ofFn (n := n * rows) fun k ↦ + M.getD ((r0 + k.val % rows) * w + c0 + k.val / rows) 0) := by + let T (k : Nat) : UInt32 := M.getD ((r0 + k % rows) * w + c0 + k / rows) 0 + have hrw : rows * w ≤ (r0 + rows) * w := Nat.mul_le_mul_right w (by omega) + have hnr : n * rows ≤ w * rows := Nat.mul_le_mul_right rows (by omega) + have hwr : w * rows = rows * w := Nat.mul_comm w rows + have hN : 4 * (n * rows) < USize.size := by omega + have hu (n : Nat) (hn : n ≤ M.size) : n.toUSize.toNat = n := USize.toNat_ofNat_of_lt' (by omega) + have htile (rt : Nat) (hrt : rt < rows / 16) (ct : Nat) + (hct : ct ∈ List.range (n / 16)) : + Fill T (fun k ↦ ∃ u < 16, ∃ t < 16, k = (16 * ct + u) * rows + (16 * rt + t)) (n * rows) + (fun b ↦ tileStrided (encode M) ((r0 + 16 * rt) * w + c0 + 16 * ct).toUSize w.toUSize b + (16 * ct * rows + 16 * rt).toUSize rows.toUSize) := by + have hct := List.mem_range.mp hct + have h1 : (r0 + 16 * rt + 16) * w ≤ (r0 + rows) * w := Nat.mul_le_mul_right w (by omega) + have h2 : (16 * ct + 16) * rows ≤ n * rows := Nat.mul_le_mul_right rows (by omega) + have e1 : (r0 + 16 * rt + 16) * w = (r0 + 16 * rt) * w + 16 * w := by ring + have e2 : (16 * ct + 16) * rows = 16 * ct * rows + 16 * rows := by ring + have hq : (r0 + 16 * rt) * w + c0 + 16 * ct ≤ M.size := by omega + have hp : 16 * ct * rows + 16 * rt ≤ M.size := by omega + refine (fill_tileStrided T (n * rows) M _ _ _ _ (by omega) hN ?_ ?_ ?_).mono ?_ + · rw [hu _ hq, hu _ (by omega)]; omega + · rw [hu _ hp, hu _ (by omega)]; omega + · intro u hu' t ht + rw [hu _ hq, hu _ (by omega), hu _ hp, hu _ (by omega)] + have hd := divMod_add (16 * ct + u) (16 * rt + t) rows (by omega) + simp only [T, show 16 * ct * rows + 16 * rt + u * rows + t = + (16 * ct + u) * rows + (16 * rt + t) by ring, hd.1, hd.2] + congr 1 + ring + · rintro k ⟨u, hu', t, ht, rfl⟩ + exact ⟨u, hu', t, ht, by rw [hu _ hp, hu _ (by omega)]; ring⟩ + have hedge (j : Nat) (hj : j ∈ List.range n) (r : Nat) + (hr : r ∈ List.range' (if j < 16 * (n / 16) then 16 * (rows / 16) else 0) + (rows - if j < 16 * (n / 16) then 16 * (rows / 16) else 0)) : + Fill T (· = j * rows + r) (n * rows) + (fun b ↦ write b (j * rows + r) (read (encode M) ((r0 + r) * w + c0 + j))) := by + have hj := List.mem_range.mp hj + have hr : r < rows := by have := List.mem_range'_1.mp hr; split at this <;> omega + have h1 : (r0 + r + 1) * w ≤ (r0 + rows) * w := Nat.mul_le_mul_right w (by omega) + have h2 : (j + 1) * rows ≤ n * rows := Nat.mul_le_mul_right rows (by omega) + have e1 : (r0 + r + 1) * w = (r0 + r) * w + w := by ring + have e2 : (j + 1) * rows = j * rows + rows := by ring + refine fill_write T (n * rows) _ _ hN (by omega) ?_ + have hd := divMod_add j r rows hr + rw [read_encodeWords M _ (by rw [size_encode_uint32]; exact hs) (by omega)] + simp only [T, hd.1, hd.2] + have hall := (Fill.foldl T _ (n * rows) (List.range ((rows / 16 + 63) / 64)) _ fun rb _ ↦ + Fill.foldl T _ (n * rows) _ _ fun ct hct ↦ + Fill.foldl T _ (n * rows) (List.range' (64 * rb) (min (64 * rb + 64) (rows / 16) - 64 * rb)) + _ fun rt hrt ↦ htile rt (by have := List.mem_range'_1.mp hrt; omega) ct hct).comp + (Fill.foldl T _ (n * rows) _ _ fun j hj ↦ Fill.foldl T _ (n * rows) _ _ (hedge j hj)) + unfold transposeRows + simp only [Std.Legacy.Range.forIn_eq_forIn_range', Std.Legacy.Range.size, + Nat.sub_zero, Nat.add_sub_cancel, Nat.div_one, ← List.range_eq_range', + List.forIn_pure_yield_eq_foldl, bind_pure, Id.run_pure, pure_bind, zeroWords_eq] + refine hall.eq_ofFn (fun k hk ↦ ?_) _ Array.size_replicate + have hrows : 0 < rows := Nat.pos_of_ne_zero (by rintro rfl; simp at hk) + have hk' := Nat.div_add_mod k rows + have hj : k / rows < n := (Nat.div_lt_iff_lt_mul hrows).mpr hk + have hr := Nat.mod_lt k hrows + by_cases hin : k / rows < 16 * (n / 16) ∧ k % rows < 16 * (rows / 16) + · refine Or.inl ⟨k % rows / 16 / 64, List.mem_range.mpr (by omega), k / rows / 16, + List.mem_range.mpr (by omega), k % rows / 16, List.mem_range'_1.mpr (by omega), + k / rows % 16, Nat.mod_lt _ (by decide), k % rows % 16, Nat.mod_lt _ (by decide), ?_⟩ + rw [show 16 * (k / rows / 16) + k / rows % 16 = k / rows by omega, + show 16 * (k % rows / 16) + k % rows % 16 = k % rows by omega, Nat.mul_comm] + omega + · refine Or.inr ⟨k / rows, List.mem_range.mpr hj, k % rows, List.mem_range'_1.mpr ?_, ?_⟩ + · split <;> omega + · rw [Nat.mul_comm]; omega + +/-! ### Output transpose -/ + +/-- A checked line of eight words reads as the bounds-proved load. -/ +theorem line8Fits_readAt {b : ByteArray} {q : USize} (h : Line8Fits b q) (k : Nat) (hk : k < 8) + (hp) : b.ugetUInt32LE ((q + OfNat.ofNat k) * 4) hp = readAt b q (OfNat.ofNat k) := by + have hf := line8Fits_toNat h k hk + have hb := toNat_usize_le b + have hU := b.usize.toNat_lt_size + have h4 := usize_numeral 4 (by decide) + have h8 := h.1 + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, usize_numeral 8 (by decide)] at h8 + have h2 := h.2 + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ h.1, USize.toNat_div, h4, + usize_numeral 8 (by decide)] at h2 + have hc : 4 ≤ b.usize ∧ (q + OfNat.ofNat k) * 4 ≤ b.usize - 4 := by + have h4' : 4 ≤ b.usize := by rw [USize.le_iff_toNat_le, h4]; omega + refine ⟨h4', ?_⟩ + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ h4', h4, hf.1] + omega + unfold readAt + rw [dite_eq_left_of_eq_true (eq_true hc)] + +/-- The eight-source tile kernel stores its eight transposed lines one after another. -/ +theorem tile8_eq_foldl (s : Nat → ByteArray) (q : USize) (out : ByteArray) (p ps : USize) + (hs : ∀ t < 8, Line8Fits (s t) q) (ho : Tile8Fits out p ps) (hn : out.size < USize.size) : + tile8 (s 0) (s 1) (s 2) (s 3) (s 4) (s 5) (s 6) (s 7) q out p ps = + (List.range 8).foldl (fun b u ↦ + let g (t : Nat) : UInt32 := readAt (s t) q (OfNat.ofNat u) + storeWords b ((p + OfNat.ofNat u * ps) * 4) 8 (g 0) (g 1) (g 2) (g 3) (g 4) (g 5) (g 6) + (g 7) 0 0 0 0 0 0 0 0) out := by + unfold tile8 + rw [dite_eq_left_of_eq_true (eq_true ⟨hs 0 (by decide), hs 1 (by decide), hs 2 (by decide), + hs 3 (by decide), hs 4 (by decide), hs 5 (by decide), hs 6 (by decide), hs 7 (by decide), + ho⟩)] + simp only [store8_val _ _ _ _ _ _ _ _ _ _ _ hn, touchLast, touch_eq] + simp (disch := decide) only [line8Fits_readAt (hs 0 (by decide)), + line8Fits_readAt (hs 1 (by decide)), line8Fits_readAt (hs 2 (by decide)), + line8Fits_readAt (hs 3 (by decide)), line8Fits_readAt (hs 4 (by decide)), + line8Fits_readAt (hs 5 (by decide)), line8Fits_readAt (hs 6 (by decide)), + line8Fits_readAt (hs 7 (by decide))] + rfl + +/-- The eight-source tile kernel fills the transposed tile with its targets. -/ +theorem fill_tile8 (T : Nat → UInt32) (N : Nat) (s : Nat → ByteArray) (V : Nat → Array UInt32) + (q p ps : USize) (hsV : ∀ t < 8, s t = encode (V t)) + (hV : ∀ t < 8, q.toNat + 8 ≤ (V t).size ∧ 4 * (V t).size < USize.size) + (hN : 4 * N < USize.size) (hp : p.toNat + 7 * ps.toNat + 8 ≤ N) + (hT : ∀ u < 8, ∀ t < 8, (V t).getD (q.toNat + u) 0 = T (p.toNat + u * ps.toNat + t)) : + Fill T (fun k ↦ ∃ u < 8, ∃ t < 8, k = p.toNat + u * ps.toNat + t) N + (fun b ↦ tile8 (s 0) (s 1) (s 2) (s 3) (s 4) (s 5) (s 6) (s 7) q b p ps) := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hpu (u : Nat) (hu : u < 8) : (p + OfNat.ofNat u * ps).toNat = p.toNat + u * ps.toNat := by + have := Nat.mul_le_mul_right ps.toNat (show u ≤ 7 by omega) + exact usize_add_mul_numeral p ps u (by omega) (by omega) + let f (b : ByteArray) (u : Nat) : ByteArray := + let g (t : Nat) : UInt32 := readAt (s t) q (OfNat.ofNat u) + storeWords b ((p + OfNat.ofNat u * ps) * 4) 8 (g 0) (g 1) (g 2) (g 3) (g 4) (g 5) (g 6) + (g 7) 0 0 0 0 0 0 0 0 + have hstep (u : Nat) (hu : u ∈ List.range 8) : + Fill T (fun k ↦ p.toNat + u * ps.toNat ≤ k ∧ k < p.toNat + u * ps.toNat + 8) N (f · u) := by + have hu := List.mem_range.mp hu + have := Nat.mul_le_mul_right ps.toNat (show u ≤ 7 by omega) + refine fill_storeWords8 T N _ (p.toNat + u * ps.toNat) (fun t ↦ readAt (s t) q (OfNat.ofNat u)) + ?_ hN (by omega) fun t ht ↦ ?_ + · rw [USize.toNat_mul, hpu u hu, usize_numeral 4 (by decide), hsize, + Nat.mod_eq_of_lt (by omega)] + omega + · have hq := hV t ht + rw [hsV t ht, readAt_encodeWords _ _ _ (by + rw [size_encode_uint32, usize_numeral u (by omega)]; omega), usize_numeral u (by omega), + hT u hu t ht] + have hs (t : Nat) (ht : t < 8) : Line8Fits (s t) q := by + have hq := hV t ht + have hu : (s t).usize.toNat = 4 * (V t).size := by + rw [hsV t ht, ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' (by + rw [size_encode_uint32]; exact hq.2), size_encode_uint32] + have h4 := usize_numeral 4 (by decide) + have h8 := usize_numeral 8 (by decide) + have h1 : 8 ≤ (s t).usize / 4 := by + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, h8, hu]; omega + refine ⟨h1, ?_⟩ + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ h1, USize.toNat_div, h4, h8, hu] + omega + refine ((Fill.foldl T _ N (List.range 8) f hstep).congr_encode fun o ho ↦ + (tile8_eq_foldl s q (encode o) p ps hs (by + have hu : (encode o).usize.toNat = 4 * N := by + rw [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' (by + rw [size_encode_uint32]; omega), size_encode_uint32, ho] + have := Nat.mul_le_mul_right ps.toNat (show 7 ≤ 7 by omega) + rw [tile8Fits_iff, hu] + omega) (by rw [size_encode_uint32]; omega)).symm).mono ?_ + rintro k ⟨u, hu, t, ht, rfl⟩ + exact ⟨u, List.mem_range.mpr hu, by omega, by omega⟩ + +/-- `count` tiles fill `count` consecutive tiles of eight output rows. -/ +theorem fill_rowsTiles (T : Nat → UInt32) (N : Nat) (s : Nat → ByteArray) + (V : Nat → Array UInt32) (w : USize) (hsV : ∀ t < 8, s t = encode (V t)) + (hN : 4 * N < USize.size) (count : Nat) (q p : USize) + (hV : ∀ t < 8, q.toNat + 8 * count ≤ (V t).size ∧ 4 * (V t).size < USize.size) + (hp : ∀ c < count, p.toNat + 8 * c * w.toNat + 7 * w.toNat + 8 ≤ N) + (hT : ∀ c < count, ∀ u < 8, ∀ t < 8, + (V t).getD (q.toNat + 8 * c + u) 0 = T (p.toNat + 8 * c * w.toNat + u * w.toNat + t)) : + Fill T (fun k ↦ ∃ c < count, ∃ u < 8, ∃ t < 8, k = p.toNat + 8 * c * w.toNat + u * w.toNat + t) + N (fun b ↦ rowsTiles (s 0) (s 1) (s 2) (s 3) (s 4) (s 5) (s 6) (s 7) w count q p b) := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + induction count generalizing q p with + | zero => exact (Fill.id T N).mono fun k ⟨c, hc, _⟩ ↦ absurd hc (Nat.not_lt_zero c) + | succ count ih => + have h0 := fill_tile8 T N s V q p w hsV (fun t ht ↦ ⟨by have := hV t ht; omega, (hV t ht).2⟩) + hN (by have := hp 0 (by omega); omega) fun u hu t ht ↦ by + have := hT 0 (by omega) u hu t ht + simpa using this + have hrest : Fill T (fun k ↦ ∃ c < count, ∃ u < 8, ∃ t < 8, + k = p.toNat + 8 * (c + 1) * w.toNat + u * w.toNat + t) N + (fun b ↦ rowsTiles (s 0) (s 1) (s 2) (s 3) (s 4) (s 5) (s 6) (s 7) w count (q + 8) + (p + 8 * w) b) := by + rcases Nat.eq_zero_or_pos count with rfl | hc + · exact (Fill.id T N).mono fun k ⟨c, hc, _⟩ ↦ absurd hc (Nat.not_lt_zero c) + have hq8 : (q + 8).toNat = q.toNat + 8 := by + have := hV 0 (by decide) + rw [USize.toNat_add, usize_numeral 8 (by decide), hsize, Nat.mod_eq_of_lt (by omega)] + have hp8 : (p + 8 * w).toNat = p.toNat + 8 * w.toNat := by + have := hp 1 (by omega) + exact usize_add_mul_numeral p w 8 (by decide) (by omega) + refine (ih (q + 8) (p + 8 * w) (fun t ht ↦ ⟨by rw [hq8]; have := hV t ht; omega, + (hV t ht).2⟩) (fun c hc ↦ ?_) fun c hc u hu t ht ↦ ?_).mono ?_ + · rw [hp8] + have := hp (c + 1) (by omega) + rw [show 8 * (c + 1) * w.toNat = 8 * c * w.toNat + 8 * w.toNat by ring] at this + omega + · rw [hq8, hp8, show q.toNat + 8 + 8 * c + u = q.toNat + 8 * (c + 1) + u by ring, + hT (c + 1) (by omega) u hu t ht] + congr 1 + ring + · rintro k ⟨c, hc, u, hu, t, ht, rfl⟩ + exact ⟨c, hc, u, hu, t, ht, by rw [hp8]; ring⟩ + refine (h0.comp hrest).mono ?_ + rintro k ⟨c, hc, u, hu, t, ht, rfl⟩ + rcases c with _ | c + · exact Or.inl ⟨u, hu, t, ht, by ring⟩ + · exact Or.inr ⟨c, by omega, u, hu, t, ht, rfl⟩ + +/-- Packed word arrays keep the empty default. -/ +theorem getD_map_encodeWords (C : Array (Array UInt32)) (j : Nat) : + (C.map encode).getD j ByteArray.empty = encode (C.getD j #[]) := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_map] + cases C[j]? <;> rfl + +/-- Rows `r0, …, r0 + rows - 1` of the matrix with columns `C`: word `r * C.size + j` is word +`r0 + r` of column `j`. A block size dividing the rows, `rows < 512 ∨ 512 ∣ rows`, lets the tiles +and edges cover every word. -/ +theorem rowsOfColumns_encodeWords (C : Array (Array UInt32)) (r0 rows : Nat) + (hC : ∀ j < C.size, r0 + rows ≤ (C.getD j #[]).size ∧ 4 * (C.getD j #[]).size < USize.size) + (hN : 4 * (rows * C.size) < USize.size) (hrows : rows < 512 ∨ 512 ∣ rows) : + rowsOfColumns (C.map encode) r0 rows = encode (Array.ofFn (n := rows * C.size) fun k ↦ + (C.getD (k.val % C.size) #[]).getD (r0 + k.val / C.size) 0) := by + let w := C.size + have hwC : w = C.size := rfl + have hN' : 4 * (rows * w) < USize.size := hN + let T (k : Nat) : UInt32 := (C.getD (k % w) #[]).getD (r0 + k / w) 0 + let B := min rows 512 + have hB : (rows < 512 ∧ B = rows) ∨ (512 ≤ rows ∧ B = 512 ∧ 512 ∣ rows) := by + simp only [B]; omega + have htile (rb : Nat) (hrb : rb ∈ List.range (rows / B)) (ct : Nat) + (hct : ct ∈ List.range (w / 8)) : + Fill T (fun k ↦ ∃ c < B / 8, ∃ u < 8, ∃ t < 8, k = (rb * B + 8 * c + u) * w + (8 * ct + t)) + (rows * w) (fun b ↦ rowsTiles ((C.map encode).getD (8 * ct + 0) .empty) + ((C.map encode).getD (8 * ct + 1) .empty) ((C.map encode).getD (8 * ct + 2) .empty) + ((C.map encode).getD (8 * ct + 3) .empty) ((C.map encode).getD (8 * ct + 4) .empty) + ((C.map encode).getD (8 * ct + 5) .empty) ((C.map encode).getD (8 * ct + 6) .empty) + ((C.map encode).getD (8 * ct + 7) .empty) w.toUSize (B / 8) (r0 + rb * B).toUSize + (rb * B * w + 8 * ct).toUSize b) := by + have hrb := List.mem_range.mp hrb + have hct := List.mem_range.mp hct + have hB0 : 0 < B := Nat.pos_of_ne_zero (by rintro h; rw [h, Nat.div_zero] at hrb; omega) + have hrb' : (rb + 1) * B ≤ rows := (Nat.le_div_iff_mul_le hB0).mp hrb + have e0 : (rb + 1) * B = rb * B + B := by ring + have hw : (8 * ct + 8) * 1 ≤ w := by omega + have hrw : (rb + 1) * B * w ≤ rows * w := Nat.mul_le_mul_right w hrb' + have h1 : 1 * w ≤ (rb + 1) * B * w := Nat.mul_le_mul_right w (by nlinarith) + have hwu : w.toUSize.toNat = w := USize.toNat_ofNat_of_lt' (by omega) + have hcol (t : Nat) (ht : t < 8) := hC (8 * ct + t) (by omega) + have hq : (r0 + rb * B).toUSize.toNat = r0 + rb * B := USize.toNat_ofNat_of_lt' (by + have := hcol 0 (by decide) + have : rb * B ≤ rows := by nlinarith + omega) + have e1 : (rb + 1) * B * w = rb * B * w + B * w := by ring + have hBw : 8 * (B / 8) * w ≤ B * w := Nat.mul_le_mul_right w (by omega) + have hp : (rb * B * w + 8 * ct).toUSize.toNat = rb * B * w + 8 * ct := + USize.toNat_ofNat_of_lt' (by omega) + refine (fill_rowsTiles T (rows * w) (fun t ↦ (C.map encode).getD (8 * ct + t) .empty) + (fun t ↦ C.getD (8 * ct + t) #[]) w.toUSize (fun t _ ↦ getD_map_encodeWords C _) + (by omega) (B / 8) _ _ (fun t ht ↦ ⟨?_, (hcol t ht).2⟩) (fun c hc ↦ ?_) + fun c hc u hu t ht ↦ ?_).mono ?_ + · rw [hq]; have := (hcol t ht).1; omega + · rw [hp, hwu] + have : (rb * B + 8 * c + 8) * w ≤ rows * w := Nat.mul_le_mul_right w (by omega) + have e : (rb * B + 8 * c + 8) * w = rb * B * w + 8 * c * w + 8 * w := by ring + omega + · rw [hq, hp, hwu] + have hd := divMod_add (rb * B + 8 * c + u) (8 * ct + t) w (by omega) + simp only [T, show rb * B * w + 8 * ct + 8 * c * w + u * w + t = + (rb * B + 8 * c + u) * w + (8 * ct + t) by ring, hd.1, hd.2] + congr 1 + omega + · rintro k ⟨c, hc, u, hu, t, ht, rfl⟩ + exact ⟨c, hc, u, hu, t, ht, by rw [hp, hwu]; ring⟩ + have hedge (r : Nat) (hr : r ∈ List.range rows) (j : Nat) + (hj : j ∈ List.range' (if r < 8 * (rows / 8) then 8 * (w / 8) else 0) + (w - if r < 8 * (rows / 8) then 8 * (w / 8) else 0)) : + Fill T (· = r * w + j) (rows * w) + (fun b ↦ write b (r * w + j) (read ((C.map encode).getD j .empty) (r0 + r))) := by + have hr := List.mem_range.mp hr + have hj : j < w := by have := List.mem_range'_1.mp hj; split at this <;> omega + have h2 : (r + 1) * w ≤ rows * w := Nat.mul_le_mul_right w (by omega) + have e2 : (r + 1) * w = r * w + w := by ring + refine fill_write T (rows * w) _ _ hN (by omega) ?_ + have hd := divMod_add r j w hj + have hc := hC j hj + rw [getD_map_encodeWords, read_encodeWords _ _ (by rw [size_encode_uint32]; exact hc.2) + (by omega)] + simp only [T, hd.1, hd.2] + have hall := (Fill.foldl T _ (rows * w) _ _ fun rb hrb ↦ + Fill.foldl T _ (rows * w) _ _ (htile rb hrb)).comp + (Fill.foldl T _ (rows * w) _ _ fun r hr ↦ Fill.foldl T _ (rows * w) _ _ (hedge r hr)) + unfold rowsOfColumns + simp only [Std.Legacy.Range.forIn_eq_forIn_range', Std.Legacy.Range.size, + Nat.sub_zero, Nat.add_sub_cancel, Nat.div_one, ← List.range_eq_range', + List.forIn_pure_yield_eq_foldl, bind_pure, Id.run_pure, pure_bind, zeroWords_eq, + Array.size_map] + refine hall.eq_ofFn (fun k hk ↦ ?_) _ Array.size_replicate + have hw : 0 < w := Nat.pos_of_ne_zero (by rintro h; rw [h] at hk; simp at hk) + have hk' := Nat.div_add_mod k w + have hR : k / w < rows := (Nat.div_lt_iff_lt_mul hw).mpr hk + have hJ := Nat.mod_lt k hw + by_cases hin : k / w < 8 * (rows / 8) ∧ k % w < 8 * (w / 8) + · have hc : k / w % B < 8 * (B / 8) := by + rcases hB with ⟨_, hB⟩ | ⟨_, hB, _⟩ <;> rw [hB] + · rw [Nat.mod_eq_of_lt hR]; exact hin.1 + · omega + have hrb : k / w / B < rows / B := by + rcases hB with ⟨_, hB⟩ | ⟨_, hB, _⟩ <;> rw [hB] + · rw [Nat.div_eq_of_lt hR, Nat.div_self (by omega)]; decide + · omega + refine Or.inl ⟨k / w / B, List.mem_range.mpr hrb, k % w / 8, List.mem_range.mpr (by omega), + k / w % B / 8, by omega, k / w % B % 8, Nat.mod_lt _ (by decide), k % w % 8, + Nat.mod_lt _ (by decide), ?_⟩ + have := Nat.div_add_mod (k / w) B + rw [show k / w / B * B + 8 * (k / w % B / 8) + k / w % B % 8 = k / w by + rw [Nat.mul_comm]; omega, show 8 * (k % w / 8) + k % w % 8 = k % w by omega, Nat.mul_comm] + omega + · refine Or.inr ⟨k / w, List.mem_range.mpr hR, k % w, List.mem_range'_1.mpr ?_, ?_⟩ + · split <;> omega + · rw [Nat.mul_comm]; omega + +/-! ### Input columns -/ + +/-- Appending one range of words of a packed array. -/ +theorem copySlice_encodeWords (a o : Array UInt32) (i c : Nat) : + (encode a).copySlice (4 * i) (encode o) (encode o).size (4 * c) = + encode (o ++ a.extract i (i + c)) := + PackedRepr.copySlice_encode a o i c + +/-- Gathering word range `j` of every piece concatenates the ranges. -/ +theorem gatherColumn_encodeWords (P : Nat → Array UInt32) (R rows j : Nat) + (hP : ∀ k < R, j * rows + rows ≤ (P k).size) : + gatherColumn ((Array.range R).map fun k ↦ encode (P k)) rows j = + encode (Array.ofFn (n := R * rows) fun i ↦ + (P (i.val / rows)).getD (j * rows + i.val % rows) 0) := by + unfold gatherColumn + rw [Array.foldl_map, ← Array.foldl_toList, Array.toList_range, Array.size_map, Array.size_range, + show ByteArray.emptyWithCapacity (4 * (R * rows)) = encode (#[] : Array UInt32) from rfl] + induction R with + | zero => + simp only [List.range_zero, List.foldl_nil] + congr 1 + exact (Array.eq_empty_of_size_eq_zero (by simp)).symm + | succ R ih => + rw [List.range_succ, List.foldl_append, ih fun k hk ↦ hP k (by omega), List.foldl_cons, + List.foldl_nil, copySlice_encodeWords] + congr 1 + have hR := hP R (by omega) + apply array_eq_of_getD _ _ 0 (by + rw [Array.size_append, Array.size_ofFn, Array.size_extract, Array.size_ofFn] + rw [Nat.succ_mul]; omega) + intro i + rw [getD_append_eq, Array.size_ofFn] + by_cases hi : i < R * rows + · rw [ite_eq_left_of_eq_true _ _ (eq_true hi), getD_ofFn_bounded _ _ hi, + getD_ofFn_bounded _ _ (by rw [Nat.succ_mul]; omega)] + · rw [ite_eq_right_of_eq_false _ _ (eq_false hi)] + by_cases hi' : i < (R + 1) * rows + · have hrows : 0 < rows := Nat.pos_of_ne_zero (by rintro rfl; simp at hi') + have hd := divMod_add R (i - R * rows) rows (by rw [Nat.succ_mul] at hi'; omega) + rw [getD_ofFn_bounded _ _ hi', + getD_extract _ _ _ _ _ hR (by rw [Nat.succ_mul] at hi'; omega)] + rw [show R * rows + (i - R * rows) = i by omega] at hd + rw [hd.1, hd.2] + · rw [Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by + rw [Array.size_extract]; rw [Nat.succ_mul] at hi'; omega), + Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by rw [Array.size_ofFn]; omega)] + +/-- The columns of a matrix from its transposed row ranges of columns `c0, …, c0 + n - 1`: entry +`i` of column `j` is entry `(i, c0 + j)` of the matrix. -/ +theorem gatherColumn_transposeRows (M : Array UInt32) (w c0 n R rows j : Nat) (hn : c0 + n ≤ w) + (hj : j < n) (hM : R * rows * w ≤ M.size) (hs : 4 * M.size < USize.size) : + gatherColumn ((Array.range R).map fun k ↦ transposeRows (encode M) w c0 n (k * rows) rows) + rows j = encode (Array.ofFn (n := R * rows) fun i ↦ M.getD (i.val * w + c0 + j) 0) := by + have hpiece (k : Nat) (hk : k < R) : (k * rows + rows) * w ≤ M.size := by + have := Nat.mul_le_mul_right w (show k * rows + rows ≤ R * rows by + have := Nat.mul_le_mul_right rows (show k + 1 ≤ R by omega) + rw [Nat.succ_mul] at this; exact this) + omega + have hm : (Array.range R).map (fun k ↦ transposeRows (encode M) w c0 n (k * rows) rows) = + (Array.range R).map fun k ↦ encode (Array.ofFn (n := n * rows) fun i ↦ + M.getD ((k * rows + i.val % rows) * w + c0 + i.val / rows) 0) := by + apply Array.ext (by simp only [Array.size_map]) + intro k h1 _ + simp only [Array.size_map, Array.size_range] at h1 + simp only [Array.getElem_map, Array.getElem_range] + exact transposeRows_encodeWords M w c0 n _ rows hn (hpiece k h1) hs + rw [hm, gatherColumn_encodeWords _ R rows j fun k hk ↦ by + rw [Array.size_ofFn] + have := Nat.mul_le_mul_right rows (show j + 1 ≤ n by omega) + rw [Nat.succ_mul] at this; exact this] + congr 1 + apply Array.ext (by simp only [Array.size_ofFn]) + intro i h1 _ + simp only [Array.size_ofFn] at h1 + have hrows : 0 < rows := Nat.pos_of_ne_zero (by rintro rfl; simp at h1) + have hr := Nat.mod_lt i hrows + have hd := divMod_add j (i % rows) rows hr + have hjr : j * rows + i % rows < n * rows := by + have := Nat.mul_le_mul_right rows (show j + 1 ≤ n by omega) + rw [Nat.succ_mul] at this; omega + simp only [Array.getElem_ofFn] + rw [getD_ofFn_bounded _ _ hjr] + dsimp only + rw [hd.1, hd.2, show i / rows * rows + i % rows = i by + rw [Nat.mul_comm]; exact Nat.div_add_mod i rows] + +end CompPoly.CPolynomial.NTTFast.Packed.Native diff --git a/CompPoly/Univariate/NTTFast/Packed/Decoder.lean b/CompPoly/Univariate/NTTFast/Packed/Decoder.lean new file mode 100644 index 00000000..e3dba347 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Decoder.lean @@ -0,0 +1,63 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.FieldStorage +import all CompPoly.Univariate.NTTFast.Packed.Native +public import CompPoly.Univariate.NTTFast.Reverse32 + +/-! # Natural-order reads of a bit-reversed transform + +`decodedFields` reads a bit-reversed array in natural order, optionally scaled; `Native.decode` +computes it from packed words, for leaves too small for the fused final layers. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +/-- The natural-order read of a bit-reversed array, optionally scaled. -/ +def decodedFields (logN : Nat) (a : Array (FastField p)) + (factor : FastField p) (inverse : Bool) : Array (FastField p) := + Array.ofFn (fun i : Fin (2 ^ logN) ↦ + let x := a.getD (NTT.Transform.bitRevNat logN i.val) 0 + if inverse then factor * x else x) + +/-- The thirty-two-bit index calculation agrees with bit reversal on every bounded transform + index. -/ +theorem Native.reverse_index (logN i : Nat) (hlog : logN ≤ 32) (hi : i < 2 ^ logN) : + (reverse32 i.toUInt32 >>> (32 - logN).toUInt32).toNat = + NTT.Transform.bitRevNat logN i := by + by_cases hz : logN = 0 + · subst logN + have he : i = 0 := by simp only [Nat.pow_zero] at hi; omega + subst i + decide + · exact reverse32_shift_eq_bitRevNat logN i (by omega) hlog + +/-- Ordinary packed output conversion computes exactly reversed indexing and optional scaling. -/ +theorem Native.decode_encode (logN : Nat) (a : Array (FastField p)) + (factor : FastField p) (inverse : Bool) (hlog : logN ≤ 32) + (hs : a.size = 2 ^ logN) (hu : (encode a).size < USize.size) : + Native.decode p logN (encode a) factor.val inverse = decodedFields logN a factor inverse := by + change tabulate (fun i : Fin (2 ^ logN) ↦ + ofWord p (if inverse then mul p factor.val + (Native.read (encode a) (reverse32 i.val.toUInt32 >>> (32 - logN).toUInt32).toNat) + else Native.read (encode a) + (reverse32 i.val.toUInt32 >>> (32 - logN).toUInt32).toNat)) = _ + rw [tabulate_eq] + unfold decodedFields + apply congrArg Array.ofFn + funext i + rw [Native.reverse_index logN i.val hlog i.isLt, + Native.read_encode a _ hu (by rw [hs]; exact NTT.Transform.bitRevNat_lt _ _)] + cases inverse <;> simp only [Bool.false_eq_true, ite_false, ite_true, mul_val, ofWord_val] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/DifSpec.lean b/CompPoly/Univariate/NTTFast/Packed/DifSpec.lean new file mode 100644 index 00000000..3e616cf2 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/DifSpec.lean @@ -0,0 +1,95 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Correctness.DIF +import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas +public import CompPoly.Univariate.NTTFast.Packed.Decoder +import Mathlib.Tactic.Ring + +/-! # The bit-reversed DFT + +`difSpec` is the output of a DIF transform: the DFT in bit-reversed order. The radix-four plan +computes it, and its natural-order read `decodedFields` is the DFT, scaled for an inverse +transform. +-/ + +@[expose] public section +open scoped BigOperators +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +/-- The DFT formula with a natural-number evaluation index. -/ +def dftValue [Field R] (D : NTT.Domain R) (a : Array R) (k : Nat) : R := + ∑ i ∈ Finset.range D.n, a.getD i 0 * D.omega ^ (k * i) + +/-- Natural-number and bounded-index DFT formulas have the same sum. -/ +theorem dftValue_eq_nttAt [Field R] (D : NTT.Domain R) (a : Array R) (k : D.Idx) : + dftValue D a k.val = NTT.Forward.nttAt D a k := by + symm + exact Fin.sum_univ_eq_sum_range (fun i ↦ a.getD i 0 * D.omega ^ (k.val * i)) D.n + +/-- The ordinary bit-reversed mathematical forward output. -/ +def difSpec [Field R] (D : NTT.Domain R) (a : Array R) : Array R := + NTT.Transform.bitRevPermute D (NTT.Forward.forwardSpec D a) + +/-- The mathematical DIF output has one value per domain point. -/ +@[simp] theorem size_difSpec [Field R] (D : NTT.Domain R) (a : Array R) : + (difSpec D a).size = D.n := by + simp only [difSpec, NTT.Transform.bitRevPermute, Array.size_ofFn] + +/-- Reading a bounded DIF output evaluates the DFT at the reversed index. -/ +theorem getD_difSpec [Field R] (D : NTT.Domain R) (a : Array R) (i : Nat) (hi : i < D.n) : + (difSpec D a).getD i 0 = dftValue D a (NTT.Transform.bitRevNat D.logN i) := by + have hb : NTT.Transform.bitRevNat D.logN i < D.n := NTT.Transform.bitRevNat_lt _ _ + simp only [difSpec, NTT.Transform.bitRevPermute] + rw [getD_ofFn_bounded _ i hi 0] + simp only [NTT.Forward.forwardSpec] + rw [getD_ofFn_bounded _ _ hb 0] + exact (dftValue_eq_nttAt D a ⟨_, hb⟩).symm + +/-- An already domain-sized array is unchanged by zero-padding. -/ +theorem loadNaturalArray_eq_self [Field R] (D : NTT.Domain R) (a : Array R) + (hs : a.size = D.n) : NTT.loadNaturalArray D a = a := by + apply Array.ext (by rw [NTT.size_loadNaturalArray, hs]) + intro i hi hj + rw [NTT.getElem_loadNaturalArray] + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hj, Option.getD_some] + +/-- The ordinary radix-four pipeline computes the mathematical bit-reversed DFT. -/ +theorem planStages_eq_difSpec [Field R] (D : NTT.Domain R) (a : Array R) (hs : a.size = D.n) : + Plan.runStagesDIFRadix4WithTwiddles D (Plan.twiddleTable D) a = difSpec D a := by + have h := Plan.runStagesDIFRadix4WithTwiddles_correct D a + rw [loadNaturalArray_eq_self D a hs] at h + exact h + +/-- Decoding an unnormalized DIF result restores natural-order DFT evaluations. -/ +theorem decodedFields_difSpec (D : NTT.Domain (FastField p)) + (a : Array (FastField p)) (factor : FastField p) (inverse : Bool) : + decodedFields D.logN (difSpec D a) factor inverse = + if inverse then (NTT.Forward.forwardSpec D a).map (fun x ↦ factor * x) + else NTT.Forward.forwardSpec D a := by + cases inverse <;> + simp only [decodedFields, Bool.false_eq_true, ite_false, ite_true, + NTT.Forward.forwardSpec, Array.map_ofFn, Function.comp_def] + all_goals + apply congrArg Array.ofFn + funext i + rw [getD_difSpec D a _ (NTT.Transform.bitRevNat_lt _ _), + NTT.Transform.bitRevNat_involutive D.logN i.val i.isLt, dftValue_eq_nttAt D a i] + +/-- The domain of `2 ^ k`-th powers of the root, `2 ^ k` times smaller. -/ +def powDomain [Field R] (D : NTT.Domain R) (k : Nat) (hk : k ≤ D.logN) : NTT.Domain R where + logN := D.logN - k + omega := D.omega ^ 2 ^ k + primitive := by + apply IsPrimitiveRoot.pow D.n_pos D.primitive + change 2 ^ D.logN = 2 ^ k * 2 ^ (D.logN - k) + rw [← Nat.pow_add, Nat.add_sub_cancel' hk] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Expressions.lean b/CompPoly/Univariate/NTTFast/Packed/Expressions.lean new file mode 100644 index 00000000..d41efe56 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Expressions.lean @@ -0,0 +1,36 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.FieldStorage +import CompPoly.Univariate.NTTFast.Packed.Unrolled + +/-! # Field-level models of the packed scalar kernels -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed.Expressions + +/-- Read one coordinate in the field-level kernel model. -/ +def readField [Zero R] (a : Array R) (i o : USize) : R := + a.getD (i.toNat + o.toNat) 0 +/-- Addition in the field-level kernel model. -/ +def fieldAdd [Add R] (x y : R) : R := x + y +/-- Subtraction in the field-level kernel model. -/ +def fieldSub [Sub R] (x y : R) : R := x - y +/-- Multiplication in the field-level kernel model. -/ +def fieldMul [Mul R] (x y : R) : R := x * y + +/-- Sixteen independent radix-four butterflies, with four batched array stores. -/ +def step16Field [Zero R] [Add R] [Sub R] [Mul R] (th tl : Array R) (j j1 i0 i1 i2 i3 : USize) + (b : Array R) : Array R := + step16_fields% + +/-- Field-array expression graph of the fused final four DIF layers. -/ +def leaf16Field [Zero R] [Add R] [Sub R] [Mul R] (t3 t2 t1 : Array R) (i : USize) (b : Array R) : + Array R := + leaf16_fields% + +end CompPoly.CPolynomial.NTTFast.Packed.Expressions diff --git a/CompPoly/Univariate/NTTFast/Packed/FieldStorage.lean b/CompPoly/Univariate/NTTFast/Packed/FieldStorage.lean new file mode 100644 index 00000000..4ae9c634 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/FieldStorage.lean @@ -0,0 +1,200 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all Init.Data.Array.Basic +import all CompPoly.Univariate.NTTFast.Packed.Native +public import CompPoly.Univariate.NTTFast.Packed.StorageLemmas +public import CompPoly.Univariate.NTTFast.Packed.Arrays +public import CompPoly.Fields.Montgomery.Native32Bytes + +/-! # Packed Montgomery field arrays -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode size_encode_uint32) + +/-- Packed field coordinates are their packed Montgomery words. -/ +theorem encode_eq_map_val (a : Array (FastField p)) : + encode a = encode (a.map Subtype.val) := by + apply ByteArray.ext + simp only [encode, Array.toList_map, List.flatMap_map] + rfl + +@[simp] theorem size_encode_fields (a : Array (FastField p)) : + (encode a).size = 4 * a.size := by + rw [encode_eq_map_val, size_encode_uint32, Array.size_map] + +/-- Reading a mapped coordinate preserves the field array's zero default. -/ +theorem getD_map_val (a : Array (FastField p)) (i : Nat) : + (a.map Subtype.val).getD i 0 = (a.getD i 0).val := by + by_cases hi : i < a.size + · have hm : i < (a.map Subtype.val).size := by simpa only [Array.size_map] using hi + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hi, + Array.getElem?_eq_getElem hm, Option.getD_some, Array.getElem_map] + · have hm : ¬i < (a.map Subtype.val).size := by simpa only [Array.size_map] using hi + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (Nat.le_of_not_lt hi), + Array.getElem?_eq_none (Nat.le_of_not_lt hm), Option.getD_none] + rfl + +/-- Mapping coordinates commutes with extracting field-array segments. -/ +theorem map_val_extract (a : Array (FastField p)) (first last : Nat) : + (a.extract first last).map Subtype.val = (a.map Subtype.val).extract first last := by + apply Array.toList_inj.mp + simp only [Array.toList_map, Array.toList_extract, List.extract, List.map_take, List.map_drop] + +/-- The compiled bounded word read returns the exact field coordinate. -/ +theorem Native.readUOffset_encode (a : Array (FastField p)) (i o : USize) (h) : + Native.readUOffset (encode a) i o h = (a.getD (i.toNat + o.toNat) 0).val := by + revert h + simp only [encode_eq_map_val] at * + intro h + exact (Native.readUOffset_encodeWords (a.map Subtype.val) i o h).trans (getD_map_val a _) + +/-- A checked ordinary read agrees with the represented field coordinate. -/ +theorem Native.read_encode (a : Array (FastField p)) (i : Nat) + (hs : (encode a).size < USize.size) (hi : i < a.size) : + Native.read (encode a) i = (a.getD i 0).val := by + simp only [encode_eq_map_val] at hs ⊢ + exact (Native.read_encodeWords (a.map Subtype.val) i hs + (by simpa only [Array.size_map] using hi)).trans (getD_map_val a i) + +/-- A field-coordinate batch store realizes the corresponding array splice. -/ +theorem Native.write16U_encode (a : Array (FastField p)) (i : USize) + (hs : (encode a).size < USize.size) (hi : i.toNat + 16 ≤ a.size) + (x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 : FastField p) : + Native.write16U (encode a) i x0.val x1.val x2.val x3.val x4.val x5.val x6.val x7.val + x8.val x9.val x10.val x11.val x12.val x13.val x14.val x15.val = + encode (splice a i.toNat #[x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, + x14, x15]) := by + have hmul : i.toNat * 4 < USize.size := by rw [size_encode_fields] at hs; omega + have ho : (i * 4).toNat = 4 * i.toNat := by + rw [USize.toNat_mul, Native.usize_numeral 4 (by decide), Nat.mod_eq_of_lt hmul] + omega + have hc : (16 : UInt8).toNat ≤ 16 := by decide + have hvalues : (#[x0.val, x1.val, x2.val, x3.val, x4.val, x5.val, x6.val, x7.val, x8.val, + x9.val, x10.val, x11.val, x12.val, x13.val, x14.val, x15.val] : Array UInt32) = (#[x0, x1, + x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15]).map Subtype.val := by + apply Array.toList_inj.mp + simp only [Array.toList_map] + rfl + unfold Native.write16U + simp only [encode_eq_map_val] at hs ⊢ + rw [Native.storeWords_replace_encodeWords (a.map Subtype.val) (i * 4) 16 i.toNat hc ho + (by simpa only [Array.size_map, show (16 : UInt8).toNat = 16 by decide] using hi) hs] + simp only [show (16 : UInt8).toNat = 16 by decide] + change encode + ((a.map Subtype.val).extract 0 i.toNat ++ #[x0.val, x1.val, x2.val, x3.val, x4.val, x5.val, + x6.val, x7.val, x8.val, x9.val, x10.val, x11.val, x12.val, x13.val, x14.val, + x15.val].extract 0 16 ++ + (a.map Subtype.val).extract (i.toNat + 16) (a.map Subtype.val).size) = _ + have hv : #[x0.val, x1.val, x2.val, x3.val, x4.val, x5.val, x6.val, x7.val, x8.val, x9.val, + x10.val, x11.val, x12.val, x13.val, x14.val, x15.val].extract 0 16 = #[x0.val, x1.val, + x2.val, x3.val, x4.val, x5.val, x6.val, x7.val, x8.val, x9.val, x10.val, x11.val, x12.val, + x13.val, x14.val, x15.val] := Array.extract_size + rw [hv, hvalues] + simp only [splice, Array.map_append, map_val_extract, Array.size_map] + rfl + +/-- A bounded single-coordinate write implements the corresponding field-array update. -/ +theorem Native.write_encode (a : Array (FastField p)) (i : Nat) + (hs : (encode a).size < USize.size) (hi : i < a.size) + (x : FastField p) : + Native.write (encode a) i x.val = encode (a.setIfInBounds i x) := by + have hin : i < USize.size := by rw [size_encode_fields] at hs; omega + have hmul : i * 4 < USize.size := by rw [size_encode_fields] at hs; omega + have ho : (i.toUSize * 4).toNat = 4 * i := by + rw [USize.toNat_mul, USize.toNat_ofNat_of_lt' hin, + Native.usize_numeral 4 (by decide), Nat.mod_eq_of_lt hmul] + omega + unfold Native.write + simp only [encode_eq_map_val] at * + rw [Native.storeWords_replace_encodeWords (a.map Subtype.val) (i.toUSize * 4) 1 i (by decide) ho + (by simp only [Array.size_map, show (1 : UInt8).toNat = 1 by decide]; omega) hs] + simp only [show (1 : UInt8).toNat = 1 by decide] + change encode ((a.map Subtype.val).extract 0 i ++ #[x.val] ++ + (a.map Subtype.val).extract (i + 1) (a.map Subtype.val).size) = _ + rw [← splice_singleton a i x hi] + simp only [splice, Array.map_append, map_val_extract, Array.size_map, Array.size_singleton, + Array.map_singleton] + +/-- A word store on a written prefix followed by zeros extends the prefix. -/ +theorem Native.write_prefix (P : Array UInt32) (n : Nat) (x : UInt32) + (hs : 4 * (P.size + n + 1) < USize.size) : + Native.write (encode (P ++ Array.replicate (n + 1) 0)) P.size x = + encode (P.push x ++ Array.replicate n 0) := by + have hin : P.size < USize.size := by omega + have hmul : P.size * 4 < USize.size := by omega + have ho : (P.size.toUSize * 4).toNat = 4 * P.size := by + rw [USize.toNat_mul, USize.toNat_ofNat_of_lt' hin, + Native.usize_numeral 4 (by decide), Nat.mod_eq_of_lt hmul] + omega + unfold Native.write + rw [Native.storeWords_replace_encodeWords _ (P.size.toUSize * 4) 1 P.size (by decide) ho + (by simp only [Array.size_append, Array.size_replicate, show (1 : UInt8).toNat = 1 by decide] + omega) + (by rw [size_encode_uint32, Array.size_append, Array.size_replicate]; omega)] + simp only [show (1 : UInt8).toNat = 1 by decide] + change encode ((P ++ Array.replicate (n + 1) 0).extract 0 P.size ++ #[x] ++ + (P ++ Array.replicate (n + 1) 0).extract (P.size + 1) + (P ++ Array.replicate (n + 1) 0).size) = _ + rw [Array.extract_append, Array.extract_append] + simp only [Array.extract_size, Nat.zero_sub, Nat.sub_self, Array.extract_zero, + Array.append_empty, Array.append_singleton, Array.size_append, Array.size_replicate, + Array.extract_replicate, Nat.add_sub_cancel_left, Nat.min_self, Nat.add_sub_cancel] + rw [Array.extract_eq_empty_of_le (by omega), Array.empty_append] + +private theorem ofFn_push (f : Nat → UInt32) (i : Nat) : + (Array.ofFn fun k : Fin i ↦ f k.val).push (f i) = Array.ofFn fun k : Fin (i + 1) ↦ f k.val := by + rw [Array.ofFn_succ] + rfl + +/-- Generating the remaining words of a buffer whose prefix is already written. -/ +theorem Native.generateGo_encodeWords (f : Nat → UInt32) (n : Nat) : ∀ (i : Nat), + 4 * (i + n) < USize.size → + Native.generateGo f n i + (encode (Array.ofFn (fun k : Fin i ↦ f k.val) ++ Array.replicate n 0)) = + encode (Array.ofFn (fun k : Fin (i + n) ↦ f k.val)) := by + induction n with + | zero => + intro i _ + simp only [Native.generateGo, Array.replicate_zero, Array.append_empty, Nat.add_zero] + | succ n ih => + intro i hs + have hP := Native.write_prefix (Array.ofFn (fun k : Fin i ↦ f k.val)) n (f i) + (by rw [Array.size_ofFn]; omega) + rw [Array.size_ofFn] at hP + rw [Native.generateGo, hP, ofFn_push, ih (i + 1) (by omega), + show i + 1 + n = i + (n + 1) by omega] + +/-- The packed generator is the encoding of its indexed array. -/ +theorem Native.generate_eq (n : Nat) (f : Nat → UInt32) : + Native.generate n f = encode (Array.ofFn (fun i : Fin n ↦ f i.val)) := by + unfold Native.generate + split + · rename_i hs + have h := Native.generateGo_encodeWords f n 0 (by omega) + rw [Array.ofFn_zero, Array.empty_append, Nat.zero_add] at h + rw [Native.zeroWords_eq, h] + · rfl + +/-- The executable encoder realizes the packed field-array representation. -/ +theorem Native.encodeFields_eq (a : Array (FastField p)) : + Native.encodeFields a = encode a := by + rw [Native.encodeFields, Native.generate_eq, encode_eq_map_val] + congr 1 + apply Array.ext + · simp only [Array.size_ofFn, Array.size_map] + · intro j h1 h2 + simp only [Array.getElem_ofFn, Array.getElem_map, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_getElem (show j < a.size by simpa only [Array.size_ofFn] using h1), + Option.getD_some] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Folds.lean b/CompPoly/Univariate/NTTFast/Packed/Folds.lean new file mode 100644 index 00000000..68a3c272 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Folds.lean @@ -0,0 +1,55 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.FieldStorage + +/-! # Representation-preserving packed folds -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +/-- Reading a packed twiddle table agrees with packing the corresponding field table entry. -/ +theorem getD_map_encode (tw : Array (Array (FastField p))) (stage : Nat) : + (tw.map encode).getD stage ByteArray.empty = encode (tw.getD stage #[]) := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_map] + cases he : tw[stage]? with + | none => simp only [Option.map_none, Option.getD_none, PackedRepr.encode_empty] + | some values => rfl + +/-- A fold transports through packing while its field-array invariant is preserved. -/ +theorem foldl_encode (xs : List Nat) + (inv : Array (FastField p) → Prop) + (raw : ByteArray → Nat → ByteArray) + (field : Array (FastField p) → Nat → Array (FastField p)) + (hf : ∀ i ∈ xs, ∀ a, inv a → raw (encode a) i = encode (field a i)) + (hp : ∀ i ∈ xs, ∀ a, inv a → inv (field a i)) + (a : Array (FastField p)) (ha : inv a) : + xs.foldl raw (encode a) = encode (xs.foldl field a) := by + induction xs generalizing a with + | nil => rfl + | cons i xs ih => + rw [List.foldl_cons, List.foldl_cons, hf i (List.mem_cons_self ..) a ha] + exact ih (fun j hj b hb ↦ hf j (List.mem_cons_of_mem i hj) b hb) + (fun j hj b hb ↦ hp j (List.mem_cons_of_mem i hj) b hb) + _ (hp i (List.mem_cons_self ..) a ha) + +/-- A sequential fold preserves any invariant maintained by each visited step. -/ +theorem foldl_invariant (xs : List ι) (p : α → Prop) (f : α → ι → α) + (hf : ∀ i ∈ xs, ∀ a, p a → p (f a i)) (a : α) (ha : p a) : + p (xs.foldl f a) := by + induction xs generalizing a with + | nil => exact ha + | cons i xs ih => + exact ih (fun j hj b hb ↦ hf j (List.mem_cons_of_mem i hj) b hb) + _ (hf i (List.mem_cons_self ..) a ha) + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Instances.lean b/CompPoly/Univariate/NTTFast/Packed/Instances.lean new file mode 100644 index 00000000..74134b80 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Instances.lean @@ -0,0 +1,60 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.CosetLde +public import CompPoly.Univariate.NTTFast.Packed.Plan +public import CompPoly.Fields.BabyBear.Fast +public import CompPoly.Fields.KoalaBear.Fast + +/-! # Compiled packed transforms of the 32-bit Montgomery fields + +`PackedNTT` and `PackedCosetLde` instances for KoalaBear and BabyBear. Each sets `run` to a +definition applying the generic `Native.run` or `Native.cosetLde` at the field's modulus, so the +compiler specializes the kernels here, with the field's constants as immediates, and this module +belongs to the CPU-targeted `CompPolyPackedNative` library. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 8000000 in +/-- The packed transform specialized to KoalaBear. -/ +def runKoalaBear (tw : Array ByteArray) (logN : Nat) (parallel : Bool) (nInv : UInt32) + (inverse : Bool) (a : ByteArray) : ByteArray := + Native.run KoalaBear.fieldSize tw logN parallel nInv inverse a + +instance : PackedNTT KoalaBear.fieldSize := ⟨runKoalaBear, rfl⟩ + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 8000000 in +/-- The batched coset LDE specialized to KoalaBear. -/ +def cosetLdeKoalaBear (P : LdePlan KoalaBear.fieldSize) (w : Nat) (ms : Array ByteArray) : + Array ByteArray := + Native.cosetLde P w ms + +instance : PackedCosetLde KoalaBear.fieldSize := ⟨cosetLdeKoalaBear, rfl⟩ + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 8000000 in +/-- The packed transform specialized to BabyBear. -/ +def runBabyBear (tw : Array ByteArray) (logN : Nat) (parallel : Bool) (nInv : UInt32) + (inverse : Bool) (a : ByteArray) : ByteArray := + Native.run BabyBear.fieldSize tw logN parallel nInv inverse a + +instance : PackedNTT BabyBear.fieldSize := ⟨runBabyBear, rfl⟩ + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 8000000 in +/-- The batched coset LDE specialized to BabyBear. -/ +def cosetLdeBabyBear (P : LdePlan BabyBear.fieldSize) (w : Nat) (ms : Array ByteArray) : + Array ByteArray := + Native.cosetLde P w ms + +instance : PackedCosetLde BabyBear.fieldSize := ⟨cosetLdeBabyBear, rfl⟩ + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/KernelCorrectness.lean b/CompPoly/Univariate/NTTFast/Packed/KernelCorrectness.lean new file mode 100644 index 00000000..2cd0699a --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/KernelCorrectness.lean @@ -0,0 +1,67 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.Expressions +public import CompPoly.Univariate.NTTFast.Packed.LeafLayers +import Mathlib.Tactic.IntervalCases + +/-! # Mathematical correctness of the fused final layers -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed.Expressions + +/-- The four final DIF layers on one sixteen-coordinate block. -/ +def leaf16Stages [Field R] (t3 t2 t1 : Array R) (a : Array R) + (base : Nat) : Array R := + layer16_1 #[1] (layer16_2 t1 (layer16_4 t2 (layer16_8 t3 a base) base) base) base + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 600000 in +/-- The fused final-layer expression graph equals four ordinary DIF stages. -/ +theorem leaf16Field_eq_stages [Field R] (t3 t2 t1 : Array R) + (a : Array R) (i : USize) (hi : i.toNat + 16 ≤ a.size) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) : + leaf16Field t3 t2 t1 i a = leaf16Stages t3 t2 t1 a i.toNat := by + unfold leaf16Field leaf16Stages readField fieldAdd fieldSub fieldMul + simp only [Native.usize_numeral 0 (by decide), Native.usize_numeral 1 (by decide), + Native.usize_numeral 2 (by decide), Native.usize_numeral 3 (by decide), Native.usize_numeral + 4 (by decide), Native.usize_numeral 5 (by decide), Native.usize_numeral 6 (by decide), + Native.usize_numeral 7 (by decide), Native.usize_numeral 8 (by decide), Native.usize_numeral + 9 (by decide), Native.usize_numeral 10 (by decide), Native.usize_numeral 11 (by decide), + Native.usize_numeral 12 (by decide), Native.usize_numeral 13 (by decide), + Native.usize_numeral 14 (by decide), Native.usize_numeral 15 (by decide)] + apply array_eq_of_getD _ _ (0 : R) + · simp (disch := omega) only [size_splice16, size_layer16_1, size_layer16_2, + size_layer16_4, size_layer16_8] + · intro k + rw [getD_splice _ _ _ (by change i.toNat + 16 ≤ a.size; exact hi)] + by_cases hk : i.toNat ≤ k ∧ k < i.toNat + 16 + · obtain ⟨offset, ho, he⟩ : ∃ offset, offset < 16 ∧ k = i.toNat + offset := by + exact ⟨k - i.toNat, by omega, by omega⟩ + subst k + interval_cases offset <;> + simp (disch := (first | (simp -failIfUnchanged only [size_layer16_1, size_layer16_2, + size_layer16_4, size_layer16_8] <;> omega) | fail)) only + [getD_layer16_1, getD_layer16_2, getD_layer16_4, getD_layer16_8, getD_layer16_1_zero, + getD_layer16_2_zero, + getD_layer16_4_zero, getD_layer16_8_zero, layer16Value, + Nat.add_assoc, Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, + Nat.reduceLT, Nat.add_sub_cancel_left, Nat.sub_self, Nat.add_zero, + ite_eq_left, ite_eq_right, ↓reduceIte, getD_toArray, size_literal16, + List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some, h3, h2, h1, one_mul] + · simp (disch := (first | (simp -failIfUnchanged only [size_layer16_1, size_layer16_2, + size_layer16_4, size_layer16_8] <;> omega) | fail)) only + [getD_layer16_1_outside, getD_layer16_2_outside, getD_layer16_4_outside, + getD_layer16_8_outside, size_literal16, ite_eq_right, ite_self] + +/-- Local final layers preserve the working array's length. -/ +@[simp] theorem size_leaf16Stages [Field R] (t3 t2 t1 : Array R) (a : Array R) + (base : Nat) : (leaf16Stages t3 t2 t1 a base).size = a.size := by + simp only [leaf16Stages, size_layer16_1, size_layer16_2, + size_layer16_4, size_layer16_8] + +end CompPoly.CPolynomial.NTTFast.Packed.Expressions diff --git a/CompPoly/Univariate/NTTFast/Packed/KernelModels.lean b/CompPoly/Univariate/NTTFast/Packed/KernelModels.lean new file mode 100644 index 00000000..6d750d32 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/KernelModels.lean @@ -0,0 +1,39 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.FieldStorage +import CompPoly.Univariate.NTTFast.Packed.Unrolled + +/-! # Field-level models of the packed scalar kernels -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +/-- Read one coordinate in the field-level kernel model. -/ +def readField (a : Array (FastField p)) (i o : USize) : FastField p := + a.getD (i.toNat + o.toNat) 0 +/-- Addition in the field-level kernel model. -/ +def fieldAdd (x y : FastField p) : FastField p := x + y +/-- Subtraction in the field-level kernel model. -/ +def fieldSub (x y : FastField p) : FastField p := x - y +/-- Multiplication in the field-level kernel model. -/ +def fieldMul (x y : FastField p) : FastField p := x * y + +/-- Sixteen independent radix-four butterflies, with four batched array stores. -/ +def step16Field (th tl : Array (FastField p)) (j j1 i0 i1 i2 i3 : USize) + (b : Array (FastField p)) : Array (FastField p) := + step16_fields% + +/-- Field-array expression graph of the fused final four DIF layers. -/ +def leaf16Field (t3 t2 t1 : Array (FastField p)) (i : USize) (b : Array + (FastField p)) : Array (FastField p) := + leaf16_fields% + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/KernelRefinement.lean b/CompPoly/Univariate/NTTFast/Packed/KernelRefinement.lean new file mode 100644 index 00000000..c6a786cc --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/KernelRefinement.lean @@ -0,0 +1,40 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Packed.Native +import all CompPoly.Univariate.NTTFast.Packed.KernelModels +public import CompPoly.Univariate.NTTFast.Packed.KernelModels + +/-! # Refinement of packed arithmetic kernels to field-array kernels -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +/-- The vectorizable word kernel is exactly sixteen field butterflies. -/ +theorem Native.step16_encode (th tl : Array (FastField p)) + (j j1 i0 i1 i2 i3 : USize) (a : Array (FastField p)) (h) : + Native.step16 p (encode th) (encode tl) j j1 i0 i1 i2 i3 (encode a) h = + encode (step16Field th tl j j1 i0 i1 i2 i3 a) := by + rcases h with ⟨hi0, hi1, hi2, hi3, hj1, hj, hjl, ha, hth, htl⟩ + have h0 : i0.toNat + 16 ≤ a.size := by rw [size_encode_fields] at hi0; omega + have h1 : i1.toNat + 16 ≤ a.size := by rw [size_encode_fields] at hi1; omega + have h2 : i2.toNat + 16 ≤ a.size := by rw [size_encode_fields] at hi2; omega + have h3 : i3.toNat + 16 ≤ a.size := by rw [size_encode_fields] at hi3; omega + rw [size_encode_fields] at ha + unfold Native.step16 + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val] + simp (config := { maxDischargeDepth := 8 }) + (disch := (simp_all (config := { maxDischargeDepth := 8 }) only + [size_encode_fields, size_splice16])) only [Native.write16U_encode] + rfl + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/KernelSpecialization.lean b/CompPoly/Univariate/NTTFast/Packed/KernelSpecialization.lean new file mode 100644 index 00000000..f9b6425f --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/KernelSpecialization.lean @@ -0,0 +1,50 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.KernelModels +public import CompPoly.Univariate.NTTFast.Packed.Expressions + +/-! # Specialization of abstract kernel expression graphs -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +/-- The concrete read has the abstract read's scalar semantics. -/ +theorem readField_eq_expression (a : Array (FastField p)) (i o : USize) : + readField a i o = Expressions.readField a i o := by rfl +/-- The concrete add specializes abstract addition. -/ +theorem fieldAdd_eq_expression (x y : FastField p) : + fieldAdd x y = Expressions.fieldAdd x y := by rfl +/-- The concrete subtract specializes abstract subtraction. -/ +theorem fieldSub_eq_expression (x y : FastField p) : + fieldSub x y = Expressions.fieldSub x y := by rfl +/-- The concrete multiply specializes abstract multiplication. -/ +theorem fieldMul_eq_expression (x y : FastField p) : + fieldMul x y = Expressions.fieldMul x y := by rfl + +/-- The concrete field model specializes the abstract expression graph. -/ +theorem step16Field_eq_expression (th tl : Array (FastField p)) (j j1 i0 i1 i2 i3 : USize) + (b : Array (FastField p)) : + step16Field th tl j j1 i0 i1 i2 i3 b = Expressions.step16Field th tl j j1 i0 i1 i2 i3 b := by + unfold Expressions.step16Field + simp only [← readField_eq_expression, ← fieldAdd_eq_expression, ← fieldSub_eq_expression, ← + fieldMul_eq_expression] + rfl + +/-- The concrete field model specializes the abstract expression graph. -/ +theorem leaf16Field_eq_expression (t3 t2 t1 : Array (FastField p)) (i : USize) (b : Array + (FastField p)) : + leaf16Field t3 t2 t1 i b = Expressions.leaf16Field t3 t2 t1 i b := by + unfold Expressions.leaf16Field + simp only [← readField_eq_expression, ← fieldAdd_eq_expression, ← fieldSub_eq_expression, ← + fieldMul_eq_expression] + rfl + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/LeafLayers.lean b/CompPoly/Univariate/NTTFast/Packed/LeafLayers.lean new file mode 100644 index 00000000..1d83f3dd --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/LeafLayers.lean @@ -0,0 +1,217 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Plan +public import CompPoly.Univariate.NTTFast.Packed.ButterflyLemmas +public import CompPoly.Univariate.NTTFast.Packed.Arithmetic +import Mathlib.Tactic.IntervalCases + +/-! # Coordinate semantics of the four final DIF layers -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- A single radix-two layer inside a sixteen-coordinate block. -/ +def layer16Value [Field R] (tw : Array R) (half : Nat) + (a : Array R) (base offset : Nat) : R := + let first := base + offset / (2 * half) * (2 * half) + offset % half + if offset % (2 * half) < half then + a.getD first 0 + a.getD (first + half) 0 + else tw.getD (offset % half) 0 * (a.getD first 0 - a.getD (first + half) 0) + +/-- The layer with butterfly half-width 8 on one sixteen-coordinate block. -/ +def layer16_8 [Field R] (tw : Array R) (a : Array R) + (base : Nat) : Array R := + let a := Plan.butterflyDIFInner tw 8 0 (base + 0) (base + 8) a + a + +/-- One complete local layer preserves the array size. -/ +@[simp] theorem size_layer16_8 [Field R] (tw : Array R) (a : Array R) (base : Nat) : + (layer16_8 tw a base).size = a.size := by + unfold layer16_8 + simp only [Plan.size_butterflyDIFInner] + +set_option maxRecDepth 4096 in +set_option maxHeartbeats 300000 in +/-- The local layer has the ordinary pairwise butterfly formula. -/ +theorem getD_layer16_8 [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (offset : Nat) (ho : offset < 16) : + (layer16_8 tw a base).getD (base + offset) 0 = layer16Value tw 8 a base offset := by + unfold layer16_8 layer16Value + interval_cases offset <;> + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_left, getD_butterflyDIFInner_right, + Nat.add_assoc, Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, + Nat.reduceLT, Nat.reduceSub, Nat.add_sub_cancel_left, Nat.add_sub_add_left, + Nat.sub_self, Nat.add_zero, Nat.zero_add, ↓reduceIte] + +/-- The first coordinate uses offset zero, even when simplification has removed `+ 0`. -/ +theorem getD_layer16_8_zero [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) : + (layer16_8 tw a base).getD base 0 = layer16Value tw 8 a base 0 := by + simpa only [Nat.add_zero] using getD_layer16_8 tw a base hs 0 (by decide) + +/-- Coordinates outside the block are unchanged by the local layer. -/ +theorem getD_layer16_8_outside [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (k : Nat) + (hk : ¬(base ≤ k ∧ k < base + 16)) : + (layer16_8 tw a base).getD k 0 = a.getD k 0 := by + unfold layer16_8 + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_outside] + +/-- The layer with butterfly half-width 4 on one sixteen-coordinate block. -/ +def layer16_4 [Field R] (tw : Array R) (a : Array R) + (base : Nat) : Array R := + let a := Plan.butterflyDIFInner tw 4 0 (base + 0) (base + 4) a + let a := Plan.butterflyDIFInner tw 4 0 (base + 8) (base + 12) a + a + +/-- One complete local layer preserves the array size. -/ +@[simp] theorem size_layer16_4 [Field R] (tw : Array R) (a : Array R) (base : Nat) : + (layer16_4 tw a base).size = a.size := by + unfold layer16_4 + simp only [Plan.size_butterflyDIFInner] + +set_option maxRecDepth 4096 in +set_option maxHeartbeats 300000 in +/-- The local layer has the ordinary pairwise butterfly formula. -/ +theorem getD_layer16_4 [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (offset : Nat) (ho : offset < 16) : + (layer16_4 tw a base).getD (base + offset) 0 = layer16Value tw 4 a base offset := by + unfold layer16_4 layer16Value + interval_cases offset <;> + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_left, getD_butterflyDIFInner_right, getD_butterflyDIFInner_outside, + Nat.add_assoc, Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, + Nat.reduceLT, Nat.reduceSub, Nat.add_sub_cancel_left, Nat.add_sub_add_left, + Nat.sub_self, Nat.add_zero, Nat.zero_add, ↓reduceIte] + +/-- The first coordinate uses offset zero, even when simplification has removed `+ 0`. -/ +theorem getD_layer16_4_zero [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) : + (layer16_4 tw a base).getD base 0 = layer16Value tw 4 a base 0 := by + simpa only [Nat.add_zero] using getD_layer16_4 tw a base hs 0 (by decide) + +/-- Coordinates outside the block are unchanged by the local layer. -/ +theorem getD_layer16_4_outside [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (k : Nat) + (hk : ¬(base ≤ k ∧ k < base + 16)) : + (layer16_4 tw a base).getD k 0 = a.getD k 0 := by + unfold layer16_4 + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_outside] + +/-- The layer with butterfly half-width 2 on one sixteen-coordinate block. -/ +def layer16_2 [Field R] (tw : Array R) (a : Array R) + (base : Nat) : Array R := + let a := Plan.butterflyDIFInner tw 2 0 (base + 0) (base + 2) a + let a := Plan.butterflyDIFInner tw 2 0 (base + 4) (base + 6) a + let a := Plan.butterflyDIFInner tw 2 0 (base + 8) (base + 10) a + let a := Plan.butterflyDIFInner tw 2 0 (base + 12) (base + 14) a + a + +/-- One complete local layer preserves the array size. -/ +@[simp] theorem size_layer16_2 [Field R] (tw : Array R) (a : Array R) (base : Nat) : + (layer16_2 tw a base).size = a.size := by + unfold layer16_2 + simp only [Plan.size_butterflyDIFInner] + +set_option maxRecDepth 4096 in +set_option maxHeartbeats 300000 in +/-- The local layer has the ordinary pairwise butterfly formula. -/ +theorem getD_layer16_2 [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (offset : Nat) (ho : offset < 16) : + (layer16_2 tw a base).getD (base + offset) 0 = layer16Value tw 2 a base offset := by + unfold layer16_2 layer16Value + interval_cases offset <;> + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_left, getD_butterflyDIFInner_right, getD_butterflyDIFInner_outside, + Nat.add_assoc, Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, + Nat.reduceLT, Nat.reduceSub, Nat.add_sub_cancel_left, Nat.add_sub_add_left, + Nat.sub_self, Nat.add_zero, Nat.zero_add, ↓reduceIte] + +/-- The first coordinate uses offset zero, even when simplification has removed `+ 0`. -/ +theorem getD_layer16_2_zero [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) : + (layer16_2 tw a base).getD base 0 = layer16Value tw 2 a base 0 := by + simpa only [Nat.add_zero] using getD_layer16_2 tw a base hs 0 (by decide) + +/-- Coordinates outside the block are unchanged by the local layer. -/ +theorem getD_layer16_2_outside [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (k : Nat) + (hk : ¬(base ≤ k ∧ k < base + 16)) : + (layer16_2 tw a base).getD k 0 = a.getD k 0 := by + unfold layer16_2 + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_outside] + +/-- The layer with butterfly half-width 1 on one sixteen-coordinate block. -/ +def layer16_1 [Field R] (tw : Array R) (a : Array R) + (base : Nat) : Array R := + let a := Plan.butterflyDIFInner tw 1 0 (base + 0) (base + 1) a + let a := Plan.butterflyDIFInner tw 1 0 (base + 2) (base + 3) a + let a := Plan.butterflyDIFInner tw 1 0 (base + 4) (base + 5) a + let a := Plan.butterflyDIFInner tw 1 0 (base + 6) (base + 7) a + let a := Plan.butterflyDIFInner tw 1 0 (base + 8) (base + 9) a + let a := Plan.butterflyDIFInner tw 1 0 (base + 10) (base + 11) a + let a := Plan.butterflyDIFInner tw 1 0 (base + 12) (base + 13) a + let a := Plan.butterflyDIFInner tw 1 0 (base + 14) (base + 15) a + a + +/-- One complete local layer preserves the array size. -/ +@[simp] theorem size_layer16_1 [Field R] (tw : Array R) (a : Array R) (base : Nat) : + (layer16_1 tw a base).size = a.size := by + unfold layer16_1 + simp only [Plan.size_butterflyDIFInner] + +set_option maxRecDepth 4096 in +set_option maxHeartbeats 300000 in +/-- The local layer has the ordinary pairwise butterfly formula. -/ +theorem getD_layer16_1 [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (offset : Nat) (ho : offset < 16) : + (layer16_1 tw a base).getD (base + offset) 0 = layer16Value tw 1 a base offset := by + unfold layer16_1 layer16Value + interval_cases offset <;> + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_left, getD_butterflyDIFInner_right, getD_butterflyDIFInner_outside, + Nat.add_assoc, Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, + Nat.reduceLT, + Nat.sub_self, Nat.add_zero, ↓reduceIte] + +/-- The first coordinate uses offset zero, even when simplification has removed `+ 0`. -/ +theorem getD_layer16_1_zero [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) : + (layer16_1 tw a base).getD base 0 = layer16Value tw 1 a base 0 := by + simpa only [Nat.add_zero] using getD_layer16_1 tw a base hs 0 (by decide) + +/-- Coordinates outside the block are unchanged by the local layer. -/ +theorem getD_layer16_1_outside [Field R] (tw : Array R) (a : Array R) + (base : Nat) (hs : base + 16 ≤ a.size) (k : Nat) + (hk : ¬(base ≤ k ∧ k < base + 16)) : + (layer16_1 tw a base).getD k 0 = a.getD k 0 := by + unfold layer16_1 + simp (disch := (first + | (simp -failIfUnchanged only [Plan.size_butterflyDIFInner] <;> omega) + | fail)) only + [getD_butterflyDIFInner_outside] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/LeafStages.lean b/CompPoly/Univariate/NTTFast/Packed/LeafStages.lean new file mode 100644 index 00000000..0883a242 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/LeafStages.lean @@ -0,0 +1,192 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.Expressions +public import CompPoly.Univariate.NTTFast.Packed.LeafLayers +public import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas +import CompPoly.Univariate.NTTFast.Packed.Unrolled +import Mathlib.Tactic.IntervalCases + +/-! # The final DIF layers one at a time + +The stage kernels run each of the four final DIF layers over all sixteen-word blocks of a leaf +before the next, each block stored as two halves of eight words. Each stage's field model, a +block's two halves spliced in by `splice8`, is the layer (`leafStage8Field_eq` and its siblings). +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- Entries `i, …, i + 7` replaced by `v0, …, v7` where they exist: an eight-entry splice that keeps +the size. -/ +def splice8 (a : Array α) (i : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : α) : Array α := + ((((((((a.setIfInBounds i v0).setIfInBounds (i + 1) v1).setIfInBounds (i + 2) v2).setIfInBounds + (i + 3) v3).setIfInBounds (i + 4) v4).setIfInBounds (i + 5) v5).setIfInBounds (i + 6) + v6).setIfInBounds (i + 7) v7) + +@[simp] theorem size_splice8 (a : Array α) (i : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : α) : + (splice8 a i v0 v1 v2 v3 v4 v5 v6 v7).size = a.size := by + simp only [splice8, Array.size_setIfInBounds] + +/-- Inside its range, an in-bounds `splice8` reads its values. -/ +theorem getD_splice8_in (a : Array α) (i : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : α) + (h : i + 8 ≤ a.size) (k : Nat) (d : α) (hk : i ≤ k ∧ k < i + 8) : + (splice8 a i v0 v1 v2 v3 v4 v5 v6 v7).getD k d = + #[v0, v1, v2, v3, v4, v5, v6, v7].getD (k - i) d := by + obtain ⟨o, ho, rfl⟩ : ∃ o, o < 8 ∧ k = i + o := ⟨k - i, by omega, by omega⟩ + unfold splice8 + interval_cases o <;> + simp (disch := (first | omega | (simp only [Array.size_setIfInBounds]; omega))) only + [getD_setIfInBounds, Nat.add_sub_cancel_left, Nat.add_zero, Nat.add_left_cancel_iff, + Nat.reduceEqDiff, Nat.left_eq_add, Nat.succ_ne_zero, Nat.sub_self, ↓reduceIte, + getD_toArray, List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some] + +/-- Outside its range, `splice8` keeps the entries. -/ +theorem getD_splice8_out (a : Array α) (i : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : α) (k : Nat) (d : α) + (hk : k < i ∨ i + 8 ≤ k) : + (splice8 a i v0 v1 v2 v3 v4 v5 v6 v7).getD k d = a.getD k d := by + simp only [splice8, Array.getD_eq_getD_getElem?, Array.getElem?_setIfInBounds] + repeat rw [ite_eq_right (by omega)] + +/-- An eight-entry literal has eight entries. -/ +@[simp] theorem size_literal8 (v0 v1 v2 v3 v4 v5 v6 v7 : α) : + (#[v0, v1, v2, v3, v4, v5, v6, v7]).size = 8 := rfl + +/-- In bounds, `splice8` is the splice of its eight values. -/ +theorem splice8_eq_splice (a : Array α) (i : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : α) + (h : i + 8 ≤ a.size) : + splice8 a i v0 v1 v2 v3 v4 v5 v6 v7 = splice a i #[v0, v1, v2, v3, v4, v5, v6, v7] := by + apply array_eq_of_getD _ _ v0 (by rw [size_splice8, size_splice _ _ _ h]) + intro k + rw [getD_splice _ _ _ h, size_literal8] + by_cases hk : i ≤ k ∧ k < i + 8 + · rw [getD_splice8_in _ _ _ _ _ _ _ _ _ _ h _ _ hk, ite_eq_right (by omega), + ite_eq_left hk.2] + · rw [getD_splice8_out _ _ _ _ _ _ _ _ _ _ _ _ (by omega)] + split_ifs <;> first | rfl | omega + +namespace Expressions + +/-- The first of the four final DIF layers on the sixteen-coordinate block at `base`, as the stage +kernel computes it. -/ +def leafStage8Field [Zero R] [Add R] [Sub R] [Mul R] (t : Array R) (base : Nat) (b : Array R) : + Array R := + leafStage8_fields% + +/-- The second of the four final DIF layers on the block at `base`. -/ +def leafStage4Field [Zero R] [Add R] [Sub R] [Mul R] (t : Array R) (base : Nat) (b : Array R) : + Array R := + leafStage4_fields% + +/-- The third of the four final DIF layers on the block at `base`. -/ +def leafStage2Field [Zero R] [Add R] [Sub R] [Mul R] (t : Array R) (base : Nat) (b : Array R) : + Array R := + leafStage2_fields% + +/-- The last DIF layer on the block at `base`. -/ +def leafStage1Field [Zero R] [Add R] [Sub R] (base : Nat) (b : Array R) : Array R := + leafStage1_fields% + +@[simp] theorem size_leafStage8Field [Zero R] [Add R] [Sub R] [Mul R] (t b : Array R) + (base : Nat) : (leafStage8Field t base b).size = b.size := by + simp only [leafStage8Field, size_splice8] + +set_option maxHeartbeats 1000000 in +/-- The first stage's model is the first of the four final layers. -/ +theorem leafStage8Field_eq [Field R] (t b : Array R) (base : Nat) (hs : base + 16 ≤ b.size) : + leafStage8Field t base b = layer16_8 t b base := by + apply array_eq_of_getD _ _ (0 : R) (by rw [size_leafStage8Field, size_layer16_8]) + intro k + by_cases hk : base ≤ k ∧ k < base + 16 + · obtain ⟨o, ho, rfl⟩ : ∃ o, o < 16 ∧ k = base + o := ⟨k - base, by omega, by omega⟩ + rw [getD_layer16_8 t b base hs o ho] + unfold leafStage8Field layer16Value + interval_cases o <;> + simp (disch := (first | omega | (simp only [size_splice8]; omega))) only [getD_splice8_in, + getD_splice8_out, fieldAdd, fieldSub, fieldMul, getD_toArray, Nat.add_assoc, + Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, Nat.reduceLT, + Nat.add_sub_cancel_left, Nat.add_sub_add_left, Nat.reduceSub, Nat.sub_self, Nat.add_zero, + List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some, ↓reduceIte] + · rw [getD_layer16_8_outside t b base hs k hk] + unfold leafStage8Field + simp (disch := omega) only [getD_splice8_out] + +@[simp] theorem size_leafStage4Field [Zero R] [Add R] [Sub R] [Mul R] (t b : Array R) + (base : Nat) : (leafStage4Field t base b).size = b.size := by + simp only [leafStage4Field, size_splice8] + +set_option maxHeartbeats 1000000 in +/-- The second stage's model is the second of the four final layers. -/ +theorem leafStage4Field_eq [Field R] (t b : Array R) (base : Nat) (hs : base + 16 ≤ b.size) : + leafStage4Field t base b = layer16_4 t b base := by + apply array_eq_of_getD _ _ (0 : R) (by rw [size_leafStage4Field, size_layer16_4]) + intro k + by_cases hk : base ≤ k ∧ k < base + 16 + · obtain ⟨o, ho, rfl⟩ : ∃ o, o < 16 ∧ k = base + o := ⟨k - base, by omega, by omega⟩ + rw [getD_layer16_4 t b base hs o ho] + unfold leafStage4Field layer16Value + interval_cases o <;> + simp (disch := (first | omega | (simp only [size_splice8]; omega))) only [getD_splice8_in, + getD_splice8_out, fieldAdd, fieldSub, fieldMul, getD_toArray, Nat.add_assoc, + Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, Nat.reduceLT, + Nat.add_sub_cancel_left, Nat.add_sub_add_left, Nat.reduceSub, Nat.sub_self, Nat.add_zero, + List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some, ↓reduceIte] + · rw [getD_layer16_4_outside t b base hs k hk] + unfold leafStage4Field + simp (disch := omega) only [getD_splice8_out] + +@[simp] theorem size_leafStage2Field [Zero R] [Add R] [Sub R] [Mul R] (t b : Array R) + (base : Nat) : (leafStage2Field t base b).size = b.size := by + simp only [leafStage2Field, size_splice8] + +set_option maxHeartbeats 1000000 in +/-- The third stage's model is the third of the four final layers. -/ +theorem leafStage2Field_eq [Field R] (t b : Array R) (base : Nat) (hs : base + 16 ≤ b.size) : + leafStage2Field t base b = layer16_2 t b base := by + apply array_eq_of_getD _ _ (0 : R) (by rw [size_leafStage2Field, size_layer16_2]) + intro k + by_cases hk : base ≤ k ∧ k < base + 16 + · obtain ⟨o, ho, rfl⟩ : ∃ o, o < 16 ∧ k = base + o := ⟨k - base, by omega, by omega⟩ + rw [getD_layer16_2 t b base hs o ho] + unfold leafStage2Field layer16Value + interval_cases o <;> + simp (disch := (first | omega | (simp only [size_splice8]; omega))) only [getD_splice8_in, + getD_splice8_out, fieldAdd, fieldSub, fieldMul, getD_toArray, Nat.add_assoc, + Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, Nat.reduceLT, + Nat.add_sub_cancel_left, Nat.add_sub_add_left, Nat.reduceSub, Nat.sub_self, Nat.add_zero, + List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some, ↓reduceIte] + · rw [getD_layer16_2_outside t b base hs k hk] + unfold leafStage2Field + simp (disch := omega) only [getD_splice8_out] + +@[simp] theorem size_leafStage1Field [Zero R] [Add R] [Sub R] (b : Array R) + (base : Nat) : (leafStage1Field base b).size = b.size := by + simp only [leafStage1Field, size_splice8] + +set_option maxHeartbeats 1000000 in +/-- The last stage's model is the last of the four final layers. -/ +theorem leafStage1Field_eq [Field R] (b : Array R) (base : Nat) (hs : base + 16 ≤ b.size) : + leafStage1Field base b = layer16_1 #[1] b base := by + apply array_eq_of_getD _ _ (0 : R) (by rw [size_leafStage1Field, size_layer16_1]) + intro k + by_cases hk : base ≤ k ∧ k < base + 16 + · obtain ⟨o, ho, rfl⟩ : ∃ o, o < 16 ∧ k = base + o := ⟨k - base, by omega, by omega⟩ + rw [getD_layer16_1 #[1] b base hs o ho] + unfold leafStage1Field layer16Value + interval_cases o <;> + simp (disch := (first | omega | (simp only [size_splice8]; omega))) only [getD_splice8_in, + getD_splice8_out, fieldAdd, fieldSub, getD_toArray, one_mul, Nat.add_assoc, + Nat.reduceAdd, Nat.reduceMul, Nat.reduceDiv, Nat.reduceMod, Nat.reduceLT, + Nat.add_sub_cancel_left, Nat.add_sub_add_left, Nat.reduceSub, Nat.sub_self, Nat.add_zero, + List.getElem?_cons_zero, List.getElem?_cons_succ, Option.getD_some, ↓reduceIte] + · rw [getD_layer16_1_outside #[1] b base hs k hk] + unfold leafStage1Field + simp (disch := omega) only [getD_splice8_out] + +end Expressions + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/LocalLayers.lean b/CompPoly/Univariate/NTTFast/Packed/LocalLayers.lean new file mode 100644 index 00000000..80068842 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/LocalLayers.lean @@ -0,0 +1,115 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Correctness.Radix4DIF +public import CompPoly.Univariate.NTTFast.Packed.KernelCorrectness + +/-! # Local layer independence and global stage fusion -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- A consecutive group of ordinary radix-two butterfly blocks, at an explicit offset. -/ +def localLayer [Field R] (tw : Array R) (half blocks base : Nat) (a : Array R) : Array R := + (List.range blocks).foldl (fun b block ↦ + Plan.butterflyDIFInner tw half 0 (base + block * (2 * half)) + (base + block * (2 * half) + half) b) a + +/-- Nonoverlapping local layers can be scheduled in either order. -/ +theorem localLayer_comm [Field R] (ta tb : Array R) (ha na ba hb nb bb : Nat) + (hha : 0 < ha) (hhb : 0 < hb) (hsep : bb + nb * (2 * hb) ≤ ba) (a : Array R) : + localLayer ta ha na ba (localLayer tb hb nb bb a) = + localLayer tb hb nb bb (localLayer ta ha na ba a) := by + unfold localLayer + apply Plan.foldl_commute_foldl + intro i j hi hj x + symm + apply Plan.butterflyDIFInner_comm + intro k hk l hl + have hib : i * (2 * ha) + 2 * ha ≤ na * (2 * ha) := by nlinarith + have hjb : j * (2 * hb) + 2 * hb ≤ nb * (2 * hb) := by nlinarith + refine ⟨?_, ?_, ?_, ?_⟩ <;> nlinarith + +/-- The unrolled half-width-eight leaf layer is one ordinary offset block. -/ +theorem layer16_8_eq_localLayer [Field R] (tw : Array R) (a : Array R) (base : Nat) : + layer16_8 tw a base = localLayer tw 8 1 base a := by rfl + +/-- The unrolled half-width-four leaf layer is two ordinary offset blocks. -/ +theorem layer16_4_eq_localLayer [Field R] (tw : Array R) (a : Array R) (base : Nat) : + layer16_4 tw a base = localLayer tw 4 2 base a := by rfl + +/-- The unrolled half-width-two leaf layer is four ordinary offset blocks. -/ +theorem layer16_2_eq_localLayer [Field R] (tw : Array R) (a : Array R) (base : Nat) : + layer16_2 tw a base = localLayer tw 2 4 base a := by rfl + +/-- The unrolled half-width-one leaf layer is eight ordinary offset blocks. -/ +theorem layer16_1_eq_localLayer [Field R] (tw : Array R) (a : Array R) (base : Nat) : + layer16_1 tw a base = localLayer tw 1 8 base a := by rfl + +/-- Consecutive groups of indexed operations traverse the same full interval. -/ +theorem foldl_range_groups (f : α → Nat → α) (count blocks : Nat) (a : α) : + (List.range blocks).foldl (fun b block ↦ + (List.range count).foldl (fun c k ↦ f c (count * block + k)) b) a = + (List.range (count * blocks)).foldl f a := by + induction blocks with + | zero => rfl + | succ blocks ih => + rw [List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil, ih] + symm + rw [Nat.mul_succ, List.range_eq_range', Plan.foldl_range'_append_split, + Nat.zero_add, ← List.range_eq_range'] + exact Plan.foldl_range'_eq_range_add (fun k b ↦ f b k) _ _ _ + +/-- Traversing local layers in consecutive sixteen-coordinate blocks is one global stage. -/ +theorem fold_localLayer [Field R] (tw : Array R) (half count blocks : Nat) + (hw : count * (2 * half) = 16) (a : Array R) : + (List.range blocks).foldl (fun b block ↦ localLayer tw half count (block * 16) b) a = + Plan.butterflyDIFBlocks tw (2 * half) half (count * blocks) 0 a := by + unfold localLayer + have hindex (block k : Nat) : block * 16 + k * (2 * half) = + (count * block + k) * (2 * half) := by rw [← hw]; ring + simp_rw [hindex] + rw [foldl_range_groups (fun b k ↦ Plan.butterflyDIFInner tw half 0 + (k * (2 * half)) (k * (2 * half) + half) b) count blocks a] + symm + simpa only [← List.range_eq_range'] using + Plan.butterflyDIFBlocks_eq_foldl_inner tw (2 * half) half (count * blocks) + (count * blocks) 0 a (by omega) + +/-- Interleaving all four local final layers is equivalent to four whole-array stages. -/ +theorem fold_leaf16Stages [Field R] (t3 t2 t1 : Array R) (blocks : Nat) (a : Array R) : + (List.range blocks).foldl (fun b block ↦ + Expressions.leaf16Stages t3 t2 t1 b (block * 16)) a = + Plan.butterflyDIFBlocks #[1] 2 1 (8 * blocks) 0 + (Plan.butterflyDIFBlocks t1 4 2 (4 * blocks) 0 + (Plan.butterflyDIFBlocks t2 8 4 (2 * blocks) 0 + (Plan.butterflyDIFBlocks t3 16 8 blocks 0 a))) := by + simp only [Expressions.leaf16Stages, layer16_1_eq_localLayer, layer16_2_eq_localLayer, + layer16_4_eq_localLayer, layer16_8_eq_localLayer] + rw [Plan.foldl_quad] + · rw [fold_localLayer _ 8 1 _ (by decide), fold_localLayer _ 4 2 _ (by decide), + fold_localLayer _ 2 4 _ (by decide), fold_localLayer _ 1 8 _ (by decide)] + simp only [Nat.one_mul] + all_goals + intro i j hij hj x + apply localLayer_comm <;> (first | decide | omega) + +/-- Fusing the final sixteen-coordinate blocks is exactly two ordinary radix-four passes. -/ +theorem fold_leaf16Stages_eq_radix4 [Field R] (t3 t2 t1 : Array R) (blocks : Nat) + (a : Array R) (hs : blocks * 16 ≤ a.size) : + (List.range blocks).foldl (fun b block ↦ + Expressions.leaf16Stages t3 t2 t1 b (block * 16)) a = + Plan.butterflyDIFRadix4Blocks t1 #[1] 4 1 (4 * blocks) 0 + (Plan.butterflyDIFRadix4Blocks t3 t2 16 4 blocks 0 a) := by + rw [fold_leaf16Stages] + symm + rw [Plan.butterflyDIFRadix4Blocks_eq_two_dif_blocks t1 #[1] 1 (4 * blocks) _ + (by decide) (by simp only [Plan.size_butterflyDIFRadix4Blocks]; omega), + Plan.butterflyDIFRadix4Blocks_eq_two_dif_blocks t3 t2 4 blocks a (by decide) hs] + simp only [← Nat.mul_assoc, Nat.reduceMul] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/LoopRefinement.lean b/CompPoly/Univariate/NTTFast/Packed/LoopRefinement.lean new file mode 100644 index 00000000..59f46038 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/LoopRefinement.lean @@ -0,0 +1,112 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Plan +import all CompPoly.Univariate.NTTFast.Packed.Native +public import CompPoly.Univariate.NTTFast.Packed.KernelRefinement +public import CompPoly.Univariate.NTTFast.Packed.BatchCorrectness +public import CompPoly.Univariate.NTTFast.Packed.KernelSpecialization + +/-! # Packed scalar loop refinement -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +/-- The packed scalar radix-four tail agrees with the ordinary field-array loop. -/ +theorem Native.inner_encode (th tl : Array (FastField p)) (q j i0 i1 i2 i3 : Nat) + (a : Array (FastField p)) + (ha : (encode a).size < USize.size) + (hth : (encode th).size < USize.size) (htl : (encode tl).size < USize.size) + (hthi : 2 * q ≤ th.size) (htli : q ≤ tl.size) + (hi : i0 + (q - j) ≤ a.size ∧ i1 + (q - j) ≤ a.size ∧ + i2 + (q - j) ≤ a.size ∧ i3 + (q - j) ≤ a.size) : + Native.inner p (encode th) (encode tl) q j i0 i1 i2 i3 (encode a) = + encode (Plan.butterflyDIFRadix4Inner th tl q j i0 i1 i2 i3 a) := by + rw [Native.inner, Plan.butterflyDIFRadix4Inner] + split + · rename_i hj + simp (config := { maxDischargeDepth := 8 }) + (disch := (simp_all only [Array.size_setIfInBounds, size_encode_fields] <;> omega)) only + [Native.read_encode, add_val, sub_val, mul_val, Native.write_encode] + exact Native.inner_encode th tl q (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) _ + (by simpa only [size_encode_fields, Array.size_setIfInBounds] using ha) + hth htl hthi htli (by simp only [Array.size_setIfInBounds]; omega) + · rfl +termination_by q - j +decreasing_by omega + + +/-- The hoisted batch loop depends only on the buffer's value. -/ +theorem Native.inner16Loop_congr {th tl : ByteArray} {n : Nat} {j j1 i0 i1 i2 i3 : USize} + {b b' : ByteArray} (h) (e : b = b') : + Native.inner16Loop p th tl n j j1 i0 i1 i2 i3 b h = + Native.inner16Loop p th tl n j j1 i0 i1 i2 i3 b' (e ▸ h) := by + subst e + rfl + +/-- The hoisted batch loop executes `16 * n` cells. -/ +theorem Native.inner16Loop_encode (th tl : Array (FastField p)) (q : Nat) : + ∀ (n : Nat) (j j1 i0 i1 i2 i3 : USize) (a : Array (FastField p)) (h), + j1.toNat = j.toNat + q → i3.toNat + 16 * n ≤ a.size → i0.toNat + 16 * n ≤ i1.toNat → + i1.toNat + 16 * n ≤ i2.toNat → i2.toNat + 16 * n ≤ i3.toNat → + Native.inner16Loop p (encode th) (encode tl) n j j1 i0 i1 i2 i3 (encode a) h = + encode (quadCells th tl q j.toNat i0.toNat i1.toNat i2.toNat i3.toNat a (16 * n)) := by + intro n + induction n with + | zero => intro j j1 i0 i1 i2 i3 a h _ _ _ _ _; simp only [Native.inner16Loop, quadCells] + | succ n ih => + intro j j1 i0 i1 i2 i3 a h hj1 hs h01 h12 h23 + have hb := h.2.2.2.2.2.2.2.1 + have ht := h.2.2.2.2.2.2.2.2.1 + rw [Native.inner16Loop, Native.inner16Loop_congr _ ((Native.step16_encode th tl + j j1 i0 i1 i2 i3 a _).trans (by + rw [step16Field_eq_expression, Expressions.step16Field_eq_quadCells th tl q j j1 i0 i1 i2 + i3 a hj1 (by omega) (by omega) (by omega) (by omega)]))] + rw [ih _ _ _ _ _ _ _ _ (by + rw [usize_add16 j _ (by omega) ht, usize_add16 j1 _ (by omega) ht, hj1] + omega) + (by rw [usize_add16 i3 _ (by omega) hb, size_quadCells]; omega) + (by rw [usize_add16 i0 _ (by omega) hb, usize_add16 i1 _ (by omega) hb]; omega) + (by rw [usize_add16 i1 _ (by omega) hb, usize_add16 i2 _ (by omega) hb]; omega) + (by rw [usize_add16 i2 _ (by omega) hb, usize_add16 i3 _ (by omega) hb]; omega)] + rw [usize_add16 j _ (by omega) ht, usize_add16 i0 _ (by omega) hb, + usize_add16 i1 _ (by omega) hb, usize_add16 i2 _ (by omega) hb, + usize_add16 i3 _ (by omega) hb, ← quadCells_add, Nat.mul_succ, Nat.add_comm (16 * n)] + +/-- Batched dispatch and the scalar tail together execute the ordinary inner loop. -/ +theorem Native.inner16_encode (th tl : Array (FastField p)) (q j i0 i1 i2 i3 : Nat) + (a : Array (FastField p)) + (ha : (encode a).size < USize.size) + (hth : (encode th).size < USize.size) (htl : (encode tl).size < USize.size) + (hthi : 2 * q ≤ th.size) (htli : q ≤ tl.size) + (hs : i3 + (q - j) ≤ a.size) (h01 : i0 + (q - j) ≤ i1) + (h12 : i1 + (q - j) ≤ i2) (h23 : i2 + (q - j) ≤ i3) : + Native.inner16 p (encode th) (encode tl) q j i0 i1 i2 i3 (encode a) = + encode (Plan.butterflyDIFRadix4Inner th tl q j i0 i1 i2 i3 a) := by + have hn : 16 * ((q - j) / 16) ≤ q - j := Nat.mul_div_le _ _ + rw [Native.inner16] + split + · rename_i h + rw [Native.inner16Loop_encode th tl q _ _ _ _ _ _ _ a _ (by simp only [USize.toNat_ofNatLT]) + (by simp only [USize.toNat_ofNatLT]; omega) (by simp only [USize.toNat_ofNatLT]; omega) + (by simp only [USize.toNat_ofNatLT]; omega) (by simp only [USize.toNat_ofNatLT]; omega)] + simp only [USize.toNat_ofNatLT] + rw [Native.inner_encode th tl q _ _ _ _ _ _ + (by simpa only [size_encode_fields, size_quadCells] using ha) hth htl hthi htli + (by simp only [size_quadCells]; omega)] + rw [butterflyDIFRadix4Inner_eq_quadCells, butterflyDIFRadix4Inner_eq_quadCells, + ← quadCells_add] + have he : 16 * ((q - j) / 16) + (q - (j + 16 * ((q - j) / 16))) = q - j := by omega + rw [he] + · exact Native.inner_encode th tl q j i0 i1 i2 i3 a ha hth htl hthi htli (by omega) + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Native.lean b/CompPoly/Univariate/NTTFast/Packed/Native.lean new file mode 100644 index 00000000..c1c55708 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Native.lean @@ -0,0 +1,406 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module +public import CompPoly.Univariate.NTTFast.Packed.Arithmetic +public import CompPoly.Data.ByteArray.Packed +public import CompPoly.Univariate.NTTFast.Natural +public import CompPoly.Univariate.NTTFast.Reverse32 +import CompPoly.Univariate.NTTFast.Packed.Unrolled +/-! # Packed native-storage FFT kernels over 32-bit Montgomery fields +All arithmetic, scheduling and storage are Lean. Packed words are read and written with the +`ByteArray` little-endian `UInt32` accessors of `CompPoly.Data.ByteArray.Pack`, Lean core's +proposed `ugetUInt32LE` / `usetUInt32LE`. +-/ +@[expose] public section +open CompPoly +namespace CompPoly.CPolynomial.NTTFast.Packed.Native +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) +theorem usize_numeral (n : Nat) (h : n < 4294967296) : + (OfNat.ofNat n : USize).toNat = n := by + rw [USize.toNat_ofNat] + exact Nat.mod_eq_of_lt (lt_of_lt_of_le h USize.le_size) +/-- A proved byte range rules out wrapping in the machine-sized byte offset. -/ +theorem wordOffset_toNat (i o : USize) (h : 4 * (i.toNat + o.toNat) + 3 < USize.size) : + ((i + o) * 4).toNat = 4 * (i.toNat + o.toNat) := by + have hn : i.toNat + o.toNat < USize.size := by omega + rw [USize.toNat_mul, USize.toNat_add, Nat.mod_eq_of_lt hn, + usize_numeral 4 (by decide), Nat.mod_eq_of_lt (show (i.toNat + o.toNat) * 4 < USize.size by + omega)] + omega +/-- Store word `k` of a batch at byte offset `off + d`, `d = 4 k`, when `k < c`, keeping the +size. -/ +@[inline] def putWord {n : Nat} (s : {x : ByteArray // x.size = n}) (off : USize) (c k : Nat) + (d : USize) (hd : d.toNat = 4 * k) (v : UInt32) + (h : off.toNat + 4 * c ≤ n ∧ off.toNat + 4 * c < USize.size) : + {x : ByteArray // x.size = n} := + if hk : k < c then + have hu : (off + d).toNat = off.toNat + 4 * k := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_add, hd, hsize, Nat.mod_eq_of_lt (by omega)] + ⟨s.1.usetUInt32LE (off + d) v (by rw [hu, s.2]; omega), + by rw [ByteArray.size_usetUInt32LE, s.2]⟩ + else s +/-- Store the first `c` of sixteen words from byte offset `off`. -/ +@[inline] def putWords {n : Nat} (s : {x : ByteArray // x.size = n}) (off : USize) (c : Nat) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) + (h : off.toNat + 4 * c ≤ n ∧ off.toNat + 4 * c < USize.size) : ByteArray := + let s := putWord s off c 0 0 (by rw [usize_numeral _ (by decide)]) v0 h + let s := putWord s off c 1 4 (by rw [usize_numeral _ (by decide)]) v1 h + let s := putWord s off c 2 8 (by rw [usize_numeral _ (by decide)]) v2 h + let s := putWord s off c 3 12 (by rw [usize_numeral _ (by decide)]) v3 h + let s := putWord s off c 4 16 (by rw [usize_numeral _ (by decide)]) v4 h + let s := putWord s off c 5 20 (by rw [usize_numeral _ (by decide)]) v5 h + let s := putWord s off c 6 24 (by rw [usize_numeral _ (by decide)]) v6 h + let s := putWord s off c 7 28 (by rw [usize_numeral _ (by decide)]) v7 h + let s := putWord s off c 8 32 (by rw [usize_numeral _ (by decide)]) v8 h + let s := putWord s off c 9 36 (by rw [usize_numeral _ (by decide)]) v9 h + let s := putWord s off c 10 40 (by rw [usize_numeral _ (by decide)]) v10 h + let s := putWord s off c 11 44 (by rw [usize_numeral _ (by decide)]) v11 h + let s := putWord s off c 12 48 (by rw [usize_numeral _ (by decide)]) v12 h + let s := putWord s off c 13 52 (by rw [usize_numeral _ (by decide)]) v13 h + let s := putWord s off c 14 56 (by rw [usize_numeral _ (by decide)]) v14 h + (putWord s off c 15 60 (by rw [usize_numeral _ (by decide)]) v15 h).1 +theorem size_putWords {n : Nat} (s : {x : ByteArray // x.size = n}) (off : USize) (c : Nat) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) (h) : + (putWords s off c v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 h).size = n := by + unfold putWords + dsimp only + exact (putWord _ off c 15 60 (by rw [usize_numeral _ (by decide)]) v15 h).property +theorem toNat_usize_le (b : ByteArray) : b.usize.toNat ≤ b.size := by + simp only [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat'] + exact Nat.mod_le _ _ +/-- `x + y` does not wrap. -/ +theorem usize_add_toNat_of_le (x y : USize) (h : x ≤ x + y) : + (x + y).toNat = x.toNat + y.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.le_iff_toNat_le, USize.toNat_add, hsize] at h + rw [USize.toNat_add, hsize] + have hx := x.toNat_lt_size + have hy := y.toNat_lt_size + by_cases hlt : x.toNat + y.toNat < USize.size + · exact Nat.mod_eq_of_lt hlt + · rw [Nat.mod_eq_sub_mod (by omega), Nat.mod_eq_of_lt (by omega)] at h + omega +/-- Overwrite at most sixteen little-endian words, preserving shared inputs. The range checks run +on machine words. -/ +@[inline] def storeWords (b : ByteArray) (offset : USize) (count : UInt8) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) : ByteArray := + if hc : count.toNat > 16 then b else + have h4 : (4 * count.toNat).toUSize.toNat = 4 * count.toNat := + USize.toNat_ofNat_of_lt' (by have := USize.le_size; omega) + if h : offset ≤ offset + (4 * count.toNat).toUSize ∧ + offset + (4 * count.toNat).toUSize ≤ b.usize then + have hn := usize_add_toNat_of_le _ _ h.1 + have hb := toNat_usize_le b + have he := USize.le_iff_toNat_le.mp h.2 + putWords ⟨b, rfl⟩ offset count.toNat v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 + ⟨by omega, by have := (offset + (4 * count.toNat).toUSize).toNat_lt_size; omega⟩ + else b +/-- Word `i + o`, or zero unless its four bytes are in range. -/ +@[inline] def readAt (b : @& ByteArray) (i o : USize) : UInt32 := + if h : 4 ≤ b.usize ∧ (i + o) * 4 ≤ b.usize - 4 then + b.ugetUInt32LE ((i + o) * 4) (by + have hb := toNat_usize_le b + rw [USize.le_iff_toNat_le, USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ h.1] at h + rw [usize_numeral 4 (by decide)] at h + omega) + else 0 +/-- Ordinary reads use the checked word read. -/ +@[inline] def read (b : @& ByteArray) (i : @& Nat) : UInt32 := + readAt b i.toUSize 0 +/-- Single-word writes use the same batch storage primitive. -/ +@[inline] def write (b : ByteArray) (i : @& Nat) (v : UInt32) : ByteArray := + storeWords b (i.toUSize * 4) 1 v 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +/-- Overwrite sixteen words from word index `i`. -/ +@[inline] def write16U (b : ByteArray) (i : USize) (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 + v14 v15 : UInt32) : ByteArray := + storeWords b (i * 4) 16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 +/-- A buffer of `n` bytes. -/ +abbrev Buf (n : Nat) : Type := { x : ByteArray // x.size = n } + +/-- Overwrite eight words of a buffer from byte offset `o`, with the range proved instead of +checked. -/ +@[inline] def store8 {n : Nat} (b : Buf n) (o : USize) (v0 v1 v2 v3 v4 v5 v6 v7 : UInt32) + (h : o.toNat + 4 * 8 ≤ n ∧ o.toNat + 4 * 8 < USize.size) : Buf n := + ⟨putWords b o 8 v0 v1 v2 v3 v4 v5 v6 v7 0 0 0 0 0 0 0 0 h, size_putWords ..⟩ + +/-- The proved store is the checked one. -/ +theorem store8_val {n : Nat} (b : Buf n) (o : USize) (v0 v1 v2 v3 v4 v5 v6 v7 : UInt32) (h) + (hs : n < USize.size) : + (store8 b o v0 v1 v2 v3 v4 v5 v6 v7 h).1 = + storeWords b.1 o 8 v0 v1 v2 v3 v4 v5 v6 v7 0 0 0 0 0 0 0 0 := by + obtain ⟨b, rfl⟩ := b + have h8 : (8 : UInt8).toNat = 8 := rfl + have h32 : (32 : Nat).toUSize.toNat = 32 := + USize.toNat_ofNat_of_lt' (by have := USize.le_size; omega) + have hu : b.usize.toNat = b.size := by + rw [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' (by omega)] + have hadd : (o + (32 : Nat).toUSize).toNat = o.toNat + 32 := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_add, h32, hsize, Nat.mod_eq_of_lt (by omega)] + unfold storeWords + split + · rename_i hc + rw [h8] at hc + omega + · split + · simp only [h8, store8] + · rename_i hn + exfalso + apply hn + rw [h8, USize.le_iff_toNat_le, USize.le_iff_toNat_le, hadd, hu] + omega + +/-- A buffer of `n` zero words, for kernels that store their output words in place. -/ +def zeroWords (n : Nat) : ByteArray := ByteArray.zeros (4 * n) +/-- Store `f i` up to `f (i + n - 1)` at words `i` up to `i + n - 1`. -/ +@[specialize] def generateGo (f : Nat → UInt32) : Nat → Nat → ByteArray → ByteArray + | 0, _, b => b + | n + 1, i, b => generateGo f n (i + 1) (write b i (f i)) +/-- The words `f 0, …, f (n - 1)`, overwriting a zero buffer word by word. -/ +@[specialize] def generate (n : Nat) (f : Nat → UInt32) : ByteArray := + if 4 * n < USize.size then generateGo f n 0 (zeroWords n) + else encode (Array.ofFn fun i : Fin n ↦ f i.val) +/-- Pack a field array. -/ +def encodeFields (a : Array (FastField p)) : ByteArray := + generate a.size fun i ↦ (a.getD i 0).val +@[specialize] +def inner (p : ℕ) [Mont32Field p] (th tl : ByteArray) (q j i0 i1 i2 i3 : Nat) + (b : ByteArray) : ByteArray := + if j < q then + let x0 := read b i0 + let x1 := read b i1 + let x2 := read b i2 + let x3 := read b i3 + let a0 := add p x0 x2 + let a1 := add p x1 x3 + let a2 := mul p (read th j) (sub p x0 x2) + let a3 := mul p (read th (j + q)) (sub p x1 x3) + let b := write b i0 (add p a0 a1) + let b := write b i1 (mul p (read tl j) (sub p a0 a1)) + let b := write b i2 (add p a2 a3) + let b := write b i3 (mul p (read tl j) (sub p a2 a3)) + inner p th tl q (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) b + else b +termination_by q - j +decreasing_by omega +@[inline] def readUOffset (b : @& ByteArray) (i : USize) (o : USize) + (h : 4 * (i.toNat + o.toNat) + 3 < b.size ∧ b.size < USize.size) : UInt32 := + b.ugetUInt32LE ((i + o) * 4) (by rw [wordOffset_toNat i o (by omega)]; omega) +/-- The word at byte offset `o` of `b` written back unchanged. The result is exclusive, so a +kernel that starts from it lets the compiler drop the exclusivity checks of its later stores: its +loads then interleave with them instead of being held, spilled, until the first one. -/ +@[inline] def touch (b : ByteArray) (o : USize) (h : o.toNat + 4 ≤ b.size) : Buf b.size := + ⟨b.usetUInt32LE o (b.ugetUInt32LE o h) h, ByteArray.size_usetUInt32LE ..⟩ + +theorem touch_eq (b : ByteArray) (o : USize) (h) : (touch b o h).1 = b := + PackedRepr.setUInt32LE!_getUInt32LE! b o.toNat h + +/-- The last word of a buffer of at least four bytes fits it. -/ +theorem lastWord_bound (b : ByteArray) (h : 4 ≤ b.usize) : (b.usize - 4).toNat + 4 ≤ b.size := by + have hb := toNat_usize_le b + rw [USize.le_iff_toNat_le, usize_numeral 4 (by decide)] at h + rw [USize.toNat_sub_of_le _ _ (by rw [USize.le_iff_toNat_le, usize_numeral 4 (by decide)]; omega), + usize_numeral 4 (by decide)] + omega + +/-- The buffer with its last word written back unchanged: exclusive, like `touch`, and away from +the words a kernel reads, so that its loads do not wait for the store. -/ +@[inline] def touchLast (b : ByteArray) (h : 4 ≤ b.usize) : Buf b.size := + touch b (b.usize - 4) (lastWord_bound b h) + +@[noinline, specialize] +def step16 (p : ℕ) [Mont32Field p] (th tl : @& ByteArray) (j j1 i0 i1 i2 i3 : USize) (b : ByteArray) + (h : 4 * (i0.toNat + 15) + 3 < b.size ∧ + 4 * (i1.toNat + 15) + 3 < b.size ∧ + 4 * (i2.toNat + 15) + 3 < b.size ∧ + 4 * (i3.toNat + 15) + 3 < b.size ∧ + 4 * (j1.toNat + 15) + 3 < th.size ∧ + 4 * (j.toNat + 15) + 3 < th.size ∧ + 4 * (j.toNat + 15) + 3 < tl.size ∧ + b.size < USize.size ∧ + th.size < USize.size ∧ + tl.size < USize.size) : ByteArray := + step16_words% +/-- Batch stores keep the buffer size. -/ +theorem size_storeWords (b : ByteArray) (offset : USize) (count : UInt8) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) : + (storeWords b offset count v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 + v15).size = b.size := by + unfold storeWords + split + · rfl + · split + · exact size_putWords _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + · rfl + +/-- The sixteen-cell kernel keeps the buffer size. -/ +theorem size_step16 (th tl : ByteArray) (j j1 i0 i1 i2 i3 : USize) (b : ByteArray) (h) : + (step16 p th tl j j1 i0 i1 i2 i3 b h).size = b.size := by + unfold step16 + simp only [write16U, size_storeWords] + +/-- Advancing a machine index by one batch inside a buffer cannot wrap. -/ +theorem usize_add16 (i : USize) (n : Nat) (h : i.toNat + 16 ≤ n) (hn : n < USize.size) : + (i + 16).toNat = i.toNat + 16 := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + simp only [USize.toNat_add, USize.reduceToNat, hsize] + exact Nat.mod_eq_of_lt (by omega) + +/-- `n` sixteen-cell batches at machine indices whose ranges were checked once. -/ +@[specialize] +def inner16Loop (p : ℕ) [Mont32Field p] (th tl : @& ByteArray) : (n : Nat) → + (j j1 i0 i1 i2 i3 : USize) → + (b : ByteArray) → 4 * (i0.toNat + 16 * n) ≤ b.size ∧ 4 * (i1.toNat + 16 * n) ≤ b.size ∧ + 4 * (i2.toNat + 16 * n) ≤ b.size ∧ 4 * (i3.toNat + 16 * n) ≤ b.size ∧ + 4 * (j1.toNat + 16 * n) ≤ th.size ∧ 4 * (j.toNat + 16 * n) ≤ th.size ∧ + 4 * (j.toNat + 16 * n) ≤ tl.size ∧ b.size < USize.size ∧ th.size < USize.size ∧ + tl.size < USize.size → ByteArray + | 0, _, _, _, _, _, _, b, _ => b + | n + 1, j, j1, i0, i1, i2, i3, b, h => + let b' := step16 p th tl j j1 i0 i1 i2 i3 b (by omega) + inner16Loop p th tl n (j + 16) (j1 + 16) (i0 + 16) (i1 + 16) (i2 + 16) (i3 + 16) b' (by + have hs : b'.size = b.size := size_step16 .. + have hb := h.2.2.2.2.2.2.2.1 + have ht := h.2.2.2.2.2.2.2.2.1 + rw [hs, usize_add16 j th.size (by omega) ht, usize_add16 j1 th.size (by omega) ht, + usize_add16 i0 b.size (by omega) hb, usize_add16 i1 b.size (by omega) hb, + usize_add16 i2 b.size (by omega) hb, usize_add16 i3 b.size (by omega) hb] + omega) + +/-- The batch loop keeps the buffer size. -/ +theorem size_inner16Loop (th tl : ByteArray) : ∀ (n : Nat) (j j1 i0 i1 i2 i3 : USize) + (b : ByteArray) (h), (inner16Loop p th tl n j j1 i0 i1 i2 i3 b h).size = b.size + | 0, _, _, _, _, _, _, _, _ => rfl + | n + 1, j, j1, i0, i1, i2, i3, b, h => by + rw [inner16Loop, size_inner16Loop th tl n, size_step16] + +/-- `x + y` does not wrap below the machine word size. -/ +theorem usize_add_toNat (x y : USize) (h : x.toNat + y.toNat < USize.size) : + (x + y).toNat = x.toNat + y.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_add, hsize, Nat.mod_eq_of_lt h] + +/-- `k * y` does not wrap below the machine word size. -/ +theorem usize_mul_toNat (k : Nat) (y : USize) (hk : k < 4294967296) + (h : k * y.toNat < USize.size) : (OfNat.ofNat k * y : USize).toNat = k * y.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [USize.toNat_mul, usize_numeral k hk, hsize, Nat.mod_eq_of_lt h] + +/-- `count` radix-four blocks of quarter `q = 16 n` from word `base`, each in `n` batches of +sixteen cells, after one range check for all of them. -/ +@[specialize] +def blocks16Loop (p : ℕ) [Mont32Field p] (th tl : @& ByteArray) (n : Nat) (q : USize) : + (count : Nat) → (base : USize) → (b : ByteArray) → + 4 * (base.toNat + 4 * q.toNat * count) ≤ b.size ∧ 16 * n = q.toNat ∧ + 4 * (2 * q.toNat) ≤ th.size ∧ 4 * q.toNat ≤ tl.size ∧ b.size < USize.size ∧ + th.size < USize.size ∧ tl.size < USize.size → ByteArray + | 0, _, b, _ => b + | count + 1, base, b, h => + have hc : 4 * q.toNat ≤ 4 * q.toNat * (count + 1) := Nat.le_mul_of_pos_right _ (by omega) + have h1 : (base + q).toNat = base.toNat + q.toNat := usize_add_toNat _ _ (by omega) + have h2 : (base + 2 * q).toNat = base.toNat + 2 * q.toNat := by + rw [usize_add_toNat _ _ (by rw [usize_mul_toNat 2 q (by decide) (by omega)]; omega), + usize_mul_toNat 2 q (by decide) (by omega)] + have h3 : (base + 3 * q).toNat = base.toNat + 3 * q.toNat := by + rw [usize_add_toNat _ _ (by rw [usize_mul_toNat 3 q (by decide) (by omega)]; omega), + usize_mul_toNat 3 q (by decide) (by omega)] + have h4 : (base + 4 * q).toNat = base.toNat + 4 * q.toNat := by + rw [usize_add_toNat _ _ (by rw [usize_mul_toNat 4 q (by decide) (by omega)]; omega), + usize_mul_toNat 4 q (by decide) (by omega)] + let b' := inner16Loop p th tl n 0 q base (base + q) (base + 2 * q) (base + 3 * q) b (by + rw [h1, h2, h3, USize.toNat_zero] + omega) + blocks16Loop p th tl n q count (base + 4 * q) b' (by + rw [size_inner16Loop, h4, Nat.mul_succ] at * + omega) + +/-- The radix-four inner loop: whole sixteen-cell batches after one range check, then the +scalar tail. -/ +@[specialize] +def inner16 (p : ℕ) [Mont32Field p] (th tl : ByteArray) (q j i0 i1 i2 i3 : Nat) + (b : ByteArray) : ByteArray := + let n := (q - j) / 16 + if h : 4 * (i0 + 16 * n) ≤ b.size ∧ 4 * (i1 + 16 * n) ≤ b.size ∧ + 4 * (i2 + 16 * n) ≤ b.size ∧ 4 * (i3 + 16 * n) ≤ b.size ∧ + 4 * (j + q + 16 * n) ≤ th.size ∧ 4 * (j + 16 * n) ≤ th.size ∧ + 4 * (j + 16 * n) ≤ tl.size ∧ b.size < USize.size ∧ th.size < USize.size ∧ + tl.size < USize.size then + inner p th tl q (j + 16 * n) (i0 + 16 * n) (i1 + 16 * n) (i2 + 16 * n) (i3 + 16 * n) + (inner16Loop p th tl n (USize.ofNatLT j (by omega)) (USize.ofNatLT (j + q) (by omega)) + (USize.ofNatLT i0 (by omega)) (USize.ofNatLT i1 (by omega)) + (USize.ofNatLT i2 (by omega)) (USize.ofNatLT i3 (by omega)) b + (by simp only [USize.toNat_ofNatLT]; exact h)) + else inner p th tl q j i0 i1 i2 i3 b +/-- Sixteen radix-two DIF butterflies between the words from `i0` and from `i1`, with twiddles +from word `j` of `w`. -/ +@[noinline, specialize] +def pair16 (p : ℕ) [Mont32Field p] (w : @& ByteArray) (j i0 i1 : USize) (b : ByteArray) + (h : 4 * (i0.toNat + 15) + 3 < b.size ∧ 4 * (i1.toNat + 15) + 3 < b.size ∧ + 4 * (j.toNat + 15) + 3 < w.size ∧ b.size < USize.size ∧ w.size < USize.size) : + ByteArray := + pair16_words% + +theorem size_pair16 (w : ByteArray) (j i0 i1 : USize) (b : ByteArray) (h) : + (pair16 p w j i0 i1 b h).size = b.size := by + unfold pair16 + simp only [write16U, size_storeWords] + +/-- `n` batches of sixteen radix-two butterflies after one range check. -/ +@[specialize] +def pair16Loop (p : ℕ) [Mont32Field p] (w : @& ByteArray) : (n : Nat) → (j i0 i1 : USize) → + (b : ByteArray) → + 4 * (i0.toNat + 16 * n) ≤ b.size ∧ 4 * (i1.toNat + 16 * n) ≤ b.size ∧ + 4 * (j.toNat + 16 * n) ≤ w.size ∧ b.size < USize.size ∧ w.size < USize.size → ByteArray + | 0, _, _, _, b, _ => b + | n + 1, j, i0, i1, b, h => + let b' := pair16 p w j i0 i1 b (by omega) + pair16Loop p w n (j + 16) (i0 + 16) (i1 + 16) b' (by + have hs : b'.size = b.size := size_pair16 .. + rw [hs, usize_add16 j w.size (by omega) h.2.2.2.2, usize_add16 i0 b.size (by omega) h.2.2.2.1, + usize_add16 i1 b.size (by omega) h.2.2.2.1] + omega) + +/-- Radix-two DIF butterflies `j, …, q - 1` between the words from `i0` and from `i1`, with +twiddles from word `j` of `w`: the scalar loop. -/ +@[specialize] +def pairInner (p : ℕ) [Mont32Field p] (w : @& ByteArray) (q j i0 i1 : Nat) + (b : ByteArray) : ByteArray := + if j < q then + let x := read b i0 + let y := read b i1 + pairInner p w q (j + 1) (i0 + 1) (i1 + 1) + (write (write b i0 (add p x y)) i1 (mul p (read w j) (sub p x y))) + else b +termination_by q - j + +/-- The radix-two inner loop: whole sixteen-butterfly batches after one range check, then the +scalar tail. -/ +@[specialize] +def pairs (p : ℕ) [Mont32Field p] (w : @& ByteArray) (q j i0 i1 : Nat) + (b : ByteArray) : ByteArray := + let n := (q - j) / 16 + if h : 4 * (i0 + 16 * n) ≤ b.size ∧ 4 * (i1 + 16 * n) ≤ b.size ∧ 4 * (j + 16 * n) ≤ w.size ∧ + b.size < USize.size ∧ w.size < USize.size then + pairInner p w q (j + 16 * n) (i0 + 16 * n) (i1 + 16 * n) + (pair16Loop p w n (USize.ofNatLT j (by omega)) (USize.ofNatLT i0 (by omega)) + (USize.ofNatLT i1 (by omega)) b (by simp only [USize.toNat_ofNatLT]; exact h)) + else pairInner p w q j i0 i1 b +@[inline] def readWord (b : @& ByteArray) (i : USize) : UInt32 := + readAt b i 0 +@[specialize] def decode (p : ℕ) [Mont32Field p] (logN : Nat) (b : ByteArray) (nInv : UInt32) + (inverse : Bool) : Array (FastField p) := + let shift := (32 - logN).toUInt32 + tabulate (fun i : Fin (2 ^ logN) ↦ + let j := (reverse32 i.val.toUInt32 >>> shift).toNat + let x := read b j + let x := if inverse then mul p nInv x else x + if h : x.toNat < p then ⟨x, h⟩ else 0) +end CompPoly.CPolynomial.NTTFast.Packed.Native diff --git a/CompPoly/Univariate/NTTFast/Packed/NativeLast.lean b/CompPoly/Univariate/NTTFast/Packed/NativeLast.lean new file mode 100644 index 00000000..0f997b2f --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/NativeLast.lean @@ -0,0 +1,376 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.NativeOrder +import CompPoly.Univariate.NTTFast.Packed.Unrolled + +/-! # Fused final leaf layers and leaf reversal + +The last four DIF layers of a leaf act inside sixteen-word blocks, and the leaf reversal then +reads one word from each of sixteen blocks `L` words apart, `L = 2 ^ (m - 4)`. `lastHalf` runs +both for eight such blocks at once, one block per vector lane, and writes eight words of each of +sixteen natural-order lines; two calls per batch of sixteen block columns complete the lines. + +The layers above run as radix-four passes from the top; an odd leaf first runs its single +radix-two layer, so every leaf of at least `2 ^ 8` words ends with the fused final layers. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed.Native +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +/-- `1 <<< k` on machine words, for `k < 32`. -/ +theorem one_shiftLeft_toNat (k : USize) (hk : k.toNat < 32) : + ((1 : USize) <<< k).toNat = 2 ^ k.toNat := by + have hb : 32 ≤ System.Platform.numBits := by + rcases System.Platform.numBits_eq with h | h <;> omega + rw [USize.toNat_shiftLeft, USize.toNat_one, Nat.one_shiftLeft, + Nat.mod_eq_of_lt (show k.toNat < System.Platform.numBits by omega), + Nat.mod_eq_of_lt (Nat.pow_lt_pow_right (by omega) (by omega))] + +/-- The output check of `lastHalf` in natural numbers. -/ +theorem lastHalf_check {out : ByteArray} {shift : UInt32} {half : USize} + (h : 0 < shift ∧ shift < 32 ∧ half < 2 ∧ + (1 : USize) <<< (32 - shift).toUSize ≤ out.usize / 64) : + 0 < shift.toNat ∧ shift.toNat < 32 ∧ half.toNat < 2 ∧ + 64 * 2 ^ (32 - shift.toNat) ≤ out.usize.toNat := by + obtain ⟨h0, hs, hh, hl⟩ := h + rw [UInt32.lt_iff_toNat_lt, show (0 : UInt32).toNat = 0 from rfl] at h0 + rw [UInt32.lt_iff_toNat_lt, show (32 : UInt32).toNat = 32 from rfl] at hs + rw [USize.lt_iff_toNat_lt, usize_numeral 2 (by decide)] at hh + have hk : (32 - shift).toUSize.toNat = 32 - shift.toNat := by + rw [UInt32.toNat_toUSize, UInt32.toNat_sub_of_le _ _ (by + rw [UInt32.le_iff_toNat_le]; change shift.toNat ≤ 32; omega)] + rfl + rw [USize.le_iff_toNat_le, one_shiftLeft_toNat _ (by rw [hk]; omega), hk, USize.toNat_div, + usize_numeral 64 (by decide)] at hl + exact ⟨h0, hs, hh, by omega⟩ + +/-- Lines `y >>> shift` of sixteen words fit an output of `2 ^ (32 - shift)` lines. -/ +theorem lastHalf_store {out : ByteArray} {shift : UInt32} {half : USize} + (h : 0 < shift ∧ shift < 32 ∧ half < 2 ∧ + (1 : USize) <<< (32 - shift).toUSize ≤ out.usize / 64) + (y : UInt32) : + (4 * (16 * (y >>> shift).toUSize + 8 * half)).toNat + 4 * 8 ≤ out.size ∧ + (4 * (16 * (y >>> shift).toUSize + 8 * half)).toNat + 4 * 8 < USize.size := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hb := toNat_usize_le out + have hU := out.usize.toNat_lt_size + obtain ⟨_, hs, hh, hl⟩ := lastHalf_check h + have hr : (y >>> shift).toNat < 2 ^ (32 - shift.toNat) := by + rw [UInt32.toNat_shiftRight, Nat.mod_eq_of_lt (show shift.toNat < 32 by omega), + Nat.shiftRight_eq_div_pow, Nat.div_lt_iff_lt_mul (Nat.two_pow_pos _), + ← Nat.pow_add, show 32 - shift.toNat + shift.toNat = 32 by omega] + exact y.toNat_lt + have hr' : (y >>> shift).toUSize.toNat = (y >>> shift).toNat := UInt32.toNat_toUSize _ + have e : (4 * (16 * (y >>> shift).toUSize + 8 * half)).toNat = + 4 * (16 * (y >>> shift).toNat + 8 * half.toNat) := by + simp only [USize.toNat_add, USize.toNat_mul, USize.reduceToNat, hr', hsize] + rw [Nat.mod_eq_of_lt (show 16 * (y >>> shift).toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 8 * half.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 16 * (y >>> shift).toNat + 8 * half.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 4 * (16 * (y >>> shift).toNat + 8 * half.toNat) < USize.size by + omega)] + rw [e] + omega + +/-- An output with room for one line has room for the touched last word. -/ +theorem lastHalf_usize {out : ByteArray} {shift : UInt32} {half : USize} + (h : 0 < shift ∧ shift < 32 ∧ half < 2 ∧ + (1 : USize) <<< (32 - shift).toUSize ≤ out.usize / 64) : + 4 ≤ out.usize := by + have hl := (lastHalf_check h).2.2.2 + have := Nat.one_le_two_pow (n := 32 - shift.toNat) + rw [USize.le_iff_toNat_le, usize_numeral 4 (by decide)] + omega + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The last four DIF layers of the eight blocks at `i0, …, i7`, written to `out` as +half `half` of the natural-order lines `bitrev (16 c + t)`: lane `k` supplies word +`8 half + k` of each line, optionally scaled. The lanes are independent, so the compiler +runs them as vector lanes. -/ +@[noinline, specialize] +def lastHalf (p : ℕ) [Mont32Field p] (t3 t2 t1 b : @& ByteArray) (i0 i1 i2 i3 i4 i5 i6 i7 c : USize) + (shift : UInt32) (half : USize) (factor : UInt32) (scale : Bool) (out : ByteArray) + (h : 4 * (i0.toNat + 15) + 3 < b.size ∧ + 4 * (i1.toNat + 15) + 3 < b.size ∧ + 4 * (i2.toNat + 15) + 3 < b.size ∧ + 4 * (i3.toNat + 15) + 3 < b.size ∧ + 4 * (i4.toNat + 15) + 3 < b.size ∧ + 4 * (i5.toNat + 15) + 3 < b.size ∧ + 4 * (i6.toNat + 15) + 3 < b.size ∧ + 4 * (i7.toNat + 15) + 3 < b.size ∧ + b.size < USize.size ∧ + 31 < t3.size ∧ + t3.size < USize.size ∧ + 15 < t2.size ∧ + t2.size < USize.size ∧ + 7 < t1.size ∧ + t1.size < USize.size) : + ByteArray := + if hout : 0 < shift ∧ shift < 32 ∧ half < 2 ∧ + (1 : USize) <<< (32 - shift).toUSize ≤ out.usize / 64 then + lastHalf_words% + else out + +/-- Block `v`'s word index at batch `c` stays in range. -/ +theorem laneBound (v : USize) (hv : v.toNat < 16) (Lw c : USize) (b : ByteArray) + (hL : 64 * Lw.toNat ≤ b.size) (hb : b.size < USize.size) + (hc : 16 * c.toNat + 16 ≤ Lw.toNat) : 4 * ((v * Lw + 16 * c).toNat + 15) + 3 < b.size := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hvL : v.toNat * Lw.toNat ≤ 15 * Lw.toNat := Nat.mul_le_mul_right _ (by omega) + simp only [USize.toNat_add, USize.toNat_mul, USize.reduceToNat, hsize] + rw [Nat.mod_eq_of_lt (show v.toNat * Lw.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 16 * c.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show v.toNat * Lw.toNat + 16 * c.toNat < USize.size by omega)] + omega + +/-- Batches `c, …, c + n - 1` of sixteen block columns, both halves each. -/ +@[specialize] +def lastRevGo (p : ℕ) [Mont32Field p] (t3 t2 t1 b : @& ByteArray) (Lw : USize) + (shift factor : UInt32) (scale : Bool) + (hb : 64 * Lw.toNat ≤ b.size ∧ b.size < USize.size ∧ 31 < t3.size ∧ t3.size < USize.size ∧ + 15 < t2.size ∧ t2.size < USize.size ∧ 7 < t1.size ∧ t1.size < USize.size) : + Nat → USize → ByteArray → ByteArray + | 0, _, out => out + | n + 1, c, out => + if hc : 16 * c.toNat + 16 ≤ Lw.toNat then + let out := lastHalf p t3 t2 t1 b (0 * Lw + 16 * c) (8 * Lw + 16 * c) (4 * Lw + 16 * c) + (12 * Lw + 16 * c) + (2 * Lw + 16 * c) (10 * Lw + 16 * c) (6 * Lw + 16 * c) (14 * Lw + 16 * c) + c shift 0 factor scale out + ⟨laneBound 0 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 8 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 4 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 12 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 2 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 10 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 6 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 14 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + hb.2.1, hb.2.2⟩ + let out := lastHalf p t3 t2 t1 b (1 * Lw + 16 * c) (9 * Lw + 16 * c) (5 * Lw + 16 * c) + (13 * Lw + 16 * c) + (3 * Lw + 16 * c) (11 * Lw + 16 * c) (7 * Lw + 16 * c) (15 * Lw + 16 * c) + c shift 1 factor scale out + ⟨laneBound 1 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 9 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 5 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 13 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 3 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 11 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 7 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + laneBound 15 (by simp only [USize.reduceToNat]; omega) Lw c b hb.1 hb.2.1 hc, + hb.2.1, hb.2.2⟩ + lastRevGo p t3 t2 t1 b Lw shift factor scale hb n (c + 1) out + else out + +/-- The last four DIF layers of a `2 ^ m`-word leaf whose earlier layers are done, in natural +order, optionally scaled. The output starts as a copy of the input, which the first store makes +because the input stays borrowed; every word is then overwritten. -/ +@[specialize] +def lastRev (p : ℕ) [Mont32Field p] (t3 t2 t1 b : @& ByteArray) (m : Nat) (factor : UInt32) + (scale : Bool) : + ByteArray := + if h : 2 ^ (m - 4) < USize.size ∧ 64 * 2 ^ (m - 4) ≤ b.size ∧ b.size < USize.size ∧ + 31 < t3.size ∧ t3.size < USize.size ∧ 15 < t2.size ∧ t2.size < USize.size ∧ + 7 < t1.size ∧ t1.size < USize.size then + lastRevGo p t3 t2 t1 b (USize.ofNatLT (2 ^ (m - 4)) h.1) (32 - (m - 4)).toUInt32 factor scale + (by simpa only [USize.toNat_ofNatLT] using h.2) (2 ^ (m - 4) / 16) 0 b + else b + +/-- `blocks` radix-four blocks of quarter `q`, the first at word `0`: for quarters of whole +sixteen-cell batches one range check covers all blocks, otherwise each block checks its own. -/ +@[specialize] +def radix4Blocks (p : ℕ) [Mont32Field p] (th tl : @& ByteArray) (q blocks : Nat) + (a : ByteArray) : ByteArray := + if h : q % 16 = 0 ∧ 4 * (4 * q * blocks) ≤ a.size ∧ 4 * (2 * q) ≤ th.size ∧ + 4 * q ≤ tl.size ∧ a.size < USize.size ∧ th.size < USize.size ∧ tl.size < USize.size then + blocks16Loop p th tl (q / 16) (USize.ofNatLT q (by omega)) blocks 0 a (by + simp only [USize.toNat_ofNatLT, USize.toNat_zero] + omega) + else Id.run do + let mut a := a + for block in [:blocks] do + a := inner16 p th tl q 0 (block * (4 * q)) (block * (4 * q) + q) (block * (4 * q) + 2 * q) + (block * (4 * q) + 3 * q) a + return a + +/-- The leading layers of a `2 ^ m`-word leaf: for odd `m` the radix-two top layer, then `count` +radix-four passes. -/ +@[specialize] +def leafLayers (p : ℕ) [Mont32Field p] (tw : @& Array ByteArray) (m count : Nat) + (a : ByteArray) : ByteArray := + let a := if m % 2 = 1 then pairs p (tw.getD (m - 1) .empty) (2 ^ (m - 1)) 0 0 (2 ^ (m - 1)) a + else a + (List.range count).foldl (fun a pass ↦ + let high := m - 1 - m % 2 - 2 * pass + let q := 2 ^ (high - 1) + radix4Blocks p (tw.getD high .empty) (tw.getD (high - 1) .empty) q (2 ^ m / (4 * q)) a) a + +/-- A complete `2 ^ m`-word leaf in natural order, optionally scaled. Leaves of at least `2 ^ 8` +words end with the fused final layers, smaller ones with a scalar reversal. -/ +@[specialize] +def leafNatural (p : ℕ) [Mont32Field p] (tw : Array ByteArray) (m : Nat) (factor : UInt32) + (scale : Bool) + (a : ByteArray) : ByteArray := + if 8 ≤ m then + lastRev p (tw.getD 3 .empty) (tw.getD 2 .empty) (tw.getD 1 .empty) (leafLayers p tw m + (m / 2 - 2) a) + m factor scale + else encodeFields (decode p m (leafLayers p tw m (m / 2) a) factor scale) + +/-- The last four DIF layers of the sixteen-word block at `i`, in place: the block's outputs stay +in DIF (bit-reversed) order. -/ +@[noinline, specialize] +def leaf16 (p : ℕ) [Mont32Field p] (t3 t2 t1 : @& ByteArray) (i : USize) (b : ByteArray) + (h : 4 * (i.toNat + 15) + 3 < b.size ∧ b.size < USize.size ∧ + 31 < t3.size ∧ t3.size < USize.size ∧ 15 < t2.size ∧ t2.size < USize.size ∧ + 7 < t1.size ∧ t1.size < USize.size) : ByteArray := + leaf16_words% + +/-- Four sixteen-word blocks from word `i` of `b`, and `K` twiddles in `t`: the range checks of a +stage kernel. -/ +abbrev LeafStageFits (b t : ByteArray) (i : USize) (K : Nat) : Prop := + 4 * (i.toNat + 64) ≤ b.size ∧ b.size < USize.size ∧ 4 * K ≤ t.size ∧ t.size < USize.size + +/-- A stage kernel's buffer has room for the touched last word. -/ +theorem leafStage_touch {b t : ByteArray} {i : USize} {K : Nat} (h : LeafStageFits b t i K) : + 4 ≤ b.usize := by + obtain ⟨h1, h2, -, -⟩ := h + have hu : b.usize.toNat = b.size := by + rw [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' h2] + rw [USize.le_iff_toNat_le, hu, usize_numeral 4 (by decide)] + omega + +/-- Word `k` of a stage kernel's blocks fits its buffer, of the same size as `b`. -/ +theorem leafStage_read {b t o : ByteArray} {i : USize} {K : Nat} (h : LeafStageFits b t i K) + (ho : o.size = b.size) (k : Nat) (hk : k < 64) : + 4 * (i.toNat + (OfNat.ofNat k : USize).toNat) + 3 < o.size ∧ o.size < USize.size := by + obtain ⟨h1, h2, -, -⟩ := h + rw [usize_numeral k (by omega), ho] + exact ⟨by omega, h2⟩ + +/-- Twiddle `k < K` of a stage kernel fits its table. -/ +theorem leafStage_twiddle {b t : ByteArray} {i : USize} {K : Nat} (h : LeafStageFits b t i K) + (k : Nat) (hk : k < K) (hk8 : k < 8) : + 4 * ((0 : USize).toNat + (OfNat.ofNat k : USize).toNat) + 3 < t.size ∧ + t.size < USize.size := by + obtain ⟨-, -, h3, h4⟩ := h + rw [usize_numeral k (by omega), USize.toNat_zero] + exact ⟨by omega, h4⟩ + +/-- The eight words from word `i + c` of a stage kernel's blocks fit its buffer. -/ +theorem leafStage_store {b t : ByteArray} {i : USize} {K : Nat} (h : LeafStageFits b t i K) + (c : Nat) (hc : c + 8 ≤ 64) : + ((i + OfNat.ofNat c) * 4).toNat + 4 * 8 ≤ b.size ∧ + ((i + OfNat.ofNat c) * 4).toNat + 4 * 8 < USize.size := by + obtain ⟨h1, h2, -, -⟩ := h + have hc' : (OfNat.ofNat c : USize).toNat = c := usize_numeral c (by omega) + rw [wordOffset_toNat _ _ (by rw [hc']; omega), hc'] + exact ⟨by omega, by omega⟩ + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The first of the four final DIF layers in place on the four blocks from word `i`: words `k` and +`k + 8` of a block, the difference times twiddle `k` of `t3`. -/ +@[noinline, specialize] +def leafStage8 (p : ℕ) [Mont32Field p] (t : @& ByteArray) (i : USize) (b : ByteArray) + (h : LeafStageFits b t i 8) : Buf b.size := + leafStage8_words% + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The second of the four final DIF layers in place: words `k` and `k + 4` of a half block, the +difference times twiddle `k` of `t2`. -/ +@[noinline, specialize] +def leafStage4 (p : ℕ) [Mont32Field p] (t : @& ByteArray) (i : USize) (b : ByteArray) + (h : LeafStageFits b t i 4) : Buf b.size := + leafStage4_words% + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The third of the four final DIF layers in place: words `k` and `k + 2` of a quarter block, the +difference times twiddle `k` of `t1`. -/ +@[noinline, specialize] +def leafStage2 (p : ℕ) [Mont32Field p] (t : @& ByteArray) (i : USize) (b : ByteArray) + (h : LeafStageFits b t i 2) : Buf b.size := + leafStage2_words% + +set_option maxRecDepth 8192 in +set_option maxHeartbeats 4000000 in +/-- The last DIF layer in place: words `2 k` and `2 k + 1`, without twiddles. -/ +@[noinline, specialize] +def leafStage1 (p : ℕ) [Mont32Field p] (t : @& ByteArray) (i : USize) (b : ByteArray) + (h : LeafStageFits b t i 0) : Buf b.size := + leafStage1_words% + +/-- `count` groups of four blocks from word `i` through the stage kernel `k`, after one range +check. -/ +@[specialize] +def leafStageLoop (K : Nat) + (k : (t : ByteArray) → (i : USize) → (b : ByteArray) → LeafStageFits b t i K → Buf b.size) + (t : @& ByteArray) : (count : Nat) → (i : USize) → (b : ByteArray) → + 4 * (i.toNat + 64 * count) ≤ b.size ∧ b.size < USize.size ∧ 4 * K ≤ t.size ∧ + t.size < USize.size → ByteArray + | 0, _, b, _ => b + | count + 1, i, b, h => + have h64 : (i + 64).toNat = i.toNat + 64 := by + rw [usize_add_toNat _ _ (by rw [usize_numeral 64 (by decide)]; omega), + usize_numeral 64 (by decide)] + let b' := k t i b ⟨by omega, h.2.1, h.2.2⟩ + leafStageLoop K k t count (i + 64) b'.1 (by rw [b'.2, h64]; omega) + +/-- The stage loop keeps the buffer size. -/ +theorem size_leafStageLoop (K : Nat) + (k : (t : ByteArray) → (i : USize) → (b : ByteArray) → LeafStageFits b t i K → Buf b.size) + (t : ByteArray) : ∀ (count : Nat) (i : USize) (b : ByteArray) (h), + (leafStageLoop K k t count i b h).size = b.size + | 0, _, _, _ => rfl + | count + 1, i, b, h => by + rw [leafStageLoop, size_leafStageLoop K k t count, (k t i b _).2] + +/-- `leaf16` on the blocks at words `0, 16, …, 16 * (blocks - 1)`: the four layers each in one pass +over the groups of four blocks, after one range check, then the remaining blocks one by one. -/ +@[specialize] +def leaf16Blocks (p : ℕ) [Mont32Field p] (t3 t2 t1 : @& ByteArray) (blocks : Nat) + (a : ByteArray) : ByteArray := Id.run do + let mut a := a + let n := blocks / 4 + if h : 4 * (64 * n) ≤ a.size ∧ a.size < USize.size ∧ + 4 * 8 ≤ t3.size ∧ t3.size < USize.size ∧ 4 * 4 ≤ t2.size ∧ t2.size < USize.size ∧ + 4 * 2 ≤ t1.size ∧ t1.size < USize.size then + have h0 : (0 : USize).toNat = 0 := USize.toNat_zero + let a8 := leafStageLoop 8 (leafStage8 p) t3 n 0 a (by omega) + let a4 := leafStageLoop 4 (leafStage4 p) t2 n 0 a8 (by + rw [size_leafStageLoop]; omega) + let a2 := leafStageLoop 2 (leafStage2 p) t1 n 0 a4 (by + rw [size_leafStageLoop, size_leafStageLoop]; omega) + a := leafStageLoop 0 (leafStage1 p) t1 n 0 a2 (by + rw [size_leafStageLoop, size_leafStageLoop, size_leafStageLoop]; omega) + for block in [4 * n:blocks] do + let i := 16 * block + if h : 4 * (i + 15) + 3 < a.size ∧ a.size < USize.size ∧ + 31 < t3.size ∧ t3.size < USize.size ∧ 15 < t2.size ∧ t2.size < USize.size ∧ + 7 < t1.size ∧ t1.size < USize.size then + a := leaf16 p t3 t2 t1 (USize.ofNatLT i (by omega)) a + (by simpa only [USize.toNat_ofNatLT] using h) + return a + +/-- A complete `2 ^ m`-word leaf in DIF order, the natural order with bit-reversed indices: from +`2 ^ 4` words its last four layers run in place on sixteen-word blocks. -/ +@[specialize] +def leafReversed (p : ℕ) [Mont32Field p] (tw : @& Array ByteArray) (m : Nat) + (a : ByteArray) : ByteArray := + if 4 ≤ m then + leaf16Blocks p (tw.getD 3 .empty) (tw.getD 2 .empty) (tw.getD 1 .empty) (2 ^ m / 16) + (leafLayers p tw m (m / 2 - 2) a) + else leafLayers p tw m (m / 2) a + +end CompPoly.CPolynomial.NTTFast.Packed.Native diff --git a/CompPoly/Univariate/NTTFast/Packed/NativeLastCorrectness.lean b/CompPoly/Univariate/NTTFast/Packed/NativeLastCorrectness.lean new file mode 100644 index 00000000..4371dc56 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/NativeLastCorrectness.lean @@ -0,0 +1,880 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Packed.NativeLast +public import CompPoly.Univariate.NTTFast.Packed.NativeLast +public import CompPoly.Univariate.NTTFast.Packed.KernelRefinement +public import CompPoly.Univariate.NTTFast.Packed.KernelCorrectness +public import CompPoly.Univariate.NTTFast.Packed.KernelSpecialization +public import CompPoly.Univariate.NTTFast.Packed.Twiddles +public import CompPoly.Univariate.NTTFast.Packed.LocalLayers +public import CompPoly.Univariate.NTTFast.Packed.PassRefinement +public import CompPoly.Univariate.NTTFast.Packed.Radix2Refinement +public import CompPoly.Univariate.NTTFast.Packed.NativeOrderCorrectness +import all CompPoly.Univariate.NTTFast.Packed.NativeOrder +import all CompPoly.Univariate.NTTFast.Packed.Native +import CompPoly.Univariate.NTTFast.Packed.Unrolled + +/-! # Correctness of the fused final leaf layers + +`laneLine` is the field-level expression graph of one lane of `lastHalf`: the four final DIF +layers of one sixteen-word block, exactly as in `leaf16Field`. A kernel call writes, for each +`t < 16`, eight words of line `bitrev (16 c + t)` (`writeLines`); the batch loop covers every +line once, so `lastRev` is the bit-reversed, optionally scaled result of the final layers. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode size_encode_uint32) + +/-- Sixteen field coordinates. -/ +structure FieldLine (p : ℕ) [Mont32Field p] where + w0 : FastField p + w1 : FastField p + w2 : FastField p + w3 : FastField p + w4 : FastField p + w5 : FastField p + w6 : FastField p + w7 : FastField p + w8 : FastField p + w9 : FastField p + w10 : FastField p + w11 : FastField p + w12 : FastField p + w13 : FastField p + w14 : FastField p + w15 : FastField p + +/-- Coordinate `t` of a line. -/ +def FieldLine.get (y : FieldLine p) : Nat → FastField p + | 0 => y.w0 + | 1 => y.w1 + | 2 => y.w2 + | 3 => y.w3 + | 4 => y.w4 + | 5 => y.w5 + | 6 => y.w6 + | 7 => y.w7 + | 8 => y.w8 + | 9 => y.w9 + | 10 => y.w10 + | 11 => y.w11 + | 12 => y.w12 + | 13 => y.w13 + | 14 => y.w14 + | _ => y.w15 + +/-- The four final DIF layers of the block at `i`, as in `leaf16Field`. -/ +def laneLine (t3 t2 t1 : Array (FastField p)) (i : USize) + (b : Array (FastField p)) : FieldLine p := + leaf16_line% + +/-- The fused leaf kernel is the lane graph followed by one splice. -/ +theorem leaf16Field_eq_laneLine (t3 t2 t1 : Array (FastField p)) (i : USize) + (b : Array (FastField p)) : + leaf16Field t3 t2 t1 i b = splice b i.toNat + #[(laneLine t3 t2 t1 i b).w0, (laneLine t3 t2 t1 i b).w1, (laneLine t3 t2 t1 i b).w2, + (laneLine t3 t2 t1 i b).w3, (laneLine t3 t2 t1 i b).w4, (laneLine t3 t2 t1 i b).w5, + (laneLine t3 t2 t1 i b).w6, (laneLine t3 t2 t1 i b).w7, (laneLine t3 t2 t1 i b).w8, + (laneLine t3 t2 t1 i b).w9, (laneLine t3 t2 t1 i b).w10, (laneLine t3 t2 t1 i b).w11, + (laneLine t3 t2 t1 i b).w12, (laneLine t3 t2 t1 i b).w13, (laneLine t3 t2 t1 i b).w14, + (laneLine t3 t2 t1 i b).w15] := by + unfold leaf16Field laneLine + rfl + +/-- A literal line returns its coordinates. -/ +theorem getD_lit_line (y : FieldLine p) (k : Nat) (hk : k < 16) : + (#[y.w0, y.w1, y.w2, y.w3, y.w4, y.w5, y.w6, y.w7, y.w8, y.w9, y.w10, y.w11, y.w12, y.w13, + y.w14, y.w15] : Array (FastField p)).getD k 0 = y.get k := by + interval_cases k <;> rfl + +/-- Coordinate `t` of a lane is the fused kernel's coordinate `i + t`. -/ +theorem getD_leaf16Field_laneLine (t3 t2 t1 : Array (FastField p)) (i : USize) + (b : Array (FastField p)) (hi : i.toNat + 16 ≤ b.size) (t : Nat) (ht : t < 16) : + (leaf16Field t3 t2 t1 i b).getD (i.toNat + t) 0 = (laneLine t3 t2 t1 i b).get t := by + have hsz : ∀ v : Array (FastField p), v.size = 16 → i.toNat + v.size ≤ b.size := by + intro v hv; omega + rw [leaf16Field_eq_laneLine, getD_splice _ _ _ (hsz _ rfl)] + rw [ite_eq_right_of_eq_false _ _ (eq_false (show ¬(i.toNat + t < i.toNat) by omega))] + simp only [List.size_toArray, List.length_cons, List.length_nil, Nat.reduceAdd, + show i.toNat + t < i.toNat + 16 by omega, ↓reduceIte, Nat.add_sub_cancel_left] + exact getD_lit_line _ t ht + +/-- Coordinate `t` of a lane is coordinate `i + t` of the four final DIF layers at `i`. -/ +theorem laneLine_get (t3 t2 t1 : Array (FastField p)) (i : USize) + (b : Array (FastField p)) (hi : i.toNat + 16 ≤ b.size) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) (t : Nat) + (ht : t < 16) : + (laneLine t3 t2 t1 i b).get t = + (Expressions.leaf16Stages t3 t2 t1 b i.toNat).getD (i.toNat + t) 0 := by + rw [← getD_leaf16Field_laneLine t3 t2 t1 i b hi t ht, leaf16Field_eq_expression, + Expressions.leaf16Field_eq_stages t3 t2 t1 b i hi h3 h2 h1] + +/-- Eight words stored at word `idx`, if they fit; a model of a guarded half-line store. -/ +def spliceHalf (O : Array UInt32) (idx : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : UInt32) : Array UInt32 := + if idx + 8 ≤ O.size then + O.extract 0 idx ++ #[v0, v1, v2, v3, v4, v5, v6, v7, 0, 0, 0, 0, 0, 0, 0, 0].extract 0 8 ++ + O.extract (idx + 8) O.size + else O + +@[simp] theorem size_spliceHalf (O : Array UInt32) (idx : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : UInt32) : + (spliceHalf O idx v0 v1 v2 v3 v4 v5 v6 v7).size = O.size := by + unfold spliceHalf + split + · simp only [Array.size_append, Array.size_extract, List.size_toArray, List.length_cons, + List.length_nil] + omega + · rfl + +/-- A half-line store into packed words. -/ +theorem storeWords_half (O : Array UInt32) (off : USize) (idx : Nat) (ho : off.toNat = 4 * idx) + (hs : (encode O).size < USize.size) (v0 v1 v2 v3 v4 v5 v6 v7 : UInt32) : + Native.storeWords (encode O) off 8 v0 v1 v2 v3 v4 v5 v6 v7 0 0 0 0 0 0 0 0 = + encode (spliceHalf O idx v0 v1 v2 v3 v4 v5 v6 v7) := by + unfold spliceHalf + split + · rename_i hi + exact Native.storeWords_replace_encodeWords O off 8 idx (by decide) ho hi hs _ _ _ _ _ _ _ _ _ + _ _ _ + _ _ _ _ + · rename_i hi + have hf := (Native.storeWords_fits O off 8 idx (by decide) ho hs).not.mpr hi + simp only [Native.storeWords_eq, ↓reduceIte, + show (8 : UInt8).toNat = 8 from rfl, show ¬(8 > 16) by decide] at hf ⊢ + exact ite_eq_right_iff.mpr (fun h ↦ absurd h hf) + +/-- The byte offset of half `half` of line `bitrev (16 c + T)`. -/ +theorem lineOffset (mm : Nat) (hm : 0 < mm) (h32 : mm ≤ 32) (c T half : USize) + (hT : T.toNat < 16) (hh : half.toNat < 2) (hc : 16 * c.toNat + 16 ≤ 2 ^ mm) + (hu : 64 * 2 ^ mm < USize.size) : + (4 * (16 * (reverse32 (16 * c + T).toUInt32 >>> (32 - mm).toUInt32).toUSize + + 8 * half)).toNat = + 4 * (16 * NTT.Transform.bitRevNat mm (16 * c.toNat + T.toNat) + 8 * half.toNat) := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have h32' : 2 ^ mm ≤ 2 ^ 32 := Nat.pow_le_pow_right (by omega) h32 + have hx : (16 * c + T).toNat = 16 * c.toNat + T.toNat := by + simp only [USize.toNat_add, USize.toNat_mul, USize.reduceToNat, hsize] + rw [Nat.mod_eq_of_lt (show 16 * c.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 16 * c.toNat + T.toNat < USize.size by omega)] + have hr : (reverse32 (16 * c + T).toUInt32 >>> (32 - mm).toUInt32).toUSize.toNat = + NTT.Transform.bitRevNat mm (16 * c.toNat + T.toNat) := by + rw [UInt32.toNat_toUSize, show (16 * c + T).toUInt32 = (16 * c + T).toNat.toUInt32 from + UInt32.toFin_inj.mp rfl, hx] + exact reverse32_shift_eq_bitRevNat mm _ hm h32 + have hg := NTT.Transform.bitRevNat_lt mm (16 * c.toNat + T.toNat) + simp only [USize.toNat_add, USize.toNat_mul, USize.reduceToNat, hr, hsize] + rw [Nat.mod_eq_of_lt (show 16 * NTT.Transform.bitRevNat mm (16 * c.toNat + T.toNat) < + USize.size by omega), + Nat.mod_eq_of_lt (show 8 * half.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 16 * NTT.Transform.bitRevNat mm (16 * c.toNat + T.toNat) + + 8 * half.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 4 * (16 * NTT.Transform.bitRevNat mm (16 * c.toNat + T.toNat) + + 8 * half.toNat) < USize.size by omega)] + +/-- Lane `k` of eight. -/ +def pick8 (i0 i1 i2 i3 i4 i5 i6 i7 : USize) : Nat → USize + | 0 => i0 + | 1 => i1 + | 2 => i2 + | 3 => i3 + | 4 => i4 + | 5 => i5 + | 6 => i6 + | _ => i7 + +/-- Half `half` of the sixteen lines `bitrev (16 c + t)`, word `8 half + k` from `val t k`. -/ +def lineStores (O : Array UInt32) (mm c half : Nat) (val : Nat → Nat → UInt32) : Array UInt32 := + (List.range 16).foldl (fun O t ↦ spliceHalf O + (16 * NTT.Transform.bitRevNat mm (16 * c + t) + 8 * half) + (val t 0) (val t 1) (val t 2) (val t 3) (val t 4) (val t 5) (val t 6) (val t 7)) O + +/-- An output of `2 ^ mm` lines passes the output check of `lastHalf`. -/ +theorem lastHalf_check_of (out : ByteArray) (mm : Nat) (half : USize) (hm : 0 < mm) + (hm32 : mm < 32) (hh : half.toNat < 2) (hO : 64 * 2 ^ mm ≤ out.size) + (hs : out.size < USize.size) : + 0 < (32 - mm).toUInt32 ∧ (32 - mm).toUInt32 < 32 ∧ half < 2 ∧ + (1 : USize) <<< (32 - (32 - mm).toUInt32).toUSize ≤ out.usize / 64 := by + have hk : (32 - mm).toUInt32.toNat = 32 - mm := by + rw [Nat.toUInt32_eq, UInt32.toNat_ofNat', Nat.mod_eq_of_lt (by omega)] + have hk' : (32 - (32 - mm).toUInt32).toUSize.toNat = mm := by + rw [UInt32.toNat_toUSize, UInt32.toNat_sub_of_le _ _ (by + rw [UInt32.le_iff_toNat_le, hk]; change 32 - mm ≤ 32; omega), hk] + change 32 - (32 - mm) = mm + omega + have hu : out.usize.toNat = out.size := by + rw [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' hs] + refine ⟨?_, ?_, ?_, ?_⟩ + · rw [UInt32.lt_iff_toNat_lt, hk]; change 0 < 32 - mm; omega + · rw [UInt32.lt_iff_toNat_lt, hk]; change 32 - mm < 32; omega + · rw [USize.lt_iff_toNat_lt, Native.usize_numeral 2 (by decide)]; exact hh + · rw [USize.le_iff_toNat_le, Native.one_shiftLeft_toNat _ (by rw [hk']; exact hm32), hk', + USize.toNat_div, Native.usize_numeral 64 (by decide), hu] + omega + +set_option maxRecDepth 100000 in +set_option maxHeartbeats 4000000 in +/-- One kernel call stores the final layers of its eight lanes into sixteen half lines. -/ +theorem lastHalf_encodeWords (t3 t2 t1 U : Array (FastField p)) (i0 i1 i2 i3 i4 i5 i6 i7 c + half : + USize) (mm : Nat) (f : FastField p) (sc : Bool) (O : Array UInt32) + (hm : 0 < mm) (hm32 : mm < 32) (hh : half.toNat < 2) (hc : 16 * c.toNat + 16 ≤ 2 ^ mm) + (hu : 64 * 2 ^ mm < USize.size) (hO : O.size = 16 * 2 ^ mm) + (hs : (encode O).size < USize.size) (h) : + Native.lastHalf p (encode t3) (encode t2) (encode t1) (encode U) + i0 i1 i2 i3 i4 i5 i6 i7 c (32 - mm).toUInt32 half f.val sc (encode O) h = + encode (lineStores O mm c.toNat half.toNat fun t k ↦ + Native.scaleWord p f.val sc + ((laneLine t3 t2 t1 (pick8 i0 i1 i2 i3 i4 i5 i6 i7 k) U).get t).val) := by + have hm32 : mm ≤ 32 := by omega + unfold Native.lastHalf + rw [dite_eq_left_of_eq_true (eq_true (lastHalf_check_of _ mm half hm (by omega) hh + (by rw [size_encode_uint32, hO]; omega) hs))] + simp only [Native.store8_val _ _ _ _ _ _ _ _ _ _ _ hs, Native.touchLast, Native.touch_eq] + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 0 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + hs] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 1 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 2 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 3 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 4 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 5 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 6 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 7 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 8 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 9 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 10 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 11 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 12 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 13 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 14 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + rw [storeWords_half _ _ _ + (lineOffset mm hm hm32 c 15 half (by simp only [USize.reduceToNat]; omega) hh hc hu) + (by simp only [size_encode_uint32, size_spliceHalf] at hs ⊢; exact hs)] + unfold lineStores + rw [show List.range 16 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] from rfl] + simp only [List.foldl_cons, List.foldl_nil, laneLine, FieldLine.get, pick8, + readField, fieldAdd, fieldSub, fieldMul, USize.reduceToNat, Nat.add_zero, Nat.zero_add] + +/-- In range, a half-line store is a splice of eight words. -/ +theorem spliceHalf_eq (O : Array UInt32) (idx : Nat) (v0 v1 v2 v3 v4 v5 v6 v7 : UInt32) + (hi : idx + 8 ≤ O.size) : + spliceHalf O idx v0 v1 v2 v3 v4 v5 v6 v7 = splice O idx #[v0, v1, v2, v3, v4, v5, v6, v7] := by + unfold spliceHalf splice + rw [ite_eq_left_of_eq_true _ _ (eq_true hi)] + rfl + +/-- A stored half line changes exactly its eight words. -/ +theorem getD_spliceHalf (O : Array UInt32) (idx : Nat) (g : Nat → UInt32) (hi : idx + 8 ≤ O.size) + (n : Nat) : + (spliceHalf O idx (g 0) (g 1) (g 2) (g 3) (g 4) (g 5) (g 6) (g 7)).getD n 0 = + if idx ≤ n ∧ n < idx + 8 then g (n - idx) else O.getD n 0 := by + rw [spliceHalf_eq _ _ _ _ _ _ _ _ _ _ hi, getD_splice _ _ _ (by + simp only [List.size_toArray, List.length_cons, List.length_nil]; omega)] + simp only [List.size_toArray, List.length_cons, List.length_nil, Nat.reduceAdd] + by_cases h1 : n < idx + · rw [ite_eq_left_of_eq_true _ _ (eq_true h1), + ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + · rw [ite_eq_right_of_eq_false _ _ (eq_false h1)] + by_cases h2 : n < idx + 8 + · rw [ite_eq_left_of_eq_true _ _ (eq_true h2), ite_eq_left_of_eq_true _ _ (eq_true ⟨by omega, + h2⟩)] + obtain ⟨k, rfl, hk⟩ : ∃ k, n = idx + k ∧ k < 8 := ⟨n - idx, by omega, by omega⟩ + rw [Nat.add_sub_cancel_left] + interval_cases k <;> rfl + · rw [ite_eq_right_of_eq_false _ _ (eq_false h2), + ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + +@[simp] theorem size_lineStores (O : Array UInt32) (mm c half : Nat) (val : Nat → Nat → UInt32) : + (lineStores O mm c half val).size = O.size := by + unfold lineStores + suffices h : ∀ (l : List Nat) (A : Array UInt32), A.size = O.size → + (l.foldl (fun O t ↦ spliceHalf O (16 * NTT.Transform.bitRevNat mm (16 * c + t) + 8 * half) + (val t 0) (val t 1) (val t 2) (val t 3) (val t 4) (val t 5) (val t 6) (val t 7)) + A).size = O.size from h _ O rfl + intro l + induction l with + | nil => intro A hA; exact hA + | cons t l ih => intro A hA; exact ih _ (by rw [size_spliceHalf, hA]) + +/-- One kernel call: word `n` is set iff its line reverses into batch `c` and its half is +`half`; then it is lane `n % 8` at coordinate `bitrev (n / 16) % 16`. -/ +theorem getD_lineStores (O : Array UInt32) (mm c half : Nat) (val : Nat → Nat → UInt32) + (hO : O.size = 16 * 2 ^ mm) (hc : 16 * c + 16 ≤ 2 ^ mm) (hh : half < 2) (n : Nat) : + (lineStores O mm c half val).getD n 0 = + if n < 16 * 2 ^ mm ∧ NTT.Transform.bitRevNat mm (n / 16) / 16 = c ∧ n % 16 / 8 = half then + val (NTT.Transform.bitRevNat mm (n / 16) % 16) (n % 8) + else O.getD n 0 := by + unfold lineStores + suffices h : ∀ T ≤ 16, + ((List.range T).foldl (fun O t ↦ spliceHalf O + (16 * NTT.Transform.bitRevNat mm (16 * c + t) + 8 * half) + (val t 0) (val t 1) (val t 2) (val t 3) (val t 4) (val t 5) (val t 6) (val t 7)) + O).getD n 0 = + if n < 16 * 2 ^ mm ∧ NTT.Transform.bitRevNat mm (n / 16) / 16 = c ∧ + NTT.Transform.bitRevNat mm (n / 16) % 16 < T ∧ n % 16 / 8 = half then + val (NTT.Transform.bitRevNat mm (n / 16) % 16) (n % 8) + else O.getD n 0 by + rw [h 16 le_rfl] + congr 1 + apply propext + have := Nat.mod_lt (NTT.Transform.bitRevNat mm (n / 16)) (show 0 < 16 by decide) + constructor + · intro hx; exact ⟨hx.1, hx.2.1, hx.2.2.2⟩ + · intro hx; exact ⟨hx.1, hx.2.1, this, hx.2.2⟩ + intro T + induction T with + | zero => + intro _ + simp only [List.range_zero, List.foldl_nil, Nat.not_lt_zero, false_and, and_false, + ↓reduceIte] + | succ T ih => + intro hT + rw [List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil] + have hsz : ((List.range T).foldl (fun O t ↦ spliceHalf O + (16 * NTT.Transform.bitRevNat mm (16 * c + t) + 8 * half) + (val t 0) (val t 1) (val t 2) (val t 3) (val t 4) (val t 5) (val t 6) (val t 7)) + O).size = O.size := by + have := size_lineStores O mm c half val + clear ih + induction T with + | zero => rfl + | succ T ih' => + rw [List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil, size_spliceHalf, + ih' (by omega)] + have hg := NTT.Transform.bitRevNat_lt mm (16 * c + T) + rw [getD_spliceHalf _ _ (val T) (by rw [hsz, hO]; omega), ih (by omega)] + have hinv : NTT.Transform.bitRevNat mm (NTT.Transform.bitRevNat mm (16 * c + T)) = 16 * c + T := + NTT.Transform.bitRevNat_involutive _ _ (by omega) + by_cases hn : n < 16 * 2 ^ mm ∧ NTT.Transform.bitRevNat mm (n / 16) = 16 * c + T + · have hq : n / 16 = NTT.Transform.bitRevNat mm (16 * c + T) := by + rw [← hn.2, NTT.Transform.bitRevNat_involutive _ _ (by omega)] + by_cases hhalf : n % 16 / 8 = half + · rw [ite_eq_left_of_eq_true _ _ (eq_true (by omega)), + ite_eq_left_of_eq_true _ _ (eq_true ⟨hn.1, by omega, by omega, hhalf⟩), hn.2] + congr 1 <;> omega + · rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (by omega)), + ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + · have hout : ¬(16 * NTT.Transform.bitRevNat mm (16 * c + T) + 8 * half ≤ n ∧ + n < 16 * NTT.Transform.bitRevNat mm (16 * c + T) + 8 * half + 8) := by + intro hin + apply hn + have hq : n / 16 = NTT.Transform.bitRevNat mm (16 * c + T) := by omega + exact ⟨by omega, by rw [hq, hinv]⟩ + rw [ite_eq_right_of_eq_false _ _ (eq_false hout)] + by_cases hn2 : n < 16 * 2 ^ mm ∧ NTT.Transform.bitRevNat mm (n / 16) / 16 = c + · have hne : NTT.Transform.bitRevNat mm (n / 16) ≠ 16 * c + T := fun h ↦ hn ⟨hn2.1, h⟩ + congr 1 + apply propext + constructor + · intro hx; exact ⟨hx.1, hx.2.1, by omega, hx.2.2.2⟩ + · intro hx; exact ⟨hx.1, hx.2.1, by omega, hx.2.2.2⟩ + · rw [ite_eq_right_of_eq_false _ _ (eq_false (by tauto)), + ite_eq_right_of_eq_false _ _ (eq_false (by tauto))] + +/-- A lane's word index. -/ +theorem laneIndex (v Lw c : USize) (hv : v.toNat < 16) (hc : 16 * c.toNat + 16 ≤ Lw.toNat) + (hL : 16 * Lw.toNat < USize.size) : + (v * Lw + 16 * c).toNat = v.toNat * Lw.toNat + 16 * c.toNat := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hvL : v.toNat * Lw.toNat ≤ 15 * Lw.toNat := Nat.mul_le_mul_right _ (by omega) + simp only [USize.toNat_add, USize.toNat_mul, USize.reduceToNat, hsize] + rw [Nat.mod_eq_of_lt (show v.toNat * Lw.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show 16 * c.toNat < USize.size by omega), + Nat.mod_eq_of_lt (show v.toNat * Lw.toNat + 16 * c.toNat < USize.size by omega)] + +/-- Lane `k` of half `0` reads block `bitrev₄ k`. -/ +theorem pick8_lanes0 (Lw c : USize) (hc : 16 * c.toNat + 16 ≤ Lw.toNat) + (hL : 16 * Lw.toNat < USize.size) (k : Nat) (hk : k < 8) : + (pick8 (0 * Lw + 16 * c) (8 * Lw + 16 * c) (4 * Lw + 16 * c) (12 * Lw + 16 * c) + (2 * Lw + 16 * c) + (10 * Lw + 16 * c) (6 * Lw + 16 * c) (14 * Lw + 16 * c) k).toNat = + NTT.Transform.bitRevNat 4 k * Lw.toNat + 16 * c.toNat := by + interval_cases k <;> simp only [pick8] <;> + rw [laneIndex _ _ _ (by simp only [USize.reduceToNat]; omega) hc hL] <;> + simp only [USize.reduceToNat] <;> rfl + +/-- Lane `k` of half `1` reads block `bitrev₄ (8 + k)`. -/ +theorem pick8_lanes1 (Lw c : USize) (hc : 16 * c.toNat + 16 ≤ Lw.toNat) + (hL : 16 * Lw.toNat < USize.size) (k : Nat) (hk : k < 8) : + (pick8 (1 * Lw + 16 * c) (9 * Lw + 16 * c) (5 * Lw + 16 * c) (13 * Lw + 16 * c) + (3 * Lw + 16 * c) + (11 * Lw + 16 * c) (7 * Lw + 16 * c) (15 * Lw + 16 * c) k).toNat = + NTT.Transform.bitRevNat 4 (8 + k) * Lw.toNat + 16 * c.toNat := by + interval_cases k <;> simp only [pick8] <;> + rw [laneIndex _ _ _ (by simp only [USize.reduceToNat]; omega) hc hL] <;> + simp only [USize.reduceToNat] <;> rfl + +/-- Coordinate `t` of the four final DIF layers of the block at `i`. -/ +def lastVal (t3 t2 t1 U : Array (FastField p)) (i t : Nat) : FastField p := + (Expressions.leaf16Stages t3 t2 t1 U i).getD (i + t) 0 + +/-- The natural-order word `w` of a fused leaf: line `w / 16` reverses to block column +`r / 16` and coordinate `r % 16`, and position `w % 16` reverses to block `bitrev₄ (w % 16)`. -/ +def lastTarget (t3 t2 t1 U : Array (FastField p)) (mm : Nat) (f : FastField p) + (sc : Bool) (w : Nat) : UInt32 := + let r := NTT.Transform.bitRevNat mm (w / 16) + Native.scaleWord p f.val sc (lastVal t3 t2 t1 U + (NTT.Transform.bitRevNat 4 (w % 16) * 2 ^ mm + 16 * (r / 16)) (r % 16)).val + +/-- The batch loop: after batches `c, …, c + n - 1`, exactly the words whose lines reverse into +them hold their final values. -/ +theorem lastRevGo_encodeWords (t3 t2 t1 U : Array (FastField p)) (mm : Nat) (Lw : USize) + (f : FastField p) (sc : Bool) (hb) (hL : Lw.toNat = 2 ^ mm) (hm : 0 < mm) + (hm32 : mm < 32) (hu : 64 * 2 ^ mm < USize.size) (hU : U.size = 16 * 2 ^ mm) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) : + ∀ (n : Nat) (c : USize) (O : Array UInt32), O.size = 16 * 2 ^ mm → + 16 * (c.toNat + n) ≤ 2 ^ mm → + ∃ O', Native.lastRevGo p (encode t3) (encode t2) (encode t1) (encode U) Lw + (32 - mm).toUInt32 f.val sc hb n c (encode O) = encode O' ∧ + O'.size = O.size ∧ ∀ w, O'.getD w 0 = + if w < 16 * 2 ^ mm ∧ c.toNat ≤ NTT.Transform.bitRevNat mm (w / 16) / 16 ∧ + NTT.Transform.bitRevNat mm (w / 16) / 16 < c.toNat + n then + lastTarget t3 t2 t1 U mm f sc w + else O.getD w 0 := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + intro n + induction n with + | zero => + intro c O _ _ + refine ⟨O, rfl, rfl, fun w ↦ ?_⟩ + rw [ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + | succ n ih => + intro c O hO hcn + have hc : 16 * c.toNat + 16 ≤ Lw.toNat := by rw [hL]; omega + have hc' : 16 * c.toNat + 16 ≤ 2 ^ mm := by omega + have hLu : 16 * Lw.toNat < USize.size := by rw [hL]; omega + have hs : (encode O).size < USize.size := by rw [size_encode_uint32, hO]; omega + rw [Native.lastRevGo, dite_eq_left_of_eq_true (eq_true hc)] + dsimp only + rw [lastHalf_encodeWords t3 t2 t1 U _ _ _ _ _ _ _ _ c 0 mm f sc O hm hm32 + (by simp only [USize.reduceToNat]; omega) hc' hu hO hs] + rw [lastHalf_encodeWords t3 t2 t1 U _ _ _ _ _ _ _ _ c 1 mm f sc _ hm hm32 + (by simp only [USize.reduceToNat]; omega) hc' hu (by rw [size_lineStores, hO]) + (by rw [size_encode_uint32, size_lineStores, hO]; omega)] + have hc1 : (c + 1).toNat = c.toNat + 1 := Plan.usize_add_one c (by omega) + let V0 := fun t k ↦ Native.scaleWord p f.val sc ((laneLine t3 t2 t1 + (pick8 (0 * Lw + 16 * c) (8 * Lw + 16 * c) (4 * Lw + 16 * c) (12 * Lw + 16 * c) + (2 * Lw + 16 * c) (10 * Lw + 16 * c) (6 * Lw + 16 * c) (14 * Lw + 16 * c) k) U).get t).val + let V1 := fun t k ↦ Native.scaleWord p f.val sc ((laneLine t3 t2 t1 + (pick8 (1 * Lw + 16 * c) (9 * Lw + 16 * c) (5 * Lw + 16 * c) (13 * Lw + 16 * c) + (3 * Lw + 16 * c) (11 * Lw + 16 * c) (7 * Lw + 16 * c) (15 * Lw + 16 * c) k) U).get t).val + obtain ⟨O', hO'1, hO'2, hO'3⟩ := ih (c + 1) + (lineStores (lineStores O mm c.toNat (USize.toNat 0) V0) mm c.toNat (USize.toNat 1) V1) + (by rw [size_lineStores, size_lineStores, hO]) (by rw [hc1]; omega) + refine ⟨O', hO'1, by rw [hO'2, size_lineStores, size_lineStores], fun w ↦ ?_⟩ + rw [hO'3 w, hc1] + simp only [USize.reduceToNat] + rw [getD_lineStores _ _ _ _ _ (by rw [size_lineStores, hO]) hc' (by decide), + getD_lineStores _ _ _ _ _ hO hc' (by decide)] + have hlane (h : Nat) (i : USize) (hi : i.toNat = NTT.Transform.bitRevNat 4 (8 * h + w % 8) * + 2 ^ mm + 16 * c.toNat) (hh : h < 2) (hwh : w % 16 / 8 = h) + (hr : NTT.Transform.bitRevNat mm (w / 16) / 16 = c.toNat) : + Native.scaleWord p f.val sc ((laneLine t3 t2 t1 i U).get + (NTT.Transform.bitRevNat mm (w / 16) % 16)).val = lastTarget t3 t2 t1 U mm f sc w := by + have hb4 := NTT.Transform.bitRevNat_lt 4 (8 * h + w % 8) + have hb4' : NTT.Transform.bitRevNat 4 (8 * h + w % 8) * 2 ^ mm ≤ 15 * 2 ^ mm := + Nat.mul_le_mul_right _ (by omega) + rw [laneLine_get t3 t2 t1 i U (by rw [hi, hU]; omega) h3 h2 h1 _ (Nat.mod_lt _ (by decide))] + unfold lastTarget lastVal + dsimp only + rw [hi, show 8 * h + w % 8 = w % 16 by omega, hr] + set r := NTT.Transform.bitRevNat mm (w / 16) with hr0 + by_cases hA : w < 16 * 2 ^ mm ∧ c.toNat + 1 ≤ r / 16 ∧ r / 16 < c.toNat + 1 + n + · rw [ite_eq_left_of_eq_true _ _ (eq_true hA), ite_eq_left_of_eq_true _ _ + (eq_true (show w < 16 * 2 ^ mm ∧ c.toNat ≤ r / 16 ∧ r / 16 < c.toNat + (n + 1) by + omega))] + rw [ite_eq_right_of_eq_false _ _ (eq_false hA)] + by_cases hB : w < 16 * 2 ^ mm ∧ r / 16 = c.toNat ∧ w % 16 / 8 = 1 + · rw [ite_eq_left_of_eq_true _ _ (eq_true hB), ite_eq_left_of_eq_true _ _ + (eq_true (show w < 16 * 2 ^ mm ∧ c.toNat ≤ r / 16 ∧ r / 16 < c.toNat + (n + 1) by + omega))] + exact hlane 1 _ (by rw [pick8_lanes1 Lw c hc hLu _ (Nat.mod_lt _ (by decide)), hL]) + (by decide) hB.2.2 hB.2.1 + rw [ite_eq_right_of_eq_false _ _ (eq_false hB)] + by_cases hC : w < 16 * 2 ^ mm ∧ r / 16 = c.toNat ∧ w % 16 / 8 = 0 + · rw [ite_eq_left_of_eq_true _ _ (eq_true hC), ite_eq_left_of_eq_true _ _ + (eq_true (show w < 16 * 2 ^ mm ∧ c.toNat ≤ r / 16 ∧ r / 16 < c.toNat + (n + 1) by + omega))] + exact hlane 0 _ (by rw [pick8_lanes0 Lw c hc hLu _ (Nat.mod_lt _ (by decide)), hL, + Nat.mul_zero, Nat.zero_add]) + (by decide) hC.2.2 hC.2.1 + rw [ite_eq_right_of_eq_false _ _ (eq_false hC), ite_eq_right_of_eq_false _ _ + (eq_false (show ¬(w < 16 * 2 ^ mm ∧ c.toNat ≤ r / 16 ∧ r / 16 < c.toNat + (n + 1)) by + omega))] + +/-- The fused final layers of a leaf, in natural order. -/ +theorem lastRev_encodeWords (t3 t2 t1 U : Array (FastField p)) (m : Nat) (f : + FastField p) + (sc : Bool) (hm : 8 ≤ m) (hm36 : m < 36) (hu : 4 * 2 ^ m < USize.size) (hU : U.size = 2 ^ m) + (ht3 : t3.size = 8) (ht2 : t2.size = 4) (ht1 : t1.size = 2) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) : + Native.lastRev p (encode t3) (encode t2) (encode t1) (encode U) m f.val sc = + encode (Array.ofFn (n := 2 ^ m) fun w ↦ lastTarget t3 t2 t1 U (m - 4) f sc w) := by + have hp : 2 ^ m = 16 * 2 ^ (m - 4) := by + rw [show m = m - 4 + 4 by omega, Nat.pow_add]; simp only [Nat.add_sub_cancel]; omega + have hpos := Nat.two_pow_pos (m - 4) + have h16 : 16 ∣ 2 ^ (m - 4) := by + rw [show m - 4 = 4 + (m - 4 - 4) by omega, Nat.pow_add]; exact Nat.dvd_mul_right _ _ + have hg : 2 ^ (m - 4) < USize.size ∧ 64 * 2 ^ (m - 4) ≤ (encode U).size ∧ + (encode U).size < USize.size ∧ 31 < (encode t3).size ∧ + (encode t3).size < USize.size ∧ 15 < (encode t2).size ∧ + (encode t2).size < USize.size ∧ 7 < (encode t1).size ∧ + (encode t1).size < USize.size := by + simp only [size_encode_fields, hU, ht3, ht2, ht1]; omega + unfold Native.lastRev + rw [dite_eq_left_of_eq_true (eq_true hg)] + obtain ⟨O', hO'1, hO'2, hO'3⟩ := lastRevGo_encodeWords t3 t2 t1 U (m - 4) + (USize.ofNatLT (2 ^ (m - 4)) hg.1) f sc _ (by simp only [USize.toNat_ofNatLT]) (by omega) + (by omega) (by omega) (by omega) h3 h2 h1 (2 ^ (m - 4) / 16) 0 (U.map Subtype.val) + (by rw [Array.size_map, hU, hp]) (by simp only [USize.toNat_zero, Nat.zero_add]; omega) + conv_lhs => arg 13; rw [encode_eq_map_val] + rw [hO'1] + congr 1 + apply array_eq_of_getD _ _ 0 (by rw [hO'2, Array.size_map, Array.size_ofFn, hU]) + intro w + rw [hO'3 w] + by_cases hw : w < 16 * 2 ^ (m - 4) + · have hr := NTT.Transform.bitRevNat_lt (m - 4) (w / 16) + rw [ite_eq_left_of_eq_true _ _ (eq_true ⟨hw, by simp only [USize.toNat_zero]; omega, + by simp only [USize.toNat_zero, Nat.zero_add]; omega⟩), + getD_ofFn_bounded _ _ (by omega)] + · rw [ite_eq_right_of_eq_false _ _ (eq_false (fun h ↦ hw h.1))] + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_map, + Array.getElem?_eq_none (show U.size ≤ w by omega), Option.map_none, Option.getD_none, + Array.getElem?_eq_none (show (Array.ofFn (n := 2 ^ m) fun w ↦ + lastTarget t3 t2 t1 U (m - 4) f sc w).size ≤ w by rw [Array.size_ofFn]; omega)] + +/-- The four final layers of a block leave the rest of the array unchanged. -/ +theorem getD_leaf16Stages_outside (t3 t2 t1 a : Array (FastField p)) (base : Nat) + (hs : base + 16 ≤ a.size) (k : Nat) (hk : ¬(base ≤ k ∧ k < base + 16)) : + (Expressions.leaf16Stages t3 t2 t1 a base).getD k 0 = a.getD k 0 := by + have s8 : base + 16 ≤ (layer16_8 t3 a base).size := by rw [size_layer16_8]; exact hs + have s4 : base + 16 ≤ (layer16_4 t2 (layer16_8 t3 a base) base).size := by + rw [size_layer16_4]; exact s8 + have s2 : base + 16 ≤ (layer16_2 t1 (layer16_4 t2 (layer16_8 t3 a base) base) base).size := by + rw [size_layer16_2]; exact s4 + unfold Expressions.leaf16Stages + rw [getD_layer16_1_outside _ _ _ s2 _ hk, getD_layer16_2_outside _ _ _ s4 _ hk, + getD_layer16_4_outside _ _ _ s8 _ hk, getD_layer16_8_outside _ _ _ hs _ hk] + +/-- Inside its block, the four final layers depend only on the block's coordinates. -/ +theorem lastVal_congr (t3 t2 t1 a a' : Array (FastField p)) (base : Nat) + (hs : base + 16 ≤ a.size) (hs' : base + 16 ≤ a'.size) (hu : a.size < USize.size) + (hagree : ∀ o < 16, a.getD (base + o) 0 = a'.getD (base + o) 0) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) (t : Nat) + (ht : t < 16) : lastVal t3 t2 t1 a base t = lastVal t3 t2 t1 a' base t := by + have hb : (USize.ofNatLT base (by omega)).toNat = base := USize.toNat_ofNatLT .. + unfold lastVal + rw [← hb, ← laneLine_get t3 t2 t1 _ a (by rw [hb]; omega) h3 h2 h1 t ht, + ← laneLine_get t3 t2 t1 _ a' (by rw [hb]; omega) h3 h2 h1 t ht] + have he : laneLine t3 t2 t1 (USize.ofNatLT base (by omega)) a = + laneLine t3 t2 t1 (USize.ofNatLT base (by omega)) a' := by + have hagree' : ∀ o : USize, o.toNat < 16 → + a.getD ((USize.ofNatLT base (by omega)).toNat + o.toNat) 0 = + a'.getD ((USize.ofNatLT base (by omega)).toNat + o.toNat) 0 := by + intro o ho; rw [hb]; exact hagree _ ho + unfold laneLine readField + simp (disch := (simp only [USize.reduceToNat]; omega)) only [hagree'] + rw [he] + +/-- The fold of final-layer blocks, block by block. -/ +theorem getD_fold_leafBlocks (t3 t2 t1 U : Array (FastField p)) (B : Nat) + (hs : U.size = 16 * B) (hu : U.size < USize.size) + (h3 : t3.getD 0 0 = 1) (h2 : t2.getD 0 0 = 1) (h1 : t1.getD 0 0 = 1) : + ∀ K ≤ B, + ((List.range K).foldl (fun b block ↦ Expressions.leaf16Stages t3 t2 t1 b (block * 16)) + U).size = U.size ∧ ∀ k < B, ∀ t < 16, + ((List.range K).foldl (fun b block ↦ Expressions.leaf16Stages t3 t2 t1 b (block * 16)) + U).getD (16 * k + t) 0 = + if k < K then lastVal t3 t2 t1 U (16 * k) t else U.getD (16 * k + t) 0 := by + intro K + induction K with + | zero => + intro _ + refine ⟨rfl, fun k _ t _ ↦ ?_⟩ + rw [ite_eq_right_of_eq_false _ _ (eq_false (Nat.not_lt_zero k))] + rfl + | succ K ih => + intro hK + obtain ⟨hsz, hv⟩ := ih (by omega) + rw [List.range_succ, List.foldl_append, List.foldl_cons, List.foldl_nil] + refine ⟨by rw [Expressions.size_leaf16Stages, hsz], fun k hk t ht ↦ ?_⟩ + by_cases hkK : k = K + · subst k + rw [ite_eq_left_of_eq_true _ _ (eq_true (Nat.lt_succ_self K))] + rw [show 16 * K + t = K * 16 + t by omega] + change lastVal t3 t2 t1 _ (K * 16) t = _ + rw [Nat.mul_comm 16 K] + apply lastVal_congr _ _ _ _ _ _ (by rw [hsz]; omega) (by omega) (by rw [hsz]; exact hu) + _ h3 h2 h1 t ht + intro o ho + have := hv K (by omega) o ho + rw [Nat.mul_comm 16 K, ite_eq_right_of_eq_false _ _ (eq_false (Nat.lt_irrefl K))] at this + exact this + · rw [getD_leaf16Stages_outside _ _ _ _ _ (by rw [hsz]; omega) _ (by omega), hv k hk t ht] + by_cases hk2 : k < K + · rw [ite_eq_left_of_eq_true _ _ (eq_true hk2), + ite_eq_left_of_eq_true _ _ (eq_true (by omega))] + · rw [ite_eq_right_of_eq_false _ _ (eq_false hk2), + ite_eq_right_of_eq_false _ _ (eq_false (by omega))] + +/-- The fused final layers and reversal of a leaf are the natural-order read of its last two +radix-four passes, as sixteen-word blocks. -/ +theorem Native.lastRev_encode (t3 t2 t1 U : Array (FastField p)) (m : Nat) + (f : FastField p) (sc : Bool) (hm : 8 ≤ m) (hm32 : m ≤ 32) + (hu : 4 * 2 ^ m < USize.size) (hUs : U.size = 2 ^ m) + (h3 : t3.size = 8) (h2 : t2.size = 4) (h1 : t1.size = 2) + (o3 : t3.getD 0 0 = 1) (o2 : t2.getD 0 0 = 1) (o1 : t1.getD 0 0 = 1) : + Native.lastRev p (encode t3) (encode t2) (encode t1) (encode U) m f.val sc = + encode (decodedFields m ((List.range (2 ^ m / 16)).foldl (fun b block ↦ + Expressions.leaf16Stages t3 t2 t1 b (block * 16)) U) f sc) := by + rw [lastRev_encodeWords _ _ _ U m f sc hm (by omega) hu hUs h3 h2 h1 o3 o2 o1] + let W := (List.range (2 ^ m / 16)).foldl (fun b block ↦ + Expressions.leaf16Stages t3 t2 t1 b (block * 16)) U + have hp : 2 ^ m = 16 * (2 ^ m / 16) := by + have : 16 ∣ 2 ^ m := by + rw [show m = 4 + (m - 4) by omega, Nat.pow_add]; exact Nat.dvd_mul_right _ _ + omega + have hfold := getD_fold_leafBlocks t3 t2 t1 U + (2 ^ m / 16) (by rw [hUs]; exact hp) (by omega) o3 o2 + o1 (2 ^ m / 16) le_rfl + have hWs : W.size = 2 ^ m := by rw [hfold.1, hUs] + simp only [encode_eq_map_val] at * + congr 1 + apply array_eq_of_getD _ _ 0 (by + rw [Array.size_ofFn, Array.size_map, decodedFields, Array.size_ofFn]) + intro w + by_cases hw : w < 2 ^ m + · rw [getD_ofFn_bounded _ _ hw, getD_map_val, decodedFields, getD_ofFn_bounded _ _ hw] + unfold lastTarget + dsimp only + have h4 : m - 4 + 4 = m := by omega + have hwm : w % 16 < 2 ^ 4 := Nat.mod_lt _ (by decide) + have hcat := bitRevNat_concat (m - 4) 4 (w / 16) (w % 16) hwm + rw [h4, show 2 ^ 4 * (w / 16) + w % 16 = w by omega] at hcat + rw [hcat] + generalize hv : NTT.Transform.bitRevNat 4 (w % 16) = v + generalize hr : NTT.Transform.bitRevNat (m - 4) (w / 16) = r + have hvl : v < 16 := by rw [← hv]; exact NTT.Transform.bitRevNat_lt 4 _ + have hrl : r < 2 ^ (m - 4) := by rw [← hr]; exact NTT.Transform.bitRevNat_lt _ _ + have hL : 2 ^ (m - 4) = 16 * (2 ^ (m - 4) / 16) := by + have : 16 ∣ 2 ^ (m - 4) := by + rw [show m - 4 = 4 + (m - 4 - 4) by omega, Nat.pow_add]; exact Nat.dvd_mul_right _ _ + omega + have hB : 2 ^ m / 16 = 2 ^ (m - 4) := by + rw [show m = m - 4 + 4 by omega, Nat.pow_add]; simp only [Nat.add_sub_cancel]; omega + generalize hq : 2 ^ (m - 4) / 16 = q at hL + have hk : v * q + r / 16 < 2 ^ m / 16 := by + have : v * q + q ≤ 16 * q := by + have := Nat.mul_le_mul_right q (show v + 1 ≤ 16 by omega) + rw [Nat.add_mul, Nat.one_mul] at this; exact this + omega + have e1 : 2 ^ (m - 4) * v + r = 16 * (v * q + r / 16) + r % 16 := by + rw [hL]; have := Nat.div_add_mod r 16; nlinarith + have e2 : v * 2 ^ (m - 4) + 16 * (r / 16) = 16 * (v * q + r / 16) := by + rw [hL]; ring + rw [e1, e2, hfold.2 _ hk _ (Nat.mod_lt _ (by decide)), + ite_eq_left_of_eq_true _ _ (eq_true hk)] + cases sc <;> simp only [scaleWord, Bool.false_eq_true, ↓reduceIte, mul_val] + · rw [Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by rw [Array.size_ofFn]; omega), + Array.getD_eq_getD_getElem?, Array.getElem?_eq_none (by + rw [Array.size_map, decodedFields, Array.size_ofFn]; omega)] + +/-- The block loop packs the ordinary radix-four blocks. -/ +theorem Native.radix4Blocks_encode (th tl : Array (FastField p)) (q blocks : Nat) + (a : Array (FastField p)) + (ha : (encode a).size < USize.size) + (hth : (encode th).size < USize.size) (htl : (encode tl).size < USize.size) + (hthi : 2 * q ≤ th.size) (htli : q ≤ tl.size) (hbound : blocks * (4 * q) ≤ a.size) : + Native.radix4Blocks p (encode th) (encode tl) q blocks (encode a) = + encode (Plan.butterflyDIFRadix4Blocks th tl (4 * q) q blocks 0 a) := by + unfold Native.radix4Blocks + split + · rename_i h + rw [Native.blocks16Loop_encode_pass th tl (q / 16) _ (by rw [USize.toNat_ofNatLT]; omega), + USize.toNat_ofNatLT] + · simp only [Std.Legacy.Range.forIn_eq_forIn_range', Std.Legacy.Range.size, + Nat.sub_zero, Nat.add_sub_cancel, Nat.div_one, ← List.range_eq_range', + List.forIn_pure_yield_eq_foldl, bind_pure, Id.run_pure] + exact Native.fold_inner16_encode th tl q blocks a ha hth htl hthi htli hbound + +/-- The field-array semantics of `leafLayers`. -/ +def fieldLayers (tw : Array (Array (FastField p))) (m count : Nat) + (X : Array (FastField p)) : Array (FastField p) := + let X := if m % 2 = 1 then + Plan.butterflyDIFInner (tw.getD (m - 1) #[]) (2 ^ (m - 1)) 0 0 (2 ^ (m - 1)) X else X + (List.range count).foldl (fun b pass ↦ + let high := m - 1 - m % 2 - 2 * pass + let q := 2 ^ (high - 1) + Plan.butterflyDIFRadix4Blocks (tw.getD high #[]) (tw.getD (high - 1) #[]) (4 * q) q + (2 ^ m / (4 * q)) 0 b) X + +@[simp] theorem size_fieldLayers (tw : Array (Array (FastField p))) (m count : Nat) + (X : Array (FastField p)) : (fieldLayers tw m count X).size = X.size := by + unfold fieldLayers + apply foldl_invariant _ (fun b : Array (FastField p) ↦ b.size = X.size) + · intro _ _ b hb + rw [Plan.size_butterflyDIFRadix4Blocks, hb] + · split <;> simp only [Plan.size_butterflyDIFInner] + +/-- The leading leaf layers on packed words agree with their field-array semantics. -/ +theorem Native.leafLayers_encode (tw : Array (Array (FastField p))) (m count : Nat) + (X : Array (FastField p)) (hc : count ≤ m / 2) (htw : TwiddleSizes m tw) + (hs : X.size = 2 ^ m) (hu : 4 * 2 ^ m < USize.size) : + Native.leafLayers p (tw.map encode) m count (encode X) = encode (fieldLayers tw m count X) := by + have hX : (if m % 2 = 1 then + Native.pairs p ((tw.map encode).getD (m - 1) .empty) (2 ^ (m - 1)) 0 0 + (2 ^ (m - 1)) (encode X) else encode X) = encode (if m % 2 = 1 then + Plan.butterflyDIFInner (tw.getD (m - 1) #[]) (2 ^ (m - 1)) 0 0 (2 ^ (m - 1)) X else X) := by + split + · rename_i ho + have hm : 2 ^ m = 2 * 2 ^ (m - 1) := by + rw [← Nat.pow_succ']; congr 1; omega + rw [getD_map_encode] + exact Native.pairs_encode _ X _ 0 0 _ (by rwa [size_encode_fields, hs]) + (by rw [size_encode_fields, htw _ (by omega)]; omega) + (by rw [htw _ (by omega)]) (Nat.zero_le _) (by omega) (by omega) + · rfl + unfold Native.leafLayers fieldLayers + dsimp only + rw [hX] + apply foldl_encode (List.range count) (fun b ↦ b.size = 2 ^ m) + · intro pass hp b hb + have hp' := List.mem_range.mp hp + have hh : 1 ≤ m - 1 - m % 2 - 2 * pass ∧ m - 1 - m % 2 - 2 * pass < m := by omega + have hq : 2 ^ (m - 1 - m % 2 - 2 * pass) = 2 * 2 ^ (m - 1 - m % 2 - 2 * pass - 1) := by + rw [← Nat.pow_succ']; congr 1; omega + have hle : 2 ^ (m - 1 - m % 2 - 2 * pass) ≤ 2 ^ m := Nat.pow_le_pow_right (by omega) (by omega) + rw [getD_map_encode, getD_map_encode] + exact Native.radix4Blocks_encode _ _ _ _ b (by rwa [size_encode_fields, hb]) + (by rw [size_encode_fields, htw _ hh.2]; omega) + (by rw [size_encode_fields, htw _ (by omega)]; omega) + (by rw [htw _ hh.2]; omega) (by rw [htw _ (by omega)]) + (by rw [hb]; exact Nat.div_mul_le_self _ _) + · intro _ _ b hb + rw [Plan.size_butterflyDIFRadix4Blocks, hb] + · split <;> simp only [Plan.size_butterflyDIFInner, hs] + +/-- All leaf layers are the mixed radix-four DIF stages. -/ +theorem fieldLayers_eq (D : NTT.Domain (FastField p)) + (tw : Array (Array (FastField p))) (X : Array (FastField p)) (hs : X.size = D.n) : + fieldLayers tw D.logN (D.logN / 2) X = Plan.runStagesDIFRadix4WithTwiddles D tw X := by + have hpass (high : Nat) (b : Array (FastField p)) : + Plan.butterflyDIFRadix4Blocks (tw.getD high #[]) (tw.getD (high - 1) #[]) + (4 * 2 ^ (high - 1)) (2 ^ (high - 1)) (2 ^ D.logN / (4 * 2 ^ (high - 1))) 0 b = + Plan.butterflyRadix4StageDIFWithTwiddles D (high - 1) (tw.getD high #[]) + (tw.getD (high - 1) #[]) b := by + unfold Plan.butterflyRadix4StageDIFWithTwiddles + rw [show 2 ^ (high - 1 + 2) = 4 * 2 ^ (high - 1) by rw [Nat.pow_add]; omega] + rfl + unfold fieldLayers + by_cases ho : D.logN % 2 = 1 + · rw [← Plan.runStagesDIFRadix4WithTwiddles_eq_top D tw X hs ho] + simp only [ho, ↓reduceIte] + have htop : Plan.butterflyDIFInner (tw.getD (D.logN - 1) #[]) (2 ^ (D.logN - 1)) 0 0 + (2 ^ (D.logN - 1)) X = + Plan.butterflyStageDIFWithTwiddles D (D.logN - 1) (tw.getD (D.logN - 1) #[]) X := by + have h1 : D.n / 2 ^ (D.logN - 1 + 1) = 1 := by + rw [NTT.Domain.n, show D.logN - 1 + 1 = D.logN by omega, Nat.div_self (Nat.two_pow_pos _)] + unfold Plan.butterflyStageDIFWithTwiddles + dsimp only + rw [h1, Plan.butterflyDIFBlocks] + simp only [Nat.zero_lt_one, ↓reduceIte, Nat.zero_mul, Nat.zero_add] + rw [Plan.butterflyDIFBlocks] + simp only [Nat.lt_irrefl, ↓reduceIte] + rw [htop] + apply Plan.foldl_range_congr + intro pass _ b + rw [hpass, show D.logN - 1 - 1 - 2 * pass = D.logN - 2 - 2 * pass by omega] + · simp only [ho, ↓reduceIte, Plan.runStagesDIFRadix4WithTwiddles, + Std.Legacy.Range.forIn_eq_forIn_range', Std.Legacy.Range.size, Nat.sub_zero, + Nat.add_sub_cancel, Nat.div_one, ← List.range_eq_range', List.forIn_pure_yield_eq_foldl, + pure_bind, Id.run_pure] + apply Plan.foldl_range_congr + intro pass _ b + rw [hpass, show D.logN % 2 = 0 by omega, Nat.sub_zero] + +/-- A natural-order leaf is the natural-order read of the mixed radix-four DIF stages, +optionally scaled. -/ +theorem Native.leafNatural_encode (D : NTT.Domain (FastField p)) + (tw : Array (Array (FastField p))) (X : Array (FastField p)) + (f : FastField p) (sc : Bool) (htw : TwiddleSizes D.logN tw) + (hone : TwiddleOnes D.logN tw) (hs : X.size = D.n) (hu : 4 * D.n < USize.size) + (h32 : D.logN ≤ 32) : + Native.leafNatural p (tw.map encode) D.logN f.val sc (encode X) = + encode (decodedFields D.logN (Plan.runStagesDIFRadix4WithTwiddles D tw X) f sc) := by + have hn : X.size = 2 ^ D.logN := hs + have hu' : 4 * 2 ^ D.logN < USize.size := hu + rw [← fieldLayers_eq D tw X hs] + unfold Native.leafNatural + split + · rename_i hm + have hzero : tw.getD 0 #[] = #[1] := + twiddle_zero_eq_one _ (by simpa using htw 0 (by omega)) (hone 0 (by omega)) + let U := fieldLayers tw D.logN (D.logN / 2 - 2) X + have hUs : U.size = 2 ^ D.logN := by simp only [U, size_fieldLayers, hn] + rw [Native.leafLayers_encode tw _ _ X (by omega) htw hn hu', getD_map_encode, + getD_map_encode, getD_map_encode, + Native.lastRev_encode _ _ _ U _ f sc hm h32 hu' hUs (by simpa using htw 3 (by omega)) + (by simpa using htw 2 (by omega)) (by simpa using htw 1 (by omega)) (hone 3 (by omega)) + (hone 2 (by omega)) (hone 1 (by omega)), + fold_leaf16Stages_eq_radix4 _ _ _ _ U (by rw [hUs]; exact Nat.div_mul_le_self _ _)] + congr 2 + have hrange : List.range (D.logN / 2) = + List.range (D.logN / 2 - 2) ++ [D.logN / 2 - 2, D.logN / 2 - 1] := by + obtain ⟨P, hP⟩ : ∃ P, D.logN / 2 = P + 2 := ⟨D.logN / 2 - 2, by omega⟩ + rw [hP, List.range_succ, List.range_succ, List.append_assoc] + rfl + have h3 : D.logN - 1 - D.logN % 2 - 2 * (D.logN / 2 - 2) = 3 := by omega + have h1 : D.logN - 1 - D.logN % 2 - 2 * (D.logN / 2 - 1) = 1 := by omega + have hd : 16 ∣ 2 ^ D.logN := by simpa using Nat.pow_dvd_pow 2 (show 4 ≤ D.logN by omega) + unfold fieldLayers + rw [hrange, List.foldl_append, List.foldl_cons, List.foldl_cons, List.foldl_nil] + simp only [h3, h1, Nat.reduceSub, Nat.reducePow, hzero] + rw [show 2 ^ D.logN / 4 = 4 * (2 ^ D.logN / 16) by omega] + rfl + · rw [Native.leafLayers_encode tw _ _ X (le_refl _) htw hn hu', + Native.decode_encode _ _ f sc h32 (by rw [size_fieldLayers, hn]) + (by rw [size_encode_fields, size_fieldLayers, hn]; exact hu'), + Native.encodeFields_eq] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/NativeOrder.lean b/CompPoly/Univariate/NTTFast/Packed/NativeOrder.lean new file mode 100644 index 00000000..75675286 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/NativeOrder.lean @@ -0,0 +1,112 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.Native +import CompPoly.Univariate.NTTFast.Packed.Unrolled + +/-! # Parallel natural-order output for the packed FFT + +The sixteen task leaves of the packed FFT hold a bit-reversed transform. Each leaf is +first reversed locally, in parallel. Natural order is then a sixteen-way interleave of +the reversed leaves, which parallel tasks compute as cache-line `16 × 16` transposes. +The task outputs are packed buffers, so joining them is a byte copy. +-/ + +@[expose] public section +open CompPoly +namespace CompPoly.CPolynomial.NTTFast.Packed.Native +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +/-- Apply the optional inverse normalization to one Montgomery word. -/ +@[inline] +def scaleWord (p : ℕ) [Mont32Field p] (factor : UInt32) (scale : Bool) (x : UInt32) : UInt32 := + if scale then mul p factor x else x + +/-- Sixteen consecutive words of one buffer. -/ +structure Line where + w0 : UInt32 + w1 : UInt32 + w2 : UInt32 + w3 : UInt32 + w4 : UInt32 + w5 : UInt32 + w6 : UInt32 + w7 : UInt32 + w8 : UInt32 + w9 : UInt32 + w10 : UInt32 + w11 : UInt32 + w12 : UInt32 + w13 : UInt32 + w14 : UInt32 + w15 : UInt32 + +/-- Words `q, …, q + 15` fit in `b`, checked on machine words. -/ +abbrev LineFits (b : ByteArray) (q : USize) : Prop := 16 ≤ b.usize / 4 ∧ q ≤ b.usize / 4 - 16 + +theorem lineFits_toNat {b : ByteArray} {q : USize} (h : LineFits b q) (k : Nat) (hk : k < 16) : + ((q + OfNat.ofNat k) * 4).toNat = 4 * (q.toNat + k) ∧ 4 * (q.toNat + k) + 4 ≤ b.size := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hb := toNat_usize_le b + have hu := b.usize.toNat_lt_size + have h16 : (16 : USize).toNat = 16 := usize_numeral 16 (by decide) + have h4 : (4 : USize).toNat = 4 := usize_numeral 4 (by decide) + obtain ⟨h1, h2⟩ := h + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, h16] at h1 + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ (by + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, h16]; exact h1), USize.toNat_div, h4, h16] + at h2 + have hk' : (OfNat.ofNat k : USize).toNat = k := usize_numeral k (by omega) + have hadd : (q + OfNat.ofNat k).toNat = q.toNat + k := by + rw [USize.toNat_add, hk', hsize, Nat.mod_eq_of_lt (by omega)] + refine ⟨?_, by omega⟩ + rw [USize.toNat_mul, hadd, h4, hsize, Nat.mod_eq_of_lt (by omega)] + omega + +/-- Read one aligned line of sixteen words, whose range was checked; inlining removes the +record. -/ +@[inline] def line (b : @& ByteArray) (q : USize) (h : LineFits b q) : Line := + line_words% + +/-- Append the transpose of one `16 × 16` tile: word `u` of every stream, for each `u`. +Reading whole lines first keeps sixteen equally aligned streams from evicting each other. -/ +@[noinline] def transposeStep (s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 : + @& ByteArray) (q : USize) (out : ByteArray) (p : USize) : ByteArray := + transposeStep_words% + +/-- Store `count` transposed tiles, starting at stream word `q` and output word `p`, the end of +`out`. Each tile first extends `out` by one zero block, which the tile then overwrites while it +is in cache. -/ +def transposeGo (s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 : @& ByteArray) : + Nat → USize → USize → ByteArray → ByteArray + | 0, _, _, out => out + | count + 1, q, p, out => + let n := out.size + let out := ByteArray.zeroBlock.copySlice 0 out n 1024 false + transposeGo s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 count (q + 16) (p + 256) + (transposeStep s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 q out p) + +/-- Natural-order words `16 * q` up to `16 * (q + 16 * count)` from reversed leaves `r`. +Output word `16 * j + c` is word `j` of leaf `bitrev₄ c`. -/ +def interleaveRange (r : @& Array ByteArray) (q count : Nat) : ByteArray := + transposeGo (r.getD 0 .empty) (r.getD 8 .empty) (r.getD 4 .empty) (r.getD 12 .empty) + (r.getD 2 .empty) (r.getD 10 .empty) (r.getD 6 .empty) (r.getD 14 .empty) + (r.getD 1 .empty) (r.getD 9 .empty) (r.getD 5 .empty) (r.getD 13 .empty) + (r.getD 3 .empty) (r.getD 11 .empty) (r.getD 7 .empty) (r.getD 15 .empty) + count q.toUSize 0 (ByteArray.emptyWithCapacity (1024 * count)) + +/-- Natural-order packed words from sixteen locally reversed leaves of `2 ^ (logN - 4)` words, +`12 ≤ logN`. Sixteen interleave tasks run in parallel; the join copies bytes. -/ +def naturalLeaves (r : Array ByteArray) (logN : Nat) : ByteArray := + let count := 2 ^ (logN - 4) / 256 + let blocks := (Array.range 16).map fun t ↦ Task.spawn fun _ ↦ + interleaveRange r (16 * count * t) count + blocks.foldl (fun acc t ↦ acc ++ t.get) (ByteArray.emptyWithCapacity (4 * 2 ^ logN)) + +end CompPoly.CPolynomial.NTTFast.Packed.Native diff --git a/CompPoly/Univariate/NTTFast/Packed/NativeOrderCorrectness.lean b/CompPoly/Univariate/NTTFast/Packed/NativeOrderCorrectness.lean new file mode 100644 index 00000000..d88bc68b --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/NativeOrderCorrectness.lean @@ -0,0 +1,347 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.StorageLemmas +public import CompPoly.Univariate.NTTFast.Packed.BitReversal +import all CompPoly.Univariate.NTTFast.Packed.Native +import all CompPoly.Univariate.NTTFast.Packed.NativeOrder +public import CompPoly.Univariate.NTTFast.Packed.NativeOrder +public import CompPoly.Univariate.NTTFast.Packed.Rows +import CompPoly.Univariate.NTTFast.Packed.ArrayLemmas +public import CompPoly.Univariate.NTTFast.Packed.DifSpec + +/-! # Correctness of the parallel natural-order output + +Every loop of `NativeOrder` appends sixteen words at a time. Their results are stated as +`rows`, a concatenation of sixteen-word literals with a single indexing lemma, so the +tile, leaf-reversal and join proofs reduce to index arithmetic. +-/ + +@[expose] public section +open CompPoly +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode size_encode_uint32) + +namespace Native + +/-- A line inside a packed word array that fits machine indices passes the range check. -/ +theorem lineFits_encodeWords (v : Array UInt32) (q : USize) (h : q.toNat + 16 ≤ v.size) + (hs : (encode v).size < USize.size) : LineFits (encode v) q := by + have hu : (encode v).usize.toNat = 4 * v.size := by + rw [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' hs, size_encode_uint32] + have h16 : (16 : USize).toNat = 16 := usize_numeral 16 (by decide) + have h4 : (4 : USize).toNat = 4 := usize_numeral 4 (by decide) + have h1 : 16 ≤ (encode v).usize / 4 := by + rw [USize.le_iff_toNat_le, USize.toNat_div, h4, h16, hu]; omega + refine ⟨h1, ?_⟩ + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ h1, USize.toNat_div, h4, h16, hu] + omega + +/-- One aligned line of a packed word array. -/ +theorem line_encodeWords (v : Array UInt32) (q : USize) (hl) : + line (encode v) q hl = ⟨v.getD (q.toNat + 0) 0, v.getD (q.toNat + 1) 0, v.getD (q.toNat + 2) 0, + v.getD (q.toNat + 3) 0, v.getD (q.toNat + 4) 0, v.getD (q.toNat + 5) 0, + v.getD (q.toNat + 6) 0, v.getD (q.toNat + 7) 0, v.getD (q.toNat + 8) 0, + v.getD (q.toNat + 9) 0, v.getD (q.toNat + 10) 0, v.getD (q.toNat + 11) 0, + v.getD (q.toNat + 12) 0, v.getD (q.toNat + 13) 0, v.getD (q.toNat + 14) 0, + v.getD (q.toNat + 15) 0⟩ := by + unfold line + simp only [ugetUInt32LE_eq, (lineFits_toNat hl 0 (by decide)).1, + (lineFits_toNat hl 1 (by decide)).1, (lineFits_toNat hl 2 (by decide)).1, + (lineFits_toNat hl 3 (by decide)).1, (lineFits_toNat hl 4 (by decide)).1, + (lineFits_toNat hl 5 (by decide)).1, (lineFits_toNat hl 6 (by decide)).1, + (lineFits_toNat hl 7 (by decide)).1, (lineFits_toNat hl 8 (by decide)).1, + (lineFits_toNat hl 9 (by decide)).1, (lineFits_toNat hl 10 (by decide)).1, + (lineFits_toNat hl 11 (by decide)).1, (lineFits_toNat hl 12 (by decide)).1, + (lineFits_toNat hl 13 (by decide)).1, (lineFits_toNat hl 14 (by decide)).1, + (lineFits_toNat hl 15 (by decide)).1, PackedRepr.getUInt32LE!_encode] + +/-- The `c`-th of sixteen word arrays. -/ +def sel16 (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : Array UInt32) (c : Nat) : + Array UInt32 := + #[v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15].getD c #[] + +/-- A sixteen-element literal is its own leading segment. -/ +theorem extract_literal16 (x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 : UInt32) : + (#[x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15] : Array UInt32).extract + 0 16 = #[x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15] := + Array.extract_size + +/-- A sixteen-word store at the end of a written prefix extends the prefix. -/ +theorem write16U_pack_cursor (o Z : Array UInt32) (p : USize) (hp : p.toNat = o.size) + (hZ : 16 ≤ Z.size) (hs : (encode (o ++ Z)).size < USize.size) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) : + write16U (encode (o ++ Z)) p v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 = + encode (o ++ #[v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15] ++ + Z.extract 16 Z.size) := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [size_encode_uint32, Array.size_append] at hs + unfold write16U + rw [storeWords_replace_encodeWords (o ++ Z) (p * 4) 16 o.size (by decide) + (by rw [USize.toNat_mul, usize_numeral 4 (by decide), hsize, Nat.mod_eq_of_lt (by omega)] + omega) + (by simp only [Array.size_append, show (16 : UInt8).toNat = 16 from rfl]; omega) + (by rw [size_encode_uint32, Array.size_append]; omega)] + simp only [show (16 : UInt8).toNat = 16 from rfl, extract_literal16] + have := splice_cursor o Z + #[v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15] hZ + simp only [splice, List.size_toArray, List.length_cons, List.length_nil] at this + rw [this] + +/-- Adding a small numeral to a machine index that cannot wrap. -/ +theorem usize_add_numeral (q : USize) (k : Nat) (hk : k < 4294967296) + (h : q.toNat + k < USize.size) : (q + OfNat.ofNat k).toNat = q.toNat + k := by + rw [USize.toNat_add, usize_numeral k hk, Nat.mod_eq_of_lt h] + +set_option maxHeartbeats 1000000 in +/-- One transposed tile stores rows `q, …, q + 15` of the sixteen-stream interleave at output +word `p`, the end of the written prefix. -/ +theorem transposeStep_encodeWords (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 o Z : + Array UInt32) (q p : USize) + (h : ∀ c < 16, q.toNat + 16 ≤ + (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c).size) + (hs : ∀ c < 16, + (encode (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c)).size < USize.size) + (hp : p.toNat = o.size) (hZ : 256 ≤ Z.size) (ho : (encode (o ++ Z)).size < USize.size) : + transposeStep (encode v0) (encode v1) (encode v2) (encode v3) (encode v4) (encode v5) (encode + v6) + (encode v7) (encode v8) (encode v9) (encode v10) (encode v11) (encode v12) (encode v13) + (encode v14) + (encode v15) q (encode (o ++ Z)) p = + encode (o ++ rows (fun u c ↦ + (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c).getD (q.toNat + u) 0) + 16 ++ Z.extract 256 Z.size) := by + have hsz := ho + rw [size_encode_uint32, Array.size_append] at hsz + have hf : ∀ c < 16, + LineFits (encode (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c)) q := + fun c hc ↦ lineFits_encodeWords _ q (h c hc) (hs c hc) + have hcond := And.intro (hf 0 (by decide)) <| And.intro (hf 1 (by decide)) <| + And.intro (hf 2 (by decide)) <| And.intro (hf 3 (by decide)) <| + And.intro (hf 4 (by decide)) <| And.intro (hf 5 (by decide)) <| + And.intro (hf 6 (by decide)) <| And.intro (hf 7 (by decide)) <| + And.intro (hf 8 (by decide)) <| And.intro (hf 9 (by decide)) <| + And.intro (hf 10 (by decide)) <| And.intro (hf 11 (by decide)) <| + And.intro (hf 12 (by decide)) <| And.intro (hf 13 (by decide)) <| + And.intro (hf 14 (by decide)) (hf 15 (by decide)) + unfold transposeStep + split + swap + · rename_i hneg; exact absurd hcond hneg + simp only [line_encodeWords] + rw [write16U_pack_cursor _ _ p ?hp0 ?hz0 ?hs0] + rw [write16U_pack_cursor _ _ (p + 16) ?hp1 ?hz1 ?hs1] + rw [write16U_pack_cursor _ _ (p + 32) ?hp2 ?hz2 ?hs2] + rw [write16U_pack_cursor _ _ (p + 48) ?hp3 ?hz3 ?hs3] + rw [write16U_pack_cursor _ _ (p + 64) ?hp4 ?hz4 ?hs4] + rw [write16U_pack_cursor _ _ (p + 80) ?hp5 ?hz5 ?hs5] + rw [write16U_pack_cursor _ _ (p + 96) ?hp6 ?hz6 ?hs6] + rw [write16U_pack_cursor _ _ (p + 112) ?hp7 ?hz7 ?hs7] + rw [write16U_pack_cursor _ _ (p + 128) ?hp8 ?hz8 ?hs8] + rw [write16U_pack_cursor _ _ (p + 144) ?hp9 ?hz9 ?hs9] + rw [write16U_pack_cursor _ _ (p + 160) ?hp10 ?hz10 ?hs10] + rw [write16U_pack_cursor _ _ (p + 176) ?hp11 ?hz11 ?hs11] + rw [write16U_pack_cursor _ _ (p + 192) ?hp12 ?hz12 ?hs12] + rw [write16U_pack_cursor _ _ (p + 208) ?hp13 ?hz13 ?hs13] + rw [write16U_pack_cursor _ _ (p + 224) ?hp14 ?hz14 ?hs14] + rw [write16U_pack_cursor _ _ (p + 240) ?hp15 ?hz15 ?hs15] + · simp only [extract_extract_tail, Nat.reduceAdd] + simp only [rows, lit16, Array.append_assoc, Array.empty_append] + rfl + all_goals (try rw [usize_add_numeral]) + all_goals (try simp only [size_encode_uint32, Array.size_append, Array.size_extract, + List.size_toArray, + List.length_cons, List.length_nil]) + all_goals omega + +/-- Repeated transposed tiles append consecutive interleave rows to a written prefix. -/ +theorem transposeGo_encodeWords (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : Array + UInt32) + (count : Nat) (q p : USize) (o : Array UInt32) + (h : ∀ c < 16, q.toNat + 16 * count ≤ + (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c).size) + (hs : ∀ c < 16, + (encode (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c)).size < USize.size) + (hp : p.toNat = o.size) (ho : 4 * (o.size + 256 * count) < USize.size) : + transposeGo (encode v0) (encode v1) (encode v2) (encode v3) (encode v4) (encode v5) (encode + v6) (encode v7) + (encode v8) (encode v9) (encode v10) (encode v11) (encode v12) (encode v13) (encode v14) + (encode v15) + count q p (encode o) = + encode (o ++ rows (fun u c ↦ + (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c).getD (q.toNat + u) 0) + (16 * count)) := by + induction count generalizing q p o with + | zero => simp only [transposeGo, rows, Array.append_empty] + | succ count ih => + have hq : q.toNat + 16 < USize.size := by + have h0 := h 0 (by decide) + have hs0 := hs 0 (by decide) + rw [size_encode_uint32] at hs0 + omega + have hZ : (Array.replicate 256 (0 : UInt32)).extract 256 + (Array.replicate 256 (0 : UInt32)).size = #[] := + Array.extract_eq_empty_of_le (by simp) + have hoZ : (encode (o ++ Array.replicate 256 0)).size < USize.size := by + rw [size_encode_uint32, Array.size_append, Array.size_replicate] + exact Nat.lt_of_le_of_lt (by omega) ho + rw [transposeGo, extendZeros_encodeWords, + transposeStep_encodeWords v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 + o (Array.replicate 256 0) q p (fun c hc ↦ by have := h c hc; omega) hs hp + (by rw [Array.size_replicate]) hoZ, + hZ, Array.append_empty, + ih (q + 16) (p + 256) (o ++ rows (fun u c ↦ + (sel16 v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 c).getD (q.toNat + u) 0) 16) + (fun c hc ↦ by rw [usize_add_numeral q 16 (by decide) hq]; have := h c hc; omega) + (by rw [usize_add_numeral p 256 (by decide) (by omega), Array.size_append, size_rows, + hp]) + (by rw [Array.size_append, size_rows]; omega), + show 16 * (count + 1) = 16 + 16 * count by omega, rows_add, Array.append_assoc, + usize_add_numeral q 16 (by decide) hq] + simp only [Nat.add_assoc] + +/-- The interleave streams are the reversed leaves in four-bit reversed order. -/ +theorem sel16_bitRev (V : Nat → Array UInt32) (c : Nat) (hc : c < 16) : + sel16 (V 0) (V 8) (V 4) (V 12) (V 2) (V 10) (V 6) (V 14) (V 1) (V 9) (V 5) (V 13) (V 3) + (V 11) (V 7) (V 15) c = V (NTT.Transform.bitRevNat 4 c) := by + interval_cases c <;> rfl + +/-- One interleave block: rows `q` up to `q + 16 * count` of the sixteen reversed leaves. -/ +theorem interleaveRange_encodeWords (V : Nat → Array UInt32) (M q count : Nat) + (hV : ∀ l < 16, (V l).size = M) (hq : q + 16 * count ≤ M) (hs : 64 * M < USize.size) : + interleaveRange ((Array.range 16).map fun l ↦ encode (V l)) q count = + encode (rows (fun u c ↦ (V (NTT.Transform.bitRevNat 4 c)).getD (q + u) 0) (16 * count)) := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hqu : q.toUSize.toNat = q := USize.toNat_ofNat_of_lt' (by omega) + unfold interleaveRange + simp only [getD_map_range _ 16 0 (by decide), getD_map_range _ 16 1 (by decide), + getD_map_range _ 16 2 (by decide), getD_map_range _ 16 3 (by decide), + getD_map_range _ 16 4 (by decide), getD_map_range _ 16 5 (by decide), + getD_map_range _ 16 6 (by decide), getD_map_range _ 16 7 (by decide), + getD_map_range _ 16 8 (by decide), getD_map_range _ 16 9 (by decide), + getD_map_range _ 16 10 (by decide), getD_map_range _ 16 11 (by decide), + getD_map_range _ 16 12 (by decide), getD_map_range _ 16 13 (by decide), + getD_map_range _ 16 14 (by decide), getD_map_range _ 16 15 (by decide)] + rw [show ByteArray.emptyWithCapacity (1024 * count) = encode (#[] : Array UInt32) from rfl, + transposeGo_encodeWords (V 0) (V 8) (V 4) (V 12) (V 2) (V 10) (V 6) + (V 14) (V 1) (V 9) (V 5) (V 13) (V 3) (V 11) (V 7) (V 15) count q.toUSize 0 #[] + (fun c hc ↦ by rw [sel16_bitRev V c hc, hqu, hV _ (NTT.Transform.bitRevNat_lt 4 c)]; exact hq) + (fun c hc ↦ by + rw [sel16_bitRev V c hc, size_encode_uint32, hV _ (NTT.Transform.bitRevNat_lt 4 c)]; omega) + rfl (by rw [Array.size_empty]; omega), + Array.empty_append, hqu] + congr 1 + exact rows_congr _ _ _ (fun j c hc ↦ by rw [sel16_bitRev V c hc]) + +/-- Joining the interleave blocks in task order concatenates their rows. -/ +theorem foldl_blocks (g : Nat → Nat → UInt32) (B : Nat → ByteArray) (count k : Nat) + (o : Array UInt32) + (hB : ∀ t < k, B t = encode (rows (fun u c ↦ g (16 * count * t + u) c) (16 * count))) : + (Array.range k).foldl (fun acc t ↦ acc ++ B t) (encode o) = + encode (o ++ rows g (16 * count * k)) := by + induction k with + | zero => + rw [show Array.range 0 = #[] from Array.eq_empty_of_size_eq_zero Array.size_range] + simp only [Array.foldl_empty, Nat.mul_zero, rows, Array.append_empty] + | succ k ih => + rw [Array.range_succ, Array.foldl_append, ih (fun t ht ↦ hB t (by omega))] + have hs : (#[k] : Array Nat).foldl (fun acc t ↦ acc ++ B t) + (encode (o ++ rows g (16 * count * k))) = + encode (o ++ rows g (16 * count * k)) ++ B k := by simp + rw [hs, hB k (by omega), ← PackedRepr.encode_append, Nat.mul_succ, rows_add, Array.append_assoc] + +/-- Interleaving sixteen reversed leaves: output word `16 * q + c` is word `q` of leaf +`bitrev₄ c`. -/ +theorem naturalLeaves_encodeWords (V : Nat → Array UInt32) (logN : Nat) (h12 : 12 ≤ logN) + (hV : ∀ l < 16, (V l).size = 2 ^ (logN - 4)) (hs : 4 * 2 ^ logN < USize.size) : + naturalLeaves ((Array.range 16).map fun l ↦ encode (V l)) logN = + encode (Array.ofFn (n := 2 ^ logN) fun i ↦ + (V (NTT.Transform.bitRevNat 4 (i % 16))).getD (i / 16) 0) := by + obtain ⟨m, rfl⟩ : ∃ m, logN = m + 4 := ⟨logN - 4, by omega⟩ + have hsub : m + 4 - 4 = m := by omega + have hpow : 2 ^ (m + 4) = 16 * 2 ^ m := by rw [Nat.pow_add]; omega + have hm8 : 2 ^ m = 256 * (2 ^ m / 256) := by + have : 256 ∣ 2 ^ m := by + rw [show m = 8 + (m - 8) by omega, Nat.pow_add] + exact Nat.dvd_mul_right _ _ + omega + have hV' : ∀ l < 16, (V l).size = 2 ^ m := by + intro l hl + rw [hV l hl, hsub] + unfold naturalLeaves + simp only [Task.spawn, hsub] + rw [Array.foldl_map, emptyWithCapacity_eq_encodeWords, + foldl_blocks (fun u c ↦ (V (NTT.Transform.bitRevNat 4 c)).getD u 0) _ (2 ^ m / 256) 16 #[] + (fun t ht ↦ by + rw [interleaveRange_encodeWords V (2 ^ m) _ _ hV' (by + have := Nat.mul_le_mul_left (16 * (2 ^ m / 256)) (show t + 1 ≤ 16 by omega) + rw [Nat.mul_add, Nat.mul_one] at this + omega) (by omega)]), + Array.empty_append] + congr 1 + apply array_eq_of_getD _ _ 0 (by rw [size_rows, Array.size_ofFn, hpow]; omega) + intro i + by_cases hi : i < 2 ^ (m + 4) + · rw [getD_rows _ _ _ (by omega), getD_ofFn_bounded _ _ hi] + · rw [Array.getD_eq_getD_getElem?, Array.getD_eq_getD_getElem?, + Array.getElem?_eq_none (by rw [size_rows]; omega), + Array.getElem?_eq_none (by rw [Array.size_ofFn]; omega)] + +/-- Interleaving the natural-order reads of sixteen blocks reads the whole array in natural +order. -/ +theorem naturalLeaves_encode (S : Array (FastField p)) (logN : Nat) + (factor : FastField p) (sc : Bool) (h12 : 12 ≤ logN) (h32 : logN ≤ 32) + (hS : S.size = 2 ^ logN) (hs : 4 * 2 ^ logN < USize.size) : + naturalLeaves ((Array.range 16).map fun l ↦ encode (decodedFields (logN - 4) + (S.extract (l * 2 ^ (logN - 4)) ((l + 1) * 2 ^ (logN - 4))) factor sc)) logN = + encode (decodedFields logN S factor sc) := by + obtain ⟨m, rfl⟩ : ∃ m, logN = m + 4 := ⟨logN - 4, by omega⟩ + have hsub : m + 4 - 4 = m := by omega + have hpow : 2 ^ (m + 4) = 16 * 2 ^ m := by rw [Nat.pow_add]; omega + let V : Nat → Array UInt32 := fun l ↦ (decodedFields m + (S.extract (l * 2 ^ m) ((l + 1) * 2 ^ m)) factor sc).map Subtype.val + have hV : ∀ l < 16, (V l).size = 2 ^ (m + 4 - 4) := by + intro l _ + simp only [V, Array.size_map, decodedFields, Array.size_ofFn, hsub] + have hr : ((Array.range 16).map fun l ↦ encode (decodedFields (m + 4 - 4) + (S.extract (l * 2 ^ (m + 4 - 4)) ((l + 1) * 2 ^ (m + 4 - 4))) factor sc)) = + (Array.range 16).map fun l ↦ encode (V l) := by + simp only [hsub, V, ← encode_eq_map_val] + rw [hr, naturalLeaves_encodeWords V (m + 4) h12 hV hs, encode_eq_map_val] + congr 1 + simp only [decodedFields, Array.map_ofFn] + apply Array.ext (by simp only [Array.size_ofFn]) + intro i h1 _ + simp only [Array.size_ofFn] at h1 + rw [Array.getElem_ofFn, Array.getElem_ofFn] + simp only [Function.comp_apply] + have hc := Nat.mod_lt i (show 0 < 16 by decide) + have hbr := NTT.Transform.bitRevNat_lt 4 (i % 16) + have hbq := NTT.Transform.bitRevNat_lt m (i / 16) + have hq : i / 16 < 2 ^ m := by omega + have hle : NTT.Transform.bitRevNat 4 (i % 16) * 2 ^ m + 2 ^ m ≤ S.size := by + rw [hS, hpow] + have := Nat.mul_le_mul_right (2 ^ m) (show NTT.Transform.bitRevNat 4 (i % 16) + 1 ≤ 16 by + omega) + rw [Nat.add_mul, Nat.one_mul] at this + exact this + simp only [V] + rw [getD_map_val, decodedFields, getD_ofFn_bounded _ _ hq, + show (NTT.Transform.bitRevNat 4 (i % 16) + 1) * 2 ^ m = + NTT.Transform.bitRevNat 4 (i % 16) * 2 ^ m + 2 ^ m by ring, + getD_extract _ _ _ _ _ hle hbq] + have hcat := bitRevNat_concat m 4 (i / 16) (i % 16) hc + rw [show 2 ^ 4 * (i / 16) + i % 16 = i by omega] at hcat + rw [show NTT.Transform.bitRevNat 4 (i % 16) * 2 ^ m + NTT.Transform.bitRevNat m (i / 16) = + NTT.Transform.bitRevNat (m + 4) i by rw [hcat]; ring] + +end Native +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/PassRefinement.lean b/CompPoly/Univariate/NTTFast/Packed/PassRefinement.lean new file mode 100644 index 00000000..59eeb3bc --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/PassRefinement.lean @@ -0,0 +1,105 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Correctness.Radix4DIF +public import CompPoly.Univariate.NTTFast.Packed.LoopRefinement +public import CompPoly.Univariate.NTTFast.Packed.Folds + +/-! # Packed radix-four pass refinement -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +/-- All vectorized blocks of one pass agree with the ordinary radix-four block loop. -/ +theorem Native.fold_inner16_encode (th tl : Array (FastField p)) + (q blocks : Nat) (a : Array (FastField p)) + (ha : (encode a).size < USize.size) + (hth : (encode th).size < USize.size) (htl : (encode tl).size < USize.size) + (hthi : 2 * q ≤ th.size) (htli : q ≤ tl.size) + (hbound : blocks * (4 * q) ≤ a.size) : + (List.range blocks).foldl (fun acc block ↦ + Native.inner16 p (encode th) (encode tl) q 0 (block * (4 * q)) + (block * (4 * q) + q) (block * (4 * q) + 2 * q) (block * (4 * q) + 3 * q) acc) + (encode a) = + encode (Plan.butterflyDIFRadix4Blocks th tl (4 * q) q blocks 0 a) := by + rw [Plan.butterflyDIFRadix4Blocks_eq_foldl_inner th tl (4 * q) q blocks blocks 0 a (by omega), + ← List.range_eq_range'] + apply foldl_encode (List.range blocks) (fun b ↦ b.size = a.size) + · intro block hb b hbs + have hb' : block < blocks := List.mem_range.mp hb + have hbase : block * (4 * q) + 4 * q ≤ a.size := by + calc + _ = (block + 1) * (4 * q) := by rw [Nat.add_mul, Nat.one_mul] + _ ≤ blocks * (4 * q) := Nat.mul_le_mul_right (4 * q) (by omega) + _ ≤ a.size := hbound + exact Native.inner16_encode th tl q 0 _ _ _ _ b + (by simpa only [size_encode_fields, hbs] using ha) hth htl hthi htli + (by omega) (by omega) (by omega) (by omega) + · intro block hb b hbs + simpa only [Plan.size_butterflyDIFRadix4Inner] using hbs + · rfl + +/-- The checked block loop depends only on the buffer's value. -/ +theorem Native.blocks16Loop_congr {th tl : ByteArray} {n : Nat} {q : USize} {count : Nat} + {base : USize} {b b' : ByteArray} (h) (e : b = b') : + Native.blocks16Loop p th tl n q count base b h = + Native.blocks16Loop p th tl n q count base b' (e ▸ h) := by + subst e + rfl + +/-- The block loop checked once runs the ordinary radix-four blocks `block, …`. -/ +theorem Native.blocks16Loop_encode (th tl : Array (FastField p)) (n : Nat) (q : USize) + (hq : 16 * n = q.toNat) : ∀ (count block : Nat) (base : USize) (a : Array (FastField p)) (h), + base.toNat = block * (4 * q.toNat) → + Native.blocks16Loop p (encode th) (encode tl) n q count base (encode a) h = + encode ((List.range' block count).foldl (fun acc blk ↦ + Plan.butterflyDIFRadix4Inner th tl q.toNat 0 (blk * (4 * q.toNat)) + (blk * (4 * q.toNat) + q.toNat) (blk * (4 * q.toNat) + 2 * q.toNat) + (blk * (4 * q.toNat) + 3 * q.toNat) acc) a) := by + intro count + induction count with + | zero => intro _ _ _ _ _; rfl + | succ count ih => + intro block base a h hb + have ha := h.1 + have hs := h.2.2.2.2.1 + rw [size_encode_fields] at ha hs + have hc : 4 * q.toNat ≤ 4 * q.toNat * (count + 1) := Nat.le_mul_of_pos_right _ (by omega) + have h1 : (base + q).toNat = base.toNat + q.toNat := + Native.usize_add_toNat _ _ (by omega) + have hm (k : Nat) (hk : k ≤ 4) : (OfNat.ofNat k * q : USize).toNat = k * q.toNat := + Native.usize_mul_toNat k q (by omega) (by + have := Nat.mul_le_mul_right q.toNat hk; omega) + have h2 : (base + 2 * q).toNat = base.toNat + 2 * q.toNat := by + rw [Native.usize_add_toNat _ _ (by rw [hm 2 (by decide)]; omega), hm 2 (by decide)] + have h3 : (base + 3 * q).toNat = base.toNat + 3 * q.toNat := by + rw [Native.usize_add_toNat _ _ (by rw [hm 3 (by decide)]; omega), hm 3 (by decide)] + have h4 : (base + 4 * q).toNat = base.toNat + 4 * q.toNat := by + rw [Native.usize_add_toNat _ _ (by rw [hm 4 (by decide)]; omega), hm 4 (by decide)] + rw [Native.blocks16Loop, Native.blocks16Loop_congr _ (Native.inner16Loop_encode th tl q.toNat + n 0 q base (base + q) (base + 2 * q) (base + 3 * q) a _ + (by rw [USize.toNat_zero, Nat.zero_add]) (by rw [h3]; omega) (by rw [h1]; omega) + (by rw [h1, h2]; omega) (by rw [h2, h3]; omega))] + rw [ih (block + 1) _ _ _ (by rw [h4, hb, Nat.add_mul, Nat.one_mul])] + rw [List.range'_succ, List.foldl_cons, butterflyDIFRadix4Inner_eq_quadCells, Nat.sub_zero, + USize.toNat_zero, h1, h2, h3, hb, hq] + +/-- The block loop checked once runs a whole radix-four pass. -/ +theorem Native.blocks16Loop_encode_pass (th tl : Array (FastField p)) (n : Nat) (q : USize) + (hq : 16 * n = q.toNat) (blocks : Nat) (a : Array (FastField p)) (h) : + Native.blocks16Loop p (encode th) (encode tl) n q blocks 0 (encode a) h = + encode (Plan.butterflyDIFRadix4Blocks th tl (4 * q.toNat) q.toNat blocks 0 a) := by + rw [Native.blocks16Loop_encode th tl n q hq blocks 0 0 a h (by rw [USize.toNat_zero, + Nat.zero_mul]), Plan.butterflyDIFRadix4Blocks_eq_foldl_inner th tl (4 * q.toNat) q.toNat + blocks blocks 0 a (by omega)] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Plan.lean b/CompPoly/Univariate/NTTFast/Packed/Plan.lean new file mode 100644 index 00000000..f455cbaf --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Plan.lean @@ -0,0 +1,121 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.ColumnsCorrectness + +/-! # Proved parallel transforms with packed native storage + +The transforms work over the fast carrier `FastField p` of any `Mont32Field p`. The kernels are +generic and `@[specialize]`d; `PackedNTT p` holds the kernel compiled for one modulus, so that a +field's instance in the native library (`Packed/Instances.lean`) runs loops with the modulus as +constants, whatever module calls the plan. + +The correctness theorems cover the complete Lean pipeline. Packed words are read and written +with the `ByteArray` word accessors proposed for Lean core (`CompPoly.Data.ByteArray.Pack`), +whose inline C remains a runtime trust assumption; arithmetic, tasks and ordering are Lean. +`forwardPacked`/`inversePacked` take and return packed Montgomery words: `encode` packs a field +array and `decode` reads one back. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +/-- The packed transform `Native.run p` as compiled for one modulus: an instance sets `run` to +`Native.run p` at its concrete `p`, where the compiler specializes the kernels. -/ +class PackedNTT (p : ℕ) [Mont32Field p] where + /-- The compiled transform. -/ + run : Array ByteArray → Nat → Bool → UInt32 → Bool → ByteArray → ByteArray + run_eq : run = Native.run p := by rfl + +/-- A reusable packed transform plan with certified cached twiddles and index bounds. -/ +structure Plan (p : ℕ) [Mont32Field p] where + domain : NTT.Domain (FastField p) + log_bound : domain.logN ≤ 32 + byte_bound : 4 * domain.n < USize.size + nInv : FastField p + nInv_eq : nInv = domain.nInv + forwardTwiddles : Array ByteArray + inverseTwiddles : Array ByteArray + forward_eq : forwardTwiddles = (NTTFast.Plan.twiddleTable domain).map encode + inverse_eq : inverseTwiddles = (NTTFast.Plan.twiddleTable domain.inverse).map encode + +namespace Plan + +/-- Encode both twiddle tables once; no plan construction is needed inside hot loops. -/ +def ofDomain (D : NTT.Domain (FastField p)) (h32 : D.logN ≤ 32) + (hu : 4 * D.n < USize.size) : Plan p where + domain := D + log_bound := h32 + byte_bound := hu + nInv := D.nInv + nInv_eq := rfl + forwardTwiddles := (NTTFast.Plan.twiddleTable D).map Native.encodeFields + inverseTwiddles := (NTTFast.Plan.twiddleTable D.inverse).map Native.encodeFields + forward_eq := by + apply congrArg (Array.map · _) + funext a + exact Native.encodeFields_eq a + inverse_eq := by + apply congrArg (Array.map · _) + funext a + exact Native.encodeFields_eq a + +/-- Natural-order forward transform of `n` packed Montgomery words: if `parallel`, from `2 ^ 15` +words, column tasks run the top two radix-four passes and sixteen leaf tasks the rest; otherwise, +and for smaller transforms, one leaf runs on the calling thread. -/ +@[inline] def forwardPacked [PackedNTT p] (P : Plan p) (a : ByteArray) (parallel : Bool := true) : + ByteArray := + PackedNTT.run p P.forwardTwiddles P.domain.logN parallel P.nInv.val false a + +/-- Natural-order inverse transform of `n` packed Montgomery words, including normalization. -/ +@[inline] def inversePacked [PackedNTT p] (P : Plan p) (a : ByteArray) (parallel : Bool := true) : + ByteArray := + PackedNTT.run p P.inverseTwiddles P.domain.logN parallel P.nInv.val true a + +private theorem normalized_inverse [Field R] (D : NTT.Domain R) (a : Array R) : + (NTT.Forward.forwardSpec D.inverse a).map (fun x ↦ D.nInv * x) = + NTT.Inverse.inverseSpec D a := by + have h : (NTT.Forward.forwardSpec D.inverse a).map (fun x ↦ D.nInv * x) = + NTT.Inverse.normalize D (NTT.Forward.forwardSpec D.inverse a) := by + simp only [NTT.Forward.forwardSpec, Array.map_ofFn, NTT.Inverse.normalize, + Function.comp_def] + apply congrArg Array.ofFn + funext i + rw [getD_ofFn_bounded _ _ i.isLt] + rw [h, NTT.Inverse.normalize_forwardSpec_inverse_eq_inverseSpec] + +private theorem inverse_logN [Field R] (D : NTT.Domain R) : + D.inverse.logN = D.logN := rfl + +private theorem inverse_n [Field R] (D : NTT.Domain R) : D.inverse.n = D.n := + congrArg (fun logN ↦ 2 ^ logN) (inverse_logN D) + +/-- The packed forward transform equals the mathematical DFT of every domain-sized input. -/ +theorem forwardPacked_correct [PackedNTT p] (P : Plan p) (a : Array (FastField p)) (parallel : Bool) + (hs : a.size = P.domain.n) : + P.forwardPacked (encode a) parallel = encode (NTT.Forward.forwardSpec P.domain a) := by + rw [forwardPacked, PackedNTT.run_eq, P.forward_eq, P.nInv_eq] + exact Native.run_dft P.domain parallel P.domain.nInv a false hs P.log_bound P.byte_bound + +/-- The packed inverse transform equals the normalized inverse DFT of every domain-sized input. -/ +theorem inversePacked_correct [PackedNTT p] (P : Plan p) (a : Array (FastField p)) (parallel : Bool) + (hs : a.size = P.domain.n) : + P.inversePacked (encode a) parallel = encode (NTT.Inverse.inverseSpec P.domain a) := by + rw [inversePacked, PackedNTT.run_eq, P.inverse_eq, P.nInv_eq] + have h := Native.run_dft P.domain.inverse parallel P.domain.nInv a true + (hs.trans (inverse_n P.domain).symm) ((inverse_logN P.domain).symm ▸ P.log_bound) + ((inverse_n P.domain).symm ▸ P.byte_bound) + rw [inverse_logN] at h + rw [h, ite_eq_left rfl, normalized_inverse] + +end Plan +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/QuadCells.lean b/CompPoly/Univariate/NTTFast/Packed/QuadCells.lean new file mode 100644 index 00000000..5de2a05a --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/QuadCells.lean @@ -0,0 +1,79 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Plan +import all CompPoly.Univariate.NTTFast.Packed.Radix4Lemmas +public import CompPoly.Univariate.NTTFast.Packed.Radix4Lemmas + +/-! # Explicit-length batches of independent radix-four cells -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- Execute an explicit number of cells while keeping the twiddle stride fixed. -/ +def quadCells [Zero R] [Add R] [Sub R] [Mul R] + (th tl : Array R) (q j i0 i1 i2 i3 : Nat) (a : Array R) : Nat → Array R + | 0 => a + | count + 1 => + let outputs := quadAt th tl q j i0 i1 i2 i3 a 0 + let b := (((a.setIfInBounds i0 outputs.1).setIfInBounds i1 outputs.2.1).setIfInBounds i2 + outputs.2.2.1).setIfInBounds i3 outputs.2.2.2 + quadCells th tl q (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) b count + +/-- Executing a batch preserves the working array's size. -/ +@[simp] theorem size_quadCells [Zero R] [Add R] [Sub R] [Mul R] + (th tl : Array R) (q j i0 i1 i2 i3 : Nat) (a : Array R) (count : Nat) : + (quadCells th tl q j i0 i1 i2 i3 a count).size = a.size := by + induction count generalizing j i0 i1 i2 i3 a with + | zero => rfl + | succ count ih => simp only [quadCells, ih, Array.size_setIfInBounds] + +/-- A disjoint batch has the independent-cell coordinate formula. -/ +theorem getD_quadCells [Field R] (th tl : Array R) (q j i0 i1 i2 i3 : Nat) + (a : Array R) (count : Nat) (hs : i3 + count ≤ a.size) + (h01 : i0 + count ≤ i1) (h12 : i1 + count ≤ i2) + (h23 : i2 + count ≤ i3) (k : Nat) : + (quadCells th tl q j i0 i1 i2 i3 a count).getD k 0 = + quadValue th tl q j i0 i1 i2 i3 count a k := by + induction count generalizing j i0 i1 i2 i3 a with + | zero => simp (disch := omega) only [quadCells, quadValue, Nat.add_zero, ite_eq_right] + | succ count ih => + rw [quadCells, ih _ _ _ _ _ _ (by simp only [Array.size_setIfInBounds]; omega) + (by omega) (by omega) (by omega)] + exact quadValue_step th tl q j i0 i1 i2 i3 count a hs h01 h12 h23 k + +/-- The ordinary inner loop executes exactly its remaining number of cells. -/ +theorem butterflyDIFRadix4Inner_eq_quadCells [Field R] (th tl : Array R) + (q j i0 i1 i2 i3 : Nat) (a : Array R) : + Plan.butterflyDIFRadix4Inner th tl q j i0 i1 i2 i3 a = + quadCells th tl q j i0 i1 i2 i3 a (q - j) := by + rw [Plan.butterflyDIFRadix4Inner] + split + · rename_i hj + have he : q - j = (q - (j + 1)) + 1 := by omega + rw [he, quadCells] + simpa only [quadAt, quadOutputs, Nat.add_zero, Array.set!] using + butterflyDIFRadix4Inner_eq_quadCells th tl q (j + 1) + (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) _ + · have he : q - j = 0 := by omega + rw [he, quadCells] +termination_by q - j +decreasing_by omega + +/-- Splitting a batch after a fixed number of cells changes only the loop counters. -/ +theorem quadCells_add [Zero R] [Add R] [Sub R] [Mul R] (th tl : Array R) + (q j i0 i1 i2 i3 : Nat) (a : Array R) (left right : Nat) : + quadCells th tl q j i0 i1 i2 i3 a (left + right) = + quadCells th tl q (j + left) (i0 + left) (i1 + left) (i2 + left) (i3 + left) + (quadCells th tl q j i0 i1 i2 i3 a left) right := by + induction left generalizing j i0 i1 i2 i3 a with + | zero => simp only [Nat.zero_add, Nat.add_zero, quadCells] + | succ left ih => + rw [Nat.succ_add, quadCells, quadCells, ih] + simp only [Nat.add_assoc, Nat.add_comm 1 left] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Radix2Refinement.lean b/CompPoly/Univariate/NTTFast/Packed/Radix2Refinement.lean new file mode 100644 index 00000000..b3b3bd4f --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Radix2Refinement.lean @@ -0,0 +1,188 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Plan +import all CompPoly.Univariate.NTTFast.Packed.Native +public import CompPoly.Univariate.NTTFast.Packed.FieldStorage +public import CompPoly.Univariate.NTTFast.Packed.ButterflyLemmas +public import CompPoly.Univariate.NTTFast.Packed.Rows +public import CompPoly.Univariate.NTTFast.Packed.Arrays + +/-! # Refinement of the packed radix-two loop + +`Native.pairs` runs `Plan.butterflyDIFInner` on packed words: sixteen-butterfly batches with +`pair16`, then the scalar `pairInner`. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +open CompPoly.PackedRepr (encode) + +local notation "F" => FastField p + +/-- One step of the inner DIF loop. -/ +theorem butterflyDIFInner_succ [Field R] (tw : Array R) (limit j i0 i1 : Nat) (a : Array R) + (h : j < limit) : + Plan.butterflyDIFInner tw limit j i0 i1 a = + Plan.butterflyDIFInner tw limit (j + 1) (i0 + 1) (i1 + 1) + ((a.set! i0 (a.getD i0 0 + a.getD i1 0)).set! i1 + (tw.getD j 0 * (a.getD i0 0 - a.getD i1 0))) := by + rw [Plan.butterflyDIFInner] + simp only [h, ↓reduceIte] + +/-- An inner DIF loop runs its first `k` butterflies, then the rest. -/ +theorem butterflyDIFInner_split [Field R] (tw : Array R) (limit : Nat) : + ∀ (k j i0 i1 : Nat) (a : Array R), j + k ≤ limit → + Plan.butterflyDIFInner tw limit j i0 i1 a = + Plan.butterflyDIFInner tw limit (j + k) (i0 + k) (i1 + k) + (Plan.butterflyDIFInner tw (j + k) j i0 i1 a) := by + intro k + induction k with + | zero => + intro j i0 i1 a _ + have hz : Plan.butterflyDIFInner tw j j i0 i1 a = a := by + rw [Plan.butterflyDIFInner] + simp only [Nat.lt_irrefl, ↓reduceIte] + simp only [Nat.add_zero, hz] + | succ k ih => + intro j i0 i1 a hk + rw [butterflyDIFInner_succ tw limit j i0 i1 a (by omega), + butterflyDIFInner_succ tw (j + (k + 1)) j i0 i1 a (by omega), + ih (j + 1) (i0 + 1) (i1 + 1) _ (by omega)] + simp only [show ∀ x, x + 1 + k = x + (k + 1) from fun x ↦ by omega] + +/-- The field-array semantics of sixteen radix-two butterflies. -/ +def pair16Field (w : Array F) (j i0 i1 : Nat) (a : Array F) : Array F := + splice (splice a i0 (lit16 fun t ↦ a.getD (i0 + t) 0 + a.getD (i1 + t) 0)) i1 + (lit16 fun t ↦ w.getD (j + t) 0 * (a.getD (i0 + t) 0 - a.getD (i1 + t) 0)) + +/-- The radix-two word kernel is its field-array semantics. -/ +theorem Native.pair16_encode (w a : Array F) (j i0 i1 : USize) (h) : + Native.pair16 p (encode w) j i0 i1 (encode a) h = + encode (pair16Field w j.toNat i0.toNat i1.toNat a) := by + obtain ⟨h0, h1, -, hs, -⟩ := h + rw [size_encode_fields] at h0 h1 + unfold Native.pair16 + simp only [Native.readUOffset_encode, add_val, sub_val, mul_val] + rw [Native.write16U_encode _ _ hs (by omega), Native.write16U_encode _ _ (by + rw [size_encode_fields, size_splice16 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ (by omega)] + rwa [size_encode_fields] at hs) (by + rw [size_splice16 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ (by omega)]; omega)] + simp only [USize.reduceToNat, Nat.add_zero] + rfl + +/-- Sixteen disjoint radix-two butterflies are sixteen steps of the inner DIF loop. -/ +theorem pair16Field_eq (w a : Array F) (j i0 i1 : Nat) (h01 : i0 + 16 ≤ i1) + (hs : i1 + 16 ≤ a.size) : + pair16Field w j i0 i1 a = Plan.butterflyDIFInner w (j + 16) j i0 i1 a := by + have hs1 : i0 + (lit16 fun t ↦ a.getD (i0 + t) 0 + a.getD (i1 + t) 0).size ≤ a.size := by + rw [size_lit16]; omega + have hsz : (splice a i0 (lit16 fun t ↦ a.getD (i0 + t) 0 + a.getD (i1 + t) 0)).size = + a.size := size_splice _ _ _ hs1 + have he : j + 16 - j = 16 := by omega + apply array_eq_of_getD _ _ 0 (by + rw [pair16Field, size_splice _ _ _ (by rw [size_lit16, hsz]; omega), hsz, + Plan.size_butterflyDIFInner]) + intro k + unfold pair16Field + rw [getD_splice _ _ _ (by rw [size_lit16, hsz]; omega), getD_splice _ _ _ hs1, size_lit16, + size_lit16] + by_cases hk1 : i1 ≤ k ∧ k < i1 + 16 + · simp only [show ¬k < i1 by omega, show k < i1 + 16 by omega, ↓reduceIte] + rw [getD_lit16 _ _ (by omega), + getD_butterflyDIFInner_right _ _ _ _ _ _ (by omega) (by omega) _ (by rw [he]; omega), + show i1 + (k - i1) = k by omega] + · by_cases hk0 : i0 ≤ k ∧ k < i0 + 16 + · simp only [show k < i1 by omega, show ¬k < i0 by omega, show k < i0 + 16 by omega, + ↓reduceIte] + rw [getD_lit16 _ _ (by omega), + getD_butterflyDIFInner_left _ _ _ _ _ _ (by omega) (by omega) _ (by rw [he]; omega), + show i0 + (k - i0) = k by omega] + · rw [getD_butterflyDIFInner_outside _ _ _ _ _ _ (by omega) (by omega) _ (by rw [he]; omega) + (by rw [he]; omega)] + by_cases hlt : k < i1 + · by_cases hlt0 : k < i0 + · simp only [hlt, hlt0, ↓reduceIte] + · simp only [hlt, hlt0, show ¬k < i0 + 16 by omega, ↓reduceIte] + · simp only [hlt, show ¬k < i1 + 16 by omega, show ¬k < i0 by omega, + show ¬k < i0 + 16 by omega, ↓reduceIte] + +/-- The hoisted batch loop depends only on the buffer's value. -/ +theorem Native.pair16Loop_congr {w : ByteArray} {n : Nat} {j i0 i1 : USize} {b b' : ByteArray} + (h) (e : b = b') : + Native.pair16Loop p w n j i0 i1 b h = Native.pair16Loop p w n j i0 i1 b' (e ▸ h) := by + subst e + rfl + +/-- The batch loop runs `16 * n` butterflies of the inner DIF loop. -/ +theorem Native.pair16Loop_encode (w : Array F) : + ∀ (n : Nat) (j i0 i1 : USize) (a : Array F) (h), i0.toNat + 16 * n ≤ i1.toNat → + i1.toNat + 16 * n ≤ a.size → + Native.pair16Loop p (encode w) n j i0 i1 (encode a) h = + encode (Plan.butterflyDIFInner w (j.toNat + 16 * n) j.toNat i0.toNat i1.toNat a) := by + intro n + induction n with + | zero => + intro j i0 i1 a h _ _ + rw [Native.pair16Loop, Nat.mul_zero, Nat.add_zero, Plan.butterflyDIFInner] + simp only [Nat.lt_irrefl, ↓reduceIte] + | succ n ih => + intro j i0 i1 a h h01 hs + have hb := h.2.2.2.1 + have hw := h.2.2.2.2 + rw [Native.pair16Loop, Native.pair16Loop_congr _ ((Native.pair16_encode w a j i0 i1 _).trans + (by rw [pair16Field_eq w a _ _ _ (by omega) (by omega)]))] + rw [ih _ _ _ _ _ (by rw [usize_add16 i0 _ (by omega) hb, usize_add16 i1 _ (by omega) hb]; omega) + (by rw [usize_add16 i1 _ (by omega) hb, Plan.size_butterflyDIFInner]; omega), + usize_add16 j _ (by omega) hw, usize_add16 i0 _ (by omega) hb, + usize_add16 i1 _ (by omega) hb, + butterflyDIFInner_split w (j.toNat + 16 * (n + 1)) 16 j.toNat i0.toNat i1.toNat a (by omega), + show j.toNat + 16 + 16 * n = j.toNat + 16 * (n + 1) by omega] + +/-- The scalar radix-two loop agrees with the ordinary inner DIF loop. -/ +theorem Native.pairInner_encode (w a : Array F) (q j i0 i1 : Nat) + (ha : (encode a).size < USize.size) (hw : (encode w).size < USize.size) (hwi : q ≤ w.size) + (hi : i0 + (q - j) ≤ a.size ∧ i1 + (q - j) ≤ a.size) : + Native.pairInner p (encode w) q j i0 i1 (encode a) = + encode (Plan.butterflyDIFInner w q j i0 i1 a) := by + rw [Native.pairInner, Plan.butterflyDIFInner] + split + · rename_i hj + simp (config := { maxDischargeDepth := 8 }) + (disch := (simp_all only [Array.size_setIfInBounds, size_encode_fields] <;> omega)) only + [Native.read_encode, add_val, sub_val, mul_val, Native.write_encode, + Array.set!_eq_setIfInBounds] + exact Native.pairInner_encode w _ q (j + 1) (i0 + 1) (i1 + 1) + (by simpa only [size_encode_fields, Array.size_setIfInBounds] using ha) hw hwi + (by simp only [Array.size_setIfInBounds]; omega) + · rfl +termination_by q - j +decreasing_by omega + +/-- The packed radix-two loop agrees with the ordinary inner DIF loop. -/ +theorem Native.pairs_encode (w a : Array F) (q j i0 i1 : Nat) + (ha : (encode a).size < USize.size) (hw : (encode w).size < USize.size) (hwi : q ≤ w.size) + (hj : j ≤ q) (hs : i1 + (q - j) ≤ a.size) (h01 : i0 + (q - j) ≤ i1) : + Native.pairs p (encode w) q j i0 i1 (encode a) = + encode (Plan.butterflyDIFInner w q j i0 i1 a) := by + have hn : 16 * ((q - j) / 16) ≤ q - j := Nat.mul_div_le _ _ + rw [Native.pairs] + split + · rw [Native.pair16Loop_encode w _ _ _ _ a _ (by simp only [USize.toNat_ofNatLT]; omega) + (by simp only [USize.toNat_ofNatLT]; omega)] + simp only [USize.toNat_ofNatLT] + rw [Native.pairInner_encode w _ q _ _ _ (by + rwa [size_encode_fields, Plan.size_butterflyDIFInner, ← size_encode_fields]) hw hwi + (by rw [Plan.size_butterflyDIFInner]; omega), + ← butterflyDIFInner_split w q _ j i0 i1 a (by omega)] + · exact Native.pairInner_encode w a q j i0 i1 ha hw hwi (by omega) + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Radix4Lemmas.lean b/CompPoly/Univariate/NTTFast/Packed/Radix4Lemmas.lean new file mode 100644 index 00000000..7a8175b3 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Radix4Lemmas.lean @@ -0,0 +1,119 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Plan +public import CompPoly.Univariate.NTTFast.Packed.ButterflyLemmas + +/-! # Coordinate semantics of independent radix-four DIF cells -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed + +/-- The four scalar outputs of two adjacent DIF layers. -/ +def quadOutputs [Add R] [Sub R] [Mul R] (w0 w1 w : R) (x0 x1 x2 x3 : R) : R × R × R × R := + let a0 := x0 + x2 + let a1 := x1 + x3 + let a2 := w0 * (x0 - x2) + let a3 := w1 * (x1 - x3) + (a0 + a1, w * (a0 - a1), a2 + a3, w * (a2 - a3)) + +/-- One independent cell reads four input coordinates at the same lane offset. -/ +def quadAt [Zero R] [Add R] [Sub R] [Mul R] (th tl : Array R) (q j i0 i1 i2 i3 : Nat) + (a : Array R) (offset : Nat) : R × R × R × R := + quadOutputs (th.getD (j + offset) 0) (th.getD (j + offset + q) 0) (tl.getD (j + offset) 0) + (a.getD (i0 + offset) 0) (a.getD (i1 + offset) 0) + (a.getD (i2 + offset) 0) (a.getD (i3 + offset) 0) + +/-- Pointwise result of a complete range of independent radix-four cells. -/ +def quadValue [Zero R] [Add R] [Sub R] [Mul R] (th tl : Array R) (q j i0 i1 i2 i3 count : Nat) + (a : Array R) (k : Nat) : R := + if i0 ≤ k ∧ k < i0 + count then (quadAt th tl q j i0 i1 i2 i3 a (k - i0)).1 else + if i1 ≤ k ∧ k < i1 + count then (quadAt th tl q j i0 i1 i2 i3 a (k - i1)).2.1 else + if i2 ≤ k ∧ k < i2 + count then (quadAt th tl q j i0 i1 i2 i3 a (k - i2)).2.2.1 else + if i3 ≤ k ∧ k < i3 + count then (quadAt th tl q j i0 i1 i2 i3 a (k - i3)).2.2.2 else + a.getD k 0 + +set_option maxHeartbeats 1200000 in +private theorem quadValue_step [Field R] (th tl : Array R) (q j i0 i1 i2 i3 count : Nat) + (a : Array R) (hs : i3 + (count + 1) ≤ a.size) + (h01 : i0 + (count + 1) ≤ i1) (h12 : i1 + (count + 1) ≤ i2) + (h23 : i2 + (count + 1) ≤ i3) (k : Nat) : + let outputs := quadAt th tl q j i0 i1 i2 i3 a 0 + let b := (((a.setIfInBounds i0 outputs.1).setIfInBounds i1 outputs.2.1).setIfInBounds i2 + outputs.2.2.1).setIfInBounds i3 outputs.2.2.2 + quadValue th tl q (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) count b k = + quadValue th tl q j i0 i1 i2 i3 (count + 1) a k := by + dsimp only + unfold quadValue quadAt quadOutputs + simp (config := { maxSteps := 1000000 }) (disch := (simp -failIfUnchanged only + [Array.size_setIfInBounds] <;> omega)) only + [getD_setIfInBounds] + by_cases hk0 : k = i0 + · subst k + simp (disch := omega) only [Nat.sub_self, Nat.add_zero, ite_eq_left, ite_eq_right, ↓reduceIte] + · by_cases hk1 : k = i1 + · subst k + simp (disch := omega) only [Nat.sub_self, Nat.add_zero, ite_eq_left, ite_eq_right, ↓reduceIte] + · by_cases hk2 : k = i2 + · subst k + simp (disch := omega) only [Nat.sub_self, Nat.add_zero, ite_eq_left, ite_eq_right, + ↓reduceIte] + · by_cases hk3 : k = i3 + · subst k + simp (disch := omega) only [Nat.sub_self, Nat.add_zero, ite_eq_left, ite_eq_right, + ↓reduceIte] + · by_cases hr0 : i0 ≤ k ∧ k < i0 + (count + 1) + · have he0 : i0 + 1 + (k - (i0 + 1)) = i0 + (k - i0) := by omega + have he1 : i1 + 1 + (k - (i0 + 1)) = i1 + (k - i0) := by omega + have he2 : i2 + 1 + (k - (i0 + 1)) = i2 + (k - i0) := by omega + have he3 : i3 + 1 + (k - (i0 + 1)) = i3 + (k - i0) := by omega + have hew : j + 1 + (k - (i0 + 1)) = j + (k - i0) := by omega + simp (disch := omega) only [he0, he1, he2, he3, ite_eq_left, ite_eq_right] + · by_cases hr1 : i1 ≤ k ∧ k < i1 + (count + 1) + · have he0 : i0 + 1 + (k - (i1 + 1)) = i0 + (k - i1) := by omega + have he1 : i1 + 1 + (k - (i1 + 1)) = i1 + (k - i1) := by omega + have he2 : i2 + 1 + (k - (i1 + 1)) = i2 + (k - i1) := by omega + have he3 : i3 + 1 + (k - (i1 + 1)) = i3 + (k - i1) := by omega + have hew : j + 1 + (k - (i1 + 1)) = j + (k - i1) := by omega + simp (disch := omega) only [he0, he1, he2, he3, hew, ite_eq_left, ite_eq_right] + · by_cases hr2 : i2 ≤ k ∧ k < i2 + (count + 1) + · have he0 : i0 + 1 + (k - (i2 + 1)) = i0 + (k - i2) := by omega + have he1 : i1 + 1 + (k - (i2 + 1)) = i1 + (k - i2) := by omega + have he2 : i2 + 1 + (k - (i2 + 1)) = i2 + (k - i2) := by omega + have he3 : i3 + 1 + (k - (i2 + 1)) = i3 + (k - i2) := by omega + have hew : j + 1 + (k - (i2 + 1)) = j + (k - i2) := by omega + simp (disch := omega) only [he0, he1, he2, he3, hew, ite_eq_left, ite_eq_right] + · by_cases hr3 : i3 ≤ k ∧ k < i3 + (count + 1) + · have he0 : i0 + 1 + (k - (i3 + 1)) = i0 + (k - i3) := by omega + have he1 : i1 + 1 + (k - (i3 + 1)) = i1 + (k - i3) := by omega + have he2 : i2 + 1 + (k - (i3 + 1)) = i2 + (k - i3) := by omega + have he3 : i3 + 1 + (k - (i3 + 1)) = i3 + (k - i3) := by omega + have hew : j + 1 + (k - (i3 + 1)) = j + (k - i3) := by omega + simp (disch := omega) only [he0, he1, he2, he3, hew, ite_eq_left, ite_eq_right] + · simp (disch := omega) only [ite_eq_left, ite_eq_right] + +/-- The ordinary radix-four inner loop has the independent-cell coordinate formula. -/ +theorem getD_butterflyDIFRadix4Inner [Field R] (th tl : Array R) (q j i0 i1 i2 i3 : Nat) + (a : Array R) (hs : i3 + (q - j) ≤ a.size) + (h01 : i0 + (q - j) ≤ i1) (h12 : i1 + (q - j) ≤ i2) + (h23 : i2 + (q - j) ≤ i3) (k : Nat) : + (Plan.butterflyDIFRadix4Inner th tl q j i0 i1 i2 i3 a).getD k 0 = + quadValue th tl q j i0 i1 i2 i3 (q - j) a k := by + rw [Plan.butterflyDIFRadix4Inner] + split + · rename_i hj + have he : q - j = (q - (j + 1)) + 1 := by omega + rw [getD_butterflyDIFRadix4Inner th tl q (j + 1) (i0 + 1) (i1 + 1) (i2 + 1) (i3 + 1) _ + (by simp only [Array.size_set!]; omega) (by omega) (by omega) (by omega), he] + exact quadValue_step th tl q j i0 i1 i2 i3 (q - (j + 1)) a + (by omega) (by omega) (by omega) (by omega) k + · have he : q - j = 0 := by omega + simp (disch := omega) only [he, quadValue, Nat.add_zero, ite_eq_right] +termination_by q - j +decreasing_by omega + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Rows.lean b/CompPoly/Univariate/NTTFast/Packed/Rows.lean new file mode 100644 index 00000000..8268989e --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Rows.lean @@ -0,0 +1,91 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Data.ByteArray.Packed +public import Mathlib.Tactic.IntervalCases + +/-! # Sixteen-word rows + +The packed loops append sixteen words at a time. Their results are stated as `rows`, a +concatenation of sixteen-entry literals with a single indexing lemma. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open CompPoly.PackedRepr (encode) + +/-- Sixteen values of a function as an array literal. -/ +def lit16 (g : Nat → α) : Array α := + #[g 0, g 1, g 2, g 3, g 4, g 5, g 6, g 7, g 8, g 9, g 10, g 11, g 12, g 13, g 14, g 15] + +@[simp] theorem size_lit16 (g : Nat → α) : (lit16 g).size = 16 := rfl + +theorem getD_lit16 (g : Nat → α) (c : Nat) (hc : c < 16) (d : α) : + (lit16 g).getD c d = g c := by + interval_cases c <;> rfl + +/-- Concatenated sixteen-entry rows; row `j` holds `g j 0, …, g j 15`. -/ +def rows (g : Nat → Nat → α) : Nat → Array α + | 0 => #[] + | k + 1 => rows g k ++ lit16 (g k) + +@[simp] theorem size_rows (g : Nat → Nat → α) (k : Nat) : (rows g k).size = 16 * k := by + induction k with + | zero => rfl + | succ k ih => + simp only [rows, Array.size_append, ih, size_lit16] + omega + +theorem getD_append_eq (a b : Array α) (i : Nat) (d : α) : + (a ++ b).getD i d = if i < a.size then a.getD i d else b.getD (i - a.size) d := by + simp only [Array.getD_eq_getD_getElem?] + split + · rw [Array.getElem?_append_left ‹_›] + · rw [Array.getElem?_append_right (by omega)] + +theorem getD_rows (g : Nat → Nat → α) (k i : Nat) (hi : i < 16 * k) (d : α) : + (rows g k).getD i d = g (i / 16) (i % 16) := by + induction k with + | zero => omega + | succ k ih => + simp only [rows] + rw [getD_append_eq, size_rows] + split + · exact ih ‹_› + · rw [getD_lit16 _ _ (by omega)] + have h1 : i / 16 = k := by omega + have h2 : i % 16 = i - 16 * k := by omega + rw [h1, h2] + +theorem rows_add (g : Nat → Nat → α) (a b : Nat) : + rows g (a + b) = rows g a ++ rows (fun j ↦ g (a + j)) b := by + induction b with + | zero => simp only [Nat.add_zero, rows, Array.append_empty] + | succ b ih => + rw [← Nat.add_assoc, rows, ih, rows, Array.append_assoc] + +theorem rows_congr (g g' : Nat → Nat → α) (k : Nat) (h : ∀ j c, c < 16 → g j c = g' j c) : + rows g k = rows g' k := by + induction k with + | zero => rfl + | succ k ih => + simp only [rows, ih, lit16] + rw [h k 0 (by decide), h k 1 (by decide), h k 2 (by decide), h k 3 (by decide), + h k 4 (by decide), h k 5 (by decide), h k 6 (by decide), h k 7 (by decide), + h k 8 (by decide), h k 9 (by decide), h k 10 (by decide), h k 11 (by decide), + h k 12 (by decide), h k 13 (by decide), h k 14 (by decide), h k 15 (by decide)] + +theorem getD_map_range (F : Nat → α) (n k : Nat) (hk : k < n) (d : α) : + ((Array.range n).map F).getD k d = F k := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_map, Array.getElem?_range, hk, + ↓reduceIte, Option.map_some, Option.getD_some] + +/-- Pack an empty array, whatever capacity was reserved. -/ +theorem Native.emptyWithCapacity_eq_encodeWords (k : Nat) : + ByteArray.emptyWithCapacity k = encode (#[] : Array UInt32) := rfl + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/StorageLemmas.lean b/CompPoly/Univariate/NTTFast/Packed/StorageLemmas.lean new file mode 100644 index 00000000..173b0e82 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/StorageLemmas.lean @@ -0,0 +1,250 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Packed.Native +public import CompPoly.Univariate.NTTFast.Packed.Native + +/-! # Word-storage refinement lemmas for the native FFT -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed.Native +open CompPoly.PackedRepr (encode size_encode_uint32) + +theorem ugetUInt32LE_eq (a : ByteArray) (i : USize) (h) : + a.ugetUInt32LE i h = a.getUInt32LE! i.toNat := rfl + +theorem usetUInt32LE_eq (a : ByteArray) (i : USize) (v : UInt32) (h) : + a.usetUInt32LE i v h = a.setUInt32LE! i.toNat v := rfl + +/-- A bounded offset read preserves the represented word array. -/ +theorem readUOffset_encodeWords (words : Array UInt32) (i o : USize) (h) : + readUOffset (encode words) i o h = words.getD (i.toNat + o.toNat) 0 := by + have hb : 4 * (i.toNat + o.toNat) + 3 < USize.size := h.1.trans h.2 + rw [readUOffset, ugetUInt32LE_eq, wordOffset_toNat i o hb, PackedRepr.getUInt32LE!_encode] + +/-- Checked reads agree with the same array entry whenever the slot is in bounds. -/ +theorem readAt_encodeWords (words : Array UInt32) (i o : USize) + (h : 4 * (i.toNat + o.toNat) + 3 < (encode words).size ∧ + (encode words).size < USize.size) : + readAt (encode words) i o = words.getD (i.toNat + o.toNat) 0 := by + have hb : 4 * (i.toNat + o.toNat) + 3 < USize.size := h.1.trans h.2 + have hu : (encode words).usize.toNat = (encode words).size := by + rw [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' h.2] + have h4 := usize_numeral 4 (by decide) + have hc : 4 ≤ (encode words).usize ∧ (i + o) * 4 ≤ (encode words).usize - 4 := by + have h4' : 4 ≤ (encode words).usize := by + rw [USize.le_iff_toNat_le, hu, h4]; omega + refine ⟨h4', ?_⟩ + rw [USize.le_iff_toNat_le, USize.toNat_sub_of_le _ _ h4', hu, h4, wordOffset_toNat i o hb] + omega + rw [readAt, dite_eq_left_of_eq_true (eq_true hc), ugetUInt32LE_eq, wordOffset_toNat i o hb, + PackedRepr.getUInt32LE!_encode] + +/-- Ordinary checked reads preserve valid packed word coordinates. -/ +theorem read_encodeWords (words : Array UInt32) (index : Nat) + (hs : (encode words).size < USize.size) (hi : index < words.size) : + read (encode words) index = words.getD index 0 := by + have hk : 4 * index + 3 < (encode words).size := by rw [size_encode_uint32]; omega + have hin : index < USize.size := by rw [size_encode_uint32] at hs; omega + have hu : index.toUSize.toNat = index := USize.toNat_ofNat_of_lt' hin + have hzero : (0 : USize).toNat = 0 := rfl + unfold read + have h := readAt_encodeWords words index.toUSize 0 + (by simp only [hu, hzero, Nat.add_zero]; exact ⟨hk, hs⟩) + simpa only [hu, hzero, Nat.add_zero] using h + +/-! ### Batch stores -/ + +/-- The words of a batch. -/ +abbrev batch (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) : Array UInt32 := + #[v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15] + +theorem extract_push_getElem (V : Array UInt32) (k : Nat) (hk : k < V.size) : + V.extract 0 k ++ #[V[k]] = V.extract 0 (k + 1) := by + apply Array.ext + · simp only [Array.size_append, Array.size_extract, List.size_toArray, List.length_cons, + List.length_nil] + omega + · intro j h1 h2 + simp only [Array.getElem_append, Array.getElem_extract, Array.size_extract] + split_ifs <;> first | omega | rfl | + (simp only [Array.getElem_singleton]; congr 1; simp only [Array.size_extract] at *; omega) + +/-- One word of a batch store, as a step of a growing byte-range replacement. -/ +theorem putWord_eq {n : Nat} (s : {x : ByteArray // x.size = n}) (off : USize) (c k : Nat) + (d : USize) (hd : d.toNat = 4 * k) + (V : Array UInt32) (hk : k < V.size) (h) (b : ByteArray) (hb : b.size = n) + (hs : s.1 = ByteArray.replaceAt b off.toNat (encode (V.extract 0 (min k c)))) : + (putWord s off c k d hd V[k] h).1 = + ByteArray.replaceAt b off.toNat (encode (V.extract 0 (min (k + 1) c))) := by + unfold putWord + split + · rename_i hkc + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + have hu : (off + d).toNat = off.toNat + 4 * k := by + rw [USize.toNat_add, hd, hsize, Nat.mod_eq_of_lt (by omega)] + have hp : (encode (V.extract 0 k)).size = 4 * k := by + rw [size_encode_uint32, Array.size_extract]; omega + simp only [usetUInt32LE_eq, hu, hs, Nat.min_eq_left (show k ≤ c by omega), + Nat.min_eq_left (show k + 1 ≤ c by omega)] + rw [PackedRepr.setUInt32LE!_eq_replaceAt_encode _ _ _ (by + rw [ByteArray.size_replaceAt _ _ _ (by rw [hp]; omega)]; omega), + ← hp, ByteArray.replaceAt_replaceAt _ _ _ _ (by rw [hp, size_encode_uint32]; simp only + [List.size_toArray, List.length_cons, List.length_nil]; omega), + ← PackedRepr.encode_append, extract_push_getElem V k hk] + · rename_i hkc + rw [hs, Nat.min_eq_right (show c ≤ k by omega), Nat.min_eq_right (show c ≤ k + 1 by omega)] + +/-- A batch store replaces the batch's byte range. -/ +theorem putWords_eq (b : ByteArray) (off : USize) (c : Nat) (hc : c ≤ 16) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) (h) : + putWords ⟨b, rfl⟩ off c v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 h = + ByteArray.replaceAt b off.toNat (encode + ((batch v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15).extract 0 c)) := by + let V := batch v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 + have e : (⟨b, rfl⟩ : {x : ByteArray // x.size = b.size}).1 = + ByteArray.replaceAt b off.toNat (encode (V.extract 0 (min 0 c))) := by + rw [Nat.min_eq_left (Nat.zero_le _), Array.extract_zero] + exact (ByteArray.replaceAt_empty b _ (by omega)).symm + have e0 := putWord_eq _ off c 0 0 (by rw [usize_numeral _ (by decide)]) V + (show 0 < 16 by decide) h b rfl e + have e1 := putWord_eq _ off c 1 4 (by rw [usize_numeral _ (by decide)]) V + (show 1 < 16 by decide) h b rfl e0 + have e2 := putWord_eq _ off c 2 8 (by rw [usize_numeral _ (by decide)]) V + (show 2 < 16 by decide) h b rfl e1 + have e3 := putWord_eq _ off c 3 12 (by rw [usize_numeral _ (by decide)]) V + (show 3 < 16 by decide) h b rfl e2 + have e4 := putWord_eq _ off c 4 16 (by rw [usize_numeral _ (by decide)]) V + (show 4 < 16 by decide) h b rfl e3 + have e5 := putWord_eq _ off c 5 20 (by rw [usize_numeral _ (by decide)]) V + (show 5 < 16 by decide) h b rfl e4 + have e6 := putWord_eq _ off c 6 24 (by rw [usize_numeral _ (by decide)]) V + (show 6 < 16 by decide) h b rfl e5 + have e7 := putWord_eq _ off c 7 28 (by rw [usize_numeral _ (by decide)]) V + (show 7 < 16 by decide) h b rfl e6 + have e8 := putWord_eq _ off c 8 32 (by rw [usize_numeral _ (by decide)]) V + (show 8 < 16 by decide) h b rfl e7 + have e9 := putWord_eq _ off c 9 36 (by rw [usize_numeral _ (by decide)]) V + (show 9 < 16 by decide) h b rfl e8 + have e10 := putWord_eq _ off c 10 40 (by rw [usize_numeral _ (by decide)]) V + (show 10 < 16 by decide) h b rfl e9 + have e11 := putWord_eq _ off c 11 44 (by rw [usize_numeral _ (by decide)]) V + (show 11 < 16 by decide) h b rfl e10 + have e12 := putWord_eq _ off c 12 48 (by rw [usize_numeral _ (by decide)]) V + (show 12 < 16 by decide) h b rfl e11 + have e13 := putWord_eq _ off c 13 52 (by rw [usize_numeral _ (by decide)]) V + (show 13 < 16 by decide) h b rfl e12 + have e14 := putWord_eq _ off c 14 56 (by rw [usize_numeral _ (by decide)]) V + (show 14 < 16 by decide) h b rfl e13 + have e15 := putWord_eq _ off c 15 60 (by rw [usize_numeral _ (by decide)]) V + (show 15 < 16 by decide) h b rfl e14 + rw [Nat.min_eq_right (show c ≤ 15 + 1 by omega)] at e15 + exact e15 + +/-- The batch store is a byte-range replacement by the batch's first `count` words. -/ +theorem storeWords_eq (b : ByteArray) (offset : USize) (count : UInt8) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) : + storeWords b offset count v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 = + if count.toNat > 16 then b else + if offset ≤ offset + (4 * count.toNat).toUSize ∧ + offset + (4 * count.toNat).toUSize ≤ b.usize then + ByteArray.replaceAt b offset.toNat (encode + ((batch v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15).extract 0 count.toNat)) + else b := by + unfold storeWords + split + · rfl + · rename_i hc + split + · rw [putWords_eq _ _ count.toNat (by omega)] + · rfl + +/-- In a packed buffer that fits machine indices, a batch's range check is its slot bound. -/ +theorem storeWords_fits (words : Array UInt32) (offset : USize) (count : UInt8) (index : Nat) + (hc : count.toNat ≤ 16) (ho : offset.toNat = 4 * index) + (hs : (encode words).size < USize.size) : + (offset ≤ offset + (4 * count.toNat).toUSize ∧ + offset + (4 * count.toNat).toUSize ≤ (encode words).usize) ↔ + index + count.toNat ≤ words.size := by + have hsize : (2 : Nat) ^ System.Platform.numBits = USize.size := rfl + rw [size_encode_uint32] at hs + have hu : (encode words).usize.toNat = 4 * words.size := by + rw [ByteArray.usize, Nat.toUSize, USize.toNat_ofNat_of_lt' (by rw [size_encode_uint32]; omega), + size_encode_uint32] + have h4 : (4 * count.toNat).toUSize.toNat = 4 * count.toNat := + USize.toNat_ofNat_of_lt' (by have := USize.le_size; omega) + rw [USize.le_iff_toNat_le, USize.le_iff_toNat_le, hu, USize.toNat_add, h4, hsize, ho] + constructor + · rintro ⟨h1, h2⟩ + by_cases hw : 4 * index + 4 * count.toNat < USize.size + · rw [Nat.mod_eq_of_lt hw] at h2; omega + · have ho' := offset.toNat_lt_size + have hle := USize.le_size + rw [ho] at ho' + rw [Nat.mod_eq_sub_mod (by omega), Nat.mod_eq_of_lt + (show 4 * index + 4 * count.toNat - USize.size < USize.size by omega)] at h1 + omega + · intro h + rw [Nat.mod_eq_of_lt (by omega)] + omega + +/-- The overwrite primitive changes precisely the specified word slots. -/ +theorem storeWords_replace_encodeWords (words : Array UInt32) (offset : USize) + (count : UInt8) (index : Nat) (hc : count.toNat ≤ 16) + (ho : offset.toNat = 4 * index) (hi : index + count.toNat ≤ words.size) + (hs : (encode words).size < USize.size) + (v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 : UInt32) : + storeWords (encode words) offset count + v0 v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 = + encode (words.extract 0 index ++ + #[v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15].extract + 0 count.toNat ++ words.extract (index + count.toNat) words.size) := by + have hf := (storeWords_fits words offset count index hc ho hs).mpr hi + have hv : (#[v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15].extract + 0 count.toNat).size = count.toNat := by + rw [Array.size_extract] + change min count.toNat 16 - 0 = count.toNat + omega + simp only [storeWords_eq, Nat.not_lt.mpr hc, ↓reduceIte, hf, ho, + PackedRepr.replaceAt_encode_uint32, hv, and_self] + +/-! ### Zero buffers -/ + +/-- `m` zero bytes. -/ +abbrev zeroBuffer (m : Nat) : ByteArray := ⟨Array.replicate m 0⟩ + +theorem zeroBuffer_eq_encodeWords (n : Nat) : + zeroBuffer (4 * n) = encode (Array.replicate n (0 : UInt32)) := by + apply ByteArray.ext + simp only [encode, ← List.toArray_replicate] + congr 1 + induction n with + | zero => rfl + | succ n ih => + rw [List.replicate_succ, List.flatMap_cons, ← ih, + show 4 * (n + 1) = 4 + 4 * n by ring, List.replicate_add] + rfl + +/-- `n` zero words are the packing of `n` zeros. -/ +theorem zeroWords_eq (n : Nat) : + zeroWords n = encode (Array.replicate n (0 : UInt32)) := by + rw [← zeroBuffer_eq_encodeWords, zeroWords, ByteArray.zeros_eq] + +/-- Extending a packed buffer by one zero block appends 256 zero words. -/ +theorem extendZeros_encodeWords (o : Array UInt32) : + ByteArray.zeroBlock.copySlice 0 (encode o) (encode o).size 1024 false = + encode (o ++ Array.replicate 256 0) := by + have hz : ByteArray.zeroBlock.data.size = 4096 := Array.size_replicate + have hb : ByteArray.zeroBlock.extract 0 (0 + 1024) = zeroBuffer (4 * 256) := by + apply ByteArray.ext + simp only [ByteArray.data_extract, ByteArray.zeroBlock, Array.extract_replicate] + rfl + rw [ByteArray.copySlice_eq_append, ByteArray.extract_zero_size, hb, + (ByteArray.extract_eq_empty_iff).mpr (by simp only [ByteArray.size, hz]; omega), + ByteArray.append_empty, PackedRepr.encode_append, zeroBuffer_eq_encodeWords] + +end CompPoly.CPolynomial.NTTFast.Packed.Native diff --git a/CompPoly/Univariate/NTTFast/Packed/Twiddles.lean b/CompPoly/Univariate/NTTFast/Packed/Twiddles.lean new file mode 100644 index 00000000..eb145a84 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Twiddles.lean @@ -0,0 +1,49 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.Arithmetic +public import CompPoly.Univariate.NTTFast.Plan +import all CompPoly.Univariate.NTTFast.Correctness.Basic + +/-! # Shape and identity invariants of twiddle tables -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Packed +open Montgomery.Native32 (Mont32Field FastField) + +variable {p : ℕ} [Mont32Field p] + +/-- Every stored twiddle table has its stage's butterfly width. -/ +def TwiddleSizes (logN : Nat) (tw : Array (Array (FastField p))) : Prop := + ∀ stage < logN, (tw.getD stage #[]).size = 2 ^ stage + +/-- Every twiddle stage starts at the multiplicative identity. -/ +def TwiddleOnes (logN : Nat) (tw : Array (Array (FastField p))) : Prop := + ∀ stage < logN, (tw.getD stage #[]).getD 0 0 = 1 + +/-- A one-coordinate twiddle table beginning at one is the identity table. -/ +theorem twiddle_zero_eq_one (tw : Array (FastField p)) + (hs : tw.size = 1) (hone : tw.getD 0 0 = 1) : tw = #[1] := by + apply Array.ext (by simpa only [Array.size_singleton] using hs) + intro i hi hj + have he : i = 0 := by omega + subst i + simpa only [Array.getD_eq_getD_getElem?, Array.getElem?_eq_getElem hi, + Option.getD_some, Array.getElem_singleton] using hone + +/-- Ordinary cached domain twiddles satisfy both packed-stage invariants. -/ +theorem twiddleTable_invariants (D : NTT.Domain (FastField p)) : + TwiddleSizes D.logN (Plan.twiddleTable D) ∧ + TwiddleOnes D.logN (Plan.twiddleTable D) := by + constructor + · intro stage hs + rw [Plan.twiddleTable_getD_eq_twiddlePowers D stage hs, Plan.twiddlePowers_size] + · intro stage hs + rw [Plan.twiddleTable_getD_eq_twiddlePowers D stage hs, + Plan.twiddlePowers_getD_eq_pow D stage 0 (Nat.two_pow_pos stage), pow_zero] + +end CompPoly.CPolynomial.NTTFast.Packed diff --git a/CompPoly/Univariate/NTTFast/Packed/Unrolled.lean b/CompPoly/Univariate/NTTFast/Packed/Unrolled.lean new file mode 100644 index 00000000..589ccf85 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Packed/Unrolled.lean @@ -0,0 +1,434 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +/-! # Generated straight-line kernels + +The packed kernels are straight-line code over sixteen words, which the C compiler turns into +vector instructions, and the proofs relate each to a field-array model of the same shape. Both +are generated here from one description, in two dialects: `words` reads and writes packed +words with bounds proofs, `fields` reads and splices field arrays. Each generator expands to the +`let`s of its kernel, referring to the binders of the definition it is used in by name; word +arithmetic refers to the modulus `p` of a `Mont32Field p` instance. +-/ + +namespace CompPoly.CPolynomial.NTTFast.Packed.Unrolled +open Lean + +/-- Packed words with proved bounds, or field arrays. -/ +meta inductive Dialect where + | words + | fields + +meta def name (s : String) : Ident := mkIdent (Name.mkSimple s) + +meta def lit (k : Nat) : Term := Syntax.mkNumLit (toString k) + +/-- Word `k` after index `idx` of `arr`. -/ +meta def read (d : Dialect) (arr idx : Term) (k : Nat) : MacroM Term := + match d with + | .words => `($(name "readUOffset") $arr $idx $(lit k) + (by simp only [USize.reduceToNat]; omega)) + | .fields => `($(name "readField") $arr $idx $(lit k)) + +meta def add (d : Dialect) (x y : Term) : MacroM Term := + match d with + | .words => `($(name "add") $(name "p") $x $y) + | .fields => `($(name "fieldAdd") $x $y) + +meta def sub (d : Dialect) (x y : Term) : MacroM Term := + match d with + | .words => `($(name "sub") $(name "p") $x $y) + | .fields => `($(name "fieldSub") $x $y) + +meta def mul (d : Dialect) (x y : Term) : MacroM Term := + match d with + | .words => `($(name "mul") $(name "p") $x $y) + | .fields => `($(name "fieldMul") $x $y) + +/-- Overwrite sixteen words of `b` from index `idx`. -/ +meta def store16 (d : Dialect) (b idx : Term) (vs : Array Term) : MacroM Term := + match d with + | .words => `($(name "write16U") $b $idx $vs*) + | .fields => `($(name "splice") $b ($idx).toNat #[$vs,*]) + +/-- `let x := v` for each binding, then `body`. -/ +meta def lets (bs : Array (Ident × Term)) (body : Term) : MacroM Term := + bs.foldrM (fun (x, v) b ↦ `(let $x:ident := $v; $b)) body + +/-- Sixteen radix-four DIF butterflies across the quarters at `i0, …, i3` with twiddles from +`th` at `j`, `j1` and `tl` at `j`: lane `k` reads `x0_k, …, x3_k`, combines them into +`a0_k, …, a3_k` and outputs `y0_k, …, y3_k`. -/ +meta def step16 (d : Dialect) (lanes : Bool := true) : MacroM Term := do + let b := name "b" + let v (s : String) (q k : Nat) : Term := name s!"{s}{q}_{k}" + let mut bs := #[] + for k in [0:16] do + let tw (arr j : String) := read d (name arr) (name j) (if lanes then k else 0) + for q in [0:4] do + bs := bs.push (name s!"x{q}_{k}", ← read d b (name s!"i{q}") k) + bs := bs.push (name s!"a0_{k}", ← add d (v "x" 0 k) (v "x" 2 k)) + bs := bs.push (name s!"a1_{k}", ← add d (v "x" 1 k) (v "x" 3 k)) + bs := bs.push (name s!"a2_{k}", ← mul d (← tw "th" "j") (← sub d (v "x" 0 k) (v "x" 2 k))) + bs := bs.push (name s!"a3_{k}", ← mul d (← tw "th" "j1") (← sub d (v "x" 1 k) (v "x" 3 k))) + bs := bs.push (name s!"y0_{k}", ← add d (v "a" 0 k) (v "a" 1 k)) + bs := bs.push (name s!"y1_{k}", ← mul d (← tw "tl" "j") (← sub d (v "a" 0 k) (v "a" 1 k))) + bs := bs.push (name s!"y2_{k}", ← add d (v "a" 2 k) (v "a" 3 k)) + bs := bs.push (name s!"y3_{k}", ← mul d (← tw "tl" "j") (← sub d (v "a" 2 k) (v "a" 3 k))) + for q in [0:4] do + bs := bs.push (b, ← store16 d b (name s!"i{q}") ((Array.range 16).map (v "y" q))) + lets bs b + +/-- The twiddles `w{s}_t` of layer `s + 1` of a sixteen-point block, from `t3`, `t2`, `t1`. -/ +meta def twiddles (d : Dialect) (s : Nat) (uniform : Bool := false) : + MacroM (Array (Ident × Term)) := do + let tws := #["t3", "t2", "t1"] + let mut bs := #[] + for t in [(if uniform && s < 3 then 0 else 1):8 >>> s] do + bs := bs.push (name s!"w{s}_{t}", ← read d (name tws[s]!) (lit 0) t) + return bs + +/-- The four final DIF layers of a sixteen-point block: layer `s` pairs points `k` and +`k + 2 ^ (4 - s)`; the difference of every pair but the first in its group is multiplied by +twiddle `w{s - 1}_t`. `x s k` names point `k` after layer `s`; `tw s` are the twiddle reads +emitted before layer `s + 1`. -/ +meta def layers (d : Dialect) (x : Nat → Nat → String) (tw : Nat → MacroM (Array (Ident × Term))) + (uniform : Bool := false) : MacroM (Array (Ident × Term)) := do + let mut bs := #[] + for s in [1:5] do + bs := bs ++ (← tw (s - 1)) + let h := 16 >>> s + for group in [0:8 / h] do + let g := 2 * h * group + for t in [0:h] do + let lo : Term := name (x (s - 1) (g + t)) + let hi : Term := name (x (s - 1) (g + t + h)) + bs := bs.push (name (x s (g + t)), ← add d lo hi) + let diff ← sub d lo hi + bs := bs.push (name (x s (g + t + h)), + ← if t = 0 && (!uniform || s = 4) then pure diff else mul d (name s!"w{s - 1}_{t}") diff) + return bs + +/-- A sixteen-point block at `i` of `b` through its four final DIF layers, twiddles read before +each layer; `finish` receives the outputs. -/ +meta def leaf16 (d : Dialect) (finish : Array Term → MacroM Term) : MacroM Term := do + let mut bs := #[] + for k in [0:16] do + bs := bs.push (name s!"x0_{k}", ← read d (name "b") (name "i") k) + bs := bs ++ (← layers d (fun s k ↦ s!"x{s}_{k}") (twiddles d)) + lets bs (← finish ((Array.range 16).map fun k ↦ (name s!"x4_{k}" : Term))) + +/-- Eight sixteen-point blocks at `i0, …, i7` through their four final DIF layers, one per vector +lane; word `t` of every lane goes to half `half` of the natural-order line `bitrev (16 c + t)`, +optionally scaled. The touched output takes proved stores, given `hout`. -/ +meta def lastHalf : MacroM Term := do + let mut bs := #[] + for s in [0:3] do + bs := bs ++ (← twiddles .words s) + for v in [0:8] do + for k in [0:16] do + bs := bs.push (name s!"x{v}_0_{k}", ← read .words (name "b") (name s!"i{v}") k) + bs := bs ++ (← layers .words (fun s k ↦ s!"x{v}_{s}_{k}") fun _ ↦ pure #[]) + let store (t : Nat) : MacroM Term := do + let ws ← (Array.range 8).mapM fun v ↦ + `($(name "scaleWord") $(name "p") $(name "factor") $(name "scale") + $(name s!"x{v}_4_{t}")) + `($(name "store8") $(name "o") + (4 * (16 * ($(name "reverse32") (16 * $(name "c") + $(lit t)).toUInt32 >>> + $(name "shift")).toUSize + 8 * $(name "half"))) $ws* + ($(name "lastHalf_store") $(name "hout") + ($(name "reverse32") (16 * $(name "c") + $(lit t)).toUInt32))) + bs := bs.push (name "o", ← `($(name "touchLast") $(name "out") + ($(name "lastHalf_usize") $(name "hout")))) + for t in [0:16] do + bs := bs.push (name "o", ← store t) + lets bs (← `($(name "o").1)) + +/-- Sixteen radix-two DIF butterflies between the words from `i0` and from `i1`, with twiddles +from `w` at `j`. -/ +meta def pair16 : MacroM Term := do + let b := name "b" + let mut bs := #[] + for k in [0:16] do + bs := bs.push (name s!"x{k}", ← read .words b (name "i0") k) + bs := bs.push (name s!"y{k}", ← read .words b (name "i1") k) + bs := bs.push (name s!"w{k}", ← read .words (name "w") (name "j") k) + let sums ← (Array.range 16).mapM fun k ↦ add .words (name s!"x{k}") (name s!"y{k}") + let diffs ← (Array.range 16).mapM fun k ↦ do + mul .words (name s!"w{k}") (← sub .words (name s!"x{k}") (name s!"y{k}")) + bs := bs.push (b, ← store16 .words b (name "i0") sums) + lets bs (← store16 .words b (name "i1") diffs) + +/-- Word `k` from `i` of `T` times the word of `c` at its index. -/ +meta def scaledRead (i : String) (k : Nat) : MacroM Term := do + mul .words (← read .words (name "T") (name i) k) (← read .words (name "c") (name i) k) + +/-- Sixteen radix-two DIF butterflies out of place: the words from `i0` and from `i1` of `c`, each +multiplied by the word of `T` at its index, combined with twiddles from `w` at `j` and stored at the +same indices of `out`. -/ +meta def pair16Scaled : MacroM Term := do + let mut bs := #[] + for k in [0:16] do + bs := bs.push (name s!"x{k}", ← scaledRead "i0" k) + bs := bs.push (name s!"y{k}", ← scaledRead "i1" k) + bs := bs.push (name s!"w{k}", ← read .words (name "w") (name "j") k) + let sums ← (Array.range 16).mapM fun k ↦ add .words (name s!"x{k}") (name s!"y{k}") + let diffs ← (Array.range 16).mapM fun k ↦ do + mul .words (name s!"w{k}") (← sub .words (name s!"x{k}") (name s!"y{k}")) + bs := bs.push (name "out", ← store16 .words (name "out") (name "i0") sums) + lets bs (← store16 .words (name "out") (name "i1") diffs) + +/-- The sixteen words of `b` from word `q`, given `h : LineFits b q`. -/ +meta def line : MacroM Term := do + let ws ← (Array.range 16).mapM fun k ↦ + `($(name "b").ugetUInt32LE (($(name "q") + $(lit k)) * 4) + (by rw [($(name "lineFits_toNat") $(name "h") $(lit k) (by decide)).1] + exact ($(name "lineFits_toNat") $(name "h") $(lit k) (by decide)).2)) + `(⟨$ws,*⟩) + +/-- The transpose of a `16 × 16` tile of the streams `s0, …, s15` at word `q`, stored from word +`p` of `out` if every stream's line fits. -/ +meta def transposeStep : MacroM Term := do + let h := name "h" + let mut bs := #[] + for t in [0:16] do + let mut proj : Term := h + for _ in [0:t] do + proj ← `($proj.2) + if t < 15 then proj ← `($proj.1) + bs := bs.push (name s!"x{t}", ← `($(name "line") $(name s!"s{t}") $(name "q") $proj)) + let store (u : Nat) : MacroM Term := do + let p : Term ← if u = 0 then pure (name "p") else `($(name "p") + $(lit (16 * u))) + let vs ← (Array.range 16).mapM fun t ↦ `($(name s!"x{t}").$(name s!"w{u}"):ident) + `($(name "write16U") $(name "out") $p $vs*) + for u in [0:15] do + bs := bs.push (name "out", ← store u) + let fits ← (Array.range 16).mapM fun t ↦ `($(name "LineFits") $(name s!"s{t}") $(name "q")) + let cond ← fits[:15].foldrM (fun f c ↦ `($f ∧ $c)) fits[15]! + let body ← lets bs (← store 15) + `(if $h:ident : $cond then $body else $(name "out")) + +/-- The transpose of a `16 × 16` tile whose line `t` is words `q + t * qs, …` of `s`, stored from +word `p` of `out` with lines `ps` words apart, if every source line fits. -/ +meta def tile16 : MacroM Term := do + let h := name "h" + let src (t : Nat) : MacroM (Term × Term) := do + pure (name "s", ← `($(name "q") + $(lit t) * $(name "qs"))) + let mut bs := #[] + for t in [0:16] do + let mut proj : Term := h + for _ in [0:t] do + proj ← `($proj.2) + if t < 15 then proj ← `($proj.1) + let (s, q) ← src t + bs := bs.push (name s!"x{t}", ← `($(name "line") $s $q $proj)) + let store (u : Nat) : MacroM Term := do + let vs ← (Array.range 16).mapM fun t ↦ `($(name s!"x{t}").$(name s!"w{u}"):ident) + `($(name "write16U") $(name "out") ($(name "p") + $(lit u) * $(name "ps")) $vs*) + for u in [0:15] do + bs := bs.push (name "out", ← store u) + let fits ← (Array.range 16).mapM fun t ↦ do + let (s, q) ← src t + `($(name "LineFits") $s $q) + let cond ← fits[:15].foldrM (fun f c ↦ `($f ∧ $c)) fits[15]! + let body ← lets bs (← store 15) + `(if $h:ident : $cond then $body else $(name "out")) + +/-- The products of sixteen words of `t` and of `a` from `i`, stored from `i` in `out`. -/ +meta def mul16 : MacroM Term := do + let mut bs := #[] + for k in [0:16] do + bs := bs.push (name s!"x{k}", + ← mul .words (← read .words (name "t") (name "i") k) (← read .words (name "a") (name "i") k)) + lets bs (← store16 .words (name "out") (name "i") ((Array.range 16).map fun k ↦ name s!"x{k}")) + +/-- The transpose of an `8 × 8` tile: source line `t` is words `q, …, q + 7` of `s{t}`; line `u` +of the transpose is stored from word `p + u * ps` of `out`, if every line fits. The touched output +takes proved stores, and the sources proved loads. -/ +meta def tile8 : MacroM Term := do + let h := name "h" + let proj (t : Nat) : MacroM Term := do + let mut pr : Term := h + for _ in [0:t] do + pr ← `($pr.2) + if t < 8 then pr ← `($pr.1) + pure pr + let mut bs := #[] + bs := bs.push (name "o", ← `($(name "touchLast") $(name "out") + ($(name "tile8Fits_usize") $(← proj 8)))) + let ho ← proj 8 + for u in [0:8] do + for t in [0:8] do + let ht ← proj t + bs := bs.push (name s!"x{t}_{u}", ← `($(name s!"s{t}").ugetUInt32LE + (($(name "q") + $(lit u)) * 4) + (by rw [($(name "line8Fits_toNat") $ht $(lit u) (by decide)).1] + exact ($(name "line8Fits_toNat") $ht $(lit u) (by decide)).2))) + let vs ← (Array.range 8).mapM fun t ↦ pure (name s!"x{t}_{u}" : Term) + bs := bs.push (name "o", ← `($(name "store8") $(name "o") + (($(name "p") + $(lit u) * $(name "ps")) * 4) $vs* + ($(name "tile8Fits_toNat") $ho $(lit u) (by decide)))) + let fits ← (Array.range 8).mapM fun t ↦ `($(name "Line8Fits") $(name s!"s{t}") $(name "q")) + let cond ← fits.foldrM (fun f c ↦ `($f ∧ $c)) + (← `($(name "Tile8Fits") $(name "out") $(name "p") $(name "ps"))) + let body ← lets bs (← `($(name "o").1)) + `(if $h:ident : $cond then $body else $(name "out")) + + +/-- One radix-two layer on sixteen-word blocks in place: lane `l` of a block pairs its words `lo[l]` +and `hi[l]` and keeps their sum at `lo[l]` and their difference at `hi[l]`, multiplied by twiddle +`tw[l]` of `t` if there are twiddles. The words dialect runs the four blocks from word `i` of `b`: +it touches the buffer, reads every block, then stores each block as two halves, with the range +checks proved from `h`. The fields dialect runs one block at word `base`. -/ +meta def leafStage (d : Dialect) (lo hi : List Nat) (tw : Option (List Nat)) : MacroM Term := do + let blocks := match d with | .words => 4 | .fields => 1 + let mut bs := #[] + let word (v k : Nat) : MacroM Term := match d with + | .words => `($(name "readUOffset") $(name "o").1 $(name "i") $(lit (16 * v + k)) + ($(name "leafStage_read") $(name "h") $(name "o").2 $(lit (16 * v + k)) (by decide))) + | .fields => `($(name "b").getD ($(name "base") + $(lit k)) 0) + if d matches .words then + bs := bs.push (name "o", ← `($(name "touchLast") $(name "b") + ($(name "leafStage_touch") $(name "h")))) + if let some ts := tw then + for t in ts.eraseDups do + bs := bs.push (name s!"w{t}", ← match d with + | .words => `($(name "readUOffset") $(name "t") 0 $(lit t) + ($(name "leafStage_twiddle") $(name "h") $(lit t) (by decide) (by decide))) + | .fields => `($(name "t").getD $(lit t) 0)) + for v in [0:blocks] do + for l in [0:8] do + bs := bs.push (name s!"x{v}_{l}", ← word v lo[l]!) + bs := bs.push (name s!"y{v}_{l}", ← word v hi[l]!) + for l in [0:8] do + bs := bs.push (name s!"s{v}_{l}", ← add d (name s!"x{v}_{l}") (name s!"y{v}_{l}")) + let diff ← sub d (name s!"x{v}_{l}") (name s!"y{v}_{l}") + bs := bs.push (name s!"d{v}_{l}", ← match tw with + | some ts => mul d (name s!"w{ts[l]!}") diff + | none => pure diff) + for v in [0:blocks] do + let mut vals : Array Term := Array.replicate 16 (name s!"s{v}_0") + for l in [0:8] do + vals := vals.set! lo[l]! (name s!"s{v}_{l}") + vals := vals.set! hi[l]! (name s!"d{v}_{l}") + for half in [0, 8] do + let vs := vals[half:half + 8].toArray + match d with + | .words => + bs := bs.push (name "o", ← `($(name "store8") $(name "o") + (($(name "i") + $(lit (16 * v + half))) * 4) $vs* + ($(name "leafStage_store") $(name "h") $(lit (16 * v + half)) (by decide)))) + | .fields => + let pos : Term ← if half = 0 then pure (name "base") else `($(name "base") + $(lit half)) + bs := bs.push (name "b", ← `($(name "splice8") $(name "b") $pos $vs*)) + match d with + | .words => lets bs (← `($(name "o"))) + | .fields => lets bs (name "b") + +end CompPoly.CPolynomial.NTTFast.Packed.Unrolled + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of a sixteen-lane radix-four kernel on packed words. -/ +macro "step16_words%" : term => step16 .words + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of a sixteen-lane radix-four kernel on field arrays. -/ +macro "step16_fields%" : term => step16 .fields + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The final DIF layers of the block at `i` of a field array, spliced back. -/ +macro "leaf16_fields%" : term => + leaf16 .fields fun ys ↦ `($(name "splice") $(name "b") ($(name "i")).toNat #[$ys,*]) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The final DIF layers of the block at `i` of a field array, as a line. -/ +macro "leaf16_line%" : term => leaf16 .fields fun ys ↦ `(⟨$ys,*⟩) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The final DIF layers of the block at `i` of packed words, written back in place. -/ +macro "leaf16_words%" : term => leaf16 .words (store16 .words (name "b") (name "i")) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of the fused final-layer kernel on packed words. -/ +macro "lastHalf_words%" : term => lastHalf + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of the sixteen-lane radix-two kernel on packed words. -/ +macro "pair16_words%" : term => pair16 + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The sixteen words of a line. -/ +macro "line_words%" : term => line + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of the `16 × 16` tile transpose on packed words. -/ +macro "transposeStep_words%" : term => transposeStep + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of the `16 × 16` tile transpose of one strided source on packed words. -/ +macro "tile16_strided%" : term => tile16 + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of the sixteen-word product kernel on packed words. -/ +macro "mul16_words%" : term => mul16 + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of a sixteen-lane radix-four kernel on packed words sharing one set of twiddles. -/ +macro "row16_words%" : term => step16 .words (lanes := false) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of the `8 × 8` tile transpose of eight sources on packed words. -/ +macro "tile8_sources%" : term => tile8 + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer one of four in place: words `k` and `k + 8` of every block. -/ +macro "leafStage8_words%" : term => + leafStage .words [0, 1, 2, 3, 4, 5, 6, 7] [8, 9, 10, 11, 12, 13, 14, 15] + (some [0, 1, 2, 3, 4, 5, 6, 7]) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer one of four on one block of field values. -/ +macro "leafStage8_fields%" : term => + leafStage .fields [0, 1, 2, 3, 4, 5, 6, 7] [8, 9, 10, 11, 12, 13, 14, 15] + (some [0, 1, 2, 3, 4, 5, 6, 7]) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer two of four in place: words `k` and `k + 4` of every half block. -/ +macro "leafStage4_words%" : term => + leafStage .words [0, 1, 2, 3, 8, 9, 10, 11] [4, 5, 6, 7, 12, 13, 14, 15] + (some [0, 1, 2, 3, 0, 1, 2, 3]) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer two of four on one block of field values. -/ +macro "leafStage4_fields%" : term => + leafStage .fields [0, 1, 2, 3, 8, 9, 10, 11] [4, 5, 6, 7, 12, 13, 14, 15] + (some [0, 1, 2, 3, 0, 1, 2, 3]) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer three of four in place: words `k` and `k + 2` of every quarter block. -/ +macro "leafStage2_words%" : term => + leafStage .words [0, 1, 4, 5, 8, 9, 12, 13] [2, 3, 6, 7, 10, 11, 14, 15] + (some [0, 1, 0, 1, 0, 1, 0, 1]) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer three of four on one block of field values. -/ +macro "leafStage2_fields%" : term => + leafStage .fields [0, 1, 4, 5, 8, 9, 12, 13] [2, 3, 6, 7, 10, 11, 14, 15] + (some [0, 1, 0, 1, 0, 1, 0, 1]) + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer four of four in place: words `2 k` and `2 k + 1`, without twiddles. -/ +macro "leafStage1_words%" : term => + leafStage .words [0, 2, 4, 6, 8, 10, 12, 14] [1, 3, 5, 7, 9, 11, 13, 15] none + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- Layer four of four on one block of field values. -/ +macro "leafStage1_fields%" : term => + leafStage .fields [0, 2, 4, 6, 8, 10, 12, 14] [1, 3, 5, 7, 9, 11, 13, 15] none + +open CompPoly.CPolynomial.NTTFast.Packed.Unrolled in +/-- The body of the scaled out-of-place radix-two kernel. -/ +macro "pair16Scaled_words%" : term => pair16Scaled diff --git a/CompPoly/Univariate/NTTFast/Parallel.lean b/CompPoly/Univariate/NTTFast/Parallel.lean new file mode 100644 index 00000000..03ed891e --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Parallel.lean @@ -0,0 +1,380 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +import all CompPoly.Univariate.NTTFast.Correctness.Basic +import all CompPoly.Univariate.NTTFast.Plan +public import CompPoly.Univariate.NTTFast.Natural + +/-! # Parallel natural-order transforms with independent array segments + +Large transforms split the stages into a sequential prefix/suffix and independent +segments. Tasks own separate arrays; the refinement proofs use butterfly locality. +-/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast.Plan +variable {R : Type*} + +private theorem getD_append_left [Zero R] (a b : Array R) (i : Nat) (h : i < a.size) : + (a ++ b).getD i 0 = a.getD i 0 := by + simp only [Array.getD_eq_getD_getElem?, Array.getElem?_append_left h] + +private theorem getD_append_right [Zero R] (a b : Array R) (i : Nat) : + (a ++ b).getD (a.size + i) 0 = b.getD i 0 := by + simp only [Array.getD_eq_getD_getElem?] + rw [Array.getElem?_append_right (xs := a) (ys := b) (i := a.size + i) (by omega)] + simp only [Nat.add_sub_cancel_left] + +private theorem setAppendLeft (a b : Array R) (i : Nat) (v : R) (h : i < a.size) : + (a ++ b).set! i v = a.set! i v ++ b := by + simp only [Array.set!, Array.setIfInBounds_append_left h] + +private theorem setAppendRight (a b : Array R) (i : Nat) (v : R) : + (a ++ b).set! (a.size + i) v = a ++ b.set! i v := by + rw [Array.set!, Array.setIfInBounds_append_right (xs := a) (ys := b) + (i := a.size + i) (by omega)] + simp only [Nat.add_sub_cancel_left, Array.set!] + +/-- Butterflies confined to the left array leave the right array unchanged. -/ +theorem butterflyDIFRadix4Inner_append_left [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) + (a b : Array R) + (ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size) : + butterflyDIFRadix4Inner th tl limit j i0 i1 i2 i3 (a ++ b) = + butterflyDIFRadix4Inner th tl limit j i0 i1 i2 i3 a ++ b := by + conv_lhs => rw [butterflyDIFRadix4Inner.eq_def] + conv_rhs => rw [butterflyDIFRadix4Inner.eq_def] + split + · rename_i hj + have h0 : i0 < a.size := by omega + have h1 : i1 < a.size := by omega + have h2 : i2 < a.size := by omega + have h3 : i3 < a.size := by omega + simp only [getD_append_left a b i0 h0, getD_append_left a b i1 h1, + getD_append_left a b i2 h2, getD_append_left a b i3 h3, + setAppendLeft, Array.size_set!, h0, h1, h2, h3] + exact butterflyDIFRadix4Inner_append_left th tl limit (j + 1) (i0 + 1) (i1 + 1) + (i2 + 1) (i3 + 1) _ b (by simp only [Array.size_set!] at *; omega) + · rfl +termination_by limit - j +decreasing_by omega + +/-- Shifted butterfly indices operate only on the appended right array. -/ +theorem butterflyDIFRadix4Inner_append_right [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) + (a b : Array R) : + butterflyDIFRadix4Inner th tl limit j (a.size + i0) (a.size + i1) + (a.size + i2) (a.size + i3) (a ++ b) = + a ++ butterflyDIFRadix4Inner th tl limit j i0 i1 i2 i3 b := by + conv_lhs => rw [butterflyDIFRadix4Inner.eq_def] + conv_rhs => rw [butterflyDIFRadix4Inner.eq_def] + split + · simp only [getD_append_right, setAppendRight] + simpa only [Nat.add_assoc] using + butterflyDIFRadix4Inner_append_right th tl limit (j + 1) (i0 + 1) (i1 + 1) + (i2 + 1) (i3 + 1) a _ + · rfl +termination_by limit - j +decreasing_by omega + +/-- Butterflies confined to the left array leave the right array unchanged. -/ +theorem butterflyDITRadix4Inner_append_left [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) + (a b : Array R) + (ha : i0 + (limit - j) ≤ a.size ∧ i1 + (limit - j) ≤ a.size ∧ + i2 + (limit - j) ≤ a.size ∧ i3 + (limit - j) ≤ a.size) : + butterflyDITRadix4Inner tl th limit j i0 i1 i2 i3 (a ++ b) = + butterflyDITRadix4Inner tl th limit j i0 i1 i2 i3 a ++ b := by + conv_lhs => rw [butterflyDITRadix4Inner.eq_def] + conv_rhs => rw [butterflyDITRadix4Inner.eq_def] + split + · rename_i hj + have h0 : i0 < a.size := by omega + have h1 : i1 < a.size := by omega + have h2 : i2 < a.size := by omega + have h3 : i3 < a.size := by omega + simp only [getD_append_left a b i0 h0, getD_append_left a b i1 h1, + getD_append_left a b i2 h2, getD_append_left a b i3 h3, + setAppendLeft, Array.size_set!, h0, h1, h2, h3] + exact butterflyDITRadix4Inner_append_left th tl limit (j + 1) (i0 + 1) (i1 + 1) + (i2 + 1) (i3 + 1) _ b (by simp only [Array.size_set!] at *; omega) + · rfl +termination_by limit - j +decreasing_by omega + +/-- Shifted butterfly indices operate only on the appended right array. -/ +theorem butterflyDITRadix4Inner_append_right [Field R] (th tl : Array R) (limit j i0 i1 i2 i3 : Nat) + (a b : Array R) : + butterflyDITRadix4Inner tl th limit j (a.size + i0) (a.size + i1) + (a.size + i2) (a.size + i3) (a ++ b) = + a ++ butterflyDITRadix4Inner tl th limit j i0 i1 i2 i3 b := by + conv_lhs => rw [butterflyDITRadix4Inner.eq_def] + conv_rhs => rw [butterflyDITRadix4Inner.eq_def] + split + · simp only [getD_append_right, setAppendRight] + simpa only [Nat.add_assoc] using + butterflyDITRadix4Inner_append_right th tl limit (j + 1) (i0 + 1) (i1 + 1) + (i2 + 1) (i3 + 1) a _ + · rfl +termination_by limit - j +decreasing_by omega + +/-- Complete left-hand blocks leave an appended suffix unchanged. -/ +theorem butterflyDIFRadix4Blocks_append_left [Field R] (th tl : Array R) (q blocks block : Nat) + (a b : Array R) (ha : blocks * (4 * q) ≤ a.size) : + butterflyDIFRadix4Blocks th tl (4 * q) q blocks block (a ++ b) = + butterflyDIFRadix4Blocks th tl (4 * q) q blocks block a ++ b := by + conv_lhs => rw [butterflyDIFRadix4Blocks.eq_def] + conv_rhs => rw [butterflyDIFRadix4Blocks.eq_def] + split + · rename_i hb + have hs : (block + 1) * (4 * q) ≤ a.size := + (Nat.mul_le_mul_right (4 * q) (by omega)).trans ha + dsimp only + rw [butterflyDIFRadix4Inner_append_left th tl q 0 (block * (4 * q)) (block * (4 * q) + q) + (block * (4 * q) + 2 * q) (block * (4 * q) + 3 * q) a b (by + simp only [Nat.sub_zero]; rw [Nat.add_mul, Nat.one_mul] at hs; omega)] + exact butterflyDIFRadix4Blocks_append_left th tl q blocks (block + 1) _ b + (by simpa only [size_butterflyDIFRadix4Inner] using ha) + · rfl +termination_by blocks - block +decreasing_by omega + +/-- Offset block traversal agrees with traversal of the right-hand segment. -/ +theorem butterflyDIFRadix4Blocks_append_right [Field R] (th tl : Array R) + (q left blocks block : Nat) + (a b : Array R) (ha : a.size = left * (4 * q)) : + butterflyDIFRadix4Blocks th tl (4 * q) q (left + blocks) (left + block) (a ++ b) = + a ++ butterflyDIFRadix4Blocks th tl (4 * q) q blocks block b := by + conv_lhs => rw [butterflyDIFRadix4Blocks.eq_def] + conv_rhs => rw [butterflyDIFRadix4Blocks.eq_def] + simp only [Nat.add_lt_add_iff_left] + split + · have h0 : (left + block) * (4 * q) = a.size + block * (4 * q) := by + rw [Nat.add_mul, ha] + simp only [h0, Nat.add_assoc] + rw [butterflyDIFRadix4Inner_append_right] + simpa only [Nat.add_assoc] using butterflyDIFRadix4Blocks_append_right th tl q left blocks + (block + 1) a _ ha + · rfl +termination_by blocks - block +decreasing_by omega + +private theorem butterflyDIFRadix4Blocks_split [Field R] (th tl : Array R) (q m n block : Nat) + (a : Array R) (hb : block ≤ m) : + butterflyDIFRadix4Blocks th tl (4 * q) q (m + n) block a = + butterflyDIFRadix4Blocks th tl (4 * q) q (m + n) m + (butterflyDIFRadix4Blocks th tl (4 * q) q m block a) := by + by_cases h : block < m + · conv_lhs => rw [butterflyDIFRadix4Blocks.eq_def] + conv_rhs => rw [show butterflyDIFRadix4Blocks th tl (4 * q) q m block a = _ from + butterflyDIFRadix4Blocks.eq_def ..] + simp only [h, show block < m + n by omega, ↓reduceIte] + exact butterflyDIFRadix4Blocks_split th tl q m n (block + 1) _ (by omega) + · have he : block = m := by omega + subst block + conv_rhs => rw [show butterflyDIFRadix4Blocks th tl (4 * q) q m m a = _ from + butterflyDIFRadix4Blocks.eq_def ..] + simp only [Nat.lt_irrefl, ↓reduceIte] +termination_by m - block +decreasing_by omega + +/-- Complete aligned blocks can be transformed independently. -/ +theorem butterflyDIFRadix4Blocks_append [Field R] (th tl : Array R) (q m n : Nat) (a b : Array R) + (ha : a.size = m * (4 * q)) : + butterflyDIFRadix4Blocks th tl (4 * q) q (m + n) 0 (a ++ b) = + butterflyDIFRadix4Blocks th tl (4 * q) q m 0 a ++ + butterflyDIFRadix4Blocks th tl (4 * q) q n 0 b := by + rw [butterflyDIFRadix4Blocks_split th tl q m n 0 _ (by omega), + butterflyDIFRadix4Blocks_append_left th tl q m 0 a b (by omega)] + simpa only [Nat.add_zero] using + butterflyDIFRadix4Blocks_append_right th tl q m n 0 _ b + (by simpa only [size_butterflyDIFRadix4Blocks] using ha) + +/-- Complete left-hand blocks leave an appended suffix unchanged. -/ +theorem butterflyDITRadix4Blocks_append_left [Field R] (th tl : Array R) (q blocks block : Nat) + (a b : Array R) (ha : blocks * (4 * q) ≤ a.size) : + butterflyDITRadix4Blocks tl th (4 * q) q blocks block (a ++ b) = + butterflyDITRadix4Blocks tl th (4 * q) q blocks block a ++ b := by + conv_lhs => rw [butterflyDITRadix4Blocks.eq_def] + conv_rhs => rw [butterflyDITRadix4Blocks.eq_def] + split + · rename_i hb + have hs : (block + 1) * (4 * q) ≤ a.size := + (Nat.mul_le_mul_right (4 * q) (by omega)).trans ha + dsimp only + rw [butterflyDITRadix4Inner_append_left th tl q 0 (block * (4 * q)) (block * (4 * q) + q) + (block * (4 * q) + 2 * q) (block * (4 * q) + 3 * q) a b (by + simp only [Nat.sub_zero]; rw [Nat.add_mul, Nat.one_mul] at hs; omega)] + exact butterflyDITRadix4Blocks_append_left th tl q blocks (block + 1) _ b + (by simpa only [size_butterflyDITRadix4Inner] using ha) + · rfl +termination_by blocks - block +decreasing_by omega + +/-- Offset block traversal agrees with traversal of the right-hand segment. -/ +theorem butterflyDITRadix4Blocks_append_right [Field R] (th tl : Array R) + (q left blocks block : Nat) + (a b : Array R) (ha : a.size = left * (4 * q)) : + butterflyDITRadix4Blocks tl th (4 * q) q (left + blocks) (left + block) (a ++ b) = + a ++ butterflyDITRadix4Blocks tl th (4 * q) q blocks block b := by + conv_lhs => rw [butterflyDITRadix4Blocks.eq_def] + conv_rhs => rw [butterflyDITRadix4Blocks.eq_def] + simp only [Nat.add_lt_add_iff_left] + split + · have h0 : (left + block) * (4 * q) = a.size + block * (4 * q) := by + rw [Nat.add_mul, ha] + simp only [h0, Nat.add_assoc] + rw [butterflyDITRadix4Inner_append_right] + simpa only [Nat.add_assoc] using butterflyDITRadix4Blocks_append_right th tl q left blocks + (block + 1) a _ ha + · rfl +termination_by blocks - block +decreasing_by omega + +private theorem butterflyDITRadix4Blocks_split [Field R] (th tl : Array R) (q m n block : Nat) + (a : Array R) (hb : block ≤ m) : + butterflyDITRadix4Blocks tl th (4 * q) q (m + n) block a = + butterflyDITRadix4Blocks tl th (4 * q) q (m + n) m + (butterflyDITRadix4Blocks tl th (4 * q) q m block a) := by + by_cases h : block < m + · conv_lhs => rw [butterflyDITRadix4Blocks.eq_def] + conv_rhs => rw [show butterflyDITRadix4Blocks tl th (4 * q) q m block a = _ from + butterflyDITRadix4Blocks.eq_def ..] + simp only [h, show block < m + n by omega, ↓reduceIte] + exact butterflyDITRadix4Blocks_split th tl q m n (block + 1) _ (by omega) + · have he : block = m := by omega + subst block + conv_rhs => rw [show butterflyDITRadix4Blocks tl th (4 * q) q m m a = _ from + butterflyDITRadix4Blocks.eq_def ..] + simp only [Nat.lt_irrefl, ↓reduceIte] +termination_by m - block +decreasing_by omega + +/-- Complete aligned blocks can be transformed independently. -/ +theorem butterflyDITRadix4Blocks_append [Field R] (th tl : Array R) (q m n : Nat) (a b : Array R) + (ha : a.size = m * (4 * q)) : + butterflyDITRadix4Blocks tl th (4 * q) q (m + n) 0 (a ++ b) = + butterflyDITRadix4Blocks tl th (4 * q) q m 0 a ++ + butterflyDITRadix4Blocks tl th (4 * q) q n 0 b := by + rw [butterflyDITRadix4Blocks_split th tl q m n 0 _ (by omega), + butterflyDITRadix4Blocks_append_left th tl q m 0 a b (by omega)] + simpa only [Nat.add_zero] using + butterflyDITRadix4Blocks_append_right th tl q m n 0 _ b + (by simpa only [size_butterflyDITRadix4Blocks] using ha) + +end CompPoly.CPolynomial.NTTFast.Plan + +namespace CompPoly.CPolynomial.NTTFast.Parallel +variable {R : Type*} + +/-- Execute one planned radix-four pass on all complete blocks. -/ +@[inline] def step [Field R] [DecidableEq R] (inverse : Bool) (tw : Array (Array R)) + (low : Nat) (a : Array R) : Array R := + let q := 2 ^ low + if inverse then + Plan.stageBlocksDIT (tw.getD (low + 1) #[]) (tw.getD low #[]) + (4 * q) q (a.size / (4 * q)) 0 a + else + Plan.stageBlocksDIF (tw.getD (low + 1) #[]) (tw.getD low #[]) + (4 * q) q (a.size / (4 * q)) 0 a + +@[simp] theorem size_step [Field R] [DecidableEq R] (inverse : Bool) + (tw : Array (Array R)) (low : Nat) (a : Array R) : + (step inverse tw low a).size = a.size := by + cases inverse <;> simp only [step, Bool.false_eq_true, ↓reduceIte, + Plan.stageBlocksDIF_eq, Plan.checkedBlocks_eq, + Plan.stageBlocksDIT_eq, Plan.checkedDITBlocks_eq, + Plan.size_butterflyDIFRadix4Blocks, Plan.size_butterflyDITRadix4Blocks] + +/-- Aligned segments have no butterfly dependencies across their boundary. -/ +theorem step_append [Field R] [DecidableEq R] (inverse : Bool) (tw : Array (Array R)) + (low : Nat) (a b : Array R) (ha : 4 * 2 ^ low ∣ a.size) + (hb : 4 * 2 ^ low ∣ b.size) : + step inverse tw low (a ++ b) = step inverse tw low a ++ step inverse tw low b := by + have hq : 0 < 4 * 2 ^ low := Nat.mul_pos (by omega) (Nat.two_pow_pos low) + have hcount : (a.size + b.size) / (4 * 2 ^ low) = + a.size / (4 * 2 ^ low) + b.size / (4 * 2 ^ low) := by + rw [← Nat.div_mul_cancel ha, ← Nat.div_mul_cancel hb, ← Nat.add_mul, + Nat.mul_div_cancel _ hq, Nat.mul_div_cancel _ hq, Nat.mul_div_cancel _ hq] + cases inverse <;> simp only [step, Bool.false_eq_true, ↓reduceIte, + Plan.stageBlocksDIF_eq, Plan.checkedBlocks_eq, + Plan.stageBlocksDIT_eq, Plan.checkedDITBlocks_eq, Array.size_append, hcount] + · exact Plan.butterflyDIFRadix4Blocks_append _ _ _ _ _ _ _ (Nat.div_mul_cancel ha).symm + · exact Plan.butterflyDITRadix4Blocks_append _ _ _ _ _ _ _ (Nat.div_mul_cancel ha).symm + +/-- Run a sequence of planned radix-four passes. -/ +@[inline, specialize] def runPasses [Field R] [DecidableEq R] (inverse : Bool) + (tw : Array (Array R)) (passes : List Nat) (a : Array R) : Array R := + passes.foldl (fun a low ↦ step inverse tw low a) a + +@[simp] theorem size_runPasses [Field R] [DecidableEq R] (inverse : Bool) + (tw : Array (Array R)) (passes : List Nat) (a : Array R) : + (runPasses inverse tw passes a).size = a.size := by + induction passes generalizing a with + | nil => rfl + | cons low rest ih => + change (runPasses inverse tw rest (step inverse tw low a)).size = a.size + rw [ih, size_step] + +/-- An aligned sequence of stages can run independently on each segment. -/ +theorem runPasses_append [Field R] [DecidableEq R] (inverse : Bool) (tw : Array (Array R)) + (passes : List Nat) (a b : Array R) + (h : ∀ low ∈ passes, 4 * 2 ^ low ∣ a.size ∧ 4 * 2 ^ low ∣ b.size) : + runPasses inverse tw passes (a ++ b) = + runPasses inverse tw passes a ++ runPasses inverse tw passes b := by + induction passes generalizing a b with + | nil => rfl + | cons low rest ih => + have hl := h low (List.mem_cons_self ..) + change runPasses inverse tw rest (step inverse tw low (a ++ b)) = + runPasses inverse tw rest (step inverse tw low a) ++ + runPasses inverse tw rest (step inverse tw low b) + rw [step_append inverse tw low a b hl.1 hl.2] + apply ih + intro k hk + simpa only [size_step] using h k (List.mem_cons_of_mem _ hk) + +end CompPoly.CPolynomial.NTTFast.Parallel + +namespace CompPoly.CPolynomial.NTTFast.Parallel +variable {R : Type*} + +/-- Descending fused passes for a forward transform. -/ +def forwardPasses [Field R] (D : NTT.Domain R) : List Nat := + (List.range (D.logN / 2)).map (fun pass ↦ D.logN - 1 - 2 * pass - 1) + +private theorem blockSize_eq (low : Nat) : 2 ^ (low + 2) = 4 * 2 ^ low := by + rw [pow_add] + omega + +private theorem forwardPasses_eq [Field R] [DecidableEq R] (D : NTT.Domain R) + (tw : Array (Array R)) (a : Array R) (ha : a.size = D.n) : + runPasses false tw (forwardPasses D) a = + (List.range (D.logN / 2)).foldl (fun a pass ↦ + let high := D.logN - 1 - 2 * pass + let low := high - 1 + Plan.stageBlocksDIF (tw.getD high #[]) (tw.getD low #[]) + (2 ^ (low + 2)) (2 ^ low) (D.n / (2 ^ (low + 2))) 0 a) a := by + simp only [runPasses, forwardPasses, List.foldl_map] + apply Plan.foldl_range_congr_inv (p := fun b : Array R ↦ b.size = D.n) + · intro pass hp b hb + have hhigh : (D.logN - 1 - 2 * pass - 1) + 1 = D.logN - 1 - 2 * pass := by omega + simp only [step, Bool.false_eq_true, ↓reduceIte, hb, hhigh, blockSize_eq] + · intro pass hp b hb + simpa only [size_step] using hb + · exact ha + +/-- Splitting forward passes retains the checked stage loop, including odd logs. -/ +theorem forwardStages_eq [Field R] [DecidableEq R] (D : NTT.Domain R) + (tw : Array (Array R)) (a : Array R) (ha : a.size = D.n) : + (if D.logN % 2 = 1 then + Plan.butterflyStageDIFWithTwiddles D 0 (tw.getD 0 #[]) + (runPasses false tw (forwardPasses D) a) + else runPasses false tw (forwardPasses D) a) = Plan.checkedStages D tw a := by + rw [forwardPasses_eq D tw a ha] + by_cases h : D.logN % 2 = 1 <;> simp [Plan.checkedStages, List.range_eq_range', h] + +end CompPoly.CPolynomial.NTTFast.Parallel diff --git a/CompPoly/Univariate/NTTFast/Permutation.lean b/CompPoly/Univariate/NTTFast/Permutation.lean new file mode 100644 index 00000000..02eaf116 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Permutation.lean @@ -0,0 +1,59 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTT.Transform +public import CompPoly.Data.Array.Involution + +/-! # Bit reversal as an array index involution -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTT.Transform +private theorem testBit_one (i : Nat) : (1 : Nat).testBit i = decide (i = 0) := by + simpa only [Nat.pow_zero, eq_comm] using (Nat.testBit_two_pow (n := 0) (m := i)) + +/-- Reversal reads bit positions in the opposite order within the selected width. -/ +theorem bitRevNat_testBit (bits i j : Nat) : + (bitRevNat bits i).testBit j = (decide (j < bits) && i.testBit (bits - 1 - j)) := by + induction bits generalizing i j with + | zero => simp only [bitRevNat, Nat.zero_testBit, Nat.not_lt_zero, decide_false, Bool.false_and] + | succ bits ih => + simp only [bitRevNat, Nat.testBit_or, Nat.testBit_shiftLeft, Nat.testBit_and, + testBit_one, ih, Nat.testBit_shiftRight] + by_cases hj : j < bits + · have hn : ¬ j ≥ bits := by omega + have hj' : j < bits + 1 := by omega + simp only [hj, hn, hj', decide_true, decide_false, Bool.false_and, Bool.true_and, + Bool.false_or] + congr 1 + omega + · by_cases he : j = bits + · subst j + simp only [Nat.le_refl, decide_true, Nat.sub_self, Bool.and_true, + Nat.lt_irrefl, decide_false, Bool.false_and, Bool.or_false, + Nat.lt_succ_self, Bool.true_and, Nat.add_sub_cancel] + · have hn : j ≥ bits := by omega + have hj' : ¬ j < bits + 1 := by omega + have hsub : j - bits ≠ 0 := by omega + simp only [hj, hn, hj', hsub, decide_false, decide_true, + Bool.and_false, Bool.false_and, Bool.or_false] + +/-- Reversing twice restores every index within the selected width. -/ +theorem bitRevNat_involutive (bits i : Nat) (hi : i < 2 ^ bits) : + bitRevNat bits (bitRevNat bits i) = i := by + apply Nat.eq_of_testBit_eq + intro j + rw [bitRevNat_testBit] + by_cases hj : j < bits + · have hh : bits - 1 - j < bits := by omega + rw [bitRevNat_testBit] + simp only [hj, hh, decide_true, Bool.true_and] + congr 1 + omega + · have hpow : 2 ^ bits ≤ 2 ^ j := Nat.pow_le_pow_right (by decide) (by omega) + simp only [hj, decide_false, Bool.false_and, + Nat.testBit_lt_two_pow (lt_of_lt_of_le hi hpow)] +end CompPoly.CPolynomial.NTT.Transform diff --git a/CompPoly/Univariate/NTTFast/Reverse32.lean b/CompPoly/Univariate/NTTFast/Reverse32.lean new file mode 100644 index 00000000..45b34318 --- /dev/null +++ b/CompPoly/Univariate/NTTFast/Reverse32.lean @@ -0,0 +1,101 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTT.Transform +import Mathlib.Tactic.IntervalCases + +/-! # Thirty-two-bit word reversal for bit-reversed indexing -/ + +@[expose] public section +namespace CompPoly.CPolynomial.NTTFast + +/-- Reverse the thirty-two bits of a word. -/ +@[inline] def reverse32 (x : UInt32) : UInt32 := + let x : UInt32 := ((x >>> (1 : UInt32)) &&& (0x55555555 : UInt32)) ||| ((x &&& (0x55555555 : + UInt32)) <<< (1 : UInt32)) + let x : UInt32 := ((x >>> (2 : UInt32)) &&& (0x33333333 : UInt32)) ||| ((x &&& (0x33333333 : + UInt32)) <<< (2 : UInt32)) + let x : UInt32 := ((x >>> (4 : UInt32)) &&& (0x0f0f0f0f : UInt32)) ||| ((x &&& (0x0f0f0f0f : + UInt32)) <<< (4 : UInt32)) + let x : UInt32 := ((x >>> (8 : UInt32)) &&& (0x00ff00ff : UInt32)) ||| ((x &&& (0x00ff00ff : + UInt32)) <<< (8 : UInt32)) + (x >>> (16 : UInt32)) ||| (x <<< (16 : UInt32)) + +/-- The mask-and-shift word permutation reverses exactly thirty-two bits. -/ +theorem reverse32_toBitVec (x : UInt32) : (reverse32 x).toBitVec = x.toBitVec.reverse := by + have h32 : (32 : BitVec 32).toNat = 32 := by decide + unfold reverse32 + simp only [UInt32.toBitVec_or, UInt32.toBitVec_and, UInt32.toBitVec_shiftRight, + UInt32.toBitVec_shiftLeft, UInt32.toBitVec_ofNat] + apply BitVec.eq_of_getLsbD_eq + intro i hi + interval_cases i <;> + simp only [BitVec.getLsbD_or, BitVec.getLsbD_and, BitVec.getLsbD_shiftLeft', + BitVec.ushiftRight_eq', BitVec.getLsbD_ushiftRight, BitVec.getLsbD_reverse, + BitVec.getMsbD_eq_getLsbD, BitVec.getLsbD_ofNat, BitVec.toNat_umod, h32, + BitVec.toNat_ofNat, Nat.reduceMod, Nat.reducePow, Nat.reduceAdd, Nat.reduceSub, + Nat.reduceLT, Nat.testBit_eq_decide_div_mod_eq, Nat.reduceDiv, Nat.reduceEqDiff, + decide_true, decide_false, Bool.false_and, Bool.true_and, Bool.and_false, Bool.and_true, + Bool.false_or, Bool.or_false, Bool.not_true, Bool.not_false] + + +/-- The low `bits` input bits appear in reversed order and all higher output bits vanish. -/ +theorem bitRevNat_testBit (bits i k : Nat) : + (NTT.Transform.bitRevNat bits i).testBit k = + if k < bits then i.testBit (bits - 1 - k) else false := by + have hone (t : Nat) : Nat.testBit 1 t = decide (t = 0) := by + simpa only [Nat.pow_zero, eq_comm] using (Nat.testBit_two_pow (n := 0) (m := t)) + induction bits generalizing i k with + | zero => simp only [NTT.Transform.bitRevNat, Nat.zero_testBit, Nat.not_lt_zero, ite_false] + | succ bits ih => + rw [NTT.Transform.bitRevNat, Nat.testBit_or, Nat.testBit_shiftLeft, + Nat.testBit_and, hone, ih] + by_cases hk : k < bits + · have hkle : ¬k ≥ bits := by omega + have hk1 : k < bits + 1 := by omega + have he : 1 + (bits - 1 - k) = bits + 1 - 1 - k := by omega + simp only [hk, hk1, hkle, decide_false, Bool.false_and, Bool.false_or, + ite_true, Nat.testBit_shiftRight, he] + · by_cases hke : k = bits + · subst k + simp only [Nat.le_refl, decide_true, Nat.sub_self, Bool.true_and, + Nat.lt_irrefl, ite_false, Bool.or_false, Nat.lt_succ_self, ite_true, + Nat.add_sub_cancel, Nat.testBit_shiftRight, Bool.and_true] + · have hkl : k ≥ bits := by omega + have hkn : ¬k < bits + 1 := by omega + have hks : ¬k - bits = 0 := by omega + simp only [hk, hkl, hkn, hks, decide_true, decide_false, Bool.and_false, + Bool.or_false, ite_false] + +/-- A shifted thirty-two-bit reversal computes the transform's bounded index permutation. -/ +theorem reverse32_shift_eq_bitRevNat (bits i : Nat) (hb : 0 < bits) (h32 : bits ≤ 32) : + (reverse32 i.toUInt32 >>> (32 - bits).toUInt32).toNat = + NTT.Transform.bitRevNat bits i := by + have hs : 32 - bits < 32 := by omega + have hword : (32 - bits).toUInt32.toBitVec.toNat = 32 - bits := by + simp only [Nat.toUInt32_eq, UInt32.toBitVec_ofNat', BitVec.toNat_ofNat] + exact Nat.mod_eq_of_lt (by omega) + have hmod : ((32 - bits).toUInt32.toBitVec % (32 : BitVec 32)).toNat = 32 - bits := by + rw [BitVec.toNat_umod, hword] + change (32 - bits) % 32 = 32 - bits + exact Nat.mod_eq_of_lt hs + apply Nat.eq_of_testBit_eq + intro k + rw [← UInt32.toNat_toBitVec, BitVec.testBit_toNat, UInt32.toBitVec_shiftRight, + reverse32_toBitVec, BitVec.ushiftRight_eq', BitVec.getLsbD_ushiftRight, hmod, + BitVec.getLsbD_reverse, BitVec.getMsbD_eq_getLsbD, bitRevNat_testBit] + by_cases hk : k < bits + · have ht : 32 - bits + k < 32 := by omega + have he : 32 - 1 - (32 - bits + k) = bits - 1 - k := by omega + have hi : bits - 1 - k < 32 := by omega + simp only [ht, hk, he, decide_true, Bool.true_and, ite_true, + Nat.toUInt32_eq, UInt32.toBitVec_ofNat', BitVec.getLsbD_ofNat] + simp only [hi, decide_true, Bool.true_and] + · have ht : ¬32 - bits + k < 32 := by omega + simp only [ht, hk, decide_false, Bool.false_and, ite_false] + +end CompPoly.CPolynomial.NTTFast diff --git a/README.md b/README.md index a8bacf82..bd8786fd 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ import CompPoly - **Multilinear (`CMlPolynomial`, `CMlPolynomialEval`)**: coefficient and Boolean-hypercube evaluation representations, basis conversions, and equivalence to Mathlib's multilinear polynomial surface. - **Univariate (`CPolynomial`)**: full ring structure, core operations (`C`, `X`, `monomial`, `coeff`, `eval`, `eval₂`, degree/leading/support), `ringEquiv` to Mathlib `Polynomial`, Lagrange interpolation (`basis`, `interpolate`, `interpolatePow`), and fixed-domain barycentric interpolation for repeated-query evaluation. - **Bivariate (`CBivariate`)**: specialized `CPolynomial (CPolynomial R)` API with `X`, `Y`, `monomialXY`, evaluation, leading coefficients, `swap`, and equivalence to `Polynomial (Polynomial R)`. -- **Fields**: broad set of finite-field instances (BabyBear, KoalaBear, Goldilocks, Mersenne31, Secp256k1, and the BN254/BLS12-381/BLS12-377 scalar fields), native-word Montgomery arithmetic for the 31-bit and 255-bit-and-below primes, a verified single-word `UInt64` Goldilocks implementation, computable extension fields `F[X]/f` for an arbitrary monic modulus, binary tower support, and additive NTT infrastructure. +- **Fields**: broad set of finite-field instances (BabyBear, KoalaBear, Goldilocks, Mersenne31, Secp256k1, and the BN254/BLS12-381/BLS12-377 scalar fields), native-word Montgomery arithmetic for the 31-bit primes and for primes up to 256 bits, a verified single-word `UInt64` Goldilocks implementation, computable extension fields `F[X]/f` for an arbitrary monic modulus, binary tower support, and additive NTT infrastructure. - **Coding theory**: Reed-Solomon encoding identified with the forward NTT, Gao's unique decoder with soundness/completeness and a farness certificate on refusal, and a Guruswami-Sudan list decoder with pluggable interpolation and root-finding backends, all proved correct. - **Linear algebra**: executable dense matrices (Gauss-Jordan, homogeneous kernels) and polynomial matrices with Mulders-Storjohann shifted row reduction, each with a proven-equal allocation-efficient fast path. diff --git a/ROADMAP.md b/ROADMAP.md index 7c16e14f..2863f814 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -109,14 +109,16 @@ CompPoly aims to be the premier formally verified library for computable polynom - ✅ Single-word `UInt32` Montgomery carrier for 31-bit primes (`Montgomery/Native32.lean`, `Montgomery/Native32Field.lean`, `Mont32Field`), instantiated by `BabyBear/Fast.lean` and `KoalaBear/Fast.lean` - - ✅ Eight-limb Montgomery carrier with CIOS multiplication for moduli below - `2^255` (`Montgomery/Native64x8*.lean`, `Mont64x8Field`), instantiated by - `BN254/Fast.lean`, `BLS12_381/Fast.lean`, and `BLS12_377/Fast.lean` + - ✅ Four-limb Montgomery carrier with 64-bit-radix CIOS multiplication for any + prime modulus below `2^256` (`Montgomery/Native64x4*.lean`, `Mont64x4Field`), + instantiated by `BN254/Fast.lean`, `BLS12_381/Fast.lean`, `BLS12_377/Fast.lean`, + `Pasta/Fast.lean` and `Secp256k1/Fast.lean`; the earlier eight-limb carrier + (`Montgomery/Native64x8*.lean`) remains behind `ScalarFft.lean` - ✅ Single-word `UInt64` carriers for 64-bit and 31-bit primes outside the Montgomery bounds (`Goldilocks/Fast.lean`, `Mersenne31/Fast.lean`), reducing via the modulus identity rather than Montgomery residues - - ✅ Checked binary-GCD inversion for the eight-limb fields - (`Montgomery/Native64x8Inv.lean`, [eprint 2020/972](https://eprint.iacr.org/2020/972)), + - ✅ Checked binary-GCD inversion for the four-limb fields + (`Montgomery/Native64x4Inv.lean`, [eprint 2020/972](https://eprint.iacr.org/2020/972)), benchmarked against `ZMod` extended Euclid and Fermat in `fields-mont64x8-*-inv` - 🔄 64-bit-radix Montgomery layer, so Goldilocks and Hachi (`2^32 - 99`) gain a `FastField` base; `Mont32Field` requires modulus < `2^31` @@ -153,7 +155,7 @@ CompPoly aims to be the premier formally verified library for computable polynom - ✅ Basic, reproducible evaluation benchmark executable (`lake exe CompPolyBench`; see `bench/README.md`) - ✅ CI build/run with artifact upload (GitHub Actions `lean_action_ci.yml`) - ✅ Operation-level coverage: base-field `mul`/`add`/`inv`/`pow` over - KoalaBear, BabyBear, Mersenne31 and Goldilocks, the eight-limb Montgomery + KoalaBear, BabyBear, Mersenne31 and Goldilocks, the four-limb Montgomery multiply, the binary tower's table-driven kernels, the standalone multiplicative NTT over `n = 2^8 … 2^16`, Reed-Solomon encoding, and the schoolbook/NTT crossover. Field and kernel rows are chained and reported @@ -305,6 +307,7 @@ CompPoly aims to be the premier formally verified library for computable polynom encoder 6.7× - Mixed-radix NTT for subgroups whose order is not a power of two: not planned without a consumer. The subproduct tree already covers such node sets + - ✅ Batched coset low-degree extension of packed row-major matrices over any 32-bit Montgomery field, in the bit-reversed row order of Plonky3's two-adic FRI commitment (`NTTFast/Packed/CosetLde.lean`). `Native.joinRows_cosetLde` proves every column's extension equal to its Lagrange interpolant evaluated on the coset (`NTTFast/Packed/CosetLdeCorrectness.lean`). Benchmarked against Plonky3's `coset_lde_batch` with `--suite lde` 4. **Proof ergonomics: simp/grind sets + tactics** - 🔄 Identify rewrite bottlenecks when porting Mathlib poly proofs → CompPoly. Recorded in [`docs/wiki/representations-and-bridges.md`](docs/wiki/representations-and-bridges.md#proof-api-simp-and-grind-sets): diff --git a/bench/CompPolyBench/Fields/Arith.lean b/bench/CompPolyBench/Fields/Arith.lean index 3044d7bb..723b06b0 100644 --- a/bench/CompPolyBench/Fields/Arith.lean +++ b/bench/CompPolyBench/Fields/Arith.lean @@ -1,7 +1,7 @@ /- Copyright (c) 2026 CompPoly Contributors. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Derek Sorensen +Authors: Derek Sorensen, Gregor Mitscha-Baude -/ module @@ -13,7 +13,7 @@ public import CompPolyBench.Harness.Chain `mul`, `add`, `inv` and `pow` over the four small prime fields a STARK prover spends its time in — KoalaBear, BabyBear, Mersenne31 and Goldilocks — and -`mul` over the three eight-limb pairing scalar fields, BN254, BLS12-381 and +`mul` over the three four-limb pairing scalar fields, BN254, BLS12-381 and BLS12-377. Each group runs the canonical `ZMod` implementation beside the verified native-word one on the same inputs, so the group digest cross-checks the two. @@ -47,8 +47,9 @@ Reported separately, as Plonky3's `benchmark_mul_latency` and `benchmark_mul_throughput` are, because they are different numbers and a prover is bounded by different ones in different places. The latency row chains the operation against a fixed operand, so each step waits on the last. The -throughput row runs ten independent accumulators, so the pipeline can overlap -them. `mul` and `add` carry both; `inv` and `pow` carry latency only, matching +throughput row uses parallel accumulators: two fixed-operand chains for BN254, +ten lanes for the other fields. `mul` and `add` carry both; `inv` and `pow` carry +latency only, matching the peer, and use a shorter chain because one operation already costs tens of multiplications. -/ @@ -70,7 +71,7 @@ orders of magnitude slower again. Still long enough that the harness floor is under a thousandth of the row. -/ def expChainRounds : Nat := unrollBlock -/-- Latency-chain depth for the eight-limb scalar fields. +/-- Latency-chain depth for the four-limb scalar fields. A quarter of `chainRounds`: a 256-bit Montgomery multiply is an order of magnitude more than a 32-bit one and the canonical row three further orders, @@ -136,20 +137,35 @@ def chainShape (rounds : Nat) : String := (fun i ↦ chainLatency op rounds (rep.pool.getD (i % fieldPoolSize) rep.constant)) rep.checksum (sink := rep.sink) -/-- Time ten independent chains of `op` over one representation. -/ +/-- Throughput patterns: small scalar ring, or two independent multi-limb chains. -/ +inductive ThroughputPattern where + | ring10 + | parallel2 + +/-- Number of arithmetic operations, excluding the final combination. -/ +def ThroughputPattern.units (pattern : ThroughputPattern) (rounds : Nat) : Nat := + match pattern with + | .ring10 => throughputUnitsOf rounds + | .parallel2 => 8 * (rounds / 4) + +/-- Time parallel chains of `op` over one representation. -/ @[specialize] def chainThroughputRow {F : Type} (fieldTag opTag method cls : String) - (rounds : Nat) (rep : ChainRep F) (op : F → F → F) (preset : BenchPreset) : IO BenchRecord := + (rounds : Nat) (rep : ChainRep F) (op : F → F → F) (preset : BenchPreset) + (pattern : ThroughputPattern) : IO BenchRecord := + let constant := rep.constant runTimedSpec { name := s!"{fieldTag}-{opTag}-{rep.suffix}", representation := rep.representation, - method := method, field := rep.field, inputShape := chainShape (throughputUnitsOf rounds), - digestIterations := digestPeriod fieldPoolSize, workUnits := throughputUnitsOf rounds, + method := method, field := rep.field, inputShape := chainShape (pattern.units rounds), + digestIterations := digestPeriod fieldPoolSize, workUnits := pattern.units rounds, digestClass := cls } preset (fun i ↦ let seed (k : Nat) : F := rep.pool.getD ((i + k) % fieldPoolSize) rep.constant - chainThroughput op op rounds - (seed 0) (seed 1) (seed 2) (seed 3) (seed 4) - (seed 5) (seed 6) (seed 7) (seed 8) (seed 9)) + match pattern with + | .ring10 => chainThroughput op op rounds + (seed 0) (seed 1) (seed 2) (seed 3) (seed 4) + (seed 5) (seed 6) (seed 7) (seed 8) (seed 9) + | .parallel2 => chainThroughputPair op rounds constant (seed 0) (seed 1)) rep.checksum (sink := rep.sink) /-- Time a binary field operation, both shapes, over both representations. -/ @@ -157,7 +173,7 @@ def chainShape (rounds : Nat) : String := (latencyRounds tputRounds : Nat) (slow : ChainRep S) (slowOp : S → S → S) (fast : ChainRep F) (fastOp : F → F → F) - (preset : BenchPreset) : IO BenchGroup := do + (preset : BenchPreset) (pattern : ThroughputPattern := .ring10) : IO BenchGroup := do -- Bound to locals, not read through `slow.constant` inside the lambda. A -- projection there is lifted into the operation itself, so the chain pays a -- `lean_ctor_get` and an unbox per operation. Out-of-order execution hides @@ -170,9 +186,9 @@ def chainShape (rounds : Nat) : String := let fastLatency ← chainLatencyRow fieldTag opTag s!"{opTag} (latency)" "latency" latencyRounds fast (fun x ↦ fastOp x fastConstant) preset let slowThroughput ← chainThroughputRow fieldTag opTag s!"{opTag} (throughput)" - "throughput" tputRounds slow slowOp preset + "throughput" tputRounds slow slowOp preset pattern let fastThroughput ← chainThroughputRow fieldTag opTag s!"{opTag} (throughput)" - "throughput" tputRounds fast fastOp preset + "throughput" tputRounds fast fastOp preset pattern pure { groupKey := groupKey, title := title, records := #[slowLatency, fastLatency, slowThroughput, fastThroughput] } @@ -412,15 +428,16 @@ private def runGoldilocksPow (preset : BenchPreset) (gen : StdGen) : fast Goldilocks.Fast.add (Goldilocks.Fast.pow · powExponent) preset pure (group, gen) -/-! ### Eight-limb pairing scalar fields +/-! ### Four-limb pairing scalar fields -`mul` only. Inversion over these carriers already has a group of its own in -`Fields/Montgomery.lean`, which compares three algorithms rather than two -representations. Both rows need an explicit sink: the canonical value is a -254- to 255-bit bignum, and `sinkMont64x8` reads two limbs instead of +BN254 covers add, mul, inv and pow; the other carriers cover mul. The inversion +groups in `Fields/Montgomery.lean` compare algorithms on individual inputs, +whereas the BN254 row here measures a dependent chain using checked binary GCD. +Both representations need an explicit sink: the canonical value is a +254- to 255-bit bignum, and `sinkMont64x4` reads two limbs instead of reassembling one. -/ -/-- BN254 scalar operands, canonical and eight-limb, from one pool. -/ +/-- BN254 scalar operands, canonical and four-limb, from one pool. -/ private def bn254Reps (gen : StdGen) : ChainRep BN254.ScalarField × ChainRep BN254.Fast.ScalarField × StdGen := let (values, gen) := (zmodArray BN254.scalarFieldSize fieldPoolSize false).run gen @@ -428,12 +445,12 @@ private def bn254Reps (gen : StdGen) : let fastPool := bn254FastArray pool ({ representation := "ZMod", field := "BN254.ScalarField", suffix := "zmod", pool := pool, constant := pool.getD 0 1, checksum := checksumZMod, sink := sinkZMod }, - { representation := "Limbs8", field := "BN254.Fast.ScalarField", suffix := "fast", + { representation := "Limbs4", field := "BN254.Fast.ScalarField", suffix := "fast", pool := fastPool, constant := fastPool.getD 0 1, checksum := checksumBn254Fast, - sink := sinkMont64x8 }, + sink := sinkMont64x4 }, gen) -/-- BLS12-381 scalar operands, canonical and eight-limb, from one pool. -/ +/-- BLS12-381 scalar operands, canonical and four-limb, from one pool. -/ private def bls12_381Reps (gen : StdGen) : ChainRep BLS12_381.ScalarField × ChainRep BLS12_381.Fast.ScalarField × StdGen := let (values, gen) := (zmodArray BLS12_381.scalarFieldSize fieldPoolSize false).run gen @@ -441,12 +458,12 @@ private def bls12_381Reps (gen : StdGen) : let fastPool := bls12_381FastArray pool ({ representation := "ZMod", field := "BLS12_381.ScalarField", suffix := "zmod", pool := pool, constant := pool.getD 0 1, checksum := checksumZMod, sink := sinkZMod }, - { representation := "Limbs8", field := "BLS12_381.Fast.ScalarField", suffix := "fast", + { representation := "Limbs4", field := "BLS12_381.Fast.ScalarField", suffix := "fast", pool := fastPool, constant := fastPool.getD 0 1, checksum := checksumBls12_381Fast, - sink := sinkMont64x8 }, + sink := sinkMont64x4 }, gen) -/-- BLS12-377 scalar operands, canonical and eight-limb, from one pool. -/ +/-- BLS12-377 scalar operands, canonical and four-limb, from one pool. -/ private def bls12_377Reps (gen : StdGen) : ChainRep BLS12_377.ScalarField × ChainRep BLS12_377.Fast.ScalarField × StdGen := let (values, gen) := (zmodArray BLS12_377.scalarFieldSize fieldPoolSize false).run gen @@ -454,9 +471,35 @@ private def bls12_377Reps (gen : StdGen) : let fastPool := bls12_377FastArray pool ({ representation := "ZMod", field := "BLS12_377.ScalarField", suffix := "zmod", pool := pool, constant := pool.getD 0 1, checksum := checksumZMod, sink := sinkZMod }, - { representation := "Limbs8", field := "BLS12_377.Fast.ScalarField", suffix := "fast", + { representation := "Limbs4", field := "BLS12_377.Fast.ScalarField", suffix := "fast", pool := fastPool, constant := fastPool.getD 0 1, checksum := checksumBls12_377Fast, - sink := sinkMont64x8 }, + sink := sinkMont64x4 }, + gen) + +/-- secp256k1 scalar-field operands, canonical and four-limb, from one pool. -/ +private def secp256k1ScalarReps (gen : StdGen) : + ChainRep Secp256k1.ScalarField × ChainRep Secp256k1.Fast.ScalarField × StdGen := + let (values, gen) := (zmodArray Secp256k1.scalarFieldSize fieldPoolSize false).run gen + let pool := nonzeroPool values + let fastPool := secp256k1ScalarFastArray pool + ({ representation := "ZMod", field := "Secp256k1.ScalarField", suffix := "zmod", + pool := pool, constant := pool.getD 0 1, checksum := checksumZMod, sink := sinkZMod }, + { representation := "Limbs4", field := "Secp256k1.Fast.ScalarField", suffix := "fast", + pool := fastPool, constant := fastPool.getD 0 1, + checksum := checksumSecp256k1ScalarFast, sink := sinkMont64x4 }, + gen) + +/-- secp256k1 base-field operands, canonical and four-limb, from one pool. -/ +private def secp256k1BaseReps (gen : StdGen) : + ChainRep Secp256k1.BaseField × ChainRep Secp256k1.Fast.BaseField × StdGen := + let (values, gen) := (zmodArray Secp256k1.baseFieldSize fieldPoolSize false).run gen + let pool := nonzeroPool values + let fastPool := secp256k1BaseFastArray pool + ({ representation := "ZMod", field := "Secp256k1.BaseField", suffix := "zmod", + pool := pool, constant := pool.getD 0 1, checksum := checksumZMod, sink := sinkZMod }, + { representation := "Limbs4", field := "Secp256k1.Fast.BaseField", suffix := "fast", + pool := fastPool, constant := fastPool.getD 0 1, + checksum := checksumSecp256k1BaseFast, sink := sinkMont64x4 }, gen) /-- Time BN254 scalar multiplication. -/ @@ -464,8 +507,38 @@ private def runBn254Mul (preset : BenchPreset) (gen : StdGen) : IO (BenchGroup × StdGen) := do let (slow, fast, gen) := bn254Reps gen let group ← runBinOpGroup "fields-bn254-mul" "BN254 scalar multiplication" - "bn254" "mul" heavyChainRounds heavyThroughputRounds - slow (· * ·) fast Montgomery.Native64x8.FastField.mul preset + "bn254" "mul" heavyChainRounds 160 + slow (· * ·) fast Montgomery.Native64x4.FastField.mul preset .parallel2 + pure (group, gen) + +/-- Time BN254 scalar addition. -/ +private def runBn254Add (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := bn254Reps gen + let group ← runBinOpGroup "fields-bn254-add" "BN254 scalar addition" + "bn254" "add" heavyChainRounds 160 + slow (· + ·) fast Montgomery.Native64x4.FastField.add preset .parallel2 + pure (group, gen) + +/-- Time BN254 scalar inversion through the checked binary-GCD implementation. -/ +private def runBn254Inv (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := bn254Reps gen + let group ← runUnOpGroup "fields-bn254-inv" "BN254 scalar inversion" + "bn254" "inv" "inv (xgcd)" "inv (checked binary GCD)" + slow (· + ·) (·⁻¹) + fast Montgomery.Native64x4.FastField.add Montgomery.Native64x4.FastField.invGcd preset + pure (group, gen) + +/-- Time BN254 scalar exponentiation with the shared fixed exponent. -/ +private def runBn254Pow (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := bn254Reps gen + let group ← runUnOpGroup "fields-bn254-pow" "BN254 scalar exponentiation" + "bn254" "pow" "pow (binary ladder)" "pow (binary ladder)" + slow (· + ·) (npowBinRec powExponent ·) + fast Montgomery.Native64x4.FastField.add + (Montgomery.Native64x4.FastField.pow · powExponent) preset pure (group, gen) /-- Time BLS12-381 scalar multiplication. -/ @@ -474,7 +547,7 @@ private def runBls12_381Mul (preset : BenchPreset) (gen : StdGen) : let (slow, fast, gen) := bls12_381Reps gen let group ← runBinOpGroup "fields-bls12-381-mul" "BLS12-381 scalar multiplication" "bls12-381" "mul" heavyChainRounds heavyThroughputRounds - slow (· * ·) fast Montgomery.Native64x8.FastField.mul preset + slow (· * ·) fast Montgomery.Native64x4.FastField.mul preset pure (group, gen) /-- Time BLS12-377 scalar multiplication. -/ @@ -483,7 +556,62 @@ private def runBls12_377Mul (preset : BenchPreset) (gen : StdGen) : let (slow, fast, gen) := bls12_377Reps gen let group ← runBinOpGroup "fields-bls12-377-mul" "BLS12-377 scalar multiplication" "bls12-377" "mul" heavyChainRounds heavyThroughputRounds - slow (· * ·) fast Montgomery.Native64x8.FastField.mul preset + slow (· * ·) fast Montgomery.Native64x4.FastField.mul preset + pure (group, gen) + +/-- Time BLS12-381 scalar addition. -/ +private def runBls12_381Add (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := bls12_381Reps gen + let group ← runBinOpGroup "fields-bls12-381-add" "BLS12-381 scalar addition" + "bls12-381" "add" chainRounds throughputRounds + slow (· + ·) fast Montgomery.Native64x4.FastField.add preset + pure (group, gen) + +/-- Time BLS12-377 scalar addition. -/ +private def runBls12_377Add (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := bls12_377Reps gen + let group ← runBinOpGroup "fields-bls12-377-add" "BLS12-377 scalar addition" + "bls12-377" "add" chainRounds throughputRounds + slow (· + ·) fast Montgomery.Native64x4.FastField.add preset + pure (group, gen) + +/-- Time secp256k1 scalar-field multiplication. -/ +private def runSecp256k1ScalarMul (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := secp256k1ScalarReps gen + let group ← runBinOpGroup "fields-secp256k1-scalar-mul" + "secp256k1 scalar-field multiplication" + "secp256k1-scalar" "mul" heavyChainRounds heavyThroughputRounds + slow (· * ·) fast Montgomery.Native64x4.FastField.mul preset + pure (group, gen) + +/-- Time secp256k1 scalar-field addition. -/ +private def runSecp256k1ScalarAdd (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := secp256k1ScalarReps gen + let group ← runBinOpGroup "fields-secp256k1-scalar-add" "secp256k1 scalar-field addition" + "secp256k1-scalar" "add" chainRounds throughputRounds + slow (· + ·) fast Montgomery.Native64x4.FastField.add preset + pure (group, gen) + +/-- Time secp256k1 base-field multiplication. -/ +private def runSecp256k1BaseMul (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := secp256k1BaseReps gen + let group ← runBinOpGroup "fields-secp256k1-base-mul" "secp256k1 base-field multiplication" + "secp256k1-base" "mul" heavyChainRounds heavyThroughputRounds + slow (· * ·) fast Montgomery.Native64x4.FastField.mul preset + pure (group, gen) + +/-- Time secp256k1 base-field addition. -/ +private def runSecp256k1BaseAdd (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + let (slow, fast, gen) := secp256k1BaseReps gen + let group ← runBinOpGroup "fields-secp256k1-base-add" "secp256k1 base-field addition" + "secp256k1-base" "add" chainRounds throughputRounds + slow (· + ·) fast Montgomery.Native64x4.FastField.add preset pure (group, gen) /-- Registry entries for the base-field arithmetic benchmarks. -/ @@ -525,7 +653,26 @@ def fieldArithTasks : List BenchTask := [ BenchTask.fromGroupRunner ⟨"fields-bls12-381-mul", "BLS12-381 scalar multiplication"⟩ runBls12_381Mul, BenchTask.fromGroupRunner ⟨"fields-bls12-377-mul", "BLS12-377 scalar multiplication"⟩ - runBls12_377Mul + runBls12_377Mul, + BenchTask.fromGroupRunner ⟨"fields-bn254-add", "BN254 scalar addition"⟩ + runBn254Add, + BenchTask.fromGroupRunner ⟨"fields-bn254-inv", "BN254 scalar inversion"⟩ + runBn254Inv, + BenchTask.fromGroupRunner ⟨"fields-bn254-pow", "BN254 scalar exponentiation"⟩ + runBn254Pow, + BenchTask.fromGroupRunner ⟨"fields-bls12-381-add", "BLS12-381 scalar addition"⟩ + runBls12_381Add, + BenchTask.fromGroupRunner ⟨"fields-bls12-377-add", "BLS12-377 scalar addition"⟩ + runBls12_377Add, + BenchTask.fromGroupRunner + ⟨"fields-secp256k1-scalar-mul", "secp256k1 scalar-field multiplication"⟩ + runSecp256k1ScalarMul, + BenchTask.fromGroupRunner ⟨"fields-secp256k1-scalar-add", "secp256k1 scalar-field addition"⟩ + runSecp256k1ScalarAdd, + BenchTask.fromGroupRunner ⟨"fields-secp256k1-base-mul", "secp256k1 base-field multiplication"⟩ + runSecp256k1BaseMul, + BenchTask.fromGroupRunner ⟨"fields-secp256k1-base-add", "secp256k1 base-field addition"⟩ + runSecp256k1BaseAdd ] end CompPolyBench diff --git a/bench/CompPolyBench/Fields/Binary/Tower.lean b/bench/CompPolyBench/Fields/Binary/Tower.lean index 6ec94b00..b6b6a2f3 100644 --- a/bench/CompPolyBench/Fields/Binary/Tower.lean +++ b/bench/CompPolyBench/Fields/Binary/Tower.lean @@ -11,11 +11,9 @@ public import CompPoly.Fields.Binary.Tower.Fast.Multilinear /-! # Binary tower field benchmarks -Times GF(2^128) multiplication and inversion, concrete tower vs packed-word -implementation, cross-checked by the group checksum. Sub-microsecond rows include the -harness's fixed per-iteration cost (roughly 0.4 us), so they are regression -indicators, not operation latencies. The coefficient-evaluation group compares the generic eager -product accumulator with concrete coefficient evaluation, using complete output words. +The coefficient-evaluation group compares the generic eager product accumulator with +concrete coefficient evaluation, using complete output words. Scalar operations live in +`Tower/Scalar.lean` and use workloads matched with Binius. -/ public section @@ -24,58 +22,6 @@ open ConcreteBinaryTower namespace CompPolyBench -/-- Input-shape label shared by the tower benchmarks. -/ -private def towerShape : String := "64 random 128-bit elements, pairwise" - -/-- Limb-fold checksum for packed tower elements; avoids building the 128-bit value. -/ -def checksumFastBT128 (x : Fast.FastBT128) : Nat := x.lo.toNat ^^^ x.hi.toNat - -/-- The same limb fold on the concrete representation. -/ -def checksumConcreteBt128 (x : ConcreteBTField 7) : Nat := - ConcreteBTField.toNat x % 2 ^ 64 ^^^ ConcreteBTField.toNat x >>> 64 - -/-- Pairwise operand sampler over a fixed pool. -/ -@[inline] private def towerSampler {E : Type} (xs : Array E) (one : E) : Nat → E × E := - fun i ↦ (xs.getD (i % xs.size) one, xs.getD ((i + 17) % xs.size) one) - -/-- Time one GF(2^128) operation over the spec and the packed implementation. -/ -@[specialize] private def runTowerGroup (groupKey title method : String) - (concreteOp : ConcreteBTField 7 → ConcreteBTField 7 → ConcreteBTField 7) - (fastOp : Fast.FastBT128 → Fast.FastBT128 → Fast.FastBT128) - (preset : BenchPreset) (gen : StdGen) : IO (BenchGroup × StdGen) := do - let (values, gen) := (randomNatArray 64 (2 ^ 128 - 1)).run gen - let concreteSample := towerSampler - (values.map fun n ↦ (fromNat n : ConcreteBTField 7)) (fromNat 1) - let fastSample := towerSampler (values.map Fast.FastBT128.ofNat) (.ofNat 1) - let checksumIterations := digestPeriod values.size - let concreteRecord ← runTimedSpec - { name := "tower-bt128", representation := "ConcreteBTField", - method := (method ++ " (ConcreteBTField)"), field := "GF(2^128)", inputShape := towerShape, - digestIterations := checksumIterations } - preset (fun i ↦ let (a, b) := concreteSample i; concreteOp a b) - checksumConcreteBt128 - let fastRecord ← runTimedSpec - { name := "tower-bt128-fast", representation := "FastBT128", - method := (method ++ " (FastBT128)"), field := "GF(2^128)", inputShape := towerShape, - digestIterations := checksumIterations } - preset (fun i ↦ let (a, b) := fastSample i; fastOp a b) checksumFastBT128 - pure ({ groupKey := groupKey, title := title, - records := #[concreteRecord, fastRecord] }, gen) - -/-- Run the GF(2^128) multiplication benchmark. -/ -private def runTowerMul (preset : BenchPreset) (gen : StdGen) : - IO (BenchGroup × StdGen) := do - runTowerGroup "fields-tower-bt128-mul" "Binary tower multiplication (GF(2^128))" "mul" - concrete_mul Fast.FastBT128.mul - preset gen - -/-- Run the GF(2^128) inversion benchmark. -/ -private def runTowerInv (preset : BenchPreset) (gen : StdGen) : - IO (BenchGroup × StdGen) := do - runTowerGroup "fields-tower-bt128-inv" "Binary tower inversion (GF(2^128))" "inv" - (fun a _ ↦ concrete_inv a) (fun a _ ↦ a.inv) - preset gen - /-- Compare eager packed coefficient accumulation with concrete evaluation at eight points. -/ private def runTowerCoeffEval (preset : BenchPreset) (gen : StdGen) : IO (BenchGroup × StdGen) := do @@ -106,12 +52,6 @@ private def runTowerCoeffEval (preset : BenchPreset) (gen : StdGen) : /-- Registry entries for the binary tower benchmarks. -/ def towerTasks : List BenchTask := [ - BenchTask.fromGroupRunner - ⟨"fields-tower-bt128-mul", "Binary tower multiplication (GF(2^128))"⟩ - runTowerMul, - BenchTask.fromGroupRunner - ⟨"fields-tower-bt128-inv", "Binary tower inversion (GF(2^128))"⟩ - runTowerInv, BenchTask.fromGroupRunner ⟨"fields-tower-bt128-coeff-eval", "Binary tower coefficient evaluation (GF(2^128))"⟩ runTowerCoeffEval diff --git a/bench/CompPolyBench/Fields/Binary/Tower/Scalar.lean b/bench/CompPolyBench/Fields/Binary/Tower/Scalar.lean index 8da7b900..86f98c7e 100644 --- a/bench/CompPolyBench/Fields/Binary/Tower/Scalar.lean +++ b/bench/CompPolyBench/Fields/Binary/Tower/Scalar.lean @@ -1,31 +1,22 @@ /- Copyright (c) 2026 CompPoly Contributors. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Derek Sorensen +Authors: Derek Sorensen, Gregor Mitscha-Baude -/ module public import CompPolyBench.Fields.Arith -public import CompPoly.Fields.Binary.Tower.FastDefs +public import CompPoly.Fields.Binary.Tower.Fast /-! -# Binary tower scalar kernels: table-driven against recursive - -`Tower/FastDefs.lean` carries each of `GF(2^8)` multiplication, `GF(2^64)` -multiplication and `GF(2^64)` inversion twice: once as the recursive tower -construction and once driven by a precomputed table. The two are proved equal, -and nothing measured which is faster — which is the whole reason the table -exists. - -All six are `UInt64 → UInt64` or `UInt64 → UInt64 → UInt64` on a bare machine -word, so they are the cleanest possible chain targets: no carrier to unbox and -no allocation. `fields-tower-bt128-mul` and `-inv` remain the group for the -128-bit packed representation against its `BitVec` spec; these are a level -below that. - -The table rows load a `ByteArray` at module initialisation, so their first -iterations touch cold memory. The calibration ramp doubles as warmup and is -long enough that this does not reach the samples. +# Binary tower operations matched with Binius + +Fast 8-, 64-, and 128-bit Fan–Paar tower arithmetic. Multiplication has a 64-step +latency chain and two independent 32-step throughput chains. Inversion mixes a fixed +word by XOR before each of 64 inversions. Squaring uses 63 dependent operations, +which is not a whole Frobenius cycle in any selected field. Only the final result is observed. +Lean's refinements live in `Tower/Fast.lean`; the Rust driver validates cross-language +chain results before timing. Field words encode tower coefficients, not field numerals. -/ public section @@ -34,75 +25,83 @@ open ConcreteBinaryTower namespace CompPolyBench -/-- Word-level operands for the tower's scalar kernels. - -`GF(2^64)` addition is `xor`, so `nonzeroPool` is not needed: the inversion -chain's step already mixes with a fixed operand, and `inv64 0 = 0` is only -reachable if the seed and the constant coincide. -/ -private def towerWordRep (suffix representation field : String) (pool : Array UInt64) : - ChainRep UInt64 := - { representation := representation, field := field, suffix := suffix, - pool := pool, constant := pool.getD 0 1, - checksum := fun x ↦ x.toNat, sink := u64Sink } - -/-- Draw a pool of nonzero words below `2 ^ bits` from the group's stream. - -The bound is not cosmetic. `mul8T_eq_mul8` (`Tower/Fast.lean:441`) holds only -for operands below `2 ^ 8`, because `mul8T` indexes a 65536-entry table with -`(a <<< 8) + b`; fed a full machine word it reads out of range, `get!` returns -zero, and the group reports a digest mismatch and a meaningless time. The -level-6 kernels take the whole word. -/ -private def towerWordPool (bits : Nat) (gen : StdGen) : Array UInt64 × StdGen := - let (values, gen) := (randomNatArray fieldPoolSize (2 ^ bits - 2)).run gen - (values.map fun n ↦ UInt64.ofNat (n + 1), gen) - -/-- Time `GF(2^8)` multiplication, recursive against table-driven. -/ -private def runTowerMul8 (preset : BenchPreset) (gen : StdGen) : - IO (BenchGroup × StdGen) := do - let (pool, gen) := towerWordPool 8 gen - let group ← runBinOpGroup "fields-tower-bt8-mul" - "Binary tower multiplication (GF(2^8)), table against recursive" - "tower-bt8" "mul" chainRounds throughputRounds - (towerWordRep "rec" "UInt64" "GF(2^8) recursive" pool) Fast.mul8 - (towerWordRep "table" "UInt64" "GF(2^8) table" pool) Fast.mul8T preset - pure (group, gen) - -/-- Time `GF(2^64)` multiplication, recursive against table-driven. -/ -private def runTowerMul64 (preset : BenchPreset) (gen : StdGen) : - IO (BenchGroup × StdGen) := do - let (pool, gen) := towerWordPool 64 gen - let group ← runBinOpGroup "fields-tower-bt64-mul" - "Binary tower multiplication (GF(2^64)), table against recursive" - "tower-bt64" "mul" chainRounds throughputRounds - (towerWordRep "rec" "UInt64" "GF(2^64) recursive" pool) Fast.mul64 - (towerWordRep "table" "UInt64" "GF(2^64) table" pool) Fast.mul64T preset - pure (group, gen) - -/-- Time `GF(2^64)` inversion, recursive against table-driven. -/ -private def runTowerInv64 (preset : BenchPreset) (gen : StdGen) : +/-- Shared deterministic word pool. Excluding zero and one avoids trivial mul constants. -/ +def towerBenchPool (bits : Nat) (gen : StdGen) : Array Nat × StdGen := + let (xs, gen) := (randomNatArray fieldPoolSize (2 ^ bits - 3)).run gen + (xs.map (· + 2), gen) + +/-- Sixty-three dependent squares, unrolled seven at a time to avoid an identity block. -/ +@[specialize] def towerSquareChain {F : Type} (square : F → F) (x : F) : F := + let rec @[specialize] go (n : Nat) (x : F) : F := + match n with + | 0 => x + | n + 1 => go n (square (square (square (square (square (square (square x))))))) + go 9 x + +/-- Run one operation on a known fast representation; Rust consumes identical fixtures. -/ +@[specialize] private def runTowerOperation {F : Type} (bits : Nat) (opTag : String) + (rep : ChainRep F) (add mul : F → F → F) (square inv : F → F) + (preset : BenchPreset) : IO BenchGroup := do + let tag := s!"tower-bt{bits}" + let constant := rep.constant + let records ← match opTag with + | "mul" => do + let latency ← chainLatencyRow tag opTag "mul (latency)" "latency" 64 rep + (fun x ↦ mul x constant) preset + let throughput ← chainThroughputRow tag opTag "mul (throughput)" "throughput" + 32 rep mul preset .parallel2 + pure #[latency, throughput] + | "square" => do + let row ← runTimedSpec + { name := s!"{tag}-square-fast", representation := rep.representation, + method := "square (63 dependent steps)", field := rep.field, + inputShape := chainShape 63, digestIterations := digestPeriod fieldPoolSize, + workUnits := 63, digestClass := "latency" } + preset + (fun i ↦ towerSquareChain square (rep.pool.getD (i % fieldPoolSize) constant)) + rep.checksum (sink := rep.sink) + pure #[row] + | "inv" => do + let row ← chainLatencyRow tag opTag "inv (XOR-mixed latency)" "latency" 64 rep + (fun x ↦ inv (add x constant)) preset + pure #[row] + | _ => throw <| IO.userError s!"unknown binary tower operation: {opTag}" + pure { groupKey := s!"fields-{tag}-{opTag}", title := s!"GF(2^{bits}) tower {opTag}", records } + +/-- Machine-word operands for a scalar tower level. -/ +private def wordRep (bits : Nat) (xs : Array Nat) : ChainRep UInt64 := + let pool := xs.map UInt64.ofNat + { representation := "UInt64 table kernels", field := s!"GF(2^{bits}) Fan–Paar tower", + suffix := "fast", pool, constant := pool.getD 0 2, + checksum := UInt64.toNat, sink := u64Sink } + +/-- Select a tower level outside the timed region, preserving statically known operations. -/ +private def runTower (bits : Nat) (opTag : String) (preset : BenchPreset) (gen : StdGen) : IO (BenchGroup × StdGen) := do - let (pool, gen) := towerWordPool 64 gen - let group ← runUnOpGroup "fields-tower-bt64-inv-word" - "Binary tower inversion (GF(2^64)), table against recursive" - "tower-bt64" "inv" "inv (recursive)" "inv (table)" - (towerWordRep "rec" "UInt64" "GF(2^64) recursive" pool) (· ^^^ ·) Fast.inv64 - (towerWordRep "table" "UInt64" "GF(2^64) table" pool) (· ^^^ ·) Fast.inv64T preset + let (xs, gen) := towerBenchPool bits gen + let group ← match bits with + | 8 => + runTowerOperation 8 opTag (wordRep 8 xs) (· ^^^ ·) + Fast.mul8T Fast.sq8T Fast.inv8T preset + | 64 => + runTowerOperation 64 opTag (wordRep 64 xs) (· ^^^ ·) + Fast.mul64T Fast.sq64T Fast.inv64T preset + | 128 => + let pool := xs.map Fast.FastBT128.ofNat + let rep : ChainRep Fast.FastBT128 := + { representation := "FastBT128", field := "GF(2^128) Fan–Paar tower", suffix := "fast", + pool, constant := pool.getD 0 (.ofNat 2), checksum := Fast.FastBT128.toNat, + sink := fun x ↦ x.lo ^^^ x.hi } + runTowerOperation 128 opTag rep Fast.FastBT128.add Fast.FastBT128.mul + Fast.FastBT128.square Fast.FastBT128.inv preset + | _ => throw <| IO.userError s!"unsupported binary tower width: {bits}" pure (group, gen) -/-- Registry entries for the tower's scalar kernels. -/ -def towerScalarTasks : List BenchTask := [ - BenchTask.fromGroupRunner - ⟨"fields-tower-bt8-mul", - "Binary tower multiplication (GF(2^8)), table against recursive"⟩ - runTowerMul8, - BenchTask.fromGroupRunner - ⟨"fields-tower-bt64-mul", - "Binary tower multiplication (GF(2^64)), table against recursive"⟩ - runTowerMul64, - BenchTask.fromGroupRunner - ⟨"fields-tower-bt64-inv-word", - "Binary tower inversion (GF(2^64)), table against recursive"⟩ - runTowerInv64 -] +/-- Three representative tower sizes, with shared group names in fixtures and Rust. -/ +def towerScalarTasks : List BenchTask := + [8, 64, 128].flatMap fun bits ↦ + ["mul", "square", "inv"].map fun op ↦ + BenchTask.fromGroupRunner + ⟨s!"fields-tower-bt{bits}-{op}", s!"GF(2^{bits}) tower {op}"⟩ (runTower bits op) end CompPolyBench diff --git a/bench/CompPolyBench/Fields/Extension.lean b/bench/CompPolyBench/Fields/Extension.lean index 93551b12..92e083fe 100644 --- a/bench/CompPolyBench/Fields/Extension.lean +++ b/bench/CompPolyBench/Fields/Extension.lean @@ -8,6 +8,7 @@ module public import CompPolyBench.Common public import CompPoly.Fields.BabyBear.Ext4 public import CompPoly.Fields.KoalaBear.Ext4 +public import CompPoly.Fields.KoalaBear.FastExt4 public import CompPoly.Fields.KoalaBear.Ext5 public import CompPoly.Fields.KoalaBear.Ext6 @@ -82,26 +83,55 @@ private def extSampler {F : Type*} [Ring F] {P : ExtensionParams F} let xs : Array (Ext P) := Array.ofFn (n := extPoolSize) fun i ↦ elem i.val fun i ↦ (xs.getD (i % extPoolSize) 1, xs.getD ((i + 17) % extPoolSize) 1) +/-- The fast element with the coordinates of a spec `KoalaBear.Ext4` element. -/ +private def toFastExt4 (a : KoalaBear.Ext4) : KoalaBear.Fast.Ext4 := + ⟨KoalaBear.Fast.ofField (Ext.coeffNat a 0), KoalaBear.Fast.ofField (Ext.coeffNat a 1), + KoalaBear.Fast.ofField (Ext.coeffNat a 2), KoalaBear.Fast.ofField (Ext.coeffNat a 3)⟩ + +/-- `checksumKoalaBearExt4` of the same element. -/ +private def checksumFastExt4 (x : KoalaBear.Fast.Ext4) : Nat := + [x.c0, x.c1, x.c2, x.c3].foldl (fun acc z ↦ acc + (Montgomery.Native32.FastField.toField z).val) 0 + +/-- Time the spec `KoalaBear.Ext4` and the fast `KoalaBear.Fast.Ext4` on the same operands, as +one group; the harness checks that both produce the same checksum. -/ +private def runKoalaBearExt4Pair (groupKey title name specMethod fastMethod : String) + (specOp : KoalaBear.Ext4 → KoalaBear.Ext4 → KoalaBear.Ext4) + (fastOp : KoalaBear.Fast.Ext4 → KoalaBear.Fast.Ext4 → KoalaBear.Fast.Ext4) + (preset : BenchPreset) (gen : StdGen) : IO (BenchGroup × StdGen) := do + let (values, gen) := (koalaBearArray 256 false).run gen + let sample := extSampler (P := KoalaBear.ext4Params.toExtensionParams) values + let fastXs : Array (KoalaBear.Fast.Ext4 × KoalaBear.Fast.Ext4) := + Array.ofFn (n := extPoolSize) fun i ↦ + let (a, b) := sample i.val + (toFastExt4 a, toFastExt4 b) + let spec ← runTimedSpec + { name := name, representation := "Extension.Ext", method := specMethod, + field := "KoalaBear.Ext4", inputShape := extShape 4, + digestIterations := digestPeriod extPoolSize } + preset (fun i ↦ let (a, b) := sample i; specOp a b) checksumKoalaBearExt4 + let fast ← runTimedSpec + { name := name ++ "-fast", representation := "KoalaBear.Fast.Ext4", method := fastMethod, + field := "KoalaBear.Ext4", inputShape := extShape 4, + digestIterations := digestPeriod extPoolSize } + preset (fun i ↦ let (a, b) := fastXs[i % extPoolSize]!; fastOp a b) + checksumFastExt4 + -- The raw Montgomery words: the default sink would convert to the spec field per iteration. + (sink := fun x ↦ (x.c0.val ^^^ x.c1.val ^^^ x.c2.val ^^^ x.c3.val).toUInt64) + pure ({ groupKey := groupKey, title := title, records := #[spec, fast] }, gen) + /-- Run the KoalaBear degree-4 multiplication benchmark. -/ private def runKoalaBearExt4Mul (preset : BenchPreset) (gen : StdGen) : - IO (BenchGroup × StdGen) := do - let (values, gen) := (koalaBearArray 256 false).run gen - runExtOp "fields-extension-koalabear-ext4-mul" - "Degree-4 extension multiplication (KoalaBear)" "extension-mul" "mul" "KoalaBear.Ext4" - (extShape 4) - checksumKoalaBearExt4 (extSampler (P := KoalaBear.ext4Params.toExtensionParams) values) (· * ·) - preset gen + IO (BenchGroup × StdGen) := + runKoalaBearExt4Pair "fields-extension-koalabear-ext4-mul" + "Degree-4 extension multiplication (KoalaBear)" "extension-mul" "mul" + "mul (one reduction per coefficient)" (· * ·) (· * ·) preset gen /-- Run the KoalaBear degree-4 inversion benchmark. -/ private def runKoalaBearExt4Inv (preset : BenchPreset) (gen : StdGen) : - IO (BenchGroup × StdGen) := do - let (values, gen) := (koalaBearArray 256 false).run gen - runExtOp "fields-extension-koalabear-ext4-inv" - "Degree-4 extension inversion (KoalaBear)" "extension-inv" "inv (Fermat)" "KoalaBear.Ext4" - (extShape 4) - checksumKoalaBearExt4 (extSampler (P := KoalaBear.ext4Params.toExtensionParams) values) - (fun a _ ↦ a⁻¹) - preset gen + IO (BenchGroup × StdGen) := + runKoalaBearExt4Pair "fields-extension-koalabear-ext4-inv" + "Degree-4 extension inversion (KoalaBear)" "extension-inv" "inv (Fermat)" "inv (norm)" + (fun a _ ↦ a⁻¹) (fun a _ ↦ a⁻¹) preset gen /-- Run the BabyBear degree-4 multiplication benchmark. -/ private def runBabyBearExt4Mul (preset : BenchPreset) (gen : StdGen) : diff --git a/bench/CompPolyBench/Fields/Inputs.lean b/bench/CompPolyBench/Fields/Inputs.lean index b588e003..ff7eecc3 100644 --- a/bench/CompPolyBench/Fields/Inputs.lean +++ b/bench/CompPolyBench/Fields/Inputs.lean @@ -10,13 +10,14 @@ public import CompPoly.Fields.Mersenne31 public import CompPoly.Fields.BN254 public import CompPoly.Fields.BLS12_381 public import CompPoly.Fields.BLS12_377 +public import CompPoly.Fields.Secp256k1 /-! # Field inputs, checksums and sinks Per-field benchmark scaffolding for the concrete fields that -`CompPolyBench.Common` does not already carry: Mersenne31 and the three -eight-limb Montgomery scalar fields. +`CompPolyBench.Common` does not already carry: Mersenne31 and the +four-limb Montgomery fields. These live here rather than in `CompPolyBench.Common` because that module is imported by every benchmark, and the field modules below are needed by a @@ -62,13 +63,13 @@ The carrier is an `abbrev` for a `Subtype`, so dot notation would resolve to def checksumMersenne31Fast (x : Mersenne31.Fast.Field) : Nat := Mersenne31.Fast.toNat x -/-! ## Eight-limb Montgomery scalar fields +/-! ## Four-limb Montgomery fields -BN254, BLS12-381 and BLS12-377. The canonical value is a 254- to 255-bit -bignum, so both representations need an explicit sink: `sinkZMod` for the -canonical side and `sinkMont64x8` for the fast one. -/ +BN254, BLS12-381, BLS12-377 and the two secp256k1 fields. The canonical value +is a 254- to 256-bit bignum, so both representations need an explicit sink: +`sinkZMod` for the canonical side and `sinkMont64x4` for the fast one. -/ -/-- Convert BN254 field inputs to the native eight-limb representation. -/ +/-- Convert BN254 field inputs to the native four-limb representation. -/ def bn254FastArray (xs : Array BN254.ScalarField) : Array BN254.Fast.ScalarField := xs.map BN254.Fast.ofField @@ -76,7 +77,7 @@ def bn254FastArray (xs : Array BN254.ScalarField) : Array BN254.Fast.ScalarField def checksumBn254Fast (x : BN254.Fast.ScalarField) : Nat := x.toNat -/-- Convert BLS12-381 field inputs to the native eight-limb representation. -/ +/-- Convert BLS12-381 field inputs to the native four-limb representation. -/ def bls12_381FastArray (xs : Array BLS12_381.ScalarField) : Array BLS12_381.Fast.ScalarField := xs.map BLS12_381.Fast.ofField @@ -85,7 +86,7 @@ def bls12_381FastArray (xs : Array BLS12_381.ScalarField) : def checksumBls12_381Fast (x : BLS12_381.Fast.ScalarField) : Nat := x.toNat -/-- Convert BLS12-377 field inputs to the native eight-limb representation. -/ +/-- Convert BLS12-377 field inputs to the native four-limb representation. -/ def bls12_377FastArray (xs : Array BLS12_377.ScalarField) : Array BLS12_377.Fast.ScalarField := xs.map BLS12_377.Fast.ofField @@ -94,14 +95,32 @@ def bls12_377FastArray (xs : Array BLS12_377.ScalarField) : def checksumBls12_377Fast (x : BLS12_377.Fast.ScalarField) : Nat := x.toNat -/-- Sink an eight-limb Montgomery element by two of its limbs. +/-- Convert secp256k1 scalar-field inputs to the native four-limb representation. -/ +def secp256k1ScalarFastArray (xs : Array Secp256k1.ScalarField) : + Array Secp256k1.Fast.ScalarField := + xs.map Secp256k1.Fast.ofScalarField + +/-- Convert a fast secp256k1 scalar-field element to a checksum word. -/ +def checksumSecp256k1ScalarFast (x : Secp256k1.Fast.ScalarField) : Nat := + x.toNat + +/-- Convert secp256k1 base-field inputs to the native four-limb representation. -/ +def secp256k1BaseFastArray (xs : Array Secp256k1.BaseField) : + Array Secp256k1.Fast.BaseField := + xs.map Secp256k1.Fast.ofBaseField + +/-- Convert a fast secp256k1 base-field element to a checksum word. -/ +def checksumSecp256k1BaseFast (x : Secp256k1.Fast.BaseField) : Nat := + x.toNat + +/-- Sink a four-limb Montgomery element by two of its limbs. `toNat` reassembles a 256-bit bignum, which costs more than the multiplication under test. The limbs are already unboxed `UInt64` fields of the carrier, so this is two loads and an exclusive or. Lossy by construction; the untimed digest is what establishes correctness. -/ -@[inline] def sinkMont64x8 {modulus : Nat} [Montgomery.Native64x8.Mont64x8Field modulus] - (x : Montgomery.Native64x8.FastField modulus) : UInt64 := - (Subtype.val x).l0 ^^^ (Subtype.val x).l7 +@[inline] def sinkMont64x4 {modulus : Nat} [Montgomery.Native64x4.Mont64x4Field modulus] + (x : Montgomery.Native64x4.FastField modulus) : UInt64 := + (Subtype.val x).l0 ^^^ (Subtype.val x).l3 end CompPolyBench diff --git a/bench/CompPolyBench/Fields/Montgomery.lean b/bench/CompPolyBench/Fields/Montgomery.lean index 6c9ff10b..8f5ed258 100644 --- a/bench/CompPolyBench/Fields/Montgomery.lean +++ b/bench/CompPolyBench/Fields/Montgomery.lean @@ -9,11 +9,12 @@ public import CompPolyBench.Common public import CompPoly.Fields.BN254 public import CompPoly.Fields.BLS12_381 public import CompPoly.Fields.BLS12_377 +public import CompPoly.Fields.Secp256k1 /-! # Scalar-field inversion benchmarks -Times inversion over the eight-limb Montgomery scalar fields. Each group runs three +Times inversion over the four-limb Montgomery scalar fields. Each group runs three implementations of the same operation on shared inputs: the canonical `ZMod` inverse, the checked binary-GCD inverse, and Fermat exponentiation, so the group checksum cross-checks all three. @@ -21,7 +22,7 @@ cross-checks all three. public section -open Montgomery.Native64x8 (FastField Mont64x8Field GcdData) +open Montgomery.Native64x4 (FastField Mont64x4Field GcdData) namespace CompPolyBench @@ -29,7 +30,7 @@ namespace CompPolyBench private def scalarInvShape : String := "256 random elements" /-- Time the three inversion implementations of one scalar field as a single group. -/ -private def runScalarInv (modulus : Nat) [Mont64x8Field modulus] [GcdData modulus] +private def runScalarInv (modulus : Nat) [Mont64x4Field modulus] [GcdData modulus] (groupKey title fieldName fastFieldName : String) (preset : BenchPreset) (gen : StdGen) : IO (BenchGroup × StdGen) := do let (values, gen) := (zmodArray modulus 256 false).run gen @@ -40,12 +41,12 @@ private def runScalarInv (modulus : Nat) [Mont64x8Field modulus] [GcdData modulu field := fieldName, inputShape := scalarInvShape, digestIterations := checksumIterations } preset (fun i ↦ (values.getD (i % values.size) 1)⁻¹) checksumZMod let gcdRecord ← runTimedSpec - { name := "scalar-inv-gcd", representation := "Mont64x8", method := "inv (binary GCD)", + { name := "scalar-inv-gcd", representation := "Mont64x4", method := "inv (binary GCD)", field := fastFieldName, inputShape := scalarInvShape, digestIterations := checksumIterations } preset (fun i ↦ (fastValues.getD (i % fastValues.size) 1).invGcd) (fun x ↦ x.toNat) let fermatRecord ← runTimedSpec - { name := "scalar-inv-fermat", representation := "Mont64x8", method := "inv (Fermat)", + { name := "scalar-inv-fermat", representation := "Mont64x4", method := "inv (Fermat)", field := fastFieldName, inputShape := scalarInvShape, digestIterations := checksumIterations } preset (fun i ↦ (fastValues.getD (i % fastValues.size) 1).inv) (fun x ↦ x.toNat) @@ -73,7 +74,21 @@ private def runBls12_377ScalarInv (preset : BenchPreset) (gen : StdGen) : "Scalar-field inversion (BLS12-377)" "BLS12_377.ScalarField" "BLS12_377.Fast.ScalarField" preset gen -/-- Registry entries for the scalar-field inversion benchmarks. -/ +/-- Run the secp256k1 scalar-field inversion benchmark. -/ +private def runSecp256k1ScalarInv (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + runScalarInv Secp256k1.scalarFieldSize "fields-mont64x8-secp256k1-scalar-inv" + "Scalar-field inversion (secp256k1)" "Secp256k1.ScalarField" "Secp256k1.Fast.ScalarField" + preset gen + +/-- Run the secp256k1 base-field inversion benchmark. -/ +private def runSecp256k1BaseInv (preset : BenchPreset) (gen : StdGen) : + IO (BenchGroup × StdGen) := do + runScalarInv Secp256k1.baseFieldSize "fields-mont64x8-secp256k1-base-inv" + "Base-field inversion (secp256k1)" "Secp256k1.BaseField" "Secp256k1.Fast.BaseField" + preset gen + +/-- Registry entries for the four-limb field inversion benchmarks. -/ def montgomeryInvTasks : List BenchTask := [ BenchTask.fromGroupRunner ⟨"fields-mont64x8-bn254-inv", "Scalar-field inversion (BN254)"⟩ @@ -83,7 +98,13 @@ def montgomeryInvTasks : List BenchTask := [ runBls12_381ScalarInv, BenchTask.fromGroupRunner ⟨"fields-mont64x8-bls12-377-inv", "Scalar-field inversion (BLS12-377)"⟩ - runBls12_377ScalarInv + runBls12_377ScalarInv, + BenchTask.fromGroupRunner + ⟨"fields-mont64x8-secp256k1-scalar-inv", "Scalar-field inversion (secp256k1)"⟩ + runSecp256k1ScalarInv, + BenchTask.fromGroupRunner + ⟨"fields-mont64x8-secp256k1-base-inv", "Base-field inversion (secp256k1)"⟩ + runSecp256k1BaseInv ] end CompPolyBench diff --git a/bench/CompPolyBench/Harness/Chain.lean b/bench/CompPolyBench/Harness/Chain.lean index 67be6a27..0fe0fd86 100644 --- a/bench/CompPolyBench/Harness/Chain.lean +++ b/bench/CompPolyBench/Harness/Chain.lean @@ -1,7 +1,7 @@ /- Copyright (c) 2026 CompPoly Contributors. All rights reserved. Released under Apache 2.0 license as described in the file LICENSE. -Authors: Derek Sorensen +Authors: Derek Sorensen, Gregor Mitscha-Baude -/ module @@ -196,4 +196,22 @@ bindings: see the note at the top of this file on `Prod` not erasing. -/ go n a4 b4 c4 d4 e4 f4 g4 h4 i4 j4 go (rounds / throughputUnroll) a b c d e f g h i j +/-- Two independent chains sharing a fixed operand, unrolled four rounds at a time. -/ +@[specialize] def chainThroughputPair {F : Type} (op : F → F → F) + (rounds : Nat) (constant a b : F) : F := + let rec @[specialize] go (n : Nat) (a b : F) : F := + match n with + | 0 => op a b + | n + 1 => + let a := op a constant + let b := op b constant + let a := op a constant + let b := op b constant + let a := op a constant + let b := op b constant + let a := op a constant + let b := op b constant + go n a b + go (rounds / 4) a b + end CompPolyBench diff --git a/bench/CompPolyBench/Univariate/Basic.lean b/bench/CompPolyBench/Univariate/Basic.lean index 53b91a09..7cb18a18 100644 --- a/bench/CompPolyBench/Univariate/Basic.lean +++ b/bench/CompPolyBench/Univariate/Basic.lean @@ -401,9 +401,11 @@ private def runKoalaBearUnivariateMonicRemainderMedium (preset : BenchPreset) (g /-- Benchmark dense Goldilocks univariate evaluation. -/ private def runGoldilocksUnivariateDense (preset : BenchPreset) (gen : StdGen) : IO (BenchGroup × StdGen) := do - runDenseUnivariateZMod - Goldilocks.fieldSize "univariate-dense-goldilocks" "goldilocks" "Goldilocks.Field" - "Goldilocks" preset gen + runDenseUnivariateWithFast + "univariate-dense-goldilocks" "Goldilocks" "Goldilocks.Field" "Goldilocks.Fast.Field" + (fun size ↦ zmodArray Goldilocks.fieldSize size false) goldilocksFastArray + checksumZMod checksumGoldilocksFast + preset gen /-- Benchmark dense BN254 univariate evaluation. -/ private def runBn254UnivariateDense (preset : BenchPreset) (gen : StdGen) : diff --git a/bench/CompPolyBench/Univariate/NTT/Transform.lean b/bench/CompPolyBench/Univariate/NTT/Transform.lean index d1bb24b7..fa06ef3c 100644 --- a/bench/CompPolyBench/Univariate/NTT/Transform.lean +++ b/bench/CompPolyBench/Univariate/NTT/Transform.lean @@ -8,7 +8,7 @@ module public import CompPolyBench.Univariate.Common public import CompPoly.Univariate.NTT.Forward public import CompPoly.Univariate.NTT.Inverse -public import CompPoly.Univariate.NTTFast.Plan +public import CompPoly.Univariate.NTTFast.ButterflyDIT /-! # Multiplicative NTT benchmarks diff --git a/bench/CompPolyFieldFixtures.lean b/bench/CompPolyFieldFixtures.lean new file mode 100644 index 00000000..5d33eb48 --- /dev/null +++ b/bench/CompPolyFieldFixtures.lean @@ -0,0 +1,61 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPolyBench.Fields.Binary.Tower.Scalar + +/-! +# Field comparison inputs + +Export the exact operand pools and chain parameters used by the existing field groups. +Rust consumes these before timing; cross-language result digests catch workload drift. +-/ + +public section + +open Lean CompPoly CompPolyBench + +/-- Fixed-width little-endian coordinate bytes, independent of runtime representation. -/ +def canonicalBytes (width value : Nat) : Array Nat := + (Array.range width).map fun i ↦ (value >>> (8 * i)) % 256 + +/-- Export the selected groups as JSONL on stdout. -/ +def main : IO Unit := do + for (field, tag, modulus, width, operations) in [ + ("koalabear", "koalabear", KoalaBear.fieldSize, 4, ["add", "mul", "inv", "pow"]), + ("mersenne31", "mersenne31", Mersenne31.fieldSize, 4, ["add", "mul", "inv", "pow"]), + ("goldilocks", "goldilocks", Goldilocks.fieldSize, 8, ["add", "mul", "inv", "pow"]), + ("bn254-scalar", "bn254", BN254.scalarFieldSize, 32, ["add", "mul", "inv", "pow"])] do + for operation in operations do + let key := s!"fields-{tag}-{operation}" + let (values, _) := (zmodArray modulus fieldPoolSize false).run (genFor key) + let inputs := values.map fun x ↦ canonicalBytes width (if x.val = 0 then 1 else x.val) + let json := Lean.Json.mkObj [ + ("encoding", toJson "field-coordinates-le-v1"), + ("basis", toJson "canonical-integer"), + ("group_key", toJson key), ("field", toJson field), + ("operation", toJson operation), ("modulus", toJson (canonicalBytes width modulus)), + ("inputs", toJson inputs), ("exponent", toJson powExponent), + ("latency_rounds", toJson + (if operation == "inv" || operation == "pow" then expChainRounds + else if tag == "bn254" then heavyChainRounds else chainRounds)), + ("throughput_rounds", toJson + (if tag == "bn254" then 160 else throughputRounds))] + IO.println json.compress + for bits in [8, 64, 128] do + for operation in ["mul", "square", "inv"] do + let field := s!"tower-bt{bits}" + let key := s!"fields-{field}-{operation}" + let (values, _) := towerBenchPool bits (genFor key) + IO.println <| (Lean.Json.mkObj [ + ("encoding", toJson "field-coordinates-le-v1"), + ("basis", toJson "fan-paar-tower"), + ("group_key", toJson key), ("field", toJson field), + ("operation", toJson operation), ("modulus", toJson (#[] : Array Nat)), + ("inputs", toJson (values.map (canonicalBytes (bits / 8)))), + ("exponent", toJson (0 : Nat)), + ("latency_rounds", toJson (if operation == "square" then 63 else 64 : Nat)), + ("throughput_rounds", toJson (32 : Nat))]).compress diff --git a/bench/CompPolyInterpolateBench.lean b/bench/CompPolyInterpolateBench.lean new file mode 100644 index 00000000..6daa8bd6 --- /dev/null +++ b/bench/CompPolyInterpolateBench.lean @@ -0,0 +1,81 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPolyBench.Common +public import CompPoly.Fields.KoalaBear.InterpolateCoset +public import CompPoly.Univariate.NTT.KoalaBear + +/-! # KoalaBear coset interpolation against Plonky3's `interpolate_coset` -/ + +public section +open CompPoly CompPolyBench + +/-- Read canonical 32-bit coordinates without silently reducing malformed fixtures. -/ +def interpolateCoordinates (bytes : ByteArray) : IO (Array KoalaBear.Fast.Field) := do + if bytes.size % 4 != 0 then throw <| IO.userError "unaligned interpolation fixture" + let mut values := Array.mkEmpty (bytes.size / 4) + for i in [:bytes.size / 4] do + let mut v := 0 + for j in [:4] do + v := v * 256 + bytes[4 * i + (3 - j)]!.toNat + if v ≥ 2130706433 then throw <| IO.userError "noncanonical interpolation coordinate" + values := values.push (KoalaBear.Fast.ofField (v : KoalaBear.Field)) + return values + +/-- The digest of all output coordinates, in the order Plonky3 lists them. -/ +def checksumExt4Array (xs : Array KoalaBear.Fast.Ext4) : Nat := + xs.foldl (fun acc e ↦ [e.c0, e.c1, e.c2, e.c3].foldl + (fun acc c ↦ mixChecksum acc (checksumKoalaBearFast c)) acc) 0 + +/-- Time `interpolateCosetPacked` or the proved reference `interpolateCoset` on a fixture: root, +shift, two points, then the row-major matrix. -/ +def main (args : List String) : IO UInt32 := do + let [path, size, widthArg, validate, impl] := args | + throw <| IO.userError + "usage: CompPolyInterpolateBench FIXTURE LOG_N WIDTH true|false packed|reference" + if impl != "packed" && impl != "reference" then throw <| IO.userError "invalid implementation" + let some logN := size.toNat? | throw <| IO.userError "invalid log size" + let some width := widthArg.toNat? | throw <| IO.userError "invalid width" + if validate != "true" && validate != "false" then throw <| IO.userError "invalid validation flag" + if h : logN ≤ KoalaBear.twoAdicity then + let n := 2 ^ logN + let words ← interpolateCoordinates (← IO.FS.readBinFile path) + if words.size != 10 + n * width then throw <| IO.userError "incorrect fixture length" + let ω := (CPolynomial.NTT.KoalaBear.fastDomainOfLogN logN h).omega + if words.getD 0 0 != ω then throw <| IO.userError "incorrect root" + let shift := words.getD 1 0 + let point (k : Nat) : KoalaBear.Fast.Ext4 := + ⟨words.getD (2 + 4 * k) 0, words.getD (3 + 4 * k) 0, words.getD (4 + 4 * k) 0, + words.getD (5 + 4 * k) 0⟩ + let points := #[point 0, point 1] + let evals := words.extract 10 words.size + -- The packed path reads Montgomery words, like Plonky3's matrix storage. + let packed := evals.foldl (fun b (x : KoalaBear.Fast.Field) ↦ + let w := x.val + b.push w.toUInt8 |>.push (w >>> 8).toUInt8 |>.push (w >>> 16).toUInt8 + |>.push (w >>> 24).toUInt8) (ByteArray.emptyWithCapacity (4 * evals.size)) + validateOnlyRef.set (validate == "true") + let spec : BenchSpec := if impl == "packed" then + { name := s!"interpolate-koalabear-{logN}-{width}-packed", representation := "ByteArray", + method := "base-field weights in blocks, lazy column sums, parallel row ranges", + field := "koalabear", inputShape := s!"{n} rows × {width} columns", + digestIterations := 2, digestClass := "interpolate" } + else + { name := s!"interpolate-koalabear-{logN}-{width}-reference", + representation := "Array KoalaBear.Fast.Field", + method := "proved reference: batch inversion, per-column sums", field := "koalabear", + inputShape := s!"{n} rows × {width} columns", digestIterations := 2, + digestClass := "interpolate" } + let run : Nat → Array KoalaBear.Fast.Ext4 := if impl == "packed" then + fun i ↦ Montgomery.Native32.interpolateCosetPacked logN ω shift width packed points[i % 2]! + else fun i ↦ Montgomery.Native32.interpolateCoset logN ω shift width evals points[i % 2]! + let row ← runTimedSpec spec .large run checksumExt4Array + (sink := fun xs ↦ (xs[0]!.c0.val.toUInt64) ^^^ (xs.size.toUInt64 <<< 32)) + let record : BenchRecord := { row with groupKey := s!"interpolate-koalabear-{logN}-{width}" } + IO.println record.toJsonLine + return 0 + else throw <| IO.userError "unsupported domain" diff --git a/bench/CompPolyLdeBench.lean b/bench/CompPolyLdeBench.lean new file mode 100644 index 00000000..1082bbd9 --- /dev/null +++ b/bench/CompPolyLdeBench.lean @@ -0,0 +1,77 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPolyBench.Univariate.Common +public import CompPoly.Univariate.NTTFast.Packed.Instances +public import CompPoly.Univariate.NTT.KoalaBear + +/-! # KoalaBear coset low-degree extension against Plonky3's `coset_lde_batch` -/ + +public section +open CompPoly CompPolyBench + +/-- Read canonical 32-bit coordinates without silently reducing malformed fixtures. -/ +def ldeCoordinates (bytes : ByteArray) : IO (Array KoalaBear.Fast.Field) := do + if bytes.size % 4 != 0 then throw <| IO.userError "unaligned LDE fixture" + let mut values := Array.mkEmpty (bytes.size / 4) + for i in [:bytes.size / 4] do + let mut v := 0 + for j in [:4] do + v := v * 256 + bytes[4 * i + (3 - j)]!.toNat + if v ≥ 2130706433 then throw <| IO.userError "noncanonical LDE coordinate" + values := values.push (KoalaBear.Fast.ofField (v : KoalaBear.Field)) + return values + +/-- Time `cosetLde` on a fixture: the root of the extended domain, the shift, then two row-major +matrices. -/ +def main (args : List String) : IO UInt32 := do + let [path, logRows, widthArg, bitsArg, validate] := args | + throw <| IO.userError "usage: CompPolyLdeBench FIXTURE LOG_H WIDTH ADDED_BITS true|false" + let some logH := logRows.toNat? | throw <| IO.userError "invalid log height" + let some width := widthArg.toNat? | throw <| IO.userError "invalid width" + let some bits := bitsArg.toNat? | throw <| IO.userError "invalid added bits" + if validate != "true" && validate != "false" then throw <| IO.userError "invalid validation flag" + if h : logH + bits ≤ KoalaBear.twoAdicity then + let domain := CPolynomial.NTT.KoalaBear.fastDomainOfLogN (logH + bits) h + let rows := 2 ^ logH + let words ← ldeCoordinates (← IO.FS.readBinFile path) + if words.size != 2 + 2 * rows * width then throw <| IO.userError "incorrect fixture length" + if words.getD 0 0 != domain.omega then throw <| IO.userError "incorrect root" + have h32 : domain.logN ≤ 32 := by + change logH + bits ≤ 32 + have : KoalaBear.twoAdicity = 24 := rfl + omega + have hu : 4 * domain.n < USize.size := by + have hp : 2 ^ (logH + bits) ≤ 2 ^ 24 := Nat.pow_le_pow_right (by decide) h + have := USize.le_size + change 4 * 2 ^ (logH + bits) < USize.size + omega + let plan := CPolynomial.NTTFast.Packed.LdePlan.ofDomain domain bits + (by change bits ≤ logH + bits; omega) (words.getD 1 0) h32 hu + -- The matrices are packed outside timing, as Montgomery words like Plonky3's storage, each + -- in one chunk of rows. + let inputs := #[words.extract 2 (2 + rows * width), words.extract (2 + rows * width) + words.size].map fun m ↦ #[CPolynomial.NTTFast.Packed.Native.encodeFields m] + let word (b : ByteArray) (i : Nat) : UInt64 := + (CPolynomial.NTTFast.Packed.Native.read b i).toUInt64 + validateOnlyRef.set (validate == "true") + let spec : BenchSpec := + { name := s!"lde-koalabear-{logH}-{width}-{bits}-packed", + representation := "Array ByteArray (row chunks)", + method := "transposed columns, inverse then per-coset DIF transforms, parallel tasks", + field := "koalabear", inputShape := s!"{rows} rows × {width} columns, blowup 2^{bits}", + digestIterations := 2, digestClass := "lde" } + -- The digest reads the joined rows, outside timing. + let row ← runTimedSpec spec .large + (fun i ↦ plan.cosetLde width inputs[i % 2]!) + (fun cs ↦ checksumArray checksumKoalaBearFast (CompPoly.PackedRepr.decode + (R := KoalaBear.Fast.Field) (CPolynomial.NTTFast.Packed.Native.joinRows cs))) + (sink := fun cs ↦ sinkStep (word cs[0]! 0) cs.size.toUInt64) + let record : BenchRecord := { row with groupKey := s!"lde-koalabear-{logH}-{width}-{bits}" } + IO.println record.toJsonLine + return 0 + else throw <| IO.userError "unsupported domain" diff --git a/bench/CompPolyNTTBench.lean b/bench/CompPolyNTTBench.lean new file mode 100644 index 00000000..bd56f222 --- /dev/null +++ b/bench/CompPolyNTTBench.lean @@ -0,0 +1,97 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPolyBench.Univariate.Common +public import CompPoly.Univariate.NTTFast.NaturalParallel +public import CompPoly.Fields.Montgomery.Native32Bytes +public import CompPoly.Univariate.NTTFast.Packed.Instances + +/-! # Natural-order KoalaBear NTT benchmarks against optimized Plonky3 -/ + +public section +open CompPoly CompPolyBench + +/-- Read canonical 32-bit coordinates without silently reducing malformed fixtures. -/ +def nttCoordinates (bytes : ByteArray) : IO (Array KoalaBear.Fast.Field) := do + if bytes.size % 4 != 0 then throw <| IO.userError "unaligned NTT fixture" + let mut values := Array.mkEmpty (bytes.size / 4) + for i in [:bytes.size / 4] do + let mut v := 0 + for j in [:4] do + v := v * 256 + bytes[4 * i + (3 - j)]!.toNat + if v ≥ 2130706433 then throw <| IO.userError "noncanonical NTT coordinate" + values := values.push (KoalaBear.Fast.ofField (v : KoalaBear.Field)) + return values + +/-- Export the certified root or time a transform on two varying fixture inputs. -/ +def main (args : List String) : IO UInt32 := do + if let ["--root", size] := args then + let some logN := size.toNat? | throw <| IO.userError "invalid log size" + if h : logN ≤ KoalaBear.twoAdicity then + IO.println (CPolynomial.NTT.KoalaBear.fastDomainOfLogN logN h).omega.toNat + return 0 + else throw <| IO.userError "unsupported domain" + let [path, size, direction, validate] := args | + throw <| IO.userError "usage: CompPolyNTTBench FIXTURE LOG_N forward|inverse true|false" + let some logN := size.toNat? | throw <| IO.userError "invalid log size" + if direction != "forward" && direction != "inverse" then + throw <| IO.userError "invalid direction" + if validate != "true" && validate != "false" then throw <| IO.userError "invalid validation flag" + if h : logN ≤ KoalaBear.twoAdicity then + let domain := CPolynomial.NTT.KoalaBear.fastDomainOfLogN logN h + let n := 2 ^ logN + let values ← nttCoordinates (← IO.FS.readBinFile path) + if values.size != 1 + 2 * n then throw <| IO.userError "incorrect NTT fixture length" + if values.getD 0 0 != domain.omega then throw <| IO.userError "incorrect NTT root" + let inputs := #[values.extract 1 (n + 1), values.extract (n + 1) (2 * n + 1)] + let workers := ((← IO.getEnv "LEAN_NUM_THREADS").bind String.toNat?).getD 16 + if workers == 0 then throw <| IO.userError "LEAN_NUM_THREADS must be positive" + let parallel := workers > 1 + let variant := (← IO.getEnv "COMPPOLY_NTT_IMPL").getD "packed" + have h32 : domain.logN ≤ 32 := by + change logN ≤ 32 + have : KoalaBear.twoAdicity = 24 := rfl + omega + have hu : 4 * domain.n < USize.size := by + have hp : 2 ^ logN ≤ 2 ^ 24 := Nat.pow_le_pow_right (by decide) h + have := USize.le_size + change 4 * 2 ^ logN < USize.size + omega + validateOnlyRef.set (validate == "true") + let spec (method representation : String) : BenchSpec := + { name := s!"ntt-koalabear-{logN}-{direction}-fast", representation := representation, + method := method, field := "koalabear", + inputShape := s!"{n} elements", + digestIterations := 2, digestClass := direction } + -- Both plans take and return packed Montgomery words, like Plonky3's contiguous field + -- vectors; inputs are packed outside timing. + let words := inputs.map CPolynomial.NTTFast.Packed.Native.encodeFields + let word (b : ByteArray) (i : Nat) : UInt64 := + (CPolynomial.NTTFast.Packed.Native.read b i).toUInt64 + let packed (method : String) (perform : ByteArray → ByteArray) := + runTimedSpec (spec method "ByteArray") .large + (fun i ↦ perform words[i % 2]!) + (fun b ↦ checksumArray checksumKoalaBearFast + (CompPoly.PackedRepr.decode (R := KoalaBear.Fast.Field) b)) + (sink := fun b ↦ sinkStep (sinkStep (sinkStep (word b 0) (word b (n / 3))) + (word b (2 * n / 3))) (word b (n - 1))) + let row ← match variant with + | "packed" => + let plan := CPolynomial.NTTFast.Packed.Plan.ofDomain domain h32 hu + packed "proved packed parallel radix-4, ByteArray word accessors" fun b ↦ + if direction == "forward" then plan.forwardPacked b parallel + else plan.inversePacked b parallel + | "externless" => + let plan := CPolynomial.NTTFast.NaturalPlan.ofDomain domain + packed "proved externless parallel radix-4" fun b ↦ + if direction == "forward" then plan.forwardPacked b parallel + else plan.inversePacked b parallel + | _ => throw <| IO.userError "COMPPOLY_NTT_IMPL must be packed or externless" + let record : BenchRecord := { row with groupKey := s!"ntt-koalabear-{logN}-{direction}" } + IO.println record.toJsonLine + return 0 + else throw <| IO.userError "unsupported domain" diff --git a/bench/README.md b/bench/README.md index e8ea325e..5e9e8296 100644 --- a/bench/README.md +++ b/bench/README.md @@ -126,7 +126,7 @@ Roughly by area, with representative group prefixes: | Extension fields | `fields-extension-*-mul`, `fields-extension-*-inv` | | Binary tower fields | `fields-tower-bt128-*`: `BitVec` spec vs packed-word implementation | | Base-field arithmetic | `fields-{koalabear,babybear,mersenne31,goldilocks}-{mul,add,inv,pow}`: canonical `ZMod` vs native-word, latency and throughput | -| Pairing scalar multiplication | `fields-{bn254,bls12-381,bls12-377}-mul` | +| Four-limb field arithmetic | `fields-{bn254,bls12-381,bls12-377}-{mul,add}`, `fields-secp256k1-{scalar,base}-{mul,add}`: canonical `ZMod` vs four-limb Montgomery | | Scalar-field inversion | `fields-mont64x8-*-inv`: `ZMod` extended Euclid vs checked binary GCD vs Fermat | | Binary tower scalar kernels | `fields-tower-bt{8,64}-*`: table-driven vs recursive | | Multiplicative NTT | `ntt-{koalabear,babybear}-l*` over `n = 2^8 … 2^16`, plus `ntt-plan-koalabear` | @@ -195,7 +195,7 @@ in different places, and named as Plonky3 names them: - **latency** — each operation depends on the last, so the pipeline cannot overlap two; -- **throughput** — ten independent accumulators, so it can. +- **throughput** — parallel accumulators: two for BN254, ten for other fields. Every row of a group must agree on `workUnits`, because the count describes the *problem* and not the implementation; a group whose rows disagree fails the @@ -302,6 +302,8 @@ Benchmarks → Run workflow** with a preset and optional group list, a `/bench` comment on a PR from a repo member, or automatically on any PR touching `bench/**`. Results are posted as a PR comment and uploaded as an artifact. +For fork PR runs, the report is available in the Actions summary and artifact; automatic commenting is skipped because the token is read-only. Ordinary PR comments do not cancel benchmark jobs; only a new eligible benchmark job supersedes a running one. + They are kept out of the blocking path deliberately, though not for the reason you might expect. *Within* one run the shared runner is actually steadier than a busy laptop — median MAD 0.2% against 1.4% locally — but severe outliers are @@ -316,3 +318,101 @@ Both tracks default to the group list in `bench/ci-groups.txt` — one key per line, `#` comments ignored. Neither runs every registered group, so **a new group must be added there to be covered**. An unknown key fails the run, so a renamed group is caught rather than silently dropped. + +## Rust field comparison + +Run `python3 scripts/bench-fields.py --suite all --cpu 0` from the repository root on Linux. For scalar field suites, choose an available logical CPU: the driver pins itself and both runners there, builds with one job, and runs Lean and Rust sequentially. It validates matching result digests and operation counts before five paired timing runs in alternating order. `--validate-only` skips timing; `--skip-build` requires a matching build record from a previous driver run. + +| Suite | Selected fields and operations | Rust library | +| --- | --- | --- | +| `small-prime` (default) | KoalaBear, Mersenne31, Goldilocks: add/mul latency and throughput, inv/exp latency | Plonky3 0.4.2 | +| `large-prime` | BN254 scalar field: add/mul latency and throughput, inv/exp latency | arkworks 0.5.0 (`ark_bn254::Fr`) | +| `binary` | 8-, 64-, 128-bit Fan–Paar towers: mul latency/throughput, square/inv latency | Binius 0.2.0, pinned Git revision | +| `ntt` | KoalaBear forward and inverse NTT; 2^12, 2^16, 2^20 elements | Plonky3 0.4.2 (`p3-dft`, parallel feature) | +| `interpolate` | KoalaBear coset values evaluated at a degree-4 extension point; 2^12, 2^16, 2^20 rows × 1 and 16 columns | Plonky3 0.4.2 (`p3-interpolation`, parallel feature) | +| `lde` | KoalaBear batched coset low-degree extension in commitment order; 2^14 to 2^20 rows × 8 to 256 columns, blowup 2 and 4 | Plonky3 0.4.2 (`p3-dft`, `p3-monty-31`, parallel feature) | +| `all` | The 36 scalar cases plus the NTT, coset interpolation and LDE suites | All three libraries | + +One Cargo project under `bench/rust/` shares the measurement and chain harness. Library-specific modules decode inputs and supply canonical checksums and cheap result sinks. Rust nightly-2026-02-26 is pinned for Binius’s x86 support, and Cargo.lock pins dependencies. The driver defaults `RUSTFLAGS` to `-C target-cpu=native` and records its value and the host instruction features; CI builds with the same flag. CI validates all selected suites on every PR; it does not gate on relative speed. Tables show fast Lean against Rust. Prime-field reference implementations still participate in validation and runs, but are omitted from the tables. Binary rows time only the verified fast Lean implementation and Binius; their cross-language result agreement is checked before timing. + +Each run creates an ignored `bench/out/fields-*` directory containing exact fixtures, raw samples, a machine/toolchain manifest, and `report.md` suitable for a PR comment. Timings are advisory, especially on a shared host. + +### Inputs and field correspondence + +`CompPolyFieldFixtures` exports pools in `field-coordinates-le-v1` format with an explicit `basis`. Prime-field operands use `canonical-integer`: fixed-width little-endian integers in `[0, p)`, with the modulus supplied separately. Binary operands use `fan-paar-tower`: little-endian tower coefficients in 1, 8, or 16 bytes, with an empty modulus array. These coordinates are not characteristic-two field numerals. Rust checks the encoding, basis, width, field identity, and workload parameters before decoding. The full coordinate word contributes to the untimed checksum, including both limbs at 128 bits. + +BN254 here means the **scalar** field with modulus `21888242871839275222246405745257275088548364400416034343698204186575808495617`, matching `ark_bn254::Fr`, not `Fq`. This common prime and canonical integer representation establish the correspondence. Future binary-field comparisons must additionally establish a basis mapping; equal bit patterns alone do not establish that correspondence. + +### Workloads + +Rust matches existing Lean batch shapes: 1,280 small-prime add/mul operations, 320 BN254 additions/multiplications, or 64 inv/exp steps with an added constant between steps. Small-prime throughput updates ten lanes from their previous values and combines them with nine extra operations at the end. The reported divisors exclude that merge, matching Lean. Exp uses `0x5A5A5A5A`; inverse maps zero to zero. Only the final batch result is consumed. BN254's timed sink samples native Montgomery words, avoiding canonical conversion or serialization in the timed loop. Rust uses scalar library APIs with native CPU target flags; arkworks parallel and optional assembly features are disabled. + +BN254 inversion uses `FastField.invGcd`, the checked binary-GCD path, rather than the default Fermat inverse. Arkworks uses its field `inverse`. Each chain step adds the same fixed constant before inversion; zero maps to zero. BN254 exponentiation uses the same 32-bit exponent `0x5A5A5A5A` as the small fields, not a random full-width exponent. + +BN254 add/mul throughput uses two independent chains with the same fixed second operand as latency, 160 steps per lane (320 operations total), four rounds unrolled per loop, and one final combining operation excluded from the divisor. This avoids the ten-lane ring’s excessive live state for multi-limb values. Small-prime throughput retains its ten-lane ring. + +The BN254 throughput configuration was selected manually on Rust, then fixed identically in Lean. It is not selected independently per language or tuned during benchmark runs. Rust screening covered 1, 2, 3, 4, 6, and 8 lanes with 1, 4, and 8 rounds unrolled; two lanes avoided the spill overhead of wider configurations. Shortlisted unroll factors were checked again in the production Rust runner. + +### Binary tower workloads + +`--suite binary` selects three representation boundaries: the 8-bit table base, a 64-bit machine word, and the two-word 128-bit carrier. CompPoly's tower matches Binius's `BinaryField8b`, `BinaryField64b`, and `BinaryField128b`. The dependency is the original [Binius field library](https://github.com/IrreducibleOSS/binius/tree/47675e19c86c0fb676f75437073a77af8e337938/crates/field), pinned because Binius64 uses different field representations. CPU-supported instructions are enabled, but the workloads use scalar field types, not explicit packed SIMD batches. + +Multiplication uses 64 dependent steps or two independent 32-step chains with the same fixed operand; the final combining multiply is timed but excluded from the divisor, identically in both languages. Inversion uses 64 dependent `inv(x XOR b)` steps, with zero mapped to zero. Squaring uses 63 dependent squares, unrolled seven at a time. Neither the block nor the complete chain is a whole Frobenius cycle at any selected size. No per-operation result accumulation occurs. The pool holds 64 deterministic full-width words, excluding zero and one to avoid trivial multiplication constants. + +The `fields-tower-bt{8,64,128}-{mul,square,inv}` groups replace the old recursive/table comparison and pairwise 128-bit scalar workloads. Historical best-time rows from those workloads are not comparable with these chains. Lean's arithmetic refinements remain in `Tower/Fast.lean`; matching complete chain digests and operation counts establishes the new runtime correspondence with Binius. + +### Build provenance + +The driver records the source commit and source-content hash (including untracked, nonignored files), toolchain versions, build environment, native CPU features, and hashes of all three executables in `.lake/build/field-bench-build.json`. `--skip-build` rejects missing or mismatched records and asks for a normal run; normal runs use Lake/Cargo’s incremental builds. The run manifest copies that build record instead of inferring binary provenance from the current environment. Sources and executable hashes are checked again after validation and timing. Build flags must also match, including `CARGO_ENCODED_RUSTFLAGS` if set. + +## KoalaBear NTT against optimized Plonky3 + +```bash +python3 scripts/bench-fields.py --suite ntt +python3 scripts/bench-fields.py --suite ntt --validate-only +``` + +This suite measures two proved Lean pipelines against Plonky3's `Radix2DFTSmallBatch` public `dft`/`idft` API, pinned to `p3-dft` 0.4.2. `NTTFast.Packed.Plan` stores packed words with the `ByteArray` word accessors proposed for Lean core; `NTTFast.NaturalPlan` uses core byte operations only. Both are parallel, take and return packed Montgomery words, and have complete forward/inverse refinement theorems. Rust uses the library's optimized algorithm and native packed arithmetic; there is no custom Rust transform. Six workloads cover forward and inverse transforms at 2^12, 2^16 and 2^20 elements, with all three implementations displayed in the report. The default budget is one worker per available physical core, at most sixteen and rounded down to a power of two; SMT siblings are not used, because they measure how each implementation shares a core rather than how it scales. `--cpus` chooses a power-of-two worker allocation; `--cpu` selects one worker. Plonky3's `parallel` feature and native SIMD are enabled. This suite is included in `--suite all` and its CI validation gate. + +Both sides consume and return natural-order arrays. Lean's plan internally uses bit-reversed spectra, so its ordering adapter is included in timing. Plonky3's public API includes its own ordering work. Plan construction, twiddle tables and fixture decoding are outside timing; input copying, output disposal and inverse normalization are inside. This compares optimized implementations of the same mathematical transform, allowing different algorithms and arithmetic schedules. It replaces the original matched-schedule scalar Rust baseline. + +Fixtures use canonical little-endian 32-bit coordinates: the certified root exported by Lean, followed by two input arrays. Rust verifies that root equals the library's selected root. Every output element contributes to an untimed digest; only four output positions feed the timed sink. Validation covers tiny odd/even log sizes and large transforms. The Rust API contract test uses an independent direct transform to check root direction, ordering and normalization. Reports and build/fixture provenance, including the selected Rust implementation and packing width, are saved under the output directory. + +### Lean natural-order NTT plans + +Import `CompPoly.Univariate.NTTFast.NaturalParallel` and construct `NTTFast.NaturalPlan.ofDomain domain` once, for any field with a `PackedRepr` instance. `plan.forwardPacked words parallel` and `plan.inversePacked words parallel` take and return packed elements (`PackedRepr.encode`); `forwardPacked_eq` proves `plan.forwardPacked (encode a) w = encode (plan.forward a)`, and `inversePacked_eq` the same for the inverse, where `plan.forward` and `plan.inverse` are the proved sequential transforms on field arrays. With `parallel` (the default) and from 2^15 elements, column tasks run the top two radix-four passes and sixteen leaf tasks the remaining layers, directly on the packed buffer; otherwise, and for smaller transforms, the sequential transform runs as one leaf. The plan caches bit-reversal indices as well as the twiddle tables. Shorter inputs are zero-padded and longer ones truncated. + +The imported butterfly modules also install proved compiler substitutions for `NTTFast.Plan.forwardImpl` and `inverseImpl`. Array bounds are checked once per radix-four block; the hot loop uses `USize` indices and bounds-proved accesses. The original total functions cover invalid ranges, and natural-number indexing covers ranges exceeding the machine-word limit. No new external functions or compiler changes are used. The NTT benchmark executable and the dedicated `CompPolyPackedNative` arithmetic module target the native CPU, matching Rust's native targeting; their object files must be rebuilt when moving machines. Release proof artifacts remain platform independent. + + +### Packed parallel NTT plans + +Import `CompPoly.Univariate.NTTFast.Packed.Instances`. The plan is generic over the fast carrier `FastField p` of any `Mont32Field p`; calling it needs a `PackedNTT p` instance, which that module provides for KoalaBear and BabyBear. Construct `NTTFast.Packed.Plan.ofDomain domain log_bound byte_bound` once, with proofs that `domain.logN ≤ 32` and `4 * domain.n < USize.size`. Then call `plan.forwardPacked words parallel` or `plan.inversePacked words parallel` on `domain.n` packed little-endian Montgomery words, like Plonky3's contiguous field vectors: `PackedRepr.encode` packs a field array and `PackedRepr.decode` reads one back. The pipeline is the extern-free plan's: with `parallel` (the default) and from 2^15 elements, column tasks run the top two radix-four passes and sixteen leaf tasks the rest, each ending with a kernel that fuses the last four layers with the leaf's bit reversal, and sixteen tasks interleave the leaves into natural order; otherwise, and for smaller transforms, one leaf runs. `forwardPacked_correct` and `inversePacked_correct` prove both entry points equal to the mathematical DFT/inverse DFT for every domain-sized input and worker count. Plans cache packed twiddles; timed calls include arithmetic, task joins, ordering and inverse normalization. + +The direct executable defaults to the packed plan (`COMPPOLY_NTT_IMPL=packed`). Set `COMPPOLY_NTT_IMPL=externless` for the extern-free plan; `LEAN_NUM_THREADS` sets the worker budget of both, and the runner passes `parallel` when it is above one. The comparison driver runs all variants automatically, with the same fixtures and CPU allocation as Rust; packed inputs are encoded outside timing, as Rust's raw inputs are. + +The packed path reads and writes words with `ByteArray.ugetUInt32LE` and `ByteArray.usetUInt32LE`, backported in `CompPoly/Data/ByteArray/Pack.lean` from [lean4#14053](https://github.com/leanprover/lean4/pull/14053) with the same names, signatures and Lean definitions. Until a toolchain ships them, their inline C is trusted to match those definitions; it contains no field arithmetic or scheduling. The C loads and stores with a fixed-size `memcpy` (byte-swapped on big-endian hosts) and, after a store, tells the compiler that the array is still unshared, so that a run of stores keeps one sharing check. `lake exe CompPolyNativeSmoke` checks the compiled accessors against bytewise Lean operations at every offset, including out-of-range offsets and shared inputs. The logical refinement proofs are kernel checked without `native_decide`, `sorry`, `implemented_by` or new `csimp` substitutions. + +## KoalaBear coset interpolation against Plonky3 + +```bash +python3 scripts/bench-fields.py --suite interpolate +python3 scripts/bench-fields.py --suite interpolate --validate-only +``` + +This suite compares two Lean implementations with Plonky3's stock `p3_interpolation::interpolate_coset` (0.4.2) over `BinomialExtensionField`. Each call takes a row-major `2^logN × width` matrix whose columns hold values of polynomials on the coset `s · ⟨ω⟩`, and returns every column's polynomial evaluated at a point of the quartic extension, by the barycentric formula `(zⁿ - sⁿ) / (n sⁿ) · ∑ᵢ xᵢ / (z - xᵢ) · p(xᵢ)`. `KoalaBear.Fast.interpolateCosetPacked` reads the matrix as Montgomery words in a `ByteArray`, like Plonky3's `RowMajorMatrix`, and runs on parallel tasks; `KoalaBear.Fast.interpolateCoset` is the sequential proved reference. `toSpec_interpolateCoset` proves the reference equal to `aeval z` of each column's polynomial, and `interpolateCosetPacked_eq` proves the packed version equal to the reference. All three compute the coset nodes, the inversions and the weights inside every call; Plonky3 uses its parallel feature and native SIMD. + +Each fixture holds the subgroup root, the shift (KoalaBear's multiplicative generator `3`), two extension points that alternate between iterations, and the matrix, as canonical little-endian 32-bit words. The root must equal Lean's certified root and Plonky3's two-adic generator. `CompPolyInterpolateBench` and `comppoly-field-bench --interpolate` read the same fixture; full output digests agree before timing, including single-row and narrow matrices. Workers follow the same physical-core selection as the NTT suite, and every invocation warms up for 200 ms. + +The packed version reads and writes words with the packed NTT's backported `ByteArray` word accessors and nothing else outside the kernel proof, so `interpolateCosetPacked_eq` and the accessors' compiled checks cover it. + +## KoalaBear coset LDE against Plonky3 + +```bash +python3 scripts/bench-fields.py --suite lde +python3 scripts/bench-fields.py --suite lde --validate-only +``` + +This suite compares `NTTFast.Packed.LdePlan.cosetLde` (KoalaBear instance of the generic `Native.cosetLde`) with Plonky3's stock `TwoAdicSubgroupDft::coset_lde_batch` (0.4.2), followed by `bit_reverse_rows().to_row_major_matrix()` as in `TwoAdicFriPcs::commit`, for each of `Radix2DitParallel`, Monty31's `RecursiveDft` and `Radix2DFTSmallBatch`. Each call extends every column of a row-major `2^logH × width` matrix, the values of a polynomial on the subgroup of order `2^logH`, to the coset `3 · ⟨ω⟩` of order `2^(logH + bits)`, and returns the rows in bit-reversed order. The Lean side reads the matrix as Montgomery words in chunks of rows (`Native.rowChunks`, here one chunk) and returns row chunks, which a commitment hashes in parallel; `joinRows` copies them into one buffer. `Native.joinRows_cosetLde` proves the joined chunks equal to the specification `ldeSpec`: row `r` holds every column's Lagrange interpolant on the subgroup, at `shift · ω ^ bitrev r`. + +Each fixture holds the root of the extended domain, the shift and two matrices that alternate between iterations, as canonical little-endian 32-bit words. The root must equal Lean's certified root and Plonky3's two-adic generator. `CompPolyLdeBench` and `comppoly-field-bench --lde` read the same fixture; full output digests of all four implementations agree before timing, including single-row, odd-height and narrow matrices. Plans and twiddles are built outside timing; input copies, transposes, transforms and output assembly are timed. Workers follow the NTT suite's physical-core selection. diff --git a/bench/ci-groups.txt b/bench/ci-groups.txt index 4a84e213..17852ccb 100644 --- a/bench/ci-groups.txt +++ b/bench/ci-groups.txt @@ -58,6 +58,8 @@ fields-extension-babybear-ext4-inv fields-mont64x8-bn254-inv fields-mont64x8-bls12-381-inv fields-mont64x8-bls12-377-inv +fields-mont64x8-secp256k1-scalar-inv +fields-mont64x8-secp256k1-base-inv fields-goldilocks-mul fields-goldilocks-inv fields-koalabear-mul @@ -67,9 +69,12 @@ fields-koalabear-pow fields-babybear-mul fields-mersenne31-mul fields-bn254-mul +fields-bn254-add +fields-secp256k1-scalar-mul +fields-secp256k1-base-mul fields-tower-bt8-mul fields-tower-bt64-mul -fields-tower-bt64-inv-word +fields-tower-bt64-inv ntt-koalabear-l12 ntt-babybear-l12 ntt-plan-koalabear @@ -80,3 +85,11 @@ univariate-interp-coset-koalabear-l12 univariate-barycentric-koalabear-l8 univariate-mul-crossover-64 univariate-mul-crossover-256 +fields-bn254-add +fields-bn254-inv +fields-bn254-pow + +fields-tower-bt8-square +fields-tower-bt8-inv +fields-tower-bt64-square +fields-tower-bt128-square diff --git a/bench/rust/.gitignore b/bench/rust/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/bench/rust/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/bench/rust/Cargo.lock b/bench/rust/Cargo.lock new file mode 100644 index 00000000..c4a3c555 --- /dev/null +++ b/bench/rust/Cargo.lock @@ -0,0 +1,1153 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "ark-bn254" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69eab57e8d2663efa5c63135b2af4f396d66424f88954c21104125ab6b3e6bc" +dependencies = [ + "ark-ec", + "ark-ff", + "ark-std", +] + +[[package]] +name = "ark-ec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d68f2d516162846c1238e755a7c4d131b892b70cc70c471a8e3ca3ed818fce" +dependencies = [ + "ahash", + "ark-ff", + "ark-poly", + "ark-serialize", + "ark-std", + "educe", + "fnv", + "hashbrown", + "itertools 0.13.0", + "num-bigint", + "num-integer", + "num-traits", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70" +dependencies = [ + "ark-ff-asm", + "ark-ff-macros", + "ark-serialize", + "ark-std", + "arrayvec", + "digest", + "educe", + "itertools 0.13.0", + "num-bigint", + "num-traits", + "paste", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-ff-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-poly" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579305839da207f02b89cd1679e50e67b4331e2f9294a57693e5051b7703fe27" +dependencies = [ + "ahash", + "ark-ff", + "ark-serialize", + "ark-std", + "educe", + "fnv", + "hashbrown", +] + +[[package]] +name = "ark-serialize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" +dependencies = [ + "ark-serialize-derive", + "ark-std", + "arrayvec", + "digest", + "num-bigint", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-std" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" +dependencies = [ + "num-traits", + "rand 0.8.8", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "auto_impl" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "binius_field" +version = "0.2.0" +source = "git+https://github.com/IrreducibleOSS/binius?rev=47675e19c86c0fb676f75437073a77af8e337938#47675e19c86c0fb676f75437073a77af8e337938" +dependencies = [ + "binius_maybe_rayon", + "binius_utils", + "bytemuck", + "cfg-if", + "derive_more", + "getrandom", + "rand 0.9.5", + "seq-macro", + "subtle", + "thiserror", + "tracing", + "trait-set", + "transpose", +] + +[[package]] +name = "binius_maybe_rayon" +version = "0.2.0" +source = "git+https://github.com/IrreducibleOSS/binius?rev=47675e19c86c0fb676f75437073a77af8e337938#47675e19c86c0fb676f75437073a77af8e337938" +dependencies = [ + "cfg-if", + "either", + "itertools 0.14.0", +] + +[[package]] +name = "binius_utils" +version = "0.2.0" +source = "git+https://github.com/IrreducibleOSS/binius?rev=47675e19c86c0fb676f75437073a77af8e337938#47675e19c86c0fb676f75437073a77af8e337938" +dependencies = [ + "auto_impl", + "binius_maybe_rayon", + "bytemuck", + "bytes", + "cfg-if", + "generic-array", + "itertools 0.14.0", + "thiserror", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "comppoly-field-bench" +version = "0.1.0" +dependencies = [ + "ark-bn254", + "ark-ff", + "binius_field", + "p3-dft", + "p3-field", + "p3-goldilocks", + "p3-interpolation", + "p3-koala-bear", + "p3-matrix", + "p3-mersenne-31", + "p3-monty-31", + "rayon", + "serde", + "serde_json", +] + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "crypto-common", +] + +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "enum-ordinalize" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89dd01549b09589510cf0647475075d12071456586d70f5c75c98ae2a5537677" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "p3-challenger" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20e42ba74a49c08c6e99f74cd9b343bfa31aa5721fea55079b18e3fd65f1dcbc" +dependencies = [ + "p3-field", + "p3-maybe-rayon", + "p3-monty-31", + "p3-symmetric", + "p3-util", + "tracing", +] + +[[package]] +name = "p3-dft" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e63fa5eb1bd12a240089e72ae3fe10350944d9c166d00a3bfd2a1794db65cf5c" +dependencies = [ + "itertools 0.14.0", + "p3-field", + "p3-matrix", + "p3-maybe-rayon", + "p3-util", + "spin", + "tracing", +] + +[[package]] +name = "p3-field" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebfdb6ef992ae64e9e8f449ac46516ffa584f11afbdf9ee244288c2a633cdf4" +dependencies = [ + "itertools 0.14.0", + "num-bigint", + "p3-maybe-rayon", + "p3-util", + "paste", + "rand 0.9.5", + "serde", + "tracing", +] + +[[package]] +name = "p3-goldilocks" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64716244b5612622d4e78a4f48b74f6d3bb7b4085b7b6b25364b1dfca7198c66" +dependencies = [ + "num-bigint", + "p3-challenger", + "p3-dft", + "p3-field", + "p3-mds", + "p3-poseidon2", + "p3-symmetric", + "p3-util", + "paste", + "rand 0.9.5", + "serde", +] + +[[package]] +name = "p3-interpolation" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d877565a94a527c89459fc8ccb0eb58769d8c86456575d1315a1651bd24616d" +dependencies = [ + "p3-field", + "p3-matrix", + "p3-maybe-rayon", + "p3-util", +] + +[[package]] +name = "p3-koala-bear" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5113f50002c56006685b7d7ae12db568150aa1d4bfb092b883d64ece20138042" +dependencies = [ + "p3-challenger", + "p3-field", + "p3-monty-31", + "p3-poseidon2", + "p3-symmetric", + "rand 0.9.5", +] + +[[package]] +name = "p3-matrix" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5542f96504dae8100c91398fb1e3f5ec669eb9c73d9e0b018a93b5fe32bad230" +dependencies = [ + "itertools 0.14.0", + "p3-field", + "p3-maybe-rayon", + "p3-util", + "rand 0.9.5", + "serde", + "tracing", + "transpose", +] + +[[package]] +name = "p3-maybe-rayon" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54afab3883d8a14676b492709d6c4e9fa535c36718b737db0817aacfaaaa11f6" +dependencies = [ + "rayon", +] + +[[package]] +name = "p3-mds" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038763af23df9da653065867fd85b38626079031576c86fd537097e5be6a0da0" +dependencies = [ + "p3-dft", + "p3-field", + "p3-symmetric", + "p3-util", + "rand 0.9.5", +] + +[[package]] +name = "p3-mersenne-31" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a16c377cdb22083de5d0ee1eb4377a137a3988dbb58050f875791cb47b5bbc" +dependencies = [ + "itertools 0.14.0", + "num-bigint", + "p3-challenger", + "p3-dft", + "p3-field", + "p3-matrix", + "p3-mds", + "p3-poseidon2", + "p3-symmetric", + "p3-util", + "paste", + "rand 0.9.5", + "serde", +] + +[[package]] +name = "p3-monty-31" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a981d60da3d8cbf8561014e2c186068578405fd69098fa75b43d4afb364a47" +dependencies = [ + "itertools 0.14.0", + "num-bigint", + "p3-dft", + "p3-field", + "p3-matrix", + "p3-maybe-rayon", + "p3-mds", + "p3-poseidon2", + "p3-symmetric", + "p3-util", + "paste", + "rand 0.9.5", + "serde", + "spin", + "tracing", + "transpose", +] + +[[package]] +name = "p3-poseidon2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "903b73e4f9a7781a18561c74dc169cf03333497b57a8dd02aaeb130c0f386599" +dependencies = [ + "p3-field", + "p3-mds", + "p3-symmetric", + "p3-util", + "rand 0.9.5", +] + +[[package]] +name = "p3-symmetric" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd788f04e86dd5c35dd87cad29eefdb6371d2fd5f7664451382eeacae3c3ed0" +dependencies = [ + "itertools 0.14.0", + "p3-field", + "serde", +] + +[[package]] +name = "p3-util" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911154accf66034b0eec4452956c088f92a200b37a8225c1caed74cfbd38cc8d" +dependencies = [ + "serde", + "transpose", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +dependencies = [ + "lock_api", +] + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "trait-set" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79e2e9c9ab44c6d7c20d5976961b47e8f49ac199154daa514b77cd1ab536625" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/bench/rust/Cargo.toml b/bench/rust/Cargo.toml new file mode 100644 index 00000000..cfa9c1cf --- /dev/null +++ b/bench/rust/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "comppoly-field-bench" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +rayon = "=1.11.0" +binius_field = { git = "https://github.com/IrreducibleOSS/binius", rev = "47675e19c86c0fb676f75437073a77af8e337938" } +p3-field = "=0.4.2" +p3-dft = { version = "=0.4.2", features = ["parallel"] } +p3-interpolation = "=0.4.2" +p3-matrix = "=0.4.2" +p3-koala-bear = "=0.4.2" +p3-monty-31 = "=0.4.2" +p3-mersenne-31 = "=0.4.2" +p3-goldilocks = "=0.4.2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +ark-ff = { version = "=0.5.0", default-features = false } +ark-bn254 = { version = "=0.5.0", default-features = false, features = ["scalar_field"] } + +[profile.release] +lto = true +codegen-units = 1 diff --git a/bench/rust/rust-toolchain.toml b/bench/rust/rust-toolchain.toml new file mode 100644 index 00000000..4cbc2b69 --- /dev/null +++ b/bench/rust/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "nightly-2026-02-26" +profile = "minimal" +components = ["rustfmt"] diff --git a/bench/rust/src/binary.rs b/bench/rust/src/binary.rs new file mode 100644 index 00000000..428e5f08 --- /dev/null +++ b/bench/rust/src/binary.rs @@ -0,0 +1,120 @@ +//! Fan–Paar tower coefficients, matching CompPoly's ConcreteBinaryTower basis. +use crate::{ + harness::{latency, measure, throughput_pair, BenchValue, DIGEST_MODULUS}, + Fixture, +}; +use binius_field::{BinaryField128b, BinaryField64b, BinaryField8b, Field}; +use std::hint::black_box; + +macro_rules! value { + ($field:ty) => { + impl BenchValue for $field { + fn checksum(self) -> u128 { + // Reduce the complete coordinate word, including both 128-bit limbs. + self.val() as u128 % DIGEST_MODULUS + } + fn sink(self) -> u64 { + let word = self.val() as u128; + word as u64 ^ (word >> 64) as u64 + } + } + }; +} +value!(BinaryField8b); +value!(BinaryField64b); +value!(BinaryField128b); + +#[inline(always)] +fn square_chain(mut x: F) -> F { + for _ in 0..9 { + x = x + .square() + .square() + .square() + .square() + .square() + .square() + .square(); + } + x +} + +fn run_field(fixture: &Fixture, validate_only: bool, xs: Vec) { + let constant = black_box(xs[0]); + match fixture.operation.as_str() { + "mul" => { + measure(fixture, "latency", 64, validate_only, |i| { + latency(|x| x * constant, 64, xs[i % 64]) + }); + measure(fixture, "throughput", 64, validate_only, |i| { + throughput_pair(|a, b| a * b, constant, 32, xs[i % 64], xs[(i + 1) % 64]) + }); + } + "square" => measure(fixture, "latency", 63, validate_only, |i| { + square_chain(xs[i % 64]) + }), + "inv" => measure(fixture, "latency", 64, validate_only, |i| { + latency(|x| (x + constant).invert_or_zero(), 64, xs[i % 64]) + }), + _ => panic!("unsupported binary operation"), + } +} + +pub fn run(fixture: &Fixture, validate_only: bool) { + let bits = match fixture.field.as_str() { + "tower-bt8" => 8, + "tower-bt64" => 64, + "tower-bt128" => 128, + _ => panic!("unsupported tower"), + }; + assert_eq!(fixture.encoding, "field-coordinates-le-v1"); + assert_eq!(fixture.basis, "fan-paar-tower"); + assert!(fixture.modulus.is_empty()); + assert_eq!( + fixture.group_key, + format!("fields-{}-{}", fixture.field, fixture.operation) + ); + assert_eq!(fixture.inputs.len(), 64); + assert_eq!(fixture.exponent, 0); + assert_eq!(fixture.throughput_rounds, 32); + assert_eq!( + fixture.latency_rounds, + if fixture.operation == "square" { 63 } else { 64 } + ); + let words: Vec = fixture + .inputs + .iter() + .map(|bytes| { + assert_eq!(bytes.len(), bits / 8); + let mut word = [0; 16]; + word[..bytes.len()].copy_from_slice(bytes); + let word = u128::from_le_bytes(word); + assert!(word >= 2, "trivial multiplication input"); + word + }) + .collect(); + match bits { + 8 => run_field( + fixture, + validate_only, + words + .into_iter() + .map(|x| BinaryField8b::new(x as u8)) + .collect(), + ), + 64 => run_field( + fixture, + validate_only, + words + .into_iter() + .map(|x| BinaryField64b::new(x as u64)) + .collect(), + ), + 128 => run_field( + fixture, + validate_only, + words.into_iter().map(BinaryField128b::new).collect(), + ), + _ => unreachable!(), + } +} diff --git a/bench/rust/src/harness.rs b/bench/rust/src/harness.rs new file mode 100644 index 00000000..2c875ca8 --- /dev/null +++ b/bench/rust/src/harness.rs @@ -0,0 +1,206 @@ +use crate::Fixture; +use serde_json::json; +use std::{hint::black_box, time::Instant}; + +pub const DIGEST_MODULUS: u128 = 18446744073709551557; + +/// Canonical digest outside timing; cheap native-word observation after each batch. +pub trait BenchValue { + fn checksum(self) -> u128; + fn sink(self) -> u64; +} + +#[inline(always)] +fn apply8(op: impl Fn(F) -> F, x: F) -> F { + op(op(op(op(op(op(op(op(x)))))))) +} + +#[inline(always)] +pub fn latency(op: impl Fn(F) -> F + Copy, rounds: usize, mut x: F) -> F { + for _ in 0..rounds / 64 { + x = apply8(|x| apply8(op, x), x); + } + x +} + +#[inline(always)] +pub fn throughput(op: impl Fn(F, F) -> F, rounds: usize, xs: [F; 10]) -> F { + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h, mut i, mut j] = xs; + macro_rules! step { + () => { + (a, b, c, d, e, f, g, h, i, j) = ( + op(a, b), + op(b, c), + op(c, d), + op(d, e), + op(e, f), + op(f, g), + op(g, h), + op(h, i), + op(i, j), + op(j, a), + ); + }; + } + for _ in 0..rounds / 4 { + step!(); + step!(); + step!(); + step!(); + } + op(op(op(op(a, b), op(c, d)), op(op(e, f), op(g, h))), op(i, j)) +} + +/// Two independent chains with a shared fixed operand, for multi-limb fields. +#[inline(always)] +pub fn throughput_pair( + op: impl Fn(F, F) -> F, + constant: F, + rounds: usize, + mut a: F, + mut b: F, +) -> F { + macro_rules! step { + () => { + a = op(a, constant); + b = op(b, constant); + }; + } + for _ in 0..rounds / 4 { + step!(); + step!(); + step!(); + step!(); + } + op(a, b) +} + +pub fn measure( + fixture: &Fixture, + mode: &str, + units: usize, + validate_only: bool, + run: impl Fn(usize) -> F, +) { + measure_workload( + &fixture.group_key, + mode, + units, + 64, + MEDIUM, + validate_only, + run, + ); +} + +/// Warmup and sample count, matching the Lean harness's presets of the same name. +#[derive(Clone, Copy)] +pub struct Budget { + pub warmup_nanos: u64, + pub samples: usize, +} +pub const MEDIUM: Budget = Budget { + warmup_nanos: 50_000_000, + samples: 20, +}; +/// Parallel workloads need the longer warmup: idle worker cores take a few hundred +/// milliseconds of load before the frequency governor raises their clocks. +pub const LARGE: Budget = Budget { + warmup_nanos: 200_000_000, + samples: 50, +}; + +/// Shared timing and digest machinery, with workload-specific validation period. +pub fn measure_workload( + key: &str, + mode: &str, + units: usize, + period: usize, + budget: Budget, + validate_only: bool, + run: impl Fn(usize) -> F, +) { + // Strong agreement check is separate from timing. One canonical result per chain. + let checksum = (0..period).fold(0u128, |acc, i| { + (acc * 16777619 + run(i).checksum() + 97) % 18446744073709551557 + }); + let mut samples = Vec::new(); + let mut sink = 0u64; + let mut iters = 1usize; + let mut warmup_iterations = 0usize; + if !validate_only { + let mut time = |iterations: usize| { + let start = Instant::now(); + for i in 0..iterations { + // Opaque seed index prevents caching a finite pool of chain outputs. + let result = black_box(run(black_box(i))).sink(); + sink = (sink ^ result) + .wrapping_mul(0x9E3779B97F4A7C15) + .rotate_left(27); + } + black_box(sink); + start.elapsed().as_nanos() as u64 + }; + let mut elapsed = 0; + for _ in 0..40 { + let nanos = time(iters).max(1); + elapsed += nanos; + warmup_iterations += iters; + if elapsed >= budget.warmup_nanos { + iters = ((1_000_000u128 * iters as u128) / nanos as u128).max(1) as usize; + break; + } + iters *= 2; + } + for _ in 0..budget.samples { + samples.push(time(iters) as f64 * 1000.0 / iters as f64); + } + } + println!( + "{}", + json!({ + "group_key": key, "mode": mode, "work_units": units, + "checksum": checksum.to_string(), "samples_picos": samples, + "iters_per_sample": iters, "warmup_iterations": warmup_iterations, + "sink_digest": sink.to_string(), + }) + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chain_shapes_match_simple_loops() { + let modulus = 101u128; + for multiply in [false, true] { + let op = |a, b| { + if multiply { + (a * b) % modulus + } else { + (a + b) % modulus + } + }; + let mut expected = 7; + for _ in 0..1280 { + expected = op(expected, 13); + } + assert_eq!(latency(|x| op(x, 13), 1280, 7), expected); + let initial = std::array::from_fn(|i| i as u128 + 1); + let mut lanes = initial; + for _ in 0..128 { + lanes = std::array::from_fn(|i| op(lanes[i], lanes[(i + 1) % 10])); + } + let expected = lanes.into_iter().reduce(op).unwrap(); + assert_eq!(throughput(op, 128, initial), expected); + let mut a = 7; + let mut b = 19; + for _ in 0..160 { + a = op(a, 13); + b = op(b, 13); + } + assert_eq!(throughput_pair(op, 13, 160, 7, 19), op(a, b)); + } + } +} diff --git a/bench/rust/src/interpolate.rs b/bench/rust/src/interpolate.rs new file mode 100644 index 00000000..6a7bd398 --- /dev/null +++ b/bench/rust/src/interpolate.rs @@ -0,0 +1,67 @@ +//! Plonky3's stock `interpolate_coset`: evaluate the columns of a matrix of coset values at an +//! extension-field point. +use crate::harness::{measure_workload, BenchValue, DIGEST_MODULUS, LARGE}; +use p3_field::extension::BinomialExtensionField; +use p3_field::{BasedVectorSpace, PrimeCharacteristicRing, PrimeField32, TwoAdicField}; +use p3_interpolation::interpolate_coset; +use p3_koala_bear::KoalaBear as F; +use p3_matrix::dense::RowMajorMatrix; + +type E = BinomialExtensionField; + +struct Output(Vec); +impl BenchValue for Output { + fn checksum(self) -> u128 { + self.0.iter().fold(0, |acc, e| { + let coords: &[F] = e.as_basis_coefficients_slice(); + coords.iter().fold(acc, |acc, x| { + (acc * 16777619 + x.as_canonical_u32() as u128 + 97) % DIGEST_MODULUS + }) + }) + } + fn sink(self) -> u64 { + let coords: &[F] = self.0[0].as_basis_coefficients_slice(); + coords[0].as_canonical_u32() as u64 ^ ((self.0.len() as u64) << 32) + } +} + +pub fn run(args: &[String]) { + assert_eq!( + args.len(), + 4, + "--interpolate FIXTURE LOG_N WIDTH true|false" + ); + let log_n: usize = args[1].parse().unwrap(); + assert!(log_n <= 24); + let n = 1 << log_n; + let width: usize = args[2].parse().unwrap(); + let validate: bool = args[3].parse().unwrap(); + let bytes = std::fs::read(&args[0]).unwrap(); + assert_eq!(bytes.len(), 4 * (10 + n * width)); + let words: Vec = bytes + .chunks_exact(4) + .map(|v| { + let v = u32::from_le_bytes(v.try_into().unwrap()); + assert!(v < 2130706433); + F::from_u32(v) + }) + .collect(); + assert_eq!( + words[0], + F::two_adic_generator(log_n), + "fixture/library root mismatch" + ); + let shift = words[1]; + let point = |k: usize| E::from_basis_coefficients_slice(&words[2 + 4 * k..6 + 4 * k]).unwrap(); + let points = [point(0), point(1)]; + let matrix = RowMajorMatrix::new(words[10..].to_vec(), width); + measure_workload( + &format!("interpolate-koalabear-{log_n}-{width}"), + "interpolate", + 1, + 2, + LARGE, + validate, + |i| Output(interpolate_coset(&matrix, shift, points[i % 2])), + ); +} diff --git a/bench/rust/src/large_prime.rs b/bench/rust/src/large_prime.rs new file mode 100644 index 00000000..e54b7920 --- /dev/null +++ b/bench/rust/src/large_prime.rs @@ -0,0 +1,98 @@ +//! BN254 scalar field Fr (not the curve's base field Fq). +use crate::{ + harness::{latency, measure, throughput_pair, BenchValue, DIGEST_MODULUS}, + Fixture, +}; +use ark_bn254::Fr; +use ark_ff::{AdditiveGroup, BigInteger, Field, PrimeField}; +use std::hint::black_box; + +impl BenchValue for Fr { + fn checksum(self) -> u128 { + // Match Lean's full canonical Nat modulo the digest prime; do not truncate. + self.into_bigint() + .to_bytes_le() + .iter() + .rev() + .fold(0, |n, b| (n * 256 + *b as u128) % DIGEST_MODULUS) + } + + fn sink(self) -> u64 { + // Same low/high 32-bit Montgomery words as Lean's eight-limb sink. + // No conversion out of Montgomery form or serialization during timing. + (self.0 .0[0] & 0xffff_ffff) ^ (self.0 .0[3] >> 32) + } +} + +#[inline(always)] +fn add(a: Fr, b: Fr) -> Fr { + a + b +} + +#[inline(always)] +fn mul(a: Fr, b: Fr) -> Fr { + a * b +} + +pub fn run(fixture: &Fixture, validate_only: bool) { + assert_eq!( + fixture.group_key, + format!("fields-bn254-{}", fixture.operation) + ); + assert_eq!(fixture.modulus, Fr::MODULUS.to_bytes_le()); + fixture.validate_inputs(32); + let binary = matches!(fixture.operation.as_str(), "add" | "mul"); + assert_eq!(fixture.latency_rounds, if binary { 320 } else { 64 }); + assert_eq!(fixture.throughput_rounds, 160); + assert_eq!(fixture.exponent, 0x5A5A5A5A); + let xs: Vec = fixture + .inputs + .iter() + .map(|x| Fr::from_le_bytes_mod_order(x)) + .collect(); + let b = black_box(xs[0]); + macro_rules! binary { + ($op:expr) => {{ + measure(fixture, "latency", 320, validate_only, |i| { + latency(|x| $op(x, b), 320, xs[i % 64]) + }); + measure(fixture, "throughput", 320, validate_only, |i| { + throughput_pair($op, b, 160, xs[i % 64], xs[(i + 1) % 64]) + }); + }}; + } + match fixture.operation.as_str() { + "add" => binary!(add), + "mul" => binary!(mul), + "inv" => measure(fixture, "latency", 64, validate_only, |i| { + latency(|x| (x + b).inverse().unwrap_or(Fr::ZERO), 64, xs[i % 64]) + }), + "pow" => measure(fixture, "latency", 64, validate_only, |i| { + latency(|x| (x + b).pow([0x5A5A5A5Au64]), 64, xs[i % 64]) + }), + _ => panic!("unsupported operation"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn canonical_boundary_roundtrips() { + for x in [Fr::from(0u64), Fr::from(1u64), -Fr::from(1u64)] { + let bytes = x.into_bigint().to_bytes_le(); + assert!(crate::canonical(&bytes, &Fr::MODULUS.to_bytes_le())); + assert_eq!(Fr::from_le_bytes_mod_order(&bytes), x); + } + // p-1 is far above u64: check that the digest sees the whole integer. + let expected = Fr::MODULUS + .to_bytes_le() + .iter() + .rev() + .fold(0u128, |n, b| (n * 256 + *b as u128) % DIGEST_MODULUS); + assert_eq!( + (-Fr::from(1u64)).checksum(), + (expected + DIGEST_MODULUS - 1) % DIGEST_MODULUS + ); + } +} diff --git a/bench/rust/src/lde.rs b/bench/rust/src/lde.rs new file mode 100644 index 00000000..2e45cc36 --- /dev/null +++ b/bench/rust/src/lde.rs @@ -0,0 +1,133 @@ +//! Plonky3's stock `coset_lde_batch`, returning rows in the bit-reversed order that +//! `TwoAdicFriPcs::commit` commits. +use crate::harness::{measure_workload, BenchValue, DIGEST_MODULUS, LARGE}; +use p3_dft::{Radix2DFTSmallBatch, Radix2DitParallel, TwoAdicSubgroupDft}; +use p3_field::{PrimeCharacteristicRing, PrimeField32, TwoAdicField}; +use p3_koala_bear::KoalaBear as F; +use p3_matrix::bitrev::BitReversibleMatrix; +use p3_matrix::dense::RowMajorMatrix; +use p3_matrix::Matrix; +use p3_monty_31::dft::RecursiveDft; + +struct Output(RowMajorMatrix); +impl BenchValue for Output { + fn checksum(self) -> u128 { + self.0.values.iter().fold(0, |acc, x| { + (acc * 16777619 + x.as_canonical_u32() as u128 + 97) % DIGEST_MODULUS + }) + } + fn sink(self) -> u64 { + let v = &self.0.values; + v[0].as_canonical_u32() as u64 ^ ((v[v.len() - 1].as_canonical_u32() as u64) << 32) + } +} + +/// The LDE in commitment order, as `TwoAdicFriPcs::commit` computes it. +fn committed>( + dft: &D, + m: RowMajorMatrix, + added_bits: usize, + shift: F, +) -> Output { + Output(dft.coset_lde_batch(m, added_bits, shift).bit_reverse_rows().to_row_major_matrix()) +} + +pub fn run(args: &[String]) { + assert_eq!( + args.len(), + 6, + "--lde FIXTURE LOG_H WIDTH ADDED_BITS dit-parallel|recursive|small-batch true|false" + ); + let log_h: usize = args[1].parse().unwrap(); + let width: usize = args[2].parse().unwrap(); + let added_bits: usize = args[3].parse().unwrap(); + assert!(log_h + added_bits <= 24); + let h = 1 << log_h; + let validate: bool = args[5].parse().unwrap(); + let bytes = std::fs::read(&args[0]).unwrap(); + assert_eq!(bytes.len(), 4 * (2 + 2 * h * width)); + let words: Vec = bytes + .chunks_exact(4) + .map(|v| { + let v = u32::from_le_bytes(v.try_into().unwrap()); + assert!(v < 2130706433); + F::from_u32(v) + }) + .collect(); + assert_eq!( + words[0], + F::two_adic_generator(log_h + added_bits), + "fixture/library root mismatch" + ); + let shift = words[1]; + let inputs = [ + RowMajorMatrix::new(words[2..2 + h * width].to_vec(), width), + RowMajorMatrix::new(words[2 + h * width..].to_vec(), width), + ]; + let name = format!("lde-koalabear-{log_h}-{width}-{added_bits}"); + // Twiddles are precomputed (or cached by the warmup) outside timing; the input copy each + // call consumes is timed, like the NTT comparison's. + let measure = |run: &dyn Fn(RowMajorMatrix) -> Output| { + measure_workload(&name, "lde", 1, 2, LARGE, validate, |i| run(inputs[i % 2].clone())) + }; + match args[4].as_str() { + "dit-parallel" => { + let dft = Radix2DitParallel::::default(); + measure(&|m| committed(&dft, m, added_bits, shift)) + } + "recursive" => { + let dft = RecursiveDft::::new(h << added_bits); + measure(&|m| committed(&dft, m, added_bits, shift)) + } + "small-batch" => { + let dft = Radix2DFTSmallBatch::::new(h << added_bits); + measure(&|m| committed(&dft, m, added_bits, shift)) + } + other => panic!("unknown DFT {other}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use p3_field::Field; + + // Committed row r of every DFT is column j's interpolant at shift · ω^bitrev(r). + #[test] + fn committed_rows_are_bit_reversed_coset_evaluations() { + for (log_h, width, added_bits) in [(0, 1, 1), (1, 3, 1), (3, 2, 2), (4, 5, 1)] { + let h = 1usize << log_h; + let log_n = log_h + added_bits; + let values: Vec = (0..h * width).map(|k| F::from_u32(7 * k as u32 + 1)).collect(); + let m = RowMajorMatrix::new(values.clone(), width); + let shift = F::from_u32(3); + let wh = F::two_adic_generator(log_h); + let w = F::two_adic_generator(log_n); + let outputs = [ + committed(&Radix2DitParallel::::default(), m.clone(), added_bits, shift).0, + committed(&RecursiveDft::::new(h << added_bits), m.clone(), added_bits, shift).0, + committed(&Radix2DFTSmallBatch::::new(h << added_bits), m, added_bits, shift).0, + ]; + for r in 0..h << added_bits { + let i = r.reverse_bits() >> (usize::BITS as usize - log_n); + let x = shift * w.exp_u64(i as u64); + for j in 0..width { + // Lagrange interpolation of column j on ⟨ω_h⟩, evaluated at x. + let mut sum = F::ZERO; + for k in 0..h { + let xk = wh.exp_u64(k as u64); + let mut basis = F::ONE; + for l in (0..h).filter(|&l| l != k) { + let xl = wh.exp_u64(l as u64); + basis *= (x - xl) * (xk - xl).inverse(); + } + sum += basis * values[k * width + j]; + } + for out in &outputs { + assert_eq!(out.values[r * width + j], sum, "log_h={log_h} r={r} j={j}"); + } + } + } + } + } +} diff --git a/bench/rust/src/main.rs b/bench/rust/src/main.rs new file mode 100644 index 00000000..157898e6 --- /dev/null +++ b/bench/rust/src/main.rs @@ -0,0 +1,76 @@ +//! Matched CompPoly field workloads using Plonky3 and arkworks. +mod binary; +mod harness; +mod large_prime; +mod interpolate; +mod lde; +mod ntt; +mod small_prime; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Fixture { + encoding: String, + basis: String, + group_key: String, + field: String, + operation: String, + modulus: Vec, + inputs: Vec>, + exponent: u64, + latency_rounds: usize, + throughput_rounds: usize, +} + +impl Fixture { + fn validate_inputs(&self, width: usize) { + assert_eq!(self.encoding, "field-coordinates-le-v1"); + assert_eq!(self.basis, "canonical-integer"); + assert_eq!(self.modulus.len(), width); + assert_eq!(self.inputs.len(), 64); + for bytes in &self.inputs { + assert!(canonical(bytes, &self.modulus), "noncanonical field input"); + } + } +} + +/// Fixed-width, little-endian integer in [0, modulus), never silently reduced. +fn canonical(bytes: &[u8], modulus: &[u8]) -> bool { + bytes.len() == modulus.len() && bytes.iter().rev().cmp(modulus.iter().rev()).is_lt() +} + +fn main() { + let args: Vec<_> = std::env::args().skip(1).collect(); + if args.first().map(String::as_str) == Some("--interpolate") { + interpolate::run(&args[1..]); + return; + } + if args.first().map(String::as_str) == Some("--lde") { + lde::run(&args[1..]); + return; + } + if args.first().map(String::as_str) == Some("--ntt") { + ntt::run(&args[1..]); + return; + } + assert!( + args.len() == 1 || (args.len() == 2 && args[1] == "--validate-only"), + "usage: comppoly-field-bench FIXTURES.jsonl [--validate-only]" + ); + let source = std::fs::read_to_string(&args[0]).expect("read fixtures"); + for line in source.lines() { + let fixture: Fixture = serde_json::from_str(line).expect("parse fixture"); + match fixture.field.as_str() { + "koalabear" => small_prime::run::(&fixture, args.len() == 2), + "mersenne31" => { + small_prime::run::(&fixture, args.len() == 2) + } + "goldilocks" => { + small_prime::run::(&fixture, args.len() == 2) + } + "bn254-scalar" => large_prime::run(&fixture, args.len() == 2), + "tower-bt8" | "tower-bt64" | "tower-bt128" => binary::run(&fixture, args.len() == 2), + _ => panic!("unsupported field"), + } + } +} diff --git a/bench/rust/src/ntt.rs b/bench/rust/src/ntt.rs new file mode 100644 index 00000000..3f81582a --- /dev/null +++ b/bench/rust/src/ntt.rs @@ -0,0 +1,110 @@ +//! Existing optimized Plonky3 NTT API, with natural-order inputs and outputs. +use crate::harness::{measure_workload, BenchValue, DIGEST_MODULUS, LARGE}; +use p3_dft::{Radix2DFTSmallBatch, TwoAdicSubgroupDft}; +use p3_field::{Field, PackedValue, PrimeCharacteristicRing, PrimeField32, TwoAdicField}; +use p3_koala_bear::KoalaBear as F; + +struct Output(Vec); +impl BenchValue for Output { + fn checksum(self) -> u128 { + self.0.into_iter().fold(0, |acc, x| { + (acc * 16777619 + x.as_canonical_u32() as u128 + 97) % DIGEST_MODULUS + }) + } + fn sink(self) -> u64 { + let n = self.0.len(); + let mix = |acc: u64, x: u64| (acc ^ x).wrapping_mul(0x9E3779B97F4A7C15).rotate_left(27); + [n / 3, 2 * n / 3, n - 1] + .into_iter() + .fold(self.0[0].as_canonical_u32() as u64, |acc, i| { + mix(acc, self.0[i].as_canonical_u32() as u64) + }) + } +} + +pub fn run(args: &[String]) { + if args == ["--info"] { + println!( + "{}", + serde_json::json!({ + "implementation": "p3_dft::Radix2DFTSmallBatch", + "packing_width": ::Packing::WIDTH, + "workers": rayon::current_num_threads(), + }) + ); + return; + } + assert_eq!( + args.len(), + 4, + "--ntt FIXTURE LOG_N forward|inverse true|false" + ); + let log_n: usize = args[1].parse().unwrap(); + assert!(log_n <= 24); + let n = 1 << log_n; + assert!(matches!(args[2].as_str(), "forward" | "inverse")); + let validate: bool = args[3].parse().unwrap(); + let bytes = std::fs::read(&args[0]).unwrap(); + assert_eq!(bytes.len(), 4 * (1 + 2 * n)); + let values: Vec<_> = bytes + .chunks_exact(4) + .map(|v| { + let v = u32::from_le_bytes(v.try_into().unwrap()); + assert!(v < 2130706433); + F::from_u32(v) + }) + .collect(); + let root = values[0]; + assert_eq!( + root, + F::two_adic_generator(log_n), + "fixture/library root mismatch" + ); + // Precompute forward and inverse twiddles before validation and timing. + // Both the parallel feature and native packed arithmetic are enabled. + let plan = Radix2DFTSmallBatch::::new(n); + let inputs = [&values[1..n + 1], &values[n + 1..]]; + measure_workload( + &format!("ntt-koalabear-{log_n}-{}", args[2]), + &args[2], + 1, + 2, + LARGE, + validate, + |i| { + Output(if args[2] == "forward" { + plan.dft(inputs[i % 2].to_vec()) + } else { + plan.idft(inputs[i % 2].to_vec()) + }) + }, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + // Check the selected API's root, natural ordering and normalization contract. + #[test] + fn library_api_matches_direct_transform() { + for log_n in 0..=5 { + let n = 1 << log_n; + let root = F::two_adic_generator(log_n); + let plan = Radix2DFTSmallBatch::::new(n); + let input: Vec<_> = (0..n) + .map(|i| F::from_u32((i * i + 7 * i + 3) as u32)) + .collect(); + let output = plan.dft(input.clone()); + for (k, actual) in output.iter().enumerate() { + let expected: F = input + .iter() + .enumerate() + .map(|(j, x)| *x * root.exp_u64((j * k) as u64)) + .sum(); + assert_eq!(*actual, expected); + } + assert_eq!(plan.idft(output), input); + } + } +} diff --git a/bench/rust/src/small_prime.rs b/bench/rust/src/small_prime.rs new file mode 100644 index 00000000..74ce286b --- /dev/null +++ b/bench/rust/src/small_prime.rs @@ -0,0 +1,66 @@ +use crate::{ + harness::{latency, measure, throughput, BenchValue}, + Fixture, +}; +use p3_field::PrimeField64; +use std::hint::black_box; + +macro_rules! value { + ($field:ty) => { + impl BenchValue for $field { + fn checksum(self) -> u128 { + self.as_canonical_u64() as u128 + } + fn sink(self) -> u64 { + self.as_canonical_u64() + } + } + }; +} +value!(p3_koala_bear::KoalaBear); +value!(p3_mersenne_31::Mersenne31); +value!(p3_goldilocks::Goldilocks); + +pub fn run(fixture: &Fixture, validate_only: bool) { + let width = if F::ORDER_U64 > u32::MAX as u64 { 8 } else { 4 }; + assert_eq!(fixture.modulus, F::ORDER_U64.to_le_bytes()[..width]); + assert_eq!(fixture.inputs.len(), 64); + fixture.validate_inputs(width); + assert_eq!(fixture.exponent, 0x5A5A5A5A); + assert_eq!(fixture.throughput_rounds, 128); + let binary = matches!(fixture.operation.as_str(), "add" | "mul"); + assert_eq!(fixture.latency_rounds, if binary { 1280 } else { 64 }); + let xs: Vec = fixture + .inputs + .iter() + .map(|x| { + F::from_u64(u64::from_le_bytes({ + let mut word = [0; 8]; + word[..width].copy_from_slice(x); + word + })) + }) + .collect(); + let b = black_box(xs[0]); + macro_rules! binary { + ($op:expr) => {{ + measure(fixture, "latency", 1280, validate_only, |i| { + latency(|x| $op(x, b), 1280, xs[i % 64]) + }); + measure(fixture, "throughput", 1280, validate_only, |i| { + throughput($op, 128, std::array::from_fn(|k| xs[(i + k) % 64])) + }); + }}; + } + match fixture.operation.as_str() { + "add" => binary!(|a, b| a + b), + "mul" => binary!(|a, b| a * b), + "inv" => measure(fixture, "latency", 64, validate_only, |i| { + latency(|x| (x + b).try_inverse().unwrap_or(F::ZERO), 64, xs[i % 64]) + }), + "pow" => measure(fixture, "latency", 64, validate_only, |i| { + latency(|x| (x + b).exp_u64(0x5A5A5A5A), 64, xs[i % 64]) + }), + _ => panic!("unsupported operation"), + } +} diff --git a/docs/wiki/autoresearch.md b/docs/wiki/autoresearch.md index 25bf24e2..b7518ac8 100644 --- a/docs/wiki/autoresearch.md +++ b/docs/wiki/autoresearch.md @@ -193,10 +193,10 @@ its rows there. | Target | Groups | Fast implementation | Gate | Idea | |---|---|---|---|---| | `Ext.mul`, O(d³) → O(d²) | `fields-extension-{koalabear-ext4,babybear-ext4,koalabear-ext5,koalabear-ext6}-mul` | `CompPoly/Fields/Extension/Arithmetic.lean` (`mulTbl`, `red`) | `mul_eq_mulTbl` (`@[csimp]`), `toQuot_mul` in `CompPoly/Fields/Extension/Bridge.lean` | first loop run (`docs/bench-audit-2026.md` §12.9): `Fin.foldl` sums and an O(d²) table kept, net ~2× at d=4; the O(d²) convolution lost to a constant that is not arithmetic, so the next step is the runtime instance construction on this path, then the convolution again. Single-row groups: the build and the cross-binary digest are the only gates | -| Base-field arithmetic | `fields-{koalabear,babybear,mersenne31,goldilocks}-{mul,add,inv,pow}`, `fields-{bn254,bls12-381,bls12-377}-mul` | `CompPoly/Fields/Montgomery/Native32Field.lean`, `CompPoly/Fields/Goldilocks/Fast.lean`, `CompPoly/Fields/Mersenne31/Fast.lean`, `CompPoly/Fields/Montgomery/Native64x8Mul.lean` | `toField_*`, `ringEquiv`, `instField` | two to four digest-cross-checked rows per group; the best-behaved targets | -| Eight-limb inversion | `fields-mont64x8-{bn254,bls12-381,bls12-377}-inv` | `CompPoly/Fields/Montgomery/Native64x8InvDefs.lean` | `invGcdRaw_eq_inv` and its bounds chain in `CompPoly/Fields/Montgomery/Native64x8Inv.lean` | three algorithms in one group, one digest class | +| Base-field arithmetic | `fields-{koalabear,babybear,mersenne31,goldilocks}-{mul,add,inv,pow}`, `fields-{bn254,bls12-381,bls12-377}-{mul,add}`, `fields-secp256k1-{scalar,base}-{mul,add}` | `CompPoly/Fields/Montgomery/Native32Field.lean`, `CompPoly/Fields/Goldilocks/Fast.lean`, `CompPoly/Fields/Mersenne31/Fast.lean`, `CompPoly/Fields/Montgomery/Native64x4Mul.lean` | `toField_*`, `ringEquiv`, `instField` | two to four digest-cross-checked rows per group; the best-behaved targets | +| Four-limb inversion | `fields-mont64x8-{bn254,bls12-381,bls12-377,secp256k1-scalar,secp256k1-base}-inv` | `CompPoly/Fields/Montgomery/Native64x4InvDefs.lean` | `invGcdRaw_eq_inv` and its bounds chain in `CompPoly/Fields/Montgomery/Native64x4Inv.lean` | three algorithms in one group, one digest class | | Many-polynomial evaluation | `univariate-many-one-point-koalabear`, `univariate-dense-*` | `CompPoly/Univariate/ManyEval/Basic.lean`, `CompPoly/Univariate/Raw/Ops.lean` | `evalManyHorner_eq_map_eval`, `evalManySharedPowers_eq_map_eval`, `eval₂_horner_eq_eval₂` | plain array loops with short refinement proofs | -| Tower scalar arithmetic | `fields-tower-bt{8,64}-mul`, `fields-tower-bt64-inv-word`, `fields-tower-bt128-{mul,inv}` | `CompPoly/Fields/Binary/Tower/FastDefs.lean` | `mul64T_eq_mul64`, `inv64T_eq_inv64`, `toConcrete_*` in `CompPoly/Fields/Binary/Tower/Fast.lean` | table twins already close to optimal; low headroom | +| Tower scalar arithmetic | `fields-tower-bt{8,64,128}-{mul,square,inv}` | `CompPoly/Fields/Binary/Tower/FastDefs.lean` | `mul64T_eq_mul64`, `inv64T_eq_inv64`, `toConcrete_*` in `CompPoly/Fields/Binary/Tower/Fast.lean` | matched scalar workloads against Binius; separate `mul32T`/`mul64T` inline trials at `1aa011e` produced no `faster` verdicts; `mul64T` made 64-bit multiplication latency 5.1% slower, so neither was retained | | Additive NTT | `additive-ntt-btf3-l2-r2`, `additive-ntt-btf3-l4-r2`, `additive-ntt-btf4-l7-r2` | `CompPoly/Fields/Binary/AdditiveNTT/Impl.lean` | `computableAdditiveNTTFast_eq_computableAdditiveNTT` | the `btf4` group is fast-only; rely on the build | | NTTFast stages | `ntt-{koalabear,babybear}-l{8,10,12,14,16}`, `ntt-plan-koalabear` | `CompPoly/Univariate/NTTFast/Plan.lean` | `CompPoly/Univariate/NTTFast/Correctness/Pipeline.lean` | largest surface and largest proof burden; last | diff --git a/docs/wiki/benchmark-best-times.md b/docs/wiki/benchmark-best-times.md index 6980baf1..d54d5d04 100644 --- a/docs/wiki/benchmark-best-times.md +++ b/docs/wiki/benchmark-best-times.md @@ -16,6 +16,8 @@ overwritten. | Date | Commit | PR | Component | Rows | Note | |---|---|---|---|---|---| | 2026-09-21 | `1247810` | #371 | all | every group | first baseline: one full run at `--medium` | +| 2026-10-23 | `92c5d21` | #395, #401 | packed NTT, coset LDE | none | the packed kernels are timed outside the suite: coset LDE of 2^15 × 2633 words 17% faster on one thread, 11% on sixteen (Ryzen 7 3700X, [benchmarking.md](benchmarking.md#proved-stores-and-checks-once-per-loop-2026-10-23)) | +| 2026-10-23 | `c469bb4` | #401 | packed NTT, coset LDE | none | the packed kernels are timed outside the suite: CompStark's Keccak extension by columns, 2^15 × 2633 words, 22% faster on one thread and 27% on sixteen (Ryzen 7 3700X, [benchmarking.md](benchmarking.md#montgomery-products-by-high-words-leaf-stages-blocked-input-transposes-and-fused-coset-products-2026-10-23)) | ## How to read and how to update the tables @@ -235,6 +237,8 @@ against recursive (`CompPoly/Fields/Binary/Tower/FastDefs.lean`). the recursive definitions' floor, so the documented headroom is low. *Peer:* Binius, packed off. +The scalar rows below record the historical workloads at the listed commits. The matched Binius suite now uses different chains and fast-row names; these old times are not current baselines. + | Group | Row | Shape | Median | Per unit | Spread | Best at | |---|---|---|---:|---:|---:|---| | `fields-tower-bt128-mul` | `tower-bt128` · mul (ConcreteBTField) | 64 random 128-bit elements, pairwise | 5.81 ms | - | ±0.5% | `1247810` | diff --git a/docs/wiki/benchmarking.md b/docs/wiki/benchmarking.md index 5e16ef8e..d50893a1 100644 --- a/docs/wiki/benchmarking.md +++ b/docs/wiki/benchmarking.md @@ -81,7 +81,7 @@ Three ways to get timings: **Actions → Benchmarks → Run workflow** with a pr and optional group list; a `/bench` comment on a PR from a repo member, optionally followed by a group list; or automatically on a PR touching `bench/**`, since a change to the harness itself should be measured. Results -arrive as a PR comment and an artifact. +arrive as a PR comment and an artifact. Fork PR runs publish the report in the Actions summary and artifact without attempting a comment with their read-only token. Ordinary PR comments do not cancel active benchmark jobs; concurrency applies only after the benchmark job's comment filter. One thing the canary needs: it compares timed totals, so under `--validate-only` it would pass vacuously against a zero floor. `runTimed` therefore takes a @@ -277,8 +277,7 @@ A field operation is one or two nanoseconds and the harness floor is about combinators in `bench/CompPolyBench/Harness/Chain.lean` perform it `workUnits` times per iteration instead, and the report divides, giving the **per-unit** cost. Two chain shapes are reported, named as Plonky3 names them: *latency*, where -each operation depends on the last, and *throughput*, with ten independent -accumulators the pipeline can overlap. +each operation depends on the last, and *throughput*, with parallel accumulators (two for BN254, ten for other fields). Three properties of those combinators are load-bearing, and the obvious alternative is measurably wrong in each case: @@ -365,10 +364,576 @@ Recorded so they are not rediscovered. The audit and plan live in build. Marking the instance `noncomputable` is not the fix: `Extension.Ext` takes `[Fintype F]` and its operations then stop compiling, so the repair is to `CompPoly/Fields/Extension/` rather than to the instance. -- No external yardstick yet. Peers and the "on par" bar live in +- External comparisons beyond the small-field suite below remain to be added. Peers and the "on par" bar live in [`docs/bench-audit-2026.md` §13](../bench-audit-2026.md#13-external-comparison-targets): measure Plonky3 (scalar, SIMD off) for the small fields and multiplicative NTT, Binius for towers and the additive NTT, arkworks / gnark-crypto for pairing scalars. "On par" means within ~2–5× of those *scalar* kernels on the same CPU, not packed AVX-512 or a whole-prover bench. Do not cite published cycle tables. + +## Fields against Rust + +`python3 scripts/bench-fields.py --suite all --cpu 0` compares selected Lean groups with pinned Rust libraries. `--suite small-prime` covers KoalaBear, Mersenne31, and Goldilocks against Plonky3; `--suite large-prime` covers BN254 scalar add/mul latency and throughput, plus inv/exp latency, against arkworks. `--suite binary` covers 8-, 64-, and 128-bit Fan–Paar tower mul latency/throughput and square/inv latency against pinned Binius. `all` selects the 36 scalar cases and the NTT, coset interpolation and LDE suites. The default suite is `small-prime`. Choose an available logical CPU with `--cpu` for scalar workloads; use `--cpus` for the NTT. Lean and Rust are measured sequentially. + +The driver exports fixed-width little-endian coordinate byte inputs with an explicit basis from Lean and checks cross-language result digests and operation counts before timing. BN254 is explicitly matched to arkworks `Fr` by its modulus. Input decoding and canonical result checks are outside timing. See [the benchmark README](../../bench/README.md#rust-field-comparison) for field correspondence, pinned versions, and workload details. + +Five paired runs alternate executable order. The output directory under `bench/out/` contains two compact tables in `report.md`, raw samples, exact inputs, and machine/toolchain metadata. Tables report the median of run medians and the median absolute deviation between runs. These are same-machine library comparisons, not historical regression comparisons; other workloads on the host can affect them. + +`--validate-only` skips timing, and CI uses it with `--suite all` on every PR. `--skip-build` requires a matching build record from a previous driver run. + +BN254 add/mul throughput uses two independent chains with the same fixed second operand as latency, 160 steps per lane (320 operations total), four rounds unrolled per loop, and one final combining operation excluded from the divisor. This avoids the ten-lane ring’s excessive live state for multi-limb values. Small-prime throughput retains its ten-lane ring. + +The BN254 throughput configuration was selected manually on Rust, then fixed identically in Lean. It is not selected independently per language or tuned during benchmark runs. Rust screening covered 1, 2, 3, 4, 6, and 8 lanes with 1, 4, and 8 rounds unrolled; two lanes avoided the spill overhead of wider configurations. Shortlisted unroll factors were checked again in the production Rust runner. + +### Eight-limb BN254 kernel optimization + +The checked binary-GCD inverse uses native `Int64` transition coefficients inside each divstep chunk, converting to `Int` at the chunk boundary. The integer kernel remains available for the coefficient-bound proofs and regression comparisons. The native candidate still passes the same boundedness, canonicality, and multiplication checks before use; the proved Fermat fallback handles a rejected candidate. + +On the Ryzen 7 3700X, pinned to CPU 11, five interleaved `--small` A/B rounds against `76f4a91` measured inversion at 0.915× baseline time (8.14 → 7.44 μs per chain step), with matching digests and no `SUSPECT`. All 4,096 inversion inputs in the comparison workload accepted the native candidate. These host-specific observations do not update the separate M3 reference-machine best-time tables. + +Conditional reduction compares bounded limbs from the most significant end and stops at the first difference. The five-pair BN254 A/B run against the signed-word-inversion baseline measured multiplication throughput at 0.883× baseline time (65.64 → 57.95 ns/op), with no judged regressions in the other BN254 rows. + +Binary exponentiation calls the inline `square` directly, exposing equal operands to code generation while retaining the definitionally equal binary-recursion algorithm. Five interleaved A/B rounds against `7c0bcd7` measured exponentiation at 0.916× baseline time (2.74 → 2.51 μs per chain step), with no `SUSPECT` or other row classified as slower. + +Compact eight-`UInt32` storage, branch-free reduction, reversed multiplication operands, and borrowed-operand kernels were also screened. None was retained: compact storage improved addition but slowed multiplication, branch-free reduction regressed other operations, and the operand variants offered no reliable improvement. + +### Four-limb BN254 kernel optimization + +The Rust-comparison branch incorporates PR #391's four-limb, radix-`2^64` carrier. The benchmark workloads, lane counts, and cross-language operation counts remain unchanged. The eight-limb measurements above describe the preceding implementation, not the current carrier. + +The four-limb checked inverse reuses the native `Int64` divstep kernel. Five interleaved A/B rounds against merge commit `856f88e` measured inversion at 0.896× baseline time (6.75 → 6.05 μs per chain step), with no `SUSPECT` or other row classified as slower. The canonicality and multiplication checks and the proved Fermat fallback remain in place. + +Standalone multiplication screening compared swapped high-product operands, balanced partial-product sums, reordered carry accumulation, reassociated high-word additions, lexicographic final reduction, and reconstruction of a carry from the low product. Every variant checked 1,024 input pairs against the original kernel before timing. None showed a sufficient improvement to adopt; the reassociation gave only about 2–3%, while reconstructing the carry lengthened the dependency chain and slowed multiplication. These screening results do not replace the matched Lean/Rust measurements. + +The previous explicit-squaring exponentiation change was ported and tested in two forms. Inlining `square` regressed the exponentiation row to 1.164× baseline time; a separately specialized `square` measured 1.013× and was classified as unchanged. Neither was retained. The four-limb implementation keeps its original binary-exponentiation definition. + +### Addition ownership + +The four-limb BN254 addition chain originally tried to reuse its shared fixed operand. Generated code allocated a new result and released the old accumulator at every step. Specializing `FastField.add` while explicitly borrowing its second operand makes the first operand available for reuse. The benchmark loop then passes the accumulator onward without per-step reference-count increments or releases; only a shared input requires allocation. + +Five interleaved A/B rounds against `4502b00` measured addition latency at 0.382× baseline time (13.13 → 5.01 ns/op) and the two-lane row at 0.445× (12.27 → 5.46 ns/op), with no `SUSPECT` or other BN254 row classified as slower. This changes ownership and code generation, not field arithmetic or the matched benchmark workloads. The field carrier still lives in a heap object; reusing it does not give the register-only representation available to Rust. + +### One-bit carry encoding + +The four-limb `adc` and `sbb` helpers combine their overflow flags with bitwise OR. With a one-bit incoming carry or borrow, at most one flag can be set; the word-level specifications prove the same exact sum/difference identities and one-bit output bounds. OR makes the output bit range visible to native code generation, avoiding the extra arithmetic needed when the two flags are added. The multiply-accumulate helper keeps addition because both of its overflow flags may be set. + +Five interleaved A/B rounds against `17a1eb9`, on the same Ryzen 7 3700X pinned to CPU 11, measured addition latency at 0.773× baseline time (5.06 → 3.91 ns/op) and the two-lane row at 0.780× (5.50 → 4.29 ns/op). All digests matched, with no `SUSPECT` or other BN254 row classified as slower. The benchmark harness and matched Rust workloads are unchanged. + +Additional `inline`, `always_inline`, and `macro_inline` variants were screened but not retained: none reliably improved the borrowed, specialized addition boundary, and inlining the original operand order brought back per-step allocations. A standalone scalar-state control removed per-step calls and allocations but still took about 4.4 ns with the old carry encoding, versus about 5 ns for the ordinary chain. This isolates a substantial arithmetic/code-generation cost beyond object handling; it is a diagnostic, not a replacement benchmark result. + +### Spare-bit addition and scalar diagnostics + +For moduli below `2^255`, the four-limb addition now omits the impossible carry out of the top limb. The modulus's top bit selects the path; specialization resolves it for concrete fields. The correctness proof uses canonical input bounds to establish that the carry is zero. Full-width moduli retain the carry-aware path. + +Five interleaved A/B rounds against `b69a497` measured BN254 addition latency at 0.904× baseline time (3.86 → 3.49 ns/op) and the two-lane row at 0.897× (4.23 → 3.80 ns/op). All digests matched, with no `SUSPECT` or other BN254 row classified as slower. No benchmark workload changed. + +Assembly inspection identified a further difference from arkworks: arkworks compares against the modulus before subtracting, whereas the retained Lean addition calculates the difference before choosing the result. An ignored scalar-state diagnostic using the same 320 additions and fixture inputs reached approximately 1.77 ns/op by combining the spare-bit optimization with comparison before subtraction. It keeps limbs in scalar loop parameters and has no per-addition calls or field objects. This is a screening result, not the ordinary field API or a replacement row in the Lean/Rust report. + +The corresponding ordinary-API variants were not retained. Conditional subtraction introduced allocation on the reduction branch; moving construction after a tuple-valued branch, changing operand order, masking the modulus, and extracting subtraction into a separate function failed to beat the retained implementation. Thus the scalar diagnostic demonstrates arithmetic close to Rust, while obtaining it through the boxed field API remains open. + +### Scalar limb API for specialized hot loops + +`Montgomery.Native64x4.Scalar` in `CompPoly/Fields/Montgomery/Native64x4Defs.lean` exposes `add`, `sub`, `mul`, and `square` with separate `UInt64` limb arguments. These are inline entry points to the existing verified arithmetic, with the same input bounds and Montgomery representation. The first four arguments are the modulus limbs; multiplication and squaring also take the Montgomery negative inverse. Results are four-word tuples. + +Immediately destructure each result and carry its words as separate loop parameters. Construct a `Limbs4` or field value at the boundary. For example, a multiplication step inside such a loop is: + +```lean +let (r0, r1, r2, r3) := Montgomery.Native64x4.Scalar.mul + q0 q1 q2 q3 negInv a0 a1 a2 a3 b0 b1 b2 b3 +-- Continue with r0, r1, r2, r3 as separate scalar parameters. +``` + +The arguments are little-endian Montgomery residues, not canonical integers. Obtain the modulus limbs and negative inverse from `Mont64x4Field`; unpack existing fast field values through their `.val` limbs. Reconstructing a field value also requires the usual proof that the residue is below the modulus. The wrappers unfold directly to the existing operations, so their bounds and refinement lemmas remain applicable. + +Inlining and immediate destructuring allow the compiler to eliminate the intermediate tuples and limb objects. Storing the tuple as the loop state, passing it through a non-inlined function, or calling through an unspecialized function argument can reintroduce boxing. Inspect generated code and measure the actual loop: eliminating allocations can still lose to register spills or code layout, particularly for multiplication. + +A complete repeated-squaring loop illustrates the state layout: + +```lean +open Montgomery.Native64x4 + +def repeatedSquare (q : Limbs4) (negInv : UInt64) (n : Nat) (x : Limbs4) : Limbs4 := + let rec go (n : Nat) (x0 x1 x2 x3 : UInt64) : Limbs4 := + match n with + | 0 => ⟨x0, x1, x2, x3⟩ + | n + 1 => + let (s0, s1, s2, s3) := Scalar.square q.l0 q.l1 q.l2 q.l3 negInv x0 x1 x2 x3 + go n s0 s1 s2 s3 + go n x.l0 x.l1 x.l2 x.l3 +``` + +Exponentiation was tested as a possible production example, with three scalar-state variants. Against `7912071`, five interleaved A/B rounds per variant measured the binary-recursion version at 1.079× baseline time (1.85 → 2.00 μs), explicit right-to-left tail recursion at 1.042× (1.88 → 1.96 μs), and left-to-right recursion with a fixed base at 1.077× (1.87 → 2.01 μs). All BN254 digests matched. None was retained: the existing `FastField.pow` remains the production implementation. The scalar API makes unboxed loops expressible, but the multiplication-heavy exponentiation experiments did not establish a speedup. + +### Dedicated four-limb squaring for exponentiation + +The next BN254 pass screened dedicated squaring, scalar loop layouts, native-word exponents, and windowed exponentiation. Sharing the ten distinct limb products across the original four CIOS rounds was the winning square implementation. A full-product Comba square also helped in isolation, but less; special diagonal-product formulas regressed relative to ordinary `mulHi`. The retained `squareCached_eq_mul` theorem proves exact equality to the original multiplication for arbitrary limb inputs, and the field-level compiler rewrite preserves the original binary-exponentiation specification. + +Five interleaved production A/B rounds against `a4dd24a` measured BN254 exponentiation at 0.928× baseline time (1.863 → 1.729 μs per chain step), classified `faster` without `SUSPECT`. The fast add, multiply, and inversion rows were classified unchanged. Inputs, exponent, chain length, and benchmark code stayed unchanged. + +Four additional scalar loop layouts varied whether multiplication and squaring were inlined or outlined. All lost to the boxed control in the standalone screening. The fully inlined loop had a 280-byte stack frame, compared with 120 bytes in the boxed multiplication routine, consistent with register pressure from keeping both four-word states live. These are static assembly observations, not a measured attribution of cycles: hardware performance counters were unavailable to this user. Array-based windows of widths two through five and a width-three version with explicit table entries also lost. A native-word exponent alone gave only a small screening improvement. Screening results were used to choose production candidates, not as replacements for the matched Lean/Rust benchmark. + +A second retained change skips the initial multiply-by-one and the last unused square, without narrowing the natural-number exponent. Against the dedicated-square baseline, five production A/B pairs measured 0.922× time (1.757 → 1.619 μs), again `faster` without `SUSPECT`. Both loops are proved against field exponentiation through `toField` for every natural-number exponent. The two A/B ratios compound to about 14% less exponentiation time. + +### Parameter boundaries for canonical four-limb arithmetic + +The BN254 lazy-evaluation investigation also exposed an issue in ordinary canonical multiplication: embedding modulus words prevented Clang from recognizing widening products in Montgomery reduction. `FastField.mulKernel` and `FastField.squareKernel` keep those words dynamic across `@[noinline]` boundaries. The underlying `Native64x4` and `Scalar` APIs remain inlineable for custom loops. Arithmetic, canonical reduction, and the existing correctness theorems are unchanged; the field proofs unfold the new wrappers. + +Five alternating A/B pairs against `45d6384` on CPU 9 of the Ryzen 7 3700X, using the medium preset and unchanged benchmark workloads, measured: + +| BN254 operation | Before | After | Time reduction | +| --- | ---: | ---: | ---: | +| Multiplication latency | 35.01 ns/op | 20.86 ns/op | 40.4% | +| Multiplication throughput workload | 35.10 ns/op | 22.52 ns/op | 35.9% | +| Exponentiation chain step | 1.617 μs | 0.985 μs | 39.1% | +| Inversion chain step | 5.763 μs | 5.713 μs | Classified unchanged | + +Both multiplication rows and exponentiation were classified `faster`. Across BN254 and multiplication in BLS12-381 scalar, BLS12-377 scalar, secp256k1 scalar and secp256k1 base, the comparison judged 30 rows: 11 faster, 19 unchanged, none slower, no digest mismatch or `SUSPECT`. Harness drift stayed within 3%. The other fields' fast multiplication rows all improved. These are measurements on this Linux host, not replacements for the Apple reference-machine best-times table. + +The final multiplication kernel contains 32 widening multiplies and four low-word multiplies; the symmetric square contains 26 widening multiplies and four low-word multiplies. Neither contains partial-product shifts. An isolated earlier experiment changing multiplication alone reduced exponentiation time by 12.5%; adding the squaring boundary then reduced it by another 30%. Final figures above compare both boundaries directly against the original executable. No compiler flags, external functions, benchmark schedules or lazy-reduction rules changed. + +`Native64x4.mulUnreduced` exposes the shared four-round product before normalization, retaining all five intermediate limbs. Its `mulUnreduced_spec` theorem gives the range bound and Montgomery congruence with a canonical first operand and an arbitrary four-limb second operand. Canonical multiplication applies `condSubWide`; specialized lazy loops can use the same core and defer normalization, proving that the carry limb vanishes when the modulus bound permits it. The primitive stays inline, while the canonical `mulKernel` and `squareKernel` parameter boundaries remain out of line. + +### Goldilocks kernels from PR #392 + +The benchmark branch incorporates [PR #392](https://github.com/Verified-zkEVM/CompPoly/pull/392). Goldilocks uses a revised wide-product expression, a separate cold borrow path in reduction, shift/subtract for the reduction's middle term, and subtraction-based canonical addition. Inversion and exponentiation keep congruent, potentially noncanonical words inside the chain and canonicalize once at the end. The carrier and public field operations remain canonical. The PR's KoalaBear/BabyBear `conditionalSubtract` inlining was already present on this branch. + +Five interleaved A/B pairs on CPU 11 of the Ryzen 7 3700X compared `79a3ea0` with merge `770f992`, using the unchanged field-operation workloads: + +| Goldilocks operation | Before (ns/op) | After (ns/op) | After / before | +| --- | ---: | ---: | ---: | +| Add latency | 1.268 | 0.747 | 0.589 | +| Add throughput | 0.667 | 0.341 | 0.512 | +| Mul latency | 3.771 | 3.154 | 0.836 | +| Mul throughput | 1.200 | 1.262 | 1.052 | +| Inversion | 432.806 | 183.945 | 0.425 | +| Exponentiation | 146.258 | 89.556 | 0.612 | + +Multiplication throughput regressed; a second five-pair run confirmed 1.180 → 1.239 ns (1.050×). The other Goldilocks rows were classified faster. Addition throughput also received `SUSPECT: below chain floor`: the current comparer selects the latency chain floor for every row, including throughput. KoalaBear addition throughput, whose implementation was unchanged, received the same flag. This flag remains a limitation of that A/B result; the harness was not changed to remove it. + +A separate five-pair Lean/Rust run on clean `770f992` passed all 24 result checks. Goldilocks measured 0.99× Rust's time for add latency, 1.28× for mul latency, 0.44× for inversion, and 0.87× for exponentiation; throughput ratios were 0.59× for add and 1.45× for mul. All timings include the same operations and use the same inputs in both languages. Local artifacts are in `bench/out/goldilocks-pr392-20260930/` and A/B runs `bench/out/ab/260930-135904/` and `bench/out/ab/260930-140026/`. + +## Optimized Plonky3 NTT comparison + +`python3 scripts/bench-fields.py --suite ntt` compares the proved KoalaBear `NTTFast.NaturalPlan` forward/inverse transforms against the existing optimized `p3_dft::Radix2DFTSmallBatch` API, at 2^12, 2^16 and 2^20 elements. Plonky3 uses native packed arithmetic and its parallel feature. The default budget is one worker per available physical core, at most sixteen, without SMT siblings (`--cpus` overrides it; `--cpu` selects one worker). Both sides receive the same CPU allocation; the existing Lean plan remains sequential. Plans are outside timing. Input copying, ordering conversions and inverse normalization are timed. Both APIs use natural-order arrays and the same root; Lean's bit-reversal adapter is therefore part of the measured cost. Different algorithms and arithmetic schedules are allowed in this library comparison, replacing the original custom scalar Rust baseline. This suite is included in `--suite all` and CI validation. See [the operator guide](../../bench/README.md#koalabear-ntt-against-optimized-plonky3) for fixtures, validation and timing boundaries. + +### Lean natural-order NTT plans + +Import `CompPoly.Univariate.NTTFast.Natural` and construct `NTTFast.NaturalPlan.ofDomain domain` once, then call `plan.forward input` or `plan.inverse input`. The plan caches bit-reversal indices as well as the existing twiddle tables. Its forward path reuses correctly sized inputs until the first copy-on-write update; shorter inputs are zero-padded and longer ones truncated. Bit reversal swaps each pair once in the working array instead of gathering a second array; a shared input still receives the required copy-on-write copy. Normalization uses a specialized scalar loop. Both operations are proved equal to the existing natural-order pipelines for every input array. + +The optimized loops are selected explicitly through the new plan. Importing it does not register `csimp` replacements for the existing `NTTFast.Plan.forwardImpl` or `NTTFast.Plan.inverseImpl` entry points; their equivalence theorems remain ordinary proofs. + +The butterfly modules expose bounds-proved loops selected by the natural-order plan. Array ranges are checked once per complete radix-four stage, and the hot traversal uses `USize` indices. The quarter-one stage checks its unit twiddles once and omits their multiplications; these fast entry points require `DecidableEq R` for that check. The original total functions cover invalid ranges, and natural-number indexing covers ranges exceeding the machine-word limit. No new external functions or compiler changes are used. The NTT benchmark executable targets the native CPU, matching Rust's native targeting; the library retains portable build settings. + +The retained Lean implementation is sequential. Experiments with per-stage parallel gathering, recursive task splitting, byte-packed storage, two-lane software arithmetic, and sixteen-element blocks did not close the large-transform gap. The block prototype generated AVX2 instructions, but packing, allocation and sharing costs limited the gain; it is not part of the verified implementation. Plonky3 combines packed arithmetic with cache-local groups of layers and parallel mutable slices. A competitive Lean parallel version needs to address the storage and data-movement costs together with the butterfly schedule. + +### NTT storage and compiler investigation (2026-10-02) + +Follow-up experiments on Lean 4.34.0 separated storage costs from field arithmetic. These are engineering diagnostics, not new verified library implementations. The first experiment below used the verified `51bb43a` implementation as its control; the Rust executable remains frozen from `8a34661`. The subsequent in-place permutation improvement and further experiments are recorded below. + +**Generic arrays do not allocate one object per KoalaBear value on this 64-bit host.** The subtype proof is erased and `UInt32` is a tagged immediate in an eight-byte array slot. Rust stores four-byte field values contiguously. The sixteen-lane prototype instead puts sixteen native `UInt32` fields in a heap record; this enables vectorization but introduces record allocation and sharing costs. See Lean's [`lean_box_uint32`, array accessors and exclusivity check](https://github.com/leanprover/lean4/blob/293d5d0c0c3f3dded4688b3ccd6a33939ac5102b/src/include/lean/lean.h). + +The blocked prototype spent approximately 8–11 ms unpacking a million-element result in phase screens. Borrowed reads, compact permutation tables, fixed-lane gathers, parallel unpacking, local conversion before task joins, and cached leaf twiddles did not produce a decisive improvement. Whole-transform screens remained around 31–35 ms. These phase measurements include memory traffic and indexing; they do not isolate a single instruction cost. + +A separate byte-buffer prototype tested native 32-bit reads, batch stores, and appends while keeping all field arithmetic in Lean. The table below isolates the append change: both experimental variants use the same word reads, bounds handling, sixteen-way unrolled butterflies, batch stores and sixteen-task schedule. Only one appends each word with four existing byte pushes; the other appends a word with one capacity/ownership check and a four-byte store, falling back to the byte pushes when necessary. + +Million-element complete transforms, milliseconds; median of five alternating run rounds ± between-run MAD. Ryzen 7 3700X, 8 physical cores / 16 logical CPUs, same CPU allocation and fixtures for all variants; shared host. Full output digests matched the frozen Rust executable before and during timing. Conversion, copying, output ordering and inverse normalization are included; plans are excluded. + +| Implementation | Forward | Inverse | +|---|---:|---:| +| Retained verified Lean | 43.860 ± 0.151 | 48.450 ± 0.208 | +| Experimental native storage, four byte appends | 49.498 ± 0.816 | 51.128 ± 1.714 | +| Experimental native storage, one word append | 31.233 ± 0.215 | 30.711 ± 0.726 | +| Frozen parallel SIMD Plonky3 | 3.355 ± 0.016 | 3.299 ± 0.017 | + +The word append reduces this prototype's time by 37–40%, but leaves it roughly nine times slower than Plonky3. The native prototype has no full FFT refinement proof, and its word-copy implementation was only tested on this little-endian host. It is not installed in the library or the default benchmark. In that first experiment, native storage alone did not establish vectorized arithmetic: the sixteen-way byte-buffer inner loop did not emit the AVX2 arithmetic seen in the scalar-record prototype. A scalar byte-buffer screen with the word append also took about 31 ms. + +Two compiler/runtime findings guide further work: + +- **Record reuse depends on code shape.** A small scalar-record update reuses its allocation, while the large butterfly's generated C allocated new records. Splitting the update into smaller non-inlined record-update helpers enabled reuse, but the extra calls, moves and private copies made the complete transform slower. Better scalar replacement and reuse across larger expressions are compiler investigation targets; no minimal compiler fix or Rust-parity result has been established. The relevant compiler passes are [`ResetReuse`](https://github.com/leanprover/lean4/blob/293d5d0c0c3f3dded4688b3ccd6a33939ac5102b/src/Lean/Compiler/LCNF/ResetReuse.lean) and [`InferBorrow`](https://github.com/leanprover/lean4/blob/293d5d0c0c3f3dded4688b3ccd6a33939ac5102b/src/Lean/Compiler/LCNF/InferBorrow.lean). +- **Multithreaded objects are not considered exclusive.** `lean_is_exclusive` returns false for objects marked multithreaded, even when a later reference count might suggest a sole owner. This matters for mutation/reuse of records crossing task boundaries. An ownership-preserving task handoff could help, but shared reachable objects make it a runtime design problem; clearing the flag is not a safe shortcut. + +There was also a source-level specialization trap: placing `@[specialize]` on a concrete unpacking wrapper with no higher-order parameters left a generic callback in its element loop. Removing that annotation allowed the inner array builder to specialize and removed a large experimental regression. The [`Specialize` pass](https://github.com/leanprover/lean4/blob/293d5d0c0c3f3dded4688b3ccd6a33939ac5102b/src/Lean/Compiler/LCNF/Specialize.lean) explains that annotated bodies defer specialization to their call sites. Use the attribute on the higher-order builder, and inspect the resulting loop; it is not a blanket request to optimize any function. + +The most concrete small runtime extension is a packed UInt32 buffer API with proof-bounded reads, word appends and batch updates that check ownership once. It would remove repeated byte operations without moving field arithmetic into C. Achieving Plonky3-like performance still needs vectorizable kernels, cache-local layers and cheap partitioning of work; none of the measurements establishes that a single extern or compiler patch solves the full gap. + +### In-place permutation and vectorized storage follow-up + +The verified natural-order plan now swaps each bit-reversed pair once in its working array. A generic `Array.permuteInvolution_eq_map` theorem proves agreement with gathering for bounded involutive index tables; bit-reversal involution instantiates it. Inputs of other lengths retain the previous padding/truncation behavior. The arithmetic and Rust implementation are unchanged, and this improvement uses existing Lean primitives only. + +Five alternating single-worker rounds on CPU 9 compared the swap implementation against `51bb43a`. Complete-transform medians in milliseconds: + +| Elements | Direction | Previous gather | Pair swaps | New / old | +|---:|---|---:|---:|---:| +| 4,096 | forward | 0.0941 | 0.0929 | 0.988 | +| 4,096 | inverse | 0.1102 | 0.1116 | 1.013 | +| 65,536 | forward | 1.9714 | 1.8191 | 0.923 | +| 65,536 | inverse | 2.2476 | 2.1392 | 0.952 | +| 1,048,576 | forward | 44.6186 | 37.6438 | 0.844 | +| 1,048,576 | inverse | 48.4924 | 43.3625 | 0.894 | + +The repository comparer classified three rows faster and three unchanged, with no slower, mismatched or suspect rows. These transform records do not include harness floor/drift rows, so those checks were unavailable. After proving and integrating the change into `NaturalPlan`, a separate three-pair check reproduced the gains (million-point new/old ratios 0.825 forward and 0.923 inverse on the shared host). + +The native-storage experiment also advanced beyond the earlier 31 ms prototype. Moving `Nat`→`USize` conversions outside the sixteen-lane butterfly kernel enabled AVX2 arithmetic: the earlier machine code had 116 big-Nat conversion call sites and no vector multiplies; the machine-index kernel had 42 vector-multiply instructions. Batched sixteen-word partition appends and a 4×16 tiled output permutation reduced data-movement costs. All field arithmetic still comes from Lean; the experimental externs provide word storage operations and batched array output writes. + +A separate five-round interleaved comparison, sixteen workers on the same host, gave these million-point complete-transform medians ± between-run MAD: + +| Implementation | Forward (ms) | Inverse (ms) | +|---|---:|---:| +| Verified `51bb43a` control | 43.872 ± 0.289 | 48.018 ± 0.458 | +| Experimental vector kernel and batched partitioning | 21.903 ± 0.159 | 22.155 ± 0.084 | +| Same experiment with 4×16 tiled output permutation | 15.976 ± 0.303 | 17.592 ± 0.598 | +| Frozen Plonky3 | 3.341 ± 0.026 | 3.314 ± 0.008 | + +All full-output digests matched. These native prototypes remain outside the library and default benchmark: they lack a complete FFT refinement proof, and their word-copy externs were only exercised on the little-endian host. They narrow the measured gap to about 4.8–5.3×, without establishing Rust parity. A larger 16×16 output tile and an eight-lane kernel did worse in their screens; replacing conditional normalization with unsigned minimum did not improve the paired timings. Constant-twiddle transform variants were inconclusive because host contention rose during their screens. + +A bounds-proved packed-buffer read can avoid the conversion problem without a compiler change: its byte-size bound implies that the `Nat` index is a tagged immediate, allowing direct unboxing as in existing bounded array accessors. This version also generated vector multiplies and was within 2–4% of the explicit `USize` prototype in a three-pair single-worker check. The bound is essential; directly unboxing an arbitrary `Nat` would be incorrect. + +A separate diagnostic copied the Lean runtime header into a temporary compiler overlay. Marking the read-only big-Nat conversion `pure` alone did not change code generation. Also outlining its small-Nat wrapper enabled vectorization and reduced the Nat-indexed prototype's time by about 8–10%, but remained slower than using `USize` in the source. This is not a validated general-purpose runtime patch. Neither the installed toolchain nor the committed build settings were modified. The most direct extension remains proof-bounded packed UInt32 storage with batched ownership checks, paired with vectorizable Lean kernels and cache-conscious data movement. + +### Four/eight-core follow-up and fused final layers + +Further native-storage experiments used four physical cores (CPUs 2–5) or eight (CPUs 0–7), without SMT siblings. Sixteen leaf tasks stayed fixed. Four cores gave a steady final run of the verified library, but did not eliminate contention in every prototype run; the eight-core inverse comparison became substantially noisier as other work increased. + +The strongest new prototype fuses the last four FFT layers into a sixteen-coefficient kernel. Values stay in scalar locals between layers, and multiplications by unit twiddles are omitted. The previous prototype reread and rewrote the buffer between those small layers. All field arithmetic is still Lean; storage uses the earlier experimental word primitives. Full-output digests matched Plonky3 and the previous implementation at all eight validation sizes, in both directions. + +Five alternating rounds per core allocation, million-element complete transforms, milliseconds ± between-run MAD: + +| Physical cores | Direction | Previous native prototype | Fused final layers | Frozen Plonky3 | Fused / Rust | +|---:|---|---:|---:|---:|---:| +| 4 | forward | 30.290 ± 2.386 | 20.769 ± 1.897 | 6.618 ± 0.176 | 3.14× | +| 4 | inverse | 36.738 ± 2.443 | 25.196 ± 1.736 | 7.136 ± 0.923 | 3.53× | +| 8 | forward | 24.684 ± 1.027 | 15.164 ± 1.348 | 4.143 ± 0.052 | 3.66× | +| 8 | inverse | 42.291 ± 7.754 | 37.584 ± 4.611 | 5.347 ± 1.221 | 7.03× | + +The four-core repository comparison classified both rows faster, with strict separation of all five baseline/candidate runs, matching digests and no suspect rows. Its harness floor/drift checks were unavailable. The eight-core forward runs also separated strictly; inverse runs overlapped across changing host load. The four- and eight-core measurements were separate runs, so their absolute differences are not an isolated scaling experiment. The native FFT still lacks a complete refinement proof and is not installed in the library or default runner. + +Several narrower experiments did not earn a retained change. Keeping task outputs as separate leaf buffers removed concatenation copies, but its forward results were inconsistent. Batched input conversion improved forward by only about 1%. Reading each word using four bounds-proved `ByteArray.uget` calls produced expensive vector byte shuffles and was 3–4× slower than the word-read prototype in its three-pair screen; delaying Lean inlining did not recover the word-read performance. This gives a concrete reason to investigate a dedicated word-read primitive rather than assuming the byte operations will combine efficiently in the vectorized kernel. + +Precomputed [Shoup-style multiplication](https://www.libntl.shoup.net/doc/ZZ.cpp.html) for fixed twiddles reduced wide vector-multiply instructions in the large kernel from 48 to 16, but did not improve complete-transform performance. It enlarged the twiddle tables and changed the kernel's register demands; those costs were not isolated. An eight-lane Shoup kernel also regressed forward in its screen. Using Shoup only in the fused small kernel improved forward by about 9% in three pairs, while inverse was unchanged; this was not taken through a five-round gate. The selected prototype keeps Montgomery arithmetic. + +A fresh standard-driver run on clean `58d39a7`, four physical cores, measured the verified library at 36.851 ± 0.132 ms forward and 43.037 ± 0.121 ms inverse for a million points; Plonky3 measured 7.003 ± 0.278 and 6.770 ± 0.151 ms respectively. These verified results are separate from the native prototypes. Reproduce with `python3 scripts/bench-fields.py --suite ntt --cpus 2,3,4,5`. No Rust source, build flags or executable changed during the experiments. The next implementation work is a packed storage API with explicit word reads and batched updates, and refinement proofs for the fused transform. + + +### NTT extern minimization (2026-10-02) + +The selected packed-storage prototype now uses two new externs, totaling 774 bytes of inline C, down from eight and 7,414 bytes. One reads a UInt32 word with checked or proof-bounded indexing; the other writes or appends up to sixteen words, checking bounds, ownership and capacity once. Nat-indexed fallback reads/writes use Lean definitions, and a bounds-proved Lean output-store loop replaces the field-array extern. Its guard uses unsigned comparisons with a kernel-checked no-wrap lemma, avoiding big-Nat bounds checks in the common path. The remaining arithmetic and transform logic are Lean. + +A four-physical-core comparison gave the following million-point times, milliseconds; medians of seven alternating rounds. Both parallel implementations had four workers, and the native prototype kept sixteen leaf tasks. The verified implementation remains sequential. + +| Elements | Direction | Plonky3 | Verified Lean, no new externs | Lean prototype, 2 externs | +|---:|---|---:|---:|---:| +| 1,048,576 | forward | 6.273 ms | 37.012 ms (5.90×) | 14.847 ms (2.37×) | +| 1,048,576 | inverse | 6.186 ms | 42.908 ms (6.94×) | 16.540 ms (2.67×) | + +The repository comparer classified all six prototype rows at log sizes 12, 16 and 20 as unchanged, with no slower, missing, mismatched or suspect rows; the practical threshold is 5% and requires strict separation to judge a change. Small rows had five rounds, million-point rows seven; harness floor/drift groups were absent. Million-point new/old median ratios were 0.978 forward and 0.951 inverse. This establishes no detected regression, not a proof of identical performance. The new prototype also passed 1,408 compiled storage agreement checks covering shared input preservation, capacity growth, partial batches and invalid ranges; all FFT output digests matched across eight validation sizes in both directions. + +Further removal did sacrifice performance in the tested replacements. Replacing native word reads with four proof-bounded Lean byte reads increased million-point time by roughly 6×. A direct, unrolled Lean writer using the existing byte primitives was about 3–3.5× slower; the generic Lean storage fallback was slower still. Moving only copy/growth fallback handling into Lean reduced C to 648 bytes but regressed small transforms and was rejected. These screens do not prove that a different storage representation or compiler implementation could never avoid the remaining externs. + +At this stage the prototype was not fully refined or installed in the default runner. The retained storage primitives and their Lean definitions now live in `CompPoly/Univariate/NTTFast/Packed/Native.lean`; compiled agreement checks are in `tests/CompPolyTests/NTT/NativeStorage.lean`. The complete proof and current reproduction command are documented below under "Complete packed parallel FFT refinement". + + +### Input fusion and kernel normalization (2026-10-02) + +The selected native-storage prototype fuses public field-array reads with the first butterfly split, removing its separate serial packing pass. A bounds-proved, inlined accessor reads the sixteen input lanes; the generated right-split kernel has vector multiplies and no reference-count call sites. Inverse scaling is folded into a separate sixteen-coefficient final-layer kernel, so it runs within the parallel FFT tasks before their final packed stores. The decoder then only reorders and converts those already-normalized values. Both changes are Lean; the two storage externs and their 774 bytes of inline C are unchanged. + +Kernel normalization is used when the leaf transform has an even log size of at least four. Other sizes retain normalization in the decoder. A zero task depth uses the existing packing path. The final source includes no `sorry`, but still lacks a complete FFT refinement theorem; it remains outside the verified library and default benchmark. + +The final comparison froze both prototypes, the verified no-new-extern implementation and the unchanged Plonky3 executable before timing. It used four physical cores (CPUs 2–5), four workers, sixteen leaf tasks, seven alternating rounds for million-point transforms and five for smaller sizes. Complete-transform medians, with conversion and normalization included: + +| Elements | Direction | Previous prototype | Combined prototype | New / old | +|---:|---|---:|---:|---:| +| 4,096 | forward | 0.2228 ms | 0.2009 ms | 0.902 | +| 4,096 | inverse | 0.2124 ms | 0.1999 ms | 0.941 | +| 65,536 | forward | 0.9522 ms | 0.8447 ms | 0.887 | +| 65,536 | inverse | 1.1092 ms | 0.8323 ms | 0.750 | +| 1,048,576 | forward | 14.9700 ms | 13.8785 ms | 0.927 | +| 1,048,576 | inverse | 17.3561 ms | 14.1433 ms | 0.815 | + +The repository comparer classified all three inverse rows faster and all three forward rows unchanged, with no slower, missing, mismatched or suspect rows. The million-point inverse gain was 18.5%, with strict separation of the seven baseline/candidate runs; the forward median fell 7.3%, but overlapping runs did not establish a gain under the comparer. Harness floor/drift groups were absent. Rust measured 7.729 / 6.601 ms and verified Lean 37.300 / 42.655 ms at one million points, forward/inverse. The shared host remained busy, so those Rust figures should not be compared with older runs to infer a Rust implementation change; its executable SHA256 stayed `83479b757394874fc7304c974b13f48461176691ca9711a144876648ccc18229`. + +All output digests agreed with verified Lean and Plonky3 at log sizes 0, 1, 3, 4, 5, 12, 14, 15, 16, 17 and 20 in both directions. Additional agreement checks used task depths 0, 1, 2, 4 and 5 at log size 5; 0, 1, 2, 3, 4, 8 and 10 at log size 12; and 0, 1, 2, 3 and 5 at log size 20. These exercise unfused input paths, odd-sized leaves and leaves too small for kernel normalization. The unchanged externs also passed 1,408 compiled agreement checks against their Lean storage specifications. + +Independent screens informed the final choice. Input fusion alone improved the 65,536-point rows but did not establish a million-point forward gain. Kernel normalization alone lowered the million-point inverse median by about 11% in three rounds. Building four natural-order field arrays and concatenating them regressed; replacing the concatenation tree with a single bounds-proved assembly loop also regressed. Returning packed natural-order chunks avoided the field-array joins but added a complete output pass and did not improve the screen. The selected version keeps the existing tiled output conversion and performs inverse normalization before the output stage, rather than adding output tasks. + +The retained input-fusion and kernel-normalization implementation is now tracked in `CompPoly/Univariate/NTTFast/Packed/Native.lean`. The complete proof and standard three-way reproduction command are documented below under "Complete packed parallel FFT refinement"; the historical experiment sources and drivers were local, ignored artifacts. + +### Externless NTT loop optimization (2026-10-02) + +The retained natural-order plan now checks ranges once per stage, advances block bases with machine-word arithmetic, and specializes the quarter-one radix-four stage to remove three unit-twiddle multiplications per butterfly. Inverse normalization reuses its working array with one bounds-proved write per coefficient. Separate size, coefficient and traversal lemmas prove these loops equal to the original total functions, including malformed tables and padding/truncation cases. There are no new externs, compiler changes, `implemented_by` routes or `csimp` registrations. + +Each retained optimization passed compiled digest validation against the frozen Plonky3 executable and an interleaved timing comparison. Seven-round gates require a median improvement of at least 5% and every candidate invocation faster than every baseline invocation for a `faster` verdict. No row was marked `SUSPECT`; these dedicated NTT rows do not include harness drift or timing-floor canaries. Measurements use four physical Ryzen 7 3700X cores as the allowed CPU set; this verified implementation still executes sequentially. + +Rejected variants included batched four-output stores, four-lane arithmetic batches, 32-bit permutation caches, bounds-proved direct gathers, moving normalization into the input gather and four-layer scalar fusion. The fused kernel improved the larger transforms by only 3–5%, below the retention gate. Parallel array prototypes remain research: splitting, gathering and joining must be considered together, and any retained parallel implementation still needs a complete refinement proof. Sources, frozen binaries and paired measurements from this pass are under the ignored local `bench/out/ntt-externless-investigation-oct2/` directory. Reproduce the retained library path with `python3 scripts/bench-fields.py --suite ntt --cpus 2,3,4,5`. + +### Proved parallel natural-order NTT + +`NTTFast.NaturalPlan.forwardParallel` and `inverseParallel` use up to `2 ^ logWorkers` independent array segments (default `logWorkers = 4`). Forward runs the largest fused stages before splitting; inverse runs the independent smaller stages before joining and finishing the largest stages. Tasks have separate mutable arrays and use dependency joins rather than blocking worker threads. The complete forward and inverse functions are proved equal to the existing scalar natural-order APIs, with no compiler substitutions or new externs. + +The NTT comparison runner sets `logWorkers` from `LEAN_NUM_THREADS`, which the driver sets to the selected CPU count. Transforms below 2^18 elements use the scalar loop to avoid task and copy costs; aligned segments stop splitting below 16,384 elements. All selected CPU counts remain configurable with `--cpus`. + +On the Ryzen 7 3700X (eight physical cores / sixteen hardware threads), seven alternating million-point rounds on CPUs 0–15 measured 28.581 ms forward and 29.785 ms inverse for the proved parallel implementation, versus 31.087 ms and 34.621 ms for the scalar control. Both large rows passed the 5% retention gate with strict separation; four smaller scalar rows were unchanged. The comparison copies aligned the old/new method labels; original samples, checksums and work-unit counts were preserved. Harness drift and timing-floor canaries were absent from these dedicated NTT rows. + +The unchanged native-storage prototype measured 12.468 / 12.385 ms, and frozen stock Plonky3 measured 3.377 / 3.338 ms, forward/inverse. All complete output digests matched at log sizes 0, 1, 3, 4, 5, 12, 14, 15, 16, 17, 18, 19 and 20. The large odd size exercises the parallel path and the final radix-two stage together. The measured source was clean `8ad5024`; raw local measurements and executable hashes are in ignored `bench/out/ntt-externless-investigation-oct2/proved-parallel-sixteen-workers.json`. + +Reproduce the verified comparison with `python3 scripts/bench-fields.py --suite ntt --cpus 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15`. Input copying, ordering, output disposal and inverse normalization are included; plans and decoding are outside timing. Sharing, copies, joins and the larger sequential stages still limit scaling. This pass adds no externs or compiler changes. + + +### Native prototype parallel buffer collection + +The selected native-storage prototype now returns an array of packed leaf buffers from its parallel task tree. Task joins copy only buffer references. After the tasks finish, one preallocated packed buffer collects all leaves in order, and the existing tiled decoder produces natural-order field output. This transfers the externless implementation’s single-assembly strategy while preserving the native prototype’s fused input loading, vectorized arithmetic and fused inverse normalization. Its two storage externs and their 774 bytes of inline C are unchanged. + +The new `splitChunks_eq`, `splitInputChunks_eq` and `run_eq_before_collection` theorems prove that collection and assembly preserve the previous prototype’s packed buffer and final output for every input and task depth. Their axiom dependencies are only `propext`, `Classical.choice` and `Quot.sound`; there is no `sorry` or native-compiler trust. These are scheduling/output-equivalence proofs, not the complete FFT refinement theorem, which remains pending for the native prototype. The verified library and default runner remain the proved externless implementation. + +Five alternating rounds compared the candidate with the previous native prototype and frozen Plonky3 at one million points, separately using eight physical cores (CPUs 0–7) and sixteen hardware threads (CPUs 0–15) on the Ryzen 7 3700X. Each invocation supplies a median; “mean” below averages those invocation medians. Both means and medians improved in both directions at both worker counts: + +| Workers | Direction | Previous mean | Selected mean | Previous median | Selected median | +|---:|---|---:|---:|---:|---:| +| 8 | forward | 12.181 ms | 11.453 ms | 12.106 ms | 10.891 ms | +| 8 | inverse | 11.497 ms | 10.985 ms | 11.435 ms | 10.875 ms | +| 16 | forward | 12.268 ms | 11.336 ms | 12.027 ms | 11.306 ms | +| 16 | inverse | 12.299 ms | 11.296 ms | 12.368 ms | 11.174 ms | + +The automatic 5% gate classified all four rows `same`, because timing ranges overlapped or the improvement fell below 5%. Gregor explicitly chose to retain the change based on the consistently lower averages across worker counts. The newer sixteen-worker sweep measured frozen Rust at 3.331 / 3.280 ms forward/inverse, within 2% of the earlier reference sweep. Full-output digests matched the previous prototype and Rust at thirteen log sizes, 0, 1, 3, 4, 5, 12, 14, 15, 16, 17, 18, 19 and 20, in both directions; the verified implementation has matching digests at those sizes. The storage externs passed 1,408 compiled agreement checks. + +The retained collection implementation and its `splitChunks_eq`, `splitInputChunks_eq` and `run_eq_before_collection` equivalence proofs now live in `CompPoly/Univariate/NTTFast/Packed/Native.lean`. The complete refinement and standard three-way reproduction command are documented below under "Complete packed parallel FFT refinement"; reproducing the current implementation requires no local experiment files. + +A later attempt to rerun all implementations and smaller sizes encountered a concurrent host build: load rose above 26 and even frozen Rust varied from roughly 6 to 156 ms between invocations. That sweep was stopped, retained as `bench/out/ntt-native-port-oct2/interrupted-busy-host-sweep.json`, and excluded from published results. The PR’s selected native numbers therefore come from the earlier stable paired sweep; verified Lean and its displayed Rust reference retain their earlier measurements. The default verified benchmark source was restored and rebuilt, and the frozen Rust executable was not changed. + +### Complete packed parallel FFT refinement + +The selected native-storage implementation is now in `CompPoly/Univariate/NTTFast/Packed/`, with a complete kernel-checked forward and inverse refinement. `Packed.Plan.forward_correct` and `inverse_correct` relate the public total array APIs directly to `NTT.Forward.forwardSpec` and `NTT.Inverse.inverseSpec`, including zero-padding/truncation, arbitrary split depth, natural ordering and exactly one inverse normalization. The arithmetic, fused sixteen-lane leaves, recursive parallel split tree, single buffer collection and tiled decoder are all covered. Only the same two inline C storage replacements remain runtime trust assumptions; their bytewise Lean definitions provide the logical model. Word copies require a little-endian host. + +For review, start with `CompPoly/Univariate/NTTFast/Packed/Plan.lean` (public API) and, since the split tree and tiled decoder were replaced (see "One column pipeline for both plans" below), `CompPoly/Univariate/NTTFast/Packed/ColumnsCorrectness.lean` (whole pipeline). Local arithmetic and loop lemmas are split into the other modules to keep kernel checking bounded. The concrete and generic expression graphs are separate to avoid expanding the concrete field dictionary during butterfly algebra proofs. The new `Data/Bytes/UInt32.lean` codec proves the four-byte little-endian storage representation. + +The normal NTT benchmark now measures both proved Lean variants beside unchanged stock Plonky3; no ignored prototype source is required to reproduce it. Run `python3 scripts/bench-fields.py --suite ntt --cpus 0,1,2,3,4,5,6,7`. Use sixteen CPUs for the sixteen-worker comparison. `NTT_DEPTH=4` is the default packed split tree; change it to vary the number of leaf tasks independently of the worker budget. `CompPolyPackedNative` compiles only the packed arithmetic module with `-march=native`; the benchmark root has the same flag and proof artifacts remain platform independent. The public API and kernel bodies preserve the selected prototype's arithmetic and task schedule. + +`lake exe CompPolyNativeSmoke` also checks the actual C storage implementations against independent bytewise Lean operations. These runtime checks cover the remaining extern trust boundary rather than re-testing results already guaranteed by the field/FFT proofs. + +The complete proof port was measured with the normal driver in five alternating three-way rounds at eight and sixteen workers. Million-point forward/inverse medians at eight workers were 11.095 / 10.828 ms packed Lean, 28.514 / 32.876 ms externless Lean and 3.670 / 3.826 ms Plonky3. At sixteen workers they were 11.636 / 11.607 ms packed Lean, 27.464 / 29.124 ms externless Lean and 3.318 / 3.343 ms Plonky3. The Rust executable retained its frozen SHA-256. Separate five-round prototype/proved comparisons found no slowdown at either worker count; full-output digests agreed at thirteen log sizes in both directions. All 1,548 compiled storage checks, library build, tests, style/import/documentation checks and the axiom sweep passed, with zero sorry/nonstandard-axiom taint across 11,986 declarations. Local reports are under ignored `bench/out/ntt-proved-packed-{eight,sixteen}/`; the normal driver reproduces their three-way tables. + +### Parallel natural-order output and packed I/O (2026-10-06) + +Phase timings of the packed pipeline at one million points and eight workers showed that the serial tiled decoder took about 4.5 ms of an 11 ms forward transform, while all FFT arithmetic and task joins took about 4.9 ms. Single-worker packed Lean took 20 ms against 10 ms for Plonky3, so most of the gap was scaling, not arithmetic. `NTTFast.Packed.NativeOrder` replaces the serial decoder for sixteen-leaf trees (`depth = 4`, `12 ≤ logN ≤ 28`): sixteen tasks reverse each leaf locally, then sixteen tasks interleave the reversed leaves into natural order with cache-line `16 × 16` transposes. Task outputs are packed buffers, so the join is a byte copy. The field-array API then decodes the natural-order buffer in one sequential pass; the new `forwardPacked`/`inversePacked` API returns the packed buffer directly. Other shapes keep the tiled decoder. `NativeOrderCorrectness` proves the new path equal to `decodedFields`, and `runFields_dft`/`runPacked_dft` give the complete DFT equalities used by the plan theorems. No externs were added; the module joins `CompPolyPackedNative`. + +Rejected and diagnostic variants, measured in throwaway executables: + +- A decoder that gathers bit-reversed words directly into natural-order blocks took 16 ms serially for one million words: each read misses, and each output block touches one word per cache line of the source. +- `Array` append is an element-by-element push. Joining sixteen field-array blocks took 2.7 ms, as long as decoding everything serially, so parallel tasks must join packed buffers. +- `lean_array_push` is an out-of-line runtime call. Decoding into a preallocated array with bounds-proved `uset` and a `USize` loop took 2.3 ms instead of 2.7–3.1 ms; a `Nat` fuel counter alone added about 0.7 ms. This 2.3 ms, including 0.7 ms to allocate and zero the output, is the remaining serial floor of the field-array API. +- Reading the sixteen leaves one word at a time aliased sixteen equally aligned streams to the same L1 sets: 0.74 ms per interleave block versus 0.09 ms for the line-wise transpose. +- A fused radix-sixteen first pass over sixteen strided rows (`n/16` apart) was slower than the four split levels, for the same aliasing reason. A cache-blocked four-step layout (sixteen-column blocks in L2, 1,024-point rows in L1) was correct but did about 25 ms of total work against 19 ms for the split tree, mostly per-leaf overhead and bit reversal of 1,024 small leaves. + +Five alternating rounds with the normal driver, one million points, milliseconds: + +| Workers | Direction | Packed I/O Lean | Packed Lean | Plonky3 | Packed I/O / Rust | Packed / Rust | +|---:|---|---:|---:|---:|---:|---:| +| 8 | forward | 6.274 | 10.375 | 4.017 | 1.56× | 2.58× | +| 8 | inverse | 6.185 | 10.303 | 3.954 | 1.56× | 2.61× | +| 16 | forward | 6.696 | 10.596 | 3.364 | 1.99× | 3.15× | +| 16 | inverse | 6.492 | 10.312 | 3.284 | 1.98× | 3.14× | + +Before this change, packed Lean measured 11.1–11.6 ms (3.5× Plonky3 at sixteen workers). The field-array API improves by about 1 ms; packed I/O removes the remaining serial decode. Small transforms still lose to task overhead: at 4,096 points the packed variants take 0.33–0.44 ms against 0.07–0.09 ms for externless Lean and Plonky3. Reproduce with `python3 scripts/bench-fields.py --suite ntt --cpus 0,1,2,3,4,5,6,7`; local reports are under ignored `bench/out/ntt-natural-order-{eight,sixteen}/`. + +`Plan.defaultDepth` now chooses the packed task tree from the size: serial up to 2^12 elements, four leaves up to 2^14 and sixteen leaves above. A depth sweep at 2^12–2^16 found the serial path fastest at 2^12, four leaves fastest at 2^13–2^14 and sixteen leaves fastest from 2^15. In five paired rounds against the previous commit at eight workers, 4,096-point transforms fell from 0.31–0.34 ms to 0.06–0.07 ms (strict separation), on par with Plonky3's 0.066 ms; larger sizes use the unchanged depth four. `NTT_DEPTH` still overrides the depth. + +A single-pass replacement for the leaf reversal and interleave was also proved and measured, then rejected. Writing natural index `i = a * 2 ^ (logN - 4) + 16 * b + c`, tile `b` reads one whole line from each leaf and appends one whole line to each of sixteen per-`a` accumulators, so no assembly or per-word reversal is needed. In isolation it took 1.09 ms instead of 1.4 ms at eight workers, but five paired end-to-end rounds at eight and sixteen workers showed no difference beyond the 5% noise of this shared host, so the simpler committed version stays. In the same investigation, out-of-place radix-2 and radix-4 splits that write two or four outputs per pass were slower than separate `splitLeft`/`splitRight` passes (1.9 and 3.4 ms against 1.2 ms per level pair at 2^20), and splitting the top tree levels into more tasks did not reduce the 4 ms tree time. With the host load near three, total CPU work rather than the critical path limits the eight-worker time: one million points cost about 9 ms of leaf FFTs, 5 ms of split levels and 4.5 ms of reordering, against Plonky3's 10 ms for a complete single-threaded transform. + +### Sliced split tree (2026-10-06) + +The binary split tree ran its top levels with only two, four and eight tasks, and each node read its input twice: `splitLeft` and `splitRight` each stream both halves. `NTTFast.Packed.SliceTree` runs every split level as `P` chunk tasks. A chunk reads a pair of input ranges once, with two proof-bounded sixteen-lane kernels (`pairLeft`, `pairRight`) called back to back so the second read hits L1, and appends both children's outputs. Children keep the chunk outputs as slices and pair them again, so no level concatenates its input; with `P = 2 ^ (depth - 1)` each leaf receives one slice. The first level of the field-array API reads the field array directly (`pairInputLeft`, `pairInputRight`). `Native.sliceCount` enables slicing from 2^18 elements; smaller transforms, and shapes the slicing does not cover, keep the binary tree. `SliceTreeCorrectness` proves that the sliced trees assemble to exactly the leaves of `splitChunks` and `splitInputChunks` for every depth and chunk count, so the existing tree theorems give the complete DFT equalities. No externs were added. + +One fused pass with two outputs took 0.62 ms per split level at 2^20 serially, against 0.52 + 0.66 ms for the separate passes. Two attempts failed first: a single fused kernel computing both outputs did not vectorize its multiplications (48 scalar `imul`) and took 1.9 ms, and kernels with checked word reads took 3.6 ms because each read's branch blocks vectorization. Paired five- to seven-round comparisons against the previous commit, one million points: + +| Workers | Direction | Packed I/O before → after | Packed before → after | +|---:|---|---:|---:| +| 8 | forward | 6.25 → 5.40 ms (strict) | 9.78 → 9.81 ms (overlap) | +| 8 | inverse | 6.39 → 5.46 ms (strict) | 9.76 → 9.61 ms (overlap) | +| 16 | forward | 6.54 → 5.65 ms (strict) | 10.29 → 9.59 ms (strict) | +| 16 | inverse | 6.71 → 5.58 ms (strict) | 10.29 → 9.57 ms (strict) | + +The field-array API gains less because its sequential decode is unchanged. At 2^16 the binary tree remains (all paired rows overlapped). + +### Leaf-local reversal (2026-10-06) + +Each leaf task now also reverses its own leaf (`leafRev`, one `reverse32` per sixteen words with strided reads), so the reordering no longer waits for all leaves, assembles a buffer or spawns a second task wave; `naturalLeaves` interleaves the reversed leaves directly. `sliceChunks` and `sliceInputChunks` take the leaf post-processing function, and their theorems became array equalities: the leaves equal the `splitChunks` leaves, post-processed. `splitChunks_leaves` and `splitInputChunks_leaves` characterize every leaf as its packed block of the DIF output, which `naturalLeaves_packFields` decodes. Where the tree falls back to `splitChunks`, the post-processing runs as parallel tasks after the leaves. Five to seven paired rounds against the sliced-tree commit, one million points, eight workers: packed I/O 5.38 → 4.92 ms forward and 5.43 → 4.80 ms inverse, the field-array API 9.15 → 8.35 and 9.30 → 8.50 ms, all with strict separation; 2^16 rows were unchanged. A radix-2 leaf kernel, which keeps fewer values live than the radix-four `step16`, was slower (0.70 against 0.56 ms per 2^16 leaf), so the arithmetic kernels are unchanged. + +The radix-four kernel `step16` and the fused final-layer kernels stored through `write16 b i.toNat`, a machine index converted to a natural number and back. The conversion's cold path calls into the runtime, so LLVM spilled the live vector registers around every store: 263 of `step16`'s roughly 720 instructions were `vmovdqu`. `write16U` stores at the machine index directly (110 `vmovdqu`), and `write16U_eq` reduces it to `write16` in the refinement proofs. A serial 2^16 leaf went from 0.573 to 0.526 ms; seven paired rounds at one million points and eight workers measured packed I/O 4.82 → 4.60 ms forward (strict) and 4.84 → 4.74 ms inverse (overlapping), with the other rows unchanged within noise. + +Two leaf-kernel alternatives were measured and rejected. The fused kernel `leaf16` for the last four layers compiles to scalar code (51 `imul`), because its butterflies stay inside a sixteen-word block. Reversing the leaf before those layers turns them into stride-`2 ^ (m - 4)` butterflies across rows with broadcast twiddles, which could run sixteen columns as vector lanes; both prototypes of that column kernel (record-valued line operations, and per-lane bindings with checked reads) compiled to scalar code and made a 2^16 leaf slower (0.72–0.88 against 0.63 ms). Leaves of odd log size keep a scalar final radix-two layer and cost about twice as much per element-layer as even ones; the benchmarked sizes use even leaves. Allocating the field-array output in a task spawned at the start, to overlap its zero fill with the transform, made one-million-point transforms 1.6× slower: an array returned by a task is marked multithreaded, so the first `uset` copies all eight megabytes. + +### Column-parallel externless NTT (2026-10-07) + +`NTTFast.NaturalPlan.forwardParallel` and `inverseParallel` (now in `NTTFast/NaturalParallel.lean`) route exactly sized even transforms of at least `2 ^ (logWorkers + 5)` elements through `NTTFast/Columns.lean`. The array is viewed as sixteen rows; `2 ^ (logWorkers + 1)` column tasks copy their column range and run the two top radix-four passes locally, sixteen leaf tasks each collect one row, run the remaining passes and reverse their own bit order, and one sequential pass interleaves the leaves into natural order. The inverse runs the same forward transform with the inverse twiddles and scales inside the leaves. Every coefficient crosses task boundaries twice instead of the segment plan's serial top passes and permutation. `ColumnsCorrectness.lean` proves the transform equal to the scalar stages followed by the bit-reversal permutation, so both entry points keep their equality theorems with the scalar natural-order APIs; `NaturalPlan` now carries `Plan.WellFormed` for the inverse. + +Paired 2^20 runs against the previous segment plan (five alternating rounds): on CPUs 0–7 forward 27.9 → 24.7 ms and inverse 28.2 → 20.9 ms (strict separation); on CPUs 0–15 forward 26.9 → 26.1 ms and inverse 28.1 → 26.7 ms (overlapping). Phase timing shows the remaining cost is mostly runtime sharing work, not arithmetic: all column tasks together compute in about 8.6 ms serially, but returning their arrays costs about 3 ms of wall time even on sixteen workers, because the runtime marks every task result for sharing element by element while holding the task manager's global lock (`resolve_core` in `src/runtime/object.cpp`). Marking a million-entry boxed array costs about 2.7 ms on one thread. + +The column and leaf tasks now return `ByteArray`s of raw words (`Word32Repr`, see [serialization](serialization.md)), so no boxed array crosses a task boundary; `forwardParallel` and `inverseParallel` require the instance, which `Native32.FastField` provides. Leaf `l` stores its words after `16 l` padding words: reading the sixteen equally aligned leaves at the same index mapped them to the same L1 sets, and the padding cut the sequential interleave of a million entries from 9.4 to 2.8 ms in the prototype. The interleave also checks the leaf sizes once instead of per read, which was about 8 ns per word in this loop. Five alternating 2^20 rounds against the boxed-array commit: on CPUs 0–15 forward 25.8 → 13.0 ms and inverse 26.2 → 12.7 ms; on CPUs 0–7 forward 19.0 → 10.4 ms and inverse 20.6 → 11.5 ms, all with strict separation. + +With the cheaper transfer, the column path also wins at smaller sizes: a sweep measured 2^15 at 0.41 against 0.82 ms serially and 2^16 at 0.60 against 1.42 ms, while 2^13 stayed faster serially (0.18 against 0.25 ms). `forwardParallel` and `inverseParallel` now take the column path for even sizes from 2^16, with `columnCount` = `2 ^ min (logWorkers + 1) (logN - 12)` tasks so each keeps at least 256 columns, and keep the segment plan for odd sizes from 2^18. Five alternating rounds on CPUs 0–7: 2^16 forward 1.40 → 0.76 ms and inverse 1.59 → 0.78 ms (strict), 2^12 and 2^20 unchanged within noise. + +Leaf tasks now reverse their bit order straight into the byte array (`reverseEncode`), instead of building a reversed field array and encoding it: 1.73 → 1.52 ms per serial 2^16 leaf. The main thread also allocates the output array before it waits for the leaves, overlapping the zero fill and its page faults with the tasks (about 0.4 ms in a prototype). Seven alternating 2^20 rounds on CPUs 0–7: forward 10.9 → 10.0 ms and inverse 11.4 → 9.9 ms; five rounds on CPUs 0–15: 10.6 → 10.1 and 11.5 → 9.4 ms, overlapping ranges on a busy host. Two other ideas were measured and rejected. Running the later leaf passes depth-first on cache-sized sub-blocks saved nothing, single-threaded or with sixteen leaves in parallel, for both plans. A fused radix-16 kernel, two radix-four layers per load and store, was 60% slower: its sixteen power-of-two-strided streams alias in L1, as the interleave's did. Pipelining the leaves in two waves, so the first half of the interleave overlaps the second wave, lost more parallelism than it saved. + +### Physical-core scaling (2026-10-07) + +On the eight-core Ryzen, sixteen workers only add SMT siblings, so they measure how each implementation shares a core rather than how it scales: Plonky3 gained slightly there while both Lean plans lost slightly. The NTT driver now defaults to physical cores, and the reference comparison uses one, four and eight workers. One normal-driver run per worker count at `f846670`, million-point forward / inverse, milliseconds: + +| Workers | Packed I/O Lean | Packed Lean | Externless Lean | Plonky3 | +|---:|---:|---:|---:|---:| +| 1 | 14.82 / 15.19 | 17.26 / 17.60 | 29.33 / 32.19 | 9.19 / 8.95 | +| 4 | 4.10 / 4.16 | 8.08 / 8.24 | 12.83 / 13.24 | 2.86 / 2.78 | +| 8 | 4.17 / 4.11 | 7.69 / 7.82 | 9.65 / 10.22 | 3.31 / 3.18 | + +Plonky3 and packed I/O Lean stop gaining at four workers. CPUs 0–3 share one L3 complex; eight workers span both, so the plateau is probably memory and cross-complex traffic rather than arithmetic, though this was not isolated. The externless plan, about 1.5 ns per element and layer in scalar code, keeps scaling to eight. Single-threaded, packed I/O Lean is 1.6–1.7× Plonky3 and externless Lean 3.2–3.6×; this per-core gap, not the eight-worker ratio, is the remaining work. Local reports are under ignored `bench/out/ntt-20261007-014257/`, `-014452/` and `-014637/`, reproduced with `--cpus 0`, `--cpus 0,1,2,3` and `--cpus 0,1,2,3,4,5,6,7`. + +### Fused final layers and leaf reversal (2026-10-07) + +A single-threaded profile of the packed plan at one million points (15.5 ms against Plonky3's 9.2 ms) split into the top four split levels (3.8 ms, 0.9 ns per element and layer), the leaves' radix-four passes (5.4 ms, 0.43 ns), the scalar kernel `leaf16` for the last four layers (2.8 ms, 0.67 ns), the leaf reversal (1.8 ms) and the interleave (1.8 ms). Vectorizing `leaf16` alone would save at most about 1 ms, so `NTTFast/Packed/NativeLast.lean` fuses it with the reversal instead. `lastHalf` reads eight sixteen-word blocks `L` words apart (`L = 2 ^ (m - 4)`), runs the four final layers on all eight at once, one block per vector lane with broadcast twiddles, and writes eight words of each of sixteen natural-order lines; two calls per batch of sixteen block columns complete the lines. The generated kernel compiles to vector code (`vpmuludq`, `vpaddd`, `vpminud`) and replaces both `leaf16` and the `leafRev` pass for even leaves of at least `2 ^ 8` words: a 2^16 leaf went from 0.63 to 0.48 ms. A sixteen-lane version was as fast but took about two minutes to compile; a version written over a record-valued line helper compiled to scalar code again. The kernel scales inverse outputs itself, so the natural-order path no longer normalizes inside `stages`. The output buffer starts as the borrowed input leaf, which the first store copies once. + +`sliceChunks` and `sliceInputChunks` take the leaf computation as a parameter; their theorems assume it equals `post ∘ stages` on every leaf-sized input, and `leafNatural_packFields` proves that for the fused leaf. `NativeLastCorrectness` relates the kernel to a field-level lane model that shares `leaf16Field`'s expression graph, characterizes one kernel call word by word (`getD_lineStores`), and shows by induction over the batches that every natural-order word holds its bit-reversed final-layer value. `runPacked`, `runFields` and their DFT theorems moved there from `NativeOrder`. + +One normal-driver run per worker count, million-point forward / inverse, milliseconds, packed I/O Lean against Plonky3: one worker 12.41 / 12.72 against 9.36 / 9.10 (was 14.82 / 15.19); four workers 3.47 / 3.52 against 2.86 / 2.79 (was 4.10 / 4.16); eight workers 3.58 / 3.69 against 3.87 / 3.19 (was 4.17 / 4.11), where Plonky3's forward row was noisy (± 0.34 ms). The field-array API improved by 0.5–0.8 ms at every worker count. + + +### Split loops with hoisted bounds (2026-10-07) + +`pairGo` and `pairInputGo`, the batch loops of the split levels, checked the bounds of every sixteen-word batch and converted Nat indices to `USize` on each step. They now check the whole range once and hand off to `pairLoop` / `pairInputLoop`, which step `USize` indices under a proof that carries the remaining range; `usize_add16` shows the increments cannot wrap. An isolated split tree with identity leaves went from 3.2–3.7 to 1.9–2.5 ms on one worker. Seven alternating rounds of the normal driver, million-point forward / inverse, milliseconds, packed I/O: one worker 12.56 / 12.86 → 11.72 / 11.92, four workers 3.48 / 3.62 → 3.31 / 3.39; the field-array API was unchanged within noise. + +The leaves' radix-four passes got the same treatment: `inner16` checks the range of all whole sixteen-cell batches once and runs `inner16Loop` over `USize` indices, then the scalar tail. Its proof needed the buffer size across a kernel call (`size_step16`, from `size_storeWords_overwrite`). Seven alternating rounds, million-point forward / inverse, milliseconds, packed I/O: one worker 11.39 / 11.68 → 10.74 / 11.08, four workers 3.29 / 3.35 → 3.16 / 3.20. + +### Warmup for parallel transforms (2026-10-07) + +With the `schedutil` governor, idle worker cores take a few hundred milliseconds of load before their clocks rise from 2.2 to about 4.1 GHz, and the medium preset's 50 ms warmup often ended before that. The NTT suite now uses the large budget, 200 ms of warmup and 50 samples, on both sides (`BenchPreset.large` in Lean, `harness::LARGE` in Rust). Five alternating rounds at `f4bb734`, million-point forward / inverse, milliseconds: packed I/O Lean 9.96 / 10.53 against Plonky3 9.29 / 9.00 on one worker and 2.92 / 3.01 against 2.89 / 2.81 on four, as before; on eight workers 2.93 / 2.95 against 3.30 / 3.17 (was 3.48 / 3.45), and the externless plan 7.89 / 7.98 (was 9.81 / 9.78). + +### Coset interpolation against Plonky3 (2026-10-07) + +`python3 scripts/bench-fields.py --suite interpolate` compares `Montgomery.Native32.interpolateCoset` over KoalaBear with Plonky3's `p3_interpolation::interpolate_coset` over the quartic extension; see [the benchmark operator guide](../../bench/README.md#koalabear-coset-interpolation-against-plonky3). The first measurement times the proved reference, which is sequential and works on lists of boxed extension elements, against parallel Plonky3 on eight workers. Three alternating rounds, milliseconds per call, Lean / Plonky3: 2^12 rows 0.67 / 0.20 (one column) and 1.78 / 0.16 (sixteen columns), 2^16 rows 12.0 / 0.93 and 28.3 / 0.63, 2^20 rows 249 / 8.05 and 531 / 9.15. + +### Packed coset interpolation (2026-10-07) + +`Montgomery.Native32.interpolateCosetPacked` ([`InterpolateCosetPacked.lean`](../../CompPoly/Fields/Montgomery/InterpolateCosetPacked.lean)) replaces the reference's lists of boxed extension elements with raw words: + +* **Base-field weights.** For `t = z₀ - x`, `(z - x)⁻¹ = (b₀ + t b₁ + t² b₂ + t³) / N(t)`, where the extension constants `bₖ` depend on `z` only and `N(t)` is the norm of `z - x` (`Ext4.smul_inv_sub_ofBase`). Each row's weight is the four base-field values `x tᵏ / N`, only base-field norms need inverting, and a column's sum is `∑ₖ Sₖ bₖ` for four base-field sums `Sₖ`. A row costs about twelve base-field multiplications. +* **Blocks of 512 rows.** A forward pass stores each row's node, `t²`, norm and the prefix product of the norms in a scratch `ByteArray`; one Fermat inversion per block inverts the product; a backward pass overwrites each row with its four weights. +* **Lazy column sums.** `Montgomery.Native32.LazyAcc` holds an unreduced 64-bit sum below `p · 2^32`: each product of a raw word and a weight costs a multiplication, an addition and one conditional subtraction, and each block ends with one Montgomery reduction per sum. Four adjacent columns share each row's weights. +* **Tasks.** Eight row ranges run as tasks, the caller taking the lowest; their sums add up at the end. + +Interleaved runs chose 512-row blocks (256 was up to 8% slower, 1024 up to 20%) and eight tasks (sixteen oversubscribe eight cores and ran 1.4× slower). The module is compiled with `-march=native` in `CompPolyPackedNative`, like the packed NTT kernels; without it the four-column loop ran almost twice as slow. + +`interpolateCosetPacked_eq` ([`InterpolateCosetPackedCorrectness.lean`](../../CompPoly/Fields/Montgomery/InterpolateCosetPackedCorrectness.lean)) proves it equal to the reference for every point off the coset. The proof follows the loops: the two passes as row updates of the packed buffer with the batch-inversion invariant, `dot4` as four `dot1` loops, `dot1` against a finite sum, then blocks, leaves and the task tree. Five rounds of `--suite interpolate` on eight workers, milliseconds per call: + +| Rows | Columns | Lean packed | Lean reference | Plonky3 | +|---:|---:|---:|---:|---:| +| 2^12 | 1 | 0.08 | 0.66 | 0.20 | +| 2^12 | 16 | 0.09 | 1.74 | 0.15 | +| 2^16 | 1 | 0.31 | 12.03 | 0.89 | +| 2^16 | 16 | 0.48 | 27.71 | 0.63 | +| 2^20 | 1 | 3.40 | 247.83 | 8.06 | +| 2^20 | 16 | 5.82 | 522.99 | 8.79 | + +At 2^20 rows some samples take twice the median, when one of the eight row ranges starts late on the shared host; the minimum is about 2.8 ms for one column. + +### Core's proposed word accessors and in-place outputs (2026-10-08) + +The packed plan's two storage externs are gone. It now reads and writes words with `ByteArray.ugetUInt32LE`/`usetUInt32LE` as proposed for Lean core in lean4#14053, backported in `CompPoly.Data.ByteArray.Pack` with the same names, signatures, Lean definitions and lemmas; the backport's C uses fixed-size `memcpy` loads and stores and states after each store that the array is still unshared, so a run of stores keeps one sharing check. Without an extern for appends, every kernel now stores its 16-word batches at a cursor into an output buffer pre-sized with zeros, instead of appending. `zeroWords` fills a reserved buffer by copying a separate 4 KiB zero block (copying a buffer into itself makes the destination shared, so every doubling copied it); the transpose extends its output by one block right before each tile overwrites it, and `encode` overwrites a zero buffer word by word. Five alternating rounds at `828e1f8`, million-point forward / inverse, milliseconds, packed I/O Lean against Plonky3: 10.06 / 10.55 against 9.24 / 8.96 on one worker (was 9.96 / 10.53 with the externs), 3.47 / 3.42 against 2.87 / 2.79 on four (was 2.92 / 3.01), 2.53 / 2.57 against 3.29 / 3.16 on eight (was 2.93 / 2.95). 2¹² and 2¹⁶ are at or below their previous times. The four-worker row is placement-sensitive: on CPUs 0, 1, 4 and 5 the new build took 3.63 ms against 4.66 ms for the extern build. + +### One column pipeline for both plans (2026-10-13) + +The packed plan now runs the extern-free plan's pipeline (`NTTFast/Packed/Columns.lean`): from 2^15 elements and with more than one worker, column tasks gather their columns with byte copies and run the top two radix-four passes with the vectorized `inner16` kernel, sixteen leaf tasks run the rest with `leafNatural` (the fused last four layers and leaf reversal), and `naturalLeaves` interleaves the leaves; smaller transforms and a single worker run one leaf. The split tree, slice tree, tiled decoder, `Plan.defaultDepth`/`NTT_DEPTH` and the in-stage normalization they needed are gone, about 3,300 lines with their proofs. `ColumnsCorrectness.lean` proves each stage packs the corresponding stage of the field-level column transform and reuses `Columns.columnTask_col` and the leaf lemmas of the extern-free proof. Leaves below sixteen words read their bit-reversed stages with the scalar `decode`, since `leafRev` works in sixteen-word batches. + +Both plans now use sixteen column tasks, `columnCount = 2 ^ min 4 (logN - 12)`, at every worker count. With `2 ^ (logWorkers + 1)` tasks, four workers gave each column task 512 KiB of rows plus 480 KiB of gathered twiddles at 2^20, beyond the 512 KiB L2 of the Ryzen 7 3700X. A sweep of 4 to 64 tasks found sixteen best or within noise for both plans at two, four and eight workers. On one worker the column path costs 50% more than one leaf (2^20: 11.9 against 7.8 ms), so the single-worker switch stays. + +Six alternating rounds against the previous packed plan (forward, ms; inverse within a few percent of forward): on CPUs 0–7, 2^12 0.079 → 0.020, 2^14 0.314 → 0.087, 2^16 0.410 → 0.246, 2^18 0.660 → 0.552, 2^20 2.53 → 2.42; on CPUs 0–3, 2^16 0.393 → 0.268, 2^18 0.687 → 0.672 (fourteen rounds), 2^20 3.54 → 3.23; on CPU 0, 2^16 1.44 → 0.385, 2^20 10.8 → 7.69. Below 2^15 the gain comes from running one leaf instead of a task tree. + +### Odd leaves through the fused path (2026-10-13) + +An odd-size leaf used to run its radix-four passes from the top, then a scalar radix-two layer and a separate reversal pass, and cost about 3.5 times as much per element and layer as an even leaf: one worker took 12.9 ms at 2^19 against 7.5 ms at 2^20. Odd leaves now run their radix-two layer first, with a sixteen-lane kernel (`pair16`), then radix-four passes down to the same fused final layers and reversal (`lastRev`) as even leaves. `runStagesDIFRadix4WithTwiddles_eq_top` proves the two orders equal: both are the same chain of radix-two stages. Leaves below 2^8 words run all passes and the scalar `decode`, so the sequential `stages` pipeline, the in-block `leaf16` kernel and the separate `leafRev` pass are gone with their proofs. + +Five alternating rounds, forward, ms: on CPU 0, 2^15 0.621 → 0.192, 2^17 2.93 → 0.878, 2^19 13.2 → 3.92; on CPUs 0–7, 2^17 0.749 → 0.353, 2^19 2.96 → 1.18. Odd sizes now cost the same per element and layer as even ones; even sizes are unchanged within noise. The radix-four block loop is a `for` loop: folding over `List.range` allocated up to 16k list cells per pass and cost 3–5% on one worker. + +### Stack alignment of the packed kernels (2026-10-13) + +The fused final-layer kernel `lastHalf` spills AVX registers to its stack frame with unaligned 32-byte moves, and its speed depended on the stack alignment it inherited: two builds with byte-identical kernels at identical addresses differed by 20% on one worker (2^17: 0.86 against 1.05 ms), with equal instruction counts and cache misses. The same binary also moved between the two speeds when the environment changed, because that shifts the main thread's stack. `CompPolyPackedNative` now compiles with `-mstackrealign`, so every kernel realigns its frame and spills with aligned moves; timings are then stable across builds and environments, within 2% of the fast case. When an A/B comparison of the packed plan shows a change that the diff cannot explain, compare the hot kernels' disassembly before trusting it. + +The column-parallel transforms of both plans are now built from three task combinators in `CompPoly/Data/Tasks.lean`: `spawnAll` runs independent tasks, `spawnAfter` runs tasks once a set of tasks is done, and `join` collects results. `map_get_spawnAll` and `map_get_spawnAfter` state their results as plain maps, which replaced the task bookkeeping in each transform's proof. + +### Generated kernels (2026-10-13) + +The unrolled packed kernels (`step16`, `pair16`, `lastHalf`, `transposeStep`, `line`) and their field-array models (`step16Field`, `leaf16Field`, `laneLine`) are generated by the macros of `CompPoly/Univariate/NTTFast/Packed/Unrolled.lean`: one description per kernel family, in a packed-word dialect with bounds proofs and a field-array dialect. The expansions are the terms that were written out before, so the refinement proofs are unchanged and the generated C is identical; about 1,700 lines of unrolled source are gone. A new kernel shape is a new generator, not another hand-written block of sixteen lanes. + +### Batched coset LDE (2026-10-16) + +`python3 scripts/bench-fields.py --suite lde` compares `NTTFast.Packed.LdePlan.cosetLde` with Plonky3's `coset_lde_batch` in the bit-reversed row order its FRI commitment commits; see [the operator guide](../../bench/README.md#koalabear-coset-lde-against-plonky3). In that order the output is `2 ^ bits` blocks of `h` rows, and block `t` is the coset `shift · ω ^ bitrev t · ⟨ω ^ 2 ^ bits⟩` in DIF order. So each column takes one unnormalized inverse transform and one DIF transform per block of its coefficients scaled by that block's coset table, with no padding and no reversal. The input comes in chunks of consecutive rows of equal heights (`Native.rowChunks`), so that a caller producing rows in parallel need not join them. Tasks transpose row ranges of the chunks into columns with `16 × 16` tiles, transform the columns, and transpose each block's rows back with `8 × 8` tiles as soon as its columns are done. The result stays in row chunks; `joinRows` adds one copy of the output. + +`Native.joinRows_cosetLde` ([`CosetLdeCorrectness.lean`](../../CompPoly/Univariate/NTTFast/Packed/CosetLdeCorrectness.lean)) proves the joined chunks equal to `ldeSpec`, each column's `CLagrange.interpolatePow` at `shift · ω ^ bitrev r`. The transposes ([`CosetLdeTransposes.lean`](../../CompPoly/Univariate/NTTFast/Packed/CosetLdeTransposes.lean)) are proved by covering: `Fill T S N g` says that `g` overwrites words only with their targets `T`, among them every word of `S`, and fills compose along folds, so tile kernels and edge loops that together cover the output compute it regardless of order and overlap. + +Packed `add`, `sub` and `mul` now reduce with an unsigned minimum, which compiles to `vpminud` and made the transform kernels about 10% faster. Eight physical cores of a Ryzen 7 3700X, three alternating rounds on the shared host, milliseconds per call; Plonky3 is its fastest DFT for the shape: + +| Rows | Columns | Blowup | Lean | Plonky3 | Lean / Plonky3 | +|---:|---:|---:|---:|---:|---:| +| 2^14 | 64 | 2 | 4.23 | 2.48 | 1.71× | +| 2^16 | 8 | 2 | 2.84 | 1.65 | 1.72× | +| 2^16 | 64 | 2 | 17.7 | 16.8 | 1.05× | +| 2^16 | 256 | 2 | 84.7 | 86.7 | 0.98× | +| 2^18 | 64 | 2 | 81.9 | 88.2 | 0.93× | +| 2^20 | 8 | 2 | 68.3 | 57.3 | 1.19× | +| 2^20 | 64 | 2 | 451 | 364 | 1.24× | +| 2^16 | 64 | 4 | 29.7 | 30.6 | 0.97× | + +On one thread Plonky3's batched DFT is two to three times faster per element: its hand-written AVX2 butterflies keep the data in registers, while about a quarter of the instructions in Lean's sixteen-lane radix-four kernel are spills. Rejected variants: eight-lane kernels (1.45× slower per pass), row-wise transforms over many columns at once, grouping several columns per task (5% faster at 2^14 × 64, slower at 2^16 × 8), `8 × 8` tiles for the input transpose, and nested parallel transforms inside column tasks. Fusing the coset scaling into the first forward pass would save about 8%. + +### Packed transforms over any 32-bit Montgomery field (2026-10-21) + +The packed plan and its proofs are generic over `Mont32Field p`, working on `FastField p` words. The raw word operations of `CompPoly/Univariate/NTTFast/Packed/Arithmetic.lean` take the modulus and `montgomeryNegInv` from the instance, `ofWord p` checks `< p`, and the kernels take `p` as their first argument (the generators of `CompPoly/Univariate/NTTFast/Packed/Unrolled.lean` emit `add p x y`). Every kernel is `@[specialize]`; at a concrete instance the compiler copies the call tree with the modulus as immediates, and the copies' hot kernels (`step16`, `pair16`, `lastHalf`) disassemble to the same instructions as the KoalaBear-only kernels before. + +So that the specialized code is compiled once, with the native library's flags, `Plan.forwardPacked` and `inversePacked` call the class `PackedNTT p`, whose `run` is `Native.run p` (`run_eq`). `CompPoly/Univariate/NTTFast/Packed/Instances.lean`, a root of `CompPolyPackedNative`, sets it for KoalaBear and BabyBear to a definition applying `Native.run` at the field's modulus, where the specialization happens. A new field needs one such instance. `lake exe CompPolyNativeSmoke` checks both compiled instances against the DFT specification and the parallel path against the sequential one. + +### Coset interpolation over any 32-bit Montgomery field (2026-10-21) + +`interpolateCoset` and `interpolateCosetPacked` ([`Montgomery/InterpolateCoset.lean`](../../CompPoly/Fields/Montgomery/InterpolateCoset.lean) and its `Packed` modules) are generic over `Mont32Field p` with a `Binomial4 p` extension, and their spec statements live in `Ext4.Spec p`; the weights' constants use `Binomial4.mulW`. As for the packed NTT, the loops are `@[specialize]`d and `interpolateCosetPacked` calls the class `PackedInterpolation p`, whose instances in `KoalaBear/InterpolateCoset.lean` and `BabyBear/InterpolateCoset.lean`, the roots of `CompPolyInterpolateNative`, compile them at the field's modulus. `lake exe CompPolyNativeSmoke` checks both instances against the reference. + +### Coset LDE over any 32-bit Montgomery field (2026-10-21) + +`LdePlan p` and `Native.cosetLde` are generic over `Mont32Field p`, with `@[specialize]`d kernels like the packed NTT; the minimum-form `add`, `sub` and `mul` take the modulus from the instance, and `reduceOnce_eq` holds for every word. `LdePlan.cosetLde` calls the class `PackedCosetLde p`, whose KoalaBear and BabyBear instances live beside the `PackedNTT` ones in `CompPoly/Univariate/NTTFast/Packed/Instances.lean`; `LdePlan.joinRows_cosetLde` restates the correctness theorem for it. `lake exe CompPolyNativeSmoke` checks both instances against `ldeSpec`. + +### Proved stores and checks once per loop (2026-10-23) + +A `ByteArray` store first checks that the array is unshared and copies it otherwise. In a kernel that loads its words and then stores, that copy is a call on the path of the first store, so the compiler keeps every loaded value across it and spills them. `Native.touchLast` writes the buffer's last word back unchanged before the kernel loads anything: the check happens with nothing live, the stores after it keep the array unshared, and `Native.store8` stores with proved bounds instead of checked ones, so no later store has a branch or a call. `lastHalf` checks its output once per call on machine words and then runs that way, which halved its stack accesses and made the packed NTT at `2 ^ 16` 6% faster (inverse, one thread). `tile8` checks its eight lines and its output tile once and stores each output line as soon as its eight words are loaded: 17% less time in the kernel. The same treatment did not pay for `step16`, whose radix-four passes are bound by memory and lost 3% with fewer spills but later loads, nor for the strided `16 × 16` input tile. + +`leaf16x2` runs the final four layers of two blocks at once and multiplies every butterfly of the first three layers by its twiddle, the first twiddle of each layer being one (`TwiddleOnes`), so that each block's layers are uniform and the compiler vectorizes them; the scalar `leaf16` stays for a last odd block. `Native.leaf16x2_encode` proves the kernel equal to `leaf16Field` on its two blocks in turn. The range checks of whole loops moved out of their bodies: `radix4Blocks` checks a pass once for quarters of whole sixteen-cell batches and runs `blocks16Loop` on machine words, `leaf16Blocks` runs `leaf16x2Loop` and `mulWords` runs `mul16Loop` the same way, each with a refinement lemma by induction on the loop. + +CompStark's extension of the Keccak trace (2633 columns, blowup 2, `CompStarkBench keccak-lde`), milliseconds, medians of three alternating rounds on the Ryzen 7 3700X; Plonky3 is `compstark-bench lde` built for the native CPU: + +| Rows | Threads | Before | After | Plonky3 | +|---:|---:|---:|---:|---:| +| 2^15 | 16 | 487 | 434 | 346 | +| 2^15 | 1 | 2423 | 2013 | 856 | +| 2^16 | 16 | 899 | 821 | 970 | +| 2^16 | 1 | 4911 | 4239 | 1747 | + +### Montgomery products by high words, leaf stages, blocked input transposes and fused coset products (2026-10-23) + +Packed `mul` subtracts the high word of `q p` from the high word of the product `z = x y`, the quotient `q = x y p⁻¹ mod 2 ^ 32` a product of words, and corrects the sign with `sub`'s unsigned minimum; `mul_eq_reduceOnce` proves it the Montgomery reduction `(z + (-z p⁻¹ mod 2 ^ 32) p) / 2 ^ 32`, reduced once. The compiler keeps every lane in 32-bit vectors, four `vpmuludq` and two `vpmulld` per eight lanes, where the Montgomery sum widened each lane to 64 bits and reduced it with 64-bit comparisons: the packed coset LDE of `2 ^ 15 × 1024` words runs 24% faster on one thread. + +The four final DIF layers of a forward leaf are four passes over the leaf (`leafStage8`, `leafStage4`, `leafStage2`, `leafStage1`). Each pass calls its kernel once per four sixteen-word blocks; the kernel touches the buffer, reads its blocks, pairs each block's words in eight vector lanes and stores every block as two halves with proved stores, so that a layer is one vector operation per lane group and the shuffles that gather its pairs. The passes are the layers block by block (`fold_leaf16Stages`), each kernel the layer's model on its four blocks (`leafStage8_encode` and its siblings), each model the layer (`leafStage8Field_eq` and its siblings in [`LeafStages.lean`](../../CompPoly/Univariate/NTTFast/Packed/LeafStages.lean)). The models splice with `splice8`, which keeps the size whatever the bounds, so that no proof computes the size of nested splices. The passes take about half the time of a kernel running all four layers on two blocks, in which the compiler shuffled scalars. + +`transposeRows` moves its tiles down a block of up to 1024 rows before across, so that each column of the transposed range receives four kilobytes per block instead of one line per sixteen rows: the transpose of `2 ^ 15` rows of 2633 words in sixteen ranges takes 150 ms on one thread instead of 440, of `2 ^ 16` rows 560 ms instead of 1080. + +A coset block of an odd leaf runs the leaf's radix-two layer out of place on the coefficients, each word multiplied by its coset-table word as it is read (`pair16Scaled`; `pairsScaled_encode` proves the pass the first layer of the products), instead of storing the products in a pass of their own: 3% fewer instructions for the coset LDE. For an even leaf the products still come first: the same fusion into its first radix-four pass, twelve streams a power of two apart, ran 50% slower. + +`step16` takes about 85 cycles per call, the same with its buffer in L1 as in L2: it is bound by the vector multiplier (32 `vpmuludq` and 14 `vpmulld` per call) and by dispatch, not by memory. Proved stores after a touch with the lanes in halves (10% fewer instructions) and quotient factors `w p⁻¹` precomputed per twiddle (six fewer `vpmulld`) each changed its time by less than 2%. + +CompStark's extension of the Keccak trace by columns (2633 columns, blowup 2, `CompStarkBench keccak-lde` at CompStark `8c47e95` with CompPoly `f689986` before, and with CompPoly `e0465e5` after), milliseconds, medians of six alternating rounds (four for `2 ^ 16` rows on one thread) on the Ryzen 7 3700X under other load; Plonky3 is `compstark-bench lde` built for the native CPU: + +| Rows | Threads | Before | After | Plonky3 | +|---:|---:|---:|---:|---:| +| 2^15 | 16 | 285 | 208 | 322 | +| 2^15 | 1 | 1675 | 1311 | 897 | +| 2^16 | 16 | 532 | 420 | 703 | +| 2^16 | 1 | 3502 | 2534 | 1863 | + +On one thread the extension of `2 ^ 15` rows is 1.46 times Plonky3's (1.24 times by the fastest rounds), of `2 ^ 16` rows 1.36 times. Most of what remains comes from the column layout itself: the input transpose and gather, which Plonky3's row-major batches do not need, the final four layers of every leaf, whose pairs lie within a vector and need shuffles or lane transposes where Plonky3's layers across columns share one broadcast twiddle, and per-lane twiddle loads. + +### Input transposes by column range (2026-10-23) + +`transposeRows` transposes a range of columns `c0, …, c0 + n - 1` of its row-major input, so that a caller can transpose and extend the columns of a wide matrix range by range: the transposed rows of a range are freed once that range's columns are extended, instead of the whole transposed copy living until the last column. CompStark's extension of the Keccak trace transposes ranges of 256 columns, all ranges at once so that the input is freed early and the columns of each range are extended while their transposed rows are still warm. The peak memory of a whole Keccak proof of `2 ^ 18` rows falls from 9.4 GB to 7.1 GB; the extension's time is unchanged within the shared host's noise (`CompStarkBench keccak-lde`, `2 ^ 15` and `2 ^ 16` rows, sixteen threads). `transposeRows_encodeWords` and `gatherColumn_transposeRows` state the range; `cosetLde` transposes all columns as one range. diff --git a/docs/wiki/build-cache.md b/docs/wiki/build-cache.md index a332465d..6169b5e9 100644 --- a/docs/wiki/build-cache.md +++ b/docs/wiki/build-cache.md @@ -65,7 +65,7 @@ builds the library, and runs `lake upload `. The build step is `rm -rf .lake/build && lake build CompPoly`, and the narrow target matters. `lake upload` packs the *whole* build directory, so a full `lake build` would also ship the benchmark executable's Linux-only `bin/` and `.o` output — bloating the -asset and contradicting the platform-independence claim above. +asset and contradicting the platform-independence claim above. It also keeps the `-march=native` objects of the packed NTT kernels out of the archive: `lake build CompPoly` compiles no C, so a consumer's build compiles the kernels for its own CPU. To attach an asset to a release that predates this workflow, or to replace one, use **Actions → Add Lean release tag → Run workflow** and give it the tag. diff --git a/docs/wiki/field-extensions.md b/docs/wiki/field-extensions.md index 8d246504..e9f2f07f 100644 --- a/docs/wiki/field-extensions.md +++ b/docs/wiki/field-extensions.md @@ -506,6 +506,19 @@ None of these is hidden behind an abstraction that makes replacing it awkward, a guarded by the `#guard` regressions in [`tests/CompPolyTests/Fields/Extension/Arithmetic.lean`](../../tests/CompPolyTests/Fields/Extension/Arithmetic.lean). +### Fast quartic binomial extensions (2026-10-07, generic since 2026-10-21) + +`Montgomery.Native32.Ext4 p` ([`Montgomery/Native32Ext4.lean`](../../CompPoly/Fields/Montgomery/Native32Ext4.lean)) is `FastField p [X] / (X^4 - W)` as a structure of four Montgomery coefficients, for any `Mont32Field p` with a `Binomial4 p` instance. `Binomial4` supplies `W`, a multiplication `mulW` by it (the product by default) with `mulW_eq`, and the irreducibility of `X^4 - C W` over `FastField p`. The type lives outside the generic `Ext` framework: a second `Mul` instance on `Ext P` over the fast carrier would make a diamond, and `@[csimp]` cannot specialize `Ext.mul` to one `P`. + +The spec field is `Ext4.Spec p = Ext (Ext4.P p)`, the framework's `Ext` over the fast base field `FastField p` itself for the binomial `Ext4.params p`, so a consumer never meets `ZMod p`. It is a field through the instances `Fact (Nat.card (FastField p) = p)` (`FastField` is a `Fintype` with `p` elements, `card_fastField`) and `Fact (Irreducible …)` from `Binomial4.irreducible`. `toSpec` maps into it, `specEquiv p : Ext4 p ≃+* Ext4.Spec p` is the ring isomorphism, and `toSpec_add`, `toSpec_mul`, `toSpec_inv`, `toSpec_div`, `toSpec_ofBase` and `toSpec_smul` give agreement operation by operation. The byte codec is the four coefficient encodings, identical to the spec field's (`toBytes_toSpec`). + +* Multiplication folds `W` into the left factors through `mulW`, so every output coefficient is a sum of four products, which `Montgomery.Native32.mulAdd4` reduces once: four products stay below `2 · p · 2^32` for every modulus below `2^31`, so one conditional subtraction of `p · 2^32` brings the sum into `reduce`'s range. +* Inversion is the norm-based inverse from item 3 above, specialized to the binomial: with `σ : X ↦ -X`, `a · σ(a) = g₀ + g₁ X²` lies in `F(X²)`, and `a⁻¹ = σ(a) (g₀ - g₁ X²) / (g₀² - W g₁²)` needs one base-field inversion. That the norm of a nonzero element is nonzero follows from `σ` being an injective ring endomorphism of the quotient field and from `W` not being a square (`W_not_square`), which the irreducibility of `X^4 - W` implies. + +Instances: KoalaBear `X^4 - 3` with `mulW x = x + x + x` ([`KoalaBear/FastExt4.lean`](../../CompPoly/Fields/KoalaBear/FastExt4.lean), `KoalaBear.Fast.Ext4`) and BabyBear `X^4 - 11` with the default product ([`BabyBear/FastExt4.lean`](../../CompPoly/Fields/BabyBear/FastExt4.lean), `BabyBear.Fast.Ext4`). Each transfers the irreducibility certificate of its `ZMod` spec field (`KoalaBear.ext4Params_poly_irreducible`, `BabyBear.ext4Params_poly_irreducible`) to the fast carrier along `ringEquiv` with `irreducible_map_ringEquiv` ([`Extension/Map.lean`](../../CompPoly/Fields/Extension/Map.lean)), so no new kernel computation is needed. + +Known answers from Plonky3's `BinomialExtensionField` and `BinomialExtensionField` are checked in [`tests/CompPolyTests/Fields/KoalaBear/FastExt4.lean`](../../tests/CompPolyTests/Fields/KoalaBear/FastExt4.lean) and [`tests/CompPolyTests/Fields/BabyBear/FastExt4.lean`](../../tests/CompPolyTests/Fields/BabyBear/FastExt4.lean). The `fields-extension-koalabear-ext4-mul` and `-inv` benchmark groups time the `ZMod` spec `KoalaBear.Ext4` and the fast type on the same operands and cross-check their checksums. On the eight-core Ryzen server, one record per call including the harness's operand lookup and the result allocation: multiplication 7.3 µs (spec) and 28 ns (fast), inversion 913 µs (spec, Fermat) and 161 ns (fast, norm). + ## Base Field Caveats `Hachi` (`2^32 - 99`) has no `FastField` Montgomery path: `Mont32Field` requires diff --git a/docs/wiki/quickstart.md b/docs/wiki/quickstart.md index 73e0ad54..07871042 100644 --- a/docs/wiki/quickstart.md +++ b/docs/wiki/quickstart.md @@ -134,7 +134,9 @@ come from the on-demand Benchmarks workflow. See `lean-toolchain` plus `lake-manifest.json`, and `.lake/build` keyed additionally per commit. A dependency-cache miss is not expensive, because `lean-action` runs `lake exe cache get` for us, so Mathlib's oleans are downloaded rather than - compiled. + compiled. The restored `.lake/build` may come from a runner with another CPU, so + `scripts/drop-native-objects.sh` deletes its `-march=native` objects and executables + before building. - [`../../.github/workflows/linting.yml`](../../.github/workflows/linting.yml) runs the style linter on changed `.lean` files in PRs and push builds. - [`../../.github/workflows/check_imports.yml`](../../.github/workflows/check_imports.yml) diff --git a/docs/wiki/serialization.md b/docs/wiki/serialization.md index fa94291c..6efe68d4 100644 --- a/docs/wiki/serialization.md +++ b/docs/wiki/serialization.md @@ -68,7 +68,7 @@ it on numerals: `ByteCodec.width BabyBear.Field = 4` is `by decide`. | BabyBear, KoalaBear, Mersenne31 | 4 | | Goldilocks, `BF64` | 8 | | `BF128` | 16 | -| BN254, BLS12-377, BLS12-381, Pasta, secp256k1 scalar fields | 32 | +| BN254, BLS12-377, BLS12-381, Pasta, secp256k1 | 32 | | `AesField`, tower level 3 | 1 | | `Ext P` | `P.d` times the base width | @@ -104,7 +104,7 @@ instance. Dumping the stored Montgomery word would be faster and wrong. | Carrier | Instance module | Agreement lemma | |---|---|---| | `Native32.FastField` (BabyBear, KoalaBear) | `CompPoly/Fields/Montgomery/Native32Bytes.lean` | `FastField.toBytes_ofField` | -| `Native64x8.FastField` (BN254, BLS12-377/381, Pasta) | `CompPoly/Fields/Montgomery/Native64x8Bytes.lean` | `FastField.toBytes_ofField` | +| `Native64x4.FastField` (BN254, BLS12-377/381, Pasta, secp256k1) | `CompPoly/Fields/Montgomery/Native64x4Bytes.lean` | `FastField.toBytes_ofField` | | `Goldilocks.Fast.Field` | `CompPoly/Fields/Goldilocks/Bytes.lean` | `toBytes_ofField` | | `Mersenne31.Fast.Field` | `CompPoly/Fields/Mersenne31/Bytes.lean` | `toBytes_ofField` | | `FastBT128` against `ConcreteBTField 7` | `CompPoly/Fields/Binary/Tower/Bytes.lean` | `toByteArray_toConcrete` | @@ -194,3 +194,9 @@ multilinear, and those have the dense codec. For a variable-length type, give it `DelimitedCodec` instead, built from the list, pair, and vector codecs, with `Valid` stating what fits in the framing, and prove the one law from the laws of the parts. + +## Packed arrays + +`PackedRepr R` (`CompPoly/Data/ByteArray/Packed.lean`) stores every `R` in `width` bytes. It is not a codec of the value: it stores the carrier as is, for example the Montgomery residue of `Native32.FastField`, where `ByteCodec` encodes the canonical value. `toBytes` and `ofBytes` are the specification, with `ofBytes (toBytes x) = x`; `read` and `write` are the fast accesses at a byte offset, proved equal to it. `PackedRepr.ofWord32` builds the instance of a one-word carrier with core byte operations only, so code that uses `PackedRepr` needs no extern; `UInt32` and every `Native32.FastField` use it. The packed NTT kernels read and write words with the backported `ByteArray.ugetUInt32LE`/`usetUInt32LE` instead. + +A packed array is a plain `ByteArray`: `PackedRepr.encode` packs an `Array R` and `PackedRepr.decode` reads one back. Its lemmas describe slots (`read_encode`, `write_encode`, `get_setAt`) and byte ranges on slot boundaries (`encode_extract`, `replaceAt_encode`). Fast code shares packed arrays between tasks, which the runtime marks as shared at once, unlike arrays of boxed values; the packed NTT and the column-parallel NTT store all their intermediate results this way. `PackedRepr.alloc` and `ByteArray.zeros` allocate zero-filled buffers in place. diff --git a/lakefile.lean b/lakefile.lean index 447e6f6e..a438daa1 100644 --- a/lakefile.lean +++ b/lakefile.lean @@ -32,6 +32,26 @@ lean_lib CompPoly where -- genuinely is platform-specific. Mathlib uses the same mechanism. platformIndependent := true +/-- CPU-targeted packed FFT kernels; the proof artifacts remain platform independent. -/ +lean_lib CompPolyPackedNative where + roots := #[`CompPoly.Univariate.NTTFast.Packed.Native, + `CompPoly.Univariate.NTTFast.Packed.NativeOrder, + `CompPoly.Univariate.NTTFast.Packed.NativeLast, + `CompPoly.Univariate.NTTFast.Packed.Columns, + `CompPoly.Univariate.NTTFast.Packed.CosetLde, + `CompPoly.Univariate.NTTFast.Packed.Instances] + platformIndependent := true + -- The kernels spill AVX registers; realigning the stack keeps the spill slots 32-byte + -- aligned whatever the caller's frame, which otherwise moved timings by 20% between builds. + moreLeancArgs := #["-march=native", "-mstackrealign"] + +/-- CPU-targeted packed interpolation loops; realigning their stack measured 2.5% slower. -/ +lean_lib CompPolyInterpolateNative where + roots := #[`CompPoly.Fields.BabyBear.InterpolateCoset, + `CompPoly.Fields.KoalaBear.InterpolateCoset] + platformIndependent := true + moreLeancArgs := #["-march=native"] + lean_lib CompPolyTests where srcDir := "tests" @@ -46,6 +66,26 @@ lean_lib CompPolyBenchLib where lean_exe CompPolyBench where srcDir := "bench" +/-- Inputs for the Rust comparison, generated by the existing field benchmark helpers. -/ +lean_exe CompPolyFieldFixtures where + srcDir := "bench" + +/-- Matched forward/inverse NTT runner for the shared Rust comparison. -/ +lean_exe CompPolyNTTBench where + srcDir := "bench" + -- Match the Rust benchmark's native CPU targeting; library artifacts remain portable. + moreLeancArgs := #["-march=native"] + +/-- KoalaBear coset interpolation runner for the Plonky3 `interpolate_coset` comparison. -/ +lean_exe CompPolyInterpolateBench where + srcDir := "bench" + moreLeancArgs := #["-march=native"] + +/-- KoalaBear coset low-degree extension runner for the Plonky3 `coset_lde_batch` comparison. -/ +lean_exe CompPolyLdeBench where + srcDir := "bench" + moreLeancArgs := #["-march=native"] + /-- Kernel-level axiom / `sorry` accounting with a committed regression baseline (`scripts/axiom_baseline.json`). Runtime-imports the built CompPoly oleans, so run it after `lake build`. See `scripts/AxiomSweep.lean`. -/ diff --git a/scripts/README.md b/scripts/README.md index 786948a9..9b9d026e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -124,6 +124,10 @@ previously recorded baseline without rerunning that baseline in the same job. This supports [`../.github/workflows/lean_action_ci.yml`](../.github/workflows/lean_action_ci.yml). +### `drop-native-objects.sh` + +Deletes the build outputs compiled with `-march=native` (the packed kernels and the benchmarks linking them) and `.lake/build/bin`, so the next `lake build` recompiles them for the current host. CI runs it after restoring `.lake/build`, which may come from a runner with a different CPU. + ### `bench-ab.sh` Driver for the optimisation loop in @@ -180,3 +184,7 @@ lake build ```bash python3 ./scripts/check-docs-integrity.py ``` + +### Rust field comparison + +`python3 scripts/bench-fields.py --suite all --cpu 0` checks Lean/Rust agreement and writes compact comparison tables plus raw samples under `bench/out/`. Select `small-prime`, `large-prime`, `binary`, `ntt`, `interpolate`, or `all`; the default is `small-prime`. Use `--validate-only` for correctness checks and `--skip-build` to reuse a verified build from a previous driver run. The `ntt` suite uses `--cpus` for a configurable worker budget (default: all physical cores, up to sixteen, without SMT siblings) in its KoalaBear transform comparison against optimized parallel Plonky3; `--cpu` selects one worker; `scripts/bench_ntt.py` implements it with the same build verification. The `interpolate` suite compares coset interpolation at an extension point against Plonky3's `interpolate_coset`, with the same worker selection; `scripts/bench_interpolate.py` implements it. See [the benchmark README](../bench/README.md#rust-field-comparison). diff --git a/scripts/bench-fields.py b/scripts/bench-fields.py new file mode 100644 index 00000000..df14b49f --- /dev/null +++ b/scripts/bench-fields.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Validate and compare selected Lean/Rust field workloads.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import time +import tomllib +import bench_interpolate +import bench_lde +import bench_ntt + +ROOT = Path(__file__).resolve().parents[1] +SUITES = { + "small-prime": {"koalabear": ("KoalaBear", "Plonky3", ("add", "mul", "inv", "pow")), + "mersenne31": ("Mersenne31", "Plonky3", ("add", "mul", "inv", "pow")), + "goldilocks": ("Goldilocks", "Plonky3", ("add", "mul", "inv", "pow"))}, + "large-prime": {"bn254": ("BN254 scalar", "arkworks", ("add", "mul", "inv", "pow"))}, + "binary": {f"tower-bt{bits}": (f"Binary tower {bits}", "Binius", ("mul", "square", "inv")) + for bits in (8, 64, 128)}, +} + + +def selection(suite): + fields = {f: spec for name, values in SUITES.items() + if suite == "all" or suite == name for f, spec in values.items()} + groups = [f"fields-{f}-{op}" for f, (_, _, ops) in fields.items() for op in ops] + expected = {(g, mode) for g in groups for mode in + (("latency", "throughput") if g.endswith(("-add", "-mul")) else ("latency",))} + return fields, groups, expected + + +def command(args, cwd=ROOT): + return subprocess.check_output(args, cwd=cwd, text=True).strip() + + +BUILD_RECORD = ROOT / ".lake/build/field-bench-build.json" +EXECUTABLES = (".lake/build/bin/CompPolyBench", ".lake/build/bin/CompPolyFieldFixtures", + ".lake/build/bin/CompPolyNTTBench", ".lake/build/bin/CompPolyInterpolateBench", + ".lake/build/bin/CompPolyLdeBench", + "bench/rust/target/release/comppoly-field-bench") + + +def file_hash(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def build_context(): + # Include untracked sources and file contents: a dirty boolean is insufficient. + paths = subprocess.check_output( + ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], cwd=ROOT + ).decode().split("\0") + digest = hashlib.sha256() + for name in sorted(set(paths) - {""}): + path = ROOT / name + digest.update(name.encode() + b"\0") + digest.update((file_hash(path) if path.is_file() else "missing").encode() + b"\0") + cpu = Path("/proc/cpuinfo").read_text().split("\n\n", 1)[0] + return { + "commit": command(["git", "rev-parse", "HEAD"]), + "dirty": bool(command(["git", "status", "--porcelain", "--untracked-files=normal"])), + "source_sha256": digest.hexdigest(), + "lean_version": command(["lake", "env", "lean", "--version"]), + "rust_version": command(["rustc", "--version"], ROOT / "bench/rust"), + "rustflags": os.environ.get("RUSTFLAGS", ""), + "build_environment": {k: v for k, v in sorted(os.environ.items()) + if k.startswith(("RUST", "CARGO", "LEAN", "LAKE")) + or k in ("CC", "CXX", "CFLAGS", "CXXFLAGS", "LDFLAGS", "PATH")}, + "native_cpu": [line for line in cpu.splitlines() + if line.startswith(("model name", "flags"))], + } + + +def executable_hashes(): + return {name: file_hash(ROOT / name) for name in EXECUTABLES} + + +def verify_build(record, context): + if record["context"] != context or record["executables"] != executable_hashes(): + raise ValueError("benchmark build does not match sources, toolchains, flags, CPU, or binaries") + + +def prepare_build(skip): + context = build_context() + if skip: + try: + record = json.loads(BUILD_RECORD.read_text()) + verify_build(record, context) + except (OSError, ValueError, KeyError, TypeError) as error: + raise SystemExit(f"Cannot reuse benchmark build: {error}. Run without --skip-build.") + return record + # A failed/interrupted build must not leave a valid-looking old record. + BUILD_RECORD.unlink(missing_ok=True) + subprocess.run(["lake", "build", "CompPolyBench", "CompPolyFieldFixtures", "CompPolyNTTBench", "CompPolyInterpolateBench", "CompPolyLdeBench"], cwd=ROOT, check=True) + subprocess.run(["cargo", "build", "--release", "--locked", "-j", "1"], cwd=ROOT / "bench/rust", check=True) + if build_context() != context: + raise SystemExit("Sources or build settings changed during the build; rerun before measuring.") + record = {"context": context, "executables": executable_hashes()} + temporary = BUILD_RECORD.with_suffix(".tmp") + temporary.write_text(json.dumps(record, indent=2) + "\n") + temporary.replace(BUILD_RECORD) + return record + + +def rows(path): + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def indexed(records, expected, lean=False): + selected = {} + for row in records: + if lean and not row["name"].endswith("-fast"): + continue + mode = (row["digest_class"] or "latency") if lean else row["mode"] + key = row["group_key"], mode + if key in selected: + raise ValueError(f"duplicate case: {key}") + selected[key] = row + if selected.keys() != expected: + raise ValueError(f"unexpected cases: missing {expected - selected.keys()}, extra {selected.keys() - expected}") + return selected + + +def compare(lean, rust): + if lean.keys() != rust.keys(): + raise ValueError("Lean/Rust case sets differ") + for key in lean: + for prop in ("checksum", "work_units"): + if int(lean[key][prop]) != int(rust[key][prop]): + raise ValueError(f"{key}: {prop} differs: {lean[key][prop]} vs {rust[key][prop]}") + + +def report(out, measurements, manifest, fields, expected): + lines = ["## Fields: fast Lean vs Rust", "", + "Rust uses Plonky3 for small primes, arkworks for BN254 scalar arithmetic, and Binius for binary towers.", "", + "Nanoseconds per operation; **lower is better**. Values are the median of run medians; ± is the median absolute deviation between runs. Ratio = Lean / Rust (>1 means Rust is faster).", ""] + for mode in ("latency", "throughput"): + lines += [f"### {mode.title()}", "", "| Field | Operation | Fast Lean (ns) | Rust (ns) | Lean / Rust |", + "| :--- | :--- | ---: | ---: | ---: |"] + for field, (title, _library, operations) in fields.items(): + for op in operations: + key = f"fields-{field}-{op}", mode + if key not in expected: + continue + values = [] + for language in ("lean", "rust"): + runs = [statistics.median(pair[language][key]["samples_picos"]) / + (1000 * pair[language][key]["work_units"]) for pair in measurements] + median = statistics.median(runs) + mad = statistics.median(abs(x - median) for x in runs) + values.append((median, mad)) + (lean, lm), (rust, rm) = values + lines.append(f"| {title} | {'exp' if op == 'pow' else op} | {lean:.2f} ± {lm:.2f} | {rust:.2f} ± {rm:.2f} | {lean/rust:.2f}× |") + lines.append("") + lines += ["### Machine and method", "", + f"- **CPU:** {manifest['cpu_model']}; {manifest['logical_cpus']} logical CPUs. Both runners pinned to logical CPU {manifest['cpu']}, sequentially, with one thread (SMT siblings: {manifest['smt_siblings']}).", + f"- **Memory:** {manifest['memory_gib']:.1f} GiB. **OS:** {manifest['os']}; kernel {manifest['kernel']} ({manifest['architecture']}).", + f"- **Toolchains:** {manifest['lean_version']}; {manifest['rust_version']}; Plonky3 0.4.2, arkworks 0.5.0, and Binius 0.2.0 (revision `{manifest['binius_revision']}`). Rust release, LTO, one codegen unit; RUSTFLAGS={manifest['rustflags']!r}.", + f"- **Source:** `{manifest['commit']}`; tracked files dirty: {manifest['dirty']}. Fixture SHA-256: `{manifest['fixture_sha256']}`.", + f"- **Sampling:** {len(measurements)} paired runs, alternating Lean/Rust order; each case uses 50 ms warmup and 20 samples targeting 1 ms each. All {len(expected)} untimed result digests agree with Lean; prime-field groups also check their reference implementations.", + "- **Workloads:** small-prime add/mul use 1,280 operations per batch; BN254 add/mul use 320. Small-prime throughput uses a ten-lane ring and nine final combining operations; BN254 uses two independent chains, four rounds unrolled, with the same fixed operand as latency and one final combining operation. This configuration was selected manually on Rust and fixed identically in Lean. Combining operations are outside the batch divisor, matching Lean. Inv/exp use 64 dependent steps of `inv(x + b)` / `(x + b)^0x5A5A5A5A`, so their times include one add per step. Only the final batch result is consumed. BN254 inversion uses CompPoly’s checked binary-GCD implementation and arkworks’ inverse.", + "- **Binary workloads:** Fan–Paar tower coefficients in little-endian bytes, with the same basis on both sides. Multiplication uses 64 dependent steps or two independent 32-step chains plus one final multiply (excluded from the divisor). Squaring uses 63 dependent squares, unrolled seven at a time; 63 is not a whole Frobenius cycle at any selected size. Inversion uses 64 steps of `inv(x XOR b)`, including the XOR. Only the final batch result is consumed. Scalar Binius field types are used, with supported CPU instructions enabled; there is no explicit packed-SIMD workload.", + f"- **CPU instruction support:** {', '.join(manifest['cpu_features'])}. The archived Binius tower library is pinned because Binius64 uses different field representations.", + "- **Shared host:** other jobs may contend for the CPU, SMT sibling, caches, or boost budget. These are observations under load, not isolated-machine speed claims.", ""] + (out / "report.md").write_text("\n".join(lines)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--suite", choices=[*SUITES, "ntt", "interpolate", "lde", "all"], default="small-prime") + parser.add_argument("--validate-only", action="store_true") + parser.add_argument("--skip-build", action="store_true", help="reuse executables only if recorded build metadata and hashes match") + parser.add_argument("--cpu", type=int, help="logical CPU; default: first allowed CPU") + parser.add_argument("--cpus", help="distinct logical CPU IDs for the NTT (one worker each; SMT allowed), e.g. 8,9,10,11") + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--out-dir", type=Path) + args = parser.parse_args() + if args.runs < 3: + parser.error("use at least three paired runs") + allowed = os.sched_getaffinity(0) + if args.suite == "ntt": + import sys + bench_ntt.run(args, sys.modules[__name__], allowed) + return + if args.suite == "interpolate": + import sys + bench_interpolate.run(args, sys.modules[__name__], allowed) + return + if args.suite == "lde": + import sys + bench_lde.run(args, sys.modules[__name__], allowed) + return + fields, groups, expected = selection(args.suite) + cpu = min(allowed) if args.cpu is None else args.cpu + if cpu not in allowed: + parser.error(f"CPU {cpu} is outside allowed affinity") + os.sched_setaffinity(0, {cpu}) + os.environ["LEAN_NUM_THREADS"] = "1" + os.environ["CARGO_BUILD_JOBS"] = "1" + os.environ.setdefault("RUSTFLAGS", "-C target-cpu=native") + out = (args.out_dir or ROOT / "bench/out" / time.strftime(f"fields-{args.suite}-%Y%m%d-%H%M%S")).resolve() + out.mkdir(parents=True, exist_ok=False) + build = prepare_build(args.skip_build) + fixtures = out / "fixtures.jsonl" + exported = [json.loads(line) for line in command( + [str(ROOT / ".lake/build/bin/CompPolyFieldFixtures")]).splitlines()] + selected = [row for row in exported if row["group_key"] in groups] + if len(selected) != len(groups) or {row["group_key"] for row in selected} != set(groups): + raise ValueError("missing or duplicate fixtures") + fixtures.write_text("".join(json.dumps(row) + "\n" for row in selected)) + rust_exe = ROOT / "bench/rust/target/release/comppoly-field-bench" + + def run(language, label, validate): + directory = out / label / language + directory.mkdir(parents=True) + if language == "lean": + cmd = [str(ROOT / ".lake/build/bin/CompPolyBench"), "--medium", "--json-only", + "--groups", ",".join(groups), "--out-dir", str(directory)] + else: + cmd = [str(rust_exe), str(fixtures)] + if validate: + cmd.append("--validate-only") + with (directory / "stdout.jsonl").open("w") as log: + subprocess.run(cmd, cwd=ROOT, stdout=log, check=True) + if language == "lean": + files = list(directory.glob("results-*.jsonl")) + if len(files) != 1: + raise ValueError("expected one Lean result file") + return indexed(rows(files[0]), expected, lean=True) + return indexed(rows(directory / "stdout.jsonl"), expected) + + compare(run("lean", "validation", True), run("rust", "validation", True)) + print(f"All {len(expected)} cases: Lean/Rust checksums and operation counts agree.", flush=True) + verify_build(build, build_context()) + if args.validate_only: + if args.suite == "all": + run_remaining_suites(args, allowed, out) + return + lock = tomllib.loads((ROOT / "bench/rust/Cargo.lock").read_text()) + binius = next(p for p in lock["package"] if p["name"] == "binius_field") + cpu_flags = next(line.split(":", 1)[1].split() for line in Path("/proc/cpuinfo").read_text().splitlines() if line.startswith("flags")) + manifest = { + "binius_revision": binius["source"].split("#")[1], + "cpu_features": [f for f in ("sse2", "ssse3", "sse4_1", "avx", "avx2", "pclmulqdq", "gfni", "avx512f") if f in cpu_flags], + "suite": args.suite, + "commit": build["context"]["commit"], + "dirty": build["context"]["dirty"], + "build": build, + "cpu": cpu, "logical_cpus": os.cpu_count(), + "smt_siblings": Path(f"/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list").read_text().strip(), + "cpu_model": next(line.split(":", 1)[1].strip() for line in Path("/proc/cpuinfo").read_text().splitlines() if line.startswith("model name")), + "memory_gib": int(Path("/proc/meminfo").read_text().splitlines()[0].split()[1]) / 1024**2, + "os": platform.freedesktop_os_release()["PRETTY_NAME"], "kernel": platform.release(), + "architecture": platform.machine(), "lean_version": build["context"]["lean_version"], + "rust_version": build["context"]["rust_version"], + "rustflags": build["context"]["rustflags"], "runs": args.runs, + "fixture_sha256": hashlib.sha256(fixtures.read_bytes()).hexdigest(), + "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "load_start": os.getloadavg(), + } + measurements = [] + for i in range(args.runs): + pair = {} + for language in (("lean", "rust") if i % 2 == 0 else ("rust", "lean")): + print(f"Run {i+1}/{args.runs}: {language}, CPU {cpu}", flush=True) + pair[language] = run(language, f"run-{i+1}", False) + compare(pair["lean"], pair["rust"]) + measurements.append(pair) + verify_build(build, build_context()) + manifest["load_end"] = os.getloadavg() + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + report(out, measurements, manifest, fields, expected) + print(f"Report: {out / 'report.md'}") + if args.suite == "all": + run_remaining_suites(args, allowed, out) + + +def run_remaining_suites(args, allowed, out): + import copy + import sys + ntt_args = copy.copy(args) + ntt_args.out_dir = out / "ntt" + ntt_args.skip_build = True + bench_ntt.run(ntt_args, sys.modules[__name__], allowed) + interpolate_args = copy.copy(args) + interpolate_args.out_dir = out / "interpolate" + interpolate_args.skip_build = True + bench_interpolate.run(interpolate_args, sys.modules[__name__], allowed) + lde_args = copy.copy(args) + lde_args.out_dir = out / "lde" + lde_args.skip_build = True + bench_lde.run(lde_args, sys.modules[__name__], allowed) + + +if __name__ == "__main__": + main() diff --git a/scripts/bench_interpolate.py b/scripts/bench_interpolate.py new file mode 100644 index 00000000..fd23a097 --- /dev/null +++ b/scripts/bench_interpolate.py @@ -0,0 +1,127 @@ +"""Lean coset interpolation versus Plonky3's `interpolate_coset`.""" +import json +import os +from pathlib import Path +import platform +import random +import statistics +import subprocess +import time + + +IMPLS = ("packed", "reference", "rust") +P = 2130706433 + + +def run(args, common, allowed): + if args.cpus: + cpus = [int(c) for c in args.cpus.split(",")] + elif args.cpu is not None: + cpus = [args.cpu] + else: + physical = {} + for c in sorted(allowed): + siblings = Path(f"/sys/devices/system/cpu/cpu{c}/topology/thread_siblings_list").read_text() + physical.setdefault(siblings, c) + preferred = list(physical.values()) + cpus = preferred[:min(16, 2 ** (len(preferred).bit_length() - 1))] + workers = len(cpus) + if not workers or workers & (workers - 1) or len(set(cpus)) != workers or not set(cpus) <= allowed: + raise SystemExit("select a power-of-two number of distinct available CPUs") + physical_cores = len({Path(f"/sys/devices/system/cpu/cpu{c}/topology/thread_siblings_list").read_text() for c in cpus}) + os.sched_setaffinity(0, {cpus[0]}) + os.environ["LEAN_NUM_THREADS"] = "1" + os.environ["CARGO_BUILD_JOBS"] = "1" + os.environ.setdefault("RUSTFLAGS", "-C target-cpu=native") + out = (args.out_dir or common.ROOT / "bench/out" / time.strftime("interpolate-%Y%m%d-%H%M%S")).resolve() + out.mkdir(parents=True, exist_ok=False) + build = common.prepare_build(args.skip_build) + lean = common.ROOT / ".lake/build/bin/CompPolyInterpolateBench" + roots_exe = common.ROOT / ".lake/build/bin/CompPolyNTTBench" + rust = common.ROOT / "bench/rust/target/release/comppoly-field-bench" + os.sched_setaffinity(0, set(cpus)) + runtime_env = dict(os.environ, LEAN_NUM_THREADS=str(workers), RAYON_NUM_THREADS=str(workers)) + # Tiny shapes exercise single-row and narrow matrices; only the large ones are timed. + timed = [(12, 1), (12, 16), (16, 1), (16, 16), (20, 1), (20, 16)] + shapes = [(0, 1), (1, 1), (3, 2), (5, 3), *timed] + fixtures = {} + roots = {} + for log_n, width in shapes: + if log_n not in roots: + roots[log_n] = int(common.command([str(roots_exe), "--root", str(log_n)])) + rng = random.Random(f"comppoly-interpolate-koalabear-v1:{log_n}:{width}") + path = out / f"koalabear-{log_n}-{width}.bin" + with path.open("wb") as stream: + # Root, shift (KoalaBear's multiplicative generator), two extension points, matrix. + header = [roots[log_n], 3] + [rng.randrange(P) for _ in range(8)] + for value in header + [rng.randrange(P) for _ in range(2 ** log_n * width)]: + stream.write(value.to_bytes(4, "little")) + fixtures[log_n, width] = path + manifest = { + "build": build, "cpus": cpus, "workers": workers, "physical_cores": physical_cores, "runs": args.runs, + "packed": {"implementation": "KoalaBear.Fast.interpolateCosetPacked (parallel row ranges, packed words)", "leanc_args": ["-march=native"]}, + "reference": {"implementation": "KoalaBear.Fast.interpolateCoset (proved reference, sequential)", "leanc_args": ["-march=native"]}, + "rust": {"implementation": "p3_interpolation::interpolate_coset", "extension": "BinomialExtensionField", "matrix": "RowMajorMatrix", "parallel": True}, + "cpu_model": next(s.split(":", 1)[1].strip() for s in Path("/proc/cpuinfo").read_text().splitlines() if s.startswith("model name")), + "memory_gib": int(Path("/proc/meminfo").read_text().splitlines()[0].split()[1]) / 1024**2, + "os": platform.freedesktop_os_release()["PRETTY_NAME"], "kernel": platform.release(), + "fixture_format": "field-coordinates-le-v1; canonical u32 root, shift, two degree-4 points, then a row-major 2^log_n × width matrix", + "roots": roots, "fixture_sha256": {f"{k[0]}-{k[1]}": common.file_hash(v) for k, v in fixtures.items()}, + "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "load_start": os.getloadavg(), + } + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + def measure(language, log_n, width, label, validate): + cmd = [str(rust), "--interpolate"] if language == "rust" else [str(lean)] + cmd += [str(fixtures[log_n, width]), str(log_n), str(width), str(validate).lower()] + if language != "rust": + cmd.append(language) + result = subprocess.check_output(cmd, cwd=common.ROOT, env=runtime_env, text=True) + (out / f"{label}-{log_n}-{width}-{language}.jsonl").write_text(result) + row = json.loads(result) + if row["group_key"] != f"interpolate-koalabear-{log_n}-{width}" or row["work_units"] != 1: + raise ValueError("unexpected interpolation workload") + return row + + results = {} + for log_n, width in shapes: + validated = [measure(lang, log_n, width, "validation", True) for lang in IMPLS] + expected = int(validated[0]["checksum"]) + if any(expected != int(row["checksum"]) for row in validated[1:]): + raise ValueError(f"interpolation checksum mismatch: log_n={log_n}, width={width}") + print(f"Validated KoalaBear coset interpolation: 2^{log_n} × {width}", flush=True) + if args.validate_only or (log_n, width) not in timed: + continue + for i in range(args.runs): + for language in IMPLS[i % 3:] + IMPLS[:i % 3]: + row = measure(language, log_n, width, f"run-{i + 1}", False) + if int(row["checksum"]) != expected: + raise ValueError("interpolation timing-run checksum mismatch") + results.setdefault((log_n, width, language), []).append(statistics.median(row["samples_picos"]) / 1e9) + print(f"Measured KoalaBear coset interpolation: 2^{log_n} × {width}", flush=True) + common.verify_build(build, common.build_context()) + manifest["load_end"] = os.getloadavg() + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + if args.validate_only: + return + lines = ["# KoalaBear coset interpolation: Lean vs Plonky3", "", "Milliseconds per call; median of run medians ± between-run MAD. Each call evaluates every column of a row-major matrix of values on a coset at a degree-4 extension point. Lean / Plonky3 > 1 means Plonky3 is faster.", "", + "| Rows | Columns | Lean packed | Lean reference | Plonky3 | Lean packed / Plonky3 |", "|---:|---:|---:|---:|---:|---:|"] + for log_n, width in timed: + medians, cells = [], [] + for language in IMPLS: + samples = results[log_n, width, language] + median = statistics.median(samples) + mad = statistics.median(abs(x - median) for x in samples) + medians.append(median) + cells.append(f"{median:.4f} ± {mad:.4f}") + lines.append(f"| {2**log_n:,} | {width} | {' | '.join(cells)} | {medians[0] / medians[2]:.2f}× |") + lines += ["", "## Machine and method", "", + f"- {manifest['cpu_model']}; {manifest['memory_gib']:.1f} GiB; {manifest['os']}, kernel {manifest['kernel']}; {workers} workers on {physical_cores} physical cores, logical CPUs {cpus}.", + f"- {build['context']['lean_version']}; {build['context']['rust_version']}; Rust flags {build['context']['rustflags']!r}. Source `{build['context']['commit']}`, dirty={build['context']['dirty']}.", + "- Lean packed runs `KoalaBear.Fast.interpolateCosetPacked` on the matrix's Montgomery words in a `ByteArray`: row ranges on parallel tasks, blocks of 512 rows with base-field weights from one inversion per block, lazily reduced column sums four columns at a time. `interpolateCosetPacked_eq` proves it equal to the reference.", + "- Lean reference runs the proved `KoalaBear.Fast.interpolateCoset` sequentially: coset nodes by prefix products, one batch inversion over lists, the weights xᵢ/(z - xᵢ), one dot product per column, and a final scale by (zⁿ - sⁿ)/(n sⁿ). Plonky3 calls `p3_interpolation::interpolate_coset` 0.4.2 on a `RowMajorMatrix` with `BinomialExtensionField`, with its parallel feature and native SIMD.", + "- All compute the coset nodes, inversions and weights inside every call. Fixture decoding and matrix construction are outside timing.", + "- Two deterministic points alternate to prevent result hoisting. Full output digests of all three implementations agree before timing, including single-row and narrow matrices.", + f"- {args.runs} rounds in rotating order, 200 ms warmup and 50 samples per invocation. Shared-host load {manifest['load_start']} → {manifest['load_end']}; other work may affect timings.", ""] + (out / "report.md").write_text("\n".join(lines)) + print(f"Report: {out / 'report.md'}") diff --git a/scripts/bench_lde.py b/scripts/bench_lde.py new file mode 100644 index 00000000..899e3279 --- /dev/null +++ b/scripts/bench_lde.py @@ -0,0 +1,138 @@ +"""Lean coset low-degree extension versus Plonky3's `coset_lde_batch`.""" +import json +import os +from pathlib import Path +import platform +import random +import statistics +import subprocess +import time + + +LEAN = ("lean",) +RUST = ("dit-parallel", "recursive", "small-batch") +IMPLS = LEAN + RUST +P = 2130706433 + + +def run(args, common, allowed): + if args.cpus: + cpus = [int(c) for c in args.cpus.split(",")] + elif args.cpu is not None: + cpus = [args.cpu] + else: + physical = {} + for c in sorted(allowed): + siblings = Path(f"/sys/devices/system/cpu/cpu{c}/topology/thread_siblings_list").read_text() + physical.setdefault(siblings, c) + preferred = list(physical.values()) + cpus = preferred[:min(16, 2 ** (len(preferred).bit_length() - 1))] + workers = len(cpus) + if not workers or workers & (workers - 1) or len(set(cpus)) != workers or not set(cpus) <= allowed: + raise SystemExit("select a power-of-two number of distinct available CPUs") + physical_cores = len({Path(f"/sys/devices/system/cpu/cpu{c}/topology/thread_siblings_list").read_text() for c in cpus}) + os.sched_setaffinity(0, {cpus[0]}) + os.environ["LEAN_NUM_THREADS"] = "1" + os.environ["CARGO_BUILD_JOBS"] = "1" + os.environ.setdefault("RUSTFLAGS", "-C target-cpu=native") + out = (args.out_dir or common.ROOT / "bench/out" / time.strftime("lde-%Y%m%d-%H%M%S")).resolve() + out.mkdir(parents=True, exist_ok=False) + build = common.prepare_build(args.skip_build) + lean = common.ROOT / ".lake/build/bin/CompPolyLdeBench" + roots_exe = common.ROOT / ".lake/build/bin/CompPolyNTTBench" + rust = common.ROOT / "bench/rust/target/release/comppoly-field-bench" + os.sched_setaffinity(0, set(cpus)) + runtime_env = dict(os.environ, LEAN_NUM_THREADS=str(workers), RAYON_NUM_THREADS=str(workers)) + # (log height, width, added bits). Tiny shapes exercise single rows, odd heights and widths off + # the tile sizes; only the large ones are timed. + timed = [(14, 64, 1), (16, 8, 1), (16, 64, 1), (16, 256, 1), (18, 64, 1), (20, 8, 1), (20, 64, 1), (16, 64, 2)] + shapes = [(0, 1, 1), (1, 3, 1), (3, 2, 2), (5, 9, 1), (6, 17, 2), (7, 4, 3), (9, 7, 1), *timed] + fixtures = {} + roots = {} + for log_h, width, bits in shapes: + log_n = log_h + bits + if log_n not in roots: + roots[log_n] = int(common.command([str(roots_exe), "--root", str(log_n)])) + rng = random.Random(f"comppoly-lde-koalabear-v1:{log_h}:{width}:{bits}") + path = out / f"koalabear-{log_h}-{width}-{bits}.bin" + with path.open("wb") as stream: + # Root of the extended domain, shift (KoalaBear's multiplicative generator), then two + # row-major matrices. + for value in [roots[log_n], 3]: + stream.write(value.to_bytes(4, "little")) + stream.write(b"".join(rng.randrange(P).to_bytes(4, "little") for _ in range(2 * 2 ** log_h * width))) + fixtures[log_h, width, bits] = path + manifest = { + "build": build, "cpus": cpus, "workers": workers, "physical_cores": physical_cores, "runs": args.runs, + "lean": {"implementation": "NTTFast.Packed.Native.cosetLde (row chunks)", "leanc_args": ["-march=native", "-mstackrealign"]}, + "rust": {"implementation": "TwoAdicSubgroupDft::coset_lde_batch(..).bit_reverse_rows().to_row_major_matrix()", + "dfts": ["Radix2DitParallel", "p3_monty_31::dft::RecursiveDft", "Radix2DFTSmallBatch"], "parallel": True}, + "cpu_model": next(s.split(":", 1)[1].strip() for s in Path("/proc/cpuinfo").read_text().splitlines() if s.startswith("model name")), + "memory_gib": int(Path("/proc/meminfo").read_text().splitlines()[0].split()[1]) / 1024**2, + "os": platform.freedesktop_os_release()["PRETTY_NAME"], "kernel": platform.release(), + "fixture_format": "field-coordinates-le-v1; canonical u32 root of the extended domain, shift, then two row-major 2^log_h × width matrices", + "roots": roots, "fixture_sha256": {f"{k[0]}-{k[1]}-{k[2]}": common.file_hash(v) for k, v in fixtures.items()}, + "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "load_start": os.getloadavg(), + } + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + def measure(impl, shape, label, validate): + log_h, width, bits = shape + args_ = [str(fixtures[shape]), str(log_h), str(width), str(bits)] + if impl in RUST: + cmd = [str(rust), "--lde", *args_, impl, str(validate).lower()] + else: + cmd = [str(lean), *args_, str(validate).lower()] + result = subprocess.check_output(cmd, cwd=common.ROOT, env=runtime_env, text=True) + (out / f"{label}-{log_h}-{width}-{bits}-{impl}.jsonl").write_text(result) + row = json.loads(result) + if row["group_key"] != f"lde-koalabear-{log_h}-{width}-{bits}" or row["work_units"] != 1: + raise ValueError("unexpected LDE workload") + return row + + results = {} + for shape in shapes: + validated = [measure(impl, shape, "validation", True) for impl in IMPLS] + expected = int(validated[0]["checksum"]) + if any(expected != int(row["checksum"]) for row in validated[1:]): + raise ValueError(f"LDE checksum mismatch: {shape}") + print(f"Validated KoalaBear coset LDE: 2^{shape[0]} × {shape[1]}, blowup 2^{shape[2]}", flush=True) + if args.validate_only or shape not in timed: + continue + for i in range(args.runs): + for impl in IMPLS[i % len(IMPLS):] + IMPLS[:i % len(IMPLS)]: + row = measure(impl, shape, f"run-{i + 1}", False) + if int(row["checksum"]) != expected: + raise ValueError("LDE timing-run checksum mismatch") + results.setdefault((shape, impl), []).append(statistics.median(row["samples_picos"]) / 1e9) + print(f"Measured KoalaBear coset LDE: 2^{shape[0]} × {shape[1]}, blowup 2^{shape[2]}", flush=True) + common.verify_build(build, common.build_context()) + manifest["load_end"] = os.getloadavg() + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + if args.validate_only: + return + + def cell(shape, impl): + samples = results[shape, impl] + median = statistics.median(samples) + return median, f"{median:.2f} ± {statistics.median(abs(x - median) for x in samples):.2f}" + + lines = ["# KoalaBear coset low-degree extension: Lean vs Plonky3", "", + "Milliseconds per call; median of run medians ± between-run MAD. Each call extends every column of a row-major matrix of values on the subgroup of order 2^log_h to the coset `3 · ⟨ω⟩` of order 2^(log_h + bits), and returns the rows in the bit-reversed order Plonky3's two-adic FRI commitment commits. Lean / Plonky3 > 1 means Plonky3 is faster; Plonky3 is its fastest DFT for the shape.", "", + "| Rows | Columns | Blowup | Lean | DitParallel | Recursive | SmallBatch | Lean / Plonky3 |", + "|---:|---:|---:|---:|---:|---:|---:|---:|"] + for shape in timed: + cells = {impl: cell(shape, impl) for impl in IMPLS} + best = min(cells[impl][0] for impl in RUST) + lines.append(f"| 2^{shape[0]} | {shape[1]} | 2^{shape[2]} | {' | '.join(cells[impl][1] for impl in IMPLS)} | " + f"{cells['lean'][0] / best:.2f}× |") + lines += ["", "## Machine and method", "", + f"- {manifest['cpu_model']}; {manifest['memory_gib']:.1f} GiB; {manifest['os']}, kernel {manifest['kernel']}; {workers} workers on {physical_cores} physical cores, logical CPUs {cpus}.", + f"- {build['context']['lean_version']}; {build['context']['rust_version']}; Rust flags {build['context']['rustflags']!r}. Source `{build['context']['commit']}`, dirty={build['context']['dirty']}.", + "- Lean runs `NTTFast.Packed.Native.cosetLde` on the matrix's Montgomery words in one chunk of rows: tasks transpose row ranges into columns, run one unnormalized inverse transform per column and one DIF transform per column and coset of the scaled coefficients, whose DIF order is the commitment order, and transpose each block's rows as soon as its columns are done. It returns those row chunks, where Plonky3's threads write one buffer in place; joining the chunks (`joinRows`) would add one copy of the output.", + "- Plonky3 0.4.2 runs `coset_lde_batch` on a `RowMajorMatrix` with each of `Radix2DitParallel`, Monty31's `RecursiveDft` and `Radix2DFTSmallBatch`, then `bit_reverse_rows().to_row_major_matrix()`, exactly as `TwoAdicFriPcs::commit`; for the first two the bit reversal is free. Parallel feature and native SIMD enabled.", + "- Plans and twiddles are built outside timing (Plonky3's caches are warmed by the untimed digest pass). Input copies, transposes, all transforms and output assembly are timed.", + "- Two deterministic matrices alternate. Full output digests of all four implementations agree before timing, including single-row, odd-height and narrow matrices.", + f"- {args.runs} rounds in rotating order, 200 ms warmup and 50 samples per invocation. Shared-host load {manifest['load_start']} → {manifest['load_end']}; other work may affect timings.", ""] + (out / "report.md").write_text("\n".join(lines)) + print(f"Report: {out / 'report.md'}") diff --git a/scripts/bench_ntt.py b/scripts/bench_ntt.py new file mode 100644 index 00000000..5b42802a --- /dev/null +++ b/scripts/bench_ntt.py @@ -0,0 +1,127 @@ +"""Lean planned NTT versus the optimized Plonky3 API.""" +import json +import os +from pathlib import Path +import platform +import random +import statistics +import subprocess +import time + + +IMPLS = ("packed", "externless", "rust") + + +def run(args, common, allowed): + if args.cpus: + cpus = [int(c) for c in args.cpus.split(",")] + elif args.cpu is not None: + cpus = [args.cpu] + else: + physical = {} + for c in sorted(allowed): + siblings = Path(f"/sys/devices/system/cpu/cpu{c}/topology/thread_siblings_list").read_text() + physical.setdefault(siblings, c) + preferred = list(physical.values()) + cpus = preferred[:min(16, 2 ** (len(preferred).bit_length() - 1))] + workers = len(cpus) + if not workers or workers & (workers - 1) or len(set(cpus)) != workers or not set(cpus) <= allowed: + raise SystemExit("select a power-of-two number of distinct available CPUs") + physical_cores = len({Path(f"/sys/devices/system/cpu/cpu{c}/topology/thread_siblings_list").read_text() for c in cpus}) + os.sched_setaffinity(0, {cpus[0]}) + os.environ["LEAN_NUM_THREADS"] = "1" + os.environ["CARGO_BUILD_JOBS"] = "1" + os.environ.setdefault("RUSTFLAGS", "-C target-cpu=native") + out = (args.out_dir or common.ROOT / "bench/out" / time.strftime("ntt-%Y%m%d-%H%M%S")).resolve() + out.mkdir(parents=True, exist_ok=False) + build = common.prepare_build(args.skip_build) + lean = common.ROOT / ".lake/build/bin/CompPolyNTTBench" + rust = common.ROOT / "bench/rust/target/release/comppoly-field-bench" + os.sched_setaffinity(0, set(cpus)) + runtime_env = dict(os.environ, LEAN_NUM_THREADS=str(workers), RAYON_NUM_THREADS=str(workers)) + rust_ntt = json.loads(subprocess.check_output([str(rust), "--ntt", "--info"], env=runtime_env, text=True)) + if rust_ntt["workers"] != workers: + raise ValueError("Plonky3 worker count mismatch") + # Odd sizes exercise the radix-2 layer, 2^15 and 2^17 the odd column path; only the + # sizes in `sizes` are timed. + sizes = (12, 16, 20) + fixtures = {} + roots = {} + for log_n in (0, 1, 3, 4, 5, 15, 17, *sizes): + root = int(common.command([str(lean), "--root", str(log_n)])) + roots[log_n] = root + rng = random.Random(f"comppoly-ntt-koalabear-v1:{log_n}") + path = out / f"koalabear-{log_n}.bin" + with path.open("wb") as stream: + stream.write(root.to_bytes(4, "little")) + for i in range(2 * 2**log_n): + value = (0, 1, 2130706432)[i] if i < 3 else rng.randrange(2130706433) + stream.write(value.to_bytes(4, "little")) + fixtures[log_n] = path + manifest = { + "build": build, "cpus": cpus, "workers": workers, "physical_cores": physical_cores, "runs": args.runs, + "rust_ntt": rust_ntt, "lean_ntt": {"packed": {"implementation": "NTTFast.Packed.Plan.forwardPacked/inversePacked", "max_arithmetic_workers": workers, "storage": "ByteArray.ugetUInt32LE/usetUInt32LE (lean4#14053 backport)"}, "externless": {"implementation": "NTTFast.NaturalPlan.forwardPacked/inversePacked", "max_arithmetic_workers": workers}, "serial_below_log_n": 15, "leanc_args": ["-march=native"]}, "ordering": "natural input and output", + "cpu_model": next(s.split(":", 1)[1].strip() for s in Path("/proc/cpuinfo").read_text().splitlines() if s.startswith("model name")), + "memory_gib": int(Path("/proc/meminfo").read_text().splitlines()[0].split()[1]) / 1024**2, + "os": platform.freedesktop_os_release()["PRETTY_NAME"], "kernel": platform.release(), + "fixture_format": "field-coordinates-le-v1; canonical u32 root, then two arrays of 2^log_n coefficients", + "roots": roots, "fixture_sha256": {str(k): common.file_hash(v) for k, v in fixtures.items()}, + "started_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "load_start": os.getloadavg(), + } + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + def measure(language, log_n, direction, label, validate): + cmd = [str(rust), "--ntt"] if language == "rust" else [str(lean)] + cmd += [str(fixtures[log_n]), str(log_n), direction, str(validate).lower()] + env = dict(runtime_env, COMPPOLY_NTT_IMPL=language) + result = subprocess.check_output(cmd, cwd=common.ROOT, env=env, text=True) + (out / f"{label}-{log_n}-{direction}-{language}.jsonl").write_text(result) + row = json.loads(result) + if row["group_key"] != f"ntt-koalabear-{log_n}-{direction}" or row["work_units"] != 1: + raise ValueError("unexpected NTT workload") + return row + + results = {} + for log_n in fixtures: + for direction in ("forward", "inverse"): + validated = [measure(lang, log_n, direction, "validation", True) for lang in IMPLS] + expected = int(validated[0]["checksum"]) + if any(expected != int(row["checksum"]) for row in validated[1:]): + raise ValueError(f"NTT checksum mismatch: log_n={log_n}, {direction}") + print(f"Validated KoalaBear NTT: 2^{log_n}, {direction}", flush=True) + if args.validate_only or log_n not in sizes: + continue + for i in range(args.runs): + for language in (IMPLS if i % 2 == 0 else IMPLS[::-1]): + row = measure(language, log_n, direction, f"run-{i + 1}", False) + if int(row["checksum"]) != expected: + raise ValueError("NTT timing-run checksum mismatch") + results.setdefault((log_n, direction, language), []).append(statistics.median(row["samples_picos"]) / 1e9) + print(f"Measured KoalaBear NTT: 2^{log_n}, {direction}", flush=True) + common.verify_build(build, common.build_context()) + manifest["load_end"] = os.getloadavg() + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + if args.validate_only: + return + lines = ["# KoalaBear NTT: Lean vs optimized Plonky3", "", "Milliseconds per complete transform; median of paired run medians ± between-run MAD. All implementations receive the same CPU budget. All Lean pipelines are fully proved: both take and return packed Montgomery words; packed Lean reads and writes them with the `ByteArray` word accessors proposed for Lean core, externless Lean with core byte operations only. Lean / Rust > 1 means Rust is faster.", "", + "| Elements | Direction | Packed Lean | Externless Lean | Plonky3 | Packed / Rust | Externless / Rust |", "|---:|---|---:|---:|---:|---:|---:|"] + for log_n in sizes: + for direction in ("forward", "inverse"): + medians, cells = [], [] + for language in IMPLS: + samples = results[log_n, direction, language] + median = statistics.median(samples) + mad = statistics.median(abs(x - median) for x in samples) + medians.append(median) + cells.append(f"{median:.4f} ± {mad:.4f}") + lines.append(f"| {2**log_n:,} | {direction} | {' | '.join(cells)} | {' | '.join(f'{m / medians[2]:.2f}×' for m in medians[:2])} |") + lines += ["", "## Machine and method", "", + f"- {manifest['cpu_model']}; {manifest['memory_gib']:.1f} GiB; {manifest['os']}, kernel {manifest['kernel']}; {workers} workers on {physical_cores} physical cores, logical CPUs {cpus}.", + f"- {build['context']['lean_version']}; {build['context']['rust_version']}; Rust flags {build['context']['rustflags']!r}. Source `{build['context']['commit']}`, dirty={build['context']['dirty']}.", + f"- Both Lean plans run the same pipeline on packed Montgomery words: from 2^15 elements, column tasks run the top two radix-four passes and sixteen leaf tasks the rest with leaf-local bit reversal, before a parallel interleave into natural order; smaller transforms and a single worker run the sequential transform as one leaf. Packed Lean (NTTFast.Packed.Plan) computes on packed words with vectorized kernels, fusing the last four layers of eight leaf blocks with the leaf reversal, and its native modules use -march=native. Externless Lean (NTTFast.NaturalPlan) computes on field arrays and exchanges packed words between tasks. Rust calls {rust_ntt['implementation']} from p3-dft 0.4.2 directly, with packing width {rust_ntt['packing_width']}; native SIMD is enabled. Plonky3's parallel feature is enabled. This compares different algorithms implementing the same transform.", + "- All APIs use natural-order inputs and outputs. Forward maps coefficients to evaluations; inverse maps evaluations to coefficients, including 1/n normalization. The fixture root must equal both Lean's certified root and Plonky3's selected root.", + "- Plan construction, twiddle tables, cached permutation indices and fixture decoding are outside timing. Input copying, arithmetic, output disposal and ordering conversions are timed, including each Lean ordering adapter. Inverse normalization is timed. Inputs remain reusable and unchanged in all implementations.", + "- Two deterministic inputs alternate to prevent result hoisting. Full output digests are checked outside timing; a four-position output sink is used inside timing. Native validation includes tiny odd/even sizes and zero/one/near-modulus coordinates.", + f"- {args.runs} alternating four-way rounds, 200 ms warmup and 50 samples per invocation, so idle worker cores reach full clocks before timing. Shared-host load {manifest['load_start']} → {manifest['load_end']}; other work may affect timings.", ""] + (out / "report.md").write_text("\n".join(lines)) + print(f"Report: {out / 'report.md'}") diff --git a/scripts/drop-native-objects.sh b/scripts/drop-native-objects.sh new file mode 100755 index 00000000..21e39931 --- /dev/null +++ b/scripts/drop-native-objects.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Delete the build outputs compiled with `-march=native`, and the executables linking them, so +# the next `lake build` recompiles them for this host. CI restores `.lake/build` from runs on +# other runner CPUs, whose host-tuned objects can hold instructions this one lacks (SIGILL). + +set -euo pipefail + +[ -d .lake/build ] || exit 0 +{ grep -rlF --include='*.trace' -e '-march=native' .lake/build || true; } | while read -r trace; do + rm -f "${trace%.trace}" "${trace%.trace}".* +done +rm -rf .lake/build/bin diff --git a/scripts/style-exceptions.txt b/scripts/style-exceptions.txt index 30d15a69..3fc86b04 100644 --- a/scripts/style-exceptions.txt +++ b/scripts/style-exceptions.txt @@ -15,3 +15,4 @@ CompPoly/Data/Nat/Bitwise.lean : line 1 : ERR_NUM_LIN : 1700 file contains 1537 CompPoly/Bivariate/GuruswamiSudan/Root/Alekhnovich/Correctness.lean : line 1 : ERR_NUM_LIN : 2600 file contains 2421 lines, try to split it up : ERR_NUM_LIN CompPoly/Data/ExtTreeMap/DTreeMap.lean : line 1 : ERR_LIN : Line has more than 100 characters : ERR_LIN CompPoly/Univariate/Basic.lean : line 1 : ERR_NUM_LIN : 1700 file contains 1516 lines, try to split it up : ERR_NUM_LIN +CompPoly/Univariate/NTTFast/Packed/Native.lean : line 1 : ERR_NUM_LIN : 1700 file contains 1516 lines, try to split it up : ERR_NUM_LIN diff --git a/tests/CompPolyNativeSmoke.lean b/tests/CompPolyNativeSmoke.lean index f054e859..1449665b 100644 --- a/tests/CompPolyNativeSmoke.lean +++ b/tests/CompPolyNativeSmoke.lean @@ -9,12 +9,17 @@ public import CompPoly.Fields.Binary.Aes.Ghash public import CompPoly.Fields.Binary.BF128Ghash.Impl public import CompPoly.Fields.Binary.BF64.Ext3 public import CompPoly.Fields.Binary.Tower.Fast.Multilinear +public import CompPolyTests.Fields.Montgomery.InterpolateCosetPacked +public import CompPolyTests.NTT.NativeStorage +public import CompPolyTests.NTT.Packed /-! # Native field startup and arithmetic checks This executable checks canonical arithmetic in AES, BF64, its cubic extension, GHASH, -and the binary tower, including packed coefficient evaluation. +and the binary tower, including packed coefficient evaluation. It also checks +the backported `ByteArray` word accessors under the packed FFT against independent bytewise +Lean operations, and runs the packed FFT and coset interpolation over KoalaBear and BabyBear. The test guide documents resource limits for native initialization and execution. Unlike compile-time guards, this target exercises the linked executable's module initializers. The extension product uses the reference vector from the existing BF64 regression tests. @@ -163,6 +168,9 @@ def run : IO Unit := do check "GHASH reduction" ((high * BF128Ghash.ofBitVec (2#128)).toBitVec == 0x87#128) check "GHASH named inverse" (BF128Ghash.invItohTsujii high == expectedInverse) checkGhashOperations high expectedInverse + CompPolyTests.NTT.NativeStorage.run + CompPolyTests.NTT.Packed.run + CompPolyTests.Fields.Montgomery.InterpolateCosetPacked.run IO.println "Native field startup and arithmetic checks passed." end CompPolyTests.NativeSmoke diff --git a/tests/CompPolyTests.lean b/tests/CompPolyTests.lean index e402bb5d..c8f39b0c 100644 --- a/tests/CompPolyTests.lean +++ b/tests/CompPolyTests.lean @@ -38,6 +38,7 @@ public import CompPolyTests.Fields.BLS12_377.Fast public import CompPolyTests.Fields.BLS12_381.Fast public import CompPolyTests.Fields.BN254.Fast public import CompPolyTests.Fields.BabyBear.Fast +public import CompPolyTests.Fields.BabyBear.FastExt4 public import CompPolyTests.Fields.Binary.AdditiveNTT.Basis public import CompPolyTests.Fields.Binary.AdditiveNTT.Compatibility public import CompPolyTests.Fields.Binary.AdditiveNTT.Executable @@ -78,6 +79,8 @@ public import CompPolyTests.Fields.Extension.RawArithmetic public import CompPolyTests.Fields.Goldilocks.Bytes public import CompPolyTests.Fields.Goldilocks.Fast public import CompPolyTests.Fields.KoalaBear.Fast +public import CompPolyTests.Fields.KoalaBear.FastExt4 +public import CompPolyTests.Fields.KoalaBear.InterpolateCoset public import CompPolyTests.Fields.Mersenne31.Bytes public import CompPolyTests.Fields.Mersenne31.Circle public import CompPolyTests.Fields.Mersenne31.Fast @@ -85,6 +88,7 @@ public import CompPolyTests.Fields.Mersenne31.Instances public import CompPolyTests.Fields.Montgomery.Bytes public import CompPolyTests.Fields.Pasta.Fast public import CompPolyTests.Fields.PrattCertificate +public import CompPolyTests.Fields.Secp256k1.Fast public import CompPolyTests.LinearAlgebra.Dense public import CompPolyTests.LinearAlgebra.PolynomialMatrix.Approximant public import CompPolyTests.LinearAlgebra.TensorProduct.Basis diff --git a/tests/CompPolyTests/Fields/BLS12_377/Fast.lean b/tests/CompPolyTests/Fields/BLS12_377/Fast.lean index c41f0c5c..38d50653 100644 --- a/tests/CompPolyTests/Fields/BLS12_377/Fast.lean +++ b/tests/CompPolyTests/Fields/BLS12_377/Fast.lean @@ -10,7 +10,7 @@ public meta import CompPoly.Fields.BLS12_377.Fast /-! # Fast BLS12-377 Scalar Field Tests -Regression checks for the executable eight-limb Montgomery representation: the stored +Regression checks for the executable four-limb Montgomery representation: the stored residues, literal round trips, the field operations, the checked binary-GCD inversion, and agreement with the canonical `BLS12_377.ScalarField` model through `toField`. -/ @@ -20,13 +20,13 @@ public meta section namespace BLS12_377.Fast open BLS12_377 (scalarFieldSize) -open Montgomery.Native64x8 +open Montgomery.Native64x4 set_option maxRecDepth 4000 -- Stored Montgomery residues. -#guard (0 : ScalarField).val = Limbs8.zero -#guard (1 : ScalarField).val = Mont64x8Field.rModModulus scalarFieldSize +#guard (0 : ScalarField).val = Limbs4.zero +#guard (1 : ScalarField).val = Mont64x4Field.rModModulus scalarFieldSize -- Numeric literals reduce modulo the prime; `toNat` exits Montgomery form. #guard (37 : ScalarField).toNat = 37 @@ -72,12 +72,12 @@ set_option maxRecDepth 4000 = ((2 ^ 200 + 12345 : BLS12_377.ScalarField)⁻¹) #guard (987654321 : ScalarField).invGcd.toField = ((987654321 : BLS12_377.ScalarField)⁻¹) #guard (37 : ScalarField).invGcd.val - = gcdInvCandidate scalarFieldSize (Mont64x8Field.modulusLimbs scalarFieldSize) - (Mont64x8Field.montgomeryNegInv scalarFieldSize) (37 : ScalarField).val + = gcdInvCandidate scalarFieldSize (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (37 : ScalarField).val -- The Fermat fallback, exercised directly (the fast path never takes it). -#guard montPow (Mont64x8Field.modulusLimbs scalarFieldSize) - (Mont64x8Field.montgomeryNegInv scalarFieldSize) (Mont64x8Field.rModModulus scalarFieldSize) +#guard montPow (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (Mont64x4Field.rModModulus scalarFieldSize) (37 : ScalarField).val (scalarFieldSize - 2) = ((37 : ScalarField)⁻¹).val diff --git a/tests/CompPolyTests/Fields/BLS12_381/Fast.lean b/tests/CompPolyTests/Fields/BLS12_381/Fast.lean index b722f5d0..3bd855f6 100644 --- a/tests/CompPolyTests/Fields/BLS12_381/Fast.lean +++ b/tests/CompPolyTests/Fields/BLS12_381/Fast.lean @@ -10,7 +10,7 @@ public meta import CompPoly.Fields.BLS12_381.Fast /-! # Fast BLS12-381 Scalar Field Tests -Regression checks for the executable eight-limb Montgomery representation: the stored +Regression checks for the executable four-limb Montgomery representation: the stored residues, literal round trips, the field operations, the checked binary-GCD inversion, and agreement with the canonical `BLS12_381.ScalarField` model through `toField`. -/ @@ -20,13 +20,13 @@ public meta section namespace BLS12_381.Fast open BLS12_381 (scalarFieldSize) -open Montgomery.Native64x8 +open Montgomery.Native64x4 set_option maxRecDepth 4000 -- Stored Montgomery residues. -#guard (0 : ScalarField).val = Limbs8.zero -#guard (1 : ScalarField).val = Mont64x8Field.rModModulus scalarFieldSize +#guard (0 : ScalarField).val = Limbs4.zero +#guard (1 : ScalarField).val = Mont64x4Field.rModModulus scalarFieldSize -- Numeric literals reduce modulo the prime; `toNat` exits Montgomery form. #guard (37 : ScalarField).toNat = 37 @@ -72,12 +72,12 @@ set_option maxRecDepth 4000 = ((2 ^ 200 + 12345 : BLS12_381.ScalarField)⁻¹) #guard (987654321 : ScalarField).invGcd.toField = ((987654321 : BLS12_381.ScalarField)⁻¹) #guard (37 : ScalarField).invGcd.val - = gcdInvCandidate scalarFieldSize (Mont64x8Field.modulusLimbs scalarFieldSize) - (Mont64x8Field.montgomeryNegInv scalarFieldSize) (37 : ScalarField).val + = gcdInvCandidate scalarFieldSize (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (37 : ScalarField).val -- The Fermat fallback, exercised directly (the fast path never takes it). -#guard montPow (Mont64x8Field.modulusLimbs scalarFieldSize) - (Mont64x8Field.montgomeryNegInv scalarFieldSize) (Mont64x8Field.rModModulus scalarFieldSize) +#guard montPow (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (Mont64x4Field.rModModulus scalarFieldSize) (37 : ScalarField).val (scalarFieldSize - 2) = ((37 : ScalarField)⁻¹).val diff --git a/tests/CompPolyTests/Fields/BN254/Fast.lean b/tests/CompPolyTests/Fields/BN254/Fast.lean index 9a512db4..2dcd53ca 100644 --- a/tests/CompPolyTests/Fields/BN254/Fast.lean +++ b/tests/CompPolyTests/Fields/BN254/Fast.lean @@ -10,7 +10,7 @@ public meta import CompPoly.Fields.BN254.Fast /-! # Fast BN254 Scalar Field Tests -Regression checks for the executable eight-limb Montgomery representation: the stored +Regression checks for the executable four-limb Montgomery representation: the stored residues, literal round trips, the field operations, the checked binary-GCD inversion, and agreement with the canonical `BN254.ScalarField` model through `toField`. -/ @@ -20,13 +20,13 @@ public meta section namespace BN254.Fast open BN254 (scalarFieldSize) -open Montgomery.Native64x8 +open Montgomery.Native64x4 set_option maxRecDepth 4000 -- Stored Montgomery residues. -#guard (0 : ScalarField).val = Limbs8.zero -#guard (1 : ScalarField).val = Mont64x8Field.rModModulus scalarFieldSize +#guard (0 : ScalarField).val = Limbs4.zero +#guard (1 : ScalarField).val = Mont64x4Field.rModModulus scalarFieldSize -- Numeric literals reduce modulo the prime; `toNat` exits Montgomery form. #guard (37 : ScalarField).toNat = 37 @@ -71,13 +71,19 @@ set_option maxRecDepth 4000 #guard ((2 ^ 200 + 12345 : ScalarField).invGcd).toField = ((2 ^ 200 + 12345 : BN254.ScalarField)⁻¹) #guard (987654321 : ScalarField).invGcd.toField = ((987654321 : BN254.ScalarField)⁻¹) #guard (37 : ScalarField).invGcd.val - = gcdInvCandidate scalarFieldSize (Mont64x8Field.modulusLimbs scalarFieldSize) - (Mont64x8Field.montgomeryNegInv scalarFieldSize) (37 : ScalarField).val + = gcdInvCandidate scalarFieldSize (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (37 : ScalarField).val -- The Fermat fallback, exercised directly (the fast path never takes it). -#guard montPow (Mont64x8Field.modulusLimbs scalarFieldSize) - (Mont64x8Field.montgomeryNegInv scalarFieldSize) (Mont64x8Field.rModModulus scalarFieldSize) +#guard montPow (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (Mont64x4Field.rModModulus scalarFieldSize) (37 : ScalarField).val (scalarFieldSize - 2) = ((37 : ScalarField)⁻¹).val +-- Signed-word divsteps agree with the integer kernel at carry/sign boundaries. +#guard ([0, 1, 0x8000000000000000, 0xffffffffffffffff] : List UInt64).all fun a ↦ + ([1, 0x7fffffffffffffff, 0xffffffffffffffff] : List UInt64).all fun b ↦ + Montgomery.Native64x8.gcdInnerFast 31 a b 1 0 0 1 == + Montgomery.Native64x8.gcdInner 31 a b 1 0 0 1 + end BN254.Fast diff --git a/tests/CompPolyTests/Fields/BabyBear/FastExt4.lean b/tests/CompPolyTests/Fields/BabyBear/FastExt4.lean new file mode 100644 index 00000000..450f7b6f --- /dev/null +++ b/tests/CompPolyTests/Fields/BabyBear/FastExt4.lean @@ -0,0 +1,58 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public meta import CompPoly.Fields.BabyBear.FastExt4 + +/-! +# Fast BabyBear quartic extension tests + +Compiled checks of `BabyBear.Fast.Ext4`: known answers computed with Plonky3's +`BinomialExtensionField` (`p3-field` 0.4.2), agreement with the spec field +`Ext4.Spec` over the fast base field, and the byte codec. +-/ + +public meta section + +namespace CompPolyTests.Fields.BabyBear.FastExt4 + +open CompPoly BabyBear.Fast Montgomery.Native32 + +/-- An element from canonical coordinates. -/ +def mk (a b c d : Nat) : Ext4 := + ⟨ofField (a : BabyBear.Field), ofField (b : BabyBear.Field), ofField (c : BabyBear.Field), + ofField (d : BabyBear.Field)⟩ + +/-- Canonical coordinates. -/ +def coords (x : Ext4) : List Nat := + [x.c0, x.c1, x.c2, x.c3].map fun c ↦ (FastField.toField c).val + +def a : Ext4 := mk 1 2 3 4 +def b : Ext4 := mk 2013265920 123456789 987654321 5 + +-- Known answers from Plonky3. +#guard coords (a * b) == [1785904840, 1301662735, 1234568116, 332413089] +#guard coords a⁻¹ == [1587469345, 920666518, 1160282443, 647153706] +#guard coords b⁻¹ == [1787456803, 987985264, 1366459329, 1869413295] + +-- Field laws on the fast carrier. +#guard a * a⁻¹ == 1 +#guard b * b⁻¹ == 1 +#guard (0 : Ext4)⁻¹ == 0 +#guard (a + b) * b == a * b + b * b +#guard (a - b) * a == a * a - b * a +#guard a / b * b == a + +-- Agreement with the spec field, including its Fermat inverse. +#guard Ext4.toSpec (a * b) == Ext4.toSpec a * Ext4.toSpec b +#guard Ext4.toSpec a⁻¹ == (Ext4.toSpec a)⁻¹ +#guard Ext4.ofSpec (Ext4.toSpec b) == b + +-- The codec round-trips and matches the spec field's encoding. +#guard ByteCodec.ofBytes? (ByteCodec.toBytes b) == some b +#guard (ByteCodec.toBytes (Ext4.toSpec b)).toList == (ByteCodec.toBytes b).toList + +end CompPolyTests.Fields.BabyBear.FastExt4 diff --git a/tests/CompPolyTests/Fields/Goldilocks/Fast.lean b/tests/CompPolyTests/Fields/Goldilocks/Fast.lean index 5d4c5cf1..8b0e8f17 100644 --- a/tests/CompPolyTests/Fields/Goldilocks/Fast.lean +++ b/tests/CompPolyTests/Fields/Goldilocks/Fast.lean @@ -25,6 +25,12 @@ private def p : Nat := Goldilocks.fieldSize #guard toNat (ofNat p) = 0 #guard toNat (ofNat (p + 73)) = 73 #guard toNat (ofUInt64 (UInt64.ofNat (UInt64.size - 1))) = 4294967294 +-- Unreduced words are the same element as their residue. +#guard (ofUInt64 0xFFFFFFFF00000001 : Field) = 0 +#guard (ofUInt64 0xFFFFFFFF00000002 : Field) = 1 +#guard (ofUInt64 0xFFFFFFFF00000002 : Field) * (7 : Field) = 7 +#guard toNat ((ofUInt64 0xFFFFFFFFFFFFFFFF : Field) + (ofUInt64 0xFFFFFFFFFFFFFFFF : Field)) + = 2 * (2 ^ 32 - 2) #guard toNat ((ofNat (p - 1)) + (4 : Field)) = 3 #guard toNat ((ofNat (p - 1)) + (ofNat (p - 1))) = p - 2 #guard toNat ((17 : Field) - (6 : Field)) = 11 diff --git a/tests/CompPolyTests/Fields/KoalaBear/FastExt4.lean b/tests/CompPolyTests/Fields/KoalaBear/FastExt4.lean new file mode 100644 index 00000000..fdb5890d --- /dev/null +++ b/tests/CompPolyTests/Fields/KoalaBear/FastExt4.lean @@ -0,0 +1,58 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public meta import CompPoly.Fields.KoalaBear.FastExt4 + +/-! +# Fast KoalaBear quartic extension tests + +Compiled checks of `KoalaBear.Fast.Ext4`: known answers computed with Plonky3's +`BinomialExtensionField` (`p3-field` 0.4.2), agreement with the spec field +`Ext4.Spec` over the fast base field, and the byte codec. +-/ + +public meta section + +namespace CompPolyTests.Fields.KoalaBear.FastExt4 + +open CompPoly KoalaBear.Fast Montgomery.Native32 + +/-- An element from canonical coordinates. -/ +def mk (a b c d : Nat) : Ext4 := + ⟨ofField (a : KoalaBear.Field), ofField (b : KoalaBear.Field), ofField (c : KoalaBear.Field), + ofField (d : KoalaBear.Field)⟩ + +/-- Canonical coordinates. -/ +def coords (x : Ext4) : List Nat := + [x.c0, x.c1, x.c2, x.c3].map fun c ↦ (FastField.toField c).val + +def a : Ext4 := mk 1 2 3 4 +def b : Ext4 := mk 2130706432 123456789 987654321 5 + +-- Known answers from Plonky3. +#guard coords (a * b) == [1847544654, 1321776519, 1234567956, 214972577] +#guard coords a⁻¹ == [476435702, 408373459, 502227710, 126094261] +#guard coords b⁻¹ == [1900592659, 1697951523, 2029979453, 1087580901] + +-- Field laws on the fast carrier. +#guard a * a⁻¹ == 1 +#guard b * b⁻¹ == 1 +#guard (0 : Ext4)⁻¹ == 0 +#guard (a + b) * b == a * b + b * b +#guard (a - b) * a == a * a - b * a +#guard a / b * b == a + +-- Agreement with the spec field, including its Fermat inverse. +#guard Ext4.toSpec (a * b) == Ext4.toSpec a * Ext4.toSpec b +#guard Ext4.toSpec a⁻¹ == (Ext4.toSpec a)⁻¹ +#guard Ext4.ofSpec (Ext4.toSpec b) == b + +-- The codec round-trips and matches the spec field's encoding. +#guard ByteCodec.ofBytes? (ByteCodec.toBytes b) == some b +#guard (ByteCodec.toBytes (Ext4.toSpec b)).toList == (ByteCodec.toBytes b).toList + +end CompPolyTests.Fields.KoalaBear.FastExt4 diff --git a/tests/CompPolyTests/Fields/KoalaBear/InterpolateCoset.lean b/tests/CompPolyTests/Fields/KoalaBear/InterpolateCoset.lean new file mode 100644 index 00000000..4560eb2b --- /dev/null +++ b/tests/CompPolyTests/Fields/KoalaBear/InterpolateCoset.lean @@ -0,0 +1,56 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public meta import CompPoly.Fields.KoalaBear.InterpolateCoset + +/-! +# Coset interpolation tests + +Evaluate two polynomials of degree below 8 on the coset `3 · ⟨ω⟩` of the order-8 subgroup, run +`interpolateCoset`, and compare with Horner evaluation of the same coefficients at +`z` in the spec field `Ext4.Spec`. +-/ + +public meta section + +namespace CompPolyTests.Fields.KoalaBear.InterpolateCoset + +open CompPoly KoalaBear.Fast Montgomery.Native32 + +def logN : Nat := 3 +def n : Nat := 2 ^ logN +def ω : Field := KoalaBear.Fast.twoAdicGenerators.getD logN 0 +def s : Field := ofField 3 +def z : Ext4 := ⟨ofField 5, ofField 6, ofField 7, ofField 8⟩ + +/-- Coefficients of the two column polynomials, low degree first. -/ +def coeffs : Array (Array Nat) := + #[#[1, 2, 3, 4, 5, 6, 7, 8], #[2130706432, 0, 123456789, 42, 0, 0, 987654321, 5]] + +def hornerBase (c : Array Nat) (x : Field) : Field := + c.foldr (fun (a : Nat) acc ↦ acc * x + ofField (a : KoalaBear.Field)) 0 + +/-- Row-major values: row `i` holds both polynomials at `s ωⁱ`. -/ +def evals : Array Field := Id.run do + let mut out := #[] + for i in [:n] do + let x := s * Montgomery.Native32.pow ω i + for c in coeffs do + out := out.push (hornerBase c x) + return out + +def hornerSpec (c : Array Nat) (x : Ext4.Spec KoalaBear.fieldSize) : + Ext4.Spec KoalaBear.fieldSize := + c.foldr (fun (a : Nat) acc ↦ acc * x + CompPoly.Extension.Ext.ofBase (ofField a)) 0 + +def result : Array Ext4 := interpolateCoset logN ω s 2 evals z + +#guard result.size == 2 +#guard Ext4.toSpec result[0]! == hornerSpec coeffs[0]! (Ext4.toSpec z) +#guard Ext4.toSpec result[1]! == hornerSpec coeffs[1]! (Ext4.toSpec z) + +end CompPolyTests.Fields.KoalaBear.InterpolateCoset diff --git a/tests/CompPolyTests/Fields/Montgomery/Bytes.lean b/tests/CompPolyTests/Fields/Montgomery/Bytes.lean index bb042bc6..cd5a08f6 100644 --- a/tests/CompPolyTests/Fields/Montgomery/Bytes.lean +++ b/tests/CompPolyTests/Fields/Montgomery/Bytes.lean @@ -6,13 +6,13 @@ Authors: Derek Sorensen module public meta import CompPoly.Fields.Montgomery.Native32Bytes -public meta import CompPoly.Fields.Montgomery.Native64x8Bytes +public meta import CompPoly.Fields.Montgomery.Native64x4Bytes public meta import CompPoly.Fields.BabyBear.Fast public meta import CompPoly.Fields.KoalaBear.Fast public meta import CompPoly.Fields.BN254.Fast public meta import CompPoly.Fields.BLS12_381.Fast public import CompPoly.Fields.Montgomery.Native32Bytes -public import CompPoly.Fields.Montgomery.Native64x8Bytes +public import CompPoly.Fields.Montgomery.Native64x4Bytes public import CompPoly.Fields.BabyBear.Fast public import CompPoly.Fields.KoalaBear.Fast public import CompPoly.Fields.BN254.Fast @@ -79,7 +79,7 @@ example (n : BabyBear.Field) : Montgomery.Native32.FastField.toBytes_ofField n example (n : BN254.ScalarField) : ByteCodec.toBytes (BN254.Fast.ofField n) = ByteCodec.toBytes n := - Montgomery.Native64x8.FastField.toBytes_ofField n + Montgomery.Native64x4.FastField.toBytes_ofField n example (x : BabyBear.Fast.Field) : (Deserialize.deserialize (serialize x : ByteArray) : BabyBear.Fast.Field) = x := CanonicalNat.deserialize_serialize_byteArray rfl x diff --git a/tests/CompPolyTests/Fields/Montgomery/InterpolateCosetPacked.lean b/tests/CompPolyTests/Fields/Montgomery/InterpolateCosetPacked.lean new file mode 100644 index 00000000..7024ab9f --- /dev/null +++ b/tests/CompPolyTests/Fields/Montgomery/InterpolateCosetPacked.lean @@ -0,0 +1,48 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Fields.BabyBear.InterpolateCoset +public import CompPoly.Fields.KoalaBear.InterpolateCoset +public import CompPoly.Univariate.NTT.BabyBear +public import CompPoly.Univariate.NTT.KoalaBear + +/-! # Packed coset interpolation over KoalaBear and BabyBear + +`interpolateCosetPacked` is generic over `Mont32Field p` with a quartic binomial extension; these +checks run its compiled KoalaBear and BabyBear instances against the reference +`interpolateCoset` on `2 ^ 10` rows and five columns, which exercises the four-column and +single-column sums and two tasks. The packed words go through the inline C of the `ByteArray` +word accessors, so the checks run compiled, in `lake exe CompPolyNativeSmoke`. +-/ + +public section +namespace CompPolyTests.Fields.Montgomery.InterpolateCosetPacked + +open CompPoly Montgomery.Native32 + +/-- Packed and reference interpolation agree on a deterministic matrix. -/ +def agrees (p : ℕ) [Mont32Field p] [Binomial4 p] [PackedInterpolation p] (ω : FastField p) : + Bool := + let logN := 10 + let width := 5 + let values : Array (FastField p) := Array.ofFn fun i : Fin (2 ^ logN * width) ↦ + ((i.val * 7919 + 13) ^ 3 : ℕ) + let words := PackedRepr.encode values + let s : FastField p := 3 + let z : Ext4 p := ⟨5, 6, 7, 8⟩ + interpolateCosetPacked logN ω s width words z == + interpolateCoset logN ω s width (decodeWords p words) z + +/-- Check the KoalaBear and BabyBear instances. -/ +def run : IO Unit := do + unless agrees _ (CPolynomial.NTT.KoalaBear.fastDomainOfLogN 10 (by decide)).omega do + throw <| IO.userError "packed interpolation mismatch: KoalaBear" + unless agrees _ (CPolynomial.NTT.BabyBear.fastDomainOfLogN 10 (by decide)).omega do + throw <| IO.userError "packed interpolation mismatch: BabyBear" + IO.println "packed coset interpolation: KoalaBear and BabyBear checks passed" + +end CompPolyTests.Fields.Montgomery.InterpolateCosetPacked diff --git a/tests/CompPolyTests/Fields/Pasta/Fast.lean b/tests/CompPolyTests/Fields/Pasta/Fast.lean index 5ea4a2b7..315274bf 100644 --- a/tests/CompPolyTests/Fields/Pasta/Fast.lean +++ b/tests/CompPolyTests/Fields/Pasta/Fast.lean @@ -10,7 +10,7 @@ public meta import CompPoly.Fields.Pasta /-! # Fast Pasta Field Tests -Regression checks for the raw eight-limb operations and the Pallas/Vesta fast-field +Regression checks for the raw four-limb operations and the Pallas/Vesta fast-field instantiations against externally computed values. -/ @@ -18,48 +18,43 @@ public meta section namespace CompPolyTests.Fields.Pasta -open _root_.Montgomery.Native64x8 +open _root_.Montgomery.Native64x4 -private def q : Limbs8 := Vesta.Fast.instMont64x8Field.modulusLimbs +private def q : Limbs4 := Vesta.Fast.instMont64x4Field.modulusLimbs -private def negInv : UInt64 := Vesta.Fast.instMont64x8Field.montgomeryNegInv +private def negInv : UInt64 := Vesta.Fast.instMont64x4Field.montgomeryNegInv -private def rMod : Limbs8 := Vesta.Fast.instMont64x8Field.rModModulus +private def rMod : Limbs4 := Vesta.Fast.instMont64x4Field.rModModulus private def modulus : ℕ := Vesta.baseFieldSize -private def a : Limbs8 := - ⟨0x90abcdef, 0x12345678, 0x90abcdef, 0x12345678, 0x90abcdef, 0x12345678, 0x90abcdef, - 0x12345678⟩ +private def a : Limbs4 := + ⟨0x1234567890abcdef, 0x1234567890abcdef, 0x1234567890abcdef, 0x1234567890abcdef⟩ -private def b : Limbs8 := - ⟨0x98765432, 0x1fedcba0, 0x98765432, 0x1fedcba0, 0x98765432, 0x1fedcba0, 0x98765432, - 0xfedcba0⟩ +private def b : Limbs4 := + ⟨0x1fedcba098765432, 0x1fedcba098765432, 0x1fedcba098765432, 0xfedcba098765432⟩ -private def montA : Limbs8 := - ⟨0x4657fff9, 0xb89185c2, 0x7fa63616, 0xac5206fd, 0xca71bcaa, 0x4a1382e8, 0xd2416c1e, - 0x3ec0c11b⟩ +private def montA : Limbs4 := + ⟨0xb89185c24657fff9, 0xac5206fd7fa63616, 0x4a1382e8ca71bcaa, 0x3ec0c11bd2416c1e⟩ -private def montB : Limbs8 := - ⟨0x96717045, 0xa13e865, 0xeda4684, 0x5fa53078, 0xadca6010, 0x8c0bc7ff, 0x6af4be5b, - 0x2bde1e93⟩ +private def montB : Limbs4 := + ⟨0xa13e86596717045, 0x5fa530780eda4684, 0x8c0bc7ffadca6010, 0x2bde1e936af4be5b⟩ -private def montAB : Limbs8 := - ⟨0x5bfbc390, 0xa6073176, 0x8f8933a9, 0xbed1aec1, 0x630437f1, 0xba615782, 0x92dc822b, - 0x77a4b3f⟩ +private def montAB : Limbs4 := + ⟨0xa60731765bfbc390, 0xbed1aec18f8933a9, 0xba615782630437f1, 0x77a4b3f92dc822b⟩ #guard q.toNat = modulus -#guard condSub q q = Limbs8.zero -#guard condSub q Limbs8.zero = Limbs8.zero +#guard condSub q q = Limbs4.zero +#guard condSub q Limbs4.zero = Limbs4.zero #guard (add q a b).toNat = (a.toNat + b.toNat) % modulus -#guard add q a (neg q a) = Limbs8.zero +#guard add q a (neg q a) = Limbs4.zero #guard (sub q a b).toNat = (a.toNat + (modulus - b.toNat)) % modulus #guard (sub q b a).toNat = (b.toNat + (modulus - a.toNat)) % modulus #guard (neg q a).toNat = (modulus - a.toNat) % modulus -#guard neg q Limbs8.zero = Limbs8.zero +#guard neg q Limbs4.zero = Limbs4.zero #guard mul q negInv montA montB = montAB #guard mul q negInv rMod rMod = rMod -#guard mul q negInv rMod Limbs8.zero = Limbs8.zero +#guard mul q negInv rMod Limbs4.zero = Limbs4.zero #guard square q negInv rMod = rMod private abbrev F := Vesta.Fast.Field @@ -77,26 +72,23 @@ private abbrev F := Vesta.Fast.Field #guard ((37 : F) / 37).toNat = 1 #guard ((0 : F)⁻¹).toNat = 0 -private def pq : Limbs8 := Pallas.Fast.instMont64x8Field.modulusLimbs +private def pq : Limbs4 := Pallas.Fast.instMont64x4Field.modulusLimbs -private def pNegInv : UInt64 := Pallas.Fast.instMont64x8Field.montgomeryNegInv +private def pNegInv : UInt64 := Pallas.Fast.instMont64x4Field.montgomeryNegInv -private def pMontA : Limbs8 := - ⟨0x73c65f1d, 0x4e0a938e, 0xd71d5fef, 0x6a1193b, 0xe42540dc, 0xcb40dac3, 0xc6cffd09, - 0x335fa2c8⟩ +private def pMontA : Limbs4 := + ⟨0x4e0a938e73c65f1d, 0x6a1193bd71d5fef, 0xcb40dac3e42540dc, 0x335fa2c8c6cffd09⟩ -private def pMontB : Limbs8 := - ⟨0x5153d947, 0x26211be3, 0x38e82d52, 0xcb949678, 0xb6f80c7e, 0xf369686f, 0x7c2907b8, - 0xd91e296⟩ +private def pMontB : Limbs4 := + ⟨0x26211be35153d947, 0xcb94967838e82d52, 0xf369686fb6f80c7e, 0xd91e2967c2907b8⟩ -private def pMontAB : Limbs8 := - ⟨0x8d6b3f08, 0xe5c0528e, 0x22646fad, 0xaec941cc, 0xfec4a4c0, 0x4f5e09aa, 0x69b735b5, - 0x179b8aa1⟩ +private def pMontAB : Limbs4 := + ⟨0xe5c0528e8d6b3f08, 0xaec941cc22646fad, 0x4f5e09aafec4a4c0, 0x179b8aa169b735b5⟩ #guard pq.toNat = Pallas.baseFieldSize #guard mul pq pNegInv pMontA pMontB = pMontAB #guard (add pq a b).toNat = (a.toNat + b.toNat) % Pallas.baseFieldSize -#guard condSub pq pq = Limbs8.zero +#guard condSub pq pq = Limbs4.zero private abbrev G := Pallas.Fast.Field diff --git a/tests/CompPolyTests/Fields/Secp256k1/Fast.lean b/tests/CompPolyTests/Fields/Secp256k1/Fast.lean new file mode 100644 index 00000000..2e1e7bfb --- /dev/null +++ b/tests/CompPolyTests/Fields/Secp256k1/Fast.lean @@ -0,0 +1,163 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Georgios Raikos +-/ +module + +public meta import CompPoly.Fields.Secp256k1.Fast + +/-! +# Fast secp256k1 Field Tests + +Regression checks for the four-limb Montgomery scalar and base fields, whose moduli have +bit 255 set: residues, literal round trips, field operations including the carry paths, +the checked binary-GCD inversion, and agreement with the canonical `ZMod` models. +-/ + +public meta section + +namespace Secp256k1.Fast + +open Secp256k1 (scalarFieldSize baseFieldSize) +open Montgomery.Native64x4 + +set_option maxRecDepth 4000 + +/-! ## Scalar field -/ + +-- Stored Montgomery residues. +#guard (0 : ScalarField).val = Limbs4.zero +#guard (1 : ScalarField).val = Mont64x4Field.rModModulus scalarFieldSize + +-- Numeric literals reduce modulo the prime; `toNat` exits Montgomery form. +#guard (37 : ScalarField).toNat = 37 +#guard (scalarFieldSize : ScalarField).toNat = 0 +#guard (scalarFieldSize + 37 : ScalarField).toNat = 37 + +-- Addition (with and without wraparound). +#guard ((scalarFieldSize - 1 : ScalarField) + 2).toNat = 1 +#guard ((scalarFieldSize - 1 : ScalarField) + (scalarFieldSize - 1 : ScalarField)).toNat + = scalarFieldSize - 2 + +-- Subtraction (with and without borrow). +#guard ((9 : ScalarField) - 5).toNat = 4 +#guard ((5 : ScalarField) - 9).toNat = scalarFieldSize - 4 + +-- Negation. +#guard (-(0 : ScalarField)).toNat = 0 +#guard (-(1 : ScalarField)).toNat = scalarFieldSize - 1 + +-- Multiplication and squaring (`(-1) * (-1) = 1`). +#guard ((scalarFieldSize - 1 : ScalarField) * (scalarFieldSize - 1 : ScalarField)).toNat + = 1 +#guard ((12345 : ScalarField) * 12345).toNat = 152399025 +#guard ((scalarFieldSize - 2 : ScalarField) * (scalarFieldSize - 3 : ScalarField)).toField + = ((scalarFieldSize - 2 : Secp256k1.ScalarField) * (scalarFieldSize - 3)) +#guard ((2 ^ 255 + 12345 : ScalarField) * (2 ^ 255 + 67890 : ScalarField)).toField + = ((2 ^ 255 + 12345 : Secp256k1.ScalarField) * (2 ^ 255 + 67890)) + +-- Exponentiation, including agreement with the canonical field. +#guard ((37 : ScalarField) ^ 0).toNat = 1 +#guard ((37 : ScalarField) ^ 1).toNat = 37 +#guard ((123456789 : ScalarField) ^ 17).toField = ((123456789 : Secp256k1.ScalarField) ^ 17) +#guard ((123456789 : ScalarField) ^ 255).toField + = ((123456789 : Secp256k1.ScalarField) ^ 255) + +-- Fermat inversion and division (`0⁻¹ = 0`, `x⁻¹ * x = 1`, `x / x = 1`). +#guard ((0 : ScalarField)⁻¹).toNat = 0 +#guard ((37 : ScalarField)⁻¹ * 37).toNat = 1 +#guard ((37 : ScalarField) / 37).toNat = 1 +#guard ((37 : ScalarField)⁻¹).toField = ((37 : Secp256k1.ScalarField)⁻¹) +#guard ((37 : ScalarField) ^ (-3 : Int)).toField = ((37 : Secp256k1.ScalarField) ^ (-3 : Int)) + +-- The checked binary-GCD inversion agrees with the Fermat inverse; the raw-candidate guard +-- detects a silent fallback. +#guard ((37 : ScalarField).invGcd * 37).toNat = 1 +#guard ((scalarFieldSize - 1 : ScalarField).invGcd).toField + = ((scalarFieldSize - 1 : Secp256k1.ScalarField)⁻¹) +#guard ((2 ^ 200 + 12345 : ScalarField).invGcd).toField + = ((2 ^ 200 + 12345 : Secp256k1.ScalarField)⁻¹) +#guard (987654321 : ScalarField).invGcd.toField = ((987654321 : Secp256k1.ScalarField)⁻¹) +#guard (37 : ScalarField).invGcd.val + = gcdInvCandidate scalarFieldSize (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (37 : ScalarField).val +#guard (2 ^ 255 + 12345 : ScalarField).invGcd.val + = gcdInvCandidate scalarFieldSize (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) (2 ^ 255 + 12345 : ScalarField).val + +-- The Fermat fallback, exercised directly (the fast path never takes it). +#guard montPow (Mont64x4Field.modulusLimbs scalarFieldSize) + (Mont64x4Field.montgomeryNegInv scalarFieldSize) + (Mont64x4Field.rModModulus scalarFieldSize) + (37 : ScalarField).val (scalarFieldSize - 2) + = ((37 : ScalarField)⁻¹).val + +/-! ## Base field -/ + +-- Stored Montgomery residues. +#guard (0 : BaseField).val = Limbs4.zero +#guard (1 : BaseField).val = Mont64x4Field.rModModulus baseFieldSize + +-- Numeric literals reduce modulo the prime; `toNat` exits Montgomery form. +#guard (37 : BaseField).toNat = 37 +#guard (baseFieldSize : BaseField).toNat = 0 +#guard (baseFieldSize + 37 : BaseField).toNat = 37 + +-- Addition (with and without wraparound). +#guard ((baseFieldSize - 1 : BaseField) + 2).toNat = 1 +#guard ((baseFieldSize - 1 : BaseField) + (baseFieldSize - 1 : BaseField)).toNat + = baseFieldSize - 2 + +-- Subtraction (with and without borrow). +#guard ((9 : BaseField) - 5).toNat = 4 +#guard ((5 : BaseField) - 9).toNat = baseFieldSize - 4 + +-- Negation. +#guard (-(0 : BaseField)).toNat = 0 +#guard (-(1 : BaseField)).toNat = baseFieldSize - 1 + +-- Multiplication and squaring (`(-1) * (-1) = 1`). +#guard ((baseFieldSize - 1 : BaseField) * (baseFieldSize - 1 : BaseField)).toNat = 1 +#guard ((12345 : BaseField) * 12345).toNat = 152399025 +#guard ((baseFieldSize - 2 : BaseField) * (baseFieldSize - 3 : BaseField)).toField + = ((baseFieldSize - 2 : Secp256k1.BaseField) * (baseFieldSize - 3)) +#guard ((2 ^ 255 + 12345 : BaseField) * (2 ^ 255 + 67890 : BaseField)).toField + = ((2 ^ 255 + 12345 : Secp256k1.BaseField) * (2 ^ 255 + 67890)) + +-- Exponentiation, including agreement with the canonical field. +#guard ((37 : BaseField) ^ 0).toNat = 1 +#guard ((37 : BaseField) ^ 1).toNat = 37 +#guard ((123456789 : BaseField) ^ 17).toField = ((123456789 : Secp256k1.BaseField) ^ 17) +#guard ((123456789 : BaseField) ^ 255).toField = ((123456789 : Secp256k1.BaseField) ^ 255) + +-- Fermat inversion and division (`0⁻¹ = 0`, `x⁻¹ * x = 1`, `x / x = 1`). +#guard ((0 : BaseField)⁻¹).toNat = 0 +#guard ((37 : BaseField)⁻¹ * 37).toNat = 1 +#guard ((37 : BaseField) / 37).toNat = 1 +#guard ((37 : BaseField)⁻¹).toField = ((37 : Secp256k1.BaseField)⁻¹) +#guard ((37 : BaseField) ^ (-3 : Int)).toField = ((37 : Secp256k1.BaseField) ^ (-3 : Int)) + +-- The checked binary-GCD inversion agrees with the Fermat inverse; the raw-candidate guard +-- detects a silent fallback. +#guard ((37 : BaseField).invGcd * 37).toNat = 1 +#guard ((baseFieldSize - 1 : BaseField).invGcd).toField + = ((baseFieldSize - 1 : Secp256k1.BaseField)⁻¹) +#guard ((2 ^ 200 + 12345 : BaseField).invGcd).toField + = ((2 ^ 200 + 12345 : Secp256k1.BaseField)⁻¹) +#guard (987654321 : BaseField).invGcd.toField = ((987654321 : Secp256k1.BaseField)⁻¹) +#guard (37 : BaseField).invGcd.val + = gcdInvCandidate baseFieldSize (Mont64x4Field.modulusLimbs baseFieldSize) + (Mont64x4Field.montgomeryNegInv baseFieldSize) (37 : BaseField).val +#guard (2 ^ 255 + 12345 : BaseField).invGcd.val + = gcdInvCandidate baseFieldSize (Mont64x4Field.modulusLimbs baseFieldSize) + (Mont64x4Field.montgomeryNegInv baseFieldSize) (2 ^ 255 + 12345 : BaseField).val + +-- The Fermat fallback, exercised directly (the fast path never takes it). +#guard montPow (Mont64x4Field.modulusLimbs baseFieldSize) + (Mont64x4Field.montgomeryNegInv baseFieldSize) + (Mont64x4Field.rModModulus baseFieldSize) + (37 : BaseField).val (baseFieldSize - 2) + = ((37 : BaseField)⁻¹).val + +end Secp256k1.Fast diff --git a/tests/CompPolyTests/NTT/NativeStorage.lean b/tests/CompPolyTests/NTT/NativeStorage.lean new file mode 100644 index 00000000..97e5c098 --- /dev/null +++ b/tests/CompPolyTests/NTT/NativeStorage.lean @@ -0,0 +1,51 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Data.ByteArray.Pack +public import CompPoly.Data.Bytes.LittleEndian + +/-! # Compiled word-accessor agreement checks + +These exercise the inline C of the backported `ByteArray` word accessors, which is outside the +kernel proof. Bytewise Lean operations provide the oracle, including out-of-range offsets and +shared inputs. The packed FFT's batch stores and reads are Lean definitions over these accessors, +proved against the byte-level `PackedRepr.encode` specification. +-/ + +public section +namespace CompPolyTests.NTT.NativeStorage + +/-- Check the native word accessors against independent bytewise reads and writes. -/ +def run : IO Unit := do + let mut cases := 0 + for size in [0, 1, 3, 4, 5, 8, 64, 65] do + let source := (List.range size).foldl (fun b n ↦ b.push (n * 37 + 11).toUInt8) + (ByteArray.emptyWithCapacity size) + for off in [:size + 3] do + let fits := off + 4 ≤ size + let expected := if fits then + UInt32.ofNat (CompPoly.Bytes.ofListLE ((source.toList.drop off).take 4)) else 0 + if source.getUInt32LE! off != expected then + throw <| IO.userError s!"getUInt32LE! mismatch: {size}/{off}" + let v : UInt32 := 0x89abcdef + let written := if fits then + (List.range 4).foldl (fun b q ↦ b.set! (off + q) (v >>> (8 * q).toUInt32).toUInt8) source + else source + if source.setUInt32LE! off v != written then + throw <| IO.userError s!"setUInt32LE! mismatch: {size}/{off}" + if h : off + 4 ≤ source.size ∧ off < USize.size then + have hu : off.toUSize.toNat = off := USize.toNat_ofNat_of_lt' h.2 + if source.ugetUInt32LE off.toUSize (by rw [hu]; exact h.1) != expected then + throw <| IO.userError s!"ugetUInt32LE mismatch: {size}/{off}" + if source.usetUInt32LE off.toUSize v (by rw [hu]; exact h.1) != written then + throw <| IO.userError s!"usetUInt32LE mismatch: {size}/{off}" + if source.toList != (List.range size).map fun n ↦ (n * 37 + 11).toUInt8 then + throw <| IO.userError "word store mutated shared input" + cases := cases + 1 + IO.println s!"word accessor agreement: {cases} checks passed" + +end CompPolyTests.NTT.NativeStorage diff --git a/tests/CompPolyTests/NTT/Packed.lean b/tests/CompPolyTests/NTT/Packed.lean new file mode 100644 index 00000000..9b1e7ec9 --- /dev/null +++ b/tests/CompPolyTests/NTT/Packed.lean @@ -0,0 +1,92 @@ +/- +Copyright (c) 2026 CompPoly Contributors. All rights reserved. +Released under Apache 2.0 license as described in the file LICENSE. +Authors: Gregor Mitscha-Baude +-/ +module + +public import CompPoly.Univariate.NTTFast.Packed.CosetLdeCorrectness +public import CompPoly.Univariate.NTTFast.Packed.Instances +public import CompPoly.Univariate.NTT.BabyBear +public import CompPoly.Univariate.NTT.KoalaBear + +/-! # Packed transforms over KoalaBear and BabyBear + +The packed plan is generic over `Mont32Field p`; these checks run its compiled KoalaBear and +BabyBear instances. Small sizes are compared with the DFT specification, covering the scalar +leaf and the fused final layers; `2 ^ 15` points exercise the column-parallel path, against the +sequential leaf and through a round trip. The batched coset LDE of a `2 ^ 5 × 3` matrix with +blowup `2` is compared with its specification. The packed words go through the inline C of the +`ByteArray` word accessors, so the checks run compiled, in `lake exe CompPolyNativeSmoke`. +-/ + +public section +namespace CompPolyTests.NTT.Packed + +open CompPoly CompPoly.CPolynomial +open Montgomery.Native32 (Mont32Field FastField) +open CompPoly.CPolynomial.NTTFast.Packed (Plan PackedNTT LdePlan PackedCosetLde ldeSpec) +open CompPoly.PackedRepr (encode) + +/-- A deterministic input of `n` elements. -/ +def input (p : ℕ) [Mont32Field p] (n : ℕ) : Array (FastField p) := + Array.ofFn fun i : Fin n ↦ ((i.val * 7919 + 13) ^ 3 : ℕ) + +theorem hu {n : ℕ} (h : n ≤ 27) : 4 * 2 ^ n < USize.size := by + have := Nat.pow_le_pow_right (show 0 < 2 by decide) h + have := USize.le_size + omega + +/-- Forward and inverse packed transforms agree with the specification. -/ +def agreesWithSpec (p : ℕ) [Mont32Field p] [PackedNTT p] (D : NTT.Domain (FastField p)) + (h32 : D.logN ≤ 32) (hu : 4 * D.n < USize.size) : Bool := + let P := Plan.ofDomain D h32 hu + let a := input p D.n + P.forwardPacked (encode a) == encode (NTT.Forward.forwardSpec D a) && + P.inversePacked (encode a) == encode (NTT.Inverse.inverseSpec D a) + +/-- The column-parallel transform agrees with the sequential one and inverts. -/ +def parallelAgrees (p : ℕ) [Mont32Field p] [PackedNTT p] (D : NTT.Domain (FastField p)) + (h32 : D.logN ≤ 32) (hu : 4 * D.n < USize.size) : Bool := + let P := Plan.ofDomain D h32 hu + let a := encode (input p D.n) + let y := P.forwardPacked a true + y == P.forwardPacked a false && P.inversePacked y true == a + +/-- The batched coset LDE of a `2 ^ 5 × 3` matrix with blowup `2`, given whole and in four chunks +of rows, agrees with the specification. -/ +def ldeAgrees (p : ℕ) [Mont32Field p] [PackedCosetLde p] (D : NTT.Domain (FastField p)) + (hb : 1 ≤ D.logN) (h32 : D.logN ≤ 32) (hu : 4 * D.n < USize.size) : Bool := + let P := LdePlan.ofDomain D 1 hb 3 h32 hu + let M := input p (2 ^ P.logH * 3) + [0, 2].all fun k ↦ CPolynomial.NTTFast.Packed.Native.joinRows + (P.cosetLde 3 (CPolynomial.NTTFast.Packed.Native.rowChunks M 3 P.logH k)) == + encode (ldeSpec P 3 M) + +def koalaBear (logN : ℕ) (h : logN ≤ KoalaBear.twoAdicity) : NTT.Domain KoalaBear.Fast.Field := + NTT.KoalaBear.fastDomainOfLogN logN h + +def babyBear (logN : ℕ) (h : logN ≤ BabyBear.twoAdicity) : NTT.Domain BabyBear.Fast.Field := + NTT.BabyBear.fastDomainOfLogN logN h + +/-- Check the KoalaBear and BabyBear packed transforms. -/ +def run : IO Unit := do + let check (name : String) (ok : Bool) : IO Unit := + unless ok do throw <| IO.userError s!"packed NTT mismatch: {name}" + check "KoalaBear 2^3" (agreesWithSpec _ (koalaBear 3 (by decide)) (by decide) (hu (by decide))) + check "KoalaBear 2^8" (agreesWithSpec _ (koalaBear 8 (by decide)) (by decide) (hu (by decide))) + check "KoalaBear 2^9" (agreesWithSpec _ (koalaBear 9 (by decide)) (by decide) (hu (by decide))) + check "KoalaBear 2^15" + (parallelAgrees _ (koalaBear 15 (by decide)) (by decide) (hu (by decide))) + check "BabyBear 2^3" (agreesWithSpec _ (babyBear 3 (by decide)) (by decide) (hu (by decide))) + check "BabyBear 2^8" (agreesWithSpec _ (babyBear 8 (by decide)) (by decide) (hu (by decide))) + check "BabyBear 2^9" (agreesWithSpec _ (babyBear 9 (by decide)) (by decide) (hu (by decide))) + check "BabyBear 2^15" + (parallelAgrees _ (babyBear 15 (by decide)) (by decide) (hu (by decide))) + check "KoalaBear LDE" + (ldeAgrees _ (koalaBear 6 (by decide)) (by decide) (by decide) (hu (by decide))) + check "BabyBear LDE" + (ldeAgrees _ (babyBear 6 (by decide)) (by decide) (by decide) (hu (by decide))) + IO.println "packed NTT: KoalaBear and BabyBear checks passed" + +end CompPolyTests.NTT.Packed diff --git a/tests/README.md b/tests/README.md index 8a016905..26368b28 100644 --- a/tests/README.md +++ b/tests/README.md @@ -69,3 +69,7 @@ the highest bit, and the generic coefficient-evaluation entry point. A same-widt counterexample checks that the two field presentations remain distinct. The corresponding `CompPolyTests/Fields/Binary/Tower/ProductAccumulation.lean` regression also checks the universal refinement statements and coefficient ordering. + +### Packed FFT native storage + +`CompPolyTests/NTT/NativeStorage.lean` and `CompPolyTests/NTT/Packed.lean` run as part of `lake exe CompPolyNativeSmoke`. Its compiled comparisons exercise the inline C of the backported `ByteArray` word accessors (`CompPoly/Data/ByteArray/Pack.lean`) against independent bytewise Lean operations, at every offset including out-of-range ones, and check that shared inputs are not mutated. These are checks of the runtime trust boundary; the packed FFT's batch stores and reads are Lean definitions over the accessors, covered with the FFT mathematics by the kernel refinement theorems. `Packed.lean` runs the packed FFT's compiled KoalaBear and BabyBear instances against the DFT specification and its parallel path against the sequential one.