Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wip] feat: Halo2 compatible vanilla Hashers #1560

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+4,701 −2,582
Open

[wip] feat: Halo2 compatible vanilla Hashers #1560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6c28dd0
feat: add Pasta compatible hashers
DrPeterVanNostrand Feb 2, 2022
7edf855
feat: pasta gpu support
DrPeterVanNostrand Mar 3, 2022
ee726db
build: use pasta_curves version implementing ec_gpu::GpuField
DrPeterVanNostrand Mar 24, 2022
020b406
feat: upgrade bellperson (and neptune)
cryptonemo Apr 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ jobs:
cargo +$(cat rust-toolchain) test -p storage-proofs-update --features isolated-testing --release test_empty_sector_update_circuit_8kib
cargo +$(cat rust-toolchain) test -p storage-proofs-update --features isolated-testing --release test_empty_sector_update_circuit_16kib
cargo +$(cat rust-toolchain) test -p storage-proofs-update --features isolated-testing --release test_empty_sector_update_circuit_32kib
cargo +$(cat rust-toolchain) test -p storage-proofs-update --features isolated-testing --release test_empty_sector_update_poseidon_circuit_1kib
cargo +$(cat rust-toolchain) test -p storage-proofs-update --features isolated-testing --release test_empty_sector_update_poseidon_circuit_8kib
no_output_timeout: 30m

test_ignored_release:
Expand Down
7 changes: 4 additions & 3 deletions fil-proofs-param/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ serde = { version = "1.0", features = ["rc", "derive"] }
serde_json = "1.0"
ff = "0.11.0"
blake2b_simd = "0.5"
bellperson = "0.18.0"
bellperson = "0.19"
log = "0.4.7"
fil_logger = "0.1"
env_proxy = "0.4"
Expand All @@ -49,6 +49,7 @@ indicatif = "0.15.0"
group = "0.11.0"
dialoguer = "0.8.0"
clap = "2.33.3"
blstrs = { version = "0.4.0", features = ["gpu"] }

[dependencies.reqwest]
version = "0.10"
Expand All @@ -68,5 +69,5 @@ cpu-profile = ["gperftools"]
heap-profile = ["gperftools/heap"]
simd = ["storage-proofs-core/simd"]
asm = ["storage-proofs-core/asm"]
cuda = ["storage-proofs-core/cuda", "storage-proofs-porep/cuda", "storage-proofs-post/cuda", "storage-proofs-update/cuda", "bellperson/cuda"]
opencl = ["storage-proofs-core/opencl", "storage-proofs-porep/opencl", "storage-proofs-post/opencl", "storage-proofs-update/opencl", "bellperson/opencl"]
cuda = ["storage-proofs-core/cuda", "storage-proofs-porep/cuda", "storage-proofs-post/cuda", "storage-proofs-update/cuda"]
opencl = ["storage-proofs-core/opencl", "storage-proofs-porep/opencl", "storage-proofs-post/opencl", "storage-proofs-update/opencl"]
66 changes: 40 additions & 26 deletions fil-proofs-param/src/bin/paramcache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ use std::env;
use std::process::exit;
use std::str::FromStr;

use blstrs::Scalar as Fr;
use dialoguer::{theme::ColorfulTheme, MultiSelect};
use filecoin_hashers::{Domain, Hasher};
use filecoin_proofs::{
constants::{
DefaultPieceHasher, POREP_PARTITIONS, PUBLISHED_SECTOR_SIZES, WINDOW_POST_CHALLENGE_COUNT,
Expand All @@ -22,14 +24,14 @@ use storage_proofs_core::{
};
use storage_proofs_porep::stacked::{StackedCircuit, StackedCompound, StackedDrg};
use storage_proofs_post::fallback::{FallbackPoSt, FallbackPoStCircuit, FallbackPoStCompound};
use storage_proofs_update::constants::TreeRHasher;
use storage_proofs_update::{
circuit::EmptySectorUpdateCircuit, compound::EmptySectorUpdateCompound, EmptySectorUpdate,
PublicParams,
};
use storage_proofs_update::{constants::TreeRHasher, EmptySectorUpdateCompound};
use structopt::StructOpt;

fn cache_porep_params<Tree: 'static + MerkleTreeTrait>(porep_config: PoRepConfig) {
fn cache_porep_params<Tree>(porep_config: PoRepConfig)
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: Groth16 params can only be generated for the field Fr.

{
info!("generating PoRep groth params");

let public_params = public_params(
Expand Down Expand Up @@ -66,7 +68,11 @@ fn cache_porep_params<Tree: 'static + MerkleTreeTrait>(porep_config: PoRepConfig
.expect("failed to get verifying key");
}

fn cache_winning_post_params<Tree: 'static + MerkleTreeTrait>(post_config: &PoStConfig) {
fn cache_winning_post_params<Tree>(post_config: &PoStConfig)
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
{
info!("generating Winning-PoSt groth params");

let public_params = winning_post_public_params::<Tree>(post_config)
Expand All @@ -92,7 +98,11 @@ fn cache_winning_post_params<Tree: 'static + MerkleTreeTrait>(post_config: &PoSt
.expect("failed to get verifying key");
}

fn cache_window_post_params<Tree: 'static + MerkleTreeTrait>(post_config: &PoStConfig) {
fn cache_window_post_params<Tree>(post_config: &PoStConfig)
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
{
info!("generating Window-PoSt groth params");

let public_params = window_post_public_params::<Tree>(post_config)
Expand All @@ -118,32 +128,36 @@ fn cache_window_post_params<Tree: 'static + MerkleTreeTrait>(post_config: &PoStC
.expect("failed to get verifying key");
}

fn cache_empty_sector_update_params<Tree: 'static + MerkleTreeTrait<Hasher = TreeRHasher>>(
porep_config: PoRepConfig,
) {
fn cache_empty_sector_update_params<Tree>(porep_config: PoRepConfig)
where
Tree: 'static + MerkleTreeTrait<Hasher = TreeRHasher<Fr>>,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ESU can only use TreeRs whose hasher is TreeRHasher.

{
info!("generating EmptySectorUpdate groth params");

let public_params: storage_proofs_update::PublicParams =
PublicParams::from_sector_size(u64::from(porep_config.sector_size));
let public_params =
storage_proofs_update::PublicParams::from_sector_size(u64::from(porep_config.sector_size));

let circuit = <EmptySectorUpdateCompound<Tree> as CompoundProof<
EmptySectorUpdate<Tree>,
EmptySectorUpdateCircuit<Tree>,
>>::blank_circuit(&public_params);
let circuit = EmptySectorUpdateCompound::<
Tree::Arity,
Tree::SubTreeArity,
Tree::TopTreeArity,
>::blank_circuit(&public_params);

let _ = <EmptySectorUpdateCompound<Tree> as CompoundProof<
EmptySectorUpdate<Tree>,
EmptySectorUpdateCircuit<Tree>,
>>::groth_params::<OsRng>(Some(&mut OsRng), &public_params)
let _ = EmptySectorUpdateCompound::<
Tree::Arity,
Tree::SubTreeArity,
Tree::TopTreeArity,
>::groth_params(Some(&mut OsRng), &public_params)
.expect("failed to get groth params");

let _ = <EmptySectorUpdateCompound<Tree>>::get_param_metadata(circuit, &public_params)
let _ = EmptySectorUpdateCompound::get_param_metadata(circuit, &public_params)
.expect("failed to get metadata");

let _ = <EmptySectorUpdateCompound<Tree> as CompoundProof<
EmptySectorUpdate<Tree>,
EmptySectorUpdateCircuit<Tree>,
>>::verifying_key::<OsRng>(Some(&mut OsRng), &public_params)
let _ = EmptySectorUpdateCompound::<
Tree::Arity,
Tree::SubTreeArity,
Tree::TopTreeArity,
>::verifying_key(Some(&mut OsRng), &public_params)
.expect("failed to get verifying key");
}

Expand Down
4 changes: 2 additions & 2 deletions fil-proofs-tooling/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ regex = "1.3.7"
commandspec = "0.12.2"
chrono = { version = "0.4.7", features = ["serde"] }
memmap = "0.7.0"
bellperson = "0.18.0"
bellperson = "0.19"
rand = "0.8"
tempfile = "3.0.8"
cpu-time = "1.0.0"
Expand All @@ -53,7 +53,7 @@ fdlimit = "0.2.0"
dialoguer = "0.8.0"
structopt = "0.3.12"
humansize = "1.1.0"
blstrs = "0.4.0"
blstrs = { version = "0.4.0", features = ["gpu"] }

[features]
default = ["opencl", "measurements"]
Expand Down
22 changes: 13 additions & 9 deletions fil-proofs-tooling/src/bin/benchy/prodbench.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,11 @@ fn measure_porep_circuit(i: &ProdbenchInputs) -> usize {
api_version: i.api_version(),
};

let pp = StackedDrg::<ProdbenchTree, Sha256Hasher>::setup(&sp).expect("failed to setup DRG");
let pp =
StackedDrg::<ProdbenchTree, Sha256Hasher<Fr>>::setup(&sp).expect("failed to setup DRG");

let mut cs = BenchCS::<Fr>::new();
<StackedCompound<_, _> as CompoundProof<StackedDrg<ProdbenchTree, Sha256Hasher>, _>>::blank_circuit(
<StackedCompound<_, _> as CompoundProof<StackedDrg<ProdbenchTree, Sha256Hasher<Fr>>, _>>::blank_circuit(
&pp,
)
.synthesize(&mut cs)
Expand Down Expand Up @@ -333,18 +334,21 @@ fn cache_porep_params(porep_config: PoRepConfig) {

{
let circuit = <StackedCompound<ProdbenchTree, _> as CompoundProof<
StackedDrg<ProdbenchTree, Sha256Hasher>,
StackedDrg<ProdbenchTree, Sha256Hasher<Fr>>,
_,
>>::blank_circuit(&public_params);
StackedCompound::<ProdbenchTree, Sha256Hasher>::get_param_metadata(circuit, &public_params)
.expect("cannot get param metadata");
StackedCompound::<ProdbenchTree, Sha256Hasher<Fr>>::get_param_metadata(
circuit,
&public_params,
)
.expect("cannot get param metadata");
}
{
let circuit = <StackedCompound<ProdbenchTree, _> as CompoundProof<
StackedDrg<ProdbenchTree, Sha256Hasher>,
StackedDrg<ProdbenchTree, Sha256Hasher<Fr>>,
_,
>>::blank_circuit(&public_params);
StackedCompound::<ProdbenchTree, Sha256Hasher>::get_groth_params(
StackedCompound::<ProdbenchTree, Sha256Hasher<Fr>>::get_groth_params(
Some(&mut XorShiftRng::from_seed(SEED)),
circuit,
&public_params,
Expand All @@ -353,11 +357,11 @@ fn cache_porep_params(porep_config: PoRepConfig) {
}
{
let circuit = <StackedCompound<ProdbenchTree, _> as CompoundProof<
StackedDrg<ProdbenchTree, Sha256Hasher>,
StackedDrg<ProdbenchTree, Sha256Hasher<Fr>>,
_,
>>::blank_circuit(&public_params);

StackedCompound::<ProdbenchTree, Sha256Hasher>::get_verifying_key(
StackedCompound::<ProdbenchTree, Sha256Hasher<Fr>>::get_verifying_key(
Some(&mut XorShiftRng::from_seed(SEED)),
circuit,
&public_params,
Expand Down
18 changes: 14 additions & 4 deletions fil-proofs-tooling/src/bin/benchy/window_post.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ use std::time::{SystemTime, UNIX_EPOCH};

use anyhow::{ensure, Context};
use bincode::{deserialize, serialize};
use blstrs::Scalar as Fr;
use fil_proofs_tooling::measure::FuncMeasurement;
use fil_proofs_tooling::shared::{PROVER_ID, RANDOMNESS, TICKET_BYTES};
use fil_proofs_tooling::{measure, Metadata};
use filecoin_hashers::{Domain, Hasher};
use filecoin_proofs::constants::{
POREP_PARTITIONS, WINDOW_POST_CHALLENGE_COUNT, WINDOW_POST_SECTOR_COUNT,
};
Expand Down Expand Up @@ -96,15 +98,19 @@ fn get_porep_config(sector_size: u64, api_version: ApiVersion) -> PoRepConfig {
}
}

fn run_pre_commit_phases<Tree: 'static + MerkleTreeTrait>(
fn run_pre_commit_phases<Tree>(
sector_size: u64,
api_version: ApiVersion,
cache_dir: PathBuf,
skip_precommit_phase1: bool,
skip_precommit_phase2: bool,
test_resume: bool,
skip_staging: bool,
) -> anyhow::Result<((u64, u64), (u64, u64), (u64, u64))> {
) -> anyhow::Result<((u64, u64), (u64, u64), (u64, u64))>
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: for now, only bench using the field Fr.

{
let (seal_pre_commit_phase1_measurement_cpu_time, seal_pre_commit_phase1_measurement_wall_time): (u64, u64) = if skip_precommit_phase1 {
// generate no-op measurements
(0, 0)
Expand Down Expand Up @@ -335,7 +341,7 @@ fn run_pre_commit_phases<Tree: 'static + MerkleTreeTrait>(
}

#[allow(clippy::too_many_arguments)]
pub fn run_window_post_bench<Tree: 'static + MerkleTreeTrait>(
pub fn run_window_post_bench<Tree>(
sector_size: u64,
api_version: ApiVersion,
cache_dir: PathBuf,
Expand All @@ -345,7 +351,11 @@ pub fn run_window_post_bench<Tree: 'static + MerkleTreeTrait>(
skip_commit_phase1: bool,
skip_commit_phase2: bool,
test_resume: bool,
) -> anyhow::Result<()> {
) -> anyhow::Result<()>
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
{
let (
(seal_pre_commit_phase1_cpu_time_ms, seal_pre_commit_phase1_wall_time_ms),
(
Expand Down
10 changes: 8 additions & 2 deletions fil-proofs-tooling/src/bin/benchy/window_post_fake.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ use std::collections::BTreeMap;
use std::fs::{remove_dir_all, remove_file};
use std::io::stdout;

use blstrs::Scalar as Fr;
use fil_proofs_tooling::shared::{create_replica, PROVER_ID, RANDOMNESS};
use fil_proofs_tooling::{measure, Metadata};
use filecoin_hashers::{Domain, Hasher};
use filecoin_proofs::constants::{WINDOW_POST_CHALLENGE_COUNT, WINDOW_POST_SECTOR_COUNT};
use filecoin_proofs::types::{PoStConfig, SectorSize};
use filecoin_proofs::{
Expand Down Expand Up @@ -45,11 +47,15 @@ impl Report {
}
}

pub fn run_window_post_bench<Tree: 'static + MerkleTreeTrait>(
pub fn run_window_post_bench<Tree>(
sector_size: u64,
fake_replica: bool,
api_version: ApiVersion,
) -> anyhow::Result<()> {
) -> anyhow::Result<()>
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
{
let arbitrary_porep_id = [66; 32];
let sector_count = *WINDOW_POST_SECTOR_COUNT
.read()
Expand Down
10 changes: 8 additions & 2 deletions fil-proofs-tooling/src/bin/benchy/winning_post.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ use std::fs::{remove_dir_all, remove_file};
use std::io::stdout;

use anyhow::anyhow;
use blstrs::Scalar as Fr;
use fil_proofs_tooling::shared::{create_replica, PROVER_ID, RANDOMNESS};
use fil_proofs_tooling::{measure, Metadata};
use filecoin_hashers::{Domain, Hasher};
use filecoin_proofs::constants::{WINNING_POST_CHALLENGE_COUNT, WINNING_POST_SECTOR_COUNT};
use filecoin_proofs::types::PoStConfig;
use filecoin_proofs::{
Expand Down Expand Up @@ -48,11 +50,15 @@ impl Report {
}
}

pub fn run_fallback_post_bench<Tree: 'static + MerkleTreeTrait>(
pub fn run_fallback_post_bench<Tree>(
sector_size: u64,
fake_replica: bool,
api_version: ApiVersion,
) -> anyhow::Result<()> {
) -> anyhow::Result<()>
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
{
if WINNING_POST_SECTOR_COUNT != 1 {
return Err(anyhow!(
"This benchmark only works with WINNING_POST_SECTOR_COUNT == 1"
Expand Down
19 changes: 16 additions & 3 deletions fil-proofs-tooling/src/bin/circuitinfo/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use std::str::FromStr;
use bellperson::{util_cs::bench_cs::BenchCS, Circuit};
use blstrs::Scalar as Fr;
use dialoguer::{theme::ColorfulTheme, MultiSelect};
use filecoin_hashers::{Domain, Hasher};
use filecoin_proofs::{
parameters::{public_params, window_post_public_params, winning_post_public_params},
with_shape, DefaultPieceHasher, PaddedBytesAmount, PoRepConfig, PoRepProofPartitions,
Expand Down Expand Up @@ -36,7 +37,11 @@ fn circuit_info<C: Circuit<Fr>>(circuit: C) -> CircuitInfo {
}
}

fn get_porep_info<Tree: 'static + MerkleTreeTrait>(porep_config: PoRepConfig) -> CircuitInfo {
fn get_porep_info<Tree>(porep_config: PoRepConfig) -> CircuitInfo
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: Groth16 circuits can only be created using the field Fr.

{
info!("PoRep info");

let public_params = public_params(
Expand All @@ -55,7 +60,11 @@ fn get_porep_info<Tree: 'static + MerkleTreeTrait>(porep_config: PoRepConfig) ->
circuit_info(circuit)
}

fn get_winning_post_info<Tree: 'static + MerkleTreeTrait>(post_config: &PoStConfig) -> CircuitInfo {
fn get_winning_post_info<Tree>(post_config: &PoStConfig) -> CircuitInfo
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
{
info!("Winning PoSt info");

let post_public_params = winning_post_public_params::<Tree>(post_config)
Expand All @@ -69,7 +78,11 @@ fn get_winning_post_info<Tree: 'static + MerkleTreeTrait>(post_config: &PoStConf
circuit_info(circuit)
}

fn get_window_post_info<Tree: 'static + MerkleTreeTrait>(post_config: &PoStConfig) -> CircuitInfo {
fn get_window_post_info<Tree>(post_config: &PoStConfig) -> CircuitInfo
where
Tree: 'static + MerkleTreeTrait,
<Tree::Hasher as Hasher>::Domain: Domain<Field = Fr>,
{
info!("Window PoSt info");

let post_public_params = window_post_public_params::<Tree>(post_config)
Expand Down
Loading