Complete Basefold verifier and Basefold e2e integration

Cargo.toml

@@ -10,26 +10,26 @@ openvm-native-circuit = { git = "https://github.com/scroll-tech/openvm.git", bra
 openvm-native-compiler = { git = "https://github.com/scroll-tech/openvm.git", branch = "feat/native_multi_observe", default-features = false }
 openvm-native-compiler-derive = { git = "https://github.com/scroll-tech/openvm.git", branch = "feat/native_multi_observe", default-features = false }
 openvm-native-recursion = { git = "https://github.com/scroll-tech/openvm.git", branch = "feat/native_multi_observe", default-features = false }
-openvm-stark-backend = { git = "https://github.com/openvm-org/stark-backend.git", tag = "v1.0.0", default-features = false }
-openvm-stark-sdk = { git = "https://github.com/openvm-org/stark-backend.git", tag = "v1.0.0", default-features = false }
+openvm-stark-backend = { git = "https://github.com/openvm-org/stark-backend.git", rev = "f48090c9febd021f8ee0349bc929a775fb1fa3ad", default-features = false }
+openvm-stark-sdk = { git = "https://github.com/openvm-org/stark-backend.git", rev = "f48090c9febd021f8ee0349bc929a775fb1fa3ad", default-features = false }
 
 rand = { version = "0.8.5", default-features = false }
 itertools = { version = "0.13.0", default-features = false }
 bincode = "1.3.3"
 tracing = "0.1.40"
 
 # Plonky3
-p3-air = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-baby-bear = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-field = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-commit = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-matrix = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-util = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-challenger = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-monty-31 = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-symmetric = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-fri = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
-p3-goldilocks = { git = "https://github.com/Plonky3/plonky3", rev = "1ba4e5c" }
+p3-air = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-baby-bear = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-field = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-commit = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-matrix = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-util = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-challenger = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-monty-31 = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-symmetric = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-fri = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
+p3-goldilocks = { git = "https://github.com/Plonky3/plonky3", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
 
 # WHIR
 ark-std = { version = "0.5", features = ["std"] }
@@ -38,26 +38,26 @@ ark-poly = "0.5"
 ark-serialize = "0.5"
 
 # Ceno
-ceno_mle = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support", package = "multilinear_extensions" }
-ceno_sumcheck = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support", package = "sumcheck" }
-ceno_transcript = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support", package = "transcript" }
-ceno_witness = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support", package = "witness" }
-ceno_zkvm = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support" }
-ceno_emul = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support" }
-mpcs = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support" }
-ff_ext = { git = "https://github.com/scroll-tech/ceno.git", branch = "feat/smaller_field_support" }
+ceno_mle = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc", package = "multilinear_extensions" }
+ceno_sumcheck = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc", package = "sumcheck" }
+ceno_transcript = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc", package = "transcript" }
+ceno_witness = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc", package = "witness" }
+ceno_zkvm = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc" }
+ceno_emul = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc" }
+mpcs = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc" }
+ff_ext = { git = "https://github.com/scroll-tech/ceno.git", branch = "build/smaller_field_support_plonky3_539bbc" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 
 [features]
 bench-metrics = ["openvm-circuit/bench-metrics"]
 
-[patch."https://github.com/scroll-tech/ceno.git"]
-ceno_mle = { path = "../ceno/multilinear_extensions", package = "multilinear_extensions" }
-ceno_sumcheck = { path = "../ceno/sumcheck", package = "sumcheck" }
-ceno_transcript = { path = "../ceno/transcript", package = "transcript" }
-ceno_witness = { path = "../ceno/witness", package = "witness" }
-ceno_zkvm = { path = "../ceno/ceno_zkvm" }
-ceno_emul = { path = "../ceno/ceno_emul" }
-mpcs = { path = "../ceno/mpcs" }
-ff_ext = { path = "../ceno/ff_ext" }
+# [patch."https://github.com/scroll-tech/ceno.git"]
+# ceno_mle = { path = "../ceno/multilinear_extensions", package = "multilinear_extensions" }
+# ceno_sumcheck = { path = "../ceno/sumcheck", package = "sumcheck" }
+# ceno_transcript = { path = "../ceno/transcript", package = "transcript" }
+# ceno_witness = { path = "../ceno/witness", package = "witness" }
+# ceno_zkvm = { path = "../ceno/ceno_zkvm" }
+# ceno_emul = { path = "../ceno/ceno_emul" }
+# mpcs = { path = "../ceno/mpcs" }
+# ff_ext = { path = "../ceno/ff_ext" }

src/basefold_verifier/basefold.rs

@@ -1,5 +1,9 @@
+use std::collections::BTreeMap;
+
+use itertools::Itertools;
 use mpcs::basefold::BasefoldProof as InnerBasefoldProof;
 use openvm_native_compiler::{asm::AsmConfig, prelude::*};
+use openvm_native_compiler_derive::iter_zip;
 use openvm_native_recursion::hints::{Hintable, VecAutoHintable};
 use openvm_stark_sdk::p3_baby_bear::BabyBear;
 use p3_field::extension::BinomialExtensionField;
@@ -78,6 +82,7 @@ pub struct BasefoldProof {
     pub final_message: Vec<Vec<E>>,
     pub query_opening_proof: QueryOpeningProofs,
     pub sumcheck_proof: Vec<IOPProverMessage>,
+    pub pow_witness: F,
 }
 
 #[derive(DslVariable, Clone)]
@@ -86,6 +91,7 @@ pub struct BasefoldProofVariable<C: Config> {
     pub final_message: Array<C, Array<C, Ext<C::F, C::EF>>>,
     pub query_opening_proof: QueryOpeningProofsVariable<C>,
     pub sumcheck_proof: Array<C, IOPProverMessageVariable<C>>,
+    pub pow_witness: Felt<C::F>,
 }
 
 impl Hintable<InnerConfig> for BasefoldProof {
@@ -95,11 +101,13 @@ impl Hintable<InnerConfig> for BasefoldProof {
         let final_message = Vec::<Vec<E>>::read(builder);
         let query_opening_proof = QueryOpeningProofs::read(builder);
         let sumcheck_proof = Vec::<IOPProverMessage>::read(builder);
+        let pow_witness = F::read(builder);
         BasefoldProofVariable {
             commits,
             final_message,
             query_opening_proof,
             sumcheck_proof,
+            pow_witness,
         }
     }
 
@@ -109,6 +117,7 @@ impl Hintable<InnerConfig> for BasefoldProof {
         stream.extend(self.final_message.write());
         stream.extend(self.query_opening_proof.write());
         stream.extend(self.sumcheck_proof.write());
+        stream.extend(self.pow_witness.write());
         stream
     }
 }
@@ -126,6 +135,7 @@ impl From<InnerBasefoldProof<E>> for BasefoldProof {
             sumcheck_proof: proof.sumcheck_proof.map_or(vec![], |proof| {
                 proof.into_iter().map(|proof| proof.into()).collect()
             }),
+            pow_witness: proof.pow_witness,
         }
     }
 }
@@ -173,20 +183,41 @@ pub struct Round {
 pub struct RoundVariable<C: Config> {
     pub commit: BasefoldCommitmentVariable<C>,
     pub openings: Array<C, RoundOpeningVariable<C>>,
+    pub perm: Array<C, Var<C::N>>,
 }
 
 impl Hintable<InnerConfig> for Round {
     type HintVariable = RoundVariable<InnerConfig>;
     fn read(builder: &mut Builder<InnerConfig>) -> Self::HintVariable {
         let commit = BasefoldCommitment::read(builder);
         let openings = Vec::<RoundOpening>::read(builder);
-        RoundVariable { commit, openings }
+        let perm = Vec::<usize>::read(builder);
+        RoundVariable {
+            commit,
+            openings,
+            perm,
+        }
     }
 
     fn write(&self) -> Vec<Vec<<InnerConfig as Config>::N>> {
+        let mut perm = vec![0; self.openings.len()];
+        self.openings
+            .iter()
+            .enumerate()
+            // the original order
+            .map(|(i, opening)| (i, opening.num_var))
+            .sorted_by(|(_, nv_a), (_, nv_b)| Ord::cmp(nv_b, nv_a))
+            .enumerate()
+            // j is the new index where i is the original index
+            .map(|(j, (i, _))| (i, j))
+            .for_each(|(i, j)| {
+                perm[i] = j;
+            });
         let mut stream = vec![];
         stream.extend(self.commit.write());
         stream.extend(self.openings.write());
+        stream.extend(perm.write());
+
         stream
     }
 }

src/basefold_verifier/mod.rs

@@ -7,3 +7,4 @@ pub(crate) mod rs;
 pub(crate) mod structs;
 // pub(crate) mod field;
 pub(crate) mod utils;
+pub(crate) mod verifier;

src/basefold_verifier/query_phase.rs

@@ -298,7 +298,7 @@ pub struct PackedCodeword<C: Config> {
     pub high: Ext<C::F, C::EF>,
 }
 
-pub(crate) fn batch_verifier_query_phase<C: Config + Debug>(
+pub(crate) fn batch_verifier_query_phase<C: Config>(
     builder: &mut Builder<C>,
     input: QueryPhaseVerifierInputVariable<C>,
 ) {
@@ -344,7 +344,7 @@ pub(crate) fn batch_verifier_query_phase<C: Config + Debug>(
     let final_codeword = encode_small(builder, final_rmm);
 
     let log2_max_codeword_size: Var<C::N> =
-        builder.eval(input.max_num_var.clone() + get_rate_log::<C>());
+        builder.eval(input.max_num_var.clone() + Usize::from(get_rate_log()));
 
     let zero: Ext<C::F, C::EF> = builder.constant(C::EF::ZERO);
 
@@ -391,12 +391,9 @@ pub(crate) fn batch_verifier_query_phase<C: Config + Debug>(
                         let j = j_vec[0];
                         let mat_j = builder.get(&opened_values, j);
                         let num_var_j = builder.get(&round.openings, j).num_var;
-                        let height_j =
-                            builder.eval(num_var_j + get_rate_log::<C>() - Usize::from(1));
+                        let height_j = builder.eval(num_var_j + Usize::from(get_rate_log() - 1));
 
-                        // TODO: use permutation to get the index
-                        // let permuted_index = builder.get(&perm, j);
-                        let permuted_j = j;
+                        let permuted_j = builder.get(&round.perm, j);
 
                         builder.set_value(&perm_opened_values, permuted_j, mat_j);
                         builder.set_value(&dimensions, permuted_j, height_j);
@@ -423,7 +420,7 @@ pub(crate) fn batch_verifier_query_phase<C: Config + Debug>(
                     let opened_value = builder.iter_ptr_get(&opened_values, ptr_vec[0]);
                     let opening = builder.iter_ptr_get(&round.openings, ptr_vec[1]);
                     let log2_height: Var<C::N> =
-                        builder.eval(opening.num_var + get_rate_log::<C>() - Usize::from(1));
+                        builder.eval(opening.num_var + Usize::from(get_rate_log() - 1));
                     let width = opening.point_and_evals.evals.len();
 
                     let batch_coeffs_next_offset: Var<C::N> =
@@ -466,7 +463,7 @@ pub(crate) fn batch_verifier_query_phase<C: Config + Debug>(
             // fold 1st codeword
             let cur_num_var: Var<C::N> = builder.eval(input.max_num_var.clone());
             let log2_height: Var<C::N> =
-                builder.eval(cur_num_var + get_rate_log::<C>() - Usize::from(1));
+                builder.eval(cur_num_var + Usize::from(get_rate_log() - 1));
 
             let r = builder.get(&input.fold_challenges, 0);
             let codeword = builder.get(&reduced_codeword_by_height, log2_height);
@@ -639,7 +636,7 @@ pub(crate) fn batch_verifier_query_phase<C: Config + Debug>(
             let point = &point_and_evals.point;
 
             let num_vars_evaluated: Var<C::N> =
-                builder.eval(point.fs.len() - get_basecode_msg_size_log::<C>());
+                builder.eval(point.fs.len() - Usize::from(get_basecode_msg_size_log()));
             let final_message = builder.get(&input.proof.final_message, j);
 
             // coeff is the eq polynomial evaluated at the first challenge.len() variables

src/basefold_verifier/rs.rs

@@ -93,12 +93,16 @@ impl<C: Config> DenseMatrixVariable<C> {
     }
 }
 
-pub fn get_rate_log<C: Config>() -> Usize<C::N> {
-    Usize::from(1)
+pub fn get_rate_log() -> usize {
+    1
 }
 
-pub fn get_basecode_msg_size_log<C: Config>() -> Usize<C::N> {
-    Usize::from(0)
+pub fn get_basecode_msg_size_log() -> usize {
+    0
+}
+
+pub fn get_num_queries() -> usize {
+    100
 }
 
 pub fn verifier_folding_coeffs_level<C: Config>(

src/basefold_verifier/structs.rs

@@ -113,7 +113,7 @@ pub fn get_base_codeword_dimensions<C: Config>(
         let fixed_num_polys = tmp.fixed_num_polys;
         // wit_dim
         // let width = builder.eval(witin_num_polys * Usize::from(2));
-        let height_exp = builder.eval(witin_num_vars + get_rate_log::<C>() - Usize::from(1));
+        let height_exp = builder.eval(witin_num_vars + Usize::from(get_rate_log() - 1));
         // let height = pow_2(builder, height_exp);
         // let next_wit: DimensionsVariable<C> = DimensionsVariable { width, height };
         // Only keep the height because the width is not needed in the mmcs batch
@@ -129,7 +129,7 @@ pub fn get_base_codeword_dimensions<C: Config>(
             .then(|builder| {
                 // let width = builder.eval(fixed_num_polys.clone() * Usize::from(2));
                 let height_exp =
-                    builder.eval(fixed_num_vars.clone() + get_rate_log::<C>() - Usize::from(1));
+                    builder.eval(fixed_num_vars.clone() + Usize::from(get_rate_log() - 1));
                 // XXX: more efficient pow implementation
                 // let height = pow_2(builder, height_exp);
                 // let next_fixed: DimensionsVariable<C> = DimensionsVariable { width, height };