From e830f42195c172de018a98ba091f52ec24d55852 Mon Sep 17 00:00:00 2001 From: Shahar Papini Date: Thu, 9 May 2024 10:42:03 +0300 Subject: [PATCH] MerkleVerifier --- .github/workflows/cairo-ci.yaml | 2 +- stwo_cairo_verifier/.tool-versions | 2 +- stwo_cairo_verifier/src/lib.cairo | 1 + stwo_cairo_verifier/src/utils.cairo | 71 +++++ stwo_cairo_verifier/src/vcs.cairo | 76 +---- stwo_cairo_verifier/src/vcs/hasher.cairo | 81 ++++++ stwo_cairo_verifier/src/vcs/verifier.cairo | 315 +++++++++++++++++++++ 7 files changed, 472 insertions(+), 76 deletions(-) create mode 100644 stwo_cairo_verifier/src/utils.cairo create mode 100644 stwo_cairo_verifier/src/vcs/hasher.cairo create mode 100644 stwo_cairo_verifier/src/vcs/verifier.cairo diff --git a/.github/workflows/cairo-ci.yaml b/.github/workflows/cairo-ci.yaml index eb19383e..cba05a5c 100644 --- a/.github/workflows/cairo-ci.yaml +++ b/.github/workflows/cairo-ci.yaml @@ -25,6 +25,6 @@ jobs: - uses: actions/checkout@v3 - uses: software-mansion/setup-scarb@v1 with: - scarb-version: "nightly-2024-04-24" + scarb-version: "nightly-2024-06-01" - run: scarb fmt --check - run: scarb test diff --git a/stwo_cairo_verifier/.tool-versions b/stwo_cairo_verifier/.tool-versions index 13f722f4..56c8837f 100644 --- a/stwo_cairo_verifier/.tool-versions +++ b/stwo_cairo_verifier/.tool-versions @@ -1 +1 @@ -scarb nightly-2024-04-24 +scarb nightly-2024-06-01 diff --git a/stwo_cairo_verifier/src/lib.cairo b/stwo_cairo_verifier/src/lib.cairo index 583a65af..9f0258fb 100644 --- a/stwo_cairo_verifier/src/lib.cairo +++ b/stwo_cairo_verifier/src/lib.cairo @@ -1,4 +1,5 @@ mod fields; +mod utils; mod vcs; pub use fields::BaseField; diff --git a/stwo_cairo_verifier/src/utils.cairo b/stwo_cairo_verifier/src/utils.cairo new file mode 100644 index 00000000..4d126fe3 --- /dev/null +++ b/stwo_cairo_verifier/src/utils.cairo @@ -0,0 +1,71 @@ +use core::array::SpanTrait; +use core::traits::PanicDestruct; +use core::option::OptionTrait; +use core::box::BoxTrait; +use core::dict::Felt252DictEntryTrait; +use core::dict::Felt252DictTrait; +use core::iter::Iterator; + +#[generate_trait] +pub impl DictImpl, +PanicDestruct> of DictTrait { + fn replace(ref self: Felt252Dict, key: felt252, new_value: T) -> T { + let (entry, value) = self.entry(key); + self = entry.finalize(new_value); + value + } +} + +#[generate_trait] +pub impl OptBoxImpl of OptBoxTrait { + fn as_unboxed(self: Option>) -> Option { + match self { + Option::Some(value) => Option::Some(value.unbox()), + Option::None => Option::None, + } + } +} + +#[generate_trait] +pub impl ArrayImpl, +Drop> of ArrayExTrait { + fn pop_n(ref self: Array, mut n: usize) -> Array { + let mut res = array![]; + while n != 0 { + if let Option::Some(value) = self.pop_front() { + res.append(value); + } else { + break; + } + n -= 1; + }; + res + } + fn max<+PartialOrd>(mut self: @Array) -> Option<@T> { + self.span().max() + } +} + +#[generate_trait] +pub impl SpanImpl of SpanExTrait { + fn next_if_eq<+PartialEq>(ref self: Span, other: @T) -> Option<@T> { + if let Option::Some(value) = self.get(0) { + if value.unbox() == other { + return self.pop_front(); + } + } + Option::None + } + + fn max<+PartialOrd, +Copy>(mut self: Span) -> Option<@T> { + let mut max = self.pop_front()?; + loop { + if let Option::Some(next) = self.pop_front() { + if *next > *max { + max = next; + } + } else { + break; + } + }; + Option::Some(max) + } +} diff --git a/stwo_cairo_verifier/src/vcs.cairo b/stwo_cairo_verifier/src/vcs.cairo index 7fb23b0c..b39f1168 100644 --- a/stwo_cairo_verifier/src/vcs.cairo +++ b/stwo_cairo_verifier/src/vcs.cairo @@ -1,74 +1,2 @@ -use core::array::ArrayTrait; -use core::option::OptionTrait; -use core::poseidon::poseidon_hash_span; -use stwo_cairo_verifier::BaseField; - -// A Merkle node hash is a hash of: -// [left_child_hash, right_child_hash], column0_value, column1_value, ... -// "[]" denotes optional values. -// The largest Merkle layer has no left and right child hashes. The rest of the layers have -// children hashes. -// At each layer, the tree may have multiple columns of the same length as the layer. -// Each node in that layer contains one value from each column. -pub trait MerkleHasher { - type Hash; - // Hashes a single Merkle node. - fn hash_node( - children_hashes: Option<(Self::Hash, Self::Hash)>, column_values: Array, - ) -> Self::Hash; -} - -// 8 M31 elements fit in a hash, since 31*8 = 242 < 252. -const M31_ELS_IN_HASH: usize = 8; -const M31_ELS_IN_HASH_MINUS1: usize = M31_ELS_IN_HASH - 1; -const M31_IN_HASH_SHIFT: felt252 = 0x80000000; // 2**31. -pub impl PoseidonMerkleHasher of MerkleHasher { - type Hash = felt252; - - fn hash_node( - children_hashes: Option<(Self::Hash, Self::Hash)>, mut column_values: Array, - ) -> Self::Hash { - let mut hash_array: Array = Default::default(); - if let Option::Some((x, y)) = children_hashes { - hash_array.append(x); - hash_array.append(y); - } - - // Pad column_values to a multiple of 8. - let mut pad_len = M31_ELS_IN_HASH_MINUS1 - - ((column_values.len() + M31_ELS_IN_HASH_MINUS1) % M31_ELS_IN_HASH); - while pad_len > 0 { - column_values.append(core::num::traits::Zero::zero()); - pad_len = M31_ELS_IN_HASH_MINUS1 - - ((column_values.len() + M31_ELS_IN_HASH_MINUS1) % M31_ELS_IN_HASH); - }; - - while !column_values.is_empty() { - let mut word = 0; - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); - }; - - poseidon_hash_span(hash_array.span()) - } -} - -#[cfg(test)] -mod tests { - use super::PoseidonMerkleHasher; - use stwo_cairo_verifier::fields::m31::{m31}; - - #[test] - fn test_m31() { - assert_eq!( - PoseidonMerkleHasher::hash_node(Option::None, array![m31(0), m31(1)]), - 973835572668429495915136902981656666590582180872133591629269551720657739196 - ); - } -} +mod hasher; +mod verifier; diff --git a/stwo_cairo_verifier/src/vcs/hasher.cairo b/stwo_cairo_verifier/src/vcs/hasher.cairo new file mode 100644 index 00000000..adfd0a5d --- /dev/null +++ b/stwo_cairo_verifier/src/vcs/hasher.cairo @@ -0,0 +1,81 @@ +use core::array::ArrayTrait; +use core::option::OptionTrait; +use core::poseidon::poseidon_hash_span; +use stwo_cairo_verifier::BaseField; + +// A Merkle node hash is a hash of: +// [left_child_hash, right_child_hash], column0_value, column1_value, ... +// "[]" denotes optional values. +// The largest Merkle layer has no left and right child hashes. The rest of the layers have +// children hashes. +// At each layer, the tree may have multiple columns of the same length as the layer. +// Each node in that layer contains one value from each column. +pub trait MerkleHasher { + type Hash; + // Hashes a single Merkle node. + fn hash_node( + children_hashes: Option<(Self::Hash, Self::Hash)>, column_values: Array, + ) -> Self::Hash; +} + +// 8 M31 elements fit in a hash, since 31*8 = 242 < 252. +const M31_ELEMENETS_IN_HASH: usize = 8; +const M31_ELEMENETS_IN_HASH_MINUS1: usize = M31_ELEMENETS_IN_HASH - 1; +const M31_IN_HASH_SHIFT: felt252 = 0x80000000; // 2**31. +pub impl PoseidonMerkleHasher of MerkleHasher { + type Hash = felt252; + + fn hash_node( + children_hashes: Option<(Self::Hash, Self::Hash)>, mut column_values: Array, + ) -> Self::Hash { + let mut hash_array: Array = Default::default(); + if let Option::Some((x, y)) = children_hashes { + hash_array.append(x); + hash_array.append(y); + } + + // Pad column_values to a multiple of 8. + let mut pad_len = M31_ELEMENETS_IN_HASH_MINUS1 + - ((column_values.len() + M31_ELEMENETS_IN_HASH_MINUS1) % M31_ELEMENETS_IN_HASH); + while pad_len > 0 { + column_values.append(core::num::traits::Zero::zero()); + pad_len = M31_ELEMENETS_IN_HASH_MINUS1 + - ((column_values.len() + M31_ELEMENETS_IN_HASH_MINUS1) % M31_ELEMENETS_IN_HASH); + }; + + while !column_values.is_empty() { + let mut word = 0; + // Hash M31_ELEMENETS_IN_HASH = 8 M31 elements into a single field element. + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + word = word * M31_IN_HASH_SHIFT + column_values.pop_front().unwrap().inner.into(); + hash_array.append(word); + }; + + poseidon_hash_span(hash_array.span()) + } +} + +#[cfg(test)] +mod tests { + use super::PoseidonMerkleHasher; + use stwo_cairo_verifier::fields::m31::{m31}; + + #[test] + fn test_m31() { + assert_eq!( + PoseidonMerkleHasher::hash_node(Option::None, array![m31(0), m31(1)]), + 2552053700073128806553921687214114320458351061521275103654266875084493044716 + ); + + assert_eq!( + PoseidonMerkleHasher::hash_node(Option::Some((1, 2)), array![m31(3)]), + 159358216886023795422515519110998391754567506678525778721401012606792642769 + ); + } +} diff --git a/stwo_cairo_verifier/src/vcs/verifier.cairo b/stwo_cairo_verifier/src/vcs/verifier.cairo new file mode 100644 index 00000000..9f435366 --- /dev/null +++ b/stwo_cairo_verifier/src/vcs/verifier.cairo @@ -0,0 +1,315 @@ +use core::dict::Felt252DictTrait; +use core::result::ResultTrait; +use stwo_cairo_verifier::utils::SpanExTrait; +use core::option::OptionTrait; +use core::array::ArrayTrait; +use core::array::SpanTrait; +use core::array::ToSpanTrait; +use core::dict::Felt252Dict; +use core::dict::Felt252DictEntryTrait; +use core::nullable::NullableTrait; +use core::cmp::min; + +use stwo_cairo_verifier::BaseField; +use stwo_cairo_verifier::fields::m31::m31; +use stwo_cairo_verifier::utils::{ArrayExTrait, DictTrait, OptBoxTrait}; +use stwo_cairo_verifier::vcs::hasher::MerkleHasher; + +pub struct MerkleDecommitment { + /// Hash values that the verifier needs but cannot deduce from previous computations, in the + /// order they are needed. + pub hash_witness: Array, + /// Column values that the verifier needs but cannot deduce from previous computations, in the + /// order they are needed. + /// This complements the column values that were queried. These must be supplied directly to + /// the verifier. + pub column_witness: Array, +} +impl MerkleDecommitmentDrop> of Drop>; + +pub struct MerkleVerifier { + pub root: H::Hash, + pub column_log_sizes: Array, +} +impl MerkleVerifierDrop> of Drop>; + +trait MerkleVerifierTrait { + /// Verifies the decommitment of the columns. + /// + /// # Arguments + /// + /// * `queries_per_log_size` - A map from log_size to a vector of queries for columns of that + /// log_size. + /// * `queried_values` - An array of spans of queried values. For each column, there is a + /// span of queried values to that column. + /// * `decommitment` - The decommitment object containing the witness and column values. + /// + /// # Errors + /// + /// Returns an error if any of the following conditions are met: + /// + /// * The witness is too long (not fully consumed). + /// * The witness is too short (missing values). + /// * The column values are too long (not fully consumed). + /// * The column values are too short (missing values). + /// * The computed root does not match the expected root. + /// + /// # Returns + /// + /// Returns `Ok(())` if the decommitment is successfully verified. + fn verify( + self: @MerkleVerifier, + queries_per_log_size: Felt252Dict>>, + queried_values: Array>, + decommitment: MerkleDecommitment, + ) -> Result<(), MerkleVerificationError>; + fn cols_by_size(self: @MerkleVerifier) -> Felt252Dict>>; +} + +impl MerkleVerifierImpl< + impl H: MerkleHasher, +Copy, +Drop, +PartialEq +> of MerkleVerifierTrait { + fn verify( + self: @MerkleVerifier, + mut queries_per_log_size: Felt252Dict>>, + queried_values: Array>, + mut decommitment: MerkleDecommitment, + ) -> Result<(), MerkleVerificationError> { + let MerkleDecommitment::< + H + > { hash_witness: mut hash_witness, column_witness: mut column_witness, } = + decommitment; + + let queried_values = @queried_values; + let mut layer_log_size = *self.column_log_sizes.max().unwrap(); + let mut cols_by_size = Self::cols_by_size(self); + + let mut prev_layer_hashes: Array<(usize, H::Hash)> = array![]; + let mut is_first_layer = true; + loop { + // Prepare write buffer for queries to the current layer. This will propagate to the + // next layer. + let mut layer_total_queries = array![]; + + // Prepare read buffer for queried values to the current layer. + let mut layer_cols = cols_by_size + .replace(layer_log_size.into(), Default::default()) + .deref_or(array![]); + let n_columns_in_layer = layer_cols.len(); + + let mut layer_queried_values = array![]; + + while let Option::Some(column_index) = layer_cols.pop_front() { + layer_queried_values.append(*queried_values[column_index]); + }; + + let layer_queried_values = @layer_queried_values; + + // Extract the requested queries to the current layer. + let mut col_query_index: u32 = 0; + let mut layer_column_queries = queries_per_log_size + .replace(layer_log_size.into(), Default::default(),) + .deref_or(array![].span()); + + // Merge previous layer queries and column queries. + let res = loop { + // Fetch the next query. + let current_query = if let Option::Some(current_query) = + next_decommitment_node(layer_column_queries, @prev_layer_hashes) { + current_query + } else { + break Result::Ok(()); + }; + + let node_hashes = if is_first_layer { + Option::None + } else { + let left_hash = if let Option::Some(val) = + fetch_prev_node_hash( + ref prev_layer_hashes, ref hash_witness, current_query * 2 + ) { + val + } else { + break Result::Err(MerkleVerificationError::WitnessTooShort); + }; + + let right_hash = if let Option::Some(val) = + fetch_prev_node_hash( + ref prev_layer_hashes, ref hash_witness, current_query * 2 + 1 + ) { + val + } else { + break Result::Err(MerkleVerificationError::WitnessTooShort); + }; + Option::Some((left_hash, right_hash)) + }; + + // If the column values were queried, read them from `queried_value`. + let column_values = if layer_column_queries.next_if_eq(@current_query).is_some() { + let mut res = array![]; + let mut i = 0; + while i != n_columns_in_layer { + let queried_column = layer_queried_values[i]; + res.append(*queried_column[col_query_index]); + i += 1; + }; + col_query_index += 1; + res + } else { + column_witness.pop_n(n_columns_in_layer) + }; + + if column_values.len() != n_columns_in_layer { + break Result::Err(MerkleVerificationError::WitnessTooShort); + } + + layer_total_queries + .append((current_query, H::hash_node(node_hashes, column_values))); + }; + + if let Result::Err(err) = res { + break Result::Err(err); + } + + prev_layer_hashes = layer_total_queries; + if layer_log_size == 0 { + break Result::Ok(()); + } + is_first_layer = false; + layer_log_size -= 1; + }?; + + // Check that all witnesses and values have been consumed. + if !hash_witness.is_empty() { + return Result::Err(MerkleVerificationError::WitnessTooLong); + } + if !column_witness.is_empty() { + return Result::Err(MerkleVerificationError::WitnessTooLong); + } + + let (_, computed_root) = prev_layer_hashes.pop_front().unwrap(); + + if @computed_root != self.root { + return Result::Err(MerkleVerificationError::RootMismatch); + } + + Result::Ok(()) + } + + fn cols_by_size(self: @MerkleVerifier) -> Felt252Dict>> { + let mut column_log_sizes = self.column_log_sizes.span(); + let mut res_dict = Default::default(); + let mut col_index = 0; + let mut max_size = 0; + while !column_log_sizes.is_empty() { + let col_size = *column_log_sizes.pop_front().unwrap(); + if col_size > max_size { + max_size = col_size; + } + let (res_dict_entry, value) = res_dict.entry(col_size.into()); + let mut value = value.deref_or(array![]); + value.append(col_index); + res_dict = res_dict_entry.finalize(NullableTrait::new(value)); + col_index += 1; + }; + + res_dict + } +} + +fn next_decommitment_node( + layer_queries: Span, prev_queries: @Array<(u32, H)>, +) -> Option { + // Fetch the next query. + let layer_query_head = layer_queries.get(0).as_unboxed(); + let prev_query_head = if let Option::Some((prev_query, _)) = prev_queries.get(0).as_unboxed() { + Option::Some(*prev_query / 2) + } else { + Option::None + }; + + match (layer_query_head, prev_query_head) { + (Option::None, Option::None) => { Option::None }, + (Option::Some(column_query), Option::None) => { Option::Some(*column_query) }, + (Option::None, Option::Some(prev_query)) => { Option::Some(prev_query) }, + ( + Option::Some(column_query), Option::Some(prev_query) + ) => { Option::Some(min(*column_query, prev_query)) }, + } +} + +/// Fetches the hash of the next node from the previous layer in the Merkle tree. +/// Either from the computed values or from the witness. +fn fetch_prev_node_hash, +Drop>( + ref prev_layer_hashes: Array<(u32, H)>, ref hash_witness: Array, expected_query: u32 +) -> Option { + // If the child was computed, use that value. + if let Option::Some((q, h)) = prev_layer_hashes.get(0).as_unboxed() { + if *q == expected_query { + prev_layer_hashes.pop_front().unwrap(); + return Option::Some(*h); + } + } + // If the child was not computed, read it from the witness. + if let Option::Some(h) = hash_witness.pop_front() { + return Option::Some(h); + } + Option::None +} + +#[derive(Copy, Drop, Debug)] +pub enum MerkleVerificationError { + WitnessTooShort, + WitnessTooLong, + ColumnValuesTooLong, + ColumnValuesTooShort, + RootMismatch, +} + + +use core::poseidon::poseidon_hash_span; +#[test] +fn test_verifier() { + let s = poseidon_hash_span(array![1, 2].span()); + println!("{}", s); + let root = 0x06e3a2499c5ee8a2a66f536f30640b9b67cb50092642003b64a60c401e280214; + let column_log_sizes = array![4, 3, 4, 3, 3, 3, 4, 4, 3, 3]; + let decommitment = MerkleDecommitment { + hash_witness: array![ + 0x037056abc40b9e8c2a67826f54a8c379b0b3ef46629e6a19609e1144bf230f36, + 0x068708ce1c3fc019a43494bd262e87fc70e5c1f68f42881f120fe90ea2bf2201, + 0x01270a97c943188a4aa8a839687ff6d2681b070d1d1627466b93843ad26f4cb2, + 0x06be4322e37fe02371c14436674765da25109e9bc3af4a683c9afea63eb3bdc3, + 0x0360c78816d1d60758c67c011dcd82396a2ccf85fe49ea45667e3cb9feca3f40, + 0x01b4e5f9533e652324ab6b5747edc3343db8f1b9432cdcf2e5ea54fa156ba483, + 0x04a389ddc8e37da68b73c185460f372a5ed8a09eab0f51c63578776db8d1b5ae, + 0x03adfd255329a9a3d49792362f34630fd6b04cc7efdb3a6a175c70b988915cdc, + ], + column_witness: array![ + m31(885772305), + m31(94648313), + m31(604384470), + m31(957953858), + m31(608524802), + m31(428382412), + ] + }; + let mut queries_per_log_size = Default::default(); + queries_per_log_size.insert(3, NullableTrait::new(array![2, 5, 7].span())); + queries_per_log_size.insert(4, NullableTrait::new(array![7, 11, 14].span())); + let queried_values = array![ + array![m31(720125469), m31(997644238), m31(194302184)].span(), + array![m31(122725140), m31(840979908), m31(658446453)].span(), + array![m31(968171809), m31(100529415), m31(1057594968)].span(), + array![m31(1012109813), m31(428994537), m31(992269493)].span(), + array![m31(766295003), m31(28706943), m31(967997322)].span(), + array![m31(552345729), m31(696999129), m31(287489501)].span(), + array![m31(364669117), m31(933029034), m31(285391207)].span(), + array![m31(996158769), m31(69309287), m31(420798739)].span(), + array![m31(650584843), m31(942699537), m31(310081088)].span(), + array![m31(71167745), m31(330264928), m31(409791388)].span() + ]; + MerkleVerifier { root, column_log_sizes, } + .verify(queries_per_log_size, queried_values, decommitment,) + .expect('verification failed'); +}