feat: merge-train/barretenberg (#19887)

BEGIN_COMMIT_OVERRIDE
feat: support JSON input files for bb verify command (#19800)
fix: update bootstrap.sh to use new JSON field names
chore: Update `index.js` so that `HAS_ZK` and `PUBLIC_INPUTS` variables
must always be set in tests (#19884)
chore: pippenger int audit (#19302)
chore: deduplicate batch affine addition trick (#19788)
chore: transcript+codec+poseidon2 fixes (#19419)
chore!: explicitly constrain inputs and intermediate witnesses (#19826)
fix: exclude nlohmann/json from WASM builds in json_output.hpp
chore: translator circuit builder and flavor audit (#19798)
Revert "fix: exclude nlohmann/json from WASM builds in json_output.hpp"
Revert "feat: support JSON input files for bb verify command (#19800)"
Revert "fix: update bootstrap.sh to use new JSON field names"
END_COMMIT_OVERRIDE

Author: johnathan79717

barretenberg/acir_tests/sol-test/src/index.js

@@ -27,14 +27,6 @@ const GEMINI_CHALLENGE_IN_SUBGROUP = "0x835eb8f7";
 // 5. Run the test against the deployed contract
 // 6. Kill the anvil instance
 
-const getEnvVarCanBeUndefined = (envvar) => {
-  const varVal = process.env[envvar];
-  if (!varVal) {
-    return false;
-  }
-  return varVal;
-};
-
 const getEnvVar = (envvar) => {
   const varVal = process.env[envvar];
   if (!varVal) {
@@ -55,7 +47,7 @@ const [test, verifier] = await Promise.all([
   fsPromises.readFile(verifierPath, encoding),
 ]);
 
-const hasZK = getEnvVarCanBeUndefined("HAS_ZK");
+const hasZK = getEnvVar("HAS_ZK");
 
 export const compilationInput = {
   language: "Solidity",
@@ -230,7 +222,6 @@ const killAnvil = () => {
   console.log(testName, " complete");
 };
 
-// TODO(https://github.com/AztecProtocol/barretenberg/issues/1316): Clean this code up. We are trying to use this logic for three different flows: bb plonk, bb honk, and bbjs honk, and all three have different setups.
 try {
   const proofPath = getEnvVar("PROOF");
 
@@ -239,7 +230,7 @@ try {
   const proof = readFileSync(proofPath);
   proofStr = proof.toString("hex");
 
-  let publicInputsPath = getEnvVarCanBeUndefined("PUBLIC_INPUTS");
+  let publicInputsPath = getEnvVar("PUBLIC_INPUTS");
   var publicInputs = [];
   let numExtraPublicInputs = 0;
   let extraPublicInputs = [];
@@ -251,15 +242,11 @@ try {
     [numExtraPublicInputs, extraPublicInputs] = readPublicInputs(proofAsFields);
   }
 
-  // Read public inputs from binary file if available
-  if (publicInputsPath) {
-    const publicInputsBinary = readFileSync(publicInputsPath);
-    const innerPublicInputs = binaryToFields(publicInputsBinary);
-    publicInputs = innerPublicInputs.concat(extraPublicInputs);
-  } else {
-    // for plonk, the extraPublicInputs are all of the public inputs
-    publicInputs = extraPublicInputs;
-  }
+  // Read public inputs from binary file
+  const publicInputsBinary = readFileSync(publicInputsPath);
+  const innerPublicInputs = binaryToFields(publicInputsBinary);
+  publicInputs = innerPublicInputs.concat(extraPublicInputs);
+
   proofStr = proofStr.substring(64 * numExtraPublicInputs);
   proofStr = "0x" + proofStr;
 

barretenberg/cpp/CLAUDE.md

@@ -2,6 +2,14 @@ succint aztec-packages cheat sheet.
 
 THE PROJECT ROOT IS AT TWO LEVELS ABOVE THIS FOLDER. Typically, the repository is at ~/aztec-packages. all advice is from the root.
 
+# Git workflow for barretenberg
+
+**IMPORTANT**: When comparing branches or looking at diffs for barretenberg work, use `merge-train/barretenberg` as the base branch, NOT `master`. The master branch is often outdated for barretenberg development.
+
+Examples:
+- `git diff merge-train/barretenberg...HEAD` (not `git diff master...HEAD`)
+- `git log merge-train/barretenberg..HEAD` (not `git log master..HEAD`)
+
 Run ./bootstrap.sh at the top-level to be sure the repo fully builds.
 Bootstrap scripts can be called with relative paths e.g. ../barretenberg/bootstrap.sh
 
@@ -95,3 +103,24 @@ BB_VERBOSE=1 yarn-project/scripts/run_test.sh ivc-integration/src/rollup_ivc_int
 ````
 
 When making barretenberg changes, ensure these tests still pass.
+
+## Benchmarking:
+
+**IMPORTANT**: In the barretenberg context, "bench" or "benchmark" almost always means running `benchmark_remote.sh` for the given target on a remote benchmarking machine.
+
+To run benchmarks for a specific target:
+```bash
+cd barretenberg/cpp
+./scripts/benchmark_remote.sh <target_name>
+```
+
+Common benchmark targets:
+- `pippenger_bench` - MSM/Pippenger benchmarks
+- `ultra_honk_bench` - Ultra Honk prover benchmarks
+- `commitment_schemes_bench` - Commitment scheme benchmarks
+
+The remote benchmark script:
+- Runs on a dedicated benchmarking machine for consistent results
+- Automatically builds the target if needed
+- Returns performance metrics and timing data
+- Should be used instead of local benchmarks for performance validation

barretenberg/cpp/scripts/compare_branch_vs_baseline_remote.sh

@@ -16,7 +16,7 @@ PRESET=${3:-clang20}
 BUILD_DIR=${4:-build}
 HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16}
 
-BASELINE_BRANCH="master"
+BASELINE_BRANCH="${BASELINE_BRANCH:-merge-train/barretenberg}"
 BENCH_TOOLS_DIR="$BUILD_DIR/_deps/benchmark-src/tools"
 
 if [ ! -z "$(git status --untracked-files=no --porcelain)" ]; then

barretenberg/cpp/scripts/test_chonk_standalone_vks_havent_changed.sh

@@ -13,7 +13,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-chonk-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-chonk-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-chonk-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="275ed862"
+pinned_short_hash="e3757861"
 pinned_chonk_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-chonk-inputs-${pinned_short_hash}.tar.gz"
 
 script_path="$(cd "$(dirname "${BASH_SOURCE[0]}")/scripts" && pwd)/$(basename "${BASH_SOURCE[0]}")"

barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp

@@ -238,7 +238,7 @@ TEST_F(IPATest, ShpleminiIPAWithoutShift)
                                    mle_opening_point,
                                    ck);
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run the full prover PCS protocol:
     // Compute:
@@ -250,7 +250,7 @@ TEST_F(IPATest, ShpleminiIPAWithoutShift)
     const auto opening_claim = ShplonkProver::prove(ck, prover_opening_claims, prover_transcript);
     PCS::compute_opening_proof(ck, opening_claim, prover_transcript);
 
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     std::array<Fr, log_n> padding_indicator_array;
     std::ranges::fill(padding_indicator_array, Fr{ 1 });
@@ -277,7 +277,7 @@ TEST_F(IPATest, ShpleminiIPAWithShift)
                                    /*num_to_be_shifted*/ 1,
                                    mle_opening_point,
                                    ck);
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run the full prover PCS protocol:
 
@@ -289,7 +289,7 @@ TEST_F(IPATest, ShpleminiIPAWithShift)
     const auto opening_claim = ShplonkProver::prove(ck, prover_opening_claims, prover_transcript);
     PCS::compute_opening_proof(ck, opening_claim, prover_transcript);
 
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     std::array<Fr, log_n> padding_indicator_array;
     std::ranges::fill(padding_indicator_array, Fr{ 1 });
@@ -318,7 +318,7 @@ TEST_F(IPATest, ShpleminiIPAShiftsRemoval)
                                    mle_opening_point,
                                    ck);
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run the full prover PCS protocol:
 
@@ -344,7 +344,7 @@ TEST_F(IPATest, ShpleminiIPAShiftsRemoval)
     // since commitments to poly2, poly3 and their shifts are the same group elements, we simply combine the scalar
     // multipliers of commitment2 and commitment3 in one place and remove the entries of the commitments and scalars
     // vectors corresponding to the "shifted" commitment
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     std::array<Fr, log_n> padding_indicator_array;
     std::ranges::fill(padding_indicator_array, Fr{ 1 });

barretenberg/cpp/src/barretenberg/commitment_schemes/kzg/kzg.test.cpp

@@ -41,11 +41,11 @@ class KZGTest : public CommitmentTest<Curve> {
 
         auto opening_claim = OpeningClaim<Curve>{ opening_pair, commitment };
 
-        auto prover_transcript = NativeTranscript::prover_init_empty();
+        auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
         PCS::compute_opening_proof(ck, { witness, opening_pair }, prover_transcript);
 
-        auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+        auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
         const auto pairing_points = PCS::reduce_verify(opening_claim, verifier_transcript);
 
         EXPECT_EQ(vk.pairing_check(pairing_points[0], pairing_points[1]), true);
@@ -83,12 +83,12 @@ TEST_F(KZGTest, WrongEvaluationFails)
     const Fr wrong_evaluation = evaluation + Fr::random_element();
     // Prove with the wrong evaluation
     Commitment commitment = ck.commit(witness);
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
     PCS::compute_opening_proof(ck, { witness, { challenge, wrong_evaluation } }, prover_transcript);
 
     auto opening_claim = OpeningClaim<Curve>{ { challenge, wrong_evaluation }, commitment };
     // Run the verifier
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
     auto pairing_point = PCS::reduce_verify(opening_claim, verifier_transcript);
     // Make sure that the pairing check fails
     EXPECT_EQ(vk.pairing_check(pairing_point[0], pairing_point[1]), false);
@@ -162,11 +162,11 @@ TEST_F(KZGTest, SingleInLagrangeBasis)
     auto opening_pair = OpeningPair<Curve>{ challenge, evaluation };
     auto opening_claim = OpeningClaim<Curve>{ opening_pair, commitment };
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     PCS::compute_opening_proof(ck, { witness_polynomial, opening_pair }, prover_transcript);
 
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
     auto pairing_points = PCS::reduce_verify(opening_claim, verifier_transcript);
 
     EXPECT_EQ(vk.pairing_check(pairing_points[0], pairing_points[1]), true);
@@ -183,7 +183,7 @@ TEST_F(KZGTest, ShpleminiKzgWithShift)
                                    mle_opening_point,
                                    ck);
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run the full prover PCS protocol:
 
@@ -204,7 +204,7 @@ TEST_F(KZGTest, ShpleminiKzgWithShift)
 
     // Run the full verifier PCS protocol with genuine opening claims (genuine commitment, genuine evaluation)
 
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     // Gemini verifier output:
     // - claim: d+1 commitments to Fold_{r}^(0), Fold_{-r}^(0), Fold^(l), d+1 evaluations a_0_pos, a_l, l = 0:d-1
@@ -238,7 +238,7 @@ TEST_F(KZGTest, ShpleminiKzgWithShiftAndInterleaving)
                                    /*num_interleaved*/ 3,
                                    /*num_to_be_interleaved*/ 2);
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run the full prover PCS protocol:
 
@@ -259,7 +259,7 @@ TEST_F(KZGTest, ShpleminiKzgWithShiftAndInterleaving)
 
     // Run the full verifier PCS protocol with genuine opening claims (genuine commitment, genuine evaluation)
 
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     // Gemini verifier output:
     // - claim: d+1 commitments to Fold_{r}^(0), Fold_{-r}^(0), Fold^(l), d+1 evaluations a_0_pos, a_l, l = 0:d-1
@@ -294,7 +294,7 @@ TEST_F(KZGTest, ShpleminiKzgShiftsRemoval)
                                    mle_opening_point,
                                    ck);
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run the full prover PCS protocol:
 
@@ -315,7 +315,7 @@ TEST_F(KZGTest, ShpleminiKzgShiftsRemoval)
 
     // Run the full verifier PCS protocol with genuine opening claims (genuine commitment, genuine evaluation)
 
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
     // the index of the first commitment to a polynomial to be shifted in the union of unshifted_commitments and
     // shifted_commitments. in our case, it is poly2
     const size_t to_be_shifted_commitments_start = 2;

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.test.cpp

@@ -277,7 +277,7 @@ TYPED_TEST(ShpleminiTest, ShpleminiZKNoSumcheckOpenings)
     using CK = typename TypeParam::CommitmentKey;
 
     // Initialize transcript and commitment key
-    auto prover_transcript = TypeParam::Transcript::prover_init_empty();
+    auto prover_transcript = TypeParam::Transcript::test_prover_init_empty();
 
     // SmallSubgroupIPAProver requires at least CURVE::SUBGROUP_SIZE + 3 elements in the ck.
     static constexpr size_t log_subgroup_size = static_cast<size_t>(numeric::get_msb(Curve::SUBGROUP_SIZE));
@@ -322,7 +322,7 @@ TYPED_TEST(ShpleminiTest, ShpleminiZKNoSumcheckOpenings)
     }
 
     // Initialize verifier's transcript
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     // Start populating Verifier's array of Libra commitments
     std::array<Commitment, NUM_LIBRA_COMMITMENTS> libra_commitments = {};
@@ -391,7 +391,7 @@ TYPED_TEST(ShpleminiTest, ShpleminiZKWithSumcheckOpenings)
     // Generate Sumcheck challenge
     std::vector<Fr> challenge = this->random_evaluation_point(this->log_n);
 
-    auto prover_transcript = TypeParam::Transcript::prover_init_empty();
+    auto prover_transcript = TypeParam::Transcript::test_prover_init_empty();
 
     // Generate masking polynomials for Sumcheck Round Univariates
     ZKSumcheckData<TypeParam> zk_sumcheck_data(this->log_n, prover_transcript, ck);
@@ -430,7 +430,7 @@ TYPED_TEST(ShpleminiTest, ShpleminiZKWithSumcheckOpenings)
     }
 
     // Initialize verifier's transcript
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     std::array<Commitment, NUM_LIBRA_COMMITMENTS> libra_commitments = {};
     libra_commitments[0] =
@@ -517,7 +517,7 @@ TYPED_TEST(ShpleminiTest, HighDegreeAttackAccept)
     MockClaimGenerator<Curve> mock_claims(
         this->n, std::vector{ std::move(poly) }, std::vector<Fr>{ claimed_multilinear_eval }, ck);
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run Shplemini prover
     const auto opening_claim =
@@ -531,7 +531,7 @@ TYPED_TEST(ShpleminiTest, HighDegreeAttackAccept)
     }
 
     // Verifier side
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     std::vector<Fr> padding_indicator_array(small_log_n, Fr{ 1 });
 
@@ -584,7 +584,7 @@ TYPED_TEST(ShpleminiTest, HighDegreeAttackReject)
     MockClaimGenerator<Curve> mock_claims(
         big_n, std::vector{ std::move(poly) }, std::vector<Fr>{ claimed_multilinear_eval }, ck);
 
-    auto prover_transcript = NativeTranscript::prover_init_empty();
+    auto prover_transcript = NativeTranscript::test_prover_init_empty();
 
     // Run Shplemini prover
     const auto opening_claim = ShpleminiProver::prove(big_n, mock_claims.polynomial_batcher, u, ck, prover_transcript);
@@ -597,7 +597,7 @@ TYPED_TEST(ShpleminiTest, HighDegreeAttackReject)
     }
 
     // Verifier side
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     std::vector<Fr> padding_indicator_array(small_log_n, Fr{ 1 });
 
@@ -636,7 +636,7 @@ TYPED_TEST(ShpleminiTest, LibraConsistencyCheckFailsOnCorruptedEvaluation)
     using CK = typename TypeParam::CommitmentKey;
 
     // Initialize transcript and commitment key
-    auto prover_transcript = TypeParam::Transcript::prover_init_empty();
+    auto prover_transcript = TypeParam::Transcript::test_prover_init_empty();
 
     // SmallSubgroupIPAProver requires at least CURVE::SUBGROUP_SIZE + 3 elements in the ck.
     static constexpr size_t log_subgroup_size = static_cast<size_t>(numeric::get_msb(Curve::SUBGROUP_SIZE));
@@ -684,7 +684,7 @@ TYPED_TEST(ShpleminiTest, LibraConsistencyCheckFailsOnCorruptedEvaluation)
     }
 
     // Initialize verifier's transcript
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     // Start populating Verifier's array of Libra commitments
     std::array<Commitment, NUM_LIBRA_COMMITMENTS> libra_commitments = {};
@@ -741,7 +741,7 @@ void run_libra_tampering_test(ShpleminiTest<TypeParam>* test,
     using Commitment = typename Curve::AffineElement;
     using CK = typename TypeParam::CommitmentKey;
 
-    auto prover_transcript = TypeParam::Transcript::prover_init_empty();
+    auto prover_transcript = TypeParam::Transcript::test_prover_init_empty();
 
     static constexpr size_t log_subgroup_size = static_cast<size_t>(numeric::get_msb(Curve::SUBGROUP_SIZE));
     CK ck = create_commitment_key<CK>(std::max<size_t>(test->n, 1ULL << (log_subgroup_size + 1)));
@@ -776,7 +776,7 @@ void run_libra_tampering_test(ShpleminiTest<TypeParam>* test,
         KZG<Curve>::compute_opening_proof(test->ck(), opening_claim, prover_transcript);
     }
 
-    auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+    auto verifier_transcript = NativeTranscript::test_verifier_init_empty(prover_transcript);
 
     std::array<Commitment, NUM_LIBRA_COMMITMENTS> libra_commitments = {};
     libra_commitments[0] =