Process and verify merkle proofs (and multiproof) with custom hash function

.changeset/spotty-queens-own.md

@@ -0,0 +1,5 @@
+---
+'openzeppelin-solidity': minor
+---
+
+`MerkleProof`: Add variations of `verify`, `processProof`, `multiProofVerify` and `processMultiProof` (and equivalent calldata version) with support for custom hashing functions.

contracts/utils/structs/MerkleTree.sol

@@ -19,6 +19,10 @@ import {Panic} from "../Panic.sol";
  * * Zero value: The value that represents an empty leaf. Used to avoid regular zero values to be part of the tree.
  * * Hashing function: A cryptographic hash function used to produce internal nodes. Defaults to {Hashes-commutativeKeccak256}.
  *
+ * NOTE: while this library supports building merkle trees using a non-commutative custom hashing function, proof of
+ * inclusion for such trees will not be verifiable using the {MerkleProof} library. The {MerkleProof} library only
+ * supports commutative hashing functions.
+ *
  * _Available since v5.1._
  */
 library MerkleTree {

scripts/generate/run.js

@@ -32,6 +32,7 @@ function generateFromTemplate(file, template, outputPrefix = '') {
 
 // Contracts
 for (const [file, template] of Object.entries({
+  'utils/cryptography/MerkleProof.sol': './templates/MerkleProof.js',
   'utils/math/SafeCast.sol': './templates/SafeCast.js',
   'utils/structs/EnumerableSet.sol': './templates/EnumerableSet.js',
   'utils/structs/EnumerableMap.sol': './templates/EnumerableMap.js',

scripts/generate/templates/MerkleProof.js

@@ -0,0 +1,178 @@
+const format = require('../format-lines');
+const { OPTS } = require('./MerkleProof.opts');
+
+const DEFAULT_HASH = 'Hashes.commutativeKeccak256';
+
+const formatArgsSingleLine = (...args) => args.filter(Boolean).join(', ');
+const formatArgsMultiline = (...args) => '\n' + format(args.filter(Boolean).join(',\0').split('\0'));
+
+// TEMPLATE
+const header = `\
+pragma solidity ^0.8.20;
+
+import {Hashes} from "./Hashes.sol";
+
+/**
+ * @dev These functions deal with verification of Merkle Tree proofs.
+ *
+ * The tree and the proofs can be generated using our
+ * https://github.com/OpenZeppelin/merkle-tree[JavaScript library].
+ * You will find a quickstart guide in the readme.
+ *
+ * WARNING: You should avoid using leaf values that are 64 bytes long prior to
+ * hashing, or use a hash function other than keccak256 for hashing leaves.
+ * This is because the concatenation of a sorted pair of internal nodes in
+ * the Merkle tree could be reinterpreted as a leaf value.
+ * OpenZeppelin's JavaScript library generates Merkle trees that are safe
+ * against this attack out of the box.
+ *
+ * NOTE: This library support verification of proof for merkle tree built using
+ * custom hashing functions. This is limited to commutative hashing functions.
+ * The verification of inclusion proofs for merkle tree build using non commutative
+ * hashing function require additional logic that this library does not provide.
+ */
+`;
+
+const errors = `\
+/**
+ *@dev The multiproof provided is not valid.
+ */
+error MerkleProofInvalidMultiproof();
+`;
+
+/* eslint-disable max-len */
+const templateProof = ({ suffix, location, visibility, hash }) => `\
+/**
+ * @dev Returns true if a \`leaf\` can be proved to be a part of a Merkle tree
+ * defined by \`root\`. For this, a \`proof\` must be provided, containing
+ * sibling hashes on the branch from the leaf to the root of the tree. Each
+ * pair of leaves and each pair of pre-images are assumed to be sorted.
+ *
+ * This version handles proofs in ${location} with ${hash ? 'a custom' : 'the default'} hashing function.
+ */
+function verify${suffix}(${(hash ? formatArgsMultiline : formatArgsSingleLine)(
+  `bytes32[] ${location} proof`,
+  'bytes32 root',
+  'bytes32 leaf',
+  hash && `function(bytes32, bytes32) view returns (bytes32) ${hash}`,
+)}) internal ${visibility} returns (bool) {
+    return processProof(proof, leaf${hash ? `, ${hash}` : ''}) == root;
+}
+
+/**
+ * @dev Returns the rebuilt hash obtained by traversing a Merkle tree up
+ * from \`leaf\` using \`proof\`. A \`proof\` is valid if and only if the rebuilt
+ * hash matches the root of the tree. When processing the proof, the pairs
+ * of leafs & pre-images are assumed to be sorted.
+ *
+ * This version handles proofs in ${location} with ${hash ? 'a custom' : 'the default'} hashing function.
+ */
+function processProof${suffix}(${(hash ? formatArgsMultiline : formatArgsSingleLine)(
+  `bytes32[] ${location} proof`,
+  'bytes32 leaf',
+  hash && `function(bytes32, bytes32) view returns (bytes32) ${hash}`,
+)}) internal ${visibility} returns (bytes32) {
+    bytes32 computedHash = leaf;
+    for (uint256 i = 0; i < proof.length; i++) {
+        computedHash = ${hash ?? DEFAULT_HASH}(computedHash, proof[i]);
+    }
+    return computedHash;
+}
+`;
+
+const templateMultiProof = ({ suffix, location, visibility, hash }) => `\
+/**
+ * @dev Returns true if the \`leaves\` can be simultaneously proven to be a part of a Merkle tree defined by
+ * \`root\`, according to \`proof\` and \`proofFlags\` as described in {processMultiProof}.
+ *
+ * This version handles multiproofs in ${location} with ${hash ? 'a custom' : 'the default'} hashing function.
+ *
+ * CAUTION: Not all Merkle trees admit multiproofs. See {processMultiProof} for details.
+ */
+function multiProofVerify${suffix}(${formatArgsMultiline(
+  `bytes32[] ${location} proof`,
+  `bool[] ${location} proofFlags`,
+  'bytes32 root',
+  `bytes32[] ${location} leaves`,
+  hash && `function(bytes32, bytes32) view returns (bytes32) ${hash}`,
+)}) internal ${visibility} returns (bool) {
+    return processMultiProof(proof, proofFlags, leaves${hash ? `, ${hash}` : ''}) == root;
+}
+
+/**
+ * @dev Returns the root of a tree reconstructed from \`leaves\` and sibling nodes in \`proof\`. The reconstruction
+ * proceeds by incrementally reconstructing all inner nodes by combining a leaf/inner node with either another
+ * leaf/inner node or a proof sibling node, depending on whether each \`proofFlags\` item is true or false
+ * respectively.
+ *
+ * This version handles multiproofs in ${location} with ${hash ? 'a custom' : 'the default'} hashing function.
+ *
+ * CAUTION: Not all Merkle trees admit multiproofs. To use multiproofs, it is sufficient to ensure that: 1) the tree
+ * is complete (but not necessarily perfect), 2) the leaves to be proven are in the opposite order they are in the
+ * tree (i.e., as seen from right to left starting at the deepest layer and continuing at the next layer).
+ */
+function processMultiProof${suffix}(${formatArgsMultiline(
+  `bytes32[] ${location} proof`,
+  `bool[] ${location} proofFlags`,
+  `bytes32[] ${location} leaves`,
+  hash && `function(bytes32, bytes32) view returns (bytes32) ${hash}`,
+)}) internal ${visibility} returns (bytes32 merkleRoot) {
+    // This function rebuilds the root hash by traversing the tree up from the leaves. The root is rebuilt by
+    // consuming and producing values on a queue. The queue starts with the \`leaves\` array, then goes onto the
+    // \`hashes\` array. At the end of the process, the last hash in the \`hashes\` array should contain the root of
+    // the Merkle tree.
+    uint256 leavesLen = leaves.length;
+
+    // Check proof validity.
+    if (leavesLen + proof.length != proofFlags.length + 1) {
+        revert MerkleProofInvalidMultiproof();
+    }
+
+    // The xxxPos values are "pointers" to the next value to consume in each array. All accesses are done using
+    // \`xxx[xxxPos++]\`, which return the current value and increment the pointer, thus mimicking a queue's "pop".
+    bytes32[] memory hashes = new bytes32[](proofFlags.length);
+    uint256 leafPos = 0;
+    uint256 hashPos = 0;
+    uint256 proofPos = 0;
+    // At each step, we compute the next hash using two values:
+    // - a value from the "main queue". If not all leaves have been consumed, we get the next leaf, otherwise we
+    //   get the next hash.
+    // - depending on the flag, either another value from the "main queue" (merging branches) or an element from the
+    //   \`proof\` array.
+    for (uint256 i = 0; i < proofFlags.length; i++) {
+        bytes32 a = leafPos < leavesLen ? leaves[leafPos++] : hashes[hashPos++];
+        bytes32 b = proofFlags[i]
+            ? (leafPos < leavesLen ? leaves[leafPos++] : hashes[hashPos++])
+            : proof[proofPos++];
+        hashes[i] = ${hash ?? DEFAULT_HASH}(a, b);
+    }
+
+    if (proofFlags.length > 0) {
+        if (proofPos != proof.length) {
+            revert MerkleProofInvalidMultiproof();
+        }
+        unchecked {
+            return hashes[proofFlags.length - 1];
+        }
+    } else if (leavesLen > 0) {
+        return leaves[0];
+    } else {
+        return proof[0];
+    }
+}
+`;
+/* eslint-enable max-len */
+
+// GENERATE
+module.exports = format(
+  header.trimEnd(),
+  'library MerkleProof {',
+  format(
+    [].concat(
+      errors,
+      OPTS.flatMap(opts => templateProof(opts)),
+      OPTS.flatMap(opts => templateMultiProof(opts)),
+    ),
+  ).trimEnd(),
+  '}',
+);

scripts/generate/templates/MerkleProof.opts.js

@@ -0,0 +1,11 @@
+const { product } = require('../../helpers');
+
+const OPTS = product(
+  [
+    { suffix: '', location: 'memory' },
+    { suffix: 'Calldata', location: 'calldata' },
+  ],
+  [{ visibility: 'pure' }, { visibility: 'view', hash: 'hasher' }],
+).map(objs => Object.assign({}, ...objs));
+
+module.exports = { OPTS };

test/utils/cryptography/MerkleProof.test.js

@@ -1,6 +1,7 @@
 const { ethers } = require('hardhat');
 const { expect } = require('chai');
 const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers');
+const { PANIC_CODES } = require('@nomicfoundation/hardhat-chai-matchers/panic');
 const { StandardMerkleTree } = require('@openzeppelin/merkle-tree');
 
 const toElements = str => str.split('').map(e => [e]);
@@ -27,6 +28,8 @@ describe('MerkleProof', function () {
       const hash = merkleTree.leafHash(['A']);
       const proof = merkleTree.getProof(['A']);
 
+      expect(await this.mock.$processProof(proof, hash)).to.equal(root);
+      expect(await this.mock.$processProofCalldata(proof, hash)).to.equal(root);
       expect(await this.mock.$verify(proof, root, hash)).to.be.true;
       expect(await this.mock.$verifyCalldata(proof, root, hash)).to.be.true;
 
@@ -35,6 +38,9 @@ describe('MerkleProof', function () {
         ethers.toBeArray(merkleTree.leafHash(['A'])),
         ethers.toBeArray(merkleTree.leafHash(['B'])),
       );
+
+      expect(await this.mock.$processProof(proof.slice(1), noSuchLeaf)).to.equal(root);
+      expect(await this.mock.$processProofCalldata(proof.slice(1), noSuchLeaf)).to.equal(root);
       expect(await this.mock.$verify(proof.slice(1), root, noSuchLeaf)).to.be.true;
       expect(await this.mock.$verifyCalldata(proof.slice(1), root, noSuchLeaf)).to.be.true;
     });
@@ -47,6 +53,8 @@ describe('MerkleProof', function () {
       const hash = correctMerkleTree.leafHash(['a']);
       const proof = otherMerkleTree.getProof(['d']);
 
+      expect(await this.mock.$processProof(proof, hash)).to.not.equal(root);
+      expect(await this.mock.$processProofCalldata(proof, hash)).to.not.equal(root);
       expect(await this.mock.$verify(proof, root, hash)).to.be.false;
       expect(await this.mock.$verifyCalldata(proof, root, hash)).to.be.false;
     });
@@ -59,6 +67,8 @@ describe('MerkleProof', function () {
       const proof = merkleTree.getProof(['a']);
       const badProof = proof.slice(0, -1);
 
+      expect(await this.mock.$processProof(badProof, hash)).to.not.equal(root);
+      expect(await this.mock.$processProofCalldata(badProof, hash)).to.not.equal(root);
       expect(await this.mock.$verify(badProof, root, hash)).to.be.false;
       expect(await this.mock.$verifyCalldata(badProof, root, hash)).to.be.false;
     });
@@ -72,6 +82,8 @@ describe('MerkleProof', function () {
       const { proof, proofFlags, leaves } = merkleTree.getMultiProof(toElements('bdf'));
       const hashes = leaves.map(e => merkleTree.leafHash(e));
 
+      expect(await this.mock.$processMultiProof(proof, proofFlags, hashes)).to.equal(root);
+      expect(await this.mock.$processMultiProofCalldata(proof, proofFlags, hashes)).to.equal(root);
       expect(await this.mock.$multiProofVerify(proof, proofFlags, root, hashes)).to.be.true;
       expect(await this.mock.$multiProofVerifyCalldata(proof, proofFlags, root, hashes)).to.be.true;
     });
@@ -84,6 +96,8 @@ describe('MerkleProof', function () {
       const { proof, proofFlags, leaves } = otherMerkleTree.getMultiProof(toElements('ghi'));
       const hashes = leaves.map(e => merkleTree.leafHash(e));
 
+      expect(await this.mock.$processMultiProof(proof, proofFlags, hashes)).to.not.equal(root);
+      expect(await this.mock.$processMultiProofCalldata(proof, proofFlags, hashes)).to.not.equal(root);
       expect(await this.mock.$multiProofVerify(proof, proofFlags, root, hashes)).to.be.false;
       expect(await this.mock.$multiProofVerifyCalldata(proof, proofFlags, root, hashes)).to.be.false;
     });
@@ -101,6 +115,14 @@ describe('MerkleProof', function () {
       const hashE = merkleTree.leafHash(['e']); // incorrect (not part of the tree)
       const fill = ethers.randomBytes(32);
 
+      await expect(
+        this.mock.$processMultiProof([hashB, fill, hashCD], [false, false, false], [hashA, hashE]),
+      ).to.be.revertedWithCustomError(this.mock, 'MerkleProofInvalidMultiproof');
+
+      await expect(
+        this.mock.$processMultiProofCalldata([hashB, fill, hashCD], [false, false, false], [hashA, hashE]),
+      ).to.be.revertedWithCustomError(this.mock, 'MerkleProofInvalidMultiproof');
+
       await expect(
         this.mock.$multiProofVerify([hashB, fill, hashCD], [false, false, false], root, [hashA, hashE]),
       ).to.be.revertedWithCustomError(this.mock, 'MerkleProofInvalidMultiproof');
@@ -123,13 +145,21 @@ describe('MerkleProof', function () {
       const hashE = merkleTree.leafHash(['e']); // incorrect (not part of the tree)
       const fill = ethers.randomBytes(32);
 
+      await expect(
+        this.mock.$processMultiProof([hashB, fill, hashCD], [false, false, false, false], [hashE, hashA]),
+      ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS);
+
+      await expect(
+        this.mock.$processMultiProofCalldata([hashB, fill, hashCD], [false, false, false, false], [hashE, hashA]),
+      ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS);
+
       await expect(
         this.mock.$multiProofVerify([hashB, fill, hashCD], [false, false, false, false], root, [hashE, hashA]),
-      ).to.be.revertedWithPanic(0x32);
+      ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS);
 
       await expect(
         this.mock.$multiProofVerifyCalldata([hashB, fill, hashCD], [false, false, false, false], root, [hashE, hashA]),
-      ).to.be.revertedWithPanic(0x32);
+      ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS);
     });
 
     it('limit case: works for tree containing a single leaf', async function () {
@@ -139,6 +169,8 @@ describe('MerkleProof', function () {
       const { proof, proofFlags, leaves } = merkleTree.getMultiProof(toElements('a'));
       const hashes = leaves.map(e => merkleTree.leafHash(e));
 
+      expect(await this.mock.$processMultiProof(proof, proofFlags, hashes)).to.equal(root);
+      expect(await this.mock.$processMultiProofCalldata(proof, proofFlags, hashes)).to.equal(root);
       expect(await this.mock.$multiProofVerify(proof, proofFlags, root, hashes)).to.be.true;
       expect(await this.mock.$multiProofVerifyCalldata(proof, proofFlags, root, hashes)).to.be.true;
     });
@@ -147,6 +179,8 @@ describe('MerkleProof', function () {
       const merkleTree = StandardMerkleTree.of(toElements('abcd'), ['string']);
 
       const root = merkleTree.root;
+      expect(await this.mock.$processMultiProof([root], [], [])).to.equal(root);
+      expect(await this.mock.$processMultiProofCalldata([root], [], [])).to.equal(root);
       expect(await this.mock.$multiProofVerify([root], [], root, [])).to.be.true;
       expect(await this.mock.$multiProofVerifyCalldata([root], [], root, [])).to.be.true;
     });
@@ -161,6 +195,14 @@ describe('MerkleProof', function () {
       const maliciousProof = [leave, leave];
       const maliciousProofFlags = [true, true, false];
 
+      await expect(
+        this.mock.$processMultiProof(maliciousProof, maliciousProofFlags, maliciousLeaves),
+      ).to.be.revertedWithCustomError(this.mock, 'MerkleProofInvalidMultiproof');
+
+      await expect(
+        this.mock.$processMultiProofCalldata(maliciousProof, maliciousProofFlags, maliciousLeaves),
+      ).to.be.revertedWithCustomError(this.mock, 'MerkleProofInvalidMultiproof');
+
       await expect(
         this.mock.$multiProofVerify(maliciousProof, maliciousProofFlags, root, maliciousLeaves),
       ).to.be.revertedWithCustomError(this.mock, 'MerkleProofInvalidMultiproof');