diff --git a/src/chains/ethereum/ethereum/src/@types/merkle-patricia-tree/index.d.ts b/src/chains/ethereum/ethereum/src/@types/merkle-patricia-tree/index.d.ts index 098dcd79ab..c637c58105 100644 --- a/src/chains/ethereum/ethereum/src/@types/merkle-patricia-tree/index.d.ts +++ b/src/chains/ethereum/ethereum/src/@types/merkle-patricia-tree/index.d.ts @@ -20,13 +20,17 @@ declare module "merkle-patricia-tree" { constructor(db: Database, root: Buffer); get(key: LargeNumber, cb: Callback): void; put(key: LargeNumber, value: LargeNumber, cb: Callback): void; - copy(): Trie; + copy(): CheckpointTrie; checkpoint(): void; commit(cb: Callback): void; revert(cb: Callback): void; createScratchReadStream(scratch: Database): ScratchReadStream; - static prove(trie: Trie, key: LargeNumber, cb: Callback): void; + static prove( + trie: CheckpointTrie, + key: LargeNumber, + cb: Callback + ): void; static verifyProof( rootHash: LargeNumber, key: LargeNumber, diff --git a/src/chains/ethereum/ethereum/src/@types/merkle-patricia-tree/secure.d.ts b/src/chains/ethereum/ethereum/src/@types/merkle-patricia-tree/secure.d.ts new file mode 100644 index 0000000000..6c165f342f --- /dev/null +++ b/src/chains/ethereum/ethereum/src/@types/merkle-patricia-tree/secure.d.ts @@ -0,0 +1,11 @@ +declare module "merkle-patricia-tree/secure" { + import { CheckpointTrie } from "merkle-patricia-tree"; + export default class SecureTrie extends CheckpointTrie { + copy(): SecureTrie; + static prove( + trie: SecureTrie, + key: LargeNumber, + cb: Callback + ): void; + } +} diff --git a/src/chains/ethereum/ethereum/src/api.ts b/src/chains/ethereum/ethereum/src/api.ts index 9e3c42a3b8..02dbc5f81a 100644 --- a/src/chains/ethereum/ethereum/src/api.ts +++ b/src/chains/ethereum/ethereum/src/api.ts @@ -1900,7 +1900,7 @@ export default class EthereumApi implements types.Api { * return storage data given a starting key and max number of entries to return. * * @param blockHash DATA, 32 Bytes - hash of a block - * @param txIndex QUANTITY - integer of the transaction index position + * @param transactionIndex QUANTITY - the index of the transaction in the block * @param contractAddress DATA, 20 Bytes - address of the contract * @param startKey DATA - hash of the start key for grabbing storage entries * @param maxResult integer of maximum number of storage entries to return @@ -1912,14 +1912,14 @@ export default class EthereumApi implements types.Api { blockHash: string | Buffer, transactionIndex: number, contractAddress: string, - keyStart: string | Buffer, + startKey: string | Buffer, maxResult: number ) { return this.#blockchain.storageRangeAt( blockHash, transactionIndex, contractAddress, - keyStart, + startKey, maxResult ); } diff --git a/src/chains/ethereum/ethereum/src/blockchain.ts b/src/chains/ethereum/ethereum/src/blockchain.ts index 975c4f683e..4dc8438361 100644 --- a/src/chains/ethereum/ethereum/src/blockchain.ts +++ b/src/chains/ethereum/ethereum/src/blockchain.ts @@ -17,7 +17,11 @@ import { RuntimeError, RETURN_TYPES, Snapshots, - StepEvent + StepEvent, + StorageKeys, + StorageRangeResult, + StorageRecords, + RangedStorageKeys } from "@ganache/ethereum-utils"; import TransactionManager from "./data-managers/transaction-manager"; import SecureTrie from "merkle-patricia-tree/secure"; @@ -39,7 +43,8 @@ const { RPCQUANTITY_EMPTY, BUFFER_32_ZERO, BUFFER_256_ZERO, - RPCQUANTITY_ZERO + RPCQUANTITY_ZERO, + findInsertPosition } = utils; type SimulationTransaction = { @@ -325,7 +330,7 @@ export default class Blockchain extends Emittery.Typed< }: { block: Block; serialized: Buffer; - storageKeys: Map; + storageKeys: StorageKeys; }) => { const { blocks } = this; blocks.latest = block; @@ -371,8 +376,8 @@ export default class Blockchain extends Emittery.Typed< }); // save storage keys to the database - storageKeys.forEach((value, key) => { - this.storageKeys.put(key, value); + storageKeys.forEach(value => { + this.storageKeys.put(value.hashedKey, value.key); }); blockLogs.blockNumber = blockNumberQ; @@ -448,7 +453,7 @@ export default class Blockchain extends Emittery.Typed< #handleNewBlockData = async (blockData: { block: Block; serialized: Buffer; - storageKeys: Map; + storageKeys: StorageKeys; }) => { this.#blockBeingSavedPromise = this.#blockBeingSavedPromise .then(() => this.#saveNewBlock(blockData)) @@ -866,11 +871,11 @@ export default class Blockchain extends Emittery.Typed< transaction: Transaction, options: TransactionTraceOptions, keys?: Buffer[], - contractAddress?: string + contractAddress?: Buffer ) => { let currentDepth = -1; const storageStack: TraceStorageMap[] = []; - const storage = {}; + const storage: StorageRecords = {}; // TODO: gas could go theoretically go over Number.MAX_SAFE_INTEGER. // (Ganache v2 didn't handle this possibility either, so it hasn't been @@ -995,42 +1000,42 @@ export default class Blockchain extends Emittery.Typed< } }; - let txHashCurrentlyProcessing: string = null; - const transactionHash = Data.from(transaction.hash()).toString(); + const transactionHash = transaction.hash(); + let txHashCurrentlyProcessing: Buffer = null; - const beforeTxListener = async (tx: Transaction) => { - txHashCurrentlyProcessing = Data.from(tx.hash()).toString(); - if (txHashCurrentlyProcessing == transactionHash) { + const beforeTxListener = (tx: Transaction) => { + txHashCurrentlyProcessing = tx.hash(); + if (txHashCurrentlyProcessing.equals(transactionHash)) { if (keys && contractAddress) { - keys.forEach(async key => { - // get the raw key using the hashed key - let rawKey = await this.#database.storageKeys.get( - Data.from(key).toString() - ); - - vm.stateManager.getContractStorage( - Address.from(contractAddress).toBuffer(), - rawKey, - (err: Error, result: Buffer) => { - if (err) { - throw err; + const database = this.#database; + return Promise.all( + keys.map(async key => { + // get the raw key using the hashed key + const rawKey: Buffer = await database.storageKeys.get(key); + + vm.stateManager.getContractStorage( + contractAddress, + rawKey, + (err: Error, result: Buffer) => { + if (err) { + throw err; + } + + storage[Data.from(key, key.length).toString()] = { + key: Data.from(rawKey, rawKey.length), + value: Data.from(result, 32) + }; } - - const keccakHashedKey = Data.from(key).toJSON(); - storage[keccakHashedKey] = { - key: Data.from(rawKey).toJSON(), - value: Data.from(result, 32).toJSON() - }; - } - ); - }); + ); + }) + ); } vm.on("step", stepListener); } }; const afterTxListener = () => { - if (txHashCurrentlyProcessing == transactionHash) { + if (txHashCurrentlyProcessing.equals(transactionHash)) { removeListeners(); } }; @@ -1221,17 +1226,7 @@ export default class Blockchain extends Emittery.Typed< contractAddress: string, startKey: string | Buffer, maxResult: number - ) { - type StorageRangeResult = { - nextKey: null | string; - storage: any; - }; - - const result: StorageRangeResult = { - nextKey: null, - storage: {} - }; - + ): Promise { // #1 - get block information const targetBlock = await this.blocks.getByHash(blockHash); @@ -1254,10 +1249,8 @@ export default class Blockchain extends Emittery.Typed< ); // get the contractAddress account storage trie - const addressDataPromise = this.getFromTrie( - trie, - Address.from(contractAddress).toBuffer() - ); + const contractAddressBuffer = Address.from(contractAddress).toBuffer(); + const addressDataPromise = this.getFromTrie(trie, contractAddressBuffer); const addressData = await addressDataPromise; if (!addressData) { throw new Error(`account ${contractAddress} doesn't exist`); @@ -1274,42 +1267,49 @@ export default class Blockchain extends Emittery.Typed< Buffer /*codeHash*/ ])[2]; - let keys: Buffer[] = []; - return new Promise((resolve, reject) => { - storageTrie - .createReadStream() - .on("data", data => { - keys.push(data.key); - }) - .on("end", () => { - // #4 - sort and filter keys - const sortedKeys = keys.sort((a, b) => Buffer.compare(a, b)); - - // find starting point in array of sorted keys - const startKeyBuffer = Data.from(startKey).toBuffer(); - keys = sortedKeys.filter(key => { - if (Buffer.compare(startKeyBuffer, key) <= 0) { - return key; - } - }); + return new Promise((resolve, reject) => { + const startKeyBuffer = Data.from(startKey).toBuffer(); + const compare = (a: Buffer, b: Buffer) => a.compare(b) < 0; + + const keys: Buffer[] = []; + const handleData = ({ key }) => { + // ignore anything that comes before our starting point + if (startKeyBuffer.compare(key) > 0) return; + + // #4 - sort and filter keys + // insert the key exactly where it needs to go in the array + const position = findInsertPosition(keys, key, compare); + // ignore if the value couldn't possibly be relevant + if (position > maxResult) return; + keys.splice(position, 0, key); + }; - // only take the maximum number of entries requested - keys = keys.slice(0, maxResult + 1); - if (keys.length > maxResult) { - // assign nextKey and remove it from array of keys - const nextKey = keys.pop(); - result.nextKey = Data.from(nextKey).toJSON(); - } + const handleEnd = () => { + if (keys.length > maxResult) { + // we collected too much data, so we've got to trim it a bit + resolve({ + // only take the maximum number of entries requested + keys: keys.slice(0, maxResult), + // assign nextKey + nextKey: Data.from(keys[maxResult]) + }); + } else { + resolve({ + keys, + nextKey: null + }); + } + }; - resolve(keys); - }); + const rs = storageTrie.createReadStream(); + rs.on("data", handleData).on("error", reject).on("end", handleEnd); }); }; - const keys = await getStorageKeys(); + const { keys, nextKey } = await getStorageKeys(); // #5 - rerun every transaction in that block prior to and including the requested transaction // prepare block to be run in traceTransaction - const transactionHashBuffer = Data.from(transaction.hash()).toBuffer(); + const transactionHashBuffer = transaction.hash(); const newBlock = this.#prepareNextBlock( targetBlock, parentBlock, @@ -1318,7 +1318,7 @@ export default class Blockchain extends Emittery.Typed< // get storage data given a set of keys const options = { disableMemory: true, - disableStack: false, + disableStack: true, disableStorage: false }; @@ -1327,13 +1327,15 @@ export default class Blockchain extends Emittery.Typed< newBlock, transaction, options, - keys as Buffer[], - contractAddress + keys, + contractAddressBuffer ); - result.storage = storage; // #6 - send back results - return result; + return { + storage, + nextKey + }; } /** diff --git a/src/chains/ethereum/ethereum/src/database.ts b/src/chains/ethereum/ethereum/src/database.ts index 4872ceb7d6..48a8ffa728 100644 --- a/src/chains/ethereum/ethereum/src/database.ts +++ b/src/chains/ethereum/ethereum/src/database.ts @@ -49,7 +49,10 @@ export default class Database extends Emittery { } #initialize = async () => { - const levelupOptions: any = { valueEncoding: "binary" }; + const levelupOptions: any = { + keyEncoding: "binary", + valueEncoding: "binary" + }; const store = this.#options.db; let db: levelup.LevelUp; if (store) { diff --git a/src/chains/ethereum/ethereum/src/miner/miner.ts b/src/chains/ethereum/ethereum/src/miner/miner.ts index 3c7d7ff4b4..2945bbd4b3 100644 --- a/src/chains/ethereum/ethereum/src/miner/miner.ts +++ b/src/chains/ethereum/ethereum/src/miner/miner.ts @@ -7,7 +7,8 @@ import { RETURN_TYPES, Executables, TraceDataFactory, - StepEvent + StepEvent, + StorageKeys } from "@ganache/ethereum-utils"; import { utils, Quantity, Data } from "@ganache/utils"; import { promisify } from "util"; @@ -39,7 +40,7 @@ export default class Miner extends Emittery.Typed< block: { block: Block; serialized: Buffer; - storageKeys: Map; + storageKeys: StorageKeys; }; }, "idle" @@ -179,7 +180,7 @@ export default class Miner extends Emittery.Typed< let keepMining = true; const priced = this.#priced; const legacyInstamine = this.#options.legacyInstamine; - const storageKeys = new Map(); + const storageKeys: StorageKeys = new Map(); let blockTransactions: Transaction[]; do { keepMining = false; @@ -231,9 +232,9 @@ export default class Miner extends Emittery.Typed< if (event.opcode.name === "SSTORE") { const key = TraceData.from( event.stack[event.stack.length - 1].toArrayLike(Buffer) - ); - const hashedKey = Data.from(keccak(key.toBuffer())).toString(); - storageKeys.set(hashedKey, key.toBuffer()); + ).toBuffer(); + const hashedKey = keccak(key); + storageKeys.set(hashedKey.toString(), { key, hashedKey }); } next(); }; diff --git a/src/chains/ethereum/utils/src/index.ts b/src/chains/ethereum/utils/src/index.ts index bbd62dd7f3..f03b4f1242 100644 --- a/src/chains/ethereum/utils/src/index.ts +++ b/src/chains/ethereum/utils/src/index.ts @@ -21,3 +21,4 @@ export * from "./types/snapshots"; export * from "./types/step-event"; export * from "./types/subscriptions"; export * from "./types/tuple-from-union"; +export * from "./types/debug-storage"; diff --git a/src/chains/ethereum/utils/src/things/runtime-block.ts b/src/chains/ethereum/utils/src/things/runtime-block.ts index 1b8766a499..606540fc85 100644 --- a/src/chains/ethereum/utils/src/things/runtime-block.ts +++ b/src/chains/ethereum/utils/src/things/runtime-block.ts @@ -6,6 +6,7 @@ import { encode as rlpEncode, decode as rlpDecode } from "rlp"; import { Transaction } from "./transaction"; import { Address } from "./address"; import { KECCAK256_RLP_ARRAY } from "ethereumjs-util"; +import { StorageKeys } from "../types/debug-storage"; const { BUFFER_EMPTY } = utils; @@ -206,7 +207,7 @@ export class RuntimeBlock { gasUsed: Buffer, extraData: Data, transactions: Transaction[], - storageKeys: Map + storageKeys: StorageKeys ) { const { header } = this; const rawHeader = [ diff --git a/src/chains/ethereum/utils/src/types/debug-storage.ts b/src/chains/ethereum/utils/src/types/debug-storage.ts new file mode 100644 index 0000000000..5ec08cd5ea --- /dev/null +++ b/src/chains/ethereum/utils/src/types/debug-storage.ts @@ -0,0 +1,18 @@ +import { Data } from "@ganache/utils"; + +export type StorageRecords = Record< + string, + { + key: Data; + value: Data; + } +>; + +export type StorageRangeResult = { + nextKey: Data | null; + storage: StorageRecords; +}; + +export type StorageKeys = Map; + +export type RangedStorageKeys = { keys: Buffer[]; nextKey: Data }; diff --git a/src/packages/utils/src/utils/find-insert-position.ts b/src/packages/utils/src/utils/find-insert-position.ts new file mode 100644 index 0000000000..94a8a568ce --- /dev/null +++ b/src/packages/utils/src/utils/find-insert-position.ts @@ -0,0 +1,63 @@ +/** + * AKA `upper_bound` + * + * The elements are compared using `comp`. The elements in the range must + * already be sorted according to this same criterion (`comp`), or at least + * partitioned with respect to val. + * + * The function optimizes the number of comparisons performed by comparing + * non-consecutive elements of the sorted range. + * + * The index into the `array` returned by this function will always be greater than + * the index of the last-occurrence of `val`. + * + * On average, logarithmic in the distance of the length of the array: Performs + * approximately `log2(N)+1` element comparisons (where `N` is this length). + * + * @param array + * @param val Value of the upper bound to search for in the range. + * @param comp A function that accepts two arguments (the first is always + * `val`, and the second from the given `array`) and returns bool. The value + * returned indicates whether the first argument is considered to go before the + * second. + * + * @returns The index to the upper bound position for `val` in the range. If no + * element in the range compares greater than `val`, the function returns + * `array.length`. + */ +export function findInsertPosition( + array: T[], + val: T, + comp: (a: T, b: T) => boolean +): number { + // `count` tracks the number of elements that remain to be searched + let count = array.length; + // `insertPosition` tracks the best insert position for the element we know + // about _so far_ + let insertPosition = 0; + // `offset` tracks the start position of the elements that remain to be + // searched + let offset = 0; + while (count > 0) { + // find the middle element between `offset` and `count` + const step = (count / 2) | 0; // ()`| 0` rounds towards 0) + offset += step; + + // compare our val to the "middle element" (`array[offset]`) + if (!comp(val, array[offset])) { + // `val` should come _after_ the element at `array[offset]`. + // * update our `insertPosition` to the index immediately after + // `array[offset]` + // * shrink our search range + // This narrows our search the elements to the right of `array[offset]`. + insertPosition = ++offset; + count -= step + 1; + } else { + // `val` should come before the element at `array[offset]`: + // This narrows the search the elements to the left of `array[offset]`. + count = step; + offset = insertPosition; + } + } + return insertPosition; +} diff --git a/src/packages/utils/src/utils/index.ts b/src/packages/utils/src/utils/index.ts index 28c0681c74..62b5d75c06 100644 --- a/src/packages/utils/src/utils/index.ts +++ b/src/packages/utils/src/utils/index.ts @@ -8,3 +8,4 @@ export * from "./uint-to-buffer"; export * from "./constants"; export * from "./buffer-to-key"; export * from "./keccak"; +export * from "./find-insert-position";