add a non-sequencing node to localnet and test it

added test cases for the following: * assert that non-sequencing tip is close to the sequencer * assert that non-sequencing hvm btc tip is the same as the sequencer * walk the l2 chain and assert that the output root is the same all of the way down, ignore sync status * assert that safe and final blocks are progressing * repeat tests for both sequencer and non-sequencer (deploying contracts, sending txs, bridging, etc) fixes #508
hemilabs · ClaytonNorthey92 · Jul 2, 2025 · May 19, 2025 · Jun 6, 2025 · Jun 6, 2025
commit 76b66faa3decfb6638681e29ded423b27ca2f2d3
@@ -270,6 +270,7 @@ services:
       ADMIN_PRIVATE_KEY: "${ADMIN_PRIVATE_KEY}"
       OP_GETH_L1_RPC: "http://geth-l1:8545"
       HVM_PHASE0_TIMESTAMP: ${HVM_PHASE0_TIMESTAMP}
+      GETH_NODEKEYHEX: "76e3e48b25c512e1036069538041df813b759fc67d4756b80ace2586b123e913"
     working_dir: "/tmp"
     command:
       - "sh"
@@ -289,6 +290,7 @@ services:
       - 8546:8546
     networks:
       e2e:
+        ipv4_address: 192.169.199.8
 
   op-node:
     build:
@@ -326,8 +328,6 @@ services:
       - "--p2p.no-discovery"
       - "--p2p.priv.path=/tmp/op-node-priv-key.txt"
       - "--p2p.sequencer.key=${ADMIN_PRIVATE_KEY}"
-      - "--p2p.static=/ip4/192.169.198.7/tcp/9222/p2p/16Uiu2HAkx8gegEci9Jk2GDJ92S7HuF7odZCgg9mbos18qiqNEpFz"
-      - "--p2p.static=/ip4/192.169.197.7/tcp/9222/p2p/16Uiu2HAmVqqZGwbuWDffuQo2jCLBSULod13CZuoACZQS8WTKEL2X"
       - "--log.level=debug"
       - "--log.format=terminal"
       - "--override.ecotone=1725868497"
@@ -346,6 +346,92 @@ services:
     networks:
       e2e:
         ipv4_address: 192.169.199.7
+    ports:
+      - "8547:8547"
+
+  op-node-non-sequencing:
+    build:
+      dockerfile: "optimism-stack.Dockerfile"
+      context: "."
+    depends_on:
+      op-node:
+        condition: "service_started"
+    healthcheck:
+      start_period: 180s
+    environment:
+      OP_NODE_BSS_WS: "http://bssd:8081/v1/ws"
+    command:
+      - "op-node/bin/op-node"
+      - "--l2=ws://op-geth-l2-non-sequencing:8551"
+      - "--l2.jwt-secret=/tmp/jwt.hex"
+      - "--verifier.l1-confs=0"
+      - "--rollup.config=/l2configs/rollup.json"
+      - "--rpc.addr=0.0.0.0"
+      - "--rpc.port=8547"
+      - "--rpc.enable-admin"
+      - "--l1=http://geth-l1:8545"
+      - "--l1.rpckind=standard"
+      - "--l1.trustrpc"
+      - "--log.level=info"
+      - "--l1.trustrpc=true"
+      - "--l1.http-poll-interval=1s"
+      - "--p2p.priv.path=/tmp/op-node-priv-key.txt"
+      - "--p2p.sequencer.key=${ADMIN_PRIVATE_KEY}"
+      - "--p2p.static=/ip4/192.169.199.7/tcp/9222/p2p/16Uiu2HAmGCJv5C97ZcdMr6pCCQFqdNXvwE8k6RgTd7vWu4WtVUmr"
+      - "--log.level=debug"
+      - "--log.format=terminal"
+      - "--override.ecotone=1725868497"
+      - "--override.canyon=1725868497"
+      - "--override.delta=1725868497"
+      - "--override.pectrablobschedule=1744238662"
+      - "--override.isthmus=${HVM_PHASE0_TIMESTAMP}"
+      - "--override.holocene=${HVM_PHASE0_TIMESTAMP}"
+      - "--override.granite=${HVM_PHASE0_TIMESTAMP}"
+      - "--override.fjord=${HVM_PHASE0_TIMESTAMP}"
+      - "--l1.beacon.ignore=true"
+    volumes:
+      - "l2configs:/l2configs"
+      - "./jwt.hex:/tmp/jwt.hex"
+    networks:
+      e2e:
+    ports:
+      - "18547:8547"
+
+  op-geth-l2-non-sequencing:
+    build:
+      dockerfile: "optimism-stack.Dockerfile"
+      context: "."
+    depends_on:
+      op-geth-l2:
+        condition: "service_started"
+    healthcheck:
+      test: ["CMD-SHELL", "ls /tmp/datadir/geth.ipc"]
+      interval: 5s
+      retries: 300
+      start_period: 30s
+    environment:
+      ADMIN_PRIVATE_KEY: "${ADMIN_PRIVATE_KEY}"
+      OP_GETH_L1_RPC: "http://geth-l1:8545"
+      HVM_PHASE0_TIMESTAMP: ${HVM_PHASE0_TIMESTAMP}
+    working_dir: "/tmp"
+    command:
+      - "sh"
+      - "/tmp/entrypointl2.sh"
+    volumes:
+      - "./e2e/keystore:/tmp/keystore:ro"
+      - "./e2e/passwords.txt:/tmp/passwords.txt:ro"
+      - "./jwt.hex:/tmp/jwt.hex:ro"
+      - "./entrypointl2.sh:/tmp/entrypointl2.sh"
+      - "./genesisl2.sh:/tmp/genesisl2.sh"
+      - "./output:/tmp/output"
+      - "l2configs:/l2configs"
+      - "./deploy-config.json:/git/optimism/packages/contracts-bedrock/deploy-config/devnetL1.json"
+      - "./prestate-proof.json:/git/optimism/op-program/bin/prestate-proof.json"
+      - {type: "tmpfs", target: "/tmp"}
+    networks:
+      e2e:
+    ports:
+      - "18546:8546"
 
   op-batcher:
     build:

@@ -5,7 +5,7 @@
 
 set -xe
 
-/bin/geth init --datadir /tmp/datadir /l2configs/genesis.json
+/bin/geth init --datadir /tmp/datadir --state.scheme hash /l2configs/genesis.json 
 
 BESTBLOCKHASH=$(curl --data-binary '{"jsonrpc": "1.0", "id": "curltest", "method": "getbestblockhash", "params": []}' -H 'content-type: text/plain;' http://user:password@bitcoind:18443/ | jq '.result')
 
@@ -19,6 +19,25 @@ BLOCKHEADER=$(curl --data-binary "{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", \
 
 echo "setting hvm genesis to $BLOCKHEADER:$BLOCKHEIGHT"
 
+filecontents=$(cat << EOF
+[Node.P2P]
+MaxPeers = 50
+NoDiscovery = false
+DiscoveryV5 = true
+StaticNodes = [
+'enode://a121ecb08858c9e681f0414d079953ec233ffd667acaf66abebad310df3dd4a025a89fe25c3b79a148083cd93c639108cb5b0193a591ee20bd362a7258a3e8d1@192.169.199.8:30303'
+]
+TrustedNodes = [
+'enode://a121ecb08858c9e681f0414d079953ec233ffd667acaf66abebad310df3dd4a025a89fe25c3b79a148083cd93c639108cb5b0193a591ee20bd362a7258a3e8d1@192.169.199.8:30303'
+]
+EOF
+)
+
+
+echo "will use toml file contents: $filecontents"
+
+echo $filecontents > ./config.toml
+
 /bin/geth \
  --keystore \
  /tmp/keystore \
@@ -38,7 +57,7 @@ echo "setting hvm genesis to $BLOCKHEADER:$BLOCKHEIGHT"
  --ws.api=debug,eth,txpool,net,engine,miner \
  --syncmode=full  \
  --nodiscover  \
- --maxpeers=0 \
+ --maxpeers=50 \
  --networkid=901 \
  --authrpc.vhosts="*"  \
  --rpc.allow-unprotected-txs \
@@ -67,7 +86,11 @@ echo "setting hvm genesis to $BLOCKHEADER:$BLOCKHEIGHT"
  --override.fjord=$HVM_PHASE0_TIMESTAMP \
  --verbosity=5 \
  --unlock='78697c88847dfbbb40523e42c1f2e28a13a170be' \
- --rpc.enabledeprecatedpersonal
+ --rpc.enabledeprecatedpersonal \
+ --gcmode=archive \
+ --state.scheme=hash \
+ --config=./config.toml \
+ --verbosity=5
  # Clayton note: this fixes the mismatched state.scheme, but is it the correct
 # thing to do?
 #  --gcmode=archive  \
@@ -16,6 +16,7 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/ethereum-optimism/optimism v0.0.0-00010101000000-000000000000
 	github.com/ethereum/go-ethereum v1.15.9
+	github.com/go-test/deep v1.1.1
 	github.com/gosuri/uilive v0.0.4
 	github.com/hemilabs/heminetwork v1.0.0
 	github.com/jedib0t/go-pretty/v6 v6.5.8

@@ -38,6 +38,9 @@ type state struct {
 	lastBatcherPublicationHash  string
 	batcherPublicationCount     int
 	popMinerBalance             string // very large number
+	tipDiff                     int
+	tipHash                     string
+	tipHashNonSequencing        string
 }
 
 type jsonOutput struct {
@@ -47,6 +50,8 @@ type jsonOutput struct {
 	LastBatcherPublicationHash  string `json:"last_batcher_publication_hash"`
 	BatcherPublicationCount     uint64 `json:"batcher_publication_count"`
 	PopMinerHemiBalance         string `json:"pop_miner_hemi_balance"`
+	TipHash                     string `json:"tip_hash"`
+	TipHashNonSequencing        string `json:"tip_hash_non_sequencing"`
 }
 
 func main() {
@@ -114,6 +119,12 @@ func render(ctx context.Context, s *state, w table.Writer, mtx *sync.Mutex) {
 
 		w.AppendRow(table.Row{"pop miner $HEMI balance", fmt.Sprintf("%s", s.popMinerBalance)})
 
+		w.AppendRow(table.Row{"sequencer vs non-sequencer tip diff", fmt.Sprintf("%d", s.tipDiff)})
+
+		w.AppendRow(table.Row{"sequencer tip", fmt.Sprintf("%s", s.tipHash)})
+
+		w.AppendRow(table.Row{"non-sequencer tip", fmt.Sprintf("%s", s.tipHashNonSequencing)})
+
 		for _, c := range s.containersRunning {
 			w.AppendRow(table.Row{fmt.Sprintf("container %s", c), "running"})
 		}
@@ -294,6 +305,14 @@ func monitorRolledUpTxs(ctx context.Context, s *state, mtx *sync.Mutex) {
 		console.log(Number.parseInt(hexValue, 16));
 	`
 
+	tipJs := `
+		console.log(eth.blockNumber)
+	`
+
+	tipHashJs := `
+		console.log(eth.getBlock(eth.blockNumber).hash)
+	`
+
 	runJs := func(jsi string, layer string, ipcPath string, replica string) string {
 		prefix := "op-"
 		if layer == "l1" {
@@ -319,17 +338,45 @@ func monitorRolledUpTxs(ctx context.Context, s *state, mtx *sync.Mutex) {
 		return strings.Split(string(output), "\n")[0]
 	}
 
+	runJsToInt := func(jsi string, layer string, ipcPath string, replica string) int {
+		val := runJs(jsi, layer, ipcPath, replica)
+		intVal, err := strconv.Atoi(val)
+		if err != nil {
+			panic(fmt.Sprintf("error converting to int: %s", err))
+		}
+
+		return intVal
+	}
+
 	for {
 		first := runJs(firstBatcherTxBlockJs, "l1", "geth.ipc", "1")
 		last := runJs(lastBatcherTxBlockJs, "l1", "geth.ipc", "1")
 		count := runJs(batcherPublicationCountJs, "l1", "geth.ipc", "1")
-		popMinerBalance := runJs(popMinerBalanceJs, "l2", "datadir/geth.ipc", "1")
+
+		popMinerBalance := runJsToInt(popMinerBalanceJs, "l2", "datadir/geth.ipc", "1")
+		popMinerBalanceNonSequencing := runJsToInt(popMinerBalanceJs, "l2-non-sequencing", "datadir/geth.ipc", "1")
+
+		tip := runJsToInt(tipJs, "l2", "datadir/geth.ipc", "1")
+		tipNonSequencing := runJsToInt(tipJs, "l2-non-sequencing", "datadir/geth.ipc", "1")
+
+		tipHash := runJs(tipHashJs, "l2", "datadir/geth.ipc", "1")
+		tipHashNonSequencing := runJs(tipHashJs, "l2-non-sequencing", "datadir/geth.ipc", "1")
+
+		// choose the min of these values; the values should be going up.  if
+		// one node is not keeping up, it will have a (noticeably) lower balance
+		if popMinerBalanceNonSequencing < popMinerBalance {
+			popMinerBalance = popMinerBalanceNonSequencing
+		}
+
+		tipDiff := tip - tipNonSequencing
 
 		mtx.Lock()
 		s.firstBatcherPublicationHash = first
 		s.lastBatcherPublicationHash = last
-
-		s.popMinerBalance = popMinerBalance
+		s.tipDiff = tipDiff
+		s.popMinerBalance = fmt.Sprintf("%d", popMinerBalance)
+		s.tipHash = tipHash
+		s.tipHashNonSequencing = tipHashNonSequencing
 		var err error
 		s.batcherPublicationCount, err = strconv.Atoi(count)
 		if err != nil {
@@ -371,6 +418,8 @@ func dumpJson(mtx *sync.Mutex, s *state) string {
 		LastBatcherPublicationHash:  s.lastBatcherPublicationHash,
 		BatcherPublicationCount:     uint64(s.batcherPublicationCount),
 		PopMinerHemiBalance:         s.popMinerBalance,
+		TipHash:                     s.tipHash,
+		TipHashNonSequencing:        s.tipHashNonSequencing,
 	}
 	mtx.Unlock()