Jg/p2p rewrite

.circleci/config.yml

@@ -1,12 +1,13 @@
 version: 2.1
 orbs:
   go: circleci/[email protected]
+  slack: circleci/[email protected]
 
 jobs:
   # This job builds the hive executable and stores it in the workspace.
   build:
     docker:
-      - image: cimg/go:1.17
+      - image: cimg/go:1.19
     steps:
       # Build it.
       - checkout
@@ -19,37 +20,50 @@ jobs:
           root: .
           paths: ["hive"]
 
-  # This job runs the smoke test simulations. This requires a virtual
-  # machine instead of the container-based build environment because
-  # hive needs to be able to talk to the docker containers it creates.
-  smoke-tests:
+  # The below job runs the optimism test simulations. This requires a virtual
+  # machine instead of the container-based build environment because hive needs
+  # to be able to talk to the docker containers it creates.
+  run-hive-sim:
     machine:
-      image: ubuntu-2004:202201-02
+      image: ubuntu-2204:2022.07.1
+      docker_layer_caching: true
+    parameters:
+      sim:
+        type: string
+        default: optimism
+        description: Sims to run
     steps:
       - checkout
       - attach_workspace: {at: "/tmp/build"}
       - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum"
+          command: |
+            /tmp/build/hive \
+            -sim=<<parameters.sim>> \
+            -sim.loglevel=5 \
+            -docker.pull=true \
+            -client=go-ethereum,op-geth_optimism-history,op-proposer_develop,op-batcher_develop,op-node_develop |& tee /tmp/build/hive.log || echo "failed."
       - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum"
-
-  # This job also runs the smoke test simulations, but against a remote dockerd.
-  smoke-tests-remote-docker:
-    docker:
-      - image: cimg/base:2022.04
-    steps:
-      - checkout
-      - attach_workspace: {at: "/tmp/build"}
-      - setup_remote_docker: {version: 20.10.14}
+          command: |
+            tar -cvf /tmp/workspace.tgz -C /home/circleci/project /home/circleci/project/workspace
+          name: "Archive workspace"
+      - store_artifacts:
+          path: /tmp/workspace.tgz
+          destination: hive-workspace.tgz
       - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum --loglevel 5"
-      - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum --loglevel 5"
+          command: "! grep 'pass.*=false' /tmp/build/hive.log"
+      - slack/notify:
+          channel: C03N11M0BBN
+          event: fail
+          template: basic_fail_1
+      - slack/notify:
+          channel: C03N11M0BBN
+          event: pass
+          template: basic_success_1
 
   # This job runs the go unit tests.
   go-test:
     docker:
-      - image: cimg/go:1.17
+      - image: cimg/go:1.19
     steps:
       # Get the source.
       - checkout
@@ -64,16 +78,41 @@ jobs:
           name: "hiveproxy module tests"
           command: "go test -cover ./..."
           working_directory: "./hiveproxy"
-      - run:
-          name: "Compile Go simulators"
-          command: ".circleci/compile-simulators.sh"
+# some upstream simulators are broken, don't use latest, geth, etc.
+#      - run:
+#          name: "Compile Go simulators"
+#          command: ".circleci/compile-simulators.sh"
 
 workflows:
   main:
     jobs:
       - go-test
       - build
-      - smoke-tests:
+      - run-hive-sim:
+          name: hive-test-l1ops
           requires: ["build"]
-      - smoke-tests-remote-docker:
+          context: slack
+          sim: optimism/l1ops
+      - run-hive-sim:
+          name: hive-test-p2p
+          requires: [ "build" ]
+          context: slack
+          sim: optimism/p2p
+      - run-hive-sim:
+          name: hive-test-rpc
+          requires: [ "build" ]
+          context: slack
+          sim: optimism/rpc
+  scheduled:
+    triggers:
+      - schedule:
+          # run every 4 hours
+          cron: "0 0,4,8,12,16 * * *"
+          filters:
+            branches:
+              only: [ "optimism" ]
+    jobs:
+      - build
+      - run-hive-sim:
           requires: ["build"]
+          context: slack

.circleci/upgrade-hivesim.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-version=latest
+version=optimism
 if [ -n "$1" ]; then
     version="$1"
 fi
@@ -12,7 +12,7 @@ for d in $sims; do
     (
         set -e
         cd $d
-        go get -d "github.com/ethereum/hive@$version"
-        go mod tidy -compat=1.17
+        go get -d "github.com/ethereum-optimism/hive@$version"
+        go mod tidy
     )
 done

Makefile

@@ -0,0 +1,31 @@
+hive:
+	go build -o ./hive ./hive.go
+.PHONY: hive
+
+mod-tidy:
+	cd simulators/optimism/l1ops && go mod tidy && cd .. && \
+	cd p2p && go mod tidy && cd .. && \
+	cd rpc && go mod tidy && \
+	cd ../../../ && go mod tidy
+.PHONY: mod-tidy
+
+contracts:
+	cd contracts && \
+		solc --bin --abi --overwrite -o . ./SimpleERC20.sol && \
+		solc --bin --abi --overwrite -o . ./Failure.sol
+.PHONY: contracts
+
+bindings: contracts
+	abigen \
+		--abi ./contracts/SimpleERC20.abi \
+		--bin ./contracts/SimpleERC20.bin \
+		--pkg bindings \
+		--type "SimpleERC20" \
+		--out ./optimism/bindings/simple_erc20.go
+	abigen \
+		--abi ./contracts/Failure.abi \
+		--bin ./contracts/Failure.bin \
+		--pkg bindings \
+		--type "Failure" \
+		--out ./optimism/bindings/failure.go
+.PHONY: bindings

clients/go-ethereum/geth.sh

@@ -98,8 +98,8 @@ mv /genesis.json /genesis-input.json
 jq -f /mapper.jq /genesis-input.json > /genesis.json
 
 # Dump genesis
-echo "Supplied genesis state:"
-cat /genesis.json
+#echo "Supplied genesis state:"
+#cat /genesis.json
 
 # Initialize the local testchain with the genesis state
 echo "Initializing database with genesis state..."

clients/op-batcher/Dockerfile

@@ -0,0 +1,11 @@
+ARG branch=develop
+FROM us-central1-docker.pkg.dev/bedrock-goerli-development/images/op-batcher:$branch
+
+RUN apk add bash
+
+COPY ./entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+RUN op-batcher --version 2>/dev/null | head -1 > /version.txt
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

clients/op-batcher/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Hive requires us to prefix env vars with "HIVE_"
+# Iterate the env, find all HIVE_UNPACK_ vars, and remove the HIVE_UNPACK_ prefix.
+while IFS='=' read -r -d '' n v; do
+    if [[ "$n" == HIVE_UNPACK_* ]]; then
+        name=${n#"HIVE_UNPACK_"}  # remove the HIVE_UNPACK_ prefix
+        echo "$name=$v"
+        declare -gx "$name=$v"
+    fi
+done < <(env -0)
+
+exec op-batcher

clients/op-batcher/hive.yaml

@@ -0,0 +1,2 @@
+roles:
+  - "op-batcher"

clients/op-geth/Dockerfile

@@ -0,0 +1,11 @@
+ARG branch=optimism-history
+FROM ethereumoptimism/op-geth:$branch
+
+RUN apk add curl jq bash
+RUN apk add --no-cache ca-certificates
+RUN geth console --exec 'console.log(admin.nodeInfo.name)' --maxpeers=0 --nodiscover --dev 2>/dev/null | head -1 > /version.txt
+
+ADD entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

clients/op-geth/entrypoint.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+set -exu
+
+# note: geth wants an integer log level (see L1 hive definition)
+VERBOSITY=${HIVE_ETH1_LOGLEVEL:-3}
+
+GETH_DATA_DIR=/db
+GETH_CHAINDATA_DIR="$GETH_DATA_DIR/geth/chaindata"
+
+CHAIN_ID=$(cat /genesis.json | jq -r .config.chainId)
+
+if [ ! -d "$GETH_CHAINDATA_DIR" ]; then
+	echo "$GETH_CHAINDATA_DIR missing, running init"
+	echo "Initializing genesis."
+	geth --verbosity="$VERBOSITY" init \
+		--datadir="$GETH_DATA_DIR" \
+		"/genesis.json"
+else
+	echo "$GETH_CHAINDATA_DIR exists."
+fi
+
+# We must set miner.gaslimit to the gas limit in genesis
+# in the command below!
+GAS_LIMIT_HEX=$(jq -r .gasLimit < /genesis.json | sed s/0x//i | tr '[:lower:]' '[:upper:]')
+GAS_LIMIT=$(echo "obase=10; ibase=16; $GAS_LIMIT_HEX" | bc)
+
+
+EXTRA_FLAGS="--rollup.disabletxpoolgossip=true"
+
+# We check for env variables that may not be bound so we need to disable `set -u` for this section.
+set +u
+if [ "$HIVE_OP_GETH_SEQUENCER_HTTP" != "" ]; then
+    EXTRA_FLAGS="$EXTRA_FLAGS --rollup.sequencerhttp $HIVE_OP_GETH_SEQUENCER_HTTP"
+fi
+set -u
+
+# Warning: Archive mode is required, otherwise old trie nodes will be
+# pruned within minutes of starting the devnet.
+
+geth \
+	--datadir="$GETH_DATA_DIR" \
+	--verbosity="$VERBOSITY" \
+	--http \
+	--http.corsdomain="*" \
+	--http.vhosts="*" \
+	--http.addr=0.0.0.0 \
+	--http.port=8545 \
+	--http.api=web3,debug,eth,txpool,net,engine \
+	--ws \
+	--ws.addr=0.0.0.0 \
+	--ws.port=8546 \
+	--ws.origins="*" \
+	--ws.api=debug,eth,txpool,net,engine \
+	--authrpc.jwtsecret="/hive/input/jwt-secret.txt" \
+	--authrpc.port=8551 \
+	--authrpc.addr=0.0.0.0 \
+	--syncmode=full \
+	--nodiscover \
+	--maxpeers=0 \
+	--miner.gaslimit=$GAS_LIMIT \
+	--networkid="$CHAIN_ID" \
+	--password="$GETH_DATA_DIR"/password \
+	--allow-insecure-unlock \
+	--gcmode=archive \
+	$EXTRA_FLAGS \
+	"$@"

clients/op-geth/hive.yaml

@@ -0,0 +1,2 @@
+roles:
+  - "op-l2"

clients/op-node/Dockerfile

@@ -0,0 +1,11 @@
+ARG branch=develop
+FROM us-central1-docker.pkg.dev/bedrock-goerli-development/images/op-node:$branch
+
+RUN apk add bash
+
+COPY ./entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+RUN op-node --version 2>/dev/null | head -1 > /version.txt
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

clients/op-node/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Hive requires us to prefix env vars with "HIVE_"
+# Iterate the env, find all HIVE_UNPACK_ vars, and remove the HIVE_UNPACK_ prefix.
+while IFS='=' read -r -d '' n v; do
+    if [[ "$n" == HIVE_UNPACK_* ]]; then
+        name=${n#"HIVE_UNPACK_"}  # remove the HIVE_UNPACK_ prefix
+        echo "$name=$v"
+        declare -gx "$name=$v"
+    fi
+done < <(env -0)
+
+op-node
+
+

clients/op-node/hive.yaml

@@ -0,0 +1,2 @@
+roles:
+  - "op-node"

clients/op-proposer/Dockerfile

@@ -0,0 +1,11 @@
+ARG branch=develop
+FROM us-central1-docker.pkg.dev/bedrock-goerli-development/images/op-proposer:$branch
+
+RUN apk add bash
+
+COPY ./entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+RUN op-proposer --version 2>/dev/null | head -1 > /version.txt
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

clients/op-proposer/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Hive requires us to prefix env vars with "HIVE_"
+# Iterate the env, find all HIVE_UNPACK_ vars, and remove the HIVE_UNPACK_ prefix.
+while IFS='=' read -r -d '' n v; do
+    if [[ "$n" == HIVE_UNPACK_* ]]; then
+        name=${n#"HIVE_UNPACK_"}  # remove the HIVE_UNPACK_ prefix
+        echo "$name=$v"
+        declare -gx "$name=$v"
+    fi
+done < <(env -0)
+
+exec op-proposer

clients/op-proposer/hive.yaml

@@ -0,0 +1,2 @@
+roles:
+  - "op-proposer"

clients/prysm-bn/prysm_bn.sh

@@ -73,6 +73,7 @@ echo Starting Prysm Beacon Node
     --jwt-secret=/jwtsecret \
     --min-sync-peers=1 \
     --subscribe-all-subnets=true \
+    --enable-debug-rpc-endpoints=true \
     $metrics_option \
     --deposit-contract="${HIVE_ETH2_CONFIG_DEPOSIT_CONTRACT_ADDRESS:-0x1111111111111111111111111111111111111111}" \
     --contract-deployment-block="${HIVE_ETH2_DEPOSIT_DEPLOY_BLOCK_NUMBER:-0}" \

cmd/hivechain/generate.go

@@ -217,7 +217,7 @@ func (cfg generatorConfig) generateAndSave(path string, blockModifier func(i int
 	}
 
 	// Import the chain. This runs all block validation rules.
-	blockchain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil, nil)
+	blockchain, err := core.NewBlockChain(db, nil, &cfg.genesis, nil, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		return fmt.Errorf("can't create blockchain: %v", err)
 	}