diff --git a/.github/scripts/check_finalization.sh b/.github/scripts/check_finalization.sh index 36d21ad191..1611056317 100755 --- a/.github/scripts/check_finalization.sh +++ b/.github/scripts/check_finalization.sh @@ -1,12 +1,12 @@ #!/bin/bash -RPC_HOST=127.0.0.1 -RPC_PORT=9933 +RPC_HOST=${RPC_HOST:-127.0.0.1} +RPC_PORT=${RPC_PORT:-9933} LAST_FINALIZED="" -VALIDATOR=damian +VALIDATOR=${VALIDATOR:-damian} while [[ "$LAST_FINALIZED" =~ "0x0" ]] || [[ -z "$LAST_FINALIZED" ]]; do - block_hash=$(docker run --network container:$VALIDATOR appropriate/curl:latest \ + block_hash=$(docker run --rm --network container:$VALIDATOR appropriate/curl:latest \ -H "Content-Type: application/json" \ -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getFinalizedHead"}' http://$RPC_HOST:$RPC_PORT | jq '.result') ret_val=$? @@ -15,7 +15,7 @@ while [[ "$LAST_FINALIZED" =~ "0x0" ]] || [[ -z "$LAST_FINALIZED" ]]; do continue fi - finalized_block=$(docker run --network container:$VALIDATOR appropriate/curl:latest \ + finalized_block=$(docker run --rm --network container:$VALIDATOR appropriate/curl:latest \ -H "Content-Type: application/json" \ -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock", "params": ['$block_hash']}' http://$RPC_HOST:$RPC_PORT | jq '.result.block.header.number') @@ -25,9 +25,9 @@ while [[ "$LAST_FINALIZED" =~ "0x0" ]] || [[ -z "$LAST_FINALIZED" ]]; do continue else LAST_FINALIZED=$finalized_block - echo "Last finalized block number: $LAST_FINALIZED" fi done +echo "Last finalized block number: $LAST_FINALIZED" exit $? diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh index e8ebae3372..ca40484618 100755 --- a/.github/scripts/run_consensus.sh +++ b/.github/scripts/run_consensus.sh @@ -6,6 +6,7 @@ set -euo pipefail # change when increasing the number of node containers NODE_COUNT=5 MIN_VALIDATOR_COUNT=4 +DOCKER_COMPOSE=${DOCKER_COMPOSE:-"docker/docker-compose.yml"} OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""} # default minimum validator count @@ -83,19 +84,20 @@ function generate_bootnode_peer_id { function run_containers { local authorities_count="$1" - local override_file="$2" + local docker_compose_file="$2" + local override_file="$3" echo "Running ${authorities_count} containers..." if [[ -z ${override_file} ]]; then - docker-compose -f docker/docker-compose.yml up -d + docker-compose -f "${docker_compose_file}" up -d else - docker-compose -f docker/docker-compose.yml -f "${override_file}" up -d + docker-compose -f "${docker_compose_file}" -f "${override_file}" up -d fi } authorities=$(generate_authorities ${NODE_COUNT}) generate_chainspec "${authorities[@]}" "${MIN_VALIDATOR_COUNT}" generate_bootnode_peer_id ${authorities[0]} -run_containers ${NODE_COUNT} "${OVERRIDE_DOCKER_COMPOSE}" +run_containers ${NODE_COUNT} "${DOCKER_COMPOSE}" "${OVERRIDE_DOCKER_COMPOSE}" exit $? diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index 94299ca870..a1859c6839 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -54,6 +54,45 @@ jobs: retention-days: 7 + build-cliain-image: + name: Build docker image for cliain + runs-on: ubuntu-20.04 + steps: + - name: GIT | Checkout source code + uses: actions/checkout@v2 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + + - name: Restore cache + uses: ./.github/actions/restore-cache + with: + target-key: cliain + cache-version: v2 + cargo-targets: bin/cliain/target/ + + - name: Cargo | Build release binary + run: | + cd bin/cliain && cargo build --release + + - name: Build docker image + run: | + cd bin/cliain + docker build --tag cliain:latest -f ./Dockerfile . + docker save -o cliain.tar cliain:latest + + - name: Upload test docker image + uses: actions/upload-artifact@v2 + with: + name: cliain-docker + path: ./bin/cliain/cliain.tar + if-no-files-found: error + retention-days: 7 + + - name: Cleanup cache + uses: ./.github/actions/post-cache + + check-determinism: needs: [build-new-node] name: Verify runtime build determinism @@ -279,6 +318,7 @@ jobs: test-case: fee_calculation timeout-minutes: 2 + run-e2e-validators-rotate: needs: [build-test-docker, build-test-client] name: Run validators rotation test @@ -485,6 +525,66 @@ jobs: follow-up-finalization-check: true timeout-minutes: 15 + run-e2e-version-upgrade-catchup: + needs: [build-test-docker, build-cliain-image] + name: Run series of tests where some of the nodes need to do version-upgrade during catch-up + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - nodes: "Node1" + ports: "9934" + ext_status: "finalized" + upgrade_before_disable: "true" + + - nodes: "Node1" + ports: "9934" + ext_status: "finalized" + upgrade_before_disable: "false" + + - nodes: "Node1:Node2" + ports: "9934:9935" + ext_status: "in-block" + upgrade_before_disable: "true" + + - nodes: "Node1:Node2" + ports: "9934:9935" + ext_status: "in-block" + upgrade_before_disable: "false" + steps: + - name: Checkout source code + uses: actions/checkout@v2 + + - name: Download artifact with docker image for aleph-node + uses: actions/download-artifact@v2 + with: + name: aleph-test-docker + + - name: Load node docker image + shell: bash + run: docker load -i aleph-node.tar + + - name: Download artifact with docker image for cliain + uses: actions/download-artifact@v2 + with: + name: cliain-docker + + - name: Load cliain docker image + shell: bash + run: docker load -i cliain.tar + + - name: Call catchup_test.sh + env: + UPGRADE_BLOCK: 31 + NODES: ${{ matrix.nodes }} + PORTS: ${{ matrix.ports }} + EXT_STATUS: ${{ matrix.ext_status }} + UPGRADE_BEFORE_DISABLE: ${{ matrix.upgrade_before_disable }} + DOCKER_COMPOSE: docker/docker-compose.bridged.yml + + run: | + ./scripts/catchup_version_upgrade_test.sh + check-e2e-test-suite-completion: needs: [ run-e2e-finalization-test, @@ -510,6 +610,7 @@ jobs: run-e2e-ban-manual, run-e2e-version-upgrade, run-e2e-failing-version-upgrade, + run-e2e-version-upgrade-catchup, ] name: Check e2e test suite completion runs-on: ubuntu-20.04 diff --git a/aleph-client/Cargo.toml b/aleph-client/Cargo.toml index 7f71b82ebe..9439f1960a 100644 --- a/aleph-client/Cargo.toml +++ b/aleph-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aleph_client" -version = "1.11.0" +version = "1.12.0" edition = "2021" license = "Apache 2.0" diff --git a/aleph-client/src/lib.rs b/aleph-client/src/lib.rs index cca7702d24..6b327c5350 100644 --- a/aleph-client/src/lib.rs +++ b/aleph-client/src/lib.rs @@ -57,7 +57,7 @@ pub use treasury::{ propose as make_treasury_proposal, reject as reject_treasury_proposal, staking_treasury_payout, treasury_account, }; -pub use version_upgrade::{schedule_upgrade, Version}; +pub use version_upgrade::{schedule_upgrade, schedule_upgrade_with_state, Version}; pub use vesting::{ get_schedules, merge_schedules, vest, vest_other, vested_transfer, VestingError, VestingSchedule, diff --git a/aleph-client/src/version_upgrade.rs b/aleph-client/src/version_upgrade.rs index 5265aaf6d3..f180d5437d 100644 --- a/aleph-client/src/version_upgrade.rs +++ b/aleph-client/src/version_upgrade.rs @@ -6,10 +6,11 @@ use crate::{try_send_xt, AnyConnection, RootConnection}; pub type Version = u32; -pub fn schedule_upgrade( +pub fn schedule_upgrade_with_state( connection: &RootConnection, version: Version, session: SessionIndex, + state: XtStatus, ) -> Result<(), ApiClientError> { let connection = connection.as_connection(); let upgrade_call = compose_call!( @@ -30,7 +31,15 @@ pub fn schedule_upgrade( &connection, xt, Some("schedule finality version change"), - XtStatus::Finalized, + state, ) .map(|_| ()) } + +pub fn schedule_upgrade( + connection: &RootConnection, + version: Version, + session: SessionIndex, +) -> Result<(), ApiClientError> { + schedule_upgrade_with_state(connection, version, session, XtStatus::Finalized) +} diff --git a/bin/cliain/src/commands.rs b/bin/cliain/src/commands.rs index adeea04533..ab49ce095b 100644 --- a/bin/cliain/src/commands.rs +++ b/bin/cliain/src/commands.rs @@ -3,9 +3,9 @@ use std::{ path::{Path, PathBuf}, }; -use aleph_client::BlockNumber; -use clap::{Args, Subcommand}; -use primitives::{Balance, CommitteeSeats}; +use aleph_client::{BlockNumber, XtStatus}; +use clap::{clap_derive::ValueEnum, Args, Subcommand}; +use primitives::{Balance, CommitteeSeats, SessionIndex}; use serde::{Deserialize, Serialize}; use sp_core::H256; use substrate_api_client::AccountId; @@ -111,6 +111,8 @@ pub struct ChangeValidatorArgs { pub committee_size: Option, } +pub type Version = u32; + impl std::str::FromStr for ChangeValidatorArgs { type Err = serde_json::Error; @@ -124,6 +126,21 @@ impl std::str::FromStr for ChangeValidatorArgs { } } +#[derive(Debug, Clone, ValueEnum)] +pub enum ExtrinsicState { + InBlock, + Finalized, +} + +impl From for XtStatus { + fn from(state: ExtrinsicState) -> Self { + match state { + ExtrinsicState::InBlock => XtStatus::InBlock, + ExtrinsicState::Finalized => XtStatus::Finalized, + } + } +} + #[derive(Debug, Clone, Subcommand)] pub enum Command { /// Staking call to bond stash with controller @@ -336,4 +353,16 @@ pub enum Command { /// Code can only be removed by its original uploader (its owner) and only if it is not used by any contract. /// API signature: https://polkadot.js.org/docs/substrate/extrinsics/#removecodecode_hash-h256 ContractRemoveCode(ContractRemoveCode), + + /// Schedules a version upgrade of the network. + VersionUpgradeSchedule { + #[clap(long)] + version: Version, + + #[clap(long)] + session: SessionIndex, + + #[clap(long, value_enum, default_value_t=ExtrinsicState::Finalized)] + expected_state: ExtrinsicState, + }, } diff --git a/bin/cliain/src/lib.rs b/bin/cliain/src/lib.rs index b69fea25fd..6a50a9b4ad 100644 --- a/bin/cliain/src/lib.rs +++ b/bin/cliain/src/lib.rs @@ -8,6 +8,7 @@ mod staking; mod transfer; mod treasury; mod validators; +mod version_upgrade; mod vesting; use aleph_client::{ @@ -27,6 +28,7 @@ pub use treasury::{ approve as treasury_approve, propose as treasury_propose, reject as treasury_reject, }; pub use validators::change_validators; +pub use version_upgrade::schedule_upgrade; pub use vesting::{vest, vest_other, vested_transfer}; pub struct ConnectionConfig { diff --git a/bin/cliain/src/main.rs b/bin/cliain/src/main.rs index 74338c9384..0e2bea0f39 100644 --- a/bin/cliain/src/main.rs +++ b/bin/cliain/src/main.rs @@ -8,9 +8,9 @@ use clap::Parser; use cliain::{ bond, call, change_validators, finalize, force_new_era, instantiate, instantiate_with_code, next_session_keys, nominate, owner_info, prepare_keys, prompt_password_hidden, remove_code, - rotate_keys, set_emergency_finalizer, set_keys, set_staking_limits, transfer, treasury_approve, - treasury_propose, treasury_reject, update_runtime, upload_code, validate, vest, vest_other, - vested_transfer, Command, ConnectionConfig, + rotate_keys, schedule_upgrade, set_emergency_finalizer, set_keys, set_staking_limits, transfer, + treasury_approve, treasury_propose, treasury_reject, update_runtime, upload_code, validate, + vest, vest_other, vested_transfer, Command, ConnectionConfig, }; use log::{error, info}; use sp_core::Pair; @@ -182,6 +182,14 @@ fn main() { Ok(result) => println!("{:?}", result), Err(why) => error!("Contract remove code failed {:?}", why), }, + Command::VersionUpgradeSchedule { + version, + session: session_for_upgrade, + expected_state, + } => match schedule_upgrade(cfg.into(), version, session_for_upgrade, expected_state) { + Ok(_) => {} + Err(why) => error!("Unable to schedule an upgrade {:?}", why), + }, } } diff --git a/bin/cliain/src/version_upgrade.rs b/bin/cliain/src/version_upgrade.rs new file mode 100644 index 0000000000..9f8b07f868 --- /dev/null +++ b/bin/cliain/src/version_upgrade.rs @@ -0,0 +1,20 @@ +use aleph_client::{schedule_upgrade_with_state, RootConnection}; +use anyhow::Error; +use primitives::SessionIndex; + +use crate::commands::{ExtrinsicState, Version}; + +pub fn schedule_upgrade( + connection: RootConnection, + version: Version, + session_for_upgrade: SessionIndex, + expected_state: ExtrinsicState, +) -> anyhow::Result<()> { + schedule_upgrade_with_state( + &connection, + version, + session_for_upgrade, + expected_state.into(), + ) + .map_err(Error::new) +} diff --git a/docker/common.yml b/docker/common.yml index d41d7f41b2..59a04adf45 100644 --- a/docker/common.yml +++ b/docker/common.yml @@ -1,7 +1,6 @@ services: AlephNodeService: image: aleph-node:latest - network_mode: host environment: - PURGE_BEFORE_START=true - RUST_LOG=info diff --git a/docker/docker-compose.base.yml b/docker/docker-compose.base.yml new file mode 100644 index 0000000000..6a159a6f19 --- /dev/null +++ b/docker/docker-compose.base.yml @@ -0,0 +1,79 @@ +# When increasing number of nodes in this file, change default value of --validators-count param in e2e-tests/src/config.rs + +services: + Node0: + extends: + file: common.yml + service: AlephBootNode + container_name: Node0 + environment: + - RPC_PORT=9933 + - WS_PORT=9943 + - PORT=30333 + - VALIDATOR_PORT=30343 + - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30343 + - NAME=Node0 + + Node1: + extends: + file: common.yml + service: AlephNonBootNode + container_name: Node1 + environment: + - RPC_PORT=9934 + - WS_PORT=9944 + - PORT=30334 + - VALIDATOR_PORT=30344 + - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30344 + - NAME=Node1 + # key derived from "//1" + - BASE_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o + - NODE_KEY_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o/p2p_secret + + Node2: + extends: + file: common.yml + service: AlephNonBootNode + container_name: Node2 + environment: + - RPC_PORT=9935 + - WS_PORT=9945 + - PORT=30335 + - VALIDATOR_PORT=30345 + - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30345 + - NAME=Node2 + # key derived from "//2" + - BASE_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9 + - NODE_KEY_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9/p2p_secret + + Node3: + extends: + file: common.yml + service: AlephNonBootNode + container_name: Node3 + environment: + - RPC_PORT=9936 + - WS_PORT=9946 + - PORT=30336 + - VALIDATOR_PORT=30346 + - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30346 + - NAME=Node3 + # key derived from "//3" + - BASE_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK + - NODE_KEY_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK/p2p_secret + + Node4: + extends: + file: common.yml + service: AlephNonBootNode + container_name: Node4 + environment: + - RPC_PORT=9937 + - WS_PORT=9947 + - PORT=30337 + - VALIDATOR_PORT=30347 + - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30347 + - NAME=Node4 + # key derived from "//4" + - BASE_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW + - NODE_KEY_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW/p2p_secret diff --git a/docker/docker-compose.bridged.yml b/docker/docker-compose.bridged.yml new file mode 100644 index 0000000000..2ee3ddc132 --- /dev/null +++ b/docker/docker-compose.bridged.yml @@ -0,0 +1,62 @@ +services: + Node0: + extends: + file: docker-compose.base.yml + service: Node0 + networks: + - main + - Node0 + + Node1: + extends: + file: docker-compose.base.yml + service: Node1 + networks: + - main + - Node1 + environment: + - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID + + Node2: + extends: + file: docker-compose.base.yml + service: Node2 + networks: + - main + - Node2 + environment: + - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID + + Node3: + extends: + file: docker-compose.base.yml + service: Node3 + networks: + - main + - Node3 + environment: + - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID + + Node4: + extends: + file: docker-compose.base.yml + service: Node4 + networks: + - main + - Node4 + environment: + - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID + +networks: + main: + name: main-network + Node0: + name: Node0-network + Node1: + name: Node1-network + Node2: + name: Node2-network + Node3: + name: Node3-network + Node4: + name: Node4-network diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c2957f7be7..154031e825 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,73 +3,30 @@ services: Node0: extends: - file: common.yml - service: AlephBootNode - container_name: Node0 - environment: - - RPC_PORT=9933 - - WS_PORT=9943 - - PORT=30333 - - VALIDATOR_PORT=30343 - - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30343 - - NAME=Node0 + file: docker-compose.base.yml + service: Node0 + network_mode: host Node1: extends: - file: common.yml - service: AlephNonBootNode - container_name: Node1 - environment: - - RPC_PORT=9934 - - WS_PORT=9944 - - PORT=30334 - - VALIDATOR_PORT=30344 - - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30344 - - NAME=Node1 - - BASE_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o - - NODE_KEY_PATH=/data/5GBNeWRhZc2jXu7D55rBimKYDk8PGk8itRYFTPfC8RJLKG5o/p2p_secret + file: docker-compose.base.yml + service: Node1 + network_mode: host Node2: extends: - file: common.yml - service: AlephNonBootNode - container_name: Node2 - environment: - - RPC_PORT=9935 - - WS_PORT=9945 - - PORT=30335 - - VALIDATOR_PORT=30345 - - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30345 - - NAME=Node2 - - BASE_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9 - - NODE_KEY_PATH=/data/5Dfis6XL8J2P6JHUnUtArnFWndn62SydeP8ee8sG2ky9nfm9/p2p_secret + file: docker-compose.base.yml + service: Node2 + network_mode: host Node3: extends: - file: common.yml - service: AlephNonBootNode - container_name: Node3 - environment: - - RPC_PORT=9936 - - WS_PORT=9946 - - PORT=30336 - - VALIDATOR_PORT=30346 - - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30346 - - NAME=Node3 - - BASE_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK - - NODE_KEY_PATH=/data/5F4H97f7nQovyrbiq4ZetaaviNwThSVcFobcA5aGab6167dK/p2p_secret + file: docker-compose.base.yml + service: Node3 + network_mode: host Node4: extends: - file: common.yml - service: AlephNonBootNode - container_name: Node4 - environment: - - RPC_PORT=9937 - - WS_PORT=9947 - - PORT=30337 - - VALIDATOR_PORT=30347 - - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30347 - - NAME=Node4 - - BASE_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW - - NODE_KEY_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW/p2p_secret + file: docker-compose.base.yml + service: Node4 + network_mode: host diff --git a/e2e-tests/Cargo.lock b/e2e-tests/Cargo.lock index a5f686c588..30eb393cb1 100644 --- a/e2e-tests/Cargo.lock +++ b/e2e-tests/Cargo.lock @@ -121,7 +121,7 @@ dependencies = [ [[package]] name = "aleph_client" -version = "1.11.0" +version = "1.12.0" dependencies = [ "ac-node-api", "ac-primitives", diff --git a/scripts/catchup_version_upgrade_test.sh b/scripts/catchup_version_upgrade_test.sh new file mode 100755 index 0000000000..baeb1a3c5b --- /dev/null +++ b/scripts/catchup_version_upgrade_test.sh @@ -0,0 +1,205 @@ +#!/bin/bash + +set -euo pipefail + +INIT_BLOCK=${INIT_BLOCK:-3} +UPGRADE_BLOCK=${UPGRADE_BLOCK:-31} +UPGRADE_VERSION=${UPGRADE_VERSION:-1} +NODES=${NODES:-"Node1:Node2"} +PORTS=${PORTS:-9934:9935} +UPGRADE_BEFORE_DISABLE=${UPGRADE_BEFORE_DISABLE:-false} +SEED=${SEED:-"//0"} +ALL_NODES=${ALL_NODES:-"Node0:Node1:Node2:Node3:Node4"} +ALL_NODES_PORTS=${ALL_NODES_PORTS:-"9933:9934:9935:9936:9937"} +WAIT_BLOCKS=${WAIT_BLOCKS:-30} +EXT_STATUS=${EXT_STATUS:-"in-block"} + +function log() { + echo $1 1>&2 +} + +function into_array() { + result=() + local tmp=$IFS + IFS=: + for e in $1; do + result+=($e) + done + IFS=$tmp +} + +function initialize { + wait_for_finalized_block $1 $2 $3 +} + +function wait_for_finalized_block() { + local block_to_be_finalized=$1 + local node=$2 + local port=$3 + + while [[ $(get_best_finalized $node $port) -le $block_to_be_finalized ]]; do + sleep 3 + done +} + +function get_best_finalized { + local validator=$1 + local rpc_port=$2 + + local best_finalized=$(VALIDATOR=$validator RPC_HOST="127.0.0.1" RPC_PORT=$rpc_port ./.github/scripts/check_finalization.sh | sed 's/Last finalized block number: "\(.*\)"/\1/') + printf "%d" $best_finalized +} + +function set_upgrade_session { + local session=$1 + local version=$2 + local validator=$3 + local port=$4 + local seed=$5 + local status=$6 + + docker run --rm --network container:$validator cliain:latest --node 127.0.0.1:$port --seed $seed version-upgrade-schedule --version $version --session $session --expected-state $status +} + +function check_if_disconnected() { + local -n nodes=$1 + local -n ports=$2 + + log "checking if nodes are disconnected" + + for i in "${!nodes[@]}"; do + local node=${nodes[$i]} + local port=${ports[$i]} + + log "checking if node $node is disconnected" + + last_finalized=$(get_best_finalized $node $port) + log "last finalized block at node $node is $last_finalized" + + last_block=$(get_last_block $node $port) + log "last block at node $node is $last_block" + + # what else we can do? + log "sleeping for 20 seconds" + sleep 20 + + new_finalized=$(get_best_finalized $node $port) + log "newest finalized block at node $node after waiting is $new_finalized" + + if [[ $(($new_finalized - $last_finalized)) -ge 1 ]]; then + log "somehow a disconnected node $node was able to finalize new blocks" + exit -1 + fi + done +} + +function connect_nodes { + local -n nodes=$1 + for node in ${nodes[@]}; do + docker network connect main-network $node + done +} + +function disconnect_nodes { + local -n nodes=$1 + + for node in ${nodes[@]}; do + log "disconnecting node $node..." + docker network disconnect main-network $node + log "node $node disconnected" + done +} + +function wait_for_block { + local block=$1 + local validator=$2 + local rpc_port=$3 + + local last_block="" + while [[ -z "$last_block" ]]; do + last_block=$(docker run --rm --network container:$validator appropriate/curl:latest \ + -H "Content-Type: application/json" \ + -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$block'}' http://127.0.0.1:$rpc_port | jq '.result') + done +} + +function get_last_block { + local validator=$1 + local rpc_port=$2 + + local last_block_number=$(docker run --rm --network container:$validator appropriate/curl:latest \ + -H "Content-Type: application/json" \ + -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock"}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number') + printf "%d" $last_block_number +} + +function check_finalization { + local block_to_check=$1 + local -n nodes=$2 + local -n ports=$3 + + log "checking finalization for block $block_to_check" + + for i in "${!nodes[@]}"; do + local node=${nodes[$i]} + local rpc_port=${ports[$i]} + + log "checking finalization at node $node" + wait_for_finalized_block $block_to_check $node $rpc_port + done +} + +into_array $NODES +NODES=(${result[@]}) + +into_array $PORTS +PORTS=(${result[@]}) + +into_array $ALL_NODES +ALL_NODES=(${result[@]}) + +into_array "$ALL_NODES_PORTS" +ALL_NODES_PORTS=(${result[@]}) + +log "initializing nodes..." +DOCKER_COMPOSE=./docker/docker-compose.bridged.yml ./.github/scripts/run_consensus.sh 1>&2 +log "awaiting finalization of $INIT_BLOCK blocks..." +initialize $INIT_BLOCK "Node0" 9933 +log "nodes initialized" + +last_block=$(get_last_block "Node0" 9933) +block_for_upgrade=$(($UPGRADE_BLOCK + $last_block)) +if [[ $UPGRADE_BEFORE_DISABLE = true ]]; then + log "setting upgrade at $block_for_upgrade block for version $UPGRADE_VERSION before disconnecting" + set_upgrade_session $block_for_upgrade $UPGRADE_VERSION "Node0" 9943 $SEED $EXT_STATUS +fi + +log "disconnecting nodes..." +disconnect_nodes NODES +log "verifying if nodes are properly disconnected..." +check_if_disconnected NODES PORTS +log "nodes disconnected" + +last_block=$(get_last_block "Node0" 9933) +block_for_upgrade=$(($UPGRADE_BLOCK + $last_block)) +if [[ $UPGRADE_BEFORE_DISABLE = false ]]; then + log "setting upgrade at $block_for_upgrade block for version $UPGRADE_VERSION" + set_upgrade_session $block_for_upgrade $UPGRADE_VERSION "Node0" 9943 $SEED $EXT_STATUS +fi + +last_block=$(get_last_block "Node0" 9933) +awaited_block=$(($WAIT_BLOCKS+$block_for_upgrade)) +log "awaiting block $awaited_block" +wait_for_block $awaited_block "Node0" 9933 +log "awaiting finished" + +log "connecting nodes..." +connect_nodes NODES +log "nodes connected" + +last_block=$(get_last_block "Node0" 9933) +log "checking finalization..." +check_finalization $(($awaited_block+1)) ALL_NODES ALL_NODES_PORTS +log "finalization checked" + +exit $?