diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 72b360833..5ac8aa1a5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -18,6 +18,7 @@ steps: - "evm_loader.log" - "faucet.log" - "airdropper.log" + - "indexer.log" - wait diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index 13799692a..90113540b 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -46,6 +46,7 @@ function cleanup_docker { if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi if docker logs faucet >faucet.log 2>&1; then echo "faucet logs saved"; fi if docker logs airdropper >airdropper.log 2>&1; then echo "airdropper logs saved"; fi + if docker logs indexer >indexer.log 2>&1; then echo "indexer logs saved"; fi echo "\nCleanup docker-compose..." docker-compose -f proxy/docker-compose-test.yml down diff --git a/proxy/__main__.py b/proxy/__main__.py index 9d4d1861f..019f9e733 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -13,10 +13,12 @@ from .proxy import entry_point import os from .indexer.airdropper import run_airdropper +from .indexer.indexer import run_indexer from solana.rpc.api import Client if __name__ == '__main__': airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True'] + indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True'] if airdropper_mode: print("Will run in airdropper mode") solana_url = os.environ['SOLANA_URL'] @@ -39,5 +41,12 @@ neon_decimals, pp_solana_url, max_conf) + elif indexer_mode: + print("Will run in indexer mode") + + solana_url = os.environ['SOLANA_URL'] + evm_loader_id = os.environ['EVM_LOADER'] + + run_indexer(solana_url, evm_loader_id) else: entry_point() diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 0c89e9ba4..968219111 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -76,6 +76,8 @@ services: condition: service_healthy evm_loader: condition: service_completed_successfully + indexer: + condition: service_started ports: - 127.0.0.1:9090:9090 expose: @@ -137,6 +139,25 @@ services: faucet: condition: service_started + indexer: + container_name: indexer + image: neonlabsorg/proxy:${REVISION} + environment: + SOLANA_URL: http://solana:8899 + POSTGRES_DB: neon-db + POSTGRES_USER: neon-proxy + POSTGRES_PASSWORD: neon-proxy-pass + CONFIG: ci + hostname: indexer + depends_on: + postgres: + condition: service_healthy + evm_loader: + condition: service_completed_successfully + networks: + - net + entrypoint: proxy/run-indexer.sh + networks: net: diff --git a/proxy/proxy.py b/proxy/proxy.py index ea3dbcdf3..55bd3f968 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -22,8 +22,6 @@ from .http.handler import HttpProtocolHandler from multiprocessing import Process -from .indexer.indexer import run_indexer -from proxy.environment import SOLANA_URL, EVM_LOADER_ID class Proxy: @@ -43,10 +41,6 @@ def delete_pid_file(self) -> None: os.remove(self.flags.pid_file) def __enter__(self) -> 'Proxy': - self.indexer = Process(target=run_indexer, - args=(SOLANA_URL, - EVM_LOADER_ID,)) - self.indexer.start() self.acceptors = AcceptorPool( flags=self.flags, work_klass=HttpProtocolHandler diff --git a/proxy/run-indexer.sh b/proxy/run-indexer.sh new file mode 100755 index 000000000..ba3cd0628 --- /dev/null +++ b/proxy/run-indexer.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ -z "$EVM_LOADER" ]; then + echo "Extracting EVM_LOADER address from keypair file..." + export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) + echo "EVM_LOADER=$EVM_LOADER" +fi +export INDEXER_MODE='true' + +source proxy/run-set-env.sh + +python3 -m proxy diff --git a/proxy/run-proxy.sh b/proxy/run-proxy.sh index 011aad25d..7aaaf01c7 100755 --- a/proxy/run-proxy.sh +++ b/proxy/run-proxy.sh @@ -1,75 +1,6 @@ #!/bin/bash -set -xeo pipefail - -date - -if [ "$CONFIG" == "ci" ]; then - [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://solana:8899" - [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=100000 - [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.5" - [[ -z "$CONTINUE_COUNT_FACTOR" ]] && export CONTINUE_COUNT_FACTOR="3" - [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 - [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="postgres" - [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=10 - [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 - [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" - [[ -z "$START_SLOT" ]] && export START_SLOT="LATEST" - [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=10 - [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=2 -elif [ "$CONFIG" == "local" ]; then - [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://localhost:8899" - [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=0 - [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.9" - [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 - [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="localhost" - [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=10 - [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 - [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" - [[ -z "$START_SLOT" ]] && export START_SLOT=0 - [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=10 - [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=2 -elif [ "$CONFIG" == "devnet" ]; then - [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.devnet.solana.com" - [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU - [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 - [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="10" - [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=1 - [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="localhost" - [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=60 - [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 - [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" - [[ -z "$START_SLOT" ]] && export START_SLOT="LATEST" - [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=30 - [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=16 -elif [ "$CONFIG" == "testnet" ]; then - [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.testnet.solana.com" - [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU - [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 - [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="15" - [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE="1" - [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="localhost" - [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=60 - [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 - [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" - [[ -z "$START_SLOT" ]] && export START_SLOT="LATEST" - [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=30 - [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=16 -elif [ "$CONFIG" != "custom" ]; then - exit 1 -fi - -[[ -z "$SOLANA_URL" ]] && echo "SOLANA_URL is not set" && exit 1 -[[ -z "$EVM_LOADER" ]] && echo "EVM_LOADER is not set" && exit 1 - -solana config set -u $SOLANA_URL - -isArg() { case "$1" in "$2"|"$2="*) true;; *) false;; esac } -EXTRA_ARGS_TIMEOUT=' --timeout 300' -for val in $EXTRA_ARGS; do - isArg $val '--timeout' && EXTRA_ARGS_TIMEOUT='' -done -EXTRA_ARGS+=$EXTRA_ARGS_TIMEOUT +source proxy/run-set-env.sh echo run-proxy python3 -m proxy --hostname 0.0.0.0 --port 9090 --enable-web-server --plugins proxy.plugin.SolanaProxyPlugin $EXTRA_ARGS diff --git a/proxy/run-set-env.sh b/proxy/run-set-env.sh new file mode 100755 index 000000000..886ff94b2 --- /dev/null +++ b/proxy/run-set-env.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -xeo pipefail + +date + +if [ "$CONFIG" == "ci" ]; then + [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://solana:8899" + [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=100000 + [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.5" + [[ -z "$CONTINUE_COUNT_FACTOR" ]] && export CONTINUE_COUNT_FACTOR="3" + [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 + [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="postgres" + [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=10 + [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 + [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" + [[ -z "$START_SLOT" ]] && export START_SLOT="LATEST" + [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=10 + [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=2 +elif [ "$CONFIG" == "local" ]; then + [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://localhost:8899" + [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=0 + [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.9" + [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 + [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="localhost" + [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=10 + [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 + [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" + [[ -z "$START_SLOT" ]] && export START_SLOT=0 + [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=10 + [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=2 +elif [ "$CONFIG" == "devnet" ]; then + [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.devnet.solana.com" + [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU + [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 + [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="10" + [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=1 + [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="localhost" + [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=60 + [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 + [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" + [[ -z "$START_SLOT" ]] && export START_SLOT="LATEST" + [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=30 + [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=16 +elif [ "$CONFIG" == "testnet" ]; then + [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.testnet.solana.com" + [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU + [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 + [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="15" + [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE="1" + [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="localhost" + [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=60 + [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 + [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" + [[ -z "$START_SLOT" ]] && export START_SLOT="LATEST" + [[ -z "$CONFIRM_TIMEOUT" ]] && export CONFIRM_TIMEOUT=30 + [[ -z "$PERM_ACCOUNT_LIMIT" ]] && export PERM_ACCOUNT_LIMIT=16 +elif [ "$CONFIG" != "custom" ]; then + exit 1 +fi + +[[ -z "$SOLANA_URL" ]] && echo "SOLANA_URL is not set" && exit 1 +[[ -z "$EVM_LOADER" ]] && echo "EVM_LOADER is not set" && exit 1 + +solana config set -u $SOLANA_URL + +isArg() { case "$1" in "$2"|"$2="*) true;; *) false;; esac } +EXTRA_ARGS_TIMEOUT=' --timeout 300' +for val in $EXTRA_ARGS; do + isArg $val '--timeout' && EXTRA_ARGS_TIMEOUT='' +done +EXTRA_ARGS+=$EXTRA_ARGS_TIMEOUT