From 851cd9c92faa2acfe1f8e31bef9bc403425cdd15 Mon Sep 17 00:00:00 2001 From: ivanl Date: Sun, 7 Nov 2021 00:17:49 +0300 Subject: [PATCH 1/2] get rid of ETH_TOKEN_MINT and COLLATERAL_POOL_BASE variables in source code --- proxy/environment.py | 60 +++++++++++++++++++++++++ proxy/indexer/solana_receipts_update.py | 4 +- proxy/indexer/utils.py | 9 ++-- proxy/plugin/solana_rest_api.py | 5 ++- proxy/plugin/solana_rest_api_tools.py | 53 +--------------------- proxy/test_environment.py | 2 +- 6 files changed, 70 insertions(+), 63 deletions(-) create mode 100644 proxy/environment.py diff --git a/proxy/environment.py b/proxy/environment.py new file mode 100644 index 000000000..df86e2042 --- /dev/null +++ b/proxy/environment.py @@ -0,0 +1,60 @@ +import os +import subprocess +import logging +from solana.publickey import PublicKey + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") +evm_loader_id = os.environ.get("EVM_LOADER") +neon_cli_timeout = float(os.environ.get("NEON_CLI_TIMEOUT", "0.1")) + +class solana_cli: + def call(self, *args): + try: + cmd = ["solana", + "--url", solana_url, + ] + list(args) + print(cmd) + return subprocess.check_output(cmd, universal_newlines=True) + except subprocess.CalledProcessError as err: + import sys + logger.debug("ERR: solana error {}".format(err)) + raise + + +class neon_cli: + def call(self, *args): + try: + cmd = ["neon-cli", + "--commitment=recent", + "--url", solana_url, + "--evm_loader={}".format(evm_loader_id), + ] + list(args) + print(cmd) + return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) + except subprocess.CalledProcessError as err: + import sys + logger.debug("ERR: neon-cli error {}".format(err)) + raise + +def read_elf_params(out_dict): + logger.debug('load for solana_url={} and evm_loader_id={}'.format(solana_url, evm_loader_id)) + res = solana_cli().call('program', 'dump', evm_loader_id, './evm_loader.dump') + substr = "Wrote program to " + path = "" + for line in res.splitlines(): + if line.startswith(substr): + path = line[len(substr):].strip() + if path == "": + raise Exception("cannot program dump for ", evm_loader_id) + for param in neon_cli().call("neon-elf-params", path).splitlines(): + if param.startswith('NEON_') and '=' in param: + v = param.split('=') + out_dict[v[0]] = v[1] + +ELF_PARAMS = {} +read_elf_params(ELF_PARAMS) +COLLATERAL_POOL_BASE = ELF_PARAMS.get("NEON_POOL_BASE") +ETH_TOKEN_MINT_ID: PublicKey = PublicKey(ELF_PARAMS.get("NEON_TOKEN_MINT")) diff --git a/proxy/indexer/solana_receipts_update.py b/proxy/indexer/solana_receipts_update.py index e1de7858b..6e9502ca6 100644 --- a/proxy/indexer/solana_receipts_update.py +++ b/proxy/indexer/solana_receipts_update.py @@ -8,6 +8,7 @@ from multiprocessing.dummy import Pool as ThreadPool from sqlitedict import SqliteDict from typing import Dict, Union +from proxy.environment import solana_url, evm_loader_id try: @@ -16,11 +17,8 @@ from .utils import check_error, get_trx_results, get_trx_receipts, LogDB, Canceller -solana_url = os.environ.get("SOLANA_URL", "https://api.devnet.solana.com") -evm_loader_id = os.environ.get("EVM_LOADER", "eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU") PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2")) - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index dcba390d0..e008e0632 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -19,11 +19,8 @@ from spl.token.constants import TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address from web3.auto.gethdev import w3 +from proxy.environment import solana_url, evm_loader_id, ETH_TOKEN_MINT_ID - -solana_url = os.environ.get("SOLANA_URL", "https://api.devnet.solana.com") -evm_loader_id = os.environ.get("EVM_LOADER", "eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU") -ETH_TOKEN_MINT_ID = os.environ.get("ETH_TOKEN_MINT", "89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g") sysvarclock = "SysvarC1ock11111111111111111111111111111111" sysinstruct = "Sysvar1nstructions1111111111111111111111111" keccakprog = "KeccakSecp256k11111111111111111111111111111" @@ -303,7 +300,7 @@ def __init__(self): self.client = Client(solana_url) self.operator = self.signer.public_key() - self.operator_token = get_associated_token_address(PublicKey(self.operator), PublicKey(ETH_TOKEN_MINT_ID)) + self.operator_token = get_associated_token_address(PublicKey(self.operator), ETH_TOKEN_MINT_ID) def call(self, *args): @@ -321,7 +318,7 @@ def call(self, *args): def unlock_accounts(self, blocked_storages): readonly_accs = [ PublicKey(evm_loader_id), - PublicKey(ETH_TOKEN_MINT_ID), + ETH_TOKEN_MINT_ID, PublicKey(TOKEN_PROGRAM_ID), PublicKey(sysvarclock), PublicKey(sysinstruct), diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 1d85409b0..7d1abb36e 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -25,8 +25,8 @@ import base58 import traceback import threading -from .solana_rest_api_tools import EthereumAddress, create_account_with_seed, evm_loader_id, getTokens, \ - getAccountInfo, solana_cli, call_signed, solana_url, call_emulated, \ +from .solana_rest_api_tools import EthereumAddress, create_account_with_seed, getTokens, \ + getAccountInfo, call_signed, call_emulated, \ Trx, EthereumError, create_collateral_pool_address, getTokenAddr, STORAGE_SIZE, neon_config_load, MINIMAL_GAS_PRICE from solana.rpc.commitment import Commitment, Confirmed from web3 import Web3 @@ -35,6 +35,7 @@ import os from ..indexer.utils import get_trx_results, LogDB from sqlitedict import SqliteDict +from proxy.environment import evm_loader_id, solana_cli, solana_url logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index c861e15e9..ccde6f766 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -6,7 +6,6 @@ import random import re import struct -import subprocess import time from datetime import datetime from hashlib import sha256 @@ -29,20 +28,17 @@ from spl.token.constants import ACCOUNT_LEN, ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address, create_associated_token_account, transfer2, Transfer2Params from web3.auto import w3 - +from proxy.environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params from .eth_proto import Trx logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") -evm_loader_id = os.environ.get("EVM_LOADER") -COLLATERAL_POOL_BASE = os.environ.get("COLLATERAL_POOL_BASE") + NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) #evm_loader_id = "EfyDoGDRPy7wrLfSLyXrbhiAG6NmufMk1ytap13gLy1" location_bin = ".deploy_contract.bin" confirmation_check_delay = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) -neon_cli_timeout = float(os.environ.get("NEON_CLI_TIMEOUT", "0.1")) USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "YES") == "YES" CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) @@ -57,10 +53,6 @@ incinerator = "1nc1nerator11111111111111111111111111111111" system = "11111111111111111111111111111111" -ETH_TOKEN_MINT_ID: PublicKey = PublicKey( - os.environ.get("ETH_TOKEN_MINT", "HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU") -) - STORAGE_SIZE = 128*1024 ACCOUNT_INFO_LAYOUT = cStruct( @@ -245,34 +237,7 @@ def emulator(contract, sender, data, value): return neon_cli().call("emulate", sender, contract, data, value) -class solana_cli: - def call(self, *args): - try: - cmd = ["solana", - "--url", solana_url, - ] + list(args) - print(cmd) - return subprocess.check_output(cmd, universal_newlines=True) - except subprocess.CalledProcessError as err: - import sys - logger.debug("ERR: solana error {}".format(err)) - raise - -class neon_cli: - def call(self, *args): - try: - cmd = ["neon-cli", - "--commitment=recent", - "--url", solana_url, - "--evm_loader={}".format(evm_loader_id), - ] + list(args) - print(cmd) - return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) - except subprocess.CalledProcessError as err: - import sys - logger.debug("ERR: neon-cli error {}".format(err)) - raise def confirm_transaction(client, tx_sig, confirmations=0): """Confirm a transaction.""" @@ -318,20 +283,6 @@ def ether2seed(ether, program_id, base): logger.debug('ether2program: {} {} => {} (seed {})'.format(ether, 255, acc, seed)) return (acc, 255, seed) -def read_elf_params(out_dict): - logger.debug('load for solana_url={} and evm_loader_id={}'.format(solana_url, evm_loader_id)) - res = solana_cli().call('program', 'dump', evm_loader_id, './evm_loader.dump') - substr = "Wrote program to " - path = "" - for line in res.splitlines(): - if line.startswith(substr): - path = line[len(substr):].strip() - if path == "": - raise Exception("cannot program dump for ", evm_loader_id) - for param in neon_cli().call("neon-elf-params", path).splitlines(): - if param.startswith('NEON_') and '=' in param: - v = param.split('=') - out_dict[v[0]] = v[1] def neon_config_load(ethereum_model): try: diff --git a/proxy/test_environment.py b/proxy/test_environment.py index 36000be41..9cf96cb4f 100644 --- a/proxy/test_environment.py +++ b/proxy/test_environment.py @@ -1,7 +1,7 @@ import unittest import os from web3 import Web3 -from proxy.plugin.solana_rest_api_tools import read_elf_params +from proxy.environment import read_elf_params proxy_url = os.environ.get('PROXY_URL', 'http://127.0.0.1:9090/solana') proxy = Web3(Web3.HTTPProvider(proxy_url)) From eb71937d983cb569c314424cb7c93ca900cd9473 Mon Sep 17 00:00:00 2001 From: ivanl Date: Sun, 7 Nov 2021 00:39:20 +0300 Subject: [PATCH 2/2] edit run-proxy --- proxy/run-proxy.sh | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/proxy/run-proxy.sh b/proxy/run-proxy.sh index f356cd3ee..e03413fe3 100755 --- a/proxy/run-proxy.sh +++ b/proxy/run-proxy.sh @@ -7,8 +7,6 @@ date if [ "$CONFIG" == "ci" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://solana:8899" [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=deploy - [[ -z "$ETH_TOKEN_MINT" ]] && export ETH_TOKEN_MINT=deploy - [[ -z "$COLLATERAL_POOL_BASE" ]] && export COLLATERAL_POOL_BASE=deploy [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=100 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=100000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.5" @@ -18,8 +16,6 @@ if [ "$CONFIG" == "ci" ]; then elif [ "$CONFIG" == "local" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://localhost:8899" [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=deploy - [[ -z "$ETH_TOKEN_MINT" ]] && export ETH_TOKEN_MINT=deploy - [[ -z "$COLLATERAL_POOL_BASE" ]] && export COLLATERAL_POOL_BASE=deploy [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=10 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=0 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.9" @@ -27,8 +23,6 @@ elif [ "$CONFIG" == "local" ]; then elif [ "$CONFIG" == "devnet" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.devnet.solana.com" [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU - [[ -z "$ETH_TOKEN_MINT" ]] && export ETH_TOKEN_MINT=89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g - [[ -z "$COLLATERAL_POOL_BASE" ]] && export COLLATERAL_POOL_BASE=7SBdHNeF9FFYySEoszpjZXXQsAiwa5Lzpsz6nUJWusEx [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=0 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="10" @@ -36,8 +30,6 @@ elif [ "$CONFIG" == "devnet" ]; then elif [ "$CONFIG" == "testnet" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.testnet.solana.com" [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU - [[ -z "$ETH_TOKEN_MINT" ]] && export ETH_TOKEN_MINT=89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g - [[ -z "$COLLATERAL_POOL_BASE" ]] && export COLLATERAL_POOL_BASE=7SBdHNeF9FFYySEoszpjZXXQsAiwa5Lzpsz6nUJWusEx [[ -z "$NEW_USER_AIRDROP_AMOUNT" ]] && export NEW_USER_AIRDROP_AMOUNT=0 [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=90000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="15" @@ -48,8 +40,6 @@ fi [[ -z "$SOLANA_URL" ]] && echo "SOLANA_URL is not set" && exit 1 [[ -z "$EVM_LOADER" ]] && echo "EVM_LOADER is not set" && exit 1 -[[ -z "$ETH_TOKEN_MINT" ]] && echo "ETH_TOKEN_MINT is not set" && exit 1 -[[ -z "$COLLATERAL_POOL_BASE" ]] && echo "COLLATERAL_POOL_BASE is not set" && exit 1 echo SOLANA_URL=$SOLANA_URL @@ -96,22 +86,15 @@ fi echo "EVM_LOADER=$EVM_LOADER" -if [ "$ETH_TOKEN_MINT" == "deploy" ]; then - echo "ETH_TOKEN_MINT is set to load. A new token will be created. Creating token..." - export ETH_TOKEN_MINT=$(/spl/bin/spl-token create-token --owner /spl/bin/test_token_owner -- /spl/bin/test_token_keypair | grep -Po 'Creating token \K[^\n]*') -fi - +echo "A new token will be created. Creating token..." +export ETH_TOKEN_MINT=$(/spl/bin/spl-token create-token --owner /spl/bin/test_token_owner -- /spl/bin/test_token_keypair | grep -Po 'Creating token \K[^\n]*') echo "ETH_TOKEN_MINT=$ETH_TOKEN_MINT" -if [ "$COLLATERAL_POOL_BASE" == "deploy" ]; then - echo "COLLATERAL_POOL_BASE is set to create. A new collateral pool accounts will be created. Creating accounts..." - #generate collateral pool accounts - solana -k /spl/bin/collateral-pool-keypair.json airdrop 1000 - python3 /spl/bin/collateral_pool_generator.py /spl/bin/collateral-pool-keypair.json - export COLLATERAL_POOL_BASE=$(solana-keygen pubkey -f /spl/bin/collateral-pool-keypair.json) -fi - +echo "A new collateral pool accounts will be created. Creating accounts..." +solana -k /spl/bin/collateral-pool-keypair.json airdrop 1000 +python3 /spl/bin/collateral_pool_generator.py /spl/bin/collateral-pool-keypair.json +export COLLATERAL_POOL_BASE=$(solana-keygen pubkey -f /spl/bin/collateral-pool-keypair.json) echo "COLLATERAL_POOL_BASE=$COLLATERAL_POOL_BASE"