From e496b270d663d8b9cb26005d11d18606857fb9dd Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 16 Nov 2021 14:12:20 +0300 Subject: [PATCH 01/54] VERSION = (0, 4, 0) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0d2c67516..94ebe2b28 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ """ from setuptools import setup, find_packages -VERSION = (2, 2, 0) +VERSION = (0, 4, 0) __version__ = '.'.join(map(str, VERSION[0:3])) __description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on Network monitoring, controls & Application development, testing, debugging.''' From fe41d64db0eae980a1b4f7539664b80c6acbf579 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 16 Nov 2021 14:25:23 +0300 Subject: [PATCH 02/54] EVM_LOADER_REVISION:=v0.4.0 --- .buildkite/steps/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index d8becc316..b9575b58f 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -4,7 +4,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) set ${SOLANA_REVISION:=v1.7.9-resources} -set ${EVM_LOADER_REVISION:=stable} +set ${EVM_LOADER_REVISION:=v0.4.0} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally docker pull neonlabsorg/solana:${SOLANA_REVISION} From 5467c50e3f65d90f60f4e497242332b4c88bba89 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 16 Nov 2021 14:33:13 +0300 Subject: [PATCH 03/54] EVM_LOADER_REVISION:=stable --- .buildkite/steps/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index b9575b58f..d8becc316 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -4,7 +4,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) set ${SOLANA_REVISION:=v1.7.9-resources} -set ${EVM_LOADER_REVISION:=v0.4.0} +set ${EVM_LOADER_REVISION:=stable} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally docker pull neonlabsorg/solana:${SOLANA_REVISION} From 3c6f4bcadff8324bae0723734aad72fa3b9cee00 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 18 Nov 2021 11:02:56 +0300 Subject: [PATCH 04/54] 289 implement eth get storage at (#298) * create eth_getStorageAt prototype * implement eth_getStorageAt * fix test * Fix review issues, add UTs * handle neon-cli errors * add test Co-authored-by: ivanl --- proxy/plugin/solana_rest_api.py | 17 ++++++++++++++- proxy/test_eth_sendRawTransaction.py | 32 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 54a19ecf4..ba0b1fb45 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -34,7 +34,7 @@ import os from ..indexer.utils import get_trx_results, LogDB from ..indexer.sql_dict import SQLDict -from proxy.environment import evm_loader_id, solana_cli, solana_url +from proxy.environment import evm_loader_id, solana_cli, solana_url, neon_cli logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -195,6 +195,21 @@ def getBlockBySlot(self, slot, full): } return ret + def eth_getStorageAt(self, account, position, block_identifier): + '''Retrieves storage data by given position + Currently supports only 'latest' block + ''' + if block_identifier != "latest": + logger.debug(f"Block type '{block_identifier}' is not supported yet") + raise RuntimeError(f"Not supported block identifier: {block_identifier}") + + try: + value = neon_cli().call('get-storage-at', account, position) + return value + except Exception as err: + logger.debug(f"Neon-cli failed to execute: {err}") + return '0x00' + def eth_getBlockByHash(self, trx_hash, full): """Returns information about a block by hash. trx_hash - Hash of a block. diff --git a/proxy/test_eth_sendRawTransaction.py b/proxy/test_eth_sendRawTransaction.py index 35ac90651..12e3f4633 100644 --- a/proxy/test_eth_sendRawTransaction.py +++ b/proxy/test_eth_sendRawTransaction.py @@ -428,6 +428,38 @@ def test_07_execute_long_transaction(self): print('times_to_calculate:', times_to_calculate) print('time_duration:', time_duration) + def test_get_storage_at(self): + print("\nhttps://github.com/neonlabsorg/proxy-model.py/issues/289") + right_nonce = proxy.eth.get_transaction_count(proxy.eth.default_account) + value_to_store = 452356 + trx_store = self.storage_contract.functions.store(value_to_store).buildTransaction({'nonce': right_nonce}) + print('trx_store:', trx_store) + trx_store_signed = proxy.eth.account.sign_transaction(trx_store, eth_account.key) + print('trx_store_signed:', trx_store_signed) + trx_store_hash = proxy.eth.send_raw_transaction(trx_store_signed.rawTransaction) + print('trx_store_hash:', trx_store_hash.hex()) + trx_store_receipt = proxy.eth.wait_for_transaction_receipt(trx_store_hash) + print('trx_store_receipt:', trx_store_receipt) + + number_pos = 0 + value_received = proxy.eth.get_storage_at(self.storage_contract.address, number_pos, "latest") + print('eth_getStorageAt existing address and index => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), value_to_store) + + non_existing_pos = 12 + value_received = proxy.eth.get_storage_at(self.storage_contract.address, non_existing_pos, "latest") + print('eth_getStorageAt existing address and non-existing index => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) + + non_exising_address = b'\xe1\xda\xb7\xa6\x17\x6f\x87\x68\xF5\x3a\x42\x5f\x29\x61\x73\x60\x5e\xd5\x08\x32' + value_received = proxy.eth.get_storage_at(non_exising_address, non_existing_pos, "latest") + print('eth_getStorageAt non-existing address => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) + + not_a_contract_address = proxy.eth.default_account + value_received = proxy.eth.get_storage_at(not_a_contract_address, 0, "latest") + print('eth_getStorageAt not_a_contract_address address => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) if __name__ == '__main__': unittest.main() From 6e1864e98554f2c5480d6beb614906f3c8f5aa11 Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Thu, 18 Nov 2021 17:18:47 +0500 Subject: [PATCH 05/54] #256 create and airdrop eth account (#259) * constants and utils * solana rest api tools imports * comments and tabs * rename calling getTokens function * Meet SolanaErrors * Implement creating account on getting balance * Move tests and add get_from_dict test * Add test "Metamask creates an account" * Just not ot loose changes * implemented * Fix tests on CI * set NEW_USER_AIRDROP_AMOUNT on CI * improve logging * improve logging of SendTransactionError * spit and polish * extend airdrop tests * spit and polish * spit and polish * move tests * move tests * Get rid off extra data.py * Improve logging * spit and polish * spit and polish * spit and polish * spit and polish * Pass MINIMAL_GAS_PRICE int airdrop tests * spit and polish * move test_operator_spending.py * move test_operator_spending.py * spit and polish * spit and polish * Fix message printing * spit and polish * spit and polish * spit and polish * spit and polish * use error instead of debug * Revert "constants and utils" This reverts commit 505653656a5406d64592c4fa2ebcf8ff1aed8edd. # Conflicts: # proxy/testing/test_eth_sendRawTransaction.py * Emphasize meaning of trx extending functions This reverts commit 505653656a5406d64592c4fa2ebcf8ff1aed8edd. # Conflicts: # proxy/testing/test_eth_sendRawTransaction.py * Resolve @otselik remarks # Conflicts: # proxy/testing/test_eth_sendRawTransaction.py * rollback common/utils.py * Rollback some changes * Resolve remarks * Use exception to check result of get_token_balance_gwei * just not to loose changes * spit and polish * Update tests * Simplify airdrop processing * Spit and polish * Spit and polish * Spit and polish * Freeze changes up * Isolate errors * spit and polish * spit and polish Co-authored-by: rozhkovdmitrii --- .buildkite/steps/deploy-test.sh | 1 + proxy/common_neon/__init__.py | 0 proxy/common_neon/errors.py | 11 + proxy/common_neon/utils.py | 13 + proxy/environment.py | 4 +- proxy/plugin/solana_rest_api.py | 79 +++--- proxy/plugin/solana_rest_api_tools.py | 253 +++++++++++------- proxy/testing/__init__.py | 2 + .../testing/test_airdropping_eth_accounts.py | 103 +++++++ proxy/{ => testing}/test_cancel_hanged.py | 1 - .../test_create_account_block.py | 0 proxy/{ => testing}/test_environment.py | 0 .../test_erc20_wrapper_contract.py | 6 +- .../test_eth_getBlockByNumber.py | 0 proxy/{ => testing}/test_eth_getLogs.py | 0 .../test_eth_sendRawTransaction.py | 29 +- proxy/{ => testing}/test_neon_faucet.py | 0 proxy/{ => testing}/test_operator_spending.py | 0 .../test_resize_storage_account.py | 0 proxy/{ => testing}/test_user_stories.py | 0 proxy/testing/test_utils.py | 16 ++ .../{ => testing}/test_web3_clientVersion.py | 0 22 files changed, 360 insertions(+), 158 deletions(-) create mode 100644 proxy/common_neon/__init__.py create mode 100644 proxy/common_neon/errors.py create mode 100644 proxy/common_neon/utils.py create mode 100644 proxy/testing/test_airdropping_eth_accounts.py rename proxy/{ => testing}/test_cancel_hanged.py (99%) rename proxy/{ => testing}/test_create_account_block.py (100%) rename proxy/{ => testing}/test_environment.py (100%) rename proxy/{ => testing}/test_erc20_wrapper_contract.py (99%) rename proxy/{ => testing}/test_eth_getBlockByNumber.py (100%) rename proxy/{ => testing}/test_eth_getLogs.py (100%) rename proxy/{ => testing}/test_eth_sendRawTransaction.py (97%) rename proxy/{ => testing}/test_neon_faucet.py (100%) rename proxy/{ => testing}/test_operator_spending.py (100%) rename proxy/{ => testing}/test_resize_storage_account.py (100%) rename proxy/{ => testing}/test_user_stories.py (100%) create mode 100644 proxy/testing/test_utils.py rename proxy/{ => testing}/test_web3_clientVersion.py (100%) diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index 11a199b15..f42f7dffd 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -78,6 +78,7 @@ docker run --rm -ti --network=container:proxy \ -e EVM_LOADER \ -e SOLANA_URL \ -e EXTRA_GAS=100000 \ + -e NEW_USER_AIRDROP_AMOUNT=100 \ --entrypoint ./proxy/deploy-test.sh \ ${EXTRA_ARGS:-} \ $PROXY_IMAGE \ diff --git a/proxy/common_neon/__init__.py b/proxy/common_neon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/proxy/common_neon/errors.py b/proxy/common_neon/errors.py new file mode 100644 index 000000000..e590afa64 --- /dev/null +++ b/proxy/common_neon/errors.py @@ -0,0 +1,11 @@ +from enum import Enum + + +class SolanaErrors(Enum): + AccountNotFound = "Invalid param: could not find account" + + +class SolanaAccountNotFoundError(Exception): + """Provides special error processing""" + def __init__(self): + super().__init__(SolanaErrors.AccountNotFound.value) diff --git a/proxy/common_neon/utils.py b/proxy/common_neon/utils.py new file mode 100644 index 000000000..be93827e1 --- /dev/null +++ b/proxy/common_neon/utils.py @@ -0,0 +1,13 @@ +from typing import Dict, Optional, Any + + +def get_from_dict(src: Dict, *path) -> Optional[Any]: + """Provides smart getting values from python dictionary""" + val = src + for key in path: + if not isinstance(val, dict): + return None + val = val.get(key) + if val is None: + return None + return val diff --git a/proxy/environment.py b/proxy/environment.py index df86e2042..7ba268c0a 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -16,7 +16,7 @@ def call(self, *args): cmd = ["solana", "--url", solana_url, ] + list(args) - print(cmd) + logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, universal_newlines=True) except subprocess.CalledProcessError as err: import sys @@ -32,7 +32,7 @@ def call(self, *args): "--url", solana_url, "--evm_loader={}".format(evm_loader_id), ] + list(args) - print(cmd) + logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) except subprocess.CalledProcessError as err: import sys diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index ba0b1fb45..ef13a4748 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -8,9 +8,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import List, Tuple +from typing import List, Tuple, Optional +import copy import json import unittest +import eth_utils import rlp import solana from solana.account import Account as sol_Account @@ -20,13 +22,14 @@ from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes from .eth_proto import Trx as EthTrx -from solana.rpc.api import Client as SolanaClient +from solana.rpc.api import Client as SolanaClient, SendTransactionError as SolanaTrxError from sha3 import keccak_256 import base58 import traceback import threading -from .solana_rest_api_tools import EthereumAddress, getTokens, getAccountInfo, \ - call_signed, call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE + +from .solana_rest_api_tools import EthereumAddress, get_token_balance_or_airdrop, getAccountInfo, call_signed, \ + call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE from solana.rpc.commitment import Commitment, Confirmed from web3 import Web3 import logging @@ -34,7 +37,7 @@ import os from ..indexer.utils import get_trx_results, LogDB from ..indexer.sql_dict import SQLDict -from proxy.environment import evm_loader_id, solana_cli, solana_url, neon_cli +from ..environment import evm_loader_id, solana_cli, solana_url, neon_cli logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -46,23 +49,7 @@ class EthereumModel: def __init__(self): - # Initialize user account - res = solana_cli().call('config', 'get') - substr = "Keypair Path: " - path = "" - for line in res.splitlines(): - if line.startswith(substr): - path = line[len(substr):].strip() - if path == "": - raise Exception("cannot get keypair path") - - with open(path.strip(), mode='r') as file: - pk = (file.read()) - nums = list(map(int, pk.strip("[] \n").split(','))) - nums = nums[0:32] - values = bytes(nums) - self.signer = sol_Account(values) - + self.signer = self.get_solana_account() self.client = SolanaClient(solana_url) self.logs_db = LogDB() @@ -77,7 +64,27 @@ def __init__(self): logger.debug("worker id {}".format(self.proxy_id)) neon_config_load(self) - pass + + + @staticmethod + def get_solana_account() -> Optional[sol_Account]: + solana_account: Optional[sol_Account] = None + res = solana_cli().call('config', 'get') + substr = "Keypair Path: " + path = "" + for line in res.splitlines(): + if line.startswith(substr): + path = line[len(substr):].strip() + if path == "": + raise Exception("cannot get keypair path") + + with open(path.strip(), mode='r') as file: + pk = (file.read()) + nums = list(map(int, pk.strip("[] \n").split(','))) + nums = nums[0:32] + values = bytes(nums) + solana_account = sol_Account(values) + return solana_account def web3_clientVersion(self): neon_config_load(self) @@ -120,7 +127,6 @@ def process_block_tag(self, tag): slot = int(tag, 16) return slot - def eth_blockNumber(self): slot = self.client.get_slot(commitment=Confirmed)['result'] logger.debug("eth_blockNumber %s", hex(slot)) @@ -132,9 +138,9 @@ def eth_getBalance(self, account, tag): """ eth_acc = EthereumAddress(account) logger.debug('eth_getBalance: %s %s', account, eth_acc) - balance = getTokens(self.client, self.signer, evm_loader_id, eth_acc, self.signer.public_key()) + balance = get_token_balance_or_airdrop(self.client, self.signer, evm_loader_id, eth_acc) - return hex(balance*10**9) + return hex(balance * eth_utils.denoms.gwei) def eth_getLogs(self, obj): fromBlock = None @@ -366,7 +372,7 @@ def eth_getTransactionByHash(self, trxId, block_info = None): "s": eth_trx[8], } - logger.debug ("eth_getTransactionByHash: %s", json.dumps(ret, indent=3)) + logger.debug("eth_getTransactionByHash: %s", json.dumps(ret, indent=3)) return ret def eth_getCode(self, param, param1): @@ -455,8 +461,8 @@ def eth_sendRawTransaction(self, rawTrx): return eth_signature - except solana.rpc.api.SendTransactionError as err: - logger.debug("eth_sendRawTransaction solana.rpc.api.SendTransactionError:%s", err.result) + except SolanaTrxError as err: + self._log_transaction_error(err, logger) raise except EthereumError as err: logger.debug("eth_sendRawTransaction EthereumError:%s", err) @@ -465,6 +471,14 @@ def eth_sendRawTransaction(self, rawTrx): logger.debug("eth_sendRawTransaction type(err):%s, Exception:%s", type(err), err) raise + def _log_transaction_error(self, error: SolanaTrxError, logger): + result = copy.deepcopy(error.result) + logs = result.get("data", {}).get("logs", []) + result.get("data", {}).update({"logs": ["\n\t" + log for log in logs]}) + log_msg = str(result).replace("\\n\\t", "\n\t") + logger.error(f"Got SendTransactionError: {log_msg}") + + class JsonEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, bytearray): @@ -475,12 +489,11 @@ def default(self, obj): class SolanaContractTests(unittest.TestCase): + def setUp(self): self.model = EthereumModel() self.owner = '0xc1566af4699928fdf9be097ca3dc47ece39f8f8e' self.token1 = '0x49a449cd7fd8fbcf34d103d98f2c05245020e35b' -# self.assertEqual(self.getBalance(self.owner), 1000*10**18) -# self.assertEqual(self.getBalance(self.token1), 0) def getBalance(self, account): return int(self.model.eth_getBalance(account, 'latest'), 16) @@ -534,7 +547,6 @@ def test_transferTokens(self): self.assertTrue(receiptId in block['transactions']) - class SolanaProxyPlugin(HttpWebServerBasePlugin): """Extend in-built Web Server to add Reverse Proxy capabilities. """ @@ -571,7 +583,7 @@ def process_request(self, request): try: method = getattr(self.model, request['method']) response['result'] = method(*request['params']) - except solana.rpc.api.SendTransactionError as err: + except SolanaTrxError as err: traceback.print_exc() response['error'] = err.result except EthereumError as err: @@ -595,7 +607,6 @@ def handle_request(self, request: HttpParser) -> None: }))) return - # print('headers', request.headers) logger.debug('<<< %s 0x%x %s', threading.get_ident(), id(self.model), request.body.decode('utf8')) response = None diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 8a748d717..42752a6d7 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -9,30 +9,37 @@ import time from datetime import datetime from hashlib import sha256 -from typing import NamedTuple +from typing import NamedTuple, Optional, Union, Dict, Tuple +import psycopg2 import rlp from base58 import b58decode, b58encode from construct import Bytes, Int8ul, Int32ul, Int64ul from construct import Struct as cStruct from eth_keys import keys as eth_keys +import eth_utils + from sha3 import keccak_256 -from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT -from solana._layouts.system_instructions import InstructionType as SystemInstructionType +from web3.auto import w3 + +from solana.account import Account as SolanaAccount from solana.blockhash import Blockhash -from solana.publickey import PublicKey -from solana.rpc.api import Client, SendTransactionError +from solana.rpc.api import Client as SolanaClient, SendTransactionError from solana.rpc.commitment import Commitment, Confirmed from solana.rpc.types import TxOpts from solana.sysvar import * from solana.transaction import AccountMeta, Transaction, TransactionInstruction +from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT +from solana._layouts.system_instructions import InstructionType as SystemInstructionType + from spl.token.constants import ACCOUNT_LEN, ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address, create_associated_token_account, transfer2, Transfer2Params -from web3.auto import w3 -from proxy.environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params + +from ..environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params +from ..common_neon.utils import get_from_dict +from ..common_neon.errors import * from .eth_proto import Trx from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob from ..indexer.sql_dict import POSTGRES_USER, POSTGRES_HOST, POSTGRES_DB, POSTGRES_PASSWORD -import psycopg2 logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -59,7 +66,7 @@ incinerator = "1nc1nerator11111111111111111111111111111111" system = "11111111111111111111111111111111" -STORAGE_SIZE = 128*1024 +STORAGE_SIZE = 128 * 1024 ACCOUNT_INFO_LAYOUT = cStruct( "type" / Int8ul, @@ -193,6 +200,7 @@ def __init__(self, caller_token, eth_accounts, eth_trx): self.collateral_pool_index_buf = collateral_pool_index.to_bytes(4, 'little') self.collateral_pool_address = create_collateral_pool_address(collateral_pool_index) + class AccountInfo(NamedTuple): ether: eth_keys.PublicKey trx_count: int @@ -203,6 +211,7 @@ def frombytes(data): cont = ACCOUNT_INFO_LAYOUT.parse(data) return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account)) + def create_account_layout(lamports, space, ether, nonce): return bytes.fromhex("02000000")+CREATE_ACCOUNT_LAYOUT.build(dict( lamports=lamports, @@ -211,6 +220,7 @@ def create_account_layout(lamports, space, ether, nonce): nonce=nonce )) + def write_holder_layout(nonce, offset, data): return (bytes.fromhex('12')+ nonce.to_bytes(8, byteorder='little')+ @@ -246,11 +256,11 @@ def get_account_info(client, storage_account): def accountWithSeed(base, seed, program): - # logger.debug(type(base), str(base), type(seed), str(seed), type(program), str(program)) result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(program)).digest()) logger.debug('accountWithSeed %s', str(result)) return result + def createAccountWithSeedTrx(funding, base, seed, lamports, space, program): seed_str = str(seed, 'utf8') data = SYSTEM_INSTRUCTIONS_LAYOUT.build( @@ -393,8 +403,6 @@ def emulator(contract, sender, data, value): return neon_cli().call("emulate", sender, contract, data, value) - - def confirm_transaction(client, tx_sig, confirmations=0): """Confirm a transaction.""" TIMEOUT = 30 # 30 seconds pylint: disable=invalid-name @@ -407,27 +415,27 @@ def confirm_transaction(client, tx_sig, confirmations=0): status = resp['result']['value'][0] if status and (status['confirmationStatus'] == 'finalized' or \ status['confirmationStatus'] == 'confirmed' and status['confirmations'] >= confirmations): -# logger.debug('Confirmed transaction:', resp) return time.sleep(confirmation_check_delay) elapsed_time += confirmation_check_delay - #if not resp["result"]: raise RuntimeError("could not confirm transaction: ", tx_sig) - #return resp + def solana2ether(public_key): from web3 import Web3 return bytes(Web3.keccak(bytes.fromhex(public_key))[-20:]) + def ether2program(ether, program_id, base): if isinstance(ether, str): - if ether.startswith('0x'): - ether = ether[2:] + pass + elif isinstance(ether, EthereumAddress): + ether = str(ether) else: ether = ether.hex() output = neon_cli().call("create-program-address", ether) items = output.rstrip().split(' ') - return (items[0], int(items[1])) + return items[0], int(items[1]) def ether2seed(ether, program_id, base): @@ -437,7 +445,7 @@ def ether2seed(ether, program_id, base): seed = b58encode(bytes.fromhex(ether)) acc = accountWithSeed(base, seed, PublicKey(program_id)) logger.debug('ether2program: {} {} => {} (seed {})'.format(ether, 255, acc, seed)) - return (acc, 255, seed) + return acc, 255, seed def neon_config_load(ethereum_model): @@ -480,6 +488,7 @@ def call_emulated(contract_id, caller_id, data=None, value=None): raise Exception("evm emulator error ", result) return result + def extract_measurements_from_receipt(receipt): log_messages = receipt['result']['meta']['logMessages'] transaction = receipt['result']['transaction'] @@ -541,6 +550,7 @@ def get_measurements(result): logger.error("Can't get measurements %s"%err) logger.info("Failed result: %s"%json.dumps(result, indent=3)) + def send_transaction(client, trx, signer, eth_trx=None, reason=None): result = client.send_transaction(trx, signer, opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed)) confirm_transaction(client, result["result"]) @@ -548,11 +558,13 @@ def send_transaction(client, trx, signer, eth_trx=None, reason=None): update_transaction_cost(result, eth_trx, reason=reason) return result + def send_measured_transaction(client, trx, signer, eth_trx, reason): result = send_transaction(client, trx, signer, eth_trx=eth_trx, reason=reason) get_measurements(result) return result + def check_if_program_exceeded_instructions(err_result): err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" err_budget = "failed: Computational budget exceeded" @@ -592,6 +604,7 @@ def call_continue(signer, client, perm_accs, trx_info, steps): return sol_instr_21_cancel(signer, client, perm_accs, trx_info) + def call_continue_iterative(signer, client, perm_accs, trx_info, step_count): while True: logger.debug("Continue iterative step:") @@ -781,6 +794,7 @@ def update_transaction_cost(receipt, eth_trx, extra_sol_trx=False, reason=None): reason if reason else '' ) + def create_account_list_by_emulate(signer, client, eth_trx): sender_ether = bytes.fromhex(eth_trx.sender()) add_keys_05 = [] @@ -801,8 +815,7 @@ def create_account_list_by_emulate(signer, client, eth_trx): for acc_desc in output_json["accounts"]: if acc_desc["new"] == False: - address = bytes.fromhex(acc_desc["address"][2:]) - if acc_desc["code_size_current"] is not None and acc_desc["code_size"] is not None: + if acc_desc.get("code_size_current") is not None and acc_desc.get("code_size") is not None: if acc_desc["code_size"] > acc_desc["code_size_current"]: code_size = acc_desc["code_size"] + 2048 seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) @@ -846,7 +859,6 @@ def create_account_list_by_emulate(signer, client, eth_trx): success = True logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) break - # wait for unlock account time.sleep(1) count = count+1 @@ -867,26 +879,9 @@ def create_account_list_by_emulate(signer, client, eth_trx): code_size = acc_desc["code_size"] + 2048 code_account_balance = client.get_minimum_balance_for_rent_exemption(code_size)["result"] trx.add(createAccountWithSeedTrx(signer.public_key(), signer.public_key(), seed, code_account_balance, code_size, PublicKey(evm_loader_id))) - # add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=acc_desc["writable"])) code_account_writable = acc_desc["writable"] - (create_trx, solana_address, token_address) = createEtherAccountTrx(client, address, evm_loader_id, signer, code_account) - trx.add(create_trx) - - if address == sender_ether and NEW_USER_AIRDROP_AMOUNT > 0: - trx.add(transfer2(Transfer2Params( - amount=NEW_USER_AIRDROP_AMOUNT*1_000_000_000, - decimals=9, - dest=get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID), - mint=ETH_TOKEN_MINT_ID, - owner=signer.public_key(), - program_id=TOKEN_PROGRAM_ID, - source=getTokenAddr(signer.public_key()), - ))) - logger.debug("Token transfer to %s as ethereum 0x%s amount %s", - get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID), - acc_desc["address"], - str(NEW_USER_AIRDROP_AMOUNT)) + extend_trx_with_create_and_airdrop(signer, EthereumAddress(address), code_account, trx=trx) if address == to_address: contract_sol = PublicKey(acc_desc["account"]) @@ -914,7 +909,6 @@ def create_account_list_by_emulate(signer, client, eth_trx): if acc_desc["contract"]: add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) - for token_account in output_json["token_accounts"]: add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) @@ -936,7 +930,7 @@ def create_account_list_by_emulate(signer, client, eth_trx): trx_info = TransactionInfo(caller_token, eth_accounts, eth_trx) - return (trx_info, sender_ether, trx) + return trx_info, sender_ether, trx def call_signed(signer, client, eth_trx, steps): @@ -984,6 +978,7 @@ def call_signed(signer, client, eth_trx, steps): finally: del perm_accs + def call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx): if len(create_acc_trx.instructions): precall_txs = Transaction() @@ -1037,52 +1032,6 @@ def call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, st return call_continue(signer, client, perm_accs, trx_info, steps) -def createEtherAccountTrx(client, ether, evm_loader_id, signer, code_acc=None): - if isinstance(ether, str): - if ether.startswith('0x'): ether = ether[2:] - else: ether = ether.hex() - (sol, nonce) = ether2program(ether, evm_loader_id, signer.public_key()) - associated_token = get_associated_token_address(PublicKey(sol), ETH_TOKEN_MINT_ID) - logger.debug('createEtherAccount: {} {} => {}'.format(ether, nonce, sol)) - logger.debug('associatedTokenAccount: {}'.format(associated_token)) - base = signer.public_key() - data=bytes.fromhex('02000000')+CREATE_ACCOUNT_LAYOUT.build(dict( - lamports=0, - space=0, - ether=bytes.fromhex(ether), - nonce=nonce)) - trx = Transaction() - if code_acc is None: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(sol), is_signer=False, is_writable=True), - AccountMeta(pubkey=associated_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - else: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(sol), is_signer=False, is_writable=True), - AccountMeta(pubkey=associated_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - return (trx, sol, associated_token) - def createERC20TokenAccountTrx(signer, token_info): trx = Transaction() trx.add(TransactionInstruction( @@ -1102,7 +1051,6 @@ def createERC20TokenAccountTrx(signer, token_info): return trx - def write_trx_to_holder_account(signer, client, holder, acc_id, eth_trx): msg = eth_trx.signature() + len(eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + eth_trx.unsigned_msg() @@ -1130,6 +1078,7 @@ def write_trx_to_holder_account(signer, client, holder, acc_id, eth_trx): update_transaction_cost(result, eth_trx, reason='WriteHolder') logger.debug("confirmed: %s", rcpt) + def _getAccountData(client, account, expected_length, owner=None): info = client.get_account_info(account, commitment=Confirmed)['result']['value'] if info is None: @@ -1140,37 +1089,137 @@ def _getAccountData(client, account, expected_length, owner=None): raise Exception("Wrong data length for account data {}".format(account)) return data + def getAccountInfo(client, eth_acc, base_account): (account_sol, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader_id, base_account) info = _getAccountData(client, account_sol, ACCOUNT_INFO_LAYOUT.sizeof()) return AccountInfo.frombytes(info) + def getLamports(client, evm_loader, eth_acc, base_account): (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) return int(client.get_balance(account, commitment=Confirmed)['result']['value']) -def getTokens(client, signer, evm_loader, eth_acc, base_account): - (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) - token_account = get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) - balance = client.get_token_account_balance(token_account, commitment=Confirmed) - if 'error' in balance: - if NEW_USER_AIRDROP_AMOUNT > 0: - return NEW_USER_AIRDROP_AMOUNT * 1_000_000_000 - else: - logger.debug("'error' in balance:") - return 0 +def make_create_eth_account_trx(signer: SolanaAccount, eth_address: EthereumAddress, evm_loader_id, code_acc=None) \ + -> Tuple[Transaction, PublicKey]: + + solana_address, nonce = ether2program(eth_address, evm_loader_id, signer.public_key()) + token_acc_address = get_associated_token_address(PublicKey(solana_address), ETH_TOKEN_MINT_ID) + logger.debug(f'Create eth account: {eth_address}, sol account: {solana_address}, token_acc_address: {token_acc_address}, nonce: {nonce}') + + base = signer.public_key() + data = bytes.fromhex('02000000') + CREATE_ACCOUNT_LAYOUT.build(dict(lamports=0, + space=0, + ether=bytes(eth_address), + nonce=nonce)) + trx = Transaction() + if code_acc is None: + trx.add(TransactionInstruction( + program_id=evm_loader_id, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), + AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=system, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), + ])) + else: + trx.add(TransactionInstruction( + program_id=evm_loader_id, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), + AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), + AccountMeta(pubkey=system, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), + ])) + return trx, token_acc_address + + +def make_transfer_instruction(owner_pda_account: SolanaAccount, associated_token_account: PublicKey) -> TransactionInstruction: + owner_pda_address = owner_pda_account.public_key() + owner_associated_token_account = getTokenAddr(owner_pda_address) + transfer_instruction = transfer2(Transfer2Params(source=owner_associated_token_account, + owner=owner_pda_address, + dest=associated_token_account, + amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei, + decimals=9, + mint=ETH_TOKEN_MINT_ID, + program_id=TOKEN_PROGRAM_ID)) + logger.debug(f"Token transfer from token: {owner_associated_token_account}, owned by: {owner_pda_address}, to token: " + f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}") + return transfer_instruction + + +def extend_trx_with_create_and_airdrop(signer: SolanaAccount, eth_account: EthereumAddress, code_acc=None, *, trx): + create_trx, associated_token_account = make_create_eth_account_trx(signer, eth_account, evm_loader_id, code_acc) + trx.add(create_trx) + if NEW_USER_AIRDROP_AMOUNT <= 0: + return + transfer_instruction = make_transfer_instruction(signer, associated_token_account) + trx.add(transfer_instruction) + + +def create_eth_account_and_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress): + trx = Transaction() + extend_trx_with_create_and_airdrop(signer, eth_account, trx=trx) + result = send_transaction(client, trx, signer) + error = result.get("error") + if error is not None: + logger.error(f"Failed to create eth_account and airdrop: {eth_account}, error occurred: {error}") + raise Exception("Create eth_account error") + + +def get_token_balance_gwei(client: SolanaClient, pda_account: str) -> int: + associated_token_account = get_associated_token_address(PublicKey(pda_account), ETH_TOKEN_MINT_ID) + rpc_response = client.get_token_account_balance(associated_token_account, commitment=Confirmed) + error = rpc_response.get('error') + if error is not None: + message = error.get("message") + if message == SolanaErrors.AccountNotFound.value: + raise SolanaAccountNotFoundError() + logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, " + f"got get_token_account_balance error: \"{message}\"") + raise Exception("Getting balance error") + + balance = get_from_dict(rpc_response, "result", "value", "amount") + if balance is None: + logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, response: {rpc_response}") + raise Exception("Unexpected get_balance response") + return int(balance) + + +def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, evm_loader: str, eth_account: EthereumAddress) -> int: + associated_token_account, nonce = ether2program(bytes(eth_account).hex(), evm_loader, signer.public_key()) + logger.debug(f"Get balance for eth account: {eth_account} aka: {associated_token_account}") + + try: + return get_token_balance_gwei(client, associated_token_account) + except SolanaAccountNotFoundError: + logger.debug(f"Account not found: {eth_account} aka: {associated_token_account} - create") + create_eth_account_and_airdrop(client, signer, eth_account) + return get_token_balance_gwei(client, associated_token_account) - return int(balance['result']['value']['amount']) def getTokenAddr(account): return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) + def make_instruction_data_from_tx(instruction, private_key=None): if isinstance(instruction, dict): - if instruction['chainId'] == None: + if instruction.get('chainId') is None: raise Exception("chainId value is needed in input dict") - if private_key == None: + if private_key is None: raise Exception("Needed private key for transaction creation from fields") signed_tx = w3.eth.account.sign_transaction(instruction, private_key) diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py index 232621f0b..0bfb50319 100644 --- a/proxy/testing/__init__.py +++ b/proxy/testing/__init__.py @@ -8,3 +8,5 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import sys +sys.path.append("/spl/bin/") # TODO: get rid off this workaround all related modules should either be installed as package or be linked from submodule diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py new file mode 100644 index 000000000..c03259ccf --- /dev/null +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -0,0 +1,103 @@ +import unittest +import os +import solcx + +import eth_account +import eth_typing +import eth_utils +from eth_account.account import LocalAccount + +from web3 import Web3, eth as web3_eth +from solana.rpc.api import Client as SolanaClient + +from ..plugin.solana_rest_api import EthereumModel +from ..plugin.solana_rest_api_tools import get_token_balance_gwei, EthereumAddress, ether2program + + +class TestAirdroppingEthAccounts(unittest.TestCase): + + @classmethod + def setUpClass(cls) -> None: + cls._EVM_LOADER_ID = os.environ.get("EVM_LOADER") + new_user_airdrop_amount = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) + cls._EXPECTED_BALANCE_WEI = eth_utils.to_wei(new_user_airdrop_amount, 'ether') + cls._MINIMAL_GAS_PRICE = int(os.environ.get("MINIMAL_GAS_PRICE", 1)) * eth_utils.denoms.gwei + + proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') + cls._web3 = Web3(Web3.HTTPProvider(proxy_url)) + solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") + cls._solana_client = SolanaClient(solana_url) + cls._host_solana_account = EthereumModel.get_solana_account() + + def test_airdrop_on_get_balance(self): + account: LocalAccount = eth_account.account.Account.create() + block_number: eth_typing.BlockNumber = self._web3.eth.get_block_number() + actual_balance_wei = self._web3.eth.get_balance(account.address, block_identifier=block_number) + self.assertEqual(self._EXPECTED_BALANCE_WEI, actual_balance_wei) + + def test_airdrop_on_deploy(self): + contract_owner: LocalAccount = self._web3.eth.account.create() + contract = self._compile_and_deploy_contract(contract_owner, self._CONTRACT_STORAGE_SOURCE) + actual_balance_wei = self._get_balance_wei(contract.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, actual_balance_wei) + + def test_airdrop_onto_wrapped_new_address(self): + contract_owner: LocalAccount = self._web3.eth.account.create() + contract = self._compile_and_deploy_contract(contract_owner, self._WRAPPER_CONTRACT_STORAGE_SOURCE) + nested_contract_address = contract.functions.getNested().call() + nested_actual_balance = self._get_balance_wei(nested_contract_address) + wrapper_actual_balance = self._get_balance_wei(contract.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, wrapper_actual_balance) + self.assertEqual(self._EXPECTED_BALANCE_WEI, nested_actual_balance) + + def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str) -> web3_eth.Contract: + compiled_sol = solcx.compile_source(source) + contract_id, contract_interface = compiled_sol.popitem() + contract = self._web3.eth.contract(abi=contract_interface['abi'], bytecode=contract_interface['bin']) + nonce = self._web3.eth.get_transaction_count(contract_owner.address) + chain_id = self._web3.eth.chain_id + trx_signed = self._web3.eth.account.sign_transaction( + dict(nonce=nonce, chainId=chain_id, gas=987654321, gasPrice=self._MINIMAL_GAS_PRICE, to='', value=0, data=contract.bytecode), + contract_owner.key) + trx_hash = self._web3.eth.send_raw_transaction(trx_signed.rawTransaction) + trx_receipt = self._web3.eth.wait_for_transaction_receipt(trx_hash) + contract = self._web3.eth.contract(address=trx_receipt.contractAddress, abi=contract.abi) + return contract + + def _get_balance_wei(self, eth_acc: str) -> int: + pub_key = self._host_solana_account.public_key() + token_owner_account, nonce = ether2program(eth_acc, self._EVM_LOADER_ID, pub_key) + balance = get_token_balance_gwei(self._solana_client, token_owner_account) + self.assertIsNotNone(balance) + self.assertIsInstance(balance, int) + return balance * eth_utils.denoms.gwei + + _CONTRACT_STORAGE_SOURCE = ''' + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.7.0 <0.9.0; + contract Storage { + uint256 number; + function store(uint256 num) public { + number = num; + } + function retrieve() public view returns (uint256) { + return number; + } + } + ''' + + _WRAPPER_CONTRACT_STORAGE_SOURCE = ''' + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.7.0 <0.9.0; + contract Wrapper { + address private nested_address; + constructor() { + Nested nested = new Nested(); + nested_address = address(nested); + } + function getNested() public view returns (address) { + return nested_address; + } + } + contract Nested {} + ''' diff --git a/proxy/test_cancel_hanged.py b/proxy/testing/test_cancel_hanged.py similarity index 99% rename from proxy/test_cancel_hanged.py rename to proxy/testing/test_cancel_hanged.py index 241f15728..1e91a1381 100644 --- a/proxy/test_cancel_hanged.py +++ b/proxy/testing/test_cancel_hanged.py @@ -3,7 +3,6 @@ from proxy.plugin.solana_rest_api_tools import sysinstruct, ETH_TOKEN_MINT_ID, system, send_transaction, MINIMAL_GAS_PRICE -sys.path.append("/spl/bin/") os.environ['SOLANA_URL'] = "http://solana:8899" os.environ['EVM_LOADER'] = "53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io" os.environ['ETH_TOKEN_MINT'] = "HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU" diff --git a/proxy/test_create_account_block.py b/proxy/testing/test_create_account_block.py similarity index 100% rename from proxy/test_create_account_block.py rename to proxy/testing/test_create_account_block.py diff --git a/proxy/test_environment.py b/proxy/testing/test_environment.py similarity index 100% rename from proxy/test_environment.py rename to proxy/testing/test_environment.py diff --git a/proxy/test_erc20_wrapper_contract.py b/proxy/testing/test_erc20_wrapper_contract.py similarity index 99% rename from proxy/test_erc20_wrapper_contract.py rename to proxy/testing/test_erc20_wrapper_contract.py index de8b61ba5..75030130b 100644 --- a/proxy/test_erc20_wrapper_contract.py +++ b/proxy/testing/test_erc20_wrapper_contract.py @@ -15,7 +15,7 @@ from solana.account import Account as SolanaAccount from solana.publickey import PublicKey -from proxy.plugin.solana_rest_api_tools import createERC20TokenAccountTrx, createEtherAccountTrx +from proxy.plugin.solana_rest_api_tools import createERC20TokenAccountTrx # install_solc(version='latest') install_solc(version='0.7.6') @@ -66,7 +66,7 @@ event Transfer(address indexed from, address indexed to, uint256 value); event Approval(address indexed owner, address indexed spender, uint256 value); - + function approveSolana(bytes32 spender, uint64 value) external returns (bool); event ApprovalSolana(address indexed owner, bytes32 indexed spender, uint64 value); } @@ -147,7 +147,7 @@ def deploy_erc20_wrapper_contract(self): compiled_wrapper = compile_source(ERC20_WRAPPER_SOURCE) wrapper_id, wrapper_interface = compiled_wrapper.popitem() self.wrapper = wrapper_interface - + erc20 = proxy.eth.contract(abi=self.wrapper['abi'], bytecode=wrapper_interface['bin']) nonce = proxy.eth.get_transaction_count(proxy.eth.default_account) tx = {'nonce': nonce} diff --git a/proxy/test_eth_getBlockByNumber.py b/proxy/testing/test_eth_getBlockByNumber.py similarity index 100% rename from proxy/test_eth_getBlockByNumber.py rename to proxy/testing/test_eth_getBlockByNumber.py diff --git a/proxy/test_eth_getLogs.py b/proxy/testing/test_eth_getLogs.py similarity index 100% rename from proxy/test_eth_getLogs.py rename to proxy/testing/test_eth_getLogs.py diff --git a/proxy/test_eth_sendRawTransaction.py b/proxy/testing/test_eth_sendRawTransaction.py similarity index 97% rename from proxy/test_eth_sendRawTransaction.py rename to proxy/testing/test_eth_sendRawTransaction.py index 12e3f4633..9b6791e72 100644 --- a/proxy/test_eth_sendRawTransaction.py +++ b/proxy/testing/test_eth_sendRawTransaction.py @@ -1,5 +1,7 @@ import unittest import os + +import eth_utils from web3 import Web3 from solcx import install_solc @@ -237,8 +239,6 @@ def test_04_execute_with_bad_nonce(self): def test_05_transfer_one_gwei(self): print("\ntest_05_transfer_one_gwei") - one_gwei = 1_000_000_000 - eth_account_alice = proxy.eth.account.create('alice') eth_account_bob = proxy.eth.account.create('bob') print('eth_account_alice.address:', eth_account_alice.address) @@ -254,7 +254,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_alice.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -271,7 +271,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -285,7 +285,7 @@ def test_05_transfer_one_gwei(self): bob_balance_before_transfer = proxy.eth.get_balance(eth_account_bob.address) print('alice_balance_before_transfer:', alice_balance_before_transfer) print('bob_balance_before_transfer:', bob_balance_before_transfer) - print('one_gwei:', one_gwei) + print('one_gwei:', eth_utils.denoms.gwei) trx_transfer = proxy.eth.account.sign_transaction(dict( nonce=proxy.eth.get_transaction_count(eth_account_alice.address), @@ -293,7 +293,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account_alice.key ) @@ -307,8 +307,8 @@ def test_05_transfer_one_gwei(self): bob_balance_after_transfer = proxy.eth.get_balance(eth_account_bob.address) print('alice_balance_after_transfer:', alice_balance_after_transfer) print('bob_balance_after_transfer:', bob_balance_after_transfer) - self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - one_gwei) - self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + one_gwei) + self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - eth_utils.denoms.gwei) + self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + eth_utils.denoms.gwei) # @unittest.skip("a.i.") def test_06_transfer_one_and_a_half_gweis(self): @@ -319,8 +319,6 @@ def test_06_transfer_one_and_a_half_gweis(self): print('eth_account_alice.address:', eth_account_alice.address) print('eth_account_bob.address:', eth_account_bob.address) - one_gwei = 1_000_000_000 - if True: print("add funds to alice and bob") @@ -331,7 +329,7 @@ def test_06_transfer_one_and_a_half_gweis(self): gas=987654321, gasPrice=0, to=eth_account_alice.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -348,7 +346,7 @@ def test_06_transfer_one_and_a_half_gweis(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -386,10 +384,9 @@ def test_06_transfer_one_and_a_half_gweis(self): print('alice_balance_after_transfer:', alice_balance_after_transfer) print('bob_balance_after_transfer:', bob_balance_after_transfer) print('check https://github.com/neonlabsorg/neon-evm/issues/210') - one_gwei = 1_000_000_000 - print('one_gwei:', one_gwei) - self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - one_gwei) - self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + one_gwei) + print('one_gwei:', eth_utils.denoms.gwei) + self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - eth_utils.denoms.gwei) + self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + eth_utils.denoms.gwei) @unittest.skip("a.i.") def test_07_execute_long_transaction(self): diff --git a/proxy/test_neon_faucet.py b/proxy/testing/test_neon_faucet.py similarity index 100% rename from proxy/test_neon_faucet.py rename to proxy/testing/test_neon_faucet.py diff --git a/proxy/test_operator_spending.py b/proxy/testing/test_operator_spending.py similarity index 100% rename from proxy/test_operator_spending.py rename to proxy/testing/test_operator_spending.py diff --git a/proxy/test_resize_storage_account.py b/proxy/testing/test_resize_storage_account.py similarity index 100% rename from proxy/test_resize_storage_account.py rename to proxy/testing/test_resize_storage_account.py diff --git a/proxy/test_user_stories.py b/proxy/testing/test_user_stories.py similarity index 100% rename from proxy/test_user_stories.py rename to proxy/testing/test_user_stories.py diff --git a/proxy/testing/test_utils.py b/proxy/testing/test_utils.py new file mode 100644 index 000000000..137ffbf01 --- /dev/null +++ b/proxy/testing/test_utils.py @@ -0,0 +1,16 @@ +import unittest +from ..common_neon.utils import get_from_dict + + +class TestUtils(unittest.TestCase): + + def test_get_from_dict(self): + test_dict = {"a": {"b": {"c": 1}}} + + self.assertEqual(1, get_from_dict(test_dict, "a", "b", "c")) + self.assertEqual({"b": {"c": 1}}, get_from_dict(test_dict, "a")) + + self.assertIsNone(get_from_dict(test_dict, "b", "c", "a")) + self.assertIsNone(get_from_dict(None, "a")) + self.assertIsNone(get_from_dict(555, "a")) + self.assertIsNone(get_from_dict({}, "a")) diff --git a/proxy/test_web3_clientVersion.py b/proxy/testing/test_web3_clientVersion.py similarity index 100% rename from proxy/test_web3_clientVersion.py rename to proxy/testing/test_web3_clientVersion.py From 4880060c84b9c28a97f7e3ef1e3666092d418727 Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Fri, 19 Nov 2021 16:58:35 +0500 Subject: [PATCH 06/54] #305 Remove extra args from ether2program (#309) #305 Remove extra args from ether2program --- proxy/plugin/solana_rest_api.py | 4 ++-- proxy/plugin/solana_rest_api_tools.py | 19 ++++++++++--------- .../testing/test_airdropping_eth_accounts.py | 3 +-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index ef13a4748..53f7d217a 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -138,7 +138,7 @@ def eth_getBalance(self, account, tag): """ eth_acc = EthereumAddress(account) logger.debug('eth_getBalance: %s %s', account, eth_acc) - balance = get_token_balance_or_airdrop(self.client, self.signer, evm_loader_id, eth_acc) + balance = get_token_balance_or_airdrop(self.client, self.signer, eth_acc) return hex(balance * eth_utils.denoms.gwei) @@ -272,7 +272,7 @@ def eth_call(self, obj, tag): def eth_getTransactionCount(self, account, tag): logger.debug('eth_getTransactionCount: %s', account) try: - acc_info = getAccountInfo(self.client, EthereumAddress(account), self.signer.public_key()) + acc_info = getAccountInfo(self.client, EthereumAddress(account)) return hex(int.from_bytes(acc_info.trx_count, 'little')) except Exception as err: print("Can't get account info: %s"%err) diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 42752a6d7..e20fdc923 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -228,6 +228,7 @@ def write_holder_layout(nonce, offset, data): len(data).to_bytes(8, byteorder='little')+ data) + def get_account_info(client, storage_account): opts = { "encoding": "base64", @@ -426,7 +427,7 @@ def solana2ether(public_key): return bytes(Web3.keccak(bytes.fromhex(public_key))[-20:]) -def ether2program(ether, program_id, base): +def ether2program(ether): if isinstance(ether, str): pass elif isinstance(ether, EthereumAddress): @@ -1090,21 +1091,21 @@ def _getAccountData(client, account, expected_length, owner=None): return data -def getAccountInfo(client, eth_acc, base_account): - (account_sol, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader_id, base_account) +def getAccountInfo(client, eth_account: EthereumAddress): + account_sol, nonce = ether2program(eth_account) info = _getAccountData(client, account_sol, ACCOUNT_INFO_LAYOUT.sizeof()) return AccountInfo.frombytes(info) -def getLamports(client, evm_loader, eth_acc, base_account): - (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) - return int(client.get_balance(account, commitment=Confirmed)['result']['value']) +def getLamports(client, eth_account): + pda_account, nonce = ether2program(eth_account) + return int(client.get_balance(pda_account, commitment=Confirmed)['result']['value']) def make_create_eth_account_trx(signer: SolanaAccount, eth_address: EthereumAddress, evm_loader_id, code_acc=None) \ -> Tuple[Transaction, PublicKey]: - solana_address, nonce = ether2program(eth_address, evm_loader_id, signer.public_key()) + solana_address, nonce = ether2program(eth_address) token_acc_address = get_associated_token_address(PublicKey(solana_address), ETH_TOKEN_MINT_ID) logger.debug(f'Create eth account: {eth_address}, sol account: {solana_address}, token_acc_address: {token_acc_address}, nonce: {nonce}') @@ -1199,8 +1200,8 @@ def get_token_balance_gwei(client: SolanaClient, pda_account: str) -> int: return int(balance) -def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, evm_loader: str, eth_account: EthereumAddress) -> int: - associated_token_account, nonce = ether2program(bytes(eth_account).hex(), evm_loader, signer.public_key()) +def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress) -> int: + associated_token_account, nonce = ether2program(eth_account) logger.debug(f"Get balance for eth account: {eth_account} aka: {associated_token_account}") try: diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py index c03259ccf..ea9692586 100644 --- a/proxy/testing/test_airdropping_eth_accounts.py +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -65,8 +65,7 @@ def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str return contract def _get_balance_wei(self, eth_acc: str) -> int: - pub_key = self._host_solana_account.public_key() - token_owner_account, nonce = ether2program(eth_acc, self._EVM_LOADER_ID, pub_key) + token_owner_account, nonce = ether2program(eth_acc) balance = get_token_balance_gwei(self._solana_client, token_owner_account) self.assertIsNotNone(balance) self.assertIsInstance(balance, int) From 531a9e2d4313f874f9e8e4cfb4ebe924f559eade Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Tue, 23 Nov 2021 21:58:49 +0500 Subject: [PATCH 07/54] Change default logDB postgress pass (#317) Co-authored-by: rozhkovdmitrii --- proxy/indexer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index 7b45f725e..49d129795 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -175,7 +175,7 @@ class LogDB: def __init__(self): POSTGRES_DB = os.environ.get("POSTGRES_DB", "neon-db") POSTGRES_USER = os.environ.get("POSTGRES_USER", "neon-proxy") - POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "neon-proxy") + POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "neon-proxy-pass") POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost") self.conn = psycopg2.connect( From 6f3cf9ac74c6bccc8837da240dd3b0e1fd00630c Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Wed, 24 Nov 2021 10:33:13 +0500 Subject: [PATCH 08/54] #311 Create account and airdrop on gas estimation if it's preset Implement creating account on getting balance and airdrop Co-authored-by: rozhkovdmitrii --- proxy/plugin/solana_rest_api.py | 15 +++++------- proxy/plugin/solana_rest_api_tools.py | 23 +++++++++++++++++++ .../testing/test_airdropping_eth_accounts.py | 20 ++++++++++++---- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 53f7d217a..2c1a44dcc 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -29,12 +29,11 @@ import threading from .solana_rest_api_tools import EthereumAddress, get_token_balance_or_airdrop, getAccountInfo, call_signed, \ - call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE + call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE, estimate_gas from solana.rpc.commitment import Commitment, Confirmed from web3 import Web3 import logging from ..core.acceptor.pool import proxy_id_glob -import os from ..indexer.utils import get_trx_results, LogDB from ..indexer.sql_dict import SQLDict from ..environment import evm_loader_id, solana_cli, solana_url, neon_cli @@ -45,7 +44,6 @@ modelInstanceLock = threading.Lock() modelInstance = None -EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) class EthereumModel: def __init__(self): @@ -105,12 +103,11 @@ def eth_gasPrice(self): def eth_estimateGas(self, param): try: - caller_id = param['from'] if 'from' in param else "0x0000000000000000000000000000000000000000" - contract_id = param['to'] if 'to' in param else "deploy" - data = param['data'] if 'data' in param else "None" - value = param['value'] if 'value' in param else "" - result = call_emulated(contract_id, caller_id, data, value) - return result['used_gas']+EXTRA_GAS + caller_id = param.get('from', "0x0000000000000000000000000000000000000000") + contract_id = param.get('to', "deploy") + data = param.get('data', "None") + value = param.get('value', "") + return estimate_gas(self.client, self.signer, contract_id, EthereumAddress(caller_id), data, value) except Exception as err: logger.debug("Exception on eth_estimateGas: %s", err) raise diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index e20fdc923..26a98068c 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -100,6 +100,9 @@ AccountMeta(pubkey=sysvarclock, is_signer=False, is_writable=False), ] +EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) + + class SQLCost(): def __init__(self): @@ -1253,3 +1256,23 @@ def make_instruction_data_from_tx(instruction, private_key=None): return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) else: raise Exception("function gets ") + + +def is_account_exists(client: SolanaClient, eth_account: EthereumAddress) -> bool: + pda_account, nonce = ether2program(eth_account) + info = client.get_account_info(pda_account, commitment=Confirmed) + value = get_from_dict(info, "result", "value") + return value is not None + + +def estimate_gas(client: SolanaClient, signer: SolanaAccount, contract_id: str, caller_eth_account: EthereumAddress, + data: str = None, value: str = None): + if not is_account_exists(client, caller_eth_account): + create_eth_account_and_airdrop(client, signer, caller_eth_account) + result = call_emulated(contract_id, str(caller_eth_account), data, value) + used_gas = result.get("used_gas") + if used_gas is None: + logger.error(f"Failed estimate_gas, unexpected result, by contract_id: {contract_id}, caller_eth_account: " + f"{caller_eth_account}, data: {data}, value: {value}, emulation result: {result}") + raise Exception("Bad estimate_gas result") + return used_gas + EXTRA_GAS diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py index ea9692586..9fc778585 100644 --- a/proxy/testing/test_airdropping_eth_accounts.py +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -11,7 +11,7 @@ from solana.rpc.api import Client as SolanaClient from ..plugin.solana_rest_api import EthereumModel -from ..plugin.solana_rest_api_tools import get_token_balance_gwei, EthereumAddress, ether2program +from ..plugin.solana_rest_api_tools import get_token_balance_gwei, ether2program class TestAirdroppingEthAccounts(unittest.TestCase): @@ -50,9 +50,19 @@ def test_airdrop_onto_wrapped_new_address(self): self.assertEqual(self._EXPECTED_BALANCE_WEI, wrapper_actual_balance) self.assertEqual(self._EXPECTED_BALANCE_WEI, nested_actual_balance) + def test_airdrop_on_deploy_estimation(self): + owner_eth_account: LocalAccount = self._web3.eth.account.create() + compile_result = solcx.compile_source(self._CONTRACT_STORAGE_SOURCE) + _, contract_interface = compile_result.popitem() + contract_data = contract_interface.get("bin") + self.assertIsNotNone(contract_data) + self._web3.eth.estimate_gas({"from": owner_eth_account.address, "data": contract_data}) + owner_balance = self._get_balance_wei(owner_eth_account.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, owner_balance) + def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str) -> web3_eth.Contract: - compiled_sol = solcx.compile_source(source) - contract_id, contract_interface = compiled_sol.popitem() + compile_result = solcx.compile_source(source) + contract_id, contract_interface = compile_result.popitem() contract = self._web3.eth.contract(abi=contract_interface['abi'], bytecode=contract_interface['bin']) nonce = self._web3.eth.get_transaction_count(contract_owner.address) chain_id = self._web3.eth.chain_id @@ -64,8 +74,8 @@ def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str contract = self._web3.eth.contract(address=trx_receipt.contractAddress, abi=contract.abi) return contract - def _get_balance_wei(self, eth_acc: str) -> int: - token_owner_account, nonce = ether2program(eth_acc) + def _get_balance_wei(self, eth_account: str) -> int: + token_owner_account, nonce = ether2program(eth_account) balance = get_token_balance_gwei(self._solana_client, token_owner_account) self.assertIsNotNone(balance) self.assertIsInstance(balance, int) From 452a86d78fa8e00a9265b17429c53bc6e759948e Mon Sep 17 00:00:00 2001 From: sinev-valentine <37595780+sinev-valentine@users.noreply.github.com> Date: Wed, 24 Nov 2021 15:41:08 +0300 Subject: [PATCH 09/54] neonbals/neon-evm#371 Update STORAGE_ACCOUNT_INFO_LAYOUT (#315) --- proxy/indexer/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index 49d129795..c83b38588 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -128,6 +128,8 @@ def get_trx_receipts(unsigned_msg, signature): "evm_data_size" / Int64ul, "gas_used_and_paid" / Int64ul, "number_of_payments" / Int64ul, + "sign" / Bytes(65), + ) def get_account_list(client, storage_account): From 08ebfbc404b019fc9487b609d3528c6145ad57ab Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Wed, 24 Nov 2021 18:04:11 +0300 Subject: [PATCH 10/54] 319 add neon cli version handler (#325) Neon-proxy returns the neon-cli version for the "neon_cli_version" (like "web3_clientVersion") request in format Neon-cli/v- For example: Neon-cli/v0.4.0-rc0-abc26675a5f979eeeeff3c886ebf4fcd310ce064 --- proxy/environment.py | 12 ++++++-- proxy/plugin/solana_rest_api.py | 3 ++ proxy/testing/test_neon_cli_version.py | 39 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 proxy/testing/test_neon_cli_version.py diff --git a/proxy/environment.py b/proxy/environment.py index 7ba268c0a..b06465798 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -19,7 +19,6 @@ def call(self, *args): logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, universal_newlines=True) except subprocess.CalledProcessError as err: - import sys logger.debug("ERR: solana error {}".format(err)) raise @@ -35,7 +34,16 @@ def call(self, *args): logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) except subprocess.CalledProcessError as err: - import sys + logger.debug("ERR: neon-cli error {}".format(err)) + raise + + def version(self): + try: + cmd = ["neon-cli", + "--version"] + logger.debug("Calling: " + " ".join(cmd)) + return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True).split()[1] + except subprocess.CalledProcessError as err: logger.debug("ERR: neon-cli error {}".format(err)) raise diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 2c1a44dcc..53655b7cf 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -93,6 +93,9 @@ def eth_chainId(self): # NEON_CHAIN_ID is a string in decimal form return hex(int(self.neon_config_dict['NEON_CHAIN_ID'])) + def neon_cli_version(self): + return neon_cli().version() + def net_version(self): neon_config_load(self) # NEON_CHAIN_ID is a string in decimal form diff --git a/proxy/testing/test_neon_cli_version.py b/proxy/testing/test_neon_cli_version.py new file mode 100644 index 000000000..66b82d68e --- /dev/null +++ b/proxy/testing/test_neon_cli_version.py @@ -0,0 +1,39 @@ +import unittest +import os +import requests +import json +import inspect + +from proxy.environment import neon_cli + +proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') +headers = {'Content-type': 'application/json'} + + +def get_line_number(): + cf = inspect.currentframe() + return cf.f_back.f_lineno + + +class TestNeonProxyVersion(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + def test_01_neon_cli_version(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/319") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "neon_cli_version", + "params": [] + })).text) + print('response:', response) + neon_cli_version = response['result'] + print('neon_cli_version:', neon_cli_version) + self.assertEqual(neon_cli_version, neon_cli().version()) + + +if __name__ == '__main__': + unittest.main() From fb01a4e9f6a8a28d2d593375e354a6f5cc7e93b4 Mon Sep 17 00:00:00 2001 From: sinev-valentine <37595780+sinev-valentine@users.noreply.github.com> Date: Wed, 24 Nov 2021 19:29:02 +0300 Subject: [PATCH 11/54] fix storage account check (#327) --- proxy/plugin/solana_rest_api_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 26a98068c..29207a332 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -58,6 +58,7 @@ COLLATERALL_POOL_MAX=10 EMPTY_STORAGE_TAG=0 +FINALIZED_STORAGE_TAG=5 sysvarclock = "SysvarC1ock11111111111111111111111111111111" sysinstruct = "Sysvar1nstructions1111111111111111111111111" @@ -334,8 +335,8 @@ def create_multiple_accounts_with_seed(client, funding, base, seeds, sizes): raise Exception("insufficient balance") if PublicKey(owner) != PublicKey(evm_loader_id): raise Exception("wrong owner") - if tag != EMPTY_STORAGE_TAG: - raise Exception("not empty") + if tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}: + raise Exception("not empty, not finalized") if len(trx.instructions) > 0: send_transaction(client, trx, funding) From ed83d904adf5c8c7f98bb16b65cc278ae8972ac1 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Thu, 25 Nov 2021 08:06:48 +0300 Subject: [PATCH 12/54] #320 add neon proxy version handler (#323) * Introduce neon_proxy_version handler * Introduce test_01_neon_proxy_version * Use proxy/plugin/solana_rest_api.py * Use /opt/proxy/plugin/solana_rest_api.py * Add echo PROXY_REVISION to check * set ${PROXY_REVISION:=${REVISION}} * Debugging * Debugging * Do the code clearly * Set setUpClass empty --- .buildkite/steps/build-image.sh | 1 + Dockerfile | 2 ++ proxy/plugin/solana_rest_api.py | 5 +++ proxy/testing/test_neon_proxy_version.py | 39 ++++++++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 proxy/testing/test_neon_proxy_version.py diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index 1492f34ad..29483beae 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -15,4 +15,5 @@ docker pull neonlabsorg/evm_loader:${EVM_LOADER_REVISION} docker build -t neonlabsorg/proxy:${REVISION} \ --build-arg SOLANA_REVISION=${SOLANA_REVISION} \ --build-arg EVM_LOADER_REVISION=${EVM_LOADER_REVISION} \ + --build-arg PROXY_REVISION=${REVISION} \ . diff --git a/Dockerfile b/Dockerfile index 6bffddcc6..8697f2f64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli FROM neonlabsorg/evm_loader:${EVM_LOADER_REVISION} AS spl FROM ubuntu:20.04 +ARG PROXY_REVISION RUN apt update && \ DEBIAN_FRONTEND=noninteractive apt -y install \ @@ -41,6 +42,7 @@ COPY --from=spl /opt/solana_utils.py \ COPY --from=spl /opt/neon-cli /spl/bin/emulator COPY . /opt +RUN sed -i 's/NEON_PROXY_REVISION_TO_BE_REPLACED/'"$PROXY_REVISION"'/g' /opt/proxy/plugin/solana_rest_api.py COPY proxy/operator-keypair.json /root/.config/solana/id.json RUN cd /usr/local/lib/python3.8/dist-packages/ && patch -p0 Optional[sol_Account]: solana_account = sol_Account(values) return solana_account + def neon_proxy_version(self): + return 'Neon-proxy/v' + NEON_PROXY_PKG_VERSION + '-' + NEON_PROXY_REVISION + def web3_clientVersion(self): neon_config_load(self) return self.neon_config_dict['web3_clientVersion'] diff --git a/proxy/testing/test_neon_proxy_version.py b/proxy/testing/test_neon_proxy_version.py new file mode 100644 index 000000000..c5ed29e40 --- /dev/null +++ b/proxy/testing/test_neon_proxy_version.py @@ -0,0 +1,39 @@ +import unittest +import os +import requests +import json +import inspect + +from proxy.plugin.solana_rest_api import NEON_PROXY_PKG_VERSION, NEON_PROXY_REVISION + +proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') +headers = {'Content-type': 'application/json'} + + +def get_line_number(): + cf = inspect.currentframe() + return cf.f_back.f_lineno + + +class TestNeonProxyVersion(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + def test_01_neon_proxy_version(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/320") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "neon_proxy_version", + "params": [] + })).text) + print('response:', response) + neon_proxy_version = response['result'] + print('neon_proxy_version:', neon_proxy_version) + self.assertEqual(neon_proxy_version, 'Neon-proxy/v' + NEON_PROXY_PKG_VERSION + '-' + NEON_PROXY_REVISION) + + +if __name__ == '__main__': + unittest.main() From dea01f4a9d3f3f4f4509191953ea4c2ab5427277 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 25 Nov 2021 12:22:50 +0300 Subject: [PATCH 13/54] 313 concurrent execution of solana program dump (#314) * fix * get rid of old-style neon-elf-params * Get rid of dump-files * Specify solana url explicitely Co-authored-by: ivanl --- proxy/deploy-test.sh | 3 +-- proxy/environment.py | 11 +---------- proxy/plugin/solana_rest_api.py | 6 +++++- proxy/run-test-proxy.sh | 3 +-- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/proxy/deploy-test.sh b/proxy/deploy-test.sh index a0df058f2..3ebbfda0f 100755 --- a/proxy/deploy-test.sh +++ b/proxy/deploy-test.sh @@ -5,8 +5,7 @@ echo "Deploy test..." solana config set -u $SOLANA_URL solana address || solana-keygen new --no-passphrase -solana program dump "$EVM_LOADER" ./evm_loader.dump -export $(/spl/bin/neon-cli --evm_loader "$EVM_LOADER" neon-elf-params ./evm_loader.dump) +export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) curl -v --header "Content-Type: application/json" --data '{"method":"eth_blockNumber","id":1,"jsonrpc":"2.0","params":[]}' $PROXY_URL diff --git a/proxy/environment.py b/proxy/environment.py index b06465798..794f099a8 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -48,16 +48,7 @@ def version(self): raise def read_elf_params(out_dict): - logger.debug('load for solana_url={} and evm_loader_id={}'.format(solana_url, evm_loader_id)) - res = solana_cli().call('program', 'dump', evm_loader_id, './evm_loader.dump') - substr = "Wrote program to " - path = "" - for line in res.splitlines(): - if line.startswith(substr): - path = line[len(substr):].strip() - if path == "": - raise Exception("cannot program dump for ", evm_loader_id) - for param in neon_cli().call("neon-elf-params", path).splitlines(): + for param in neon_cli().call("neon-elf-params").splitlines(): if param.startswith('NEON_') and '=' in param: v = param.split('=') out_dict[v[0]] = v[1] diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 9597efc0b..0372daa8d 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -128,8 +128,12 @@ def process_block_tag(self, tag): slot = int(self.client.get_slot(commitment=Confirmed)["result"]) elif tag in ('earliest', 'pending'): raise Exception("Invalid tag {}".format(tag)) - else: + elif isinstance(tag, str): slot = int(tag, 16) + elif isinstance(tag, int): + slot = tag + else: + raise Exception(f'Failed to parse block tag: {tag}') return slot def eth_blockNumber(self): diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index 5df1ad4ab..4b9af9199 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -9,8 +9,7 @@ solana config set -u $SOLANA_URL echo "Dumping evm_loader and extracting ELF parameters" export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) -solana program dump "$EVM_LOADER" ./evm_loader.dump -export $(/spl/bin/neon-cli --evm_loader="$EVM_LOADER" neon-elf-params ./evm_loader.dump) +export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader="$EVM_LOADER" neon-elf-params) /spl/bin/create-test-accounts.sh 1 From 024bf02ed5e8266ad735c275ba833f4d31501029 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 25 Nov 2021 15:43:09 +0300 Subject: [PATCH 14/54] fix scripts (#329) Co-authored-by: ivanl --- proxy/deploy-test.sh | 2 +- proxy/run-test-proxy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/deploy-test.sh b/proxy/deploy-test.sh index 3ebbfda0f..b47dc651f 100755 --- a/proxy/deploy-test.sh +++ b/proxy/deploy-test.sh @@ -5,7 +5,7 @@ echo "Deploy test..." solana config set -u $SOLANA_URL solana address || solana-keygen new --no-passphrase -export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) +export $(/spl/bin/neon-cli --commitment confirmed --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) curl -v --header "Content-Type: application/json" --data '{"method":"eth_blockNumber","id":1,"jsonrpc":"2.0","params":[]}' $PROXY_URL diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index 4b9af9199..fdb38c24b 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -9,7 +9,7 @@ solana config set -u $SOLANA_URL echo "Dumping evm_loader and extracting ELF parameters" export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) -export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader="$EVM_LOADER" neon-elf-params) +export $(/spl/bin/neon-cli --commitment confirmed --url $SOLANA_URL --evm_loader="$EVM_LOADER" neon-elf-params) /spl/bin/create-test-accounts.sh 1 From cf9112d299cfcd46f7d3ad716d34cc0300e2e494 Mon Sep 17 00:00:00 2001 From: mich-master <86723787+mich-master@users.noreply.github.com> Date: Fri, 26 Nov 2021 18:24:16 +0300 Subject: [PATCH 15/54] #318 JSON_RPC "params" field may be omitted (#322) * #318 JSON-RPC "params " field may be omitted --- proxy/plugin/solana_rest_api.py | 3 ++- proxy/testing/test_user_stories.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 0372daa8d..f84a80c6d 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -591,7 +591,8 @@ def process_request(self, request): } try: method = getattr(self.model, request['method']) - response['result'] = method(*request['params']) + params = request.get('params', []) + response['result'] = method(*params) except SolanaTrxError as err: traceback.print_exc() response['error'] = err.result diff --git a/proxy/testing/test_user_stories.py b/proxy/testing/test_user_stories.py index 45b837223..e6001740e 100644 --- a/proxy/testing/test_user_stories.py +++ b/proxy/testing/test_user_stories.py @@ -91,6 +91,18 @@ def test_04_check_eth_estimateGas_on_deploying_a_contract_with_the_empty_data_an print('used_gas:', used_gas) self.assertEqual(used_gas, 53001 + EXTRA_GAS) + def test_05_check_params_omitted(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/318") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "eth_chainId" + })).text) + print('response:', response) + chain_id = int(response['result'], 0) + print('chain_id:', chain_id) + self.assertEqual(chain_id, 111) if __name__ == '__main__': unittest.main() From fa9e43e7dbb6c45392d5b379dc1e29911ba8d7e8 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 2 Dec 2021 08:30:51 +0300 Subject: [PATCH 16/54] #336 indexer refactoring (#340) * cherrypick part of changes * create indexer.py * remove solana_receipts_update.py * Fix inspection issues * fix last issue Co-authored-by: ivanl --- .buildkite/steps/deploy-test.sh | 4 + proxy/docker-compose-test.yml | 2 +- .../{solana_receipts_update.py => indexer.py} | 167 ++++-------------- proxy/indexer/indexer_base.py | 149 ++++++++++++++++ proxy/proxy.py | 7 +- 5 files changed, 196 insertions(+), 133 deletions(-) rename proxy/indexer/{solana_receipts_update.py => indexer.py} (84%) create mode 100644 proxy/indexer/indexer_base.py diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index f42f7dffd..d90039ef3 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -79,6 +79,10 @@ docker run --rm -ti --network=container:proxy \ -e SOLANA_URL \ -e EXTRA_GAS=100000 \ -e NEW_USER_AIRDROP_AMOUNT=100 \ + -e POSTGRES_DB=neon-db \ + -e POSTGRES_USER=neon-proxy \ + -e POSTGRES_PASSWORD=neon-proxy-pass \ + -e POSTGRES_HOST=postgres \ --entrypoint ./proxy/deploy-test.sh \ ${EXTRA_ARGS:-} \ $PROXY_IMAGE \ diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index d40820a89..3d98cb7ef 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -46,7 +46,7 @@ services: POSTGRES_PASSWORD: neon-proxy-pass hostname: postgres healthcheck: - test: [ CMD-SHELL, "pg_isready" ] + test: [ CMD-SHELL, "pg_isready -h postgres -p 5432" ] interval: 5s timeout: 10s retries: 10 diff --git a/proxy/indexer/solana_receipts_update.py b/proxy/indexer/indexer.py similarity index 84% rename from proxy/indexer/solana_receipts_update.py rename to proxy/indexer/indexer.py index 2bc67b1e2..9ca772648 100644 --- a/proxy/indexer/solana_receipts_update.py +++ b/proxy/indexer/indexer.py @@ -1,13 +1,11 @@ +from proxy.indexer.indexer_base import logger, IndexerBase, PARALLEL_REQUESTS import base58 import rlp import json import os import time import logging -from solana.rpc.api import Client from multiprocessing.dummy import Pool as ThreadPool -from typing import Dict, Union -from proxy.environment import solana_url, evm_loader_id try: @@ -17,16 +15,7 @@ from .utils import check_error, get_trx_results, get_trx_receipts, LogDB, Canceller from .sql_dict import SQLDict - -PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2")) CANCEL_TIMEOUT = int(os.environ.get("CANCEL_TIMEOUT", "60")) - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -DEVNET_HISTORY_START = "7BdwyUQ61RUZP63HABJkbW66beLk22tdXnP69KsvQBJekCPVaHoJY47Rw68b3VV1UbQNHxX3uxUSLfiJrfy2bTn" -HISTORY_START = [DEVNET_HISTORY_START] - UPDATE_BLOCK_COUNT = PARALLEL_REQUESTS * 16 class HolderStruct: @@ -57,131 +46,34 @@ def __init__(self, eth_trx, eth_signature, from_address, got_result, signatures, self.slot = slot -class Indexer: - def __init__(self): - self.client = Client(solana_url) +class Indexer(IndexerBase): + def __init__(self, + solana_url, + evm_loader_id, + log_level = 'INFO'): + IndexerBase.__init__(self, solana_url, evm_loader_id, log_level) + self.canceller = Canceller() self.logs_db = LogDB() self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash") - self.transaction_receipts = SQLDict(tablename="known_transactions") self.ethereum_trx = SQLDict(tablename="ethereum_transactions") self.eth_sol_trx = SQLDict(tablename="ethereum_solana_transactions") self.sol_eth_trx = SQLDict(tablename="solana_ethereum_transactions") self.constants = SQLDict(tablename="constants") - self.last_slot = 0 - self.current_slot = 0 - self.transaction_order = [] if 'last_block' not in self.constants: self.constants['last_block'] = 0 self.blocked_storages = {} - self.counter_ = 0 - - def run(self, loop = True): - while (True): - try: - logger.debug("Start indexing") - self.gather_unknown_transactions() - logger.debug("Process receipts") - self.process_receipts() - logger.debug("Start getting blocks") - self.gather_blocks() - logger.debug("Unlock accounts") - self.canceller.unlock_accounts(self.blocked_storages) - self.blocked_storages = {} - except Exception as err: - logger.debug("Got exception while indexing. Type(err):%s, Exception:%s", type(err), err) - - - def gather_unknown_transactions(self): - poll_txs = set() - ordered_txs = [] - - minimal_tx = None - continue_flag = True - current_slot = self.client.get_slot(commitment="confirmed")["result"] - maximum_slot = self.last_slot - minimal_slot = current_slot - - percent = 0 - - counter = 0 - while (continue_flag): - opts: Dict[str, Union[int, str]] = {} - if minimal_tx: - opts["before"] = minimal_tx - opts["commitment"] = "confirmed" - result = self.client._provider.make_request("getSignaturesForAddress", evm_loader_id, opts) - logger.debug("{:>3} get_signatures_for_address {}".format(counter, len(result["result"]))) - counter += 1 - - if len(result["result"]) == 0: - logger.debug("len(result['result']) == 0") - break - - for tx in result["result"]: - solana_signature = tx["signature"] - slot = tx["slot"] - - if solana_signature in HISTORY_START: - logger.debug(solana_signature) - continue_flag = False - break - - ordered_txs.append(solana_signature) - - if solana_signature not in self.transaction_receipts: - poll_txs.add(solana_signature) - - if slot < minimal_slot: - minimal_slot = slot - minimal_tx = solana_signature - - if slot > maximum_slot: - maximum_slot = slot - if slot < self.last_slot: - continue_flag = False - break - logger.debug("start getting receipts") - pool = ThreadPool(PARALLEL_REQUESTS) - pool.map(self.get_tx_receipts, poll_txs) - - if len(self.transaction_order): - index = 0 - try: - index = ordered_txs.index(self.transaction_order[0]) - except ValueError: - self.transaction_order = ordered_txs + self.transaction_order - else: - self.transaction_order = ordered_txs[:index] + self.transaction_order - else: - self.transaction_order = ordered_txs - - self.last_slot = maximum_slot - self.current_slot = current_slot - - self.counter_ = 0 - - - def get_tx_receipts(self, solana_signature): - # trx = None - retry = True - - while retry: - try: - trx = self.client.get_confirmed_transaction(solana_signature)['result'] - self.transaction_receipts[solana_signature] = trx - retry = False - except Exception as err: - logger.debug(err) - time.sleep(1) - - self.counter_ += 1 - if self.counter_ % 100 == 0: - logger.debug(self.counter_) - - # return (solana_signature, trx) + def process_functions(self): + IndexerBase.process_functions(self) + logger.debug("Process receipts") + self.process_receipts() + logger.debug("Start getting blocks") + self.gather_blocks() + logger.debug("Unlock accounts") + self.canceller.unlock_accounts(self.blocked_storages) + self.blocked_storages = {} def process_receipts(self): @@ -209,7 +101,7 @@ def process_receipts(self): if trx['transaction']['message']['instructions'] is not None: for instruction in trx['transaction']['message']['instructions']: - if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] != evm_loader_id: + if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] != self.evm_loader_id: continue if check_error(trx): @@ -574,12 +466,27 @@ def get_block(self, slot): return (slot, block_hash) -def run_indexer(): +def run_indexer(solana_url, + evm_loader_id, + log_level = 'DEBUG'): logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s') logger.setLevel(logging.DEBUG) - indexer = Indexer() - indexer.run(False) + logger.info(f"""Running indexer with params: + solana_url: {solana_url}, + evm_loader_id: {evm_loader_id}, + log_level: {log_level}""") + + indexer = Indexer(solana_url, + evm_loader_id, + log_level) + indexer.run() if __name__ == "__main__": - run_indexer() + solana_url = os.environ.get('SOLANA_URL', 'http://localhost:8899') + evm_loader_id = os.environ.get('EVM_LOADER_ID', '53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io') + log_level = os.environ.get('LOG_LEVEL', 'INFO') + + run_indexer(solana_url, + evm_loader_id, + log_level) diff --git a/proxy/indexer/indexer_base.py b/proxy/indexer/indexer_base.py new file mode 100644 index 000000000..cda952bbf --- /dev/null +++ b/proxy/indexer/indexer_base.py @@ -0,0 +1,149 @@ +import os +import time +import logging +from solana.rpc.api import Client +from multiprocessing.dummy import Pool as ThreadPool +from typing import Dict, Union + +try: + from sql_dict import SQLDict +except ImportError: + from .sql_dict import SQLDict + + +PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2")) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +DEVNET_HISTORY_START = "7BdwyUQ61RUZP63HABJkbW66beLk22tdXnP69KsvQBJekCPVaHoJY47Rw68b3VV1UbQNHxX3uxUSLfiJrfy2bTn" +HISTORY_START = [DEVNET_HISTORY_START] + + +log_levels = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARN': logging.WARN, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'FATAL': logging.FATAL, + 'CRITICAL': logging.CRITICAL +} + +class IndexerBase: + def __init__(self, + solana_url, + evm_loader_id, + log_level): + logger.setLevel(log_levels.get(log_level, logging.INFO)) + + self.evm_loader_id = evm_loader_id + self.client = Client(solana_url) + self.transaction_receipts = SQLDict(tablename="known_transactions") + self.last_slot = 0 + self.current_slot = 0 + self.transaction_order = [] + self.counter_ = 0 + + + def run(self): + while (True): + try: + self.process_functions() + except Exception as err: + logger.warning("Got exception while indexing. Type(err):%s, Exception:%s", type(err), err) + + + def process_functions(self): + logger.debug("Start indexing") + self.gather_unknown_transactions() + + + def gather_unknown_transactions(self): + poll_txs = set() + ordered_txs = [] + + minimal_tx = None + continue_flag = True + current_slot = self.client.get_slot(commitment="confirmed")["result"] + maximum_slot = self.last_slot + minimal_slot = current_slot + + counter = 0 + while (continue_flag): + opts: Dict[str, Union[int, str]] = {} + if minimal_tx: + opts["before"] = minimal_tx + opts["commitment"] = "confirmed" + result = self.client._provider.make_request("getSignaturesForAddress", self.evm_loader_id, opts) + logger.debug("{:>3} get_signatures_for_address {}".format(counter, len(result["result"]))) + counter += 1 + + if len(result["result"]) == 0: + logger.debug("len(result['result']) == 0") + break + + for tx in result["result"]: + solana_signature = tx["signature"] + slot = tx["slot"] + + if solana_signature in HISTORY_START: + logger.debug(solana_signature) + continue_flag = False + break + + ordered_txs.append(solana_signature) + + if solana_signature not in self.transaction_receipts: + poll_txs.add(solana_signature) + + if slot < minimal_slot: + minimal_slot = slot + minimal_tx = solana_signature + + if slot > maximum_slot: + maximum_slot = slot + + if slot < self.last_slot: + continue_flag = False + break + + logger.debug("start getting receipts") + pool = ThreadPool(PARALLEL_REQUESTS) + pool.map(self.get_tx_receipts, poll_txs) + + if len(self.transaction_order): + index = 0 + try: + index = ordered_txs.index(self.transaction_order[0]) + except ValueError: + self.transaction_order = ordered_txs + self.transaction_order + else: + self.transaction_order = ordered_txs[:index] + self.transaction_order + else: + self.transaction_order = ordered_txs + + self.last_slot = maximum_slot + self.current_slot = current_slot + + self.counter_ = 0 + + + def get_tx_receipts(self, solana_signature): + # trx = None + retry = True + + while retry: + try: + trx = self.client.get_confirmed_transaction(solana_signature)['result'] + self.transaction_receipts[solana_signature] = trx + retry = False + except Exception as err: + logger.debug(err) + time.sleep(1) + + self.counter_ += 1 + if self.counter_ % 100 == 0: + logger.debug(self.counter_) + + # return (solana_signature, trx) diff --git a/proxy/proxy.py b/proxy/proxy.py index c63281665..b4eb08753 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -23,7 +23,8 @@ from .http.handler import HttpProtocolHandler from multiprocessing import Process -from .indexer.solana_receipts_update import run_indexer +from .indexer.indexer import run_indexer +from proxy.environment import solana_url, evm_loader_id logger = logging.getLogger(__name__) @@ -45,7 +46,9 @@ def delete_pid_file(self) -> None: os.remove(self.flags.pid_file) def __enter__(self) -> 'Proxy': - self.indexer = Process(target=run_indexer) + self.indexer = Process(target=run_indexer, + args=(solana_url, + evm_loader_id,)) self.indexer.start() self.acceptors = AcceptorPool( flags=self.flags, From c7ef79351480f18e72fdcf6666dd110b384afd2a Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Thu, 2 Dec 2021 14:38:57 +0300 Subject: [PATCH 17/54] #333 fix indexer errors (#334) * #333 fix continue blocked accounts * #333 continue_table fixes * #333 fixes * #333 remove filling db to lent indexer work properly * #333 sync --- proxy/indexer/indexer.py | 59 ++++++++++++++++++++++++++++++---------- proxy/indexer/utils.py | 4 +++ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index 9ca772648..52c9fcf81 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -178,8 +178,8 @@ def process_receipts(self): # logger.debug("rlp.exceptions.RLPException") pass except Exception as err: - if str(err).startswith("unhashable type"): - # logger.debug("unhashable type") + if str(err).startswith("nonhashable type"): + # logger.debug("nonhashable type") pass elif str(err).startswith("unsupported operand type"): # logger.debug("unsupported operand type") @@ -268,9 +268,9 @@ def process_receipts(self): continue_result = continue_table[storage_account] if continue_result.accounts != blocked_accounts: logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") - continue_result.signatures.append(signature) trx_table[eth_signature].got_result = continue_result.results - trx_table[eth_signature].signatures = continue_result.signatures + trx_table[eth_signature].signatures += continue_result.signatures + del continue_table[storage_account] elif instruction_data[0] == 0x0a or instruction_data[0] == 0x14: # Continue or ContinueV02 @@ -281,14 +281,14 @@ def process_receipts(self): blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] if instruction_data[0] == 0x14: # logger.debug("{:>10} {:>6} ContinueV02 0x{}".format(slot, counter, instruction_data.hex())) - blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] + blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] got_result = get_trx_results(trx) if storage_account in continue_table: continue_table[storage_account].signatures.append(signature) - if got_result: - if continue_table[storage_account].results: + if got_result is not None: + if continue_table[storage_account].results is not None: logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE") if continue_table[storage_account].accounts != blocked_accounts: logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") @@ -305,10 +305,8 @@ def process_receipts(self): # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeV02 0x{}".format(slot, counter, instruction_data.hex())) blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]] - holder_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][1]] - blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] if storage_account in continue_table: continue_table[storage_account].signatures.append(signature) @@ -328,7 +326,9 @@ def process_receipts(self): # logger.debug("{:>10} {:>6} Cancel 0x{}".format(slot, counter, instruction_data.hex())) storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] - continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot'])) + blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] + + continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), blocked_accounts) elif instruction_data[0] == 0x0d: # logger.debug("{:>10} {:>6} PartialCallOrContinueFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex())) @@ -360,6 +360,14 @@ def process_receipts(self): slot ) + if storage_account in continue_table: + continue_result = continue_table[storage_account] + trx_table[eth_signature].signatures += continue_result.signatures + if continue_result.results is not None: + trx_table[eth_signature].got_result = continue_result.results + + del continue_table[storage_account] + elif instruction_data[0] == 0x0e: # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeOrContinue 0x{}".format(slot, counter, instruction_data.hex())) @@ -379,7 +387,7 @@ def process_receipts(self): logger.error("Strange behavior. Pay attention. HOLDER ACCOUNT NOT FOUND") holder_table[holder_account] = HolderStruct(storage_account) - if got_result: + if got_result is not None: if continue_table[storage_account].results: logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE") if continue_table[storage_account].accounts != blocked_accounts: @@ -396,11 +404,15 @@ def process_receipts(self): pass for eth_signature, trx_struct in trx_table.items(): - if trx_struct.got_result: + if trx_struct.got_result is not None: self.submit_transaction(trx_struct) - elif trx_struct.storage: - if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT: - self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts) + elif trx_struct.storage is not None: + if not self.submit_transaction_part(trx_struct): + if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT: + logger.debug("Probably blocked") + logger.debug(trx_struct.eth_signature) + logger.debug(trx_struct.signatures) + self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts) else: logger.error(trx_struct) @@ -433,6 +445,23 @@ def submit_transaction(self, trx_struct): logger.debug(trx_struct.eth_signature + " " + status) + def submit_transaction_part(self, trx_struct): + ''' Check if transaction was allready submitted by proxy. ''' + eth_signature = trx_struct.eth_signature + ethereum_trx = self.ethereum_trx.get(eth_signature, None) + if ethereum_trx is not None: + signatures = self.eth_sol_trx.get(eth_signature, []) + signatures = signatures + trx_struct.signatures + self.eth_sol_trx[eth_signature] = signatures + for idx, sig in enumerate(signatures): + self.sol_eth_trx[sig] = { + 'idx': idx, + 'eth': eth_signature, + } + return True + return False + + def gather_blocks(self): max_slot = self.client.get_slot(commitment="recent")["result"] diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index c83b38588..cc579daa0 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -350,6 +350,10 @@ def unlock_accounts(self, blocked_storages): if blocked_accs is None: logger.error("blocked_accs is None") continue + if acc_list is None: + logger.error("acc_list is None. Storage is empty") + logger.error(storage) + continue eth_trx = rlp.decode(bytes.fromhex(eth_trx), EthTrx) if acc_list != blocked_accs: From e74e5687ac05b6139b4f7622caccd999bbb93e6a Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Thu, 2 Dec 2021 16:47:14 +0300 Subject: [PATCH 18/54] #291 Proxy refactoring (#324) * #291 extract transaction sender class * #291 move perm accs to transaction sender * #291 fix state * #291 fix errors * #291 merge fixes * #291 refactoring * #291 move EXTRA_GAS to environment * #291 capitalize CONFIRMATION_CHECK_DELAY * #291 sort imports * #291 relative paths * #291 Should be fixed in #326 * #291 testing chnages * fix storage account check * #291 rename `trx_with_create_and_airdrop` -> `make_trx_with_create_and_airdrop` * #291 pull request fixes * #291 merge fix * #291 rename operator and associated token accounts Co-authored-by: sinev-valentine --- .buildkite/steps/build-image.sh | 2 +- Dockerfile | 2 +- proxy/common_neon/address.py | 70 + proxy/common_neon/constants.py | 12 + proxy/common_neon/costs.py | 93 ++ proxy/common_neon/emulator_interactor.py | 34 + proxy/common_neon/errors.py | 11 + proxy/common_neon/layouts.py | 39 + proxy/common_neon/neon_instruction.py | 375 +++++ proxy/common_neon/solana_interactor.py | 214 +++ proxy/common_neon/transaction_sender.py | 420 ++++++ proxy/docker-compose-test.yml | 2 +- proxy/environment.py | 18 +- proxy/indexer/indexer.py | 3 +- proxy/indexer/utils.py | 60 +- proxy/plugin/solana_rest_api.py | 44 +- proxy/plugin/solana_rest_api_tools.py | 1207 +---------------- proxy/proxy.py | 6 +- proxy/run-test-proxy.sh | 2 +- .../testing/test_airdropping_eth_accounts.py | 3 +- proxy/testing/test_erc20_wrapper_contract.py | 6 +- ...anged.py => test_indexer_cancel_hanged.py} | 10 +- proxy/testing/test_resize_storage_account.py | 4 +- proxy/testing/test_web3_clientVersion.py | 5 - 24 files changed, 1376 insertions(+), 1266 deletions(-) create mode 100644 proxy/common_neon/address.py create mode 100644 proxy/common_neon/constants.py create mode 100644 proxy/common_neon/costs.py create mode 100644 proxy/common_neon/emulator_interactor.py create mode 100644 proxy/common_neon/layouts.py create mode 100644 proxy/common_neon/neon_instruction.py create mode 100644 proxy/common_neon/solana_interactor.py create mode 100644 proxy/common_neon/transaction_sender.py rename proxy/testing/{test_cancel_hanged.py => test_indexer_cancel_hanged.py} (95%) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index 29483beae..a67344b9f 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -3,7 +3,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) -set ${SOLANA_REVISION:=v1.7.9-resources} +set ${SOLANA_REVISION:=v1.7.9-testnet} set ${EVM_LOADER_REVISION:=latest} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally diff --git a/Dockerfile b/Dockerfile index 8697f2f64..a1abf84f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG SOLANA_REVISION=v1.7.9-resources +ARG SOLANA_REVISION=v1.7.9-testnet ARG EVM_LOADER_REVISION=latest FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli diff --git a/proxy/common_neon/address.py b/proxy/common_neon/address.py new file mode 100644 index 000000000..462ca8564 --- /dev/null +++ b/proxy/common_neon/address.py @@ -0,0 +1,70 @@ +import logging +import random + +from eth_keys import keys as eth_keys +from hashlib import sha256 +from solana.publickey import PublicKey +from spl.token.instructions import get_associated_token_address +from typing import NamedTuple + +from .layouts import ACCOUNT_INFO_LAYOUT +from ..environment import neon_cli, ETH_TOKEN_MINT_ID, EVM_LOADER_ID + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class EthereumAddress: + def __init__(self, data, private=None): + if isinstance(data, str): + data = bytes(bytearray.fromhex(data[2:])) + self.data = data + self.private = private + + @staticmethod + def random(): + letters = '0123456789abcdef' + data = bytearray.fromhex(''.join([random.choice(letters) for k in range(64)])) + pk = eth_keys.PrivateKey(data) + return EthereumAddress(pk.public_key.to_canonical_address(), pk) + + def __str__(self): + return '0x'+self.data.hex() + + def __repr__(self): + return self.__str__() + + def __bytes__(self): return self.data + + +def accountWithSeed(base, seed): + result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(PublicKey(EVM_LOADER_ID))).digest()) + return result + + +def ether2program(ether): + if isinstance(ether, str): + pass + elif isinstance(ether, EthereumAddress): + ether = str(ether) + else: + ether = ether.hex() + output = neon_cli().call("create-program-address", ether) + items = output.rstrip().split(' ') + return items[0], int(items[1]) + + +def getTokenAddr(account): + return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) + + +class AccountInfo(NamedTuple): + ether: eth_keys.PublicKey + trx_count: int + code_account: PublicKey + + @staticmethod + def frombytes(data): + cont = ACCOUNT_INFO_LAYOUT.parse(data) + return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account)) diff --git a/proxy/common_neon/constants.py b/proxy/common_neon/constants.py new file mode 100644 index 000000000..e16cfbd36 --- /dev/null +++ b/proxy/common_neon/constants.py @@ -0,0 +1,12 @@ +KECCAK_PROGRAM = "KeccakSecp256k11111111111111111111111111111" +INCINERATOR_PUBKEY = "1nc1nerator11111111111111111111111111111111" +SYSVAR_INSTRUCTION_PUBKEY = "Sysvar1nstructions1111111111111111111111111" + +STORAGE_SIZE = 128*1024 + +ACCOUNT_SEED_VERSION=b'\1' + +COLLATERALL_POOL_MAX=10 + +EMPTY_STORAGE_TAG=0 +FINALIZED_STORAGE_TAG=5 diff --git a/proxy/common_neon/costs.py b/proxy/common_neon/costs.py new file mode 100644 index 000000000..2012570b1 --- /dev/null +++ b/proxy/common_neon/costs.py @@ -0,0 +1,93 @@ +import base58 +import psycopg2 + +from ..environment import EVM_LOADER_ID +from ..indexer.sql_dict import POSTGRES_USER, POSTGRES_HOST, POSTGRES_DB, POSTGRES_PASSWORD + +class SQLCost(): + def __init__(self): + + self.conn = psycopg2.connect( + dbname=POSTGRES_DB, + user=POSTGRES_USER, + password=POSTGRES_PASSWORD, + host=POSTGRES_HOST + ) + + self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = self.conn.cursor() + cur.execute(''' + CREATE TABLE IF NOT EXISTS OPERATOR_COST + ( + hash char(64), + cost bigint, + used_gas bigint, + sender char(40), + to_address char(40) , + sig char(100), + status varchar(100), + reason varchar(100) + )''' + ) + + def close(self): + self.conn.close() + + def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason): + cur = self.conn.cursor() + cur.execute(''' + INSERT INTO OPERATOR_COST (hash, cost, used_gas, sender, to_address, sig, status, reason) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s) + ''', + (hash, cost, used_gas, sender, to_address, sig, status, reason) + ) + + +class CostSingleton(object): + def __new__(cls): + if not hasattr(cls, 'instance'): + cls.instance = super(CostSingleton, cls).__new__(cls) + cls.instance.operator_cost = SQLCost() + return cls.instance + + +def update_transaction_cost(receipt, eth_trx, extra_sol_trx=False, reason=None): + cost = receipt['result']['meta']['preBalances'][0] - receipt['result']['meta']['postBalances'][0] + if eth_trx: + hash = eth_trx.hash_signed().hex() + sender = eth_trx.sender() + to_address = eth_trx.toAddress.hex() if eth_trx.toAddress else "None" + else: + hash = None + sender = None + to_address = None + + sig = receipt['result']['transaction']['signatures'][0] + used_gas=None + + tx_info = receipt['result'] + accounts = tx_info["transaction"]["message"]["accountKeys"] + evm_loader_instructions = [] + + for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: + evm_loader_instructions.append(idx) + + for inner in (tx_info['meta']['innerInstructions']): + if inner["index"] in evm_loader_instructions: + for event in inner['instructions']: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: + used_gas = base58.b58decode(event['data'])[2:10] + used_gas = int().from_bytes(used_gas, "little") + + table = CostSingleton().operator_cost + table.insert( + hash, + cost, + used_gas if used_gas else 0, + sender, + to_address, + sig, + 'extra' if extra_sol_trx else 'ok', + reason if reason else '' + ) diff --git a/proxy/common_neon/emulator_interactor.py b/proxy/common_neon/emulator_interactor.py new file mode 100644 index 000000000..7f5897f4d --- /dev/null +++ b/proxy/common_neon/emulator_interactor.py @@ -0,0 +1,34 @@ +import json +import logging + +from .errors import EthereumError +from ..environment import neon_cli + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def call_emulated(contract_id, caller_id, data=None, value=None): + output = emulator(contract_id, caller_id, data, value) + logger.debug("call_emulated %s %s %s %s return %s", contract_id, caller_id, data, value, output) + result = json.loads(output) + exit_status = result['exit_status'] + if exit_status == 'revert': + result_value = result['result'] + if len(result_value) < 8 or result_value[:8] != '08c379a0': + raise EthereumError(code=3, message='execution reverted') + + offset = int(result_value[8:8+64], 16) + length = int(result_value[8+64:8+64+64], 16) + message = str(bytes.fromhex(result_value[8+offset*2+64:8+offset*2+64+length*2]), 'utf8') + raise EthereumError(code=3, message='execution reverted: '+message, data='0x'+result_value) + if result["exit_status"] != "succeed": + raise Exception("evm emulator error ", result) + return result + + +def emulator(contract, sender, data, value): + data = data or "none" + value = value or "" + return neon_cli().call("emulate", sender, contract, data, value) diff --git a/proxy/common_neon/errors.py b/proxy/common_neon/errors.py index e590afa64..4c17e8a9b 100644 --- a/proxy/common_neon/errors.py +++ b/proxy/common_neon/errors.py @@ -1,6 +1,17 @@ from enum import Enum +class EthereumError(Exception): + def __init__(self, code, message, data=None): + self.code = code + self.message = message + self.data = data + + def getError(self): + error = {'code': self.code, 'message': self.message} + if self.data: error['data'] = self.data + return error + class SolanaErrors(Enum): AccountNotFound = "Invalid param: could not find account" diff --git a/proxy/common_neon/layouts.py b/proxy/common_neon/layouts.py new file mode 100644 index 000000000..a48171ab7 --- /dev/null +++ b/proxy/common_neon/layouts.py @@ -0,0 +1,39 @@ + +from construct import Bytes, Int8ul, Int64ul +from construct import Struct + +STORAGE_ACCOUNT_INFO_LAYOUT = Struct( + # "tag" / Int8ul, + "caller" / Bytes(20), + "nonce" / Int64ul, + "gas_limit" / Int64ul, + "gas_price" / Int64ul, + "slot" / Int64ul, + "operator" / Bytes(32), + "accounts_len" / Int64ul, + "executor_data_size" / Int64ul, + "evm_data_size" / Int64ul, + "gas_used_and_paid" / Int64ul, + "number_of_payments" / Int64ul, + "sign" / Bytes(65), +) + +ACCOUNT_INFO_LAYOUT = Struct( + "type" / Int8ul, + "ether" / Bytes(20), + "nonce" / Int8ul, + "trx_count" / Bytes(8), + "code_account" / Bytes(32), + "is_rw_blocked" / Int8ul, + "rw_blocked_acc" / Bytes(32), + "eth_token_account" / Bytes(32), + "ro_blocked_cnt" / Int8ul, +) + + +CREATE_ACCOUNT_LAYOUT = Struct( + "lamports" / Int64ul, + "space" / Int64ul, + "ether" / Bytes(20), + "nonce" / Int8ul +) diff --git a/proxy/common_neon/neon_instruction.py b/proxy/common_neon/neon_instruction.py new file mode 100644 index 000000000..8242d97cb --- /dev/null +++ b/proxy/common_neon/neon_instruction.py @@ -0,0 +1,375 @@ +import eth_utils +import logging +import struct + +from sha3 import keccak_256 +from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT, InstructionType +from solana.publickey import PublicKey +from solana.system_program import SYS_PROGRAM_ID +from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY +from solana.transaction import AccountMeta, TransactionInstruction, Transaction +from spl.token.constants import ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID +from spl.token.instructions import transfer2, Transfer2Params +from typing import Tuple + +from .address import accountWithSeed, ether2program, getTokenAddr, EthereumAddress +from .constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM, COLLATERALL_POOL_MAX +from .layouts import CREATE_ACCOUNT_LAYOUT +from ..environment import EVM_LOADER_ID, ETH_TOKEN_MINT_ID , COLLATERAL_POOL_BASE, NEW_USER_AIRDROP_AMOUNT + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +obligatory_accounts = [ + AccountMeta(pubkey=EVM_LOADER_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_CLOCK_PUBKEY, is_signer=False, is_writable=False), +] + + +def create_account_with_seed_layout(base, seed, lamports, space): + return SYSTEM_INSTRUCTIONS_LAYOUT.build( + dict( + instruction_type = InstructionType.CREATE_ACCOUNT_WITH_SEED, + args=dict( + base=bytes(base), + seed=dict(length=len(seed), chars=seed), + lamports=lamports, + space=space, + program_id=bytes(PublicKey(EVM_LOADER_ID)) + ) + ) + ) + + +def create_account_layout(lamports, space, ether, nonce): + return bytes.fromhex("02000000")+CREATE_ACCOUNT_LAYOUT.build(dict( + lamports=lamports, + space=space, + ether=ether, + nonce=nonce + )) + + +def write_holder_layout(nonce, offset, data): + return (bytes.fromhex('12')+ + nonce.to_bytes(8, byteorder='little')+ + offset.to_bytes(4, byteorder='little')+ + len(data).to_bytes(8, byteorder='little')+ + data) + + +def make_keccak_instruction_data(check_instruction_index, msg_len, data_start): + if check_instruction_index > 255 and check_instruction_index < 0: + raise Exception("Invalid index for instruction - {}".format(check_instruction_index)) + + check_count = 1 + eth_address_size = 20 + signature_size = 65 + eth_address_offset = data_start + signature_offset = eth_address_offset + eth_address_size + message_data_offset = signature_offset + signature_size + + data = struct.pack("B", check_count) + data += struct.pack(" Tuple[Transaction, PublicKey]: + pda_account, nonce = ether2program(eth_address) + neon_token_account = getTokenAddr(PublicKey(pda_account)) + logger.debug(f'Create eth account: {eth_address}, sol account: {pda_account}, neon_token_account: {neon_token_account}, nonce: {nonce}') + + base = self.operator_account + data = create_account_layout(0, 0, bytes(eth_address), nonce) + trx = Transaction() + if code_acc is None: + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(pda_account), is_signer=False, is_writable=True), + AccountMeta(pubkey=neon_token_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ])) + else: + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(pda_account), is_signer=False, is_writable=True), + AccountMeta(pubkey=neon_token_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ])) + return trx, neon_token_account + + + def createERC20TokenAccountTrx(self, token_info) -> Transaction: + trx = Transaction() + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=bytes.fromhex('0F'), + keys=[ + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["key"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["owner"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["contract"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["mint"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ] + )) + + return trx + + + def make_transfer_instruction(self, associated_token_account: PublicKey) -> TransactionInstruction: + transfer_instruction = transfer2(Transfer2Params( + source=self.operator_neon_address, + owner=self.operator_account, + dest=associated_token_account, + amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei, + decimals=9, + mint=ETH_TOKEN_MINT_ID, + program_id=TOKEN_PROGRAM_ID + )) + logger.debug(f"Token transfer from token: {self.operator_neon_address}, owned by: {self.operator_account}, to token: " + f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}") + return transfer_instruction + + + def make_trx_with_create_and_airdrop(self, eth_account, code_acc=None) -> Transaction: + trx = Transaction() + create_trx, associated_token_account = self.make_create_eth_account_trx(eth_account, code_acc) + trx.add(create_trx) + if NEW_USER_AIRDROP_AMOUNT <= 0: + return trx + transfer_instruction = self.make_transfer_instruction(associated_token_account) + trx.add(transfer_instruction) + + return trx + + + def make_resize_instruction(self, acc_desc, code_account_new, seed) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("11") + bytes(seed), # 17- ResizeStorageAccount + keys = [ + AccountMeta(pubkey=PublicKey(acc_desc["account"]), is_signer=False, is_writable=True), + ( + AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=True) + if acc_desc["contract"] else + AccountMeta(pubkey=PublicKey("11111111111111111111111111111111"), is_signer=False, is_writable=False) + ), + AccountMeta(pubkey=code_account_new, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=False) + ], + ) + + + def make_write_transaction(self, offset: int, data: bytes) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=write_holder_layout(self.perm_accs_id, offset, data), + keys=[ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=False), + ] + )) + + + def make_keccak_instruction(self, check_instruction_index, msg_len, data_start) -> TransactionInstruction: + return TransactionInstruction( + program_id=KECCAK_PROGRAM, + data=make_keccak_instruction_data(check_instruction_index, msg_len, data_start), + keys=[ + AccountMeta(pubkey=KECCAK_PROGRAM, is_signer=False, is_writable=False), + ] + ) + + + def make_05_call_instruction(self) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("05") + self.collateral_pool_index_buf + self.msg, + keys = [ + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + obligatory_accounts + ) + + + def make_noniterative_call_transaction(self, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 5)) + trx.add(self.make_05_call_instruction()) + return trx + + + def make_partial_call_instruction(self) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("13") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little") + self.msg, + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + ) + + + def make_iterative_call_transaction(self, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 13)) + trx.add(self.make_partial_call_instruction()) + return trx + + + def make_call_from_account_instruction(self) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("16") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little"), + keys = [ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) + + + def make_continue_instruction(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("14") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + if index: + data = data + index.to_bytes(8, byteorder="little") + + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = data, + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) + + + def make_cancel_instruction(self) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py new file mode 100644 index 000000000..ffb02e5d4 --- /dev/null +++ b/proxy/common_neon/solana_interactor.py @@ -0,0 +1,214 @@ +import base58 +import base64 +import json +import logging +import re +import time + +from solana.rpc.commitment import Confirmed +from solana.rpc.types import TxOpts + +from .costs import update_transaction_cost +from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class SolanaInteractor: + def __init__(self, signer, client) -> None: + self.signer = signer + self.client = client + + + def get_operator_key(self): + return self.signer.public_key() + + + def get_account_info(self, storage_account): + opts = { + "encoding": "base64", + "commitment": "confirmed", + "dataSlice": { + "offset": 0, + "length": 16, + } + } + + result = self.client._provider.make_request("getAccountInfo", str(storage_account), opts) + logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) + + info = result['result']['value'] + if info is None: + logger.debug("Can't get information about {}".format(storage_account)) + return None + + data = base64.b64decode(info['data'][0]) + + account_tag = data[0] + lamports = info['lamports'] + owner = info['owner'] + + return (account_tag, lamports, owner) + + + def get_sol_balance(self, account): + return self.client.get_balance(account, commitment=Confirmed)['result']['value'] + + + def get_rent_exempt_balance_for_size(self, size): + return self.client.get_minimum_balance_for_rent_exemption(size, commitment=Confirmed)["result"] + + + def _getAccountData(self, account, expected_length, owner=None): + info = self.client.get_account_info(account, commitment=Confirmed)['result']['value'] + if info is None: + raise Exception("Can't get information about {}".format(account)) + + data = base64.b64decode(info['data'][0]) + if len(data) < expected_length: + raise Exception("Wrong data length for account data {}".format(account)) + return data + + + def send_transaction(self, trx, eth_trx, reason=None): + reciept = self.send_transaction_unconfirmed(trx) + result = self.collect_result(reciept, eth_trx, reason) + return result + + + def send_transaction_unconfirmed(self, trx): + result = self.client.send_transaction(trx, self.signer, opts=TxOpts(preflight_commitment=Confirmed))["result"] + return result + + def collect_result(self, reciept, eth_trx, reason=None): + self.confirm_transaction(reciept) + result = self.client.get_confirmed_transaction(reciept) + update_transaction_cost(result, eth_trx, reason) + return result + + + def send_measured_transaction(self, trx, eth_trx, reason): + result = self.send_transaction(trx, eth_trx, reason=reason) + self.get_measurements(result) + return result + + + # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh) + def get_measurements(self, result): + try: + measurements = self.extract_measurements_from_receipt(result) + for m in measurements: logger.info(json.dumps(m)) + except Exception as err: + logger.error("Can't get measurements %s"%err) + logger.info("Failed result: %s"%json.dumps(result, indent=3)) + + + def confirm_transaction(self, tx_sig, confirmations=0): + """Confirm a transaction.""" + TIMEOUT = 30 # 30 seconds pylint: disable=invalid-name + elapsed_time = 0 + while elapsed_time < TIMEOUT: + logger.debug('confirm_transaction for %s', tx_sig) + resp = self.client.get_signature_statuses([tx_sig]) + logger.debug('confirm_transaction: %s', resp) + if resp["result"]: + status = resp['result']['value'][0] + if status and (status['confirmationStatus'] == 'finalized' or \ + status['confirmationStatus'] == 'confirmed' and status['confirmations'] >= confirmations): + return + time.sleep(CONFIRMATION_CHECK_DELAY) + elapsed_time += CONFIRMATION_CHECK_DELAY + raise RuntimeError("could not confirm transaction: ", tx_sig) + + + def collect_results(self, receipts, eth_trx=None, reason=None): + results = [] + for rcpt in receipts: + results.append(self.collect_result(rcpt, eth_trx, reason)) + return results + + @staticmethod + def extract_measurements_from_receipt(receipt): + log_messages = receipt['result']['meta']['logMessages'] + transaction = receipt['result']['transaction'] + accounts = transaction['message']['accountKeys'] + instructions = [] + for instr in transaction['message']['instructions']: + program = accounts[instr['programIdIndex']] + instructions.append({ + 'accs': [accounts[acc] for acc in instr['accounts']], + 'program': accounts[instr['programIdIndex']], + 'data': base58.b58decode(instr['data']).hex() + }) + + pattern = re.compile('Program ([0-9A-Za-z]+) (.*)') + messages = [] + for log in log_messages: + res = pattern.match(log) + if res: + (program, reason) = res.groups() + if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) + messages[-1]['logs'].append(log) + + for instr in instructions: + if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue + if messages[0]['program'] != instr['program']: + raise Exception('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) + instr['logs'] = messages.pop(0)['logs'] + exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) + if not exit_result: raise Exception("Can't get exit result") + instr['result'] = exit_result.group(1) + + if instr['program'] == EVM_LOADER_ID: + memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) + instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) + if not (memory_result and instruction_result): + raise Exception("Can't parse measurements for evm_loader") + instr['measurements'] = { + 'instructions': instruction_result.group(1), + 'memory': memory_result.group(1) + } + + result = [] + for instr in instructions: + if instr['program'] == EVM_LOADER_ID: + result.append({ + 'program':instr['program'], + 'measurements':instr['measurements'], + 'result':instr['result'], + 'data':instr['data'] + }) + return result + + +def check_if_program_exceeded_instructions(err_result): + err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" + err_budget = "failed: Computational budget exceeded" + + if err_result['data']['logs'][-1].find(err_instruction) >= 0 or \ + err_result['data']['logs'][-2].find(err_instruction) >= 0 or \ + err_result['data']['logs'][-1].find(err_budget) >= 0: + return True + return False + + +def check_if_continue_returned(result): + tx_info = result['result'] + accounts = tx_info["transaction"]["message"]["accountKeys"] + evm_loader_instructions = [] + + for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: + evm_loader_instructions.append(idx) + + for inner in (tx_info['meta']['innerInstructions']): + if inner["index"] in evm_loader_instructions: + for event in inner['instructions']: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: + instruction = base58.b58decode(event['data'])[:1] + if int().from_bytes(instruction, "little") == 6: # OnReturn evmInstruction code + return tx_info['transaction']['signatures'][0] + + return None diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py new file mode 100644 index 000000000..5300c6a85 --- /dev/null +++ b/proxy/common_neon/transaction_sender.py @@ -0,0 +1,420 @@ +import json +import logging +import os +import rlp +import time + +from base58 import b58encode +from sha3 import keccak_256 +from solana.publickey import PublicKey +from solana.rpc.api import SendTransactionError +from solana.sysvar import * +from solana.transaction import AccountMeta, Transaction + +from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob + +from .address import accountWithSeed, AccountInfo, getTokenAddr +from .constants import STORAGE_SIZE, EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG, ACCOUNT_SEED_VERSION +from .emulator_interactor import call_emulated +from .layouts import ACCOUNT_INFO_LAYOUT +from .neon_instruction import NeonInstruction +from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_if_program_exceeded_instructions +from ..environment import EVM_LOADER_ID +from ..plugin.eth_proto import Trx as EthTrx + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class TransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, eth_trx: EthTrx, steps: int) -> None: + self.sender = solana_interactor + self.eth_trx = eth_trx + self.steps = steps + + self.instruction = NeonInstruction(self.sender.get_operator_key()) + + + def execute(self): + self.create_account_list_by_emulate() + + noniterative_executor = self.create_noniterative_executor() + + call_iterative = False + call_from_holder = False + + if not self.eth_trx.toAddress: + call_from_holder = True + else: + try: + logger.debug("Try single trx call") + return noniterative_executor.call_signed_noniterative() + except Exception as err: + logger.debug(str(err)) + errStr = str(err) + if "Program failed to complete" in errStr or "Computational budget exceeded" in errStr: + logger.debug("Program exceeded instructions") + call_iterative = True + elif str(err).startswith("transaction too large:"): + logger.debug("Transaction too large, call call_signed_with_holder_acc():") + call_from_holder = True + else: + raise + + self.init_perm_accs() + iterative_executor = self.create_iterative_executor() + try: + if call_iterative: + try: + return iterative_executor.call_signed_iterative() + except Exception as err: + logger.debug(str(err)) + if str(err).startswith("transaction too large:"): + logger.debug("Transaction too large, call call_signed_with_holder_acc():") + call_from_holder = True + else: + raise + + if call_from_holder: + return iterative_executor.call_signed_with_holder_acc() + finally: + self.free_perm_accs() + + + def create_instruction_constructor(self): + return NeonInstruction(self.sender.get_operator_key(), self.eth_trx, self.eth_accounts, self.caller_token) + + + def create_noniterative_executor(self): + self.instruction.init_eth_trx(self.eth_trx, self.eth_accounts, self.caller_token) + return NoniterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx) + + + def create_iterative_executor(self): + self.instruction.init_iterative(self.storage, self.holder, self.perm_accs_id) + return IterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx, self.steps) + + + def init_perm_accs(self): + while True: + with new_acc_id_glob.get_lock(): + try: + self.perm_accs_id = acc_list_glob.pop(0) + except IndexError: + self.perm_accs_id = new_acc_id_glob.value + new_acc_id_glob.value += 1 + + logger.debug("LOCK RESOURCES {}".format(self.perm_accs_id)) + + acc_id_bytes = self.perm_accs_id.to_bytes((self.perm_accs_id.bit_length() + 7) // 8, 'big') + + storage_seed = keccak_256(b"storage" + acc_id_bytes).hexdigest()[:32] + storage_seed = bytes(storage_seed, 'utf8') + + holder_seed = keccak_256(b"holder" + acc_id_bytes).hexdigest()[:32] + holder_seed = bytes(holder_seed, 'utf8') + + try: + self.storage, self.holder = self.create_multiple_accounts_with_seed( + seeds=[storage_seed, holder_seed], + sizes=[STORAGE_SIZE, STORAGE_SIZE] + ) + except Exception as err: + logger.warn("Account is locked err({}) id({}) owner({})".format(str(err), self.perm_accs_id, self.sender.get_operator_key())) + else: + break + + + def free_perm_accs(self): + logger.debug("FREE RESOURCES {}".format(self.perm_accs_id)) + with new_acc_id_glob.get_lock(): + acc_list_glob.append(self.perm_accs_id) + + + def create_account_with_seed(self, seed, storage_size): + account = accountWithSeed(self.sender.get_operator_key(), seed) + + if self.sender.get_sol_balance(account) == 0: + minimum_balance = self.sender.get_rent_exempt_balance_for_size(storage_size) + logger.debug("Minimum balance required for account {}".format(minimum_balance)) + + trx = Transaction() + trx.add(self.instruction.create_account_with_seed_trx(account, seed, minimum_balance, storage_size)) + self.sender.send_transaction(trx, eth_trx=self.eth_trx, reason='createAccountWithSeed') + + return account + + + def create_multiple_accounts_with_seed(self, seeds, sizes): + accounts = [] + trx = Transaction() + + for seed, storage_size in zip(seeds, sizes): + account = accountWithSeed(self.sender.get_operator_key(), seed) + accounts.append(account) + + minimum_balance = self.sender.get_rent_exempt_balance_for_size(storage_size) + + account_info = self.sender.get_account_info(account) + if account_info is None: + logger.debug("Minimum balance required for account {}".format(minimum_balance)) + + trx.add(self.instruction.create_account_with_seed_trx(account, seed, minimum_balance, storage_size)) + else: + (tag, lamports, owner) = account_info + if lamports < minimum_balance: + raise Exception("insufficient balance") + if PublicKey(owner) != PublicKey(EVM_LOADER_ID): + raise Exception("wrong owner") + if tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}: + raise Exception("not empty, not finalized") + + if len(trx.instructions) > 0: + self.sender.send_transaction(trx, eth_trx=self.eth_trx, reason='createAccountWithSeed') + + return accounts + + + def create_account_list_by_emulate(self): + sender_ether = bytes.fromhex(self.eth_trx.sender()) + add_keys_05 = [] + self.create_acc_trx = Transaction() + + if not self.eth_trx.toAddress: + to_address_arg = "deploy" + to_address = keccak_256(rlp.encode((bytes.fromhex(self.eth_trx.sender()), self.eth_trx.nonce))).digest()[-20:] + else: + to_address_arg = self.eth_trx.toAddress.hex() + to_address = self.eth_trx.toAddress + + logger.debug("send_addr: %s", self.eth_trx.sender()) + logger.debug("dest_addr: %s", to_address.hex()) + + output_json = call_emulated(to_address_arg, sender_ether.hex(), self.eth_trx.callData.hex(), hex(self.eth_trx.value)) + logger.debug("emulator returns: %s", json.dumps(output_json, indent=3)) + + # resize storage account + resize_instr = [] + for acc_desc in output_json["accounts"]: + if acc_desc["new"] == False: + if acc_desc["code_size_current"] is not None and acc_desc["code_size"] is not None: + if acc_desc["code_size"] > acc_desc["code_size_current"]: + code_size = acc_desc["code_size"] + 2048 + seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) + code_account_new = accountWithSeed(self.sender.get_operator_key(), seed) + + logger.debug("creating new code_account with increased size %s", code_account_new) + self.create_account_with_seed(seed, code_size) + logger.debug("resized account is created %s", code_account_new) + + resize_instr.append(self.instruction.make_resize_instruction(acc_desc, code_account_new, seed)) + # replace code_account + acc_desc["contract"] = code_account_new + + for instr in resize_instr: + logger.debug("code and storage migration, account %s from %s to %s", instr.keys[0].pubkey, instr.keys[1].pubkey, instr.keys[2].pubkey) + + tx = Transaction().add(instr) + success = False + count = 0 + + while count < 2: + logger.debug("attemt: %d", count) + + self.sender.send_transaction(tx, eth_trx=self.eth_trx, reason='resize_storage_account') + info = self.sender._getAccountData(instr.keys[0].pubkey, ACCOUNT_INFO_LAYOUT.sizeof()) + info_data = AccountInfo.frombytes(info) + if info_data.code_account == instr.keys[2].pubkey: + success = True + logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) + break + # wait for unlock account + time.sleep(1) + count = count+1 + + if success == False: + raise Exception("Can't resize storage account. Account is blocked {}".format(instr.keys[0].pubkey)) + + for acc_desc in output_json["accounts"]: + address = bytes.fromhex(acc_desc["address"][2:]) + + code_account = None + code_account_writable = False + if acc_desc["new"]: + logger.debug("Create solana accounts for %s: %s %s", acc_desc["address"], acc_desc["account"], acc_desc["contract"]) + if acc_desc["code_size"]: + seed = b58encode(ACCOUNT_SEED_VERSION+address) + code_account = accountWithSeed(self.sender.get_operator_key(), seed) + logger.debug(" with code account %s", code_account) + code_size = acc_desc["code_size"] + 2048 + code_account_balance = self.sender.get_rent_exempt_balance_for_size(code_size) + self.create_acc_trx.add(self.instruction.create_account_with_seed_trx(code_account, seed, code_account_balance, code_size)) + # add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=acc_desc["writable"])) + code_account_writable = acc_desc["writable"] + + create_trx = self.instruction.make_trx_with_create_and_airdrop(address, code_account) + self.create_acc_trx.add(create_trx) + + if address == to_address: + contract_sol = PublicKey(acc_desc["account"]) + if acc_desc["new"]: + code_sol = code_account + code_writable = code_account_writable + else: + if acc_desc["contract"] != None: + code_sol = PublicKey(acc_desc["contract"]) + code_writable = acc_desc["writable"] + else: + code_sol = None + code_writable = None + + elif address == sender_ether: + sender_sol = PublicKey(acc_desc["account"]) + else: + add_keys_05.append(AccountMeta(pubkey=acc_desc["account"], is_signer=False, is_writable=True)) + token_account = getTokenAddr(PublicKey(acc_desc["account"])) + add_keys_05.append(AccountMeta(pubkey=token_account, is_signer=False, is_writable=True)) + if acc_desc["new"]: + if code_account: + add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=code_account_writable)) + else: + if acc_desc["contract"]: + add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) + + + for token_account in output_json["token_accounts"]: + add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) + + if token_account["new"]: + self.create_acc_trx.add(self.instruction.createERC20TokenAccountTrx(token_account)) + + for account_meta in output_json["solana_accounts"]: + add_keys_05.append(AccountMeta(pubkey=PublicKey(account_meta["pubkey"]), is_signer=account_meta["is_signer"], is_writable=account_meta["is_writable"])) + + self.caller_token = getTokenAddr(PublicKey(sender_sol)) + + self.eth_accounts = [ + AccountMeta(pubkey=contract_sol, is_signer=False, is_writable=True), + AccountMeta(pubkey=getTokenAddr(contract_sol), is_signer=False, is_writable=True), + ] + ([AccountMeta(pubkey=code_sol, is_signer=False, is_writable=code_writable)] if code_sol != None else []) + [ + AccountMeta(pubkey=sender_sol, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + ] + add_keys_05 + + +class NoniterativeTransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx): + self.sender = solana_interactor + self.instruction = neon_instruction + self.create_acc_trx = create_acc_trx + self.eth_trx = eth_trx + + + def call_signed_noniterative(self): + call_txs_05 = Transaction() + if len(self.create_acc_trx.instructions) > 0: + call_txs_05.add(self.create_acc_trx) + call_txs_05.add(self.instruction.make_noniterative_call_transaction(len(call_txs_05.instructions))) + result = self.sender.send_measured_transaction(call_txs_05, self.eth_trx, 'CallFromRawEthereumTX') + return result['result']['transaction']['signatures'][0] + + +class IterativeTransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx, steps: int): + self.sender = solana_interactor + self.instruction = neon_instruction + self.create_acc_trx = create_acc_trx + self.eth_trx = eth_trx + self.steps = steps + + + def call_signed_iterative(self): + if len(self.create_acc_trx.instructions): + precall_txs = Transaction() + precall_txs.add(self.create_acc_trx) + self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') + + call_txs = self.instruction.make_iterative_call_transaction() + + logger.debug("Partial call") + self.sender.send_measured_transaction(call_txs, self.eth_trx, 'PartialCallFromRawEthereumTXv02') + + return self.call_continue() + + + def call_signed_with_holder_acc(self): + self.write_trx_to_holder_account() + if len(self.create_acc_trx.instructions): + precall_txs = Transaction() + precall_txs.add(self.create_acc_trx) + self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'create_accounts_for_deploy') + + # ExecuteTrxFromAccountDataIterative + logger.debug("ExecuteTrxFromAccountDataIterative:") + call_txs = self.instruction.make_call_from_account_instruction() + self.sender.send_measured_transaction(call_txs, self.eth_trx, 'ExecuteTrxFromAccountDataIterativeV02') + + return self.call_continue() + + + def write_trx_to_holder_account(self): + msg = self.eth_trx.signature() + len(self.eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + self.eth_trx.unsigned_msg() + + # Write transaction to transaction holder account + offset = 0 + receipts = [] + rest = msg + while len(rest): + (part, rest) = (rest[:1000], rest[1000:]) + # logger.debug("sender_sol %s %s %s", sender_sol, holder, acc.public_key()) + trx = self.instruction.make_write_transaction(offset, part) + receipts.append(self.sender.send_transaction_unconfirmed(trx)) + offset += len(part) + + logger.debug("receipts %s", receipts) + self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason='WriteHolder') + + + def call_continue(self): + try: + return self.call_continue_step_by_step() + except Exception as err: + logger.error("call_continue_step_by_step exception:") + logger.debug(str(err)) + + return self.call_cancel() + + + def call_continue_step_by_step(self): + while True: + logger.debug("Continue iterative step:") + result = self.call_continue_step() + signature = check_if_continue_returned(result) + if signature is not None: + return signature + + + def call_continue_step(self): + step_count = self.steps + while step_count > 0: + trx = self.instruction.make_continue_instruction(step_count) + + logger.debug("Step count {}".format(step_count)) + try: + result = self.sender.send_measured_transaction(trx, self.eth_trx, 'ContinueV02') + return result + except SendTransactionError as err: + if check_if_program_exceeded_instructions(err.result): + step_count = int(step_count * 90 / 100) + else: + raise + raise Exception("Can't execute even one EVM instruction") + + + def call_cancel(self): + trx = self.instruction.make_cancel_instruction() + + logger.debug("Cancel") + result = self.sender.send_measured_transaction(trx, self.eth_trx, 'CancelWithNonce') + return result['result']['transaction']['signatures'][0] diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 3d98cb7ef..729b4affa 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -3,7 +3,7 @@ version: "3" services: solana: container_name: solana - image: neonlabsorg/solana:${SOLANA_REVISION:-v1.7.9-resources} + image: neonlabsorg/solana:${SOLANA_REVISION:-v1.7.9-testnet} environment: SOLANA_URL: http://solana:8899 RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug diff --git a/proxy/environment.py b/proxy/environment.py index 794f099a8..526ce53cf 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -6,15 +6,23 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") -evm_loader_id = os.environ.get("EVM_LOADER") +SOLANA_URL = os.environ.get("SOLANA_URL", "http://localhost:8899") +EVM_LOADER_ID = os.environ.get("EVM_LOADER") neon_cli_timeout = float(os.environ.get("NEON_CLI_TIMEOUT", "0.1")) +NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) +CONFIRMATION_CHECK_DELAY = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) +USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "NO") == "YES" +CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) +TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) +MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 +EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) + class solana_cli: def call(self, *args): try: cmd = ["solana", - "--url", solana_url, + "--url", SOLANA_URL, ] + list(args) logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, universal_newlines=True) @@ -28,8 +36,8 @@ def call(self, *args): try: cmd = ["neon-cli", "--commitment=recent", - "--url", solana_url, - "--evm_loader={}".format(evm_loader_id), + "--url", SOLANA_URL, + "--evm_loader={}".format(EVM_LOADER_ID), ] + list(args) logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index 52c9fcf81..fe6bd1342 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -1,8 +1,9 @@ from proxy.indexer.indexer_base import logger, IndexerBase, PARALLEL_REQUESTS import base58 -import rlp import json +import logging import os +import rlp import time import logging from multiprocessing.dummy import Pool as ThreadPool diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index cc579daa0..60425b7a8 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -3,10 +3,10 @@ import json import logging import os -import rlp import psycopg2 +import rlp import subprocess -from construct import Struct, Bytes, Int64ul + from eth_utils import big_endian_to_int from ethereum.transactions import Transaction as EthTrx from ethereum.utils import sha3 @@ -15,18 +15,16 @@ from solana.rpc.api import Client from solana.rpc.commitment import Confirmed from solana.rpc.types import TxOpts +from solana.system_program import SYS_PROGRAM_ID +from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY from solana.transaction import AccountMeta, Transaction, TransactionInstruction from spl.token.constants import TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address from web3.auto.gethdev import w3 -from proxy.environment import solana_url, evm_loader_id, ETH_TOKEN_MINT_ID -sysvarclock = "SysvarC1ock11111111111111111111111111111111" -sysinstruct = "Sysvar1nstructions1111111111111111111111111" -keccakprog = "KeccakSecp256k11111111111111111111111111111" -rentid = "SysvarRent111111111111111111111111111111111" -incinerator = "1nc1nerator11111111111111111111111111111111" -system = "11111111111111111111111111111111" +from ..common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM +from ..common_neon.layouts import STORAGE_ACCOUNT_INFO_LAYOUT +from ..environment import SOLANA_URL, EVM_LOADER_ID, ETH_TOKEN_MINT_ID logger = logging.getLogger(__name__) @@ -46,7 +44,7 @@ def get_trx_results(trx): accounts = trx["transaction"]["message"]["accountKeys"] evm_loader_instructions = [] for idx, instruction in enumerate(trx["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: evm_loader_instructions.append(idx) slot = trx['slot'] @@ -60,7 +58,7 @@ def get_trx_results(trx): for inner in (trx['meta']['innerInstructions']): if inner["index"] in evm_loader_instructions: for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: log = base58.b58decode(event['data']) instruction = log[:1] if (int().from_bytes(instruction, "little") == 7): # OnEvent evmInstruction code @@ -115,22 +113,6 @@ def get_trx_receipts(unsigned_msg, signature): return (trx_raw.hex(), eth_signature, from_address) -STORAGE_ACCOUNT_INFO_LAYOUT = Struct( - # "tag" / Int8ul, - "caller" / Bytes(20), - "nonce" / Int64ul, - "gas_limit" / Int64ul, - "gas_price" / Int64ul, - "slot" / Int64ul, - "operator" / Bytes(32), - "accounts_len" / Int64ul, - "executor_data_size" / Int64ul, - "evm_data_size" / Int64ul, - "gas_used_and_paid" / Int64ul, - "number_of_payments" / Int64ul, - "sign" / Bytes(65), - -) def get_account_list(client, storage_account): opts = { @@ -311,7 +293,7 @@ def __init__(self): values = bytes(numbs) self.signer = Account(values) - self.client = Client(solana_url) + self.client = Client(SOLANA_URL) self.operator = self.signer.public_key() self.operator_token = get_associated_token_address(PublicKey(self.operator), ETH_TOKEN_MINT_ID) @@ -320,7 +302,7 @@ def __init__(self): def call(self, *args): try: cmd = ["solana", - "--url", solana_url, + "--url", SOLANA_URL, ] + list(args) logger.debug(cmd) return subprocess.check_output(cmd, universal_newlines=True) @@ -331,15 +313,15 @@ def call(self, *args): def unlock_accounts(self, blocked_storages): readonly_accs = [ - PublicKey(evm_loader_id), + PublicKey(EVM_LOADER_ID), ETH_TOKEN_MINT_ID, PublicKey(TOKEN_PROGRAM_ID), - PublicKey(sysvarclock), - PublicKey(sysinstruct), - PublicKey(keccakprog), - PublicKey(rentid), - PublicKey(incinerator), - PublicKey(system), + PublicKey(SYSVAR_CLOCK_PUBKEY), + PublicKey(SYSVAR_INSTRUCTION_PUBKEY), + PublicKey(KECCAK_PROGRAM), + PublicKey(SYSVAR_RENT_PUBKEY), + PublicKey(INCINERATOR_PUBKEY), + PublicKey(SYS_PROGRAM_ID), ] for storage, trx_accs in blocked_storages.items(): (eth_trx, blocked_accs) = trx_accs @@ -366,15 +348,15 @@ def unlock_accounts(self, blocked_storages): AccountMeta(pubkey=self.operator, is_signer=True, is_writable=True), AccountMeta(pubkey=self.operator_token, is_signer=False, is_writable=True), AccountMeta(pubkey=acc_list[4], is_signer=False, is_writable=True), - AccountMeta(pubkey=incinerator, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False) + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False) ] for acc in acc_list: keys.append(AccountMeta(pubkey=acc, is_signer=False, is_writable=(False if acc in readonly_accs else True))) trx = Transaction() trx.add(TransactionInstruction( - program_id=evm_loader_id, + program_id=EVM_LOADER_ID, data=bytearray.fromhex("15") + eth_trx[0].to_bytes(8, 'little'), keys=keys )) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index f84a80c6d..e7f9afd19 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -8,35 +8,37 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import List, Tuple, Optional +import base58 import copy -import json -import unittest import eth_utils +import json +import logging import rlp -import solana -from solana.account import Account as sol_Account -from ..common.utils import socket_connection, text_, build_http_response +import threading +import traceback +import unittest + +from ..common.utils import build_http_response from ..http.codes import httpStatusCodes from ..http.parser import HttpParser from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes -from .eth_proto import Trx as EthTrx +from solana.account import Account as sol_Account from solana.rpc.api import Client as SolanaClient, SendTransactionError as SolanaTrxError -from sha3 import keccak_256 -import base58 -import traceback -import threading - -from .solana_rest_api_tools import EthereumAddress, get_token_balance_or_airdrop, getAccountInfo, call_signed, \ - call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE, estimate_gas -from solana.rpc.commitment import Commitment, Confirmed +from solana.rpc.commitment import Confirmed +from typing import List, Tuple, Optional from web3 import Web3 -import logging + +from .eth_proto import Trx as EthTrx +from .solana_rest_api_tools import getAccountInfo, call_signed, neon_config_load, \ + get_token_balance_or_airdrop, estimate_gas +from ..common_neon.address import EthereumAddress +from ..common_neon.emulator_interactor import call_emulated +from ..common_neon.errors import EthereumError from ..core.acceptor.pool import proxy_id_glob -from ..indexer.utils import get_trx_results, LogDB +from ..environment import neon_cli, solana_cli, SOLANA_URL, MINIMAL_GAS_PRICE from ..indexer.sql_dict import SQLDict -from ..environment import evm_loader_id, solana_cli, solana_url, neon_cli +from ..indexer.utils import get_trx_results, LogDB logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -50,7 +52,7 @@ class EthereumModel: def __init__(self): self.signer = self.get_solana_account() - self.client = SolanaClient(solana_url) + self.client = SolanaClient(SOLANA_URL) self.logs_db = LogDB() self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash") @@ -410,8 +412,8 @@ def eth_sendRawTransaction(self, rawTrx): nonce = int(self.eth_getTransactionCount('0x' + sender, None), base=16) - logger.debug('Eth Sender trx nonce: %s', nonce) - logger.debug('Operator nonce: %s', trx.nonce) + logger.debug('Eth Sender trx nonce in solana: %s', nonce) + logger.debug('Eth Sender trx nonce in transaction: %s', trx.nonce) if (int(nonce) != int(trx.nonce)): raise EthereumError(-32002, 'Verifying nonce before send transaction: Error processing Instruction 1: invalid program argument' diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 29207a332..73085b405 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -1,458 +1,27 @@ -import base58 import base64 -import json import logging -import os -import random -import re -import struct -import time -from datetime import datetime -from hashlib import sha256 -from typing import NamedTuple, Optional, Union, Dict, Tuple -import psycopg2 -import rlp -from base58 import b58decode, b58encode -from construct import Bytes, Int8ul, Int32ul, Int64ul -from construct import Struct as cStruct -from eth_keys import keys as eth_keys -import eth_utils - -from sha3 import keccak_256 -from web3.auto import w3 +from datetime import datetime from solana.account import Account as SolanaAccount -from solana.blockhash import Blockhash -from solana.rpc.api import Client as SolanaClient, SendTransactionError -from solana.rpc.commitment import Commitment, Confirmed -from solana.rpc.types import TxOpts -from solana.sysvar import * -from solana.transaction import AccountMeta, Transaction, TransactionInstruction -from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT -from solana._layouts.system_instructions import InstructionType as SystemInstructionType - -from spl.token.constants import ACCOUNT_LEN, ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID -from spl.token.instructions import get_associated_token_address, create_associated_token_account, transfer2, Transfer2Params - -from ..environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params +from solana.publickey import PublicKey +from solana.rpc.api import Client as SolanaClient +from solana.rpc.commitment import Confirmed + +from ..common_neon.address import ether2program, getTokenAddr, EthereumAddress, AccountInfo +from ..common_neon.errors import SolanaAccountNotFoundError, SolanaErrors +from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT +from ..common_neon.neon_instruction import NeonInstruction +from ..common_neon.solana_interactor import SolanaInteractor +from ..common_neon.transaction_sender import TransactionSender +from ..common_neon.emulator_interactor import call_emulated from ..common_neon.utils import get_from_dict -from ..common_neon.errors import * -from .eth_proto import Trx -from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob -from ..indexer.sql_dict import POSTGRES_USER, POSTGRES_HOST, POSTGRES_DB, POSTGRES_PASSWORD +from ..environment import read_elf_params, TIMEOUT_TO_RELOAD_NEON_CONFIG, EXTRA_GAS + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) -location_bin = ".deploy_contract.bin" -confirmation_check_delay = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) -USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "NO") == "YES" -CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) -TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) -MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 - -ACCOUNT_SEED_VERSION=b'\1' - -COLLATERALL_POOL_MAX=10 - -EMPTY_STORAGE_TAG=0 -FINALIZED_STORAGE_TAG=5 - -sysvarclock = "SysvarC1ock11111111111111111111111111111111" -sysinstruct = "Sysvar1nstructions1111111111111111111111111" -keccakprog = "KeccakSecp256k11111111111111111111111111111" -rentid = "SysvarRent111111111111111111111111111111111" -incinerator = "1nc1nerator11111111111111111111111111111111" -system = "11111111111111111111111111111111" - -STORAGE_SIZE = 128 * 1024 - -ACCOUNT_INFO_LAYOUT = cStruct( - "type" / Int8ul, - "ether" / Bytes(20), - "nonce" / Int8ul, - "trx_count" / Bytes(8), - "code_account" / Bytes(32), - "is_rw_blocked" / Int8ul, - "rw_blocked_acc" / Bytes(32), - "eth_token_account" / Bytes(32), - "ro_blocked_cnt" / Int8ul, -) - -CODE_INFO_LAYOUT = cStruct( - "tag" / Int8ul, - "owner" / Bytes(20), - "code_size" / Bytes(4), -) - -CREATE_ACCOUNT_LAYOUT = cStruct( - "lamports" / Int64ul, - "space" / Int64ul, - "ether" / Bytes(20), - "nonce" / Int8ul -) - -obligatory_accounts = [ - AccountMeta(pubkey=evm_loader_id, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=sysvarclock, is_signer=False, is_writable=False), -] - -EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) - - -class SQLCost(): - def __init__(self): - - self.conn = psycopg2.connect( - dbname=POSTGRES_DB, - user=POSTGRES_USER, - password=POSTGRES_PASSWORD, - host=POSTGRES_HOST - ) - - self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - cur = self.conn.cursor() - cur.execute(''' - CREATE TABLE IF NOT EXISTS OPERATOR_COST - ( - hash char(64), - cost bigint, - used_gas bigint, - sender char(40), - to_address char(40) , - sig char(100), - status varchar(100), - reason varchar(100) - )''' - ) - - def close(self): - self.conn.close() - - def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason): - cur = self.conn.cursor() - cur.execute(''' - INSERT INTO OPERATOR_COST (hash, cost, used_gas, sender, to_address, sig, status, reason) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s) - ''', - (hash, cost, used_gas, sender, to_address, sig, status, reason) - ) - -class CostSingleton(object): - def __new__(cls): - if not hasattr(cls, 'instance'): - cls.instance = super(CostSingleton, cls).__new__(cls) - cls.instance.operator_cost = SQLCost() - return cls.instance - -class PermanentAccounts: - def __init__(self, client, signer): - while True: - with new_acc_id_glob.get_lock(): - try: - self.acc_id = acc_list_glob.pop(0) - except IndexError: - self.acc_id = new_acc_id_glob.value - new_acc_id_glob.value += 1 - - logger.debug("LOCK RESOURCES {}".format(self.acc_id)) - - self.operator = signer.public_key() - self.operator_token = getTokenAddr(self.operator) - - acc_id_bytes = self.acc_id.to_bytes((self.acc_id.bit_length() + 7) // 8, 'big') - - storage_seed = keccak_256(b"storage" + acc_id_bytes).hexdigest()[:32] - storage_seed = bytes(storage_seed, 'utf8') - - holder_seed = keccak_256(b"holder" + acc_id_bytes).hexdigest()[:32] - holder_seed = bytes(holder_seed, 'utf8') - - try: - self.storage, self.holder = create_multiple_accounts_with_seed( - client, - funding=signer, - base=signer, - seeds=[storage_seed, holder_seed], - sizes=[STORAGE_SIZE, STORAGE_SIZE] - ) - except Exception as err: - logger.warn("Account is locked err({}) id({}) owner({})".format(str(err), self.acc_id, signer.public_key())) - else: - break - - def __del__(self): - logger.debug("FREE RESOURCES {}".format(self.acc_id)) - with new_acc_id_glob.get_lock(): - acc_list_glob.append(self.acc_id) - - -class TransactionInfo: - def __init__(self, caller_token, eth_accounts, eth_trx): - self.eth_trx = eth_trx - - self.caller_token = caller_token - self.eth_accounts = eth_accounts - self.nonce = eth_trx.nonce - - hash = keccak_256(eth_trx.unsigned_msg()).digest() - collateral_pool_index = int().from_bytes(hash[:4], "little") % COLLATERALL_POOL_MAX - self.collateral_pool_index_buf = collateral_pool_index.to_bytes(4, 'little') - self.collateral_pool_address = create_collateral_pool_address(collateral_pool_index) - - -class AccountInfo(NamedTuple): - ether: eth_keys.PublicKey - trx_count: int - code_account: PublicKey - - @staticmethod - def frombytes(data): - cont = ACCOUNT_INFO_LAYOUT.parse(data) - return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account)) - - -def create_account_layout(lamports, space, ether, nonce): - return bytes.fromhex("02000000")+CREATE_ACCOUNT_LAYOUT.build(dict( - lamports=lamports, - space=space, - ether=ether, - nonce=nonce - )) - - -def write_holder_layout(nonce, offset, data): - return (bytes.fromhex('12')+ - nonce.to_bytes(8, byteorder='little')+ - offset.to_bytes(4, byteorder='little')+ - len(data).to_bytes(8, byteorder='little')+ - data) - - -def get_account_info(client, storage_account): - opts = { - "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 16, - } - } - - result = client._provider.make_request("getAccountInfo", str(storage_account), opts) - logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) - - info = result['result']['value'] - if info is None: - logger.debug("Can't get information about {}".format(storage_account)) - return None - - data = base64.b64decode(info['data'][0]) - - account_tag = data[0] - lamports = info['lamports'] - owner = info['owner'] - - return (account_tag, lamports, owner) - - -def accountWithSeed(base, seed, program): - result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(program)).digest()) - logger.debug('accountWithSeed %s', str(result)) - return result - - -def createAccountWithSeedTrx(funding, base, seed, lamports, space, program): - seed_str = str(seed, 'utf8') - data = SYSTEM_INSTRUCTIONS_LAYOUT.build( - dict( - instruction_type = SystemInstructionType.CREATE_ACCOUNT_WITH_SEED, - args=dict( - base=bytes(base), - seed=dict(length=len(seed_str), chars=seed_str), - lamports=lamports, - space=space, - program_id=bytes(program) - ) - ) - ) - logger.debug("createAccountWithSeedTrx %s %s %s", type(base), base, data.hex()) - created = accountWithSeed(base, seed, PublicKey(program)) - logger.debug("created %s", created) - return TransactionInstruction( - keys=[ - AccountMeta(pubkey=funding, is_signer=True, is_writable=True), - AccountMeta(pubkey=created, is_signer=False, is_writable=True), - AccountMeta(pubkey=base, is_signer=True, is_writable=False), - ], - program_id=system, - data=data - ) - - -def create_collateral_pool_address(collateral_pool_index): - COLLATERAL_SEED_PREFIX = "collateral_seed_" - seed = COLLATERAL_SEED_PREFIX + str(collateral_pool_index) - return accountWithSeed(PublicKey(COLLATERAL_POOL_BASE), str.encode(seed), PublicKey(evm_loader_id)) - - -def create_account_with_seed(client, funding, base, seed, storage_size, eth_trx=None): - account = accountWithSeed(base.public_key(), seed, PublicKey(evm_loader_id)) - - if client.get_balance(account, commitment=Confirmed)['result']['value'] == 0: - minimum_balance = client.get_minimum_balance_for_rent_exemption(storage_size, commitment=Confirmed)["result"] - logger.debug("Minimum balance required for account {}".format(minimum_balance)) - - trx = Transaction() - trx.add(createAccountWithSeedTrx(funding.public_key(), base.public_key(), seed, minimum_balance, storage_size, PublicKey(evm_loader_id))) - send_transaction(client, trx, funding, eth_trx=eth_trx, reason='createAccountWithSeed') - - return account - - -def create_multiple_accounts_with_seed(client, funding, base, seeds, sizes): - accounts = [] - trx = Transaction() - - for seed, storage_size in zip(seeds, sizes): - account = accountWithSeed(base.public_key(), seed, PublicKey(evm_loader_id)) - accounts.append(account) - - minimum_balance = client.get_minimum_balance_for_rent_exemption(storage_size, commitment=Confirmed)["result"] - - account_info = get_account_info(client, account) - if account_info is None: - logger.debug("Minimum balance required for account {}".format(minimum_balance)) - - trx.add(createAccountWithSeedTrx(funding.public_key(), base.public_key(), seed, minimum_balance, storage_size, PublicKey(evm_loader_id))) - else: - (tag, lamports, owner) = account_info - if lamports < minimum_balance: - raise Exception("insufficient balance") - if PublicKey(owner) != PublicKey(evm_loader_id): - raise Exception("wrong owner") - if tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}: - raise Exception("not empty, not finalized") - - if len(trx.instructions) > 0: - send_transaction(client, trx, funding) - - return accounts - - -def make_keccak_instruction_data(check_instruction_index, msg_len, data_start): - if check_instruction_index > 255 and check_instruction_index < 0: - raise Exception("Invalid index for instruction - {}".format(check_instruction_index)) - - check_count = 1 - eth_address_size = 20 - signature_size = 65 - eth_address_offset = data_start - signature_offset = eth_address_offset + eth_address_size - message_data_offset = signature_offset + signature_size - - data = struct.pack("B", check_count) - data += struct.pack("= confirmations): - return - time.sleep(confirmation_check_delay) - elapsed_time += confirmation_check_delay - raise RuntimeError("could not confirm transaction: ", tx_sig) - - -def solana2ether(public_key): - from web3 import Web3 - return bytes(Web3.keccak(bytes.fromhex(public_key))[-20:]) - - -def ether2program(ether): - if isinstance(ether, str): - pass - elif isinstance(ether, EthereumAddress): - ether = str(ether) - else: - ether = ether.hex() - output = neon_cli().call("create-program-address", ether) - items = output.rstrip().split(' ') - return items[0], int(items[1]) - - -def ether2seed(ether, program_id, base): - if isinstance(ether, str): - if ether.startswith('0x'): ether = ether[2:] - else: ether = ether.hex() - seed = b58encode(bytes.fromhex(ether)) - acc = accountWithSeed(base, seed, PublicKey(program_id)) - logger.debug('ether2program: {} {} => {} (seed {})'.format(ether, 255, acc, seed)) - return acc, 255, seed - - def neon_config_load(ethereum_model): try: ethereum_model.neon_config_dict @@ -475,614 +44,12 @@ def neon_config_load(ethereum_model): logger.debug(ethereum_model.neon_config_dict) -def call_emulated(contract_id, caller_id, data=None, value=None): - output = emulator(contract_id, caller_id, data, value) - logger.debug("call_emulated %s %s %s %s return %s", contract_id, caller_id, data, value, output) - result = json.loads(output) - exit_status = result['exit_status'] - if exit_status == 'revert': - result_value = result['result'] - if len(result_value) < 8 or result_value[:8] != '08c379a0': - raise EthereumError(code=3, message='execution reverted') - - offset = int(result_value[8:8+64], 16) - length = int(result_value[8+64:8+64+64], 16) - message = str(bytes.fromhex(result_value[8+offset*2+64:8+offset*2+64+length*2]), 'utf8') - raise EthereumError(code=3, message='execution reverted: '+message, data='0x'+result_value) - if result["exit_status"] != "succeed": - raise Exception("evm emulator error ", result) - return result - - -def extract_measurements_from_receipt(receipt): - log_messages = receipt['result']['meta']['logMessages'] - transaction = receipt['result']['transaction'] - accounts = transaction['message']['accountKeys'] - instructions = [] - for instr in transaction['message']['instructions']: - program = accounts[instr['programIdIndex']] - instructions.append({ - 'accs': [accounts[acc] for acc in instr['accounts']], - 'program': accounts[instr['programIdIndex']], - 'data': b58decode(instr['data']).hex() - }) - - pattern = re.compile('Program ([0-9A-Za-z]+) (.*)') - messages = [] - for log in log_messages: - res = pattern.match(log) - if res: - (program, reason) = res.groups() - if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) - messages[-1]['logs'].append(log) - - for instr in instructions: - if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue - if messages[0]['program'] != instr['program']: - raise Exception('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) - instr['logs'] = messages.pop(0)['logs'] - exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) - if not exit_result: raise Exception("Can't get exit result") - instr['result'] = exit_result.group(1) - - if instr['program'] == evm_loader_id: - memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) - instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) - if not (memory_result and instruction_result): - raise Exception("Can't parse measurements for evm_loader") - instr['measurements'] = { - 'instructions': instruction_result.group(1), - 'memory': memory_result.group(1) - } - - result = [] - for instr in instructions: - if instr['program'] == evm_loader_id: - result.append({ - 'program':instr['program'], - 'measurements':instr['measurements'], - 'result':instr['result'], - 'data':instr['data'] - }) - return result - -# Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh) -def get_measurements(result): - try: - measurements = extract_measurements_from_receipt(result) - for m in measurements: logger.info(json.dumps(m)) - except Exception as err: - logger.error("Can't get measurements %s"%err) - logger.info("Failed result: %s"%json.dumps(result, indent=3)) - - -def send_transaction(client, trx, signer, eth_trx=None, reason=None): - result = client.send_transaction(trx, signer, opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed)) - confirm_transaction(client, result["result"]) - result = client.get_confirmed_transaction(result["result"]) - update_transaction_cost(result, eth_trx, reason=reason) - return result - - -def send_measured_transaction(client, trx, signer, eth_trx, reason): - result = send_transaction(client, trx, signer, eth_trx=eth_trx, reason=reason) - get_measurements(result) - return result - - -def check_if_program_exceeded_instructions(err_result): - err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" - err_budget = "failed: Computational budget exceeded" - - if err_result['data']['logs'][-1].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-2].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-1].find(err_budget) >= 0: - return True - return False - - -def check_if_continue_returned(result): - tx_info = result['result'] - accounts = tx_info["transaction"]["message"]["accountKeys"] - evm_loader_instructions = [] - - for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: - evm_loader_instructions.append(idx) - - for inner in (tx_info['meta']['innerInstructions']): - if inner["index"] in evm_loader_instructions: - for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: - instruction = base58.b58decode(event['data'])[:1] - if int().from_bytes(instruction, "little") == 6: # OnReturn evmInstruction code - return (True, tx_info['transaction']['signatures'][0]) - return (False, ()) - - -def call_continue(signer, client, perm_accs, trx_info, steps): - try: - return call_continue_iterative(signer, client, perm_accs, trx_info, steps) - except Exception as err: - logger.debug("call_continue_iterative exception:") - logger.debug(str(err)) - - return sol_instr_21_cancel(signer, client, perm_accs, trx_info) - - -def call_continue_iterative(signer, client, perm_accs, trx_info, step_count): - while True: - logger.debug("Continue iterative step:") - result = sol_instr_10_continue(signer, client, perm_accs, trx_info, step_count) - (succeed, signature) = check_if_continue_returned(result) - if succeed: - return signature - - -def sol_instr_10_continue(signer, client, perm_accs, trx_info, initial_step_count): - step_count = initial_step_count - while step_count > 0: - trx = Transaction() - trx.add(make_continue_instruction(signer, perm_accs, trx_info, step_count)) - - logger.debug("Step count {}".format(step_count)) - try: - result = send_measured_transaction(client, trx, signer, trx_info.eth_trx, 'ContinueV02') - return result - except SendTransactionError as err: - if check_if_program_exceeded_instructions(err.result): - step_count = int(step_count * 90 / 100) - else: - raise - raise Exception("Can't execute even one EVM instruction") - - -def sol_instr_21_cancel(signer, client, perm_accs, trx_info): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - trx = Transaction() - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=bytearray.fromhex("15") + trx_info.nonce.to_bytes(8, 'little'), - keys=[ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=incinerator, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - )) - - logger.debug("Cancel") - result = send_measured_transaction(client, trx, signer, trx_info.eth_trx, 'CancelWithNonce') - return result['result']['transaction']['signatures'][0] - - -def make_partial_call_instruction(signer, perm_accs, trx_info, step_count, call_data): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("13") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little") + call_data, - keys = [ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_continue_instruction(signer, perm_accs, trx_info, step_count, index=None): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - data = bytearray.fromhex("14") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little") - if index: - data = data + index.to_bytes(8, byteorder="little") - - return TransactionInstruction( - program_id = evm_loader_id, - data = data, - keys = [ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_call_from_account_instruction(signer, perm_accs, trx_info, step_count = 0): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("16") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little"), - keys = [ - AccountMeta(pubkey=perm_accs.holder, is_signer=False, is_writable=True), - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_05_call_instruction(signer, trx_info, call_data): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("05") + trx_info.collateral_pool_index_buf + call_data, - keys = [ - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + obligatory_accounts - ) - - -def update_transaction_cost(receipt, eth_trx, extra_sol_trx=False, reason=None): - cost = receipt['result']['meta']['preBalances'][0] - receipt['result']['meta']['postBalances'][0] - if eth_trx: - hash = eth_trx.hash_signed().hex() - sender = eth_trx.sender() - to_address = eth_trx.toAddress.hex() if eth_trx.toAddress else "None" - else: - hash = None - sender = None - to_address = None - - sig = receipt['result']['transaction']['signatures'][0] - used_gas=None - - tx_info = receipt['result'] - accounts = tx_info["transaction"]["message"]["accountKeys"] - evm_loader_instructions = [] - - for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: - evm_loader_instructions.append(idx) - - for inner in (tx_info['meta']['innerInstructions']): - if inner["index"] in evm_loader_instructions: - for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: - used_gas = base58.b58decode(event['data'])[2:10] - used_gas = int().from_bytes(used_gas, "little") - - table = CostSingleton().operator_cost - table.insert( - hash, - cost, - used_gas if used_gas else 0, - sender, - to_address, - sig, - 'extra' if extra_sol_trx else 'ok', - reason if reason else '' - ) - - -def create_account_list_by_emulate(signer, client, eth_trx): - sender_ether = bytes.fromhex(eth_trx.sender()) - add_keys_05 = [] - trx = Transaction() - - if not eth_trx.toAddress: - to_address_arg = "deploy" - to_address = keccak_256(rlp.encode((bytes.fromhex(eth_trx.sender()), eth_trx.nonce))).digest()[-20:] - else: - to_address_arg = eth_trx.toAddress.hex() - to_address = eth_trx.toAddress - - output_json = call_emulated(to_address_arg, sender_ether.hex(), eth_trx.callData.hex(), hex(eth_trx.value)) - logger.debug("emulator returns: %s", json.dumps(output_json, indent=3)) - - # resize storage account - resize_instr = [] - for acc_desc in output_json["accounts"]: - if acc_desc["new"] == False: - - if acc_desc.get("code_size_current") is not None and acc_desc.get("code_size") is not None: - if acc_desc["code_size"] > acc_desc["code_size_current"]: - code_size = acc_desc["code_size"] + 2048 - seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) - code_account_new = accountWithSeed(signer.public_key(), seed, PublicKey(evm_loader_id)) - - logger.debug("creating new code_account with increased size %s", code_account_new) - create_account_with_seed(client, signer, signer, seed, code_size, eth_trx); - logger.debug("resized account is created %s", code_account_new) - - resize_instr.append(TransactionInstruction( - keys=[ - AccountMeta(pubkey=PublicKey(acc_desc["account"]), is_signer=False, is_writable=True), - ( - AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=True) - if acc_desc["contract"] else - AccountMeta(pubkey=PublicKey("11111111111111111111111111111111"), is_signer=False, is_writable=False) - ), - AccountMeta(pubkey=code_account_new, is_signer=False, is_writable=True), - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=False) - ], - program_id=evm_loader_id, - data=bytearray.fromhex("11")+bytes(seed) # 17- ResizeStorageAccount - )) - # replace code_account - acc_desc["contract"] = code_account_new - - for instr in resize_instr: - logger.debug("code and storage migration, account %s from %s to %s", instr.keys[0].pubkey, instr.keys[1].pubkey, instr.keys[2].pubkey) - - tx = Transaction().add(instr) - success = False - count = 0 - - while count < 2: - logger.debug("attemt: %d", count) - - send_transaction(client, tx, signer, eth_trx=eth_trx, reason='resize_storage_account') - info = _getAccountData(client, instr.keys[0].pubkey, ACCOUNT_INFO_LAYOUT.sizeof()) - info_data = AccountInfo.frombytes(info) - if info_data.code_account == instr.keys[2].pubkey: - success = True - logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) - break - time.sleep(1) - count = count+1 - - if success == False: - raise Exception("Can't resize storage account. Account is blocked {}".format(instr.keys[0].pubkey)) - - for acc_desc in output_json["accounts"]: - address = bytes.fromhex(acc_desc["address"][2:]) - - code_account = None - code_account_writable = False - if acc_desc["new"]: - logger.debug("Create solana accounts for %s: %s %s", acc_desc["address"], acc_desc["account"], acc_desc["contract"]) - if acc_desc["code_size"]: - seed = b58encode(ACCOUNT_SEED_VERSION+address) - code_account = accountWithSeed(signer.public_key(), seed, PublicKey(evm_loader_id)) - logger.debug(" with code account %s", code_account) - code_size = acc_desc["code_size"] + 2048 - code_account_balance = client.get_minimum_balance_for_rent_exemption(code_size)["result"] - trx.add(createAccountWithSeedTrx(signer.public_key(), signer.public_key(), seed, code_account_balance, code_size, PublicKey(evm_loader_id))) - code_account_writable = acc_desc["writable"] - - extend_trx_with_create_and_airdrop(signer, EthereumAddress(address), code_account, trx=trx) - - if address == to_address: - contract_sol = PublicKey(acc_desc["account"]) - if acc_desc["new"]: - code_sol = code_account - code_writable = code_account_writable - else: - if acc_desc["contract"] != None: - code_sol = PublicKey(acc_desc["contract"]) - code_writable = acc_desc["writable"] - else: - code_sol = None - code_writable = None - - elif address == sender_ether: - sender_sol = PublicKey(acc_desc["account"]) - else: - add_keys_05.append(AccountMeta(pubkey=acc_desc["account"], is_signer=False, is_writable=True)) - token_account = get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID) - add_keys_05.append(AccountMeta(pubkey=token_account, is_signer=False, is_writable=True)) - if acc_desc["new"]: - if code_account: - add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=code_account_writable)) - else: - if acc_desc["contract"]: - add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) - - for token_account in output_json["token_accounts"]: - add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) - - if token_account["new"]: - trx.add(createERC20TokenAccountTrx(signer, token_account)) - - for account_meta in output_json["solana_accounts"]: - add_keys_05.append(AccountMeta(pubkey=PublicKey(account_meta["pubkey"]), is_signer=account_meta["is_signer"], is_writable=account_meta["is_writable"])) - - caller_token = get_associated_token_address(PublicKey(sender_sol), ETH_TOKEN_MINT_ID) - - eth_accounts = [ - AccountMeta(pubkey=contract_sol, is_signer=False, is_writable=True), - AccountMeta(pubkey=get_associated_token_address(contract_sol, ETH_TOKEN_MINT_ID), is_signer=False, is_writable=True), - ] + ([AccountMeta(pubkey=code_sol, is_signer=False, is_writable=code_writable)] if code_sol != None else []) + [ - AccountMeta(pubkey=sender_sol, is_signer=False, is_writable=True), - AccountMeta(pubkey=caller_token, is_signer=False, is_writable=True), - ] + add_keys_05 - - trx_info = TransactionInfo(caller_token, eth_accounts, eth_trx) - - return trx_info, sender_ether, trx - - def call_signed(signer, client, eth_trx, steps): - - (trx_info, sender_ether, create_acc_trx) = create_account_list_by_emulate(signer, client, eth_trx) - - call_iterative = False - call_from_holder = False - - if not eth_trx.toAddress: - call_from_holder = True - else: - msg = sender_ether + eth_trx.signature() + eth_trx.unsigned_msg() - - try: - logger.debug("Try single trx call") - return call_signed_noniterative(signer, client, eth_trx, trx_info, msg, create_acc_trx) - except Exception as err: - logger.debug(str(err)) - errStr = str(err) - if "Program failed to complete" in errStr or "Computational budget exceeded" in errStr: - logger.debug("Program exceeded instructions") - call_iterative = True - elif str(err).startswith("transaction too large:"): - logger.debug("Transaction too large, call call_signed_with_holder_acc():") - call_from_holder = True - else: - raise - - perm_accs = PermanentAccounts(client, signer) - try: - if call_iterative: - try: - return call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx) - except Exception as err: - logger.debug(str(err)) - if str(err).startswith("transaction too large:"): - logger.debug("Transaction too large, call call_signed_with_holder_acc():") - call_from_holder = True - else: - raise - - if call_from_holder: - return call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, steps, create_acc_trx) - finally: - del perm_accs - - -def call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx): - if len(create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(create_acc_trx) - send_measured_transaction(client, precall_txs, signer, eth_trx, 'CreateAccountsForTrx') - - precall_txs = Transaction() - precall_txs.add(TransactionInstruction( - program_id=keccakprog, - data=make_keccak_instruction_data(len(precall_txs.instructions)+1, len(eth_trx.unsigned_msg()), data_start=13), - keys=[ - AccountMeta(pubkey=keccakprog, is_signer=False, is_writable=False), - ])) - precall_txs.add(make_partial_call_instruction(signer, perm_accs, trx_info, 0, msg)) - - logger.debug("Partial call") - send_measured_transaction(client, precall_txs, signer, eth_trx, 'PartialCallFromRawEthereumTXv02') - - return call_continue(signer, client, perm_accs, trx_info, steps) - - -def call_signed_noniterative(signer, client, eth_trx, trx_info, msg, create_acc_trx): - call_txs_05 = Transaction() - call_txs_05.add(create_acc_trx) - call_txs_05.add(TransactionInstruction( - program_id=keccakprog, - data=make_keccak_instruction_data(len(call_txs_05.instructions)+1, len(eth_trx.unsigned_msg()), 5), - keys=[ - AccountMeta(pubkey=keccakprog, is_signer=False, is_writable=False), - ])) - call_txs_05.add(make_05_call_instruction(signer, trx_info, msg)) - result = send_measured_transaction(client, call_txs_05, signer, eth_trx, 'CallFromRawEthereumTX') - return result['result']['transaction']['signatures'][0] - - -def call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, steps, create_acc_trx): - - write_trx_to_holder_account(signer, client, perm_accs.holder, perm_accs.acc_id, eth_trx) - if len(create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(create_acc_trx) - send_measured_transaction(client, precall_txs, signer, eth_trx, 'create_accounts_for_deploy') - - precall_txs = Transaction() - precall_txs.add(make_call_from_account_instruction(signer, perm_accs, trx_info)) - - # ExecuteTrxFromAccountDataIterative - logger.debug("ExecuteTrxFromAccountDataIterative:") - send_measured_transaction(client, precall_txs, signer, eth_trx, 'ExecuteTrxFromAccountDataIterativeV02') - - return call_continue(signer, client, perm_accs, trx_info, steps) + solana_interactor = SolanaInteractor(signer, client) + trx_sender = TransactionSender(solana_interactor, eth_trx, steps) + return trx_sender.execute() -def createERC20TokenAccountTrx(signer, token_info): - trx = Transaction() - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=bytes.fromhex('0F'), - keys=[ - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["key"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["owner"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["contract"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["mint"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - - return trx - - -def write_trx_to_holder_account(signer, client, holder, acc_id, eth_trx): - msg = eth_trx.signature() + len(eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + eth_trx.unsigned_msg() - - # Write transaction to transaction holder account - offset = 0 - receipts = [] - rest = msg - while len(rest): - (part, rest) = (rest[:1000], rest[1000:]) - trx = Transaction() - # logger.debug("sender_sol %s %s %s", sender_sol, holder, acc.public_key()) - trx.add(TransactionInstruction(program_id=evm_loader_id, - data=write_holder_layout(acc_id, offset, part), - keys=[ - AccountMeta(pubkey=holder, is_signer=False, is_writable=True), - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=False), - ])) - receipts.append(client.send_transaction(trx, signer, - opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed))["result"]) - offset += len(part) - logger.debug("receipts %s", receipts) - for rcpt in receipts: - confirm_transaction(client, rcpt) - result = client.get_confirmed_transaction(rcpt) - update_transaction_cost(result, eth_trx, reason='WriteHolder') - logger.debug("confirmed: %s", rcpt) - def _getAccountData(client, account, expected_length, owner=None): info = client.get_account_info(account, commitment=Confirmed)['result']['value'] @@ -1101,84 +68,9 @@ def getAccountInfo(client, eth_account: EthereumAddress): return AccountInfo.frombytes(info) -def getLamports(client, eth_account): - pda_account, nonce = ether2program(eth_account) - return int(client.get_balance(pda_account, commitment=Confirmed)['result']['value']) - - -def make_create_eth_account_trx(signer: SolanaAccount, eth_address: EthereumAddress, evm_loader_id, code_acc=None) \ - -> Tuple[Transaction, PublicKey]: - - solana_address, nonce = ether2program(eth_address) - token_acc_address = get_associated_token_address(PublicKey(solana_address), ETH_TOKEN_MINT_ID) - logger.debug(f'Create eth account: {eth_address}, sol account: {solana_address}, token_acc_address: {token_acc_address}, nonce: {nonce}') - - base = signer.public_key() - data = bytes.fromhex('02000000') + CREATE_ACCOUNT_LAYOUT.build(dict(lamports=0, - space=0, - ether=bytes(eth_address), - nonce=nonce)) - trx = Transaction() - if code_acc is None: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), - AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - else: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), - AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - return trx, token_acc_address - - -def make_transfer_instruction(owner_pda_account: SolanaAccount, associated_token_account: PublicKey) -> TransactionInstruction: - owner_pda_address = owner_pda_account.public_key() - owner_associated_token_account = getTokenAddr(owner_pda_address) - transfer_instruction = transfer2(Transfer2Params(source=owner_associated_token_account, - owner=owner_pda_address, - dest=associated_token_account, - amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei, - decimals=9, - mint=ETH_TOKEN_MINT_ID, - program_id=TOKEN_PROGRAM_ID)) - logger.debug(f"Token transfer from token: {owner_associated_token_account}, owned by: {owner_pda_address}, to token: " - f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}") - return transfer_instruction - - -def extend_trx_with_create_and_airdrop(signer: SolanaAccount, eth_account: EthereumAddress, code_acc=None, *, trx): - create_trx, associated_token_account = make_create_eth_account_trx(signer, eth_account, evm_loader_id, code_acc) - trx.add(create_trx) - if NEW_USER_AIRDROP_AMOUNT <= 0: - return - transfer_instruction = make_transfer_instruction(signer, associated_token_account) - trx.add(transfer_instruction) - - def create_eth_account_and_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress): - trx = Transaction() - extend_trx_with_create_and_airdrop(signer, eth_account, trx=trx) - result = send_transaction(client, trx, signer) + trx = NeonInstruction(signer.public_key()).make_trx_with_create_and_airdrop (eth_account) + result = SolanaInteractor(signer, client).send_transaction(trx, None, reason='create_eth_account_and_airdrop') error = result.get("error") if error is not None: logger.error(f"Failed to create eth_account and airdrop: {eth_account}, error occurred: {error}") @@ -1186,77 +78,34 @@ def create_eth_account_and_airdrop(client: SolanaClient, signer: SolanaAccount, def get_token_balance_gwei(client: SolanaClient, pda_account: str) -> int: - associated_token_account = get_associated_token_address(PublicKey(pda_account), ETH_TOKEN_MINT_ID) - rpc_response = client.get_token_account_balance(associated_token_account, commitment=Confirmed) + neon_token_account = getTokenAddr(PublicKey(pda_account)) + rpc_response = client.get_token_account_balance(neon_token_account, commitment=Confirmed) error = rpc_response.get('error') if error is not None: message = error.get("message") if message == SolanaErrors.AccountNotFound.value: raise SolanaAccountNotFoundError() - logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, " + logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, " f"got get_token_account_balance error: \"{message}\"") raise Exception("Getting balance error") balance = get_from_dict(rpc_response, "result", "value", "amount") if balance is None: - logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, response: {rpc_response}") + logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, response: {rpc_response}") raise Exception("Unexpected get_balance response") return int(balance) def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress) -> int: - associated_token_account, nonce = ether2program(eth_account) - logger.debug(f"Get balance for eth account: {eth_account} aka: {associated_token_account}") + solana_account, nonce = ether2program(eth_account) + logger.debug(f"Get balance for eth account: {eth_account} aka: {solana_account}") try: - return get_token_balance_gwei(client, associated_token_account) + return get_token_balance_gwei(client, solana_account) except SolanaAccountNotFoundError: - logger.debug(f"Account not found: {eth_account} aka: {associated_token_account} - create") + logger.debug(f"Account not found: {eth_account} aka: {solana_account} - create") create_eth_account_and_airdrop(client, signer, eth_account) - return get_token_balance_gwei(client, associated_token_account) - - -def getTokenAddr(account): - return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) - - -def make_instruction_data_from_tx(instruction, private_key=None): - if isinstance(instruction, dict): - if instruction.get('chainId') is None: - raise Exception("chainId value is needed in input dict") - if private_key is None: - raise Exception("Needed private key for transaction creation from fields") - - signed_tx = w3.eth.account.sign_transaction(instruction, private_key) - # logger.debug(signed_tx.rawTransaction.hex()) - _trx = Trx.fromString(signed_tx.rawTransaction) - # logger.debug(json.dumps(_trx.__dict__, cls=JsonEncoder, indent=3)) - - raw_msg = _trx.get_msg(instruction['chainId']) - sig = keys.Signature(vrs=[1 if _trx.v % 2 == 0 else 0, _trx.r, _trx.s]) - pub = sig.recover_public_key_from_msg_hash(_trx.hash()) - - # logger.debug(pub.to_hex()) - - return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) - elif isinstance(instruction, str): - if instruction[:2] == "0x": - instruction = instruction[2:] - - _trx = Trx.fromString(bytearray.fromhex(instruction)) - # logger.debug(json.dumps(_trx.__dict__, cls=JsonEncoder, indent=3)) - - raw_msg = _trx.get_msg() - sig = keys.Signature(vrs=[1 if _trx.v % 2 == 0 else 0, _trx.r, _trx.s]) - pub = sig.recover_public_key_from_msg_hash(_trx.hash()) - - data = pub.to_canonical_address() - data += sig.to_bytes() - data += raw_msg - - return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) - else: - raise Exception("function gets ") + return get_token_balance_gwei(client, solana_account) def is_account_exists(client: SolanaClient, eth_account: EthereumAddress) -> bool: diff --git a/proxy/proxy.py b/proxy/proxy.py index b4eb08753..86264a6e1 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -24,7 +24,7 @@ from multiprocessing import Process from .indexer.indexer import run_indexer -from proxy.environment import solana_url, evm_loader_id +from proxy.environment import SOLANA_URL, EVM_LOADER_ID logger = logging.getLogger(__name__) @@ -47,8 +47,8 @@ def delete_pid_file(self) -> None: def __enter__(self) -> 'Proxy': self.indexer = Process(target=run_indexer, - args=(solana_url, - evm_loader_id,)) + args=(SOLANA_URL, + EVM_LOADER_ID,)) self.indexer.start() self.acceptors = AcceptorPool( flags=self.flags, diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index fdb38c24b..602fb50e2 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -19,7 +19,7 @@ if [ "$(spl-token balance "$NEON_TOKEN_MINT" || echo 0)" -eq 0 ]; then echo 'Create balance and mint token' TOKEN_ACCOUNT=$( (spl-token create-account "$NEON_TOKEN_MINT" || true) | grep -Po 'Creating account \K[^\n]*') echo "TOKEN_ACCOUNT=$TOKEN_ACCOUNT" - spl-token mint "$NEON_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*1000)) --owner /spl/bin/evm_loader-keypair.json -- "$TOKEN_ACCOUNT" + spl-token mint "$NEON_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*100000)) --owner /spl/bin/evm_loader-keypair.json -- "$TOKEN_ACCOUNT" fi proxy/run-proxy.sh diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py index 9fc778585..2c7f5e4a7 100644 --- a/proxy/testing/test_airdropping_eth_accounts.py +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -11,7 +11,8 @@ from solana.rpc.api import Client as SolanaClient from ..plugin.solana_rest_api import EthereumModel -from ..plugin.solana_rest_api_tools import get_token_balance_gwei, ether2program +from ..plugin.solana_rest_api_tools import get_token_balance_gwei +from ..common_neon.address import ether2program class TestAirdroppingEthAccounts(unittest.TestCase): diff --git a/proxy/testing/test_erc20_wrapper_contract.py b/proxy/testing/test_erc20_wrapper_contract.py index 75030130b..b745c7e76 100644 --- a/proxy/testing/test_erc20_wrapper_contract.py +++ b/proxy/testing/test_erc20_wrapper_contract.py @@ -15,7 +15,7 @@ from solana.account import Account as SolanaAccount from solana.publickey import PublicKey -from proxy.plugin.solana_rest_api_tools import createERC20TokenAccountTrx +from proxy.common_neon.neon_instruction import NeonInstruction # install_solc(version='latest') install_solc(version='0.7.6') @@ -171,7 +171,9 @@ def create_token_accounts(self): admin_token_seeds = [ b"\1", b"ERC20Balance", bytes(self.token.pubkey), contract_address_bytes, admin_address_bytes ] admin_token_key = PublicKey.find_program_address(admin_token_seeds, evm_loader_id)[0] admin_token_info = { "key": admin_token_key, "owner": admin_address_solana, "contract": contract_address_solana, "mint": self.token.pubkey } - self.solana_client.send_transaction(createERC20TokenAccountTrx(self.solana_account, admin_token_info), self.solana_account, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) + + instr = NeonInstruction(self.solana_account.public_key()).createERC20TokenAccountTrx(admin_token_info) + self.solana_client.send_transaction(instr, self.solana_account, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) self.token.mint_to(admin_token_key, self.solana_account, 10_000_000_000_000, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) def test_erc20_name(self): diff --git a/proxy/testing/test_cancel_hanged.py b/proxy/testing/test_indexer_cancel_hanged.py similarity index 95% rename from proxy/testing/test_cancel_hanged.py rename to proxy/testing/test_indexer_cancel_hanged.py index 1e91a1381..e0324df2e 100644 --- a/proxy/testing/test_cancel_hanged.py +++ b/proxy/testing/test_indexer_cancel_hanged.py @@ -1,7 +1,8 @@ import os import sys -from proxy.plugin.solana_rest_api_tools import sysinstruct, ETH_TOKEN_MINT_ID, system, send_transaction, MINIMAL_GAS_PRICE +from proxy.common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY +from proxy.environment import ETH_TOKEN_MINT_ID, MINIMAL_GAS_PRICE os.environ['SOLANA_URL'] = "http://solana:8899" os.environ['EVM_LOADER'] = "53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io" @@ -17,6 +18,7 @@ from ethereum.utils import sha3 from solana.publickey import PublicKey from solana.rpc.commitment import Confirmed +from solana.system_program import SYS_PROGRAM_ID from solana.transaction import AccountMeta, Transaction, TransactionInstruction from solana_utils import * from solcx import install_solc @@ -73,7 +75,7 @@ class CancelTest(unittest.TestCase): @classmethod def setUpClass(cls): - print("\ntest_event.py setUpClass") + print("\ntest_cancel_hanged.py setUpClass") cls.token = SplToken(solana_url) wallet = WalletAccount(wallet_path()) @@ -183,7 +185,7 @@ def sol_instr_19_partial_call(self, storage_account, step_count, evm_instruction AccountMeta(pubkey=storage_account, is_signer=False, is_writable=True), # System instructions account: - AccountMeta(pubkey=PublicKey(sysinstruct), is_signer=False, is_writable=False), + AccountMeta(pubkey=PublicKey(SYSVAR_INSTRUCTION_PUBKEY), is_signer=False, is_writable=False), # Operator address: AccountMeta(pubkey=self.acc.public_key(), is_signer=True, is_writable=True), # Collateral pool address: @@ -193,7 +195,7 @@ def sol_instr_19_partial_call(self, storage_account, step_count, evm_instruction # User's NEON token account: AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), # System program account: - AccountMeta(pubkey=PublicKey(system), is_signer=False, is_writable=False), + AccountMeta(pubkey=PublicKey(SYS_PROGRAM_ID), is_signer=False, is_writable=False), AccountMeta(pubkey=self.reId, is_signer=False, is_writable=True), AccountMeta(pubkey=self.reId_token, is_signer=False, is_writable=True), diff --git a/proxy/testing/test_resize_storage_account.py b/proxy/testing/test_resize_storage_account.py index a2aece6fd..01e2319c1 100644 --- a/proxy/testing/test_resize_storage_account.py +++ b/proxy/testing/test_resize_storage_account.py @@ -2,8 +2,8 @@ from web3 import Web3 import unittest from solana.publickey import PublicKey -from solcx import compile_source -from solcx import install_solc +from solcx import compile_source, install_solc + proxy_url = os.environ.get('PROXY_URL', 'http://127.0.0.1:9090/solana') solana_url = os.environ.get("SOLANA_URL", "http://127.0.0.1:8899") diff --git a/proxy/testing/test_web3_clientVersion.py b/proxy/testing/test_web3_clientVersion.py index 23c8afa45..791549bef 100644 --- a/proxy/testing/test_web3_clientVersion.py +++ b/proxy/testing/test_web3_clientVersion.py @@ -1,11 +1,6 @@ import unittest import os from web3 import Web3 -from solcx import install_solc - -# install_solc(version='latest') -install_solc(version='0.7.0') -from solcx import compile_source proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') proxy = Web3(Web3.HTTPProvider(proxy_url)) From d5a501515fa817356a8da8e9cb11ed8f852fe457 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Fri, 3 Dec 2021 17:41:14 +0300 Subject: [PATCH 19/54] =?UTF-8?q?#337=20=D1=81reate=20base=20airdropper=20?= =?UTF-8?q?service=20(#343)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cherrypick part of changes * create indexer.py * remove solana_receipts_update.py * Cherry pick files from old branch * add requirement * fix refactoring issues * Fix inspection issues * fix last issue * simplify tests * add test Co-authored-by: ivanl --- proxy/indexer/airdropper.py | 175 +++++++++++ proxy/testing/mock_server.py | 28 ++ proxy/testing/test_airdropper.py | 176 +++++++++++ proxy/testing/transactions.py | 491 +++++++++++++++++++++++++++++++ requirements.txt | 1 + 5 files changed, 871 insertions(+) create mode 100644 proxy/indexer/airdropper.py create mode 100644 proxy/testing/mock_server.py create mode 100644 proxy/testing/test_airdropper.py create mode 100644 proxy/testing/transactions.py diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py new file mode 100644 index 000000000..12353fe09 --- /dev/null +++ b/proxy/indexer/airdropper.py @@ -0,0 +1,175 @@ +from proxy.indexer.indexer_base import IndexerBase, logger +import os +import requests +import base58 +import json +import logging + +try: + from utils import check_error + from sql_dict import SQLDict +except ImportError: + from .utils import check_error + from .sql_dict import SQLDict + +class Airdropper(IndexerBase): + def __init__(self, + solana_url, + evm_loader_id, + faucet_url = '', + wrapper_whitelist = [], + airdrop_amount = 10, + log_level = 'INFO'): + IndexerBase.__init__(self, solana_url, evm_loader_id, log_level) + + # collection of eth-address-to-create-accout-trx mappings + # for every addresses that was already funded with airdrop + self.airdrop_ready = SQLDict(tablename="airdrop_ready") + self.wrapper_whitelist = wrapper_whitelist + self.airdrop_amount = airdrop_amount + self.faucet_url = faucet_url + + + # helper function checking if given contract address is in whitelist + def _is_allowed_wrapper_contract(self, contract_addr): + return contract_addr in self.wrapper_whitelist + + + # helper function checking if given 'create account' corresponds to 'create erc20 token account' instruction + def _check_create_instr(self, account_keys, create_acc, create_token_acc): + # Must use the same Ethereum account + if account_keys[create_acc['accounts'][1]] != account_keys[create_token_acc['accounts'][2]]: + return False + # Must use the same token program + if account_keys[create_acc['accounts'][5]] != account_keys[create_token_acc['accounts'][6]]: + return False + # Token program must be system token program + if account_keys[create_acc['accounts'][5]] != 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA': + return False + # CreateERC20TokenAccount instruction must use ERC20-wrapper from whitelist + if not self._is_allowed_wrapper_contract(account_keys[create_token_acc['accounts'][3]]): + return False + return True + + + # helper function checking if given 'create erc20 token account' corresponds to 'token transfer' instruction + def _check_transfer(self, account_keys, create_token_acc, token_transfer) -> bool: + return account_keys[create_token_acc['accounts'][1]] == account_keys[token_transfer['accounts'][1]] + + + def _airdrop_to(self, create_acc): + eth_address = "0x" + bytearray(base58.b58decode(create_acc['data'])[20:][:20]).hex() + + if eth_address in self.airdrop_ready: # transaction already processed + return + + logger.info(f"Airdrop to address: {eth_address}") + + json_data = { 'wallet': eth_address, 'amount': self.airdrop_amount } + resp = requests.post(self.faucet_url + '/request_eth_token', json = json_data) + if not resp.ok: + logger.warning(f'Failed to airdrop: {resp.status_code}') + return + + self.airdrop_ready[eth_address] = create_acc + + + def process_trx_airdropper_mode(self, trx): + if check_error(trx): + return + + # helper function finding all instructions that satisfies predicate + def find_instructions(trx, predicate): + return [instr for instr in trx['transaction']['message']['instructions'] if predicate(instr)] + + account_keys = trx["transaction"]["message"]["accountKeys"] + + # Finding instructions specific for airdrop. + # Airdrop triggers on sequence: + # neon.CreateAccount -> neon.CreateERC20TokenAccount -> spl.Transfer (maybe shuffled) + # First: select all instructions that can form such chains + predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + and base58.b58decode(instr['data'])[0] == 0x02 + create_acc_list = find_instructions(trx, predicate) + + predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + and base58.b58decode(instr['data'])[0] == 0x0f + create_token_acc_list = find_instructions(trx, predicate) + + predicate = lambda instr: account_keys[instr['programIdIndex']] == 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' \ + and base58.b58decode(instr['data'])[0] == 0x03 + token_transfer_list = find_instructions(trx, predicate) + + # Second: Find exact chains of instructions in sets created previously + for create_acc in create_acc_list: + for create_token_acc in create_token_acc_list: + if not self._check_create_instr(account_keys, create_acc, create_token_acc): + continue + for token_transfer in token_transfer_list: + if not self._check_transfer(account_keys, create_token_acc, token_transfer): + continue + self._airdrop_to(create_acc) + + + def process_functions(self): + IndexerBase.process_functions(self) + logger.debug("Process receipts") + self.process_receipts() + + + def process_receipts(self): + counter = 0 + for signature in self.transaction_order: + counter += 1 + if signature in self.transaction_receipts: + trx = self.transaction_receipts[signature] + if trx is None: + logger.error("trx is None") + del self.transaction_receipts[signature] + continue + if 'slot' not in trx: + logger.debug("\n{}".format(json.dumps(trx, indent=4, sort_keys=True))) + exit() + if trx['transaction']['message']['instructions'] is not None: + self.process_trx_airdropper_mode(trx) + + +def run_airdropper(solana_url, + evm_loader_id, + faucet_url = '', + wrapper_whitelist = [], + airdrop_amount = 10, + log_level = 'INFO'): + logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s') + logger.setLevel(logging.DEBUG) + logger.info(f"""Running indexer with params: + solana_url: {solana_url}, + evm_loader_id: {evm_loader_id}, + log_level: {log_level}, + faucet_url: {faucet_url}, + wrapper_whitelist: {wrapper_whitelist}, + airdrop_amount: {airdrop_amount}""") + + airdropper = Airdropper(solana_url, + evm_loader_id, + faucet_url, + wrapper_whitelist, + airdrop_amount, + log_level) + airdropper.run() + + +if __name__ == "__main__": + solana_url = os.environ.get('SOLANA_URL', 'http://localhost:8899') + evm_loader_id = os.environ.get('EVM_LOADER_ID', '53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io') + faucet_url = os.environ.get('FAUCET_URL', 'http://localhost:3333') + wrapper_whitelist = os.environ.get('INDEXER_ERC20_WRAPPER_WHITELIST', '').split(',') + airdrop_amount = os.environ.get('AIRDROP_AMOUNT', 0) + log_level = os.environ.get('LOG_LEVEL', 'INFO') + + run_airdropper(solana_url, + evm_loader_id, + faucet_url, + wrapper_whitelist, + airdrop_amount, + log_level) diff --git a/proxy/testing/mock_server.py b/proxy/testing/mock_server.py new file mode 100644 index 000000000..ef8e1c34f --- /dev/null +++ b/proxy/testing/mock_server.py @@ -0,0 +1,28 @@ +import requests + +from flask import Flask, request +from threading import Thread + +class MockServer(Thread): + def __init__(self, port): + super().__init__() + self.port = port + self.app = Flask(__name__) + self.url = "http://localhost:%s" % self.port + self.app.add_url_rule("/shutdown", view_func=self._shutdown_server) + + def add_url_rule(self, url, callback, methods): + self.app.add_url_rule(url, view_func=callback, methods=methods) + + def _shutdown_server(self): + if not 'werkzeug.server.shutdown' in request.environ: + raise RuntimeError('Not running the development server') + request.environ['werkzeug.server.shutdown']() + return 'Server shutting down...' + + def run(self): + self.app.run(port=self.port) + + def shutdown_server(self): + requests.get("http://localhost:%s/shutdown" % self.port) + self.join() diff --git a/proxy/testing/test_airdropper.py b/proxy/testing/test_airdropper.py new file mode 100644 index 000000000..9a00d5279 --- /dev/null +++ b/proxy/testing/test_airdropper.py @@ -0,0 +1,176 @@ +import unittest +from proxy.testing.mock_server import MockServer +from proxy.indexer.airdropper import Airdropper +from proxy.indexer.sql_dict import SQLDict +import time +from flask import request, Response +from unittest.mock import MagicMock, patch, call, ANY +import itertools +from proxy.testing.transactions import pre_token_airdrop_trx1, pre_token_airdrop_trx2,\ + create_sol_acc_and_airdrop_trx, wrapper_whitelist, evm_loader_addr, token_airdrop_address1, \ + token_airdrop_address2, token_airdrop_address3 + +class MockFaucet(MockServer): + def __init__(self, port): + super().__init__(port) + self.request_eth_token_mock = MagicMock() + self.request_eth_token_mock.side_effect = itertools.repeat({}) + self.add_url_rule("/request_eth_token", callback=self.request_eth_token, methods=['POST']) + + def request_eth_token(self): + req = request.get_json() + return self.request_eth_token_mock(req) + + +def create_signature_for_address(signature: str): + return { + 'blockTime': 1638177745, # not make sense + 'confirmationStatus': 'finalized', + 'err': None, + 'memo': None, + 'signature': signature, + 'slot': 9748200 # not make sense + } + + +def create_get_signatures_for_address(signatures: list): + return { + 'jsonrpc': '2.0', + 'result': [ create_signature_for_address(sign) for sign in signatures ], + 'id': 1 + } + + +class Test_Airdropper(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + print("testing indexer in airdropper mode") + cls.address = 'localhost' + cls.faucet_port = 3333 + cls.airdrop_amount = 10 + + cls.faucet = MockFaucet(cls.faucet_port) + cls.faucet.start() + time.sleep(0.2) + + cls.evm_loader_id = evm_loader_addr + cls.wrapper_whitelist = wrapper_whitelist + cls.airdropper = Airdropper(f'http://{cls.address}:8899', + cls.evm_loader_id, + f'http://{cls.address}:{cls.faucet_port}', + cls.wrapper_whitelist, + cls.airdrop_amount, + 'INFO') + + + @classmethod + def tearDownClass(cls) -> None: + cls.faucet.shutdown_server() + cls.faucet.join() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_success_process_trx_with_one_airdrop(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould airdrop to new address - one target in transaction") + mock_sql_dict_contains.side_effect = [False] # new eth address + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=200, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_has_calls([call(token_airdrop_address1, ANY)]) + json_req = {'wallet': token_airdrop_address1, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_called_once_with(json_req) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(Airdropper, '_is_allowed_wrapper_contract') + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_failed_airdrop_contract_not_in_whitelist(self, + mock_sql_dict_contains, + mock_sql_dict_setitem, + mock_is_allowed_contract): + print("\n\nShould not airdrop for contract that is not in whitelist") + mock_is_allowed_contract.side_effect = [False] + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_is_allowed_contract.assert_called_once() + mock_sql_dict_contains.assert_not_called() + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_faucet_failure(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not add address to processed list due to faucet error") + mock_sql_dict_contains.side_effect = [False] # new eth address + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=400, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_not_called() + json_req = {'wallet': token_airdrop_address1, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_called_once_with(json_req) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_process_trx_with_one_airdrop_for_already_processed_address(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not airdrop to repeated address") + mock_sql_dict_contains.side_effect = [True] # eth address processed later + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_complex_transation(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould airdrop to several targets in one transaction") + mock_sql_dict_contains.side_effect = [False, False] # both targets are new + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=200, mimetype='application/json'), + Response("{}", status=200, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx2) + + mock_sql_dict_contains.assert_has_calls([call(token_airdrop_address3), + call(token_airdrop_address2)]) + mock_sql_dict_setitem.assert_has_calls([call(token_airdrop_address3, ANY), + call(token_airdrop_address2, ANY)]) + json_req1 = {'wallet': token_airdrop_address2, 'amount': self.airdrop_amount} + json_req2 = {'wallet': token_airdrop_address3, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_has_calls([call(json_req2), call(json_req1)]) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_no_airdrop_instructions(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not airdrop when instructions are inconsistent") + self.airdropper.process_trx_airdropper_mode(create_sol_acc_and_airdrop_trx) + + mock_sql_dict_contains.assert_not_called() + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + diff --git a/proxy/testing/transactions.py b/proxy/testing/transactions.py new file mode 100644 index 000000000..dec4e2bb9 --- /dev/null +++ b/proxy/testing/transactions.py @@ -0,0 +1,491 @@ + +token_program = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' +evm_loader_addr = 'eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU' +erc20_wrapper = '5H7kvhPD7GECAmf227vTPYTS7SC2PmyuVZaT5zVTx7vb' +wrapper_whitelist = [erc20_wrapper] + +token_airdrop_address1 = '0xf71c4daca893e5333982e2956c5ed9b648818376' + +# Solana transaction for simple case airdrop +pre_token_airdrop_trx1 = { + 'blockTime': 1637857371, + 'meta': { + 'err': None, + 'fee': 5000, + 'innerInstructions': [ + { + 'index': 0, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '111112fUvhuhctf7ykHr29ATacqTktVJJSG9xpkwPTuR6WcMjZZQDYhZ5k4S6Zu6C5sdsn', + 'programIdIndex': 7 + }, + { + 'accounts': [0, 2, 1, 8, 7, 9, 11], + 'data': '', + 'programIdIndex': 10 + }, + { + 'accounts': [0, 2], + 'data': '3Bxs4h24hBtQy9rw', + 'programIdIndex': 7 + }, + { + 'accounts': [2], 'data': + '9krTDU2LzCSUJuVZ', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': 'SYXsBSQy3GeifSEQSGvTbrPNposbSAiSoh1YA85wcvGKSnYg', + 'programIdIndex': 7 + }, + { + 'accounts': [2, 8, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + }, + { + 'index': 1, + 'instructions': [ + { + 'accounts': [0, 3], + 'data': '11119os1e9qSs2u7TsThXqkBSRVFxhmYaFKFZ1waB2X7armDmvK3p5GmLdUxYdg3h7QSrL', + 'programIdIndex': 7 + }, + { + 'accounts': [3, 5, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + } + ], + 'logMessages': [ + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', + 'Program log: Transfer 2039280 lamports to the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Allocate space for the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Assign the associated token account to the SPL Token program', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Initialize the associated token account', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [3]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 464826 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 24626 of 485359 compute units', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', + 'Program log: Total memory occupied: 1414', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 40680 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program log: Total memory occupied: 1536', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: Transfer', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success' + ], + 'postBalances': [5944954400, 2672640, 2039280, 2039280, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'postTokenBalances': [ + { + 'accountIndex': 2, + 'mint': '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + 'owner': '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'uiTokenAmount': { + 'amount': '0', + 'decimals': 9, + 'uiAmount': None, + 'uiAmountString': '0' + } + }, + { + 'accountIndex': 3, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'uiTokenAmount': { + 'amount': '1000000', + 'decimals': 6, + 'uiAmount': 1.0, + 'uiAmountString': '1' + } + }, + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3798000000', + 'decimals': 6, + 'uiAmount': 3798.0, + 'uiAmountString': '3798' + } + } + ], + 'preBalances': [5951710600, 0, 0, 0, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'preTokenBalances': [ + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3799000000', + 'decimals': 6, + 'uiAmount': 3799.0, + 'uiAmountString': '3799' + } + } + ], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 96659490, + 'transaction': { + 'message': { + 'accountKeys': [ + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'J4zoYXUtxNJsCXLih7D7dVkfoT9pvsYa8HtHPEEN88U', + 'AZqbo1ZCwS1grcqUdhs79YspjXME2BjrV5WCHabBS1ht', + erc20_wrapper, + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', + '11111111111111111111111111111111', + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + token_program, + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', + 'SysvarRent111111111111111111111111111111111', + evm_loader_addr + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 6, + 'numRequiredSignatures': 1 + }, + 'instructions': [ + { + 'accounts': [0, 1, 2, 7, 8, 9, 10, 11], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcPrQv1EF4uXaNUgBSq2E3sfp7PKeAH', + 'programIdIndex': 12 + }, + { + 'accounts': [0, 3, 1, 4, 5, 7, 9, 11], + 'data': 'G', + 'programIdIndex': 12 + }, + { + 'accounts': [6, 3, 0], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 9 + } + ], + 'recentBlockhash': '8UBfYxDWWcEbXQPGxPRqK4oGnb6c2XAQm4xZwzM2fxCB' + }, + 'signatures': ['3np8r1PBJW9uuw7o7P86B46zH7nW4UufQa716NUsSscJ1mN5nG6K74JHcMb6YmTmu9Les2NSe1iQyLJZwgS5RpvE'] + } + } + +token_airdrop_address2 = '0x8bd4991b9b81b3298fc6ac06b553b87c8e8582f0' +token_airdrop_address3 = '0x67d9e53d5e747b36fa6587421114aa0b0ca39753' + +# This is transaction containing 2 addresses for airdrop +# Instructions here mixed to test complex case +pre_token_airdrop_trx2 = { + 'blockTime': 1637857344, + 'meta': { + 'err': None, + 'fee': 5000, + 'innerInstructions': [ + { + 'index': 0, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '111112fUvhuhctf7ykHr29ATacqTktVJJSG9xpkwPTuR6WcMjZZQDYhZ5k4S6Zu6C5sdsn', + 'programIdIndex': 7 + }, + { + 'accounts': [0, 2, 1, 8, 7, 9, 11], + 'data': '', + 'programIdIndex': 10 + }, + { + 'accounts': [0, 2], + 'data': '3Bxs4h24hBtQy9rw', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': '9krTDU2LzCSUJuVZ', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': 'SYXsBSQy3GeifSEQSGvTbrPNposbSAiSoh1YA85wcvGKSnYg', + 'programIdIndex': 7 + }, + { + 'accounts': [2, 8, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + }, + { + 'index': 1, + 'instructions': [ + { + 'accounts': [0, 3], + 'data': '11119os1e9qSs2u7TsThXqkBSRVFxhmYaFKFZ1waB2X7armDmvK3p5GmLdUxYdg3h7QSrL', + 'programIdIndex': 7 + }, + { + 'accounts': [3, 5, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + } + ], + 'logMessages': [ + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', + 'Program log: Transfer 2039280 lamports to the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Allocate space for the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Assign the associated token account to the SPL Token program', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Initialize the associated token account', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [3]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 457797 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 27155 of 480859 compute units', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', + 'Program log: Total memory occupied: 1414', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 47709 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program log: Total memory occupied: 1536', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: Transfer', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success'], + 'postBalances': [5951710600, 2672640, 2039280, 2039280, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'postTokenBalances': [ + { + 'accountIndex': 2, + 'mint': '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + 'owner': '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', + 'uiTokenAmount': { + 'amount': '0', + 'decimals': 9, + 'uiAmount': None, + 'uiAmountString': '0' + } + }, + { + 'accountIndex': 3, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', + 'uiTokenAmount': { + 'amount': '1000000', + 'decimals': 6, + 'uiAmount': 1.0, + 'uiAmountString': '1' + } + }, + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3799000000', + 'decimals': 6, + 'uiAmount': 3799.0, + 'uiAmountString': '3799' + } + } + ], + 'preBalances': [5958466800, 0, 0, 0, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'preTokenBalances': [ + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3800000000', + 'decimals': 6, + 'uiAmount': 3800.0, + 'uiAmountString': '3800' + } + } + ], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 96659420, + 'transaction': { + 'message': { + 'accountKeys': [ + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', # 0 - funding + '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', # 1 - ETH account + 'CztEiw75xKWoJ32Nr6exUtmjQHQFD7SE7ZWckX6b55v3', # 2 - NEON token account + '6whnofFTKa6ynDAv882VpQMx6rQvo9v18QM4AGmLCHZ9', # 3 - >> ERC20 token account (wrapper balance) + erc20_wrapper, # 4 - >> contract (ERC20 wrapper) + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', # 5 - SPL token-mint account (wrapped by ERC20) + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', # 6 - SPL token from + '11111111111111111111111111111111', # 7 - System program + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', # 8 - NEON token mint + token_program, # 9 - >> Token program + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', # 10 - Associated token program + 'SysvarRent111111111111111111111111111111111', # 11 + evm_loader_addr, # 12 - >> EVM_LOADER + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', # 13 - funding + '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', # 14 - ETH account + 'J4zoYXUtxNJsCXLih7D7dVkfoT9pvsYa8HtHPEEN88U', # 15 - NEON token account + 'AZqbo1ZCwS1grcqUdhs79YspjXME2BjrV5WCHabBS1ht', # 16 - >> ERC20 token account (wrapper balance) + erc20_wrapper, # 17 - >> contract (ERC20 wrapper) + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', # 18 - SPL token-mint account (wrapped by ERC20) + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', # 19 - SPL token from + '11111111111111111111111111111111', # 20 - System program + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', # 21 - NEON token mint + token_program, # 22 - >> Token program + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', # 23 - Associated token program + 'SysvarRent111111111111111111111111111111111', # 24 + evm_loader_addr # 25 - >> EVM_LOADER + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 6, + 'numRequiredSignatures': 1 + }, + 'instructions': [ + { + # Create 2-nd ERC20 token account + 'accounts': [13, 16, 14, 17, 18, 20, 22, 24], + 'data': 'G', + 'programIdIndex': 25 + }, + { + # Create 2-nd account + 'accounts': [13, 14, 15, 20, 21, 22, 23, 24], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcF3UcpFjrvswawEUuqDYcNbh7NZsuX', + 'programIdIndex': 25 + }, + { + # Create 1-st account + 'accounts': [0, 1, 2, 7, 8, 9, 10, 11], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcHFoDD1r8SrtHfKsSB7RxGBwrn2xPk', + 'programIdIndex': 12 + }, + { + # Create 1-st ERC20 token account + 'accounts': [0, 3, 1, 4, 5, 7, 9, 11], + 'data': 'G', + 'programIdIndex': 12 + }, + { + # Transfer to 1-st ERC20 token account + 'accounts': [6, 3, 0], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 9 + }, + { + # Transfer to 2-nd ERC20 token account + 'accounts': [19, 16, 13], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 22 + } + ], + 'recentBlockhash': 'DhuNa4ts1c8jnD1u55rvQd8YKQo8t4aHwt3jUd1Anexu'}, + 'signatures': ['4WotQiXmE5AUjC9yoTE69C2CLmWa2Typ1mAuG4EzAkjYXSfYH6gQewkSfsEHGQjvXkANFKXtkUtvkqm7eq23M74V'] + } + } + +create_sol_acc_and_airdrop_trx = { + 'blockTime': 1638178743, + 'meta': { + 'err': None, + 'fee': 10000, + 'innerInstructions': [], + 'logMessages': [ + 'Program 11111111111111111111111111111111 invoke [1]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: InitializeMint', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 2833 of 200000000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success'], + 'postBalances': [999998528400, 1461600, 1009200, 1, 1130582400], + 'postTokenBalances': [], + 'preBalances': [1000000000000, 0, 1009200, 1, 1130582400], + 'preTokenBalances': [], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 15029, + 'transaction': { + 'message': { + 'accountKeys': [ + 'D4Aa2HU5kwF3nByGYk7pdjbP4n3cFFjPjYASdKUfpH4H', + 'EHDze1sDhUk7dR9iBgV4Mm3dYMk3ZQXKGHiiuVTEYaYr', + 'SysvarRent111111111111111111111111111111111', + '11111111111111111111111111111111', + 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 3, + 'numRequiredSignatures': 2 + }, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '11114XtYk9gGfZoo968fyjNUYQJKf9gdmkGoaoBpzFv4vyaSMBn3VKxZdv7mZLzoyX5YNC', + 'programIdIndex': 3 + }, + { + 'accounts': [1, 2], + 'data': '1DidxzgH8WA79wndRY2Vc8EsYn3Rf9CKWmB3tRLNtzhWUZD', + 'programIdIndex': 4 + } + ], + 'recentBlockhash': '6kT9KCR37ZWhu9fbdacwUsXAHXvPo9kYwzQsQnh9dWyW' + }, + 'signatures': [ + '4wNHEzKDpqKpQ51A3cYWGsLXAxc3cxHtk45cs1RMUYgY6bViBUu6w7VaDMSQjprbwC7AF4bMy3ejR69FAVwQWUgh', + '4n6PzpFyQ5e9PTDFTUmHoPXUJYtNqsfQgwinu5ujYeY6EigseJHGgykmzMMb8exKsC45E7RjiyahLqhbR1uQo5V5' + ] + } + } diff --git a/requirements.txt b/requirements.txt index 323f726c3..3ca6efc39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ solana==0.10.0 psycopg2-binary ethereum py-solc-x==1.1.0 +flask From 50bc6bacb97f07534d0f9ee10469fa7e658cb801 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Mon, 6 Dec 2021 11:53:14 +0300 Subject: [PATCH 20/54] #337 fix running airdropper (#347) Co-authored-by: ivanl --- proxy/__main__.py | 21 ++++++++++++++++++++- proxy/indexer/airdropper.py | 16 ---------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/proxy/__main__.py b/proxy/__main__.py index d04d8529d..18c95ec59 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -8,7 +8,26 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ + from .proxy import entry_point +import os +from .indexer.airdropper import run_airdropper if __name__ == '__main__': - entry_point() + airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True'] + if airdropper_mode: + print("Will run in airdropper mode") + solana_url = os.environ['SOLANA_URL'] + evm_loader_id = os.environ['EVM_LOADER'] + faucet_url = os.environ['FAUCET_URL'] + wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'].split(',') + airdrop_amount = int(os.environ['AIRDROP_AMOUNT']) + log_level = os.environ['LOG_LEVEL'] + run_airdropper(solana_url, + evm_loader_id, + faucet_url, + wrapper_whitelist, + airdrop_amount, + log_level) + else: + entry_point() diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py index 12353fe09..41c638cc6 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/indexer/airdropper.py @@ -157,19 +157,3 @@ def run_airdropper(solana_url, airdrop_amount, log_level) airdropper.run() - - -if __name__ == "__main__": - solana_url = os.environ.get('SOLANA_URL', 'http://localhost:8899') - evm_loader_id = os.environ.get('EVM_LOADER_ID', '53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io') - faucet_url = os.environ.get('FAUCET_URL', 'http://localhost:3333') - wrapper_whitelist = os.environ.get('INDEXER_ERC20_WRAPPER_WHITELIST', '').split(',') - airdrop_amount = os.environ.get('AIRDROP_AMOUNT', 0) - log_level = os.environ.get('LOG_LEVEL', 'INFO') - - run_airdropper(solana_url, - evm_loader_id, - faucet_url, - wrapper_whitelist, - airdrop_amount, - log_level) From a619b6607517515603353077f4b0b7f72db3a9f3 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Mon, 6 Dec 2021 17:08:32 +0300 Subject: [PATCH 21/54] #351 fix canceller droping running transactions (#352) --- .buildkite/steps/deploy-test.sh | 1 + proxy/indexer/indexer.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index d90039ef3..85ee0f509 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -92,6 +92,7 @@ docker run --rm -ti --network=host \ --entrypoint ./deploy-test.sh \ ${EXTRA_ARGS:-} \ $UNISWAP_V2_CORE_IMAGE \ + all echo "Run tests return" exit 0 diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index fe6bd1342..3254bfafa 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -28,9 +28,10 @@ def __init__(self, storage_account): class ContinueStruct: - def __init__(self, signature, results, accounts = None): + def __init__(self, signature, results, slot, accounts = None): self.signatures = [signature] self.results = results + self.slot = slot self.accounts = accounts @@ -165,7 +166,7 @@ def process_receipts(self): continue_result.signatures, storage_account, continue_result.accounts, - slot + max(slot, continue_result.slot) ) del continue_table[storage_account] @@ -271,6 +272,7 @@ def process_receipts(self): logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") trx_table[eth_signature].got_result = continue_result.results trx_table[eth_signature].signatures += continue_result.signatures + trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, continue_result.slot) del continue_table[storage_account] @@ -296,7 +298,7 @@ def process_receipts(self): continue_table[storage_account].results = got_result else: - continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts) elif instruction_data[0] == 0x0b or instruction_data[0] == 0x16: # ExecuteTrxFromAccountDataIterative ExecuteTrxFromAccountDataIterativeV02 if instruction_data[0] == 0x0b: @@ -319,7 +321,7 @@ def process_receipts(self): else: holder_table[holder_account] = HolderStruct(storage_account) else: - continue_table[storage_account] = ContinueStruct(signature, None, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, None, slot, blocked_accounts) holder_table[holder_account] = HolderStruct(storage_account) @@ -329,7 +331,7 @@ def process_receipts(self): storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] - continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), slot, blocked_accounts) elif instruction_data[0] == 0x0d: # logger.debug("{:>10} {:>6} PartialCallOrContinueFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex())) @@ -349,6 +351,7 @@ def process_receipts(self): if eth_signature in trx_table: trx_table[eth_signature].signatures.append(signature) + trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, slot) else: trx_table[eth_signature] = TransactionStruct( eth_trx, @@ -396,7 +399,7 @@ def process_receipts(self): continue_table[storage_account].results = got_result else: - continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts) holder_table[holder_account] = HolderStruct(storage_account) if instruction_data[0] > 0x16: From d2f8817119804972cd324991e821aba9bea46469 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Mon, 6 Dec 2021 17:18:16 +0300 Subject: [PATCH 22/54] 349 improve neon proxy logging to filter a request and the corresponding response by 'method' (#350) * Added the request['method'] value only in debug output on response So it allows you to filer neon-proxy logs for requests and corresponding response for a 'method' * Introduced LOG_SENDING_SOLANA_TRANSACTION by default equal 'NO'. If it is set as 'YES' and then restart neon-proxy you can see sending solana transactions in logs --- proxy/docker-compose-test.yml | 1 + proxy/environment.py | 1 + proxy/plugin/solana_rest_api.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 729b4affa..fefd28d6b 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -65,6 +65,7 @@ services: POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass NEW_USER_AIRDROP_AMOUNT: 100 + LOG_SENDING_SOLANA_TRANSACTION: "YES" CONFIG: ci hostname: proxy depends_on: diff --git a/proxy/environment.py b/proxy/environment.py index 526ce53cf..8fc3387f3 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -17,6 +17,7 @@ TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) +LOG_SENDING_SOLANA_TRANSACTION = os.environ.get("LOG_SENDING_SOLANA_TRANSACTION", "NO") == "YES" class solana_cli: def call(self, *args): diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index e7f9afd19..61f522b7a 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -639,7 +639,8 @@ def handle_request(self, request: HttpParser) -> None: traceback.print_exc() response = {'jsonrpc': '2.0', 'error': {'code': -32000, 'message': str(err)}} - logger.debug('>>> %s 0x%0x %s', threading.get_ident(), id(self.model), json.dumps(response)) + logger.debug('>>> %s 0x%0x %s %s', threading.get_ident(), id(self.model), json.dumps(response), + request['method'] if 'method' in request else '---') self.client.queue(memoryview(build_http_response( httpStatusCodes.OK, body=json.dumps(response).encode('utf8'), From 5d6dab296e84994ebc2e95544a23c5c668b20919 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Mon, 6 Dec 2021 18:29:00 +0300 Subject: [PATCH 23/54] #349 Fix using log sending solana transaction (#353) * Move LOG_SENDING_SOLANA_TRANSACTION to environment.py after merging develop * Fix after merging Co-authored-by: Dmitriy Borisenko --- proxy/common_neon/solana_interactor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index ffb02e5d4..bd0138d07 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -9,8 +9,7 @@ from solana.rpc.types import TxOpts from .costs import update_transaction_cost -from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY - +from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, LOG_SENDING_SOLANA_TRANSACTION logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -88,14 +87,15 @@ def collect_result(self, reciept, eth_trx, reason=None): update_transaction_cost(result, eth_trx, reason) return result - def send_measured_transaction(self, trx, eth_trx, reason): + if LOG_SENDING_SOLANA_TRANSACTION: + logger.debug("send_measured_transaction for reason %s: %s ", reason, trx.__dict__) result = self.send_transaction(trx, eth_trx, reason=reason) self.get_measurements(result) return result - - # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh) + # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in + # .buildkite/steps/deploy-test.sh) def get_measurements(self, result): try: measurements = self.extract_measurements_from_receipt(result) From 4045e90d28d945617e15c495cf6cfdae23c20957 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Mon, 6 Dec 2021 21:07:51 +0300 Subject: [PATCH 24/54] #295 iterative execution (#332) * #291 extract transaction sender class * #291 move perm accs to transaction sender * #291 fix state * #291 fix errors * #291 merge fixes * #291 refactoring * #291 move EXTRA_GAS to environment * #291 capitalize CONFIRMATION_CHECK_DELAY * #291 sort imports * #291 relative paths * #291 Should be fixed in #326 * #291 testing chnages * fix storage account check * #291 rename `trx_with_create_and_airdrop` -> `make_trx_with_create_and_airdrop` * #295 fix state * #291 iterative combined * #295 do not get measurments * #291 pull request fixes * #295 turn combined instructions ON * #295 make neon_instructions return transasactions * #291 merge fix * #295 get rid of `USE_COMBINED_START_CONTINUE` * #295 requested fixes * #295 call_continue_bucked refactoring * #295 fix * #295 leave only combined iterative transactions * #295 move constants into class * #295 refactoring Co-authored-by: sinev-valentine --- proxy/common_neon/errors.py | 1 + proxy/common_neon/neon_instruction.py | 58 +++++----- proxy/common_neon/solana_interactor.py | 14 ++- proxy/common_neon/transaction_sender.py | 139 +++++++++++++++++++----- proxy/environment.py | 1 - proxy/run-proxy.sh | 1 - 6 files changed, 154 insertions(+), 60 deletions(-) diff --git a/proxy/common_neon/errors.py b/proxy/common_neon/errors.py index 4c17e8a9b..30286e6a0 100644 --- a/proxy/common_neon/errors.py +++ b/proxy/common_neon/errors.py @@ -12,6 +12,7 @@ def getError(self): if self.data: error['data'] = self.data return error + class SolanaErrors(Enum): AccountNotFound = "Invalid param: could not find account" diff --git a/proxy/common_neon/neon_instruction.py b/proxy/common_neon/neon_instruction.py index 8242d97cb..db6066fa7 100644 --- a/proxy/common_neon/neon_instruction.py +++ b/proxy/common_neon/neon_instruction.py @@ -283,14 +283,17 @@ def make_noniterative_call_transaction(self, length_before: int = 0) -> Transact return trx - def make_partial_call_instruction(self) -> TransactionInstruction: - return TransactionInstruction( + def make_continue_transaction(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("14") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + if index: + data = data + index.to_bytes(8, byteorder="little") + + return Transaction().add(TransactionInstruction( program_id = EVM_LOADER_ID, - data = bytearray.fromhex("13") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little") + self.msg, + data = data, keys = [ AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), @@ -301,28 +304,19 @@ def make_partial_call_instruction(self) -> TransactionInstruction: AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), ] + obligatory_accounts - ) - - - def make_iterative_call_transaction(self, length_before: int = 0) -> Transaction: - trx = Transaction() - trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 13)) - trx.add(self.make_partial_call_instruction()) - return trx + )) - def make_call_from_account_instruction(self) -> Transaction: + def make_cancel_transaction(self) -> Transaction: return Transaction().add(TransactionInstruction( program_id = EVM_LOADER_ID, - data = bytearray.fromhex("16") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little"), + data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), keys = [ - AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), - AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), ] + self.eth_accounts + [ @@ -332,17 +326,15 @@ def make_call_from_account_instruction(self) -> Transaction: )) - def make_continue_instruction(self, steps, index=None) -> Transaction: - data = bytearray.fromhex("14") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") - if index: - data = data + index.to_bytes(8, byteorder="little") - - return Transaction().add(TransactionInstruction( + def make_partial_call_or_continue_instruction(self, steps: int = 0) -> TransactionInstruction: + data = bytearray.fromhex("0D") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + self.msg + return TransactionInstruction( program_id = EVM_LOADER_ID, data = data, keys = [ AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), @@ -353,19 +345,31 @@ def make_continue_instruction(self, steps, index=None) -> Transaction: AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), ] + obligatory_accounts - )) + ) - def make_cancel_instruction(self) -> Transaction: + def make_partial_call_or_continue_transaction(self, steps: int = 0, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 13)) + trx.add(self.make_partial_call_or_continue_instruction(steps)) + return trx + + + def make_partial_call_or_continue_from_account_data(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("0E") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder='little') + if index: + data = data + index.to_bytes(8, byteorder="little") return Transaction().add(TransactionInstruction( program_id = EVM_LOADER_ID, - data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), + data = data, keys = [ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), ] + self.eth_accounts + [ diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index bd0138d07..660ba9b25 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -5,10 +5,12 @@ import re import time +from solana.rpc.api import Client as SolanaClient from solana.rpc.commitment import Confirmed from solana.rpc.types import TxOpts from .costs import update_transaction_cost +from .utils import get_from_dict from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, LOG_SENDING_SOLANA_TRANSACTION logger = logging.getLogger(__name__) @@ -16,7 +18,7 @@ class SolanaInteractor: - def __init__(self, signer, client) -> None: + def __init__(self, signer, client: SolanaClient) -> None: self.signer = signer self.client = client @@ -194,6 +196,16 @@ def check_if_program_exceeded_instructions(err_result): return False +def check_if_storage_is_empty_error(err_result): + error_arr = get_from_dict(err_result, "data", "err", "InstructionError") + if error_arr is not None and isinstance(error_arr, list): + error_dict = error_arr[1] + if isinstance(error_dict, dict) and 'Custom' in error_dict: + if error_dict['Custom'] == 1 or error_dict['Custom'] == 4: + return True + return False + + def check_if_continue_returned(result): tx_info = result['result'] accounts = tx_info["transaction"]["message"]["accountKeys"] diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 5300c6a85..820bfb6b3 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -1,5 +1,6 @@ import json import logging +import math import os import rlp import time @@ -18,7 +19,8 @@ from .emulator_interactor import call_emulated from .layouts import ACCOUNT_INFO_LAYOUT from .neon_instruction import NeonInstruction -from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_if_program_exceeded_instructions +from .solana_interactor import SolanaInteractor, check_if_continue_returned, \ + check_if_program_exceeded_instructions, check_if_storage_is_empty_error from ..environment import EVM_LOADER_ID from ..plugin.eth_proto import Trx as EthTrx @@ -67,7 +69,7 @@ def execute(self): try: if call_iterative: try: - return iterative_executor.call_signed_iterative() + return iterative_executor.call_signed_iterative_combined() except Exception as err: logger.debug(str(err)) if str(err).startswith("transaction too large:"): @@ -77,7 +79,7 @@ def execute(self): raise if call_from_holder: - return iterative_executor.call_signed_with_holder_acc() + return iterative_executor.call_signed_with_holder_combined() finally: self.free_perm_accs() @@ -93,7 +95,7 @@ def create_noniterative_executor(self): def create_iterative_executor(self): self.instruction.init_iterative(self.storage, self.holder, self.perm_accs_id) - return IterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx, self.steps) + return IterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx, self.steps, self.steps_emulated) def init_perm_accs(self): @@ -302,6 +304,8 @@ def create_account_list_by_emulate(self): AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), ] + add_keys_05 + self.steps_emulated = output_json["steps_executed"] + class NoniterativeTransactionSender: def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx): @@ -321,53 +325,52 @@ def call_signed_noniterative(self): class IterativeTransactionSender: - def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx, steps: int): + CONTINUE_REGULAR = 'ContinueV02' + CONTINUE_COMBINED = 'PartialCallOrContinueFromRawEthereumTX' + CONTINUE_HOLDER_COMB = 'ExecuteTrxFromAccountDataIterativeOrContinue' + + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx, steps: int, steps_emulated: int): self.sender = solana_interactor self.instruction = neon_instruction self.create_acc_trx = create_acc_trx self.eth_trx = eth_trx self.steps = steps + self.steps_emulated = steps_emulated + self.instruction_type = self.CONTINUE_REGULAR - def call_signed_iterative(self): - if len(self.create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(self.create_acc_trx) - self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') - - call_txs = self.instruction.make_iterative_call_transaction() - - logger.debug("Partial call") - self.sender.send_measured_transaction(call_txs, self.eth_trx, 'PartialCallFromRawEthereumTXv02') - + def call_signed_iterative_combined(self): + self.create_accounts_for_trx() + self.instruction_type = self.CONTINUE_COMBINED return self.call_continue() - def call_signed_with_holder_acc(self): + def call_signed_with_holder_combined(self): self.write_trx_to_holder_account() - if len(self.create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(self.create_acc_trx) - self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'create_accounts_for_deploy') + self.create_accounts_for_trx() + self.instruction_type = self.CONTINUE_HOLDER_COMB + return self.call_continue() - # ExecuteTrxFromAccountDataIterative - logger.debug("ExecuteTrxFromAccountDataIterative:") - call_txs = self.instruction.make_call_from_account_instruction() - self.sender.send_measured_transaction(call_txs, self.eth_trx, 'ExecuteTrxFromAccountDataIterativeV02') - return self.call_continue() + def create_accounts_for_trx(self): + length = len(self.create_acc_trx.instructions) + if length == 0: + return + logger.debug(f"Create account for trx: {length}") + precall_txs = Transaction() + precall_txs.add(self.create_acc_trx) + self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') def write_trx_to_holder_account(self): + logger.debug('write_trx_to_holder_account') msg = self.eth_trx.signature() + len(self.eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + self.eth_trx.unsigned_msg() - # Write transaction to transaction holder account offset = 0 receipts = [] rest = msg while len(rest): (part, rest) = (rest[:1000], rest[1000:]) - # logger.debug("sender_sol %s %s %s", sender_sol, holder, acc.public_key()) trx = self.instruction.make_write_transaction(offset, part) receipts.append(self.sender.send_transaction_unconfirmed(trx)) offset += len(part) @@ -377,6 +380,19 @@ def write_trx_to_holder_account(self): def call_continue(self): + return_result = None + try: + return_result = self.call_continue_bucked() + except Exception as err: + logger.debug("call_continue_bucked_combined exception: {}".format(str(err))) + + if return_result is not None: + return return_result + + return self.call_continue_iterative() + + + def call_continue_iterative(self): try: return self.call_continue_step_by_step() except Exception as err: @@ -398,7 +414,7 @@ def call_continue_step_by_step(self): def call_continue_step(self): step_count = self.steps while step_count > 0: - trx = self.instruction.make_continue_instruction(step_count) + trx = self.instruction.make_continue_transaction(step_count) logger.debug("Step count {}".format(step_count)) try: @@ -413,8 +429,71 @@ def call_continue_step(self): def call_cancel(self): - trx = self.instruction.make_cancel_instruction() + trx = self.instruction.make_cancel_transaction() logger.debug("Cancel") result = self.sender.send_measured_transaction(trx, self.eth_trx, 'CancelWithNonce') return result['result']['transaction']['signatures'][0] + + + def call_continue_bucked(self): + logger.debug("Send bucked combined: %s", self.instruction_type) + steps = self.steps + + receipts = [] + for index in range(math.ceil(self.steps_emulated/self.steps) + self.addition_count()): + try: + trx = self.make_bucked_trx(steps, index) + receipts.append(self.sender.send_transaction_unconfirmed(trx)) + except SendTransactionError as err: + logger.error(f"Failed to call continue bucked, error: {err.result}") + if check_if_storage_is_empty_error(err.result): + pass + elif check_if_program_exceeded_instructions(err.result): + steps = int(steps * 90 / 100) + else: + raise + except Exception as err: + logger.debug(str(err)) + if str(err).startswith('failed to get recent blockhash'): + pass + else: + raise + + return self.collect_bucked_results(receipts, self.instruction_type) + + + def addition_count(self): + ''' + How many transactions are needed depending on trx type: + CONTINUE_COMBINED: 2 (1 for begin and 1 for decreased steps) + CONTINUE_HOLDER_COMB: 1 for begin + 0 otherwise + ''' + addition_count = 0 + if self.instruction_type == self.CONTINUE_COMBINED: + addition_count = 2 + elif self.instruction_type == self.CONTINUE_HOLDER_COMB: + addition_count = 1 + return addition_count + + + def make_bucked_trx(self, steps, index): + if self.instruction_type == self.CONTINUE_REGULAR: + return self.instruction.make_continue_transaction(steps, index) + elif self.instruction_type == self.CONTINUE_COMBINED: + return self.instruction.make_partial_call_or_continue_transaction(steps - index) + elif self.instruction_type == self.CONTINUE_HOLDER_COMB: + return self.instruction.make_partial_call_or_continue_from_account_data(steps, index) + else: + raise Exception("Unknown continue type: {}".format(self.instruction_type)) + + + def collect_bucked_results(self, receipts, reason): + logger.debug(f"Collected bucked results: {receipts}") + result_list = self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason=reason) + for result in result_list: + # self.sender.get_measurements(result) + signature = check_if_continue_returned(result) + if signature: + return signature diff --git a/proxy/environment.py b/proxy/environment.py index 8fc3387f3..927cb30ad 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -12,7 +12,6 @@ NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) CONFIRMATION_CHECK_DELAY = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) -USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "NO") == "YES" CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 diff --git a/proxy/run-proxy.sh b/proxy/run-proxy.sh index 103795d70..199489563 100755 --- a/proxy/run-proxy.sh +++ b/proxy/run-proxy.sh @@ -8,7 +8,6 @@ if [ "$CONFIG" == "ci" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://solana:8899" [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=100000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.5" - [[ -z "$USE_COMBINED_START_CONTINUE" ]] && export USE_COMBINED_START_CONTINUE="NO" [[ -z "$CONTINUE_COUNT_FACTOR" ]] && export CONTINUE_COUNT_FACTOR="3" [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="postgres" From 521696f164cacc5a3f34016174d1c0cbe49147ab Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Tue, 7 Dec 2021 15:08:47 +0300 Subject: [PATCH 25/54] #354 Check result for errors (#355) * #354 Check result for errors * #354 get_measurements error -> warning * #354 check for errors in get measurements * #354 get measurements from bucked transactions * #354 remove repeated code --- proxy/common_neon/solana_interactor.py | 49 +++++++++++++++++++++---- proxy/common_neon/transaction_sender.py | 14 +++++-- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 660ba9b25..56015e127 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -133,6 +133,11 @@ def collect_results(self, receipts, eth_trx=None, reason=None): @staticmethod def extract_measurements_from_receipt(receipt): + if check_for_errors(receipt): + logger.warning("Can't get measurements from receipt with error") + logger.info("Failed result: %s"%json.dumps(receipt, indent=3)) + return [] + log_messages = receipt['result']['meta']['logMessages'] transaction = receipt['result']['transaction'] accounts = transaction['message']['accountKeys'] @@ -185,19 +190,47 @@ def extract_measurements_from_receipt(receipt): return result -def check_if_program_exceeded_instructions(err_result): - err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" - err_budget = "failed: Computational budget exceeded" +def get_error_definition_from_reciept(receipt): + err_from_reciept = get_from_dict(receipt, 'result', 'meta', 'err', 'InstructionError') + if err_from_reciept is not None: + return err_from_reciept + + err_from_reciept_result = get_from_dict(receipt, 'meta', 'err', 'InstructionError') + if err_from_reciept_result is not None: + return err_from_reciept_result + + err_from_send_trx_error = get_from_dict(receipt, 'data', 'err', 'InstructionError') + if err_from_send_trx_error is not None: + return err_from_send_trx_error + + err_from_prepared_receipt = get_from_dict(receipt, 'err', 'InstructionError') + if err_from_prepared_receipt is not None: + return err_from_prepared_receipt + + return None + + - if err_result['data']['logs'][-1].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-2].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-1].find(err_budget) >= 0: +def check_for_errors(receipt): + if get_error_definition_from_reciept(receipt) is not None: return True return False -def check_if_storage_is_empty_error(err_result): - error_arr = get_from_dict(err_result, "data", "err", "InstructionError") +def check_if_program_exceeded_instructions(receipt): + error_arr = get_error_definition_from_reciept(receipt) + if error_arr is not None and isinstance(error_arr, list): + error_type = error_arr[1] + if isinstance(error_type, str): + if error_type == 'ProgramFailedToComplete': + return True + if error_type == 'ComputationalBudgetExceeded': + return True + return False + + +def check_if_storage_is_empty_error(receipt): + error_arr = get_error_definition_from_reciept(receipt) if error_arr is not None and isinstance(error_arr, list): error_dict = error_arr[1] if isinstance(error_dict, dict) and 'Custom' in error_dict: diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 820bfb6b3..3150ed885 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -19,7 +19,7 @@ from .emulator_interactor import call_emulated from .layouts import ACCOUNT_INFO_LAYOUT from .neon_instruction import NeonInstruction -from .solana_interactor import SolanaInteractor, check_if_continue_returned, \ +from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_for_errors,\ check_if_program_exceeded_instructions, check_if_storage_is_empty_error from ..environment import EVM_LOADER_ID from ..plugin.eth_proto import Trx as EthTrx @@ -321,6 +321,12 @@ def call_signed_noniterative(self): call_txs_05.add(self.create_acc_trx) call_txs_05.add(self.instruction.make_noniterative_call_transaction(len(call_txs_05.instructions))) result = self.sender.send_measured_transaction(call_txs_05, self.eth_trx, 'CallFromRawEthereumTX') + + if check_for_errors(result): + if check_if_program_exceeded_instructions(result): + raise Exception("Program failed to complete") + raise Exception(json.dumps(result['result']['meta'])) + return result['result']['transaction']['signatures'][0] @@ -359,7 +365,9 @@ def create_accounts_for_trx(self): logger.debug(f"Create account for trx: {length}") precall_txs = Transaction() precall_txs.add(self.create_acc_trx) - self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') + result = self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') + if check_for_errors(result): + raise Exception("Failed to create account for trx") def write_trx_to_holder_account(self): @@ -493,7 +501,7 @@ def collect_bucked_results(self, receipts, reason): logger.debug(f"Collected bucked results: {receipts}") result_list = self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason=reason) for result in result_list: - # self.sender.get_measurements(result) + self.sender.get_measurements(result) signature = check_if_continue_returned(result) if signature: return signature From fff455e7c7657768606001275fab2eb2e3cd9459 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Tue, 7 Dec 2021 15:09:53 +0300 Subject: [PATCH 26/54] #360 pass transaction too large upper (#361) --- proxy/common_neon/transaction_sender.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 3150ed885..c78d74d2a 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -393,6 +393,8 @@ def call_continue(self): return_result = self.call_continue_bucked() except Exception as err: logger.debug("call_continue_bucked_combined exception: {}".format(str(err))) + if str(err).startswith("transaction too large:"): + raise if return_result is not None: return return_result From 241659115bd1f3d0c769b1b157f847c71d7e3436 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 7 Dec 2021 15:31:27 +0300 Subject: [PATCH 27/54] NEON_PROXY_PKG_VERSION = '0.5.0' --- proxy/plugin/solana_rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 61f522b7a..badc941f6 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -46,7 +46,7 @@ modelInstanceLock = threading.Lock() modelInstance = None -NEON_PROXY_PKG_VERSION = '0.4.1-rc0' +NEON_PROXY_PKG_VERSION = '0.5.0' NEON_PROXY_REVISION = 'NEON_PROXY_REVISION_TO_BE_REPLACED' class EthereumModel: From 168e428a596e015b1a56743280f1b64b12636040 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 16 Nov 2021 14:12:20 +0300 Subject: [PATCH 28/54] VERSION = (0, 4, 0) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0d2c67516..94ebe2b28 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ """ from setuptools import setup, find_packages -VERSION = (2, 2, 0) +VERSION = (0, 4, 0) __version__ = '.'.join(map(str, VERSION[0:3])) __description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on Network monitoring, controls & Application development, testing, debugging.''' From 3125645f880c94c8baedfa7fa539ab8dda8c610c Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 16 Nov 2021 14:25:23 +0300 Subject: [PATCH 29/54] EVM_LOADER_REVISION:=v0.4.0 --- .buildkite/steps/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index d8becc316..b9575b58f 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -4,7 +4,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) set ${SOLANA_REVISION:=v1.7.9-resources} -set ${EVM_LOADER_REVISION:=stable} +set ${EVM_LOADER_REVISION:=v0.4.0} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally docker pull neonlabsorg/solana:${SOLANA_REVISION} From 07f175d6f9580e30d8d99b15c4e6799ee99de220 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 16 Nov 2021 14:33:13 +0300 Subject: [PATCH 30/54] EVM_LOADER_REVISION:=stable --- .buildkite/steps/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index b9575b58f..d8becc316 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -4,7 +4,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) set ${SOLANA_REVISION:=v1.7.9-resources} -set ${EVM_LOADER_REVISION:=v0.4.0} +set ${EVM_LOADER_REVISION:=stable} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally docker pull neonlabsorg/solana:${SOLANA_REVISION} From 0bbb26b16e2a462837ea80d9e8d983e45fe99c35 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 18 Nov 2021 11:02:56 +0300 Subject: [PATCH 31/54] 289 implement eth get storage at (#298) * create eth_getStorageAt prototype * implement eth_getStorageAt * fix test * Fix review issues, add UTs * handle neon-cli errors * add test Co-authored-by: ivanl --- proxy/plugin/solana_rest_api.py | 17 ++++++++++++++- proxy/test_eth_sendRawTransaction.py | 32 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 54a19ecf4..ba0b1fb45 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -34,7 +34,7 @@ import os from ..indexer.utils import get_trx_results, LogDB from ..indexer.sql_dict import SQLDict -from proxy.environment import evm_loader_id, solana_cli, solana_url +from proxy.environment import evm_loader_id, solana_cli, solana_url, neon_cli logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -195,6 +195,21 @@ def getBlockBySlot(self, slot, full): } return ret + def eth_getStorageAt(self, account, position, block_identifier): + '''Retrieves storage data by given position + Currently supports only 'latest' block + ''' + if block_identifier != "latest": + logger.debug(f"Block type '{block_identifier}' is not supported yet") + raise RuntimeError(f"Not supported block identifier: {block_identifier}") + + try: + value = neon_cli().call('get-storage-at', account, position) + return value + except Exception as err: + logger.debug(f"Neon-cli failed to execute: {err}") + return '0x00' + def eth_getBlockByHash(self, trx_hash, full): """Returns information about a block by hash. trx_hash - Hash of a block. diff --git a/proxy/test_eth_sendRawTransaction.py b/proxy/test_eth_sendRawTransaction.py index 35ac90651..12e3f4633 100644 --- a/proxy/test_eth_sendRawTransaction.py +++ b/proxy/test_eth_sendRawTransaction.py @@ -428,6 +428,38 @@ def test_07_execute_long_transaction(self): print('times_to_calculate:', times_to_calculate) print('time_duration:', time_duration) + def test_get_storage_at(self): + print("\nhttps://github.com/neonlabsorg/proxy-model.py/issues/289") + right_nonce = proxy.eth.get_transaction_count(proxy.eth.default_account) + value_to_store = 452356 + trx_store = self.storage_contract.functions.store(value_to_store).buildTransaction({'nonce': right_nonce}) + print('trx_store:', trx_store) + trx_store_signed = proxy.eth.account.sign_transaction(trx_store, eth_account.key) + print('trx_store_signed:', trx_store_signed) + trx_store_hash = proxy.eth.send_raw_transaction(trx_store_signed.rawTransaction) + print('trx_store_hash:', trx_store_hash.hex()) + trx_store_receipt = proxy.eth.wait_for_transaction_receipt(trx_store_hash) + print('trx_store_receipt:', trx_store_receipt) + + number_pos = 0 + value_received = proxy.eth.get_storage_at(self.storage_contract.address, number_pos, "latest") + print('eth_getStorageAt existing address and index => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), value_to_store) + + non_existing_pos = 12 + value_received = proxy.eth.get_storage_at(self.storage_contract.address, non_existing_pos, "latest") + print('eth_getStorageAt existing address and non-existing index => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) + + non_exising_address = b'\xe1\xda\xb7\xa6\x17\x6f\x87\x68\xF5\x3a\x42\x5f\x29\x61\x73\x60\x5e\xd5\x08\x32' + value_received = proxy.eth.get_storage_at(non_exising_address, non_existing_pos, "latest") + print('eth_getStorageAt non-existing address => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) + + not_a_contract_address = proxy.eth.default_account + value_received = proxy.eth.get_storage_at(not_a_contract_address, 0, "latest") + print('eth_getStorageAt not_a_contract_address address => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) if __name__ == '__main__': unittest.main() From 7841b07f33d9d45b68b621c711ff4c7e67b7f3f6 Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Thu, 18 Nov 2021 17:18:47 +0500 Subject: [PATCH 32/54] #256 create and airdrop eth account (#259) * constants and utils * solana rest api tools imports * comments and tabs * rename calling getTokens function * Meet SolanaErrors * Implement creating account on getting balance * Move tests and add get_from_dict test * Add test "Metamask creates an account" * Just not ot loose changes * implemented * Fix tests on CI * set NEW_USER_AIRDROP_AMOUNT on CI * improve logging * improve logging of SendTransactionError * spit and polish * extend airdrop tests * spit and polish * spit and polish * move tests * move tests * Get rid off extra data.py * Improve logging * spit and polish * spit and polish * spit and polish * spit and polish * Pass MINIMAL_GAS_PRICE int airdrop tests * spit and polish * move test_operator_spending.py * move test_operator_spending.py * spit and polish * spit and polish * Fix message printing * spit and polish * spit and polish * spit and polish * spit and polish * use error instead of debug * Revert "constants and utils" This reverts commit 505653656a5406d64592c4fa2ebcf8ff1aed8edd. # Conflicts: # proxy/testing/test_eth_sendRawTransaction.py * Emphasize meaning of trx extending functions This reverts commit 505653656a5406d64592c4fa2ebcf8ff1aed8edd. # Conflicts: # proxy/testing/test_eth_sendRawTransaction.py * Resolve @otselik remarks # Conflicts: # proxy/testing/test_eth_sendRawTransaction.py * rollback common/utils.py * Rollback some changes * Resolve remarks * Use exception to check result of get_token_balance_gwei * just not to loose changes * spit and polish * Update tests * Simplify airdrop processing * Spit and polish * Spit and polish * Spit and polish * Freeze changes up * Isolate errors * spit and polish * spit and polish Co-authored-by: rozhkovdmitrii --- .buildkite/steps/deploy-test.sh | 1 + proxy/common_neon/__init__.py | 0 proxy/common_neon/errors.py | 11 + proxy/common_neon/utils.py | 13 + proxy/environment.py | 4 +- proxy/plugin/solana_rest_api.py | 79 +++--- proxy/plugin/solana_rest_api_tools.py | 253 +++++++++++------- proxy/testing/__init__.py | 2 + .../testing/test_airdropping_eth_accounts.py | 103 +++++++ proxy/{ => testing}/test_cancel_hanged.py | 1 - .../test_create_account_block.py | 0 proxy/{ => testing}/test_environment.py | 0 .../test_erc20_wrapper_contract.py | 6 +- .../test_eth_getBlockByNumber.py | 0 proxy/{ => testing}/test_eth_getLogs.py | 0 .../test_eth_sendRawTransaction.py | 29 +- proxy/{ => testing}/test_neon_faucet.py | 0 proxy/{ => testing}/test_operator_spending.py | 0 .../test_resize_storage_account.py | 0 proxy/{ => testing}/test_user_stories.py | 0 proxy/testing/test_utils.py | 16 ++ .../{ => testing}/test_web3_clientVersion.py | 0 22 files changed, 360 insertions(+), 158 deletions(-) create mode 100644 proxy/common_neon/__init__.py create mode 100644 proxy/common_neon/errors.py create mode 100644 proxy/common_neon/utils.py create mode 100644 proxy/testing/test_airdropping_eth_accounts.py rename proxy/{ => testing}/test_cancel_hanged.py (99%) rename proxy/{ => testing}/test_create_account_block.py (100%) rename proxy/{ => testing}/test_environment.py (100%) rename proxy/{ => testing}/test_erc20_wrapper_contract.py (99%) rename proxy/{ => testing}/test_eth_getBlockByNumber.py (100%) rename proxy/{ => testing}/test_eth_getLogs.py (100%) rename proxy/{ => testing}/test_eth_sendRawTransaction.py (97%) rename proxy/{ => testing}/test_neon_faucet.py (100%) rename proxy/{ => testing}/test_operator_spending.py (100%) rename proxy/{ => testing}/test_resize_storage_account.py (100%) rename proxy/{ => testing}/test_user_stories.py (100%) create mode 100644 proxy/testing/test_utils.py rename proxy/{ => testing}/test_web3_clientVersion.py (100%) diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index 1ebae3767..7fc9d01ab 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -78,6 +78,7 @@ docker run --rm -ti --network=container:proxy \ -e EVM_LOADER \ -e SOLANA_URL \ -e EXTRA_GAS=100000 \ + -e NEW_USER_AIRDROP_AMOUNT=100 \ --entrypoint ./proxy/deploy-test.sh \ ${EXTRA_ARGS:-} \ $PROXY_IMAGE \ diff --git a/proxy/common_neon/__init__.py b/proxy/common_neon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/proxy/common_neon/errors.py b/proxy/common_neon/errors.py new file mode 100644 index 000000000..e590afa64 --- /dev/null +++ b/proxy/common_neon/errors.py @@ -0,0 +1,11 @@ +from enum import Enum + + +class SolanaErrors(Enum): + AccountNotFound = "Invalid param: could not find account" + + +class SolanaAccountNotFoundError(Exception): + """Provides special error processing""" + def __init__(self): + super().__init__(SolanaErrors.AccountNotFound.value) diff --git a/proxy/common_neon/utils.py b/proxy/common_neon/utils.py new file mode 100644 index 000000000..be93827e1 --- /dev/null +++ b/proxy/common_neon/utils.py @@ -0,0 +1,13 @@ +from typing import Dict, Optional, Any + + +def get_from_dict(src: Dict, *path) -> Optional[Any]: + """Provides smart getting values from python dictionary""" + val = src + for key in path: + if not isinstance(val, dict): + return None + val = val.get(key) + if val is None: + return None + return val diff --git a/proxy/environment.py b/proxy/environment.py index df86e2042..7ba268c0a 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -16,7 +16,7 @@ def call(self, *args): cmd = ["solana", "--url", solana_url, ] + list(args) - print(cmd) + logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, universal_newlines=True) except subprocess.CalledProcessError as err: import sys @@ -32,7 +32,7 @@ def call(self, *args): "--url", solana_url, "--evm_loader={}".format(evm_loader_id), ] + list(args) - print(cmd) + logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) except subprocess.CalledProcessError as err: import sys diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index ba0b1fb45..ef13a4748 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -8,9 +8,11 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import List, Tuple +from typing import List, Tuple, Optional +import copy import json import unittest +import eth_utils import rlp import solana from solana.account import Account as sol_Account @@ -20,13 +22,14 @@ from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes from .eth_proto import Trx as EthTrx -from solana.rpc.api import Client as SolanaClient +from solana.rpc.api import Client as SolanaClient, SendTransactionError as SolanaTrxError from sha3 import keccak_256 import base58 import traceback import threading -from .solana_rest_api_tools import EthereumAddress, getTokens, getAccountInfo, \ - call_signed, call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE + +from .solana_rest_api_tools import EthereumAddress, get_token_balance_or_airdrop, getAccountInfo, call_signed, \ + call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE from solana.rpc.commitment import Commitment, Confirmed from web3 import Web3 import logging @@ -34,7 +37,7 @@ import os from ..indexer.utils import get_trx_results, LogDB from ..indexer.sql_dict import SQLDict -from proxy.environment import evm_loader_id, solana_cli, solana_url, neon_cli +from ..environment import evm_loader_id, solana_cli, solana_url, neon_cli logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -46,23 +49,7 @@ class EthereumModel: def __init__(self): - # Initialize user account - res = solana_cli().call('config', 'get') - substr = "Keypair Path: " - path = "" - for line in res.splitlines(): - if line.startswith(substr): - path = line[len(substr):].strip() - if path == "": - raise Exception("cannot get keypair path") - - with open(path.strip(), mode='r') as file: - pk = (file.read()) - nums = list(map(int, pk.strip("[] \n").split(','))) - nums = nums[0:32] - values = bytes(nums) - self.signer = sol_Account(values) - + self.signer = self.get_solana_account() self.client = SolanaClient(solana_url) self.logs_db = LogDB() @@ -77,7 +64,27 @@ def __init__(self): logger.debug("worker id {}".format(self.proxy_id)) neon_config_load(self) - pass + + + @staticmethod + def get_solana_account() -> Optional[sol_Account]: + solana_account: Optional[sol_Account] = None + res = solana_cli().call('config', 'get') + substr = "Keypair Path: " + path = "" + for line in res.splitlines(): + if line.startswith(substr): + path = line[len(substr):].strip() + if path == "": + raise Exception("cannot get keypair path") + + with open(path.strip(), mode='r') as file: + pk = (file.read()) + nums = list(map(int, pk.strip("[] \n").split(','))) + nums = nums[0:32] + values = bytes(nums) + solana_account = sol_Account(values) + return solana_account def web3_clientVersion(self): neon_config_load(self) @@ -120,7 +127,6 @@ def process_block_tag(self, tag): slot = int(tag, 16) return slot - def eth_blockNumber(self): slot = self.client.get_slot(commitment=Confirmed)['result'] logger.debug("eth_blockNumber %s", hex(slot)) @@ -132,9 +138,9 @@ def eth_getBalance(self, account, tag): """ eth_acc = EthereumAddress(account) logger.debug('eth_getBalance: %s %s', account, eth_acc) - balance = getTokens(self.client, self.signer, evm_loader_id, eth_acc, self.signer.public_key()) + balance = get_token_balance_or_airdrop(self.client, self.signer, evm_loader_id, eth_acc) - return hex(balance*10**9) + return hex(balance * eth_utils.denoms.gwei) def eth_getLogs(self, obj): fromBlock = None @@ -366,7 +372,7 @@ def eth_getTransactionByHash(self, trxId, block_info = None): "s": eth_trx[8], } - logger.debug ("eth_getTransactionByHash: %s", json.dumps(ret, indent=3)) + logger.debug("eth_getTransactionByHash: %s", json.dumps(ret, indent=3)) return ret def eth_getCode(self, param, param1): @@ -455,8 +461,8 @@ def eth_sendRawTransaction(self, rawTrx): return eth_signature - except solana.rpc.api.SendTransactionError as err: - logger.debug("eth_sendRawTransaction solana.rpc.api.SendTransactionError:%s", err.result) + except SolanaTrxError as err: + self._log_transaction_error(err, logger) raise except EthereumError as err: logger.debug("eth_sendRawTransaction EthereumError:%s", err) @@ -465,6 +471,14 @@ def eth_sendRawTransaction(self, rawTrx): logger.debug("eth_sendRawTransaction type(err):%s, Exception:%s", type(err), err) raise + def _log_transaction_error(self, error: SolanaTrxError, logger): + result = copy.deepcopy(error.result) + logs = result.get("data", {}).get("logs", []) + result.get("data", {}).update({"logs": ["\n\t" + log for log in logs]}) + log_msg = str(result).replace("\\n\\t", "\n\t") + logger.error(f"Got SendTransactionError: {log_msg}") + + class JsonEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, bytearray): @@ -475,12 +489,11 @@ def default(self, obj): class SolanaContractTests(unittest.TestCase): + def setUp(self): self.model = EthereumModel() self.owner = '0xc1566af4699928fdf9be097ca3dc47ece39f8f8e' self.token1 = '0x49a449cd7fd8fbcf34d103d98f2c05245020e35b' -# self.assertEqual(self.getBalance(self.owner), 1000*10**18) -# self.assertEqual(self.getBalance(self.token1), 0) def getBalance(self, account): return int(self.model.eth_getBalance(account, 'latest'), 16) @@ -534,7 +547,6 @@ def test_transferTokens(self): self.assertTrue(receiptId in block['transactions']) - class SolanaProxyPlugin(HttpWebServerBasePlugin): """Extend in-built Web Server to add Reverse Proxy capabilities. """ @@ -571,7 +583,7 @@ def process_request(self, request): try: method = getattr(self.model, request['method']) response['result'] = method(*request['params']) - except solana.rpc.api.SendTransactionError as err: + except SolanaTrxError as err: traceback.print_exc() response['error'] = err.result except EthereumError as err: @@ -595,7 +607,6 @@ def handle_request(self, request: HttpParser) -> None: }))) return - # print('headers', request.headers) logger.debug('<<< %s 0x%x %s', threading.get_ident(), id(self.model), request.body.decode('utf8')) response = None diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 8a748d717..42752a6d7 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -9,30 +9,37 @@ import time from datetime import datetime from hashlib import sha256 -from typing import NamedTuple +from typing import NamedTuple, Optional, Union, Dict, Tuple +import psycopg2 import rlp from base58 import b58decode, b58encode from construct import Bytes, Int8ul, Int32ul, Int64ul from construct import Struct as cStruct from eth_keys import keys as eth_keys +import eth_utils + from sha3 import keccak_256 -from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT -from solana._layouts.system_instructions import InstructionType as SystemInstructionType +from web3.auto import w3 + +from solana.account import Account as SolanaAccount from solana.blockhash import Blockhash -from solana.publickey import PublicKey -from solana.rpc.api import Client, SendTransactionError +from solana.rpc.api import Client as SolanaClient, SendTransactionError from solana.rpc.commitment import Commitment, Confirmed from solana.rpc.types import TxOpts from solana.sysvar import * from solana.transaction import AccountMeta, Transaction, TransactionInstruction +from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT +from solana._layouts.system_instructions import InstructionType as SystemInstructionType + from spl.token.constants import ACCOUNT_LEN, ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address, create_associated_token_account, transfer2, Transfer2Params -from web3.auto import w3 -from proxy.environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params + +from ..environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params +from ..common_neon.utils import get_from_dict +from ..common_neon.errors import * from .eth_proto import Trx from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob from ..indexer.sql_dict import POSTGRES_USER, POSTGRES_HOST, POSTGRES_DB, POSTGRES_PASSWORD -import psycopg2 logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -59,7 +66,7 @@ incinerator = "1nc1nerator11111111111111111111111111111111" system = "11111111111111111111111111111111" -STORAGE_SIZE = 128*1024 +STORAGE_SIZE = 128 * 1024 ACCOUNT_INFO_LAYOUT = cStruct( "type" / Int8ul, @@ -193,6 +200,7 @@ def __init__(self, caller_token, eth_accounts, eth_trx): self.collateral_pool_index_buf = collateral_pool_index.to_bytes(4, 'little') self.collateral_pool_address = create_collateral_pool_address(collateral_pool_index) + class AccountInfo(NamedTuple): ether: eth_keys.PublicKey trx_count: int @@ -203,6 +211,7 @@ def frombytes(data): cont = ACCOUNT_INFO_LAYOUT.parse(data) return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account)) + def create_account_layout(lamports, space, ether, nonce): return bytes.fromhex("02000000")+CREATE_ACCOUNT_LAYOUT.build(dict( lamports=lamports, @@ -211,6 +220,7 @@ def create_account_layout(lamports, space, ether, nonce): nonce=nonce )) + def write_holder_layout(nonce, offset, data): return (bytes.fromhex('12')+ nonce.to_bytes(8, byteorder='little')+ @@ -246,11 +256,11 @@ def get_account_info(client, storage_account): def accountWithSeed(base, seed, program): - # logger.debug(type(base), str(base), type(seed), str(seed), type(program), str(program)) result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(program)).digest()) logger.debug('accountWithSeed %s', str(result)) return result + def createAccountWithSeedTrx(funding, base, seed, lamports, space, program): seed_str = str(seed, 'utf8') data = SYSTEM_INSTRUCTIONS_LAYOUT.build( @@ -393,8 +403,6 @@ def emulator(contract, sender, data, value): return neon_cli().call("emulate", sender, contract, data, value) - - def confirm_transaction(client, tx_sig, confirmations=0): """Confirm a transaction.""" TIMEOUT = 30 # 30 seconds pylint: disable=invalid-name @@ -407,27 +415,27 @@ def confirm_transaction(client, tx_sig, confirmations=0): status = resp['result']['value'][0] if status and (status['confirmationStatus'] == 'finalized' or \ status['confirmationStatus'] == 'confirmed' and status['confirmations'] >= confirmations): -# logger.debug('Confirmed transaction:', resp) return time.sleep(confirmation_check_delay) elapsed_time += confirmation_check_delay - #if not resp["result"]: raise RuntimeError("could not confirm transaction: ", tx_sig) - #return resp + def solana2ether(public_key): from web3 import Web3 return bytes(Web3.keccak(bytes.fromhex(public_key))[-20:]) + def ether2program(ether, program_id, base): if isinstance(ether, str): - if ether.startswith('0x'): - ether = ether[2:] + pass + elif isinstance(ether, EthereumAddress): + ether = str(ether) else: ether = ether.hex() output = neon_cli().call("create-program-address", ether) items = output.rstrip().split(' ') - return (items[0], int(items[1])) + return items[0], int(items[1]) def ether2seed(ether, program_id, base): @@ -437,7 +445,7 @@ def ether2seed(ether, program_id, base): seed = b58encode(bytes.fromhex(ether)) acc = accountWithSeed(base, seed, PublicKey(program_id)) logger.debug('ether2program: {} {} => {} (seed {})'.format(ether, 255, acc, seed)) - return (acc, 255, seed) + return acc, 255, seed def neon_config_load(ethereum_model): @@ -480,6 +488,7 @@ def call_emulated(contract_id, caller_id, data=None, value=None): raise Exception("evm emulator error ", result) return result + def extract_measurements_from_receipt(receipt): log_messages = receipt['result']['meta']['logMessages'] transaction = receipt['result']['transaction'] @@ -541,6 +550,7 @@ def get_measurements(result): logger.error("Can't get measurements %s"%err) logger.info("Failed result: %s"%json.dumps(result, indent=3)) + def send_transaction(client, trx, signer, eth_trx=None, reason=None): result = client.send_transaction(trx, signer, opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed)) confirm_transaction(client, result["result"]) @@ -548,11 +558,13 @@ def send_transaction(client, trx, signer, eth_trx=None, reason=None): update_transaction_cost(result, eth_trx, reason=reason) return result + def send_measured_transaction(client, trx, signer, eth_trx, reason): result = send_transaction(client, trx, signer, eth_trx=eth_trx, reason=reason) get_measurements(result) return result + def check_if_program_exceeded_instructions(err_result): err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" err_budget = "failed: Computational budget exceeded" @@ -592,6 +604,7 @@ def call_continue(signer, client, perm_accs, trx_info, steps): return sol_instr_21_cancel(signer, client, perm_accs, trx_info) + def call_continue_iterative(signer, client, perm_accs, trx_info, step_count): while True: logger.debug("Continue iterative step:") @@ -781,6 +794,7 @@ def update_transaction_cost(receipt, eth_trx, extra_sol_trx=False, reason=None): reason if reason else '' ) + def create_account_list_by_emulate(signer, client, eth_trx): sender_ether = bytes.fromhex(eth_trx.sender()) add_keys_05 = [] @@ -801,8 +815,7 @@ def create_account_list_by_emulate(signer, client, eth_trx): for acc_desc in output_json["accounts"]: if acc_desc["new"] == False: - address = bytes.fromhex(acc_desc["address"][2:]) - if acc_desc["code_size_current"] is not None and acc_desc["code_size"] is not None: + if acc_desc.get("code_size_current") is not None and acc_desc.get("code_size") is not None: if acc_desc["code_size"] > acc_desc["code_size_current"]: code_size = acc_desc["code_size"] + 2048 seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) @@ -846,7 +859,6 @@ def create_account_list_by_emulate(signer, client, eth_trx): success = True logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) break - # wait for unlock account time.sleep(1) count = count+1 @@ -867,26 +879,9 @@ def create_account_list_by_emulate(signer, client, eth_trx): code_size = acc_desc["code_size"] + 2048 code_account_balance = client.get_minimum_balance_for_rent_exemption(code_size)["result"] trx.add(createAccountWithSeedTrx(signer.public_key(), signer.public_key(), seed, code_account_balance, code_size, PublicKey(evm_loader_id))) - # add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=acc_desc["writable"])) code_account_writable = acc_desc["writable"] - (create_trx, solana_address, token_address) = createEtherAccountTrx(client, address, evm_loader_id, signer, code_account) - trx.add(create_trx) - - if address == sender_ether and NEW_USER_AIRDROP_AMOUNT > 0: - trx.add(transfer2(Transfer2Params( - amount=NEW_USER_AIRDROP_AMOUNT*1_000_000_000, - decimals=9, - dest=get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID), - mint=ETH_TOKEN_MINT_ID, - owner=signer.public_key(), - program_id=TOKEN_PROGRAM_ID, - source=getTokenAddr(signer.public_key()), - ))) - logger.debug("Token transfer to %s as ethereum 0x%s amount %s", - get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID), - acc_desc["address"], - str(NEW_USER_AIRDROP_AMOUNT)) + extend_trx_with_create_and_airdrop(signer, EthereumAddress(address), code_account, trx=trx) if address == to_address: contract_sol = PublicKey(acc_desc["account"]) @@ -914,7 +909,6 @@ def create_account_list_by_emulate(signer, client, eth_trx): if acc_desc["contract"]: add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) - for token_account in output_json["token_accounts"]: add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) @@ -936,7 +930,7 @@ def create_account_list_by_emulate(signer, client, eth_trx): trx_info = TransactionInfo(caller_token, eth_accounts, eth_trx) - return (trx_info, sender_ether, trx) + return trx_info, sender_ether, trx def call_signed(signer, client, eth_trx, steps): @@ -984,6 +978,7 @@ def call_signed(signer, client, eth_trx, steps): finally: del perm_accs + def call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx): if len(create_acc_trx.instructions): precall_txs = Transaction() @@ -1037,52 +1032,6 @@ def call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, st return call_continue(signer, client, perm_accs, trx_info, steps) -def createEtherAccountTrx(client, ether, evm_loader_id, signer, code_acc=None): - if isinstance(ether, str): - if ether.startswith('0x'): ether = ether[2:] - else: ether = ether.hex() - (sol, nonce) = ether2program(ether, evm_loader_id, signer.public_key()) - associated_token = get_associated_token_address(PublicKey(sol), ETH_TOKEN_MINT_ID) - logger.debug('createEtherAccount: {} {} => {}'.format(ether, nonce, sol)) - logger.debug('associatedTokenAccount: {}'.format(associated_token)) - base = signer.public_key() - data=bytes.fromhex('02000000')+CREATE_ACCOUNT_LAYOUT.build(dict( - lamports=0, - space=0, - ether=bytes.fromhex(ether), - nonce=nonce)) - trx = Transaction() - if code_acc is None: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(sol), is_signer=False, is_writable=True), - AccountMeta(pubkey=associated_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - else: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(sol), is_signer=False, is_writable=True), - AccountMeta(pubkey=associated_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - return (trx, sol, associated_token) - def createERC20TokenAccountTrx(signer, token_info): trx = Transaction() trx.add(TransactionInstruction( @@ -1102,7 +1051,6 @@ def createERC20TokenAccountTrx(signer, token_info): return trx - def write_trx_to_holder_account(signer, client, holder, acc_id, eth_trx): msg = eth_trx.signature() + len(eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + eth_trx.unsigned_msg() @@ -1130,6 +1078,7 @@ def write_trx_to_holder_account(signer, client, holder, acc_id, eth_trx): update_transaction_cost(result, eth_trx, reason='WriteHolder') logger.debug("confirmed: %s", rcpt) + def _getAccountData(client, account, expected_length, owner=None): info = client.get_account_info(account, commitment=Confirmed)['result']['value'] if info is None: @@ -1140,37 +1089,137 @@ def _getAccountData(client, account, expected_length, owner=None): raise Exception("Wrong data length for account data {}".format(account)) return data + def getAccountInfo(client, eth_acc, base_account): (account_sol, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader_id, base_account) info = _getAccountData(client, account_sol, ACCOUNT_INFO_LAYOUT.sizeof()) return AccountInfo.frombytes(info) + def getLamports(client, evm_loader, eth_acc, base_account): (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) return int(client.get_balance(account, commitment=Confirmed)['result']['value']) -def getTokens(client, signer, evm_loader, eth_acc, base_account): - (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) - token_account = get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) - balance = client.get_token_account_balance(token_account, commitment=Confirmed) - if 'error' in balance: - if NEW_USER_AIRDROP_AMOUNT > 0: - return NEW_USER_AIRDROP_AMOUNT * 1_000_000_000 - else: - logger.debug("'error' in balance:") - return 0 +def make_create_eth_account_trx(signer: SolanaAccount, eth_address: EthereumAddress, evm_loader_id, code_acc=None) \ + -> Tuple[Transaction, PublicKey]: + + solana_address, nonce = ether2program(eth_address, evm_loader_id, signer.public_key()) + token_acc_address = get_associated_token_address(PublicKey(solana_address), ETH_TOKEN_MINT_ID) + logger.debug(f'Create eth account: {eth_address}, sol account: {solana_address}, token_acc_address: {token_acc_address}, nonce: {nonce}') + + base = signer.public_key() + data = bytes.fromhex('02000000') + CREATE_ACCOUNT_LAYOUT.build(dict(lamports=0, + space=0, + ether=bytes(eth_address), + nonce=nonce)) + trx = Transaction() + if code_acc is None: + trx.add(TransactionInstruction( + program_id=evm_loader_id, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), + AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=system, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), + ])) + else: + trx.add(TransactionInstruction( + program_id=evm_loader_id, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), + AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), + AccountMeta(pubkey=system, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), + ])) + return trx, token_acc_address + + +def make_transfer_instruction(owner_pda_account: SolanaAccount, associated_token_account: PublicKey) -> TransactionInstruction: + owner_pda_address = owner_pda_account.public_key() + owner_associated_token_account = getTokenAddr(owner_pda_address) + transfer_instruction = transfer2(Transfer2Params(source=owner_associated_token_account, + owner=owner_pda_address, + dest=associated_token_account, + amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei, + decimals=9, + mint=ETH_TOKEN_MINT_ID, + program_id=TOKEN_PROGRAM_ID)) + logger.debug(f"Token transfer from token: {owner_associated_token_account}, owned by: {owner_pda_address}, to token: " + f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}") + return transfer_instruction + + +def extend_trx_with_create_and_airdrop(signer: SolanaAccount, eth_account: EthereumAddress, code_acc=None, *, trx): + create_trx, associated_token_account = make_create_eth_account_trx(signer, eth_account, evm_loader_id, code_acc) + trx.add(create_trx) + if NEW_USER_AIRDROP_AMOUNT <= 0: + return + transfer_instruction = make_transfer_instruction(signer, associated_token_account) + trx.add(transfer_instruction) + + +def create_eth_account_and_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress): + trx = Transaction() + extend_trx_with_create_and_airdrop(signer, eth_account, trx=trx) + result = send_transaction(client, trx, signer) + error = result.get("error") + if error is not None: + logger.error(f"Failed to create eth_account and airdrop: {eth_account}, error occurred: {error}") + raise Exception("Create eth_account error") + + +def get_token_balance_gwei(client: SolanaClient, pda_account: str) -> int: + associated_token_account = get_associated_token_address(PublicKey(pda_account), ETH_TOKEN_MINT_ID) + rpc_response = client.get_token_account_balance(associated_token_account, commitment=Confirmed) + error = rpc_response.get('error') + if error is not None: + message = error.get("message") + if message == SolanaErrors.AccountNotFound.value: + raise SolanaAccountNotFoundError() + logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, " + f"got get_token_account_balance error: \"{message}\"") + raise Exception("Getting balance error") + + balance = get_from_dict(rpc_response, "result", "value", "amount") + if balance is None: + logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, response: {rpc_response}") + raise Exception("Unexpected get_balance response") + return int(balance) + + +def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, evm_loader: str, eth_account: EthereumAddress) -> int: + associated_token_account, nonce = ether2program(bytes(eth_account).hex(), evm_loader, signer.public_key()) + logger.debug(f"Get balance for eth account: {eth_account} aka: {associated_token_account}") + + try: + return get_token_balance_gwei(client, associated_token_account) + except SolanaAccountNotFoundError: + logger.debug(f"Account not found: {eth_account} aka: {associated_token_account} - create") + create_eth_account_and_airdrop(client, signer, eth_account) + return get_token_balance_gwei(client, associated_token_account) - return int(balance['result']['value']['amount']) def getTokenAddr(account): return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) + def make_instruction_data_from_tx(instruction, private_key=None): if isinstance(instruction, dict): - if instruction['chainId'] == None: + if instruction.get('chainId') is None: raise Exception("chainId value is needed in input dict") - if private_key == None: + if private_key is None: raise Exception("Needed private key for transaction creation from fields") signed_tx = w3.eth.account.sign_transaction(instruction, private_key) diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py index 232621f0b..0bfb50319 100644 --- a/proxy/testing/__init__.py +++ b/proxy/testing/__init__.py @@ -8,3 +8,5 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import sys +sys.path.append("/spl/bin/") # TODO: get rid off this workaround all related modules should either be installed as package or be linked from submodule diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py new file mode 100644 index 000000000..c03259ccf --- /dev/null +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -0,0 +1,103 @@ +import unittest +import os +import solcx + +import eth_account +import eth_typing +import eth_utils +from eth_account.account import LocalAccount + +from web3 import Web3, eth as web3_eth +from solana.rpc.api import Client as SolanaClient + +from ..plugin.solana_rest_api import EthereumModel +from ..plugin.solana_rest_api_tools import get_token_balance_gwei, EthereumAddress, ether2program + + +class TestAirdroppingEthAccounts(unittest.TestCase): + + @classmethod + def setUpClass(cls) -> None: + cls._EVM_LOADER_ID = os.environ.get("EVM_LOADER") + new_user_airdrop_amount = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) + cls._EXPECTED_BALANCE_WEI = eth_utils.to_wei(new_user_airdrop_amount, 'ether') + cls._MINIMAL_GAS_PRICE = int(os.environ.get("MINIMAL_GAS_PRICE", 1)) * eth_utils.denoms.gwei + + proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') + cls._web3 = Web3(Web3.HTTPProvider(proxy_url)) + solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") + cls._solana_client = SolanaClient(solana_url) + cls._host_solana_account = EthereumModel.get_solana_account() + + def test_airdrop_on_get_balance(self): + account: LocalAccount = eth_account.account.Account.create() + block_number: eth_typing.BlockNumber = self._web3.eth.get_block_number() + actual_balance_wei = self._web3.eth.get_balance(account.address, block_identifier=block_number) + self.assertEqual(self._EXPECTED_BALANCE_WEI, actual_balance_wei) + + def test_airdrop_on_deploy(self): + contract_owner: LocalAccount = self._web3.eth.account.create() + contract = self._compile_and_deploy_contract(contract_owner, self._CONTRACT_STORAGE_SOURCE) + actual_balance_wei = self._get_balance_wei(contract.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, actual_balance_wei) + + def test_airdrop_onto_wrapped_new_address(self): + contract_owner: LocalAccount = self._web3.eth.account.create() + contract = self._compile_and_deploy_contract(contract_owner, self._WRAPPER_CONTRACT_STORAGE_SOURCE) + nested_contract_address = contract.functions.getNested().call() + nested_actual_balance = self._get_balance_wei(nested_contract_address) + wrapper_actual_balance = self._get_balance_wei(contract.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, wrapper_actual_balance) + self.assertEqual(self._EXPECTED_BALANCE_WEI, nested_actual_balance) + + def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str) -> web3_eth.Contract: + compiled_sol = solcx.compile_source(source) + contract_id, contract_interface = compiled_sol.popitem() + contract = self._web3.eth.contract(abi=contract_interface['abi'], bytecode=contract_interface['bin']) + nonce = self._web3.eth.get_transaction_count(contract_owner.address) + chain_id = self._web3.eth.chain_id + trx_signed = self._web3.eth.account.sign_transaction( + dict(nonce=nonce, chainId=chain_id, gas=987654321, gasPrice=self._MINIMAL_GAS_PRICE, to='', value=0, data=contract.bytecode), + contract_owner.key) + trx_hash = self._web3.eth.send_raw_transaction(trx_signed.rawTransaction) + trx_receipt = self._web3.eth.wait_for_transaction_receipt(trx_hash) + contract = self._web3.eth.contract(address=trx_receipt.contractAddress, abi=contract.abi) + return contract + + def _get_balance_wei(self, eth_acc: str) -> int: + pub_key = self._host_solana_account.public_key() + token_owner_account, nonce = ether2program(eth_acc, self._EVM_LOADER_ID, pub_key) + balance = get_token_balance_gwei(self._solana_client, token_owner_account) + self.assertIsNotNone(balance) + self.assertIsInstance(balance, int) + return balance * eth_utils.denoms.gwei + + _CONTRACT_STORAGE_SOURCE = ''' + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.7.0 <0.9.0; + contract Storage { + uint256 number; + function store(uint256 num) public { + number = num; + } + function retrieve() public view returns (uint256) { + return number; + } + } + ''' + + _WRAPPER_CONTRACT_STORAGE_SOURCE = ''' + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.7.0 <0.9.0; + contract Wrapper { + address private nested_address; + constructor() { + Nested nested = new Nested(); + nested_address = address(nested); + } + function getNested() public view returns (address) { + return nested_address; + } + } + contract Nested {} + ''' diff --git a/proxy/test_cancel_hanged.py b/proxy/testing/test_cancel_hanged.py similarity index 99% rename from proxy/test_cancel_hanged.py rename to proxy/testing/test_cancel_hanged.py index 241f15728..1e91a1381 100644 --- a/proxy/test_cancel_hanged.py +++ b/proxy/testing/test_cancel_hanged.py @@ -3,7 +3,6 @@ from proxy.plugin.solana_rest_api_tools import sysinstruct, ETH_TOKEN_MINT_ID, system, send_transaction, MINIMAL_GAS_PRICE -sys.path.append("/spl/bin/") os.environ['SOLANA_URL'] = "http://solana:8899" os.environ['EVM_LOADER'] = "53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io" os.environ['ETH_TOKEN_MINT'] = "HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU" diff --git a/proxy/test_create_account_block.py b/proxy/testing/test_create_account_block.py similarity index 100% rename from proxy/test_create_account_block.py rename to proxy/testing/test_create_account_block.py diff --git a/proxy/test_environment.py b/proxy/testing/test_environment.py similarity index 100% rename from proxy/test_environment.py rename to proxy/testing/test_environment.py diff --git a/proxy/test_erc20_wrapper_contract.py b/proxy/testing/test_erc20_wrapper_contract.py similarity index 99% rename from proxy/test_erc20_wrapper_contract.py rename to proxy/testing/test_erc20_wrapper_contract.py index de8b61ba5..75030130b 100644 --- a/proxy/test_erc20_wrapper_contract.py +++ b/proxy/testing/test_erc20_wrapper_contract.py @@ -15,7 +15,7 @@ from solana.account import Account as SolanaAccount from solana.publickey import PublicKey -from proxy.plugin.solana_rest_api_tools import createERC20TokenAccountTrx, createEtherAccountTrx +from proxy.plugin.solana_rest_api_tools import createERC20TokenAccountTrx # install_solc(version='latest') install_solc(version='0.7.6') @@ -66,7 +66,7 @@ event Transfer(address indexed from, address indexed to, uint256 value); event Approval(address indexed owner, address indexed spender, uint256 value); - + function approveSolana(bytes32 spender, uint64 value) external returns (bool); event ApprovalSolana(address indexed owner, bytes32 indexed spender, uint64 value); } @@ -147,7 +147,7 @@ def deploy_erc20_wrapper_contract(self): compiled_wrapper = compile_source(ERC20_WRAPPER_SOURCE) wrapper_id, wrapper_interface = compiled_wrapper.popitem() self.wrapper = wrapper_interface - + erc20 = proxy.eth.contract(abi=self.wrapper['abi'], bytecode=wrapper_interface['bin']) nonce = proxy.eth.get_transaction_count(proxy.eth.default_account) tx = {'nonce': nonce} diff --git a/proxy/test_eth_getBlockByNumber.py b/proxy/testing/test_eth_getBlockByNumber.py similarity index 100% rename from proxy/test_eth_getBlockByNumber.py rename to proxy/testing/test_eth_getBlockByNumber.py diff --git a/proxy/test_eth_getLogs.py b/proxy/testing/test_eth_getLogs.py similarity index 100% rename from proxy/test_eth_getLogs.py rename to proxy/testing/test_eth_getLogs.py diff --git a/proxy/test_eth_sendRawTransaction.py b/proxy/testing/test_eth_sendRawTransaction.py similarity index 97% rename from proxy/test_eth_sendRawTransaction.py rename to proxy/testing/test_eth_sendRawTransaction.py index 12e3f4633..9b6791e72 100644 --- a/proxy/test_eth_sendRawTransaction.py +++ b/proxy/testing/test_eth_sendRawTransaction.py @@ -1,5 +1,7 @@ import unittest import os + +import eth_utils from web3 import Web3 from solcx import install_solc @@ -237,8 +239,6 @@ def test_04_execute_with_bad_nonce(self): def test_05_transfer_one_gwei(self): print("\ntest_05_transfer_one_gwei") - one_gwei = 1_000_000_000 - eth_account_alice = proxy.eth.account.create('alice') eth_account_bob = proxy.eth.account.create('bob') print('eth_account_alice.address:', eth_account_alice.address) @@ -254,7 +254,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_alice.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -271,7 +271,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -285,7 +285,7 @@ def test_05_transfer_one_gwei(self): bob_balance_before_transfer = proxy.eth.get_balance(eth_account_bob.address) print('alice_balance_before_transfer:', alice_balance_before_transfer) print('bob_balance_before_transfer:', bob_balance_before_transfer) - print('one_gwei:', one_gwei) + print('one_gwei:', eth_utils.denoms.gwei) trx_transfer = proxy.eth.account.sign_transaction(dict( nonce=proxy.eth.get_transaction_count(eth_account_alice.address), @@ -293,7 +293,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account_alice.key ) @@ -307,8 +307,8 @@ def test_05_transfer_one_gwei(self): bob_balance_after_transfer = proxy.eth.get_balance(eth_account_bob.address) print('alice_balance_after_transfer:', alice_balance_after_transfer) print('bob_balance_after_transfer:', bob_balance_after_transfer) - self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - one_gwei) - self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + one_gwei) + self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - eth_utils.denoms.gwei) + self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + eth_utils.denoms.gwei) # @unittest.skip("a.i.") def test_06_transfer_one_and_a_half_gweis(self): @@ -319,8 +319,6 @@ def test_06_transfer_one_and_a_half_gweis(self): print('eth_account_alice.address:', eth_account_alice.address) print('eth_account_bob.address:', eth_account_bob.address) - one_gwei = 1_000_000_000 - if True: print("add funds to alice and bob") @@ -331,7 +329,7 @@ def test_06_transfer_one_and_a_half_gweis(self): gas=987654321, gasPrice=0, to=eth_account_alice.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -348,7 +346,7 @@ def test_06_transfer_one_and_a_half_gweis(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -386,10 +384,9 @@ def test_06_transfer_one_and_a_half_gweis(self): print('alice_balance_after_transfer:', alice_balance_after_transfer) print('bob_balance_after_transfer:', bob_balance_after_transfer) print('check https://github.com/neonlabsorg/neon-evm/issues/210') - one_gwei = 1_000_000_000 - print('one_gwei:', one_gwei) - self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - one_gwei) - self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + one_gwei) + print('one_gwei:', eth_utils.denoms.gwei) + self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - eth_utils.denoms.gwei) + self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + eth_utils.denoms.gwei) @unittest.skip("a.i.") def test_07_execute_long_transaction(self): diff --git a/proxy/test_neon_faucet.py b/proxy/testing/test_neon_faucet.py similarity index 100% rename from proxy/test_neon_faucet.py rename to proxy/testing/test_neon_faucet.py diff --git a/proxy/test_operator_spending.py b/proxy/testing/test_operator_spending.py similarity index 100% rename from proxy/test_operator_spending.py rename to proxy/testing/test_operator_spending.py diff --git a/proxy/test_resize_storage_account.py b/proxy/testing/test_resize_storage_account.py similarity index 100% rename from proxy/test_resize_storage_account.py rename to proxy/testing/test_resize_storage_account.py diff --git a/proxy/test_user_stories.py b/proxy/testing/test_user_stories.py similarity index 100% rename from proxy/test_user_stories.py rename to proxy/testing/test_user_stories.py diff --git a/proxy/testing/test_utils.py b/proxy/testing/test_utils.py new file mode 100644 index 000000000..137ffbf01 --- /dev/null +++ b/proxy/testing/test_utils.py @@ -0,0 +1,16 @@ +import unittest +from ..common_neon.utils import get_from_dict + + +class TestUtils(unittest.TestCase): + + def test_get_from_dict(self): + test_dict = {"a": {"b": {"c": 1}}} + + self.assertEqual(1, get_from_dict(test_dict, "a", "b", "c")) + self.assertEqual({"b": {"c": 1}}, get_from_dict(test_dict, "a")) + + self.assertIsNone(get_from_dict(test_dict, "b", "c", "a")) + self.assertIsNone(get_from_dict(None, "a")) + self.assertIsNone(get_from_dict(555, "a")) + self.assertIsNone(get_from_dict({}, "a")) diff --git a/proxy/test_web3_clientVersion.py b/proxy/testing/test_web3_clientVersion.py similarity index 100% rename from proxy/test_web3_clientVersion.py rename to proxy/testing/test_web3_clientVersion.py From ce05c8124692cfbb04f8c7504c98acf7e2c3db51 Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Fri, 19 Nov 2021 16:58:35 +0500 Subject: [PATCH 33/54] #305 Remove extra args from ether2program (#309) #305 Remove extra args from ether2program --- proxy/plugin/solana_rest_api.py | 4 ++-- proxy/plugin/solana_rest_api_tools.py | 19 ++++++++++--------- .../testing/test_airdropping_eth_accounts.py | 3 +-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index ef13a4748..53f7d217a 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -138,7 +138,7 @@ def eth_getBalance(self, account, tag): """ eth_acc = EthereumAddress(account) logger.debug('eth_getBalance: %s %s', account, eth_acc) - balance = get_token_balance_or_airdrop(self.client, self.signer, evm_loader_id, eth_acc) + balance = get_token_balance_or_airdrop(self.client, self.signer, eth_acc) return hex(balance * eth_utils.denoms.gwei) @@ -272,7 +272,7 @@ def eth_call(self, obj, tag): def eth_getTransactionCount(self, account, tag): logger.debug('eth_getTransactionCount: %s', account) try: - acc_info = getAccountInfo(self.client, EthereumAddress(account), self.signer.public_key()) + acc_info = getAccountInfo(self.client, EthereumAddress(account)) return hex(int.from_bytes(acc_info.trx_count, 'little')) except Exception as err: print("Can't get account info: %s"%err) diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 42752a6d7..e20fdc923 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -228,6 +228,7 @@ def write_holder_layout(nonce, offset, data): len(data).to_bytes(8, byteorder='little')+ data) + def get_account_info(client, storage_account): opts = { "encoding": "base64", @@ -426,7 +427,7 @@ def solana2ether(public_key): return bytes(Web3.keccak(bytes.fromhex(public_key))[-20:]) -def ether2program(ether, program_id, base): +def ether2program(ether): if isinstance(ether, str): pass elif isinstance(ether, EthereumAddress): @@ -1090,21 +1091,21 @@ def _getAccountData(client, account, expected_length, owner=None): return data -def getAccountInfo(client, eth_acc, base_account): - (account_sol, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader_id, base_account) +def getAccountInfo(client, eth_account: EthereumAddress): + account_sol, nonce = ether2program(eth_account) info = _getAccountData(client, account_sol, ACCOUNT_INFO_LAYOUT.sizeof()) return AccountInfo.frombytes(info) -def getLamports(client, evm_loader, eth_acc, base_account): - (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) - return int(client.get_balance(account, commitment=Confirmed)['result']['value']) +def getLamports(client, eth_account): + pda_account, nonce = ether2program(eth_account) + return int(client.get_balance(pda_account, commitment=Confirmed)['result']['value']) def make_create_eth_account_trx(signer: SolanaAccount, eth_address: EthereumAddress, evm_loader_id, code_acc=None) \ -> Tuple[Transaction, PublicKey]: - solana_address, nonce = ether2program(eth_address, evm_loader_id, signer.public_key()) + solana_address, nonce = ether2program(eth_address) token_acc_address = get_associated_token_address(PublicKey(solana_address), ETH_TOKEN_MINT_ID) logger.debug(f'Create eth account: {eth_address}, sol account: {solana_address}, token_acc_address: {token_acc_address}, nonce: {nonce}') @@ -1199,8 +1200,8 @@ def get_token_balance_gwei(client: SolanaClient, pda_account: str) -> int: return int(balance) -def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, evm_loader: str, eth_account: EthereumAddress) -> int: - associated_token_account, nonce = ether2program(bytes(eth_account).hex(), evm_loader, signer.public_key()) +def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress) -> int: + associated_token_account, nonce = ether2program(eth_account) logger.debug(f"Get balance for eth account: {eth_account} aka: {associated_token_account}") try: diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py index c03259ccf..ea9692586 100644 --- a/proxy/testing/test_airdropping_eth_accounts.py +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -65,8 +65,7 @@ def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str return contract def _get_balance_wei(self, eth_acc: str) -> int: - pub_key = self._host_solana_account.public_key() - token_owner_account, nonce = ether2program(eth_acc, self._EVM_LOADER_ID, pub_key) + token_owner_account, nonce = ether2program(eth_acc) balance = get_token_balance_gwei(self._solana_client, token_owner_account) self.assertIsNotNone(balance) self.assertIsInstance(balance, int) From 6ca860aafb050094b31cc7459cdb008f98ae7d69 Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Tue, 23 Nov 2021 21:58:49 +0500 Subject: [PATCH 34/54] Change default logDB postgress pass (#317) Co-authored-by: rozhkovdmitrii --- proxy/indexer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index 7b45f725e..49d129795 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -175,7 +175,7 @@ class LogDB: def __init__(self): POSTGRES_DB = os.environ.get("POSTGRES_DB", "neon-db") POSTGRES_USER = os.environ.get("POSTGRES_USER", "neon-proxy") - POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "neon-proxy") + POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "neon-proxy-pass") POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost") self.conn = psycopg2.connect( From b3b322f036713b8db2661b4b3e2d6ff9f5527048 Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Wed, 24 Nov 2021 10:33:13 +0500 Subject: [PATCH 35/54] #311 Create account and airdrop on gas estimation if it's preset Implement creating account on getting balance and airdrop Co-authored-by: rozhkovdmitrii --- proxy/plugin/solana_rest_api.py | 15 +++++------- proxy/plugin/solana_rest_api_tools.py | 23 +++++++++++++++++++ .../testing/test_airdropping_eth_accounts.py | 20 ++++++++++++---- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 53f7d217a..2c1a44dcc 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -29,12 +29,11 @@ import threading from .solana_rest_api_tools import EthereumAddress, get_token_balance_or_airdrop, getAccountInfo, call_signed, \ - call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE + call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE, estimate_gas from solana.rpc.commitment import Commitment, Confirmed from web3 import Web3 import logging from ..core.acceptor.pool import proxy_id_glob -import os from ..indexer.utils import get_trx_results, LogDB from ..indexer.sql_dict import SQLDict from ..environment import evm_loader_id, solana_cli, solana_url, neon_cli @@ -45,7 +44,6 @@ modelInstanceLock = threading.Lock() modelInstance = None -EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) class EthereumModel: def __init__(self): @@ -105,12 +103,11 @@ def eth_gasPrice(self): def eth_estimateGas(self, param): try: - caller_id = param['from'] if 'from' in param else "0x0000000000000000000000000000000000000000" - contract_id = param['to'] if 'to' in param else "deploy" - data = param['data'] if 'data' in param else "None" - value = param['value'] if 'value' in param else "" - result = call_emulated(contract_id, caller_id, data, value) - return result['used_gas']+EXTRA_GAS + caller_id = param.get('from', "0x0000000000000000000000000000000000000000") + contract_id = param.get('to', "deploy") + data = param.get('data', "None") + value = param.get('value', "") + return estimate_gas(self.client, self.signer, contract_id, EthereumAddress(caller_id), data, value) except Exception as err: logger.debug("Exception on eth_estimateGas: %s", err) raise diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index e20fdc923..26a98068c 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -100,6 +100,9 @@ AccountMeta(pubkey=sysvarclock, is_signer=False, is_writable=False), ] +EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) + + class SQLCost(): def __init__(self): @@ -1253,3 +1256,23 @@ def make_instruction_data_from_tx(instruction, private_key=None): return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) else: raise Exception("function gets ") + + +def is_account_exists(client: SolanaClient, eth_account: EthereumAddress) -> bool: + pda_account, nonce = ether2program(eth_account) + info = client.get_account_info(pda_account, commitment=Confirmed) + value = get_from_dict(info, "result", "value") + return value is not None + + +def estimate_gas(client: SolanaClient, signer: SolanaAccount, contract_id: str, caller_eth_account: EthereumAddress, + data: str = None, value: str = None): + if not is_account_exists(client, caller_eth_account): + create_eth_account_and_airdrop(client, signer, caller_eth_account) + result = call_emulated(contract_id, str(caller_eth_account), data, value) + used_gas = result.get("used_gas") + if used_gas is None: + logger.error(f"Failed estimate_gas, unexpected result, by contract_id: {contract_id}, caller_eth_account: " + f"{caller_eth_account}, data: {data}, value: {value}, emulation result: {result}") + raise Exception("Bad estimate_gas result") + return used_gas + EXTRA_GAS diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py index ea9692586..9fc778585 100644 --- a/proxy/testing/test_airdropping_eth_accounts.py +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -11,7 +11,7 @@ from solana.rpc.api import Client as SolanaClient from ..plugin.solana_rest_api import EthereumModel -from ..plugin.solana_rest_api_tools import get_token_balance_gwei, EthereumAddress, ether2program +from ..plugin.solana_rest_api_tools import get_token_balance_gwei, ether2program class TestAirdroppingEthAccounts(unittest.TestCase): @@ -50,9 +50,19 @@ def test_airdrop_onto_wrapped_new_address(self): self.assertEqual(self._EXPECTED_BALANCE_WEI, wrapper_actual_balance) self.assertEqual(self._EXPECTED_BALANCE_WEI, nested_actual_balance) + def test_airdrop_on_deploy_estimation(self): + owner_eth_account: LocalAccount = self._web3.eth.account.create() + compile_result = solcx.compile_source(self._CONTRACT_STORAGE_SOURCE) + _, contract_interface = compile_result.popitem() + contract_data = contract_interface.get("bin") + self.assertIsNotNone(contract_data) + self._web3.eth.estimate_gas({"from": owner_eth_account.address, "data": contract_data}) + owner_balance = self._get_balance_wei(owner_eth_account.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, owner_balance) + def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str) -> web3_eth.Contract: - compiled_sol = solcx.compile_source(source) - contract_id, contract_interface = compiled_sol.popitem() + compile_result = solcx.compile_source(source) + contract_id, contract_interface = compile_result.popitem() contract = self._web3.eth.contract(abi=contract_interface['abi'], bytecode=contract_interface['bin']) nonce = self._web3.eth.get_transaction_count(contract_owner.address) chain_id = self._web3.eth.chain_id @@ -64,8 +74,8 @@ def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str contract = self._web3.eth.contract(address=trx_receipt.contractAddress, abi=contract.abi) return contract - def _get_balance_wei(self, eth_acc: str) -> int: - token_owner_account, nonce = ether2program(eth_acc) + def _get_balance_wei(self, eth_account: str) -> int: + token_owner_account, nonce = ether2program(eth_account) balance = get_token_balance_gwei(self._solana_client, token_owner_account) self.assertIsNotNone(balance) self.assertIsInstance(balance, int) From c860ed01244076c45ef26cc2d136f11bca7b8c35 Mon Sep 17 00:00:00 2001 From: sinev-valentine <37595780+sinev-valentine@users.noreply.github.com> Date: Wed, 24 Nov 2021 15:41:08 +0300 Subject: [PATCH 36/54] neonbals/neon-evm#371 Update STORAGE_ACCOUNT_INFO_LAYOUT (#315) --- proxy/indexer/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index 49d129795..c83b38588 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -128,6 +128,8 @@ def get_trx_receipts(unsigned_msg, signature): "evm_data_size" / Int64ul, "gas_used_and_paid" / Int64ul, "number_of_payments" / Int64ul, + "sign" / Bytes(65), + ) def get_account_list(client, storage_account): From 9bb3f89bc7917fa2b59c799d0e23ec6340260c5d Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Wed, 24 Nov 2021 18:04:11 +0300 Subject: [PATCH 37/54] 319 add neon cli version handler (#325) Neon-proxy returns the neon-cli version for the "neon_cli_version" (like "web3_clientVersion") request in format Neon-cli/v- For example: Neon-cli/v0.4.0-rc0-abc26675a5f979eeeeff3c886ebf4fcd310ce064 --- proxy/environment.py | 12 ++++++-- proxy/plugin/solana_rest_api.py | 3 ++ proxy/testing/test_neon_cli_version.py | 39 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 proxy/testing/test_neon_cli_version.py diff --git a/proxy/environment.py b/proxy/environment.py index 7ba268c0a..b06465798 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -19,7 +19,6 @@ def call(self, *args): logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, universal_newlines=True) except subprocess.CalledProcessError as err: - import sys logger.debug("ERR: solana error {}".format(err)) raise @@ -35,7 +34,16 @@ def call(self, *args): logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) except subprocess.CalledProcessError as err: - import sys + logger.debug("ERR: neon-cli error {}".format(err)) + raise + + def version(self): + try: + cmd = ["neon-cli", + "--version"] + logger.debug("Calling: " + " ".join(cmd)) + return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True).split()[1] + except subprocess.CalledProcessError as err: logger.debug("ERR: neon-cli error {}".format(err)) raise diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 2c1a44dcc..53655b7cf 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -93,6 +93,9 @@ def eth_chainId(self): # NEON_CHAIN_ID is a string in decimal form return hex(int(self.neon_config_dict['NEON_CHAIN_ID'])) + def neon_cli_version(self): + return neon_cli().version() + def net_version(self): neon_config_load(self) # NEON_CHAIN_ID is a string in decimal form diff --git a/proxy/testing/test_neon_cli_version.py b/proxy/testing/test_neon_cli_version.py new file mode 100644 index 000000000..66b82d68e --- /dev/null +++ b/proxy/testing/test_neon_cli_version.py @@ -0,0 +1,39 @@ +import unittest +import os +import requests +import json +import inspect + +from proxy.environment import neon_cli + +proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') +headers = {'Content-type': 'application/json'} + + +def get_line_number(): + cf = inspect.currentframe() + return cf.f_back.f_lineno + + +class TestNeonProxyVersion(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + def test_01_neon_cli_version(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/319") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "neon_cli_version", + "params": [] + })).text) + print('response:', response) + neon_cli_version = response['result'] + print('neon_cli_version:', neon_cli_version) + self.assertEqual(neon_cli_version, neon_cli().version()) + + +if __name__ == '__main__': + unittest.main() From 081bc7597b7836ddd071f6153b1aee947599f23e Mon Sep 17 00:00:00 2001 From: sinev-valentine <37595780+sinev-valentine@users.noreply.github.com> Date: Wed, 24 Nov 2021 19:29:02 +0300 Subject: [PATCH 38/54] fix storage account check (#327) --- proxy/plugin/solana_rest_api_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 26a98068c..29207a332 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -58,6 +58,7 @@ COLLATERALL_POOL_MAX=10 EMPTY_STORAGE_TAG=0 +FINALIZED_STORAGE_TAG=5 sysvarclock = "SysvarC1ock11111111111111111111111111111111" sysinstruct = "Sysvar1nstructions1111111111111111111111111" @@ -334,8 +335,8 @@ def create_multiple_accounts_with_seed(client, funding, base, seeds, sizes): raise Exception("insufficient balance") if PublicKey(owner) != PublicKey(evm_loader_id): raise Exception("wrong owner") - if tag != EMPTY_STORAGE_TAG: - raise Exception("not empty") + if tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}: + raise Exception("not empty, not finalized") if len(trx.instructions) > 0: send_transaction(client, trx, funding) From 7845ca95ab935ff6522689477abbff26bbb81ab4 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Thu, 25 Nov 2021 08:06:48 +0300 Subject: [PATCH 39/54] #320 add neon proxy version handler (#323) * Introduce neon_proxy_version handler * Introduce test_01_neon_proxy_version * Use proxy/plugin/solana_rest_api.py * Use /opt/proxy/plugin/solana_rest_api.py * Add echo PROXY_REVISION to check * set ${PROXY_REVISION:=${REVISION}} * Debugging * Debugging * Do the code clearly * Set setUpClass empty --- .buildkite/steps/build-image.sh | 1 + Dockerfile | 2 ++ proxy/plugin/solana_rest_api.py | 5 +++ proxy/testing/test_neon_proxy_version.py | 39 ++++++++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 proxy/testing/test_neon_proxy_version.py diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index d8becc316..03cd1dad0 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -15,4 +15,5 @@ docker pull neonlabsorg/evm_loader:${EVM_LOADER_REVISION} docker build -t neonlabsorg/proxy:${REVISION} \ --build-arg SOLANA_REVISION=${SOLANA_REVISION} \ --build-arg EVM_LOADER_REVISION=${EVM_LOADER_REVISION} \ + --build-arg PROXY_REVISION=${REVISION} \ . diff --git a/Dockerfile b/Dockerfile index ce1d31997..6837342c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli FROM neonlabsorg/evm_loader:${EVM_LOADER_REVISION} AS spl FROM ubuntu:20.04 +ARG PROXY_REVISION RUN apt update && \ DEBIAN_FRONTEND=noninteractive apt -y install \ @@ -41,6 +42,7 @@ COPY --from=spl /opt/solana_utils.py \ COPY --from=spl /opt/neon-cli /spl/bin/emulator COPY . /opt +RUN sed -i 's/NEON_PROXY_REVISION_TO_BE_REPLACED/'"$PROXY_REVISION"'/g' /opt/proxy/plugin/solana_rest_api.py COPY proxy/operator-keypair.json /root/.config/solana/id.json RUN cd /usr/local/lib/python3.8/dist-packages/ && patch -p0 Optional[sol_Account]: solana_account = sol_Account(values) return solana_account + def neon_proxy_version(self): + return 'Neon-proxy/v' + NEON_PROXY_PKG_VERSION + '-' + NEON_PROXY_REVISION + def web3_clientVersion(self): neon_config_load(self) return self.neon_config_dict['web3_clientVersion'] diff --git a/proxy/testing/test_neon_proxy_version.py b/proxy/testing/test_neon_proxy_version.py new file mode 100644 index 000000000..c5ed29e40 --- /dev/null +++ b/proxy/testing/test_neon_proxy_version.py @@ -0,0 +1,39 @@ +import unittest +import os +import requests +import json +import inspect + +from proxy.plugin.solana_rest_api import NEON_PROXY_PKG_VERSION, NEON_PROXY_REVISION + +proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') +headers = {'Content-type': 'application/json'} + + +def get_line_number(): + cf = inspect.currentframe() + return cf.f_back.f_lineno + + +class TestNeonProxyVersion(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + def test_01_neon_proxy_version(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/320") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "neon_proxy_version", + "params": [] + })).text) + print('response:', response) + neon_proxy_version = response['result'] + print('neon_proxy_version:', neon_proxy_version) + self.assertEqual(neon_proxy_version, 'Neon-proxy/v' + NEON_PROXY_PKG_VERSION + '-' + NEON_PROXY_REVISION) + + +if __name__ == '__main__': + unittest.main() From f299c57d80123ec6cbced5e84e1ff2caa04239ae Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 25 Nov 2021 12:22:50 +0300 Subject: [PATCH 40/54] 313 concurrent execution of solana program dump (#314) * fix * get rid of old-style neon-elf-params * Get rid of dump-files * Specify solana url explicitely Co-authored-by: ivanl --- proxy/deploy-test.sh | 3 +-- proxy/environment.py | 11 +---------- proxy/plugin/solana_rest_api.py | 6 +++++- proxy/run-test-proxy.sh | 3 +-- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/proxy/deploy-test.sh b/proxy/deploy-test.sh index a0df058f2..3ebbfda0f 100755 --- a/proxy/deploy-test.sh +++ b/proxy/deploy-test.sh @@ -5,8 +5,7 @@ echo "Deploy test..." solana config set -u $SOLANA_URL solana address || solana-keygen new --no-passphrase -solana program dump "$EVM_LOADER" ./evm_loader.dump -export $(/spl/bin/neon-cli --evm_loader "$EVM_LOADER" neon-elf-params ./evm_loader.dump) +export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) curl -v --header "Content-Type: application/json" --data '{"method":"eth_blockNumber","id":1,"jsonrpc":"2.0","params":[]}' $PROXY_URL diff --git a/proxy/environment.py b/proxy/environment.py index b06465798..794f099a8 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -48,16 +48,7 @@ def version(self): raise def read_elf_params(out_dict): - logger.debug('load for solana_url={} and evm_loader_id={}'.format(solana_url, evm_loader_id)) - res = solana_cli().call('program', 'dump', evm_loader_id, './evm_loader.dump') - substr = "Wrote program to " - path = "" - for line in res.splitlines(): - if line.startswith(substr): - path = line[len(substr):].strip() - if path == "": - raise Exception("cannot program dump for ", evm_loader_id) - for param in neon_cli().call("neon-elf-params", path).splitlines(): + for param in neon_cli().call("neon-elf-params").splitlines(): if param.startswith('NEON_') and '=' in param: v = param.split('=') out_dict[v[0]] = v[1] diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 9597efc0b..0372daa8d 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -128,8 +128,12 @@ def process_block_tag(self, tag): slot = int(self.client.get_slot(commitment=Confirmed)["result"]) elif tag in ('earliest', 'pending'): raise Exception("Invalid tag {}".format(tag)) - else: + elif isinstance(tag, str): slot = int(tag, 16) + elif isinstance(tag, int): + slot = tag + else: + raise Exception(f'Failed to parse block tag: {tag}') return slot def eth_blockNumber(self): diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index 5df1ad4ab..4b9af9199 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -9,8 +9,7 @@ solana config set -u $SOLANA_URL echo "Dumping evm_loader and extracting ELF parameters" export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) -solana program dump "$EVM_LOADER" ./evm_loader.dump -export $(/spl/bin/neon-cli --evm_loader="$EVM_LOADER" neon-elf-params ./evm_loader.dump) +export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader="$EVM_LOADER" neon-elf-params) /spl/bin/create-test-accounts.sh 1 From 0a5dc2dbaa40f235c57706ecd588956f2c755cb1 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 25 Nov 2021 15:43:09 +0300 Subject: [PATCH 41/54] fix scripts (#329) Co-authored-by: ivanl --- proxy/deploy-test.sh | 2 +- proxy/run-test-proxy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/deploy-test.sh b/proxy/deploy-test.sh index 3ebbfda0f..b47dc651f 100755 --- a/proxy/deploy-test.sh +++ b/proxy/deploy-test.sh @@ -5,7 +5,7 @@ echo "Deploy test..." solana config set -u $SOLANA_URL solana address || solana-keygen new --no-passphrase -export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) +export $(/spl/bin/neon-cli --commitment confirmed --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) curl -v --header "Content-Type: application/json" --data '{"method":"eth_blockNumber","id":1,"jsonrpc":"2.0","params":[]}' $PROXY_URL diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index 4b9af9199..fdb38c24b 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -9,7 +9,7 @@ solana config set -u $SOLANA_URL echo "Dumping evm_loader and extracting ELF parameters" export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) -export $(/spl/bin/neon-cli --url $SOLANA_URL --evm_loader="$EVM_LOADER" neon-elf-params) +export $(/spl/bin/neon-cli --commitment confirmed --url $SOLANA_URL --evm_loader="$EVM_LOADER" neon-elf-params) /spl/bin/create-test-accounts.sh 1 From 7964cadda144b71187ccd3a3f57bae762e493dd3 Mon Sep 17 00:00:00 2001 From: mich-master <86723787+mich-master@users.noreply.github.com> Date: Fri, 26 Nov 2021 18:24:16 +0300 Subject: [PATCH 42/54] #318 JSON_RPC "params" field may be omitted (#322) * #318 JSON-RPC "params " field may be omitted --- proxy/plugin/solana_rest_api.py | 3 ++- proxy/testing/test_user_stories.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 0372daa8d..f84a80c6d 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -591,7 +591,8 @@ def process_request(self, request): } try: method = getattr(self.model, request['method']) - response['result'] = method(*request['params']) + params = request.get('params', []) + response['result'] = method(*params) except SolanaTrxError as err: traceback.print_exc() response['error'] = err.result diff --git a/proxy/testing/test_user_stories.py b/proxy/testing/test_user_stories.py index 45b837223..e6001740e 100644 --- a/proxy/testing/test_user_stories.py +++ b/proxy/testing/test_user_stories.py @@ -91,6 +91,18 @@ def test_04_check_eth_estimateGas_on_deploying_a_contract_with_the_empty_data_an print('used_gas:', used_gas) self.assertEqual(used_gas, 53001 + EXTRA_GAS) + def test_05_check_params_omitted(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/318") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "eth_chainId" + })).text) + print('response:', response) + chain_id = int(response['result'], 0) + print('chain_id:', chain_id) + self.assertEqual(chain_id, 111) if __name__ == '__main__': unittest.main() From 01df8a4c9175c1ffc592de6cd916cc9a7f8fa890 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Thu, 2 Dec 2021 08:30:51 +0300 Subject: [PATCH 43/54] #336 indexer refactoring (#340) * cherrypick part of changes * create indexer.py * remove solana_receipts_update.py * Fix inspection issues * fix last issue Co-authored-by: ivanl --- .buildkite/steps/deploy-test.sh | 4 + proxy/docker-compose-test.yml | 2 +- .../{solana_receipts_update.py => indexer.py} | 167 ++++-------------- proxy/indexer/indexer_base.py | 149 ++++++++++++++++ proxy/proxy.py | 7 +- 5 files changed, 196 insertions(+), 133 deletions(-) rename proxy/indexer/{solana_receipts_update.py => indexer.py} (84%) create mode 100644 proxy/indexer/indexer_base.py diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index 7fc9d01ab..85ee0f509 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -79,6 +79,10 @@ docker run --rm -ti --network=container:proxy \ -e SOLANA_URL \ -e EXTRA_GAS=100000 \ -e NEW_USER_AIRDROP_AMOUNT=100 \ + -e POSTGRES_DB=neon-db \ + -e POSTGRES_USER=neon-proxy \ + -e POSTGRES_PASSWORD=neon-proxy-pass \ + -e POSTGRES_HOST=postgres \ --entrypoint ./proxy/deploy-test.sh \ ${EXTRA_ARGS:-} \ $PROXY_IMAGE \ diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index d40820a89..3d98cb7ef 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -46,7 +46,7 @@ services: POSTGRES_PASSWORD: neon-proxy-pass hostname: postgres healthcheck: - test: [ CMD-SHELL, "pg_isready" ] + test: [ CMD-SHELL, "pg_isready -h postgres -p 5432" ] interval: 5s timeout: 10s retries: 10 diff --git a/proxy/indexer/solana_receipts_update.py b/proxy/indexer/indexer.py similarity index 84% rename from proxy/indexer/solana_receipts_update.py rename to proxy/indexer/indexer.py index 2bc67b1e2..9ca772648 100644 --- a/proxy/indexer/solana_receipts_update.py +++ b/proxy/indexer/indexer.py @@ -1,13 +1,11 @@ +from proxy.indexer.indexer_base import logger, IndexerBase, PARALLEL_REQUESTS import base58 import rlp import json import os import time import logging -from solana.rpc.api import Client from multiprocessing.dummy import Pool as ThreadPool -from typing import Dict, Union -from proxy.environment import solana_url, evm_loader_id try: @@ -17,16 +15,7 @@ from .utils import check_error, get_trx_results, get_trx_receipts, LogDB, Canceller from .sql_dict import SQLDict - -PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2")) CANCEL_TIMEOUT = int(os.environ.get("CANCEL_TIMEOUT", "60")) - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -DEVNET_HISTORY_START = "7BdwyUQ61RUZP63HABJkbW66beLk22tdXnP69KsvQBJekCPVaHoJY47Rw68b3VV1UbQNHxX3uxUSLfiJrfy2bTn" -HISTORY_START = [DEVNET_HISTORY_START] - UPDATE_BLOCK_COUNT = PARALLEL_REQUESTS * 16 class HolderStruct: @@ -57,131 +46,34 @@ def __init__(self, eth_trx, eth_signature, from_address, got_result, signatures, self.slot = slot -class Indexer: - def __init__(self): - self.client = Client(solana_url) +class Indexer(IndexerBase): + def __init__(self, + solana_url, + evm_loader_id, + log_level = 'INFO'): + IndexerBase.__init__(self, solana_url, evm_loader_id, log_level) + self.canceller = Canceller() self.logs_db = LogDB() self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash") - self.transaction_receipts = SQLDict(tablename="known_transactions") self.ethereum_trx = SQLDict(tablename="ethereum_transactions") self.eth_sol_trx = SQLDict(tablename="ethereum_solana_transactions") self.sol_eth_trx = SQLDict(tablename="solana_ethereum_transactions") self.constants = SQLDict(tablename="constants") - self.last_slot = 0 - self.current_slot = 0 - self.transaction_order = [] if 'last_block' not in self.constants: self.constants['last_block'] = 0 self.blocked_storages = {} - self.counter_ = 0 - - def run(self, loop = True): - while (True): - try: - logger.debug("Start indexing") - self.gather_unknown_transactions() - logger.debug("Process receipts") - self.process_receipts() - logger.debug("Start getting blocks") - self.gather_blocks() - logger.debug("Unlock accounts") - self.canceller.unlock_accounts(self.blocked_storages) - self.blocked_storages = {} - except Exception as err: - logger.debug("Got exception while indexing. Type(err):%s, Exception:%s", type(err), err) - - - def gather_unknown_transactions(self): - poll_txs = set() - ordered_txs = [] - - minimal_tx = None - continue_flag = True - current_slot = self.client.get_slot(commitment="confirmed")["result"] - maximum_slot = self.last_slot - minimal_slot = current_slot - - percent = 0 - - counter = 0 - while (continue_flag): - opts: Dict[str, Union[int, str]] = {} - if minimal_tx: - opts["before"] = minimal_tx - opts["commitment"] = "confirmed" - result = self.client._provider.make_request("getSignaturesForAddress", evm_loader_id, opts) - logger.debug("{:>3} get_signatures_for_address {}".format(counter, len(result["result"]))) - counter += 1 - - if len(result["result"]) == 0: - logger.debug("len(result['result']) == 0") - break - - for tx in result["result"]: - solana_signature = tx["signature"] - slot = tx["slot"] - - if solana_signature in HISTORY_START: - logger.debug(solana_signature) - continue_flag = False - break - - ordered_txs.append(solana_signature) - - if solana_signature not in self.transaction_receipts: - poll_txs.add(solana_signature) - - if slot < minimal_slot: - minimal_slot = slot - minimal_tx = solana_signature - - if slot > maximum_slot: - maximum_slot = slot - if slot < self.last_slot: - continue_flag = False - break - logger.debug("start getting receipts") - pool = ThreadPool(PARALLEL_REQUESTS) - pool.map(self.get_tx_receipts, poll_txs) - - if len(self.transaction_order): - index = 0 - try: - index = ordered_txs.index(self.transaction_order[0]) - except ValueError: - self.transaction_order = ordered_txs + self.transaction_order - else: - self.transaction_order = ordered_txs[:index] + self.transaction_order - else: - self.transaction_order = ordered_txs - - self.last_slot = maximum_slot - self.current_slot = current_slot - - self.counter_ = 0 - - - def get_tx_receipts(self, solana_signature): - # trx = None - retry = True - - while retry: - try: - trx = self.client.get_confirmed_transaction(solana_signature)['result'] - self.transaction_receipts[solana_signature] = trx - retry = False - except Exception as err: - logger.debug(err) - time.sleep(1) - - self.counter_ += 1 - if self.counter_ % 100 == 0: - logger.debug(self.counter_) - - # return (solana_signature, trx) + def process_functions(self): + IndexerBase.process_functions(self) + logger.debug("Process receipts") + self.process_receipts() + logger.debug("Start getting blocks") + self.gather_blocks() + logger.debug("Unlock accounts") + self.canceller.unlock_accounts(self.blocked_storages) + self.blocked_storages = {} def process_receipts(self): @@ -209,7 +101,7 @@ def process_receipts(self): if trx['transaction']['message']['instructions'] is not None: for instruction in trx['transaction']['message']['instructions']: - if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] != evm_loader_id: + if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] != self.evm_loader_id: continue if check_error(trx): @@ -574,12 +466,27 @@ def get_block(self, slot): return (slot, block_hash) -def run_indexer(): +def run_indexer(solana_url, + evm_loader_id, + log_level = 'DEBUG'): logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s') logger.setLevel(logging.DEBUG) - indexer = Indexer() - indexer.run(False) + logger.info(f"""Running indexer with params: + solana_url: {solana_url}, + evm_loader_id: {evm_loader_id}, + log_level: {log_level}""") + + indexer = Indexer(solana_url, + evm_loader_id, + log_level) + indexer.run() if __name__ == "__main__": - run_indexer() + solana_url = os.environ.get('SOLANA_URL', 'http://localhost:8899') + evm_loader_id = os.environ.get('EVM_LOADER_ID', '53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io') + log_level = os.environ.get('LOG_LEVEL', 'INFO') + + run_indexer(solana_url, + evm_loader_id, + log_level) diff --git a/proxy/indexer/indexer_base.py b/proxy/indexer/indexer_base.py new file mode 100644 index 000000000..cda952bbf --- /dev/null +++ b/proxy/indexer/indexer_base.py @@ -0,0 +1,149 @@ +import os +import time +import logging +from solana.rpc.api import Client +from multiprocessing.dummy import Pool as ThreadPool +from typing import Dict, Union + +try: + from sql_dict import SQLDict +except ImportError: + from .sql_dict import SQLDict + + +PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2")) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +DEVNET_HISTORY_START = "7BdwyUQ61RUZP63HABJkbW66beLk22tdXnP69KsvQBJekCPVaHoJY47Rw68b3VV1UbQNHxX3uxUSLfiJrfy2bTn" +HISTORY_START = [DEVNET_HISTORY_START] + + +log_levels = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARN': logging.WARN, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'FATAL': logging.FATAL, + 'CRITICAL': logging.CRITICAL +} + +class IndexerBase: + def __init__(self, + solana_url, + evm_loader_id, + log_level): + logger.setLevel(log_levels.get(log_level, logging.INFO)) + + self.evm_loader_id = evm_loader_id + self.client = Client(solana_url) + self.transaction_receipts = SQLDict(tablename="known_transactions") + self.last_slot = 0 + self.current_slot = 0 + self.transaction_order = [] + self.counter_ = 0 + + + def run(self): + while (True): + try: + self.process_functions() + except Exception as err: + logger.warning("Got exception while indexing. Type(err):%s, Exception:%s", type(err), err) + + + def process_functions(self): + logger.debug("Start indexing") + self.gather_unknown_transactions() + + + def gather_unknown_transactions(self): + poll_txs = set() + ordered_txs = [] + + minimal_tx = None + continue_flag = True + current_slot = self.client.get_slot(commitment="confirmed")["result"] + maximum_slot = self.last_slot + minimal_slot = current_slot + + counter = 0 + while (continue_flag): + opts: Dict[str, Union[int, str]] = {} + if minimal_tx: + opts["before"] = minimal_tx + opts["commitment"] = "confirmed" + result = self.client._provider.make_request("getSignaturesForAddress", self.evm_loader_id, opts) + logger.debug("{:>3} get_signatures_for_address {}".format(counter, len(result["result"]))) + counter += 1 + + if len(result["result"]) == 0: + logger.debug("len(result['result']) == 0") + break + + for tx in result["result"]: + solana_signature = tx["signature"] + slot = tx["slot"] + + if solana_signature in HISTORY_START: + logger.debug(solana_signature) + continue_flag = False + break + + ordered_txs.append(solana_signature) + + if solana_signature not in self.transaction_receipts: + poll_txs.add(solana_signature) + + if slot < minimal_slot: + minimal_slot = slot + minimal_tx = solana_signature + + if slot > maximum_slot: + maximum_slot = slot + + if slot < self.last_slot: + continue_flag = False + break + + logger.debug("start getting receipts") + pool = ThreadPool(PARALLEL_REQUESTS) + pool.map(self.get_tx_receipts, poll_txs) + + if len(self.transaction_order): + index = 0 + try: + index = ordered_txs.index(self.transaction_order[0]) + except ValueError: + self.transaction_order = ordered_txs + self.transaction_order + else: + self.transaction_order = ordered_txs[:index] + self.transaction_order + else: + self.transaction_order = ordered_txs + + self.last_slot = maximum_slot + self.current_slot = current_slot + + self.counter_ = 0 + + + def get_tx_receipts(self, solana_signature): + # trx = None + retry = True + + while retry: + try: + trx = self.client.get_confirmed_transaction(solana_signature)['result'] + self.transaction_receipts[solana_signature] = trx + retry = False + except Exception as err: + logger.debug(err) + time.sleep(1) + + self.counter_ += 1 + if self.counter_ % 100 == 0: + logger.debug(self.counter_) + + # return (solana_signature, trx) diff --git a/proxy/proxy.py b/proxy/proxy.py index c63281665..b4eb08753 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -23,7 +23,8 @@ from .http.handler import HttpProtocolHandler from multiprocessing import Process -from .indexer.solana_receipts_update import run_indexer +from .indexer.indexer import run_indexer +from proxy.environment import solana_url, evm_loader_id logger = logging.getLogger(__name__) @@ -45,7 +46,9 @@ def delete_pid_file(self) -> None: os.remove(self.flags.pid_file) def __enter__(self) -> 'Proxy': - self.indexer = Process(target=run_indexer) + self.indexer = Process(target=run_indexer, + args=(solana_url, + evm_loader_id,)) self.indexer.start() self.acceptors = AcceptorPool( flags=self.flags, From d9a8e51d6ab0ef87f5095107a6c1ac3640c95e8b Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Thu, 2 Dec 2021 14:38:57 +0300 Subject: [PATCH 44/54] #333 fix indexer errors (#334) * #333 fix continue blocked accounts * #333 continue_table fixes * #333 fixes * #333 remove filling db to lent indexer work properly * #333 sync --- proxy/indexer/indexer.py | 59 ++++++++++++++++++++++++++++++---------- proxy/indexer/utils.py | 4 +++ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index 9ca772648..52c9fcf81 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -178,8 +178,8 @@ def process_receipts(self): # logger.debug("rlp.exceptions.RLPException") pass except Exception as err: - if str(err).startswith("unhashable type"): - # logger.debug("unhashable type") + if str(err).startswith("nonhashable type"): + # logger.debug("nonhashable type") pass elif str(err).startswith("unsupported operand type"): # logger.debug("unsupported operand type") @@ -268,9 +268,9 @@ def process_receipts(self): continue_result = continue_table[storage_account] if continue_result.accounts != blocked_accounts: logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") - continue_result.signatures.append(signature) trx_table[eth_signature].got_result = continue_result.results - trx_table[eth_signature].signatures = continue_result.signatures + trx_table[eth_signature].signatures += continue_result.signatures + del continue_table[storage_account] elif instruction_data[0] == 0x0a or instruction_data[0] == 0x14: # Continue or ContinueV02 @@ -281,14 +281,14 @@ def process_receipts(self): blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] if instruction_data[0] == 0x14: # logger.debug("{:>10} {:>6} ContinueV02 0x{}".format(slot, counter, instruction_data.hex())) - blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] + blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] got_result = get_trx_results(trx) if storage_account in continue_table: continue_table[storage_account].signatures.append(signature) - if got_result: - if continue_table[storage_account].results: + if got_result is not None: + if continue_table[storage_account].results is not None: logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE") if continue_table[storage_account].accounts != blocked_accounts: logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") @@ -305,10 +305,8 @@ def process_receipts(self): # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeV02 0x{}".format(slot, counter, instruction_data.hex())) blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]] - holder_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][1]] - blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] if storage_account in continue_table: continue_table[storage_account].signatures.append(signature) @@ -328,7 +326,9 @@ def process_receipts(self): # logger.debug("{:>10} {:>6} Cancel 0x{}".format(slot, counter, instruction_data.hex())) storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] - continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot'])) + blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] + + continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), blocked_accounts) elif instruction_data[0] == 0x0d: # logger.debug("{:>10} {:>6} PartialCallOrContinueFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex())) @@ -360,6 +360,14 @@ def process_receipts(self): slot ) + if storage_account in continue_table: + continue_result = continue_table[storage_account] + trx_table[eth_signature].signatures += continue_result.signatures + if continue_result.results is not None: + trx_table[eth_signature].got_result = continue_result.results + + del continue_table[storage_account] + elif instruction_data[0] == 0x0e: # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeOrContinue 0x{}".format(slot, counter, instruction_data.hex())) @@ -379,7 +387,7 @@ def process_receipts(self): logger.error("Strange behavior. Pay attention. HOLDER ACCOUNT NOT FOUND") holder_table[holder_account] = HolderStruct(storage_account) - if got_result: + if got_result is not None: if continue_table[storage_account].results: logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE") if continue_table[storage_account].accounts != blocked_accounts: @@ -396,11 +404,15 @@ def process_receipts(self): pass for eth_signature, trx_struct in trx_table.items(): - if trx_struct.got_result: + if trx_struct.got_result is not None: self.submit_transaction(trx_struct) - elif trx_struct.storage: - if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT: - self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts) + elif trx_struct.storage is not None: + if not self.submit_transaction_part(trx_struct): + if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT: + logger.debug("Probably blocked") + logger.debug(trx_struct.eth_signature) + logger.debug(trx_struct.signatures) + self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts) else: logger.error(trx_struct) @@ -433,6 +445,23 @@ def submit_transaction(self, trx_struct): logger.debug(trx_struct.eth_signature + " " + status) + def submit_transaction_part(self, trx_struct): + ''' Check if transaction was allready submitted by proxy. ''' + eth_signature = trx_struct.eth_signature + ethereum_trx = self.ethereum_trx.get(eth_signature, None) + if ethereum_trx is not None: + signatures = self.eth_sol_trx.get(eth_signature, []) + signatures = signatures + trx_struct.signatures + self.eth_sol_trx[eth_signature] = signatures + for idx, sig in enumerate(signatures): + self.sol_eth_trx[sig] = { + 'idx': idx, + 'eth': eth_signature, + } + return True + return False + + def gather_blocks(self): max_slot = self.client.get_slot(commitment="recent")["result"] diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index c83b38588..cc579daa0 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -350,6 +350,10 @@ def unlock_accounts(self, blocked_storages): if blocked_accs is None: logger.error("blocked_accs is None") continue + if acc_list is None: + logger.error("acc_list is None. Storage is empty") + logger.error(storage) + continue eth_trx = rlp.decode(bytes.fromhex(eth_trx), EthTrx) if acc_list != blocked_accs: From f4b178c43f6ecba1c0d358632a885e1d9434cdf0 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Thu, 2 Dec 2021 16:47:14 +0300 Subject: [PATCH 45/54] #291 Proxy refactoring (#324) * #291 extract transaction sender class * #291 move perm accs to transaction sender * #291 fix state * #291 fix errors * #291 merge fixes * #291 refactoring * #291 move EXTRA_GAS to environment * #291 capitalize CONFIRMATION_CHECK_DELAY * #291 sort imports * #291 relative paths * #291 Should be fixed in #326 * #291 testing chnages * fix storage account check * #291 rename `trx_with_create_and_airdrop` -> `make_trx_with_create_and_airdrop` * #291 pull request fixes * #291 merge fix * #291 rename operator and associated token accounts Co-authored-by: sinev-valentine --- .buildkite/steps/build-image.sh | 2 +- Dockerfile | 2 +- proxy/common_neon/address.py | 70 + proxy/common_neon/constants.py | 12 + proxy/common_neon/costs.py | 93 ++ proxy/common_neon/emulator_interactor.py | 34 + proxy/common_neon/errors.py | 11 + proxy/common_neon/layouts.py | 39 + proxy/common_neon/neon_instruction.py | 375 +++++ proxy/common_neon/solana_interactor.py | 214 +++ proxy/common_neon/transaction_sender.py | 420 ++++++ proxy/docker-compose-test.yml | 2 +- proxy/environment.py | 18 +- proxy/indexer/indexer.py | 3 +- proxy/indexer/utils.py | 60 +- proxy/plugin/solana_rest_api.py | 44 +- proxy/plugin/solana_rest_api_tools.py | 1207 +---------------- proxy/proxy.py | 6 +- proxy/run-test-proxy.sh | 2 +- .../testing/test_airdropping_eth_accounts.py | 3 +- proxy/testing/test_erc20_wrapper_contract.py | 6 +- ...anged.py => test_indexer_cancel_hanged.py} | 10 +- proxy/testing/test_resize_storage_account.py | 4 +- proxy/testing/test_web3_clientVersion.py | 5 - 24 files changed, 1376 insertions(+), 1266 deletions(-) create mode 100644 proxy/common_neon/address.py create mode 100644 proxy/common_neon/constants.py create mode 100644 proxy/common_neon/costs.py create mode 100644 proxy/common_neon/emulator_interactor.py create mode 100644 proxy/common_neon/layouts.py create mode 100644 proxy/common_neon/neon_instruction.py create mode 100644 proxy/common_neon/solana_interactor.py create mode 100644 proxy/common_neon/transaction_sender.py rename proxy/testing/{test_cancel_hanged.py => test_indexer_cancel_hanged.py} (95%) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index 03cd1dad0..104ffc621 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -3,7 +3,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) -set ${SOLANA_REVISION:=v1.7.9-resources} +set ${SOLANA_REVISION:=v1.7.9-testnet} set ${EVM_LOADER_REVISION:=stable} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally diff --git a/Dockerfile b/Dockerfile index 6837342c8..19cbbe0e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG SOLANA_REVISION=v1.7.9-resources +ARG SOLANA_REVISION=v1.7.9-testnet ARG EVM_LOADER_REVISION=stable FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli diff --git a/proxy/common_neon/address.py b/proxy/common_neon/address.py new file mode 100644 index 000000000..462ca8564 --- /dev/null +++ b/proxy/common_neon/address.py @@ -0,0 +1,70 @@ +import logging +import random + +from eth_keys import keys as eth_keys +from hashlib import sha256 +from solana.publickey import PublicKey +from spl.token.instructions import get_associated_token_address +from typing import NamedTuple + +from .layouts import ACCOUNT_INFO_LAYOUT +from ..environment import neon_cli, ETH_TOKEN_MINT_ID, EVM_LOADER_ID + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class EthereumAddress: + def __init__(self, data, private=None): + if isinstance(data, str): + data = bytes(bytearray.fromhex(data[2:])) + self.data = data + self.private = private + + @staticmethod + def random(): + letters = '0123456789abcdef' + data = bytearray.fromhex(''.join([random.choice(letters) for k in range(64)])) + pk = eth_keys.PrivateKey(data) + return EthereumAddress(pk.public_key.to_canonical_address(), pk) + + def __str__(self): + return '0x'+self.data.hex() + + def __repr__(self): + return self.__str__() + + def __bytes__(self): return self.data + + +def accountWithSeed(base, seed): + result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(PublicKey(EVM_LOADER_ID))).digest()) + return result + + +def ether2program(ether): + if isinstance(ether, str): + pass + elif isinstance(ether, EthereumAddress): + ether = str(ether) + else: + ether = ether.hex() + output = neon_cli().call("create-program-address", ether) + items = output.rstrip().split(' ') + return items[0], int(items[1]) + + +def getTokenAddr(account): + return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) + + +class AccountInfo(NamedTuple): + ether: eth_keys.PublicKey + trx_count: int + code_account: PublicKey + + @staticmethod + def frombytes(data): + cont = ACCOUNT_INFO_LAYOUT.parse(data) + return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account)) diff --git a/proxy/common_neon/constants.py b/proxy/common_neon/constants.py new file mode 100644 index 000000000..e16cfbd36 --- /dev/null +++ b/proxy/common_neon/constants.py @@ -0,0 +1,12 @@ +KECCAK_PROGRAM = "KeccakSecp256k11111111111111111111111111111" +INCINERATOR_PUBKEY = "1nc1nerator11111111111111111111111111111111" +SYSVAR_INSTRUCTION_PUBKEY = "Sysvar1nstructions1111111111111111111111111" + +STORAGE_SIZE = 128*1024 + +ACCOUNT_SEED_VERSION=b'\1' + +COLLATERALL_POOL_MAX=10 + +EMPTY_STORAGE_TAG=0 +FINALIZED_STORAGE_TAG=5 diff --git a/proxy/common_neon/costs.py b/proxy/common_neon/costs.py new file mode 100644 index 000000000..2012570b1 --- /dev/null +++ b/proxy/common_neon/costs.py @@ -0,0 +1,93 @@ +import base58 +import psycopg2 + +from ..environment import EVM_LOADER_ID +from ..indexer.sql_dict import POSTGRES_USER, POSTGRES_HOST, POSTGRES_DB, POSTGRES_PASSWORD + +class SQLCost(): + def __init__(self): + + self.conn = psycopg2.connect( + dbname=POSTGRES_DB, + user=POSTGRES_USER, + password=POSTGRES_PASSWORD, + host=POSTGRES_HOST + ) + + self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = self.conn.cursor() + cur.execute(''' + CREATE TABLE IF NOT EXISTS OPERATOR_COST + ( + hash char(64), + cost bigint, + used_gas bigint, + sender char(40), + to_address char(40) , + sig char(100), + status varchar(100), + reason varchar(100) + )''' + ) + + def close(self): + self.conn.close() + + def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason): + cur = self.conn.cursor() + cur.execute(''' + INSERT INTO OPERATOR_COST (hash, cost, used_gas, sender, to_address, sig, status, reason) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s) + ''', + (hash, cost, used_gas, sender, to_address, sig, status, reason) + ) + + +class CostSingleton(object): + def __new__(cls): + if not hasattr(cls, 'instance'): + cls.instance = super(CostSingleton, cls).__new__(cls) + cls.instance.operator_cost = SQLCost() + return cls.instance + + +def update_transaction_cost(receipt, eth_trx, extra_sol_trx=False, reason=None): + cost = receipt['result']['meta']['preBalances'][0] - receipt['result']['meta']['postBalances'][0] + if eth_trx: + hash = eth_trx.hash_signed().hex() + sender = eth_trx.sender() + to_address = eth_trx.toAddress.hex() if eth_trx.toAddress else "None" + else: + hash = None + sender = None + to_address = None + + sig = receipt['result']['transaction']['signatures'][0] + used_gas=None + + tx_info = receipt['result'] + accounts = tx_info["transaction"]["message"]["accountKeys"] + evm_loader_instructions = [] + + for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: + evm_loader_instructions.append(idx) + + for inner in (tx_info['meta']['innerInstructions']): + if inner["index"] in evm_loader_instructions: + for event in inner['instructions']: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: + used_gas = base58.b58decode(event['data'])[2:10] + used_gas = int().from_bytes(used_gas, "little") + + table = CostSingleton().operator_cost + table.insert( + hash, + cost, + used_gas if used_gas else 0, + sender, + to_address, + sig, + 'extra' if extra_sol_trx else 'ok', + reason if reason else '' + ) diff --git a/proxy/common_neon/emulator_interactor.py b/proxy/common_neon/emulator_interactor.py new file mode 100644 index 000000000..7f5897f4d --- /dev/null +++ b/proxy/common_neon/emulator_interactor.py @@ -0,0 +1,34 @@ +import json +import logging + +from .errors import EthereumError +from ..environment import neon_cli + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def call_emulated(contract_id, caller_id, data=None, value=None): + output = emulator(contract_id, caller_id, data, value) + logger.debug("call_emulated %s %s %s %s return %s", contract_id, caller_id, data, value, output) + result = json.loads(output) + exit_status = result['exit_status'] + if exit_status == 'revert': + result_value = result['result'] + if len(result_value) < 8 or result_value[:8] != '08c379a0': + raise EthereumError(code=3, message='execution reverted') + + offset = int(result_value[8:8+64], 16) + length = int(result_value[8+64:8+64+64], 16) + message = str(bytes.fromhex(result_value[8+offset*2+64:8+offset*2+64+length*2]), 'utf8') + raise EthereumError(code=3, message='execution reverted: '+message, data='0x'+result_value) + if result["exit_status"] != "succeed": + raise Exception("evm emulator error ", result) + return result + + +def emulator(contract, sender, data, value): + data = data or "none" + value = value or "" + return neon_cli().call("emulate", sender, contract, data, value) diff --git a/proxy/common_neon/errors.py b/proxy/common_neon/errors.py index e590afa64..4c17e8a9b 100644 --- a/proxy/common_neon/errors.py +++ b/proxy/common_neon/errors.py @@ -1,6 +1,17 @@ from enum import Enum +class EthereumError(Exception): + def __init__(self, code, message, data=None): + self.code = code + self.message = message + self.data = data + + def getError(self): + error = {'code': self.code, 'message': self.message} + if self.data: error['data'] = self.data + return error + class SolanaErrors(Enum): AccountNotFound = "Invalid param: could not find account" diff --git a/proxy/common_neon/layouts.py b/proxy/common_neon/layouts.py new file mode 100644 index 000000000..a48171ab7 --- /dev/null +++ b/proxy/common_neon/layouts.py @@ -0,0 +1,39 @@ + +from construct import Bytes, Int8ul, Int64ul +from construct import Struct + +STORAGE_ACCOUNT_INFO_LAYOUT = Struct( + # "tag" / Int8ul, + "caller" / Bytes(20), + "nonce" / Int64ul, + "gas_limit" / Int64ul, + "gas_price" / Int64ul, + "slot" / Int64ul, + "operator" / Bytes(32), + "accounts_len" / Int64ul, + "executor_data_size" / Int64ul, + "evm_data_size" / Int64ul, + "gas_used_and_paid" / Int64ul, + "number_of_payments" / Int64ul, + "sign" / Bytes(65), +) + +ACCOUNT_INFO_LAYOUT = Struct( + "type" / Int8ul, + "ether" / Bytes(20), + "nonce" / Int8ul, + "trx_count" / Bytes(8), + "code_account" / Bytes(32), + "is_rw_blocked" / Int8ul, + "rw_blocked_acc" / Bytes(32), + "eth_token_account" / Bytes(32), + "ro_blocked_cnt" / Int8ul, +) + + +CREATE_ACCOUNT_LAYOUT = Struct( + "lamports" / Int64ul, + "space" / Int64ul, + "ether" / Bytes(20), + "nonce" / Int8ul +) diff --git a/proxy/common_neon/neon_instruction.py b/proxy/common_neon/neon_instruction.py new file mode 100644 index 000000000..8242d97cb --- /dev/null +++ b/proxy/common_neon/neon_instruction.py @@ -0,0 +1,375 @@ +import eth_utils +import logging +import struct + +from sha3 import keccak_256 +from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT, InstructionType +from solana.publickey import PublicKey +from solana.system_program import SYS_PROGRAM_ID +from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY +from solana.transaction import AccountMeta, TransactionInstruction, Transaction +from spl.token.constants import ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID +from spl.token.instructions import transfer2, Transfer2Params +from typing import Tuple + +from .address import accountWithSeed, ether2program, getTokenAddr, EthereumAddress +from .constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM, COLLATERALL_POOL_MAX +from .layouts import CREATE_ACCOUNT_LAYOUT +from ..environment import EVM_LOADER_ID, ETH_TOKEN_MINT_ID , COLLATERAL_POOL_BASE, NEW_USER_AIRDROP_AMOUNT + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +obligatory_accounts = [ + AccountMeta(pubkey=EVM_LOADER_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_CLOCK_PUBKEY, is_signer=False, is_writable=False), +] + + +def create_account_with_seed_layout(base, seed, lamports, space): + return SYSTEM_INSTRUCTIONS_LAYOUT.build( + dict( + instruction_type = InstructionType.CREATE_ACCOUNT_WITH_SEED, + args=dict( + base=bytes(base), + seed=dict(length=len(seed), chars=seed), + lamports=lamports, + space=space, + program_id=bytes(PublicKey(EVM_LOADER_ID)) + ) + ) + ) + + +def create_account_layout(lamports, space, ether, nonce): + return bytes.fromhex("02000000")+CREATE_ACCOUNT_LAYOUT.build(dict( + lamports=lamports, + space=space, + ether=ether, + nonce=nonce + )) + + +def write_holder_layout(nonce, offset, data): + return (bytes.fromhex('12')+ + nonce.to_bytes(8, byteorder='little')+ + offset.to_bytes(4, byteorder='little')+ + len(data).to_bytes(8, byteorder='little')+ + data) + + +def make_keccak_instruction_data(check_instruction_index, msg_len, data_start): + if check_instruction_index > 255 and check_instruction_index < 0: + raise Exception("Invalid index for instruction - {}".format(check_instruction_index)) + + check_count = 1 + eth_address_size = 20 + signature_size = 65 + eth_address_offset = data_start + signature_offset = eth_address_offset + eth_address_size + message_data_offset = signature_offset + signature_size + + data = struct.pack("B", check_count) + data += struct.pack(" Tuple[Transaction, PublicKey]: + pda_account, nonce = ether2program(eth_address) + neon_token_account = getTokenAddr(PublicKey(pda_account)) + logger.debug(f'Create eth account: {eth_address}, sol account: {pda_account}, neon_token_account: {neon_token_account}, nonce: {nonce}') + + base = self.operator_account + data = create_account_layout(0, 0, bytes(eth_address), nonce) + trx = Transaction() + if code_acc is None: + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(pda_account), is_signer=False, is_writable=True), + AccountMeta(pubkey=neon_token_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ])) + else: + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(pda_account), is_signer=False, is_writable=True), + AccountMeta(pubkey=neon_token_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ])) + return trx, neon_token_account + + + def createERC20TokenAccountTrx(self, token_info) -> Transaction: + trx = Transaction() + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=bytes.fromhex('0F'), + keys=[ + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["key"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["owner"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["contract"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["mint"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ] + )) + + return trx + + + def make_transfer_instruction(self, associated_token_account: PublicKey) -> TransactionInstruction: + transfer_instruction = transfer2(Transfer2Params( + source=self.operator_neon_address, + owner=self.operator_account, + dest=associated_token_account, + amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei, + decimals=9, + mint=ETH_TOKEN_MINT_ID, + program_id=TOKEN_PROGRAM_ID + )) + logger.debug(f"Token transfer from token: {self.operator_neon_address}, owned by: {self.operator_account}, to token: " + f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}") + return transfer_instruction + + + def make_trx_with_create_and_airdrop(self, eth_account, code_acc=None) -> Transaction: + trx = Transaction() + create_trx, associated_token_account = self.make_create_eth_account_trx(eth_account, code_acc) + trx.add(create_trx) + if NEW_USER_AIRDROP_AMOUNT <= 0: + return trx + transfer_instruction = self.make_transfer_instruction(associated_token_account) + trx.add(transfer_instruction) + + return trx + + + def make_resize_instruction(self, acc_desc, code_account_new, seed) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("11") + bytes(seed), # 17- ResizeStorageAccount + keys = [ + AccountMeta(pubkey=PublicKey(acc_desc["account"]), is_signer=False, is_writable=True), + ( + AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=True) + if acc_desc["contract"] else + AccountMeta(pubkey=PublicKey("11111111111111111111111111111111"), is_signer=False, is_writable=False) + ), + AccountMeta(pubkey=code_account_new, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=False) + ], + ) + + + def make_write_transaction(self, offset: int, data: bytes) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=write_holder_layout(self.perm_accs_id, offset, data), + keys=[ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=False), + ] + )) + + + def make_keccak_instruction(self, check_instruction_index, msg_len, data_start) -> TransactionInstruction: + return TransactionInstruction( + program_id=KECCAK_PROGRAM, + data=make_keccak_instruction_data(check_instruction_index, msg_len, data_start), + keys=[ + AccountMeta(pubkey=KECCAK_PROGRAM, is_signer=False, is_writable=False), + ] + ) + + + def make_05_call_instruction(self) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("05") + self.collateral_pool_index_buf + self.msg, + keys = [ + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + obligatory_accounts + ) + + + def make_noniterative_call_transaction(self, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 5)) + trx.add(self.make_05_call_instruction()) + return trx + + + def make_partial_call_instruction(self) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("13") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little") + self.msg, + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + ) + + + def make_iterative_call_transaction(self, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 13)) + trx.add(self.make_partial_call_instruction()) + return trx + + + def make_call_from_account_instruction(self) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("16") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little"), + keys = [ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) + + + def make_continue_instruction(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("14") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + if index: + data = data + index.to_bytes(8, byteorder="little") + + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = data, + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) + + + def make_cancel_instruction(self) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py new file mode 100644 index 000000000..ffb02e5d4 --- /dev/null +++ b/proxy/common_neon/solana_interactor.py @@ -0,0 +1,214 @@ +import base58 +import base64 +import json +import logging +import re +import time + +from solana.rpc.commitment import Confirmed +from solana.rpc.types import TxOpts + +from .costs import update_transaction_cost +from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class SolanaInteractor: + def __init__(self, signer, client) -> None: + self.signer = signer + self.client = client + + + def get_operator_key(self): + return self.signer.public_key() + + + def get_account_info(self, storage_account): + opts = { + "encoding": "base64", + "commitment": "confirmed", + "dataSlice": { + "offset": 0, + "length": 16, + } + } + + result = self.client._provider.make_request("getAccountInfo", str(storage_account), opts) + logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) + + info = result['result']['value'] + if info is None: + logger.debug("Can't get information about {}".format(storage_account)) + return None + + data = base64.b64decode(info['data'][0]) + + account_tag = data[0] + lamports = info['lamports'] + owner = info['owner'] + + return (account_tag, lamports, owner) + + + def get_sol_balance(self, account): + return self.client.get_balance(account, commitment=Confirmed)['result']['value'] + + + def get_rent_exempt_balance_for_size(self, size): + return self.client.get_minimum_balance_for_rent_exemption(size, commitment=Confirmed)["result"] + + + def _getAccountData(self, account, expected_length, owner=None): + info = self.client.get_account_info(account, commitment=Confirmed)['result']['value'] + if info is None: + raise Exception("Can't get information about {}".format(account)) + + data = base64.b64decode(info['data'][0]) + if len(data) < expected_length: + raise Exception("Wrong data length for account data {}".format(account)) + return data + + + def send_transaction(self, trx, eth_trx, reason=None): + reciept = self.send_transaction_unconfirmed(trx) + result = self.collect_result(reciept, eth_trx, reason) + return result + + + def send_transaction_unconfirmed(self, trx): + result = self.client.send_transaction(trx, self.signer, opts=TxOpts(preflight_commitment=Confirmed))["result"] + return result + + def collect_result(self, reciept, eth_trx, reason=None): + self.confirm_transaction(reciept) + result = self.client.get_confirmed_transaction(reciept) + update_transaction_cost(result, eth_trx, reason) + return result + + + def send_measured_transaction(self, trx, eth_trx, reason): + result = self.send_transaction(trx, eth_trx, reason=reason) + self.get_measurements(result) + return result + + + # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh) + def get_measurements(self, result): + try: + measurements = self.extract_measurements_from_receipt(result) + for m in measurements: logger.info(json.dumps(m)) + except Exception as err: + logger.error("Can't get measurements %s"%err) + logger.info("Failed result: %s"%json.dumps(result, indent=3)) + + + def confirm_transaction(self, tx_sig, confirmations=0): + """Confirm a transaction.""" + TIMEOUT = 30 # 30 seconds pylint: disable=invalid-name + elapsed_time = 0 + while elapsed_time < TIMEOUT: + logger.debug('confirm_transaction for %s', tx_sig) + resp = self.client.get_signature_statuses([tx_sig]) + logger.debug('confirm_transaction: %s', resp) + if resp["result"]: + status = resp['result']['value'][0] + if status and (status['confirmationStatus'] == 'finalized' or \ + status['confirmationStatus'] == 'confirmed' and status['confirmations'] >= confirmations): + return + time.sleep(CONFIRMATION_CHECK_DELAY) + elapsed_time += CONFIRMATION_CHECK_DELAY + raise RuntimeError("could not confirm transaction: ", tx_sig) + + + def collect_results(self, receipts, eth_trx=None, reason=None): + results = [] + for rcpt in receipts: + results.append(self.collect_result(rcpt, eth_trx, reason)) + return results + + @staticmethod + def extract_measurements_from_receipt(receipt): + log_messages = receipt['result']['meta']['logMessages'] + transaction = receipt['result']['transaction'] + accounts = transaction['message']['accountKeys'] + instructions = [] + for instr in transaction['message']['instructions']: + program = accounts[instr['programIdIndex']] + instructions.append({ + 'accs': [accounts[acc] for acc in instr['accounts']], + 'program': accounts[instr['programIdIndex']], + 'data': base58.b58decode(instr['data']).hex() + }) + + pattern = re.compile('Program ([0-9A-Za-z]+) (.*)') + messages = [] + for log in log_messages: + res = pattern.match(log) + if res: + (program, reason) = res.groups() + if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) + messages[-1]['logs'].append(log) + + for instr in instructions: + if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue + if messages[0]['program'] != instr['program']: + raise Exception('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) + instr['logs'] = messages.pop(0)['logs'] + exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) + if not exit_result: raise Exception("Can't get exit result") + instr['result'] = exit_result.group(1) + + if instr['program'] == EVM_LOADER_ID: + memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) + instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) + if not (memory_result and instruction_result): + raise Exception("Can't parse measurements for evm_loader") + instr['measurements'] = { + 'instructions': instruction_result.group(1), + 'memory': memory_result.group(1) + } + + result = [] + for instr in instructions: + if instr['program'] == EVM_LOADER_ID: + result.append({ + 'program':instr['program'], + 'measurements':instr['measurements'], + 'result':instr['result'], + 'data':instr['data'] + }) + return result + + +def check_if_program_exceeded_instructions(err_result): + err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" + err_budget = "failed: Computational budget exceeded" + + if err_result['data']['logs'][-1].find(err_instruction) >= 0 or \ + err_result['data']['logs'][-2].find(err_instruction) >= 0 or \ + err_result['data']['logs'][-1].find(err_budget) >= 0: + return True + return False + + +def check_if_continue_returned(result): + tx_info = result['result'] + accounts = tx_info["transaction"]["message"]["accountKeys"] + evm_loader_instructions = [] + + for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: + evm_loader_instructions.append(idx) + + for inner in (tx_info['meta']['innerInstructions']): + if inner["index"] in evm_loader_instructions: + for event in inner['instructions']: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: + instruction = base58.b58decode(event['data'])[:1] + if int().from_bytes(instruction, "little") == 6: # OnReturn evmInstruction code + return tx_info['transaction']['signatures'][0] + + return None diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py new file mode 100644 index 000000000..5300c6a85 --- /dev/null +++ b/proxy/common_neon/transaction_sender.py @@ -0,0 +1,420 @@ +import json +import logging +import os +import rlp +import time + +from base58 import b58encode +from sha3 import keccak_256 +from solana.publickey import PublicKey +from solana.rpc.api import SendTransactionError +from solana.sysvar import * +from solana.transaction import AccountMeta, Transaction + +from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob + +from .address import accountWithSeed, AccountInfo, getTokenAddr +from .constants import STORAGE_SIZE, EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG, ACCOUNT_SEED_VERSION +from .emulator_interactor import call_emulated +from .layouts import ACCOUNT_INFO_LAYOUT +from .neon_instruction import NeonInstruction +from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_if_program_exceeded_instructions +from ..environment import EVM_LOADER_ID +from ..plugin.eth_proto import Trx as EthTrx + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class TransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, eth_trx: EthTrx, steps: int) -> None: + self.sender = solana_interactor + self.eth_trx = eth_trx + self.steps = steps + + self.instruction = NeonInstruction(self.sender.get_operator_key()) + + + def execute(self): + self.create_account_list_by_emulate() + + noniterative_executor = self.create_noniterative_executor() + + call_iterative = False + call_from_holder = False + + if not self.eth_trx.toAddress: + call_from_holder = True + else: + try: + logger.debug("Try single trx call") + return noniterative_executor.call_signed_noniterative() + except Exception as err: + logger.debug(str(err)) + errStr = str(err) + if "Program failed to complete" in errStr or "Computational budget exceeded" in errStr: + logger.debug("Program exceeded instructions") + call_iterative = True + elif str(err).startswith("transaction too large:"): + logger.debug("Transaction too large, call call_signed_with_holder_acc():") + call_from_holder = True + else: + raise + + self.init_perm_accs() + iterative_executor = self.create_iterative_executor() + try: + if call_iterative: + try: + return iterative_executor.call_signed_iterative() + except Exception as err: + logger.debug(str(err)) + if str(err).startswith("transaction too large:"): + logger.debug("Transaction too large, call call_signed_with_holder_acc():") + call_from_holder = True + else: + raise + + if call_from_holder: + return iterative_executor.call_signed_with_holder_acc() + finally: + self.free_perm_accs() + + + def create_instruction_constructor(self): + return NeonInstruction(self.sender.get_operator_key(), self.eth_trx, self.eth_accounts, self.caller_token) + + + def create_noniterative_executor(self): + self.instruction.init_eth_trx(self.eth_trx, self.eth_accounts, self.caller_token) + return NoniterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx) + + + def create_iterative_executor(self): + self.instruction.init_iterative(self.storage, self.holder, self.perm_accs_id) + return IterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx, self.steps) + + + def init_perm_accs(self): + while True: + with new_acc_id_glob.get_lock(): + try: + self.perm_accs_id = acc_list_glob.pop(0) + except IndexError: + self.perm_accs_id = new_acc_id_glob.value + new_acc_id_glob.value += 1 + + logger.debug("LOCK RESOURCES {}".format(self.perm_accs_id)) + + acc_id_bytes = self.perm_accs_id.to_bytes((self.perm_accs_id.bit_length() + 7) // 8, 'big') + + storage_seed = keccak_256(b"storage" + acc_id_bytes).hexdigest()[:32] + storage_seed = bytes(storage_seed, 'utf8') + + holder_seed = keccak_256(b"holder" + acc_id_bytes).hexdigest()[:32] + holder_seed = bytes(holder_seed, 'utf8') + + try: + self.storage, self.holder = self.create_multiple_accounts_with_seed( + seeds=[storage_seed, holder_seed], + sizes=[STORAGE_SIZE, STORAGE_SIZE] + ) + except Exception as err: + logger.warn("Account is locked err({}) id({}) owner({})".format(str(err), self.perm_accs_id, self.sender.get_operator_key())) + else: + break + + + def free_perm_accs(self): + logger.debug("FREE RESOURCES {}".format(self.perm_accs_id)) + with new_acc_id_glob.get_lock(): + acc_list_glob.append(self.perm_accs_id) + + + def create_account_with_seed(self, seed, storage_size): + account = accountWithSeed(self.sender.get_operator_key(), seed) + + if self.sender.get_sol_balance(account) == 0: + minimum_balance = self.sender.get_rent_exempt_balance_for_size(storage_size) + logger.debug("Minimum balance required for account {}".format(minimum_balance)) + + trx = Transaction() + trx.add(self.instruction.create_account_with_seed_trx(account, seed, minimum_balance, storage_size)) + self.sender.send_transaction(trx, eth_trx=self.eth_trx, reason='createAccountWithSeed') + + return account + + + def create_multiple_accounts_with_seed(self, seeds, sizes): + accounts = [] + trx = Transaction() + + for seed, storage_size in zip(seeds, sizes): + account = accountWithSeed(self.sender.get_operator_key(), seed) + accounts.append(account) + + minimum_balance = self.sender.get_rent_exempt_balance_for_size(storage_size) + + account_info = self.sender.get_account_info(account) + if account_info is None: + logger.debug("Minimum balance required for account {}".format(minimum_balance)) + + trx.add(self.instruction.create_account_with_seed_trx(account, seed, minimum_balance, storage_size)) + else: + (tag, lamports, owner) = account_info + if lamports < minimum_balance: + raise Exception("insufficient balance") + if PublicKey(owner) != PublicKey(EVM_LOADER_ID): + raise Exception("wrong owner") + if tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}: + raise Exception("not empty, not finalized") + + if len(trx.instructions) > 0: + self.sender.send_transaction(trx, eth_trx=self.eth_trx, reason='createAccountWithSeed') + + return accounts + + + def create_account_list_by_emulate(self): + sender_ether = bytes.fromhex(self.eth_trx.sender()) + add_keys_05 = [] + self.create_acc_trx = Transaction() + + if not self.eth_trx.toAddress: + to_address_arg = "deploy" + to_address = keccak_256(rlp.encode((bytes.fromhex(self.eth_trx.sender()), self.eth_trx.nonce))).digest()[-20:] + else: + to_address_arg = self.eth_trx.toAddress.hex() + to_address = self.eth_trx.toAddress + + logger.debug("send_addr: %s", self.eth_trx.sender()) + logger.debug("dest_addr: %s", to_address.hex()) + + output_json = call_emulated(to_address_arg, sender_ether.hex(), self.eth_trx.callData.hex(), hex(self.eth_trx.value)) + logger.debug("emulator returns: %s", json.dumps(output_json, indent=3)) + + # resize storage account + resize_instr = [] + for acc_desc in output_json["accounts"]: + if acc_desc["new"] == False: + if acc_desc["code_size_current"] is not None and acc_desc["code_size"] is not None: + if acc_desc["code_size"] > acc_desc["code_size_current"]: + code_size = acc_desc["code_size"] + 2048 + seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) + code_account_new = accountWithSeed(self.sender.get_operator_key(), seed) + + logger.debug("creating new code_account with increased size %s", code_account_new) + self.create_account_with_seed(seed, code_size) + logger.debug("resized account is created %s", code_account_new) + + resize_instr.append(self.instruction.make_resize_instruction(acc_desc, code_account_new, seed)) + # replace code_account + acc_desc["contract"] = code_account_new + + for instr in resize_instr: + logger.debug("code and storage migration, account %s from %s to %s", instr.keys[0].pubkey, instr.keys[1].pubkey, instr.keys[2].pubkey) + + tx = Transaction().add(instr) + success = False + count = 0 + + while count < 2: + logger.debug("attemt: %d", count) + + self.sender.send_transaction(tx, eth_trx=self.eth_trx, reason='resize_storage_account') + info = self.sender._getAccountData(instr.keys[0].pubkey, ACCOUNT_INFO_LAYOUT.sizeof()) + info_data = AccountInfo.frombytes(info) + if info_data.code_account == instr.keys[2].pubkey: + success = True + logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) + break + # wait for unlock account + time.sleep(1) + count = count+1 + + if success == False: + raise Exception("Can't resize storage account. Account is blocked {}".format(instr.keys[0].pubkey)) + + for acc_desc in output_json["accounts"]: + address = bytes.fromhex(acc_desc["address"][2:]) + + code_account = None + code_account_writable = False + if acc_desc["new"]: + logger.debug("Create solana accounts for %s: %s %s", acc_desc["address"], acc_desc["account"], acc_desc["contract"]) + if acc_desc["code_size"]: + seed = b58encode(ACCOUNT_SEED_VERSION+address) + code_account = accountWithSeed(self.sender.get_operator_key(), seed) + logger.debug(" with code account %s", code_account) + code_size = acc_desc["code_size"] + 2048 + code_account_balance = self.sender.get_rent_exempt_balance_for_size(code_size) + self.create_acc_trx.add(self.instruction.create_account_with_seed_trx(code_account, seed, code_account_balance, code_size)) + # add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=acc_desc["writable"])) + code_account_writable = acc_desc["writable"] + + create_trx = self.instruction.make_trx_with_create_and_airdrop(address, code_account) + self.create_acc_trx.add(create_trx) + + if address == to_address: + contract_sol = PublicKey(acc_desc["account"]) + if acc_desc["new"]: + code_sol = code_account + code_writable = code_account_writable + else: + if acc_desc["contract"] != None: + code_sol = PublicKey(acc_desc["contract"]) + code_writable = acc_desc["writable"] + else: + code_sol = None + code_writable = None + + elif address == sender_ether: + sender_sol = PublicKey(acc_desc["account"]) + else: + add_keys_05.append(AccountMeta(pubkey=acc_desc["account"], is_signer=False, is_writable=True)) + token_account = getTokenAddr(PublicKey(acc_desc["account"])) + add_keys_05.append(AccountMeta(pubkey=token_account, is_signer=False, is_writable=True)) + if acc_desc["new"]: + if code_account: + add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=code_account_writable)) + else: + if acc_desc["contract"]: + add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) + + + for token_account in output_json["token_accounts"]: + add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) + + if token_account["new"]: + self.create_acc_trx.add(self.instruction.createERC20TokenAccountTrx(token_account)) + + for account_meta in output_json["solana_accounts"]: + add_keys_05.append(AccountMeta(pubkey=PublicKey(account_meta["pubkey"]), is_signer=account_meta["is_signer"], is_writable=account_meta["is_writable"])) + + self.caller_token = getTokenAddr(PublicKey(sender_sol)) + + self.eth_accounts = [ + AccountMeta(pubkey=contract_sol, is_signer=False, is_writable=True), + AccountMeta(pubkey=getTokenAddr(contract_sol), is_signer=False, is_writable=True), + ] + ([AccountMeta(pubkey=code_sol, is_signer=False, is_writable=code_writable)] if code_sol != None else []) + [ + AccountMeta(pubkey=sender_sol, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + ] + add_keys_05 + + +class NoniterativeTransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx): + self.sender = solana_interactor + self.instruction = neon_instruction + self.create_acc_trx = create_acc_trx + self.eth_trx = eth_trx + + + def call_signed_noniterative(self): + call_txs_05 = Transaction() + if len(self.create_acc_trx.instructions) > 0: + call_txs_05.add(self.create_acc_trx) + call_txs_05.add(self.instruction.make_noniterative_call_transaction(len(call_txs_05.instructions))) + result = self.sender.send_measured_transaction(call_txs_05, self.eth_trx, 'CallFromRawEthereumTX') + return result['result']['transaction']['signatures'][0] + + +class IterativeTransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx, steps: int): + self.sender = solana_interactor + self.instruction = neon_instruction + self.create_acc_trx = create_acc_trx + self.eth_trx = eth_trx + self.steps = steps + + + def call_signed_iterative(self): + if len(self.create_acc_trx.instructions): + precall_txs = Transaction() + precall_txs.add(self.create_acc_trx) + self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') + + call_txs = self.instruction.make_iterative_call_transaction() + + logger.debug("Partial call") + self.sender.send_measured_transaction(call_txs, self.eth_trx, 'PartialCallFromRawEthereumTXv02') + + return self.call_continue() + + + def call_signed_with_holder_acc(self): + self.write_trx_to_holder_account() + if len(self.create_acc_trx.instructions): + precall_txs = Transaction() + precall_txs.add(self.create_acc_trx) + self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'create_accounts_for_deploy') + + # ExecuteTrxFromAccountDataIterative + logger.debug("ExecuteTrxFromAccountDataIterative:") + call_txs = self.instruction.make_call_from_account_instruction() + self.sender.send_measured_transaction(call_txs, self.eth_trx, 'ExecuteTrxFromAccountDataIterativeV02') + + return self.call_continue() + + + def write_trx_to_holder_account(self): + msg = self.eth_trx.signature() + len(self.eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + self.eth_trx.unsigned_msg() + + # Write transaction to transaction holder account + offset = 0 + receipts = [] + rest = msg + while len(rest): + (part, rest) = (rest[:1000], rest[1000:]) + # logger.debug("sender_sol %s %s %s", sender_sol, holder, acc.public_key()) + trx = self.instruction.make_write_transaction(offset, part) + receipts.append(self.sender.send_transaction_unconfirmed(trx)) + offset += len(part) + + logger.debug("receipts %s", receipts) + self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason='WriteHolder') + + + def call_continue(self): + try: + return self.call_continue_step_by_step() + except Exception as err: + logger.error("call_continue_step_by_step exception:") + logger.debug(str(err)) + + return self.call_cancel() + + + def call_continue_step_by_step(self): + while True: + logger.debug("Continue iterative step:") + result = self.call_continue_step() + signature = check_if_continue_returned(result) + if signature is not None: + return signature + + + def call_continue_step(self): + step_count = self.steps + while step_count > 0: + trx = self.instruction.make_continue_instruction(step_count) + + logger.debug("Step count {}".format(step_count)) + try: + result = self.sender.send_measured_transaction(trx, self.eth_trx, 'ContinueV02') + return result + except SendTransactionError as err: + if check_if_program_exceeded_instructions(err.result): + step_count = int(step_count * 90 / 100) + else: + raise + raise Exception("Can't execute even one EVM instruction") + + + def call_cancel(self): + trx = self.instruction.make_cancel_instruction() + + logger.debug("Cancel") + result = self.sender.send_measured_transaction(trx, self.eth_trx, 'CancelWithNonce') + return result['result']['transaction']['signatures'][0] diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 3d98cb7ef..729b4affa 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -3,7 +3,7 @@ version: "3" services: solana: container_name: solana - image: neonlabsorg/solana:${SOLANA_REVISION:-v1.7.9-resources} + image: neonlabsorg/solana:${SOLANA_REVISION:-v1.7.9-testnet} environment: SOLANA_URL: http://solana:8899 RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug diff --git a/proxy/environment.py b/proxy/environment.py index 794f099a8..526ce53cf 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -6,15 +6,23 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") -evm_loader_id = os.environ.get("EVM_LOADER") +SOLANA_URL = os.environ.get("SOLANA_URL", "http://localhost:8899") +EVM_LOADER_ID = os.environ.get("EVM_LOADER") neon_cli_timeout = float(os.environ.get("NEON_CLI_TIMEOUT", "0.1")) +NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) +CONFIRMATION_CHECK_DELAY = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) +USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "NO") == "YES" +CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) +TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) +MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 +EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) + class solana_cli: def call(self, *args): try: cmd = ["solana", - "--url", solana_url, + "--url", SOLANA_URL, ] + list(args) logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, universal_newlines=True) @@ -28,8 +36,8 @@ def call(self, *args): try: cmd = ["neon-cli", "--commitment=recent", - "--url", solana_url, - "--evm_loader={}".format(evm_loader_id), + "--url", SOLANA_URL, + "--evm_loader={}".format(EVM_LOADER_ID), ] + list(args) logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index 52c9fcf81..fe6bd1342 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -1,8 +1,9 @@ from proxy.indexer.indexer_base import logger, IndexerBase, PARALLEL_REQUESTS import base58 -import rlp import json +import logging import os +import rlp import time import logging from multiprocessing.dummy import Pool as ThreadPool diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index cc579daa0..60425b7a8 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -3,10 +3,10 @@ import json import logging import os -import rlp import psycopg2 +import rlp import subprocess -from construct import Struct, Bytes, Int64ul + from eth_utils import big_endian_to_int from ethereum.transactions import Transaction as EthTrx from ethereum.utils import sha3 @@ -15,18 +15,16 @@ from solana.rpc.api import Client from solana.rpc.commitment import Confirmed from solana.rpc.types import TxOpts +from solana.system_program import SYS_PROGRAM_ID +from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY from solana.transaction import AccountMeta, Transaction, TransactionInstruction from spl.token.constants import TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address from web3.auto.gethdev import w3 -from proxy.environment import solana_url, evm_loader_id, ETH_TOKEN_MINT_ID -sysvarclock = "SysvarC1ock11111111111111111111111111111111" -sysinstruct = "Sysvar1nstructions1111111111111111111111111" -keccakprog = "KeccakSecp256k11111111111111111111111111111" -rentid = "SysvarRent111111111111111111111111111111111" -incinerator = "1nc1nerator11111111111111111111111111111111" -system = "11111111111111111111111111111111" +from ..common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM +from ..common_neon.layouts import STORAGE_ACCOUNT_INFO_LAYOUT +from ..environment import SOLANA_URL, EVM_LOADER_ID, ETH_TOKEN_MINT_ID logger = logging.getLogger(__name__) @@ -46,7 +44,7 @@ def get_trx_results(trx): accounts = trx["transaction"]["message"]["accountKeys"] evm_loader_instructions = [] for idx, instruction in enumerate(trx["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: evm_loader_instructions.append(idx) slot = trx['slot'] @@ -60,7 +58,7 @@ def get_trx_results(trx): for inner in (trx['meta']['innerInstructions']): if inner["index"] in evm_loader_instructions: for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: log = base58.b58decode(event['data']) instruction = log[:1] if (int().from_bytes(instruction, "little") == 7): # OnEvent evmInstruction code @@ -115,22 +113,6 @@ def get_trx_receipts(unsigned_msg, signature): return (trx_raw.hex(), eth_signature, from_address) -STORAGE_ACCOUNT_INFO_LAYOUT = Struct( - # "tag" / Int8ul, - "caller" / Bytes(20), - "nonce" / Int64ul, - "gas_limit" / Int64ul, - "gas_price" / Int64ul, - "slot" / Int64ul, - "operator" / Bytes(32), - "accounts_len" / Int64ul, - "executor_data_size" / Int64ul, - "evm_data_size" / Int64ul, - "gas_used_and_paid" / Int64ul, - "number_of_payments" / Int64ul, - "sign" / Bytes(65), - -) def get_account_list(client, storage_account): opts = { @@ -311,7 +293,7 @@ def __init__(self): values = bytes(numbs) self.signer = Account(values) - self.client = Client(solana_url) + self.client = Client(SOLANA_URL) self.operator = self.signer.public_key() self.operator_token = get_associated_token_address(PublicKey(self.operator), ETH_TOKEN_MINT_ID) @@ -320,7 +302,7 @@ def __init__(self): def call(self, *args): try: cmd = ["solana", - "--url", solana_url, + "--url", SOLANA_URL, ] + list(args) logger.debug(cmd) return subprocess.check_output(cmd, universal_newlines=True) @@ -331,15 +313,15 @@ def call(self, *args): def unlock_accounts(self, blocked_storages): readonly_accs = [ - PublicKey(evm_loader_id), + PublicKey(EVM_LOADER_ID), ETH_TOKEN_MINT_ID, PublicKey(TOKEN_PROGRAM_ID), - PublicKey(sysvarclock), - PublicKey(sysinstruct), - PublicKey(keccakprog), - PublicKey(rentid), - PublicKey(incinerator), - PublicKey(system), + PublicKey(SYSVAR_CLOCK_PUBKEY), + PublicKey(SYSVAR_INSTRUCTION_PUBKEY), + PublicKey(KECCAK_PROGRAM), + PublicKey(SYSVAR_RENT_PUBKEY), + PublicKey(INCINERATOR_PUBKEY), + PublicKey(SYS_PROGRAM_ID), ] for storage, trx_accs in blocked_storages.items(): (eth_trx, blocked_accs) = trx_accs @@ -366,15 +348,15 @@ def unlock_accounts(self, blocked_storages): AccountMeta(pubkey=self.operator, is_signer=True, is_writable=True), AccountMeta(pubkey=self.operator_token, is_signer=False, is_writable=True), AccountMeta(pubkey=acc_list[4], is_signer=False, is_writable=True), - AccountMeta(pubkey=incinerator, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False) + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False) ] for acc in acc_list: keys.append(AccountMeta(pubkey=acc, is_signer=False, is_writable=(False if acc in readonly_accs else True))) trx = Transaction() trx.add(TransactionInstruction( - program_id=evm_loader_id, + program_id=EVM_LOADER_ID, data=bytearray.fromhex("15") + eth_trx[0].to_bytes(8, 'little'), keys=keys )) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index f84a80c6d..e7f9afd19 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -8,35 +8,37 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import List, Tuple, Optional +import base58 import copy -import json -import unittest import eth_utils +import json +import logging import rlp -import solana -from solana.account import Account as sol_Account -from ..common.utils import socket_connection, text_, build_http_response +import threading +import traceback +import unittest + +from ..common.utils import build_http_response from ..http.codes import httpStatusCodes from ..http.parser import HttpParser from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes -from .eth_proto import Trx as EthTrx +from solana.account import Account as sol_Account from solana.rpc.api import Client as SolanaClient, SendTransactionError as SolanaTrxError -from sha3 import keccak_256 -import base58 -import traceback -import threading - -from .solana_rest_api_tools import EthereumAddress, get_token_balance_or_airdrop, getAccountInfo, call_signed, \ - call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE, estimate_gas -from solana.rpc.commitment import Commitment, Confirmed +from solana.rpc.commitment import Confirmed +from typing import List, Tuple, Optional from web3 import Web3 -import logging + +from .eth_proto import Trx as EthTrx +from .solana_rest_api_tools import getAccountInfo, call_signed, neon_config_load, \ + get_token_balance_or_airdrop, estimate_gas +from ..common_neon.address import EthereumAddress +from ..common_neon.emulator_interactor import call_emulated +from ..common_neon.errors import EthereumError from ..core.acceptor.pool import proxy_id_glob -from ..indexer.utils import get_trx_results, LogDB +from ..environment import neon_cli, solana_cli, SOLANA_URL, MINIMAL_GAS_PRICE from ..indexer.sql_dict import SQLDict -from ..environment import evm_loader_id, solana_cli, solana_url, neon_cli +from ..indexer.utils import get_trx_results, LogDB logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -50,7 +52,7 @@ class EthereumModel: def __init__(self): self.signer = self.get_solana_account() - self.client = SolanaClient(solana_url) + self.client = SolanaClient(SOLANA_URL) self.logs_db = LogDB() self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash") @@ -410,8 +412,8 @@ def eth_sendRawTransaction(self, rawTrx): nonce = int(self.eth_getTransactionCount('0x' + sender, None), base=16) - logger.debug('Eth Sender trx nonce: %s', nonce) - logger.debug('Operator nonce: %s', trx.nonce) + logger.debug('Eth Sender trx nonce in solana: %s', nonce) + logger.debug('Eth Sender trx nonce in transaction: %s', trx.nonce) if (int(nonce) != int(trx.nonce)): raise EthereumError(-32002, 'Verifying nonce before send transaction: Error processing Instruction 1: invalid program argument' diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 29207a332..73085b405 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -1,458 +1,27 @@ -import base58 import base64 -import json import logging -import os -import random -import re -import struct -import time -from datetime import datetime -from hashlib import sha256 -from typing import NamedTuple, Optional, Union, Dict, Tuple -import psycopg2 -import rlp -from base58 import b58decode, b58encode -from construct import Bytes, Int8ul, Int32ul, Int64ul -from construct import Struct as cStruct -from eth_keys import keys as eth_keys -import eth_utils - -from sha3 import keccak_256 -from web3.auto import w3 +from datetime import datetime from solana.account import Account as SolanaAccount -from solana.blockhash import Blockhash -from solana.rpc.api import Client as SolanaClient, SendTransactionError -from solana.rpc.commitment import Commitment, Confirmed -from solana.rpc.types import TxOpts -from solana.sysvar import * -from solana.transaction import AccountMeta, Transaction, TransactionInstruction -from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT -from solana._layouts.system_instructions import InstructionType as SystemInstructionType - -from spl.token.constants import ACCOUNT_LEN, ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID -from spl.token.instructions import get_associated_token_address, create_associated_token_account, transfer2, Transfer2Params - -from ..environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params +from solana.publickey import PublicKey +from solana.rpc.api import Client as SolanaClient +from solana.rpc.commitment import Confirmed + +from ..common_neon.address import ether2program, getTokenAddr, EthereumAddress, AccountInfo +from ..common_neon.errors import SolanaAccountNotFoundError, SolanaErrors +from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT +from ..common_neon.neon_instruction import NeonInstruction +from ..common_neon.solana_interactor import SolanaInteractor +from ..common_neon.transaction_sender import TransactionSender +from ..common_neon.emulator_interactor import call_emulated from ..common_neon.utils import get_from_dict -from ..common_neon.errors import * -from .eth_proto import Trx -from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob -from ..indexer.sql_dict import POSTGRES_USER, POSTGRES_HOST, POSTGRES_DB, POSTGRES_PASSWORD +from ..environment import read_elf_params, TIMEOUT_TO_RELOAD_NEON_CONFIG, EXTRA_GAS + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) -location_bin = ".deploy_contract.bin" -confirmation_check_delay = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) -USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "NO") == "YES" -CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) -TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) -MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 - -ACCOUNT_SEED_VERSION=b'\1' - -COLLATERALL_POOL_MAX=10 - -EMPTY_STORAGE_TAG=0 -FINALIZED_STORAGE_TAG=5 - -sysvarclock = "SysvarC1ock11111111111111111111111111111111" -sysinstruct = "Sysvar1nstructions1111111111111111111111111" -keccakprog = "KeccakSecp256k11111111111111111111111111111" -rentid = "SysvarRent111111111111111111111111111111111" -incinerator = "1nc1nerator11111111111111111111111111111111" -system = "11111111111111111111111111111111" - -STORAGE_SIZE = 128 * 1024 - -ACCOUNT_INFO_LAYOUT = cStruct( - "type" / Int8ul, - "ether" / Bytes(20), - "nonce" / Int8ul, - "trx_count" / Bytes(8), - "code_account" / Bytes(32), - "is_rw_blocked" / Int8ul, - "rw_blocked_acc" / Bytes(32), - "eth_token_account" / Bytes(32), - "ro_blocked_cnt" / Int8ul, -) - -CODE_INFO_LAYOUT = cStruct( - "tag" / Int8ul, - "owner" / Bytes(20), - "code_size" / Bytes(4), -) - -CREATE_ACCOUNT_LAYOUT = cStruct( - "lamports" / Int64ul, - "space" / Int64ul, - "ether" / Bytes(20), - "nonce" / Int8ul -) - -obligatory_accounts = [ - AccountMeta(pubkey=evm_loader_id, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=sysvarclock, is_signer=False, is_writable=False), -] - -EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) - - -class SQLCost(): - def __init__(self): - - self.conn = psycopg2.connect( - dbname=POSTGRES_DB, - user=POSTGRES_USER, - password=POSTGRES_PASSWORD, - host=POSTGRES_HOST - ) - - self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - cur = self.conn.cursor() - cur.execute(''' - CREATE TABLE IF NOT EXISTS OPERATOR_COST - ( - hash char(64), - cost bigint, - used_gas bigint, - sender char(40), - to_address char(40) , - sig char(100), - status varchar(100), - reason varchar(100) - )''' - ) - - def close(self): - self.conn.close() - - def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason): - cur = self.conn.cursor() - cur.execute(''' - INSERT INTO OPERATOR_COST (hash, cost, used_gas, sender, to_address, sig, status, reason) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s) - ''', - (hash, cost, used_gas, sender, to_address, sig, status, reason) - ) - -class CostSingleton(object): - def __new__(cls): - if not hasattr(cls, 'instance'): - cls.instance = super(CostSingleton, cls).__new__(cls) - cls.instance.operator_cost = SQLCost() - return cls.instance - -class PermanentAccounts: - def __init__(self, client, signer): - while True: - with new_acc_id_glob.get_lock(): - try: - self.acc_id = acc_list_glob.pop(0) - except IndexError: - self.acc_id = new_acc_id_glob.value - new_acc_id_glob.value += 1 - - logger.debug("LOCK RESOURCES {}".format(self.acc_id)) - - self.operator = signer.public_key() - self.operator_token = getTokenAddr(self.operator) - - acc_id_bytes = self.acc_id.to_bytes((self.acc_id.bit_length() + 7) // 8, 'big') - - storage_seed = keccak_256(b"storage" + acc_id_bytes).hexdigest()[:32] - storage_seed = bytes(storage_seed, 'utf8') - - holder_seed = keccak_256(b"holder" + acc_id_bytes).hexdigest()[:32] - holder_seed = bytes(holder_seed, 'utf8') - - try: - self.storage, self.holder = create_multiple_accounts_with_seed( - client, - funding=signer, - base=signer, - seeds=[storage_seed, holder_seed], - sizes=[STORAGE_SIZE, STORAGE_SIZE] - ) - except Exception as err: - logger.warn("Account is locked err({}) id({}) owner({})".format(str(err), self.acc_id, signer.public_key())) - else: - break - - def __del__(self): - logger.debug("FREE RESOURCES {}".format(self.acc_id)) - with new_acc_id_glob.get_lock(): - acc_list_glob.append(self.acc_id) - - -class TransactionInfo: - def __init__(self, caller_token, eth_accounts, eth_trx): - self.eth_trx = eth_trx - - self.caller_token = caller_token - self.eth_accounts = eth_accounts - self.nonce = eth_trx.nonce - - hash = keccak_256(eth_trx.unsigned_msg()).digest() - collateral_pool_index = int().from_bytes(hash[:4], "little") % COLLATERALL_POOL_MAX - self.collateral_pool_index_buf = collateral_pool_index.to_bytes(4, 'little') - self.collateral_pool_address = create_collateral_pool_address(collateral_pool_index) - - -class AccountInfo(NamedTuple): - ether: eth_keys.PublicKey - trx_count: int - code_account: PublicKey - - @staticmethod - def frombytes(data): - cont = ACCOUNT_INFO_LAYOUT.parse(data) - return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account)) - - -def create_account_layout(lamports, space, ether, nonce): - return bytes.fromhex("02000000")+CREATE_ACCOUNT_LAYOUT.build(dict( - lamports=lamports, - space=space, - ether=ether, - nonce=nonce - )) - - -def write_holder_layout(nonce, offset, data): - return (bytes.fromhex('12')+ - nonce.to_bytes(8, byteorder='little')+ - offset.to_bytes(4, byteorder='little')+ - len(data).to_bytes(8, byteorder='little')+ - data) - - -def get_account_info(client, storage_account): - opts = { - "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 16, - } - } - - result = client._provider.make_request("getAccountInfo", str(storage_account), opts) - logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) - - info = result['result']['value'] - if info is None: - logger.debug("Can't get information about {}".format(storage_account)) - return None - - data = base64.b64decode(info['data'][0]) - - account_tag = data[0] - lamports = info['lamports'] - owner = info['owner'] - - return (account_tag, lamports, owner) - - -def accountWithSeed(base, seed, program): - result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(program)).digest()) - logger.debug('accountWithSeed %s', str(result)) - return result - - -def createAccountWithSeedTrx(funding, base, seed, lamports, space, program): - seed_str = str(seed, 'utf8') - data = SYSTEM_INSTRUCTIONS_LAYOUT.build( - dict( - instruction_type = SystemInstructionType.CREATE_ACCOUNT_WITH_SEED, - args=dict( - base=bytes(base), - seed=dict(length=len(seed_str), chars=seed_str), - lamports=lamports, - space=space, - program_id=bytes(program) - ) - ) - ) - logger.debug("createAccountWithSeedTrx %s %s %s", type(base), base, data.hex()) - created = accountWithSeed(base, seed, PublicKey(program)) - logger.debug("created %s", created) - return TransactionInstruction( - keys=[ - AccountMeta(pubkey=funding, is_signer=True, is_writable=True), - AccountMeta(pubkey=created, is_signer=False, is_writable=True), - AccountMeta(pubkey=base, is_signer=True, is_writable=False), - ], - program_id=system, - data=data - ) - - -def create_collateral_pool_address(collateral_pool_index): - COLLATERAL_SEED_PREFIX = "collateral_seed_" - seed = COLLATERAL_SEED_PREFIX + str(collateral_pool_index) - return accountWithSeed(PublicKey(COLLATERAL_POOL_BASE), str.encode(seed), PublicKey(evm_loader_id)) - - -def create_account_with_seed(client, funding, base, seed, storage_size, eth_trx=None): - account = accountWithSeed(base.public_key(), seed, PublicKey(evm_loader_id)) - - if client.get_balance(account, commitment=Confirmed)['result']['value'] == 0: - minimum_balance = client.get_minimum_balance_for_rent_exemption(storage_size, commitment=Confirmed)["result"] - logger.debug("Minimum balance required for account {}".format(minimum_balance)) - - trx = Transaction() - trx.add(createAccountWithSeedTrx(funding.public_key(), base.public_key(), seed, minimum_balance, storage_size, PublicKey(evm_loader_id))) - send_transaction(client, trx, funding, eth_trx=eth_trx, reason='createAccountWithSeed') - - return account - - -def create_multiple_accounts_with_seed(client, funding, base, seeds, sizes): - accounts = [] - trx = Transaction() - - for seed, storage_size in zip(seeds, sizes): - account = accountWithSeed(base.public_key(), seed, PublicKey(evm_loader_id)) - accounts.append(account) - - minimum_balance = client.get_minimum_balance_for_rent_exemption(storage_size, commitment=Confirmed)["result"] - - account_info = get_account_info(client, account) - if account_info is None: - logger.debug("Minimum balance required for account {}".format(minimum_balance)) - - trx.add(createAccountWithSeedTrx(funding.public_key(), base.public_key(), seed, minimum_balance, storage_size, PublicKey(evm_loader_id))) - else: - (tag, lamports, owner) = account_info - if lamports < minimum_balance: - raise Exception("insufficient balance") - if PublicKey(owner) != PublicKey(evm_loader_id): - raise Exception("wrong owner") - if tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}: - raise Exception("not empty, not finalized") - - if len(trx.instructions) > 0: - send_transaction(client, trx, funding) - - return accounts - - -def make_keccak_instruction_data(check_instruction_index, msg_len, data_start): - if check_instruction_index > 255 and check_instruction_index < 0: - raise Exception("Invalid index for instruction - {}".format(check_instruction_index)) - - check_count = 1 - eth_address_size = 20 - signature_size = 65 - eth_address_offset = data_start - signature_offset = eth_address_offset + eth_address_size - message_data_offset = signature_offset + signature_size - - data = struct.pack("B", check_count) - data += struct.pack("= confirmations): - return - time.sleep(confirmation_check_delay) - elapsed_time += confirmation_check_delay - raise RuntimeError("could not confirm transaction: ", tx_sig) - - -def solana2ether(public_key): - from web3 import Web3 - return bytes(Web3.keccak(bytes.fromhex(public_key))[-20:]) - - -def ether2program(ether): - if isinstance(ether, str): - pass - elif isinstance(ether, EthereumAddress): - ether = str(ether) - else: - ether = ether.hex() - output = neon_cli().call("create-program-address", ether) - items = output.rstrip().split(' ') - return items[0], int(items[1]) - - -def ether2seed(ether, program_id, base): - if isinstance(ether, str): - if ether.startswith('0x'): ether = ether[2:] - else: ether = ether.hex() - seed = b58encode(bytes.fromhex(ether)) - acc = accountWithSeed(base, seed, PublicKey(program_id)) - logger.debug('ether2program: {} {} => {} (seed {})'.format(ether, 255, acc, seed)) - return acc, 255, seed - - def neon_config_load(ethereum_model): try: ethereum_model.neon_config_dict @@ -475,614 +44,12 @@ def neon_config_load(ethereum_model): logger.debug(ethereum_model.neon_config_dict) -def call_emulated(contract_id, caller_id, data=None, value=None): - output = emulator(contract_id, caller_id, data, value) - logger.debug("call_emulated %s %s %s %s return %s", contract_id, caller_id, data, value, output) - result = json.loads(output) - exit_status = result['exit_status'] - if exit_status == 'revert': - result_value = result['result'] - if len(result_value) < 8 or result_value[:8] != '08c379a0': - raise EthereumError(code=3, message='execution reverted') - - offset = int(result_value[8:8+64], 16) - length = int(result_value[8+64:8+64+64], 16) - message = str(bytes.fromhex(result_value[8+offset*2+64:8+offset*2+64+length*2]), 'utf8') - raise EthereumError(code=3, message='execution reverted: '+message, data='0x'+result_value) - if result["exit_status"] != "succeed": - raise Exception("evm emulator error ", result) - return result - - -def extract_measurements_from_receipt(receipt): - log_messages = receipt['result']['meta']['logMessages'] - transaction = receipt['result']['transaction'] - accounts = transaction['message']['accountKeys'] - instructions = [] - for instr in transaction['message']['instructions']: - program = accounts[instr['programIdIndex']] - instructions.append({ - 'accs': [accounts[acc] for acc in instr['accounts']], - 'program': accounts[instr['programIdIndex']], - 'data': b58decode(instr['data']).hex() - }) - - pattern = re.compile('Program ([0-9A-Za-z]+) (.*)') - messages = [] - for log in log_messages: - res = pattern.match(log) - if res: - (program, reason) = res.groups() - if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) - messages[-1]['logs'].append(log) - - for instr in instructions: - if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue - if messages[0]['program'] != instr['program']: - raise Exception('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) - instr['logs'] = messages.pop(0)['logs'] - exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) - if not exit_result: raise Exception("Can't get exit result") - instr['result'] = exit_result.group(1) - - if instr['program'] == evm_loader_id: - memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) - instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) - if not (memory_result and instruction_result): - raise Exception("Can't parse measurements for evm_loader") - instr['measurements'] = { - 'instructions': instruction_result.group(1), - 'memory': memory_result.group(1) - } - - result = [] - for instr in instructions: - if instr['program'] == evm_loader_id: - result.append({ - 'program':instr['program'], - 'measurements':instr['measurements'], - 'result':instr['result'], - 'data':instr['data'] - }) - return result - -# Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh) -def get_measurements(result): - try: - measurements = extract_measurements_from_receipt(result) - for m in measurements: logger.info(json.dumps(m)) - except Exception as err: - logger.error("Can't get measurements %s"%err) - logger.info("Failed result: %s"%json.dumps(result, indent=3)) - - -def send_transaction(client, trx, signer, eth_trx=None, reason=None): - result = client.send_transaction(trx, signer, opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed)) - confirm_transaction(client, result["result"]) - result = client.get_confirmed_transaction(result["result"]) - update_transaction_cost(result, eth_trx, reason=reason) - return result - - -def send_measured_transaction(client, trx, signer, eth_trx, reason): - result = send_transaction(client, trx, signer, eth_trx=eth_trx, reason=reason) - get_measurements(result) - return result - - -def check_if_program_exceeded_instructions(err_result): - err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" - err_budget = "failed: Computational budget exceeded" - - if err_result['data']['logs'][-1].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-2].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-1].find(err_budget) >= 0: - return True - return False - - -def check_if_continue_returned(result): - tx_info = result['result'] - accounts = tx_info["transaction"]["message"]["accountKeys"] - evm_loader_instructions = [] - - for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: - evm_loader_instructions.append(idx) - - for inner in (tx_info['meta']['innerInstructions']): - if inner["index"] in evm_loader_instructions: - for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: - instruction = base58.b58decode(event['data'])[:1] - if int().from_bytes(instruction, "little") == 6: # OnReturn evmInstruction code - return (True, tx_info['transaction']['signatures'][0]) - return (False, ()) - - -def call_continue(signer, client, perm_accs, trx_info, steps): - try: - return call_continue_iterative(signer, client, perm_accs, trx_info, steps) - except Exception as err: - logger.debug("call_continue_iterative exception:") - logger.debug(str(err)) - - return sol_instr_21_cancel(signer, client, perm_accs, trx_info) - - -def call_continue_iterative(signer, client, perm_accs, trx_info, step_count): - while True: - logger.debug("Continue iterative step:") - result = sol_instr_10_continue(signer, client, perm_accs, trx_info, step_count) - (succeed, signature) = check_if_continue_returned(result) - if succeed: - return signature - - -def sol_instr_10_continue(signer, client, perm_accs, trx_info, initial_step_count): - step_count = initial_step_count - while step_count > 0: - trx = Transaction() - trx.add(make_continue_instruction(signer, perm_accs, trx_info, step_count)) - - logger.debug("Step count {}".format(step_count)) - try: - result = send_measured_transaction(client, trx, signer, trx_info.eth_trx, 'ContinueV02') - return result - except SendTransactionError as err: - if check_if_program_exceeded_instructions(err.result): - step_count = int(step_count * 90 / 100) - else: - raise - raise Exception("Can't execute even one EVM instruction") - - -def sol_instr_21_cancel(signer, client, perm_accs, trx_info): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - trx = Transaction() - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=bytearray.fromhex("15") + trx_info.nonce.to_bytes(8, 'little'), - keys=[ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=incinerator, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - )) - - logger.debug("Cancel") - result = send_measured_transaction(client, trx, signer, trx_info.eth_trx, 'CancelWithNonce') - return result['result']['transaction']['signatures'][0] - - -def make_partial_call_instruction(signer, perm_accs, trx_info, step_count, call_data): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("13") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little") + call_data, - keys = [ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_continue_instruction(signer, perm_accs, trx_info, step_count, index=None): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - data = bytearray.fromhex("14") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little") - if index: - data = data + index.to_bytes(8, byteorder="little") - - return TransactionInstruction( - program_id = evm_loader_id, - data = data, - keys = [ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_call_from_account_instruction(signer, perm_accs, trx_info, step_count = 0): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("16") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little"), - keys = [ - AccountMeta(pubkey=perm_accs.holder, is_signer=False, is_writable=True), - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_05_call_instruction(signer, trx_info, call_data): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("05") + trx_info.collateral_pool_index_buf + call_data, - keys = [ - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + obligatory_accounts - ) - - -def update_transaction_cost(receipt, eth_trx, extra_sol_trx=False, reason=None): - cost = receipt['result']['meta']['preBalances'][0] - receipt['result']['meta']['postBalances'][0] - if eth_trx: - hash = eth_trx.hash_signed().hex() - sender = eth_trx.sender() - to_address = eth_trx.toAddress.hex() if eth_trx.toAddress else "None" - else: - hash = None - sender = None - to_address = None - - sig = receipt['result']['transaction']['signatures'][0] - used_gas=None - - tx_info = receipt['result'] - accounts = tx_info["transaction"]["message"]["accountKeys"] - evm_loader_instructions = [] - - for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: - evm_loader_instructions.append(idx) - - for inner in (tx_info['meta']['innerInstructions']): - if inner["index"] in evm_loader_instructions: - for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: - used_gas = base58.b58decode(event['data'])[2:10] - used_gas = int().from_bytes(used_gas, "little") - - table = CostSingleton().operator_cost - table.insert( - hash, - cost, - used_gas if used_gas else 0, - sender, - to_address, - sig, - 'extra' if extra_sol_trx else 'ok', - reason if reason else '' - ) - - -def create_account_list_by_emulate(signer, client, eth_trx): - sender_ether = bytes.fromhex(eth_trx.sender()) - add_keys_05 = [] - trx = Transaction() - - if not eth_trx.toAddress: - to_address_arg = "deploy" - to_address = keccak_256(rlp.encode((bytes.fromhex(eth_trx.sender()), eth_trx.nonce))).digest()[-20:] - else: - to_address_arg = eth_trx.toAddress.hex() - to_address = eth_trx.toAddress - - output_json = call_emulated(to_address_arg, sender_ether.hex(), eth_trx.callData.hex(), hex(eth_trx.value)) - logger.debug("emulator returns: %s", json.dumps(output_json, indent=3)) - - # resize storage account - resize_instr = [] - for acc_desc in output_json["accounts"]: - if acc_desc["new"] == False: - - if acc_desc.get("code_size_current") is not None and acc_desc.get("code_size") is not None: - if acc_desc["code_size"] > acc_desc["code_size_current"]: - code_size = acc_desc["code_size"] + 2048 - seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) - code_account_new = accountWithSeed(signer.public_key(), seed, PublicKey(evm_loader_id)) - - logger.debug("creating new code_account with increased size %s", code_account_new) - create_account_with_seed(client, signer, signer, seed, code_size, eth_trx); - logger.debug("resized account is created %s", code_account_new) - - resize_instr.append(TransactionInstruction( - keys=[ - AccountMeta(pubkey=PublicKey(acc_desc["account"]), is_signer=False, is_writable=True), - ( - AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=True) - if acc_desc["contract"] else - AccountMeta(pubkey=PublicKey("11111111111111111111111111111111"), is_signer=False, is_writable=False) - ), - AccountMeta(pubkey=code_account_new, is_signer=False, is_writable=True), - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=False) - ], - program_id=evm_loader_id, - data=bytearray.fromhex("11")+bytes(seed) # 17- ResizeStorageAccount - )) - # replace code_account - acc_desc["contract"] = code_account_new - - for instr in resize_instr: - logger.debug("code and storage migration, account %s from %s to %s", instr.keys[0].pubkey, instr.keys[1].pubkey, instr.keys[2].pubkey) - - tx = Transaction().add(instr) - success = False - count = 0 - - while count < 2: - logger.debug("attemt: %d", count) - - send_transaction(client, tx, signer, eth_trx=eth_trx, reason='resize_storage_account') - info = _getAccountData(client, instr.keys[0].pubkey, ACCOUNT_INFO_LAYOUT.sizeof()) - info_data = AccountInfo.frombytes(info) - if info_data.code_account == instr.keys[2].pubkey: - success = True - logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) - break - time.sleep(1) - count = count+1 - - if success == False: - raise Exception("Can't resize storage account. Account is blocked {}".format(instr.keys[0].pubkey)) - - for acc_desc in output_json["accounts"]: - address = bytes.fromhex(acc_desc["address"][2:]) - - code_account = None - code_account_writable = False - if acc_desc["new"]: - logger.debug("Create solana accounts for %s: %s %s", acc_desc["address"], acc_desc["account"], acc_desc["contract"]) - if acc_desc["code_size"]: - seed = b58encode(ACCOUNT_SEED_VERSION+address) - code_account = accountWithSeed(signer.public_key(), seed, PublicKey(evm_loader_id)) - logger.debug(" with code account %s", code_account) - code_size = acc_desc["code_size"] + 2048 - code_account_balance = client.get_minimum_balance_for_rent_exemption(code_size)["result"] - trx.add(createAccountWithSeedTrx(signer.public_key(), signer.public_key(), seed, code_account_balance, code_size, PublicKey(evm_loader_id))) - code_account_writable = acc_desc["writable"] - - extend_trx_with_create_and_airdrop(signer, EthereumAddress(address), code_account, trx=trx) - - if address == to_address: - contract_sol = PublicKey(acc_desc["account"]) - if acc_desc["new"]: - code_sol = code_account - code_writable = code_account_writable - else: - if acc_desc["contract"] != None: - code_sol = PublicKey(acc_desc["contract"]) - code_writable = acc_desc["writable"] - else: - code_sol = None - code_writable = None - - elif address == sender_ether: - sender_sol = PublicKey(acc_desc["account"]) - else: - add_keys_05.append(AccountMeta(pubkey=acc_desc["account"], is_signer=False, is_writable=True)) - token_account = get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID) - add_keys_05.append(AccountMeta(pubkey=token_account, is_signer=False, is_writable=True)) - if acc_desc["new"]: - if code_account: - add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=code_account_writable)) - else: - if acc_desc["contract"]: - add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) - - for token_account in output_json["token_accounts"]: - add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) - - if token_account["new"]: - trx.add(createERC20TokenAccountTrx(signer, token_account)) - - for account_meta in output_json["solana_accounts"]: - add_keys_05.append(AccountMeta(pubkey=PublicKey(account_meta["pubkey"]), is_signer=account_meta["is_signer"], is_writable=account_meta["is_writable"])) - - caller_token = get_associated_token_address(PublicKey(sender_sol), ETH_TOKEN_MINT_ID) - - eth_accounts = [ - AccountMeta(pubkey=contract_sol, is_signer=False, is_writable=True), - AccountMeta(pubkey=get_associated_token_address(contract_sol, ETH_TOKEN_MINT_ID), is_signer=False, is_writable=True), - ] + ([AccountMeta(pubkey=code_sol, is_signer=False, is_writable=code_writable)] if code_sol != None else []) + [ - AccountMeta(pubkey=sender_sol, is_signer=False, is_writable=True), - AccountMeta(pubkey=caller_token, is_signer=False, is_writable=True), - ] + add_keys_05 - - trx_info = TransactionInfo(caller_token, eth_accounts, eth_trx) - - return trx_info, sender_ether, trx - - def call_signed(signer, client, eth_trx, steps): - - (trx_info, sender_ether, create_acc_trx) = create_account_list_by_emulate(signer, client, eth_trx) - - call_iterative = False - call_from_holder = False - - if not eth_trx.toAddress: - call_from_holder = True - else: - msg = sender_ether + eth_trx.signature() + eth_trx.unsigned_msg() - - try: - logger.debug("Try single trx call") - return call_signed_noniterative(signer, client, eth_trx, trx_info, msg, create_acc_trx) - except Exception as err: - logger.debug(str(err)) - errStr = str(err) - if "Program failed to complete" in errStr or "Computational budget exceeded" in errStr: - logger.debug("Program exceeded instructions") - call_iterative = True - elif str(err).startswith("transaction too large:"): - logger.debug("Transaction too large, call call_signed_with_holder_acc():") - call_from_holder = True - else: - raise - - perm_accs = PermanentAccounts(client, signer) - try: - if call_iterative: - try: - return call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx) - except Exception as err: - logger.debug(str(err)) - if str(err).startswith("transaction too large:"): - logger.debug("Transaction too large, call call_signed_with_holder_acc():") - call_from_holder = True - else: - raise - - if call_from_holder: - return call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, steps, create_acc_trx) - finally: - del perm_accs - - -def call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx): - if len(create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(create_acc_trx) - send_measured_transaction(client, precall_txs, signer, eth_trx, 'CreateAccountsForTrx') - - precall_txs = Transaction() - precall_txs.add(TransactionInstruction( - program_id=keccakprog, - data=make_keccak_instruction_data(len(precall_txs.instructions)+1, len(eth_trx.unsigned_msg()), data_start=13), - keys=[ - AccountMeta(pubkey=keccakprog, is_signer=False, is_writable=False), - ])) - precall_txs.add(make_partial_call_instruction(signer, perm_accs, trx_info, 0, msg)) - - logger.debug("Partial call") - send_measured_transaction(client, precall_txs, signer, eth_trx, 'PartialCallFromRawEthereumTXv02') - - return call_continue(signer, client, perm_accs, trx_info, steps) - - -def call_signed_noniterative(signer, client, eth_trx, trx_info, msg, create_acc_trx): - call_txs_05 = Transaction() - call_txs_05.add(create_acc_trx) - call_txs_05.add(TransactionInstruction( - program_id=keccakprog, - data=make_keccak_instruction_data(len(call_txs_05.instructions)+1, len(eth_trx.unsigned_msg()), 5), - keys=[ - AccountMeta(pubkey=keccakprog, is_signer=False, is_writable=False), - ])) - call_txs_05.add(make_05_call_instruction(signer, trx_info, msg)) - result = send_measured_transaction(client, call_txs_05, signer, eth_trx, 'CallFromRawEthereumTX') - return result['result']['transaction']['signatures'][0] - - -def call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, steps, create_acc_trx): - - write_trx_to_holder_account(signer, client, perm_accs.holder, perm_accs.acc_id, eth_trx) - if len(create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(create_acc_trx) - send_measured_transaction(client, precall_txs, signer, eth_trx, 'create_accounts_for_deploy') - - precall_txs = Transaction() - precall_txs.add(make_call_from_account_instruction(signer, perm_accs, trx_info)) - - # ExecuteTrxFromAccountDataIterative - logger.debug("ExecuteTrxFromAccountDataIterative:") - send_measured_transaction(client, precall_txs, signer, eth_trx, 'ExecuteTrxFromAccountDataIterativeV02') - - return call_continue(signer, client, perm_accs, trx_info, steps) + solana_interactor = SolanaInteractor(signer, client) + trx_sender = TransactionSender(solana_interactor, eth_trx, steps) + return trx_sender.execute() -def createERC20TokenAccountTrx(signer, token_info): - trx = Transaction() - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=bytes.fromhex('0F'), - keys=[ - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["key"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["owner"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["contract"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["mint"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - - return trx - - -def write_trx_to_holder_account(signer, client, holder, acc_id, eth_trx): - msg = eth_trx.signature() + len(eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + eth_trx.unsigned_msg() - - # Write transaction to transaction holder account - offset = 0 - receipts = [] - rest = msg - while len(rest): - (part, rest) = (rest[:1000], rest[1000:]) - trx = Transaction() - # logger.debug("sender_sol %s %s %s", sender_sol, holder, acc.public_key()) - trx.add(TransactionInstruction(program_id=evm_loader_id, - data=write_holder_layout(acc_id, offset, part), - keys=[ - AccountMeta(pubkey=holder, is_signer=False, is_writable=True), - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=False), - ])) - receipts.append(client.send_transaction(trx, signer, - opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed))["result"]) - offset += len(part) - logger.debug("receipts %s", receipts) - for rcpt in receipts: - confirm_transaction(client, rcpt) - result = client.get_confirmed_transaction(rcpt) - update_transaction_cost(result, eth_trx, reason='WriteHolder') - logger.debug("confirmed: %s", rcpt) - def _getAccountData(client, account, expected_length, owner=None): info = client.get_account_info(account, commitment=Confirmed)['result']['value'] @@ -1101,84 +68,9 @@ def getAccountInfo(client, eth_account: EthereumAddress): return AccountInfo.frombytes(info) -def getLamports(client, eth_account): - pda_account, nonce = ether2program(eth_account) - return int(client.get_balance(pda_account, commitment=Confirmed)['result']['value']) - - -def make_create_eth_account_trx(signer: SolanaAccount, eth_address: EthereumAddress, evm_loader_id, code_acc=None) \ - -> Tuple[Transaction, PublicKey]: - - solana_address, nonce = ether2program(eth_address) - token_acc_address = get_associated_token_address(PublicKey(solana_address), ETH_TOKEN_MINT_ID) - logger.debug(f'Create eth account: {eth_address}, sol account: {solana_address}, token_acc_address: {token_acc_address}, nonce: {nonce}') - - base = signer.public_key() - data = bytes.fromhex('02000000') + CREATE_ACCOUNT_LAYOUT.build(dict(lamports=0, - space=0, - ether=bytes(eth_address), - nonce=nonce)) - trx = Transaction() - if code_acc is None: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), - AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - else: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(solana_address), is_signer=False, is_writable=True), - AccountMeta(pubkey=token_acc_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - return trx, token_acc_address - - -def make_transfer_instruction(owner_pda_account: SolanaAccount, associated_token_account: PublicKey) -> TransactionInstruction: - owner_pda_address = owner_pda_account.public_key() - owner_associated_token_account = getTokenAddr(owner_pda_address) - transfer_instruction = transfer2(Transfer2Params(source=owner_associated_token_account, - owner=owner_pda_address, - dest=associated_token_account, - amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei, - decimals=9, - mint=ETH_TOKEN_MINT_ID, - program_id=TOKEN_PROGRAM_ID)) - logger.debug(f"Token transfer from token: {owner_associated_token_account}, owned by: {owner_pda_address}, to token: " - f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}") - return transfer_instruction - - -def extend_trx_with_create_and_airdrop(signer: SolanaAccount, eth_account: EthereumAddress, code_acc=None, *, trx): - create_trx, associated_token_account = make_create_eth_account_trx(signer, eth_account, evm_loader_id, code_acc) - trx.add(create_trx) - if NEW_USER_AIRDROP_AMOUNT <= 0: - return - transfer_instruction = make_transfer_instruction(signer, associated_token_account) - trx.add(transfer_instruction) - - def create_eth_account_and_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress): - trx = Transaction() - extend_trx_with_create_and_airdrop(signer, eth_account, trx=trx) - result = send_transaction(client, trx, signer) + trx = NeonInstruction(signer.public_key()).make_trx_with_create_and_airdrop (eth_account) + result = SolanaInteractor(signer, client).send_transaction(trx, None, reason='create_eth_account_and_airdrop') error = result.get("error") if error is not None: logger.error(f"Failed to create eth_account and airdrop: {eth_account}, error occurred: {error}") @@ -1186,77 +78,34 @@ def create_eth_account_and_airdrop(client: SolanaClient, signer: SolanaAccount, def get_token_balance_gwei(client: SolanaClient, pda_account: str) -> int: - associated_token_account = get_associated_token_address(PublicKey(pda_account), ETH_TOKEN_MINT_ID) - rpc_response = client.get_token_account_balance(associated_token_account, commitment=Confirmed) + neon_token_account = getTokenAddr(PublicKey(pda_account)) + rpc_response = client.get_token_account_balance(neon_token_account, commitment=Confirmed) error = rpc_response.get('error') if error is not None: message = error.get("message") if message == SolanaErrors.AccountNotFound.value: raise SolanaAccountNotFoundError() - logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, " + logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, " f"got get_token_account_balance error: \"{message}\"") raise Exception("Getting balance error") balance = get_from_dict(rpc_response, "result", "value", "amount") if balance is None: - logger.error(f"Failed to get_token_balance_gwei by associated_token_account: {associated_token_account}, response: {rpc_response}") + logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, response: {rpc_response}") raise Exception("Unexpected get_balance response") return int(balance) def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress) -> int: - associated_token_account, nonce = ether2program(eth_account) - logger.debug(f"Get balance for eth account: {eth_account} aka: {associated_token_account}") + solana_account, nonce = ether2program(eth_account) + logger.debug(f"Get balance for eth account: {eth_account} aka: {solana_account}") try: - return get_token_balance_gwei(client, associated_token_account) + return get_token_balance_gwei(client, solana_account) except SolanaAccountNotFoundError: - logger.debug(f"Account not found: {eth_account} aka: {associated_token_account} - create") + logger.debug(f"Account not found: {eth_account} aka: {solana_account} - create") create_eth_account_and_airdrop(client, signer, eth_account) - return get_token_balance_gwei(client, associated_token_account) - - -def getTokenAddr(account): - return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) - - -def make_instruction_data_from_tx(instruction, private_key=None): - if isinstance(instruction, dict): - if instruction.get('chainId') is None: - raise Exception("chainId value is needed in input dict") - if private_key is None: - raise Exception("Needed private key for transaction creation from fields") - - signed_tx = w3.eth.account.sign_transaction(instruction, private_key) - # logger.debug(signed_tx.rawTransaction.hex()) - _trx = Trx.fromString(signed_tx.rawTransaction) - # logger.debug(json.dumps(_trx.__dict__, cls=JsonEncoder, indent=3)) - - raw_msg = _trx.get_msg(instruction['chainId']) - sig = keys.Signature(vrs=[1 if _trx.v % 2 == 0 else 0, _trx.r, _trx.s]) - pub = sig.recover_public_key_from_msg_hash(_trx.hash()) - - # logger.debug(pub.to_hex()) - - return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) - elif isinstance(instruction, str): - if instruction[:2] == "0x": - instruction = instruction[2:] - - _trx = Trx.fromString(bytearray.fromhex(instruction)) - # logger.debug(json.dumps(_trx.__dict__, cls=JsonEncoder, indent=3)) - - raw_msg = _trx.get_msg() - sig = keys.Signature(vrs=[1 if _trx.v % 2 == 0 else 0, _trx.r, _trx.s]) - pub = sig.recover_public_key_from_msg_hash(_trx.hash()) - - data = pub.to_canonical_address() - data += sig.to_bytes() - data += raw_msg - - return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) - else: - raise Exception("function gets ") + return get_token_balance_gwei(client, solana_account) def is_account_exists(client: SolanaClient, eth_account: EthereumAddress) -> bool: diff --git a/proxy/proxy.py b/proxy/proxy.py index b4eb08753..86264a6e1 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -24,7 +24,7 @@ from multiprocessing import Process from .indexer.indexer import run_indexer -from proxy.environment import solana_url, evm_loader_id +from proxy.environment import SOLANA_URL, EVM_LOADER_ID logger = logging.getLogger(__name__) @@ -47,8 +47,8 @@ def delete_pid_file(self) -> None: def __enter__(self) -> 'Proxy': self.indexer = Process(target=run_indexer, - args=(solana_url, - evm_loader_id,)) + args=(SOLANA_URL, + EVM_LOADER_ID,)) self.indexer.start() self.acceptors = AcceptorPool( flags=self.flags, diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index fdb38c24b..602fb50e2 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -19,7 +19,7 @@ if [ "$(spl-token balance "$NEON_TOKEN_MINT" || echo 0)" -eq 0 ]; then echo 'Create balance and mint token' TOKEN_ACCOUNT=$( (spl-token create-account "$NEON_TOKEN_MINT" || true) | grep -Po 'Creating account \K[^\n]*') echo "TOKEN_ACCOUNT=$TOKEN_ACCOUNT" - spl-token mint "$NEON_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*1000)) --owner /spl/bin/evm_loader-keypair.json -- "$TOKEN_ACCOUNT" + spl-token mint "$NEON_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*100000)) --owner /spl/bin/evm_loader-keypair.json -- "$TOKEN_ACCOUNT" fi proxy/run-proxy.sh diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py index 9fc778585..2c7f5e4a7 100644 --- a/proxy/testing/test_airdropping_eth_accounts.py +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -11,7 +11,8 @@ from solana.rpc.api import Client as SolanaClient from ..plugin.solana_rest_api import EthereumModel -from ..plugin.solana_rest_api_tools import get_token_balance_gwei, ether2program +from ..plugin.solana_rest_api_tools import get_token_balance_gwei +from ..common_neon.address import ether2program class TestAirdroppingEthAccounts(unittest.TestCase): diff --git a/proxy/testing/test_erc20_wrapper_contract.py b/proxy/testing/test_erc20_wrapper_contract.py index 75030130b..b745c7e76 100644 --- a/proxy/testing/test_erc20_wrapper_contract.py +++ b/proxy/testing/test_erc20_wrapper_contract.py @@ -15,7 +15,7 @@ from solana.account import Account as SolanaAccount from solana.publickey import PublicKey -from proxy.plugin.solana_rest_api_tools import createERC20TokenAccountTrx +from proxy.common_neon.neon_instruction import NeonInstruction # install_solc(version='latest') install_solc(version='0.7.6') @@ -171,7 +171,9 @@ def create_token_accounts(self): admin_token_seeds = [ b"\1", b"ERC20Balance", bytes(self.token.pubkey), contract_address_bytes, admin_address_bytes ] admin_token_key = PublicKey.find_program_address(admin_token_seeds, evm_loader_id)[0] admin_token_info = { "key": admin_token_key, "owner": admin_address_solana, "contract": contract_address_solana, "mint": self.token.pubkey } - self.solana_client.send_transaction(createERC20TokenAccountTrx(self.solana_account, admin_token_info), self.solana_account, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) + + instr = NeonInstruction(self.solana_account.public_key()).createERC20TokenAccountTrx(admin_token_info) + self.solana_client.send_transaction(instr, self.solana_account, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) self.token.mint_to(admin_token_key, self.solana_account, 10_000_000_000_000, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) def test_erc20_name(self): diff --git a/proxy/testing/test_cancel_hanged.py b/proxy/testing/test_indexer_cancel_hanged.py similarity index 95% rename from proxy/testing/test_cancel_hanged.py rename to proxy/testing/test_indexer_cancel_hanged.py index 1e91a1381..e0324df2e 100644 --- a/proxy/testing/test_cancel_hanged.py +++ b/proxy/testing/test_indexer_cancel_hanged.py @@ -1,7 +1,8 @@ import os import sys -from proxy.plugin.solana_rest_api_tools import sysinstruct, ETH_TOKEN_MINT_ID, system, send_transaction, MINIMAL_GAS_PRICE +from proxy.common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY +from proxy.environment import ETH_TOKEN_MINT_ID, MINIMAL_GAS_PRICE os.environ['SOLANA_URL'] = "http://solana:8899" os.environ['EVM_LOADER'] = "53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io" @@ -17,6 +18,7 @@ from ethereum.utils import sha3 from solana.publickey import PublicKey from solana.rpc.commitment import Confirmed +from solana.system_program import SYS_PROGRAM_ID from solana.transaction import AccountMeta, Transaction, TransactionInstruction from solana_utils import * from solcx import install_solc @@ -73,7 +75,7 @@ class CancelTest(unittest.TestCase): @classmethod def setUpClass(cls): - print("\ntest_event.py setUpClass") + print("\ntest_cancel_hanged.py setUpClass") cls.token = SplToken(solana_url) wallet = WalletAccount(wallet_path()) @@ -183,7 +185,7 @@ def sol_instr_19_partial_call(self, storage_account, step_count, evm_instruction AccountMeta(pubkey=storage_account, is_signer=False, is_writable=True), # System instructions account: - AccountMeta(pubkey=PublicKey(sysinstruct), is_signer=False, is_writable=False), + AccountMeta(pubkey=PublicKey(SYSVAR_INSTRUCTION_PUBKEY), is_signer=False, is_writable=False), # Operator address: AccountMeta(pubkey=self.acc.public_key(), is_signer=True, is_writable=True), # Collateral pool address: @@ -193,7 +195,7 @@ def sol_instr_19_partial_call(self, storage_account, step_count, evm_instruction # User's NEON token account: AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), # System program account: - AccountMeta(pubkey=PublicKey(system), is_signer=False, is_writable=False), + AccountMeta(pubkey=PublicKey(SYS_PROGRAM_ID), is_signer=False, is_writable=False), AccountMeta(pubkey=self.reId, is_signer=False, is_writable=True), AccountMeta(pubkey=self.reId_token, is_signer=False, is_writable=True), diff --git a/proxy/testing/test_resize_storage_account.py b/proxy/testing/test_resize_storage_account.py index a2aece6fd..01e2319c1 100644 --- a/proxy/testing/test_resize_storage_account.py +++ b/proxy/testing/test_resize_storage_account.py @@ -2,8 +2,8 @@ from web3 import Web3 import unittest from solana.publickey import PublicKey -from solcx import compile_source -from solcx import install_solc +from solcx import compile_source, install_solc + proxy_url = os.environ.get('PROXY_URL', 'http://127.0.0.1:9090/solana') solana_url = os.environ.get("SOLANA_URL", "http://127.0.0.1:8899") diff --git a/proxy/testing/test_web3_clientVersion.py b/proxy/testing/test_web3_clientVersion.py index 23c8afa45..791549bef 100644 --- a/proxy/testing/test_web3_clientVersion.py +++ b/proxy/testing/test_web3_clientVersion.py @@ -1,11 +1,6 @@ import unittest import os from web3 import Web3 -from solcx import install_solc - -# install_solc(version='latest') -install_solc(version='0.7.0') -from solcx import compile_source proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') proxy = Web3(Web3.HTTPProvider(proxy_url)) From 5ced8095583bfb34d379c9da843286fc27aa3f3b Mon Sep 17 00:00:00 2001 From: ivandzen Date: Fri, 3 Dec 2021 17:41:14 +0300 Subject: [PATCH 46/54] =?UTF-8?q?#337=20=D1=81reate=20base=20airdropper=20?= =?UTF-8?q?service=20(#343)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cherrypick part of changes * create indexer.py * remove solana_receipts_update.py * Cherry pick files from old branch * add requirement * fix refactoring issues * Fix inspection issues * fix last issue * simplify tests * add test Co-authored-by: ivanl --- proxy/indexer/airdropper.py | 175 +++++++++++ proxy/testing/mock_server.py | 28 ++ proxy/testing/test_airdropper.py | 176 +++++++++++ proxy/testing/transactions.py | 491 +++++++++++++++++++++++++++++++ requirements.txt | 1 + 5 files changed, 871 insertions(+) create mode 100644 proxy/indexer/airdropper.py create mode 100644 proxy/testing/mock_server.py create mode 100644 proxy/testing/test_airdropper.py create mode 100644 proxy/testing/transactions.py diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py new file mode 100644 index 000000000..12353fe09 --- /dev/null +++ b/proxy/indexer/airdropper.py @@ -0,0 +1,175 @@ +from proxy.indexer.indexer_base import IndexerBase, logger +import os +import requests +import base58 +import json +import logging + +try: + from utils import check_error + from sql_dict import SQLDict +except ImportError: + from .utils import check_error + from .sql_dict import SQLDict + +class Airdropper(IndexerBase): + def __init__(self, + solana_url, + evm_loader_id, + faucet_url = '', + wrapper_whitelist = [], + airdrop_amount = 10, + log_level = 'INFO'): + IndexerBase.__init__(self, solana_url, evm_loader_id, log_level) + + # collection of eth-address-to-create-accout-trx mappings + # for every addresses that was already funded with airdrop + self.airdrop_ready = SQLDict(tablename="airdrop_ready") + self.wrapper_whitelist = wrapper_whitelist + self.airdrop_amount = airdrop_amount + self.faucet_url = faucet_url + + + # helper function checking if given contract address is in whitelist + def _is_allowed_wrapper_contract(self, contract_addr): + return contract_addr in self.wrapper_whitelist + + + # helper function checking if given 'create account' corresponds to 'create erc20 token account' instruction + def _check_create_instr(self, account_keys, create_acc, create_token_acc): + # Must use the same Ethereum account + if account_keys[create_acc['accounts'][1]] != account_keys[create_token_acc['accounts'][2]]: + return False + # Must use the same token program + if account_keys[create_acc['accounts'][5]] != account_keys[create_token_acc['accounts'][6]]: + return False + # Token program must be system token program + if account_keys[create_acc['accounts'][5]] != 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA': + return False + # CreateERC20TokenAccount instruction must use ERC20-wrapper from whitelist + if not self._is_allowed_wrapper_contract(account_keys[create_token_acc['accounts'][3]]): + return False + return True + + + # helper function checking if given 'create erc20 token account' corresponds to 'token transfer' instruction + def _check_transfer(self, account_keys, create_token_acc, token_transfer) -> bool: + return account_keys[create_token_acc['accounts'][1]] == account_keys[token_transfer['accounts'][1]] + + + def _airdrop_to(self, create_acc): + eth_address = "0x" + bytearray(base58.b58decode(create_acc['data'])[20:][:20]).hex() + + if eth_address in self.airdrop_ready: # transaction already processed + return + + logger.info(f"Airdrop to address: {eth_address}") + + json_data = { 'wallet': eth_address, 'amount': self.airdrop_amount } + resp = requests.post(self.faucet_url + '/request_eth_token', json = json_data) + if not resp.ok: + logger.warning(f'Failed to airdrop: {resp.status_code}') + return + + self.airdrop_ready[eth_address] = create_acc + + + def process_trx_airdropper_mode(self, trx): + if check_error(trx): + return + + # helper function finding all instructions that satisfies predicate + def find_instructions(trx, predicate): + return [instr for instr in trx['transaction']['message']['instructions'] if predicate(instr)] + + account_keys = trx["transaction"]["message"]["accountKeys"] + + # Finding instructions specific for airdrop. + # Airdrop triggers on sequence: + # neon.CreateAccount -> neon.CreateERC20TokenAccount -> spl.Transfer (maybe shuffled) + # First: select all instructions that can form such chains + predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + and base58.b58decode(instr['data'])[0] == 0x02 + create_acc_list = find_instructions(trx, predicate) + + predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + and base58.b58decode(instr['data'])[0] == 0x0f + create_token_acc_list = find_instructions(trx, predicate) + + predicate = lambda instr: account_keys[instr['programIdIndex']] == 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' \ + and base58.b58decode(instr['data'])[0] == 0x03 + token_transfer_list = find_instructions(trx, predicate) + + # Second: Find exact chains of instructions in sets created previously + for create_acc in create_acc_list: + for create_token_acc in create_token_acc_list: + if not self._check_create_instr(account_keys, create_acc, create_token_acc): + continue + for token_transfer in token_transfer_list: + if not self._check_transfer(account_keys, create_token_acc, token_transfer): + continue + self._airdrop_to(create_acc) + + + def process_functions(self): + IndexerBase.process_functions(self) + logger.debug("Process receipts") + self.process_receipts() + + + def process_receipts(self): + counter = 0 + for signature in self.transaction_order: + counter += 1 + if signature in self.transaction_receipts: + trx = self.transaction_receipts[signature] + if trx is None: + logger.error("trx is None") + del self.transaction_receipts[signature] + continue + if 'slot' not in trx: + logger.debug("\n{}".format(json.dumps(trx, indent=4, sort_keys=True))) + exit() + if trx['transaction']['message']['instructions'] is not None: + self.process_trx_airdropper_mode(trx) + + +def run_airdropper(solana_url, + evm_loader_id, + faucet_url = '', + wrapper_whitelist = [], + airdrop_amount = 10, + log_level = 'INFO'): + logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s') + logger.setLevel(logging.DEBUG) + logger.info(f"""Running indexer with params: + solana_url: {solana_url}, + evm_loader_id: {evm_loader_id}, + log_level: {log_level}, + faucet_url: {faucet_url}, + wrapper_whitelist: {wrapper_whitelist}, + airdrop_amount: {airdrop_amount}""") + + airdropper = Airdropper(solana_url, + evm_loader_id, + faucet_url, + wrapper_whitelist, + airdrop_amount, + log_level) + airdropper.run() + + +if __name__ == "__main__": + solana_url = os.environ.get('SOLANA_URL', 'http://localhost:8899') + evm_loader_id = os.environ.get('EVM_LOADER_ID', '53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io') + faucet_url = os.environ.get('FAUCET_URL', 'http://localhost:3333') + wrapper_whitelist = os.environ.get('INDEXER_ERC20_WRAPPER_WHITELIST', '').split(',') + airdrop_amount = os.environ.get('AIRDROP_AMOUNT', 0) + log_level = os.environ.get('LOG_LEVEL', 'INFO') + + run_airdropper(solana_url, + evm_loader_id, + faucet_url, + wrapper_whitelist, + airdrop_amount, + log_level) diff --git a/proxy/testing/mock_server.py b/proxy/testing/mock_server.py new file mode 100644 index 000000000..ef8e1c34f --- /dev/null +++ b/proxy/testing/mock_server.py @@ -0,0 +1,28 @@ +import requests + +from flask import Flask, request +from threading import Thread + +class MockServer(Thread): + def __init__(self, port): + super().__init__() + self.port = port + self.app = Flask(__name__) + self.url = "http://localhost:%s" % self.port + self.app.add_url_rule("/shutdown", view_func=self._shutdown_server) + + def add_url_rule(self, url, callback, methods): + self.app.add_url_rule(url, view_func=callback, methods=methods) + + def _shutdown_server(self): + if not 'werkzeug.server.shutdown' in request.environ: + raise RuntimeError('Not running the development server') + request.environ['werkzeug.server.shutdown']() + return 'Server shutting down...' + + def run(self): + self.app.run(port=self.port) + + def shutdown_server(self): + requests.get("http://localhost:%s/shutdown" % self.port) + self.join() diff --git a/proxy/testing/test_airdropper.py b/proxy/testing/test_airdropper.py new file mode 100644 index 000000000..9a00d5279 --- /dev/null +++ b/proxy/testing/test_airdropper.py @@ -0,0 +1,176 @@ +import unittest +from proxy.testing.mock_server import MockServer +from proxy.indexer.airdropper import Airdropper +from proxy.indexer.sql_dict import SQLDict +import time +from flask import request, Response +from unittest.mock import MagicMock, patch, call, ANY +import itertools +from proxy.testing.transactions import pre_token_airdrop_trx1, pre_token_airdrop_trx2,\ + create_sol_acc_and_airdrop_trx, wrapper_whitelist, evm_loader_addr, token_airdrop_address1, \ + token_airdrop_address2, token_airdrop_address3 + +class MockFaucet(MockServer): + def __init__(self, port): + super().__init__(port) + self.request_eth_token_mock = MagicMock() + self.request_eth_token_mock.side_effect = itertools.repeat({}) + self.add_url_rule("/request_eth_token", callback=self.request_eth_token, methods=['POST']) + + def request_eth_token(self): + req = request.get_json() + return self.request_eth_token_mock(req) + + +def create_signature_for_address(signature: str): + return { + 'blockTime': 1638177745, # not make sense + 'confirmationStatus': 'finalized', + 'err': None, + 'memo': None, + 'signature': signature, + 'slot': 9748200 # not make sense + } + + +def create_get_signatures_for_address(signatures: list): + return { + 'jsonrpc': '2.0', + 'result': [ create_signature_for_address(sign) for sign in signatures ], + 'id': 1 + } + + +class Test_Airdropper(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + print("testing indexer in airdropper mode") + cls.address = 'localhost' + cls.faucet_port = 3333 + cls.airdrop_amount = 10 + + cls.faucet = MockFaucet(cls.faucet_port) + cls.faucet.start() + time.sleep(0.2) + + cls.evm_loader_id = evm_loader_addr + cls.wrapper_whitelist = wrapper_whitelist + cls.airdropper = Airdropper(f'http://{cls.address}:8899', + cls.evm_loader_id, + f'http://{cls.address}:{cls.faucet_port}', + cls.wrapper_whitelist, + cls.airdrop_amount, + 'INFO') + + + @classmethod + def tearDownClass(cls) -> None: + cls.faucet.shutdown_server() + cls.faucet.join() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_success_process_trx_with_one_airdrop(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould airdrop to new address - one target in transaction") + mock_sql_dict_contains.side_effect = [False] # new eth address + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=200, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_has_calls([call(token_airdrop_address1, ANY)]) + json_req = {'wallet': token_airdrop_address1, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_called_once_with(json_req) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(Airdropper, '_is_allowed_wrapper_contract') + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_failed_airdrop_contract_not_in_whitelist(self, + mock_sql_dict_contains, + mock_sql_dict_setitem, + mock_is_allowed_contract): + print("\n\nShould not airdrop for contract that is not in whitelist") + mock_is_allowed_contract.side_effect = [False] + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_is_allowed_contract.assert_called_once() + mock_sql_dict_contains.assert_not_called() + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_faucet_failure(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not add address to processed list due to faucet error") + mock_sql_dict_contains.side_effect = [False] # new eth address + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=400, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_not_called() + json_req = {'wallet': token_airdrop_address1, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_called_once_with(json_req) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_process_trx_with_one_airdrop_for_already_processed_address(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not airdrop to repeated address") + mock_sql_dict_contains.side_effect = [True] # eth address processed later + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_complex_transation(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould airdrop to several targets in one transaction") + mock_sql_dict_contains.side_effect = [False, False] # both targets are new + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=200, mimetype='application/json'), + Response("{}", status=200, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx2) + + mock_sql_dict_contains.assert_has_calls([call(token_airdrop_address3), + call(token_airdrop_address2)]) + mock_sql_dict_setitem.assert_has_calls([call(token_airdrop_address3, ANY), + call(token_airdrop_address2, ANY)]) + json_req1 = {'wallet': token_airdrop_address2, 'amount': self.airdrop_amount} + json_req2 = {'wallet': token_airdrop_address3, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_has_calls([call(json_req2), call(json_req1)]) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_no_airdrop_instructions(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not airdrop when instructions are inconsistent") + self.airdropper.process_trx_airdropper_mode(create_sol_acc_and_airdrop_trx) + + mock_sql_dict_contains.assert_not_called() + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + diff --git a/proxy/testing/transactions.py b/proxy/testing/transactions.py new file mode 100644 index 000000000..dec4e2bb9 --- /dev/null +++ b/proxy/testing/transactions.py @@ -0,0 +1,491 @@ + +token_program = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' +evm_loader_addr = 'eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU' +erc20_wrapper = '5H7kvhPD7GECAmf227vTPYTS7SC2PmyuVZaT5zVTx7vb' +wrapper_whitelist = [erc20_wrapper] + +token_airdrop_address1 = '0xf71c4daca893e5333982e2956c5ed9b648818376' + +# Solana transaction for simple case airdrop +pre_token_airdrop_trx1 = { + 'blockTime': 1637857371, + 'meta': { + 'err': None, + 'fee': 5000, + 'innerInstructions': [ + { + 'index': 0, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '111112fUvhuhctf7ykHr29ATacqTktVJJSG9xpkwPTuR6WcMjZZQDYhZ5k4S6Zu6C5sdsn', + 'programIdIndex': 7 + }, + { + 'accounts': [0, 2, 1, 8, 7, 9, 11], + 'data': '', + 'programIdIndex': 10 + }, + { + 'accounts': [0, 2], + 'data': '3Bxs4h24hBtQy9rw', + 'programIdIndex': 7 + }, + { + 'accounts': [2], 'data': + '9krTDU2LzCSUJuVZ', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': 'SYXsBSQy3GeifSEQSGvTbrPNposbSAiSoh1YA85wcvGKSnYg', + 'programIdIndex': 7 + }, + { + 'accounts': [2, 8, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + }, + { + 'index': 1, + 'instructions': [ + { + 'accounts': [0, 3], + 'data': '11119os1e9qSs2u7TsThXqkBSRVFxhmYaFKFZ1waB2X7armDmvK3p5GmLdUxYdg3h7QSrL', + 'programIdIndex': 7 + }, + { + 'accounts': [3, 5, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + } + ], + 'logMessages': [ + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', + 'Program log: Transfer 2039280 lamports to the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Allocate space for the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Assign the associated token account to the SPL Token program', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Initialize the associated token account', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [3]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 464826 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 24626 of 485359 compute units', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', + 'Program log: Total memory occupied: 1414', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 40680 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program log: Total memory occupied: 1536', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: Transfer', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success' + ], + 'postBalances': [5944954400, 2672640, 2039280, 2039280, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'postTokenBalances': [ + { + 'accountIndex': 2, + 'mint': '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + 'owner': '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'uiTokenAmount': { + 'amount': '0', + 'decimals': 9, + 'uiAmount': None, + 'uiAmountString': '0' + } + }, + { + 'accountIndex': 3, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'uiTokenAmount': { + 'amount': '1000000', + 'decimals': 6, + 'uiAmount': 1.0, + 'uiAmountString': '1' + } + }, + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3798000000', + 'decimals': 6, + 'uiAmount': 3798.0, + 'uiAmountString': '3798' + } + } + ], + 'preBalances': [5951710600, 0, 0, 0, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'preTokenBalances': [ + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3799000000', + 'decimals': 6, + 'uiAmount': 3799.0, + 'uiAmountString': '3799' + } + } + ], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 96659490, + 'transaction': { + 'message': { + 'accountKeys': [ + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'J4zoYXUtxNJsCXLih7D7dVkfoT9pvsYa8HtHPEEN88U', + 'AZqbo1ZCwS1grcqUdhs79YspjXME2BjrV5WCHabBS1ht', + erc20_wrapper, + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', + '11111111111111111111111111111111', + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + token_program, + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', + 'SysvarRent111111111111111111111111111111111', + evm_loader_addr + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 6, + 'numRequiredSignatures': 1 + }, + 'instructions': [ + { + 'accounts': [0, 1, 2, 7, 8, 9, 10, 11], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcPrQv1EF4uXaNUgBSq2E3sfp7PKeAH', + 'programIdIndex': 12 + }, + { + 'accounts': [0, 3, 1, 4, 5, 7, 9, 11], + 'data': 'G', + 'programIdIndex': 12 + }, + { + 'accounts': [6, 3, 0], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 9 + } + ], + 'recentBlockhash': '8UBfYxDWWcEbXQPGxPRqK4oGnb6c2XAQm4xZwzM2fxCB' + }, + 'signatures': ['3np8r1PBJW9uuw7o7P86B46zH7nW4UufQa716NUsSscJ1mN5nG6K74JHcMb6YmTmu9Les2NSe1iQyLJZwgS5RpvE'] + } + } + +token_airdrop_address2 = '0x8bd4991b9b81b3298fc6ac06b553b87c8e8582f0' +token_airdrop_address3 = '0x67d9e53d5e747b36fa6587421114aa0b0ca39753' + +# This is transaction containing 2 addresses for airdrop +# Instructions here mixed to test complex case +pre_token_airdrop_trx2 = { + 'blockTime': 1637857344, + 'meta': { + 'err': None, + 'fee': 5000, + 'innerInstructions': [ + { + 'index': 0, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '111112fUvhuhctf7ykHr29ATacqTktVJJSG9xpkwPTuR6WcMjZZQDYhZ5k4S6Zu6C5sdsn', + 'programIdIndex': 7 + }, + { + 'accounts': [0, 2, 1, 8, 7, 9, 11], + 'data': '', + 'programIdIndex': 10 + }, + { + 'accounts': [0, 2], + 'data': '3Bxs4h24hBtQy9rw', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': '9krTDU2LzCSUJuVZ', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': 'SYXsBSQy3GeifSEQSGvTbrPNposbSAiSoh1YA85wcvGKSnYg', + 'programIdIndex': 7 + }, + { + 'accounts': [2, 8, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + }, + { + 'index': 1, + 'instructions': [ + { + 'accounts': [0, 3], + 'data': '11119os1e9qSs2u7TsThXqkBSRVFxhmYaFKFZ1waB2X7armDmvK3p5GmLdUxYdg3h7QSrL', + 'programIdIndex': 7 + }, + { + 'accounts': [3, 5, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + } + ], + 'logMessages': [ + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', + 'Program log: Transfer 2039280 lamports to the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Allocate space for the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Assign the associated token account to the SPL Token program', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Initialize the associated token account', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [3]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 457797 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 27155 of 480859 compute units', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', + 'Program log: Total memory occupied: 1414', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 47709 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program log: Total memory occupied: 1536', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: Transfer', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success'], + 'postBalances': [5951710600, 2672640, 2039280, 2039280, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'postTokenBalances': [ + { + 'accountIndex': 2, + 'mint': '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + 'owner': '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', + 'uiTokenAmount': { + 'amount': '0', + 'decimals': 9, + 'uiAmount': None, + 'uiAmountString': '0' + } + }, + { + 'accountIndex': 3, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', + 'uiTokenAmount': { + 'amount': '1000000', + 'decimals': 6, + 'uiAmount': 1.0, + 'uiAmountString': '1' + } + }, + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3799000000', + 'decimals': 6, + 'uiAmount': 3799.0, + 'uiAmountString': '3799' + } + } + ], + 'preBalances': [5958466800, 0, 0, 0, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'preTokenBalances': [ + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3800000000', + 'decimals': 6, + 'uiAmount': 3800.0, + 'uiAmountString': '3800' + } + } + ], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 96659420, + 'transaction': { + 'message': { + 'accountKeys': [ + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', # 0 - funding + '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', # 1 - ETH account + 'CztEiw75xKWoJ32Nr6exUtmjQHQFD7SE7ZWckX6b55v3', # 2 - NEON token account + '6whnofFTKa6ynDAv882VpQMx6rQvo9v18QM4AGmLCHZ9', # 3 - >> ERC20 token account (wrapper balance) + erc20_wrapper, # 4 - >> contract (ERC20 wrapper) + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', # 5 - SPL token-mint account (wrapped by ERC20) + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', # 6 - SPL token from + '11111111111111111111111111111111', # 7 - System program + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', # 8 - NEON token mint + token_program, # 9 - >> Token program + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', # 10 - Associated token program + 'SysvarRent111111111111111111111111111111111', # 11 + evm_loader_addr, # 12 - >> EVM_LOADER + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', # 13 - funding + '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', # 14 - ETH account + 'J4zoYXUtxNJsCXLih7D7dVkfoT9pvsYa8HtHPEEN88U', # 15 - NEON token account + 'AZqbo1ZCwS1grcqUdhs79YspjXME2BjrV5WCHabBS1ht', # 16 - >> ERC20 token account (wrapper balance) + erc20_wrapper, # 17 - >> contract (ERC20 wrapper) + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', # 18 - SPL token-mint account (wrapped by ERC20) + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', # 19 - SPL token from + '11111111111111111111111111111111', # 20 - System program + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', # 21 - NEON token mint + token_program, # 22 - >> Token program + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', # 23 - Associated token program + 'SysvarRent111111111111111111111111111111111', # 24 + evm_loader_addr # 25 - >> EVM_LOADER + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 6, + 'numRequiredSignatures': 1 + }, + 'instructions': [ + { + # Create 2-nd ERC20 token account + 'accounts': [13, 16, 14, 17, 18, 20, 22, 24], + 'data': 'G', + 'programIdIndex': 25 + }, + { + # Create 2-nd account + 'accounts': [13, 14, 15, 20, 21, 22, 23, 24], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcF3UcpFjrvswawEUuqDYcNbh7NZsuX', + 'programIdIndex': 25 + }, + { + # Create 1-st account + 'accounts': [0, 1, 2, 7, 8, 9, 10, 11], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcHFoDD1r8SrtHfKsSB7RxGBwrn2xPk', + 'programIdIndex': 12 + }, + { + # Create 1-st ERC20 token account + 'accounts': [0, 3, 1, 4, 5, 7, 9, 11], + 'data': 'G', + 'programIdIndex': 12 + }, + { + # Transfer to 1-st ERC20 token account + 'accounts': [6, 3, 0], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 9 + }, + { + # Transfer to 2-nd ERC20 token account + 'accounts': [19, 16, 13], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 22 + } + ], + 'recentBlockhash': 'DhuNa4ts1c8jnD1u55rvQd8YKQo8t4aHwt3jUd1Anexu'}, + 'signatures': ['4WotQiXmE5AUjC9yoTE69C2CLmWa2Typ1mAuG4EzAkjYXSfYH6gQewkSfsEHGQjvXkANFKXtkUtvkqm7eq23M74V'] + } + } + +create_sol_acc_and_airdrop_trx = { + 'blockTime': 1638178743, + 'meta': { + 'err': None, + 'fee': 10000, + 'innerInstructions': [], + 'logMessages': [ + 'Program 11111111111111111111111111111111 invoke [1]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: InitializeMint', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 2833 of 200000000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success'], + 'postBalances': [999998528400, 1461600, 1009200, 1, 1130582400], + 'postTokenBalances': [], + 'preBalances': [1000000000000, 0, 1009200, 1, 1130582400], + 'preTokenBalances': [], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 15029, + 'transaction': { + 'message': { + 'accountKeys': [ + 'D4Aa2HU5kwF3nByGYk7pdjbP4n3cFFjPjYASdKUfpH4H', + 'EHDze1sDhUk7dR9iBgV4Mm3dYMk3ZQXKGHiiuVTEYaYr', + 'SysvarRent111111111111111111111111111111111', + '11111111111111111111111111111111', + 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 3, + 'numRequiredSignatures': 2 + }, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '11114XtYk9gGfZoo968fyjNUYQJKf9gdmkGoaoBpzFv4vyaSMBn3VKxZdv7mZLzoyX5YNC', + 'programIdIndex': 3 + }, + { + 'accounts': [1, 2], + 'data': '1DidxzgH8WA79wndRY2Vc8EsYn3Rf9CKWmB3tRLNtzhWUZD', + 'programIdIndex': 4 + } + ], + 'recentBlockhash': '6kT9KCR37ZWhu9fbdacwUsXAHXvPo9kYwzQsQnh9dWyW' + }, + 'signatures': [ + '4wNHEzKDpqKpQ51A3cYWGsLXAxc3cxHtk45cs1RMUYgY6bViBUu6w7VaDMSQjprbwC7AF4bMy3ejR69FAVwQWUgh', + '4n6PzpFyQ5e9PTDFTUmHoPXUJYtNqsfQgwinu5ujYeY6EigseJHGgykmzMMb8exKsC45E7RjiyahLqhbR1uQo5V5' + ] + } + } diff --git a/requirements.txt b/requirements.txt index 323f726c3..3ca6efc39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ solana==0.10.0 psycopg2-binary ethereum py-solc-x==1.1.0 +flask From ae7b392be68e66995b7172b216fa71be4f66fdf8 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Mon, 6 Dec 2021 11:53:14 +0300 Subject: [PATCH 47/54] #337 fix running airdropper (#347) Co-authored-by: ivanl --- proxy/__main__.py | 21 ++++++++++++++++++++- proxy/indexer/airdropper.py | 16 ---------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/proxy/__main__.py b/proxy/__main__.py index d04d8529d..18c95ec59 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -8,7 +8,26 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ + from .proxy import entry_point +import os +from .indexer.airdropper import run_airdropper if __name__ == '__main__': - entry_point() + airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True'] + if airdropper_mode: + print("Will run in airdropper mode") + solana_url = os.environ['SOLANA_URL'] + evm_loader_id = os.environ['EVM_LOADER'] + faucet_url = os.environ['FAUCET_URL'] + wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'].split(',') + airdrop_amount = int(os.environ['AIRDROP_AMOUNT']) + log_level = os.environ['LOG_LEVEL'] + run_airdropper(solana_url, + evm_loader_id, + faucet_url, + wrapper_whitelist, + airdrop_amount, + log_level) + else: + entry_point() diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py index 12353fe09..41c638cc6 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/indexer/airdropper.py @@ -157,19 +157,3 @@ def run_airdropper(solana_url, airdrop_amount, log_level) airdropper.run() - - -if __name__ == "__main__": - solana_url = os.environ.get('SOLANA_URL', 'http://localhost:8899') - evm_loader_id = os.environ.get('EVM_LOADER_ID', '53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io') - faucet_url = os.environ.get('FAUCET_URL', 'http://localhost:3333') - wrapper_whitelist = os.environ.get('INDEXER_ERC20_WRAPPER_WHITELIST', '').split(',') - airdrop_amount = os.environ.get('AIRDROP_AMOUNT', 0) - log_level = os.environ.get('LOG_LEVEL', 'INFO') - - run_airdropper(solana_url, - evm_loader_id, - faucet_url, - wrapper_whitelist, - airdrop_amount, - log_level) From 9feca6ae43a491efbae3ceb17f3aeb5391d48361 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Mon, 6 Dec 2021 17:08:32 +0300 Subject: [PATCH 48/54] #351 fix canceller droping running transactions (#352) --- proxy/indexer/indexer.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index fe6bd1342..3254bfafa 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -28,9 +28,10 @@ def __init__(self, storage_account): class ContinueStruct: - def __init__(self, signature, results, accounts = None): + def __init__(self, signature, results, slot, accounts = None): self.signatures = [signature] self.results = results + self.slot = slot self.accounts = accounts @@ -165,7 +166,7 @@ def process_receipts(self): continue_result.signatures, storage_account, continue_result.accounts, - slot + max(slot, continue_result.slot) ) del continue_table[storage_account] @@ -271,6 +272,7 @@ def process_receipts(self): logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") trx_table[eth_signature].got_result = continue_result.results trx_table[eth_signature].signatures += continue_result.signatures + trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, continue_result.slot) del continue_table[storage_account] @@ -296,7 +298,7 @@ def process_receipts(self): continue_table[storage_account].results = got_result else: - continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts) elif instruction_data[0] == 0x0b or instruction_data[0] == 0x16: # ExecuteTrxFromAccountDataIterative ExecuteTrxFromAccountDataIterativeV02 if instruction_data[0] == 0x0b: @@ -319,7 +321,7 @@ def process_receipts(self): else: holder_table[holder_account] = HolderStruct(storage_account) else: - continue_table[storage_account] = ContinueStruct(signature, None, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, None, slot, blocked_accounts) holder_table[holder_account] = HolderStruct(storage_account) @@ -329,7 +331,7 @@ def process_receipts(self): storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] - continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), slot, blocked_accounts) elif instruction_data[0] == 0x0d: # logger.debug("{:>10} {:>6} PartialCallOrContinueFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex())) @@ -349,6 +351,7 @@ def process_receipts(self): if eth_signature in trx_table: trx_table[eth_signature].signatures.append(signature) + trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, slot) else: trx_table[eth_signature] = TransactionStruct( eth_trx, @@ -396,7 +399,7 @@ def process_receipts(self): continue_table[storage_account].results = got_result else: - continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts) holder_table[holder_account] = HolderStruct(storage_account) if instruction_data[0] > 0x16: From 2f0a0df2a5df9d6b2a95ab2d6c6958baaec45266 Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Mon, 6 Dec 2021 17:18:16 +0300 Subject: [PATCH 49/54] 349 improve neon proxy logging to filter a request and the corresponding response by 'method' (#350) * Added the request['method'] value only in debug output on response So it allows you to filer neon-proxy logs for requests and corresponding response for a 'method' * Introduced LOG_SENDING_SOLANA_TRANSACTION by default equal 'NO'. If it is set as 'YES' and then restart neon-proxy you can see sending solana transactions in logs --- proxy/docker-compose-test.yml | 1 + proxy/environment.py | 1 + proxy/plugin/solana_rest_api.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 729b4affa..fefd28d6b 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -65,6 +65,7 @@ services: POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass NEW_USER_AIRDROP_AMOUNT: 100 + LOG_SENDING_SOLANA_TRANSACTION: "YES" CONFIG: ci hostname: proxy depends_on: diff --git a/proxy/environment.py b/proxy/environment.py index 526ce53cf..8fc3387f3 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -17,6 +17,7 @@ TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) +LOG_SENDING_SOLANA_TRANSACTION = os.environ.get("LOG_SENDING_SOLANA_TRANSACTION", "NO") == "YES" class solana_cli: def call(self, *args): diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index e7f9afd19..61f522b7a 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -639,7 +639,8 @@ def handle_request(self, request: HttpParser) -> None: traceback.print_exc() response = {'jsonrpc': '2.0', 'error': {'code': -32000, 'message': str(err)}} - logger.debug('>>> %s 0x%0x %s', threading.get_ident(), id(self.model), json.dumps(response)) + logger.debug('>>> %s 0x%0x %s %s', threading.get_ident(), id(self.model), json.dumps(response), + request['method'] if 'method' in request else '---') self.client.queue(memoryview(build_http_response( httpStatusCodes.OK, body=json.dumps(response).encode('utf8'), From 3b36864b6bcc25a85b075de944301bca009277ac Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Mon, 6 Dec 2021 18:29:00 +0300 Subject: [PATCH 50/54] #349 Fix using log sending solana transaction (#353) * Move LOG_SENDING_SOLANA_TRANSACTION to environment.py after merging develop * Fix after merging Co-authored-by: Dmitriy Borisenko --- proxy/common_neon/solana_interactor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index ffb02e5d4..bd0138d07 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -9,8 +9,7 @@ from solana.rpc.types import TxOpts from .costs import update_transaction_cost -from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY - +from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, LOG_SENDING_SOLANA_TRANSACTION logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -88,14 +87,15 @@ def collect_result(self, reciept, eth_trx, reason=None): update_transaction_cost(result, eth_trx, reason) return result - def send_measured_transaction(self, trx, eth_trx, reason): + if LOG_SENDING_SOLANA_TRANSACTION: + logger.debug("send_measured_transaction for reason %s: %s ", reason, trx.__dict__) result = self.send_transaction(trx, eth_trx, reason=reason) self.get_measurements(result) return result - - # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh) + # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in + # .buildkite/steps/deploy-test.sh) def get_measurements(self, result): try: measurements = self.extract_measurements_from_receipt(result) From 6bb8509c24b84f23577d2f5993e6c3de98261d47 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Mon, 6 Dec 2021 21:07:51 +0300 Subject: [PATCH 51/54] #295 iterative execution (#332) * #291 extract transaction sender class * #291 move perm accs to transaction sender * #291 fix state * #291 fix errors * #291 merge fixes * #291 refactoring * #291 move EXTRA_GAS to environment * #291 capitalize CONFIRMATION_CHECK_DELAY * #291 sort imports * #291 relative paths * #291 Should be fixed in #326 * #291 testing chnages * fix storage account check * #291 rename `trx_with_create_and_airdrop` -> `make_trx_with_create_and_airdrop` * #295 fix state * #291 iterative combined * #295 do not get measurments * #291 pull request fixes * #295 turn combined instructions ON * #295 make neon_instructions return transasactions * #291 merge fix * #295 get rid of `USE_COMBINED_START_CONTINUE` * #295 requested fixes * #295 call_continue_bucked refactoring * #295 fix * #295 leave only combined iterative transactions * #295 move constants into class * #295 refactoring Co-authored-by: sinev-valentine --- proxy/common_neon/errors.py | 1 + proxy/common_neon/neon_instruction.py | 58 +++++----- proxy/common_neon/solana_interactor.py | 14 ++- proxy/common_neon/transaction_sender.py | 139 +++++++++++++++++++----- proxy/environment.py | 1 - proxy/run-proxy.sh | 1 - 6 files changed, 154 insertions(+), 60 deletions(-) diff --git a/proxy/common_neon/errors.py b/proxy/common_neon/errors.py index 4c17e8a9b..30286e6a0 100644 --- a/proxy/common_neon/errors.py +++ b/proxy/common_neon/errors.py @@ -12,6 +12,7 @@ def getError(self): if self.data: error['data'] = self.data return error + class SolanaErrors(Enum): AccountNotFound = "Invalid param: could not find account" diff --git a/proxy/common_neon/neon_instruction.py b/proxy/common_neon/neon_instruction.py index 8242d97cb..db6066fa7 100644 --- a/proxy/common_neon/neon_instruction.py +++ b/proxy/common_neon/neon_instruction.py @@ -283,14 +283,17 @@ def make_noniterative_call_transaction(self, length_before: int = 0) -> Transact return trx - def make_partial_call_instruction(self) -> TransactionInstruction: - return TransactionInstruction( + def make_continue_transaction(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("14") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + if index: + data = data + index.to_bytes(8, byteorder="little") + + return Transaction().add(TransactionInstruction( program_id = EVM_LOADER_ID, - data = bytearray.fromhex("13") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little") + self.msg, + data = data, keys = [ AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), @@ -301,28 +304,19 @@ def make_partial_call_instruction(self) -> TransactionInstruction: AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), ] + obligatory_accounts - ) - - - def make_iterative_call_transaction(self, length_before: int = 0) -> Transaction: - trx = Transaction() - trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 13)) - trx.add(self.make_partial_call_instruction()) - return trx + )) - def make_call_from_account_instruction(self) -> Transaction: + def make_cancel_transaction(self) -> Transaction: return Transaction().add(TransactionInstruction( program_id = EVM_LOADER_ID, - data = bytearray.fromhex("16") + self.collateral_pool_index_buf + int(0).to_bytes(8, byteorder="little"), + data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), keys = [ - AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), - AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), ] + self.eth_accounts + [ @@ -332,17 +326,15 @@ def make_call_from_account_instruction(self) -> Transaction: )) - def make_continue_instruction(self, steps, index=None) -> Transaction: - data = bytearray.fromhex("14") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") - if index: - data = data + index.to_bytes(8, byteorder="little") - - return Transaction().add(TransactionInstruction( + def make_partial_call_or_continue_instruction(self, steps: int = 0) -> TransactionInstruction: + data = bytearray.fromhex("0D") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + self.msg + return TransactionInstruction( program_id = EVM_LOADER_ID, data = data, keys = [ AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), @@ -353,19 +345,31 @@ def make_continue_instruction(self, steps, index=None) -> Transaction: AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), ] + obligatory_accounts - )) + ) - def make_cancel_instruction(self) -> Transaction: + def make_partial_call_or_continue_transaction(self, steps: int = 0, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 13)) + trx.add(self.make_partial_call_or_continue_instruction(steps)) + return trx + + + def make_partial_call_or_continue_from_account_data(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("0E") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder='little') + if index: + data = data + index.to_bytes(8, byteorder="little") return Transaction().add(TransactionInstruction( program_id = EVM_LOADER_ID, - data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), + data = data, keys = [ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), ] + self.eth_accounts + [ diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index bd0138d07..660ba9b25 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -5,10 +5,12 @@ import re import time +from solana.rpc.api import Client as SolanaClient from solana.rpc.commitment import Confirmed from solana.rpc.types import TxOpts from .costs import update_transaction_cost +from .utils import get_from_dict from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, LOG_SENDING_SOLANA_TRANSACTION logger = logging.getLogger(__name__) @@ -16,7 +18,7 @@ class SolanaInteractor: - def __init__(self, signer, client) -> None: + def __init__(self, signer, client: SolanaClient) -> None: self.signer = signer self.client = client @@ -194,6 +196,16 @@ def check_if_program_exceeded_instructions(err_result): return False +def check_if_storage_is_empty_error(err_result): + error_arr = get_from_dict(err_result, "data", "err", "InstructionError") + if error_arr is not None and isinstance(error_arr, list): + error_dict = error_arr[1] + if isinstance(error_dict, dict) and 'Custom' in error_dict: + if error_dict['Custom'] == 1 or error_dict['Custom'] == 4: + return True + return False + + def check_if_continue_returned(result): tx_info = result['result'] accounts = tx_info["transaction"]["message"]["accountKeys"] diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 5300c6a85..820bfb6b3 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -1,5 +1,6 @@ import json import logging +import math import os import rlp import time @@ -18,7 +19,8 @@ from .emulator_interactor import call_emulated from .layouts import ACCOUNT_INFO_LAYOUT from .neon_instruction import NeonInstruction -from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_if_program_exceeded_instructions +from .solana_interactor import SolanaInteractor, check_if_continue_returned, \ + check_if_program_exceeded_instructions, check_if_storage_is_empty_error from ..environment import EVM_LOADER_ID from ..plugin.eth_proto import Trx as EthTrx @@ -67,7 +69,7 @@ def execute(self): try: if call_iterative: try: - return iterative_executor.call_signed_iterative() + return iterative_executor.call_signed_iterative_combined() except Exception as err: logger.debug(str(err)) if str(err).startswith("transaction too large:"): @@ -77,7 +79,7 @@ def execute(self): raise if call_from_holder: - return iterative_executor.call_signed_with_holder_acc() + return iterative_executor.call_signed_with_holder_combined() finally: self.free_perm_accs() @@ -93,7 +95,7 @@ def create_noniterative_executor(self): def create_iterative_executor(self): self.instruction.init_iterative(self.storage, self.holder, self.perm_accs_id) - return IterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx, self.steps) + return IterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx, self.steps, self.steps_emulated) def init_perm_accs(self): @@ -302,6 +304,8 @@ def create_account_list_by_emulate(self): AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), ] + add_keys_05 + self.steps_emulated = output_json["steps_executed"] + class NoniterativeTransactionSender: def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx): @@ -321,53 +325,52 @@ def call_signed_noniterative(self): class IterativeTransactionSender: - def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx, steps: int): + CONTINUE_REGULAR = 'ContinueV02' + CONTINUE_COMBINED = 'PartialCallOrContinueFromRawEthereumTX' + CONTINUE_HOLDER_COMB = 'ExecuteTrxFromAccountDataIterativeOrContinue' + + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx, steps: int, steps_emulated: int): self.sender = solana_interactor self.instruction = neon_instruction self.create_acc_trx = create_acc_trx self.eth_trx = eth_trx self.steps = steps + self.steps_emulated = steps_emulated + self.instruction_type = self.CONTINUE_REGULAR - def call_signed_iterative(self): - if len(self.create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(self.create_acc_trx) - self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') - - call_txs = self.instruction.make_iterative_call_transaction() - - logger.debug("Partial call") - self.sender.send_measured_transaction(call_txs, self.eth_trx, 'PartialCallFromRawEthereumTXv02') - + def call_signed_iterative_combined(self): + self.create_accounts_for_trx() + self.instruction_type = self.CONTINUE_COMBINED return self.call_continue() - def call_signed_with_holder_acc(self): + def call_signed_with_holder_combined(self): self.write_trx_to_holder_account() - if len(self.create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(self.create_acc_trx) - self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'create_accounts_for_deploy') + self.create_accounts_for_trx() + self.instruction_type = self.CONTINUE_HOLDER_COMB + return self.call_continue() - # ExecuteTrxFromAccountDataIterative - logger.debug("ExecuteTrxFromAccountDataIterative:") - call_txs = self.instruction.make_call_from_account_instruction() - self.sender.send_measured_transaction(call_txs, self.eth_trx, 'ExecuteTrxFromAccountDataIterativeV02') - return self.call_continue() + def create_accounts_for_trx(self): + length = len(self.create_acc_trx.instructions) + if length == 0: + return + logger.debug(f"Create account for trx: {length}") + precall_txs = Transaction() + precall_txs.add(self.create_acc_trx) + self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') def write_trx_to_holder_account(self): + logger.debug('write_trx_to_holder_account') msg = self.eth_trx.signature() + len(self.eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + self.eth_trx.unsigned_msg() - # Write transaction to transaction holder account offset = 0 receipts = [] rest = msg while len(rest): (part, rest) = (rest[:1000], rest[1000:]) - # logger.debug("sender_sol %s %s %s", sender_sol, holder, acc.public_key()) trx = self.instruction.make_write_transaction(offset, part) receipts.append(self.sender.send_transaction_unconfirmed(trx)) offset += len(part) @@ -377,6 +380,19 @@ def write_trx_to_holder_account(self): def call_continue(self): + return_result = None + try: + return_result = self.call_continue_bucked() + except Exception as err: + logger.debug("call_continue_bucked_combined exception: {}".format(str(err))) + + if return_result is not None: + return return_result + + return self.call_continue_iterative() + + + def call_continue_iterative(self): try: return self.call_continue_step_by_step() except Exception as err: @@ -398,7 +414,7 @@ def call_continue_step_by_step(self): def call_continue_step(self): step_count = self.steps while step_count > 0: - trx = self.instruction.make_continue_instruction(step_count) + trx = self.instruction.make_continue_transaction(step_count) logger.debug("Step count {}".format(step_count)) try: @@ -413,8 +429,71 @@ def call_continue_step(self): def call_cancel(self): - trx = self.instruction.make_cancel_instruction() + trx = self.instruction.make_cancel_transaction() logger.debug("Cancel") result = self.sender.send_measured_transaction(trx, self.eth_trx, 'CancelWithNonce') return result['result']['transaction']['signatures'][0] + + + def call_continue_bucked(self): + logger.debug("Send bucked combined: %s", self.instruction_type) + steps = self.steps + + receipts = [] + for index in range(math.ceil(self.steps_emulated/self.steps) + self.addition_count()): + try: + trx = self.make_bucked_trx(steps, index) + receipts.append(self.sender.send_transaction_unconfirmed(trx)) + except SendTransactionError as err: + logger.error(f"Failed to call continue bucked, error: {err.result}") + if check_if_storage_is_empty_error(err.result): + pass + elif check_if_program_exceeded_instructions(err.result): + steps = int(steps * 90 / 100) + else: + raise + except Exception as err: + logger.debug(str(err)) + if str(err).startswith('failed to get recent blockhash'): + pass + else: + raise + + return self.collect_bucked_results(receipts, self.instruction_type) + + + def addition_count(self): + ''' + How many transactions are needed depending on trx type: + CONTINUE_COMBINED: 2 (1 for begin and 1 for decreased steps) + CONTINUE_HOLDER_COMB: 1 for begin + 0 otherwise + ''' + addition_count = 0 + if self.instruction_type == self.CONTINUE_COMBINED: + addition_count = 2 + elif self.instruction_type == self.CONTINUE_HOLDER_COMB: + addition_count = 1 + return addition_count + + + def make_bucked_trx(self, steps, index): + if self.instruction_type == self.CONTINUE_REGULAR: + return self.instruction.make_continue_transaction(steps, index) + elif self.instruction_type == self.CONTINUE_COMBINED: + return self.instruction.make_partial_call_or_continue_transaction(steps - index) + elif self.instruction_type == self.CONTINUE_HOLDER_COMB: + return self.instruction.make_partial_call_or_continue_from_account_data(steps, index) + else: + raise Exception("Unknown continue type: {}".format(self.instruction_type)) + + + def collect_bucked_results(self, receipts, reason): + logger.debug(f"Collected bucked results: {receipts}") + result_list = self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason=reason) + for result in result_list: + # self.sender.get_measurements(result) + signature = check_if_continue_returned(result) + if signature: + return signature diff --git a/proxy/environment.py b/proxy/environment.py index 8fc3387f3..927cb30ad 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -12,7 +12,6 @@ NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) CONFIRMATION_CHECK_DELAY = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) -USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "NO") == "YES" CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 diff --git a/proxy/run-proxy.sh b/proxy/run-proxy.sh index 103795d70..199489563 100755 --- a/proxy/run-proxy.sh +++ b/proxy/run-proxy.sh @@ -8,7 +8,6 @@ if [ "$CONFIG" == "ci" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://solana:8899" [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=100000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.5" - [[ -z "$USE_COMBINED_START_CONTINUE" ]] && export USE_COMBINED_START_CONTINUE="NO" [[ -z "$CONTINUE_COUNT_FACTOR" ]] && export CONTINUE_COUNT_FACTOR="3" [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="postgres" From 56044eaf7ce4335104750884494e366559422c7a Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Tue, 7 Dec 2021 15:08:47 +0300 Subject: [PATCH 52/54] #354 Check result for errors (#355) * #354 Check result for errors * #354 get_measurements error -> warning * #354 check for errors in get measurements * #354 get measurements from bucked transactions * #354 remove repeated code --- proxy/common_neon/solana_interactor.py | 49 +++++++++++++++++++++---- proxy/common_neon/transaction_sender.py | 14 +++++-- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 660ba9b25..56015e127 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -133,6 +133,11 @@ def collect_results(self, receipts, eth_trx=None, reason=None): @staticmethod def extract_measurements_from_receipt(receipt): + if check_for_errors(receipt): + logger.warning("Can't get measurements from receipt with error") + logger.info("Failed result: %s"%json.dumps(receipt, indent=3)) + return [] + log_messages = receipt['result']['meta']['logMessages'] transaction = receipt['result']['transaction'] accounts = transaction['message']['accountKeys'] @@ -185,19 +190,47 @@ def extract_measurements_from_receipt(receipt): return result -def check_if_program_exceeded_instructions(err_result): - err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" - err_budget = "failed: Computational budget exceeded" +def get_error_definition_from_reciept(receipt): + err_from_reciept = get_from_dict(receipt, 'result', 'meta', 'err', 'InstructionError') + if err_from_reciept is not None: + return err_from_reciept + + err_from_reciept_result = get_from_dict(receipt, 'meta', 'err', 'InstructionError') + if err_from_reciept_result is not None: + return err_from_reciept_result + + err_from_send_trx_error = get_from_dict(receipt, 'data', 'err', 'InstructionError') + if err_from_send_trx_error is not None: + return err_from_send_trx_error + + err_from_prepared_receipt = get_from_dict(receipt, 'err', 'InstructionError') + if err_from_prepared_receipt is not None: + return err_from_prepared_receipt + + return None + + - if err_result['data']['logs'][-1].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-2].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-1].find(err_budget) >= 0: +def check_for_errors(receipt): + if get_error_definition_from_reciept(receipt) is not None: return True return False -def check_if_storage_is_empty_error(err_result): - error_arr = get_from_dict(err_result, "data", "err", "InstructionError") +def check_if_program_exceeded_instructions(receipt): + error_arr = get_error_definition_from_reciept(receipt) + if error_arr is not None and isinstance(error_arr, list): + error_type = error_arr[1] + if isinstance(error_type, str): + if error_type == 'ProgramFailedToComplete': + return True + if error_type == 'ComputationalBudgetExceeded': + return True + return False + + +def check_if_storage_is_empty_error(receipt): + error_arr = get_error_definition_from_reciept(receipt) if error_arr is not None and isinstance(error_arr, list): error_dict = error_arr[1] if isinstance(error_dict, dict) and 'Custom' in error_dict: diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 820bfb6b3..3150ed885 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -19,7 +19,7 @@ from .emulator_interactor import call_emulated from .layouts import ACCOUNT_INFO_LAYOUT from .neon_instruction import NeonInstruction -from .solana_interactor import SolanaInteractor, check_if_continue_returned, \ +from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_for_errors,\ check_if_program_exceeded_instructions, check_if_storage_is_empty_error from ..environment import EVM_LOADER_ID from ..plugin.eth_proto import Trx as EthTrx @@ -321,6 +321,12 @@ def call_signed_noniterative(self): call_txs_05.add(self.create_acc_trx) call_txs_05.add(self.instruction.make_noniterative_call_transaction(len(call_txs_05.instructions))) result = self.sender.send_measured_transaction(call_txs_05, self.eth_trx, 'CallFromRawEthereumTX') + + if check_for_errors(result): + if check_if_program_exceeded_instructions(result): + raise Exception("Program failed to complete") + raise Exception(json.dumps(result['result']['meta'])) + return result['result']['transaction']['signatures'][0] @@ -359,7 +365,9 @@ def create_accounts_for_trx(self): logger.debug(f"Create account for trx: {length}") precall_txs = Transaction() precall_txs.add(self.create_acc_trx) - self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') + result = self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') + if check_for_errors(result): + raise Exception("Failed to create account for trx") def write_trx_to_holder_account(self): @@ -493,7 +501,7 @@ def collect_bucked_results(self, receipts, reason): logger.debug(f"Collected bucked results: {receipts}") result_list = self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason=reason) for result in result_list: - # self.sender.get_measurements(result) + self.sender.get_measurements(result) signature = check_if_continue_returned(result) if signature: return signature From 0bad34cce6d0448075cf4d2d52e0fc816ea36b5b Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Tue, 7 Dec 2021 15:09:53 +0300 Subject: [PATCH 53/54] #360 pass transaction too large upper (#361) --- proxy/common_neon/transaction_sender.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 3150ed885..c78d74d2a 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -393,6 +393,8 @@ def call_continue(self): return_result = self.call_continue_bucked() except Exception as err: logger.debug("call_continue_bucked_combined exception: {}".format(str(err))) + if str(err).startswith("transaction too large:"): + raise if return_result is not None: return return_result From 9e8344cba18fc4a100068a5a2ab499c2187661fd Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin Date: Tue, 7 Dec 2021 15:31:27 +0300 Subject: [PATCH 54/54] NEON_PROXY_PKG_VERSION = '0.5.0' --- proxy/plugin/solana_rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 61f522b7a..badc941f6 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -46,7 +46,7 @@ modelInstanceLock = threading.Lock() modelInstance = None -NEON_PROXY_PKG_VERSION = '0.4.1-rc0' +NEON_PROXY_PKG_VERSION = '0.5.0' NEON_PROXY_REVISION = 'NEON_PROXY_REVISION_TO_BE_REPLACED' class EthereumModel: