diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index d8becc316..104ffc621 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -3,7 +3,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) -set ${SOLANA_REVISION:=v1.7.9-resources} +set ${SOLANA_REVISION:=v1.7.9-testnet} set ${EVM_LOADER_REVISION:=stable} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally @@ -15,4 +15,5 @@ docker pull neonlabsorg/evm_loader:${EVM_LOADER_REVISION} docker build -t neonlabsorg/proxy:${REVISION} \ --build-arg SOLANA_REVISION=${SOLANA_REVISION} \ --build-arg EVM_LOADER_REVISION=${EVM_LOADER_REVISION} \ + --build-arg PROXY_REVISION=${REVISION} \ . diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index 1ebae3767..85ee0f509 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -78,6 +78,11 @@ docker run --rm -ti --network=container:proxy \ -e EVM_LOADER \ -e SOLANA_URL \ -e EXTRA_GAS=100000 \ + -e NEW_USER_AIRDROP_AMOUNT=100 \ + -e POSTGRES_DB=neon-db \ + -e POSTGRES_USER=neon-proxy \ + -e POSTGRES_PASSWORD=neon-proxy-pass \ + -e POSTGRES_HOST=postgres \ --entrypoint ./proxy/deploy-test.sh \ ${EXTRA_ARGS:-} \ $PROXY_IMAGE \ diff --git a/Dockerfile b/Dockerfile index ce1d31997..19cbbe0e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG SOLANA_REVISION=v1.7.9-resources +ARG SOLANA_REVISION=v1.7.9-testnet ARG EVM_LOADER_REVISION=stable FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli @@ -6,6 +6,7 @@ FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli FROM neonlabsorg/evm_loader:${EVM_LOADER_REVISION} AS spl FROM ubuntu:20.04 +ARG PROXY_REVISION RUN apt update && \ DEBIAN_FRONTEND=noninteractive apt -y install \ @@ -41,6 +42,7 @@ COPY --from=spl /opt/solana_utils.py \ COPY --from=spl /opt/neon-cli /spl/bin/emulator COPY . /opt +RUN sed -i 's/NEON_PROXY_REVISION_TO_BE_REPLACED/'"$PROXY_REVISION"'/g' /opt/proxy/plugin/solana_rest_api.py COPY proxy/operator-keypair.json /root/.config/solana/id.json RUN cd /usr/local/lib/python3.8/dist-packages/ && patch -p0 255 and check_instruction_index < 0: + raise Exception("Invalid index for instruction - {}".format(check_instruction_index)) + + check_count = 1 + eth_address_size = 20 + signature_size = 65 + eth_address_offset = data_start + signature_offset = eth_address_offset + eth_address_size + message_data_offset = signature_offset + signature_size + + data = struct.pack("B", check_count) + data += struct.pack(" Tuple[Transaction, PublicKey]: + pda_account, nonce = ether2program(eth_address) + neon_token_account = getTokenAddr(PublicKey(pda_account)) + logger.debug(f'Create eth account: {eth_address}, sol account: {pda_account}, neon_token_account: {neon_token_account}, nonce: {nonce}') + + base = self.operator_account + data = create_account_layout(0, 0, bytes(eth_address), nonce) + trx = Transaction() + if code_acc is None: + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(pda_account), is_signer=False, is_writable=True), + AccountMeta(pubkey=neon_token_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ])) + else: + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=data, + keys=[ + AccountMeta(pubkey=base, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(pda_account), is_signer=False, is_writable=True), + AccountMeta(pubkey=neon_token_account, is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ])) + return trx, neon_token_account + + + def createERC20TokenAccountTrx(self, token_info) -> Transaction: + trx = Transaction() + trx.add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=bytes.fromhex('0F'), + keys=[ + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["key"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["owner"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["contract"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=PublicKey(token_info["mint"]), is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), + AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False), + ] + )) + + return trx + + + def make_transfer_instruction(self, associated_token_account: PublicKey) -> TransactionInstruction: + transfer_instruction = transfer2(Transfer2Params( + source=self.operator_neon_address, + owner=self.operator_account, + dest=associated_token_account, + amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei, + decimals=9, + mint=ETH_TOKEN_MINT_ID, + program_id=TOKEN_PROGRAM_ID + )) + logger.debug(f"Token transfer from token: {self.operator_neon_address}, owned by: {self.operator_account}, to token: " + f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}") + return transfer_instruction + + + def make_trx_with_create_and_airdrop(self, eth_account, code_acc=None) -> Transaction: + trx = Transaction() + create_trx, associated_token_account = self.make_create_eth_account_trx(eth_account, code_acc) + trx.add(create_trx) + if NEW_USER_AIRDROP_AMOUNT <= 0: + return trx + transfer_instruction = self.make_transfer_instruction(associated_token_account) + trx.add(transfer_instruction) + + return trx + + + def make_resize_instruction(self, acc_desc, code_account_new, seed) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("11") + bytes(seed), # 17- ResizeStorageAccount + keys = [ + AccountMeta(pubkey=PublicKey(acc_desc["account"]), is_signer=False, is_writable=True), + ( + AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=True) + if acc_desc["contract"] else + AccountMeta(pubkey=PublicKey("11111111111111111111111111111111"), is_signer=False, is_writable=False) + ), + AccountMeta(pubkey=code_account_new, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=False) + ], + ) + + + def make_write_transaction(self, offset: int, data: bytes) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id=EVM_LOADER_ID, + data=write_holder_layout(self.perm_accs_id, offset, data), + keys=[ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=False), + ] + )) + + + def make_keccak_instruction(self, check_instruction_index, msg_len, data_start) -> TransactionInstruction: + return TransactionInstruction( + program_id=KECCAK_PROGRAM, + data=make_keccak_instruction_data(check_instruction_index, msg_len, data_start), + keys=[ + AccountMeta(pubkey=KECCAK_PROGRAM, is_signer=False, is_writable=False), + ] + ) + + + def make_05_call_instruction(self) -> TransactionInstruction: + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("05") + self.collateral_pool_index_buf + self.msg, + keys = [ + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + obligatory_accounts + ) + + + def make_noniterative_call_transaction(self, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 5)) + trx.add(self.make_05_call_instruction()) + return trx + + + def make_continue_transaction(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("14") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + if index: + data = data + index.to_bytes(8, byteorder="little") + + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = data, + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) + + + def make_cancel_transaction(self) -> Transaction: + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) + + + def make_partial_call_or_continue_instruction(self, steps: int = 0) -> TransactionInstruction: + data = bytearray.fromhex("0D") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder="little") + self.msg + return TransactionInstruction( + program_id = EVM_LOADER_ID, + data = data, + keys = [ + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + ) + + + def make_partial_call_or_continue_transaction(self, steps: int = 0, length_before: int = 0) -> Transaction: + trx = Transaction() + trx.add(self.make_keccak_instruction(length_before + 1, len(self.eth_trx.unsigned_msg()), 13)) + trx.add(self.make_partial_call_or_continue_instruction(steps)) + return trx + + + def make_partial_call_or_continue_from_account_data(self, steps, index=None) -> Transaction: + data = bytearray.fromhex("0E") + self.collateral_pool_index_buf + steps.to_bytes(8, byteorder='little') + if index: + data = data + index.to_bytes(8, byteorder="little") + return Transaction().add(TransactionInstruction( + program_id = EVM_LOADER_ID, + data = data, + keys = [ + AccountMeta(pubkey=self.holder, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + + AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), + AccountMeta(pubkey=self.collateral_pool_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.operator_neon_address, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), + + ] + self.eth_accounts + [ + + AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), + ] + obligatory_accounts + )) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py new file mode 100644 index 000000000..56015e127 --- /dev/null +++ b/proxy/common_neon/solana_interactor.py @@ -0,0 +1,259 @@ +import base58 +import base64 +import json +import logging +import re +import time + +from solana.rpc.api import Client as SolanaClient +from solana.rpc.commitment import Confirmed +from solana.rpc.types import TxOpts + +from .costs import update_transaction_cost +from .utils import get_from_dict +from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, LOG_SENDING_SOLANA_TRANSACTION + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class SolanaInteractor: + def __init__(self, signer, client: SolanaClient) -> None: + self.signer = signer + self.client = client + + + def get_operator_key(self): + return self.signer.public_key() + + + def get_account_info(self, storage_account): + opts = { + "encoding": "base64", + "commitment": "confirmed", + "dataSlice": { + "offset": 0, + "length": 16, + } + } + + result = self.client._provider.make_request("getAccountInfo", str(storage_account), opts) + logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) + + info = result['result']['value'] + if info is None: + logger.debug("Can't get information about {}".format(storage_account)) + return None + + data = base64.b64decode(info['data'][0]) + + account_tag = data[0] + lamports = info['lamports'] + owner = info['owner'] + + return (account_tag, lamports, owner) + + + def get_sol_balance(self, account): + return self.client.get_balance(account, commitment=Confirmed)['result']['value'] + + + def get_rent_exempt_balance_for_size(self, size): + return self.client.get_minimum_balance_for_rent_exemption(size, commitment=Confirmed)["result"] + + + def _getAccountData(self, account, expected_length, owner=None): + info = self.client.get_account_info(account, commitment=Confirmed)['result']['value'] + if info is None: + raise Exception("Can't get information about {}".format(account)) + + data = base64.b64decode(info['data'][0]) + if len(data) < expected_length: + raise Exception("Wrong data length for account data {}".format(account)) + return data + + + def send_transaction(self, trx, eth_trx, reason=None): + reciept = self.send_transaction_unconfirmed(trx) + result = self.collect_result(reciept, eth_trx, reason) + return result + + + def send_transaction_unconfirmed(self, trx): + result = self.client.send_transaction(trx, self.signer, opts=TxOpts(preflight_commitment=Confirmed))["result"] + return result + + def collect_result(self, reciept, eth_trx, reason=None): + self.confirm_transaction(reciept) + result = self.client.get_confirmed_transaction(reciept) + update_transaction_cost(result, eth_trx, reason) + return result + + def send_measured_transaction(self, trx, eth_trx, reason): + if LOG_SENDING_SOLANA_TRANSACTION: + logger.debug("send_measured_transaction for reason %s: %s ", reason, trx.__dict__) + result = self.send_transaction(trx, eth_trx, reason=reason) + self.get_measurements(result) + return result + + # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in + # .buildkite/steps/deploy-test.sh) + def get_measurements(self, result): + try: + measurements = self.extract_measurements_from_receipt(result) + for m in measurements: logger.info(json.dumps(m)) + except Exception as err: + logger.error("Can't get measurements %s"%err) + logger.info("Failed result: %s"%json.dumps(result, indent=3)) + + + def confirm_transaction(self, tx_sig, confirmations=0): + """Confirm a transaction.""" + TIMEOUT = 30 # 30 seconds pylint: disable=invalid-name + elapsed_time = 0 + while elapsed_time < TIMEOUT: + logger.debug('confirm_transaction for %s', tx_sig) + resp = self.client.get_signature_statuses([tx_sig]) + logger.debug('confirm_transaction: %s', resp) + if resp["result"]: + status = resp['result']['value'][0] + if status and (status['confirmationStatus'] == 'finalized' or \ + status['confirmationStatus'] == 'confirmed' and status['confirmations'] >= confirmations): + return + time.sleep(CONFIRMATION_CHECK_DELAY) + elapsed_time += CONFIRMATION_CHECK_DELAY + raise RuntimeError("could not confirm transaction: ", tx_sig) + + + def collect_results(self, receipts, eth_trx=None, reason=None): + results = [] + for rcpt in receipts: + results.append(self.collect_result(rcpt, eth_trx, reason)) + return results + + @staticmethod + def extract_measurements_from_receipt(receipt): + if check_for_errors(receipt): + logger.warning("Can't get measurements from receipt with error") + logger.info("Failed result: %s"%json.dumps(receipt, indent=3)) + return [] + + log_messages = receipt['result']['meta']['logMessages'] + transaction = receipt['result']['transaction'] + accounts = transaction['message']['accountKeys'] + instructions = [] + for instr in transaction['message']['instructions']: + program = accounts[instr['programIdIndex']] + instructions.append({ + 'accs': [accounts[acc] for acc in instr['accounts']], + 'program': accounts[instr['programIdIndex']], + 'data': base58.b58decode(instr['data']).hex() + }) + + pattern = re.compile('Program ([0-9A-Za-z]+) (.*)') + messages = [] + for log in log_messages: + res = pattern.match(log) + if res: + (program, reason) = res.groups() + if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) + messages[-1]['logs'].append(log) + + for instr in instructions: + if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue + if messages[0]['program'] != instr['program']: + raise Exception('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) + instr['logs'] = messages.pop(0)['logs'] + exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) + if not exit_result: raise Exception("Can't get exit result") + instr['result'] = exit_result.group(1) + + if instr['program'] == EVM_LOADER_ID: + memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) + instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) + if not (memory_result and instruction_result): + raise Exception("Can't parse measurements for evm_loader") + instr['measurements'] = { + 'instructions': instruction_result.group(1), + 'memory': memory_result.group(1) + } + + result = [] + for instr in instructions: + if instr['program'] == EVM_LOADER_ID: + result.append({ + 'program':instr['program'], + 'measurements':instr['measurements'], + 'result':instr['result'], + 'data':instr['data'] + }) + return result + + +def get_error_definition_from_reciept(receipt): + err_from_reciept = get_from_dict(receipt, 'result', 'meta', 'err', 'InstructionError') + if err_from_reciept is not None: + return err_from_reciept + + err_from_reciept_result = get_from_dict(receipt, 'meta', 'err', 'InstructionError') + if err_from_reciept_result is not None: + return err_from_reciept_result + + err_from_send_trx_error = get_from_dict(receipt, 'data', 'err', 'InstructionError') + if err_from_send_trx_error is not None: + return err_from_send_trx_error + + err_from_prepared_receipt = get_from_dict(receipt, 'err', 'InstructionError') + if err_from_prepared_receipt is not None: + return err_from_prepared_receipt + + return None + + + +def check_for_errors(receipt): + if get_error_definition_from_reciept(receipt) is not None: + return True + return False + + +def check_if_program_exceeded_instructions(receipt): + error_arr = get_error_definition_from_reciept(receipt) + if error_arr is not None and isinstance(error_arr, list): + error_type = error_arr[1] + if isinstance(error_type, str): + if error_type == 'ProgramFailedToComplete': + return True + if error_type == 'ComputationalBudgetExceeded': + return True + return False + + +def check_if_storage_is_empty_error(receipt): + error_arr = get_error_definition_from_reciept(receipt) + if error_arr is not None and isinstance(error_arr, list): + error_dict = error_arr[1] + if isinstance(error_dict, dict) and 'Custom' in error_dict: + if error_dict['Custom'] == 1 or error_dict['Custom'] == 4: + return True + return False + + +def check_if_continue_returned(result): + tx_info = result['result'] + accounts = tx_info["transaction"]["message"]["accountKeys"] + evm_loader_instructions = [] + + for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: + evm_loader_instructions.append(idx) + + for inner in (tx_info['meta']['innerInstructions']): + if inner["index"] in evm_loader_instructions: + for event in inner['instructions']: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: + instruction = base58.b58decode(event['data'])[:1] + if int().from_bytes(instruction, "little") == 6: # OnReturn evmInstruction code + return tx_info['transaction']['signatures'][0] + + return None diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py new file mode 100644 index 000000000..c78d74d2a --- /dev/null +++ b/proxy/common_neon/transaction_sender.py @@ -0,0 +1,509 @@ +import json +import logging +import math +import os +import rlp +import time + +from base58 import b58encode +from sha3 import keccak_256 +from solana.publickey import PublicKey +from solana.rpc.api import SendTransactionError +from solana.sysvar import * +from solana.transaction import AccountMeta, Transaction + +from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob + +from .address import accountWithSeed, AccountInfo, getTokenAddr +from .constants import STORAGE_SIZE, EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG, ACCOUNT_SEED_VERSION +from .emulator_interactor import call_emulated +from .layouts import ACCOUNT_INFO_LAYOUT +from .neon_instruction import NeonInstruction +from .solana_interactor import SolanaInteractor, check_if_continue_returned, check_for_errors,\ + check_if_program_exceeded_instructions, check_if_storage_is_empty_error +from ..environment import EVM_LOADER_ID +from ..plugin.eth_proto import Trx as EthTrx + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class TransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, eth_trx: EthTrx, steps: int) -> None: + self.sender = solana_interactor + self.eth_trx = eth_trx + self.steps = steps + + self.instruction = NeonInstruction(self.sender.get_operator_key()) + + + def execute(self): + self.create_account_list_by_emulate() + + noniterative_executor = self.create_noniterative_executor() + + call_iterative = False + call_from_holder = False + + if not self.eth_trx.toAddress: + call_from_holder = True + else: + try: + logger.debug("Try single trx call") + return noniterative_executor.call_signed_noniterative() + except Exception as err: + logger.debug(str(err)) + errStr = str(err) + if "Program failed to complete" in errStr or "Computational budget exceeded" in errStr: + logger.debug("Program exceeded instructions") + call_iterative = True + elif str(err).startswith("transaction too large:"): + logger.debug("Transaction too large, call call_signed_with_holder_acc():") + call_from_holder = True + else: + raise + + self.init_perm_accs() + iterative_executor = self.create_iterative_executor() + try: + if call_iterative: + try: + return iterative_executor.call_signed_iterative_combined() + except Exception as err: + logger.debug(str(err)) + if str(err).startswith("transaction too large:"): + logger.debug("Transaction too large, call call_signed_with_holder_acc():") + call_from_holder = True + else: + raise + + if call_from_holder: + return iterative_executor.call_signed_with_holder_combined() + finally: + self.free_perm_accs() + + + def create_instruction_constructor(self): + return NeonInstruction(self.sender.get_operator_key(), self.eth_trx, self.eth_accounts, self.caller_token) + + + def create_noniterative_executor(self): + self.instruction.init_eth_trx(self.eth_trx, self.eth_accounts, self.caller_token) + return NoniterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx) + + + def create_iterative_executor(self): + self.instruction.init_iterative(self.storage, self.holder, self.perm_accs_id) + return IterativeTransactionSender(self.sender, self.instruction, self.create_acc_trx, self.eth_trx, self.steps, self.steps_emulated) + + + def init_perm_accs(self): + while True: + with new_acc_id_glob.get_lock(): + try: + self.perm_accs_id = acc_list_glob.pop(0) + except IndexError: + self.perm_accs_id = new_acc_id_glob.value + new_acc_id_glob.value += 1 + + logger.debug("LOCK RESOURCES {}".format(self.perm_accs_id)) + + acc_id_bytes = self.perm_accs_id.to_bytes((self.perm_accs_id.bit_length() + 7) // 8, 'big') + + storage_seed = keccak_256(b"storage" + acc_id_bytes).hexdigest()[:32] + storage_seed = bytes(storage_seed, 'utf8') + + holder_seed = keccak_256(b"holder" + acc_id_bytes).hexdigest()[:32] + holder_seed = bytes(holder_seed, 'utf8') + + try: + self.storage, self.holder = self.create_multiple_accounts_with_seed( + seeds=[storage_seed, holder_seed], + sizes=[STORAGE_SIZE, STORAGE_SIZE] + ) + except Exception as err: + logger.warn("Account is locked err({}) id({}) owner({})".format(str(err), self.perm_accs_id, self.sender.get_operator_key())) + else: + break + + + def free_perm_accs(self): + logger.debug("FREE RESOURCES {}".format(self.perm_accs_id)) + with new_acc_id_glob.get_lock(): + acc_list_glob.append(self.perm_accs_id) + + + def create_account_with_seed(self, seed, storage_size): + account = accountWithSeed(self.sender.get_operator_key(), seed) + + if self.sender.get_sol_balance(account) == 0: + minimum_balance = self.sender.get_rent_exempt_balance_for_size(storage_size) + logger.debug("Minimum balance required for account {}".format(minimum_balance)) + + trx = Transaction() + trx.add(self.instruction.create_account_with_seed_trx(account, seed, minimum_balance, storage_size)) + self.sender.send_transaction(trx, eth_trx=self.eth_trx, reason='createAccountWithSeed') + + return account + + + def create_multiple_accounts_with_seed(self, seeds, sizes): + accounts = [] + trx = Transaction() + + for seed, storage_size in zip(seeds, sizes): + account = accountWithSeed(self.sender.get_operator_key(), seed) + accounts.append(account) + + minimum_balance = self.sender.get_rent_exempt_balance_for_size(storage_size) + + account_info = self.sender.get_account_info(account) + if account_info is None: + logger.debug("Minimum balance required for account {}".format(minimum_balance)) + + trx.add(self.instruction.create_account_with_seed_trx(account, seed, minimum_balance, storage_size)) + else: + (tag, lamports, owner) = account_info + if lamports < minimum_balance: + raise Exception("insufficient balance") + if PublicKey(owner) != PublicKey(EVM_LOADER_ID): + raise Exception("wrong owner") + if tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}: + raise Exception("not empty, not finalized") + + if len(trx.instructions) > 0: + self.sender.send_transaction(trx, eth_trx=self.eth_trx, reason='createAccountWithSeed') + + return accounts + + + def create_account_list_by_emulate(self): + sender_ether = bytes.fromhex(self.eth_trx.sender()) + add_keys_05 = [] + self.create_acc_trx = Transaction() + + if not self.eth_trx.toAddress: + to_address_arg = "deploy" + to_address = keccak_256(rlp.encode((bytes.fromhex(self.eth_trx.sender()), self.eth_trx.nonce))).digest()[-20:] + else: + to_address_arg = self.eth_trx.toAddress.hex() + to_address = self.eth_trx.toAddress + + logger.debug("send_addr: %s", self.eth_trx.sender()) + logger.debug("dest_addr: %s", to_address.hex()) + + output_json = call_emulated(to_address_arg, sender_ether.hex(), self.eth_trx.callData.hex(), hex(self.eth_trx.value)) + logger.debug("emulator returns: %s", json.dumps(output_json, indent=3)) + + # resize storage account + resize_instr = [] + for acc_desc in output_json["accounts"]: + if acc_desc["new"] == False: + if acc_desc["code_size_current"] is not None and acc_desc["code_size"] is not None: + if acc_desc["code_size"] > acc_desc["code_size_current"]: + code_size = acc_desc["code_size"] + 2048 + seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) + code_account_new = accountWithSeed(self.sender.get_operator_key(), seed) + + logger.debug("creating new code_account with increased size %s", code_account_new) + self.create_account_with_seed(seed, code_size) + logger.debug("resized account is created %s", code_account_new) + + resize_instr.append(self.instruction.make_resize_instruction(acc_desc, code_account_new, seed)) + # replace code_account + acc_desc["contract"] = code_account_new + + for instr in resize_instr: + logger.debug("code and storage migration, account %s from %s to %s", instr.keys[0].pubkey, instr.keys[1].pubkey, instr.keys[2].pubkey) + + tx = Transaction().add(instr) + success = False + count = 0 + + while count < 2: + logger.debug("attemt: %d", count) + + self.sender.send_transaction(tx, eth_trx=self.eth_trx, reason='resize_storage_account') + info = self.sender._getAccountData(instr.keys[0].pubkey, ACCOUNT_INFO_LAYOUT.sizeof()) + info_data = AccountInfo.frombytes(info) + if info_data.code_account == instr.keys[2].pubkey: + success = True + logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) + break + # wait for unlock account + time.sleep(1) + count = count+1 + + if success == False: + raise Exception("Can't resize storage account. Account is blocked {}".format(instr.keys[0].pubkey)) + + for acc_desc in output_json["accounts"]: + address = bytes.fromhex(acc_desc["address"][2:]) + + code_account = None + code_account_writable = False + if acc_desc["new"]: + logger.debug("Create solana accounts for %s: %s %s", acc_desc["address"], acc_desc["account"], acc_desc["contract"]) + if acc_desc["code_size"]: + seed = b58encode(ACCOUNT_SEED_VERSION+address) + code_account = accountWithSeed(self.sender.get_operator_key(), seed) + logger.debug(" with code account %s", code_account) + code_size = acc_desc["code_size"] + 2048 + code_account_balance = self.sender.get_rent_exempt_balance_for_size(code_size) + self.create_acc_trx.add(self.instruction.create_account_with_seed_trx(code_account, seed, code_account_balance, code_size)) + # add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=acc_desc["writable"])) + code_account_writable = acc_desc["writable"] + + create_trx = self.instruction.make_trx_with_create_and_airdrop(address, code_account) + self.create_acc_trx.add(create_trx) + + if address == to_address: + contract_sol = PublicKey(acc_desc["account"]) + if acc_desc["new"]: + code_sol = code_account + code_writable = code_account_writable + else: + if acc_desc["contract"] != None: + code_sol = PublicKey(acc_desc["contract"]) + code_writable = acc_desc["writable"] + else: + code_sol = None + code_writable = None + + elif address == sender_ether: + sender_sol = PublicKey(acc_desc["account"]) + else: + add_keys_05.append(AccountMeta(pubkey=acc_desc["account"], is_signer=False, is_writable=True)) + token_account = getTokenAddr(PublicKey(acc_desc["account"])) + add_keys_05.append(AccountMeta(pubkey=token_account, is_signer=False, is_writable=True)) + if acc_desc["new"]: + if code_account: + add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=code_account_writable)) + else: + if acc_desc["contract"]: + add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) + + + for token_account in output_json["token_accounts"]: + add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) + + if token_account["new"]: + self.create_acc_trx.add(self.instruction.createERC20TokenAccountTrx(token_account)) + + for account_meta in output_json["solana_accounts"]: + add_keys_05.append(AccountMeta(pubkey=PublicKey(account_meta["pubkey"]), is_signer=account_meta["is_signer"], is_writable=account_meta["is_writable"])) + + self.caller_token = getTokenAddr(PublicKey(sender_sol)) + + self.eth_accounts = [ + AccountMeta(pubkey=contract_sol, is_signer=False, is_writable=True), + AccountMeta(pubkey=getTokenAddr(contract_sol), is_signer=False, is_writable=True), + ] + ([AccountMeta(pubkey=code_sol, is_signer=False, is_writable=code_writable)] if code_sol != None else []) + [ + AccountMeta(pubkey=sender_sol, is_signer=False, is_writable=True), + AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), + ] + add_keys_05 + + self.steps_emulated = output_json["steps_executed"] + + +class NoniterativeTransactionSender: + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx): + self.sender = solana_interactor + self.instruction = neon_instruction + self.create_acc_trx = create_acc_trx + self.eth_trx = eth_trx + + + def call_signed_noniterative(self): + call_txs_05 = Transaction() + if len(self.create_acc_trx.instructions) > 0: + call_txs_05.add(self.create_acc_trx) + call_txs_05.add(self.instruction.make_noniterative_call_transaction(len(call_txs_05.instructions))) + result = self.sender.send_measured_transaction(call_txs_05, self.eth_trx, 'CallFromRawEthereumTX') + + if check_for_errors(result): + if check_if_program_exceeded_instructions(result): + raise Exception("Program failed to complete") + raise Exception(json.dumps(result['result']['meta'])) + + return result['result']['transaction']['signatures'][0] + + +class IterativeTransactionSender: + CONTINUE_REGULAR = 'ContinueV02' + CONTINUE_COMBINED = 'PartialCallOrContinueFromRawEthereumTX' + CONTINUE_HOLDER_COMB = 'ExecuteTrxFromAccountDataIterativeOrContinue' + + def __init__(self, solana_interactor: SolanaInteractor, neon_instruction: NeonInstruction, create_acc_trx: Transaction, eth_trx: EthTrx, steps: int, steps_emulated: int): + self.sender = solana_interactor + self.instruction = neon_instruction + self.create_acc_trx = create_acc_trx + self.eth_trx = eth_trx + self.steps = steps + self.steps_emulated = steps_emulated + self.instruction_type = self.CONTINUE_REGULAR + + + def call_signed_iterative_combined(self): + self.create_accounts_for_trx() + self.instruction_type = self.CONTINUE_COMBINED + return self.call_continue() + + + def call_signed_with_holder_combined(self): + self.write_trx_to_holder_account() + self.create_accounts_for_trx() + self.instruction_type = self.CONTINUE_HOLDER_COMB + return self.call_continue() + + + def create_accounts_for_trx(self): + length = len(self.create_acc_trx.instructions) + if length == 0: + return + logger.debug(f"Create account for trx: {length}") + precall_txs = Transaction() + precall_txs.add(self.create_acc_trx) + result = self.sender.send_measured_transaction(precall_txs, self.eth_trx, 'CreateAccountsForTrx') + if check_for_errors(result): + raise Exception("Failed to create account for trx") + + + def write_trx_to_holder_account(self): + logger.debug('write_trx_to_holder_account') + msg = self.eth_trx.signature() + len(self.eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + self.eth_trx.unsigned_msg() + + offset = 0 + receipts = [] + rest = msg + while len(rest): + (part, rest) = (rest[:1000], rest[1000:]) + trx = self.instruction.make_write_transaction(offset, part) + receipts.append(self.sender.send_transaction_unconfirmed(trx)) + offset += len(part) + + logger.debug("receipts %s", receipts) + self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason='WriteHolder') + + + def call_continue(self): + return_result = None + try: + return_result = self.call_continue_bucked() + except Exception as err: + logger.debug("call_continue_bucked_combined exception: {}".format(str(err))) + if str(err).startswith("transaction too large:"): + raise + + if return_result is not None: + return return_result + + return self.call_continue_iterative() + + + def call_continue_iterative(self): + try: + return self.call_continue_step_by_step() + except Exception as err: + logger.error("call_continue_step_by_step exception:") + logger.debug(str(err)) + + return self.call_cancel() + + + def call_continue_step_by_step(self): + while True: + logger.debug("Continue iterative step:") + result = self.call_continue_step() + signature = check_if_continue_returned(result) + if signature is not None: + return signature + + + def call_continue_step(self): + step_count = self.steps + while step_count > 0: + trx = self.instruction.make_continue_transaction(step_count) + + logger.debug("Step count {}".format(step_count)) + try: + result = self.sender.send_measured_transaction(trx, self.eth_trx, 'ContinueV02') + return result + except SendTransactionError as err: + if check_if_program_exceeded_instructions(err.result): + step_count = int(step_count * 90 / 100) + else: + raise + raise Exception("Can't execute even one EVM instruction") + + + def call_cancel(self): + trx = self.instruction.make_cancel_transaction() + + logger.debug("Cancel") + result = self.sender.send_measured_transaction(trx, self.eth_trx, 'CancelWithNonce') + return result['result']['transaction']['signatures'][0] + + + def call_continue_bucked(self): + logger.debug("Send bucked combined: %s", self.instruction_type) + steps = self.steps + + receipts = [] + for index in range(math.ceil(self.steps_emulated/self.steps) + self.addition_count()): + try: + trx = self.make_bucked_trx(steps, index) + receipts.append(self.sender.send_transaction_unconfirmed(trx)) + except SendTransactionError as err: + logger.error(f"Failed to call continue bucked, error: {err.result}") + if check_if_storage_is_empty_error(err.result): + pass + elif check_if_program_exceeded_instructions(err.result): + steps = int(steps * 90 / 100) + else: + raise + except Exception as err: + logger.debug(str(err)) + if str(err).startswith('failed to get recent blockhash'): + pass + else: + raise + + return self.collect_bucked_results(receipts, self.instruction_type) + + + def addition_count(self): + ''' + How many transactions are needed depending on trx type: + CONTINUE_COMBINED: 2 (1 for begin and 1 for decreased steps) + CONTINUE_HOLDER_COMB: 1 for begin + 0 otherwise + ''' + addition_count = 0 + if self.instruction_type == self.CONTINUE_COMBINED: + addition_count = 2 + elif self.instruction_type == self.CONTINUE_HOLDER_COMB: + addition_count = 1 + return addition_count + + + def make_bucked_trx(self, steps, index): + if self.instruction_type == self.CONTINUE_REGULAR: + return self.instruction.make_continue_transaction(steps, index) + elif self.instruction_type == self.CONTINUE_COMBINED: + return self.instruction.make_partial_call_or_continue_transaction(steps - index) + elif self.instruction_type == self.CONTINUE_HOLDER_COMB: + return self.instruction.make_partial_call_or_continue_from_account_data(steps, index) + else: + raise Exception("Unknown continue type: {}".format(self.instruction_type)) + + + def collect_bucked_results(self, receipts, reason): + logger.debug(f"Collected bucked results: {receipts}") + result_list = self.sender.collect_results(receipts, eth_trx=self.eth_trx, reason=reason) + for result in result_list: + self.sender.get_measurements(result) + signature = check_if_continue_returned(result) + if signature: + return signature diff --git a/proxy/common_neon/utils.py b/proxy/common_neon/utils.py new file mode 100644 index 000000000..be93827e1 --- /dev/null +++ b/proxy/common_neon/utils.py @@ -0,0 +1,13 @@ +from typing import Dict, Optional, Any + + +def get_from_dict(src: Dict, *path) -> Optional[Any]: + """Provides smart getting values from python dictionary""" + val = src + for key in path: + if not isinstance(val, dict): + return None + val = val.get(key) + if val is None: + return None + return val diff --git a/proxy/deploy-test.sh b/proxy/deploy-test.sh index a0df058f2..b47dc651f 100755 --- a/proxy/deploy-test.sh +++ b/proxy/deploy-test.sh @@ -5,8 +5,7 @@ echo "Deploy test..." solana config set -u $SOLANA_URL solana address || solana-keygen new --no-passphrase -solana program dump "$EVM_LOADER" ./evm_loader.dump -export $(/spl/bin/neon-cli --evm_loader "$EVM_LOADER" neon-elf-params ./evm_loader.dump) +export $(/spl/bin/neon-cli --commitment confirmed --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) curl -v --header "Content-Type: application/json" --data '{"method":"eth_blockNumber","id":1,"jsonrpc":"2.0","params":[]}' $PROXY_URL diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index d40820a89..fefd28d6b 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -3,7 +3,7 @@ version: "3" services: solana: container_name: solana - image: neonlabsorg/solana:${SOLANA_REVISION:-v1.7.9-resources} + image: neonlabsorg/solana:${SOLANA_REVISION:-v1.7.9-testnet} environment: SOLANA_URL: http://solana:8899 RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug @@ -46,7 +46,7 @@ services: POSTGRES_PASSWORD: neon-proxy-pass hostname: postgres healthcheck: - test: [ CMD-SHELL, "pg_isready" ] + test: [ CMD-SHELL, "pg_isready -h postgres -p 5432" ] interval: 5s timeout: 10s retries: 10 @@ -65,6 +65,7 @@ services: POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass NEW_USER_AIRDROP_AMOUNT: 100 + LOG_SENDING_SOLANA_TRANSACTION: "YES" CONFIG: ci hostname: proxy depends_on: diff --git a/proxy/environment.py b/proxy/environment.py index df86e2042..927cb30ad 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -6,20 +6,27 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") -evm_loader_id = os.environ.get("EVM_LOADER") +SOLANA_URL = os.environ.get("SOLANA_URL", "http://localhost:8899") +EVM_LOADER_ID = os.environ.get("EVM_LOADER") neon_cli_timeout = float(os.environ.get("NEON_CLI_TIMEOUT", "0.1")) +NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) +CONFIRMATION_CHECK_DELAY = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) +CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) +TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) +MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 +EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) +LOG_SENDING_SOLANA_TRANSACTION = os.environ.get("LOG_SENDING_SOLANA_TRANSACTION", "NO") == "YES" + class solana_cli: def call(self, *args): try: cmd = ["solana", - "--url", solana_url, + "--url", SOLANA_URL, ] + list(args) - print(cmd) + logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, universal_newlines=True) except subprocess.CalledProcessError as err: - import sys logger.debug("ERR: solana error {}".format(err)) raise @@ -29,27 +36,27 @@ def call(self, *args): try: cmd = ["neon-cli", "--commitment=recent", - "--url", solana_url, - "--evm_loader={}".format(evm_loader_id), + "--url", SOLANA_URL, + "--evm_loader={}".format(EVM_LOADER_ID), ] + list(args) - print(cmd) + logger.debug("Calling: " + " ".join(cmd)) return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True) except subprocess.CalledProcessError as err: - import sys + logger.debug("ERR: neon-cli error {}".format(err)) + raise + + def version(self): + try: + cmd = ["neon-cli", + "--version"] + logger.debug("Calling: " + " ".join(cmd)) + return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True).split()[1] + except subprocess.CalledProcessError as err: logger.debug("ERR: neon-cli error {}".format(err)) raise def read_elf_params(out_dict): - logger.debug('load for solana_url={} and evm_loader_id={}'.format(solana_url, evm_loader_id)) - res = solana_cli().call('program', 'dump', evm_loader_id, './evm_loader.dump') - substr = "Wrote program to " - path = "" - for line in res.splitlines(): - if line.startswith(substr): - path = line[len(substr):].strip() - if path == "": - raise Exception("cannot program dump for ", evm_loader_id) - for param in neon_cli().call("neon-elf-params", path).splitlines(): + for param in neon_cli().call("neon-elf-params").splitlines(): if param.startswith('NEON_') and '=' in param: v = param.split('=') out_dict[v[0]] = v[1] diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py new file mode 100644 index 000000000..41c638cc6 --- /dev/null +++ b/proxy/indexer/airdropper.py @@ -0,0 +1,159 @@ +from proxy.indexer.indexer_base import IndexerBase, logger +import os +import requests +import base58 +import json +import logging + +try: + from utils import check_error + from sql_dict import SQLDict +except ImportError: + from .utils import check_error + from .sql_dict import SQLDict + +class Airdropper(IndexerBase): + def __init__(self, + solana_url, + evm_loader_id, + faucet_url = '', + wrapper_whitelist = [], + airdrop_amount = 10, + log_level = 'INFO'): + IndexerBase.__init__(self, solana_url, evm_loader_id, log_level) + + # collection of eth-address-to-create-accout-trx mappings + # for every addresses that was already funded with airdrop + self.airdrop_ready = SQLDict(tablename="airdrop_ready") + self.wrapper_whitelist = wrapper_whitelist + self.airdrop_amount = airdrop_amount + self.faucet_url = faucet_url + + + # helper function checking if given contract address is in whitelist + def _is_allowed_wrapper_contract(self, contract_addr): + return contract_addr in self.wrapper_whitelist + + + # helper function checking if given 'create account' corresponds to 'create erc20 token account' instruction + def _check_create_instr(self, account_keys, create_acc, create_token_acc): + # Must use the same Ethereum account + if account_keys[create_acc['accounts'][1]] != account_keys[create_token_acc['accounts'][2]]: + return False + # Must use the same token program + if account_keys[create_acc['accounts'][5]] != account_keys[create_token_acc['accounts'][6]]: + return False + # Token program must be system token program + if account_keys[create_acc['accounts'][5]] != 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA': + return False + # CreateERC20TokenAccount instruction must use ERC20-wrapper from whitelist + if not self._is_allowed_wrapper_contract(account_keys[create_token_acc['accounts'][3]]): + return False + return True + + + # helper function checking if given 'create erc20 token account' corresponds to 'token transfer' instruction + def _check_transfer(self, account_keys, create_token_acc, token_transfer) -> bool: + return account_keys[create_token_acc['accounts'][1]] == account_keys[token_transfer['accounts'][1]] + + + def _airdrop_to(self, create_acc): + eth_address = "0x" + bytearray(base58.b58decode(create_acc['data'])[20:][:20]).hex() + + if eth_address in self.airdrop_ready: # transaction already processed + return + + logger.info(f"Airdrop to address: {eth_address}") + + json_data = { 'wallet': eth_address, 'amount': self.airdrop_amount } + resp = requests.post(self.faucet_url + '/request_eth_token', json = json_data) + if not resp.ok: + logger.warning(f'Failed to airdrop: {resp.status_code}') + return + + self.airdrop_ready[eth_address] = create_acc + + + def process_trx_airdropper_mode(self, trx): + if check_error(trx): + return + + # helper function finding all instructions that satisfies predicate + def find_instructions(trx, predicate): + return [instr for instr in trx['transaction']['message']['instructions'] if predicate(instr)] + + account_keys = trx["transaction"]["message"]["accountKeys"] + + # Finding instructions specific for airdrop. + # Airdrop triggers on sequence: + # neon.CreateAccount -> neon.CreateERC20TokenAccount -> spl.Transfer (maybe shuffled) + # First: select all instructions that can form such chains + predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + and base58.b58decode(instr['data'])[0] == 0x02 + create_acc_list = find_instructions(trx, predicate) + + predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + and base58.b58decode(instr['data'])[0] == 0x0f + create_token_acc_list = find_instructions(trx, predicate) + + predicate = lambda instr: account_keys[instr['programIdIndex']] == 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' \ + and base58.b58decode(instr['data'])[0] == 0x03 + token_transfer_list = find_instructions(trx, predicate) + + # Second: Find exact chains of instructions in sets created previously + for create_acc in create_acc_list: + for create_token_acc in create_token_acc_list: + if not self._check_create_instr(account_keys, create_acc, create_token_acc): + continue + for token_transfer in token_transfer_list: + if not self._check_transfer(account_keys, create_token_acc, token_transfer): + continue + self._airdrop_to(create_acc) + + + def process_functions(self): + IndexerBase.process_functions(self) + logger.debug("Process receipts") + self.process_receipts() + + + def process_receipts(self): + counter = 0 + for signature in self.transaction_order: + counter += 1 + if signature in self.transaction_receipts: + trx = self.transaction_receipts[signature] + if trx is None: + logger.error("trx is None") + del self.transaction_receipts[signature] + continue + if 'slot' not in trx: + logger.debug("\n{}".format(json.dumps(trx, indent=4, sort_keys=True))) + exit() + if trx['transaction']['message']['instructions'] is not None: + self.process_trx_airdropper_mode(trx) + + +def run_airdropper(solana_url, + evm_loader_id, + faucet_url = '', + wrapper_whitelist = [], + airdrop_amount = 10, + log_level = 'INFO'): + logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s') + logger.setLevel(logging.DEBUG) + logger.info(f"""Running indexer with params: + solana_url: {solana_url}, + evm_loader_id: {evm_loader_id}, + log_level: {log_level}, + faucet_url: {faucet_url}, + wrapper_whitelist: {wrapper_whitelist}, + airdrop_amount: {airdrop_amount}""") + + airdropper = Airdropper(solana_url, + evm_loader_id, + faucet_url, + wrapper_whitelist, + airdrop_amount, + log_level) + airdropper.run() diff --git a/proxy/indexer/solana_receipts_update.py b/proxy/indexer/indexer.py similarity index 81% rename from proxy/indexer/solana_receipts_update.py rename to proxy/indexer/indexer.py index 2bc67b1e2..3254bfafa 100644 --- a/proxy/indexer/solana_receipts_update.py +++ b/proxy/indexer/indexer.py @@ -1,13 +1,12 @@ +from proxy.indexer.indexer_base import logger, IndexerBase, PARALLEL_REQUESTS import base58 -import rlp import json +import logging import os +import rlp import time import logging -from solana.rpc.api import Client from multiprocessing.dummy import Pool as ThreadPool -from typing import Dict, Union -from proxy.environment import solana_url, evm_loader_id try: @@ -17,16 +16,7 @@ from .utils import check_error, get_trx_results, get_trx_receipts, LogDB, Canceller from .sql_dict import SQLDict - -PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2")) CANCEL_TIMEOUT = int(os.environ.get("CANCEL_TIMEOUT", "60")) - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -DEVNET_HISTORY_START = "7BdwyUQ61RUZP63HABJkbW66beLk22tdXnP69KsvQBJekCPVaHoJY47Rw68b3VV1UbQNHxX3uxUSLfiJrfy2bTn" -HISTORY_START = [DEVNET_HISTORY_START] - UPDATE_BLOCK_COUNT = PARALLEL_REQUESTS * 16 class HolderStruct: @@ -38,9 +28,10 @@ def __init__(self, storage_account): class ContinueStruct: - def __init__(self, signature, results, accounts = None): + def __init__(self, signature, results, slot, accounts = None): self.signatures = [signature] self.results = results + self.slot = slot self.accounts = accounts @@ -57,131 +48,34 @@ def __init__(self, eth_trx, eth_signature, from_address, got_result, signatures, self.slot = slot -class Indexer: - def __init__(self): - self.client = Client(solana_url) +class Indexer(IndexerBase): + def __init__(self, + solana_url, + evm_loader_id, + log_level = 'INFO'): + IndexerBase.__init__(self, solana_url, evm_loader_id, log_level) + self.canceller = Canceller() self.logs_db = LogDB() self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash") - self.transaction_receipts = SQLDict(tablename="known_transactions") self.ethereum_trx = SQLDict(tablename="ethereum_transactions") self.eth_sol_trx = SQLDict(tablename="ethereum_solana_transactions") self.sol_eth_trx = SQLDict(tablename="solana_ethereum_transactions") self.constants = SQLDict(tablename="constants") - self.last_slot = 0 - self.current_slot = 0 - self.transaction_order = [] if 'last_block' not in self.constants: self.constants['last_block'] = 0 self.blocked_storages = {} - self.counter_ = 0 - - def run(self, loop = True): - while (True): - try: - logger.debug("Start indexing") - self.gather_unknown_transactions() - logger.debug("Process receipts") - self.process_receipts() - logger.debug("Start getting blocks") - self.gather_blocks() - logger.debug("Unlock accounts") - self.canceller.unlock_accounts(self.blocked_storages) - self.blocked_storages = {} - except Exception as err: - logger.debug("Got exception while indexing. Type(err):%s, Exception:%s", type(err), err) - - - def gather_unknown_transactions(self): - poll_txs = set() - ordered_txs = [] - - minimal_tx = None - continue_flag = True - current_slot = self.client.get_slot(commitment="confirmed")["result"] - maximum_slot = self.last_slot - minimal_slot = current_slot - percent = 0 - counter = 0 - while (continue_flag): - opts: Dict[str, Union[int, str]] = {} - if minimal_tx: - opts["before"] = minimal_tx - opts["commitment"] = "confirmed" - result = self.client._provider.make_request("getSignaturesForAddress", evm_loader_id, opts) - logger.debug("{:>3} get_signatures_for_address {}".format(counter, len(result["result"]))) - counter += 1 - - if len(result["result"]) == 0: - logger.debug("len(result['result']) == 0") - break - - for tx in result["result"]: - solana_signature = tx["signature"] - slot = tx["slot"] - - if solana_signature in HISTORY_START: - logger.debug(solana_signature) - continue_flag = False - break - - ordered_txs.append(solana_signature) - - if solana_signature not in self.transaction_receipts: - poll_txs.add(solana_signature) - - if slot < minimal_slot: - minimal_slot = slot - minimal_tx = solana_signature - - if slot > maximum_slot: - maximum_slot = slot - - if slot < self.last_slot: - continue_flag = False - break - - logger.debug("start getting receipts") - pool = ThreadPool(PARALLEL_REQUESTS) - pool.map(self.get_tx_receipts, poll_txs) - - if len(self.transaction_order): - index = 0 - try: - index = ordered_txs.index(self.transaction_order[0]) - except ValueError: - self.transaction_order = ordered_txs + self.transaction_order - else: - self.transaction_order = ordered_txs[:index] + self.transaction_order - else: - self.transaction_order = ordered_txs - - self.last_slot = maximum_slot - self.current_slot = current_slot - - self.counter_ = 0 - - - def get_tx_receipts(self, solana_signature): - # trx = None - retry = True - - while retry: - try: - trx = self.client.get_confirmed_transaction(solana_signature)['result'] - self.transaction_receipts[solana_signature] = trx - retry = False - except Exception as err: - logger.debug(err) - time.sleep(1) - - self.counter_ += 1 - if self.counter_ % 100 == 0: - logger.debug(self.counter_) - - # return (solana_signature, trx) + def process_functions(self): + IndexerBase.process_functions(self) + logger.debug("Process receipts") + self.process_receipts() + logger.debug("Start getting blocks") + self.gather_blocks() + logger.debug("Unlock accounts") + self.canceller.unlock_accounts(self.blocked_storages) + self.blocked_storages = {} def process_receipts(self): @@ -209,7 +103,7 @@ def process_receipts(self): if trx['transaction']['message']['instructions'] is not None: for instruction in trx['transaction']['message']['instructions']: - if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] != evm_loader_id: + if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] != self.evm_loader_id: continue if check_error(trx): @@ -272,7 +166,7 @@ def process_receipts(self): continue_result.signatures, storage_account, continue_result.accounts, - slot + max(slot, continue_result.slot) ) del continue_table[storage_account] @@ -286,8 +180,8 @@ def process_receipts(self): # logger.debug("rlp.exceptions.RLPException") pass except Exception as err: - if str(err).startswith("unhashable type"): - # logger.debug("unhashable type") + if str(err).startswith("nonhashable type"): + # logger.debug("nonhashable type") pass elif str(err).startswith("unsupported operand type"): # logger.debug("unsupported operand type") @@ -376,9 +270,10 @@ def process_receipts(self): continue_result = continue_table[storage_account] if continue_result.accounts != blocked_accounts: logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") - continue_result.signatures.append(signature) trx_table[eth_signature].got_result = continue_result.results - trx_table[eth_signature].signatures = continue_result.signatures + trx_table[eth_signature].signatures += continue_result.signatures + trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, continue_result.slot) + del continue_table[storage_account] elif instruction_data[0] == 0x0a or instruction_data[0] == 0x14: # Continue or ContinueV02 @@ -389,21 +284,21 @@ def process_receipts(self): blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] if instruction_data[0] == 0x14: # logger.debug("{:>10} {:>6} ContinueV02 0x{}".format(slot, counter, instruction_data.hex())) - blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] + blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] got_result = get_trx_results(trx) if storage_account in continue_table: continue_table[storage_account].signatures.append(signature) - if got_result: - if continue_table[storage_account].results: + if got_result is not None: + if continue_table[storage_account].results is not None: logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE") if continue_table[storage_account].accounts != blocked_accounts: logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL") continue_table[storage_account].results = got_result else: - continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts) elif instruction_data[0] == 0x0b or instruction_data[0] == 0x16: # ExecuteTrxFromAccountDataIterative ExecuteTrxFromAccountDataIterativeV02 if instruction_data[0] == 0x0b: @@ -413,10 +308,8 @@ def process_receipts(self): # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeV02 0x{}".format(slot, counter, instruction_data.hex())) blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]] - holder_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][1]] - blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]] if storage_account in continue_table: continue_table[storage_account].signatures.append(signature) @@ -428,7 +321,7 @@ def process_receipts(self): else: holder_table[holder_account] = HolderStruct(storage_account) else: - continue_table[storage_account] = ContinueStruct(signature, None, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, None, slot, blocked_accounts) holder_table[holder_account] = HolderStruct(storage_account) @@ -436,7 +329,9 @@ def process_receipts(self): # logger.debug("{:>10} {:>6} Cancel 0x{}".format(slot, counter, instruction_data.hex())) storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]] - continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot'])) + blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]] + + continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), slot, blocked_accounts) elif instruction_data[0] == 0x0d: # logger.debug("{:>10} {:>6} PartialCallOrContinueFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex())) @@ -456,6 +351,7 @@ def process_receipts(self): if eth_signature in trx_table: trx_table[eth_signature].signatures.append(signature) + trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, slot) else: trx_table[eth_signature] = TransactionStruct( eth_trx, @@ -468,6 +364,14 @@ def process_receipts(self): slot ) + if storage_account in continue_table: + continue_result = continue_table[storage_account] + trx_table[eth_signature].signatures += continue_result.signatures + if continue_result.results is not None: + trx_table[eth_signature].got_result = continue_result.results + + del continue_table[storage_account] + elif instruction_data[0] == 0x0e: # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeOrContinue 0x{}".format(slot, counter, instruction_data.hex())) @@ -487,7 +391,7 @@ def process_receipts(self): logger.error("Strange behavior. Pay attention. HOLDER ACCOUNT NOT FOUND") holder_table[holder_account] = HolderStruct(storage_account) - if got_result: + if got_result is not None: if continue_table[storage_account].results: logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE") if continue_table[storage_account].accounts != blocked_accounts: @@ -495,7 +399,7 @@ def process_receipts(self): continue_table[storage_account].results = got_result else: - continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts) + continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts) holder_table[holder_account] = HolderStruct(storage_account) if instruction_data[0] > 0x16: @@ -504,11 +408,15 @@ def process_receipts(self): pass for eth_signature, trx_struct in trx_table.items(): - if trx_struct.got_result: + if trx_struct.got_result is not None: self.submit_transaction(trx_struct) - elif trx_struct.storage: - if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT: - self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts) + elif trx_struct.storage is not None: + if not self.submit_transaction_part(trx_struct): + if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT: + logger.debug("Probably blocked") + logger.debug(trx_struct.eth_signature) + logger.debug(trx_struct.signatures) + self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts) else: logger.error(trx_struct) @@ -541,6 +449,23 @@ def submit_transaction(self, trx_struct): logger.debug(trx_struct.eth_signature + " " + status) + def submit_transaction_part(self, trx_struct): + ''' Check if transaction was allready submitted by proxy. ''' + eth_signature = trx_struct.eth_signature + ethereum_trx = self.ethereum_trx.get(eth_signature, None) + if ethereum_trx is not None: + signatures = self.eth_sol_trx.get(eth_signature, []) + signatures = signatures + trx_struct.signatures + self.eth_sol_trx[eth_signature] = signatures + for idx, sig in enumerate(signatures): + self.sol_eth_trx[sig] = { + 'idx': idx, + 'eth': eth_signature, + } + return True + return False + + def gather_blocks(self): max_slot = self.client.get_slot(commitment="recent")["result"] @@ -574,12 +499,27 @@ def get_block(self, slot): return (slot, block_hash) -def run_indexer(): +def run_indexer(solana_url, + evm_loader_id, + log_level = 'DEBUG'): logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s') logger.setLevel(logging.DEBUG) - indexer = Indexer() - indexer.run(False) + logger.info(f"""Running indexer with params: + solana_url: {solana_url}, + evm_loader_id: {evm_loader_id}, + log_level: {log_level}""") + + indexer = Indexer(solana_url, + evm_loader_id, + log_level) + indexer.run() if __name__ == "__main__": - run_indexer() + solana_url = os.environ.get('SOLANA_URL', 'http://localhost:8899') + evm_loader_id = os.environ.get('EVM_LOADER_ID', '53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io') + log_level = os.environ.get('LOG_LEVEL', 'INFO') + + run_indexer(solana_url, + evm_loader_id, + log_level) diff --git a/proxy/indexer/indexer_base.py b/proxy/indexer/indexer_base.py new file mode 100644 index 000000000..cda952bbf --- /dev/null +++ b/proxy/indexer/indexer_base.py @@ -0,0 +1,149 @@ +import os +import time +import logging +from solana.rpc.api import Client +from multiprocessing.dummy import Pool as ThreadPool +from typing import Dict, Union + +try: + from sql_dict import SQLDict +except ImportError: + from .sql_dict import SQLDict + + +PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2")) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +DEVNET_HISTORY_START = "7BdwyUQ61RUZP63HABJkbW66beLk22tdXnP69KsvQBJekCPVaHoJY47Rw68b3VV1UbQNHxX3uxUSLfiJrfy2bTn" +HISTORY_START = [DEVNET_HISTORY_START] + + +log_levels = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARN': logging.WARN, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'FATAL': logging.FATAL, + 'CRITICAL': logging.CRITICAL +} + +class IndexerBase: + def __init__(self, + solana_url, + evm_loader_id, + log_level): + logger.setLevel(log_levels.get(log_level, logging.INFO)) + + self.evm_loader_id = evm_loader_id + self.client = Client(solana_url) + self.transaction_receipts = SQLDict(tablename="known_transactions") + self.last_slot = 0 + self.current_slot = 0 + self.transaction_order = [] + self.counter_ = 0 + + + def run(self): + while (True): + try: + self.process_functions() + except Exception as err: + logger.warning("Got exception while indexing. Type(err):%s, Exception:%s", type(err), err) + + + def process_functions(self): + logger.debug("Start indexing") + self.gather_unknown_transactions() + + + def gather_unknown_transactions(self): + poll_txs = set() + ordered_txs = [] + + minimal_tx = None + continue_flag = True + current_slot = self.client.get_slot(commitment="confirmed")["result"] + maximum_slot = self.last_slot + minimal_slot = current_slot + + counter = 0 + while (continue_flag): + opts: Dict[str, Union[int, str]] = {} + if minimal_tx: + opts["before"] = minimal_tx + opts["commitment"] = "confirmed" + result = self.client._provider.make_request("getSignaturesForAddress", self.evm_loader_id, opts) + logger.debug("{:>3} get_signatures_for_address {}".format(counter, len(result["result"]))) + counter += 1 + + if len(result["result"]) == 0: + logger.debug("len(result['result']) == 0") + break + + for tx in result["result"]: + solana_signature = tx["signature"] + slot = tx["slot"] + + if solana_signature in HISTORY_START: + logger.debug(solana_signature) + continue_flag = False + break + + ordered_txs.append(solana_signature) + + if solana_signature not in self.transaction_receipts: + poll_txs.add(solana_signature) + + if slot < minimal_slot: + minimal_slot = slot + minimal_tx = solana_signature + + if slot > maximum_slot: + maximum_slot = slot + + if slot < self.last_slot: + continue_flag = False + break + + logger.debug("start getting receipts") + pool = ThreadPool(PARALLEL_REQUESTS) + pool.map(self.get_tx_receipts, poll_txs) + + if len(self.transaction_order): + index = 0 + try: + index = ordered_txs.index(self.transaction_order[0]) + except ValueError: + self.transaction_order = ordered_txs + self.transaction_order + else: + self.transaction_order = ordered_txs[:index] + self.transaction_order + else: + self.transaction_order = ordered_txs + + self.last_slot = maximum_slot + self.current_slot = current_slot + + self.counter_ = 0 + + + def get_tx_receipts(self, solana_signature): + # trx = None + retry = True + + while retry: + try: + trx = self.client.get_confirmed_transaction(solana_signature)['result'] + self.transaction_receipts[solana_signature] = trx + retry = False + except Exception as err: + logger.debug(err) + time.sleep(1) + + self.counter_ += 1 + if self.counter_ % 100 == 0: + logger.debug(self.counter_) + + # return (solana_signature, trx) diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index 7b45f725e..60425b7a8 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -3,10 +3,10 @@ import json import logging import os -import rlp import psycopg2 +import rlp import subprocess -from construct import Struct, Bytes, Int64ul + from eth_utils import big_endian_to_int from ethereum.transactions import Transaction as EthTrx from ethereum.utils import sha3 @@ -15,18 +15,16 @@ from solana.rpc.api import Client from solana.rpc.commitment import Confirmed from solana.rpc.types import TxOpts +from solana.system_program import SYS_PROGRAM_ID +from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY from solana.transaction import AccountMeta, Transaction, TransactionInstruction from spl.token.constants import TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address from web3.auto.gethdev import w3 -from proxy.environment import solana_url, evm_loader_id, ETH_TOKEN_MINT_ID -sysvarclock = "SysvarC1ock11111111111111111111111111111111" -sysinstruct = "Sysvar1nstructions1111111111111111111111111" -keccakprog = "KeccakSecp256k11111111111111111111111111111" -rentid = "SysvarRent111111111111111111111111111111111" -incinerator = "1nc1nerator11111111111111111111111111111111" -system = "11111111111111111111111111111111" +from ..common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM +from ..common_neon.layouts import STORAGE_ACCOUNT_INFO_LAYOUT +from ..environment import SOLANA_URL, EVM_LOADER_ID, ETH_TOKEN_MINT_ID logger = logging.getLogger(__name__) @@ -46,7 +44,7 @@ def get_trx_results(trx): accounts = trx["transaction"]["message"]["accountKeys"] evm_loader_instructions = [] for idx, instruction in enumerate(trx["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: + if accounts[instruction["programIdIndex"]] == EVM_LOADER_ID: evm_loader_instructions.append(idx) slot = trx['slot'] @@ -60,7 +58,7 @@ def get_trx_results(trx): for inner in (trx['meta']['innerInstructions']): if inner["index"] in evm_loader_instructions: for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: + if accounts[event['programIdIndex']] == EVM_LOADER_ID: log = base58.b58decode(event['data']) instruction = log[:1] if (int().from_bytes(instruction, "little") == 7): # OnEvent evmInstruction code @@ -115,20 +113,6 @@ def get_trx_receipts(unsigned_msg, signature): return (trx_raw.hex(), eth_signature, from_address) -STORAGE_ACCOUNT_INFO_LAYOUT = Struct( - # "tag" / Int8ul, - "caller" / Bytes(20), - "nonce" / Int64ul, - "gas_limit" / Int64ul, - "gas_price" / Int64ul, - "slot" / Int64ul, - "operator" / Bytes(32), - "accounts_len" / Int64ul, - "executor_data_size" / Int64ul, - "evm_data_size" / Int64ul, - "gas_used_and_paid" / Int64ul, - "number_of_payments" / Int64ul, -) def get_account_list(client, storage_account): opts = { @@ -175,7 +159,7 @@ class LogDB: def __init__(self): POSTGRES_DB = os.environ.get("POSTGRES_DB", "neon-db") POSTGRES_USER = os.environ.get("POSTGRES_USER", "neon-proxy") - POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "neon-proxy") + POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "neon-proxy-pass") POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost") self.conn = psycopg2.connect( @@ -309,7 +293,7 @@ def __init__(self): values = bytes(numbs) self.signer = Account(values) - self.client = Client(solana_url) + self.client = Client(SOLANA_URL) self.operator = self.signer.public_key() self.operator_token = get_associated_token_address(PublicKey(self.operator), ETH_TOKEN_MINT_ID) @@ -318,7 +302,7 @@ def __init__(self): def call(self, *args): try: cmd = ["solana", - "--url", solana_url, + "--url", SOLANA_URL, ] + list(args) logger.debug(cmd) return subprocess.check_output(cmd, universal_newlines=True) @@ -329,15 +313,15 @@ def call(self, *args): def unlock_accounts(self, blocked_storages): readonly_accs = [ - PublicKey(evm_loader_id), + PublicKey(EVM_LOADER_ID), ETH_TOKEN_MINT_ID, PublicKey(TOKEN_PROGRAM_ID), - PublicKey(sysvarclock), - PublicKey(sysinstruct), - PublicKey(keccakprog), - PublicKey(rentid), - PublicKey(incinerator), - PublicKey(system), + PublicKey(SYSVAR_CLOCK_PUBKEY), + PublicKey(SYSVAR_INSTRUCTION_PUBKEY), + PublicKey(KECCAK_PROGRAM), + PublicKey(SYSVAR_RENT_PUBKEY), + PublicKey(INCINERATOR_PUBKEY), + PublicKey(SYS_PROGRAM_ID), ] for storage, trx_accs in blocked_storages.items(): (eth_trx, blocked_accs) = trx_accs @@ -348,6 +332,10 @@ def unlock_accounts(self, blocked_storages): if blocked_accs is None: logger.error("blocked_accs is None") continue + if acc_list is None: + logger.error("acc_list is None. Storage is empty") + logger.error(storage) + continue eth_trx = rlp.decode(bytes.fromhex(eth_trx), EthTrx) if acc_list != blocked_accs: @@ -360,15 +348,15 @@ def unlock_accounts(self, blocked_storages): AccountMeta(pubkey=self.operator, is_signer=True, is_writable=True), AccountMeta(pubkey=self.operator_token, is_signer=False, is_writable=True), AccountMeta(pubkey=acc_list[4], is_signer=False, is_writable=True), - AccountMeta(pubkey=incinerator, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False) + AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), + AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False) ] for acc in acc_list: keys.append(AccountMeta(pubkey=acc, is_signer=False, is_writable=(False if acc in readonly_accs else True))) trx = Transaction() trx.add(TransactionInstruction( - program_id=evm_loader_id, + program_id=EVM_LOADER_ID, data=bytearray.fromhex("15") + eth_trx[0].to_bytes(8, 'little'), keys=keys )) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 54a19ecf4..badc941f6 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -8,33 +8,37 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -from typing import List, Tuple +import base58 +import copy +import eth_utils import json -import unittest +import logging import rlp -import solana -from solana.account import Account as sol_Account -from ..common.utils import socket_connection, text_, build_http_response +import threading +import traceback +import unittest + +from ..common.utils import build_http_response from ..http.codes import httpStatusCodes from ..http.parser import HttpParser from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes -from .eth_proto import Trx as EthTrx -from solana.rpc.api import Client as SolanaClient -from sha3 import keccak_256 -import base58 -import traceback -import threading -from .solana_rest_api_tools import EthereumAddress, getTokens, getAccountInfo, \ - call_signed, call_emulated, EthereumError, neon_config_load, MINIMAL_GAS_PRICE -from solana.rpc.commitment import Commitment, Confirmed +from solana.account import Account as sol_Account +from solana.rpc.api import Client as SolanaClient, SendTransactionError as SolanaTrxError +from solana.rpc.commitment import Confirmed +from typing import List, Tuple, Optional from web3 import Web3 -import logging + +from .eth_proto import Trx as EthTrx +from .solana_rest_api_tools import getAccountInfo, call_signed, neon_config_load, \ + get_token_balance_or_airdrop, estimate_gas +from ..common_neon.address import EthereumAddress +from ..common_neon.emulator_interactor import call_emulated +from ..common_neon.errors import EthereumError from ..core.acceptor.pool import proxy_id_glob -import os -from ..indexer.utils import get_trx_results, LogDB +from ..environment import neon_cli, solana_cli, SOLANA_URL, MINIMAL_GAS_PRICE from ..indexer.sql_dict import SQLDict -from proxy.environment import evm_loader_id, solana_cli, solana_url +from ..indexer.utils import get_trx_results, LogDB logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -42,11 +46,31 @@ modelInstanceLock = threading.Lock() modelInstance = None -EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0")) +NEON_PROXY_PKG_VERSION = '0.5.0' +NEON_PROXY_REVISION = 'NEON_PROXY_REVISION_TO_BE_REPLACED' class EthereumModel: def __init__(self): - # Initialize user account + self.signer = self.get_solana_account() + self.client = SolanaClient(SOLANA_URL) + + self.logs_db = LogDB() + self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash") + self.ethereum_trx = SQLDict(tablename="ethereum_transactions") + self.eth_sol_trx = SQLDict(tablename="ethereum_solana_transactions") + self.sol_eth_trx = SQLDict(tablename="solana_ethereum_transactions") + + with proxy_id_glob.get_lock(): + self.proxy_id = proxy_id_glob.value + proxy_id_glob.value += 1 + logger.debug("worker id {}".format(self.proxy_id)) + + neon_config_load(self) + + + @staticmethod + def get_solana_account() -> Optional[sol_Account]: + solana_account: Optional[sol_Account] = None res = solana_cli().call('config', 'get') substr = "Keypair Path: " path = "" @@ -61,23 +85,11 @@ def __init__(self): nums = list(map(int, pk.strip("[] \n").split(','))) nums = nums[0:32] values = bytes(nums) - self.signer = sol_Account(values) + solana_account = sol_Account(values) + return solana_account - self.client = SolanaClient(solana_url) - - self.logs_db = LogDB() - self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash") - self.ethereum_trx = SQLDict(tablename="ethereum_transactions") - self.eth_sol_trx = SQLDict(tablename="ethereum_solana_transactions") - self.sol_eth_trx = SQLDict(tablename="solana_ethereum_transactions") - - with proxy_id_glob.get_lock(): - self.proxy_id = proxy_id_glob.value - proxy_id_glob.value += 1 - logger.debug("worker id {}".format(self.proxy_id)) - - neon_config_load(self) - pass + def neon_proxy_version(self): + return 'Neon-proxy/v' + NEON_PROXY_PKG_VERSION + '-' + NEON_PROXY_REVISION def web3_clientVersion(self): neon_config_load(self) @@ -88,6 +100,9 @@ def eth_chainId(self): # NEON_CHAIN_ID is a string in decimal form return hex(int(self.neon_config_dict['NEON_CHAIN_ID'])) + def neon_cli_version(self): + return neon_cli().version() + def net_version(self): neon_config_load(self) # NEON_CHAIN_ID is a string in decimal form @@ -98,12 +113,11 @@ def eth_gasPrice(self): def eth_estimateGas(self, param): try: - caller_id = param['from'] if 'from' in param else "0x0000000000000000000000000000000000000000" - contract_id = param['to'] if 'to' in param else "deploy" - data = param['data'] if 'data' in param else "None" - value = param['value'] if 'value' in param else "" - result = call_emulated(contract_id, caller_id, data, value) - return result['used_gas']+EXTRA_GAS + caller_id = param.get('from', "0x0000000000000000000000000000000000000000") + contract_id = param.get('to', "deploy") + data = param.get('data', "None") + value = param.get('value', "") + return estimate_gas(self.client, self.signer, contract_id, EthereumAddress(caller_id), data, value) except Exception as err: logger.debug("Exception on eth_estimateGas: %s", err) raise @@ -116,11 +130,14 @@ def process_block_tag(self, tag): slot = int(self.client.get_slot(commitment=Confirmed)["result"]) elif tag in ('earliest', 'pending'): raise Exception("Invalid tag {}".format(tag)) - else: + elif isinstance(tag, str): slot = int(tag, 16) + elif isinstance(tag, int): + slot = tag + else: + raise Exception(f'Failed to parse block tag: {tag}') return slot - def eth_blockNumber(self): slot = self.client.get_slot(commitment=Confirmed)['result'] logger.debug("eth_blockNumber %s", hex(slot)) @@ -132,9 +149,9 @@ def eth_getBalance(self, account, tag): """ eth_acc = EthereumAddress(account) logger.debug('eth_getBalance: %s %s', account, eth_acc) - balance = getTokens(self.client, self.signer, evm_loader_id, eth_acc, self.signer.public_key()) + balance = get_token_balance_or_airdrop(self.client, self.signer, eth_acc) - return hex(balance*10**9) + return hex(balance * eth_utils.denoms.gwei) def eth_getLogs(self, obj): fromBlock = None @@ -195,6 +212,21 @@ def getBlockBySlot(self, slot, full): } return ret + def eth_getStorageAt(self, account, position, block_identifier): + '''Retrieves storage data by given position + Currently supports only 'latest' block + ''' + if block_identifier != "latest": + logger.debug(f"Block type '{block_identifier}' is not supported yet") + raise RuntimeError(f"Not supported block identifier: {block_identifier}") + + try: + value = neon_cli().call('get-storage-at', account, position) + return value + except Exception as err: + logger.debug(f"Neon-cli failed to execute: {err}") + return '0x00' + def eth_getBlockByHash(self, trx_hash, full): """Returns information about a block by hash. trx_hash - Hash of a block. @@ -251,7 +283,7 @@ def eth_call(self, obj, tag): def eth_getTransactionCount(self, account, tag): logger.debug('eth_getTransactionCount: %s', account) try: - acc_info = getAccountInfo(self.client, EthereumAddress(account), self.signer.public_key()) + acc_info = getAccountInfo(self.client, EthereumAddress(account)) return hex(int.from_bytes(acc_info.trx_count, 'little')) except Exception as err: print("Can't get account info: %s"%err) @@ -351,7 +383,7 @@ def eth_getTransactionByHash(self, trxId, block_info = None): "s": eth_trx[8], } - logger.debug ("eth_getTransactionByHash: %s", json.dumps(ret, indent=3)) + logger.debug("eth_getTransactionByHash: %s", json.dumps(ret, indent=3)) return ret def eth_getCode(self, param, param1): @@ -380,8 +412,8 @@ def eth_sendRawTransaction(self, rawTrx): nonce = int(self.eth_getTransactionCount('0x' + sender, None), base=16) - logger.debug('Eth Sender trx nonce: %s', nonce) - logger.debug('Operator nonce: %s', trx.nonce) + logger.debug('Eth Sender trx nonce in solana: %s', nonce) + logger.debug('Eth Sender trx nonce in transaction: %s', trx.nonce) if (int(nonce) != int(trx.nonce)): raise EthereumError(-32002, 'Verifying nonce before send transaction: Error processing Instruction 1: invalid program argument' @@ -440,8 +472,8 @@ def eth_sendRawTransaction(self, rawTrx): return eth_signature - except solana.rpc.api.SendTransactionError as err: - logger.debug("eth_sendRawTransaction solana.rpc.api.SendTransactionError:%s", err.result) + except SolanaTrxError as err: + self._log_transaction_error(err, logger) raise except EthereumError as err: logger.debug("eth_sendRawTransaction EthereumError:%s", err) @@ -450,6 +482,14 @@ def eth_sendRawTransaction(self, rawTrx): logger.debug("eth_sendRawTransaction type(err):%s, Exception:%s", type(err), err) raise + def _log_transaction_error(self, error: SolanaTrxError, logger): + result = copy.deepcopy(error.result) + logs = result.get("data", {}).get("logs", []) + result.get("data", {}).update({"logs": ["\n\t" + log for log in logs]}) + log_msg = str(result).replace("\\n\\t", "\n\t") + logger.error(f"Got SendTransactionError: {log_msg}") + + class JsonEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, bytearray): @@ -460,12 +500,11 @@ def default(self, obj): class SolanaContractTests(unittest.TestCase): + def setUp(self): self.model = EthereumModel() self.owner = '0xc1566af4699928fdf9be097ca3dc47ece39f8f8e' self.token1 = '0x49a449cd7fd8fbcf34d103d98f2c05245020e35b' -# self.assertEqual(self.getBalance(self.owner), 1000*10**18) -# self.assertEqual(self.getBalance(self.token1), 0) def getBalance(self, account): return int(self.model.eth_getBalance(account, 'latest'), 16) @@ -519,7 +558,6 @@ def test_transferTokens(self): self.assertTrue(receiptId in block['transactions']) - class SolanaProxyPlugin(HttpWebServerBasePlugin): """Extend in-built Web Server to add Reverse Proxy capabilities. """ @@ -555,8 +593,9 @@ def process_request(self, request): } try: method = getattr(self.model, request['method']) - response['result'] = method(*request['params']) - except solana.rpc.api.SendTransactionError as err: + params = request.get('params', []) + response['result'] = method(*params) + except SolanaTrxError as err: traceback.print_exc() response['error'] = err.result except EthereumError as err: @@ -580,7 +619,6 @@ def handle_request(self, request: HttpParser) -> None: }))) return - # print('headers', request.headers) logger.debug('<<< %s 0x%x %s', threading.get_ident(), id(self.model), request.body.decode('utf8')) response = None @@ -601,7 +639,8 @@ def handle_request(self, request: HttpParser) -> None: traceback.print_exc() response = {'jsonrpc': '2.0', 'error': {'code': -32000, 'message': str(err)}} - logger.debug('>>> %s 0x%0x %s', threading.get_ident(), id(self.model), json.dumps(response)) + logger.debug('>>> %s 0x%0x %s %s', threading.get_ident(), id(self.model), json.dumps(response), + request['method'] if 'method' in request else '---') self.client.queue(memoryview(build_http_response( httpStatusCodes.OK, body=json.dumps(response).encode('utf8'), diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 8a748d717..73085b405 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -1,443 +1,25 @@ -import base58 import base64 -import json import logging -import os -import random -import re -import struct -import time + from datetime import datetime -from hashlib import sha256 -from typing import NamedTuple -import rlp -from base58 import b58decode, b58encode -from construct import Bytes, Int8ul, Int32ul, Int64ul -from construct import Struct as cStruct -from eth_keys import keys as eth_keys -from sha3 import keccak_256 -from solana._layouts.system_instructions import SYSTEM_INSTRUCTIONS_LAYOUT -from solana._layouts.system_instructions import InstructionType as SystemInstructionType -from solana.blockhash import Blockhash +from solana.account import Account as SolanaAccount from solana.publickey import PublicKey -from solana.rpc.api import Client, SendTransactionError -from solana.rpc.commitment import Commitment, Confirmed -from solana.rpc.types import TxOpts -from solana.sysvar import * -from solana.transaction import AccountMeta, Transaction, TransactionInstruction -from spl.token.constants import ACCOUNT_LEN, ASSOCIATED_TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID -from spl.token.instructions import get_associated_token_address, create_associated_token_account, transfer2, Transfer2Params -from web3.auto import w3 -from proxy.environment import neon_cli, evm_loader_id, ETH_TOKEN_MINT_ID, COLLATERAL_POOL_BASE, read_elf_params -from .eth_proto import Trx -from ..core.acceptor.pool import new_acc_id_glob, acc_list_glob -from ..indexer.sql_dict import POSTGRES_USER, POSTGRES_HOST, POSTGRES_DB, POSTGRES_PASSWORD -import psycopg2 - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) -location_bin = ".deploy_contract.bin" -confirmation_check_delay = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1")) -USE_COMBINED_START_CONTINUE = os.environ.get("USE_COMBINED_START_CONTINUE", "NO") == "YES" -CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3")) -TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600")) -MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9 - -ACCOUNT_SEED_VERSION=b'\1' - -COLLATERALL_POOL_MAX=10 - -EMPTY_STORAGE_TAG=0 - -sysvarclock = "SysvarC1ock11111111111111111111111111111111" -sysinstruct = "Sysvar1nstructions1111111111111111111111111" -keccakprog = "KeccakSecp256k11111111111111111111111111111" -rentid = "SysvarRent111111111111111111111111111111111" -incinerator = "1nc1nerator11111111111111111111111111111111" -system = "11111111111111111111111111111111" - -STORAGE_SIZE = 128*1024 - -ACCOUNT_INFO_LAYOUT = cStruct( - "type" / Int8ul, - "ether" / Bytes(20), - "nonce" / Int8ul, - "trx_count" / Bytes(8), - "code_account" / Bytes(32), - "is_rw_blocked" / Int8ul, - "rw_blocked_acc" / Bytes(32), - "eth_token_account" / Bytes(32), - "ro_blocked_cnt" / Int8ul, -) - -CODE_INFO_LAYOUT = cStruct( - "tag" / Int8ul, - "owner" / Bytes(20), - "code_size" / Bytes(4), -) - -CREATE_ACCOUNT_LAYOUT = cStruct( - "lamports" / Int64ul, - "space" / Int64ul, - "ether" / Bytes(20), - "nonce" / Int8ul -) - -obligatory_accounts = [ - AccountMeta(pubkey=evm_loader_id, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=sysvarclock, is_signer=False, is_writable=False), -] - -class SQLCost(): - def __init__(self): - - self.conn = psycopg2.connect( - dbname=POSTGRES_DB, - user=POSTGRES_USER, - password=POSTGRES_PASSWORD, - host=POSTGRES_HOST - ) - - self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - cur = self.conn.cursor() - cur.execute(''' - CREATE TABLE IF NOT EXISTS OPERATOR_COST - ( - hash char(64), - cost bigint, - used_gas bigint, - sender char(40), - to_address char(40) , - sig char(100), - status varchar(100), - reason varchar(100) - )''' - ) - - def close(self): - self.conn.close() - - def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason): - cur = self.conn.cursor() - cur.execute(''' - INSERT INTO OPERATOR_COST (hash, cost, used_gas, sender, to_address, sig, status, reason) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s) - ''', - (hash, cost, used_gas, sender, to_address, sig, status, reason) - ) - -class CostSingleton(object): - def __new__(cls): - if not hasattr(cls, 'instance'): - cls.instance = super(CostSingleton, cls).__new__(cls) - cls.instance.operator_cost = SQLCost() - return cls.instance - -class PermanentAccounts: - def __init__(self, client, signer): - while True: - with new_acc_id_glob.get_lock(): - try: - self.acc_id = acc_list_glob.pop(0) - except IndexError: - self.acc_id = new_acc_id_glob.value - new_acc_id_glob.value += 1 - - logger.debug("LOCK RESOURCES {}".format(self.acc_id)) - - self.operator = signer.public_key() - self.operator_token = getTokenAddr(self.operator) - - acc_id_bytes = self.acc_id.to_bytes((self.acc_id.bit_length() + 7) // 8, 'big') - - storage_seed = keccak_256(b"storage" + acc_id_bytes).hexdigest()[:32] - storage_seed = bytes(storage_seed, 'utf8') - - holder_seed = keccak_256(b"holder" + acc_id_bytes).hexdigest()[:32] - holder_seed = bytes(holder_seed, 'utf8') - - try: - self.storage, self.holder = create_multiple_accounts_with_seed( - client, - funding=signer, - base=signer, - seeds=[storage_seed, holder_seed], - sizes=[STORAGE_SIZE, STORAGE_SIZE] - ) - except Exception as err: - logger.warn("Account is locked err({}) id({}) owner({})".format(str(err), self.acc_id, signer.public_key())) - else: - break - - def __del__(self): - logger.debug("FREE RESOURCES {}".format(self.acc_id)) - with new_acc_id_glob.get_lock(): - acc_list_glob.append(self.acc_id) - - -class TransactionInfo: - def __init__(self, caller_token, eth_accounts, eth_trx): - self.eth_trx = eth_trx - - self.caller_token = caller_token - self.eth_accounts = eth_accounts - self.nonce = eth_trx.nonce - - hash = keccak_256(eth_trx.unsigned_msg()).digest() - collateral_pool_index = int().from_bytes(hash[:4], "little") % COLLATERALL_POOL_MAX - self.collateral_pool_index_buf = collateral_pool_index.to_bytes(4, 'little') - self.collateral_pool_address = create_collateral_pool_address(collateral_pool_index) - -class AccountInfo(NamedTuple): - ether: eth_keys.PublicKey - trx_count: int - code_account: PublicKey - - @staticmethod - def frombytes(data): - cont = ACCOUNT_INFO_LAYOUT.parse(data) - return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account)) - -def create_account_layout(lamports, space, ether, nonce): - return bytes.fromhex("02000000")+CREATE_ACCOUNT_LAYOUT.build(dict( - lamports=lamports, - space=space, - ether=ether, - nonce=nonce - )) - -def write_holder_layout(nonce, offset, data): - return (bytes.fromhex('12')+ - nonce.to_bytes(8, byteorder='little')+ - offset.to_bytes(4, byteorder='little')+ - len(data).to_bytes(8, byteorder='little')+ - data) - -def get_account_info(client, storage_account): - opts = { - "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 16, - } - } - - result = client._provider.make_request("getAccountInfo", str(storage_account), opts) - logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) - - info = result['result']['value'] - if info is None: - logger.debug("Can't get information about {}".format(storage_account)) - return None - - data = base64.b64decode(info['data'][0]) - - account_tag = data[0] - lamports = info['lamports'] - owner = info['owner'] - - return (account_tag, lamports, owner) - - -def accountWithSeed(base, seed, program): - # logger.debug(type(base), str(base), type(seed), str(seed), type(program), str(program)) - result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(program)).digest()) - logger.debug('accountWithSeed %s', str(result)) - return result - -def createAccountWithSeedTrx(funding, base, seed, lamports, space, program): - seed_str = str(seed, 'utf8') - data = SYSTEM_INSTRUCTIONS_LAYOUT.build( - dict( - instruction_type = SystemInstructionType.CREATE_ACCOUNT_WITH_SEED, - args=dict( - base=bytes(base), - seed=dict(length=len(seed_str), chars=seed_str), - lamports=lamports, - space=space, - program_id=bytes(program) - ) - ) - ) - logger.debug("createAccountWithSeedTrx %s %s %s", type(base), base, data.hex()) - created = accountWithSeed(base, seed, PublicKey(program)) - logger.debug("created %s", created) - return TransactionInstruction( - keys=[ - AccountMeta(pubkey=funding, is_signer=True, is_writable=True), - AccountMeta(pubkey=created, is_signer=False, is_writable=True), - AccountMeta(pubkey=base, is_signer=True, is_writable=False), - ], - program_id=system, - data=data - ) - - -def create_collateral_pool_address(collateral_pool_index): - COLLATERAL_SEED_PREFIX = "collateral_seed_" - seed = COLLATERAL_SEED_PREFIX + str(collateral_pool_index) - return accountWithSeed(PublicKey(COLLATERAL_POOL_BASE), str.encode(seed), PublicKey(evm_loader_id)) - - -def create_account_with_seed(client, funding, base, seed, storage_size, eth_trx=None): - account = accountWithSeed(base.public_key(), seed, PublicKey(evm_loader_id)) - - if client.get_balance(account, commitment=Confirmed)['result']['value'] == 0: - minimum_balance = client.get_minimum_balance_for_rent_exemption(storage_size, commitment=Confirmed)["result"] - logger.debug("Minimum balance required for account {}".format(minimum_balance)) - - trx = Transaction() - trx.add(createAccountWithSeedTrx(funding.public_key(), base.public_key(), seed, minimum_balance, storage_size, PublicKey(evm_loader_id))) - send_transaction(client, trx, funding, eth_trx=eth_trx, reason='createAccountWithSeed') +from solana.rpc.api import Client as SolanaClient +from solana.rpc.commitment import Confirmed - return account +from ..common_neon.address import ether2program, getTokenAddr, EthereumAddress, AccountInfo +from ..common_neon.errors import SolanaAccountNotFoundError, SolanaErrors +from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT +from ..common_neon.neon_instruction import NeonInstruction +from ..common_neon.solana_interactor import SolanaInteractor +from ..common_neon.transaction_sender import TransactionSender +from ..common_neon.emulator_interactor import call_emulated +from ..common_neon.utils import get_from_dict +from ..environment import read_elf_params, TIMEOUT_TO_RELOAD_NEON_CONFIG, EXTRA_GAS -def create_multiple_accounts_with_seed(client, funding, base, seeds, sizes): - accounts = [] - trx = Transaction() - - for seed, storage_size in zip(seeds, sizes): - account = accountWithSeed(base.public_key(), seed, PublicKey(evm_loader_id)) - accounts.append(account) - - minimum_balance = client.get_minimum_balance_for_rent_exemption(storage_size, commitment=Confirmed)["result"] - - account_info = get_account_info(client, account) - if account_info is None: - logger.debug("Minimum balance required for account {}".format(minimum_balance)) - - trx.add(createAccountWithSeedTrx(funding.public_key(), base.public_key(), seed, minimum_balance, storage_size, PublicKey(evm_loader_id))) - else: - (tag, lamports, owner) = account_info - if lamports < minimum_balance: - raise Exception("insufficient balance") - if PublicKey(owner) != PublicKey(evm_loader_id): - raise Exception("wrong owner") - if tag != EMPTY_STORAGE_TAG: - raise Exception("not empty") - - if len(trx.instructions) > 0: - send_transaction(client, trx, funding) - - return accounts - - -def make_keccak_instruction_data(check_instruction_index, msg_len, data_start): - if check_instruction_index > 255 and check_instruction_index < 0: - raise Exception("Invalid index for instruction - {}".format(check_instruction_index)) - - check_count = 1 - eth_address_size = 20 - signature_size = 65 - eth_address_offset = data_start - signature_offset = eth_address_offset + eth_address_size - message_data_offset = signature_offset + signature_size - - data = struct.pack("B", check_count) - data += struct.pack("= confirmations): -# logger.debug('Confirmed transaction:', resp) - return - time.sleep(confirmation_check_delay) - elapsed_time += confirmation_check_delay - #if not resp["result"]: - raise RuntimeError("could not confirm transaction: ", tx_sig) - #return resp - -def solana2ether(public_key): - from web3 import Web3 - return bytes(Web3.keccak(bytes.fromhex(public_key))[-20:]) - -def ether2program(ether, program_id, base): - if isinstance(ether, str): - if ether.startswith('0x'): - ether = ether[2:] - else: - ether = ether.hex() - output = neon_cli().call("create-program-address", ether) - items = output.rstrip().split(' ') - return (items[0], int(items[1])) - - -def ether2seed(ether, program_id, base): - if isinstance(ether, str): - if ether.startswith('0x'): ether = ether[2:] - else: ether = ether.hex() - seed = b58encode(bytes.fromhex(ether)) - acc = accountWithSeed(base, seed, PublicKey(program_id)) - logger.debug('ether2program: {} {} => {} (seed {})'.format(ether, 255, acc, seed)) - return (acc, 255, seed) +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) def neon_config_load(ethereum_model): @@ -462,673 +44,12 @@ def neon_config_load(ethereum_model): logger.debug(ethereum_model.neon_config_dict) -def call_emulated(contract_id, caller_id, data=None, value=None): - output = emulator(contract_id, caller_id, data, value) - logger.debug("call_emulated %s %s %s %s return %s", contract_id, caller_id, data, value, output) - result = json.loads(output) - exit_status = result['exit_status'] - if exit_status == 'revert': - result_value = result['result'] - if len(result_value) < 8 or result_value[:8] != '08c379a0': - raise EthereumError(code=3, message='execution reverted') - - offset = int(result_value[8:8+64], 16) - length = int(result_value[8+64:8+64+64], 16) - message = str(bytes.fromhex(result_value[8+offset*2+64:8+offset*2+64+length*2]), 'utf8') - raise EthereumError(code=3, message='execution reverted: '+message, data='0x'+result_value) - if result["exit_status"] != "succeed": - raise Exception("evm emulator error ", result) - return result - -def extract_measurements_from_receipt(receipt): - log_messages = receipt['result']['meta']['logMessages'] - transaction = receipt['result']['transaction'] - accounts = transaction['message']['accountKeys'] - instructions = [] - for instr in transaction['message']['instructions']: - program = accounts[instr['programIdIndex']] - instructions.append({ - 'accs': [accounts[acc] for acc in instr['accounts']], - 'program': accounts[instr['programIdIndex']], - 'data': b58decode(instr['data']).hex() - }) - - pattern = re.compile('Program ([0-9A-Za-z]+) (.*)') - messages = [] - for log in log_messages: - res = pattern.match(log) - if res: - (program, reason) = res.groups() - if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) - messages[-1]['logs'].append(log) - - for instr in instructions: - if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue - if messages[0]['program'] != instr['program']: - raise Exception('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) - instr['logs'] = messages.pop(0)['logs'] - exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) - if not exit_result: raise Exception("Can't get exit result") - instr['result'] = exit_result.group(1) - - if instr['program'] == evm_loader_id: - memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) - instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) - if not (memory_result and instruction_result): - raise Exception("Can't parse measurements for evm_loader") - instr['measurements'] = { - 'instructions': instruction_result.group(1), - 'memory': memory_result.group(1) - } - - result = [] - for instr in instructions: - if instr['program'] == evm_loader_id: - result.append({ - 'program':instr['program'], - 'measurements':instr['measurements'], - 'result':instr['result'], - 'data':instr['data'] - }) - return result - -# Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh) -def get_measurements(result): - try: - measurements = extract_measurements_from_receipt(result) - for m in measurements: logger.info(json.dumps(m)) - except Exception as err: - logger.error("Can't get measurements %s"%err) - logger.info("Failed result: %s"%json.dumps(result, indent=3)) - -def send_transaction(client, trx, signer, eth_trx=None, reason=None): - result = client.send_transaction(trx, signer, opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed)) - confirm_transaction(client, result["result"]) - result = client.get_confirmed_transaction(result["result"]) - update_transaction_cost(result, eth_trx, reason=reason) - return result - -def send_measured_transaction(client, trx, signer, eth_trx, reason): - result = send_transaction(client, trx, signer, eth_trx=eth_trx, reason=reason) - get_measurements(result) - return result - -def check_if_program_exceeded_instructions(err_result): - err_instruction = "Program failed to complete: exceeded maximum number of instructions allowed" - err_budget = "failed: Computational budget exceeded" - - if err_result['data']['logs'][-1].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-2].find(err_instruction) >= 0 or \ - err_result['data']['logs'][-1].find(err_budget) >= 0: - return True - return False - - -def check_if_continue_returned(result): - tx_info = result['result'] - accounts = tx_info["transaction"]["message"]["accountKeys"] - evm_loader_instructions = [] - - for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: - evm_loader_instructions.append(idx) - - for inner in (tx_info['meta']['innerInstructions']): - if inner["index"] in evm_loader_instructions: - for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: - instruction = base58.b58decode(event['data'])[:1] - if int().from_bytes(instruction, "little") == 6: # OnReturn evmInstruction code - return (True, tx_info['transaction']['signatures'][0]) - return (False, ()) - - -def call_continue(signer, client, perm_accs, trx_info, steps): - try: - return call_continue_iterative(signer, client, perm_accs, trx_info, steps) - except Exception as err: - logger.debug("call_continue_iterative exception:") - logger.debug(str(err)) - - return sol_instr_21_cancel(signer, client, perm_accs, trx_info) - -def call_continue_iterative(signer, client, perm_accs, trx_info, step_count): - while True: - logger.debug("Continue iterative step:") - result = sol_instr_10_continue(signer, client, perm_accs, trx_info, step_count) - (succeed, signature) = check_if_continue_returned(result) - if succeed: - return signature - - -def sol_instr_10_continue(signer, client, perm_accs, trx_info, initial_step_count): - step_count = initial_step_count - while step_count > 0: - trx = Transaction() - trx.add(make_continue_instruction(signer, perm_accs, trx_info, step_count)) - - logger.debug("Step count {}".format(step_count)) - try: - result = send_measured_transaction(client, trx, signer, trx_info.eth_trx, 'ContinueV02') - return result - except SendTransactionError as err: - if check_if_program_exceeded_instructions(err.result): - step_count = int(step_count * 90 / 100) - else: - raise - raise Exception("Can't execute even one EVM instruction") - - -def sol_instr_21_cancel(signer, client, perm_accs, trx_info): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - trx = Transaction() - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=bytearray.fromhex("15") + trx_info.nonce.to_bytes(8, 'little'), - keys=[ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=incinerator, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - )) - - logger.debug("Cancel") - result = send_measured_transaction(client, trx, signer, trx_info.eth_trx, 'CancelWithNonce') - return result['result']['transaction']['signatures'][0] - - -def make_partial_call_instruction(signer, perm_accs, trx_info, step_count, call_data): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("13") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little") + call_data, - keys = [ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_continue_instruction(signer, perm_accs, trx_info, step_count, index=None): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - data = bytearray.fromhex("14") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little") - if index: - data = data + index.to_bytes(8, byteorder="little") - - return TransactionInstruction( - program_id = evm_loader_id, - data = data, - keys = [ - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_call_from_account_instruction(signer, perm_accs, trx_info, step_count = 0): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("16") + trx_info.collateral_pool_index_buf + step_count.to_bytes(8, byteorder="little"), - keys = [ - AccountMeta(pubkey=perm_accs.holder, is_signer=False, is_writable=True), - AccountMeta(pubkey=perm_accs.storage, is_signer=False, is_writable=True), - - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + [ - - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - ] + obligatory_accounts - ) - - -def make_05_call_instruction(signer, trx_info, call_data): - operator = signer.public_key() - operator_token = getTokenAddr(operator) - - return TransactionInstruction( - program_id = evm_loader_id, - data = bytearray.fromhex("05") + trx_info.collateral_pool_index_buf + call_data, - keys = [ - AccountMeta(pubkey=sysinstruct, is_signer=False, is_writable=False), - AccountMeta(pubkey=operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=trx_info.collateral_pool_address, is_signer=False, is_writable=True), - AccountMeta(pubkey=operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=trx_info.caller_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - - ] + trx_info.eth_accounts + obligatory_accounts - ) - - -def update_transaction_cost(receipt, eth_trx, extra_sol_trx=False, reason=None): - cost = receipt['result']['meta']['preBalances'][0] - receipt['result']['meta']['postBalances'][0] - if eth_trx: - hash = eth_trx.hash_signed().hex() - sender = eth_trx.sender() - to_address = eth_trx.toAddress.hex() if eth_trx.toAddress else "None" - else: - hash = None - sender = None - to_address = None - - sig = receipt['result']['transaction']['signatures'][0] - used_gas=None - - tx_info = receipt['result'] - accounts = tx_info["transaction"]["message"]["accountKeys"] - evm_loader_instructions = [] - - for idx, instruction in enumerate(tx_info["transaction"]["message"]["instructions"]): - if accounts[instruction["programIdIndex"]] == evm_loader_id: - evm_loader_instructions.append(idx) - - for inner in (tx_info['meta']['innerInstructions']): - if inner["index"] in evm_loader_instructions: - for event in inner['instructions']: - if accounts[event['programIdIndex']] == evm_loader_id: - used_gas = base58.b58decode(event['data'])[2:10] - used_gas = int().from_bytes(used_gas, "little") - - table = CostSingleton().operator_cost - table.insert( - hash, - cost, - used_gas if used_gas else 0, - sender, - to_address, - sig, - 'extra' if extra_sol_trx else 'ok', - reason if reason else '' - ) - -def create_account_list_by_emulate(signer, client, eth_trx): - sender_ether = bytes.fromhex(eth_trx.sender()) - add_keys_05 = [] - trx = Transaction() - - if not eth_trx.toAddress: - to_address_arg = "deploy" - to_address = keccak_256(rlp.encode((bytes.fromhex(eth_trx.sender()), eth_trx.nonce))).digest()[-20:] - else: - to_address_arg = eth_trx.toAddress.hex() - to_address = eth_trx.toAddress - - output_json = call_emulated(to_address_arg, sender_ether.hex(), eth_trx.callData.hex(), hex(eth_trx.value)) - logger.debug("emulator returns: %s", json.dumps(output_json, indent=3)) - - # resize storage account - resize_instr = [] - for acc_desc in output_json["accounts"]: - if acc_desc["new"] == False: - - address = bytes.fromhex(acc_desc["address"][2:]) - if acc_desc["code_size_current"] is not None and acc_desc["code_size"] is not None: - if acc_desc["code_size"] > acc_desc["code_size_current"]: - code_size = acc_desc["code_size"] + 2048 - seed = b58encode(ACCOUNT_SEED_VERSION + os.urandom(20)) - code_account_new = accountWithSeed(signer.public_key(), seed, PublicKey(evm_loader_id)) - - logger.debug("creating new code_account with increased size %s", code_account_new) - create_account_with_seed(client, signer, signer, seed, code_size, eth_trx); - logger.debug("resized account is created %s", code_account_new) - - resize_instr.append(TransactionInstruction( - keys=[ - AccountMeta(pubkey=PublicKey(acc_desc["account"]), is_signer=False, is_writable=True), - ( - AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=True) - if acc_desc["contract"] else - AccountMeta(pubkey=PublicKey("11111111111111111111111111111111"), is_signer=False, is_writable=False) - ), - AccountMeta(pubkey=code_account_new, is_signer=False, is_writable=True), - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=False) - ], - program_id=evm_loader_id, - data=bytearray.fromhex("11")+bytes(seed) # 17- ResizeStorageAccount - )) - # replace code_account - acc_desc["contract"] = code_account_new - - for instr in resize_instr: - logger.debug("code and storage migration, account %s from %s to %s", instr.keys[0].pubkey, instr.keys[1].pubkey, instr.keys[2].pubkey) - - tx = Transaction().add(instr) - success = False - count = 0 - - while count < 2: - logger.debug("attemt: %d", count) - - send_transaction(client, tx, signer, eth_trx=eth_trx, reason='resize_storage_account') - info = _getAccountData(client, instr.keys[0].pubkey, ACCOUNT_INFO_LAYOUT.sizeof()) - info_data = AccountInfo.frombytes(info) - if info_data.code_account == instr.keys[2].pubkey: - success = True - logger.debug("successful code and storage migration, %s", instr.keys[0].pubkey) - break - # wait for unlock account - time.sleep(1) - count = count+1 - - if success == False: - raise Exception("Can't resize storage account. Account is blocked {}".format(instr.keys[0].pubkey)) - - for acc_desc in output_json["accounts"]: - address = bytes.fromhex(acc_desc["address"][2:]) - - code_account = None - code_account_writable = False - if acc_desc["new"]: - logger.debug("Create solana accounts for %s: %s %s", acc_desc["address"], acc_desc["account"], acc_desc["contract"]) - if acc_desc["code_size"]: - seed = b58encode(ACCOUNT_SEED_VERSION+address) - code_account = accountWithSeed(signer.public_key(), seed, PublicKey(evm_loader_id)) - logger.debug(" with code account %s", code_account) - code_size = acc_desc["code_size"] + 2048 - code_account_balance = client.get_minimum_balance_for_rent_exemption(code_size)["result"] - trx.add(createAccountWithSeedTrx(signer.public_key(), signer.public_key(), seed, code_account_balance, code_size, PublicKey(evm_loader_id))) - # add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=acc_desc["writable"])) - code_account_writable = acc_desc["writable"] - - (create_trx, solana_address, token_address) = createEtherAccountTrx(client, address, evm_loader_id, signer, code_account) - trx.add(create_trx) - - if address == sender_ether and NEW_USER_AIRDROP_AMOUNT > 0: - trx.add(transfer2(Transfer2Params( - amount=NEW_USER_AIRDROP_AMOUNT*1_000_000_000, - decimals=9, - dest=get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID), - mint=ETH_TOKEN_MINT_ID, - owner=signer.public_key(), - program_id=TOKEN_PROGRAM_ID, - source=getTokenAddr(signer.public_key()), - ))) - logger.debug("Token transfer to %s as ethereum 0x%s amount %s", - get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID), - acc_desc["address"], - str(NEW_USER_AIRDROP_AMOUNT)) - - if address == to_address: - contract_sol = PublicKey(acc_desc["account"]) - if acc_desc["new"]: - code_sol = code_account - code_writable = code_account_writable - else: - if acc_desc["contract"] != None: - code_sol = PublicKey(acc_desc["contract"]) - code_writable = acc_desc["writable"] - else: - code_sol = None - code_writable = None - - elif address == sender_ether: - sender_sol = PublicKey(acc_desc["account"]) - else: - add_keys_05.append(AccountMeta(pubkey=acc_desc["account"], is_signer=False, is_writable=True)) - token_account = get_associated_token_address(PublicKey(acc_desc["account"]), ETH_TOKEN_MINT_ID) - add_keys_05.append(AccountMeta(pubkey=token_account, is_signer=False, is_writable=True)) - if acc_desc["new"]: - if code_account: - add_keys_05.append(AccountMeta(pubkey=code_account, is_signer=False, is_writable=code_account_writable)) - else: - if acc_desc["contract"]: - add_keys_05.append(AccountMeta(pubkey=acc_desc["contract"], is_signer=False, is_writable=acc_desc["writable"])) - - - for token_account in output_json["token_accounts"]: - add_keys_05.append(AccountMeta(pubkey=PublicKey(token_account["key"]), is_signer=False, is_writable=True)) - - if token_account["new"]: - trx.add(createERC20TokenAccountTrx(signer, token_account)) - - for account_meta in output_json["solana_accounts"]: - add_keys_05.append(AccountMeta(pubkey=PublicKey(account_meta["pubkey"]), is_signer=account_meta["is_signer"], is_writable=account_meta["is_writable"])) - - caller_token = get_associated_token_address(PublicKey(sender_sol), ETH_TOKEN_MINT_ID) - - eth_accounts = [ - AccountMeta(pubkey=contract_sol, is_signer=False, is_writable=True), - AccountMeta(pubkey=get_associated_token_address(contract_sol, ETH_TOKEN_MINT_ID), is_signer=False, is_writable=True), - ] + ([AccountMeta(pubkey=code_sol, is_signer=False, is_writable=code_writable)] if code_sol != None else []) + [ - AccountMeta(pubkey=sender_sol, is_signer=False, is_writable=True), - AccountMeta(pubkey=caller_token, is_signer=False, is_writable=True), - ] + add_keys_05 - - trx_info = TransactionInfo(caller_token, eth_accounts, eth_trx) - - return (trx_info, sender_ether, trx) - - def call_signed(signer, client, eth_trx, steps): + solana_interactor = SolanaInteractor(signer, client) + trx_sender = TransactionSender(solana_interactor, eth_trx, steps) + return trx_sender.execute() - (trx_info, sender_ether, create_acc_trx) = create_account_list_by_emulate(signer, client, eth_trx) - - call_iterative = False - call_from_holder = False - - if not eth_trx.toAddress: - call_from_holder = True - else: - msg = sender_ether + eth_trx.signature() + eth_trx.unsigned_msg() - - try: - logger.debug("Try single trx call") - return call_signed_noniterative(signer, client, eth_trx, trx_info, msg, create_acc_trx) - except Exception as err: - logger.debug(str(err)) - errStr = str(err) - if "Program failed to complete" in errStr or "Computational budget exceeded" in errStr: - logger.debug("Program exceeded instructions") - call_iterative = True - elif str(err).startswith("transaction too large:"): - logger.debug("Transaction too large, call call_signed_with_holder_acc():") - call_from_holder = True - else: - raise - - perm_accs = PermanentAccounts(client, signer) - try: - if call_iterative: - try: - return call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx) - except Exception as err: - logger.debug(str(err)) - if str(err).startswith("transaction too large:"): - logger.debug("Transaction too large, call call_signed_with_holder_acc():") - call_from_holder = True - else: - raise - - if call_from_holder: - return call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, steps, create_acc_trx) - finally: - del perm_accs - -def call_signed_iterative(signer, client, eth_trx, perm_accs, trx_info, steps, msg, create_acc_trx): - if len(create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(create_acc_trx) - send_measured_transaction(client, precall_txs, signer, eth_trx, 'CreateAccountsForTrx') - - precall_txs = Transaction() - precall_txs.add(TransactionInstruction( - program_id=keccakprog, - data=make_keccak_instruction_data(len(precall_txs.instructions)+1, len(eth_trx.unsigned_msg()), data_start=13), - keys=[ - AccountMeta(pubkey=keccakprog, is_signer=False, is_writable=False), - ])) - precall_txs.add(make_partial_call_instruction(signer, perm_accs, trx_info, 0, msg)) - - logger.debug("Partial call") - send_measured_transaction(client, precall_txs, signer, eth_trx, 'PartialCallFromRawEthereumTXv02') - return call_continue(signer, client, perm_accs, trx_info, steps) - - -def call_signed_noniterative(signer, client, eth_trx, trx_info, msg, create_acc_trx): - call_txs_05 = Transaction() - call_txs_05.add(create_acc_trx) - call_txs_05.add(TransactionInstruction( - program_id=keccakprog, - data=make_keccak_instruction_data(len(call_txs_05.instructions)+1, len(eth_trx.unsigned_msg()), 5), - keys=[ - AccountMeta(pubkey=keccakprog, is_signer=False, is_writable=False), - ])) - call_txs_05.add(make_05_call_instruction(signer, trx_info, msg)) - result = send_measured_transaction(client, call_txs_05, signer, eth_trx, 'CallFromRawEthereumTX') - return result['result']['transaction']['signatures'][0] - - -def call_signed_with_holder_acc(signer, client, eth_trx, perm_accs, trx_info, steps, create_acc_trx): - - write_trx_to_holder_account(signer, client, perm_accs.holder, perm_accs.acc_id, eth_trx) - if len(create_acc_trx.instructions): - precall_txs = Transaction() - precall_txs.add(create_acc_trx) - send_measured_transaction(client, precall_txs, signer, eth_trx, 'create_accounts_for_deploy') - - precall_txs = Transaction() - precall_txs.add(make_call_from_account_instruction(signer, perm_accs, trx_info)) - - # ExecuteTrxFromAccountDataIterative - logger.debug("ExecuteTrxFromAccountDataIterative:") - send_measured_transaction(client, precall_txs, signer, eth_trx, 'ExecuteTrxFromAccountDataIterativeV02') - - return call_continue(signer, client, perm_accs, trx_info, steps) - - -def createEtherAccountTrx(client, ether, evm_loader_id, signer, code_acc=None): - if isinstance(ether, str): - if ether.startswith('0x'): ether = ether[2:] - else: ether = ether.hex() - (sol, nonce) = ether2program(ether, evm_loader_id, signer.public_key()) - associated_token = get_associated_token_address(PublicKey(sol), ETH_TOKEN_MINT_ID) - logger.debug('createEtherAccount: {} {} => {}'.format(ether, nonce, sol)) - logger.debug('associatedTokenAccount: {}'.format(associated_token)) - base = signer.public_key() - data=bytes.fromhex('02000000')+CREATE_ACCOUNT_LAYOUT.build(dict( - lamports=0, - space=0, - ether=bytes.fromhex(ether), - nonce=nonce)) - trx = Transaction() - if code_acc is None: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(sol), is_signer=False, is_writable=True), - AccountMeta(pubkey=associated_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - else: - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=data, - keys=[ - AccountMeta(pubkey=base, is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(sol), is_signer=False, is_writable=True), - AccountMeta(pubkey=associated_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(code_acc), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=ETH_TOKEN_MINT_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=ASSOCIATED_TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - return (trx, sol, associated_token) - -def createERC20TokenAccountTrx(signer, token_info): - trx = Transaction() - trx.add(TransactionInstruction( - program_id=evm_loader_id, - data=bytes.fromhex('0F'), - keys=[ - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["key"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["owner"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["contract"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=PublicKey(token_info["mint"]), is_signer=False, is_writable=True), - AccountMeta(pubkey=system, is_signer=False, is_writable=False), - AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False), - AccountMeta(pubkey=rentid, is_signer=False, is_writable=False), - ])) - - return trx - - - -def write_trx_to_holder_account(signer, client, holder, acc_id, eth_trx): - msg = eth_trx.signature() + len(eth_trx.unsigned_msg()).to_bytes(8, byteorder="little") + eth_trx.unsigned_msg() - - # Write transaction to transaction holder account - offset = 0 - receipts = [] - rest = msg - while len(rest): - (part, rest) = (rest[:1000], rest[1000:]) - trx = Transaction() - # logger.debug("sender_sol %s %s %s", sender_sol, holder, acc.public_key()) - trx.add(TransactionInstruction(program_id=evm_loader_id, - data=write_holder_layout(acc_id, offset, part), - keys=[ - AccountMeta(pubkey=holder, is_signer=False, is_writable=True), - AccountMeta(pubkey=signer.public_key(), is_signer=True, is_writable=False), - ])) - receipts.append(client.send_transaction(trx, signer, - opts=TxOpts(skip_confirmation=True, preflight_commitment=Confirmed))["result"]) - offset += len(part) - logger.debug("receipts %s", receipts) - for rcpt in receipts: - confirm_transaction(client, rcpt) - result = client.get_confirmed_transaction(rcpt) - update_transaction_cost(result, eth_trx, reason='WriteHolder') - logger.debug("confirmed: %s", rcpt) def _getAccountData(client, account, expected_length, owner=None): info = client.get_account_info(account, commitment=Confirmed)['result']['value'] @@ -1140,66 +61,68 @@ def _getAccountData(client, account, expected_length, owner=None): raise Exception("Wrong data length for account data {}".format(account)) return data -def getAccountInfo(client, eth_acc, base_account): - (account_sol, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader_id, base_account) + +def getAccountInfo(client, eth_account: EthereumAddress): + account_sol, nonce = ether2program(eth_account) info = _getAccountData(client, account_sol, ACCOUNT_INFO_LAYOUT.sizeof()) return AccountInfo.frombytes(info) -def getLamports(client, evm_loader, eth_acc, base_account): - (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) - return int(client.get_balance(account, commitment=Confirmed)['result']['value']) - -def getTokens(client, signer, evm_loader, eth_acc, base_account): - (account, nonce) = ether2program(bytes(eth_acc).hex(), evm_loader, base_account) - token_account = get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) - - balance = client.get_token_account_balance(token_account, commitment=Confirmed) - if 'error' in balance: - if NEW_USER_AIRDROP_AMOUNT > 0: - return NEW_USER_AIRDROP_AMOUNT * 1_000_000_000 - else: - logger.debug("'error' in balance:") - return 0 - return int(balance['result']['value']['amount']) +def create_eth_account_and_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress): + trx = NeonInstruction(signer.public_key()).make_trx_with_create_and_airdrop (eth_account) + result = SolanaInteractor(signer, client).send_transaction(trx, None, reason='create_eth_account_and_airdrop') + error = result.get("error") + if error is not None: + logger.error(f"Failed to create eth_account and airdrop: {eth_account}, error occurred: {error}") + raise Exception("Create eth_account error") -def getTokenAddr(account): - return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) -def make_instruction_data_from_tx(instruction, private_key=None): - if isinstance(instruction, dict): - if instruction['chainId'] == None: - raise Exception("chainId value is needed in input dict") - if private_key == None: - raise Exception("Needed private key for transaction creation from fields") +def get_token_balance_gwei(client: SolanaClient, pda_account: str) -> int: + neon_token_account = getTokenAddr(PublicKey(pda_account)) + rpc_response = client.get_token_account_balance(neon_token_account, commitment=Confirmed) + error = rpc_response.get('error') + if error is not None: + message = error.get("message") + if message == SolanaErrors.AccountNotFound.value: + raise SolanaAccountNotFoundError() + logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, " + f"got get_token_account_balance error: \"{message}\"") + raise Exception("Getting balance error") - signed_tx = w3.eth.account.sign_transaction(instruction, private_key) - # logger.debug(signed_tx.rawTransaction.hex()) - _trx = Trx.fromString(signed_tx.rawTransaction) - # logger.debug(json.dumps(_trx.__dict__, cls=JsonEncoder, indent=3)) + balance = get_from_dict(rpc_response, "result", "value", "amount") + if balance is None: + logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, response: {rpc_response}") + raise Exception("Unexpected get_balance response") + return int(balance) - raw_msg = _trx.get_msg(instruction['chainId']) - sig = keys.Signature(vrs=[1 if _trx.v % 2 == 0 else 0, _trx.r, _trx.s]) - pub = sig.recover_public_key_from_msg_hash(_trx.hash()) - # logger.debug(pub.to_hex()) +def get_token_balance_or_airdrop(client: SolanaClient, signer: SolanaAccount, eth_account: EthereumAddress) -> int: + solana_account, nonce = ether2program(eth_account) + logger.debug(f"Get balance for eth account: {eth_account} aka: {solana_account}") - return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) - elif isinstance(instruction, str): - if instruction[:2] == "0x": - instruction = instruction[2:] - - _trx = Trx.fromString(bytearray.fromhex(instruction)) - # logger.debug(json.dumps(_trx.__dict__, cls=JsonEncoder, indent=3)) - - raw_msg = _trx.get_msg() - sig = keys.Signature(vrs=[1 if _trx.v % 2 == 0 else 0, _trx.r, _trx.s]) - pub = sig.recover_public_key_from_msg_hash(_trx.hash()) - - data = pub.to_canonical_address() - data += sig.to_bytes() - data += raw_msg - - return (pub.to_canonical_address(), sig.to_bytes(), raw_msg) - else: - raise Exception("function gets ") + try: + return get_token_balance_gwei(client, solana_account) + except SolanaAccountNotFoundError: + logger.debug(f"Account not found: {eth_account} aka: {solana_account} - create") + create_eth_account_and_airdrop(client, signer, eth_account) + return get_token_balance_gwei(client, solana_account) + + +def is_account_exists(client: SolanaClient, eth_account: EthereumAddress) -> bool: + pda_account, nonce = ether2program(eth_account) + info = client.get_account_info(pda_account, commitment=Confirmed) + value = get_from_dict(info, "result", "value") + return value is not None + + +def estimate_gas(client: SolanaClient, signer: SolanaAccount, contract_id: str, caller_eth_account: EthereumAddress, + data: str = None, value: str = None): + if not is_account_exists(client, caller_eth_account): + create_eth_account_and_airdrop(client, signer, caller_eth_account) + result = call_emulated(contract_id, str(caller_eth_account), data, value) + used_gas = result.get("used_gas") + if used_gas is None: + logger.error(f"Failed estimate_gas, unexpected result, by contract_id: {contract_id}, caller_eth_account: " + f"{caller_eth_account}, data: {data}, value: {value}, emulation result: {result}") + raise Exception("Bad estimate_gas result") + return used_gas + EXTRA_GAS diff --git a/proxy/proxy.py b/proxy/proxy.py index c63281665..86264a6e1 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -23,7 +23,8 @@ from .http.handler import HttpProtocolHandler from multiprocessing import Process -from .indexer.solana_receipts_update import run_indexer +from .indexer.indexer import run_indexer +from proxy.environment import SOLANA_URL, EVM_LOADER_ID logger = logging.getLogger(__name__) @@ -45,7 +46,9 @@ def delete_pid_file(self) -> None: os.remove(self.flags.pid_file) def __enter__(self) -> 'Proxy': - self.indexer = Process(target=run_indexer) + self.indexer = Process(target=run_indexer, + args=(SOLANA_URL, + EVM_LOADER_ID,)) self.indexer.start() self.acceptors = AcceptorPool( flags=self.flags, diff --git a/proxy/run-proxy.sh b/proxy/run-proxy.sh index 103795d70..199489563 100755 --- a/proxy/run-proxy.sh +++ b/proxy/run-proxy.sh @@ -8,7 +8,6 @@ if [ "$CONFIG" == "ci" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://solana:8899" [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=100000 [[ -z "$NEON_CLI_TIMEOUT" ]] && export NEON_CLI_TIMEOUT="0.5" - [[ -z "$USE_COMBINED_START_CONTINUE" ]] && export USE_COMBINED_START_CONTINUE="NO" [[ -z "$CONTINUE_COUNT_FACTOR" ]] && export CONTINUE_COUNT_FACTOR="3" [[ -z "$MINIMAL_GAS_PRICE" ]] && export MINIMAL_GAS_PRICE=0 [[ -z "$POSTGRES_HOST" ]] && export POSTGRES_HOST="postgres" diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index 5df1ad4ab..602fb50e2 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -9,8 +9,7 @@ solana config set -u $SOLANA_URL echo "Dumping evm_loader and extracting ELF parameters" export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) -solana program dump "$EVM_LOADER" ./evm_loader.dump -export $(/spl/bin/neon-cli --evm_loader="$EVM_LOADER" neon-elf-params ./evm_loader.dump) +export $(/spl/bin/neon-cli --commitment confirmed --url $SOLANA_URL --evm_loader="$EVM_LOADER" neon-elf-params) /spl/bin/create-test-accounts.sh 1 @@ -20,7 +19,7 @@ if [ "$(spl-token balance "$NEON_TOKEN_MINT" || echo 0)" -eq 0 ]; then echo 'Create balance and mint token' TOKEN_ACCOUNT=$( (spl-token create-account "$NEON_TOKEN_MINT" || true) | grep -Po 'Creating account \K[^\n]*') echo "TOKEN_ACCOUNT=$TOKEN_ACCOUNT" - spl-token mint "$NEON_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*1000)) --owner /spl/bin/evm_loader-keypair.json -- "$TOKEN_ACCOUNT" + spl-token mint "$NEON_TOKEN_MINT" $(("$NEW_USER_AIRDROP_AMOUNT"*100000)) --owner /spl/bin/evm_loader-keypair.json -- "$TOKEN_ACCOUNT" fi proxy/run-proxy.sh diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py index 232621f0b..0bfb50319 100644 --- a/proxy/testing/__init__.py +++ b/proxy/testing/__init__.py @@ -8,3 +8,5 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import sys +sys.path.append("/spl/bin/") # TODO: get rid off this workaround all related modules should either be installed as package or be linked from submodule diff --git a/proxy/testing/mock_server.py b/proxy/testing/mock_server.py new file mode 100644 index 000000000..ef8e1c34f --- /dev/null +++ b/proxy/testing/mock_server.py @@ -0,0 +1,28 @@ +import requests + +from flask import Flask, request +from threading import Thread + +class MockServer(Thread): + def __init__(self, port): + super().__init__() + self.port = port + self.app = Flask(__name__) + self.url = "http://localhost:%s" % self.port + self.app.add_url_rule("/shutdown", view_func=self._shutdown_server) + + def add_url_rule(self, url, callback, methods): + self.app.add_url_rule(url, view_func=callback, methods=methods) + + def _shutdown_server(self): + if not 'werkzeug.server.shutdown' in request.environ: + raise RuntimeError('Not running the development server') + request.environ['werkzeug.server.shutdown']() + return 'Server shutting down...' + + def run(self): + self.app.run(port=self.port) + + def shutdown_server(self): + requests.get("http://localhost:%s/shutdown" % self.port) + self.join() diff --git a/proxy/testing/test_airdropper.py b/proxy/testing/test_airdropper.py new file mode 100644 index 000000000..9a00d5279 --- /dev/null +++ b/proxy/testing/test_airdropper.py @@ -0,0 +1,176 @@ +import unittest +from proxy.testing.mock_server import MockServer +from proxy.indexer.airdropper import Airdropper +from proxy.indexer.sql_dict import SQLDict +import time +from flask import request, Response +from unittest.mock import MagicMock, patch, call, ANY +import itertools +from proxy.testing.transactions import pre_token_airdrop_trx1, pre_token_airdrop_trx2,\ + create_sol_acc_and_airdrop_trx, wrapper_whitelist, evm_loader_addr, token_airdrop_address1, \ + token_airdrop_address2, token_airdrop_address3 + +class MockFaucet(MockServer): + def __init__(self, port): + super().__init__(port) + self.request_eth_token_mock = MagicMock() + self.request_eth_token_mock.side_effect = itertools.repeat({}) + self.add_url_rule("/request_eth_token", callback=self.request_eth_token, methods=['POST']) + + def request_eth_token(self): + req = request.get_json() + return self.request_eth_token_mock(req) + + +def create_signature_for_address(signature: str): + return { + 'blockTime': 1638177745, # not make sense + 'confirmationStatus': 'finalized', + 'err': None, + 'memo': None, + 'signature': signature, + 'slot': 9748200 # not make sense + } + + +def create_get_signatures_for_address(signatures: list): + return { + 'jsonrpc': '2.0', + 'result': [ create_signature_for_address(sign) for sign in signatures ], + 'id': 1 + } + + +class Test_Airdropper(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + print("testing indexer in airdropper mode") + cls.address = 'localhost' + cls.faucet_port = 3333 + cls.airdrop_amount = 10 + + cls.faucet = MockFaucet(cls.faucet_port) + cls.faucet.start() + time.sleep(0.2) + + cls.evm_loader_id = evm_loader_addr + cls.wrapper_whitelist = wrapper_whitelist + cls.airdropper = Airdropper(f'http://{cls.address}:8899', + cls.evm_loader_id, + f'http://{cls.address}:{cls.faucet_port}', + cls.wrapper_whitelist, + cls.airdrop_amount, + 'INFO') + + + @classmethod + def tearDownClass(cls) -> None: + cls.faucet.shutdown_server() + cls.faucet.join() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_success_process_trx_with_one_airdrop(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould airdrop to new address - one target in transaction") + mock_sql_dict_contains.side_effect = [False] # new eth address + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=200, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_has_calls([call(token_airdrop_address1, ANY)]) + json_req = {'wallet': token_airdrop_address1, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_called_once_with(json_req) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(Airdropper, '_is_allowed_wrapper_contract') + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_failed_airdrop_contract_not_in_whitelist(self, + mock_sql_dict_contains, + mock_sql_dict_setitem, + mock_is_allowed_contract): + print("\n\nShould not airdrop for contract that is not in whitelist") + mock_is_allowed_contract.side_effect = [False] + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_is_allowed_contract.assert_called_once() + mock_sql_dict_contains.assert_not_called() + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_faucet_failure(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not add address to processed list due to faucet error") + mock_sql_dict_contains.side_effect = [False] # new eth address + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=400, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_not_called() + json_req = {'wallet': token_airdrop_address1, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_called_once_with(json_req) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_process_trx_with_one_airdrop_for_already_processed_address(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not airdrop to repeated address") + mock_sql_dict_contains.side_effect = [True] # eth address processed later + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx1) + + mock_sql_dict_contains.assert_called_once_with(token_airdrop_address1) + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_complex_transation(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould airdrop to several targets in one transaction") + mock_sql_dict_contains.side_effect = [False, False] # both targets are new + self.faucet.request_eth_token_mock.side_effect = [Response("{}", status=200, mimetype='application/json'), + Response("{}", status=200, mimetype='application/json')] + + self.airdropper.process_trx_airdropper_mode(pre_token_airdrop_trx2) + + mock_sql_dict_contains.assert_has_calls([call(token_airdrop_address3), + call(token_airdrop_address2)]) + mock_sql_dict_setitem.assert_has_calls([call(token_airdrop_address3, ANY), + call(token_airdrop_address2, ANY)]) + json_req1 = {'wallet': token_airdrop_address2, 'amount': self.airdrop_amount} + json_req2 = {'wallet': token_airdrop_address3, 'amount': self.airdrop_amount} + self.faucet.request_eth_token_mock.assert_has_calls([call(json_req2), call(json_req1)]) + self.faucet.request_eth_token_mock.reset_mock() + + + @patch.object(SQLDict, '__setitem__') + @patch.object(SQLDict, '__contains__') + def test_no_airdrop_instructions(self, + mock_sql_dict_contains, + mock_sql_dict_setitem): + print("\n\nShould not airdrop when instructions are inconsistent") + self.airdropper.process_trx_airdropper_mode(create_sol_acc_and_airdrop_trx) + + mock_sql_dict_contains.assert_not_called() + mock_sql_dict_setitem.assert_not_called() + self.faucet.request_eth_token_mock.assert_not_called() + self.faucet.request_eth_token_mock.reset_mock() + diff --git a/proxy/testing/test_airdropping_eth_accounts.py b/proxy/testing/test_airdropping_eth_accounts.py new file mode 100644 index 000000000..2c7f5e4a7 --- /dev/null +++ b/proxy/testing/test_airdropping_eth_accounts.py @@ -0,0 +1,113 @@ +import unittest +import os +import solcx + +import eth_account +import eth_typing +import eth_utils +from eth_account.account import LocalAccount + +from web3 import Web3, eth as web3_eth +from solana.rpc.api import Client as SolanaClient + +from ..plugin.solana_rest_api import EthereumModel +from ..plugin.solana_rest_api_tools import get_token_balance_gwei +from ..common_neon.address import ether2program + + +class TestAirdroppingEthAccounts(unittest.TestCase): + + @classmethod + def setUpClass(cls) -> None: + cls._EVM_LOADER_ID = os.environ.get("EVM_LOADER") + new_user_airdrop_amount = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0")) + cls._EXPECTED_BALANCE_WEI = eth_utils.to_wei(new_user_airdrop_amount, 'ether') + cls._MINIMAL_GAS_PRICE = int(os.environ.get("MINIMAL_GAS_PRICE", 1)) * eth_utils.denoms.gwei + + proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') + cls._web3 = Web3(Web3.HTTPProvider(proxy_url)) + solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899") + cls._solana_client = SolanaClient(solana_url) + cls._host_solana_account = EthereumModel.get_solana_account() + + def test_airdrop_on_get_balance(self): + account: LocalAccount = eth_account.account.Account.create() + block_number: eth_typing.BlockNumber = self._web3.eth.get_block_number() + actual_balance_wei = self._web3.eth.get_balance(account.address, block_identifier=block_number) + self.assertEqual(self._EXPECTED_BALANCE_WEI, actual_balance_wei) + + def test_airdrop_on_deploy(self): + contract_owner: LocalAccount = self._web3.eth.account.create() + contract = self._compile_and_deploy_contract(contract_owner, self._CONTRACT_STORAGE_SOURCE) + actual_balance_wei = self._get_balance_wei(contract.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, actual_balance_wei) + + def test_airdrop_onto_wrapped_new_address(self): + contract_owner: LocalAccount = self._web3.eth.account.create() + contract = self._compile_and_deploy_contract(contract_owner, self._WRAPPER_CONTRACT_STORAGE_SOURCE) + nested_contract_address = contract.functions.getNested().call() + nested_actual_balance = self._get_balance_wei(nested_contract_address) + wrapper_actual_balance = self._get_balance_wei(contract.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, wrapper_actual_balance) + self.assertEqual(self._EXPECTED_BALANCE_WEI, nested_actual_balance) + + def test_airdrop_on_deploy_estimation(self): + owner_eth_account: LocalAccount = self._web3.eth.account.create() + compile_result = solcx.compile_source(self._CONTRACT_STORAGE_SOURCE) + _, contract_interface = compile_result.popitem() + contract_data = contract_interface.get("bin") + self.assertIsNotNone(contract_data) + self._web3.eth.estimate_gas({"from": owner_eth_account.address, "data": contract_data}) + owner_balance = self._get_balance_wei(owner_eth_account.address) + self.assertEqual(self._EXPECTED_BALANCE_WEI, owner_balance) + + def _compile_and_deploy_contract(self, contract_owner: LocalAccount, source: str) -> web3_eth.Contract: + compile_result = solcx.compile_source(source) + contract_id, contract_interface = compile_result.popitem() + contract = self._web3.eth.contract(abi=contract_interface['abi'], bytecode=contract_interface['bin']) + nonce = self._web3.eth.get_transaction_count(contract_owner.address) + chain_id = self._web3.eth.chain_id + trx_signed = self._web3.eth.account.sign_transaction( + dict(nonce=nonce, chainId=chain_id, gas=987654321, gasPrice=self._MINIMAL_GAS_PRICE, to='', value=0, data=contract.bytecode), + contract_owner.key) + trx_hash = self._web3.eth.send_raw_transaction(trx_signed.rawTransaction) + trx_receipt = self._web3.eth.wait_for_transaction_receipt(trx_hash) + contract = self._web3.eth.contract(address=trx_receipt.contractAddress, abi=contract.abi) + return contract + + def _get_balance_wei(self, eth_account: str) -> int: + token_owner_account, nonce = ether2program(eth_account) + balance = get_token_balance_gwei(self._solana_client, token_owner_account) + self.assertIsNotNone(balance) + self.assertIsInstance(balance, int) + return balance * eth_utils.denoms.gwei + + _CONTRACT_STORAGE_SOURCE = ''' + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.7.0 <0.9.0; + contract Storage { + uint256 number; + function store(uint256 num) public { + number = num; + } + function retrieve() public view returns (uint256) { + return number; + } + } + ''' + + _WRAPPER_CONTRACT_STORAGE_SOURCE = ''' + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.7.0 <0.9.0; + contract Wrapper { + address private nested_address; + constructor() { + Nested nested = new Nested(); + nested_address = address(nested); + } + function getNested() public view returns (address) { + return nested_address; + } + } + contract Nested {} + ''' diff --git a/proxy/test_create_account_block.py b/proxy/testing/test_create_account_block.py similarity index 100% rename from proxy/test_create_account_block.py rename to proxy/testing/test_create_account_block.py diff --git a/proxy/test_environment.py b/proxy/testing/test_environment.py similarity index 100% rename from proxy/test_environment.py rename to proxy/testing/test_environment.py diff --git a/proxy/test_erc20_wrapper_contract.py b/proxy/testing/test_erc20_wrapper_contract.py similarity index 96% rename from proxy/test_erc20_wrapper_contract.py rename to proxy/testing/test_erc20_wrapper_contract.py index de8b61ba5..b745c7e76 100644 --- a/proxy/test_erc20_wrapper_contract.py +++ b/proxy/testing/test_erc20_wrapper_contract.py @@ -15,7 +15,7 @@ from solana.account import Account as SolanaAccount from solana.publickey import PublicKey -from proxy.plugin.solana_rest_api_tools import createERC20TokenAccountTrx, createEtherAccountTrx +from proxy.common_neon.neon_instruction import NeonInstruction # install_solc(version='latest') install_solc(version='0.7.6') @@ -66,7 +66,7 @@ event Transfer(address indexed from, address indexed to, uint256 value); event Approval(address indexed owner, address indexed spender, uint256 value); - + function approveSolana(bytes32 spender, uint64 value) external returns (bool); event ApprovalSolana(address indexed owner, bytes32 indexed spender, uint64 value); } @@ -147,7 +147,7 @@ def deploy_erc20_wrapper_contract(self): compiled_wrapper = compile_source(ERC20_WRAPPER_SOURCE) wrapper_id, wrapper_interface = compiled_wrapper.popitem() self.wrapper = wrapper_interface - + erc20 = proxy.eth.contract(abi=self.wrapper['abi'], bytecode=wrapper_interface['bin']) nonce = proxy.eth.get_transaction_count(proxy.eth.default_account) tx = {'nonce': nonce} @@ -171,7 +171,9 @@ def create_token_accounts(self): admin_token_seeds = [ b"\1", b"ERC20Balance", bytes(self.token.pubkey), contract_address_bytes, admin_address_bytes ] admin_token_key = PublicKey.find_program_address(admin_token_seeds, evm_loader_id)[0] admin_token_info = { "key": admin_token_key, "owner": admin_address_solana, "contract": contract_address_solana, "mint": self.token.pubkey } - self.solana_client.send_transaction(createERC20TokenAccountTrx(self.solana_account, admin_token_info), self.solana_account, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) + + instr = NeonInstruction(self.solana_account.public_key()).createERC20TokenAccountTrx(admin_token_info) + self.solana_client.send_transaction(instr, self.solana_account, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) self.token.mint_to(admin_token_key, self.solana_account, 10_000_000_000_000, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) def test_erc20_name(self): diff --git a/proxy/test_eth_getBlockByNumber.py b/proxy/testing/test_eth_getBlockByNumber.py similarity index 100% rename from proxy/test_eth_getBlockByNumber.py rename to proxy/testing/test_eth_getBlockByNumber.py diff --git a/proxy/test_eth_getLogs.py b/proxy/testing/test_eth_getLogs.py similarity index 100% rename from proxy/test_eth_getLogs.py rename to proxy/testing/test_eth_getLogs.py diff --git a/proxy/test_eth_sendRawTransaction.py b/proxy/testing/test_eth_sendRawTransaction.py similarity index 87% rename from proxy/test_eth_sendRawTransaction.py rename to proxy/testing/test_eth_sendRawTransaction.py index 35ac90651..9b6791e72 100644 --- a/proxy/test_eth_sendRawTransaction.py +++ b/proxy/testing/test_eth_sendRawTransaction.py @@ -1,5 +1,7 @@ import unittest import os + +import eth_utils from web3 import Web3 from solcx import install_solc @@ -237,8 +239,6 @@ def test_04_execute_with_bad_nonce(self): def test_05_transfer_one_gwei(self): print("\ntest_05_transfer_one_gwei") - one_gwei = 1_000_000_000 - eth_account_alice = proxy.eth.account.create('alice') eth_account_bob = proxy.eth.account.create('bob') print('eth_account_alice.address:', eth_account_alice.address) @@ -254,7 +254,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_alice.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -271,7 +271,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -285,7 +285,7 @@ def test_05_transfer_one_gwei(self): bob_balance_before_transfer = proxy.eth.get_balance(eth_account_bob.address) print('alice_balance_before_transfer:', alice_balance_before_transfer) print('bob_balance_before_transfer:', bob_balance_before_transfer) - print('one_gwei:', one_gwei) + print('one_gwei:', eth_utils.denoms.gwei) trx_transfer = proxy.eth.account.sign_transaction(dict( nonce=proxy.eth.get_transaction_count(eth_account_alice.address), @@ -293,7 +293,7 @@ def test_05_transfer_one_gwei(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account_alice.key ) @@ -307,8 +307,8 @@ def test_05_transfer_one_gwei(self): bob_balance_after_transfer = proxy.eth.get_balance(eth_account_bob.address) print('alice_balance_after_transfer:', alice_balance_after_transfer) print('bob_balance_after_transfer:', bob_balance_after_transfer) - self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - one_gwei) - self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + one_gwei) + self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - eth_utils.denoms.gwei) + self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + eth_utils.denoms.gwei) # @unittest.skip("a.i.") def test_06_transfer_one_and_a_half_gweis(self): @@ -319,8 +319,6 @@ def test_06_transfer_one_and_a_half_gweis(self): print('eth_account_alice.address:', eth_account_alice.address) print('eth_account_bob.address:', eth_account_bob.address) - one_gwei = 1_000_000_000 - if True: print("add funds to alice and bob") @@ -331,7 +329,7 @@ def test_06_transfer_one_and_a_half_gweis(self): gas=987654321, gasPrice=0, to=eth_account_alice.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -348,7 +346,7 @@ def test_06_transfer_one_and_a_half_gweis(self): gas=987654321, gasPrice=0, to=eth_account_bob.address, - value=one_gwei), + value=eth_utils.denoms.gwei), eth_account.key ) @@ -386,10 +384,9 @@ def test_06_transfer_one_and_a_half_gweis(self): print('alice_balance_after_transfer:', alice_balance_after_transfer) print('bob_balance_after_transfer:', bob_balance_after_transfer) print('check https://github.com/neonlabsorg/neon-evm/issues/210') - one_gwei = 1_000_000_000 - print('one_gwei:', one_gwei) - self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - one_gwei) - self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + one_gwei) + print('one_gwei:', eth_utils.denoms.gwei) + self.assertEqual(alice_balance_after_transfer, alice_balance_before_transfer - eth_utils.denoms.gwei) + self.assertEqual(bob_balance_after_transfer, bob_balance_before_transfer + eth_utils.denoms.gwei) @unittest.skip("a.i.") def test_07_execute_long_transaction(self): @@ -428,6 +425,38 @@ def test_07_execute_long_transaction(self): print('times_to_calculate:', times_to_calculate) print('time_duration:', time_duration) + def test_get_storage_at(self): + print("\nhttps://github.com/neonlabsorg/proxy-model.py/issues/289") + right_nonce = proxy.eth.get_transaction_count(proxy.eth.default_account) + value_to_store = 452356 + trx_store = self.storage_contract.functions.store(value_to_store).buildTransaction({'nonce': right_nonce}) + print('trx_store:', trx_store) + trx_store_signed = proxy.eth.account.sign_transaction(trx_store, eth_account.key) + print('trx_store_signed:', trx_store_signed) + trx_store_hash = proxy.eth.send_raw_transaction(trx_store_signed.rawTransaction) + print('trx_store_hash:', trx_store_hash.hex()) + trx_store_receipt = proxy.eth.wait_for_transaction_receipt(trx_store_hash) + print('trx_store_receipt:', trx_store_receipt) + + number_pos = 0 + value_received = proxy.eth.get_storage_at(self.storage_contract.address, number_pos, "latest") + print('eth_getStorageAt existing address and index => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), value_to_store) + + non_existing_pos = 12 + value_received = proxy.eth.get_storage_at(self.storage_contract.address, non_existing_pos, "latest") + print('eth_getStorageAt existing address and non-existing index => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) + + non_exising_address = b'\xe1\xda\xb7\xa6\x17\x6f\x87\x68\xF5\x3a\x42\x5f\x29\x61\x73\x60\x5e\xd5\x08\x32' + value_received = proxy.eth.get_storage_at(non_exising_address, non_existing_pos, "latest") + print('eth_getStorageAt non-existing address => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) + + not_a_contract_address = proxy.eth.default_account + value_received = proxy.eth.get_storage_at(not_a_contract_address, 0, "latest") + print('eth_getStorageAt not_a_contract_address address => ', value_received.hex()) + self.assertEqual(int.from_bytes(value_received, byteorder='big'), 0) if __name__ == '__main__': unittest.main() diff --git a/proxy/test_cancel_hanged.py b/proxy/testing/test_indexer_cancel_hanged.py similarity index 95% rename from proxy/test_cancel_hanged.py rename to proxy/testing/test_indexer_cancel_hanged.py index 241f15728..e0324df2e 100644 --- a/proxy/test_cancel_hanged.py +++ b/proxy/testing/test_indexer_cancel_hanged.py @@ -1,9 +1,9 @@ import os import sys -from proxy.plugin.solana_rest_api_tools import sysinstruct, ETH_TOKEN_MINT_ID, system, send_transaction, MINIMAL_GAS_PRICE +from proxy.common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY +from proxy.environment import ETH_TOKEN_MINT_ID, MINIMAL_GAS_PRICE -sys.path.append("/spl/bin/") os.environ['SOLANA_URL'] = "http://solana:8899" os.environ['EVM_LOADER'] = "53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io" os.environ['ETH_TOKEN_MINT'] = "HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU" @@ -18,6 +18,7 @@ from ethereum.utils import sha3 from solana.publickey import PublicKey from solana.rpc.commitment import Confirmed +from solana.system_program import SYS_PROGRAM_ID from solana.transaction import AccountMeta, Transaction, TransactionInstruction from solana_utils import * from solcx import install_solc @@ -74,7 +75,7 @@ class CancelTest(unittest.TestCase): @classmethod def setUpClass(cls): - print("\ntest_event.py setUpClass") + print("\ntest_cancel_hanged.py setUpClass") cls.token = SplToken(solana_url) wallet = WalletAccount(wallet_path()) @@ -184,7 +185,7 @@ def sol_instr_19_partial_call(self, storage_account, step_count, evm_instruction AccountMeta(pubkey=storage_account, is_signer=False, is_writable=True), # System instructions account: - AccountMeta(pubkey=PublicKey(sysinstruct), is_signer=False, is_writable=False), + AccountMeta(pubkey=PublicKey(SYSVAR_INSTRUCTION_PUBKEY), is_signer=False, is_writable=False), # Operator address: AccountMeta(pubkey=self.acc.public_key(), is_signer=True, is_writable=True), # Collateral pool address: @@ -194,7 +195,7 @@ def sol_instr_19_partial_call(self, storage_account, step_count, evm_instruction # User's NEON token account: AccountMeta(pubkey=self.caller_token, is_signer=False, is_writable=True), # System program account: - AccountMeta(pubkey=PublicKey(system), is_signer=False, is_writable=False), + AccountMeta(pubkey=PublicKey(SYS_PROGRAM_ID), is_signer=False, is_writable=False), AccountMeta(pubkey=self.reId, is_signer=False, is_writable=True), AccountMeta(pubkey=self.reId_token, is_signer=False, is_writable=True), diff --git a/proxy/testing/test_neon_cli_version.py b/proxy/testing/test_neon_cli_version.py new file mode 100644 index 000000000..66b82d68e --- /dev/null +++ b/proxy/testing/test_neon_cli_version.py @@ -0,0 +1,39 @@ +import unittest +import os +import requests +import json +import inspect + +from proxy.environment import neon_cli + +proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') +headers = {'Content-type': 'application/json'} + + +def get_line_number(): + cf = inspect.currentframe() + return cf.f_back.f_lineno + + +class TestNeonProxyVersion(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + def test_01_neon_cli_version(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/319") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "neon_cli_version", + "params": [] + })).text) + print('response:', response) + neon_cli_version = response['result'] + print('neon_cli_version:', neon_cli_version) + self.assertEqual(neon_cli_version, neon_cli().version()) + + +if __name__ == '__main__': + unittest.main() diff --git a/proxy/test_neon_faucet.py b/proxy/testing/test_neon_faucet.py similarity index 100% rename from proxy/test_neon_faucet.py rename to proxy/testing/test_neon_faucet.py diff --git a/proxy/testing/test_neon_proxy_version.py b/proxy/testing/test_neon_proxy_version.py new file mode 100644 index 000000000..c5ed29e40 --- /dev/null +++ b/proxy/testing/test_neon_proxy_version.py @@ -0,0 +1,39 @@ +import unittest +import os +import requests +import json +import inspect + +from proxy.plugin.solana_rest_api import NEON_PROXY_PKG_VERSION, NEON_PROXY_REVISION + +proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') +headers = {'Content-type': 'application/json'} + + +def get_line_number(): + cf = inspect.currentframe() + return cf.f_back.f_lineno + + +class TestNeonProxyVersion(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + def test_01_neon_proxy_version(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/320") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "neon_proxy_version", + "params": [] + })).text) + print('response:', response) + neon_proxy_version = response['result'] + print('neon_proxy_version:', neon_proxy_version) + self.assertEqual(neon_proxy_version, 'Neon-proxy/v' + NEON_PROXY_PKG_VERSION + '-' + NEON_PROXY_REVISION) + + +if __name__ == '__main__': + unittest.main() diff --git a/proxy/test_operator_spending.py b/proxy/testing/test_operator_spending.py similarity index 100% rename from proxy/test_operator_spending.py rename to proxy/testing/test_operator_spending.py diff --git a/proxy/test_resize_storage_account.py b/proxy/testing/test_resize_storage_account.py similarity index 97% rename from proxy/test_resize_storage_account.py rename to proxy/testing/test_resize_storage_account.py index a2aece6fd..01e2319c1 100644 --- a/proxy/test_resize_storage_account.py +++ b/proxy/testing/test_resize_storage_account.py @@ -2,8 +2,8 @@ from web3 import Web3 import unittest from solana.publickey import PublicKey -from solcx import compile_source -from solcx import install_solc +from solcx import compile_source, install_solc + proxy_url = os.environ.get('PROXY_URL', 'http://127.0.0.1:9090/solana') solana_url = os.environ.get("SOLANA_URL", "http://127.0.0.1:8899") diff --git a/proxy/test_user_stories.py b/proxy/testing/test_user_stories.py similarity index 93% rename from proxy/test_user_stories.py rename to proxy/testing/test_user_stories.py index 45b837223..e6001740e 100644 --- a/proxy/test_user_stories.py +++ b/proxy/testing/test_user_stories.py @@ -91,6 +91,18 @@ def test_04_check_eth_estimateGas_on_deploying_a_contract_with_the_empty_data_an print('used_gas:', used_gas) self.assertEqual(used_gas, 53001 + EXTRA_GAS) + def test_05_check_params_omitted(self): + print("https://github.com/neonlabsorg/proxy-model.py/issues/318") + response = json.loads(requests.post( + proxy_url, headers=headers, + data=json.dumps({"jsonrpc": "2.0", + "id": get_line_number(), + "method": "eth_chainId" + })).text) + print('response:', response) + chain_id = int(response['result'], 0) + print('chain_id:', chain_id) + self.assertEqual(chain_id, 111) if __name__ == '__main__': unittest.main() diff --git a/proxy/testing/test_utils.py b/proxy/testing/test_utils.py new file mode 100644 index 000000000..137ffbf01 --- /dev/null +++ b/proxy/testing/test_utils.py @@ -0,0 +1,16 @@ +import unittest +from ..common_neon.utils import get_from_dict + + +class TestUtils(unittest.TestCase): + + def test_get_from_dict(self): + test_dict = {"a": {"b": {"c": 1}}} + + self.assertEqual(1, get_from_dict(test_dict, "a", "b", "c")) + self.assertEqual({"b": {"c": 1}}, get_from_dict(test_dict, "a")) + + self.assertIsNone(get_from_dict(test_dict, "b", "c", "a")) + self.assertIsNone(get_from_dict(None, "a")) + self.assertIsNone(get_from_dict(555, "a")) + self.assertIsNone(get_from_dict({}, "a")) diff --git a/proxy/test_web3_clientVersion.py b/proxy/testing/test_web3_clientVersion.py similarity index 89% rename from proxy/test_web3_clientVersion.py rename to proxy/testing/test_web3_clientVersion.py index 23c8afa45..791549bef 100644 --- a/proxy/test_web3_clientVersion.py +++ b/proxy/testing/test_web3_clientVersion.py @@ -1,11 +1,6 @@ import unittest import os from web3 import Web3 -from solcx import install_solc - -# install_solc(version='latest') -install_solc(version='0.7.0') -from solcx import compile_source proxy_url = os.environ.get('PROXY_URL', 'http://localhost:9090/solana') proxy = Web3(Web3.HTTPProvider(proxy_url)) diff --git a/proxy/testing/transactions.py b/proxy/testing/transactions.py new file mode 100644 index 000000000..dec4e2bb9 --- /dev/null +++ b/proxy/testing/transactions.py @@ -0,0 +1,491 @@ + +token_program = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' +evm_loader_addr = 'eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU' +erc20_wrapper = '5H7kvhPD7GECAmf227vTPYTS7SC2PmyuVZaT5zVTx7vb' +wrapper_whitelist = [erc20_wrapper] + +token_airdrop_address1 = '0xf71c4daca893e5333982e2956c5ed9b648818376' + +# Solana transaction for simple case airdrop +pre_token_airdrop_trx1 = { + 'blockTime': 1637857371, + 'meta': { + 'err': None, + 'fee': 5000, + 'innerInstructions': [ + { + 'index': 0, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '111112fUvhuhctf7ykHr29ATacqTktVJJSG9xpkwPTuR6WcMjZZQDYhZ5k4S6Zu6C5sdsn', + 'programIdIndex': 7 + }, + { + 'accounts': [0, 2, 1, 8, 7, 9, 11], + 'data': '', + 'programIdIndex': 10 + }, + { + 'accounts': [0, 2], + 'data': '3Bxs4h24hBtQy9rw', + 'programIdIndex': 7 + }, + { + 'accounts': [2], 'data': + '9krTDU2LzCSUJuVZ', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': 'SYXsBSQy3GeifSEQSGvTbrPNposbSAiSoh1YA85wcvGKSnYg', + 'programIdIndex': 7 + }, + { + 'accounts': [2, 8, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + }, + { + 'index': 1, + 'instructions': [ + { + 'accounts': [0, 3], + 'data': '11119os1e9qSs2u7TsThXqkBSRVFxhmYaFKFZ1waB2X7armDmvK3p5GmLdUxYdg3h7QSrL', + 'programIdIndex': 7 + }, + { + 'accounts': [3, 5, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + } + ], + 'logMessages': [ + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', + 'Program log: Transfer 2039280 lamports to the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Allocate space for the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Assign the associated token account to the SPL Token program', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Initialize the associated token account', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [3]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 464826 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 24626 of 485359 compute units', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', + 'Program log: Total memory occupied: 1414', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 40680 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program log: Total memory occupied: 1536', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: Transfer', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success' + ], + 'postBalances': [5944954400, 2672640, 2039280, 2039280, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'postTokenBalances': [ + { + 'accountIndex': 2, + 'mint': '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + 'owner': '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'uiTokenAmount': { + 'amount': '0', + 'decimals': 9, + 'uiAmount': None, + 'uiAmountString': '0' + } + }, + { + 'accountIndex': 3, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'uiTokenAmount': { + 'amount': '1000000', + 'decimals': 6, + 'uiAmount': 1.0, + 'uiAmountString': '1' + } + }, + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3798000000', + 'decimals': 6, + 'uiAmount': 3798.0, + 'uiAmountString': '3798' + } + } + ], + 'preBalances': [5951710600, 0, 0, 0, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'preTokenBalances': [ + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3799000000', + 'decimals': 6, + 'uiAmount': 3799.0, + 'uiAmountString': '3799' + } + } + ], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 96659490, + 'transaction': { + 'message': { + 'accountKeys': [ + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', + 'J4zoYXUtxNJsCXLih7D7dVkfoT9pvsYa8HtHPEEN88U', + 'AZqbo1ZCwS1grcqUdhs79YspjXME2BjrV5WCHabBS1ht', + erc20_wrapper, + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', + '11111111111111111111111111111111', + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + token_program, + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', + 'SysvarRent111111111111111111111111111111111', + evm_loader_addr + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 6, + 'numRequiredSignatures': 1 + }, + 'instructions': [ + { + 'accounts': [0, 1, 2, 7, 8, 9, 10, 11], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcPrQv1EF4uXaNUgBSq2E3sfp7PKeAH', + 'programIdIndex': 12 + }, + { + 'accounts': [0, 3, 1, 4, 5, 7, 9, 11], + 'data': 'G', + 'programIdIndex': 12 + }, + { + 'accounts': [6, 3, 0], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 9 + } + ], + 'recentBlockhash': '8UBfYxDWWcEbXQPGxPRqK4oGnb6c2XAQm4xZwzM2fxCB' + }, + 'signatures': ['3np8r1PBJW9uuw7o7P86B46zH7nW4UufQa716NUsSscJ1mN5nG6K74JHcMb6YmTmu9Les2NSe1iQyLJZwgS5RpvE'] + } + } + +token_airdrop_address2 = '0x8bd4991b9b81b3298fc6ac06b553b87c8e8582f0' +token_airdrop_address3 = '0x67d9e53d5e747b36fa6587421114aa0b0ca39753' + +# This is transaction containing 2 addresses for airdrop +# Instructions here mixed to test complex case +pre_token_airdrop_trx2 = { + 'blockTime': 1637857344, + 'meta': { + 'err': None, + 'fee': 5000, + 'innerInstructions': [ + { + 'index': 0, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '111112fUvhuhctf7ykHr29ATacqTktVJJSG9xpkwPTuR6WcMjZZQDYhZ5k4S6Zu6C5sdsn', + 'programIdIndex': 7 + }, + { + 'accounts': [0, 2, 1, 8, 7, 9, 11], + 'data': '', + 'programIdIndex': 10 + }, + { + 'accounts': [0, 2], + 'data': '3Bxs4h24hBtQy9rw', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': '9krTDU2LzCSUJuVZ', + 'programIdIndex': 7 + }, + { + 'accounts': [2], + 'data': 'SYXsBSQy3GeifSEQSGvTbrPNposbSAiSoh1YA85wcvGKSnYg', + 'programIdIndex': 7 + }, + { + 'accounts': [2, 8, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + }, + { + 'index': 1, + 'instructions': [ + { + 'accounts': [0, 3], + 'data': '11119os1e9qSs2u7TsThXqkBSRVFxhmYaFKFZ1waB2X7armDmvK3p5GmLdUxYdg3h7QSrL', + 'programIdIndex': 7 + }, + { + 'accounts': [3, 5, 1, 11], + 'data': '2', + 'programIdIndex': 9 + } + ] + } + ], + 'logMessages': [ + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', + 'Program log: Transfer 2039280 lamports to the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Allocate space for the associated token account', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Assign the associated token account to the SPL Token program', + 'Program 11111111111111111111111111111111 invoke [3]', + 'Program 11111111111111111111111111111111 success', + 'Program log: Initialize the associated token account', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [3]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 457797 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 27155 of 480859 compute units', + 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', + 'Program log: Total memory occupied: 1414', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 47709 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + 'Program 11111111111111111111111111111111 invoke [2]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', + 'Program log: Instruction: InitializeAccount', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', + 'Program log: Total memory occupied: 1536', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', + 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: Transfer', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success'], + 'postBalances': [5951710600, 2672640, 2039280, 2039280, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'postTokenBalances': [ + { + 'accountIndex': 2, + 'mint': '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', + 'owner': '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', + 'uiTokenAmount': { + 'amount': '0', + 'decimals': 9, + 'uiAmount': None, + 'uiAmountString': '0' + } + }, + { + 'accountIndex': 3, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', + 'uiTokenAmount': { + 'amount': '1000000', + 'decimals': 6, + 'uiAmount': 1.0, + 'uiAmountString': '1' + } + }, + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3799000000', + 'decimals': 6, + 'uiAmount': 3799.0, + 'uiAmountString': '3799' + } + } + ], + 'preBalances': [5958466800, 0, 0, 0, 2672640, 1461600, 2039280, 1, 1461600, 1089991680, 898174080, 1009200, 1141440], + 'preTokenBalances': [ + { + 'accountIndex': 6, + 'mint': '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', + 'owner': 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', + 'uiTokenAmount': { + 'amount': '3800000000', + 'decimals': 6, + 'uiAmount': 3800.0, + 'uiAmountString': '3800' + } + } + ], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 96659420, + 'transaction': { + 'message': { + 'accountKeys': [ + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', # 0 - funding + '2dQ5m8pun68x5LRqG5kQtF6t3ELzNoSuJu5WYP47MqBW', # 1 - ETH account + 'CztEiw75xKWoJ32Nr6exUtmjQHQFD7SE7ZWckX6b55v3', # 2 - NEON token account + '6whnofFTKa6ynDAv882VpQMx6rQvo9v18QM4AGmLCHZ9', # 3 - >> ERC20 token account (wrapper balance) + erc20_wrapper, # 4 - >> contract (ERC20 wrapper) + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', # 5 - SPL token-mint account (wrapped by ERC20) + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', # 6 - SPL token from + '11111111111111111111111111111111', # 7 - System program + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', # 8 - NEON token mint + token_program, # 9 - >> Token program + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', # 10 - Associated token program + 'SysvarRent111111111111111111111111111111111', # 11 + evm_loader_addr, # 12 - >> EVM_LOADER + 'CVAimMqtcmSUCV4RLZSJAreDpEd7JEZmrvCVj85yaRzZ', # 13 - funding + '8utQrai6so3pWtJhyCYafWhWJ3wJmi2eSurnMPZis4Aw', # 14 - ETH account + 'J4zoYXUtxNJsCXLih7D7dVkfoT9pvsYa8HtHPEEN88U', # 15 - NEON token account + 'AZqbo1ZCwS1grcqUdhs79YspjXME2BjrV5WCHabBS1ht', # 16 - >> ERC20 token account (wrapper balance) + erc20_wrapper, # 17 - >> contract (ERC20 wrapper) + '3vxj94fSd3jrhaGAwaEKGDPEwn5Yqs81Ay5j1BcdMqSZ', # 18 - SPL token-mint account (wrapped by ERC20) + '7XVY7C79A6UQa2JUN5hpqYHg9jgsgUPF7SErtbuvbHRc', # 19 - SPL token from + '11111111111111111111111111111111', # 20 - System program + '89dre8rZjLNft7HoupGiyxu3MNftR577ZYu8bHe2kK7g', # 21 - NEON token mint + token_program, # 22 - >> Token program + 'ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL', # 23 - Associated token program + 'SysvarRent111111111111111111111111111111111', # 24 + evm_loader_addr # 25 - >> EVM_LOADER + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 6, + 'numRequiredSignatures': 1 + }, + 'instructions': [ + { + # Create 2-nd ERC20 token account + 'accounts': [13, 16, 14, 17, 18, 20, 22, 24], + 'data': 'G', + 'programIdIndex': 25 + }, + { + # Create 2-nd account + 'accounts': [13, 14, 15, 20, 21, 22, 23, 24], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcF3UcpFjrvswawEUuqDYcNbh7NZsuX', + 'programIdIndex': 25 + }, + { + # Create 1-st account + 'accounts': [0, 1, 2, 7, 8, 9, 10, 11], + 'data': 'SSX8YzB3JHrjo6vdi3AMoi7zpcHFoDD1r8SrtHfKsSB7RxGBwrn2xPk', + 'programIdIndex': 12 + }, + { + # Create 1-st ERC20 token account + 'accounts': [0, 3, 1, 4, 5, 7, 9, 11], + 'data': 'G', + 'programIdIndex': 12 + }, + { + # Transfer to 1-st ERC20 token account + 'accounts': [6, 3, 0], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 9 + }, + { + # Transfer to 2-nd ERC20 token account + 'accounts': [19, 16, 13], + 'data': '3QCwqmHZ4mdq', + 'programIdIndex': 22 + } + ], + 'recentBlockhash': 'DhuNa4ts1c8jnD1u55rvQd8YKQo8t4aHwt3jUd1Anexu'}, + 'signatures': ['4WotQiXmE5AUjC9yoTE69C2CLmWa2Typ1mAuG4EzAkjYXSfYH6gQewkSfsEHGQjvXkANFKXtkUtvkqm7eq23M74V'] + } + } + +create_sol_acc_and_airdrop_trx = { + 'blockTime': 1638178743, + 'meta': { + 'err': None, + 'fee': 10000, + 'innerInstructions': [], + 'logMessages': [ + 'Program 11111111111111111111111111111111 invoke [1]', + 'Program 11111111111111111111111111111111 success', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', + 'Program log: Instruction: InitializeMint', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 2833 of 200000000 compute units', + 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success'], + 'postBalances': [999998528400, 1461600, 1009200, 1, 1130582400], + 'postTokenBalances': [], + 'preBalances': [1000000000000, 0, 1009200, 1, 1130582400], + 'preTokenBalances': [], + 'rewards': [], + 'status': {'Ok': None} + }, + 'slot': 15029, + 'transaction': { + 'message': { + 'accountKeys': [ + 'D4Aa2HU5kwF3nByGYk7pdjbP4n3cFFjPjYASdKUfpH4H', + 'EHDze1sDhUk7dR9iBgV4Mm3dYMk3ZQXKGHiiuVTEYaYr', + 'SysvarRent111111111111111111111111111111111', + '11111111111111111111111111111111', + 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + ], + 'header': { + 'numReadonlySignedAccounts': 0, + 'numReadonlyUnsignedAccounts': 3, + 'numRequiredSignatures': 2 + }, + 'instructions': [ + { + 'accounts': [0, 1], + 'data': '11114XtYk9gGfZoo968fyjNUYQJKf9gdmkGoaoBpzFv4vyaSMBn3VKxZdv7mZLzoyX5YNC', + 'programIdIndex': 3 + }, + { + 'accounts': [1, 2], + 'data': '1DidxzgH8WA79wndRY2Vc8EsYn3Rf9CKWmB3tRLNtzhWUZD', + 'programIdIndex': 4 + } + ], + 'recentBlockhash': '6kT9KCR37ZWhu9fbdacwUsXAHXvPo9kYwzQsQnh9dWyW' + }, + 'signatures': [ + '4wNHEzKDpqKpQ51A3cYWGsLXAxc3cxHtk45cs1RMUYgY6bViBUu6w7VaDMSQjprbwC7AF4bMy3ejR69FAVwQWUgh', + '4n6PzpFyQ5e9PTDFTUmHoPXUJYtNqsfQgwinu5ujYeY6EigseJHGgykmzMMb8exKsC45E7RjiyahLqhbR1uQo5V5' + ] + } + } diff --git a/requirements.txt b/requirements.txt index 323f726c3..3ca6efc39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ solana==0.10.0 psycopg2-binary ethereum py-solc-x==1.1.0 +flask diff --git a/setup.py b/setup.py index 0d2c67516..94ebe2b28 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ """ from setuptools import setup, find_packages -VERSION = (2, 2, 0) +VERSION = (0, 4, 0) __version__ = '.'.join(map(str, VERSION[0:3])) __description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on Network monitoring, controls & Application development, testing, debugging.'''