diff --git a/Dockerfile b/Dockerfile index 95ae8fcdf..05e0fbe74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,9 @@ RUN apt update && \ pip3 install --upgrade pip && \ /bin/bash -c "source venv/bin/activate" && \ pip install -r requirements.txt && \ + pip3 install py-solc-x && \ + python3 -c "import solcx; solcx.install_solc(version='0.7.6')" && \ apt remove -y git && \ - pip install py-solc-x && \ rm -rf /var/lib/apt/lists/* COPY --from=cli /opt/solana/bin/solana \ diff --git a/proxy/__main__.py b/proxy/__main__.py index 019f9e733..41ad6dc43 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -14,7 +14,6 @@ import os from .indexer.airdropper import run_airdropper from .indexer.indexer import run_indexer -from solana.rpc.api import Client if __name__ == '__main__': airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True'] @@ -22,7 +21,6 @@ if airdropper_mode: print("Will run in airdropper mode") solana_url = os.environ['SOLANA_URL'] - evm_loader_id = os.environ['EVM_LOADER'] pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT']) faucet_url = os.environ['FAUCET_URL'] wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'] @@ -34,7 +32,6 @@ max_conf = float(os.environ.get('MAX_CONFIDENCE_INTERVAL', 0.02)) run_airdropper(solana_url, - evm_loader_id, pyth_mapping_account, faucet_url, wrapper_whitelist, @@ -45,8 +42,7 @@ print("Will run in indexer mode") solana_url = os.environ['SOLANA_URL'] - evm_loader_id = os.environ['EVM_LOADER'] - run_indexer(solana_url, evm_loader_id) + run_indexer(solana_url) else: entry_point() diff --git a/proxy/common_neon/account_whitelist.py b/proxy/common_neon/account_whitelist.py index 1c2d11bd4..2f92e6ecd 100644 --- a/proxy/common_neon/account_whitelist.py +++ b/proxy/common_neon/account_whitelist.py @@ -1,14 +1,13 @@ import traceback from datetime import datetime -import time -from proxy.environment import ELF_PARAMS, GET_WHITE_LIST_BALANCE_MAX_RETRIES, GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S +from proxy.environment import ELF_PARAMS from proxy.common_neon.permission_token import PermissionToken from solana.publickey import PublicKey -from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount from typing import Union from proxy.common_neon.address import EthereumAddress from logged_groups import logged_group +from ..common_neon.solana_interactor import SolanaInteractor NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE = int(ELF_PARAMS.get("NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE", 0)) NEON_MINIMAL_CONTRACT_ALLOWANCE_BALANCE = int(ELF_PARAMS.get("NEON_MINIMAL_CONTRACT_ALLOWANCE_BALANCE", 0)) @@ -18,10 +17,7 @@ @logged_group("neon.AccountWhitelist") class AccountWhitelist: - def __init__(self, solana: SolanaClient, payer: SolanaAccount, permission_update_int: int): - self.info(f'GET_WHITE_LIST_BALANCE_MAX_RETRIES={GET_WHITE_LIST_BALANCE_MAX_RETRIES}') - self.info(f'GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S={GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S} seconds') - self.info(f'permission_update_int={permission_update_int}') + def __init__(self, solana: SolanaInteractor, payer: SolanaAccount, permission_update_int: int): self.solana = solana self.account_cache = {} self.permission_update_int = permission_update_int @@ -43,9 +39,15 @@ def __init__(self, solana: SolanaClient, payer: SolanaAccount, permission_update PublicKey(DENIAL_TOKEN_ADDR), payer) - def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]): - allowance_balance = self.allowance_token.get_balance(ether_addr) - denial_balance = self.denial_token.get_balance(ether_addr) + def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]) -> int: + token_list = [ + self.allowance_token.get_token_account_address(ether_addr), + self.denial_token.get_token_account_address(ether_addr) + ] + + balance_list = self.solana.get_token_account_balance_list(token_list) + allowance_balance = balance_list[0] + denial_balance = balance_list[1] return allowance_balance - denial_balance def grant_permissions(self, ether_addr: Union[str, EthereumAddress], min_balance: int): @@ -106,27 +108,17 @@ def has_permission(self, ether_addr: Union[str, EthereumAddress], min_balance: i if diff < self.permission_update_int: return cached['diff'] >= min_balance - num_retries = GET_WHITE_LIST_BALANCE_MAX_RETRIES - - while True: - try: - diff = self.read_balance_diff(ether_addr) - self.account_cache[ether_addr] = { - 'last_update': current_time, - 'diff': diff - } - return diff >= min_balance - except Exception as err: - err_tb = "".join(traceback.format_tb(err.__traceback__)) - self.error(f'Failed to read permissions for {ether_addr}: ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - num_retries -= 1 - if num_retries == 0: - # This error should be forwarded to client - raise RuntimeError('Failed to read account permissions. Try to repeat later') - - self.info(f'Will retry getting whitelist balance after {GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S} seconds') - time.sleep(GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S) + try: + diff = self.read_balance_diff(ether_addr) + self.account_cache[ether_addr] = { + 'last_update': current_time, + 'diff': diff + } + return diff >= min_balance + except Exception as err: + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.error(f'Failed to read permissions for {ether_addr}: ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') def has_client_permission(self, ether_addr: Union[str, EthereumAddress]): return self.has_permission(ether_addr, NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE) diff --git a/proxy/common_neon/address.py b/proxy/common_neon/address.py index 6819e6784..a6d1fec01 100644 --- a/proxy/common_neon/address.py +++ b/proxy/common_neon/address.py @@ -1,5 +1,6 @@ +from __future__ import annotations + import random -import base64 from eth_keys import keys as eth_keys from hashlib import sha256 @@ -8,9 +9,8 @@ from typing import NamedTuple from .layouts import ACCOUNT_INFO_LAYOUT -from ..environment import neon_cli, ETH_TOKEN_MINT_ID, EVM_LOADER_ID +from ..environment import ETH_TOKEN_MINT_ID, EVM_LOADER_ID from .constants import ACCOUNT_SEED_VERSION -from solana.rpc.commitment import Confirmed class EthereumAddress: @@ -60,35 +60,17 @@ def getTokenAddr(account): return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) -class AccountInfo(NamedTuple): +class AccountInfoLayout(NamedTuple): ether: eth_keys.PublicKey trx_count: int code_account: PublicKey state: int + def is_payed(self) -> bool: + return self.state != 0 + @staticmethod - def frombytes(data): + def frombytes(data) -> AccountInfoLayout: cont = ACCOUNT_INFO_LAYOUT.parse(data) - return AccountInfo(cont.ether, int.from_bytes(cont.trx_count, 'little'), PublicKey(cont.code_account), cont.state) - - -def _getAccountData(client, account, expected_length, owner=None): - info = client.get_account_info(account, commitment=Confirmed)['result']['value'] - if info is None: - raise Exception("Can't get information about {}".format(account)) - - data = base64.b64decode(info['data'][0]) - if len(data) < expected_length: - raise Exception("Wrong data length for account data {}".format(account)) - return data - - -def getAccountInfo(client, eth_account: EthereumAddress): - account_sol, nonce = ether2program(eth_account) - info = _getAccountData(client, account_sol, ACCOUNT_INFO_LAYOUT.sizeof()) - return AccountInfo.frombytes(info) - - -def isPayed(client, address: str): - acc_info: AccountInfo = getAccountInfo(client, EthereumAddress(address)) - return acc_info.state != 0 + return AccountInfoLayout(cont.ether, int.from_bytes(cont.trx_count, 'little'), + PublicKey(cont.code_account), cont.state) diff --git a/proxy/common_neon/costs.py b/proxy/common_neon/costs.py index b3a17ed4a..17ecf518c 100644 --- a/proxy/common_neon/costs.py +++ b/proxy/common_neon/costs.py @@ -1,7 +1,7 @@ import base58 from ..environment import EVM_LOADER_ID -from ..indexer.utils import BaseDB +from ..indexer.base_db import BaseDB class SQLCost(BaseDB): diff --git a/proxy/common_neon/estimate.py b/proxy/common_neon/estimate.py index 452bf13f5..ee3125a97 100644 --- a/proxy/common_neon/estimate.py +++ b/proxy/common_neon/estimate.py @@ -21,7 +21,7 @@ def evm_step_cost(signature_cnt): @logged_group("neon.Proxy") class GasEstimate: - def __init__(self, request, db, client, evm_step_count): + def __init__(self, request, db, solana, evm_step_count): self.sender: bytes = bytes.fromhex(request.get('from', "0x%040x" % 0x0)[2:]) self.step_count = evm_step_count @@ -46,7 +46,7 @@ def __init__(self, request, db, client, evm_step_count): signed_trx = w3.eth.account.sign_transaction(unsigned_trx, eth_keys.PrivateKey(os.urandom(32))) trx = EthTrx.fromString(signed_trx.rawTransaction) - self.tx_sender = NeonTxSender(db, client, trx, steps=evm_step_count) + self.tx_sender = NeonTxSender(db, solana, trx, steps=evm_step_count) def iteration_info(self) -> Tuple[int, int]: if self.tx_sender.steps_emulated > 0: @@ -59,34 +59,30 @@ def iteration_info(self) -> Tuple[int, int]: final_steps = EVM_STEPS return final_steps, full_step_iterations - @logged_group("neon.Proxy") - def simple_neon_tx_strategy(self, *, logger): + def simple_neon_tx_strategy(self): gas = evm_step_cost(2) * (self.tx_sender.steps_emulated if self.tx_sender.steps_emulated > EVM_STEPS else EVM_STEPS) - logger.debug(f'estimate simple_neon_tx_strategy: {gas}') + self.debug(f'estimate simple_neon_tx_strategy: {gas}') return gas - @logged_group("neon.Proxy") - def iterative_neon_tx_strategy(self, *, logger): + def iterative_neon_tx_strategy(self): begin_iteration = 1 final_steps, full_step_iterations = self.iteration_info() steps = begin_iteration * EVM_STEPS + full_step_iterations * self.step_count + final_steps gas = steps * evm_step_cost(1) - logger.debug(f'estimate iterative_neon_tx_strategy: {gas}') + self.debug(f'estimate iterative_neon_tx_strategy: {gas}') return gas - @logged_group("neon.Proxy") - def holder_neon_tx_strategy(self, *, logger): + def holder_neon_tx_strategy(self): begin_iteration = 1 msg = get_holder_msg(self.tx_sender.eth_tx) holder_iterations = math.ceil(len(msg) / HOLDER_MSG_SIZE) final_steps, full_step_iterations = self.iteration_info() steps = (begin_iteration + holder_iterations) * EVM_STEPS + full_step_iterations * self.step_count + final_steps gas = steps * evm_step_cost(1) - logger.debug(f'estimate holder_neon_tx_strategy: {gas}') + self.debug(f'estimate holder_neon_tx_strategy: {gas}') return gas - @logged_group("neon.Proxy") - def allocated_space(self, *, logger): + def allocated_space(self): space = 0 for s in self.tx_sender._create_account_list: if s.NAME == NeonCreateContractTxStage.NAME: @@ -95,14 +91,13 @@ def allocated_space(self, *, logger): space += ACCOUNT_MAX_SIZE + SPL_TOKEN_ACCOUNT_SIZE + ACCOUNT_STORAGE_OVERHEAD * 2 space += self.tx_sender.unpaid_space - logger.debug(f'allocated space: {space}') + self.debug(f'allocated space: {space}') return space - @logged_group("neon.Proxy") - def estimate(self, *, logger): + def estimate(self): self.tx_sender.operator_key = PublicKey(os.urandom(32)) self.tx_sender._call_emulated(self.sender) - self.tx_sender._parse_accounts_list(); + self.tx_sender._parse_accounts_list() gas_for_trx = max(self.simple_neon_tx_strategy(), self.iterative_neon_tx_strategy(), self.holder_neon_tx_strategy()) gas_for_space = self.allocated_space() * EVM_BYTE_COST @@ -112,8 +107,8 @@ def estimate(self, *, logger): # if gas < 21000: # gas = 21000 - logger.debug(f'extra_gas: {EXTRA_GAS}') - logger.debug(f'gas_for_space: {gas_for_space}') - logger.debug(f'gas_for_trx: {gas_for_trx}') - logger.debug(f'estimated gas: {gas}') + self.debug(f'extra_gas: {EXTRA_GAS}') + self.debug(f'gas_for_space: {gas_for_space}') + self.debug(f'gas_for_trx: {gas_for_trx}') + self.debug(f'estimated gas: {gas}') return hex(gas) diff --git a/proxy/common_neon/permission_token.py b/proxy/common_neon/permission_token.py index fdec58516..145740b86 100644 --- a/proxy/common_neon/permission_token.py +++ b/proxy/common_neon/permission_token.py @@ -1,59 +1,47 @@ -from lib2to3.pgen2 import token -from spl.token.client import Token as SplToken from solana.publickey import PublicKey -from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount -from spl.token.constants import TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address from proxy.common_neon.address import EthereumAddress, ether2program from typing import Union -from solana.rpc.commitment import Confirmed from solana.transaction import Transaction -from solana.rpc.types import TxOpts import spl.token.instructions as spl_token -from proxy.common_neon.utils import get_from_dict +from proxy.common_neon.solana_interactor import SolanaInteractor, SolTxListSender from decimal import Decimal import os class PermissionToken: def __init__(self, - solana: SolanaClient, + solana: SolanaInteractor, token_mint: PublicKey, payer: SolanaAccount): self.solana = solana + self.signer = payer + self.waiter = None self.token_mint = token_mint - self.payer = payer - self.token = SplToken(self.solana, - self.token_mint, - TOKEN_PROGRAM_ID, - payer) def get_token_account_address(self, ether_addr: Union[str, EthereumAddress]): sol_addr = PublicKey(ether2program(ether_addr)[0]) - return get_associated_token_address(sol_addr, self.token.pubkey) + return get_associated_token_address(sol_addr, self.token_mint) def get_balance(self, ether_addr: Union[str, EthereumAddress]): token_account = self.get_token_account_address(ether_addr) - result = self.token.get_balance(token_account).get('result', None) - if result is None: - return 0 - return int(result['value']['amount']) + return self.solana.get_token_account_balance(token_account) def create_account_if_needed(self, ether_addr: Union[str, EthereumAddress]): token_account = self.get_token_account_address(ether_addr) - response = self.solana.get_account_info(token_account, Confirmed) - if get_from_dict(response, 'result', 'value') is not None: + info = self.solana.get_account_info(token_account) + if info is not None: return token_account txn = Transaction() create_txn = spl_token.create_associated_token_account( - payer=self.payer.public_key(), + payer=self.signer.public_key(), owner=PublicKey(ether2program(ether_addr)[0]), - mint=self.token.pubkey + mint=self.token_mint ) txn.add(create_txn) - self.token._conn.send_transaction(txn, self.payer, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) + SolTxListSender(self, [txn], 'CreateAssociatedTokenAccount(1)', skip_preflight=True).send() return token_account def mint_to(self, @@ -61,6 +49,6 @@ def mint_to(self, ether_addr: Union[str, EthereumAddress], mint_authority_file: str): token_account = self.create_account_if_needed(ether_addr) - mint_command = f'spl-token mint "{str(self.token.pubkey)}" {Decimal(amount) * pow(Decimal(10), -9)}' + mint_command = f'spl-token mint "{str(self.token_mint)}" {Decimal(amount) * pow(Decimal(10), -9)}' mint_command += f' --owner {mint_authority_file} -- "{str(token_account)}"' os.system(mint_command) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 397548595..f00499bdb 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -1,8 +1,12 @@ +from __future__ import annotations + import base58 import base64 import json import re import time +import traceback +import requests from typing import Optional @@ -10,22 +14,34 @@ from solana.publickey import PublicKey from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount -from solana.rpc.commitment import Confirmed from solana.rpc.types import RPCResponse from solana.transaction import Transaction from itertools import zip_longest from logged_groups import logged_group +from typing import Dict, Union, Any, List, NamedTuple, cast +from base58 import b58decode, b58encode from .costs import update_transaction_cost from .utils import get_from_dict, SolanaBlockInfo from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, WRITE_TRANSACTION_COST_IN_DB, SKIP_PREFLIGHT from ..environment import LOG_SENDING_SOLANA_TRANSACTION, FUZZING_BLOCKHASH, CONFIRM_TIMEOUT, FINALIZED +from ..environment import RETRY_ON_FAIL from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT from ..common_neon.address import EthereumAddress, ether2program -from ..common_neon.address import AccountInfo as NeonAccountInfo +from ..common_neon.address import AccountInfoLayout as AccountInfoLayout + -from typing import Any, List, NamedTuple, cast +class SolTxError(Exception): + def __init__(self, receipt): + self.result = receipt + error = get_error_definition_from_receipt(receipt) + if isinstance(error, list) and isinstance(error[1], str): + super().__init__(str(error[1])) + self.error = str(error[1]) + else: + super().__init__('Unknown error') + self.error = json.dumps(receipt) class AccountInfo(NamedTuple): @@ -42,16 +58,58 @@ class SendResult(NamedTuple): @logged_group("neon.Proxy") class SolanaInteractor: - def __init__(self, client: SolanaClient) -> None: - self.client = client + def __init__(self, solana_url: str) -> None: + self._client = SolanaClient(solana_url)._provider self._fuzzing_hash_cycle = False + def _make_request(self, request) -> RPCResponse: + """This method is used to make retries to send request to Solana""" + + headers = { + "Content-Type": "application/json" + } + client = self._client + + retry = 0 + while True: + try: + retry += 1 + raw_response = client.session.post(client.endpoint_uri, headers=headers, json=request) + raw_response.raise_for_status() + return raw_response + + except requests.exceptions.ConnectionError as err: + if retry > RETRY_ON_FAIL: + raise + + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.error(f'ConnectionError({retry}) on send request to Solana. ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + time.sleep(1) + + except Exception as err: + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.error('Unknown exception on send request to Solana. ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + raise + + def _send_rpc_request(self, method: str, *params: Any) -> RPCResponse: + request_id = next(self._client._request_counter) + 1 + + request = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params + } + raw_response = self._make_request(request) + return cast(RPCResponse, raw_response.json()) + def _send_rpc_batch_request(self, method: str, params_list: List[Any]) -> List[RPCResponse]: full_request_data = [] full_response_data = [] request_data = [] - client = self.client._provider - headers = {"Content-Type": "application/json"} + client = self._client for params in params_list: request_id = next(client._request_counter) + 1 @@ -61,10 +119,8 @@ def _send_rpc_batch_request(self, method: str, params_list: List[Any]) -> List[R # Protection from big payload if len(request_data) == 30 or len(full_request_data) == len(params_list): - response = client.session.post(client.endpoint_uri, headers=headers, json=request_data) - response.raise_for_status() - - response_data = cast(List[RPCResponse], response.json()) + raw_response = self._make_request(request_data) + response_data = cast(List[RPCResponse], raw_response.json()) full_response_data += response_data request_data.clear() @@ -79,22 +135,42 @@ def _send_rpc_batch_request(self, method: str, params_list: List[Any]) -> List[R return full_response_data - def get_account_info(self, storage_account) -> Optional[AccountInfo]: + def get_signatures_for_address(self, before, until, commitment='confirmed'): + opts: Dict[str, Union[int, str]] = {} + if until is not None: + opts["until"] = until + if before is not None: + opts["before"] = before + opts["commitment"] = commitment + return self._send_rpc_request("getSignaturesForAddress", EVM_LOADER_ID, opts) + + def get_confirmed_transaction(self, sol_sign: str, encoding: str = "json"): + return self._send_rpc_request("getConfirmedTransaction", sol_sign, encoding) + + def get_slot(self, commitment='confirmed') -> RPCResponse: + opts = { + 'commitment': commitment + } + return self._send_rpc_request('getSlot', opts) + + def get_account_info(self, pubkey: PublicKey, length=256, commitment='confirmed') -> Optional[AccountInfo]: opts = { "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 256, - } + "commitment": commitment, } - result = self.client._provider.make_request("getAccountInfo", str(storage_account), opts) - self.debug(f"\n{json.dumps(result, indent=4, sort_keys=True)}") + if length != 0: + opts['dataSlice'] = { + 'offset': 0, + 'length': length + } + + result = self._send_rpc_request('getAccountInfo', str(pubkey), opts) + self.debug(f"{json.dumps(result, sort_keys=True)}") info = result['result']['value'] if info is None: - self.debug(f"Can't get information about {storage_account}") + self.debug(f"Can't get information about {str(pubkey)}") return None data = base64.b64decode(info['data'][0]) @@ -105,24 +181,27 @@ def get_account_info(self, storage_account) -> Optional[AccountInfo]: return AccountInfo(account_tag, lamports, owner, data) - def get_multiple_accounts_info(self, accounts: [PublicKey]) -> [AccountInfo]: - options = { + def get_account_info_list(self, accounts: [PublicKey], length=256, commitment='confirmed') -> [AccountInfo]: + opts = { "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 16 - } + "commitment": commitment, } - result = self.client._provider.make_request("getMultipleAccounts", [str(a) for a in accounts], options) - self.debug(f"\n{json.dumps(result, indent=4, sort_keys=True)}") + + if length != 0: + opts['dataSlice'] = { + 'offset': 0, + 'length': length + } + + result = self._send_rpc_request("getMultipleAccounts", [str(a) for a in accounts], opts) + self.debug(f"{json.dumps(result, sort_keys=True)}") if result['result']['value'] is None: self.debug(f"Can't get information about {accounts}") return [] accounts_info = [] - for info in result['result']['value']: + for pubkey, info in zip(accounts, result['result']['value']): if info is None: accounts_info.append(None) else: @@ -132,10 +211,40 @@ def get_multiple_accounts_info(self, accounts: [PublicKey]) -> [AccountInfo]: return accounts_info - def get_sol_balance(self, account): - return self.client.get_balance(account, commitment=Confirmed)['result']['value'] + def get_sol_balance(self, account, commitment='confirmed'): + opts = { + "commitment": commitment + } + return self._send_rpc_request('getBalance', str(account), opts)['result']['value'] + + def get_token_account_balance(self, pubkey: Union[str, PublicKey], commitment='confirmed') -> int: + opts = { + "commitment": commitment + } + response = self._send_rpc_request("getTokenAccountBalance", str(pubkey), opts) + result = response.get('result', None) + if result is None: + return 0 + return int(result['value']['amount']) + + def get_token_account_balance_list(self, pubkey_list: [Union[str, PublicKey]], commitment: object = 'confirmed') -> [int]: + opts = { + "commitment": commitment + } + request_list = [] + for pubkey in pubkey_list: + request_list.append((str(pubkey), opts)) + + balance_list = [] + response_list = self._send_rpc_batch_request('getTokenAccountBalance', request_list) + for response in response_list: + result = response.get('result', None) + balance = int(result['value']['amount']) if result else 0 + balance_list.append(balance) - def get_neon_account_info(self, eth_account: EthereumAddress) -> Optional[NeonAccountInfo]: + return balance_list + + def get_account_info_layout(self, eth_account: EthereumAddress) -> Optional[AccountInfoLayout]: account_sol, nonce = ether2program(eth_account) info = self.get_account_info(account_sol) if info is None: @@ -143,10 +252,12 @@ def get_neon_account_info(self, eth_account: EthereumAddress) -> Optional[NeonAc elif len(info.data) < ACCOUNT_INFO_LAYOUT.sizeof(): raise RuntimeError(f"Wrong data length for account data {account_sol}: " + f"{len(info.data)} < {ACCOUNT_INFO_LAYOUT.sizeof()}") - return NeonAccountInfo.frombytes(info.data) + return AccountInfoLayout.frombytes(info.data) - def get_multiple_rent_exempt_balances_for_size(self, size_list: [int]) -> [int]: - opts = {"commitment": "confirmed"} + def get_multiple_rent_exempt_balances_for_size(self, size_list: [int], commitment='confirmed') -> [int]: + opts = { + "commitment": commitment + } request_list = [(size, opts) for size in size_list] response_list = self._send_rpc_batch_request("getMinimumBalanceForRentExemption", request_list) return [r['result'] for r in response_list] @@ -156,7 +267,29 @@ def get_block_slot_list(self, last_block_slot, limit: int, commitment='confirmed "commitment": commitment, "enconding": "json", } - return self.client._provider.make_request("getBlocksWithLimit", last_block_slot, limit, opts)['result'] + return self._send_rpc_request("getBlocksWithLimit", last_block_slot, limit, opts)['result'] + + def get_block_info(self, slot: int, commitment='confirmed') -> [SolanaBlockInfo]: + opts = { + "commitment": commitment, + "encoding": "json", + "transactionDetails": "signatures", + "rewards": False + } + + response = self._send_rpc_request('getBlock', slot, opts) + net_block = response.get('result', None) + if not net_block: + return SolanaBlockInfo(slot=slot) + + return SolanaBlockInfo( + slot=slot, + finalized=(commitment == FINALIZED), + hash='0x' + base58.b58decode(net_block['blockhash']).hex(), + parent_hash='0x' + base58.b58decode(net_block['previousBlockhash']).hex(), + time=net_block['blockTime'], + signs=net_block['signatures'] + ) def get_block_info_list(self, block_slot_list: [int], commitment='confirmed') -> [SolanaBlockInfo]: block_list = [] @@ -177,28 +310,37 @@ def get_block_info_list(self, block_slot_list: [int], commitment='confirmed') -> response_list = self._send_rpc_batch_request('getBlock', request_list) for slot, response in zip(block_slot_list, response_list): if (not response) or ('result' not in response): - continue - net_block = response['result'] - block = SolanaBlockInfo( - slot=slot, - finalized=(commitment == FINALIZED), - hash='0x' + base58.b58decode(net_block['blockhash']).hex(), - height=net_block['blockHeight'], - parent_hash='0x' + base58.b58decode(net_block['previousBlockhash']).hex(), - time=net_block['blockTime'], - signs=net_block['signatures'] - ) + block = SolanaBlockInfo( + slot=slot, + finalized=(commitment == FINALIZED), + ) + else: + net_block = response['result'] + block = SolanaBlockInfo( + slot=slot, + finalized=(commitment == FINALIZED), + hash='0x' + base58.b58decode(net_block['blockhash']).hex(), + parent_hash='0x' + base58.b58decode(net_block['previousBlockhash']).hex(), + time=net_block['blockTime'], + signs=net_block['signatures'] + ) block_list.append(block) return block_list - def get_recent_blockslot(self, commitment=Confirmed) -> int: - blockhash_resp = self.client.get_recent_blockhash(commitment=commitment) + def get_recent_blockslot(self, commitment='confirmed') -> int: + opts = { + 'commitment': commitment + } + blockhash_resp = self._send_rpc_request('getRecentBlockhash', opts) if not blockhash_resp["result"]: raise RuntimeError("failed to get recent blockhash") return blockhash_resp['result']['context']['slot'] - def get_recent_blockhash(self, commitment=Confirmed) -> Blockhash: - blockhash_resp = self.client.get_recent_blockhash(commitment=commitment) + def get_recent_blockhash(self, commitment='confirmed') -> Blockhash: + opts = { + 'commitment': commitment + } + blockhash_resp = self._send_rpc_request('getRecentBlockhash', opts) if not blockhash_resp["result"]: raise RuntimeError("failed to get recent blockhash") blockhash = blockhash_resp["result"]["value"]["blockhash"] @@ -225,7 +367,7 @@ def _fuzzing_transactions(self, signer: SolanaAccount, tx_list, tx_opts, request "rewards": False } slot = max(slot - 500, 10) - block = self.client._provider.make_request("getBlock", slot, block_opts) + block = self._send_rpc_request("getBlock", slot, block_opts) fuzzing_blockhash = Blockhash(block['result']['blockhash']) self.debug(f"fuzzing block {fuzzing_blockhash} for slot {slot}") @@ -239,11 +381,12 @@ def _fuzzing_transactions(self, signer: SolanaAccount, tx_list, tx_opts, request request_list[idx] = (base64_tx, tx_opts) return request_list - def _send_multiple_transactions_unconfirmed(self, signer: SolanaAccount, tx_list: [Transaction]) -> [str]: + def _send_multiple_transactions(self, signer: SolanaAccount, tx_list: [Transaction], + skip_preflight: bool, preflight_commitment: str) -> [str]: opts = { - "skipPreflight": SKIP_PREFLIGHT, + "skipPreflight": skip_preflight, "encoding": "base64", - "preflightCommitment": "confirmed" + "preflightCommitment": preflight_commitment } blockhash = None @@ -264,8 +407,9 @@ def _send_multiple_transactions_unconfirmed(self, signer: SolanaAccount, tx_list response_list = self._send_rpc_batch_request('sendTransaction', request_list) return [SendResult(result=r.get('result'), error=r.get('error')) for r in response_list] - def send_multiple_transactions(self, signer, tx_list, eth_tx, reason, waiter=None) -> [{}]: - send_result_list = self._send_multiple_transactions_unconfirmed(signer, tx_list) + def send_multiple_transactions(self, signer: SolanaAccount, tx_list: [], waiter, + skip_preflight: bool, preflight_commitment: str) -> [{}]: + send_result_list = self._send_multiple_transactions(signer, tx_list, skip_preflight, preflight_commitment) # Filter good transactions and wait the confirmations for them sign_list = [s.result for s in send_result_list if s.result] self._confirm_multiple_transactions(sign_list, waiter) @@ -275,46 +419,38 @@ def send_multiple_transactions(self, signer, tx_list, eth_tx, reason, waiter=Non receipt_list = [] for s in send_result_list: if s.error: + self.debug(f'Got error on preflight check of transaction: {s.error}') receipt_list.append(s.error) else: receipt_list.append(confirmed_list.pop(0)) - if WRITE_TRANSACTION_COST_IN_DB: - for receipt in receipt_list: - update_transaction_cost(receipt, eth_tx, reason) - return receipt_list - # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in - # .buildkite/steps/deploy-test.sh) - def get_measurements(self, reason, eth_tx, receipt): - if not LOG_SENDING_SOLANA_TRANSACTION: - return - - try: - self.debug(f"send multiple transactions for reason {reason}") - - measurements = self._extract_measurements_from_receipt(receipt) - for m in measurements: - self.info(f'get_measurements: {json.dumps(m)}') - except Exception as err: - self.error(f"get_measurements: can't get measurements {err}") - self.info(f"get measurements: failed result {json.dumps(receipt, indent=3)}") - def _confirm_multiple_transactions(self, sign_list: [str], waiter=None): """Confirm a transaction.""" if not len(sign_list): - self.debug(f'Got confirmed status for transactions: {sign_list}') + self.debug('No confirmations, because transaction list is empty') return + base58_sign_list: List[str] = [] + for sign in sign_list: + if isinstance(sign, str): + base58_sign_list.append(b58encode(b58decode(sign)).decode("utf-8")) + else: + base58_sign_list.append(b58encode(sign).decode("utf-8")) + + opts = { + "searchTransactionHistory": False + } + elapsed_time = 0 while elapsed_time < CONFIRM_TIMEOUT: if elapsed_time > 0: time.sleep(CONFIRMATION_CHECK_DELAY) elapsed_time += CONFIRMATION_CHECK_DELAY - response = self.client.get_signature_statuses(sign_list) - result = response['result'] + response = self._send_rpc_request("getSignatureStatuses", base58_sign_list, opts) + result = response.get('result', None) if not result: continue @@ -341,6 +477,174 @@ def _get_multiple_receipts(self, sign_list: [str]) -> [Any]: response_list = self._send_rpc_batch_request("getTransaction", request_list) return [r['result'] for r in response_list] + +@logged_group("neon.Proxy") +class SolTxListSender: + def __init__(self, sender, tx_list: [Transaction], name: str, + skip_preflight=SKIP_PREFLIGHT, preflight_commitment='confirmed'): + self._s = sender + self._name = name + self._skip_preflight = skip_preflight + self._preflight_commitment = preflight_commitment + + self._blockhash = None + self._retry_idx = 0 + self._slots_behind = 0 + self._tx_list = tx_list + self._node_behind_list = [] + self._bad_block_list = [] + self._blocked_account_list = [] + self._pending_list = [] + self._budget_exceeded_list = [] + self._storage_bad_status_list = [] + self._unknown_error_list = [] + + self._all_tx_list = [self._node_behind_list, + self._bad_block_list, + self._blocked_account_list, + self._budget_exceeded_list, + self._pending_list] + + def clear(self): + self._tx_list.clear() + for lst in self._all_tx_list: + lst.clear() + + def _get_full_list(self): + return [tx for lst in self._all_tx_list for tx in lst] + + def send(self) -> SolTxListSender: + solana = self._s.solana + signer = self._s.signer + waiter = self._s.waiter + skip = self._skip_preflight + commitment = self._preflight_commitment + + self.debug(f'start transactions sending: {self._name}') + + while (self._retry_idx < RETRY_ON_FAIL) and (len(self._tx_list)): + self._retry_idx += 1 + self._slots_behind = 0 + + receipt_list = solana.send_multiple_transactions(signer, self._tx_list, waiter, skip, commitment) + self.update_transaction_cost(receipt_list) + + success_cnt = 0 + for receipt, tx in zip(receipt_list, self._tx_list): + slots_behind = check_if_node_behind(receipt) + if slots_behind: + self._slots_behind = slots_behind + self._node_behind_list.append(tx) + elif check_if_blockhash_notfound(receipt): + self._bad_block_list.append(tx) + elif check_if_accounts_blocked(receipt): + self._blocked_account_list.append(tx) + elif check_for_errors(receipt): + if check_if_program_exceeded_instructions(receipt): + self._budget_exceeded_list.append(tx) + else: + custom = check_if_storage_is_empty_error(receipt) + if custom in (1, 4): + self._storage_bad_status_list.append(receipt) + else: + self._unknown_error_list.append(receipt) + else: + success_cnt += 1 + self._on_success_send(tx, receipt) + + self.debug(f'retry {self._retry_idx}, ' + + f'total receipts {len(receipt_list)}, ' + + f'success receipts {success_cnt}, ' + + f'node behind {len(self._node_behind_list)}, ' + f'bad blocks {len(self._bad_block_list)}, ' + + f'blocked accounts {len(self._blocked_account_list)}, ' + + f'budget exceeded {len(self._budget_exceeded_list)}, ' + + f'bad storage: {len(self._storage_bad_status_list)}, ' + + f'unknown error: {len(self._unknown_error_list)}') + + self._on_post_send() + + if len(self._tx_list): + raise RuntimeError('Run out of attempts to execute transaction') + return self + + def update_transaction_cost(self, receipt_list): + if not WRITE_TRANSACTION_COST_IN_DB: + return False + if not hasattr(self._s, 'eth_tx'): + return False + + for receipt in receipt_list: + update_transaction_cost(receipt, self._s.eth_tx, reason=self._name) + + def _on_success_send(self, tx: Transaction, receipt: {}) -> bool: + """Store the last successfully blockhash and set it in _set_tx_blockhash""" + self._blockhash = tx.recent_blockhash + return False + + def _on_post_send(self): + if len(self._unknown_error_list): + raise SolTxError(self._unknown_error_list[0]) + elif len(self._node_behind_list): + self.warning(f'Node is behind by {self._slots_behind} slots') + time.sleep(1) + elif len(self._storage_bad_status_list): + raise SolTxError(self._storage_bad_status_list[0]) + elif len(self._budget_exceeded_list): + raise RuntimeError(COMPUTATION_BUDGET_EXCEEDED) + + # There is no more retries to send transactions + if self._retry_idx >= RETRY_ON_FAIL: + if not self._is_canceled: + self._cancel() + return + + if len(self._blocked_account_list): + time.sleep(0.4) # one block time + + # force changing of recent_blockhash if Solana doesn't accept the current one + if len(self._bad_block_list): + self._blockhash = None + + # resend not-accepted transactions + self._move_txlist() + + def _set_tx_blockhash(self, tx): + """Try to keep the branch of block history""" + tx.recent_blockhash = self._blockhash + tx.signatures.clear() + + def _move_txlist(self): + full_list = self._get_full_list() + self.clear() + for tx in full_list: + self._set_tx_blockhash(tx) + self._tx_list.append(tx) + if len(self._tx_list): + self.debug(f' Resend Solana transactions: {len(self._tx_list)}') + + +@logged_group("neon.Proxy") +class Measurements: + def __init__(self): + pass + + # Do not change headers in info logs! This name used in CI measurements (see function `cleanup_docker` in + # .buildkite/steps/deploy-test.sh) + def extract(self, reason: str, receipt: {}): + if not LOG_SENDING_SOLANA_TRANSACTION: + return + + try: + self.debug(f"send multiple transactions for reason {reason}") + + measurements = self._extract_measurements_from_receipt(receipt) + for m in measurements: + self.info(f'get_measurements: {json.dumps(m)}') + except Exception as err: + self.error(f"get_measurements: can't get measurements {err}") + self.info(f"get measurements: failed result {json.dumps(receipt, indent=3)}") + def _extract_measurements_from_receipt(self, receipt): if check_for_errors(receipt): self.warning("Can't get measurements from receipt with error") @@ -365,37 +669,39 @@ def _extract_measurements_from_receipt(self, receipt): res = pattern.match(log) if res: (program, reason) = res.groups() - if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) + if reason == 'invoke [1]': messages.append({'program': program, 'logs': []}) messages[-1]['logs'].append(log) for instr in instructions: if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue if messages[0]['program'] != instr['program']: - raise ValueError('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) + raise ValueError('Invalid program in log messages: expect %s, actual %s' % ( + messages[0]['program'], instr['program'])) instr['logs'] = messages.pop(0)['logs'] - exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) + exit_result = re.match(r'Program %s (success)' % instr['program'], instr['logs'][-1]) if not exit_result: raise ValueError("Can't get exit result") instr['result'] = exit_result.group(1) if instr['program'] == EVM_LOADER_ID: memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) - instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) + instruction_result = re.match( + r'Program %s consumed ([0-9]+) of ([0-9]+) compute units' % instr['program'], instr['logs'][-2]) if not (memory_result and instruction_result): raise ValueError("Can't parse measurements for evm_loader") instr['measurements'] = { - 'instructions': instruction_result.group(1), - 'memory': memory_result.group(1) - } + 'instructions': instruction_result.group(1), + 'memory': memory_result.group(1) + } result = [] for instr in instructions: if instr['program'] == EVM_LOADER_ID: result.append({ - 'program':instr['program'], - 'measurements':instr['measurements'], - 'result':instr['result'], - 'data':instr['data'] - }) + 'program': instr['program'], + 'measurements': instr['measurements'], + 'result': instr['result'], + 'data': instr['data'] + }) return result @@ -491,7 +797,7 @@ def check_if_accounts_blocked(receipt, *, logger): logs = get_logs_from_receipt(receipt) if logs is None: logger.error("Can't get logs") - logger.info("Failed result: %s"%json.dumps(receipt, indent=3)) + logger.info(f"Failed result: {json.dumps(receipt, indent=3)}") return False ro_blocked = "trying to execute transaction on ro locked account" @@ -504,3 +810,7 @@ def check_if_accounts_blocked(receipt, *, logger): def check_if_blockhash_notfound(receipt): return (not receipt) or (get_from_dict(receipt, 'data', 'err') == 'BlockhashNotFound') + + +def check_if_node_behind(receipt): + return get_from_dict(receipt, 'data', 'numSlotsBehind') diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 753e261cc..7fffe2e25 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -16,40 +16,27 @@ from solana.transaction import AccountMeta, Transaction, PublicKey from solana.blockhash import Blockhash from solana.account import Account as SolanaAccount -from solana.rpc.api import Client as SolanaClient -from .address import accountWithSeed, getTokenAddr, EthereumAddress, isPayed +from .address import accountWithSeed, getTokenAddr, EthereumAddress from ..common_neon.errors import EthereumError from .constants import STORAGE_SIZE, EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG, ACCOUNT_SEED_VERSION from .emulator_interactor import call_emulated from .neon_instruction import NeonInstruction as NeonIxBuilder from .solana_interactor import COMPUTATION_BUDGET_EXCEEDED -from .solana_interactor import SolanaInteractor, check_for_errors, check_if_accounts_blocked +from .solana_interactor import SolanaInteractor, Measurements, SolTxListSender, SolTxError from .solana_interactor import check_if_big_transaction, check_if_program_exceeded_instructions -from .solana_interactor import get_error_definition_from_receipt, check_if_storage_is_empty_error -from .solana_interactor import check_if_blockhash_notfound from ..common_neon.eth_proto import Trx as EthTx from ..common_neon.utils import NeonTxResultInfo, NeonTxInfo -from ..environment import RETRY_ON_FAIL, EVM_LOADER_ID, PERM_ACCOUNT_LIMIT, ACCOUNT_PERMISSION_UPDATE_INT, MIN_OPERATOR_BALANCE_TO_WARN, MIN_OPERATOR_BALANCE_TO_ERR, \ - ACCOUNT_MAX_SIZE, SPL_TOKEN_ACCOUNT_SIZE, HOLDER_MSG_SIZE, ACCOUNT_STORAGE_OVERHEAD + +from ..environment import RETRY_ON_FAIL, EVM_LOADER_ID, PERM_ACCOUNT_LIMIT, ACCOUNT_PERMISSION_UPDATE_INT +from ..environment import MIN_OPERATOR_BALANCE_TO_WARN, MIN_OPERATOR_BALANCE_TO_ERR +from ..environment import ACCOUNT_MAX_SIZE, SPL_TOKEN_ACCOUNT_SIZE, HOLDER_MSG_SIZE, ACCOUNT_STORAGE_OVERHEAD from ..memdb.memdb import MemDB, NeonPendingTxInfo from ..environment import get_solana_accounts from ..common_neon.account_whitelist import AccountWhitelist from proxy.common_neon.utils import get_holder_msg -class SolanaTxError(Exception): - def __init__(self, receipt): - self.result = receipt - error = get_error_definition_from_receipt(receipt) - if isinstance(error, list) and isinstance(error[1], str): - super().__init__(str(error[1])) - self.error = str(error[1]) - else: - super().__init__('Unknown error') - self.error = json.dumps(receipt) - - class NeonTxStage(metaclass=abc.ABCMeta): NAME = 'UNKNOWN' @@ -338,7 +325,7 @@ def _create_perm_accounts(self, seed_list): stage_list = [NeonCreatePermAccount(self._s, seed, STORAGE_SIZE) for seed in seed_list] account_list = [s.sol_account for s in stage_list] - info_list = self._s.solana.get_multiple_accounts_info(account_list) + info_list = self._s.solana.get_account_info_list(account_list) balance = self._s.solana.get_multiple_rent_exempt_balances_for_size([STORAGE_SIZE])[0] for account, stage in zip(info_list, stage_list): if not account: @@ -385,14 +372,16 @@ def EthMeta(pubkey, is_writable) -> AccountMeta: @logged_group("neon.Proxy") class NeonTxSender: - def __init__(self, db: MemDB, client: SolanaClient, eth_tx: EthTx, steps: int): + def __init__(self, db: MemDB, solana: SolanaInteractor, eth_tx: EthTx, steps: int): self._db = db self.eth_tx = eth_tx self.neon_sign = '0x' + eth_tx.hash_signed().hex() self.steps = steps - self.solana = SolanaInteractor(client) + self.waiter = self + self.solana = solana self._resource_list = OperatorResourceList(self) self.resource = None + self.signer = None self.operator_key = None self.builder = None @@ -424,6 +413,7 @@ def execute(self) -> NeonTxResultInfo: def set_resource(self, resource: Optional[OperatorResourceInfo]): self.resource = resource + self.signer = resource.signer self.operator_key = resource.public_key() self.builder = NeonIxBuilder(self.operator_key) @@ -446,7 +436,7 @@ def _validate_pend_tx(self): self._pend_tx_into_db(self.solana.get_recent_blockslot()) def _validate_whitelist(self): - whitelist = AccountWhitelist(self.solana.client, ACCOUNT_PERMISSION_UPDATE_INT, self.resource.signer) + whitelist = AccountWhitelist(self.solana, ACCOUNT_PERMISSION_UPDATE_INT, self.resource.signer) if not whitelist.has_client_permission(self.eth_sender[2:]): self.warning(f'Sender account {self.eth_sender} is not allowed to execute transactions') raise Exception(f'Sender account {self.eth_sender} is not allowed to execute transactions') @@ -456,7 +446,7 @@ def _validate_whitelist(self): raise Exception(f'Contract account {self.deployed_contract} is not allowed for deployment') def _validate_tx_count(self): - info = self.solana.get_neon_account_info(EthereumAddress(self.eth_sender)) + info = self.solana.get_account_info_layout(EthereumAddress(self.eth_sender)) if not info: return @@ -494,7 +484,7 @@ def _execute(self): self.error(f'No strategy to execute the Neon transaction: {self.eth_tx}') raise RuntimeError('No strategy to execute the Neon transaction') - def on_wait_confirm(self, slot: int): + def on_wait_confirm(self, _, slot: int): self._pend_tx_into_db(slot) def _pend_tx_into_db(self, slot: int): @@ -542,7 +532,7 @@ def _call_emulated(self, sender=None): self.debug(f'destination address {self.to_address}') self._emulator_json = call_emulated(dst, src, self.eth_tx.callData.hex(), hex(self.eth_tx.value)) - self.debug(f'emulator returns: {json.dumps(self._emulator_json, indent=3)}') + self.debug(f'emulator returns: {json.dumps(self._emulator_json, sort_keys=True)}') self.steps_emulated = self._emulator_json['steps_executed'] @@ -575,7 +565,7 @@ def _parse_accounts_list(self): elif account_desc["storage_increment"]: self.unpaid_space += account_desc["storage_increment"] - if not isPayed(self.solana.client, account_desc['address']): + if not self.solana.get_account_info_layout(account_desc['address']).is_payed(): self.debug(f'found losted account {account_desc["account"]}') self.unpaid_space += ACCOUNT_MAX_SIZE + SPL_TOKEN_ACCOUNT_SIZE + ACCOUNT_STORAGE_OVERHEAD * 2 @@ -636,120 +626,6 @@ def done_account_txs(self, skip_create_accounts=False): self.create_account_tx.instructions.clear() -@logged_group("neon.Proxy") -class SolTxListSender: - def __init__(self, sender: NeonTxSender, tx_list: [Transaction], name: str): - self._s = sender - self._name = name - - self._blockhash = None - self._retry_idx = 0 - self._tx_list = tx_list - self._bad_block_list = [] - self._blocked_account_list = [] - self._pending_list = [] - self._budget_exceeded_list = [] - self._storage_bad_status_list = [] - self._unknown_error_list = [] - - self._all_tx_list = [self._bad_block_list, - self._blocked_account_list, - self._budget_exceeded_list, - self._pending_list] - - def clear(self): - self._tx_list.clear() - for lst in self._all_tx_list: - lst.clear() - - def _get_full_list(self): - return [tx for lst in self._all_tx_list for tx in lst] - - def send(self) -> SolTxListSender: - solana = self._s.solana - eth_tx = self._s.eth_tx - signer = self._s.resource.signer - - self.debug(f'Start stage: {self._name}') - - while (self._retry_idx < RETRY_ON_FAIL) and (len(self._tx_list)): - self._retry_idx += 1 - receipt_list = solana.send_multiple_transactions(signer, self._tx_list, eth_tx, self._name, self) - - success_cnt = 0 - for receipt, tx in zip(receipt_list, self._tx_list): - if check_if_blockhash_notfound(receipt): - self._bad_block_list.append(tx) - elif check_if_accounts_blocked(receipt): - self._blocked_account_list.append(tx) - elif check_for_errors(receipt): - if check_if_program_exceeded_instructions(receipt): - self._budget_exceeded_list.append(tx) - else: - custom = check_if_storage_is_empty_error(receipt) - if custom in (1, 4): - self._storage_bad_status_list.append(receipt) - else: - self._unknown_error_list.append(receipt) - else: - success_cnt += 1 - self._on_success_send(tx, receipt) - - self.debug(f'retry {self._retry_idx}, ' + - f'total receipts {len(receipt_list)}, ' + - f'success receipts {success_cnt}, ' + - f'bad blocks {len(self._bad_block_list)}, ' + - f'blocked accounts {len(self._blocked_account_list)}, ' + - f'budget exceeded {len(self._budget_exceeded_list)}, ' + - f'bad storage: {len(self._storage_bad_status_list)}, ' + - f'unknown error: {len(self._unknown_error_list)}') - - self._on_post_send() - - if len(self._tx_list): - raise RuntimeError('Run out of attempts to execute transaction') - return self - - def on_wait_confirm(self, _, slot: int): - self._s.on_wait_confirm(slot) - - def _on_success_send(self, tx: Transaction, receipt: {}): - """Store the last successfully blockhash and set it in _set_tx_blockhash""" - self._blockhash = tx.recent_blockhash - - def _on_post_send(self): - if len(self._unknown_error_list): - raise SolanaTxError(self._unknown_error_list[0]) - elif len(self._storage_bad_status_list): - raise SolanaTxError(self._storage_bad_status_list[0]) - elif len(self._budget_exceeded_list): - raise RuntimeError(COMPUTATION_BUDGET_EXCEEDED) - - if len(self._blocked_account_list): - time.sleep(0.4) # one block time - - # force changing of recent_blockhash if Solana doesn't accept the current one - if len(self._bad_block_list): - self._blockhash = None - - # resend not-accepted transactions - self._move_txlist() - - def _set_tx_blockhash(self, tx): - """Try to keep the branch of block history""" - tx.recent_blockhash = self._blockhash - tx.signatures.clear() - - def _move_txlist(self): - full_list = self._get_full_list() - self.clear() - for tx in full_list: - self._set_tx_blockhash(tx) - self._tx_list.append(tx) - if len(self._tx_list): - self.debug(f' Resend Solana transactions: {len(self._tx_list)}') - - @logged_group("neon.Proxy") class BaseNeonTxStrategy(metaclass=abc.ABCMeta): NAME = 'UNKNOWN STRATEGY' @@ -806,8 +682,7 @@ def __init__(self, strategy: BaseNeonTxStrategy, *args, **kwargs): def _on_success_send(self, tx: Transaction, receipt: {}): if not self.neon_res.is_valid(): if self.neon_res.decode(self._s.neon_sign, receipt).is_valid(): - self._s.solana.get_measurements(self._name, self._s.eth_tx, receipt) - + Measurements().extract(self._name, receipt) super()._on_success_send(tx, receipt) def _on_post_send(self): @@ -907,6 +782,7 @@ def _on_success_send(self, tx: Transaction, receipt: {}): if self._is_canceled: # Transaction with cancel is confirmed self.neon_res.canceled(receipt) + Measurements().extract(self._name, receipt) else: super()._on_success_send(tx, receipt) @@ -916,6 +792,10 @@ def _on_post_send(self): self.debug(f'Got Neon tx {"cancel" if self._is_canceled else "result"}: {self.neon_res}') return self.clear() + if len(self._node_behind_list): + self.warning(f'Node is behind by {self._slots_behind} slots') + time.sleep(1) + # Unknown error happens - cancel the transaction if len(self._unknown_error_list): self._unknown_error_list.clear() @@ -924,14 +804,14 @@ def _on_post_send(self): return # There is no more retries to send transactions - if self._retry_idx == RETRY_ON_FAIL: + if self._retry_idx >= RETRY_ON_FAIL: if not self._is_canceled: self._cancel() return # The storage has bad structure and the result isn't received! (( if len(self._storage_bad_status_list): - raise SolanaTxError(self._storage_bad_status_list[0]) + raise SolTxError(self._storage_bad_status_list[0]) # Blockhash is changed ((( if len(self._bad_block_list): diff --git a/proxy/common_neon/utils.py b/proxy/common_neon/utils.py index 20eaf0df9..99dd0fe8c 100644 --- a/proxy/common_neon/utils.py +++ b/proxy/common_neon/utils.py @@ -21,10 +21,9 @@ def str_fmt_object(obj): class SolanaBlockInfo: - def __init__(self, slot=None, finalized=False, height=None, hash=None, parent_hash=None, time=None, signs=None): + def __init__(self, slot=None, finalized=False, hash=None, parent_hash=None, time=None, signs=None): self.slot = slot self.finalized = finalized - self.height = height self.hash = hash self.parent_hash = parent_hash self.time = time @@ -63,7 +62,6 @@ def _set_defaults(self): self.return_value = bytes() self.sol_sign = None self.slot = -1 - self.block_height = -1 self.block_hash = '' self.idx = -1 @@ -101,11 +99,10 @@ def _decode_return(self, log: bytes, ix_idx: int, tx: {}): def fill_block_info(self, block: SolanaBlockInfo): self.slot = block.slot - self.block_height = block.height self.block_hash = block.hash for rec in self.logs: rec['blockHash'] = block.hash - rec['blockNumber'] = hex(block.height) + rec['blockNumber'] = hex(block.slot) def decode(self, neon_sign: str, tx: {}, ix_idx=-1) -> NeonTxResultInfo: self._set_defaults() diff --git a/proxy/environment.py b/proxy/environment.py index c60c46d54..739972b76 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -42,8 +42,6 @@ SOL_PRICE_UPDATE_INTERVAL = int(os.environ.get("SOL_PRICE_UPDATE_INTERVAL", 60)) GET_SOL_PRICE_MAX_RETRIES = int(os.environ.get("GET_SOL_PRICE_MAX_RETRIES", 3)) GET_SOL_PRICE_RETRY_INTERVAL = int(os.environ.get("GET_SOL_PRICE_RETRY_INTERVAL", 1)) -GET_WHITE_LIST_BALANCE_MAX_RETRIES = int(os.environ.get("GET_WHITE_LIST_BALANCE_MAX_RETRIES", 3)) -GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S = int(os.environ.get("GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S", 1)) INDEXER_LOG_SKIP_COUNT = int(os.environ.get("INDEXER_LOG_SKIP_COUNT", 10)) MIN_OPERATOR_BALANCE_TO_WARN = max(int(os.environ.get("MIN_OPERATOR_BALANCE_TO_WARN", 9000000000)), 9000000000) MIN_OPERATOR_BALANCE_TO_ERR = max(int(os.environ.get("MIN_OPERATOR_BALANCE_TO_ERR", 1000000000)), 1000000000) diff --git a/proxy/indexer/accounts_db.py b/proxy/indexer/accounts_db.py index a02d937eb..938791964 100644 --- a/proxy/indexer/accounts_db.py +++ b/proxy/indexer/accounts_db.py @@ -1,4 +1,4 @@ -from ..indexer.utils import BaseDB, DBQuery +from ..indexer.base_db import BaseDB, DBQuery from ..common_neon.utils import str_fmt_object diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py index bce74c661..9a37501b4 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/indexer/airdropper.py @@ -1,23 +1,18 @@ -from calendar import c from solana.publickey import PublicKey from proxy.indexer.indexer_base import IndexerBase from proxy.indexer.pythnetwork import PythNetworkClient -from proxy.indexer.utils import BaseDB -from solana.rpc.api import Client as SolanaClient +from proxy.indexer.base_db import BaseDB +from proxy.indexer.utils import check_error +from proxy.indexer.sql_dict import SQLDict import requests import base58 +import traceback from datetime import datetime from decimal import Decimal -import os from logged_groups import logged_group -from ..environment import NEON_PRICE_USD +from ..environment import NEON_PRICE_USD, EVM_LOADER_ID +from ..common_neon.solana_interactor import SolanaInteractor -try: - from utils import check_error - from sql_dict import SQLDict -except ImportError: - from .utils import check_error - from .sql_dict import SQLDict ACCOUNT_CREATION_PRICE_SOL = Decimal('0.00472692') AIRDROP_AMOUNT_SOL = ACCOUNT_CREATION_PRICE_SOL / 2 @@ -76,14 +71,11 @@ def is_airdrop_ready(self, eth_address): cur.execute(f"SELECT 1 FROM {self._table_name} WHERE eth_address = '{eth_address}'") return cur.fetchone() is not None -FINALIZED = os.environ.get('FINALIZED', 'finalized') - @logged_group("neon.Airdropper") class Airdropper(IndexerBase): def __init__(self, solana_url, - evm_loader_id, pyth_mapping_account: PublicKey, faucet_url = '', wrapper_whitelist = 'ANY', @@ -92,8 +84,9 @@ def __init__(self, max_conf = 0.1): # maximum confidence interval deviation related to price self._constants = SQLDict(tablename="constants") + solana = SolanaInteractor(solana_url) last_known_slot = self._constants.get('latest_processed_slot', None) - IndexerBase.__init__(self, solana_url, evm_loader_id, last_known_slot) + IndexerBase.__init__(self, solana, last_known_slot) self.latest_processed_slot = self.last_slot # collection of eth-address-to-create-accout-trx mappings @@ -113,7 +106,7 @@ def __init__(self, # but there will be different slot numbers so price should be updated every time self.always_reload_price = (pp_solana_url != solana_url) self.pyth_mapping_account = pyth_mapping_account - self.pyth_client = PythNetworkClient(SolanaClient(pp_solana_url)) + self.pyth_client = PythNetworkClient(SolanaInteractor(pp_solana_url)) self.neon_decimals = neon_decimals self.max_conf = Decimal(max_conf) self.session = requests.Session() @@ -122,13 +115,12 @@ def __init__(self, self.airdrop_amount_usd = None self.airdrop_amount_neon = None self.last_update_pyth_mapping = None - self.max_update_pyth_mapping_int = 60 * 60 # update once an hour - + self.max_update_pyth_mapping_int = 60 * 60 # update once an hour - def get_current_time(self): + @staticmethod + def get_current_time(): return datetime.now().timestamp() - def try_update_pyth_mapping(self): current_time = self.get_current_time() if self.last_update_pyth_mapping is None or self.last_update_pyth_mapping - current_time > self.max_update_pyth_mapping_int: @@ -136,7 +128,9 @@ def try_update_pyth_mapping(self): self.pyth_client.update_mapping(self.pyth_mapping_account) self.last_update_pyth_mapping = current_time except Exception as err: - self.warning(f'Failed to update pyth.network mapping account data: {err}') + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.warning(f'Failed to update pyth.network mapping account data ' + + f'{type(err)}, Error: {err}, Traceback: {err_tb}') return False return True @@ -147,7 +141,6 @@ def is_allowed_wrapper_contract(self, contract_addr): return True return contract_addr in self.wrapper_whitelist - # helper function checking if given 'create account' corresponds to 'create erc20 token account' instruction def check_create_instr(self, account_keys, create_acc, create_token_acc): # Must use the same Ethereum account @@ -164,27 +157,24 @@ def check_create_instr(self, account_keys, create_acc, create_token_acc): return False return True - # helper function checking if given 'create erc20 token account' corresponds to 'token transfer' instruction - def check_transfer(self, account_keys, create_token_acc, token_transfer) -> bool: + @staticmethod + def check_transfer(account_keys, create_token_acc, token_transfer) -> bool: return account_keys[create_token_acc['accounts'][1]] == account_keys[token_transfer['accounts'][1]] - def airdrop_to(self, eth_address, airdrop_galans): self.info(f"Airdrop {airdrop_galans} Galans to address: {eth_address}") json_data = { 'wallet': eth_address, 'amount': airdrop_galans } - resp = self.session.post(self.faucet_url + '/request_neon_in_galans', json = json_data) + resp = self.session.post(self.faucet_url + '/request_neon_in_galans', json=json_data) if not resp.ok: self.warning(f'Failed to airdrop: {resp.status_code}') return False return True - def process_trx_airdropper_mode(self, trx): if check_error(trx): return - # helper function finding all instructions that satisfies predicate def find_instructions(trx, predicate): return [instr for instr in trx['transaction']['message']['instructions'] if predicate(instr)] @@ -195,11 +185,11 @@ def find_instructions(trx, predicate): # Airdrop triggers on sequence: # neon.CreateAccount -> neon.CreateERC20TokenAccount -> spl.Transfer (maybe shuffled) # First: select all instructions that can form such chains - predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + predicate = lambda instr: account_keys[instr['programIdIndex']] == EVM_LOADER_ID \ and base58.b58decode(instr['data'])[0] == 0x02 create_acc_list = find_instructions(trx, predicate) - predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + predicate = lambda instr: account_keys[instr['programIdIndex']] == EVM_LOADER_ID \ and base58.b58decode(instr['data'])[0] == 0x0f create_token_acc_list = find_instructions(trx, predicate) @@ -217,18 +207,19 @@ def find_instructions(trx, predicate): continue self.schedule_airdrop(create_acc) - def get_sol_usd_price(self): should_reload = self.always_reload_price if not should_reload: - if self.recent_price == None or self.recent_price['valid_slot'] < self.current_slot: + if self.recent_price is None or self.recent_price['valid_slot'] < self.current_slot: should_reload = True if should_reload: try: self.recent_price = self.pyth_client.get_price('Crypto.SOL/USD') except Exception as err: - self.warning(f'Exception occured when reading price: {err}') + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.warning(f'Exception occured when reading price ' + + f'{type(err)}, Error: {err}, Traceback: {err_tb}') return None return self.recent_price @@ -252,7 +243,6 @@ def get_airdrop_amount_galans(self): self.info(f"Airdrop amount: ${self.airdrop_amount_usd} ({self.airdrop_amount_neon} NEONs)\n") return int(self.airdrop_amount_neon * pow(Decimal(10), self.neon_decimals)) - def schedule_airdrop(self, create_acc): eth_address = "0x" + bytearray(base58.b58decode(create_acc['data'])[20:][:20]).hex() if self.airdrop_ready.is_airdrop_ready(eth_address) or eth_address in self.airdrop_scheduled: @@ -261,7 +251,6 @@ def schedule_airdrop(self, create_acc): self.info(f'Scheduling airdrop for {eth_address}') self.airdrop_scheduled[eth_address] = { 'scheduled': self.get_current_time() } - def process_scheduled_trxs(self): # Pyth.network mapping account was never updated if not self.try_update_pyth_mapping() and self.last_update_pyth_mapping is None: @@ -287,8 +276,8 @@ def process_scheduled_trxs(self): }) for eth_address in success_addresses: - del self.airdrop_scheduled[eth_address] - + if eth_address in self.airdrop_scheduled: + del self.airdrop_scheduled[eth_address] def process_functions(self): """ @@ -299,7 +288,6 @@ def process_functions(self): self.process_receipts() self.process_scheduled_trxs() - def process_receipts(self): max_slot = 0 for slot, _, trx in self.transaction_receipts.get_trxs(self.latest_processed_slot): @@ -312,7 +300,6 @@ def process_receipts(self): @logged_group("neon.Airdropper") def run_airdropper(solana_url, - evm_loader_id, pyth_mapping_account: PublicKey, faucet_url, wrapper_whitelist = 'ANY', @@ -321,7 +308,7 @@ def run_airdropper(solana_url, max_conf = 0.1, *, logger): logger.info(f"""Running indexer with params: solana_url: {solana_url}, - evm_loader_id: {evm_loader_id}, + evm_loader_id: {EVM_LOADER_ID}, pyth.network mapping account: {pyth_mapping_account}, faucet_url: {faucet_url}, wrapper_whitelist: {wrapper_whitelist}, @@ -331,7 +318,6 @@ def run_airdropper(solana_url, try: airdropper = Airdropper(solana_url, - evm_loader_id, pyth_mapping_account, faucet_url, wrapper_whitelist, diff --git a/proxy/indexer/base_db.py b/proxy/indexer/base_db.py new file mode 100644 index 000000000..dc6d86fde --- /dev/null +++ b/proxy/indexer/base_db.py @@ -0,0 +1,75 @@ +import multiprocessing +import psycopg2 + +from typing import NamedTuple +from logged_groups import logged_group + +from .pg_common import POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST +from .pg_common import encode, decode + + +class DBQuery(NamedTuple): + column_list: list + key_list: list + order_list: list + + +class DBQueryExpression(NamedTuple): + column_expr: str + where_expr: str + where_keys: list + order_expr: str + + +@logged_group("neon.Indexer") +class BaseDB: + _create_table_lock = multiprocessing.Lock() + + def __init__(self): + self._conn = psycopg2.connect( + dbname=POSTGRES_DB, + user=POSTGRES_USER, + password=POSTGRES_PASSWORD, + host=POSTGRES_HOST + ) + self._conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + + with self._create_table_lock: + cursor = self._conn.cursor() + cursor.execute(self._create_table_sql()) + + def _create_table_sql(self) -> str: + assert False, 'No script for the table' + + def _build_expression(self, q: DBQuery) -> DBQueryExpression: + + return DBQueryExpression( + column_expr=','.join(q.column_list), + where_expr=' AND '.join(['1=1'] + [f'{name}=%s' for name, _ in q.key_list]), + where_keys=[value for _, value in q.key_list], + order_expr='ORDER BY ' + ', '.join(q.order_list) if len(q.order_list) else '', + ) + + def _fetchone(self, query: DBQuery) -> []: + e = self._build_expression(query) + + request = f''' + SELECT {e.column_expr} + FROM {self._table_name} AS a + WHERE {e.where_expr} + {e.order_expr} + LIMIT 1 + ''' + + with self._conn.cursor() as cursor: + cursor.execute(request, e.where_keys) + return cursor.fetchone() + + def __del__(self): + self._conn.close() + + def decode_list(self, v): + return [] if not v else decode(v) + + def encode_list(self, v: []): + return None if (not v) or (len(v) == 0) else encode(v) diff --git a/proxy/indexer/blocks_db.py b/proxy/indexer/blocks_db.py index 5c2ed5c67..5a6c2e8eb 100644 --- a/proxy/indexer/blocks_db.py +++ b/proxy/indexer/blocks_db.py @@ -1,25 +1,19 @@ -import psycopg2 -import psycopg2.extras -from ..indexer.utils import BaseDB, DBQuery +from typing import Optional + +from ..indexer.base_db import BaseDB, DBQuery from ..common_neon.utils import SolanaBlockInfo class SolanaBlocksDB(BaseDB): def __init__(self): BaseDB.__init__(self) + self._column_lst = ('slot', 'hash') self._full_column_lst = ('slot', 'hash', 'parent_hash', 'blocktime', 'signatures') def _create_table_sql(self) -> str: self._table_name = 'solana_block' return f""" - CREATE TABLE IF NOT EXISTS {self._table_name}_heights ( - slot BIGINT, - height BIGINT, - - UNIQUE(slot), - UNIQUE(height) - ); - CREATE TABLE IF NOT EXISTS {self._table_name}_hashes ( + CREATE TABLE IF NOT EXISTS {self._table_name} ( slot BIGINT, hash CHAR(66), @@ -32,127 +26,48 @@ def _create_table_sql(self) -> str: ); """ - def _fetch_block(self, slot, q: DBQuery) -> SolanaBlockInfo: - e = self._build_expression(q) - - request = f''' - SELECT a.slot, a.height, b.hash - FROM {self._table_name}_heights AS a - LEFT JOIN {self._table_name}_hashes AS b - ON a.slot = b.slot - WHERE {e.where_expr} - {e.order_expr} - LIMIT 1 - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, e.where_keys) - values = cursor.fetchone() - + def _block_from_value(self, slot: Optional[int], values: []) -> SolanaBlockInfo: if not values: return SolanaBlockInfo(slot=slot) return SolanaBlockInfo( finalized=True, slot=values[0], - height=values[1], - hash=values[2], + hash=values[1], ) - def _fetch_full_block(self, slot, q: DBQuery) -> SolanaBlockInfo: - e = self._build_expression(q) - - request = f''' - SELECT a.slot, a.height, b.hash, b.parent_hash, b.blocktime, b.signatures - FROM {self._table_name}_heights AS a - LEFT JOIN {self._table_name}_hashes AS b - ON a.slot = b.slot - WHERE {e.where_expr} - {e.order_expr} - LIMIT 1 - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, e.where_keys) - values = cursor.fetchone() - + def _full_block_from_value(self, slot: Optional[int], values: []) -> SolanaBlockInfo: if not values: return SolanaBlockInfo(slot=slot) return SolanaBlockInfo( finalized=True, slot=values[0], - height=values[1], - hash=values[2], - parent_hash=values[3], - time=values[4], - signs=self.decode_list(values[5]) + hash=values[1], + parent_hash=values[2], + time=values[3], + signs=self.decode_list(values[4]) ) - def get_latest_block(self) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[], order_list=['a.slot DESC']) - return self._fetch_block(None, q) - - def get_latest_block_list(self, limit: int) -> [SolanaBlockInfo]: - request = f''' - SELECT a.slot, a.height, b.hash, b.parent_hash, b.blocktime, b.signatures - FROM {self._table_name}_heights AS a - LEFT JOIN {self._table_name}_hashes AS b - ON a.slot = b.slot - ORDER BY a.slot DESC - LIMIT {limit} - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, []) - values = cursor.fetchall() - - if not values: - return [] - return [ - SolanaBlockInfo( - finalized=True, - slot=value[0], - height=value[1], - hash=value[2], - parent_hash=value[3], - time=value[4], - signs=self.decode_list(value[5]) - ) for value in values - ] - def get_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('a.slot', block_slot)], order_list=[]) - return self._fetch_block(block_slot, q) + q = DBQuery(column_list=self._column_lst, key_list=[('slot', block_slot)], order_list=[]) + return self._block_from_value(block_slot, self._fetchone(q)) def get_full_block_by_slot(self, block_slot) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('a.slot', block_slot)], order_list=[]) - return self._fetch_full_block(block_slot, q) + q = DBQuery(column_list=self._full_column_lst, key_list=[('slot', block_slot)], order_list=[]) + return self._block_from_value(block_slot, self._fetchone(q)) def get_block_by_hash(self, block_hash) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('b.hash', block_hash)], order_list=[]) - return self._fetch_block(None, q) - - def get_block_by_height(self, block_num) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('a.height', block_num)], order_list=[]) - return self._fetch_block(None, q) + q = DBQuery(column_list=self._column_lst, key_list=[('block_hash', block_hash)], order_list=[]) + return self._block_from_value(None, self._fetchone(q)) def set_block(self, block: SolanaBlockInfo): - cursor = self._conn.cursor() - cursor.execute(f''' - INSERT INTO {self._table_name}_hashes - ({', '.join(self._full_column_lst)}) - VALUES - ({', '.join(['%s' for _ in range(len(self._full_column_lst))])}) - ON CONFLICT DO NOTHING; - ''', - (block.slot, block.hash, block.parent_hash, block.time, self.encode_list(block.signs))) - - def fill_block_height(self, height, slots): with self._conn.cursor() as cursor: - psycopg2.extras.execute_values(cursor, f""" - INSERT INTO {self._table_name}_heights - (slot, height) - VALUES %s - ON CONFLICT DO NOTHING - """, ((slot, height+idx) for idx, slot in enumerate(slots)), template="(%s, %s)", page_size=1000) + cursor.execute(f''' + INSERT INTO {self._table_name} + ({', '.join(self._full_column_lst)}) + VALUES + ({', '.join(['%s' for _ in range(len(self._full_column_lst))])}) + ON CONFLICT DO NOTHING; + ''', + (block.slot, block.hash, block.parent_hash, block.time, self.encode_list(block.signs))) diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py index 6f9dfc15e..1a7d516c3 100644 --- a/proxy/indexer/canceller.py +++ b/proxy/indexer/canceller.py @@ -3,7 +3,6 @@ from logged_groups import logged_group from solana.publickey import PublicKey -from solana.rpc.api import Client from solana.system_program import SYS_PROGRAM_ID from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY from solana.transaction import AccountMeta @@ -12,7 +11,7 @@ from proxy.common_neon.constants import INCINERATOR_PUBKEY, KECCAK_PROGRAM, SYSVAR_INSTRUCTION_PUBKEY from proxy.common_neon.neon_instruction import NeonInstruction -from proxy.common_neon.solana_interactor import SolanaInteractor +from proxy.common_neon.solana_interactor import SolanaInteractor, SolTxListSender from proxy.common_neon.utils import get_from_dict from proxy.environment import ETH_TOKEN_MINT_ID, EVM_LOADER_ID, SOLANA_URL, get_solana_accounts @@ -34,15 +33,14 @@ class Canceller: def __init__(self): # Initialize user account self.signer = get_solana_accounts()[0] + self.solana = SolanaInteractor(SOLANA_URL) + self.waiter = None self._operator = self.signer.public_key() - self._client = Client(SOLANA_URL) self.operator_token = get_associated_token_address(PublicKey(self._operator), ETH_TOKEN_MINT_ID) - - self.solana = SolanaInteractor(self._client) self.builder = NeonInstruction(self._operator) - def unlock_accounts(self, blocked_storages): + tx_list = [] for storage, tx_accounts in blocked_storages.items(): (neon_tx, blocked_accounts) = tx_accounts if blocked_accounts is None: @@ -56,18 +54,17 @@ def unlock_accounts(self, blocked_storages): self.builder.init_eth_trx(neon_tx.tx, None, self.operator_token) self.builder.init_iterative(storage, None, 0) - trx = self.builder.make_cancel_transaction(keys) + tx = self.builder.make_cancel_transaction(keys) + tx_list.append(tx) + + if not len(tx_list): + return + + self.debug(f"Send Cancel: {len(tx_list)}") - self.debug(f"Send Cancel: {trx}") - try: - cancel_result = self.solana.send_multiple_transactions(self.signer, [trx], neon_tx.tx, "CancelWithNonce")[0] - self.debug(f"cancel result: {cancel_result}") - result_error = get_from_dict(cancel_result, 'meta', 'err') - if result_error: - self.error(f'Error sending cancel transaction: {result_error}') - except Exception as err: - err_tb = "".join(traceback.format_tb(err.__traceback__)) - self.error('Exception on submitting transaction. ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - else: - self.debug(f"Canceled: {blocked_accounts}") + try: + SolTxListSender(self, tx_list, f'CancelWithNonce({len(tx_list)})').send() + except Exception as err: + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.warning('Exception on submitting transaction. ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index af4bf8c26..002c49f3b 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -3,7 +3,6 @@ import base58 import time from logged_groups import logged_group, logging_context -from solana.rpc.api import Client from solana.system_program import SYS_PROGRAM_ID from ..indexer.indexer_base import IndexerBase @@ -13,6 +12,7 @@ from ..indexer.canceller import Canceller from ..common_neon.utils import NeonTxResultInfo, NeonTxInfo, str_fmt_object +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import EVM_LOADER_ID, FINALIZED, CANCEL_TIMEOUT, SOLANA_URL @@ -168,9 +168,9 @@ class ReceiptsParserState: - All instructions are removed from the _used_ixs; - If number of the smallest slot in the _used_ixs is changed, it's stored into the DB for the future restart. """ - def __init__(self, db: IndexerDB, solana_client: Client): + def __init__(self, db: IndexerDB, solana: SolanaInteractor): self._db = db - self._client = solana_client + self._solana = solana self._holder_table = {} self._tx_table = {} self._done_tx_list = [] @@ -674,71 +674,38 @@ def execute(self) -> bool: @logged_group("neon.Indexer") class BlocksIndexer: - def __init__(self, db: IndexerDB, solana_client: Client): + def __init__(self, db: IndexerDB, solana: SolanaInteractor): self.db = db - self.solana_client = solana_client + self.solana = solana self.counted_logger = MetricsToLogBuff() def gather_blocks(self): start_time = time.time() - latest_block = self.db.get_latest_block() - height = -1 - min_height = height - confirmed_blocks_len = 10000 - client = self.solana_client._provider - list_opts = {"commitment": FINALIZED} - block_opts = {"commitment": FINALIZED, "transactionDetails": "none", "rewards": False} - while confirmed_blocks_len == 10000: - confirmed_blocks = client.make_request("getBlocksWithLimit", latest_block.slot, confirmed_blocks_len, list_opts)['result'] - confirmed_blocks_len = len(confirmed_blocks) - # No more blocks - if confirmed_blocks_len == 0: - break - - # Intitialize start height - if height == -1: - first_block = client.make_request("getBlock", confirmed_blocks[0], block_opts) - height = first_block['result']['blockHeight'] - - # Validate last block height - latest_block.height = height + confirmed_blocks_len - 1 - latest_block.slot = confirmed_blocks[confirmed_blocks_len - 1] - last_block = client.make_request("getBlock", latest_block.slot, block_opts) - if not last_block['result'] or last_block['result']['blockHeight'] != latest_block.height: - self.warning(f"FAILED last_block_height {latest_block.height} " + - f"last_block_slot {latest_block.slot} " + - f"last_block {last_block}") - break - - # Everything is good - min_height = min(min_height, height) if min_height > 0 else height - self.db.fill_block_height(height, confirmed_blocks) - height = latest_block.height - + slot = self.solana.get_slot(FINALIZED)['result'] + self.db.set_latest_block(slot) gather_blocks_ms = (time.time() - start_time) * 1000 # convert this into milliseconds self.counted_logger.print( self.debug, - list_params={"gather_blocks_ms": gather_blocks_ms, "processed_height": latest_block.height - min_height}, - latest_params={"last_block_slot": latest_block.slot} + list_params={"gather_blocks_ms": gather_blocks_ms}, + latest_params={"last_block_slot": slot} ) @logged_group("neon.Indexer") class Indexer(IndexerBase): - def __init__(self, solana_url, evm_loader_id): + def __init__(self, solana_url): self.debug(f'Finalized commitment: {FINALIZED}') - self.db = IndexerDB() + solana = SolanaInteractor(solana_url) + self.db = IndexerDB(solana) last_known_slot = self.db.get_min_receipt_slot() - IndexerBase.__init__(self, solana_url, evm_loader_id, last_known_slot) + IndexerBase.__init__(self, solana, last_known_slot) self.indexed_slot = self.last_slot - self.db.set_client(self.solana_client) self.canceller = Canceller() self.blocked_storages = {} - self._init_last_height_slot() - self.block_indexer = BlocksIndexer(db=self.db, solana_client=self.solana_client) + self.block_indexer = BlocksIndexer(db=self.db, solana=solana) self.counted_logger = MetricsToLogBuff() - self.state = ReceiptsParserState(db=self.db, solana_client=self.solana_client) + self.state = ReceiptsParserState(db=self.db, solana=solana) self.ix_decoder_map = { 0x00: WriteIxDecoder(self.state), 0x01: DummyIxDecoder('Finalize', self.state), @@ -764,22 +731,6 @@ def __init__(self, solana_url, evm_loader_id): } self.def_decoder = DummyIxDecoder('Unknown', self.state) - def _init_last_height_slot(self): - last_known_slot = self.db.get_latest_block().slot - slot = self._init_last_slot('height', last_known_slot) - if last_known_slot == slot: - return - - block_opts = {"commitment": FINALIZED, "transactionDetails": "none", "rewards": False} - client = self.solana_client._provider - block = client.make_request("getBlock", slot, block_opts) - if not block['result']: - self.warning(f"Solana haven't return block information for the slot {slot}") - return - - height = block['result']['blockHeight'] - self.db.fill_block_height(height, [slot]) - def process_functions(self): self.block_indexer.gather_blocks() IndexerBase.process_functions(self) @@ -842,7 +793,7 @@ def unlock_accounts(self, tx) -> bool: self.warning(f"Transaction {tx.neon_tx} hasn't blocked accounts.") return False - storage_accounts_list = get_accounts_from_storage(self.solana_client, tx.storage_account) + storage_accounts_list = get_accounts_from_storage(self.solana, tx.storage_account) if storage_accounts_list is None: self.warning(f"Transaction {tx.neon_tx} has empty storage.") return False @@ -858,16 +809,15 @@ def unlock_accounts(self, tx) -> bool: @logged_group("neon.Indexer") -def run_indexer(solana_url, evm_loader_id, *, logger): +def run_indexer(solana_url, *, logger): logger.info(f"""Running indexer with params: solana_url: {solana_url}, - evm_loader_id: {evm_loader_id}""") + evm_loader_id: {EVM_LOADER_ID}""") - indexer = Indexer(solana_url, evm_loader_id) + indexer = Indexer(solana_url) indexer.run() if __name__ == "__main__": solana_url = SOLANA_URL - evm_loader_id = EVM_LOADER_ID - run_indexer(solana_url, evm_loader_id) + run_indexer(solana_url) diff --git a/proxy/indexer/indexer_base.py b/proxy/indexer/indexer_base.py index 1ade8fb6b..54fc2c475 100644 --- a/proxy/indexer/indexer_base.py +++ b/proxy/indexer/indexer_base.py @@ -1,26 +1,23 @@ import os import time import traceback -from solana.rpc.api import Client from multiprocessing.dummy import Pool as ThreadPool -from typing import Dict, Union from logged_groups import logged_group from .trx_receipts_storage import TrxReceiptsStorage from .utils import MetricsToLogBuff +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import RETRY_ON_FAIL_ON_GETTING_CONFIRMED_TRANSACTION -from ..environment import HISTORY_START, PARALLEL_REQUESTS, FINALIZED +from ..environment import HISTORY_START, PARALLEL_REQUESTS, FINALIZED, EVM_LOADER_ID @logged_group("neon.Indexer") class IndexerBase: def __init__(self, - solana_url, - evm_loader_id, - last_slot): - self.evm_loader_id = evm_loader_id - self.solana_client = Client(solana_url) + solana: SolanaInteractor, + last_slot: int): + self.solana = solana self.transaction_receipts = TrxReceiptsStorage('transaction_receipts') self.max_known_tx = self.transaction_receipts.max_known_trx() self.last_slot = self._init_last_slot('receipt', last_slot) @@ -36,7 +33,7 @@ def _init_last_slot(self, name: str, last_known_slot: int): - NUMBER - first start from the number, then continue from last parsed slot """ last_known_slot = 0 if not isinstance(last_known_slot, int) else last_known_slot - latest_slot = self.solana_client.get_slot(commitment=FINALIZED)["result"] + latest_slot = self.solana.get_slot(FINALIZED)["result"] start_int_slot = 0 name = f'{name} slot' @@ -87,7 +84,7 @@ def gather_unknown_transactions(self): minimal_tx = None continue_flag = True - current_slot = self.solana_client.get_slot(commitment=FINALIZED)["result"] + current_slot = self.solana.get_slot(commitment=FINALIZED)["result"] max_known_tx = self.max_known_tx @@ -137,13 +134,7 @@ def gather_unknown_transactions(self): ) def _get_signatures(self, before, until): - opts: Dict[str, Union[int, str]] = {} - if until is not None: - opts["until"] = until - if before is not None: - opts["before"] = before - opts["commitment"] = FINALIZED - result = self.solana_client._provider.make_request("getSignaturesForAddress", self.evm_loader_id, opts) + result = self.solana.get_signatures_for_address(before, until, FINALIZED) return result['result'] def _get_tx_receipts(self, solana_signature): @@ -152,7 +143,7 @@ def _get_tx_receipts(self, solana_signature): while retry > 0: try: - trx = self.solana_client.get_confirmed_transaction(solana_signature)['result'] + trx = self.solana.get_confirmed_transaction(solana_signature)['result'] self._add_trx(solana_signature, trx) retry = 0 except Exception as err: @@ -170,7 +161,7 @@ def _add_trx(self, solana_signature, trx): if trx is not None: add = False for instruction in trx['transaction']['message']['instructions']: - if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] == self.evm_loader_id: + if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] == EVM_LOADER_ID: add = True if add: self.debug((trx['slot'], solana_signature)) diff --git a/proxy/indexer/indexer_db.py b/proxy/indexer/indexer_db.py index 494ef3f21..143b77ecd 100644 --- a/proxy/indexer/indexer_db.py +++ b/proxy/indexer/indexer_db.py @@ -2,7 +2,6 @@ import traceback from logged_groups import logged_group -from solana.rpc.api import Client from typing import Optional from ..common_neon.utils import NeonTxInfo, NeonTxResultInfo, NeonTxFullInfo @@ -15,25 +14,23 @@ from ..indexer.logs_db import LogsDB from ..indexer.sql_dict import SQLDict from ..indexer.utils import get_code_from_account, get_accounts_by_neon_address +from ..common_neon.solana_interactor import SolanaInteractor @logged_group("neon.Indexer") class IndexerDB: - def __init__(self): + def __init__(self, solana: SolanaInteractor): self._logs_db = LogsDB() self._blocks_db = SolanaBlocksDB() self._txs_db = NeonTxsDB() self._account_db = NeonAccountDB() - self._client = None + self._solana = solana self._constants = SQLDict(tablename="constants") - for k in ['min_receipt_slot']: + for k in ['min_receipt_slot', 'latest_slot']: if k not in self._constants: self._constants[k] = 0 - def set_client(self, solana_client: Client): - self._client = solana_client - def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, used_ixs: [SolanaIxSignInfo]): try: block = self.get_block_by_slot(neon_res.slot) @@ -50,36 +47,25 @@ def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, us self.error('Exception on submitting transaction. ' + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - def _fill_block_from_net(self, block: SolanaBlockInfo): - opts = {"commitment": FINALIZED, "transactionDetails": "signatures", "rewards": False} - net_block = self._client._provider.make_request("getBlock", block.slot, opts) - if (not net_block) or ('result' not in net_block): - return block - - net_block = net_block['result'] - if not net_block: + def _get_block_from_net(self, block: SolanaBlockInfo) -> SolanaBlockInfo: + net_block = self._solana.get_block_info(block.slot, FINALIZED) + if not net_block.hash: return block - block.hash = '0x' + base58.b58decode(net_block['blockhash']).hex() - block.height = net_block['blockHeight'] - block.signs = net_block['signatures'] - block.parent_hash = '0x' + base58.b58decode(net_block['previousBlockhash']).hex() - block.time = net_block['blockTime'] - block.finalized = True - self.debug(f'{block}') - self._blocks_db.set_block(block) - return block + self.debug(f'{net_block}') + self._blocks_db.set_block(net_block) + return net_block def _fill_account_data_from_net(self, account: NeonAccountInfo): got_changes = False if not account.pda_account: - pda_account, code_account = get_accounts_by_neon_address(self._client, account.neon_account) + pda_account, code_account = get_accounts_by_neon_address(self._solana, account.neon_account) if pda_account: account.pda_account = pda_account account.code_account = code_account got_changes = True if account.code_account: - code = get_code_from_account(self._client, account.code_account) + code = get_code_from_account(self._solana, account.code_account) if code: account.code = code got_changes = True @@ -94,28 +80,25 @@ def _fill_account_data_from_net(self, account: NeonAccountInfo): def get_block_by_slot(self, slot) -> SolanaBlockInfo: block = self._blocks_db.get_block_by_slot(slot) if not block.hash: - self._fill_block_from_net(block) + block = self._get_block_from_net(block) return block def get_full_block_by_slot(self, slot) -> SolanaBlockInfo: block = self._blocks_db.get_full_block_by_slot(slot) if not block.parent_hash: - self._fill_block_from_net(block) + block = self._get_block_from_net(block) return block def get_latest_block(self) -> SolanaBlockInfo: - return self._blocks_db.get_latest_block() - - def get_latest_block_list(self, limit: int) -> [SolanaBlockInfo]: - return self._blocks_db.get_latest_block_list(limit) + return SolanaBlockInfo(slot=self._constants['latest_slot']) - def fill_block_height(self, number, slots): - self._blocks_db.fill_block_height(number, slots) + def set_latest_block(self, slot: int): + self._constants['latest_slot'] = slot def get_min_receipt_slot(self) -> int: return self._constants['min_receipt_slot'] - def set_min_receipt_slot(self, slot): + def set_min_receipt_slot(self, slot: int): self._constants['min_receipt_slot'] = slot def get_logs(self, from_block, to_block, addresses, topics, block_hash): @@ -124,9 +107,6 @@ def get_logs(self, from_block, to_block, addresses, topics, block_hash): def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: return self._blocks_db.get_block_by_hash(block_hash) - def get_block_by_height(self, block_height: int) -> SolanaBlockInfo: - return self._blocks_db.get_block_by_height(block_height) - def get_tx_list_by_sol_sign(self, sol_sign_list: [str]) -> [NeonTxFullInfo]: tx_list = self._txs_db.get_tx_list_by_sol_sign(sol_sign_list) block = None diff --git a/proxy/indexer/logs_db.py b/proxy/indexer/logs_db.py index f4e427399..984b67019 100644 --- a/proxy/indexer/logs_db.py +++ b/proxy/indexer/logs_db.py @@ -1,5 +1,5 @@ import json -from ..indexer.utils import BaseDB +from ..indexer.base_db import BaseDB class LogsDB(BaseDB): @@ -35,7 +35,7 @@ def push_logs(self, logs, block): ( log['address'], block.hash, - block.height, + block.slot, log['transactionHash'], int(log['transactionLogIndex'], 16), topic, diff --git a/proxy/indexer/pythnetwork.py b/proxy/indexer/pythnetwork.py index f6a7159f1..e8c4573b3 100644 --- a/proxy/indexer/pythnetwork.py +++ b/proxy/indexer/pythnetwork.py @@ -1,13 +1,13 @@ -from solana.rpc.api import Client as SolanaClient from solana.publickey import PublicKey from solana.system_program import SYS_PROGRAM_ID from decimal import Decimal -import base64 -import base58 import struct +import traceback from logged_groups import logged_group from typing import List, Union +from ..common_neon.solana_interactor import SolanaInteractor + def read_str(pos, data): length = data[pos] @@ -82,8 +82,8 @@ class PythNetworkClient: 'agg.status': { 'pos': 224, 'len': 4, 'format': ' str: from_addr CHAR(42), sol_sign CHAR(88), slot BIGINT, - block_height BIGINT, block_hash CHAR(66), idx INT, diff --git a/proxy/indexer/trx_receipts_storage.py b/proxy/indexer/trx_receipts_storage.py index a61e01def..3c4354b75 100644 --- a/proxy/indexer/trx_receipts_storage.py +++ b/proxy/indexer/trx_receipts_storage.py @@ -1,5 +1,5 @@ from proxy.indexer.pg_common import encode, decode -from proxy.indexer.utils import BaseDB +from proxy.indexer.base_db import BaseDB class TrxReceiptsStorage(BaseDB): diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index 951399020..1d7f6986b 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -1,26 +1,17 @@ from __future__ import annotations -import base64 -import multiprocessing -import psycopg2 import statistics -from typing import NamedTuple - from solana.publickey import PublicKey -from solana.rpc.api import Client -from solana.rpc.commitment import Confirmed from logged_groups import logged_group from typing import Dict, Union, Callable from ..common_neon.address import ether2program from ..common_neon.layouts import STORAGE_ACCOUNT_INFO_LAYOUT, CODE_ACCOUNT_INFO_LAYOUT, ACCOUNT_INFO_LAYOUT -from ..common_neon.utils import get_from_dict +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import INDEXER_LOG_SKIP_COUNT -from .pg_common import POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST -from .pg_common import encode, decode def check_error(trx): @@ -48,38 +39,25 @@ def get_req_id(self): return f"{self.idx}{self.sign}"[:7] - @logged_group("neon.Indexer") -def get_accounts_from_storage(client, storage_account, *, logger): - opts = { - "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 2048, - } - } - result = client._provider.make_request("getAccountInfo", str(storage_account), opts) +def get_accounts_from_storage(solana: SolanaInteractor, storage_account, *, logger): + info = solana.get_account_info(storage_account, length=0) # logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) - info = result['result']['value'] if info is None: - raise Exception("Can't get information about {}".format(storage_account)) + raise Exception(f"Can't get information about {storage_account}") - data = base64.b64decode(info['data'][0]) - - tag = data[0] - if tag in (0, 1, 4): + if info.tag in (0, 1, 4): logger.debug("Empty") return None else: logger.debug("Not empty storage") acc_list = [] - storage = STORAGE_ACCOUNT_INFO_LAYOUT.parse(data[1:]) + storage = STORAGE_ACCOUNT_INFO_LAYOUT.parse(info.data[1:]) offset = 1 + STORAGE_ACCOUNT_INFO_LAYOUT.sizeof() for _ in range(storage.accounts_len): - some_pubkey = PublicKey(data[offset:offset + 32]) + some_pubkey = PublicKey(info.data[offset:offset + 32]) acc_list.append(str(some_pubkey)) offset += 32 @@ -87,18 +65,16 @@ def get_accounts_from_storage(client, storage_account, *, logger): @logged_group("neon.Indexer") -def get_accounts_by_neon_address(client: Client, neon_address, *, logger): +def get_accounts_by_neon_address(solana: SolanaInteractor, neon_address, *, logger): pda_address, _nonce = ether2program(neon_address) - reciept = client.get_account_info(pda_address, commitment=Confirmed) - account_info = get_from_dict(reciept, 'result', 'value') - if account_info is None: - logger.debug(f"account_info is None for pda_address({pda_address}) in reciept({reciept})") + info = solana.get_account_info(pda_address, length=0) + if info is None: + logger.debug(f"account_info is None for pda_address({pda_address})") return None, None - data = base64.b64decode(account_info['data'][0]) - if len(data) < ACCOUNT_INFO_LAYOUT.sizeof(): - logger.debug(f"{len(data)} < {ACCOUNT_INFO_LAYOUT.sizeof()}") + if len(info.data) < ACCOUNT_INFO_LAYOUT.sizeof(): + logger.debug(f"{len(info.data)} < {ACCOUNT_INFO_LAYOUT.sizeof()}") return None, None - account = ACCOUNT_INFO_LAYOUT.parse(data) + account = ACCOUNT_INFO_LAYOUT.parse(info.data) code_account = None if account.code_account != [0]*32: code_account = str(PublicKey(account.code_account)) @@ -106,23 +82,21 @@ def get_accounts_by_neon_address(client: Client, neon_address, *, logger): @logged_group("neon.Indexer") -def get_code_from_account(client: Client, address, *, logger): - reciept = client.get_account_info(address, commitment=Confirmed) - code_account_info = get_from_dict(reciept, 'result', 'value') +def get_code_from_account(solana: SolanaInteractor, address, *, logger): + code_account_info = solana.get_account_info(address, length=0) if code_account_info is None: - logger.debug(f"code_account_info is None for code_address({address}) in reciept({reciept})") + logger.debug(f"code_account_info is None for code_address({address})") return None - data = base64.b64decode(code_account_info['data'][0]) - if len(data) < CODE_ACCOUNT_INFO_LAYOUT.sizeof(): + if len(code_account_info.data) < CODE_ACCOUNT_INFO_LAYOUT.sizeof(): return None - storage = CODE_ACCOUNT_INFO_LAYOUT.parse(data) + storage = CODE_ACCOUNT_INFO_LAYOUT.parse(code_account_info.data) offset = CODE_ACCOUNT_INFO_LAYOUT.sizeof() - if len(data) < offset + storage.code_size: + if len(code_account_info.data) < offset + storage.code_size: return None - return '0x' + data[offset:][:storage.code_size].hex() + return '0x' + code_account_info.data[offset:][:storage.code_size].hex() -class MetricsToLogBuff : +class MetricsToLogBuff: def __init__(self): self._reset() @@ -151,70 +125,3 @@ def print(self, logger: Callable[[str], None], list_params: Dict[str, Union[int, msg += f' {key}: {value};' logger(msg) self._reset() - - -class DBQuery(NamedTuple): - column_list: [] - key_list: [] - order_list: [] - - -class DBQueryExpression(NamedTuple): - column_expr: str - where_expr: str - where_keys: [] - order_expr: str - - -@logged_group("neon.Indexer") -class BaseDB: - _create_table_lock = multiprocessing.Lock() - - def __init__(self): - self._conn = psycopg2.connect( - dbname=POSTGRES_DB, - user=POSTGRES_USER, - password=POSTGRES_PASSWORD, - host=POSTGRES_HOST - ) - self._conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - - with self._create_table_lock: - cursor = self._conn.cursor() - cursor.execute(self._create_table_sql()) - - def _create_table_sql(self) -> str: - assert False, 'No script for the table' - - def _build_expression(self, q: DBQuery) -> DBQueryExpression: - - return DBQueryExpression( - column_expr=','.join(q.column_list), - where_expr=' AND '.join(['1=1'] + [f'{name}=%s' for name, _ in q.key_list]), - where_keys=[value for _, value in q.key_list], - order_expr='ORDER BY ' + ', '.join(q.order_list) if len(q.order_list) else '', - ) - - def _fetchone(self, query: DBQuery) -> str: - e = self._build_expression(query) - - request = f''' - SELECT {e.column_expr} - FROM {self._table_name} AS a - WHERE {e.where_expr} - {e.order_expr} - LIMIT 1 - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, e.where_keys) - return cursor.fetchone() - - def __del__(self): - self._conn.close() - - def decode_list(self, v): - return [] if not v else decode(v) - - def encode_list(self, v: []): - return None if (not v) or (len(v) == 0) else encode(v) diff --git a/proxy/memdb/blocks_db.py b/proxy/memdb/blocks_db.py index f4ed7b748..121a6e661 100644 --- a/proxy/memdb/blocks_db.py +++ b/proxy/memdb/blocks_db.py @@ -19,8 +19,7 @@ @logged_group("neon.Proxy") class RequestSolanaBlockList: - BLOCK_CACHE_LIMIT = 100 - BIG_SLOT = 1_000_000_000_000 + BLOCK_CACHE_LIMIT = (32 + 16) def __init__(self, blocks_db: MemBlocksDB): self._b = blocks_db @@ -32,8 +31,8 @@ def __init__(self, blocks_db: MemBlocksDB): self.pending_block_revision = 0 self.block_list = [] - self.first_block = SolanaBlockInfo(slot=0, height=0) - self.latest_block = SolanaBlockInfo(slot=0, height=0) + self.first_block = SolanaBlockInfo(slot=0) + self.latest_block = SolanaBlockInfo(slot=0) def execute(self) -> bool: try: @@ -54,22 +53,41 @@ def _get_latest_db_block(self): self.latest_db_block_slot = self._b.solana.get_recent_blockslot(commitment=FINALIZED) def _get_solana_block_list(self) -> bool: - slot = self.latest_db_block_slot - slot_list = [s for s in range(slot + self.BLOCK_CACHE_LIMIT, slot - 1, -1)] - self.block_list = self._b.solana.get_block_info_list(slot_list) + latest_db_slot = self.latest_db_block_slot + exist_block_dict = self._b.get_block_dict(latest_db_slot) + latest_slot = max(exist_block_dict) if len(exist_block_dict) else 0 + + block_time = 0 + slot_list = [] + self.block_list = [] + + max_slot = max(latest_db_slot + self.BLOCK_CACHE_LIMIT, latest_slot) + for slot in range(max_slot, latest_db_slot - 1, -1): + block = exist_block_dict.get(slot) + if block is None: + slot_list.append(slot) + else: + self.block_list.append(block) + block_time = max(block_time, block.time) + + solana_block_list = self._b.solana.get_block_info_list(slot_list) + for block in solana_block_list: + if not block.time: + if block.slot > latest_slot: + continue + block.time = block_time + block.hash = '0x' + os.urandom(32).hex() + block.parent_hash = '0x' + os.urandom(32).hex() + else: + block_time = max(block_time, block.time) + latest_slot = max(block.slot, latest_slot) + self.block_list.append(block) + if not len(self.block_list): return False + self.block_list.sort(key=lambda b: b.slot, reverse=True) self.latest_block = self.block_list[0] - - height = self.latest_block.height - latest_height = self._b.get_latest_block_height() - if latest_height > height: - height = latest_height - for block in self.block_list: - block.height = height - height -= 1 - self.first_block = self.block_list[len(self.block_list) - 1] return len(self.block_list) > 0 @@ -100,18 +118,18 @@ class MemBlocksDB: _active_block_revision = 0 _block_by_hash = {} - _block_by_height = {} + _block_by_slot = {} # Head and tail of cache - _first_block = SolanaBlockInfo(slot=0, height=0) - _latest_block = SolanaBlockInfo(slot=0, height=0) + _first_block = SolanaBlockInfo(slot=0) + _latest_block = SolanaBlockInfo(slot=0) _latest_db_block_slot = 0 def __init__(self, solana: SolanaInteractor, db: IndexerDB): self.db = db self.solana = solana self._update_block_dicts() - self.debug(f'Init first version of block list {len(self._block_by_height)} ' + + self.debug(f'Init first version of block list {len(self._block_by_slot)} ' + f'first block - {self._first_block}, ' + f'latest block - {self._latest_block}, ' + f'latest db block slot - {self._latest_db_block_slot}') @@ -150,12 +168,12 @@ def _fill_block_dicts(self, request: RequestSolanaBlockList): self._latest_block = request.latest_block self._latest_db_block_slot = request.latest_db_block_slot - self._block_by_height.clear() + self._block_by_slot.clear() self._block_by_hash.clear() for block in request.block_list: self._block_by_hash[block.hash] = block - self._block_by_height[block.height] = block + self._block_by_slot[block.slot] = block def _start_request(self) -> bool: last_time = self._last_time.value @@ -222,27 +240,37 @@ def _try_to_fill_blocks_from_pending_list(self): self._fill_block_dicts(request) def _update_block_dicts(self): - if not self._request_new_block_list(): - self._try_to_fill_blocks_from_pending_list() + self._try_to_fill_blocks_from_pending_list() + self._request_new_block_list() + + def get_block_dict(self, from_slot: int) -> {}: + return {slot: block for slot, block in self._block_by_slot.items() if slot > from_slot} def get_latest_block(self) -> SolanaBlockInfo: self._update_block_dicts() return self._latest_block - def get_latest_block_height(self) -> int: + def get_latest_block_slot(self) -> int: self._update_block_dicts() - return self._latest_block.height + return self._latest_block.slot def get_db_block_slot(self) -> int: self._update_block_dicts() return self._latest_db_block_slot - def get_block_by_height(self, block_height: int) -> SolanaBlockInfo: + def get_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: + self._update_block_dicts() + if block_slot > self._first_block.slot: + return self._block_by_slot.get(block_slot, SolanaBlockInfo()) + + return self.db.get_block_by_slot(block_slot) + + def get_full_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: self._update_block_dicts() - if block_height > self._first_block.height: - return self._block_by_height.get(block_height, SolanaBlockInfo()) + if block_slot > self._first_block.slot: + return self._block_by_slot.get(block_slot, SolanaBlockInfo()) - return self.db.get_block_by_height(block_height) + return self.db.get_full_block_by_slot(block_slot) def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: self._update_block_dicts() @@ -257,14 +285,9 @@ def _generate_fake_block(self, neon_res: NeonTxResultInfo) -> SolanaBlockInfo: if data: block = pickle.loads(data) else: - latest_block = self._latest_block - block_height = (latest_block.height or neon_res.slot) + 1 - block_time = (latest_block.time or 1) - block = SolanaBlockInfo( slot=neon_res.slot, - height=block_height, - time=block_time, + time=self._latest_block.time, hash='0x' + os.urandom(32).hex(), parent_hash='0x' + os.urandom(32).hex(), ) @@ -274,21 +297,15 @@ def _generate_fake_block(self, neon_res: NeonTxResultInfo) -> SolanaBlockInfo: return block def submit_block(self, neon_res: NeonTxResultInfo) -> SolanaBlockInfo: - block_list = self.solana.get_block_info_list([neon_res.slot]) - is_new_block = False - if len(block_list): - block = block_list[0] - data = pickle.dumps(block) - else: - block = SolanaBlockInfo() - data = None + block = self.solana.get_block_info(neon_res.slot) with self._last_time.get_lock(): - if not block.slot: + if not block.time: block = self._generate_fake_block(neon_res) data = pickle.dumps(block) is_new_block = True else: + data = pickle.dumps(block) is_new_block = neon_res.slot not in self._pending_block_by_slot if is_new_block: diff --git a/proxy/memdb/memdb.py b/proxy/memdb/memdb.py index 19b481d72..a156d1524 100644 --- a/proxy/memdb/memdb.py +++ b/proxy/memdb/memdb.py @@ -1,5 +1,4 @@ from logged_groups import logged_group -from solana.rpc.api import Client as SolanaClient from typing import Optional from ..indexer.indexer_db import IndexerDB @@ -14,12 +13,9 @@ @logged_group("neon.Proxy") class MemDB: - def __init__(self, client: SolanaClient): - self._client = client - - self._db = IndexerDB() - self._db.set_client(self._client) - self._solana = SolanaInteractor(client) + def __init__(self, solana: SolanaInteractor): + self._solana = solana + self._db = IndexerDB(solana) self._blocks_db = MemBlocksDB(self._solana, self._db) self._txs_db = MemTxsDB(self._db) @@ -31,14 +27,14 @@ def _before_slot(self) -> int: def get_latest_block(self) -> SolanaBlockInfo: return self._blocks_db.get_latest_block() - def get_latest_block_height(self) -> int: - return self._blocks_db.get_latest_block_height() + def get_latest_block_slot(self) -> int: + return self._blocks_db.get_latest_block_slot() - def get_block_by_height(self, block_height: int) -> SolanaBlockInfo: - return self._blocks_db.get_block_by_height(block_height) + def get_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: + return self._blocks_db.get_block_by_slot(block_slot) def get_full_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: - return self._db.get_full_block_by_slot(block_slot) + return self._blocks_db.get_full_block_by_slot(block_slot) def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: return self._blocks_db.get_block_by_hash(block_hash) diff --git a/proxy/memdb/transactions_db.py b/proxy/memdb/transactions_db.py index 474fba1e6..de042a423 100644 --- a/proxy/memdb/transactions_db.py +++ b/proxy/memdb/transactions_db.py @@ -86,9 +86,9 @@ def _has_topics(src_topics, dst_topics): with self._tx_slot.get_lock(): for data in self._tx_by_neon_sign.values(): tx = pickle.loads(data) - if from_block and tx.neon_res.block_height < from_block: + if from_block and tx.neon_res.slot < from_block: continue - if to_block and tx.neon_res.block_height > to_block: + if to_block and tx.neon_res.slot > to_block: continue if block_hash and tx.neon_res.block_hash != block_hash: continue diff --git a/proxy/plugin/gas_price_calculator.py b/proxy/plugin/gas_price_calculator.py index ce7ac2915..ad93aa05f 100644 --- a/proxy/plugin/gas_price_calculator.py +++ b/proxy/plugin/gas_price_calculator.py @@ -3,16 +3,17 @@ import time from logged_groups import logged_group from ..indexer.pythnetwork import PythNetworkClient +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import MINIMAL_GAS_PRICE, OPERATOR_FEE, NEON_PRICE_USD, \ SOL_PRICE_UPDATE_INTERVAL, GET_SOL_PRICE_MAX_RETRIES, GET_SOL_PRICE_RETRY_INTERVAL @logged_group("neon.gas_price_calculator") class GasPriceCalculator: - def __init__(self, solana_client, pyth_mapping_acc) -> None: - self.solana_client = solana_client + def __init__(self, solana: SolanaInteractor, pyth_mapping_acc) -> None: + self.solana = solana self.mapping_account = pyth_mapping_acc - self.pyth_network_client = PythNetworkClient(self.solana_client) + self.pyth_network_client = PythNetworkClient(self.solana) self.recent_sol_price_update_time = None self.min_gas_price = None diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 9edaac1e6..72bac8946 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -23,14 +23,12 @@ from ..http.parser import HttpParser from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes -from solana.rpc.api import Client as SolanaClient from typing import List, Tuple from .solana_rest_api_tools import neon_config_load, get_token_balance_or_zero from ..common_neon.transaction_sender import NeonTxSender -from ..common_neon.solana_interactor import SolanaInteractor +from ..common_neon.solana_interactor import SolanaInteractor, SolTxError from ..common_neon.address import EthereumAddress -from ..common_neon.transaction_sender import SolanaTxError from ..common_neon.emulator_interactor import call_emulated from ..common_neon.errors import EthereumError, PendingTxError from ..common_neon.estimate import GasEstimate @@ -54,13 +52,13 @@ class EthereumModel: proxy_id_glob = multiprocessing.Value('i', 0) def __init__(self): - self._client = SolanaClient(SOLANA_URL) - self._db = MemDB(self._client) + self._solana = SolanaInteractor(SOLANA_URL) + self._db = MemDB(self._solana) if PP_SOLANA_URL == SOLANA_URL: - self.gas_price_calculator = GasPriceCalculator(self._client, PYTH_MAPPING_ACCOUNT) + self.gas_price_calculator = GasPriceCalculator(self._solana, PYTH_MAPPING_ACCOUNT) else: - self.gas_price_calculator = GasPriceCalculator(SolanaClient(PP_SOLANA_URL), PYTH_MAPPING_ACCOUNT) + self.gas_price_calculator = GasPriceCalculator(SolanaInteractor(PP_SOLANA_URL), PYTH_MAPPING_ACCOUNT) self.gas_price_calculator.update_mapping() with self.proxy_id_glob.get_lock(): @@ -96,7 +94,7 @@ def eth_gasPrice(self): def eth_estimateGas(self, param): try: - calculator = GasEstimate(param, self._db, self._client, evm_step_count) + calculator = GasEstimate(param, self._db, self._solana, evm_step_count) return calculator.estimate() except EthereumError: @@ -115,17 +113,16 @@ def process_block_tag(self, tag): elif tag in ('earliest', 'pending'): raise Exception("Invalid tag {}".format(tag)) elif isinstance(tag, str): - block = SolanaBlockInfo(height=int(tag, 16)) + block = SolanaBlockInfo(slot=int(tag, 16)) elif isinstance(tag, int): - block = SolanaBlockInfo(height=tag) + block = SolanaBlockInfo(slot=tag) else: raise Exception(f'Failed to parse block tag: {tag}') return block def eth_blockNumber(self): - height = self._db.get_latest_block_height() - self.debug("eth_blockNumber %s", hex(height)) - return hex(height) + slot = self._db.get_latest_block_slot() + return hex(slot) def eth_getBalance(self, account, tag): """account - address to check for balance. @@ -133,7 +130,7 @@ def eth_getBalance(self, account, tag): """ eth_acc = EthereumAddress(account) self.debug(f'eth_getBalance: {account} {eth_acc}') - balance = get_token_balance_or_zero(self._client, eth_acc) + balance = get_token_balance_or_zero(self._solana, eth_acc) return hex(balance * eth_utils.denoms.gwei) def eth_getLogs(self, obj): @@ -151,9 +148,9 @@ def to_list(items): block_hash = None if 'fromBlock' in obj and obj['fromBlock'] != '0': - from_block = self.process_block_tag(obj['fromBlock']).height + from_block = self.process_block_tag(obj['fromBlock']).slot if 'toBlock' in obj and obj['toBlock'] != 'latest': - to_block = self.process_block_tag(obj['toBlock']).height + to_block = self.process_block_tag(obj['toBlock']).slot if 'address' in obj: addresses = to_list(obj['address']) if 'topics' in obj: @@ -226,10 +223,6 @@ def eth_getBlockByHash(self, block_hash, full): self.debug("Not found block by hash %s", block_hash) return None ret = self.getBlockBySlot(block, full, False) - if ret is not None: - self.debug("eth_getBlockByHash: %s", json.dumps(ret, indent=3)) - else: - self.debug("Not found block by hash %s", block_hash) return ret def eth_getBlockByNumber(self, tag, full): @@ -239,15 +232,9 @@ def eth_getBlockByNumber(self, tag, full): """ block = self.process_block_tag(tag) if block.slot is None: - block = self._db.get_block_by_height(block.height) - if block.slot is None: - self.debug("Not found block by number %s", tag) + self.debug(f"Not found block by number {tag}") return None ret = self.getBlockBySlot(block, full, tag == 'latest') - if ret is not None: - self.debug("eth_getBlockByNumber: %s", json.dumps(ret, indent=3)) - else: - self.debug("Not found block by number %s", tag) return ret def eth_call(self, obj, tag): @@ -278,8 +265,7 @@ def eth_call(self, obj, tag): def eth_getTransactionCount(self, account, tag): self.debug('eth_getTransactionCount: %s', account) try: - solana = SolanaInteractor(self._client) - acc_info = solana.get_neon_account_info(EthereumAddress(account)) + acc_info = self._solana.get_account_info_layout(EthereumAddress(account)) return hex(acc_info.trx_count) except Exception as err: self.debug(f"eth_getTransactionCount: Can't get account info: {err}") @@ -290,7 +276,7 @@ def _getTransactionReceipt(self, tx): "transactionHash": tx.neon_tx.sign, "transactionIndex": hex(0), "blockHash": tx.neon_res.block_hash, - "blockNumber": hex(tx.neon_res.block_height), + "blockNumber": hex(tx.neon_res.slot), "from": tx.neon_tx.addr, "to": tx.neon_tx.to_addr, "gasUsed": tx.neon_res.gas_used, @@ -301,7 +287,6 @@ def _getTransactionReceipt(self, tx): "logsBloom": "0x"+'0'*512 } - self.debug('RESULT: %s', json.dumps(result, indent=3)) return result def eth_getTransactionReceipt(self, trxId): @@ -320,7 +305,7 @@ def _getTransaction(self, tx): result = { "blockHash": r.block_hash, - "blockNumber": hex(r.block_height), + "blockNumber": hex(r.slot), "hash": t.sign, "transactionIndex": hex(0), "from": t.addr, @@ -335,7 +320,6 @@ def _getTransaction(self, tx): "s": t.s, } - self.debug("_getTransaction: %s", json.dumps(result, indent=3)) return result def eth_getTransactionByHash(self, trxId): @@ -353,15 +337,12 @@ def eth_getCode(self, account, _tag): return self._db.get_contract_code(account) def eth_sendTransaction(self, trx): - self.debug("eth_sendTransaction") - self.debug("eth_sendTransaction: type(trx):%s", type(trx)) - self.debug("eth_sendTransaction: str(trx):%s", str(trx)) - self.debug("eth_sendTransaction: trx=%s", json.dumps(trx, cls=JsonEncoder, indent=3)) + self.debug(f"eth_sendTransaction type(trx): {type(trx)}, str(trx): {str(trx)}") raise RuntimeError("eth_sendTransaction is not supported. please use eth_sendRawTransaction") def eth_sendRawTransaction(self, rawTrx): trx = EthTrx.fromString(bytearray.fromhex(rawTrx[2:])) - self.debug(f"{json.dumps(trx.as_dict(), cls=JsonEncoder, indent=3)}") + self.debug(f"{json.dumps(trx.as_dict(), cls=JsonEncoder, sort_keys=True)}") min_gas_price = self.gas_price_calculator.get_min_gas_price() if trx.gasPrice < min_gas_price: @@ -381,14 +362,14 @@ def eth_sendRawTransaction(self, rawTrx): eth_signature = '0x' + trx.hash_signed().hex() try: - tx_sender = NeonTxSender(self._db, self._client, trx, steps=evm_step_count) + tx_sender = NeonTxSender(self._db, self._solana, trx, steps=evm_step_count) tx_sender.execute() return eth_signature except PendingTxError as err: self.debug(f'{err}') return eth_signature - except SolanaTxError as err: + except SolTxError as err: err_msg = json.dumps(err.result, indent=3) self.error(f"Got SendTransactionError: {err_msg}") raise @@ -450,7 +431,7 @@ def process_request(self, request): method = getattr(self.model, request['method']) params = request.get('params', []) response['result'] = method(*params) - except SolanaTxError as err: + except SolTxError as err: # traceback.print_exc() response['error'] = err.error except EthereumError as err: @@ -459,7 +440,7 @@ def process_request(self, request): except Exception as err: err_tb = "".join(traceback.format_tb(err.__traceback__)) self.error('Exception on process request. ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') response['error'] = {'code': -32000, 'message': str(err)} return response @@ -506,9 +487,12 @@ def handle_request_impl(self, request: HttpParser) -> None: response = {'jsonrpc': '2.0', 'error': {'code': -32000, 'message': str(err)}} resp_time_ms = (time.time() - start_time)*1000 # convert this into milliseconds - self.info('handle_request >>> %s 0x%0x %s %s resp_time_ms= %s', threading.get_ident(), id(self.model), json.dumps(response), - request.get('method', '---'), - resp_time_ms) + self.info('handle_request >>> %s 0x%0x %s %s resp_time_ms= %s', + threading.get_ident(), + id(self.model), + json.dumps(response), + request.get('method', '---'), + resp_time_ms) self.client.queue(memoryview(build_http_response( httpStatusCodes.OK, body=json.dumps(response).encode('utf8'), diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index ab2630470..d6bcd7b26 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -1,14 +1,12 @@ from datetime import datetime from solana.publickey import PublicKey -from solana.rpc.api import Client as SolanaClient -from solana.rpc.commitment import Confirmed from logged_groups import logged_group from ..common_neon.address import ether2program, getTokenAddr, EthereumAddress -from ..common_neon.errors import SolanaAccountNotFoundError, SolanaErrors -from ..common_neon.utils import get_from_dict +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import read_elf_params, TIMEOUT_TO_RELOAD_NEON_CONFIG + @logged_group("neon.Proxy") def neon_config_load(ethereum_model, *, logger): try: @@ -30,42 +28,23 @@ def neon_config_load(ethereum_model, *, logger): '-' \ + ethereum_model.neon_config_dict['NEON_REVISION'] logger.debug(ethereum_model.neon_config_dict) + + @logged_group("neon.Proxy") -def get_token_balance_gwei(client: SolanaClient, pda_account: str, *, logger) -> int: +def get_token_balance_gwei(solana: SolanaInteractor, pda_account: str, *, logger) -> int: neon_token_account = getTokenAddr(PublicKey(pda_account)) - rpc_response = client.get_token_account_balance(neon_token_account, commitment=Confirmed) - error = rpc_response.get('error') - if error is not None: - message = error.get("message") - if message == SolanaErrors.AccountNotFound.value: - raise SolanaAccountNotFoundError() - logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, " - f"got get_token_account_balance error: \"{message}\"") - raise Exception("Getting balance error") - - balance = get_from_dict(rpc_response, "result", "value", "amount") - if balance is None: - logger.error( - f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, response: {rpc_response}") - raise Exception("Unexpected get_balance response") - return int(balance) + return solana.get_token_account_balance(neon_token_account) @logged_group("neon.Proxy") -def get_token_balance_or_zero(client: SolanaClient, eth_account: EthereumAddress, *, logger) -> int: +def get_token_balance_or_zero(solana: SolanaInteractor, eth_account: EthereumAddress, *, logger) -> int: solana_account, nonce = ether2program(eth_account) logger.debug(f"Get balance for eth account: {eth_account} aka: {solana_account}") - - try: - return get_token_balance_gwei(client, solana_account) - except SolanaAccountNotFoundError: - logger.debug(f"Account not found: {eth_account} aka: {solana_account} - return airdrop amount") - return 0 + return get_token_balance_gwei(solana, solana_account) -def is_account_exists(client: SolanaClient, eth_account: EthereumAddress) -> bool: +def is_account_exists(solana: SolanaInteractor, eth_account: EthereumAddress) -> bool: pda_account, nonce = ether2program(eth_account) - info = client.get_account_info(pda_account, commitment=Confirmed) - value = get_from_dict(info, "result", "value") - return value is not None + info = solana.get_account_info(pda_account) + return info is not None diff --git a/proxy/testing/test_account_whitelist.py b/proxy/testing/test_account_whitelist.py index ed225e8ae..d41220d04 100644 --- a/proxy/testing/test_account_whitelist.py +++ b/proxy/testing/test_account_whitelist.py @@ -1,44 +1,43 @@ import os -from signal import default_int_handler import unittest -from unittest import mock +from proxy.common_neon.solana_interactor import SolanaInteractor from proxy.common_neon.account_whitelist import AccountWhitelist from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount from solana.rpc.commitment import Confirmed from unittest.mock import Mock, MagicMock, patch, call -from proxy.environment import GET_WHITE_LIST_BALANCE_MAX_RETRIES - class TestAccountWhitelist(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.solana = SolanaClient(os.environ['SOLANA_URL']) + cls.solana = SolanaInteractor(os.environ['SOLANA_URL']) cls.payer = SolanaAccount() - cls.solana.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) + client = SolanaClient(os.environ['SOLANA_URL']) + client.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) cls.permission_update_int = 10 cls.testee = AccountWhitelist(cls.solana, cls.payer, cls.permission_update_int) mock_allowance_token = Mock() - mock_allowance_token.get_balance = MagicMock() + mock_allowance_token.get_token_account_address = MagicMock() mock_allowance_token.mint_to = MagicMock() cls.testee.allowance_token = mock_allowance_token mock_denial_token = Mock() - mock_denial_token.get_balance = MagicMock() + mock_denial_token.get_token_account_address = MagicMock() mock_denial_token.mint_to = MagicMock() cls.testee.denial_token = mock_denial_token def tearDown(self) -> None: - self.testee.allowance_token.get_balance.reset_mock() + self.testee.allowance_token.get_token_account_address.reset_mock() self.testee.allowance_token.mint_to.reset_mock() - self.testee.denial_token.get_balance.reset_mock() + self.testee.denial_token.get_token_account_address.reset_mock() self.testee.denial_token.mint_to.reset_mock() self.testee.account_cache = {} - def test_grant_permissions_negative_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_grant_permissions_negative_difference(self, mock_get_token_account_balance_list): """ Should mint allowance token - negative differenct """ @@ -49,16 +48,16 @@ def test_grant_permissions_negative_difference(self): expected_mint = min_balance - diff ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.grant_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.allowance_token.mint_to.assert_called_once_with(expected_mint, ether_address) - def test_grant_permissions_positive_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_grant_permissions_positive_difference(self, mock_get_token_account_balance_list): """ Should NOT mint allowance token - positive difference """ @@ -67,16 +66,16 @@ def test_grant_permissions_positive_difference(self): min_balance = 1 ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.grant_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.allowance_token.mint_to.assert_not_called() - def test_deprive_permissions_positive_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_deprive_permissions_positive_difference(self, mock_get_token_account_balance_list): """ Should mint denial token - positive difference """ @@ -87,16 +86,16 @@ def test_deprive_permissions_positive_difference(self): expected_mint = diff - min_balance + 1 ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.deprive_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.denial_token.mint_to.assert_called_once_with(expected_mint, ether_address) - def test_deprive_permissions_negative_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_deprive_permissions_negative_difference(self, mock_get_token_account_balance_list): """ Should NOT mint denial token - negative difference """ @@ -105,49 +104,28 @@ def test_deprive_permissions_negative_difference(self): min_balance = 3 ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.deprive_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.denial_token.mint_to.assert_not_called() @patch.object(AccountWhitelist, 'get_current_time') - def test_check_has_permission(self, mock_get_current_time): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_check_has_permission(self, mock_get_token_account_balance_list, mock_get_current_time): ether_address = 'Ethereum-Address' - time1 = 123 # will cause get_balance call - time2 = time1 + self.permission_update_int + 2 # will cause get_balance call - time3 = time2 + self.permission_update_int - 3 # will NOT cause get_balance call + time1 = 123 # will cause get_token_account_address call + time2 = time1 + self.permission_update_int + 2 # will cause get_token_account_address call + time3 = time2 + self.permission_update_int - 3 # will NOT cause get_token_account_address call mock_get_current_time.side_effect = [ time1, time2, time3 ] - - allowance_balance1 = 100 - denial_balance1 = 50 - allowance_balance2 = 100 - denial_balance2 = 150 - - self.testee.allowance_token.get_balance.side_effect = [allowance_balance1, allowance_balance2] - self.testee.denial_token.get_balance.side_effect = [denial_balance1, denial_balance2] + mock_get_token_account_balance_list.side_effect = [[100, 50], [100, 150]] self.assertTrue(self.testee.has_permission(ether_address, 0)) self.assertFalse(self.testee.has_permission(ether_address, 0)) self.assertFalse(self.testee.has_permission(ether_address, 0)) mock_get_current_time.assert_has_calls([call()] * 3) - self.testee.allowance_token.get_balance.assert_has_calls([call(ether_address)] * 2) - self.testee.denial_token.get_balance.assert_has_calls([call(ether_address)] * 2) - - @patch.object(AccountWhitelist, 'read_balance_diff') - @patch.object(AccountWhitelist, 'get_current_time') - def test_success_check_has_permission_after_retry_due_to_read_balance_diff_exception(self, mock_get_current_time, mock_read_balance_diff): - """ - Should retry read_balance_diff after exception - """ - self.assertGreaterEqual(GET_WHITE_LIST_BALANCE_MAX_RETRIES, 2) # Condition required to start test - ether_address = 'Ethereum-Address' - mock_get_current_time.side_effect = [0] - mock_read_balance_diff.side_effect = [Exception('TestException'), 12.3] - - self.assertTrue(self.testee.has_permission(ether_address, 0)) - self.assertTrue(mock_read_balance_diff.call_count, 2) + self.testee.allowance_token.get_token_account_address.assert_has_calls([call(ether_address)] * 2) + self.testee.denial_token.get_token_account_address.assert_has_calls([call(ether_address)] * 2) diff --git a/proxy/testing/test_airdropper.py b/proxy/testing/test_airdropper.py index 537d0a933..264bcf803 100644 --- a/proxy/testing/test_airdropper.py +++ b/proxy/testing/test_airdropper.py @@ -1,11 +1,11 @@ import os import unittest -from solana.rpc.api import Client as SolanaClient from solana.publickey import PublicKey from proxy.testing.mock_server import MockServer from proxy.indexer.airdropper import Airdropper, AIRDROP_AMOUNT_SOL, NEON_PRICE_USD from proxy.indexer.sql_dict import SQLDict +from proxy.common_neon.solana_interactor import SolanaInteractor import time from flask import request, Response from unittest.mock import Mock, MagicMock, patch, ANY @@ -13,6 +13,10 @@ import itertools from proxy.testing.transactions import pre_token_airdrop_trx, wrapper_whitelist, evm_loader_addr, token_airdrop_address + +SOLANA_URL = os.environ.get("SOLANA_URL", "http://solana:8899") + + class MockFaucet(MockServer): def __init__(self, port): super().__init__(port) @@ -54,8 +58,7 @@ def create_price_info(valid_slot: int, price: Decimal, conf: Decimal): class Test_Airdropper(unittest.TestCase): def create_airdropper(self, start_slot): os.environ['START_SLOT'] = str(start_slot) - return Airdropper(solana_url =f'http://{self.address}:8899', - evm_loader_id =self.evm_loader_id, + return Airdropper(solana_url =SOLANA_URL, pyth_mapping_account=self.pyth_mapping_account, faucet_url =f'http://{self.address}:{self.faucet_port}', wrapper_whitelist =self.wrapper_whitelist, @@ -63,7 +66,7 @@ def create_airdropper(self, start_slot): @classmethod @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def setUpClass(cls, mock_get_slot, mock_dict_get) -> None: print("testing indexer in airdropper mode") cls.address = 'localhost' @@ -73,7 +76,7 @@ def setUpClass(cls, mock_get_slot, mock_dict_get) -> None: cls.wrapper_whitelist = wrapper_whitelist cls.neon_decimals = 9 cls.airdropper = cls.create_airdropper(cls, 0) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() cls.airdropper.always_reload_price = True @@ -92,7 +95,6 @@ def setUpClass(cls, mock_get_slot, mock_dict_get) -> None: cls.mock_failed_attempts.airdrop_failed = MagicMock() cls.airdropper.failed_attempts = cls.mock_failed_attempts - def setUp(self) -> None: print(f"\n\n{self._testMethodName}\n{self._testMethodDoc}") self.faucet = MockFaucet(self.faucet_port) @@ -100,7 +102,6 @@ def setUp(self) -> None: self.airdropper.last_update_pyth_mapping = None time.sleep(0.2) - def tearDown(self) -> None: self.faucet.shutdown_server() self.faucet.join() @@ -111,7 +112,6 @@ def tearDown(self) -> None: self.mock_pyth_client.update_mapping.reset_mock() self.mock_failed_attempts.airdrop_failed.reset_mock() - def test_failed_process_trx_with_one_airdrop_price_provider_error(self): """ Should not airdrop to new address due to price provider error @@ -131,7 +131,6 @@ def test_failed_process_trx_with_one_airdrop_price_provider_error(self): self.mock_pyth_client.get_price.assert_called_once_with('Crypto.SOL/USD') self.faucet.request_neon_in_galans_mock.assert_not_called() - @patch.object(Airdropper, 'is_allowed_wrapper_contract') def test_failed_airdrop_contract_not_in_whitelist(self, mock_is_allowed_contract): """ @@ -158,7 +157,6 @@ def test_failed_airdrop_contract_not_in_whitelist(self, mock_is_allowed_contract self.mock_airdrop_ready.register_airdrop.assert_not_called() self.faucet.request_neon_in_galans_mock.assert_not_called() - def test_faucet_failure(self): """ Should not add address to processed list due to faucet error @@ -187,7 +185,6 @@ def test_faucet_failure(self): json_req = {'wallet': token_airdrop_address, 'amount': airdrop_amount} self.faucet.request_neon_in_galans_mock.assert_called_once_with(json_req) - def test_process_trx_with_one_airdrop_for_already_processed_address(self): """ Should not airdrop to repeated address @@ -212,7 +209,6 @@ def test_process_trx_with_one_airdrop_for_already_processed_address(self): self.mock_airdrop_ready.register_airdrop.assert_not_called() self.faucet.request_neon_in_galans_mock.assert_not_called() - def test_failed_airdrop_confidence_interval_too_large(self): """ Should not airdrop because confidence interval too large @@ -234,7 +230,6 @@ def test_failed_airdrop_confidence_interval_too_large(self): self.mock_airdrop_ready.register_airdrop.assert_not_called() self.faucet.request_neon_in_galans_mock.assert_not_called() - def test_update_mapping_error(self): self.mock_pyth_client.update_mapping.side_effect = [Exception('TestException')] try: @@ -244,7 +239,6 @@ def test_update_mapping_error(self): except Exception as err: self.fail(f'Excpected not throws exception but it does: {err}') - def test_get_price_error(self): self.mock_pyth_client.get_price.side_effect = [Exception('TestException')] try: @@ -255,69 +249,67 @@ def test_get_price_error(self): self.fail(f'Excpected not throws exception but it does: {err}') @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_continue(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('CONTINUE') self.assertEqual(new_airdropper.latest_processed_slot, start_slot - 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_continue_recent_slot_not_found(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [None] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('CONTINUE') self.assertEqual(new_airdropper.latest_processed_slot, start_slot + 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() - @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_start_slot_parse_error(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('Wrong value') self.assertEqual(new_airdropper.latest_processed_slot, start_slot - 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() - @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_latest(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('LATEST') self.assertEqual(new_airdropper.latest_processed_slot, start_slot + 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_number(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper(str(start_slot)) self.assertEqual(new_airdropper.latest_processed_slot, start_slot) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_big_slot_number(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper(str(start_slot + 100)) self.assertEqual(new_airdropper.latest_processed_slot, start_slot + 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() diff --git a/proxy/testing/test_neon_tx_sender.py b/proxy/testing/test_neon_tx_sender.py index 8ac10f105..3e9bc01fb 100644 --- a/proxy/testing/test_neon_tx_sender.py +++ b/proxy/testing/test_neon_tx_sender.py @@ -1,21 +1,18 @@ -import logging import os import unittest from solana.rpc.api import Client as SolanaClient -from solana.account import Account as SolanaAccount -from solana.rpc.commitment import Confirmed -from unittest.mock import Mock, MagicMock, patch, call +from unittest.mock import Mock from proxy.common_neon.eth_proto import Trx as EthTrx from proxy.common_neon.transaction_sender import NeonTxSender -from proxy.indexer.indexer_db import IndexerDB +from proxy.common_neon.solana_interactor import SolanaInteractor from proxy.memdb.memdb import MemDB class TestNeonTxSender(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.solana = SolanaClient(os.environ['SOLANA_URL']) + cls.solana = SolanaInteractor(os.environ['SOLANA_URL']) def setUp(self) -> None: trx = EthTrx.fromString(bytearray.fromhex('f8678080843ade68b194f0dafe87532d4373453b2555c644390e1b99e84c8459682f0080820102a00193e1966a82c5597942370980fb78080901ca86eb3c1b25ec600b2760cfcc94a03efcc1169e161f9a148fd4586e0bcf880648ca74075bfa7a9acc8800614fc9ff')) diff --git a/proxy/testing/test_permission_token.py b/proxy/testing/test_permission_token.py index 1259ed18f..bd609da27 100644 --- a/proxy/testing/test_permission_token.py +++ b/proxy/testing/test_permission_token.py @@ -2,9 +2,9 @@ import unittest from solana.rpc.api import Client as SolanaClient from proxy.common_neon.permission_token import PermissionToken +from proxy.common_neon.solana_interactor import SolanaInteractor from solana.publickey import PublicKey from solana.account import Account as SolanaAccount -import json from web3 import Web3 from solana.rpc.commitment import Confirmed @@ -14,7 +14,7 @@ class TestPermissionToken(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.solana = SolanaClient(os.environ['SOLANA_URL']) + cls.solana = SolanaInteractor(os.environ['SOLANA_URL']) cls.mint_authority_file = "/spl/bin/evm_loader-keypair.json" proxy_url = os.environ['PROXY_URL'] cls.proxy = Web3(Web3.HTTPProvider(proxy_url)) @@ -22,7 +22,8 @@ def setUpClass(cls) -> None: request_airdrop(cls.eth_account.address) cls.payer = SolanaAccount() - cls.solana.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) + client = SolanaClient(os.environ['SOLANA_URL']) + client.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) cls.allowance_token = PermissionToken(cls.solana, PublicKey(os.environ['NEON_PERMISSION_ALLOWANCE_TOKEN']), cls.payer) @@ -31,13 +32,13 @@ def setUpClass(cls) -> None: PublicKey(os.environ['NEON_PERMISSION_DENIAL_TOKEN']), cls.payer) - def test_get_balance_non_existing_account(self): - """ - Should return zero balance for non existing token-account - """ - new_acc = self.proxy.eth.account.create(f'test_get_balance_non_existing_account') - self.assertEqual(self.allowance_token.get_balance(new_acc.address), 0) - self.assertEqual(self.denial_token.get_balance(new_acc.address), 0) + # def test_get_balance_non_existing_account(self): + # """ + # Should return zero balance for non existing token-account + # """ + # new_acc = self.proxy.eth.account.create(f'test_get_balance_non_existing_account') + # self.assertEqual(self.allowance_token.get_balance(new_acc.address), 0) + # self.assertEqual(self.denial_token.get_balance(new_acc.address), 0) def test_mint_permission_tokens(self): """ diff --git a/proxy/testing/test_pyth_network_client.py b/proxy/testing/test_pyth_network_client.py index 1a03019da..3d12ad25e 100644 --- a/proxy/testing/test_pyth_network_client.py +++ b/proxy/testing/test_pyth_network_client.py @@ -1,14 +1,14 @@ import unittest -from unittest.mock import patch, ANY, call +from unittest.mock import patch, call from proxy.indexer.pythnetwork import PythNetworkClient -from solana.rpc.api import Client as SolanaClient +from proxy.common_neon.solana_interactor import SolanaInteractor from solana.publickey import PublicKey from time import sleep from decimal import Decimal # Will perform tests with devnet network # CI Airdropper that is already running in parallel (see docker-compose-test.yml) -# uses mainnet-beta. +# uses mainnet-beta. # PythNetworkClient will fail with 'too many requests' if trying to connect # it to the same Solana network solana_url = "https://api.devnet.solana.com" @@ -57,7 +57,7 @@ def setUpClass(cls) -> None: 'status': 1 } - cls.testee = PythNetworkClient(SolanaClient(solana_url)) + cls.testee = PythNetworkClient(SolanaInteractor(solana_url)) def update_mapping(self): self.testee.update_mapping(mapping_account) @@ -67,9 +67,9 @@ def update_mapping(self): @patch.object(PythNetworkClient, 'parse_mapping_account') @patch.object(PythNetworkClient, 'parse_prod_account') @patch.object(PythNetworkClient, 'parse_price_account') - def test_success_update_mapping(self, + def test_success_update_mapping(self, mock_parse_price_account, - mock_parse_prod_account, + mock_parse_prod_account, mock_parse_mapping_account, mock_read_pyth_acct_data): ''' @@ -97,9 +97,9 @@ def test_success_update_mapping(self, @patch.object(PythNetworkClient, 'parse_mapping_account') @patch.object(PythNetworkClient, 'parse_prod_account') @patch.object(PythNetworkClient, 'parse_price_account') - def test_continue_when_failed_prod_account(self, + def test_continue_when_failed_prod_account(self, mock_parse_price_account, - mock_parse_prod_account, + mock_parse_prod_account, mock_parse_mapping_account, mock_read_pyth_acct_data): ''' @@ -115,7 +115,7 @@ def test_continue_when_failed_prod_account(self, with self.assertRaises(Exception): # get_price for 1st product should fail self.assertEqual(self.testee.get_price(self.prod1_symbol), self.prod1_price_data) - + self.assertEqual(self.testee.get_price(self.prod2_symbol), self.prod2_price_data) mock_parse_mapping_account.assert_called_once_with(mapping_account) @@ -126,12 +126,12 @@ def test_continue_when_failed_prod_account(self, self.fail(f"Expected not throws exception but it does: {err}") - @patch.object(SolanaClient, 'get_account_info') + @patch.object(SolanaInteractor, 'get_account_info') def test_forward_exception_when_reading_mapping_account(self, mock_get_account_info): mock_get_account_info.side_effect = Exception('TestException') with self.assertRaises(Exception): self.update_mapping() - mock_get_account_info.assert_called_once_with(mapping_account) + mock_get_account_info.assert_called_once_with(mapping_account, length=0) def test_integration_success_read_price(self): diff --git a/proxy/testing/transactions.py b/proxy/testing/transactions.py index b5eec0e2d..3ff73b0ef 100644 --- a/proxy/testing/transactions.py +++ b/proxy/testing/transactions.py @@ -1,6 +1,8 @@ +from ..environment import EVM_LOADER_ID + token_program = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' -evm_loader_addr = 'eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU' +evm_loader_addr = EVM_LOADER_ID erc20_wrapper = '5H7kvhPD7GECAmf227vTPYTS7SC2PmyuVZaT5zVTx7vb' wrapper_whitelist = [erc20_wrapper] @@ -65,7 +67,7 @@ } ], 'logMessages': [ - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + f'Program {evm_loader_addr} invoke [1]', 'Program 11111111111111111111111111111111 invoke [2]', 'Program 11111111111111111111111111111111 success', 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', @@ -86,9 +88,9 @@ 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 24626 of 485359 compute units', 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', 'Program log: Total memory occupied: 1414', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 40680 of 500000 compute units', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + f'Program {evm_loader_addr} consumed 40680 of 500000 compute units', + f'Program {evm_loader_addr} success', + f'Program {evm_loader_addr} invoke [1]', 'Program 11111111111111111111111111111111 invoke [2]', 'Program 11111111111111111111111111111111 success', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', @@ -96,8 +98,8 @@ 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', 'Program log: Total memory occupied: 1536', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + f'Program {evm_loader_addr} consumed 18381 of 500000 compute units', + f'Program {evm_loader_addr} success', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', 'Program log: Instruction: Transfer', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units',