From aa2d446a1f5a9f766b68f6ab1a75b081e0f6e0d7 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Thu, 17 Feb 2022 19:30:20 +0700 Subject: [PATCH 01/16] Use slot for block number. #580 --- proxy/common_neon/solana_interactor.py | 1 - proxy/common_neon/utils.py | 7 +- proxy/indexer/blocks_db.py | 135 +++++-------------------- proxy/indexer/indexer.py | 57 +---------- proxy/indexer/indexer_db.py | 17 +--- proxy/indexer/logs_db.py | 2 +- proxy/indexer/transactions_db.py | 3 +- proxy/indexer/utils.py | 2 +- proxy/memdb/blocks_db.py | 56 +++++----- proxy/memdb/memdb.py | 10 +- proxy/memdb/transactions_db.py | 4 +- proxy/plugin/solana_rest_api.py | 44 +++----- 12 files changed, 90 insertions(+), 248 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 397548595..f094fb699 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -183,7 +183,6 @@ def get_block_info_list(self, block_slot_list: [int], commitment='confirmed') -> slot=slot, finalized=(commitment == FINALIZED), hash='0x' + base58.b58decode(net_block['blockhash']).hex(), - height=net_block['blockHeight'], parent_hash='0x' + base58.b58decode(net_block['previousBlockhash']).hex(), time=net_block['blockTime'], signs=net_block['signatures'] diff --git a/proxy/common_neon/utils.py b/proxy/common_neon/utils.py index a7f0815ac..a947f0a61 100644 --- a/proxy/common_neon/utils.py +++ b/proxy/common_neon/utils.py @@ -20,10 +20,9 @@ def str_fmt_object(obj): class SolanaBlockInfo: - def __init__(self, slot=None, finalized=False, height=None, hash=None, parent_hash=None, time=None, signs=None): + def __init__(self, slot=None, finalized=False, hash=None, parent_hash=None, time=None, signs=None): self.slot = slot self.finalized = finalized - self.height = height self.hash = hash self.parent_hash = parent_hash self.time = time @@ -62,7 +61,6 @@ def _set_defaults(self): self.return_value = bytes() self.sol_sign = None self.slot = -1 - self.block_height = -1 self.block_hash = '' self.idx = -1 @@ -100,11 +98,10 @@ def _decode_return(self, log: bytes, ix_idx: int, tx: {}): def fill_block_info(self, block: SolanaBlockInfo): self.slot = block.slot - self.block_height = block.height self.block_hash = block.hash for rec in self.logs: rec['blockHash'] = block.hash - rec['blockNumber'] = hex(block.height) + rec['blockNumber'] = hex(block.slot) def decode(self, neon_sign: str, tx: {}, ix_idx=-1) -> NeonTxResultInfo: self._set_defaults() diff --git a/proxy/indexer/blocks_db.py b/proxy/indexer/blocks_db.py index 5c2ed5c67..6c0ef882a 100644 --- a/proxy/indexer/blocks_db.py +++ b/proxy/indexer/blocks_db.py @@ -1,5 +1,5 @@ -import psycopg2 -import psycopg2.extras +from typing import Optional + from ..indexer.utils import BaseDB, DBQuery from ..common_neon.utils import SolanaBlockInfo @@ -7,19 +7,13 @@ class SolanaBlocksDB(BaseDB): def __init__(self): BaseDB.__init__(self) + self._column_lst = ('slot', 'hash') self._full_column_lst = ('slot', 'hash', 'parent_hash', 'blocktime', 'signatures') def _create_table_sql(self) -> str: self._table_name = 'solana_block' return f""" - CREATE TABLE IF NOT EXISTS {self._table_name}_heights ( - slot BIGINT, - height BIGINT, - - UNIQUE(slot), - UNIQUE(height) - ); - CREATE TABLE IF NOT EXISTS {self._table_name}_hashes ( + CREATE TABLE IF NOT EXISTS {self._table_name} ( slot BIGINT, hash CHAR(66), @@ -32,127 +26,48 @@ def _create_table_sql(self) -> str: ); """ - def _fetch_block(self, slot, q: DBQuery) -> SolanaBlockInfo: - e = self._build_expression(q) - - request = f''' - SELECT a.slot, a.height, b.hash - FROM {self._table_name}_heights AS a - LEFT JOIN {self._table_name}_hashes AS b - ON a.slot = b.slot - WHERE {e.where_expr} - {e.order_expr} - LIMIT 1 - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, e.where_keys) - values = cursor.fetchone() - + def _block_from_value(self, slot: Optional[int], values: []) -> SolanaBlockInfo: if not values: return SolanaBlockInfo(slot=slot) return SolanaBlockInfo( finalized=True, slot=values[0], - height=values[1], - hash=values[2], + hash=values[1], ) - def _fetch_full_block(self, slot, q: DBQuery) -> SolanaBlockInfo: - e = self._build_expression(q) - - request = f''' - SELECT a.slot, a.height, b.hash, b.parent_hash, b.blocktime, b.signatures - FROM {self._table_name}_heights AS a - LEFT JOIN {self._table_name}_hashes AS b - ON a.slot = b.slot - WHERE {e.where_expr} - {e.order_expr} - LIMIT 1 - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, e.where_keys) - values = cursor.fetchone() - + def _full_block_from_value(self, slot: Optional[int], values: []) -> SolanaBlockInfo: if not values: return SolanaBlockInfo(slot=slot) return SolanaBlockInfo( finalized=True, slot=values[0], - height=values[1], - hash=values[2], - parent_hash=values[3], - time=values[4], - signs=self.decode_list(values[5]) + hash=values[1], + parent_hash=values[2], + time=values[3], + signs=self.decode_list(values[4]) ) - def get_latest_block(self) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[], order_list=['a.slot DESC']) - return self._fetch_block(None, q) - - def get_latest_block_list(self, limit: int) -> [SolanaBlockInfo]: - request = f''' - SELECT a.slot, a.height, b.hash, b.parent_hash, b.blocktime, b.signatures - FROM {self._table_name}_heights AS a - LEFT JOIN {self._table_name}_hashes AS b - ON a.slot = b.slot - ORDER BY a.slot DESC - LIMIT {limit} - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, []) - values = cursor.fetchall() - - if not values: - return [] - return [ - SolanaBlockInfo( - finalized=True, - slot=value[0], - height=value[1], - hash=value[2], - parent_hash=value[3], - time=value[4], - signs=self.decode_list(value[5]) - ) for value in values - ] - def get_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('a.slot', block_slot)], order_list=[]) - return self._fetch_block(block_slot, q) + q = DBQuery(column_list=self._column_lst, key_list=[('slot', block_slot)], order_list=[]) + return self._block_from_value(block_slot, self._fetchone(q)) def get_full_block_by_slot(self, block_slot) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('a.slot', block_slot)], order_list=[]) - return self._fetch_full_block(block_slot, q) + q = DBQuery(column_list=self._full_column_lst, key_list=[('slot', block_slot)], order_list=[]) + return self._block_from_value(block_slot, self._fetchone(q)) def get_block_by_hash(self, block_hash) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('b.hash', block_hash)], order_list=[]) - return self._fetch_block(None, q) - - def get_block_by_height(self, block_num) -> SolanaBlockInfo: - q = DBQuery(column_list=[], key_list=[('a.height', block_num)], order_list=[]) - return self._fetch_block(None, q) + q = DBQuery(column_list=self._column_lst, key_list=[('block_hash', block_hash)], order_list=[]) + return self._block_from_value(None, self._fetchone(q)) def set_block(self, block: SolanaBlockInfo): - cursor = self._conn.cursor() - cursor.execute(f''' - INSERT INTO {self._table_name}_hashes - ({', '.join(self._full_column_lst)}) - VALUES - ({', '.join(['%s' for _ in range(len(self._full_column_lst))])}) - ON CONFLICT DO NOTHING; - ''', - (block.slot, block.hash, block.parent_hash, block.time, self.encode_list(block.signs))) - - def fill_block_height(self, height, slots): with self._conn.cursor() as cursor: - psycopg2.extras.execute_values(cursor, f""" - INSERT INTO {self._table_name}_heights - (slot, height) - VALUES %s - ON CONFLICT DO NOTHING - """, ((slot, height+idx) for idx, slot in enumerate(slots)), template="(%s, %s)", page_size=1000) + cursor.execute(f''' + INSERT INTO {self._table_name} + ({', '.join(self._full_column_lst)}) + VALUES + ({', '.join(['%s' for _ in range(len(self._full_column_lst))])}) + ON CONFLICT DO NOTHING; + ''', + (block.slot, block.hash, block.parent_hash, block.time, self.encode_list(block.signs))) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index af4bf8c26..e6d716b1b 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -681,45 +681,15 @@ def __init__(self, db: IndexerDB, solana_client: Client): def gather_blocks(self): start_time = time.time() - latest_block = self.db.get_latest_block() - height = -1 - min_height = height - confirmed_blocks_len = 10000 client = self.solana_client._provider - list_opts = {"commitment": FINALIZED} - block_opts = {"commitment": FINALIZED, "transactionDetails": "none", "rewards": False} - while confirmed_blocks_len == 10000: - confirmed_blocks = client.make_request("getBlocksWithLimit", latest_block.slot, confirmed_blocks_len, list_opts)['result'] - confirmed_blocks_len = len(confirmed_blocks) - # No more blocks - if confirmed_blocks_len == 0: - break - - # Intitialize start height - if height == -1: - first_block = client.make_request("getBlock", confirmed_blocks[0], block_opts) - height = first_block['result']['blockHeight'] - - # Validate last block height - latest_block.height = height + confirmed_blocks_len - 1 - latest_block.slot = confirmed_blocks[confirmed_blocks_len - 1] - last_block = client.make_request("getBlock", latest_block.slot, block_opts) - if not last_block['result'] or last_block['result']['blockHeight'] != latest_block.height: - self.warning(f"FAILED last_block_height {latest_block.height} " + - f"last_block_slot {latest_block.slot} " + - f"last_block {last_block}") - break - - # Everything is good - min_height = min(min_height, height) if min_height > 0 else height - self.db.fill_block_height(height, confirmed_blocks) - height = latest_block.height - + opts = {"commitment": FINALIZED} + slot = client.make_request('getSlot', opts)['result'] + self.db.set_latest_block(slot) gather_blocks_ms = (time.time() - start_time) * 1000 # convert this into milliseconds self.counted_logger.print( self.debug, - list_params={"gather_blocks_ms": gather_blocks_ms, "processed_height": latest_block.height - min_height}, - latest_params={"last_block_slot": latest_block.slot} + list_params={"gather_blocks_ms": gather_blocks_ms}, + latest_params={"last_block_slot": slot} ) @@ -734,7 +704,6 @@ def __init__(self, solana_url, evm_loader_id): self.db.set_client(self.solana_client) self.canceller = Canceller() self.blocked_storages = {} - self._init_last_height_slot() self.block_indexer = BlocksIndexer(db=self.db, solana_client=self.solana_client) self.counted_logger = MetricsToLogBuff() @@ -764,22 +733,6 @@ def __init__(self, solana_url, evm_loader_id): } self.def_decoder = DummyIxDecoder('Unknown', self.state) - def _init_last_height_slot(self): - last_known_slot = self.db.get_latest_block().slot - slot = self._init_last_slot('height', last_known_slot) - if last_known_slot == slot: - return - - block_opts = {"commitment": FINALIZED, "transactionDetails": "none", "rewards": False} - client = self.solana_client._provider - block = client.make_request("getBlock", slot, block_opts) - if not block['result']: - self.warning(f"Solana haven't return block information for the slot {slot}") - return - - height = block['result']['blockHeight'] - self.db.fill_block_height(height, [slot]) - def process_functions(self): self.block_indexer.gather_blocks() IndexerBase.process_functions(self) diff --git a/proxy/indexer/indexer_db.py b/proxy/indexer/indexer_db.py index 494ef3f21..061052405 100644 --- a/proxy/indexer/indexer_db.py +++ b/proxy/indexer/indexer_db.py @@ -27,7 +27,7 @@ def __init__(self): self._client = None self._constants = SQLDict(tablename="constants") - for k in ['min_receipt_slot']: + for k in ['min_receipt_slot', 'latest_slot']: if k not in self._constants: self._constants[k] = 0 @@ -61,7 +61,6 @@ def _fill_block_from_net(self, block: SolanaBlockInfo): return block block.hash = '0x' + base58.b58decode(net_block['blockhash']).hex() - block.height = net_block['blockHeight'] block.signs = net_block['signatures'] block.parent_hash = '0x' + base58.b58decode(net_block['previousBlockhash']).hex() block.time = net_block['blockTime'] @@ -104,18 +103,15 @@ def get_full_block_by_slot(self, slot) -> SolanaBlockInfo: return block def get_latest_block(self) -> SolanaBlockInfo: - return self._blocks_db.get_latest_block() + return SolanaBlockInfo(slot=self._constants['latest_slot']) - def get_latest_block_list(self, limit: int) -> [SolanaBlockInfo]: - return self._blocks_db.get_latest_block_list(limit) - - def fill_block_height(self, number, slots): - self._blocks_db.fill_block_height(number, slots) + def set_latest_block(self, slot: int): + self._constants['latest_slot'] = slot def get_min_receipt_slot(self) -> int: return self._constants['min_receipt_slot'] - def set_min_receipt_slot(self, slot): + def set_min_receipt_slot(self, slot: int): self._constants['min_receipt_slot'] = slot def get_logs(self, from_block, to_block, addresses, topics, block_hash): @@ -124,9 +120,6 @@ def get_logs(self, from_block, to_block, addresses, topics, block_hash): def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: return self._blocks_db.get_block_by_hash(block_hash) - def get_block_by_height(self, block_height: int) -> SolanaBlockInfo: - return self._blocks_db.get_block_by_height(block_height) - def get_tx_list_by_sol_sign(self, sol_sign_list: [str]) -> [NeonTxFullInfo]: tx_list = self._txs_db.get_tx_list_by_sol_sign(sol_sign_list) block = None diff --git a/proxy/indexer/logs_db.py b/proxy/indexer/logs_db.py index f4e427399..ac9a4d56e 100644 --- a/proxy/indexer/logs_db.py +++ b/proxy/indexer/logs_db.py @@ -35,7 +35,7 @@ def push_logs(self, logs, block): ( log['address'], block.hash, - block.height, + block.slot, log['transactionHash'], int(log['transactionLogIndex'], 16), topic, diff --git a/proxy/indexer/transactions_db.py b/proxy/indexer/transactions_db.py index 592a92242..d6c2b62cf 100644 --- a/proxy/indexer/transactions_db.py +++ b/proxy/indexer/transactions_db.py @@ -38,7 +38,7 @@ def set_txs(self, neon_sign: str, used_ixs: [SolanaIxSignInfo]): class NeonTxsDB(BaseDB): def __init__(self): BaseDB.__init__(self) - self._column_lst = ('neon_sign', 'from_addr', 'sol_sign', 'slot', 'block_height', 'block_hash', 'idx', + self._column_lst = ('neon_sign', 'from_addr', 'sol_sign', 'slot', 'block_hash', 'idx', 'nonce', 'gas_price', 'gas_limit', 'to_addr', 'contract', 'value', 'calldata', 'v', 'r', 's', 'status', 'gas_used', 'return_value', 'logs') self._sol_neon_txs_db = SolanaNeonTxsDB() @@ -51,7 +51,6 @@ def _create_table_sql(self) -> str: from_addr CHAR(42), sol_sign CHAR(88), slot BIGINT, - block_height BIGINT, block_hash CHAR(66), idx INT, diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index 980c779ae..1ae893992 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -198,7 +198,7 @@ def _build_expression(self, q: DBQuery) -> DBQueryExpression: order_expr='ORDER BY ' + ', '.join(q.order_list) if len(q.order_list) else '', ) - def _fetchone(self, query: DBQuery) -> str: + def _fetchone(self, query: DBQuery) -> []: e = self._build_expression(query) request = f''' diff --git a/proxy/memdb/blocks_db.py b/proxy/memdb/blocks_db.py index f4ed7b748..b85ea5af0 100644 --- a/proxy/memdb/blocks_db.py +++ b/proxy/memdb/blocks_db.py @@ -19,7 +19,7 @@ @logged_group("neon.Proxy") class RequestSolanaBlockList: - BLOCK_CACHE_LIMIT = 100 + BLOCK_CACHE_LIMIT = (32 + 16) BIG_SLOT = 1_000_000_000_000 def __init__(self, blocks_db: MemBlocksDB): @@ -32,8 +32,8 @@ def __init__(self, blocks_db: MemBlocksDB): self.pending_block_revision = 0 self.block_list = [] - self.first_block = SolanaBlockInfo(slot=0, height=0) - self.latest_block = SolanaBlockInfo(slot=0, height=0) + self.first_block = SolanaBlockInfo(slot=0) + self.latest_block = SolanaBlockInfo(slot=0) def execute(self) -> bool: try: @@ -62,13 +62,13 @@ def _get_solana_block_list(self) -> bool: self.latest_block = self.block_list[0] - height = self.latest_block.height - latest_height = self._b.get_latest_block_height() - if latest_height > height: - height = latest_height + slot = self.latest_block.slot + latest_slot = self._b.get_latest_block_slot() + if latest_slot > slot: + slot = latest_slot for block in self.block_list: - block.height = height - height -= 1 + block.slot = slot + slot -= 1 self.first_block = self.block_list[len(self.block_list) - 1] @@ -100,18 +100,18 @@ class MemBlocksDB: _active_block_revision = 0 _block_by_hash = {} - _block_by_height = {} + _block_by_slot = {} # Head and tail of cache - _first_block = SolanaBlockInfo(slot=0, height=0) - _latest_block = SolanaBlockInfo(slot=0, height=0) + _first_block = SolanaBlockInfo(slot=0) + _latest_block = SolanaBlockInfo(slot=0) _latest_db_block_slot = 0 def __init__(self, solana: SolanaInteractor, db: IndexerDB): self.db = db self.solana = solana self._update_block_dicts() - self.debug(f'Init first version of block list {len(self._block_by_height)} ' + + self.debug(f'Init first version of block list {len(self._block_by_slot)} ' + f'first block - {self._first_block}, ' + f'latest block - {self._latest_block}, ' + f'latest db block slot - {self._latest_db_block_slot}') @@ -150,12 +150,12 @@ def _fill_block_dicts(self, request: RequestSolanaBlockList): self._latest_block = request.latest_block self._latest_db_block_slot = request.latest_db_block_slot - self._block_by_height.clear() + self._block_by_slot.clear() self._block_by_hash.clear() for block in request.block_list: self._block_by_hash[block.hash] = block - self._block_by_height[block.height] = block + self._block_by_slot[block.slot] = block def _start_request(self) -> bool: last_time = self._last_time.value @@ -229,20 +229,27 @@ def get_latest_block(self) -> SolanaBlockInfo: self._update_block_dicts() return self._latest_block - def get_latest_block_height(self) -> int: + def get_latest_block_slot(self) -> int: self._update_block_dicts() - return self._latest_block.height + return self._latest_block.slot def get_db_block_slot(self) -> int: self._update_block_dicts() return self._latest_db_block_slot - def get_block_by_height(self, block_height: int) -> SolanaBlockInfo: + def get_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: self._update_block_dicts() - if block_height > self._first_block.height: - return self._block_by_height.get(block_height, SolanaBlockInfo()) + if block_slot > self._first_block.slot: + return self._block_by_slot.get(block_slot, SolanaBlockInfo()) - return self.db.get_block_by_height(block_height) + return self.db.get_block_by_slot(block_slot) + + def get_full_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: + self._update_block_dicts() + if block_slot > self._first_block.slot: + return self._block_by_slot.get(block_slot, SolanaBlockInfo()) + + return self.db.get_full_block_by_slot(block_slot) def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: self._update_block_dicts() @@ -257,14 +264,9 @@ def _generate_fake_block(self, neon_res: NeonTxResultInfo) -> SolanaBlockInfo: if data: block = pickle.loads(data) else: - latest_block = self._latest_block - block_height = (latest_block.height or neon_res.slot) + 1 - block_time = (latest_block.time or 1) - block = SolanaBlockInfo( slot=neon_res.slot, - height=block_height, - time=block_time, + time=self._latest_block.time, hash='0x' + os.urandom(32).hex(), parent_hash='0x' + os.urandom(32).hex(), ) diff --git a/proxy/memdb/memdb.py b/proxy/memdb/memdb.py index 19b481d72..9c93c5daa 100644 --- a/proxy/memdb/memdb.py +++ b/proxy/memdb/memdb.py @@ -31,14 +31,14 @@ def _before_slot(self) -> int: def get_latest_block(self) -> SolanaBlockInfo: return self._blocks_db.get_latest_block() - def get_latest_block_height(self) -> int: - return self._blocks_db.get_latest_block_height() + def get_latest_block_slot(self) -> int: + return self._blocks_db.get_latest_block_slot() - def get_block_by_height(self, block_height: int) -> SolanaBlockInfo: - return self._blocks_db.get_block_by_height(block_height) + def get_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: + return self._blocks_db.get_block_by_slot(block_slot) def get_full_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: - return self._db.get_full_block_by_slot(block_slot) + return self._blocks_db.get_full_block_by_slot(block_slot) def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: return self._blocks_db.get_block_by_hash(block_hash) diff --git a/proxy/memdb/transactions_db.py b/proxy/memdb/transactions_db.py index 474fba1e6..de042a423 100644 --- a/proxy/memdb/transactions_db.py +++ b/proxy/memdb/transactions_db.py @@ -86,9 +86,9 @@ def _has_topics(src_topics, dst_topics): with self._tx_slot.get_lock(): for data in self._tx_by_neon_sign.values(): tx = pickle.loads(data) - if from_block and tx.neon_res.block_height < from_block: + if from_block and tx.neon_res.slot < from_block: continue - if to_block and tx.neon_res.block_height > to_block: + if to_block and tx.neon_res.slot > to_block: continue if block_hash and tx.neon_res.block_hash != block_hash: continue diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 732debb75..c8533e040 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -116,17 +116,16 @@ def process_block_tag(self, tag): elif tag in ('earliest', 'pending'): raise Exception("Invalid tag {}".format(tag)) elif isinstance(tag, str): - block = SolanaBlockInfo(height=int(tag, 16)) + block = SolanaBlockInfo(slot=int(tag, 16)) elif isinstance(tag, int): - block = SolanaBlockInfo(height=tag) + block = SolanaBlockInfo(slot=tag) else: raise Exception(f'Failed to parse block tag: {tag}') return block def eth_blockNumber(self): - height = self._db.get_latest_block_height() - self.debug("eth_blockNumber %s", hex(height)) - return hex(height) + slot = self._db.get_latest_block_slot() + return hex(slot) def eth_getBalance(self, account, tag): """account - address to check for balance. @@ -152,9 +151,9 @@ def to_list(items): block_hash = None if 'fromBlock' in obj and obj['fromBlock'] != '0': - from_block = self.process_block_tag(obj['fromBlock']).height + from_block = self.process_block_tag(obj['fromBlock']).slot if 'toBlock' in obj and obj['toBlock'] != 'latest': - to_block = self.process_block_tag(obj['toBlock']).height + to_block = self.process_block_tag(obj['toBlock']).slot if 'address' in obj: addresses = to_list(obj['address']) if 'topics' in obj: @@ -227,10 +226,6 @@ def eth_getBlockByHash(self, block_hash, full): self.debug("Not found block by hash %s", block_hash) return None ret = self.getBlockBySlot(block, full, False) - if ret is not None: - self.debug("eth_getBlockByHash: %s", json.dumps(ret, indent=3)) - else: - self.debug("Not found block by hash %s", block_hash) return ret def eth_getBlockByNumber(self, tag, full): @@ -240,15 +235,9 @@ def eth_getBlockByNumber(self, tag, full): """ block = self.process_block_tag(tag) if block.slot is None: - block = self._db.get_block_by_height(block.height) - if block.slot is None: - self.debug("Not found block by number %s", tag) + self.debug(f"Not found block by number {tag}") return None ret = self.getBlockBySlot(block, full, tag == 'latest') - if ret is not None: - self.debug("eth_getBlockByNumber: %s", json.dumps(ret, indent=3)) - else: - self.debug("Not found block by number %s", tag) return ret def eth_call(self, obj, tag): @@ -291,7 +280,7 @@ def _getTransactionReceipt(self, tx): "transactionHash": tx.neon_tx.sign, "transactionIndex": hex(0), "blockHash": tx.neon_res.block_hash, - "blockNumber": hex(tx.neon_res.block_height), + "blockNumber": hex(tx.neon_res.slot), "from": tx.neon_tx.addr, "to": tx.neon_tx.to_addr, "gasUsed": tx.neon_res.gas_used, @@ -302,7 +291,6 @@ def _getTransactionReceipt(self, tx): "logsBloom": "0x"+'0'*512 } - self.debug('RESULT: %s', json.dumps(result, indent=3)) return result def eth_getTransactionReceipt(self, trxId): @@ -321,7 +309,7 @@ def _getTransaction(self, tx): result = { "blockHash": r.block_hash, - "blockNumber": hex(r.block_height), + "blockNumber": hex(r.slot), "hash": t.sign, "transactionIndex": hex(0), "from": t.addr, @@ -336,7 +324,6 @@ def _getTransaction(self, tx): "s": t.s, } - self.debug("_getTransaction: %s", json.dumps(result, indent=3)) return result def eth_getTransactionByHash(self, trxId): @@ -354,15 +341,12 @@ def eth_getCode(self, account, _tag): return self._db.get_contract_code(account) def eth_sendTransaction(self, trx): - self.debug("eth_sendTransaction") - self.debug("eth_sendTransaction: type(trx):%s", type(trx)) - self.debug("eth_sendTransaction: str(trx):%s", str(trx)) - self.debug("eth_sendTransaction: trx=%s", json.dumps(trx, cls=JsonEncoder, indent=3)) + self.debug(f"eth_sendTransaction type(trx): {type(trx)}, str(trx): {str(trx)}") raise RuntimeError("eth_sendTransaction is not supported. please use eth_sendRawTransaction") def eth_sendRawTransaction(self, rawTrx): trx = EthTrx.fromString(bytearray.fromhex(rawTrx[2:])) - self.debug(f"{json.dumps(trx.as_dict(), cls=JsonEncoder, indent=3)}") + self.debug(f"{json.dumps(trx.as_dict(), cls=JsonEncoder, sort_keys=True)}") min_gas_price = self.gas_price_calculator.get_min_gas_price() if trx.gasPrice < min_gas_price: @@ -460,7 +444,7 @@ def process_request(self, request): except Exception as err: err_tb = "".join(traceback.format_tb(err.__traceback__)) self.error('Exception on process request. ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') response['error'] = {'code': -32000, 'message': str(err)} return response @@ -508,8 +492,8 @@ def handle_request_impl(self, request: HttpParser) -> None: resp_time_ms = (time.time() - start_time)*1000 # convert this into milliseconds self.info('handle_request >>> %s 0x%0x %s %s resp_time_ms= %s', threading.get_ident(), id(self.model), json.dumps(response), - request.get('method', '---'), - resp_time_ms) + request.get('method', '---'), + resp_time_ms) self.client.queue(memoryview(build_http_response( httpStatusCodes.OK, body=json.dumps(response).encode('utf8'), From 62c4368c4c6657b408958fe78756182743dbe137 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sat, 19 Feb 2022 18:47:16 +0700 Subject: [PATCH 02/16] All connections to Solana by SolanaInteractor --- proxy/__main__.py | 6 +- proxy/common_neon/account_whitelist.py | 54 +++--- proxy/common_neon/costs.py | 2 +- proxy/common_neon/permission_token.py | 31 +--- proxy/common_neon/solana_interactor.py | 215 +++++++++++++++++----- proxy/common_neon/transaction_sender.py | 9 +- proxy/environment.py | 2 - proxy/indexer/accounts_db.py | 2 +- proxy/indexer/airdropper.py | 64 +++---- proxy/indexer/base_db.py | 75 ++++++++ proxy/indexer/blocks_db.py | 2 +- proxy/indexer/canceller.py | 6 +- proxy/indexer/indexer.py | 37 ++-- proxy/indexer/indexer_base.py | 29 +-- proxy/indexer/indexer_db.py | 39 ++-- proxy/indexer/logs_db.py | 2 +- proxy/indexer/pythnetwork.py | 48 ++--- proxy/indexer/sql_dict.py | 2 +- proxy/indexer/transactions_db.py | 3 +- proxy/indexer/trx_receipts_storage.py | 2 +- proxy/indexer/utils.py | 139 +++----------- proxy/memdb/memdb.py | 10 +- proxy/plugin/gas_price_calculator.py | 7 +- proxy/plugin/solana_rest_api.py | 16 +- proxy/plugin/solana_rest_api_tools.py | 41 +---- proxy/testing/test_account_whitelist.py | 94 ++++------ proxy/testing/test_airdropper.py | 1 - proxy/testing/test_neon_tx_sender.py | 9 +- proxy/testing/test_permission_token.py | 21 ++- proxy/testing/test_pyth_network_client.py | 20 +- 30 files changed, 474 insertions(+), 514 deletions(-) create mode 100644 proxy/indexer/base_db.py diff --git a/proxy/__main__.py b/proxy/__main__.py index 019f9e733..41ad6dc43 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -14,7 +14,6 @@ import os from .indexer.airdropper import run_airdropper from .indexer.indexer import run_indexer -from solana.rpc.api import Client if __name__ == '__main__': airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True'] @@ -22,7 +21,6 @@ if airdropper_mode: print("Will run in airdropper mode") solana_url = os.environ['SOLANA_URL'] - evm_loader_id = os.environ['EVM_LOADER'] pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT']) faucet_url = os.environ['FAUCET_URL'] wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'] @@ -34,7 +32,6 @@ max_conf = float(os.environ.get('MAX_CONFIDENCE_INTERVAL', 0.02)) run_airdropper(solana_url, - evm_loader_id, pyth_mapping_account, faucet_url, wrapper_whitelist, @@ -45,8 +42,7 @@ print("Will run in indexer mode") solana_url = os.environ['SOLANA_URL'] - evm_loader_id = os.environ['EVM_LOADER'] - run_indexer(solana_url, evm_loader_id) + run_indexer(solana_url) else: entry_point() diff --git a/proxy/common_neon/account_whitelist.py b/proxy/common_neon/account_whitelist.py index 1c2d11bd4..2f92e6ecd 100644 --- a/proxy/common_neon/account_whitelist.py +++ b/proxy/common_neon/account_whitelist.py @@ -1,14 +1,13 @@ import traceback from datetime import datetime -import time -from proxy.environment import ELF_PARAMS, GET_WHITE_LIST_BALANCE_MAX_RETRIES, GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S +from proxy.environment import ELF_PARAMS from proxy.common_neon.permission_token import PermissionToken from solana.publickey import PublicKey -from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount from typing import Union from proxy.common_neon.address import EthereumAddress from logged_groups import logged_group +from ..common_neon.solana_interactor import SolanaInteractor NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE = int(ELF_PARAMS.get("NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE", 0)) NEON_MINIMAL_CONTRACT_ALLOWANCE_BALANCE = int(ELF_PARAMS.get("NEON_MINIMAL_CONTRACT_ALLOWANCE_BALANCE", 0)) @@ -18,10 +17,7 @@ @logged_group("neon.AccountWhitelist") class AccountWhitelist: - def __init__(self, solana: SolanaClient, payer: SolanaAccount, permission_update_int: int): - self.info(f'GET_WHITE_LIST_BALANCE_MAX_RETRIES={GET_WHITE_LIST_BALANCE_MAX_RETRIES}') - self.info(f'GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S={GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S} seconds') - self.info(f'permission_update_int={permission_update_int}') + def __init__(self, solana: SolanaInteractor, payer: SolanaAccount, permission_update_int: int): self.solana = solana self.account_cache = {} self.permission_update_int = permission_update_int @@ -43,9 +39,15 @@ def __init__(self, solana: SolanaClient, payer: SolanaAccount, permission_update PublicKey(DENIAL_TOKEN_ADDR), payer) - def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]): - allowance_balance = self.allowance_token.get_balance(ether_addr) - denial_balance = self.denial_token.get_balance(ether_addr) + def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]) -> int: + token_list = [ + self.allowance_token.get_token_account_address(ether_addr), + self.denial_token.get_token_account_address(ether_addr) + ] + + balance_list = self.solana.get_token_account_balance_list(token_list) + allowance_balance = balance_list[0] + denial_balance = balance_list[1] return allowance_balance - denial_balance def grant_permissions(self, ether_addr: Union[str, EthereumAddress], min_balance: int): @@ -106,27 +108,17 @@ def has_permission(self, ether_addr: Union[str, EthereumAddress], min_balance: i if diff < self.permission_update_int: return cached['diff'] >= min_balance - num_retries = GET_WHITE_LIST_BALANCE_MAX_RETRIES - - while True: - try: - diff = self.read_balance_diff(ether_addr) - self.account_cache[ether_addr] = { - 'last_update': current_time, - 'diff': diff - } - return diff >= min_balance - except Exception as err: - err_tb = "".join(traceback.format_tb(err.__traceback__)) - self.error(f'Failed to read permissions for {ether_addr}: ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - num_retries -= 1 - if num_retries == 0: - # This error should be forwarded to client - raise RuntimeError('Failed to read account permissions. Try to repeat later') - - self.info(f'Will retry getting whitelist balance after {GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S} seconds') - time.sleep(GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S) + try: + diff = self.read_balance_diff(ether_addr) + self.account_cache[ether_addr] = { + 'last_update': current_time, + 'diff': diff + } + return diff >= min_balance + except Exception as err: + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.error(f'Failed to read permissions for {ether_addr}: ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') def has_client_permission(self, ether_addr: Union[str, EthereumAddress]): return self.has_permission(ether_addr, NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE) diff --git a/proxy/common_neon/costs.py b/proxy/common_neon/costs.py index b3a17ed4a..17ecf518c 100644 --- a/proxy/common_neon/costs.py +++ b/proxy/common_neon/costs.py @@ -1,7 +1,7 @@ import base58 from ..environment import EVM_LOADER_ID -from ..indexer.utils import BaseDB +from ..indexer.base_db import BaseDB class SQLCost(BaseDB): diff --git a/proxy/common_neon/permission_token.py b/proxy/common_neon/permission_token.py index fdec58516..00e36e2e8 100644 --- a/proxy/common_neon/permission_token.py +++ b/proxy/common_neon/permission_token.py @@ -1,59 +1,46 @@ -from lib2to3.pgen2 import token -from spl.token.client import Token as SplToken from solana.publickey import PublicKey -from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount -from spl.token.constants import TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address from proxy.common_neon.address import EthereumAddress, ether2program from typing import Union -from solana.rpc.commitment import Confirmed from solana.transaction import Transaction -from solana.rpc.types import TxOpts import spl.token.instructions as spl_token -from proxy.common_neon.utils import get_from_dict +from proxy.common_neon.solana_interactor import SolanaInteractor from decimal import Decimal import os class PermissionToken: def __init__(self, - solana: SolanaClient, + solana: SolanaInteractor, token_mint: PublicKey, payer: SolanaAccount): self.solana = solana self.token_mint = token_mint self.payer = payer - self.token = SplToken(self.solana, - self.token_mint, - TOKEN_PROGRAM_ID, - payer) def get_token_account_address(self, ether_addr: Union[str, EthereumAddress]): sol_addr = PublicKey(ether2program(ether_addr)[0]) - return get_associated_token_address(sol_addr, self.token.pubkey) + return get_associated_token_address(sol_addr, self.token_mint) def get_balance(self, ether_addr: Union[str, EthereumAddress]): token_account = self.get_token_account_address(ether_addr) - result = self.token.get_balance(token_account).get('result', None) - if result is None: - return 0 - return int(result['value']['amount']) + return self.solana.get_token_account_balance(token_account) def create_account_if_needed(self, ether_addr: Union[str, EthereumAddress]): token_account = self.get_token_account_address(ether_addr) - response = self.solana.get_account_info(token_account, Confirmed) - if get_from_dict(response, 'result', 'value') is not None: + info = self.solana.get_account_info(token_account) + if info is not None: return token_account txn = Transaction() create_txn = spl_token.create_associated_token_account( payer=self.payer.public_key(), owner=PublicKey(ether2program(ether_addr)[0]), - mint=self.token.pubkey + mint=self.token_mint ) txn.add(create_txn) - self.token._conn.send_transaction(txn, self.payer, opts=TxOpts(skip_preflight=True, skip_confirmation=False)) + self.solana.send_multiple_transactions(self.payer, [txn], skip_preflight=True) return token_account def mint_to(self, @@ -61,6 +48,6 @@ def mint_to(self, ether_addr: Union[str, EthereumAddress], mint_authority_file: str): token_account = self.create_account_if_needed(ether_addr) - mint_command = f'spl-token mint "{str(self.token.pubkey)}" {Decimal(amount) * pow(Decimal(10), -9)}' + mint_command = f'spl-token mint "{str(self.token_mint)}" {Decimal(amount) * pow(Decimal(10), -9)}' mint_command += f' --owner {mint_authority_file} -- "{str(token_account)}"' os.system(mint_command) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index f094fb699..e82ea8327 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -10,11 +10,12 @@ from solana.publickey import PublicKey from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount -from solana.rpc.commitment import Confirmed from solana.rpc.types import RPCResponse from solana.transaction import Transaction from itertools import zip_longest from logged_groups import logged_group +from typing import Dict, Union +from base58 import b58decode, b58encode from .costs import update_transaction_cost from .utils import get_from_dict, SolanaBlockInfo @@ -42,16 +43,38 @@ class SendResult(NamedTuple): @logged_group("neon.Proxy") class SolanaInteractor: - def __init__(self, client: SolanaClient) -> None: - self.client = client + def __init__(self, solana_url: str) -> None: + self._client = SolanaClient(solana_url)._provider self._fuzzing_hash_cycle = False + def _make_request(self, request) -> RPCResponse: + """This method is used to make retries to send request to Solana""" + + headers = { + "Content-Type": "application/json" + } + client = self._client + raw_response = client.session.post(client.endpoint_uri, headers=headers, json=request) + raw_response.raise_for_status() + return raw_response + + def _send_rpc_request(self, method: str, *params: Any) -> RPCResponse: + request_id = next(self._client._request_counter) + 1 + + request = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params + } + raw_response = self._make_request(request) + return cast(RPCResponse, raw_response.json()) + def _send_rpc_batch_request(self, method: str, params_list: List[Any]) -> List[RPCResponse]: full_request_data = [] full_response_data = [] request_data = [] - client = self.client._provider - headers = {"Content-Type": "application/json"} + client = self._client for params in params_list: request_id = next(client._request_counter) + 1 @@ -61,10 +84,8 @@ def _send_rpc_batch_request(self, method: str, params_list: List[Any]) -> List[R # Protection from big payload if len(request_data) == 30 or len(full_request_data) == len(params_list): - response = client.session.post(client.endpoint_uri, headers=headers, json=request_data) - response.raise_for_status() - - response_data = cast(List[RPCResponse], response.json()) + raw_response = self._make_request(request_data) + response_data = cast(List[RPCResponse], raw_response.json()) full_response_data += response_data request_data.clear() @@ -79,22 +100,42 @@ def _send_rpc_batch_request(self, method: str, params_list: List[Any]) -> List[R return full_response_data - def get_account_info(self, storage_account) -> Optional[AccountInfo]: + def get_signatures_for_address(self, before, until, commitment='confirmed'): + opts: Dict[str, Union[int, str]] = {} + if until is not None: + opts["until"] = until + if before is not None: + opts["before"] = before + opts["commitment"] = commitment + return self._send_rpc_request("getSignaturesForAddress", EVM_LOADER_ID, opts) + + def get_confirmed_transaction(self, sol_sign: str, encoding: str = "json"): + return self._send_rpc_request("getConfirmedTransaction", sol_sign, encoding) + + def get_slot(self, commitment='confirmed') -> RPCResponse: + opts = { + 'commitment': commitment + } + return self._send_rpc_request('getSlot', opts) + + def get_account_info(self, pubkey: PublicKey, length=256, commitment='confirmed') -> Optional[AccountInfo]: opts = { "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 256, - } + "commitment": commitment, } - result = self.client._provider.make_request("getAccountInfo", str(storage_account), opts) - self.debug(f"\n{json.dumps(result, indent=4, sort_keys=True)}") + if length != 0: + opts['dataSlice'] = { + 'offset': 0, + 'length': length + } + + result = self._send_rpc_request('getAccountInfo', str(pubkey), opts) + self.debug(f"{json.dumps(result, sort_keys=True)}") info = result['result']['value'] if info is None: - self.debug(f"Can't get information about {storage_account}") + self.debug(f"Can't get information about {str(pubkey)}") return None data = base64.b64decode(info['data'][0]) @@ -105,24 +146,27 @@ def get_account_info(self, storage_account) -> Optional[AccountInfo]: return AccountInfo(account_tag, lamports, owner, data) - def get_multiple_accounts_info(self, accounts: [PublicKey]) -> [AccountInfo]: - options = { + def get_multiple_accounts_info(self, accounts: [PublicKey], length=256, commitment='confirmed') -> [AccountInfo]: + opts = { "encoding": "base64", - "commitment": "confirmed", - "dataSlice": { - "offset": 0, - "length": 16 - } + "commitment": commitment, } - result = self.client._provider.make_request("getMultipleAccounts", [str(a) for a in accounts], options) - self.debug(f"\n{json.dumps(result, indent=4, sort_keys=True)}") + + if length != 0: + opts['dataSlice'] = { + 'offset': 0, + 'length': length + } + + result = self._send_rpc_request("getMultipleAccounts", [str(a) for a in accounts], opts) + self.debug(f"{json.dumps(result, sort_keys=True)}") if result['result']['value'] is None: self.debug(f"Can't get information about {accounts}") return [] accounts_info = [] - for info in result['result']['value']: + for pubkey, info in zip(accounts, result['result']['value']): if info is None: accounts_info.append(None) else: @@ -132,8 +176,38 @@ def get_multiple_accounts_info(self, accounts: [PublicKey]) -> [AccountInfo]: return accounts_info - def get_sol_balance(self, account): - return self.client.get_balance(account, commitment=Confirmed)['result']['value'] + def get_sol_balance(self, account, commitment='confirmed'): + opts = { + "commitment": commitment + } + return self._send_rpc_request('getBalance', str(account), opts)['result']['value'] + + def get_token_account_balance(self, pubkey: Union[str, PublicKey], commitment='confirmed') -> int: + opts = { + "commitment": commitment + } + response = self._send_rpc_request("getTokenAccountBalance", str(pubkey), opts) + result = response.get('result', None) + if result is None: + return 0 + return int(result['value']['amount']) + + def get_token_account_balance_list(self, pubkey_list: [Union[str, PublicKey]], commitment: object = 'confirmed') -> [int]: + opts = { + "commitment": commitment + } + request_list = [] + for pubkey in pubkey_list: + request_list.append((str(pubkey), opts)) + + balance_list = [] + response_list = self._send_rpc_batch_request('getTokenAccountBalance', request_list) + for response in response_list: + result = response.get('result', None) + balance = int(result['value']['amount']) if result else 0 + balance_list.append(balance) + + return balance_list def get_neon_account_info(self, eth_account: EthereumAddress) -> Optional[NeonAccountInfo]: account_sol, nonce = ether2program(eth_account) @@ -145,8 +219,10 @@ def get_neon_account_info(self, eth_account: EthereumAddress) -> Optional[NeonAc f"{len(info.data)} < {ACCOUNT_INFO_LAYOUT.sizeof()}") return NeonAccountInfo.frombytes(info.data) - def get_multiple_rent_exempt_balances_for_size(self, size_list: [int]) -> [int]: - opts = {"commitment": "confirmed"} + def get_multiple_rent_exempt_balances_for_size(self, size_list: [int], commitment='confirmed') -> [int]: + opts = { + "commitment": commitment + } request_list = [(size, opts) for size in size_list] response_list = self._send_rpc_batch_request("getMinimumBalanceForRentExemption", request_list) return [r['result'] for r in response_list] @@ -156,7 +232,29 @@ def get_block_slot_list(self, last_block_slot, limit: int, commitment='confirmed "commitment": commitment, "enconding": "json", } - return self.client._provider.make_request("getBlocksWithLimit", last_block_slot, limit, opts)['result'] + return self._send_rpc_request("getBlocksWithLimit", last_block_slot, limit, opts)['result'] + + def get_block_info(self, slot: int, commitment='confirmed') -> [SolanaBlockInfo]: + opts = { + "commitment": commitment, + "encoding": "json", + "transactionDetails": "signatures", + "rewards": False + } + + response = self._send_rpc_request('getBlock', slot, opts) + net_block = response.get('result', None) + if not net_block: + return SolanaBlockInfo(slot=slot) + + return SolanaBlockInfo( + slot=slot, + finalized=(commitment == FINALIZED), + hash='0x' + base58.b58decode(net_block['blockhash']).hex(), + parent_hash='0x' + base58.b58decode(net_block['previousBlockhash']).hex(), + time=net_block['blockTime'], + signs=net_block['signatures'] + ) def get_block_info_list(self, block_slot_list: [int], commitment='confirmed') -> [SolanaBlockInfo]: block_list = [] @@ -190,14 +288,20 @@ def get_block_info_list(self, block_slot_list: [int], commitment='confirmed') -> block_list.append(block) return block_list - def get_recent_blockslot(self, commitment=Confirmed) -> int: - blockhash_resp = self.client.get_recent_blockhash(commitment=commitment) + def get_recent_blockslot(self, commitment='confirmed') -> int: + opts = { + 'commitment': commitment + } + blockhash_resp = self._send_rpc_request('getRecentBlockhash', opts) if not blockhash_resp["result"]: raise RuntimeError("failed to get recent blockhash") return blockhash_resp['result']['context']['slot'] - def get_recent_blockhash(self, commitment=Confirmed) -> Blockhash: - blockhash_resp = self.client.get_recent_blockhash(commitment=commitment) + def get_recent_blockhash(self, commitment='confirmed') -> Blockhash: + opts = { + 'commitment': commitment + } + blockhash_resp = self._send_rpc_request('getRecentBlockhash', opts) if not blockhash_resp["result"]: raise RuntimeError("failed to get recent blockhash") blockhash = blockhash_resp["result"]["value"]["blockhash"] @@ -224,7 +328,7 @@ def _fuzzing_transactions(self, signer: SolanaAccount, tx_list, tx_opts, request "rewards": False } slot = max(slot - 500, 10) - block = self.client._provider.make_request("getBlock", slot, block_opts) + block = self._send_rpc_request("getBlock", slot, block_opts) fuzzing_blockhash = Blockhash(block['result']['blockhash']) self.debug(f"fuzzing block {fuzzing_blockhash} for slot {slot}") @@ -238,11 +342,12 @@ def _fuzzing_transactions(self, signer: SolanaAccount, tx_list, tx_opts, request request_list[idx] = (base64_tx, tx_opts) return request_list - def _send_multiple_transactions_unconfirmed(self, signer: SolanaAccount, tx_list: [Transaction]) -> [str]: + def _send_multiple_transactions(self, signer: SolanaAccount, tx_list: [Transaction], + skip_preflight: bool, preflight_commitment: str) -> [str]: opts = { - "skipPreflight": SKIP_PREFLIGHT, + "skipPreflight": skip_preflight, "encoding": "base64", - "preflightCommitment": "confirmed" + "preflightCommitment": preflight_commitment } blockhash = None @@ -263,8 +368,10 @@ def _send_multiple_transactions_unconfirmed(self, signer: SolanaAccount, tx_list response_list = self._send_rpc_batch_request('sendTransaction', request_list) return [SendResult(result=r.get('result'), error=r.get('error')) for r in response_list] - def send_multiple_transactions(self, signer, tx_list, eth_tx, reason, waiter=None) -> [{}]: - send_result_list = self._send_multiple_transactions_unconfirmed(signer, tx_list) + def send_multiple_transactions(self, signer, tx_list, + eth_tx=None, reason=None, waiter=None, + skip_preflight=SKIP_PREFLIGHT, preflight_commitment='confirmed') -> [{}]: + send_result_list = self._send_multiple_transactions(signer, tx_list, skip_preflight, preflight_commitment) # Filter good transactions and wait the confirmations for them sign_list = [s.result for s in send_result_list if s.result] self._confirm_multiple_transactions(sign_list, waiter) @@ -274,11 +381,12 @@ def send_multiple_transactions(self, signer, tx_list, eth_tx, reason, waiter=Non receipt_list = [] for s in send_result_list: if s.error: + self.debug(f'Got error on preflight check of transaction: {s.error}') receipt_list.append(s.error) else: receipt_list.append(confirmed_list.pop(0)) - if WRITE_TRANSACTION_COST_IN_DB: + if eth_tx and reason and WRITE_TRANSACTION_COST_IN_DB: for receipt in receipt_list: update_transaction_cost(receipt, eth_tx, reason) @@ -303,17 +411,28 @@ def get_measurements(self, reason, eth_tx, receipt): def _confirm_multiple_transactions(self, sign_list: [str], waiter=None): """Confirm a transaction.""" if not len(sign_list): - self.debug(f'Got confirmed status for transactions: {sign_list}') + self.debug('No confirmations, because transaction list is empty') return + base58_sign_list: List[str] = [] + for sign in sign_list: + if isinstance(sign, str): + base58_sign_list.append(b58encode(b58decode(sign)).decode("utf-8")) + else: + base58_sign_list.append(b58encode(sign).decode("utf-8")) + + opts = { + "searchTransactionHistory": False + } + elapsed_time = 0 while elapsed_time < CONFIRM_TIMEOUT: if elapsed_time > 0: time.sleep(CONFIRMATION_CHECK_DELAY) elapsed_time += CONFIRMATION_CHECK_DELAY - response = self.client.get_signature_statuses(sign_list) - result = response['result'] + response = self._send_rpc_request("getSignatureStatuses", base58_sign_list, opts) + result = response.get('result', None) if not result: continue @@ -490,7 +609,7 @@ def check_if_accounts_blocked(receipt, *, logger): logs = get_logs_from_receipt(receipt) if logs is None: logger.error("Can't get logs") - logger.info("Failed result: %s"%json.dumps(receipt, indent=3)) + logger.info(f"Failed result: {json.dumps(receipt, indent=3)}") return False ro_blocked = "trying to execute transaction on ro locked account" diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 38933b06b..0ca9cb94e 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -16,7 +16,6 @@ from solana.transaction import AccountMeta, Transaction, PublicKey from solana.blockhash import Blockhash from solana.account import Account as SolanaAccount -from solana.rpc.api import Client as SolanaClient from ..common_neon.address import accountWithSeed, getTokenAddr, EthereumAddress from ..common_neon.errors import EthereumError @@ -383,12 +382,12 @@ def EthMeta(pubkey, is_writable) -> AccountMeta: @logged_group("neon.Proxy") class NeonTxSender: - def __init__(self, db: MemDB, client: SolanaClient, eth_tx: EthTx, steps: int): + def __init__(self, db: MemDB, solana: SolanaInteractor, eth_tx: EthTx, steps: int): self._db = db self.eth_tx = eth_tx self.neon_sign = '0x' + eth_tx.hash_signed().hex() self.steps = steps - self.solana = SolanaInteractor(client) + self.solana = solana self._resource_list = OperatorResourceList(self) self.resource = None self.operator_key = None @@ -443,7 +442,7 @@ def _validate_pend_tx(self): self._pend_tx_into_db(self.solana.get_recent_blockslot()) def _validate_whitelist(self): - whitelist = AccountWhitelist(self.solana.client, ACCOUNT_PERMISSION_UPDATE_INT, self.resource.signer) + whitelist = AccountWhitelist(self.solana, ACCOUNT_PERMISSION_UPDATE_INT, self.resource.signer) if not whitelist.has_client_permission(self.eth_sender[2:]): self.warning(f'Sender account {self.eth_sender} is not allowed to execute transactions') raise Exception(f'Sender account {self.eth_sender} is not allowed to execute transactions') @@ -540,7 +539,7 @@ def _call_emulated(self): self.debug(f'destination address {self.to_address}') self._emulator_json = call_emulated(dst, src, self.eth_tx.callData.hex(), hex(self.eth_tx.value)) - self.debug(f'emulator returns: {json.dumps(self._emulator_json, indent=3)}') + self.debug(f'emulator returns: {json.dumps(self._emulator_json, sort_keys=True)}') self.steps_emulated = self._emulator_json['steps_executed'] diff --git a/proxy/environment.py b/proxy/environment.py index e5cf73942..1ff62f24e 100644 --- a/proxy/environment.py +++ b/proxy/environment.py @@ -42,8 +42,6 @@ SOL_PRICE_UPDATE_INTERVAL = int(os.environ.get("SOL_PRICE_UPDATE_INTERVAL", 60)) GET_SOL_PRICE_MAX_RETRIES = int(os.environ.get("GET_SOL_PRICE_MAX_RETRIES", 3)) GET_SOL_PRICE_RETRY_INTERVAL = int(os.environ.get("GET_SOL_PRICE_RETRY_INTERVAL", 1)) -GET_WHITE_LIST_BALANCE_MAX_RETRIES = int(os.environ.get("GET_WHITE_LIST_BALANCE_MAX_RETRIES", 3)) -GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S = int(os.environ.get("GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S", 1)) INDEXER_LOG_SKIP_COUNT = int(os.environ.get("INDEXER_LOG_SKIP_COUNT", 10)) MIN_OPERATOR_BALANCE_TO_WARN = max(int(os.environ.get("MIN_OPERATOR_BALANCE_TO_WARN", 9000000000)), 9000000000) MIN_OPERATOR_BALANCE_TO_ERR = max(int(os.environ.get("MIN_OPERATOR_BALANCE_TO_ERR", 1000000000)), 1000000000) diff --git a/proxy/indexer/accounts_db.py b/proxy/indexer/accounts_db.py index a02d937eb..938791964 100644 --- a/proxy/indexer/accounts_db.py +++ b/proxy/indexer/accounts_db.py @@ -1,4 +1,4 @@ -from ..indexer.utils import BaseDB, DBQuery +from ..indexer.base_db import BaseDB, DBQuery from ..common_neon.utils import str_fmt_object diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py index bce74c661..e5c3d9a78 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/indexer/airdropper.py @@ -1,23 +1,18 @@ -from calendar import c from solana.publickey import PublicKey from proxy.indexer.indexer_base import IndexerBase from proxy.indexer.pythnetwork import PythNetworkClient -from proxy.indexer.utils import BaseDB -from solana.rpc.api import Client as SolanaClient +from proxy.indexer.base_db import BaseDB +from proxy.indexer.utils import check_error +from proxy.indexer.sql_dict import SQLDict import requests import base58 +import traceback from datetime import datetime from decimal import Decimal -import os from logged_groups import logged_group -from ..environment import NEON_PRICE_USD +from ..environment import NEON_PRICE_USD, EVM_LOADER_ID +from ..common_neon.solana_interactor import SolanaInteractor -try: - from utils import check_error - from sql_dict import SQLDict -except ImportError: - from .utils import check_error - from .sql_dict import SQLDict ACCOUNT_CREATION_PRICE_SOL = Decimal('0.00472692') AIRDROP_AMOUNT_SOL = ACCOUNT_CREATION_PRICE_SOL / 2 @@ -76,14 +71,11 @@ def is_airdrop_ready(self, eth_address): cur.execute(f"SELECT 1 FROM {self._table_name} WHERE eth_address = '{eth_address}'") return cur.fetchone() is not None -FINALIZED = os.environ.get('FINALIZED', 'finalized') - @logged_group("neon.Airdropper") class Airdropper(IndexerBase): def __init__(self, solana_url, - evm_loader_id, pyth_mapping_account: PublicKey, faucet_url = '', wrapper_whitelist = 'ANY', @@ -92,8 +84,9 @@ def __init__(self, max_conf = 0.1): # maximum confidence interval deviation related to price self._constants = SQLDict(tablename="constants") + solana = SolanaInteractor(solana_url) last_known_slot = self._constants.get('latest_processed_slot', None) - IndexerBase.__init__(self, solana_url, evm_loader_id, last_known_slot) + IndexerBase.__init__(self, solana, last_known_slot) self.latest_processed_slot = self.last_slot # collection of eth-address-to-create-accout-trx mappings @@ -113,7 +106,7 @@ def __init__(self, # but there will be different slot numbers so price should be updated every time self.always_reload_price = (pp_solana_url != solana_url) self.pyth_mapping_account = pyth_mapping_account - self.pyth_client = PythNetworkClient(SolanaClient(pp_solana_url)) + self.pyth_client = PythNetworkClient(SolanaInteractor(pp_solana_url)) self.neon_decimals = neon_decimals self.max_conf = Decimal(max_conf) self.session = requests.Session() @@ -122,13 +115,12 @@ def __init__(self, self.airdrop_amount_usd = None self.airdrop_amount_neon = None self.last_update_pyth_mapping = None - self.max_update_pyth_mapping_int = 60 * 60 # update once an hour - + self.max_update_pyth_mapping_int = 60 * 60 # update once an hour - def get_current_time(self): + @staticmethod + def get_current_time(): return datetime.now().timestamp() - def try_update_pyth_mapping(self): current_time = self.get_current_time() if self.last_update_pyth_mapping is None or self.last_update_pyth_mapping - current_time > self.max_update_pyth_mapping_int: @@ -136,7 +128,9 @@ def try_update_pyth_mapping(self): self.pyth_client.update_mapping(self.pyth_mapping_account) self.last_update_pyth_mapping = current_time except Exception as err: - self.warning(f'Failed to update pyth.network mapping account data: {err}') + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.warning(f'Failed to update pyth.network mapping account data ' + + f'{type(err)}, Error: {err}, Traceback: {err_tb}') return False return True @@ -147,7 +141,6 @@ def is_allowed_wrapper_contract(self, contract_addr): return True return contract_addr in self.wrapper_whitelist - # helper function checking if given 'create account' corresponds to 'create erc20 token account' instruction def check_create_instr(self, account_keys, create_acc, create_token_acc): # Must use the same Ethereum account @@ -164,23 +157,21 @@ def check_create_instr(self, account_keys, create_acc, create_token_acc): return False return True - # helper function checking if given 'create erc20 token account' corresponds to 'token transfer' instruction - def check_transfer(self, account_keys, create_token_acc, token_transfer) -> bool: + @staticmethod + def check_transfer(account_keys, create_token_acc, token_transfer) -> bool: return account_keys[create_token_acc['accounts'][1]] == account_keys[token_transfer['accounts'][1]] - def airdrop_to(self, eth_address, airdrop_galans): self.info(f"Airdrop {airdrop_galans} Galans to address: {eth_address}") json_data = { 'wallet': eth_address, 'amount': airdrop_galans } - resp = self.session.post(self.faucet_url + '/request_neon_in_galans', json = json_data) + resp = self.session.post(self.faucet_url + '/request_neon_in_galans', json=json_data) if not resp.ok: self.warning(f'Failed to airdrop: {resp.status_code}') return False return True - def process_trx_airdropper_mode(self, trx): if check_error(trx): return @@ -195,11 +186,11 @@ def find_instructions(trx, predicate): # Airdrop triggers on sequence: # neon.CreateAccount -> neon.CreateERC20TokenAccount -> spl.Transfer (maybe shuffled) # First: select all instructions that can form such chains - predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + predicate = lambda instr: account_keys[instr['programIdIndex']] == EVM_LOADER_ID \ and base58.b58decode(instr['data'])[0] == 0x02 create_acc_list = find_instructions(trx, predicate) - predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \ + predicate = lambda instr: account_keys[instr['programIdIndex']] == EVM_LOADER_ID \ and base58.b58decode(instr['data'])[0] == 0x0f create_token_acc_list = find_instructions(trx, predicate) @@ -217,18 +208,19 @@ def find_instructions(trx, predicate): continue self.schedule_airdrop(create_acc) - def get_sol_usd_price(self): should_reload = self.always_reload_price if not should_reload: - if self.recent_price == None or self.recent_price['valid_slot'] < self.current_slot: + if self.recent_price is None or self.recent_price['valid_slot'] < self.current_slot: should_reload = True if should_reload: try: self.recent_price = self.pyth_client.get_price('Crypto.SOL/USD') except Exception as err: - self.warning(f'Exception occured when reading price: {err}') + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.warning(f'Exception occured when reading price ' + + f'{type(err)}, Error: {err}, Traceback: {err_tb}') return None return self.recent_price @@ -252,7 +244,6 @@ def get_airdrop_amount_galans(self): self.info(f"Airdrop amount: ${self.airdrop_amount_usd} ({self.airdrop_amount_neon} NEONs)\n") return int(self.airdrop_amount_neon * pow(Decimal(10), self.neon_decimals)) - def schedule_airdrop(self, create_acc): eth_address = "0x" + bytearray(base58.b58decode(create_acc['data'])[20:][:20]).hex() if self.airdrop_ready.is_airdrop_ready(eth_address) or eth_address in self.airdrop_scheduled: @@ -261,7 +252,6 @@ def schedule_airdrop(self, create_acc): self.info(f'Scheduling airdrop for {eth_address}') self.airdrop_scheduled[eth_address] = { 'scheduled': self.get_current_time() } - def process_scheduled_trxs(self): # Pyth.network mapping account was never updated if not self.try_update_pyth_mapping() and self.last_update_pyth_mapping is None: @@ -289,7 +279,6 @@ def process_scheduled_trxs(self): for eth_address in success_addresses: del self.airdrop_scheduled[eth_address] - def process_functions(self): """ Overrides IndexerBase.process_functions @@ -299,7 +288,6 @@ def process_functions(self): self.process_receipts() self.process_scheduled_trxs() - def process_receipts(self): max_slot = 0 for slot, _, trx in self.transaction_receipts.get_trxs(self.latest_processed_slot): @@ -312,7 +300,6 @@ def process_receipts(self): @logged_group("neon.Airdropper") def run_airdropper(solana_url, - evm_loader_id, pyth_mapping_account: PublicKey, faucet_url, wrapper_whitelist = 'ANY', @@ -321,7 +308,7 @@ def run_airdropper(solana_url, max_conf = 0.1, *, logger): logger.info(f"""Running indexer with params: solana_url: {solana_url}, - evm_loader_id: {evm_loader_id}, + evm_loader_id: {EVM_LOADER_ID}, pyth.network mapping account: {pyth_mapping_account}, faucet_url: {faucet_url}, wrapper_whitelist: {wrapper_whitelist}, @@ -331,7 +318,6 @@ def run_airdropper(solana_url, try: airdropper = Airdropper(solana_url, - evm_loader_id, pyth_mapping_account, faucet_url, wrapper_whitelist, diff --git a/proxy/indexer/base_db.py b/proxy/indexer/base_db.py new file mode 100644 index 000000000..dc6d86fde --- /dev/null +++ b/proxy/indexer/base_db.py @@ -0,0 +1,75 @@ +import multiprocessing +import psycopg2 + +from typing import NamedTuple +from logged_groups import logged_group + +from .pg_common import POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST +from .pg_common import encode, decode + + +class DBQuery(NamedTuple): + column_list: list + key_list: list + order_list: list + + +class DBQueryExpression(NamedTuple): + column_expr: str + where_expr: str + where_keys: list + order_expr: str + + +@logged_group("neon.Indexer") +class BaseDB: + _create_table_lock = multiprocessing.Lock() + + def __init__(self): + self._conn = psycopg2.connect( + dbname=POSTGRES_DB, + user=POSTGRES_USER, + password=POSTGRES_PASSWORD, + host=POSTGRES_HOST + ) + self._conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + + with self._create_table_lock: + cursor = self._conn.cursor() + cursor.execute(self._create_table_sql()) + + def _create_table_sql(self) -> str: + assert False, 'No script for the table' + + def _build_expression(self, q: DBQuery) -> DBQueryExpression: + + return DBQueryExpression( + column_expr=','.join(q.column_list), + where_expr=' AND '.join(['1=1'] + [f'{name}=%s' for name, _ in q.key_list]), + where_keys=[value for _, value in q.key_list], + order_expr='ORDER BY ' + ', '.join(q.order_list) if len(q.order_list) else '', + ) + + def _fetchone(self, query: DBQuery) -> []: + e = self._build_expression(query) + + request = f''' + SELECT {e.column_expr} + FROM {self._table_name} AS a + WHERE {e.where_expr} + {e.order_expr} + LIMIT 1 + ''' + + with self._conn.cursor() as cursor: + cursor.execute(request, e.where_keys) + return cursor.fetchone() + + def __del__(self): + self._conn.close() + + def decode_list(self, v): + return [] if not v else decode(v) + + def encode_list(self, v: []): + return None if (not v) or (len(v) == 0) else encode(v) diff --git a/proxy/indexer/blocks_db.py b/proxy/indexer/blocks_db.py index 6c0ef882a..5a6c2e8eb 100644 --- a/proxy/indexer/blocks_db.py +++ b/proxy/indexer/blocks_db.py @@ -1,6 +1,6 @@ from typing import Optional -from ..indexer.utils import BaseDB, DBQuery +from ..indexer.base_db import BaseDB, DBQuery from ..common_neon.utils import SolanaBlockInfo diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py index 6f9dfc15e..86e5d31a2 100644 --- a/proxy/indexer/canceller.py +++ b/proxy/indexer/canceller.py @@ -3,7 +3,6 @@ from logged_groups import logged_group from solana.publickey import PublicKey -from solana.rpc.api import Client from solana.system_program import SYS_PROGRAM_ID from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY from solana.transaction import AccountMeta @@ -35,10 +34,9 @@ def __init__(self): # Initialize user account self.signer = get_solana_accounts()[0] self._operator = self.signer.public_key() - self._client = Client(SOLANA_URL) self.operator_token = get_associated_token_address(PublicKey(self._operator), ETH_TOKEN_MINT_ID) - self.solana = SolanaInteractor(self._client) + self._solana = SolanaInteractor(SOLANA_URL) self.builder = NeonInstruction(self._operator) @@ -60,7 +58,7 @@ def unlock_accounts(self, blocked_storages): self.debug(f"Send Cancel: {trx}") try: - cancel_result = self.solana.send_multiple_transactions(self.signer, [trx], neon_tx.tx, "CancelWithNonce")[0] + cancel_result = self._solana.send_multiple_transactions(self.signer, [trx], neon_tx.tx, "CancelWithNonce")[0] self.debug(f"cancel result: {cancel_result}") result_error = get_from_dict(cancel_result, 'meta', 'err') if result_error: diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index e6d716b1b..002c49f3b 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -3,7 +3,6 @@ import base58 import time from logged_groups import logged_group, logging_context -from solana.rpc.api import Client from solana.system_program import SYS_PROGRAM_ID from ..indexer.indexer_base import IndexerBase @@ -13,6 +12,7 @@ from ..indexer.canceller import Canceller from ..common_neon.utils import NeonTxResultInfo, NeonTxInfo, str_fmt_object +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import EVM_LOADER_ID, FINALIZED, CANCEL_TIMEOUT, SOLANA_URL @@ -168,9 +168,9 @@ class ReceiptsParserState: - All instructions are removed from the _used_ixs; - If number of the smallest slot in the _used_ixs is changed, it's stored into the DB for the future restart. """ - def __init__(self, db: IndexerDB, solana_client: Client): + def __init__(self, db: IndexerDB, solana: SolanaInteractor): self._db = db - self._client = solana_client + self._solana = solana self._holder_table = {} self._tx_table = {} self._done_tx_list = [] @@ -674,16 +674,14 @@ def execute(self) -> bool: @logged_group("neon.Indexer") class BlocksIndexer: - def __init__(self, db: IndexerDB, solana_client: Client): + def __init__(self, db: IndexerDB, solana: SolanaInteractor): self.db = db - self.solana_client = solana_client + self.solana = solana self.counted_logger = MetricsToLogBuff() def gather_blocks(self): start_time = time.time() - client = self.solana_client._provider - opts = {"commitment": FINALIZED} - slot = client.make_request('getSlot', opts)['result'] + slot = self.solana.get_slot(FINALIZED)['result'] self.db.set_latest_block(slot) gather_blocks_ms = (time.time() - start_time) * 1000 # convert this into milliseconds self.counted_logger.print( @@ -695,19 +693,19 @@ def gather_blocks(self): @logged_group("neon.Indexer") class Indexer(IndexerBase): - def __init__(self, solana_url, evm_loader_id): + def __init__(self, solana_url): self.debug(f'Finalized commitment: {FINALIZED}') - self.db = IndexerDB() + solana = SolanaInteractor(solana_url) + self.db = IndexerDB(solana) last_known_slot = self.db.get_min_receipt_slot() - IndexerBase.__init__(self, solana_url, evm_loader_id, last_known_slot) + IndexerBase.__init__(self, solana, last_known_slot) self.indexed_slot = self.last_slot - self.db.set_client(self.solana_client) self.canceller = Canceller() self.blocked_storages = {} - self.block_indexer = BlocksIndexer(db=self.db, solana_client=self.solana_client) + self.block_indexer = BlocksIndexer(db=self.db, solana=solana) self.counted_logger = MetricsToLogBuff() - self.state = ReceiptsParserState(db=self.db, solana_client=self.solana_client) + self.state = ReceiptsParserState(db=self.db, solana=solana) self.ix_decoder_map = { 0x00: WriteIxDecoder(self.state), 0x01: DummyIxDecoder('Finalize', self.state), @@ -795,7 +793,7 @@ def unlock_accounts(self, tx) -> bool: self.warning(f"Transaction {tx.neon_tx} hasn't blocked accounts.") return False - storage_accounts_list = get_accounts_from_storage(self.solana_client, tx.storage_account) + storage_accounts_list = get_accounts_from_storage(self.solana, tx.storage_account) if storage_accounts_list is None: self.warning(f"Transaction {tx.neon_tx} has empty storage.") return False @@ -811,16 +809,15 @@ def unlock_accounts(self, tx) -> bool: @logged_group("neon.Indexer") -def run_indexer(solana_url, evm_loader_id, *, logger): +def run_indexer(solana_url, *, logger): logger.info(f"""Running indexer with params: solana_url: {solana_url}, - evm_loader_id: {evm_loader_id}""") + evm_loader_id: {EVM_LOADER_ID}""") - indexer = Indexer(solana_url, evm_loader_id) + indexer = Indexer(solana_url) indexer.run() if __name__ == "__main__": solana_url = SOLANA_URL - evm_loader_id = EVM_LOADER_ID - run_indexer(solana_url, evm_loader_id) + run_indexer(solana_url) diff --git a/proxy/indexer/indexer_base.py b/proxy/indexer/indexer_base.py index 1ade8fb6b..54fc2c475 100644 --- a/proxy/indexer/indexer_base.py +++ b/proxy/indexer/indexer_base.py @@ -1,26 +1,23 @@ import os import time import traceback -from solana.rpc.api import Client from multiprocessing.dummy import Pool as ThreadPool -from typing import Dict, Union from logged_groups import logged_group from .trx_receipts_storage import TrxReceiptsStorage from .utils import MetricsToLogBuff +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import RETRY_ON_FAIL_ON_GETTING_CONFIRMED_TRANSACTION -from ..environment import HISTORY_START, PARALLEL_REQUESTS, FINALIZED +from ..environment import HISTORY_START, PARALLEL_REQUESTS, FINALIZED, EVM_LOADER_ID @logged_group("neon.Indexer") class IndexerBase: def __init__(self, - solana_url, - evm_loader_id, - last_slot): - self.evm_loader_id = evm_loader_id - self.solana_client = Client(solana_url) + solana: SolanaInteractor, + last_slot: int): + self.solana = solana self.transaction_receipts = TrxReceiptsStorage('transaction_receipts') self.max_known_tx = self.transaction_receipts.max_known_trx() self.last_slot = self._init_last_slot('receipt', last_slot) @@ -36,7 +33,7 @@ def _init_last_slot(self, name: str, last_known_slot: int): - NUMBER - first start from the number, then continue from last parsed slot """ last_known_slot = 0 if not isinstance(last_known_slot, int) else last_known_slot - latest_slot = self.solana_client.get_slot(commitment=FINALIZED)["result"] + latest_slot = self.solana.get_slot(FINALIZED)["result"] start_int_slot = 0 name = f'{name} slot' @@ -87,7 +84,7 @@ def gather_unknown_transactions(self): minimal_tx = None continue_flag = True - current_slot = self.solana_client.get_slot(commitment=FINALIZED)["result"] + current_slot = self.solana.get_slot(commitment=FINALIZED)["result"] max_known_tx = self.max_known_tx @@ -137,13 +134,7 @@ def gather_unknown_transactions(self): ) def _get_signatures(self, before, until): - opts: Dict[str, Union[int, str]] = {} - if until is not None: - opts["until"] = until - if before is not None: - opts["before"] = before - opts["commitment"] = FINALIZED - result = self.solana_client._provider.make_request("getSignaturesForAddress", self.evm_loader_id, opts) + result = self.solana.get_signatures_for_address(before, until, FINALIZED) return result['result'] def _get_tx_receipts(self, solana_signature): @@ -152,7 +143,7 @@ def _get_tx_receipts(self, solana_signature): while retry > 0: try: - trx = self.solana_client.get_confirmed_transaction(solana_signature)['result'] + trx = self.solana.get_confirmed_transaction(solana_signature)['result'] self._add_trx(solana_signature, trx) retry = 0 except Exception as err: @@ -170,7 +161,7 @@ def _add_trx(self, solana_signature, trx): if trx is not None: add = False for instruction in trx['transaction']['message']['instructions']: - if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] == self.evm_loader_id: + if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] == EVM_LOADER_ID: add = True if add: self.debug((trx['slot'], solana_signature)) diff --git a/proxy/indexer/indexer_db.py b/proxy/indexer/indexer_db.py index 061052405..143b77ecd 100644 --- a/proxy/indexer/indexer_db.py +++ b/proxy/indexer/indexer_db.py @@ -2,7 +2,6 @@ import traceback from logged_groups import logged_group -from solana.rpc.api import Client from typing import Optional from ..common_neon.utils import NeonTxInfo, NeonTxResultInfo, NeonTxFullInfo @@ -15,25 +14,23 @@ from ..indexer.logs_db import LogsDB from ..indexer.sql_dict import SQLDict from ..indexer.utils import get_code_from_account, get_accounts_by_neon_address +from ..common_neon.solana_interactor import SolanaInteractor @logged_group("neon.Indexer") class IndexerDB: - def __init__(self): + def __init__(self, solana: SolanaInteractor): self._logs_db = LogsDB() self._blocks_db = SolanaBlocksDB() self._txs_db = NeonTxsDB() self._account_db = NeonAccountDB() - self._client = None + self._solana = solana self._constants = SQLDict(tablename="constants") for k in ['min_receipt_slot', 'latest_slot']: if k not in self._constants: self._constants[k] = 0 - def set_client(self, solana_client: Client): - self._client = solana_client - def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, used_ixs: [SolanaIxSignInfo]): try: block = self.get_block_by_slot(neon_res.slot) @@ -50,35 +47,25 @@ def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, us self.error('Exception on submitting transaction. ' + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - def _fill_block_from_net(self, block: SolanaBlockInfo): - opts = {"commitment": FINALIZED, "transactionDetails": "signatures", "rewards": False} - net_block = self._client._provider.make_request("getBlock", block.slot, opts) - if (not net_block) or ('result' not in net_block): - return block - - net_block = net_block['result'] - if not net_block: + def _get_block_from_net(self, block: SolanaBlockInfo) -> SolanaBlockInfo: + net_block = self._solana.get_block_info(block.slot, FINALIZED) + if not net_block.hash: return block - block.hash = '0x' + base58.b58decode(net_block['blockhash']).hex() - block.signs = net_block['signatures'] - block.parent_hash = '0x' + base58.b58decode(net_block['previousBlockhash']).hex() - block.time = net_block['blockTime'] - block.finalized = True - self.debug(f'{block}') - self._blocks_db.set_block(block) - return block + self.debug(f'{net_block}') + self._blocks_db.set_block(net_block) + return net_block def _fill_account_data_from_net(self, account: NeonAccountInfo): got_changes = False if not account.pda_account: - pda_account, code_account = get_accounts_by_neon_address(self._client, account.neon_account) + pda_account, code_account = get_accounts_by_neon_address(self._solana, account.neon_account) if pda_account: account.pda_account = pda_account account.code_account = code_account got_changes = True if account.code_account: - code = get_code_from_account(self._client, account.code_account) + code = get_code_from_account(self._solana, account.code_account) if code: account.code = code got_changes = True @@ -93,13 +80,13 @@ def _fill_account_data_from_net(self, account: NeonAccountInfo): def get_block_by_slot(self, slot) -> SolanaBlockInfo: block = self._blocks_db.get_block_by_slot(slot) if not block.hash: - self._fill_block_from_net(block) + block = self._get_block_from_net(block) return block def get_full_block_by_slot(self, slot) -> SolanaBlockInfo: block = self._blocks_db.get_full_block_by_slot(slot) if not block.parent_hash: - self._fill_block_from_net(block) + block = self._get_block_from_net(block) return block def get_latest_block(self) -> SolanaBlockInfo: diff --git a/proxy/indexer/logs_db.py b/proxy/indexer/logs_db.py index ac9a4d56e..984b67019 100644 --- a/proxy/indexer/logs_db.py +++ b/proxy/indexer/logs_db.py @@ -1,5 +1,5 @@ import json -from ..indexer.utils import BaseDB +from ..indexer.base_db import BaseDB class LogsDB(BaseDB): diff --git a/proxy/indexer/pythnetwork.py b/proxy/indexer/pythnetwork.py index f6a7159f1..4ca0e2aeb 100644 --- a/proxy/indexer/pythnetwork.py +++ b/proxy/indexer/pythnetwork.py @@ -1,13 +1,13 @@ -from solana.rpc.api import Client as SolanaClient from solana.publickey import PublicKey from solana.system_program import SYS_PROGRAM_ID from decimal import Decimal -import base64 -import base58 import struct +import traceback from logged_groups import logged_group from typing import List, Union +from ..common_neon.solana_interactor import SolanaInteractor + def read_str(pos, data): length = data[pos] @@ -82,8 +82,8 @@ class PythNetworkClient: 'agg.status': { 'pos': 224, 'len': 4, 'format': ' str: - assert False, 'No script for the table' - - def _build_expression(self, q: DBQuery) -> DBQueryExpression: - - return DBQueryExpression( - column_expr=','.join(q.column_list), - where_expr=' AND '.join(['1=1'] + [f'{name}=%s' for name, _ in q.key_list]), - where_keys=[value for _, value in q.key_list], - order_expr='ORDER BY ' + ', '.join(q.order_list) if len(q.order_list) else '', - ) - - def _fetchone(self, query: DBQuery) -> []: - e = self._build_expression(query) - - request = f''' - SELECT {e.column_expr} - FROM {self._table_name} AS a - WHERE {e.where_expr} - {e.order_expr} - LIMIT 1 - ''' - - with self._conn.cursor() as cursor: - cursor.execute(request, e.where_keys) - return cursor.fetchone() - - def __del__(self): - self._conn.close() - - def decode_list(self, v): - return [] if not v else decode(v) - - def encode_list(self, v: []): - return None if (not v) or (len(v) == 0) else encode(v) diff --git a/proxy/memdb/memdb.py b/proxy/memdb/memdb.py index 9c93c5daa..a156d1524 100644 --- a/proxy/memdb/memdb.py +++ b/proxy/memdb/memdb.py @@ -1,5 +1,4 @@ from logged_groups import logged_group -from solana.rpc.api import Client as SolanaClient from typing import Optional from ..indexer.indexer_db import IndexerDB @@ -14,12 +13,9 @@ @logged_group("neon.Proxy") class MemDB: - def __init__(self, client: SolanaClient): - self._client = client - - self._db = IndexerDB() - self._db.set_client(self._client) - self._solana = SolanaInteractor(client) + def __init__(self, solana: SolanaInteractor): + self._solana = solana + self._db = IndexerDB(solana) self._blocks_db = MemBlocksDB(self._solana, self._db) self._txs_db = MemTxsDB(self._db) diff --git a/proxy/plugin/gas_price_calculator.py b/proxy/plugin/gas_price_calculator.py index ce7ac2915..ad93aa05f 100644 --- a/proxy/plugin/gas_price_calculator.py +++ b/proxy/plugin/gas_price_calculator.py @@ -3,16 +3,17 @@ import time from logged_groups import logged_group from ..indexer.pythnetwork import PythNetworkClient +from ..common_neon.solana_interactor import SolanaInteractor from ..environment import MINIMAL_GAS_PRICE, OPERATOR_FEE, NEON_PRICE_USD, \ SOL_PRICE_UPDATE_INTERVAL, GET_SOL_PRICE_MAX_RETRIES, GET_SOL_PRICE_RETRY_INTERVAL @logged_group("neon.gas_price_calculator") class GasPriceCalculator: - def __init__(self, solana_client, pyth_mapping_acc) -> None: - self.solana_client = solana_client + def __init__(self, solana: SolanaInteractor, pyth_mapping_acc) -> None: + self.solana = solana self.mapping_account = pyth_mapping_acc - self.pyth_network_client = PythNetworkClient(self.solana_client) + self.pyth_network_client = PythNetworkClient(self.solana) self.recent_sol_price_update_time = None self.min_gas_price = None diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index c8533e040..225d211b9 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -23,7 +23,6 @@ from ..http.parser import HttpParser from ..http.websocket import WebsocketFrame from ..http.server import HttpWebServerBasePlugin, httpProtocolTypes -from solana.rpc.api import Client as SolanaClient from typing import List, Tuple from .solana_rest_api_tools import neon_config_load, get_token_balance_or_zero, estimate_gas @@ -53,13 +52,13 @@ class EthereumModel: proxy_id_glob = multiprocessing.Value('i', 0) def __init__(self): - self._client = SolanaClient(SOLANA_URL) - self._db = MemDB(self._client) + self._solana = SolanaInteractor(SOLANA_URL) + self._db = MemDB(self._solana) if PP_SOLANA_URL == SOLANA_URL: - self.gas_price_calculator = GasPriceCalculator(self._client, PYTH_MAPPING_ACCOUNT) + self.gas_price_calculator = GasPriceCalculator(self._solana, PYTH_MAPPING_ACCOUNT) else: - self.gas_price_calculator = GasPriceCalculator(SolanaClient(PP_SOLANA_URL), PYTH_MAPPING_ACCOUNT) + self.gas_price_calculator = GasPriceCalculator(SolanaInteractor(PP_SOLANA_URL), PYTH_MAPPING_ACCOUNT) self.gas_price_calculator.update_mapping() with self.proxy_id_glob.get_lock(): @@ -133,7 +132,7 @@ def eth_getBalance(self, account, tag): """ eth_acc = EthereumAddress(account) self.debug(f'eth_getBalance: {account} {eth_acc}') - balance = get_token_balance_or_zero(self._client, eth_acc) + balance = get_token_balance_or_zero(self._solana, eth_acc) return hex(balance * eth_utils.denoms.gwei) def eth_getLogs(self, obj): @@ -268,8 +267,7 @@ def eth_call(self, obj, tag): def eth_getTransactionCount(self, account, tag): self.debug('eth_getTransactionCount: %s', account) try: - solana = SolanaInteractor(self._client) - acc_info = solana.get_neon_account_info(EthereumAddress(account)) + acc_info = self._solana.get_neon_account_info(EthereumAddress(account)) return hex(acc_info.trx_count) except Exception as err: self.debug(f"eth_getTransactionCount: Can't get account info: {err}") @@ -366,7 +364,7 @@ def eth_sendRawTransaction(self, rawTrx): eth_signature = '0x' + trx.hash_signed().hex() try: - tx_sender = NeonTxSender(self._db, self._client, trx, steps=500) + tx_sender = NeonTxSender(self._db, self._solana, trx, steps=500) tx_sender.execute() return eth_signature diff --git a/proxy/plugin/solana_rest_api_tools.py b/proxy/plugin/solana_rest_api_tools.py index 71676d034..656d7fe9d 100644 --- a/proxy/plugin/solana_rest_api_tools.py +++ b/proxy/plugin/solana_rest_api_tools.py @@ -1,15 +1,11 @@ -import base64 -import eth_utils - from datetime import datetime from solana.publickey import PublicKey -from solana.rpc.api import Client as SolanaClient -from solana.rpc.commitment import Confirmed from logged_groups import logged_group from ..common_neon.address import ether2program, getTokenAddr, EthereumAddress from ..common_neon.errors import SolanaAccountNotFoundError, SolanaErrors from ..common_neon.emulator_interactor import call_emulated +from ..common_neon.solana_interactor import SolanaInteractor from ..common_neon.utils import get_from_dict from ..environment import read_elf_params, TIMEOUT_TO_RELOAD_NEON_CONFIG, EXTRA_GAS @@ -38,43 +34,22 @@ def neon_config_load(ethereum_model, *, logger): @logged_group("neon.Proxy") -def get_token_balance_gwei(client: SolanaClient, pda_account: str, *, logger) -> int: +def get_token_balance_gwei(solana: SolanaInteractor, pda_account: str, *, logger) -> int: neon_token_account = getTokenAddr(PublicKey(pda_account)) - rpc_response = client.get_token_account_balance(neon_token_account, commitment=Confirmed) - error = rpc_response.get('error') - if error is not None: - message = error.get("message") - if message == SolanaErrors.AccountNotFound.value: - raise SolanaAccountNotFoundError() - logger.error(f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, " - f"got get_token_account_balance error: \"{message}\"") - raise Exception("Getting balance error") - - balance = get_from_dict(rpc_response, "result", "value", "amount") - if balance is None: - logger.error( - f"Failed to get_token_balance_gwei by neon_token_account: {neon_token_account}, response: {rpc_response}") - raise Exception("Unexpected get_balance response") - return int(balance) + return solana.get_token_account_balance(neon_token_account) @logged_group("neon.Proxy") -def get_token_balance_or_zero(client: SolanaClient, eth_account: EthereumAddress, *, logger) -> int: +def get_token_balance_or_zero(solana: SolanaInteractor, eth_account: EthereumAddress, *, logger) -> int: solana_account, nonce = ether2program(eth_account) logger.debug(f"Get balance for eth account: {eth_account} aka: {solana_account}") - - try: - return get_token_balance_gwei(client, solana_account) - except SolanaAccountNotFoundError: - logger.debug(f"Account not found: {eth_account} aka: {solana_account} - return airdrop amount") - return 0 + return get_token_balance_gwei(solana, solana_account) -def is_account_exists(client: SolanaClient, eth_account: EthereumAddress) -> bool: +def is_account_exists(solana: SolanaInteractor, eth_account: EthereumAddress) -> bool: pda_account, nonce = ether2program(eth_account) - info = client.get_account_info(pda_account, commitment=Confirmed) - value = get_from_dict(info, "result", "value") - return value is not None + info = solana.get_account_info(pda_account) + return info is not None @logged_group("neon.Proxy") diff --git a/proxy/testing/test_account_whitelist.py b/proxy/testing/test_account_whitelist.py index ed225e8ae..d41220d04 100644 --- a/proxy/testing/test_account_whitelist.py +++ b/proxy/testing/test_account_whitelist.py @@ -1,44 +1,43 @@ import os -from signal import default_int_handler import unittest -from unittest import mock +from proxy.common_neon.solana_interactor import SolanaInteractor from proxy.common_neon.account_whitelist import AccountWhitelist from solana.rpc.api import Client as SolanaClient from solana.account import Account as SolanaAccount from solana.rpc.commitment import Confirmed from unittest.mock import Mock, MagicMock, patch, call -from proxy.environment import GET_WHITE_LIST_BALANCE_MAX_RETRIES - class TestAccountWhitelist(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.solana = SolanaClient(os.environ['SOLANA_URL']) + cls.solana = SolanaInteractor(os.environ['SOLANA_URL']) cls.payer = SolanaAccount() - cls.solana.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) + client = SolanaClient(os.environ['SOLANA_URL']) + client.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) cls.permission_update_int = 10 cls.testee = AccountWhitelist(cls.solana, cls.payer, cls.permission_update_int) mock_allowance_token = Mock() - mock_allowance_token.get_balance = MagicMock() + mock_allowance_token.get_token_account_address = MagicMock() mock_allowance_token.mint_to = MagicMock() cls.testee.allowance_token = mock_allowance_token mock_denial_token = Mock() - mock_denial_token.get_balance = MagicMock() + mock_denial_token.get_token_account_address = MagicMock() mock_denial_token.mint_to = MagicMock() cls.testee.denial_token = mock_denial_token def tearDown(self) -> None: - self.testee.allowance_token.get_balance.reset_mock() + self.testee.allowance_token.get_token_account_address.reset_mock() self.testee.allowance_token.mint_to.reset_mock() - self.testee.denial_token.get_balance.reset_mock() + self.testee.denial_token.get_token_account_address.reset_mock() self.testee.denial_token.mint_to.reset_mock() self.testee.account_cache = {} - def test_grant_permissions_negative_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_grant_permissions_negative_difference(self, mock_get_token_account_balance_list): """ Should mint allowance token - negative differenct """ @@ -49,16 +48,16 @@ def test_grant_permissions_negative_difference(self): expected_mint = min_balance - diff ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.grant_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.allowance_token.mint_to.assert_called_once_with(expected_mint, ether_address) - def test_grant_permissions_positive_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_grant_permissions_positive_difference(self, mock_get_token_account_balance_list): """ Should NOT mint allowance token - positive difference """ @@ -67,16 +66,16 @@ def test_grant_permissions_positive_difference(self): min_balance = 1 ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.grant_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.allowance_token.mint_to.assert_not_called() - def test_deprive_permissions_positive_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_deprive_permissions_positive_difference(self, mock_get_token_account_balance_list): """ Should mint denial token - positive difference """ @@ -87,16 +86,16 @@ def test_deprive_permissions_positive_difference(self): expected_mint = diff - min_balance + 1 ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.deprive_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.denial_token.mint_to.assert_called_once_with(expected_mint, ether_address) - def test_deprive_permissions_negative_difference(self): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_deprive_permissions_negative_difference(self, mock_get_token_account_balance_list): """ Should NOT mint denial token - negative difference """ @@ -105,49 +104,28 @@ def test_deprive_permissions_negative_difference(self): min_balance = 3 ether_address = 'Ethereum-Address' - self.testee.allowance_token.get_balance.side_effect = [allowance_balance] - self.testee.denial_token.get_balance.side_effect = [denial_balance] + mock_get_token_account_balance_list.side_effect = [[allowance_balance, denial_balance]] self.assertTrue(self.testee.deprive_permissions(ether_address, min_balance)) - self.testee.allowance_token.get_balance.assert_called_once_with(ether_address) - self.testee.denial_token.get_balance.assert_called_once_with(ether_address) + self.testee.allowance_token.get_token_account_address.assert_called_once_with(ether_address) + self.testee.denial_token.get_token_account_address.assert_called_once_with(ether_address) self.testee.denial_token.mint_to.assert_not_called() @patch.object(AccountWhitelist, 'get_current_time') - def test_check_has_permission(self, mock_get_current_time): + @patch.object(SolanaInteractor, 'get_token_account_balance_list') + def test_check_has_permission(self, mock_get_token_account_balance_list, mock_get_current_time): ether_address = 'Ethereum-Address' - time1 = 123 # will cause get_balance call - time2 = time1 + self.permission_update_int + 2 # will cause get_balance call - time3 = time2 + self.permission_update_int - 3 # will NOT cause get_balance call + time1 = 123 # will cause get_token_account_address call + time2 = time1 + self.permission_update_int + 2 # will cause get_token_account_address call + time3 = time2 + self.permission_update_int - 3 # will NOT cause get_token_account_address call mock_get_current_time.side_effect = [ time1, time2, time3 ] - - allowance_balance1 = 100 - denial_balance1 = 50 - allowance_balance2 = 100 - denial_balance2 = 150 - - self.testee.allowance_token.get_balance.side_effect = [allowance_balance1, allowance_balance2] - self.testee.denial_token.get_balance.side_effect = [denial_balance1, denial_balance2] + mock_get_token_account_balance_list.side_effect = [[100, 50], [100, 150]] self.assertTrue(self.testee.has_permission(ether_address, 0)) self.assertFalse(self.testee.has_permission(ether_address, 0)) self.assertFalse(self.testee.has_permission(ether_address, 0)) mock_get_current_time.assert_has_calls([call()] * 3) - self.testee.allowance_token.get_balance.assert_has_calls([call(ether_address)] * 2) - self.testee.denial_token.get_balance.assert_has_calls([call(ether_address)] * 2) - - @patch.object(AccountWhitelist, 'read_balance_diff') - @patch.object(AccountWhitelist, 'get_current_time') - def test_success_check_has_permission_after_retry_due_to_read_balance_diff_exception(self, mock_get_current_time, mock_read_balance_diff): - """ - Should retry read_balance_diff after exception - """ - self.assertGreaterEqual(GET_WHITE_LIST_BALANCE_MAX_RETRIES, 2) # Condition required to start test - ether_address = 'Ethereum-Address' - mock_get_current_time.side_effect = [0] - mock_read_balance_diff.side_effect = [Exception('TestException'), 12.3] - - self.assertTrue(self.testee.has_permission(ether_address, 0)) - self.assertTrue(mock_read_balance_diff.call_count, 2) + self.testee.allowance_token.get_token_account_address.assert_has_calls([call(ether_address)] * 2) + self.testee.denial_token.get_token_account_address.assert_has_calls([call(ether_address)] * 2) diff --git a/proxy/testing/test_airdropper.py b/proxy/testing/test_airdropper.py index 537d0a933..c7ec3883a 100644 --- a/proxy/testing/test_airdropper.py +++ b/proxy/testing/test_airdropper.py @@ -55,7 +55,6 @@ class Test_Airdropper(unittest.TestCase): def create_airdropper(self, start_slot): os.environ['START_SLOT'] = str(start_slot) return Airdropper(solana_url =f'http://{self.address}:8899', - evm_loader_id =self.evm_loader_id, pyth_mapping_account=self.pyth_mapping_account, faucet_url =f'http://{self.address}:{self.faucet_port}', wrapper_whitelist =self.wrapper_whitelist, diff --git a/proxy/testing/test_neon_tx_sender.py b/proxy/testing/test_neon_tx_sender.py index 8ac10f105..3e9bc01fb 100644 --- a/proxy/testing/test_neon_tx_sender.py +++ b/proxy/testing/test_neon_tx_sender.py @@ -1,21 +1,18 @@ -import logging import os import unittest from solana.rpc.api import Client as SolanaClient -from solana.account import Account as SolanaAccount -from solana.rpc.commitment import Confirmed -from unittest.mock import Mock, MagicMock, patch, call +from unittest.mock import Mock from proxy.common_neon.eth_proto import Trx as EthTrx from proxy.common_neon.transaction_sender import NeonTxSender -from proxy.indexer.indexer_db import IndexerDB +from proxy.common_neon.solana_interactor import SolanaInteractor from proxy.memdb.memdb import MemDB class TestNeonTxSender(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.solana = SolanaClient(os.environ['SOLANA_URL']) + cls.solana = SolanaInteractor(os.environ['SOLANA_URL']) def setUp(self) -> None: trx = EthTrx.fromString(bytearray.fromhex('f8678080843ade68b194f0dafe87532d4373453b2555c644390e1b99e84c8459682f0080820102a00193e1966a82c5597942370980fb78080901ca86eb3c1b25ec600b2760cfcc94a03efcc1169e161f9a148fd4586e0bcf880648ca74075bfa7a9acc8800614fc9ff')) diff --git a/proxy/testing/test_permission_token.py b/proxy/testing/test_permission_token.py index 1259ed18f..bd609da27 100644 --- a/proxy/testing/test_permission_token.py +++ b/proxy/testing/test_permission_token.py @@ -2,9 +2,9 @@ import unittest from solana.rpc.api import Client as SolanaClient from proxy.common_neon.permission_token import PermissionToken +from proxy.common_neon.solana_interactor import SolanaInteractor from solana.publickey import PublicKey from solana.account import Account as SolanaAccount -import json from web3 import Web3 from solana.rpc.commitment import Confirmed @@ -14,7 +14,7 @@ class TestPermissionToken(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - cls.solana = SolanaClient(os.environ['SOLANA_URL']) + cls.solana = SolanaInteractor(os.environ['SOLANA_URL']) cls.mint_authority_file = "/spl/bin/evm_loader-keypair.json" proxy_url = os.environ['PROXY_URL'] cls.proxy = Web3(Web3.HTTPProvider(proxy_url)) @@ -22,7 +22,8 @@ def setUpClass(cls) -> None: request_airdrop(cls.eth_account.address) cls.payer = SolanaAccount() - cls.solana.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) + client = SolanaClient(os.environ['SOLANA_URL']) + client.request_airdrop(cls.payer.public_key(), 1000_000_000_000, Confirmed) cls.allowance_token = PermissionToken(cls.solana, PublicKey(os.environ['NEON_PERMISSION_ALLOWANCE_TOKEN']), cls.payer) @@ -31,13 +32,13 @@ def setUpClass(cls) -> None: PublicKey(os.environ['NEON_PERMISSION_DENIAL_TOKEN']), cls.payer) - def test_get_balance_non_existing_account(self): - """ - Should return zero balance for non existing token-account - """ - new_acc = self.proxy.eth.account.create(f'test_get_balance_non_existing_account') - self.assertEqual(self.allowance_token.get_balance(new_acc.address), 0) - self.assertEqual(self.denial_token.get_balance(new_acc.address), 0) + # def test_get_balance_non_existing_account(self): + # """ + # Should return zero balance for non existing token-account + # """ + # new_acc = self.proxy.eth.account.create(f'test_get_balance_non_existing_account') + # self.assertEqual(self.allowance_token.get_balance(new_acc.address), 0) + # self.assertEqual(self.denial_token.get_balance(new_acc.address), 0) def test_mint_permission_tokens(self): """ diff --git a/proxy/testing/test_pyth_network_client.py b/proxy/testing/test_pyth_network_client.py index 1a03019da..e5ca9c9de 100644 --- a/proxy/testing/test_pyth_network_client.py +++ b/proxy/testing/test_pyth_network_client.py @@ -1,14 +1,14 @@ import unittest -from unittest.mock import patch, ANY, call +from unittest.mock import patch, call from proxy.indexer.pythnetwork import PythNetworkClient -from solana.rpc.api import Client as SolanaClient +from proxy.common_neon.solana_interactor import SolanaInteractor from solana.publickey import PublicKey from time import sleep from decimal import Decimal # Will perform tests with devnet network # CI Airdropper that is already running in parallel (see docker-compose-test.yml) -# uses mainnet-beta. +# uses mainnet-beta. # PythNetworkClient will fail with 'too many requests' if trying to connect # it to the same Solana network solana_url = "https://api.devnet.solana.com" @@ -57,7 +57,7 @@ def setUpClass(cls) -> None: 'status': 1 } - cls.testee = PythNetworkClient(SolanaClient(solana_url)) + cls.testee = PythNetworkClient(SolanaInteractor(solana_url)) def update_mapping(self): self.testee.update_mapping(mapping_account) @@ -67,9 +67,9 @@ def update_mapping(self): @patch.object(PythNetworkClient, 'parse_mapping_account') @patch.object(PythNetworkClient, 'parse_prod_account') @patch.object(PythNetworkClient, 'parse_price_account') - def test_success_update_mapping(self, + def test_success_update_mapping(self, mock_parse_price_account, - mock_parse_prod_account, + mock_parse_prod_account, mock_parse_mapping_account, mock_read_pyth_acct_data): ''' @@ -97,9 +97,9 @@ def test_success_update_mapping(self, @patch.object(PythNetworkClient, 'parse_mapping_account') @patch.object(PythNetworkClient, 'parse_prod_account') @patch.object(PythNetworkClient, 'parse_price_account') - def test_continue_when_failed_prod_account(self, + def test_continue_when_failed_prod_account(self, mock_parse_price_account, - mock_parse_prod_account, + mock_parse_prod_account, mock_parse_mapping_account, mock_read_pyth_acct_data): ''' @@ -115,7 +115,7 @@ def test_continue_when_failed_prod_account(self, with self.assertRaises(Exception): # get_price for 1st product should fail self.assertEqual(self.testee.get_price(self.prod1_symbol), self.prod1_price_data) - + self.assertEqual(self.testee.get_price(self.prod2_symbol), self.prod2_price_data) mock_parse_mapping_account.assert_called_once_with(mapping_account) @@ -126,7 +126,7 @@ def test_continue_when_failed_prod_account(self, self.fail(f"Expected not throws exception but it does: {err}") - @patch.object(SolanaClient, 'get_account_info') + @patch.object(SolanaInteractor, 'get_account_info') def test_forward_exception_when_reading_mapping_account(self, mock_get_account_info): mock_get_account_info.side_effect = Exception('TestException') with self.assertRaises(Exception): From d61b87c1702968328bb1259f37453a62f0e0f0fc Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sat, 19 Feb 2022 22:42:13 +0700 Subject: [PATCH 03/16] Add installing of solcx to Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95ae8fcdf..05e0fbe74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,9 @@ RUN apt update && \ pip3 install --upgrade pip && \ /bin/bash -c "source venv/bin/activate" && \ pip install -r requirements.txt && \ + pip3 install py-solc-x && \ + python3 -c "import solcx; solcx.install_solc(version='0.7.6')" && \ apt remove -y git && \ - pip install py-solc-x && \ rm -rf /var/lib/apt/lists/* COPY --from=cli /opt/solana/bin/solana \ From b89880de6898098d0dcda3bf2ba3f70b32cde428 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sat, 19 Feb 2022 22:48:44 +0700 Subject: [PATCH 04/16] Fix Neon EVM program version --- .buildkite/steps/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index dc1524165..738ef25b4 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -4,7 +4,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) set ${SOLANA_REVISION:=v1.8.12-testnet} -set ${EVM_LOADER_REVISION:=latest} +set ${EVM_LOADER_REVISION:=v0.6.0-rc3} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally docker pull neonlabsorg/solana:${SOLANA_REVISION} From 34d667559a65d40ba080676a4ffcc1c29c269572 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 13:54:14 +0700 Subject: [PATCH 05/16] Use SolTxSender for transaction sending. --- proxy/common_neon/permission_token.py | 9 +- proxy/common_neon/solana_interactor.py | 221 ++++++++++++++++++---- proxy/common_neon/transaction_sender.py | 144 ++------------ proxy/indexer/airdropper.py | 1 - proxy/indexer/canceller.py | 35 ++-- proxy/indexer/pythnetwork.py | 2 +- proxy/plugin/solana_rest_api.py | 12 +- proxy/testing/test_airdropper.py | 47 ++--- proxy/testing/test_pyth_network_client.py | 2 +- proxy/testing/transactions.py | 16 +- 10 files changed, 255 insertions(+), 234 deletions(-) diff --git a/proxy/common_neon/permission_token.py b/proxy/common_neon/permission_token.py index 00e36e2e8..145740b86 100644 --- a/proxy/common_neon/permission_token.py +++ b/proxy/common_neon/permission_token.py @@ -5,7 +5,7 @@ from typing import Union from solana.transaction import Transaction import spl.token.instructions as spl_token -from proxy.common_neon.solana_interactor import SolanaInteractor +from proxy.common_neon.solana_interactor import SolanaInteractor, SolTxListSender from decimal import Decimal import os @@ -15,8 +15,9 @@ def __init__(self, token_mint: PublicKey, payer: SolanaAccount): self.solana = solana + self.signer = payer + self.waiter = None self.token_mint = token_mint - self.payer = payer def get_token_account_address(self, ether_addr: Union[str, EthereumAddress]): sol_addr = PublicKey(ether2program(ether_addr)[0]) @@ -35,12 +36,12 @@ def create_account_if_needed(self, txn = Transaction() create_txn = spl_token.create_associated_token_account( - payer=self.payer.public_key(), + payer=self.signer.public_key(), owner=PublicKey(ether2program(ether_addr)[0]), mint=self.token_mint ) txn.add(create_txn) - self.solana.send_multiple_transactions(self.payer, [txn], skip_preflight=True) + SolTxListSender(self, [txn], 'CreateAssociatedTokenAccount(1)', skip_preflight=True).send() return token_account def mint_to(self, diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index e82ea8327..98446bab7 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import base58 import base64 import json @@ -14,19 +16,30 @@ from solana.transaction import Transaction from itertools import zip_longest from logged_groups import logged_group -from typing import Dict, Union +from typing import Dict, Union, Any, List, NamedTuple, cast from base58 import b58decode, b58encode from .costs import update_transaction_cost from .utils import get_from_dict, SolanaBlockInfo from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, WRITE_TRANSACTION_COST_IN_DB, SKIP_PREFLIGHT from ..environment import LOG_SENDING_SOLANA_TRANSACTION, FUZZING_BLOCKHASH, CONFIRM_TIMEOUT, FINALIZED +from ..environment import RETRY_ON_FAIL from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT from ..common_neon.address import EthereumAddress, ether2program from ..common_neon.address import AccountInfo as NeonAccountInfo -from typing import Any, List, NamedTuple, cast + +class SolTxError(Exception): + def __init__(self, receipt): + self.result = receipt + error = get_error_definition_from_receipt(receipt) + if isinstance(error, list) and isinstance(error[1], str): + super().__init__(str(error[1])) + self.error = str(error[1]) + else: + super().__init__('Unknown error') + self.error = json.dumps(receipt) class AccountInfo(NamedTuple): @@ -146,7 +159,7 @@ def get_account_info(self, pubkey: PublicKey, length=256, commitment='confirmed' return AccountInfo(account_tag, lamports, owner, data) - def get_multiple_accounts_info(self, accounts: [PublicKey], length=256, commitment='confirmed') -> [AccountInfo]: + def get_account_info_list(self, accounts: [PublicKey], length=256, commitment='confirmed') -> [AccountInfo]: opts = { "encoding": "base64", "commitment": commitment, @@ -368,9 +381,8 @@ def _send_multiple_transactions(self, signer: SolanaAccount, tx_list: [Transacti response_list = self._send_rpc_batch_request('sendTransaction', request_list) return [SendResult(result=r.get('result'), error=r.get('error')) for r in response_list] - def send_multiple_transactions(self, signer, tx_list, - eth_tx=None, reason=None, waiter=None, - skip_preflight=SKIP_PREFLIGHT, preflight_commitment='confirmed') -> [{}]: + def send_multiple_transactions(self, signer: SolanaAccount, tx_list: [], waiter, + skip_preflight: bool, preflight_commitment: str) -> [{}]: send_result_list = self._send_multiple_transactions(signer, tx_list, skip_preflight, preflight_commitment) # Filter good transactions and wait the confirmations for them sign_list = [s.result for s in send_result_list if s.result] @@ -386,28 +398,8 @@ def send_multiple_transactions(self, signer, tx_list, else: receipt_list.append(confirmed_list.pop(0)) - if eth_tx and reason and WRITE_TRANSACTION_COST_IN_DB: - for receipt in receipt_list: - update_transaction_cost(receipt, eth_tx, reason) - return receipt_list - # Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in - # .buildkite/steps/deploy-test.sh) - def get_measurements(self, reason, eth_tx, receipt): - if not LOG_SENDING_SOLANA_TRANSACTION: - return - - try: - self.debug(f"send multiple transactions for reason {reason}") - - measurements = self._extract_measurements_from_receipt(receipt) - for m in measurements: - self.info(f'get_measurements: {json.dumps(m)}') - except Exception as err: - self.error(f"get_measurements: can't get measurements {err}") - self.info(f"get measurements: failed result {json.dumps(receipt, indent=3)}") - def _confirm_multiple_transactions(self, sign_list: [str], waiter=None): """Confirm a transaction.""" if not len(sign_list): @@ -459,6 +451,157 @@ def _get_multiple_receipts(self, sign_list: [str]) -> [Any]: response_list = self._send_rpc_batch_request("getTransaction", request_list) return [r['result'] for r in response_list] + +@logged_group("neon.Proxy") +class SolTxListSender: + def __init__(self, sender, tx_list: [Transaction], name: str, + skip_preflight=SKIP_PREFLIGHT, preflight_commitment='confirmed'): + self._s = sender + self._name = name + self._skip_preflight = skip_preflight + self._preflight_commitment = preflight_commitment + + self._blockhash = None + self._retry_idx = 0 + self._tx_list = tx_list + self._bad_block_list = [] + self._blocked_account_list = [] + self._pending_list = [] + self._budget_exceeded_list = [] + self._storage_bad_status_list = [] + + self._all_tx_list = [self._bad_block_list, + self._blocked_account_list, + self._budget_exceeded_list, + self._pending_list] + + def clear(self): + self._tx_list.clear() + for lst in self._all_tx_list: + lst.clear() + + def _get_full_list(self): + return [tx for lst in self._all_tx_list for tx in lst] + + def send(self) -> SolTxListSender: + solana = self._s.solana + signer = self._s.signer + waiter = self._s.waiter + skip = self._skip_preflight + commitment = self._preflight_commitment + + self.debug(f'start transactions sending: {self._name}') + + while (self._retry_idx < RETRY_ON_FAIL) and (len(self._tx_list)): + self._retry_idx += 1 + receipt_list = solana.send_multiple_transactions(signer, self._tx_list, waiter, skip, commitment) + self.update_transaction_cost(receipt_list) + + success_cnt = 0 + for receipt, tx in zip(receipt_list, self._tx_list): + if check_if_blockhash_notfound(receipt): + self._bad_block_list.append(tx) + elif check_if_accounts_blocked(receipt): + self._blocked_account_list.append(tx) + elif check_for_errors(receipt): + if check_if_program_exceeded_instructions(receipt): + self._budget_exceeded_list.append(tx) + else: + custom = check_if_storage_is_empty_error(receipt) + if custom in (1, 4): + self._storage_bad_status_list.append(receipt) + else: + raise SolTxError(receipt) + else: + success_cnt += 1 + self._on_success_send(tx, receipt) + + self.debug(f'retry {self._retry_idx}, ' + + f'total receipts {len(receipt_list)}, ' + + f'success receipts {success_cnt}, ' + + f'bad blocks {len(self._bad_block_list)}, ' + + f'blocked accounts {len(self._blocked_account_list)}, ' + + f'budget exceeded {len(self._budget_exceeded_list)}, ' + + f'bad storage: {len(self._storage_bad_status_list)}') + + self._on_post_send() + + if len(self._tx_list): + raise RuntimeError('Run out of attempts to execute transaction') + return self + + def update_transaction_cost(self, receipt_list): + if not WRITE_TRANSACTION_COST_IN_DB: + return False + if not hasattr(self._s, 'eth_tx'): + return False + + for receipt in receipt_list: + update_transaction_cost(receipt, self._s.eth_tx, reason=self._name) + + def _on_success_send(self, tx: Transaction, receipt: {}) -> bool: + """Store the last successfully blockhash and set it in _set_tx_blockhash""" + self._blockhash = tx.recent_blockhash + return False + + def _on_post_send(self): + if len(self._storage_bad_status_list): + raise SolTxError(self._storage_bad_status_list[0]) + elif len(self._budget_exceeded_list): + raise RuntimeError(COMPUTATION_BUDGET_EXCEEDED) + + # There is no more retries to send transactions + if self._retry_idx >= RETRY_ON_FAIL: + if not self._is_canceled: + self._cancel() + return + + if len(self._blocked_account_list): + time.sleep(0.4) # one block time + + # force changing of recent_blockhash if Solana doesn't accept the current one + if len(self._bad_block_list): + self._blockhash = None + + # resend not-accepted transactions + self._move_txlist() + + def _set_tx_blockhash(self, tx): + """Try to keep the branch of block history""" + tx.recent_blockhash = self._blockhash + tx.signatures.clear() + + def _move_txlist(self): + full_list = self._get_full_list() + self.clear() + for tx in full_list: + self._set_tx_blockhash(tx) + self._tx_list.append(tx) + if len(self._tx_list): + self.debug(f' Resend Solana transactions: {len(self._tx_list)}') + + +@logged_group("neon.Proxy") +class Measurements: + def __init__(self): + pass + + # Do not change headers in info logs! This name used in CI measurements (see function `cleanup_docker` in + # .buildkite/steps/deploy-test.sh) + def extract(self, reason: str, receipt: {}): + if not LOG_SENDING_SOLANA_TRANSACTION: + return + + try: + self.debug(f"send multiple transactions for reason {reason}") + + measurements = self._extract_measurements_from_receipt(receipt) + for m in measurements: + self.info(f'get_measurements: {json.dumps(m)}') + except Exception as err: + self.error(f"get_measurements: can't get measurements {err}") + self.info(f"get measurements: failed result {json.dumps(receipt, indent=3)}") + def _extract_measurements_from_receipt(self, receipt): if check_for_errors(receipt): self.warning("Can't get measurements from receipt with error") @@ -483,37 +626,39 @@ def _extract_measurements_from_receipt(self, receipt): res = pattern.match(log) if res: (program, reason) = res.groups() - if reason == 'invoke [1]': messages.append({'program':program,'logs':[]}) + if reason == 'invoke [1]': messages.append({'program': program, 'logs': []}) messages[-1]['logs'].append(log) for instr in instructions: if instr['program'] in ('KeccakSecp256k11111111111111111111111111111',): continue if messages[0]['program'] != instr['program']: - raise ValueError('Invalid program in log messages: expect %s, actual %s' % (messages[0]['program'], instr['program'])) + raise ValueError('Invalid program in log messages: expect %s, actual %s' % ( + messages[0]['program'], instr['program'])) instr['logs'] = messages.pop(0)['logs'] - exit_result = re.match(r'Program %s (success)'%instr['program'], instr['logs'][-1]) + exit_result = re.match(r'Program %s (success)' % instr['program'], instr['logs'][-1]) if not exit_result: raise ValueError("Can't get exit result") instr['result'] = exit_result.group(1) if instr['program'] == EVM_LOADER_ID: memory_result = re.match(r'Program log: Total memory occupied: ([0-9]+)', instr['logs'][-3]) - instruction_result = re.match(r'Program %s consumed ([0-9]+) of ([0-9]+) compute units'%instr['program'], instr['logs'][-2]) + instruction_result = re.match( + r'Program %s consumed ([0-9]+) of ([0-9]+) compute units' % instr['program'], instr['logs'][-2]) if not (memory_result and instruction_result): raise ValueError("Can't parse measurements for evm_loader") instr['measurements'] = { - 'instructions': instruction_result.group(1), - 'memory': memory_result.group(1) - } + 'instructions': instruction_result.group(1), + 'memory': memory_result.group(1) + } result = [] for instr in instructions: if instr['program'] == EVM_LOADER_ID: result.append({ - 'program':instr['program'], - 'measurements':instr['measurements'], - 'result':instr['result'], - 'data':instr['data'] - }) + 'program': instr['program'], + 'measurements': instr['measurements'], + 'result': instr['result'], + 'data': instr['data'] + }) return result diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 0ca9cb94e..9d9fdcace 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -23,30 +23,17 @@ from .emulator_interactor import call_emulated from .neon_instruction import NeonInstruction as NeonIxBuilder from .solana_interactor import COMPUTATION_BUDGET_EXCEEDED -from .solana_interactor import SolanaInteractor, check_for_errors, check_if_accounts_blocked +from .solana_interactor import SolanaInteractor, Measurements, SolTxListSender, SolTxError from .solana_interactor import check_if_big_transaction, check_if_program_exceeded_instructions -from .solana_interactor import get_error_definition_from_receipt, check_if_storage_is_empty_error -from .solana_interactor import check_if_blockhash_notfound from ..common_neon.eth_proto import Trx as EthTx from ..common_neon.utils import NeonTxResultInfo, NeonTxInfo -from ..environment import RETRY_ON_FAIL, EVM_LOADER_ID, PERM_ACCOUNT_LIMIT, ACCOUNT_PERMISSION_UPDATE_INT, MIN_OPERATOR_BALANCE_TO_WARN, MIN_OPERATOR_BALANCE_TO_ERR +from ..environment import RETRY_ON_FAIL, EVM_LOADER_ID, PERM_ACCOUNT_LIMIT, ACCOUNT_PERMISSION_UPDATE_INT +from ..environment import MIN_OPERATOR_BALANCE_TO_WARN, MIN_OPERATOR_BALANCE_TO_ERR from ..memdb.memdb import MemDB, NeonPendingTxInfo from ..environment import get_solana_accounts from ..common_neon.account_whitelist import AccountWhitelist -class SolanaTxError(Exception): - def __init__(self, receipt): - self.result = receipt - error = get_error_definition_from_receipt(receipt) - if isinstance(error, list) and isinstance(error[1], str): - super().__init__(str(error[1])) - self.error = str(error[1]) - else: - super().__init__('Unknown error') - self.error = json.dumps(receipt) - - class NeonTxStage(metaclass=abc.ABCMeta): NAME = 'UNKNOWN' @@ -335,7 +322,7 @@ def _create_perm_accounts(self, seed_list): stage_list = [NeonCreatePermAccount(self._s, seed, STORAGE_SIZE) for seed in seed_list] account_list = [s.sol_account for s in stage_list] - info_list = self._s.solana.get_multiple_accounts_info(account_list) + info_list = self._s.solana.get_account_info_list(account_list) balance = self._s.solana.get_multiple_rent_exempt_balances_for_size([STORAGE_SIZE])[0] for account, stage in zip(info_list, stage_list): if not account: @@ -387,9 +374,11 @@ def __init__(self, db: MemDB, solana: SolanaInteractor, eth_tx: EthTx, steps: in self.eth_tx = eth_tx self.neon_sign = '0x' + eth_tx.hash_signed().hex() self.steps = steps + self.waiter = self self.solana = solana self._resource_list = OperatorResourceList(self) self.resource = None + self.signer = None self.operator_key = None self.builder = None @@ -420,6 +409,7 @@ def execute(self) -> NeonTxResultInfo: def set_resource(self, resource: Optional[OperatorResourceInfo]): self.resource = resource + self.signer = resource.signer self.operator_key = resource.public_key() self.builder = NeonIxBuilder(self.operator_key) @@ -490,7 +480,7 @@ def _execute(self): self.error(f'No strategy to execute the Neon transaction: {self.eth_tx}') raise RuntimeError('No strategy to execute the Neon transaction') - def on_wait_confirm(self, slot: int): + def on_wait_confirm(self, _, slot: int): self._pend_tx_into_db(slot) def _pend_tx_into_db(self, slot: int): @@ -620,116 +610,6 @@ def done_account_txs(self, skip_create_accounts=False): self.create_account_tx.instructions.clear() -@logged_group("neon.Proxy") -class SolTxListSender: - def __init__(self, sender: NeonTxSender, tx_list: [Transaction], name: str): - self._s = sender - self._name = name - - self._blockhash = None - self._retry_idx = 0 - self._tx_list = tx_list - self._bad_block_list = [] - self._blocked_account_list = [] - self._pending_list = [] - self._budget_exceeded_list = [] - self._storage_bad_status_list = [] - - self._all_tx_list = [self._bad_block_list, - self._blocked_account_list, - self._budget_exceeded_list, - self._pending_list] - - def clear(self): - self._tx_list.clear() - for lst in self._all_tx_list: - lst.clear() - - def _get_full_list(self): - return [tx for lst in self._all_tx_list for tx in lst] - - def send(self) -> SolTxListSender: - solana = self._s.solana - eth_tx = self._s.eth_tx - signer = self._s.resource.signer - - self.debug(f'Start stage: {self._name}') - - while (self._retry_idx < RETRY_ON_FAIL) and (len(self._tx_list)): - self._retry_idx += 1 - receipt_list = solana.send_multiple_transactions(signer, self._tx_list, eth_tx, self._name, self) - - success_cnt = 0 - for receipt, tx in zip(receipt_list, self._tx_list): - if check_if_blockhash_notfound(receipt): - self._bad_block_list.append(tx) - elif check_if_accounts_blocked(receipt): - self._blocked_account_list.append(tx) - elif check_for_errors(receipt): - if check_if_program_exceeded_instructions(receipt): - self._budget_exceeded_list.append(tx) - else: - custom = check_if_storage_is_empty_error(receipt) - if custom in (1, 4): - self._storage_bad_status_list.append(receipt) - else: - raise SolanaTxError(receipt) - else: - success_cnt += 1 - self._on_success_send(tx, receipt) - - self.debug(f'retry {self._retry_idx}, ' + - f'total receipts {len(receipt_list)}, ' + - f'success receipts {success_cnt}, ' + - f'bad blocks {len(self._bad_block_list)}, ' + - f'blocked accounts {len(self._blocked_account_list)}, ' + - f'budget exceeded {len(self._budget_exceeded_list)}, ' + - f'bad storage: {len(self._storage_bad_status_list)}') - - self._on_post_send() - - if len(self._tx_list): - raise RuntimeError('Run out of attempts to execute transaction') - return self - - def on_wait_confirm(self, _, slot: int): - self._s.on_wait_confirm(slot) - - def _on_success_send(self, tx: Transaction, receipt: {}): - """Store the last successfully blockhash and set it in _set_tx_blockhash""" - self._blockhash = tx.recent_blockhash - - def _on_post_send(self): - if len(self._storage_bad_status_list): - raise SolanaTxError(self._storage_bad_status_list[0]) - elif len(self._budget_exceeded_list): - raise RuntimeError(COMPUTATION_BUDGET_EXCEEDED) - - if len(self._blocked_account_list): - time.sleep(0.4) # one block time - - # force changing of recent_blockhash if Solana doesn't accept the current one - if len(self._bad_block_list): - self._blockhash = None - - # resend not-accepted transactions - self._move_txlist() - - def _set_tx_blockhash(self, tx): - """Try to keep the branch of block history""" - tx.recent_blockhash = self._blockhash - tx.signatures.clear() - - def _move_txlist(self): - full_list = self._get_full_list() - self.clear() - for tx in full_list: - self._set_tx_blockhash(tx) - self._tx_list.append(tx) - if len(self._tx_list): - self.debug(f' Resend Solana transactions: {len(self._tx_list)}') - - @logged_group("neon.Proxy") class BaseNeonTxStrategy(metaclass=abc.ABCMeta): NAME = 'UNKNOWN STRATEGY' @@ -786,8 +666,7 @@ def __init__(self, strategy: BaseNeonTxStrategy, *args, **kwargs): def _on_success_send(self, tx: Transaction, receipt: {}): if not self.neon_res.is_valid(): if self.neon_res.decode(self._s.neon_sign, receipt).is_valid(): - self._s.solana.get_measurements(self._name, self._s.eth_tx, receipt) - + Measurements().extract(self._name, receipt) super()._on_success_send(tx, receipt) def _on_post_send(self): @@ -887,6 +766,7 @@ def _on_success_send(self, tx: Transaction, receipt: {}): if self._is_canceled: # Transaction with cancel is confirmed self.neon_res.canceled(receipt) + Measurements().extract(self._name, receipt) else: super()._on_success_send(tx, receipt) @@ -897,14 +777,14 @@ def _on_post_send(self): return self.clear() # There is no more retries to send transactions - if self._retry_idx == RETRY_ON_FAIL: + if self._retry_idx >= RETRY_ON_FAIL: if not self._is_canceled: self._cancel() return # The storage has bad structure and the result isn't received! (( if len(self._storage_bad_status_list): - raise SolanaTxError(self._storage_bad_status_list[0]) + raise SolTxError(self._storage_bad_status_list[0]) # Blockhash is changed ((( if len(self._bad_block_list): diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py index e5c3d9a78..0331ea288 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/indexer/airdropper.py @@ -175,7 +175,6 @@ def airdrop_to(self, eth_address, airdrop_galans): def process_trx_airdropper_mode(self, trx): if check_error(trx): return - # helper function finding all instructions that satisfies predicate def find_instructions(trx, predicate): return [instr for instr in trx['transaction']['message']['instructions'] if predicate(instr)] diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py index 86e5d31a2..1a7d516c3 100644 --- a/proxy/indexer/canceller.py +++ b/proxy/indexer/canceller.py @@ -11,7 +11,7 @@ from proxy.common_neon.constants import INCINERATOR_PUBKEY, KECCAK_PROGRAM, SYSVAR_INSTRUCTION_PUBKEY from proxy.common_neon.neon_instruction import NeonInstruction -from proxy.common_neon.solana_interactor import SolanaInteractor +from proxy.common_neon.solana_interactor import SolanaInteractor, SolTxListSender from proxy.common_neon.utils import get_from_dict from proxy.environment import ETH_TOKEN_MINT_ID, EVM_LOADER_ID, SOLANA_URL, get_solana_accounts @@ -33,14 +33,14 @@ class Canceller: def __init__(self): # Initialize user account self.signer = get_solana_accounts()[0] + self.solana = SolanaInteractor(SOLANA_URL) + self.waiter = None self._operator = self.signer.public_key() self.operator_token = get_associated_token_address(PublicKey(self._operator), ETH_TOKEN_MINT_ID) - - self._solana = SolanaInteractor(SOLANA_URL) self.builder = NeonInstruction(self._operator) - def unlock_accounts(self, blocked_storages): + tx_list = [] for storage, tx_accounts in blocked_storages.items(): (neon_tx, blocked_accounts) = tx_accounts if blocked_accounts is None: @@ -54,18 +54,17 @@ def unlock_accounts(self, blocked_storages): self.builder.init_eth_trx(neon_tx.tx, None, self.operator_token) self.builder.init_iterative(storage, None, 0) - trx = self.builder.make_cancel_transaction(keys) + tx = self.builder.make_cancel_transaction(keys) + tx_list.append(tx) + + if not len(tx_list): + return + + self.debug(f"Send Cancel: {len(tx_list)}") - self.debug(f"Send Cancel: {trx}") - try: - cancel_result = self._solana.send_multiple_transactions(self.signer, [trx], neon_tx.tx, "CancelWithNonce")[0] - self.debug(f"cancel result: {cancel_result}") - result_error = get_from_dict(cancel_result, 'meta', 'err') - if result_error: - self.error(f'Error sending cancel transaction: {result_error}') - except Exception as err: - err_tb = "".join(traceback.format_tb(err.__traceback__)) - self.error('Exception on submitting transaction. ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - else: - self.debug(f"Canceled: {blocked_accounts}") + try: + SolTxListSender(self, tx_list, f'CancelWithNonce({len(tx_list)})').send() + except Exception as err: + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.warning('Exception on submitting transaction. ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') diff --git a/proxy/indexer/pythnetwork.py b/proxy/indexer/pythnetwork.py index 4ca0e2aeb..e8c4573b3 100644 --- a/proxy/indexer/pythnetwork.py +++ b/proxy/indexer/pythnetwork.py @@ -116,7 +116,7 @@ def read_pyth_acct_data(self, acc_addrs: Union[List[PublicKey], PublicKey]): if isinstance(acc_addrs, PublicKey): acct_values = self.solana.get_account_info(acc_addrs, length=0) elif isinstance(acc_addrs, list): - acct_values = self.solana.get_multiple_accounts_info(acc_addrs, length=0) + acct_values = self.solana.get_account_info_list(acc_addrs, length=0) else: raise Exception(f'Unsupported argument to read_pyth_acct_data: {acc_addrs}') diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 225d211b9..b46852864 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -27,9 +27,8 @@ from .solana_rest_api_tools import neon_config_load, get_token_balance_or_zero, estimate_gas from ..common_neon.transaction_sender import NeonTxSender -from ..common_neon.solana_interactor import SolanaInteractor +from ..common_neon.solana_interactor import SolanaInteractor, SolTxError from ..common_neon.address import EthereumAddress -from ..common_neon.transaction_sender import SolanaTxError from ..common_neon.emulator_interactor import call_emulated from ..common_neon.errors import EthereumError, PendingTxError from ..common_neon.eth_proto import Trx as EthTrx @@ -371,7 +370,7 @@ def eth_sendRawTransaction(self, rawTrx): except PendingTxError as err: self.debug(f'{err}') return eth_signature - except SolanaTxError as err: + except SolTxError as err: err_msg = json.dumps(err.result, indent=3) self.error(f"Got SendTransactionError: {err_msg}") raise @@ -433,7 +432,7 @@ def process_request(self, request): method = getattr(self.model, request['method']) params = request.get('params', []) response['result'] = method(*params) - except SolanaTxError as err: + except SolTxError as err: # traceback.print_exc() response['error'] = err.error except EthereumError as err: @@ -489,7 +488,10 @@ def handle_request_impl(self, request: HttpParser) -> None: response = {'jsonrpc': '2.0', 'error': {'code': -32000, 'message': str(err)}} resp_time_ms = (time.time() - start_time)*1000 # convert this into milliseconds - self.info('handle_request >>> %s 0x%0x %s %s resp_time_ms= %s', threading.get_ident(), id(self.model), json.dumps(response), + self.info('handle_request >>> %s 0x%0x %s %s resp_time_ms= %s', + threading.get_ident(), + id(self.model), + json.dumps(response), request.get('method', '---'), resp_time_ms) diff --git a/proxy/testing/test_airdropper.py b/proxy/testing/test_airdropper.py index c7ec3883a..264bcf803 100644 --- a/proxy/testing/test_airdropper.py +++ b/proxy/testing/test_airdropper.py @@ -1,11 +1,11 @@ import os import unittest -from solana.rpc.api import Client as SolanaClient from solana.publickey import PublicKey from proxy.testing.mock_server import MockServer from proxy.indexer.airdropper import Airdropper, AIRDROP_AMOUNT_SOL, NEON_PRICE_USD from proxy.indexer.sql_dict import SQLDict +from proxy.common_neon.solana_interactor import SolanaInteractor import time from flask import request, Response from unittest.mock import Mock, MagicMock, patch, ANY @@ -13,6 +13,10 @@ import itertools from proxy.testing.transactions import pre_token_airdrop_trx, wrapper_whitelist, evm_loader_addr, token_airdrop_address + +SOLANA_URL = os.environ.get("SOLANA_URL", "http://solana:8899") + + class MockFaucet(MockServer): def __init__(self, port): super().__init__(port) @@ -54,7 +58,7 @@ def create_price_info(valid_slot: int, price: Decimal, conf: Decimal): class Test_Airdropper(unittest.TestCase): def create_airdropper(self, start_slot): os.environ['START_SLOT'] = str(start_slot) - return Airdropper(solana_url =f'http://{self.address}:8899', + return Airdropper(solana_url =SOLANA_URL, pyth_mapping_account=self.pyth_mapping_account, faucet_url =f'http://{self.address}:{self.faucet_port}', wrapper_whitelist =self.wrapper_whitelist, @@ -62,7 +66,7 @@ def create_airdropper(self, start_slot): @classmethod @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def setUpClass(cls, mock_get_slot, mock_dict_get) -> None: print("testing indexer in airdropper mode") cls.address = 'localhost' @@ -72,7 +76,7 @@ def setUpClass(cls, mock_get_slot, mock_dict_get) -> None: cls.wrapper_whitelist = wrapper_whitelist cls.neon_decimals = 9 cls.airdropper = cls.create_airdropper(cls, 0) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() cls.airdropper.always_reload_price = True @@ -91,7 +95,6 @@ def setUpClass(cls, mock_get_slot, mock_dict_get) -> None: cls.mock_failed_attempts.airdrop_failed = MagicMock() cls.airdropper.failed_attempts = cls.mock_failed_attempts - def setUp(self) -> None: print(f"\n\n{self._testMethodName}\n{self._testMethodDoc}") self.faucet = MockFaucet(self.faucet_port) @@ -99,7 +102,6 @@ def setUp(self) -> None: self.airdropper.last_update_pyth_mapping = None time.sleep(0.2) - def tearDown(self) -> None: self.faucet.shutdown_server() self.faucet.join() @@ -110,7 +112,6 @@ def tearDown(self) -> None: self.mock_pyth_client.update_mapping.reset_mock() self.mock_failed_attempts.airdrop_failed.reset_mock() - def test_failed_process_trx_with_one_airdrop_price_provider_error(self): """ Should not airdrop to new address due to price provider error @@ -130,7 +131,6 @@ def test_failed_process_trx_with_one_airdrop_price_provider_error(self): self.mock_pyth_client.get_price.assert_called_once_with('Crypto.SOL/USD') self.faucet.request_neon_in_galans_mock.assert_not_called() - @patch.object(Airdropper, 'is_allowed_wrapper_contract') def test_failed_airdrop_contract_not_in_whitelist(self, mock_is_allowed_contract): """ @@ -157,7 +157,6 @@ def test_failed_airdrop_contract_not_in_whitelist(self, mock_is_allowed_contract self.mock_airdrop_ready.register_airdrop.assert_not_called() self.faucet.request_neon_in_galans_mock.assert_not_called() - def test_faucet_failure(self): """ Should not add address to processed list due to faucet error @@ -186,7 +185,6 @@ def test_faucet_failure(self): json_req = {'wallet': token_airdrop_address, 'amount': airdrop_amount} self.faucet.request_neon_in_galans_mock.assert_called_once_with(json_req) - def test_process_trx_with_one_airdrop_for_already_processed_address(self): """ Should not airdrop to repeated address @@ -211,7 +209,6 @@ def test_process_trx_with_one_airdrop_for_already_processed_address(self): self.mock_airdrop_ready.register_airdrop.assert_not_called() self.faucet.request_neon_in_galans_mock.assert_not_called() - def test_failed_airdrop_confidence_interval_too_large(self): """ Should not airdrop because confidence interval too large @@ -233,7 +230,6 @@ def test_failed_airdrop_confidence_interval_too_large(self): self.mock_airdrop_ready.register_airdrop.assert_not_called() self.faucet.request_neon_in_galans_mock.assert_not_called() - def test_update_mapping_error(self): self.mock_pyth_client.update_mapping.side_effect = [Exception('TestException')] try: @@ -243,7 +239,6 @@ def test_update_mapping_error(self): except Exception as err: self.fail(f'Excpected not throws exception but it does: {err}') - def test_get_price_error(self): self.mock_pyth_client.get_price.side_effect = [Exception('TestException')] try: @@ -254,69 +249,67 @@ def test_get_price_error(self): self.fail(f'Excpected not throws exception but it does: {err}') @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_continue(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('CONTINUE') self.assertEqual(new_airdropper.latest_processed_slot, start_slot - 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_continue_recent_slot_not_found(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [None] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('CONTINUE') self.assertEqual(new_airdropper.latest_processed_slot, start_slot + 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() - @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_start_slot_parse_error(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('Wrong value') self.assertEqual(new_airdropper.latest_processed_slot, start_slot - 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() - @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_latest(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper('LATEST') self.assertEqual(new_airdropper.latest_processed_slot, start_slot + 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_slot_number(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper(str(start_slot)) self.assertEqual(new_airdropper.latest_processed_slot, start_slot) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() @patch.object(SQLDict, 'get') - @patch.object(SolanaClient, 'get_slot') + @patch.object(SolanaInteractor, 'get_slot') def test_init_airdropper_big_slot_number(self, mock_get_slot, mock_dict_get): start_slot = 1234 mock_dict_get.side_effect = [start_slot - 1] mock_get_slot.side_effect = [{'result': start_slot + 1}] new_airdropper = self.create_airdropper(str(start_slot + 100)) self.assertEqual(new_airdropper.latest_processed_slot, start_slot + 1) - mock_get_slot.assert_called_once_with(commitment='finalized') + mock_get_slot.assert_called_once_with('finalized') mock_dict_get.assert_called() diff --git a/proxy/testing/test_pyth_network_client.py b/proxy/testing/test_pyth_network_client.py index e5ca9c9de..3d12ad25e 100644 --- a/proxy/testing/test_pyth_network_client.py +++ b/proxy/testing/test_pyth_network_client.py @@ -131,7 +131,7 @@ def test_forward_exception_when_reading_mapping_account(self, mock_get_account_i mock_get_account_info.side_effect = Exception('TestException') with self.assertRaises(Exception): self.update_mapping() - mock_get_account_info.assert_called_once_with(mapping_account) + mock_get_account_info.assert_called_once_with(mapping_account, length=0) def test_integration_success_read_price(self): diff --git a/proxy/testing/transactions.py b/proxy/testing/transactions.py index b5eec0e2d..3ff73b0ef 100644 --- a/proxy/testing/transactions.py +++ b/proxy/testing/transactions.py @@ -1,6 +1,8 @@ +from ..environment import EVM_LOADER_ID + token_program = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' -evm_loader_addr = 'eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU' +evm_loader_addr = EVM_LOADER_ID erc20_wrapper = '5H7kvhPD7GECAmf227vTPYTS7SC2PmyuVZaT5zVTx7vb' wrapper_whitelist = [erc20_wrapper] @@ -65,7 +67,7 @@ } ], 'logMessages': [ - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + f'Program {evm_loader_addr} invoke [1]', 'Program 11111111111111111111111111111111 invoke [2]', 'Program 11111111111111111111111111111111 success', 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [2]', @@ -86,9 +88,9 @@ 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 24626 of 485359 compute units', 'Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success', 'Program log: Total memory occupied: 1414', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 40680 of 500000 compute units', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU invoke [1]', + f'Program {evm_loader_addr} consumed 40680 of 500000 compute units', + f'Program {evm_loader_addr} success', + f'Program {evm_loader_addr} invoke [1]', 'Program 11111111111111111111111111111111 invoke [2]', 'Program 11111111111111111111111111111111 success', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]', @@ -96,8 +98,8 @@ 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3412 of 486179 compute units', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success', 'Program log: Total memory occupied: 1536', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU consumed 18381 of 500000 compute units', - 'Program eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU success', + f'Program {evm_loader_addr} consumed 18381 of 500000 compute units', + f'Program {evm_loader_addr} success', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]', 'Program log: Instruction: Transfer', 'Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 3120 of 200000 compute units', From fea297df45185b9cba578978fd18967d25c04cdf Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 14:44:10 +0700 Subject: [PATCH 06/16] Fix exeception on erasing not-exist key --- proxy/indexer/airdropper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py index 0331ea288..9a37501b4 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/indexer/airdropper.py @@ -276,7 +276,8 @@ def process_scheduled_trxs(self): }) for eth_address in success_addresses: - del self.airdrop_scheduled[eth_address] + if eth_address in self.airdrop_scheduled: + del self.airdrop_scheduled[eth_address] def process_functions(self): """ From f799d9472a296f197e8074e0696fe109f81c2204 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 14:44:49 +0700 Subject: [PATCH 07/16] Add reconnect on disconnect from Solana node --- proxy/common_neon/solana_interactor.py | 51 ++++++++++++++++++++++--- proxy/common_neon/transaction_sender.py | 4 ++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 98446bab7..59635eb73 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -5,6 +5,7 @@ import json import re import time +import traceback from typing import Optional @@ -67,9 +68,30 @@ def _make_request(self, request) -> RPCResponse: "Content-Type": "application/json" } client = self._client - raw_response = client.session.post(client.endpoint_uri, headers=headers, json=request) - raw_response.raise_for_status() - return raw_response + + retry = 0 + while True: + try: + retry += 1 + raw_response = client.session.post(client.endpoint_uri, headers=headers, json=request) + raw_response.raise_for_status() + return raw_response + + except ConnectionError as err: + if retry > RETRY_ON_FAIL: + raise + + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.error(f'ConnectionError({retry}) on send request to Solana. ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + time.sleep(1) + + except Exception as err: + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.error('Unknown exception on send request to Solana. ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + raise + def _send_rpc_request(self, method: str, *params: Any) -> RPCResponse: request_id = next(self._client._request_counter) + 1 @@ -463,14 +485,17 @@ def __init__(self, sender, tx_list: [Transaction], name: str, self._blockhash = None self._retry_idx = 0 + self._slots_behind = 0 self._tx_list = tx_list + self._node_behind_list = [] self._bad_block_list = [] self._blocked_account_list = [] self._pending_list = [] self._budget_exceeded_list = [] self._storage_bad_status_list = [] - self._all_tx_list = [self._bad_block_list, + self._all_tx_list = [self._node_behind_list, + self._bad_block_list, self._blocked_account_list, self._budget_exceeded_list, self._pending_list] @@ -494,12 +519,18 @@ def send(self) -> SolTxListSender: while (self._retry_idx < RETRY_ON_FAIL) and (len(self._tx_list)): self._retry_idx += 1 + self._slots_behind = 0 + receipt_list = solana.send_multiple_transactions(signer, self._tx_list, waiter, skip, commitment) self.update_transaction_cost(receipt_list) success_cnt = 0 for receipt, tx in zip(receipt_list, self._tx_list): - if check_if_blockhash_notfound(receipt): + slots_behind = check_if_node_behind(receipt) + if slots_behind: + self._slots_behind = slots_behind + self._node_behind_list.append(tx) + elif check_if_blockhash_notfound(receipt): self._bad_block_list.append(tx) elif check_if_accounts_blocked(receipt): self._blocked_account_list.append(tx) @@ -519,6 +550,7 @@ def send(self) -> SolTxListSender: self.debug(f'retry {self._retry_idx}, ' + f'total receipts {len(receipt_list)}, ' + f'success receipts {success_cnt}, ' + + f'node behind {len(self._node_behind_list)}, ' f'bad blocks {len(self._bad_block_list)}, ' + f'blocked accounts {len(self._blocked_account_list)}, ' + f'budget exceeded {len(self._budget_exceeded_list)}, ' + @@ -545,7 +577,10 @@ def _on_success_send(self, tx: Transaction, receipt: {}) -> bool: return False def _on_post_send(self): - if len(self._storage_bad_status_list): + if len(self._node_behind_list): + self.warning(f'Node is behind by {self._slots_behind} slots') + time.sleep(1) + elif len(self._storage_bad_status_list): raise SolTxError(self._storage_bad_status_list[0]) elif len(self._budget_exceeded_list): raise RuntimeError(COMPUTATION_BUDGET_EXCEEDED) @@ -767,3 +802,7 @@ def check_if_accounts_blocked(receipt, *, logger): def check_if_blockhash_notfound(receipt): return (not receipt) or (get_from_dict(receipt, 'data', 'err') == 'BlockhashNotFound') + + +def check_if_node_behind(receipt): + return get_from_dict(receipt, 'data', 'numSlotsBehind') diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 9d9fdcace..62262661d 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -776,6 +776,10 @@ def _on_post_send(self): self.debug(f'Got Neon tx {"cancel" if self._is_canceled else "result"}: {self.neon_res}') return self.clear() + if len(self._node_behind_list): + self.warning(f'Node is behind by {self._slots_behind} slots') + time.sleep(1) + # There is no more retries to send transactions if self._retry_idx >= RETRY_ON_FAIL: if not self._is_canceled: From cd676927ac9a6529fde1edadfe3f62dc2fc56c50 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 15:32:50 +0700 Subject: [PATCH 08/16] Use cache of existing blocks --- proxy/common_neon/solana_interactor.py | 24 +++++++------ proxy/memdb/blocks_db.py | 49 +++++++++++++++----------- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 59635eb73..08100b586 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -310,16 +310,20 @@ def get_block_info_list(self, block_slot_list: [int], commitment='confirmed') -> response_list = self._send_rpc_batch_request('getBlock', request_list) for slot, response in zip(block_slot_list, response_list): if (not response) or ('result' not in response): - continue - net_block = response['result'] - block = SolanaBlockInfo( - slot=slot, - finalized=(commitment == FINALIZED), - hash='0x' + base58.b58decode(net_block['blockhash']).hex(), - parent_hash='0x' + base58.b58decode(net_block['previousBlockhash']).hex(), - time=net_block['blockTime'], - signs=net_block['signatures'] - ) + block = SolanaBlockInfo( + slot=slot, + finalized=(commitment == FINALIZED), + ) + else: + net_block = response['result'] + block = SolanaBlockInfo( + slot=slot, + finalized=(commitment == FINALIZED), + hash='0x' + base58.b58decode(net_block['blockhash']).hex(), + parent_hash='0x' + base58.b58decode(net_block['previousBlockhash']).hex(), + time=net_block['blockTime'], + signs=net_block['signatures'] + ) block_list.append(block) return block_list diff --git a/proxy/memdb/blocks_db.py b/proxy/memdb/blocks_db.py index b85ea5af0..34f3f25ae 100644 --- a/proxy/memdb/blocks_db.py +++ b/proxy/memdb/blocks_db.py @@ -55,21 +55,33 @@ def _get_latest_db_block(self): def _get_solana_block_list(self) -> bool: slot = self.latest_db_block_slot - slot_list = [s for s in range(slot + self.BLOCK_CACHE_LIMIT, slot - 1, -1)] - self.block_list = self._b.solana.get_block_info_list(slot_list) + exist_block_dict = self._b.get_block_dict(slot) + + slot_list = [] + self.block_list = [] + block_time = 1 + for slot in range(slot + self.BLOCK_CACHE_LIMIT, slot - 1, -1): + block = exist_block_dict.get(slot) + if block is None: + slot_list.append(slot) + else: + self.block_list.append(block) + block_time = block.time + + solana_block_list = self._b.solana.get_block_info_list(slot_list) + for block in solana_block_list: + if not block.time: + # generate fake block + block.time = block_time + block.hash = '0x' + os.urandom(32).hex(), + block.parent_hash = '0x' + os.urandom(32).hex() + block_time = block.time + self.block_list.append(block) + if not len(self.block_list): return False self.latest_block = self.block_list[0] - - slot = self.latest_block.slot - latest_slot = self._b.get_latest_block_slot() - if latest_slot > slot: - slot = latest_slot - for block in self.block_list: - block.slot = slot - slot -= 1 - self.first_block = self.block_list[len(self.block_list) - 1] return len(self.block_list) > 0 @@ -225,6 +237,9 @@ def _update_block_dicts(self): if not self._request_new_block_list(): self._try_to_fill_blocks_from_pending_list() + def get_block_dict(self, from_slot: int) -> {}: + return {slot: block for slot, block in self._block_by_slot.items() if slot > from_slot} + def get_latest_block(self) -> SolanaBlockInfo: self._update_block_dicts() return self._latest_block @@ -276,21 +291,15 @@ def _generate_fake_block(self, neon_res: NeonTxResultInfo) -> SolanaBlockInfo: return block def submit_block(self, neon_res: NeonTxResultInfo) -> SolanaBlockInfo: - block_list = self.solana.get_block_info_list([neon_res.slot]) - is_new_block = False - if len(block_list): - block = block_list[0] - data = pickle.dumps(block) - else: - block = SolanaBlockInfo() - data = None + block = self.solana.get_block_info(neon_res.slot) with self._last_time.get_lock(): - if not block.slot: + if not block.time: block = self._generate_fake_block(neon_res) data = pickle.dumps(block) is_new_block = True else: + data = pickle.dumps(block) is_new_block = neon_res.slot not in self._pending_block_by_slot if is_new_block: From 2d230a341acf4ca8ab6224ca3641612f080ddc9c Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 15:42:25 +0700 Subject: [PATCH 09/16] Fix Neon EVM version in docker-compose --- proxy/docker-compose-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 9b4f3c1cc..a90e07159 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -26,7 +26,7 @@ services: evm_loader: container_name: evm_loader - image: neonlabsorg/evm_loader:${EVM_LOADER_REVISION:-latest} + image: neonlabsorg/evm_loader:${EVM_LOADER_REVISION:-v0.6.0-rc3} environment: - SOLANA_URL=http://solana:8899 networks: From 44162f6f5e7d576e12f11b74098a8e49dc363421 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 18:26:35 +0700 Subject: [PATCH 10/16] Fix block order in cache --- proxy/memdb/blocks_db.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/proxy/memdb/blocks_db.py b/proxy/memdb/blocks_db.py index 34f3f25ae..19cfacb59 100644 --- a/proxy/memdb/blocks_db.py +++ b/proxy/memdb/blocks_db.py @@ -20,7 +20,6 @@ @logged_group("neon.Proxy") class RequestSolanaBlockList: BLOCK_CACHE_LIMIT = (32 + 16) - BIG_SLOT = 1_000_000_000_000 def __init__(self, blocks_db: MemBlocksDB): self._b = blocks_db @@ -58,26 +57,32 @@ def _get_solana_block_list(self) -> bool: exist_block_dict = self._b.get_block_dict(slot) slot_list = [] - self.block_list = [] - block_time = 1 - for slot in range(slot + self.BLOCK_CACHE_LIMIT, slot - 1, -1): + block_list = [] + max_slot = max(slot + self.BLOCK_CACHE_LIMIT, self._b.get_latest_block_slot()) + for slot in range(max_slot, slot - 1, -1): block = exist_block_dict.get(slot) if block is None: slot_list.append(slot) else: - self.block_list.append(block) - block_time = block.time + block_list.append(block) + + self.block_list = [] + block_time = 0 solana_block_list = self._b.solana.get_block_info_list(slot_list) for block in solana_block_list: if not block.time: - # generate fake block + if not block_time: + continue block.time = block_time - block.hash = '0x' + os.urandom(32).hex(), + block.hash = '0x' + os.urandom(32).hex() block.parent_hash = '0x' + os.urandom(32).hex() - block_time = block.time + else: + block_time = block.time self.block_list.append(block) + self.block_list.extend(block_list) + if not len(self.block_list): return False @@ -234,8 +239,8 @@ def _try_to_fill_blocks_from_pending_list(self): self._fill_block_dicts(request) def _update_block_dicts(self): - if not self._request_new_block_list(): - self._try_to_fill_blocks_from_pending_list() + self._try_to_fill_blocks_from_pending_list() + self._request_new_block_list() def get_block_dict(self, from_slot: int) -> {}: return {slot: block for slot, block in self._block_by_slot.items() if slot > from_slot} From 1c3565192978eaba1df430226d9520116ad7a429 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 21:02:53 +0700 Subject: [PATCH 11/16] Change type of exception on connection problem --- proxy/common_neon/solana_interactor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 08100b586..467f6ec95 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -6,6 +6,7 @@ import re import time import traceback +import requests from typing import Optional @@ -77,13 +78,13 @@ def _make_request(self, request) -> RPCResponse: raw_response.raise_for_status() return raw_response - except ConnectionError as err: + except requests.exceptions.ConnectionError as err: if retry > RETRY_ON_FAIL: raise err_tb = "".join(traceback.format_tb(err.__traceback__)) self.error(f'ConnectionError({retry}) on send request to Solana. ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') time.sleep(1) except Exception as err: @@ -92,7 +93,6 @@ def _make_request(self, request) -> RPCResponse: f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') raise - def _send_rpc_request(self, method: str, *params: Any) -> RPCResponse: request_id = next(self._client._request_counter) + 1 From 263eb27ea99285d71f16cf27ec5c86df5086080d Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Sun, 20 Feb 2022 23:56:02 +0700 Subject: [PATCH 12/16] Create fake blocks for empty slots --- proxy/memdb/blocks_db.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/proxy/memdb/blocks_db.py b/proxy/memdb/blocks_db.py index 19cfacb59..121a6e661 100644 --- a/proxy/memdb/blocks_db.py +++ b/proxy/memdb/blocks_db.py @@ -53,39 +53,40 @@ def _get_latest_db_block(self): self.latest_db_block_slot = self._b.solana.get_recent_blockslot(commitment=FINALIZED) def _get_solana_block_list(self) -> bool: - slot = self.latest_db_block_slot - exist_block_dict = self._b.get_block_dict(slot) + latest_db_slot = self.latest_db_block_slot + exist_block_dict = self._b.get_block_dict(latest_db_slot) + latest_slot = max(exist_block_dict) if len(exist_block_dict) else 0 + block_time = 0 slot_list = [] - block_list = [] - max_slot = max(slot + self.BLOCK_CACHE_LIMIT, self._b.get_latest_block_slot()) - for slot in range(max_slot, slot - 1, -1): + self.block_list = [] + + max_slot = max(latest_db_slot + self.BLOCK_CACHE_LIMIT, latest_slot) + for slot in range(max_slot, latest_db_slot - 1, -1): block = exist_block_dict.get(slot) if block is None: slot_list.append(slot) else: - block_list.append(block) + self.block_list.append(block) + block_time = max(block_time, block.time) - self.block_list = [] - - block_time = 0 solana_block_list = self._b.solana.get_block_info_list(slot_list) for block in solana_block_list: if not block.time: - if not block_time: + if block.slot > latest_slot: continue block.time = block_time block.hash = '0x' + os.urandom(32).hex() block.parent_hash = '0x' + os.urandom(32).hex() else: - block_time = block.time + block_time = max(block_time, block.time) + latest_slot = max(block.slot, latest_slot) self.block_list.append(block) - self.block_list.extend(block_list) - if not len(self.block_list): return False + self.block_list.sort(key=lambda b: b.slot, reverse=True) self.latest_block = self.block_list[0] self.first_block = self.block_list[len(self.block_list) - 1] From b0f0aaac74d18eee859c4d08c1d1b437d3ab3259 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Mon, 21 Feb 2022 15:28:03 +0700 Subject: [PATCH 13/16] Call cancel on unknown errors for iterative transactions. #587 --- proxy/common_neon/solana_interactor.py | 10 +++++++--- proxy/common_neon/transaction_sender.py | 7 +++++++ proxy/indexer/utils.py | 7 ++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 467f6ec95..58a3ad889 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -497,6 +497,7 @@ def __init__(self, sender, tx_list: [Transaction], name: str, self._pending_list = [] self._budget_exceeded_list = [] self._storage_bad_status_list = [] + self._unknown_error_list = [] self._all_tx_list = [self._node_behind_list, self._bad_block_list, @@ -546,7 +547,7 @@ def send(self) -> SolTxListSender: if custom in (1, 4): self._storage_bad_status_list.append(receipt) else: - raise SolTxError(receipt) + self._unknown_error_list.append(receipt) else: success_cnt += 1 self._on_success_send(tx, receipt) @@ -558,7 +559,8 @@ def send(self) -> SolTxListSender: f'bad blocks {len(self._bad_block_list)}, ' + f'blocked accounts {len(self._blocked_account_list)}, ' + f'budget exceeded {len(self._budget_exceeded_list)}, ' + - f'bad storage: {len(self._storage_bad_status_list)}') + f'bad storage: {len(self._storage_bad_status_list)}, ' + + f'unknown error: {len(self._unknown_error_list)}') self._on_post_send() @@ -581,7 +583,9 @@ def _on_success_send(self, tx: Transaction, receipt: {}) -> bool: return False def _on_post_send(self): - if len(self._node_behind_list): + if len(self._unknown_error_list): + raise SolTxError(self._unknown_error_list[0]) + elif len(self._node_behind_list): self.warning(f'Node is behind by {self._slots_behind} slots') time.sleep(1) elif len(self._storage_bad_status_list): diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 62262661d..f2f8fc2d1 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -780,6 +780,13 @@ def _on_post_send(self): self.warning(f'Node is behind by {self._slots_behind} slots') time.sleep(1) + # Unknown error happens - cancel the transaction + if len(self._unknown_error_list): + self._unknown_error_list.clear() + if not self._is_canceled: + self._cancel() + return + # There is no more retries to send transactions if self._retry_idx >= RETRY_ON_FAIL: if not self._is_canceled: diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index e8e3336cf..1d7f6986b 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -47,10 +47,10 @@ def get_accounts_from_storage(solana: SolanaInteractor, storage_account, *, logg if info is None: raise Exception(f"Can't get information about {storage_account}") - if info.tag == 0: + if info.tag in (0, 1, 4): logger.debug("Empty") return None - elif info.tag == 3: + else: logger.debug("Not empty storage") acc_list = [] @@ -62,9 +62,6 @@ def get_accounts_from_storage(solana: SolanaInteractor, storage_account, *, logg offset += 32 return acc_list - else: - logger.debug("Not empty other") - return None @logged_group("neon.Indexer") From 2cedfa5119c17c4ab56222abdc3b63213de9a715 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Tue, 22 Feb 2022 00:10:58 +0700 Subject: [PATCH 14/16] Switch to latest build --- .buildkite/steps/build-image.sh | 2 +- proxy/docker-compose-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/steps/build-image.sh b/.buildkite/steps/build-image.sh index 738ef25b4..dc1524165 100755 --- a/.buildkite/steps/build-image.sh +++ b/.buildkite/steps/build-image.sh @@ -4,7 +4,7 @@ set -euo pipefail REVISION=$(git rev-parse HEAD) set ${SOLANA_REVISION:=v1.8.12-testnet} -set ${EVM_LOADER_REVISION:=v0.6.0-rc3} +set ${EVM_LOADER_REVISION:=latest} # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally docker pull neonlabsorg/solana:${SOLANA_REVISION} diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 850bc30d7..6782a2e18 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -26,7 +26,7 @@ services: evm_loader: container_name: evm_loader - image: neonlabsorg/evm_loader:${EVM_LOADER_REVISION:-v0.6.0-rc3} + image: neonlabsorg/evm_loader:${EVM_LOADER_REVISION:-latest} environment: - SOLANA_URL=http://solana:8899 networks: From 3611be8c9f3a88b547036e9324c7603b5a9ffaf0 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Tue, 22 Feb 2022 00:28:22 +0700 Subject: [PATCH 15/16] Use solana instead of client --- proxy/common_neon/estimate.py | 4 ++-- proxy/plugin/solana_rest_api.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/common_neon/estimate.py b/proxy/common_neon/estimate.py index 452bf13f5..11acb80e4 100644 --- a/proxy/common_neon/estimate.py +++ b/proxy/common_neon/estimate.py @@ -21,7 +21,7 @@ def evm_step_cost(signature_cnt): @logged_group("neon.Proxy") class GasEstimate: - def __init__(self, request, db, client, evm_step_count): + def __init__(self, request, db, solana, evm_step_count): self.sender: bytes = bytes.fromhex(request.get('from', "0x%040x" % 0x0)[2:]) self.step_count = evm_step_count @@ -46,7 +46,7 @@ def __init__(self, request, db, client, evm_step_count): signed_trx = w3.eth.account.sign_transaction(unsigned_trx, eth_keys.PrivateKey(os.urandom(32))) trx = EthTrx.fromString(signed_trx.rawTransaction) - self.tx_sender = NeonTxSender(db, client, trx, steps=evm_step_count) + self.tx_sender = NeonTxSender(db, solana, trx, steps=evm_step_count) def iteration_info(self) -> Tuple[int, int]: if self.tx_sender.steps_emulated > 0: diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 4ea7667d7..6880ae8b2 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -94,7 +94,7 @@ def eth_gasPrice(self): def eth_estimateGas(self, param): try: - calculator = GasEstimate(param, self._db, self._client, evm_step_count) + calculator = GasEstimate(param, self._db, self._solana, evm_step_count) return calculator.estimate() except EthereumError: From d538a20a5d17be88746f7961db46e9469025b35d Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Tue, 22 Feb 2022 14:59:16 +0700 Subject: [PATCH 16/16] Use solana instead of client --- proxy/common_neon/address.py | 38 +++++++------------------ proxy/common_neon/estimate.py | 33 +++++++++------------ proxy/common_neon/solana_interactor.py | 6 ++-- proxy/common_neon/transaction_sender.py | 6 ++-- proxy/plugin/solana_rest_api.py | 2 +- 5 files changed, 31 insertions(+), 54 deletions(-) diff --git a/proxy/common_neon/address.py b/proxy/common_neon/address.py index 6819e6784..a6d1fec01 100644 --- a/proxy/common_neon/address.py +++ b/proxy/common_neon/address.py @@ -1,5 +1,6 @@ +from __future__ import annotations + import random -import base64 from eth_keys import keys as eth_keys from hashlib import sha256 @@ -8,9 +9,8 @@ from typing import NamedTuple from .layouts import ACCOUNT_INFO_LAYOUT -from ..environment import neon_cli, ETH_TOKEN_MINT_ID, EVM_LOADER_ID +from ..environment import ETH_TOKEN_MINT_ID, EVM_LOADER_ID from .constants import ACCOUNT_SEED_VERSION -from solana.rpc.commitment import Confirmed class EthereumAddress: @@ -60,35 +60,17 @@ def getTokenAddr(account): return get_associated_token_address(PublicKey(account), ETH_TOKEN_MINT_ID) -class AccountInfo(NamedTuple): +class AccountInfoLayout(NamedTuple): ether: eth_keys.PublicKey trx_count: int code_account: PublicKey state: int + def is_payed(self) -> bool: + return self.state != 0 + @staticmethod - def frombytes(data): + def frombytes(data) -> AccountInfoLayout: cont = ACCOUNT_INFO_LAYOUT.parse(data) - return AccountInfo(cont.ether, int.from_bytes(cont.trx_count, 'little'), PublicKey(cont.code_account), cont.state) - - -def _getAccountData(client, account, expected_length, owner=None): - info = client.get_account_info(account, commitment=Confirmed)['result']['value'] - if info is None: - raise Exception("Can't get information about {}".format(account)) - - data = base64.b64decode(info['data'][0]) - if len(data) < expected_length: - raise Exception("Wrong data length for account data {}".format(account)) - return data - - -def getAccountInfo(client, eth_account: EthereumAddress): - account_sol, nonce = ether2program(eth_account) - info = _getAccountData(client, account_sol, ACCOUNT_INFO_LAYOUT.sizeof()) - return AccountInfo.frombytes(info) - - -def isPayed(client, address: str): - acc_info: AccountInfo = getAccountInfo(client, EthereumAddress(address)) - return acc_info.state != 0 + return AccountInfoLayout(cont.ether, int.from_bytes(cont.trx_count, 'little'), + PublicKey(cont.code_account), cont.state) diff --git a/proxy/common_neon/estimate.py b/proxy/common_neon/estimate.py index 11acb80e4..ee3125a97 100644 --- a/proxy/common_neon/estimate.py +++ b/proxy/common_neon/estimate.py @@ -59,34 +59,30 @@ def iteration_info(self) -> Tuple[int, int]: final_steps = EVM_STEPS return final_steps, full_step_iterations - @logged_group("neon.Proxy") - def simple_neon_tx_strategy(self, *, logger): + def simple_neon_tx_strategy(self): gas = evm_step_cost(2) * (self.tx_sender.steps_emulated if self.tx_sender.steps_emulated > EVM_STEPS else EVM_STEPS) - logger.debug(f'estimate simple_neon_tx_strategy: {gas}') + self.debug(f'estimate simple_neon_tx_strategy: {gas}') return gas - @logged_group("neon.Proxy") - def iterative_neon_tx_strategy(self, *, logger): + def iterative_neon_tx_strategy(self): begin_iteration = 1 final_steps, full_step_iterations = self.iteration_info() steps = begin_iteration * EVM_STEPS + full_step_iterations * self.step_count + final_steps gas = steps * evm_step_cost(1) - logger.debug(f'estimate iterative_neon_tx_strategy: {gas}') + self.debug(f'estimate iterative_neon_tx_strategy: {gas}') return gas - @logged_group("neon.Proxy") - def holder_neon_tx_strategy(self, *, logger): + def holder_neon_tx_strategy(self): begin_iteration = 1 msg = get_holder_msg(self.tx_sender.eth_tx) holder_iterations = math.ceil(len(msg) / HOLDER_MSG_SIZE) final_steps, full_step_iterations = self.iteration_info() steps = (begin_iteration + holder_iterations) * EVM_STEPS + full_step_iterations * self.step_count + final_steps gas = steps * evm_step_cost(1) - logger.debug(f'estimate holder_neon_tx_strategy: {gas}') + self.debug(f'estimate holder_neon_tx_strategy: {gas}') return gas - @logged_group("neon.Proxy") - def allocated_space(self, *, logger): + def allocated_space(self): space = 0 for s in self.tx_sender._create_account_list: if s.NAME == NeonCreateContractTxStage.NAME: @@ -95,14 +91,13 @@ def allocated_space(self, *, logger): space += ACCOUNT_MAX_SIZE + SPL_TOKEN_ACCOUNT_SIZE + ACCOUNT_STORAGE_OVERHEAD * 2 space += self.tx_sender.unpaid_space - logger.debug(f'allocated space: {space}') + self.debug(f'allocated space: {space}') return space - @logged_group("neon.Proxy") - def estimate(self, *, logger): + def estimate(self): self.tx_sender.operator_key = PublicKey(os.urandom(32)) self.tx_sender._call_emulated(self.sender) - self.tx_sender._parse_accounts_list(); + self.tx_sender._parse_accounts_list() gas_for_trx = max(self.simple_neon_tx_strategy(), self.iterative_neon_tx_strategy(), self.holder_neon_tx_strategy()) gas_for_space = self.allocated_space() * EVM_BYTE_COST @@ -112,8 +107,8 @@ def estimate(self, *, logger): # if gas < 21000: # gas = 21000 - logger.debug(f'extra_gas: {EXTRA_GAS}') - logger.debug(f'gas_for_space: {gas_for_space}') - logger.debug(f'gas_for_trx: {gas_for_trx}') - logger.debug(f'estimated gas: {gas}') + self.debug(f'extra_gas: {EXTRA_GAS}') + self.debug(f'gas_for_space: {gas_for_space}') + self.debug(f'gas_for_trx: {gas_for_trx}') + self.debug(f'estimated gas: {gas}') return hex(gas) diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 58a3ad889..f00499bdb 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -29,7 +29,7 @@ from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT from ..common_neon.address import EthereumAddress, ether2program -from ..common_neon.address import AccountInfo as NeonAccountInfo +from ..common_neon.address import AccountInfoLayout as AccountInfoLayout class SolTxError(Exception): @@ -244,7 +244,7 @@ def get_token_account_balance_list(self, pubkey_list: [Union[str, PublicKey]], c return balance_list - def get_neon_account_info(self, eth_account: EthereumAddress) -> Optional[NeonAccountInfo]: + def get_account_info_layout(self, eth_account: EthereumAddress) -> Optional[AccountInfoLayout]: account_sol, nonce = ether2program(eth_account) info = self.get_account_info(account_sol) if info is None: @@ -252,7 +252,7 @@ def get_neon_account_info(self, eth_account: EthereumAddress) -> Optional[NeonAc elif len(info.data) < ACCOUNT_INFO_LAYOUT.sizeof(): raise RuntimeError(f"Wrong data length for account data {account_sol}: " + f"{len(info.data)} < {ACCOUNT_INFO_LAYOUT.sizeof()}") - return NeonAccountInfo.frombytes(info.data) + return AccountInfoLayout.frombytes(info.data) def get_multiple_rent_exempt_balances_for_size(self, size_list: [int], commitment='confirmed') -> [int]: opts = { diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index 0e8026723..7fffe2e25 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -17,7 +17,7 @@ from solana.blockhash import Blockhash from solana.account import Account as SolanaAccount -from .address import accountWithSeed, getTokenAddr, EthereumAddress, isPayed +from .address import accountWithSeed, getTokenAddr, EthereumAddress from ..common_neon.errors import EthereumError from .constants import STORAGE_SIZE, EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG, ACCOUNT_SEED_VERSION from .emulator_interactor import call_emulated @@ -446,7 +446,7 @@ def _validate_whitelist(self): raise Exception(f'Contract account {self.deployed_contract} is not allowed for deployment') def _validate_tx_count(self): - info = self.solana.get_neon_account_info(EthereumAddress(self.eth_sender)) + info = self.solana.get_account_info_layout(EthereumAddress(self.eth_sender)) if not info: return @@ -565,7 +565,7 @@ def _parse_accounts_list(self): elif account_desc["storage_increment"]: self.unpaid_space += account_desc["storage_increment"] - if not isPayed(self.solana.client, account_desc['address']): + if not self.solana.get_account_info_layout(account_desc['address']).is_payed(): self.debug(f'found losted account {account_desc["account"]}') self.unpaid_space += ACCOUNT_MAX_SIZE + SPL_TOKEN_ACCOUNT_SIZE + ACCOUNT_STORAGE_OVERHEAD * 2 diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 6880ae8b2..72bac8946 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -265,7 +265,7 @@ def eth_call(self, obj, tag): def eth_getTransactionCount(self, account, tag): self.debug('eth_getTransactionCount: %s', account) try: - acc_info = self._solana.get_neon_account_info(EthereumAddress(account)) + acc_info = self._solana.get_account_info_layout(EthereumAddress(account)) return hex(acc_info.trx_count) except Exception as err: self.debug(f"eth_getTransactionCount: Can't get account info: {err}")