Merge branch 'develop' into 748_indexer_mem_usage

Author: otselnik

.buildkite/pipeline.yml

@@ -38,7 +38,7 @@ steps:
     commands:
       - ".buildkite/steps/full_test_suite/run_full_test_suite.sh"
     env:
-      FTS_THRESHOLD: 1700
+      FTS_THRESHOLD: 1920
       FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER}
       FTS_IMAGE: neonlabsorg/full_test_suite:develop
       FTS_USERS_NUMBER: 15
@@ -73,4 +73,4 @@ steps:
       - "create_infrastructure"
     allow_dependency_failure: true
 
-      
+

proxy/common_neon/account_whitelist.py

@@ -17,7 +17,7 @@
 
 @logged_group("neon.AccountWhitelist")
 class AccountWhitelist:
-    def __init__(self, solana: SolanaInteractor, payer: SolanaAccount, permission_update_int: int):
+    def __init__(self, solana: SolanaInteractor, permission_update_int: int):
         self.solana = solana
         self.account_cache = {}
         self.permission_update_int = permission_update_int
@@ -31,13 +31,8 @@ def __init__(self, solana: SolanaInteractor, payer: SolanaAccount, permission_up
             self.error(f'Wrong proxy configuration: allowance and denial tokens must both exist or absent!')
             raise Exception("NEON service is unhealthy. Try again later")
 
-        self.allowance_token = PermissionToken(self.solana,
-                                               PublicKey(ALLOWANCE_TOKEN_ADDR),
-                                               payer)
-
-        self.denial_token = PermissionToken(self.solana,
-                                            PublicKey(DENIAL_TOKEN_ADDR),
-                                            payer)
+        self.allowance_token = PermissionToken(self.solana, PublicKey(ALLOWANCE_TOKEN_ADDR))
+        self.denial_token = PermissionToken(self.solana, PublicKey(DENIAL_TOKEN_ADDR))
 
     def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]) -> int:
         token_list = [
@@ -50,30 +45,30 @@ def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]) -> int:
         denial_balance = balance_list[1]
         return allowance_balance - denial_balance
 
-    def grant_permissions(self, ether_addr: Union[str, EthereumAddress], min_balance: int):
+    def grant_permissions(self, ether_addr: Union[str, EthereumAddress], min_balance: int, signer: SolanaAccount):
         try:
             diff = self.read_balance_diff(ether_addr)
             if diff >= min_balance:
                 self.info(f'{ether_addr} already has permission')
                 return True
 
             to_mint = min_balance - diff
-            self.allowance_token.mint_to(to_mint, ether_addr)
+            self.allowance_token.mint_to(to_mint, ether_addr, signer)
             self.info(f'Permissions granted to {ether_addr}')
             return True
         except Exception as err:
             self.error(f'Failed to grant permissions to {ether_addr}: {type(err)}: {err}')
             return False
 
-    def deprive_permissions(self, ether_addr: Union[str, EthereumAddress], min_balance: int):
+    def deprive_permissions(self, ether_addr: Union[str, EthereumAddress], min_balance: int, signer: SolanaAccount):
         try:
             diff = self.read_balance_diff(ether_addr)
             if diff < min_balance:
                 self.info(f'{ether_addr} already deprived')
                 return True
 
             to_mint = diff - min_balance + 1
-            self.denial_token.mint_to(to_mint, ether_addr)
+            self.denial_token.mint_to(to_mint, ether_addr, signer)
             self.info(f'Permissions deprived to {ether_addr}')
             return True
         except Exception as err:

proxy/common_neon/emulator_interactor.py

@@ -3,19 +3,40 @@
 from logged_groups import logged_group
 
 from typing import Optional, Dict, Any
-from .errors import EthereumError
+from ethereum.transactions import Transaction as NeonTrx
+
 from ..environment import neon_cli, NEON_TOKEN_MINT, CHAIN_ID
 
+from .errors import EthereumError
+from .types import NeonEmulatingResult
+
 
 @logged_group("neon.Proxy")
-def call_emulated(contract_id, caller_id, data=None, value=None, *, logger):
+def call_emulated(contract_id, caller_id, data=None, value=None, *, logger) -> NeonEmulatingResult:
     output = emulator(contract_id, caller_id, data, value)
     logger.debug(f"Call emulated. contract_id: {contract_id}, caller_id: {caller_id}, data: {data}, value: {value}, return: {output}")
     result = json.loads(output)
     check_emulated_exit_status(result)
     return result
 
 
+@logged_group("neon.Proxy")
+def call_trx_emulated(neon_trx: NeonTrx, *, logger) -> NeonEmulatingResult:
+    neon_sender_acc = neon_trx.sender()
+    contract = neon_trx.contract()
+    logger.debug(f'sender address: 0x{neon_sender_acc}')
+    if contract:
+        dst = 'deploy'
+        logger.debug(f'deploy contract: {contract}')
+    else:
+        dst = neon_trx.toAddress.hex()
+        logger.debug(f'destination address {dst}')
+    logger.debug(f"Calling data: {(dst, neon_sender_acc, neon_trx.callData.hex(), hex(neon_trx.value))}")
+    emulator_json = call_emulated(dst, neon_sender_acc, neon_trx.callData.hex(), hex(neon_trx.value))
+    logger.debug(f'emulator returns: {json.dumps(emulator_json, sort_keys=True)}')
+    return emulator_json
+
+
 @logged_group("neon.Proxy")
 def check_emulated_exit_status(result: Dict[str, Any], *, logger):
     exit_status = result['exit_status']

proxy/common_neon/permission_token.py

@@ -12,12 +12,8 @@
 
 
 class PermissionToken:
-    def __init__(self,
-                 solana: SolanaInteractor,
-                 token_mint: PublicKey,
-                 payer: SolanaAccount):
+    def __init__(self, solana: SolanaInteractor, token_mint: PublicKey):
         self.solana = solana
-        self.signer = payer
         self.waiter = None
         self.token_mint = token_mint
 
@@ -29,28 +25,24 @@ def get_balance(self, ether_addr: Union[str, EthereumAddress]):
         token_account = self.get_token_account_address(ether_addr)
         return self.solana.get_token_account_balance(token_account)
 
-    def create_account_if_needed(self,
-                                 ether_addr: Union[str, EthereumAddress]):
+    def create_account_if_needed(self, ether_addr: Union[str, EthereumAddress], signer: SolanaAccount):
         token_account = self.get_token_account_address(ether_addr)
         info = self.solana.get_account_info(token_account)
         if info is not None:
             return token_account
 
         txn = TransactionWithComputeBudget()
         create_txn = spl_token.create_associated_token_account(
-            payer=self.signer.public_key(),
+            payer=signer.public_key(),
             owner=PublicKey(ether2program(ether_addr)[0]),
             mint=self.token_mint
         )
         txn.add(create_txn)
-        SolTxListSender(self, [txn], 'CreateAssociatedTokenAccount(1)', skip_preflight=True).send()
+        SolTxListSender(self, [txn], 'CreateAssociatedTokenAccount(1)', skip_preflight=True).send(signer)
         return token_account
 
-    def mint_to(self,
-                amount: int,
-                ether_addr: Union[str, EthereumAddress],
-                mint_authority_file: str):
-        token_account = self.create_account_if_needed(ether_addr)
+    def mint_to(self, amount: int, ether_addr: Union[str, EthereumAddress], mint_authority_file: str, signer: SolanaAccount):
+        token_account = self.create_account_if_needed(ether_addr, signer)
         mint_command = f'spl-token mint "{str(self.token_mint)}" {Decimal(amount) * pow(Decimal(10), -9)}'
         mint_command += f' --owner {mint_authority_file} -- "{str(token_account)}"'
         os.system(mint_command)

proxy/common_neon/solana_tx_list_sender.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from solana.account import Account as SolanaAccount
 import time
 
 from logged_groups import logged_group
@@ -50,9 +51,8 @@ def clear(self):
     def _get_full_list(self):
         return [tx for lst in self._all_tx_list for tx in lst]
 
-    def send(self) -> SolTxListSender:
+    def send(self, signer: SolanaAccount) -> SolTxListSender:
         solana = self._s.solana
-        signer = self._s.signer
         waiter = self._s.waiter
         skip = self._skip_preflight
         commitment = self._preflight_commitment

proxy/common_neon/types.py

@@ -1,3 +1,9 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Any
+
+
 class Result:
     def __init__(self, reason: str = None):
         self._reason = reason
@@ -7,3 +13,12 @@ def __bool__(self) -> bool:
 
     def __str__(self) -> str:
         return self._reason if self._reason is not None else ""
+
+
+@dataclass
+class NeonTxPrecheckResult:
+    is_underpriced_tx_without_chainid: bool
+    emulating_result: NeonEmulatingResult
+
+
+NeonEmulatingResult = Dict[str, Any]

proxy/indexer/canceller.py

@@ -42,7 +42,7 @@ def unlock_accounts(self, blocked_storages):
         self.debug(f"Send Cancel: {len(tx_list)}")
 
         try:
-            SolTxListSender(self, tx_list, f'CancelWithNonce({len(tx_list)})').send()
+            SolTxListSender(self, tx_list, f'CancelWithNonce({len(tx_list)})').send(self.signer)
         except Exception as err:
             err_tb = "".join(traceback.format_tb(err.__traceback__))
             self.warning('Exception on submitting transaction. ' +

proxy/neon_rpc_api_model/neon_rpc_api_model.py

@@ -8,20 +8,23 @@
 from web3.auto import w3
 
 from ..common_neon.address import EthereumAddress
-from ..common_neon.emulator_interactor import call_emulated
+from ..common_neon.emulator_interactor import call_emulated, call_trx_emulated
 from ..common_neon.errors import EthereumError, InvalidParamError, PendingTxError
 from ..common_neon.estimate import GasEstimate
 from ..common_neon.eth_proto import Trx as EthTrx
 from ..common_neon.keys_storage import KeyStorage
 from ..common_neon.solana_interactor import SolanaInteractor
 from ..common_neon.utils import SolanaBlockInfo
+from ..common_neon.types import NeonTxPrecheckResult, NeonEmulatingResult
 from ..environment import SOLANA_URL, PP_SOLANA_URL, PYTH_MAPPING_ACCOUNT, NEON_EVM_VERSION, NEON_EVM_REVISION, \
                           CHAIN_ID, neon_cli, EVM_STEP_COUNT
 from ..memdb.memdb import MemDB
 from ..common_neon.gas_price_calculator import GasPriceCalculator
 from ..statistics_exporter.proxy_metrics_interface import StatisticsExporter
 
 from .transaction_sender import NeonTxSender
+from .operator_resource_list import OperatorResourceList
+from .transaction_validator import NeonTxValidator
 
 NEON_PROXY_PKG_VERSION = '0.7.16-dev'
 NEON_PROXY_REVISION = 'NEON_PROXY_REVISION_TO_BE_REPLACED'
@@ -437,13 +440,14 @@ def eth_sendRawTransaction(self, rawTrx: str) -> str:
         eth_signature = '0x' + trx.hash_signed().hex()
         self.debug(f"sendRawTransaction {eth_signature}: {json.dumps(trx.as_dict(), cls=JsonEncoder, sort_keys=True)}")
 
-        min_gas_price = self.gas_price_calculator.get_min_gas_price()
-
         self._stat_tx_begin()
-
         try:
-            tx_sender = NeonTxSender(self._db, self._solana, trx, steps=EVM_STEP_COUNT, min_gas_price=min_gas_price)
-            tx_sender.execute()
+            neon_tx_precheck_result = self.precheck(trx)
+
+            tx_sender = NeonTxSender(self._db, self._solana, trx, steps=EVM_STEP_COUNT)
+            with OperatorResourceList(tx_sender):
+                tx_sender.execute(neon_tx_precheck_result)
+
             self._stat_tx_success()
             return eth_signature
 
@@ -458,6 +462,14 @@ def eth_sendRawTransaction(self, rawTrx: str) -> str:
             self._stat_tx_failed()
             raise
 
+    def precheck(self, neon_trx: EthTrx) -> NeonTxPrecheckResult:
+
+        min_gas_price = self.gas_price_calculator.get_min_gas_price()
+        neon_validator = NeonTxValidator(self._solana, neon_trx, min_gas_price)
+        precheck_result = neon_validator.precheck()
+
+        return precheck_result
+
     def _stat_tx_begin(self):
         self._stat_exporter.stat_commit_tx_begin()