NDEV-806 limit number of executors

proxy/common_neon/config.py

@@ -14,6 +14,8 @@ def __init__(self):
         self._evm_loader_id = SolPubKey(EVM_LOADER_ID)
         self._evm_step_cnt_inc_pct = self._env_decimal("EVM_STEP_COUNT_INC_PCT", "0.9")
         self._mempool_capacity = self._env_int("MEMPOOL_CAPACITY", 10, 4096)
+        self._mempool_executor_limit_cnt = self._env_int('MEMPOOL_EXECUTOR_LIMIT_CNT', 4, 1024)
+        self._mempool_cache_life_sec = self._env_int('MEMPOOL_CACHE_LIFE_SEC', 15, 15 * 60)
         self._holder_size = self._env_int("HOLDER_SIZE", 1024, 131072)  # 128*1024
         self._min_op_balance_to_warn = self._env_int("MIN_OPERATOR_BALANCE_TO_WARN", 9000000000, 9000000000)
         self._min_op_balance_to_err = self._env_int("MIN_OPERATOR_BALANCE_TO_ERR", 1000000000, 1000000000)
@@ -26,7 +28,6 @@ def __init__(self):
         self._use_earliest_block_if_0_passed = self._env_bool("USE_EARLIEST_BLOCK_IF_0_PASSED", False)
         self._account_permission_update_int = self._env_int("ACCOUNT_PERMISSION_UPDATE_INT", 10, 60 * 5)
         self._allow_underpriced_tx_wo_chainid = self._env_bool("ALLOW_UNDERPRICED_TX_WITHOUT_CHAINID", False)
-        self._contract_extra_space = self._env_int("CONTRACT_EXTRA_SPACE", 0, 2048)
         self._extra_gas_pct = self._env_decimal("EXTRA_GAS_PCT", "0.0")
         self._operator_fee = self._env_decimal("OPERATOR_FEE", "0.1")
         self._gas_price_suggested_pct = self._env_decimal("GAS_PRICE_SUGGEST_PCT", "0.01")
@@ -52,7 +53,6 @@ def __init__(self):
         self._holder_timeout = self._env_int("HOLDER_TIMEOUT", 1, 216000)  # 1 day by default
         self._indexer_log_skip_cnt = self._env_int("INDEXER_LOG_SKIP_COUNT", 1, 1000)
         self._gather_statistics = self._env_bool("GATHER_STATISTICS", False)
-        self._mempool_cache_life_sec = self._env_int('MEMPOOL_CACHE_LIFE_SEC', 15, 15 * 60)
         self._hvac_url = os.environ.get('HVAC_URL', None)
         self._hvac_token = os.environ.get('HVAC_TOKEN', None)
         self._hvac_mount = os.environ.get('HVAC_MOUNT', None)
@@ -92,6 +92,14 @@ def evm_step_cnt_inc_pct(self) -> Decimal:
     def mempool_capacity(self) -> int:
         return self._mempool_capacity
 
+    @property
+    def mempool_executor_limit_cnt(self) -> int:
+        return self._mempool_executor_limit_cnt
+
+    @property
+    def mempool_cache_life_sec(self) -> int:
+        return self._mempool_cache_life_sec
+
     @property
     def pyth_mapping_account(self) -> Optional[SolPubKey]:
         return self._pyth_mapping_account
@@ -156,10 +164,6 @@ def allow_underpriced_tx_wo_chainid(self) -> bool:
     def extra_gas_pct(self) -> Decimal:
         return self._extra_gas_pct
 
-    @property
-    def contract_extra_space(self) -> int:
-        return self._contract_extra_space
-
     @property
     def operator_fee(self) -> Decimal:
         return self._operator_fee
@@ -258,10 +262,6 @@ def indexer_log_skip_cnt(self) -> int:
     def gather_statistics(self) -> bool:
         return self._gather_statistics
 
-    @property
-    def mempool_cache_life_sec(self) -> int:
-        return self._mempool_cache_life_sec
-
     @property
     def hvac_url(self) -> Optional[str]:
         return self._hvac_url
@@ -286,7 +286,9 @@ def __str__(self):
             f"PP_SOLANA_URL: {self.pyth_solana_url}",
             f"PYTH_MAPPING_ACCOUNT: {self.pyth_mapping_account}",
             f"EVM_STEP_COUNT_INC_PCT: {self._evm_step_cnt_inc_pct},",
-            f"MP_CAPACITY: {self.mempool_capacity}",
+            f"MEMPOOL_CAPACITY: {self.mempool_capacity}",
+            f"MEMPOOL_EXECUTOR_LIMIT_CNT: {self.mempool_executor_limit_cnt}",
+            f"MEMPOOL_CACHE_LIFE_SEC: {self.mempool_cache_life_sec}",
             f"HOLDER_SIZE: {self.holder_size}",
             f"MIN_OPERATOR_BALANCE_TO_WARN: {self.min_operator_balance_to_warn}",
             f"MIN_OPERATOR_BALANCE_TO_ERR: {self.min_operator_balance_to_err}",
@@ -299,7 +301,6 @@ def __str__(self):
             f"USE_EARLIEST_BLOCK_IF_0_PASSED: {self.use_earliest_block_if_0_passed}",
             f"ACCOUNT_PERMISSION_UPDATE_INT: {self.account_permission_update_int}",
             f"ALLOW_UNDERPRICED_TX_WITHOUT_CHAINID: {self.allow_underpriced_tx_wo_chainid}",
-            f"CONTRACT_EXTRA_SPACE: {self.contract_extra_space}",
             f"EXTRA_GAS_PCT: {self.extra_gas_pct}",
             f"OPERATOR_FEE: {self.operator_fee}",
             f"GAS_PRICE_SUGGEST_PCT: {self.gas_price_suggested_pct}",
@@ -325,7 +326,6 @@ def __str__(self):
             f"HOLDER_TIMOUT: {self.holder_timeout}",
             f"INDEXER_LOG_SKIP_COUNT: {self.indexer_log_skip_cnt}",
             f"GATHER_STATISTICS: {self.gather_statistics}",
-            f"MEMPOOL_CACHE_LIFE_SEC: {self.mempool_cache_life_sec}",
             f"HVAC_URL: {self.hvac_url}",
             f"HVAC_TOKEN: {self.hvac_token}",
             f"HVAC_PATH: {self.hvac_path}",

proxy/common_neon/estimate.py

@@ -13,7 +13,7 @@
 from ..common_neon.solana_interactor import SolInteractor
 from ..common_neon.solana_alt_builder import ALTTxBuilder
 from ..common_neon.neon_instruction import NeonIxBuilder
-from ..common_neon.solana_tx import SolAccount, SolPubKey, SolAccountMeta, SolBlockhash
+from ..common_neon.solana_tx import SolAccount, SolPubKey, SolAccountMeta, SolBlockhash, SolTxSizeError
 from ..common_neon.solana_tx_legacy import SolLegacyTx
 from ..common_neon.address import NeonAddress
 
@@ -121,8 +121,10 @@ def _tx_size_cost(self) -> int:
                 pass
             elif self._execution_cost() < self._small_gas_limit:
                 return self._cached_tx_cost_size
-        except (Exception, ):
+        except SolTxSizeError:
             pass
+        except BaseException as exc:
+            self.debug('Error during pack solana tx', exc_info=exc)
 
         self._cached_tx_cost_size = self._holder_tx_cost(self._tx_builder.neon_tx_len())
         return self._cached_tx_cost_size
@@ -138,7 +140,7 @@ def _emulated_step_cnt(self) -> int:
 
     def _iterative_overhead_cost(self) -> int:
         if self._cached_overhead_cost is not None:
-            return self._cached_tx_cost_size
+            return self._cached_overhead_cost
 
         execution_cost = self._execution_cost()
         tx_size_cost = self._tx_size_cost()

proxy/common_neon/solana_tx.py

@@ -20,6 +20,10 @@
 SolTxReceipt = Dict[str, Any]
 
 
+class SolTxSizeError(Exception):
+    pass
+
+
 class SolTx(abc.ABC):
     _empty_blockhash = SolBlockhash(str(solders.hash.Hash.default()))
 
@@ -65,7 +69,7 @@ def serialize(self) -> bytes:
         assert self._is_signed, "transaction has not been signed"
         result = self._serialize()
         if len(result) > solana.transaction.PACKET_DATA_SIZE:
-            raise ValueError('Transaction too big')
+            raise SolTxSizeError('Transaction too big')
         return result
 
     def sign(self, signer: SolAccount) -> None:

proxy/mempool/executor_mng.py

@@ -43,7 +43,7 @@ def __init__(self, config: Config, user: IMPExecutorMngUser):
         self._last_id = 0
 
     async def set_executor_cnt(self, executor_count: int) -> None:
-        executor_count = max(executor_count + 1, 3)
+        executor_count = min(max(executor_count + 1, 3), self._config.mempool_executor_limit_cnt)
         diff_count = executor_count - len(self._executor_dict)
         if diff_count > 0:
             return await self._run_executors(diff_count)