Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
61 commits
Select commit Hold shift + click to select a range
36173d3
Get rid of queue based service
rozhkovdmitrii Apr 16, 2022
30e11cf
Move all data types into data.py
rozhkovdmitrii Apr 16, 2022
cbfc8bb
Simplify start mechanism
rozhkovdmitrii Apr 16, 2022
5444085
Update neon_rpc_api_model
rozhkovdmitrii Apr 16, 2022
d956dc7
mempool init
rozhkovdmitrii Apr 16, 2022
a2ada65
update mempool
rozhkovdmitrii Apr 16, 2022
6e810fe
Get rid of second getting price int test_integration_success_read_price
rozhkovdmitrii Apr 16, 2022
4675a2c
Bring processing onto asyncio
rozhkovdmitrii Apr 17, 2022
76b73d7
Implement asyncio event loop on the MemPool
rozhkovdmitrii Apr 17, 2022
dc11a9d
rename mempool_server.py
rozhkovdmitrii Apr 17, 2022
89d63c8
Correct some things
rozhkovdmitrii Apr 17, 2022
6ae513b
Spit and polish
rozhkovdmitrii Apr 17, 2022
4bfd337
Spit and polish
rozhkovdmitrii Apr 17, 2022
7d49886
Merge branch '712-mempool' into 712-restructure-processing
rozhkovdmitrii Apr 17, 2022
817df22
Spit and polish
rozhkovdmitrii Apr 18, 2022
39ad9c9
Move pickable_data server into utils
rozhkovdmitrii Apr 18, 2022
22b0ac7
Merge branch '712-mempool' into 713-send-tx-from-mempool-worker
rozhkovdmitrii Apr 21, 2022
8b9a821
Introduce a variety of approaches to transfer a pickable data, config
rozhkovdmitrii Apr 21, 2022
017bcde
Spit on MemPool and polish a little
rozhkovdmitrii Apr 21, 2022
1df5786
Move transaction sending functionality
rozhkovdmitrii Apr 21, 2022
3b725ca
.
rozhkovdmitrii Apr 21, 2022
d4b68d7
Bring emulating beck to transaction_validator
rozhkovdmitrii Apr 21, 2022
df68f14
Resolve some remarks made by @a_falaleev
rozhkovdmitrii Apr 21, 2022
406aa6b
spit and polish
rozhkovdmitrii Apr 21, 2022
88376f8
Rename neon_tx_exec_cfg
rozhkovdmitrii Apr 21, 2022
971cb11
Turn down full test suite
rozhkovdmitrii Apr 22, 2022
9dc5d44
Save changes as they are
rozhkovdmitrii Apr 23, 2022
2c0f4eb
Fix pickable data client
rozhkovdmitrii Apr 23, 2022
b4e33e5
Spit ans polish
rozhkovdmitrii Apr 25, 2022
153f3a3
Move executor_mng out from MemPool
rozhkovdmitrii Apr 25, 2022
634b358
Merge branch '712-mempool' into 713-make-get_receipt-working
rozhkovdmitrii Apr 25, 2022
2464609
Move executor_mng out from MemPool
rozhkovdmitrii Apr 25, 2022
18af023
Merge branch '712-mempool' into 713-send-tx-from-mempool-worker
rozhkovdmitrii Apr 26, 2022
779019f
Get rid of endless request processing
rozhkovdmitrii Apr 26, 2022
f0d9f1a
Just reraise exception when failed to send
rozhkovdmitrii Apr 26, 2022
6dfc5d1
Just reraise exception when failed to send
rozhkovdmitrii Apr 26, 2022
dcf699b
Just reraise exception when failed to send
rozhkovdmitrii Apr 29, 2022
e78153b
Just try to fix logs
rozhkovdmitrii Apr 29, 2022
a467a3d
Logging logs
rozhkovdmitrii Apr 29, 2022
60b713a
Provide some extra logging
rozhkovdmitrii Apr 30, 2022
925d6b2
Just add address to eth_getLogs tests filter
rozhkovdmitrii May 1, 2022
f3c99bb
Fix test_neon_tx_sender.py
rozhkovdmitrii May 1, 2022
cdcca71
enable FTS
rozhkovdmitrii May 1, 2022
5366a53
revert some changes
rozhkovdmitrii May 13, 2022
c6b6935
fix one error
rozhkovdmitrii May 13, 2022
07ccac2
fix counters store
rozhkovdmitrii May 13, 2022
a3bf585
Merge branch '712-mempool' into 713-send-tx-from-mempool-worker
rozhkovdmitrii May 16, 2022
9f37b0f
Lowercase account before getting transaction count
rozhkovdmitrii May 17, 2022
bd668e1
Use delay to make tests passed
rozhkovdmitrii May 17, 2022
62d553f
Use delay to make tests passed
rozhkovdmitrii May 17, 2022
0b6686b
Rollback some changes
rozhkovdmitrii May 18, 2022
3544bd2
Rollback some changes
rozhkovdmitrii May 18, 2022
260e8f7
Spit and polish
rozhkovdmitrii May 19, 2022
8234bb8
Spit and polish
rozhkovdmitrii May 19, 2022
0a13155
Spit and polish
rozhkovdmitrii May 19, 2022
dbf5ca1
Spit and polish
rozhkovdmitrii May 19, 2022
388f028
Spit and polish
rozhkovdmitrii May 19, 2022
8e7eaf4
Spit and polish
rozhkovdmitrii May 19, 2022
93051f0
Spit and polish
rozhkovdmitrii May 19, 2022
728880a
Spit and polish
rozhkovdmitrii May 19, 2022
3d71178
Merge branch '712-mempool' into 713-send-tx-from-mempool-worker
rozhkovdmitrii May 25, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Resolve some remarks made by @a_falaleev
  • Loading branch information
rozhkovdmitrii committed Apr 21, 2022
commit df68f14d5f354f36b8fcc7b2b173fa5217888778
10 changes: 1 addition & 9 deletions proxy/common_neon/data.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,14 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Dict, Any
from .eth_proto import Trx as NeonTx


@dataclass
class NeonTxCfg:
is_without_chainid: bool
is_underpriced_tx_without_chainid: bool
steps_executed: int


@dataclass
class MemPoolTxCfg:
neon_tx: NeonTx
neon_tx_cfg: NeonTxCfg
emulating_result: NeonEmulatingResult


NeonEmulatingResult = Dict[str, Any]


Expand Down
5 changes: 3 additions & 2 deletions proxy/common_neon/utils/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

from eth_utils import big_endian_to_int

# TODO: move it out from here
from ...environment import EVM_LOADER_ID

from ..eth_proto import Trx as EthTx


Expand Down Expand Up @@ -126,8 +129,6 @@ def fill_block_info(self, block: SolanaBlockInfo):
rec['blockNumber'] = hex(block.slot)

def decode(self, neon_sign: str, tx: {}, ix_idx=-1) -> NeonTxResultInfo:
# TODO: move it out from here
from ...environment import EVM_LOADER_ID
self._set_defaults()
meta_ixs = tx['meta']['innerInstructions']
msg = tx['transaction']['message']
Expand Down
1 change: 1 addition & 0 deletions proxy/mempool/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from .mempool_client import MemPoolClient
from .mempool_service import MemPoolService
from .mem_pool import MemPool
from .mempool_api import *

MEMPOOL_SERVICE_PORT = MemPoolService.MEMPOOL_SERVICE_PORT
MEMPOOL_SERVICE_HOST = MemPoolService.MEMPOOL_SERVICE_HOST
12 changes: 6 additions & 6 deletions proxy/mempool/mem_pool.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
from logged_groups import logged_group
from concurrent.futures import ProcessPoolExecutor

from ..common_neon.data import MemPoolTxCfg
from ..common_neon.config import IConfig

from .mempool_api import MemPoolTxRequest
from .mempool_tx_executor import MemPoolTxExecutor


Expand All @@ -16,15 +16,15 @@ def __init__(self, config: IConfig):
self._pool = ProcessPoolExecutor(self.POOL_PROC_COUNT)
self._tx_executor = MemPoolTxExecutor(config)

def send_raw_transaction(self, mempool_tx_cfg: MemPoolTxCfg) -> bool:
def send_raw_transaction(self, mempool_tx_request: MemPoolTxRequest) -> bool:
try:
self._pool.submit(MemPool._send_raw_transaction_impl, mempool_tx_cfg)
self._pool.submit(MemPool._send_raw_transaction_impl, mempool_tx_request)
except Exception as err:
print(f"Failed enqueue mempool_tx_cfg into the worker pool: {err}")
print(f"Failed enqueue mempool_tx_request into the worker pool: {err}")
return False
return True

@staticmethod
def _send_raw_transaction_impl(mempool_tx_cfg: MemPoolTxCfg) -> bool:
print(f"mempool_tx_cfg: {mempool_tx_cfg}")
def _send_raw_transaction_impl(mempool_tx_request: MemPoolTxRequest) -> bool:
print(f"mempool_tx_request: {mempool_tx_request}")
return True
11 changes: 11 additions & 0 deletions proxy/mempool/mempool_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from dataclasses import dataclass

from ..common_neon.eth_proto import Trx as NeonTx
from ..common_neon.data import NeonTxCfg, NeonEmulatingResult


@dataclass
class MemPoolTxRequest:
neon_tx: NeonTx
neon_tx_cfg: NeonTxCfg
emulating_result: NeonEmulatingResult
7 changes: 4 additions & 3 deletions proxy/mempool/mempool_client.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
from ..common_neon.data import MemPoolTxCfg
from ..common_neon.utils import AddrPickableDataClient

from .mempool_api import MemPoolTxRequest


class MemPoolClient:

def __init__(self, host: str, port: int):
self._pickable_data_client = AddrPickableDataClient((host, port))

def send_raw_transaction(self, mempool_tx_cfg: MemPoolTxCfg):
self._pickable_data_client.send_data(mempool_tx_cfg)
def send_raw_transaction(self, mempool_tx_request: MemPoolTxRequest):
self._pickable_data_client.send_data(mempool_tx_request)
10 changes: 5 additions & 5 deletions proxy/mempool/mempool_tx_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@

from ..common_neon.solana_interactor import SolanaInteractor
from ..common_neon.config import IConfig
from ..common_neon.data import MemPoolTxCfg
from ..memdb.memdb import MemDB

# TODO: NeonTxSender should be moved out from there
from .transaction_sender import NeonTxSender
from .operator_resource_list import OperatorResourceList
from .mempool_api import MemPoolTxRequest


@logged_group("neon.MemPool")
Expand All @@ -19,10 +19,10 @@ def __init__(self, config: IConfig):
self._db = MemDB(self._solana)
self._config = config

def execute_neon_tx(self, mempool_tx_cfg: MemPoolTxCfg):
neon_tx = mempool_tx_cfg.neon_tx
neon_tx_cfg = mempool_tx_cfg.neon_tx_cfg
emulating_result = mempool_tx_cfg.emulating_result
def execute_neon_tx(self, mempool_tx_request: MemPoolTxRequest):
neon_tx = mempool_tx_request.neon_tx
neon_tx_cfg = mempool_tx_request.neon_tx_cfg
emulating_result = mempool_tx_request.emulating_result
emv_step_count = self._config.get_evm_count()
tx_sender = NeonTxSender(self._db, self._solana, neon_tx, steps=emv_step_count)
with OperatorResourceList(tx_sender):
Expand Down
6 changes: 3 additions & 3 deletions proxy/mempool/transaction_sender.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from solana.blockhash import Blockhash

from ..mempool.neon_tx_stages import NeonCreateAccountTxStage, NeonCreateERC20TxStage, NeonCreateContractTxStage, \
NeonResizeContractTxStage
NeonResizeContractTxStage

from ..common_neon.compute_budget import TransactionWithComputeBudget
from ..common_neon.neon_instruction import NeonInstruction as NeonIxBuilder
Expand Down Expand Up @@ -252,7 +252,7 @@ def _validate_txsize(self) -> bool:
raise

def _validate_gas_limit(self):
if not self._neon_tx_exec_cfg.is_without_chainid:
if not self._neon_tx_exec_cfg.is_underpriced_tx_without_chainid:
return True

self.error = "Underpriced transaction without chain-id"
Expand Down Expand Up @@ -539,7 +539,7 @@ def __init__(self, *args, **kwargs):
HolderNeonTxStrategy.__init__(self, *args, **kwargs)

def _validate(self) -> bool:
if not self._neon_tx_exec_cfg.is_without_chainid:
if not self._neon_tx_exec_cfg.is_underpriced_tx_without_chainid:
self.error = 'Normal transaction'
return False

Expand Down
27 changes: 11 additions & 16 deletions proxy/neon_rpc_api_model/neon_rpc_api_model.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import json
import multiprocessing
import traceback
from typing import Optional, Union
from typing import Optional, Union, Tuple

import sha3
from logged_groups import logged_group
Expand All @@ -15,15 +15,15 @@
from ..common_neon.keys_storage import KeyStorage
from ..common_neon.solana_interactor import SolanaInteractor
from ..common_neon.utils import SolanaBlockInfo
from ..common_neon.data import MemPoolTxCfg, NeonTxCfg, NeonEmulatingResult
from ..common_neon.data import NeonTxCfg, NeonEmulatingResult
from ..common_neon.gas_price_calculator import GasPriceCalculator

from ..environment import SOLANA_URL, PP_SOLANA_URL, PYTH_MAPPING_ACCOUNT, NEON_EVM_VERSION, NEON_EVM_REVISION, \
CHAIN_ID, neon_cli

from ..memdb.memdb import MemDB
from ..statistics_exporter.proxy_metrics_interface import StatisticsExporter
from ..mempool import MemPoolClient, MEMPOOL_SERVICE_HOST, MEMPOOL_SERVICE_PORT
from ..mempool import MemPoolTxRequest, MemPoolClient, MEMPOOL_SERVICE_HOST, MEMPOOL_SERVICE_PORT

from .transaction_validator import NeonTxValidator

Expand Down Expand Up @@ -444,20 +444,19 @@ def eth_sendRawTransaction(self, rawTrx: str) -> str:

self._stat_tx_begin()
try:
emulating_result: NeonEmulatingResult = call_trx_emulated(trx)
neon_tx_cfg = self.precheck(trx, emulating_result)
neon_tx_cfg, emulating_result = self.precheck(trx)

# tx_sender = NeonTxSender(self._db, self._solana, trx, steps=EVM_STEP_COUNT)
# with OperatorResourceList(tx_sender):
# tx_sender.execute(neon_tx_cfg)

self._stat_tx_success()
mempool_tx_cfg = MemPoolTxCfg(neon_tx=trx,
neon_tx_cfg=neon_tx_cfg,
emulating_result=emulating_result)
mempool_tx_request = MemPoolTxRequest(neon_tx=trx,
neon_tx_cfg=neon_tx_cfg,
emulating_result=emulating_result)

if not self._mempool_client.send_raw_transaction(mempool_tx_cfg):
raise Exception("Failed to pass tx into MemPool")
if not self._mempool_client.send_raw_transaction(mempool_tx_request):
raise Exception("Failed to pass neon_tx into MemPool")

return eth_signature

Expand All @@ -472,14 +471,10 @@ def eth_sendRawTransaction(self, rawTrx: str) -> str:
self._stat_tx_failed()
raise

def precheck(self, neon_trx: EthTrx, emulating_result: NeonEmulatingResult) -> NeonTxCfg:

def precheck(self, neon_trx: EthTrx) -> Tuple[NeonTxCfg, NeonEmulatingResult]:
min_gas_price = self.gas_price_calculator.get_min_gas_price()
neon_validator = NeonTxValidator(self._solana, neon_trx, min_gas_price)

precheck_result = neon_validator.precheck()

return precheck_result
return neon_validator.precheck()

def _stat_tx_begin(self):
self._stat_exporter.stat_commit_tx_begin()
Expand Down
12 changes: 6 additions & 6 deletions proxy/neon_rpc_api_model/transaction_validator.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from __future__ import annotations

from typing import Tuple
from logged_groups import logged_group

from ..common_neon.eth_proto import Trx as EthTx
Expand Down Expand Up @@ -58,16 +58,16 @@ def is_underpriced_tx_without_chainid(self) -> bool:
return False
return (self._tx.gasPrice < self._min_gas_price) or (self._tx.gasLimit < self._estimated_gas)

def precheck(self) -> NeonTxCfg:
def precheck(self) -> Tuple[NeonTxCfg, NeonEmulatingResult]:
try:
self._prevalidate_tx()
emulating_result: NeonEmulatingResult = call_trx_emulated(self._tx)
self._prevalidate_emulator(emulating_result)

is_without_chainid = self.is_underpriced_tx_without_chainid()
precheck_result = NeonTxCfg(steps_executed=emulating_result["steps_executed"],
is_without_chainid=is_without_chainid)
return precheck_result
is_underpriced_tx_without_chainid = self.is_underpriced_tx_without_chainid()
neon_tx_cfg = NeonTxCfg(steps_executed=emulating_result["steps_executed"],
is_underpriced_tx_without_chainid=is_underpriced_tx_without_chainid)
return neon_tx_cfg, emulating_result

except Exception as e:
self.extract_ethereum_error(e)
Expand Down