Skip to content

Commit cec7a38

Browse files
committed
Merge remote-tracking branch 'origin/develop' into 295_iterative_execution
2 parents 0eebc1f + 5d6dab2 commit cec7a38

File tree

12 files changed

+894
-13
lines changed

12 files changed

+894
-13
lines changed

.buildkite/steps/deploy-test.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ docker run --rm -ti --network=host \
9292
--entrypoint ./deploy-test.sh \
9393
${EXTRA_ARGS:-} \
9494
$UNISWAP_V2_CORE_IMAGE \
95+
all
9596

9697
echo "Run tests return"
9798
exit 0

proxy/__main__.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,26 @@
88
:copyright: (c) 2013-present by Abhinav Singh and contributors.
99
:license: BSD, see LICENSE for more details.
1010
"""
11+
1112
from .proxy import entry_point
13+
import os
14+
from .indexer.airdropper import run_airdropper
1215

1316
if __name__ == '__main__':
14-
entry_point()
17+
airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True']
18+
if airdropper_mode:
19+
print("Will run in airdropper mode")
20+
solana_url = os.environ['SOLANA_URL']
21+
evm_loader_id = os.environ['EVM_LOADER']
22+
faucet_url = os.environ['FAUCET_URL']
23+
wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'].split(',')
24+
airdrop_amount = int(os.environ['AIRDROP_AMOUNT'])
25+
log_level = os.environ['LOG_LEVEL']
26+
run_airdropper(solana_url,
27+
evm_loader_id,
28+
faucet_url,
29+
wrapper_whitelist,
30+
airdrop_amount,
31+
log_level)
32+
else:
33+
entry_point()

proxy/common_neon/solana_interactor.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@
1111

1212
from .costs import update_transaction_cost
1313
from .utils import get_from_dict
14-
from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY
15-
14+
from ..environment import EVM_LOADER_ID, CONFIRMATION_CHECK_DELAY, LOG_SENDING_SOLANA_TRANSACTION
1615

1716
logger = logging.getLogger(__name__)
1817
logger.setLevel(logging.DEBUG)
@@ -90,14 +89,15 @@ def collect_result(self, reciept, eth_trx, reason=None):
9089
update_transaction_cost(result, eth_trx, reason)
9190
return result
9291

93-
9492
def send_measured_transaction(self, trx, eth_trx, reason):
93+
if LOG_SENDING_SOLANA_TRANSACTION:
94+
logger.debug("send_measured_transaction for reason %s: %s ", reason, trx.__dict__)
9595
result = self.send_transaction(trx, eth_trx, reason=reason)
9696
self.get_measurements(result)
9797
return result
9898

99-
100-
# Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in .buildkite/steps/deploy-test.sh)
99+
# Do not rename this function! This name used in CI measurements (see function `cleanup_docker` in
100+
# .buildkite/steps/deploy-test.sh)
101101
def get_measurements(self, result):
102102
try:
103103
measurements = self.extract_measurements_from_receipt(result)

proxy/docker-compose-test.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ services:
6565
POSTGRES_USER: neon-proxy
6666
POSTGRES_PASSWORD: neon-proxy-pass
6767
NEW_USER_AIRDROP_AMOUNT: 100
68+
LOG_SENDING_SOLANA_TRANSACTION: "YES"
6869
CONFIG: ci
6970
hostname: proxy
7071
depends_on:

proxy/environment.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600"))
1717
MINIMAL_GAS_PRICE=int(os.environ.get("MINIMAL_GAS_PRICE", 1))*10**9
1818
EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0"))
19+
LOG_SENDING_SOLANA_TRANSACTION = os.environ.get("LOG_SENDING_SOLANA_TRANSACTION", "NO") == "YES"
1920

2021
class solana_cli:
2122
def call(self, *args):

proxy/indexer/airdropper.py

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
from proxy.indexer.indexer_base import IndexerBase, logger
2+
import os
3+
import requests
4+
import base58
5+
import json
6+
import logging
7+
8+
try:
9+
from utils import check_error
10+
from sql_dict import SQLDict
11+
except ImportError:
12+
from .utils import check_error
13+
from .sql_dict import SQLDict
14+
15+
class Airdropper(IndexerBase):
16+
def __init__(self,
17+
solana_url,
18+
evm_loader_id,
19+
faucet_url = '',
20+
wrapper_whitelist = [],
21+
airdrop_amount = 10,
22+
log_level = 'INFO'):
23+
IndexerBase.__init__(self, solana_url, evm_loader_id, log_level)
24+
25+
# collection of eth-address-to-create-accout-trx mappings
26+
# for every addresses that was already funded with airdrop
27+
self.airdrop_ready = SQLDict(tablename="airdrop_ready")
28+
self.wrapper_whitelist = wrapper_whitelist
29+
self.airdrop_amount = airdrop_amount
30+
self.faucet_url = faucet_url
31+
32+
33+
# helper function checking if given contract address is in whitelist
34+
def _is_allowed_wrapper_contract(self, contract_addr):
35+
return contract_addr in self.wrapper_whitelist
36+
37+
38+
# helper function checking if given 'create account' corresponds to 'create erc20 token account' instruction
39+
def _check_create_instr(self, account_keys, create_acc, create_token_acc):
40+
# Must use the same Ethereum account
41+
if account_keys[create_acc['accounts'][1]] != account_keys[create_token_acc['accounts'][2]]:
42+
return False
43+
# Must use the same token program
44+
if account_keys[create_acc['accounts'][5]] != account_keys[create_token_acc['accounts'][6]]:
45+
return False
46+
# Token program must be system token program
47+
if account_keys[create_acc['accounts'][5]] != 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA':
48+
return False
49+
# CreateERC20TokenAccount instruction must use ERC20-wrapper from whitelist
50+
if not self._is_allowed_wrapper_contract(account_keys[create_token_acc['accounts'][3]]):
51+
return False
52+
return True
53+
54+
55+
# helper function checking if given 'create erc20 token account' corresponds to 'token transfer' instruction
56+
def _check_transfer(self, account_keys, create_token_acc, token_transfer) -> bool:
57+
return account_keys[create_token_acc['accounts'][1]] == account_keys[token_transfer['accounts'][1]]
58+
59+
60+
def _airdrop_to(self, create_acc):
61+
eth_address = "0x" + bytearray(base58.b58decode(create_acc['data'])[20:][:20]).hex()
62+
63+
if eth_address in self.airdrop_ready: # transaction already processed
64+
return
65+
66+
logger.info(f"Airdrop to address: {eth_address}")
67+
68+
json_data = { 'wallet': eth_address, 'amount': self.airdrop_amount }
69+
resp = requests.post(self.faucet_url + '/request_eth_token', json = json_data)
70+
if not resp.ok:
71+
logger.warning(f'Failed to airdrop: {resp.status_code}')
72+
return
73+
74+
self.airdrop_ready[eth_address] = create_acc
75+
76+
77+
def process_trx_airdropper_mode(self, trx):
78+
if check_error(trx):
79+
return
80+
81+
# helper function finding all instructions that satisfies predicate
82+
def find_instructions(trx, predicate):
83+
return [instr for instr in trx['transaction']['message']['instructions'] if predicate(instr)]
84+
85+
account_keys = trx["transaction"]["message"]["accountKeys"]
86+
87+
# Finding instructions specific for airdrop.
88+
# Airdrop triggers on sequence:
89+
# neon.CreateAccount -> neon.CreateERC20TokenAccount -> spl.Transfer (maybe shuffled)
90+
# First: select all instructions that can form such chains
91+
predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \
92+
and base58.b58decode(instr['data'])[0] == 0x02
93+
create_acc_list = find_instructions(trx, predicate)
94+
95+
predicate = lambda instr: account_keys[instr['programIdIndex']] == self.evm_loader_id \
96+
and base58.b58decode(instr['data'])[0] == 0x0f
97+
create_token_acc_list = find_instructions(trx, predicate)
98+
99+
predicate = lambda instr: account_keys[instr['programIdIndex']] == 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' \
100+
and base58.b58decode(instr['data'])[0] == 0x03
101+
token_transfer_list = find_instructions(trx, predicate)
102+
103+
# Second: Find exact chains of instructions in sets created previously
104+
for create_acc in create_acc_list:
105+
for create_token_acc in create_token_acc_list:
106+
if not self._check_create_instr(account_keys, create_acc, create_token_acc):
107+
continue
108+
for token_transfer in token_transfer_list:
109+
if not self._check_transfer(account_keys, create_token_acc, token_transfer):
110+
continue
111+
self._airdrop_to(create_acc)
112+
113+
114+
def process_functions(self):
115+
IndexerBase.process_functions(self)
116+
logger.debug("Process receipts")
117+
self.process_receipts()
118+
119+
120+
def process_receipts(self):
121+
counter = 0
122+
for signature in self.transaction_order:
123+
counter += 1
124+
if signature in self.transaction_receipts:
125+
trx = self.transaction_receipts[signature]
126+
if trx is None:
127+
logger.error("trx is None")
128+
del self.transaction_receipts[signature]
129+
continue
130+
if 'slot' not in trx:
131+
logger.debug("\n{}".format(json.dumps(trx, indent=4, sort_keys=True)))
132+
exit()
133+
if trx['transaction']['message']['instructions'] is not None:
134+
self.process_trx_airdropper_mode(trx)
135+
136+
137+
def run_airdropper(solana_url,
138+
evm_loader_id,
139+
faucet_url = '',
140+
wrapper_whitelist = [],
141+
airdrop_amount = 10,
142+
log_level = 'INFO'):
143+
logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s')
144+
logger.setLevel(logging.DEBUG)
145+
logger.info(f"""Running indexer with params:
146+
solana_url: {solana_url},
147+
evm_loader_id: {evm_loader_id},
148+
log_level: {log_level},
149+
faucet_url: {faucet_url},
150+
wrapper_whitelist: {wrapper_whitelist},
151+
airdrop_amount: {airdrop_amount}""")
152+
153+
airdropper = Airdropper(solana_url,
154+
evm_loader_id,
155+
faucet_url,
156+
wrapper_whitelist,
157+
airdrop_amount,
158+
log_level)
159+
airdropper.run()

proxy/indexer/indexer.py

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@ def __init__(self, storage_account):
2828

2929

3030
class ContinueStruct:
31-
def __init__(self, signature, results, accounts = None):
31+
def __init__(self, signature, results, slot, accounts = None):
3232
self.signatures = [signature]
3333
self.results = results
34+
self.slot = slot
3435
self.accounts = accounts
3536

3637

@@ -165,7 +166,7 @@ def process_receipts(self):
165166
continue_result.signatures,
166167
storage_account,
167168
continue_result.accounts,
168-
slot
169+
max(slot, continue_result.slot)
169170
)
170171

171172
del continue_table[storage_account]
@@ -271,6 +272,7 @@ def process_receipts(self):
271272
logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL")
272273
trx_table[eth_signature].got_result = continue_result.results
273274
trx_table[eth_signature].signatures += continue_result.signatures
275+
trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, continue_result.slot)
274276

275277
del continue_table[storage_account]
276278

@@ -296,7 +298,7 @@ def process_receipts(self):
296298

297299
continue_table[storage_account].results = got_result
298300
else:
299-
continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts)
301+
continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts)
300302

301303
elif instruction_data[0] == 0x0b or instruction_data[0] == 0x16: # ExecuteTrxFromAccountDataIterative ExecuteTrxFromAccountDataIterativeV02
302304
if instruction_data[0] == 0x0b:
@@ -319,7 +321,7 @@ def process_receipts(self):
319321
else:
320322
holder_table[holder_account] = HolderStruct(storage_account)
321323
else:
322-
continue_table[storage_account] = ContinueStruct(signature, None, blocked_accounts)
324+
continue_table[storage_account] = ContinueStruct(signature, None, slot, blocked_accounts)
323325
holder_table[holder_account] = HolderStruct(storage_account)
324326

325327

@@ -329,7 +331,7 @@ def process_receipts(self):
329331
storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
330332
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]]
331333

332-
continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), blocked_accounts)
334+
continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), slot, blocked_accounts)
333335

334336
elif instruction_data[0] == 0x0d:
335337
# logger.debug("{:>10} {:>6} PartialCallOrContinueFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex()))
@@ -349,6 +351,7 @@ def process_receipts(self):
349351

350352
if eth_signature in trx_table:
351353
trx_table[eth_signature].signatures.append(signature)
354+
trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, slot)
352355
else:
353356
trx_table[eth_signature] = TransactionStruct(
354357
eth_trx,
@@ -396,7 +399,7 @@ def process_receipts(self):
396399

397400
continue_table[storage_account].results = got_result
398401
else:
399-
continue_table[storage_account] = ContinueStruct(signature, got_result, blocked_accounts)
402+
continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts)
400403
holder_table[holder_account] = HolderStruct(storage_account)
401404

402405
if instruction_data[0] > 0x16:

proxy/plugin/solana_rest_api.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -639,7 +639,8 @@ def handle_request(self, request: HttpParser) -> None:
639639
traceback.print_exc()
640640
response = {'jsonrpc': '2.0', 'error': {'code': -32000, 'message': str(err)}}
641641

642-
logger.debug('>>> %s 0x%0x %s', threading.get_ident(), id(self.model), json.dumps(response))
642+
logger.debug('>>> %s 0x%0x %s %s', threading.get_ident(), id(self.model), json.dumps(response),
643+
request['method'] if 'method' in request else '---')
643644

644645
self.client.queue(memoryview(build_http_response(
645646
httpStatusCodes.OK, body=json.dumps(response).encode('utf8'),

proxy/testing/mock_server.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import requests
2+
3+
from flask import Flask, request
4+
from threading import Thread
5+
6+
class MockServer(Thread):
7+
def __init__(self, port):
8+
super().__init__()
9+
self.port = port
10+
self.app = Flask(__name__)
11+
self.url = "http://localhost:%s" % self.port
12+
self.app.add_url_rule("/shutdown", view_func=self._shutdown_server)
13+
14+
def add_url_rule(self, url, callback, methods):
15+
self.app.add_url_rule(url, view_func=callback, methods=methods)
16+
17+
def _shutdown_server(self):
18+
if not 'werkzeug.server.shutdown' in request.environ:
19+
raise RuntimeError('Not running the development server')
20+
request.environ['werkzeug.server.shutdown']()
21+
return 'Server shutting down...'
22+
23+
def run(self):
24+
self.app.run(port=self.port)
25+
26+
def shutdown_server(self):
27+
requests.get("http://localhost:%s/shutdown" % self.port)
28+
self.join()

0 commit comments

Comments
 (0)