Skip to content

#425 use Solana's block height as neon block number #426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
afalaleev merged 20 commits into from
Jan 15, 2022
Merged

#425 use Solana's block height as neon block number #426

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
4c595af
changes
otselnik Dec 30, 2021
a587c3b
Merge remote-tracking branch 'origin/develop' into 425_use_solanas_bl…
otselnik Dec 30, 2021
8b340a8
fixes
otselnik Dec 30, 2021
4e5accf
Fix review issues
Jan 5, 2022
c389404
Merge last changes from develop
afalaleev Jan 5, 2022
dd8e52a
Fix after merge develop
afalaleev Jan 6, 2022
a464957
Optimize indexing of blocks
afalaleev Jan 6, 2022
67c9e9b
Get latest block from DB
afalaleev Jan 6, 2022
d007ccc
Get block number from DB. #425
afalaleev Jan 11, 2022
ca5319b
Refactor indexer.
afalaleev Jan 11, 2022
c708d44
Remove old code
afalaleev Jan 11, 2022
6a59f31
Get results only for defined instruction
afalaleev Jan 11, 2022
da46dee
Store min parsed slot for receipts
afalaleev Jan 11, 2022
edf2c70
Save state of parsed transaction between parsing iterations
afalaleev Jan 12, 2022
da830df
Don't filter getLogs by 0 and latest values
afalaleev Jan 12, 2022
1e7c600
Add finalized stage for neon transactions and blocks
afalaleev Jan 13, 2022
ad2b4d1
Merge branch 'develop' into 425_use_solanas_block_height_as_neon_bloc…
afalaleev Jan 13, 2022
1de44e9
Add Exception to start of log messages
afalaleev Jan 14, 2022
bc8747f
Add check before encoding/decoding values to/from the DB
afalaleev Jan 14, 2022
66b9711
Fix types in DB
afalaleev Jan 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge remote-tracking branch 'origin/develop' into 425_use_solanas_bl… 
…ock_height_as_neon_block_number
  • Loading branch information
@otselnik
otselnik committed Dec 30, 2021
commit a587c3b44fd23976ad7b125de09cbe06def503c7
172 changes: 18 additions & 154 deletions proxy/indexer/indexer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,41 +209,7 @@ def process_receipts(self):
sign = instruction_data[25:90]
unsigned_msg = instruction_data[90:]

(eth_trx, eth_signature, from_address) = get_trx_receipts(unsigned_msg, sign)

got_result = get_trx_results(trx)
if got_result is not None:
trx_table[eth_signature] = TransactionStruct(
eth_trx,
eth_signature,
from_address,
got_result,
[signature],
None,
None,
slot
)
else:
logger.error("RESULT NOT FOUND IN 05\n{}".format(json.dumps(trx, indent=4, sort_keys=True)))

elif instruction_data[0] == 0x09 or instruction_data[0] == 0x13: # PartialCallFromRawEthereumTX PartialCallFromRawEthereumTXv02
# if instruction_data[0] == 0x09:
# logger.debug("{:>10} {:>6} PartialCallFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex()))
# if instruction_data[0] == 0x13:
# logger.debug("{:>10} {:>6} PartialCallFromRawEthereumTXv02 0x{}".format(slot, counter, instruction_data.hex()))


storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]]

# collateral_pool_buf = instruction_data[1:5]
# step_count = instruction_data[5:13]
# from_addr = instruction_data[13:33]

sign = instruction_data[33:98]
unsigned_msg = instruction_data[98:]

(eth_trx, eth_signature, from_address) = get_trx_receipts(unsigned_msg, sign)
(eth_trx, eth_signature, from_address) = get_trx_receipts(unsigned_msg, sign)

got_result = get_trx_results(trx)
if got_result is not None:
Expand Down Expand Up @@ -382,10 +348,12 @@ def process_receipts(self):
(eth_trx, eth_signature, from_address) = get_trx_receipts(unsigned_msg, sign)

if eth_signature in trx_table:
trx_table[eth_signature].signatures.append(signature)
trx_table[eth_signature].slot.append(slot)
if got_result is not None:
trx_table[eth_signature].got_result = got_result
trx_table[eth_signature].signatures.append(signature)
else:
trx_table[eth_signature].signatures.insert(0,signature)
else:
trx_table[eth_signature] = TransactionStruct(
eth_trx,
Expand Down Expand Up @@ -418,39 +386,12 @@ def process_receipts(self):
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]]
got_result = get_trx_results(trx)

if storage_account in continue_table:
if got_result is not None:
if continue_table[storage_account].results is not None:
logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE")
if continue_table[storage_account].accounts != blocked_accounts:
logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL")

continue_table[storage_account].results = got_result
continue_table[storage_account].signatures.insert(0, signature)
else:
continue_table[storage_account].signatures.append(signature)
else:
continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts)

elif instruction_data[0] == 0x0b or instruction_data[0] == 0x16: # ExecuteTrxFromAccountDataIterative ExecuteTrxFromAccountDataIterativeV02
if instruction_data[0] == 0x0b:
# logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterative 0x{}".format(slot, counter, instruction_data.hex()))
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]]
if instruction_data[0] == 0x16:
# logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeV02 0x{}".format(slot, counter, instruction_data.hex()))
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]]

holder_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][1]]

if storage_account in continue_table:
continue_table[storage_account].signatures.append(signature)
if storage_account in continue_table:
continue_table[storage_account].slot.append(slot)

if holder_account in holder_table:
if holder_table[holder_account].storage_account != storage_account:
logger.error("Strange behavior. Pay attention. STORAGE_ACCOUNT != STORAGE_ACCOUNT")
holder_table[holder_account] = HolderStruct(storage_account)
else:
if holder_account in holder_table:
if holder_table[holder_account].storage_account != storage_account:
logger.error("Strange behavior. Pay attention. STORAGE_ACCOUNT != STORAGE_ACCOUNT")
holder_table[holder_account] = HolderStruct(storage_account)
else:
logger.error("Strange behavior. Pay attention. HOLDER ACCOUNT NOT FOUND")
Expand All @@ -463,98 +404,20 @@ def process_receipts(self):
logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL")

continue_table[storage_account].results = got_result
continue_table[storage_account].signatures.append(signature)
else:
continue_table[storage_account].signatures.insert(0,signature)
else:
continue_table[storage_account] = ContinueStruct(signature, got_result, [slot], blocked_accounts)
holder_table[holder_account] = HolderStruct(storage_account)

storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][6:]]

continue_table[storage_account] = ContinueStruct(signature, ([], "0x0", 0, [], trx['slot']), slot, blocked_accounts)

elif instruction_data[0] == 0x0d:
# logger.debug("{:>10} {:>6} PartialCallOrContinueFromRawEthereumTX 0x{}".format(slot, counter, instruction_data.hex()))

storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]]
got_result = get_trx_results(trx)

# collateral_pool_buf = instruction_data[1:5]
# step_count = instruction_data[5:13]
# from_addr = instruction_data[13:33]

sign = instruction_data[33:98]
unsigned_msg = instruction_data[98:]

(eth_trx, eth_signature, from_address) = get_trx_receipts(unsigned_msg, sign)

if eth_signature in trx_table:
if got_result is not None:
continue_table[storage_account].signatures.insert(0, signature)
trx_table[eth_signature].got_result = got_result
else:
trx_table[eth_signature].signatures.append(signature)
trx_table[eth_signature].slot = max(trx_table[eth_signature].slot, slot)
else:
trx_table[eth_signature] = TransactionStruct(
eth_trx,
eth_signature,
from_address,
got_result,
[signature],
storage_account,
blocked_accounts,
slot
)

if storage_account in continue_table:
continue_result = continue_table[storage_account]
trx_table[eth_signature].signatures += continue_result.signatures
if continue_result.results is not None:
if trx_table[eth_signature].got_result is not None:
logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE")
trx_table[eth_signature].got_result = continue_result.results

del continue_table[storage_account]

elif instruction_data[0] == 0x0e:
# logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeOrContinue 0x{}".format(slot, counter, instruction_data.hex()))

holder_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][1]]
blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][7:]]
got_result = get_trx_results(trx)

if storage_account in continue_table:
if holder_account in holder_table:
if holder_table[holder_account].storage_account != storage_account:
logger.error("Strange behavior. Pay attention. STORAGE_ACCOUNT != STORAGE_ACCOUNT")
holder_table[holder_account] = HolderStruct(storage_account)
else:
logger.error("Strange behavior. Pay attention. HOLDER ACCOUNT NOT FOUND")
holder_table[holder_account] = HolderStruct(storage_account)

if got_result is not None:
if continue_table[storage_account].results:
logger.error("Strange behavior. Pay attention. RESULT ALREADY EXISTS IN CONTINUE TABLE")
if continue_table[storage_account].accounts != blocked_accounts:
logger.error("Strange behavior. Pay attention. BLOCKED ACCOUNTS NOT EQUAL")

continue_table[storage_account].results = got_result
continue_table[storage_account].signatures.insert(0, signature)
else:
continue_table[storage_account].signatures.append(signature)
else:
continue_table[storage_account] = ContinueStruct(signature, got_result, slot, blocked_accounts)
holder_table[holder_account] = HolderStruct(storage_account)

if instruction_data[0] > 0x16:
logger.debug("{:>10} {:>6} Unknown 0x{}".format(slot, counter, instruction_data.hex()))

pass
if instruction_data[0] > 0x16:
logger.debug("{:>10} {:>6} Unknown 0x{}".format(slot, counter, instruction_data.hex()))
pass

for eth_signature, trx_struct in trx_table.items():
if trx_struct.got_result is not None:
seen_slots.difference_update(trx_struct.slot)
self.db.submit_transaction(
self.client,
trx_struct.eth_trx,
Expand All @@ -564,8 +427,9 @@ def process_receipts(self):
trx_struct.signatures
)
elif trx_struct.storage is not None:
self.processed_slot = min(self.processed_slot, min(trx_struct.slot))
if not self.db.submit_transaction_part(trx_struct.eth_signature, trx_struct.signatures):
if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT:
if abs(max(trx_struct.slot) - self.current_slot) > CANCEL_TIMEOUT:
logger.debug("Probably blocked")
logger.debug(trx_struct.eth_signature)
logger.debug(trx_struct.signatures)
Expand Down
16 changes: 0 additions & 16 deletions proxy/indexer/sql_dict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,6 @@
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)


def encode(obj):
"""Serialize an object using pickle to a binary format accepted by SQLite."""
return psycopg2.Binary(dumps(obj, protocol=PICKLE_PROTOCOL))


def decode(obj):
"""Deserialize objects retrieved from SQLite."""
return loads(bytes(obj))


def dummy(obj):
"""Does nothing"""
return obj


class SQLDict(MutableMapping):
"""Serialize an object using pickle to a binary format accepted by SQLite."""

Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.