Merge branch 'develop' into 191-create-and-airdrop-eth-account

# Conflicts:
#	proxy/plugin/solana_rest_api.py
#	proxy/plugin/solana_rest_api_tools.py

Author: rozhkovdmitrii

Dockerfile

@@ -32,7 +32,7 @@ COPY --from=cli /opt/solana/bin/solana \
 
 COPY --from=spl /opt/solana/bin/solana /cli/bin/
 COPY --from=spl /opt/evm_loader.so /opt/evm_loader-keypair.json \
-                /opt/neon-cli /spl/bin/
+                /opt/neon-cli /opt/faucet /spl/bin/
 COPY --from=spl /opt/spl-token /opt/test_token_keypair /opt/test_token_owner /spl/bin/
 COPY --from=spl /opt/collateral_pool_generator.py \
                 /opt/solana_utils.py \

proxy/deploy-test.sh

@@ -3,8 +3,10 @@ set -xeuo pipefail
 
 echo "Deploy test..."
 
+solana config set -u $SOLANA_URL
 solana address || solana-keygen new --no-passphrase
-export $(/spl/bin/neon-cli --evm_loader JxujFZpNBPADbfw2MnPPgnnFGruzp2ELSFWPQgrjz5D neon-elf-params /spl/bin/evm_loader.so | grep NEON_REVISION | xargs)
+solana program dump "$EVM_LOADER" ./evm_loader.dump
+export $(/spl/bin/neon-cli --evm_loader "$EVM_LOADER" neon-elf-params ./evm_loader.dump)
 
 curl -v --header "Content-Type: application/json" --data '{"method":"eth_blockNumber","id":1,"jsonrpc":"2.0","params":[]}' $PROXY_URL
 

proxy/docker-compose-test.yml

@@ -5,8 +5,8 @@ services:
     container_name: solana
     image: neonlabsorg/solana:${SOLANA_REVISION:-v1.7.9-resources}
     environment:
-      - SOLANA_URL=http://solana:8899
-      - RUST_LOG=solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug
+      SOLANA_URL: http://solana:8899
+      RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug
     hostname: solana
     expose:
       - "8899"
@@ -17,11 +17,28 @@ services:
     networks:
       - net
 
+  postgres:
+    container_name: postgres
+    image: postgres:14.0
+    command: postgres -c 'max_connections=1000'
+    environment:
+      POSTGRES_DB: neon-db
+      POSTGRES_USER: neon-proxy
+      POSTGRES_PASSWORD: neon-proxy-pass
+    hostname: postgres
+    expose:
+      - "5432"
+    networks:
+      - net
+
   proxy:
     container_name: proxy
     image: neonlabsorg/proxy:${REVISION}
     environment:
-      - CONFIG=ci
+      POSTGRES_DB: neon-db
+      POSTGRES_USER: neon-proxy
+      POSTGRES_PASSWORD: neon-proxy-pass
+      CONFIG: ci
     hostname: proxy
     ports:
       - 127.0.0.1:9090:9090

proxy/environment.py

@@ -0,0 +1,60 @@
+import os
+import subprocess
+import logging
+from solana.publickey import PublicKey
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+solana_url = os.environ.get("SOLANA_URL", "http://localhost:8899")
+evm_loader_id = os.environ.get("EVM_LOADER")
+neon_cli_timeout = float(os.environ.get("NEON_CLI_TIMEOUT", "0.1"))
+
+class solana_cli:
+    def call(self, *args):
+        try:
+            cmd = ["solana",
+                   "--url", solana_url,
+                   ] + list(args)
+            print(cmd)
+            return subprocess.check_output(cmd, universal_newlines=True)
+        except subprocess.CalledProcessError as err:
+            import sys
+            logger.debug("ERR: solana error {}".format(err))
+            raise
+
+
+class neon_cli:
+    def call(self, *args):
+        try:
+            cmd = ["neon-cli",
+                   "--commitment=recent",
+                   "--url", solana_url,
+                   "--evm_loader={}".format(evm_loader_id),
+                   ] + list(args)
+            print(cmd)
+            return subprocess.check_output(cmd, timeout=neon_cli_timeout, universal_newlines=True)
+        except subprocess.CalledProcessError as err:
+            import sys
+            logger.debug("ERR: neon-cli error {}".format(err))
+            raise
+
+def read_elf_params(out_dict):
+    logger.debug('load for solana_url={} and evm_loader_id={}'.format(solana_url, evm_loader_id))
+    res = solana_cli().call('program', 'dump', evm_loader_id, './evm_loader.dump')
+    substr = "Wrote program to "
+    path = ""
+    for line in res.splitlines():
+        if line.startswith(substr):
+            path = line[len(substr):].strip()
+    if path == "":
+        raise Exception("cannot program dump for ", evm_loader_id)
+    for param in neon_cli().call("neon-elf-params", path).splitlines():
+        if param.startswith('NEON_') and '=' in param:
+            v = param.split('=')
+            out_dict[v[0]] = v[1]
+
+ELF_PARAMS = {}
+read_elf_params(ELF_PARAMS)
+COLLATERAL_POOL_BASE = ELF_PARAMS.get("NEON_POOL_BASE")
+ETH_TOKEN_MINT_ID: PublicKey = PublicKey(ELF_PARAMS.get("NEON_TOKEN_MINT"))

proxy/indexer/solana_receipts_update.py

@@ -6,20 +6,20 @@
 import logging
 from solana.rpc.api import Client
 from multiprocessing.dummy import Pool as ThreadPool
-from sqlitedict import SqliteDict
 from typing import Dict, Union
+from proxy.environment import solana_url, evm_loader_id
 
 
 try:
     from utils import check_error, get_trx_results, get_trx_receipts, LogDB, Canceller
+    from sql_dict import SQLDict
 except ImportError:
     from .utils import check_error, get_trx_results, get_trx_receipts, LogDB, Canceller
+    from .sql_dict import SQLDict
 
 
-solana_url = os.environ.get("SOLANA_URL", "https://api.devnet.solana.com")
-evm_loader_id = os.environ.get("EVM_LOADER", "eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU")
 PARALLEL_REQUESTS = int(os.environ.get("PARALLEL_REQUESTS", "2"))
-
+CANCEL_TIMEOUT = int(os.environ.get("CANCEL_TIMEOUT", "60"))
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -45,7 +45,7 @@ def __init__(self, signature, results, accounts = None):
 
 
 class TransactionStruct:
-    def __init__(self, eth_trx, eth_signature, from_address, got_result, signatures, storage, blocked_accounts):
+    def __init__(self, eth_trx, eth_signature, from_address, got_result, signatures, storage, blocked_accounts, slot):
         # logger.debug(eth_signature)
         self.eth_trx = eth_trx
         self.eth_signature = eth_signature
@@ -54,19 +54,20 @@ def __init__(self, eth_trx, eth_signature, from_address, got_result, signatures,
         self.signatures = signatures
         self.storage = storage
         self.blocked_accounts = blocked_accounts
+        self.slot = slot
 
 
 class Indexer:
     def __init__(self):
         self.client = Client(solana_url)
         self.canceller = Canceller()
-        self.logs_db = LogDB(filename="local.db")
-        self.blocks_by_hash = SqliteDict(filename="local.db", tablename="solana_blocks_by_hash", autocommit=True)
-        self.transaction_receipts = SqliteDict(filename="local.db", tablename="known_transactions", autocommit=True, encode=json.dumps, decode=json.loads)
-        self.ethereum_trx = SqliteDict(filename="local.db", tablename="ethereum_transactions", autocommit=True, encode=json.dumps, decode=json.loads)
-        self.eth_sol_trx = SqliteDict(filename="local.db", tablename="ethereum_solana_transactions", autocommit=True, encode=json.dumps, decode=json.loads)
-        self.sol_eth_trx = SqliteDict(filename="local.db", tablename="solana_ethereum_transactions", autocommit=True, encode=json.dumps, decode=json.loads)
-        self.constants = SqliteDict(filename="local.db", tablename="constants", autocommit=True)
+        self.logs_db = LogDB()
+        self.blocks_by_hash = SQLDict(tablename="solana_blocks_by_hash")
+        self.transaction_receipts = SQLDict(tablename="known_transactions")
+        self.ethereum_trx = SQLDict(tablename="ethereum_transactions")
+        self.eth_sol_trx = SQLDict(tablename="ethereum_solana_transactions")
+        self.sol_eth_trx = SQLDict(tablename="solana_ethereum_transactions")
+        self.constants = SQLDict(tablename="constants")
         self.last_slot = 0
         self.current_slot = 0
         self.transaction_order = []
@@ -271,6 +272,7 @@ def process_receipts(self):
                                                     continue_result.signatures,
                                                     storage_account,
                                                     continue_result.accounts,
+                                                    slot
                                                 )
 
                                             del continue_table[storage_account]
@@ -327,7 +329,16 @@ def process_receipts(self):
                             got_result = get_trx_results(trx)
                             if got_result is not None:
                                 # self.submit_transaction(eth_trx, eth_signature, from_address, got_result, [signature])
-                                trx_table[eth_signature] = TransactionStruct(eth_trx, eth_signature, from_address, got_result, [signature], None, None)
+                                trx_table[eth_signature] = TransactionStruct(
+                                        eth_trx,
+                                        eth_signature,
+                                        from_address,
+                                        got_result,
+                                        [signature],
+                                        None,
+                                        None,
+                                        slot
+                                    )
                             else:
                                 logger.error("RESULT NOT FOUND IN 05\n{}".format(json.dumps(trx, indent=4, sort_keys=True)))
 
@@ -357,7 +368,8 @@ def process_receipts(self):
                                     None,
                                     [signature],
                                     storage_account,
-                                    blocked_accounts
+                                    blocked_accounts,
+                                    slot
                                 )
 
                             if storage_account in continue_table:
@@ -369,11 +381,15 @@ def process_receipts(self):
                                 trx_table[eth_signature].signatures = continue_result.signatures
                                 del continue_table[storage_account]
 
-                        elif instruction_data[0] == 0x0a: # Continue
-                            # logger.debug("{:>10} {:>6} Continue 0x{}".format(slot, counter, instruction_data.hex()))
+                        elif instruction_data[0] == 0x0a or instruction_data[0] == 0x14: # Continue or ContinueV02
 
                             storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
-                            blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]]
+                            if instruction_data[0] == 0x0a:
+                                # logger.debug("{:>10} {:>6} Continue 0x{}".format(slot, counter, instruction_data.hex()))
+                                blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]]
+                            if instruction_data[0] == 0x14:
+                                # logger.debug("{:>10} {:>6} ContinueV02 0x{}".format(slot, counter, instruction_data.hex()))
+                                blocked_accounts = [trx['transaction']['message']['accountKeys'][acc_idx] for acc_idx in instruction['accounts'][5:]]
                             got_result = get_trx_results(trx)
 
                             if storage_account in continue_table:
@@ -406,13 +422,14 @@ def process_receipts(self):
                                 continue_table[storage_account].signatures.append(signature)
 
                                 if holder_account in holder_table:
-                                    holder_table[holder_account] = HolderStruct(storage_account)
+                                    if holder_table[holder_account].storage_account != storage_account:
+                                        logger.error("Strange behavior. Pay attention. STORAGE_ACCOUNT != STORAGE_ACCOUNT")
+                                        holder_table[holder_account] = HolderStruct(storage_account)
                                 else:
                                     holder_table[holder_account] = HolderStruct(storage_account)
                             else:
                                 continue_table[storage_account] =  ContinueStruct(signature, None, blocked_accounts)
                                 holder_table[holder_account] = HolderStruct(storage_account)
-                                # self.add_hunged_storage(trx, storage_account)
 
 
                         elif instruction_data[0] == 0x0c or instruction_data[0] == 0x15: # Cancel
@@ -440,7 +457,16 @@ def process_receipts(self):
                             if eth_signature in trx_table:
                                 trx_table[eth_signature].signatures.append(signature)
                             else:
-                                trx_table[eth_signature] = TransactionStruct(eth_trx, eth_signature, from_address, got_result, [signature], storage_account, blocked_accounts)
+                                trx_table[eth_signature] = TransactionStruct(
+                                        eth_trx,
+                                        eth_signature,
+                                        from_address,
+                                        got_result,
+                                        [signature],
+                                        storage_account,
+                                        blocked_accounts,
+                                        slot
+                                    )
 
                         elif instruction_data[0] == 0x0e:
                             # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeOrContinue 0x{}".format(slot, counter, instruction_data.hex()))
@@ -454,9 +480,11 @@ def process_receipts(self):
                                 continue_table[storage_account].signatures.append(signature)
 
                                 if holder_account in holder_table:
-                                    logger.error("Strange behavior. Pay attention. HOLDER ACCOUNT FOUND")
-                                    holder_table[holder_account] = HolderStruct(storage_account)
+                                    if holder_table[holder_account].storage_account != storage_account:
+                                        logger.error("Strange behavior. Pay attention. STORAGE_ACCOUNT != STORAGE_ACCOUNT")
+                                        holder_table[holder_account] = HolderStruct(storage_account)
                                 else:
+                                    logger.error("Strange behavior. Pay attention. HOLDER ACCOUNT NOT FOUND")
                                     holder_table[holder_account] = HolderStruct(storage_account)
 
                                 if got_result:
@@ -467,67 +495,8 @@ def process_receipts(self):
 
                                     continue_table[storage_account].results = got_result
                             else:
-                                got_result = get_trx_results(trx)
-                                if got_result is not None:
-                                    continue_table[storage_account] =  ContinueStruct(signature, got_result, blocked_accounts)
-                                    holder_table[holder_account] = HolderStruct(storage_account)
-                                else:
-                                    self.add_hunged_storage(trx, storage_account)
-
-                        elif instruction_data[0] == 0x13:  # PartialCallFromRawEthereumTXv02
-                            # logger.debug("{:>10} {:>6} PartialCallFromRawEthereumTXv02 0x{}".format(slot, counter, instruction_data.hex()))
-
-                            storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
-
-                            if storage_account in continue_table:
-                                # collateral_pool_buf = instruction_data[1:5]
-                                # step_count = instruction_data[5:13]
-                                # from_addr = instruction_data[13:33]
-
-                                sign = instruction_data[33:98]
-                                unsigned_msg = instruction_data[98:]
-
-                                (eth_trx, eth_signature, from_address) = get_trx_receipts(unsigned_msg, sign)
-
-                                continue_result = continue_table[storage_account]
-
-                                self.submit_transaction(eth_trx, eth_signature, from_address, continue_result.results, continue_result.signatures)
-
-                                del continue_table[storage_account]
-                            else:
-                                self.add_hunged_storage(trx, storage_account)
-
-                        elif instruction_data[0] == 0x14:  # ContinueV02
-                            # logger.debug("{:>10} {:>6} ContinueV02 0x{}".format(slot, counter, instruction_data.hex()))
-
-                            storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
-
-                            if storage_account in continue_table:
-                                continue_table[storage_account].signatures.append(signature)
-                            else:
-                                got_result = get_trx_results(trx)
-                                if got_result is not None:
-                                    continue_table[storage_account] = ContinueStruct(signature, got_result)
-                                else:
-                                    self.add_hunged_storage(trx, storage_account)
-
-                        elif instruction_data[0] == 0x16:  # ExecuteTrxFromAccountDataIterativeV02
-                            # logger.debug("{:>10} {:>6} ExecuteTrxFromAccountDataIterativeV02 0x{}".format(slot, counter, instruction_data.hex()))
-
-                            holder_account =  trx['transaction']['message']['accountKeys'][instruction['accounts'][0]]
-                            storage_account = trx['transaction']['message']['accountKeys'][instruction['accounts'][1]]
-
-                            if storage_account in continue_table:
-                                continue_table[storage_account].signatures.append(signature)
-
-                                if holder_account in holder_table:
-                                    # logger.debug("holder_account found")
-                                    # logger.debug("Strange behavior. Pay attention.")
-                                    holder_table[holder_account] = HolderStruct(storage_account)
-                                else:
-                                    holder_table[holder_account] = HolderStruct(storage_account)
-                            else:
-                                self.add_hunged_storage(trx, storage_account)
+                                continue_table[storage_account] =  ContinueStruct(signature, got_result, blocked_accounts)
+                                holder_table[holder_account] = HolderStruct(storage_account)
 
                         if instruction_data[0] > 0x16:
                             logger.debug("{:>10} {:>6} Unknown 0x{}".format(slot, counter, instruction_data.hex()))
@@ -538,12 +507,12 @@ def process_receipts(self):
             if trx_struct.got_result:
                 self.submit_transaction(trx_struct)
             elif trx_struct.storage:
-                self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts)
+                if abs(trx_struct.slot - self.current_slot) > CANCEL_TIMEOUT:
+                    self.blocked_storages[trx_struct.storage] = (trx_struct.eth_trx, trx_struct.blocked_accounts)
             else:
                 logger.error(trx_struct)
 
 
-
     def submit_transaction(self, trx_struct):
         (logs, status, gas_used, return_value, slot) = trx_struct.got_result
         (_slot, block_hash) = self.get_block(slot)
@@ -605,11 +574,6 @@ def get_block(self, slot):
         return (slot, block_hash)
 
 
-    def add_hunged_storage(self, trx, storage):
-        if abs(trx['slot'] - self.current_slot) > 16:
-            self.blocked_storages.add(storage)
-
-
 def run_indexer():
     logging.basicConfig(format='%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s')
     logger.setLevel(logging.DEBUG)