583 introduce full test suite tf

.buildkite/pipeline.yml

@@ -59,6 +59,15 @@ steps:
       FTS_IMAGE: neonlabsorg/full_test_suite:develop
     agents:
        queue: "testing"
+    artifact_paths:
+      - "proxy.log"
+      - "solana.log"
+      - "measurements.log"
+      - "evm_loader.log"
+      - "dbcreation.log"
+      - "faucet.log"
+      - "airdropper.log"
+      - "indexer.log"
 
   - wait
 

.buildkite/steps/deploy-test.sh

@@ -44,6 +44,7 @@ function cleanup_docker {
 
     if docker logs solana >solana.log 2>&1; then echo "solana logs saved"; fi
     if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi
+    if docker logs dbcreation >dbcreation.log 2>&1; then echo "dbcreation logs saved"; fi
     if docker logs faucet >faucet.log 2>&1; then echo "faucet logs saved"; fi
     if docker logs airdropper >airdropper.log 2>&1; then echo "airdropper logs saved"; fi
     if docker logs indexer >indexer.log 2>&1; then echo "indexer logs saved"; fi

Dockerfile

@@ -13,7 +13,7 @@ WORKDIR /opt
 
 RUN apt update && \
     DEBIAN_FRONTEND=noninteractive apt install -y git software-properties-common openssl curl parallel \
-                                                  ca-certificates python3-pip python3-venv && \
+                                                  ca-certificates python3-pip python3-venv postgresql-client && \
     python3 -m venv venv && \
     pip3 install --upgrade pip && \
     /bin/bash -c "source venv/bin/activate" && \

proxy/__main__.py

@@ -9,40 +9,19 @@
     :license: BSD, see LICENSE for more details.
 """
 
-from solana.publickey import PublicKey
 from .proxy import entry_point
 import os
-from .indexer.airdropper import run_airdropper
 from .indexer.indexer import run_indexer
 
 if __name__ == '__main__':
-    airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True']
-    indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True']
-    if airdropper_mode:
-        print("Will run in airdropper mode")
-        solana_url = os.environ['SOLANA_URL']
-        pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT'])
-        faucet_url = os.environ['FAUCET_URL']
-        wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST']
-        if wrapper_whitelist != 'ANY':
-            wrapper_whitelist = wrapper_whitelist.split(',')
-        neon_decimals = int(os.environ.get('NEON_DECIMALS', '9'))
+    solana_url = os.environ['SOLANA_URL']
+    evm_loader_id = os.environ['EVM_LOADER']
+    print(f"Will run with SOLANA_URL={solana_url}; EVM_LOADER={evm_loader_id}")
 
-        pp_solana_url = os.environ.get('PP_SOLANA_URL', None)
-        max_conf = float(os.environ.get('MAX_CONFIDENCE_INTERVAL', 0.02))
+    indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True']
 
-        run_airdropper(solana_url,
-                       pyth_mapping_account,
-                       faucet_url,
-                       wrapper_whitelist,
-                       neon_decimals,
-                       pp_solana_url,
-                       max_conf)
-    elif indexer_mode:
+    if indexer_mode:
         print("Will run in indexer mode")
-
-        solana_url = os.environ['SOLANA_URL']
-
         run_indexer(solana_url)
     else:
         entry_point()

proxy/airdropper/__init__.py

@@ -0,0 +1,2 @@
+from .airdropper import Airdropper, AIRDROP_AMOUNT_SOL, NEON_PRICE_USD
+from .airdropper_app import AirdropperApp

proxy/airdropper/__main__.py

@@ -0,0 +1,6 @@
+from . import AirdropperApp
+
+if __name__ == '__main__':
+    airdropper_app = AirdropperApp()
+    exit_code = airdropper_app.run()
+    exit(exit_code)

proxy/indexer/airdropper.py → proxy/airdropper/airdropper.py

@@ -1,17 +1,18 @@
 from solana.publickey import PublicKey
-from proxy.indexer.indexer_base import IndexerBase
-from proxy.indexer.pythnetwork import PythNetworkClient
-from proxy.indexer.base_db import BaseDB
-from proxy.indexer.utils import check_error
-from proxy.indexer.sql_dict import SQLDict
 import requests
 import base58
 import traceback
 from datetime import datetime
 from decimal import Decimal
 from logged_groups import logged_group
+
 from ..environment import NEON_PRICE_USD, EVM_LOADER_ID
 from ..common_neon.solana_interactor import SolanaInteractor
+from ..indexer.indexer_base import IndexerBase
+from ..indexer.pythnetwork import PythNetworkClient
+from ..indexer.base_db import BaseDB
+from ..indexer.utils import check_error
+from ..indexer.sql_dict import SQLDict
 
 
 ACCOUNT_CREATION_PRICE_SOL = Decimal('0.00472692')
@@ -20,18 +21,7 @@
 
 class FailedAttempts(BaseDB):
     def __init__(self) -> None:
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'failed_airdrop_attempts'
-        return f'''
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                attempt_time    BIGINT,
-                eth_address     TEXT,
-                reason          TEXT
-            );
-            CREATE INDEX IF NOT EXISTS failed_attempt_time_idx ON {self._table_name} (attempt_time);
-            '''
+        BaseDB.__init__(self, 'failed_airdrop_attempts')
 
     def airdrop_failed(self, eth_address, reason):
         with self._conn.cursor() as cur:
@@ -43,19 +33,7 @@ def airdrop_failed(self, eth_address, reason):
 
 class AirdropReadySet(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'airdrop_ready'
-        return f'''
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                eth_address     TEXT UNIQUE,
-                scheduled_ts    BIGINT,
-                finished_ts     BIGINT,
-                duration        INTEGER,
-                amount_galans   INTEGER
-            )
-            '''
+        BaseDB.__init__(self, 'airdrop_ready')
 
     def register_airdrop(self, eth_address: str, airdrop_info: dict):
         finished = int(datetime.now().timestamp())
@@ -293,34 +271,3 @@ def process_receipts(self):
                 self.process_trx_airdropper_mode(trx)
         self.latest_processed_slot = max(self.latest_processed_slot, max_slot)
         self._constants['latest_processed_slot'] = self.latest_processed_slot
-
-
-@logged_group("neon.Airdropper")
-def run_airdropper(solana_url,
-                   pyth_mapping_account: PublicKey,
-                   faucet_url,
-                   wrapper_whitelist = 'ANY',
-                   neon_decimals = 9,
-                   pp_solana_url = None,
-                   max_conf = 0.1, *, logger):
-    logger.info(f"""Running indexer with params:
-        solana_url: {solana_url},
-        evm_loader_id: {EVM_LOADER_ID},
-        pyth.network mapping account: {pyth_mapping_account},
-        faucet_url: {faucet_url},
-        wrapper_whitelist: {wrapper_whitelist},
-        NEON decimals: {neon_decimals},
-        Price provider solana: {pp_solana_url},
-        Max confidence interval: {max_conf}""")
-
-    try:
-        airdropper = Airdropper(solana_url,
-                                pyth_mapping_account,
-                                faucet_url,
-                                wrapper_whitelist,
-                                neon_decimals,
-                                pp_solana_url,
-                                max_conf)
-        airdropper.run()
-    except Exception as err:
-        logger.error(f'Failed to start Airdropper: {err}')

proxy/airdropper/airdropper_app.py

@@ -0,0 +1,45 @@
+import os
+from logged_groups import logged_group
+from solana.publickey import PublicKey
+
+from ..environment import EVM_LOADER_ID
+
+from .airdropper import Airdropper
+
+
+@logged_group("neon.Airdropper")
+class AirdropperApp:
+
+    def __init__(self):
+        self.info("Airdropper application is starting ...")
+        pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT'])
+        faucet_url = os.environ['FAUCET_URL']
+        wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST']
+        if wrapper_whitelist != 'ANY':
+            wrapper_whitelist = wrapper_whitelist.split(',')
+        neon_decimals = int(os.environ.get('NEON_DECIMALS', '9'))
+
+        pp_solana_url = os.environ.get('PP_SOLANA_URL', None)
+        max_conf = float(os.environ.get('MAX_CONFIDENCE_INTERVAL', 0.02))
+        solana_url = os.environ['SOLANA_URL']
+
+        self.info(f"""Construct Airdropper with params:
+                  solana_url: {solana_url},
+                  evm_loader_id: {EVM_LOADER_ID},
+                  pyth.network mapping account: {pyth_mapping_account},
+                  faucet_url: {faucet_url},
+                  wrapper_whitelist: {wrapper_whitelist},
+                  NEON decimals: {neon_decimals},
+                  Price provider solana: {pp_solana_url},
+                  Max confidence interval: {max_conf}""")
+
+        self._airdropper = Airdropper(solana_url, pyth_mapping_account, faucet_url, wrapper_whitelist, neon_decimals,
+                                      pp_solana_url, max_conf)
+
+    def run(self) -> int:
+        try:
+            self._airdropper.run()
+        except Exception as err:
+            self.error(f'Failed to start Airdropper: {err}')
+            return 1
+        return 0

proxy/common_neon/costs.py

@@ -6,23 +6,7 @@
 
 class SQLCost(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'OPERATOR_COST'
-        return f"""
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                id SERIAL PRIMARY KEY,
-                hash char(64),
-                cost bigint,
-                used_gas bigint,
-                sender char(40),
-                to_address char(40) ,
-                sig char(100),
-                status varchar(100),
-                reason varchar(100)
-            );
-            """
+        BaseDB.__init__(self, 'OPERATOR_COST')
 
     def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason):
         with self._conn.cursor() as cur: