368 move sql schema to distinct file

.buildkite/pipeline.yml

@@ -16,6 +16,7 @@ steps:
       - "solana.log"
       - "measurements.log"
       - "evm_loader.log"
+      - "dbcreation.log"
       - "faucet.log"
       - "airdropper.log"
       - "indexer.log"

.buildkite/steps/deploy-test.sh

@@ -44,6 +44,7 @@ function cleanup_docker {
 
     if docker logs solana >solana.log 2>&1; then echo "solana logs saved"; fi
     if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi
+    if docker logs dbcreation >dbcreation.log 2>&1; then echo "dbcreation logs saved"; fi
     if docker logs faucet >faucet.log 2>&1; then echo "faucet logs saved"; fi
     if docker logs airdropper >airdropper.log 2>&1; then echo "airdropper logs saved"; fi
     if docker logs indexer >indexer.log 2>&1; then echo "indexer logs saved"; fi

Dockerfile

@@ -13,7 +13,8 @@ WORKDIR /opt
 
 RUN apt update && \
     DEBIAN_FRONTEND=noninteractive apt install -y git software-properties-common openssl curl \
-                                                  ca-certificates python3-pip python3-venv && \
+                                                  ca-certificates python3-pip python3-venv \
+                                                  postgresql-client-common postgresql-client && \
     python3 -m venv venv && \
     pip3 install --upgrade pip && \
     /bin/bash -c "source venv/bin/activate" && \

proxy/__main__.py

@@ -16,11 +16,15 @@
 from .indexer.indexer import run_indexer
 
 if __name__ == '__main__':
+    solana_url = os.environ['SOLANA_URL']
+    evm_loader_id = os.environ['EVM_LOADER']
+    print(f"Will run with SOLANA_URL={solana_url}; EVM_LOADER={evm_loader_id}")
+
     airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True']
     indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True']
+
     if airdropper_mode:
         print("Will run in airdropper mode")
-        solana_url = os.environ['SOLANA_URL']
         pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT'])
         faucet_url = os.environ['FAUCET_URL']
         wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST']
@@ -40,9 +44,6 @@
                        max_conf)
     elif indexer_mode:
         print("Will run in indexer mode")
-
-        solana_url = os.environ['SOLANA_URL']
-
         run_indexer(solana_url)
     else:
         entry_point()

proxy/common_neon/costs.py

@@ -6,23 +6,7 @@
 
 class SQLCost(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'OPERATOR_COST'
-        return f"""
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                id SERIAL PRIMARY KEY,
-                hash char(64),
-                cost bigint,
-                used_gas bigint,
-                sender char(40),
-                to_address char(40) ,
-                sig char(100),
-                status varchar(100),
-                reason varchar(100)
-            );
-            """
+        BaseDB.__init__(self, 'OPERATOR_COST')
 
     def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason):
         with self._conn.cursor() as cur:

proxy/db/scheme.sql

@@ -0,0 +1,124 @@
+    CREATE TABLE IF NOT EXISTS constants (
+        key TEXT UNIQUE,
+        value BYTEA
+    );
+
+    CREATE TABLE IF NOT EXISTS airdrop_scheduled (
+        key TEXT UNIQUE,
+        value BYTEA
+    );
+
+    CREATE TABLE IF NOT EXISTS OPERATOR_COST (
+        id SERIAL PRIMARY KEY,
+        hash char(64),
+        cost bigint,
+        used_gas bigint,
+        sender char(40),
+        to_address char(40) ,
+        sig char(100),
+        status varchar(100),
+        reason varchar(100)
+    );
+
+    CREATE TABLE IF NOT EXISTS neon_accounts (
+        neon_account CHAR(42),
+        pda_account VARCHAR(50),
+        code_account VARCHAR(50),
+        slot BIGINT,
+        code TEXT,
+
+        UNIQUE(pda_account, code_account)
+    );
+
+    CREATE TABLE IF NOT EXISTS failed_airdrop_attempts (
+        attempt_time    BIGINT,
+        eth_address     TEXT,
+        reason          TEXT
+    );
+    CREATE INDEX IF NOT EXISTS failed_attempt_time_idx ON failed_airdrop_attempts (attempt_time);
+
+    CREATE TABLE IF NOT EXISTS airdrop_ready (
+        eth_address     TEXT UNIQUE,
+        scheduled_ts    BIGINT,
+        finished_ts     BIGINT,
+        duration        INTEGER,
+        amount_galans   INTEGER
+    );
+
+    CREATE TABLE IF NOT EXISTS solana_block (
+        slot BIGINT,
+        hash CHAR(66),
+
+        parent_hash CHAR(66),
+        blocktime BIGINT,
+        signatures BYTEA,
+
+        UNIQUE(slot),
+        UNIQUE(hash)
+    );
+
+    CREATE TABLE IF NOT EXISTS neon_transaction_logs (
+        address CHAR(42),
+        blockHash CHAR(66),
+        blockNumber BIGINT,
+
+        transactionHash CHAR(66),
+        transactionLogIndex INT,
+        topic TEXT,
+
+        json TEXT,
+
+        UNIQUE(blockNumber, transactionHash, transactionLogIndex)
+    );
+    CREATE INDEX IF NOT EXISTS neon_transaction_logs_block_hash ON neon_transaction_logs(blockHash);
+    CREATE INDEX IF NOT EXISTS neon_transaction_logs_address ON neon_transaction_logs(address);
+    CREATE INDEX IF NOT EXISTS neon_transaction_logs_topic ON neon_transaction_logs(topic);
+
+    CREATE TABLE IF NOT EXISTS solana_neon_transactions (
+        sol_sign CHAR(88),
+        neon_sign CHAR(66),
+        slot BIGINT,
+        idx INT,
+
+        UNIQUE(sol_sign, neon_sign, idx),
+        UNIQUE(neon_sign, sol_sign, idx)
+    );
+
+    CREATE TABLE IF NOT EXISTS neon_transactions (
+        neon_sign CHAR(66),
+        from_addr CHAR(42),
+        sol_sign CHAR(88),
+        slot BIGINT,
+        block_hash CHAR(66),
+        idx INT,
+
+        nonce VARCHAR,
+        gas_price VARCHAR,
+        gas_limit VARCHAR,
+        value VARCHAR,
+        gas_used VARCHAR,
+
+        to_addr CHAR(42),
+        contract CHAR(42),
+
+        status CHAR(3),
+
+        return_value TEXT,
+
+        v TEXT,
+        r TEXT,
+        s TEXT,
+
+        calldata TEXT,
+        logs BYTEA,
+
+        UNIQUE(neon_sign),
+        UNIQUE(sol_sign, idx)
+    );
+
+    CREATE TABLE IF NOT EXISTS transaction_receipts (
+        slot        BIGINT,
+        signature   VARCHAR(88),
+        trx         BYTEA,
+        PRIMARY KEY (slot, signature)
+    );

proxy/docker-compose-test.yml

@@ -8,6 +8,8 @@ services:
       SOLANA_URL: http://solana:8899
       RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug
     hostname: solana
+    ports:
+      - 127.0.0.1:8899:8899
     expose:
       - "8899"
       - "9900"
@@ -53,9 +55,29 @@ services:
       start_period: 5s
     expose:
       - "5432"
+    ports:
+      - "5432"
     networks:
       - net
 
+  dbcreation:
+    container_name: dbcreation
+    image: neonlabsorg/proxy:${REVISION}
+    environment:
+      SOLANA_URL: http://solana:8899
+      POSTGRES_DB: neon-db
+      POSTGRES_USER: neon-proxy
+      POSTGRES_PASSWORD: neon-proxy-pass
+      POSTGRES_HOST: postgres
+    entrypoint: proxy/run-dbcreation.sh
+    networks:
+      - net
+    depends_on:
+      postgres:
+        condition: service_healthy
+      evm_loader:
+        condition: service_completed_successfully
+
   proxy:
     container_name: proxy
     image: neonlabsorg/proxy:${REVISION}
@@ -76,6 +98,8 @@ services:
       MINIMAL_GAS_PRICE: 1
     hostname: proxy
     depends_on:
+      dbcreation:
+        condition: service_completed_successfully
       postgres:
         condition: service_healthy
       evm_loader:
@@ -100,11 +124,11 @@ services:
       WEB3_RPC_URL: 'http://proxy:9090/solana'
       WEB3_PRIVATE_KEY: ''
       NEON_ERC20_TOKENS:
-      NEON_ERC20_MAX_AMOUNT: 1000
+      NEON_ERC20_MAX_AMOUNT: 2000
       FAUCET_SOLANA_ENABLE: 'true'
       SOLANA_URL: 'http://solana:8899'
       NEON_OPERATOR_KEYFILE: '/root/.config/solana/id.json'
-      NEON_ETH_MAX_AMOUNT: 1000
+      NEON_ETH_MAX_AMOUNT: 2000
       TEST_FAUCET_INIT_NEON_BALANCE: 100000
     hostname: faucet
     ports:
@@ -142,6 +166,8 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
+      dbcreation:
+        condition: service_completed_successfully
       faucet:
         condition: service_started
 
@@ -160,6 +186,8 @@ services:
         condition: service_healthy
       evm_loader:
         condition: service_completed_successfully
+      dbcreation:
+        condition: service_completed_successfully
     networks:
       - net
     entrypoint: proxy/run-indexer.sh

proxy/indexer/accounts_db.py

@@ -16,20 +16,7 @@ def __str__(self):
 
 class NeonAccountDB(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'neon_accounts'
-        return f"""
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                neon_account CHAR(42),
-                pda_account VARCHAR(50),
-                code_account VARCHAR(50),
-                slot BIGINT,
-                code TEXT,
-
-                UNIQUE(pda_account, code_account)
-            );"""
+        BaseDB.__init__(self, 'neon_accounts')
 
     def set_acc_by_request(self, neon_account: str, pda_account: str, code_account: str, code: str):
         with self._conn.cursor() as cursor:

proxy/indexer/airdropper.py

@@ -10,6 +10,7 @@
 from datetime import datetime
 from decimal import Decimal
 from logged_groups import logged_group
+
 from ..environment import NEON_PRICE_USD, EVM_LOADER_ID
 from ..common_neon.solana_interactor import SolanaInteractor
 
@@ -20,18 +21,7 @@
 
 class FailedAttempts(BaseDB):
     def __init__(self) -> None:
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'failed_airdrop_attempts'
-        return f'''
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                attempt_time    BIGINT,
-                eth_address     TEXT,
-                reason          TEXT
-            );
-            CREATE INDEX IF NOT EXISTS failed_attempt_time_idx ON {self._table_name} (attempt_time);
-            '''
+        BaseDB.__init__(self, 'failed_airdrop_attempts')
 
     def airdrop_failed(self, eth_address, reason):
         with self._conn.cursor() as cur:
@@ -43,19 +33,7 @@ def airdrop_failed(self, eth_address, reason):
 
 class AirdropReadySet(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'airdrop_ready'
-        return f'''
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                eth_address     TEXT UNIQUE,
-                scheduled_ts    BIGINT,
-                finished_ts     BIGINT,
-                duration        INTEGER,
-                amount_galans   INTEGER
-            )
-            '''
+        BaseDB.__init__(self, 'airdrop_ready')
 
     def register_airdrop(self, eth_address: str, airdrop_info: dict):
         finished = int(datetime.now().timestamp())

proxy/indexer/base_db.py

@@ -25,7 +25,8 @@ class DBQueryExpression(NamedTuple):
 class BaseDB:
     _create_table_lock = multiprocessing.Lock()
 
-    def __init__(self):
+    def __init__(self, table_name):
+        self._table_name = table_name
         self._conn = psycopg2.connect(
             dbname=POSTGRES_DB,
             user=POSTGRES_USER,
@@ -34,13 +35,6 @@ def __init__(self):
         )
         self._conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
 
-        with self._create_table_lock:
-            cursor = self._conn.cursor()
-            cursor.execute(self._create_table_sql())
-
-    def _create_table_sql(self) -> str:
-        assert False, 'No script for the table'
-
     def _build_expression(self, q: DBQuery) -> DBQueryExpression:
 
         return DBQueryExpression(