#368 move sql schema to distinct file (#586)

* Introduce proxy/db_scheme.py

* Debug

* Introduce run-db-creation.sh

* Use entrypoint: proxy/run-db-creation.sh

* Add dbcreation.log

* Debug

* Debug

* Add dbcreation.log

* Use from ..proxy

* Debug

* Debug

* Introduce dbcreation_mode to run proxy

* Change paths

* Will run in dbcreation mode with SOLANA_URL and EVM_LOADER

* Debug

* Debug

* Debug

* SOLANA_URL: http://solana:8899

* Add logged_groups

* Debug

* Log schema

* Use psql

* Add postgresql-client-common in the docker image

* Add postgresql-client in the docker image

* Set network_mode: container:postgres

* Remove networks

* Use network_mode: service:postgres

* Remove hostname

* Remove hostname

* ports:
      - "5432:5432"

* networks:
      - net

* Add     ports:
          - "5432"

* Add    networks:
         - net

* Try to use links: - postgres

* Use export PGPASSWORD=${POSTGRES_PASSWORD} to set the password for psql
Use psql -h ${POSTGRES_HOST} ...

* Use run_dbcreation()

* Correction after the prev merge

* Correction after the prev merge

* Lowercase in function names

* Refactor to introduce scheme.sql

* Debug

* Debug

* Debug

* Add table airdrop_scheduled
Add table constants

* Revert some changes

* Fixed: psql:proxy/db/scheme.sql:73: ERROR:  syntax error at or near "{"
       LINE 1: ...IF NOT EXISTS neon_transaction_logs_block_hash ON {table_nam...

* Use run_indexer(solana_url)

* Use run_airdropper with 7 args

* Remove creation.py

* Remove creation.py

* Remove dbcreation_mode

* check it

* check it

* Catch "Operator has NO resources!" exception

* Catch "Operator has NO resources!" exception

* Get rid of create_table in sql_dict.py

* Get rid of stuff from sql_dict

* Get rid of crating table stuff from TrxReceiptsStorage

* Roll back NEON_ERC20_MAX_AMOUNT
          NEON_ETH_MAX_AMOUNT

* Spit and polish

Co-authored-by: rozhkovdmitrii <[email protected]>

Author: vasiliy-zaznobin

.buildkite/pipeline.yml

@@ -16,6 +16,7 @@ steps:
       - "solana.log"
       - "measurements.log"
       - "evm_loader.log"
+      - "dbcreation.log"
       - "faucet.log"
       - "airdropper.log"
       - "indexer.log"

.buildkite/steps/deploy-test.sh

@@ -44,6 +44,7 @@ function cleanup_docker {
 
     if docker logs solana >solana.log 2>&1; then echo "solana logs saved"; fi
     if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi
+    if docker logs dbcreation >dbcreation.log 2>&1; then echo "dbcreation logs saved"; fi
     if docker logs faucet >faucet.log 2>&1; then echo "faucet logs saved"; fi
     if docker logs airdropper >airdropper.log 2>&1; then echo "airdropper logs saved"; fi
     if docker logs indexer >indexer.log 2>&1; then echo "indexer logs saved"; fi

Dockerfile

@@ -13,7 +13,7 @@ WORKDIR /opt
 
 RUN apt update && \
     DEBIAN_FRONTEND=noninteractive apt install -y git software-properties-common openssl curl parallel \
-                                                  ca-certificates python3-pip python3-venv && \
+                                                  ca-certificates python3-pip python3-venv postgresql-client && \
     python3 -m venv venv && \
     pip3 install --upgrade pip && \
     /bin/bash -c "source venv/bin/activate" && \

proxy/__main__.py

@@ -16,11 +16,15 @@
 from .indexer.indexer import run_indexer
 
 if __name__ == '__main__':
+    solana_url = os.environ['SOLANA_URL']
+    evm_loader_id = os.environ['EVM_LOADER']
+    print(f"Will run with SOLANA_URL={solana_url}; EVM_LOADER={evm_loader_id}")
+
     airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True']
     indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True']
+
     if airdropper_mode:
         print("Will run in airdropper mode")
-        solana_url = os.environ['SOLANA_URL']
         pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT'])
         faucet_url = os.environ['FAUCET_URL']
         wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST']
@@ -40,9 +44,6 @@
                        max_conf)
     elif indexer_mode:
         print("Will run in indexer mode")
-
-        solana_url = os.environ['SOLANA_URL']
-
         run_indexer(solana_url)
     else:
         entry_point()

proxy/common_neon/costs.py

@@ -6,23 +6,7 @@
 
 class SQLCost(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'OPERATOR_COST'
-        return f"""
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                id SERIAL PRIMARY KEY,
-                hash char(64),
-                cost bigint,
-                used_gas bigint,
-                sender char(40),
-                to_address char(40) ,
-                sig char(100),
-                status varchar(100),
-                reason varchar(100)
-            );
-            """
+        BaseDB.__init__(self, 'OPERATOR_COST')
 
     def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason):
         with self._conn.cursor() as cur:

proxy/db/scheme.sql

@@ -0,0 +1,148 @@
+    CREATE TABLE IF NOT EXISTS constants (
+        key TEXT UNIQUE,
+        value BYTEA
+    );
+
+    CREATE TABLE IF NOT EXISTS airdrop_scheduled (
+        key TEXT UNIQUE,
+        value BYTEA
+    );
+
+    CREATE TABLE IF NOT EXISTS OPERATOR_COST (
+        id SERIAL PRIMARY KEY,
+        hash char(64),
+        cost bigint,
+        used_gas bigint,
+        sender char(40),
+        to_address char(40) ,
+        sig char(100),
+        status varchar(100),
+        reason varchar(100)
+    );
+
+    CREATE TABLE IF NOT EXISTS neon_accounts (
+        neon_account CHAR(42),
+        pda_account VARCHAR(50),
+        code_account VARCHAR(50),
+        slot BIGINT,
+        code TEXT,
+
+        UNIQUE(pda_account, code_account)
+    );
+
+    CREATE TABLE IF NOT EXISTS failed_airdrop_attempts (
+        attempt_time    BIGINT,
+        eth_address     TEXT,
+        reason          TEXT
+    );
+    CREATE INDEX IF NOT EXISTS failed_attempt_time_idx ON failed_airdrop_attempts (attempt_time);
+
+    CREATE TABLE IF NOT EXISTS airdrop_ready (
+        eth_address     TEXT UNIQUE,
+        scheduled_ts    BIGINT,
+        finished_ts     BIGINT,
+        duration        INTEGER,
+        amount_galans   INTEGER
+    );
+
+    CREATE TABLE IF NOT EXISTS solana_block (
+        slot BIGINT,
+        hash CHAR(66),
+
+        parent_hash CHAR(66),
+        blocktime BIGINT,
+        signatures BYTEA,
+
+        UNIQUE(slot),
+        UNIQUE(hash)
+    );
+
+    CREATE TABLE IF NOT EXISTS neon_transaction_logs (
+        address CHAR(42),
+        blockHash CHAR(66),
+        blockNumber BIGINT,
+
+        transactionHash CHAR(66),
+        transactionLogIndex INT,
+        topic TEXT,
+
+        json TEXT,
+
+        UNIQUE(blockNumber, transactionHash, transactionLogIndex)
+    );
+    CREATE INDEX IF NOT EXISTS neon_transaction_logs_block_hash ON neon_transaction_logs(blockHash);
+    CREATE INDEX IF NOT EXISTS neon_transaction_logs_address ON neon_transaction_logs(address);
+    CREATE INDEX IF NOT EXISTS neon_transaction_logs_topic ON neon_transaction_logs(topic);
+
+    CREATE TABLE IF NOT EXISTS solana_neon_transactions (
+        sol_sign CHAR(88),
+        neon_sign CHAR(66),
+        slot BIGINT,
+        idx INT,
+
+        UNIQUE(sol_sign, neon_sign, idx),
+        UNIQUE(neon_sign, sol_sign, idx)
+    );
+
+    CREATE TABLE IF NOT EXISTS neon_transactions (
+        neon_sign CHAR(66),
+        from_addr CHAR(42),
+        sol_sign CHAR(88),
+        slot BIGINT,
+        block_hash CHAR(66),
+        idx INT,
+
+        nonce VARCHAR,
+        gas_price VARCHAR,
+        gas_limit VARCHAR,
+        value VARCHAR,
+        gas_used VARCHAR,
+
+        to_addr CHAR(42),
+        contract CHAR(42),
+
+        status CHAR(3),
+
+        return_value TEXT,
+
+        v TEXT,
+        r TEXT,
+        s TEXT,
+
+        calldata TEXT,
+        logs BYTEA,
+
+        UNIQUE(neon_sign),
+        UNIQUE(sol_sign, idx)
+    );
+
+    CREATE TABLE IF NOT EXISTS transaction_receipts (
+        slot        BIGINT,
+        signature   VARCHAR(88),
+        trx         BYTEA,
+        PRIMARY KEY (slot, signature)
+    );
+
+    CREATE TABLE IF NOT EXISTS constants (
+        key TEXT UNIQUE,
+        value BYTEA
+    )
+
+    CREATE TABLE IF NOT EXISTS airdrop_scheduled (
+        key TEXT UNIQUE,
+        value BYTEA
+    )
+
+    CREATE TABLE IF NOT EXISTS transaction_receipts (
+        slot        BIGINT,
+        signature   VARCHAR(88),
+        trx         BYTEA,
+        PRIMARY KEY (slot, signature)
+    );
+
+    CREATE TABLE IF NOT EXISTS test_storage (
+        slot        BIGINT,
+        signature   VARCHAR(88),
+        trx         BYTEA,
+        PRIMARY KEY (slot, signature)
+    );

proxy/docker-compose-test.yml

@@ -8,6 +8,8 @@ services:
       SOLANA_URL: http://solana:8899
       RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug
     hostname: solana
+    ports:
+      - 127.0.0.1:8899:8899
     expose:
       - "8899"
       - "9900"
@@ -53,9 +55,29 @@ services:
       start_period: 5s
     expose:
       - "5432"
+    ports:
+      - "5432"
     networks:
       - net
 
+  dbcreation:
+    container_name: dbcreation
+    image: neonlabsorg/proxy:${REVISION}
+    environment:
+      SOLANA_URL: http://solana:8899
+      POSTGRES_DB: neon-db
+      POSTGRES_USER: neon-proxy
+      POSTGRES_PASSWORD: neon-proxy-pass
+      POSTGRES_HOST: postgres
+    entrypoint: proxy/run-dbcreation.sh
+    networks:
+      - net
+    depends_on:
+      postgres:
+        condition: service_healthy
+      evm_loader:
+        condition: service_completed_successfully
+
   proxy:
     container_name: proxy
     image: neonlabsorg/proxy:${REVISION}
@@ -77,6 +99,8 @@ services:
       OPERATOR_GAS_ACCOUNTS: 0x8966Ef2ae7A109Fd0977F5151b4607dc42929fBD;0x619d670152103a972B67a45b9Be764FF11979E4E
     hostname: proxy
     depends_on:
+      dbcreation:
+        condition: service_completed_successfully
       postgres:
         condition: service_healthy
       evm_loader:
@@ -143,6 +167,8 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
+      dbcreation:
+        condition: service_completed_successfully
       faucet:
         condition: service_started
 
@@ -161,6 +187,8 @@ services:
         condition: service_healthy
       evm_loader:
         condition: service_completed_successfully
+      dbcreation:
+        condition: service_completed_successfully
     networks:
       - net
     entrypoint: proxy/run-indexer.sh

proxy/indexer/accounts_db.py

@@ -16,20 +16,7 @@ def __str__(self):
 
 class NeonAccountDB(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'neon_accounts'
-        return f"""
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                neon_account CHAR(42),
-                pda_account VARCHAR(50),
-                code_account VARCHAR(50),
-                slot BIGINT,
-                code TEXT,
-
-                UNIQUE(pda_account, code_account)
-            );"""
+        BaseDB.__init__(self, 'neon_accounts')
 
     def set_acc_by_request(self, neon_account: str, pda_account: str, code_account: str, code: str):
         with self._conn.cursor() as cursor:

proxy/indexer/airdropper.py

@@ -10,6 +10,7 @@
 from datetime import datetime
 from decimal import Decimal
 from logged_groups import logged_group
+
 from ..environment import NEON_PRICE_USD, EVM_LOADER_ID
 from ..common_neon.solana_interactor import SolanaInteractor
 
@@ -20,18 +21,7 @@
 
 class FailedAttempts(BaseDB):
     def __init__(self) -> None:
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'failed_airdrop_attempts'
-        return f'''
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                attempt_time    BIGINT,
-                eth_address     TEXT,
-                reason          TEXT
-            );
-            CREATE INDEX IF NOT EXISTS failed_attempt_time_idx ON {self._table_name} (attempt_time);
-            '''
+        BaseDB.__init__(self, 'failed_airdrop_attempts')
 
     def airdrop_failed(self, eth_address, reason):
         with self._conn.cursor() as cur:
@@ -43,19 +33,7 @@ def airdrop_failed(self, eth_address, reason):
 
 class AirdropReadySet(BaseDB):
     def __init__(self):
-        BaseDB.__init__(self)
-
-    def _create_table_sql(self) -> str:
-        self._table_name = 'airdrop_ready'
-        return f'''
-            CREATE TABLE IF NOT EXISTS {self._table_name} (
-                eth_address     TEXT UNIQUE,
-                scheduled_ts    BIGINT,
-                finished_ts     BIGINT,
-                duration        INTEGER,
-                amount_galans   INTEGER
-            )
-            '''
+        BaseDB.__init__(self, 'airdrop_ready')
 
     def register_airdrop(self, eth_address: str, airdrop_info: dict):
         finished = int(datetime.now().timestamp())

proxy/indexer/base_db.py

@@ -23,9 +23,9 @@ class DBQueryExpression(NamedTuple):
 
 @logged_group("neon.Indexer")
 class BaseDB:
-    _create_table_lock = multiprocessing.Lock()
 
-    def __init__(self):
+    def __init__(self, table_name):
+        self._table_name = table_name
         self._conn = psycopg2.connect(
             dbname=POSTGRES_DB,
             user=POSTGRES_USER,
@@ -34,13 +34,6 @@ def __init__(self):
         )
         self._conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
 
-        with self._create_table_lock:
-            cursor = self._conn.cursor()
-            cursor.execute(self._create_table_sql())
-
-    def _create_table_sql(self) -> str:
-        assert False, 'No script for the table'
-
     def _build_expression(self, q: DBQuery) -> DBQueryExpression:
 
         return DBQueryExpression(