Merge branch 'develop' into feature/NDEV-698/move-workflow-from-bildkite

proxy/common_neon/config.py

@@ -53,6 +53,10 @@ def __init__(self):
         self._indexer_log_skip_cnt = self._env_int("INDEXER_LOG_SKIP_COUNT", 1, 1000)
         self._gather_statistics = self._env_bool("GATHER_STATISTICS", False)
         self._mempool_cache_life_sec = self._env_int('MEMPOOL_CACHE_LIFE_SEC', 15, 15 * 60)
+        self._hvac_url = os.environ.get('HVAC_URL', None)
+        self._hvac_token = os.environ.get('HVAC_TOKEN', None)
+        self._hvac_mount = os.environ.get('HVAC_MOUNT', None)
+        self._hvac_path = os.environ.get('HVAC_PATH', '')
 
         pyth_mapping_account = os.environ.get("PYTH_MAPPING_ACCOUNT", None)
         self._pyth_mapping_account = SolPubKey(pyth_mapping_account) if pyth_mapping_account is not None else None
@@ -258,6 +262,22 @@ def gather_statistics(self) -> bool:
     def mempool_cache_life_sec(self) -> int:
         return self._mempool_cache_life_sec
 
+    @property
+    def hvac_url(self) -> Optional[str]:
+        return self._hvac_url
+
+    @property
+    def hvac_token(self) -> Optional[str]:
+        return self._hvac_token
+
+    @property
+    def hvac_mount(self) -> Optional[str]:
+        return self._hvac_mount
+
+    @property
+    def hvac_path(self) -> str:
+        return self._hvac_path
+
     def __str__(self):
         return '\n        '.join([
             '',
@@ -306,5 +326,9 @@ def __str__(self):
             f"INDEXER_LOG_SKIP_COUNT: {self.indexer_log_skip_cnt}",
             f"GATHER_STATISTICS: {self.gather_statistics}",
             f"MEMPOOL_CACHE_LIFE_SEC: {self.mempool_cache_life_sec}",
+            f"HVAC_URL: {self.hvac_url}",
+            f"HVAC_TOKEN: {self.hvac_token}",
+            f"HVAC_PATH: {self.hvac_path}",
+            f"HVAC_MOUNT: {self.hvac_mount}",
             ""
         ])

proxy/common_neon/elf_params.py

@@ -110,12 +110,13 @@ def read_elf_param_dict_from_net(self, config: Config) -> ElfParams:
             if param.startswith('NEON_') and '=' in param:
                 v = param.split('=')
                 elf_param_dict.setdefault(v[0], v[1])
-        self.set_elf_param_dict(elf_param_dict)
-        return self
 
-    def set_elf_param_dict(self, elf_param_dict: Dict[str, str]) -> ElfParams:
         for param, value in elf_param_dict.items():
             if self._elf_param_dict.get(param) != value:
                 self.debug(f"new ELF param: {param}: {value}")
+        self.set_elf_param_dict(elf_param_dict)
+        return self
+
+    def set_elf_param_dict(self, elf_param_dict: Dict[str, str]) -> ElfParams:
         self._elf_param_dict = elf_param_dict
         return self

proxy/common_neon/environment_utils.py

@@ -43,48 +43,6 @@ def call(self, *args):
             raise
 
 
-@logged_group("neon.Proxy")
-def get_solana_accounts(config, *, logger) -> List[SolAccount]:
-    def read_sol_account(name) -> Optional[SolAccount]:
-        if not os.path.isfile(name):
-            return None
-        logger.debug(f"Open a sol_account file: {name}")
-        with open(name.strip(), mode='r') as d:
-            pkey = (d.read())
-            num_list = [int(v) for v in pkey.strip("[] \n").split(',')]
-            value_list = bytes(num_list[0:32])
-            return SolAccount.from_secret_key(value_list)
-
-    res = SolanaCli(config).call('config', 'get')
-    logger.debug(f"Got solana config: {res}")
-    substr = "Keypair Path: "
-    path = ""
-    for line in res.splitlines():
-        if line.startswith(substr):
-            path = line[len(substr):].strip()
-    if path == "":
-        raise Exception("cannot get keypair path")
-
-    path = path.strip()
-
-    signer_list = []
-    (file_name, file_ext) = os.path.splitext(path)
-    i = 0
-    while True:
-        i += 1
-        full_path = file_name + (str(i) if i > 1 else '') + file_ext
-        signer = read_sol_account(full_path)
-        if not signer:
-            break
-        signer_list.append(signer)
-        logger.debug(f'Add signer: {signer.public_key}')
-
-    if not len(signer_list):
-        raise Exception("No keypairs")
-    logger.debug(f"Got signer list of: {len(signer_list)} - keys")
-    return signer_list
-
-
 class NeonCli(CliBase):
     EMULATOR_LOGLEVEL = {
         logging.CRITICAL: "off",

proxy/common_neon/neon_instruction.py

@@ -130,7 +130,7 @@ def create_holder_ix(self, holder: SolPubKey) -> SolTxIx:
             data=EvmInstruction.HolderCreate.value,
         )
 
-    def make_create_eth_account_ix(self, eth_address: NeonAddress) -> SolTxIx:
+    def make_create_neon_account_ix(self, eth_address: NeonAddress) -> SolTxIx:
         if isinstance(eth_address, str):
             eth_address = NeonAddress(eth_address)
         pda_account, nonce = neon_2program(eth_address)

proxy/common_neon/operator_secret_mng.py

@@ -0,0 +1,113 @@
+import base64
+import os
+
+from typing import List, Optional
+
+import hvac
+from hvac.api.secrets_engines.kv_v2 import DEFAULT_MOUNT_POINT
+
+from logged_groups import logged_group
+
+from ..common_neon.config import Config
+from ..common_neon.environment_utils import SolanaCli
+from ..common_neon.solana_tx import SolAccount
+
+
+@logged_group("neon.Decorder")
+class OpSecretMng:
+    def __init__(self, config: Config):
+        self._config = config
+
+    def read_secret_list(self) -> List[bytes]:
+        if self._config.hvac_url is not None:
+            secret_list = self._read_secret_list_from_hvac()
+        else:
+            secret_list = self._read_secret_list_from_fs()
+
+        if len(secret_list) == 0:
+            self.warning("No secrets")
+        else:
+            self.debug(f"Got secret list of: {len(secret_list)} - keys")
+
+        return secret_list
+
+    def _read_secret_list_from_hvac(self) -> List[bytes]:
+        self.debug('Read secret keys from HashiCorp Vault...')
+
+        client = hvac.Client(url=self._config.hvac_url, token=self._config.hvac_token)
+        if not client.is_authenticated():
+            self.warning('Cannot connect to HashiCorp Vault!')
+            return []
+
+        secret_list: List[bytes] = []
+
+        mount = self._config.hvac_mount if self._config.hvac_mount is not None else DEFAULT_MOUNT_POINT
+
+        base_path = self._config.hvac_path
+        try:
+            response_list = client.secrets.kv.v2.list_secrets(path=base_path, mount_point=mount)
+        except BaseException as exc:
+            self.warning(f'Fail to read secret list from {base_path}')
+            return []
+
+        for key_name in response_list.get('data', {}).get('keys', []):
+            key_path = os.path.join(base_path, key_name)
+            try:
+                data = client.secrets.kv.v2.read_secret(path=key_path, mount_point=mount)
+                secret = data.get('data', {}).get('data', {}).get('secret_key', None)
+                if secret is None:
+                    self.warning(f'No secret_key in the path {key_path}')
+                    continue
+
+                sol_account = SolAccount.from_secret_key(base64.b64decode(secret))
+                secret_list.append(sol_account.secret_key)
+                self.debug(f'Get secret: {str(sol_account.public_key)}')
+
+            except (Exception, ):
+                self.warning(f'Fail to read secret from {key_path}')
+
+        return secret_list
+
+    def _read_secret_file(self, name: str) -> Optional[SolAccount]:
+        self.debug(f"Open a secret file: {name}")
+        with open(name.strip(), mode='r') as d:
+            pkey = (d.read())
+            num_list = [int(v) for v in pkey.strip("[] \n").split(',') if 0 <= int(v) <= 255]
+            if len(num_list) < 32:
+                self.debug(f'Wrong content in the file {name}')
+                return None
+            return SolAccount.from_secret_key(bytes(num_list[:32]))
+
+    def _read_secret_list_from_fs(self) -> List[bytes]:
+        self.debug('Read secret keys from filesystem...')
+
+        res = SolanaCli(self._config).call('config', 'get')
+        self.debug(f"Got solana config: {res}")
+        substr = "Keypair Path: "
+        path = ""
+        for line in res.splitlines():
+            if line.startswith(substr):
+                path = line[len(substr):].strip()
+        if path == "":
+            self.warning("cannot get keypair path")
+            return []
+
+        path = path.strip()
+        file_name, file_ext = os.path.splitext(path)
+
+        secret_list: List[bytes] = []
+
+        i = 0
+        while True:
+            i += 1
+            full_path = file_name + (str(i) if i > 1 else '') + file_ext
+            if not os.path.isfile(full_path):
+                break
+
+            secret = self._read_secret_file(full_path)
+            if secret is None:
+                continue
+            secret_list.append(secret.secret_key)
+            self.debug(f'Get secret: {str(secret.public_key)}')
+
+        return secret_list

proxy/common_neon/solana_tx_list_sender.py

@@ -213,7 +213,7 @@ def _decode_tx_status(self, tx: SolTx, tx_error_parser: SolTxErrorParser) -> Sol
             return SolTxSendState.Status.BadNonceError
         elif tx_error_parser.check_if_alt_uses_invalid_index():
             return SolTxSendState.Status.AltInvalidIndexError
-        elif tx_error_parser.check_if_account_already_exists():
+        elif tx_error_parser.check_if_already_finalized():
             return SolTxSendState.Status.AlreadyFinalizedError
         elif tx_error_parser.check_if_blockhash_notfound():
             if tx.recent_blockhash == self._blockhash:

proxy/docker-compose-test.yml

@@ -91,16 +91,26 @@ services:
       NEON_CLI_DEBUG_LOG: "YES"
       FUZZING_BLOCKHASH: "NO"
       CONFIG: ci
+      # Comment next two lines, if you want local stand without real gas-price
       PP_SOLANA_URL: ${CI_PP_SOLANA_URL:-https://api.devnet.solana.com}
       PYTH_MAPPING_ACCOUNT: ${CI_PYTH_MAPPING_ACCOUNT:-BmA9Z6FjioHJPpjT39QazZyhDRUdZy2ezwx4GiDdE2u2}
+      #
       MIN_OPERATOR_BALANCE_TO_WARN: 4565760000 # = 913152000 * 5 (5 storage accounts) = 4.56576 SOL
       MIN_OPERATOR_BALANCE_TO_ERR: 913152000 # = solana rent 131072 (= Rent-exempt minimum: 0.913152 SOL) SOLs to create a storage
+      # Set to 0 + Comment PYTH configuration above, if you want 0 gas-price
       MINIMAL_GAS_PRICE: 1
+      #
       ENABLE_PRIVATE_API: "YES"
       ALLOW_UNDERPRICED_TX_WITHOUT_CHAINID: "YES"
       LOG_FULL_OBJECT_INFO: "NO"
       EVM_LOADER: 53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io
       RUST_BACKTRACE: ${RUST_BACKTRACE:-0}
+      # An example of HashiCorp configuration, operator key lies in the secret_key field
+#      HVAC_URL: http://vault:8200
+#      HVAC_TOKEN: local-test-hvac-token
+#      HVAC_MOUNT: secret
+#      HVAC_PATH: neon-proxy/
+      #
     hostname: proxy
     depends_on:
       solana:
@@ -210,5 +220,17 @@ services:
       - net
     entrypoint: proxy/run-indexer.sh
 
+#  hashicorp:
+#    container_name: vault
+#    image: hashicorp/vault:latest
+#    hostname: vault
+#    expose:
+#      - 8200
+#    ports:
+#      - 8200:8200
+#    networks:
+#      - net
+#    entrypoint: vault server -dev -dev-listen-address=0.0.0.0:8200 -dev-root-token-id=local-test-hvac-token
+
 networks:
   net:

proxy/indexer/indexer.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import base64
 import time
 from collections import deque
 from typing import List, Optional, Dict, Deque, Type
@@ -10,10 +11,10 @@
 from ..common_neon.config import Config
 from ..common_neon.constants import ACTIVE_HOLDER_TAG
 from ..common_neon.data import NeonTxStatData
-from ..common_neon.environment_utils import get_solana_accounts
+from ..common_neon.operator_secret_mng import OpSecretMng
 from ..common_neon.solana_interactor import SolInteractor
 from ..common_neon.solana_neon_tx_receipt import SolTxMetaInfo, SolTxCostInfo
-from ..common_neon.solana_tx import SolPubKey
+from ..common_neon.solana_tx import SolPubKey, SolAccount
 from ..common_neon.solana_tx_error_parser import SolTxErrorParser
 from ..common_neon.utils import SolanaBlockInfo
 
@@ -36,15 +37,23 @@ def __init__(self, config: Config, indexer_stat_exporter: IIndexerStatExporter):
         self._db = IndexerDB()
         last_known_slot = self._db.get_min_receipt_block_slot()
         super().__init__(config, solana, last_known_slot)
-        self._cancel_tx_executor = CancelTxExecutor(config, solana, get_solana_accounts(config)[0])
+
+        op_secret_mng = OpSecretMng(self._config)
+        secret_list = op_secret_mng.read_secret_list()
+        sol_account = SolAccount.from_secret_key(secret_list[0]) if len(secret_list) > 0 else SolAccount()
+        self._cancel_tx_executor = CancelTxExecutor(config, solana, sol_account)
+
         self._counted_logger = MetricsToLogger()
         self._stat_exporter = indexer_stat_exporter
         self._last_stat_time = 0.0
+
         sol_tx_meta_dict = SolTxMetaDict()
         collector = FinalizedSolTxMetaCollector(config, self._solana, sol_tx_meta_dict, self._last_slot)
         self._finalized_sol_tx_collector = collector
+
         collector = ConfirmedSolTxMetaCollector(config, self._solana, sol_tx_meta_dict)
         self._confirmed_sol_tx_collector = collector
+
         self._confirmed_block_slot: Optional[int] = None
         self._neon_block_dict = NeonIndexedBlockDict()
 
@@ -84,7 +93,6 @@ def _cancel_neon_tx(self, tx: NeonIndexedTxInfo, sol_tx_meta: SolTxMetaInfo) ->
             return False
 
         holder_account = tx.storage_account
-
         holder_info = self._solana.get_holder_account_info(SolPubKey(holder_account))
         if not holder_info:
             self.warning(f'holder {holder_account} for neon tx {tx.neon_tx.sig} is empty')
@@ -103,7 +111,7 @@ def _cancel_neon_tx(self, tx: NeonIndexedTxInfo, sol_tx_meta: SolTxMetaInfo) ->
 
         if not self._cancel_tx_executor.add_blocked_holder_account(holder_info):
             self.warning(
-                f'neon tx {tx.neon_tx} uses the storage account {holder_account}' +
+                f'neon tx {tx.neon_tx} uses the storage account {holder_account} '
                 'which is already in the list on unlock'
             )
             return False