From 22b10f94ff4fbc045fb6b0bee89db8094525a827 Mon Sep 17 00:00:00 2001 From: ivandzen Date: Fri, 11 Mar 2022 09:55:01 +0300 Subject: [PATCH 01/73] 307 implement withdraw instruction (#591) * #308 Support AccountV2 * Dockerfile * add test file * remove not used file * just compiles and executes emulator * Fix NeonToken account, send method call transaction with amount * check balances before and after * add some logging * Improve test * Add test case * add require to neon contract * Fix tests * fix user stories test * rollback user stories tests * Fix review issues Co-authored-by: Anton Lisanin Co-authored-by: Ivan Loboda --- Dockerfile | 2 +- proxy/testing/test_neon_token.py | 281 +++++++++++++++++++++++++++ proxy/testing/test_price_provider.py | 0 proxy/testing/testing_helpers.py | 4 +- 4 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 proxy/testing/test_neon_token.py delete mode 100644 proxy/testing/test_price_provider.py diff --git a/Dockerfile b/Dockerfile index 55449ad83..b2308b668 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG SOLANA_REVISION=v1.8.12-testnet -ARG EVM_LOADER_REVISION=latest +ARG EVM_LOADER_REVISION=develop FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli diff --git a/proxy/testing/test_neon_token.py b/proxy/testing/test_neon_token.py new file mode 100644 index 000000000..30cb27d5d --- /dev/null +++ b/proxy/testing/test_neon_token.py @@ -0,0 +1,281 @@ +import unittest +from solcx import compile_source +from web3 import Web3 +import os +from .testing_helpers import request_airdrop +from solana.account import Account as SolanaAccount +from solana.rpc.api import Client as SolanaClient +from solana.transaction import Transaction +from solana.rpc.types import TxOpts +from solana.rpc.commitment import Confirmed +from spl.token.client import Token as SplToken +from spl.token.instructions import get_associated_token_address, create_associated_token_account +from proxy.environment import NEON_TOKEN_MINT +from spl.token.constants import TOKEN_PROGRAM_ID +from solana.rpc.commitment import Confirmed +from time import sleep +from web3 import exceptions as web3_exceptions +from random import uniform +from eth_account.signers.local import LocalAccount as NeonAccount + +NEON_TOKEN_CONTRACT = ''' +// SPDX-License-Identifier: MIT +pragma solidity >=0.5.12; + +contract NeonToken { + address constant NeonPrecompiled = 0xFF00000000000000000000000000000000000003; + + function withdraw(bytes32 spender) public payable { + (bool success, bytes memory returnData) = NeonPrecompiled.delegatecall(abi.encodeWithSignature("withdraw(bytes32)", spender)); + require(success); + } +} +''' + + +PROXY_URL = os.environ.get('PROXY_URL', 'http://127.0.0.1:9090/solana') +SOLANA_URL = os.environ.get('SOLANA_URL', 'http://solana:8899/') +proxy = Web3(Web3.HTTPProvider(PROXY_URL)) +solana = SolanaClient(SOLANA_URL) + +class TestNeonToken(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.sol_payer = SolanaAccount() + cls.deploy_contract(cls) + cls.spl_neon_token = SplToken(solana, NEON_TOKEN_MINT, TOKEN_PROGRAM_ID, cls.sol_payer) + + def create_eth_account(self, balance): + seed = f'TestAccount{uniform(0, 10000)}' + new_neon_acc = proxy.eth.account.create(seed) + request_airdrop(new_neon_acc.address, balance) + print(f"New Neon account {new_neon_acc.address} with balance {balance}") + return new_neon_acc + + def create_sol_account(self, balance = 1000_000_000_000): + new_sol_acc = SolanaAccount() + print(f"New Solana account {new_sol_acc.public_key()} with balance {balance}") + solana.request_airdrop(new_sol_acc.public_key(), balance) + return new_sol_acc + + def deploy_contract(self): + artifacts = compile_source(NEON_TOKEN_CONTRACT) + _, self.neon_token_iface = artifacts.popitem() + + self.neon_contract = proxy.eth.contract(abi=self.neon_token_iface['abi'], + bytecode=self.neon_token_iface['bin']) + + deployer = self.create_eth_account(self, 100) + proxy.eth.default_account = deployer.address + + nonce = proxy.eth.get_transaction_count(deployer.address) + tx = {'nonce': nonce} + tx_constructor = self.neon_contract.constructor().buildTransaction(tx) + tx_deploy = proxy.eth.account.sign_transaction(tx_constructor, deployer.key) + tx_deploy_hash = proxy.eth.send_raw_transaction(tx_deploy.rawTransaction) + print(f'tx_deploy_hash: {tx_deploy_hash.hex()}') + tx_deploy_receipt = proxy.eth.wait_for_transaction_receipt(tx_deploy_hash) + print(f'tx_deploy_receipt: {tx_deploy_receipt}') + print(f'deploy status: {tx_deploy_receipt.status}') + self.neon_token_address = tx_deploy_receipt.contractAddress + print(f'NeonToken contract address is: {self.neon_token_address}') + self.neon_contract = proxy.eth.contract(address=self.neon_token_address, + abi=self.neon_token_iface['abi']) + + def withdraw(self, source_acc: NeonAccount, dest_acc: SolanaAccount, withdraw_amount_alan: int): + nonce = proxy.eth.get_transaction_count(source_acc.address) + tx = {'value': withdraw_amount_alan, 'nonce': nonce} + withdraw_tx_dict = self.neon_contract.functions.withdraw(bytes(dest_acc.public_key())).buildTransaction(tx) + withdraw_tx = proxy.eth.account.sign_transaction(withdraw_tx_dict, source_acc.key) + withdraw_tx_hash = proxy.eth.send_raw_transaction(withdraw_tx.rawTransaction) + print(f'withdraw_tx_hash: {withdraw_tx_hash.hex()}') + withdraw_tx_receipt = proxy.eth.wait_for_transaction_receipt(withdraw_tx_hash) + print(f'withdraw_tx_receipt: {withdraw_tx_receipt}') + print(f'deploy status: {withdraw_tx_receipt.status}') + + def test_success_withdraw_to_non_existing_account(self): + """ + Should succesfully withdraw NEON tokens to previously non-existing Associated Token Account + """ + source_acc = self.create_eth_account(10) + dest_acc = self.create_sol_account() + + dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) + print(f"Destination token account: {dest_token_acc}") + + withdraw_amount_alan = pow(10, 18) # 1 NEON + withdraw_amount_galan = int(withdraw_amount_alan / 1_000_000_000) + + # Check source balance + source_balance_before_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance before (Alan): {source_balance_before_alan}') + + # Check destination balance (must not exist) + destination_balance_before_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance before (Galan): {destination_balance_before_galan}') + self.assertTrue(destination_balance_before_galan['error'] is not None) + + self.withdraw(source_acc, dest_acc, withdraw_amount_alan) + + # Check source balance + source_balance_after_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance after (Alan): {source_balance_after_alan}') + self.assertLess(source_balance_after_alan, source_balance_before_alan - withdraw_amount_alan) + + # Check destination balance + destination_balance_after_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance after (Galan): {destination_balance_after_galan}') + self.assertEqual(int(destination_balance_after_galan['result']['value']['amount']), withdraw_amount_galan) + + def test_success_withdraw_to_existing_account(self): + """ + Should succesfully withdraw NEON tokens to existing Associated Token Account + """ + source_acc = self.create_eth_account(10) + dest_acc = self.create_sol_account() + + # Creating destination Associated Token Account + trx = Transaction() + trx.add( + create_associated_token_account( + dest_acc.public_key(), + dest_acc.public_key(), + NEON_TOKEN_MINT + ) + ) + opts = TxOpts(skip_preflight=True, skip_confirmation=False) + solana.send_transaction(trx, dest_acc, opts=opts) + + dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) + print(f"Destination token account: {dest_token_acc}") + + withdraw_amount_alan = 2_123_000_321_000_000_000 + withdraw_amount_galan = int(withdraw_amount_alan / 1_000_000_000) + + # Check source balance + source_balance_before_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance before (Alan): {source_balance_before_alan}') + + # Check destination balance (must exist with zero balance) + resp = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + destination_balance_before_galan = int(resp['result']['value']['amount']) + print(f'Destination account balance before (Galan): {destination_balance_before_galan}') + self.assertEqual(destination_balance_before_galan, 0) + + self.withdraw(source_acc, dest_acc, withdraw_amount_alan) + + # Check source balance + source_balance_after_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance after (Alan): {source_balance_after_alan}') + self.assertLess(source_balance_after_alan, source_balance_before_alan - withdraw_amount_alan) + + # Check destination balance + resp = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + destination_balance_after_galan = int(resp['result']['value']['amount']) + print(f'Destination account balance after (Galan): {destination_balance_after_galan}') + self.assertEqual(destination_balance_after_galan, withdraw_amount_galan) + + def test_failed_withdraw_non_divisible_amount(self): + """ + Should fail withdrawal because amount not divised by 1 billion + """ + source_acc = self.create_eth_account(10) + dest_acc = self.create_sol_account() + + dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) + print(f"Destination token account: {dest_token_acc}") + + withdraw_amount_alan = pow(10, 18) + 123 # NEONs + + # Check source balance + source_balance_before_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance before (Alan): {source_balance_before_alan}') + + # Check destination balance (must not exist) + destination_balance_before_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance before (Galan): {destination_balance_before_galan}') + self.assertTrue(destination_balance_before_galan['error'] is not None) + + with self.assertRaises(web3_exceptions.ContractLogicError) as er: + self.withdraw(source_acc, dest_acc, withdraw_amount_alan) + print(f'Exception occured: {er.exception}') + + # Check source balance + source_balance_after_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance after (Alan): {source_balance_after_alan}') + self.assertEqual(source_balance_after_alan, source_balance_before_alan) + + # Check destination balance + destination_balance_after_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance after (Galan): {destination_balance_after_galan}') + self.assertTrue(destination_balance_after_galan['error'] is not None) + + def test_failed_withdraw_insufficient_balance(self): + """ + Should fail withdrawal because of insufficient balance + """ + source_acc = self.create_eth_account(1) + dest_acc = self.create_sol_account() + + dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) + print(f"Destination token account: {dest_token_acc}") + + withdraw_amount_alan = 2 * pow(10, 18) # 2 NEONs + + # Check source balance + source_balance_before_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance before (Alan): {source_balance_before_alan}') + + # Check destination balance (must not exist) + destination_balance_before_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance before (Galan): {destination_balance_before_galan}') + self.assertTrue(destination_balance_before_galan['error'] is not None) + + with self.assertRaises(ValueError) as er: + self.withdraw(source_acc, dest_acc, withdraw_amount_alan) + print(f'Exception occured: {er.exception}') + + # Check source balance + source_balance_after_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance after (Alan): {source_balance_after_alan}') + self.assertEqual(source_balance_after_alan, source_balance_before_alan) + + # Check destination balance + destination_balance_after_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance after (Galan): {destination_balance_after_galan}') + self.assertTrue(destination_balance_after_galan['error'] is not None) + + def test_failed_withdraw_all_balance(self): + """ + Should fail withdrawal all balance + """ + source_acc = self.create_eth_account(1) # 1 NEON + dest_acc = self.create_sol_account() + + dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) + print(f"Destination token account: {dest_token_acc}") + + withdraw_amount_alan = 1_000_000_000_000_000_000 # 1 NEON + + # Check source balance + source_balance_before_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance before (Alan): {source_balance_before_alan}') + + # Check destination balance (must not exist) + destination_balance_before_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance before (Galan): {destination_balance_before_galan}') + self.assertTrue(destination_balance_before_galan['error'] is not None) + + with self.assertRaises(ValueError) as er: + self.withdraw(source_acc, dest_acc, withdraw_amount_alan) + print(f'Exception occured: {er.exception}') + + # Check source balance + source_balance_after_alan = proxy.eth.get_balance(source_acc.address) + print(f'Source account balance after (Alan): {source_balance_after_alan}') + self.assertEqual(source_balance_after_alan, source_balance_before_alan) + + # Check destination balance + destination_balance_after_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) + print(f'Destination account balance after (Galan): {destination_balance_after_galan}') + self.assertTrue(destination_balance_after_galan['error'] is not None) diff --git a/proxy/testing/test_price_provider.py b/proxy/testing/test_price_provider.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/proxy/testing/testing_helpers.py b/proxy/testing/testing_helpers.py index f7411876c..a61230fbe 100644 --- a/proxy/testing/testing_helpers.py +++ b/proxy/testing/testing_helpers.py @@ -48,10 +48,10 @@ def web3(self) -> Web3: return self._web3 -def request_airdrop(address): +def request_airdrop(address, amount: int = 10): FAUCET_URL = os.environ.get('FAUCET_URL', 'http://faucet:3333') url = FAUCET_URL + '/request_neon' - data = '{"wallet": "' + address + '", "amount": 10}' + data = f'{{"wallet": "{address}", "amount": {amount}}}' r = requests.post(url, data=data) if not r.ok: print() From 0ec3c9e4fb0b5a90b9f87890802c6a5c8049f560 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 14:45:23 +0300 Subject: [PATCH 02/73] intro --- .buildkite/pipeline.yml | 7 +- tf/build.sh | 13 +++ tf/destroy.sh | 12 +++ tf/main.tf | 188 +++++++++++++++++++++++++++++++++++++++ tf/output.tf | 8 ++ tf/proxy_init.sh | 59 ++++++++++++ tf/solana_init.sh | 26 ++++++ tf/terraform.auto.tfvars | 9 ++ tf/vars.tf | 31 +++++++ 9 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 tf/build.sh create mode 100644 tf/destroy.sh create mode 100644 tf/main.tf create mode 100644 tf/output.tf create mode 100644 tf/proxy_init.sh create mode 100644 tf/solana_init.sh create mode 100644 tf/terraform.auto.tfvars create mode 100644 tf/vars.tf diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b82b2ce1f..150e8e504 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,6 +4,9 @@ steps: - ".buildkite/steps/build-image.sh" - ".buildkite/steps/upload-image.sh" + - label: ":terraform: build infrastructure" + command: + - "tf/build.sh" - wait # - label: ":cop::skin-tone-2: deploy check" @@ -54,4 +57,6 @@ steps: build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - + - label: ":terraform: destroy infrastructure" + command: + - "tf/destroy.sh" diff --git a/tf/build.sh b/tf/build.sh new file mode 100644 index 000000000..da57d4924 --- /dev/null +++ b/tf/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cd tf + +export TF_VAR_branch=$BUILDKITE_BRANCH +export TFSTATE_BUCKET="nl-ci-stands" +export TFSTATE_KEY="tests/test-$BUILDKITE_COMMIT" +export TFSTATE_REGION="us-east-2" +export TF_VAR_neon_evm_revision=latest +export TF_VAR_proxy_model_revision=latest +export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" +terraform init $TF_BACKEND_CONFIG +terraform apply --auto-approve=true diff --git a/tf/destroy.sh b/tf/destroy.sh new file mode 100644 index 000000000..15ba0ac15 --- /dev/null +++ b/tf/destroy.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd tf + +export TF_VAR_branch=$BUILDKITE_BRANCH +export TFSTATE_BUCKET="nl-ci-stands" +export TFSTATE_KEY="tests/test-$BUILDKITE_COMMIT" +export TFSTATE_REGION="us-east-2" +export TF_VAR_neon_evm_revision=latest +export TF_VAR_proxy_model_revision=latest +export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" +terraform destroy --auto-approve=true diff --git a/tf/main.tf b/tf/main.tf new file mode 100644 index 000000000..f9ddb0ed5 --- /dev/null +++ b/tf/main.tf @@ -0,0 +1,188 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } + + backend "s3" { + + } +} + +provider "aws" { + region = "us-east-2" +} + +data "aws_vpc" "default" { + default = true +} + +data "aws_key_pair" "ci-stands" { + key_name = "ci-stands" +} + +data "template_file" "solana_init" { + template = file("solana_init.sh") + + vars = { + branch = "${var.branch}" + } +} + +data "template_file" "proxy_init" { + template = file("proxy_init.sh") + + vars = { + branch = "${var.branch}" + revision = "latest" + solana_ip = aws_instance.solana.private_ip + } +} + +resource "aws_security_group" "test-stand-solana" { + name = "${var.branch} - solana group for test stand" + description = "set of rules allow incoming traffic from ci test agents for OZ tests" + vpc_id = data.aws_vpc.default.id + + ingress { + description = "allow incoming from ci test agent to SOLANA" + from_port = 0 + to_port = 65535 + protocol = "tcp" + cidr_blocks = var.allow_list + + } + + ingress { + description = "allow incoming from ci test agent to SOLANA" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = { + Name = "${var.branch}-test-stand-solana" + purpose = "ci-oz-full-tests" + } +} + +resource "aws_security_group" "test-stand-proxy" { + name = "${var.branch} - proxy group for test stand" + description = "set of rules allow incoming traffic from ci test agents for OZ tests" + vpc_id = data.aws_vpc.default.id + + ingress { + description = "allow incoming from ci test agent to PROXY" + from_port = 9090 + to_port = 9090 + protocol = "tcp" + cidr_blocks = var.allow_list + + } + + ingress { + description = "allow incoming from ci test agent to FAUCET" + from_port = 3333 + to_port = 3333 + protocol = "tcp" + cidr_blocks = var.allow_list + + } + ingress { + description = "allow incoming from ci test agent to SOLANA" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = { + Name = "${var.branch}-test-stand-proxy" + purpose = "ci-oz-full-tests" + } +} + + +resource "aws_instance" "solana" { + instance_type = var.instance_type + ami = var.ami + key_name = data.aws_key_pair.ci-stands.key_name + vpc_security_group_ids = [aws_security_group.test-stand-solana.id] + subnet_id = var.aws_subnet + + ebs_block_device { + device_name = "/dev/sda1" + volume_size = 50 + } + + user_data = data.template_file.solana_init.rendered + + tags = { + Name = "${var.branch}-test-stand-solana" + purpose = "ci-oz-full-tests" + } + +} + +resource "aws_instance" "proxy" { + instance_type = var.instance_type + ami = var.ami + key_name = data.aws_key_pair.ci-stands.key_name + vpc_security_group_ids = [aws_security_group.test-stand-proxy.id] + subnet_id = var.aws_subnet + ebs_block_device { + device_name = "/dev/sda1" + volume_size = 50 + } + //user_data = data.template_file.proxy_init.rendered + tags = { + Name = "${var.branch}-test-stand-proxy" + purpose = "ci-oz-full-tests" + } + depends_on = [ + aws_instance.solana + ] + + connection { + type = "ssh" + user = "ubuntu" + host = aws_instance.proxy.public_ip + private_key = file("1") + } + + provisioner "file" { + content = data.template_file.proxy_init.rendered + destination = "/tmp/proxy_init.sh" + } + + provisioner "remote-exec" { + inline = [ + "echo '${aws_instance.solana.private_ip}' > /tmp/solana_host", + "chmod a+x /tmp/proxy_init.sh", + "sudo /tmp/proxy_init.sh" + ] + } +} + + diff --git a/tf/output.tf b/tf/output.tf new file mode 100644 index 000000000..859b5d7bb --- /dev/null +++ b/tf/output.tf @@ -0,0 +1,8 @@ +output "solana_ip" { + value = aws_instance.solana.public_ip +} + +output "proxy_ip" { + value = aws_instance.proxy.public_ip +} + diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh new file mode 100644 index 000000000..ad8b520e5 --- /dev/null +++ b/tf/proxy_init.sh @@ -0,0 +1,59 @@ +#!/bin/bash +sudo apt-get remove docker docker-engine docker.io containerd runc +sudo apt-get update +sudo apt-get -y install ca-certificates curl gnupg lsb-release +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get -y install docker-ce docker-ce-cli containerd.io +sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +cd /opt +curl -O https://raw.githubusercontent.com/neonlabsorg/proxy-model.py/${branch}/proxy/docker-compose-test.yml + +export REVISION=${revision} +export SOLANA_URL=http:\/\/${solana_ip}:8899 + +cat > docker-compose-test.override.yml <> /tmp/output.txt + echo "attempt: $CURRENT_ATTEMPT" + ((CURRENT_ATTEMPT=CURRENT_ATTEMPT+1)) + sleep 2 +done; + +docker-compose -f docker-compose-test.yml -f docker-compose-test.override.yml up -d $SERVICES diff --git a/tf/solana_init.sh b/tf/solana_init.sh new file mode 100644 index 000000000..1349bbf20 --- /dev/null +++ b/tf/solana_init.sh @@ -0,0 +1,26 @@ +#!/bin/bash +sudo apt-get remove docker docker-engine docker.io containerd runc +sudo apt-get update +sudo apt-get -y install ca-certificates curl gnupg lsb-release +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get -y install docker-ce docker-ce-cli containerd.io +sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +cd /opt +curl -O https://raw.githubusercontent.com/neonlabsorg/proxy-model.py/${branch}/proxy/docker-compose-test.yml +cat > docker-compose-test.override.yml < Date: Fri, 11 Mar 2022 19:02:10 +0300 Subject: [PATCH 03/73] upd creds --- .buildkite/pipeline.yml | 4 ++++ tf/main.tf | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 150e8e504..d9883b5f7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -5,6 +5,8 @@ steps: - ".buildkite/steps/upload-image.sh" - label: ":terraform: build infrastructure" + agents: + queue: "testing" command: - "tf/build.sh" - wait @@ -58,5 +60,7 @@ steps: (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - label: ":terraform: destroy infrastructure" + agents: + queue: "testing" command: - "tf/destroy.sh" diff --git a/tf/main.tf b/tf/main.tf index f9ddb0ed5..11ae91e2a 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -168,7 +168,7 @@ resource "aws_instance" "proxy" { type = "ssh" user = "ubuntu" host = aws_instance.proxy.public_ip - private_key = file("1") + private_key = file("~/.ssh/ci-stands") } provisioner "file" { From 7ef74d6e404aba696f5d58f64693105bb8e8c016 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 19:05:07 +0300 Subject: [PATCH 04/73] fix --- tf/destroy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/destroy.sh b/tf/destroy.sh index 15ba0ac15..15f408de7 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -9,4 +9,5 @@ export TFSTATE_REGION="us-east-2" export TF_VAR_neon_evm_revision=latest export TF_VAR_proxy_model_revision=latest export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" +terraform init terraform destroy --auto-approve=true From 60124c2e9d0ffd0acbf0995dd872257ebeb7d0e0 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 20:00:11 +0300 Subject: [PATCH 05/73] wip --- .buildkite/pipeline.yml | 62 ++++++++++++++++++++--------------------- tf/build.sh | 6 ++++ tf/destroy.sh | 3 ++ 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d9883b5f7..fce0c89e0 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -25,39 +25,39 @@ steps: # - "airdropper.log" # - "indexer.log" - - label: ":coverage: full test suite (FTS)" -# if: | -# (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || -# (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") - commands: - - echo Full test suite container name - $${FTS_CONTAINER_NAME} - - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up - - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) - - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ - - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log - - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f - - echo Full test passing - $${FTS_RESULT} - - echo Full test threshold - $${FTS_THRESHOLD} - - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} - - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} - artifact_paths: - - allure-reports.tar.gz - - fts_${BUILDKITE_BUILD_NUMBER}.log - env: - PROXY_URL: http://proxy.night.stand.neontest.xyz/solana - FAUCET_URL: http://proxy.night.stand.neontest.xyz/request_eth_token - SOLANA_URL: http://proxy.night.stand.neontest.xyz/node-solana - FTS_THRESHOLD: 1700 - FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} - FTS_IMAGE: neonlabsorg/full_test_suite:583-full-test-suite +# - label: ":coverage: full test suite (FTS)" +# # if: | +# # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || +# # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") +# commands: +# - echo Full test suite container name - $${FTS_CONTAINER_NAME} +# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up +# - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) +# - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ +# - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log +# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f +# - echo Full test passing - $${FTS_RESULT} +# - echo Full test threshold - $${FTS_THRESHOLD} +# - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} +# - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} +# artifact_paths: +# - allure-reports.tar.gz +# - fts_${BUILDKITE_BUILD_NUMBER}.log +# env: +# PROXY_URL: http://proxy.night.stand.neontest.xyz/solana +# FAUCET_URL: http://proxy.night.stand.neontest.xyz/request_eth_token +# SOLANA_URL: http://proxy.night.stand.neontest.xyz/node-solana +# FTS_THRESHOLD: 1700 +# FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} +# FTS_IMAGE: neonlabsorg/full_test_suite:583-full-test-suite - - wait +# - wait - - label: ":floppy_disk: publish image" - command: ".buildkite/steps/publish-image.sh" - if: | - build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && - (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) +# - label: ":floppy_disk: publish image" +# command: ".buildkite/steps/publish-image.sh" +# if: | +# build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && +# (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - label: ":terraform: destroy infrastructure" agents: diff --git a/tf/build.sh b/tf/build.sh index da57d4924..1322503a5 100644 --- a/tf/build.sh +++ b/tf/build.sh @@ -11,3 +11,9 @@ export TF_VAR_proxy_model_revision=latest export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" terraform init $TF_BACKEND_CONFIG terraform apply --auto-approve=true + +terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "neon-tests-proxy-$BUILDKITE_COMMIT" +terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "neon-tests-solana-$BUILDKITE_COMMIT" + +buildkite meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" +buildkite meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" diff --git a/tf/destroy.sh b/tf/destroy.sh index 15f408de7..d4abb470b 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -11,3 +11,6 @@ export TF_VAR_proxy_model_revision=latest export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" terraform init terraform destroy --auto-approve=true + +buildkite meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" +buildkite meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" From ebf8ea0a0fb5fc121cc3fc84832555aac91077c1 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 20:06:38 +0300 Subject: [PATCH 06/73] typo --- tf/build.sh | 4 ++-- tf/destroy.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tf/build.sh b/tf/build.sh index 1322503a5..4f768ff31 100644 --- a/tf/build.sh +++ b/tf/build.sh @@ -15,5 +15,5 @@ terraform apply --auto-approve=true terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "neon-tests-proxy-$BUILDKITE_COMMIT" terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "neon-tests-solana-$BUILDKITE_COMMIT" -buildkite meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" -buildkite meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" +buildkite-agent meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" +buildkite-agent meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" diff --git a/tf/destroy.sh b/tf/destroy.sh index d4abb470b..4fd41815a 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -12,5 +12,5 @@ export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-co terraform init terraform destroy --auto-approve=true -buildkite meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" -buildkite meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" +buildkite-agent meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" +buildkite-agent meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" From 432bac780504c7501fa9d3a42aa1a5cd186a0a27 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 20:22:30 +0300 Subject: [PATCH 07/73] fix --- tf/destroy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/destroy.sh b/tf/destroy.sh index 4fd41815a..76ceca616 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -9,7 +9,7 @@ export TFSTATE_REGION="us-east-2" export TF_VAR_neon_evm_revision=latest export TF_VAR_proxy_model_revision=latest export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" -terraform init +terraform init $TF_BACKEND_CONFIG terraform destroy --auto-approve=true buildkite-agent meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" From dbe118d990c40df3ecc4e3497fce6313558dadca Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 20:48:18 +0300 Subject: [PATCH 08/73] wip --- .buildkite/pipeline.yml | 9 +++++++++ tf/build.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fce0c89e0..8cdb00795 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -25,6 +25,15 @@ steps: # - "airdropper.log" # - "indexer.log" + - label: "Check variables" + command: + - "echo $$SOLANA_IP" + - "echo $$PROXY_IP" + - "env" + env: + SOLANA_IP: "buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'" + PROXY_IP: "buildkite-agent meta-data get 'neon-tests-proxy-$$BUILDKITE_COMMIT'" + # - label: ":coverage: full test suite (FTS)" # # if: | # # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || diff --git a/tf/build.sh b/tf/build.sh index 4f768ff31..3f3b5327c 100644 --- a/tf/build.sh +++ b/tf/build.sh @@ -13,7 +13,7 @@ terraform init $TF_BACKEND_CONFIG terraform apply --auto-approve=true terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "neon-tests-proxy-$BUILDKITE_COMMIT" -terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "neon-tests-solana-$BUILDKITE_COMMIT" +terraform output --json | jq -r '.solana_ip.value' | buildkite-agent meta-data set "neon-tests-solana-$BUILDKITE_COMMIT" buildkite-agent meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" buildkite-agent meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" From 600d6cafad788a84181f9c200466b4ed480cf815 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 20:59:22 +0300 Subject: [PATCH 09/73] wip --- .buildkite/pipeline.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8cdb00795..de1e70e86 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,9 +27,11 @@ steps: - label: "Check variables" command: - - "echo $$SOLANA_IP" - - "echo $$PROXY_IP" - - "env" + - echo "SOLANA_IP: $$SOLANA_IP" + - echo "PROXY_IP: $$PROXY_IP" + - export ZZ=`buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'` + - echo "ZZ $ZZ" + env: SOLANA_IP: "buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'" PROXY_IP: "buildkite-agent meta-data get 'neon-tests-proxy-$$BUILDKITE_COMMIT'" From cb18d20d996a7fa653ce3297d46aecbc7f7b9453 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 21:07:58 +0300 Subject: [PATCH 10/73] wip --- .buildkite/pipeline.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index de1e70e86..6cf3ef994 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,10 +27,8 @@ steps: - label: "Check variables" command: - - echo "SOLANA_IP: $$SOLANA_IP" - - echo "PROXY_IP: $$PROXY_IP" - - export ZZ=`buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'` - - echo "ZZ $ZZ" + - env + - PROXY_URL=`buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'` && echo "PROXY_URL $PROXY_URL" env: SOLANA_IP: "buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'" From e9fd64c8ba7fc83f04711151a72d1cf6a90e68bf Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 21:20:34 +0300 Subject: [PATCH 11/73] wip --- .buildkite/pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6cf3ef994..ece7b11c5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -28,7 +28,8 @@ steps: - label: "Check variables" command: - env - - PROXY_URL=`buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'` && echo "PROXY_URL $PROXY_URL" + - PROXY_URL=`buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'` + - echo "PROXY_URL $$PROXY_URL" env: SOLANA_IP: "buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'" From b37c3cb193123c5a51875e3d80f43bec45104cfe Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 21:27:14 +0300 Subject: [PATCH 12/73] wip --- .buildkite/pipeline.yml | 2 +- tf/build.sh | 8 ++++---- tf/destroy.sh | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ece7b11c5..89f88ccfc 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -28,7 +28,7 @@ steps: - label: "Check variables" command: - env - - PROXY_URL=`buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'` + - PROXY_URL=`buildkite-agent meta-data get 'PROXY_IP'` - echo "PROXY_URL $$PROXY_URL" env: diff --git a/tf/build.sh b/tf/build.sh index 3f3b5327c..c7098548b 100644 --- a/tf/build.sh +++ b/tf/build.sh @@ -12,8 +12,8 @@ export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-co terraform init $TF_BACKEND_CONFIG terraform apply --auto-approve=true -terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "neon-tests-proxy-$BUILDKITE_COMMIT" -terraform output --json | jq -r '.solana_ip.value' | buildkite-agent meta-data set "neon-tests-solana-$BUILDKITE_COMMIT" +terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "PROXY_IP" +terraform output --json | jq -r '.solana_ip.value' | buildkite-agent meta-data set "SOLANA_IP" -buildkite-agent meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" -buildkite-agent meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" +buildkite-agent meta-data get "PROXY_IP" +buildkite-agent meta-data get "SOLANA_IP" diff --git a/tf/destroy.sh b/tf/destroy.sh index 76ceca616..da11f9923 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -12,5 +12,5 @@ export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-co terraform init $TF_BACKEND_CONFIG terraform destroy --auto-approve=true -buildkite-agent meta-data get "neon-tests-proxy-$BUILDKITE_COMMIT" -buildkite-agent meta-data get "neon-tests-solana-$BUILDKITE_COMMIT" +buildkite-agent meta-data get "PROXY_IP" +buildkite-agent meta-data get "SOLANA_IP" From 2f443c76ede7f2a0400785133aa5d549fd8d3d58 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 21:37:25 +0300 Subject: [PATCH 13/73] wip --- .buildkite/pipeline.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 89f88ccfc..b6fe45f65 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,9 +27,14 @@ steps: - label: "Check variables" command: - - env - - PROXY_URL=`buildkite-agent meta-data get 'PROXY_IP'` + - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` + - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` + - PROXY_URL="http://$$PROXY_ADDR:9090/solana" + - FAUCET_URL="http://$$PROXY_ADDR:3333/request_neon" + - SOLANA_URL="http://$$SOLANA_ADDR:8899" - echo "PROXY_URL $$PROXY_URL" + - echo "FAUCET_URL $$FAUCET_URL" + - echo "SOLANA_URL $$SOLANA_URL" env: SOLANA_IP: "buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'" From 885a0ad3add4f9f9701c1a28dd88b5aae299c1bd Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 22:44:50 +0300 Subject: [PATCH 14/73] upd SG --- .buildkite/pipeline.yml | 4 ++-- tf/main.tf | 18 ++++++++++++++---- tf/proxy_init.sh | 4 ++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b6fe45f65..d5adc3d3a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -29,8 +29,8 @@ steps: command: - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - - PROXY_URL="http://$$PROXY_ADDR:9090/solana" - - FAUCET_URL="http://$$PROXY_ADDR:3333/request_neon" + - PROXY_URL="http://$$PROXY_ADDR:9091/solana" + - FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - SOLANA_URL="http://$$SOLANA_ADDR:8899" - echo "PROXY_URL $$PROXY_URL" - echo "FAUCET_URL $$FAUCET_URL" diff --git a/tf/main.tf b/tf/main.tf index 11ae91e2a..2f4c5ef1a 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -41,8 +41,13 @@ data "template_file" "proxy_init" { } } +resource "random_id" "test-stand-solana" { + byte_length = 4 + prefix = "test-stand-solana-" +} + resource "aws_security_group" "test-stand-solana" { - name = "${var.branch} - solana group for test stand" + name = random_id.test-stand-solana.hex description = "set of rules allow incoming traffic from ci test agents for OZ tests" vpc_id = data.aws_vpc.default.id @@ -78,15 +83,20 @@ resource "aws_security_group" "test-stand-solana" { } } +resource "random_id" "test-stand-proxy" { + byte_length = 4 + prefix = "test-stand-solana-" +} + resource "aws_security_group" "test-stand-proxy" { - name = "${var.branch} - proxy group for test stand" + name = random_id.test-stand-proxy.hex description = "set of rules allow incoming traffic from ci test agents for OZ tests" vpc_id = data.aws_vpc.default.id ingress { description = "allow incoming from ci test agent to PROXY" from_port = 9090 - to_port = 9090 + to_port = 9091 protocol = "tcp" cidr_blocks = var.allow_list @@ -95,7 +105,7 @@ resource "aws_security_group" "test-stand-proxy" { ingress { description = "allow incoming from ci test agent to FAUCET" from_port = 3333 - to_port = 3333 + to_port = 3334 protocol = "tcp" cidr_blocks = var.allow_list diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index ad8b520e5..9261b26c7 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -30,9 +30,13 @@ services: proxy: environment: - SOLANA_URL=$SOLANA_URL + ports: + - 9091:9090 faucet: environment: - SOLANA_URL=$SOLANA_URL + ports: + - 3334:3333 airdropper: environment: - SOLANA_URL=$SOLANA_URL From d35af722675b2b80d1be582a0c87e0b713872bb0 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 22:51:29 +0300 Subject: [PATCH 15/73] enable tests --- .buildkite/pipeline.yml | 70 +++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d5adc3d3a..fa7462f70 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -25,54 +25,42 @@ steps: # - "airdropper.log" # - "indexer.log" - - label: "Check variables" - command: + + - label: ":coverage: full test suite (FTS)" +# if: | +# (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || +# (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") + commands: - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - PROXY_URL="http://$$PROXY_ADDR:9091/solana" - FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - - SOLANA_URL="http://$$SOLANA_ADDR:8899" - - echo "PROXY_URL $$PROXY_URL" - - echo "FAUCET_URL $$FAUCET_URL" - - echo "SOLANA_URL $$SOLANA_URL" - + - SOLANA_URL="http://$$SOLANA_ADDR:8899" + - echo Full test suite container name - $${FTS_CONTAINER_NAME} + - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up + - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) + - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ + - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log + - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f + - echo Full test passing - $${FTS_RESULT} + - echo Full test threshold - $${FTS_THRESHOLD} + - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} + - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} + artifact_paths: + - allure-reports.tar.gz + - fts_${BUILDKITE_BUILD_NUMBER}.log env: - SOLANA_IP: "buildkite-agent meta-data get 'neon-tests-solana-$$BUILDKITE_COMMIT'" - PROXY_IP: "buildkite-agent meta-data get 'neon-tests-proxy-$$BUILDKITE_COMMIT'" + FTS_THRESHOLD: 1700 + FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} + FTS_IMAGE: neonlabsorg/full_test_suite:583-full-test-suite -# - label: ":coverage: full test suite (FTS)" -# # if: | -# # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || -# # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") -# commands: -# - echo Full test suite container name - $${FTS_CONTAINER_NAME} -# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up -# - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) -# - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ -# - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log -# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f -# - echo Full test passing - $${FTS_RESULT} -# - echo Full test threshold - $${FTS_THRESHOLD} -# - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} -# - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} -# artifact_paths: -# - allure-reports.tar.gz -# - fts_${BUILDKITE_BUILD_NUMBER}.log -# env: -# PROXY_URL: http://proxy.night.stand.neontest.xyz/solana -# FAUCET_URL: http://proxy.night.stand.neontest.xyz/request_eth_token -# SOLANA_URL: http://proxy.night.stand.neontest.xyz/node-solana -# FTS_THRESHOLD: 1700 -# FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} -# FTS_IMAGE: neonlabsorg/full_test_suite:583-full-test-suite - -# - wait + - wait -# - label: ":floppy_disk: publish image" -# command: ".buildkite/steps/publish-image.sh" -# if: | -# build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && -# (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) + - label: ":floppy_disk: publish image" + command: ".buildkite/steps/publish-image.sh" + if: | + build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && + (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - label: ":terraform: destroy infrastructure" agents: From ba6750e064e30f8f50d63bf34f517af93da35965 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 22:58:59 +0300 Subject: [PATCH 16/73] wip --- .buildkite/pipeline.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fa7462f70..51e1f07df 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -35,7 +35,10 @@ steps: - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - PROXY_URL="http://$$PROXY_ADDR:9091/solana" - FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - - SOLANA_URL="http://$$SOLANA_ADDR:8899" + - SOLANA_URL="http://$$SOLANA_ADDR:8899" + - echo $$PROXY_URL + - echo $$FAUCET_URL + - echo $$SOLANA_URL - echo Full test suite container name - $${FTS_CONTAINER_NAME} - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) From 332c031db5879dbfb3c7fd611cb0cdf2b71c1e89 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 11 Mar 2022 23:04:28 +0300 Subject: [PATCH 17/73] wip --- .buildkite/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 51e1f07df..c603f0a97 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -33,9 +33,9 @@ steps: commands: - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - - PROXY_URL="http://$$PROXY_ADDR:9091/solana" - - FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - - SOLANA_URL="http://$$SOLANA_ADDR:8899" + - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" + - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" + - export SOLANA_URL="http://$$SOLANA_ADDR:8899" - echo $$PROXY_URL - echo $$FAUCET_URL - echo $$SOLANA_URL From 9c9447bed20e18527568131181cce63ab1ba05e1 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Sat, 12 Mar 2022 11:43:55 +0300 Subject: [PATCH 18/73] wip --- .buildkite/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e2963fa04..0d92387ac 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,9 +27,9 @@ steps: - label: ":coverage: full test suite (FTS)" - if: | - (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || - (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") +# if: | +# (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || +# (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") commands: - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` From 991cc46714f319426e93aacaed06a8cce93efb35 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Sat, 12 Mar 2022 12:01:57 +0300 Subject: [PATCH 19/73] wip --- .buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0d92387ac..8453bb797 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -72,3 +72,4 @@ steps: queue: "testing" command: - "tf/destroy.sh" + allow_dependency_failure: true From fde5590a0c0c15e1805bac6927e166dbee87feda Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Sat, 12 Mar 2022 13:05:37 +0300 Subject: [PATCH 20/73] wip --- .buildkite/pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8453bb797..1b398efee 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,6 +27,7 @@ steps: - label: ":coverage: full test suite (FTS)" + key: "full_tests" # if: | # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") @@ -72,4 +73,5 @@ steps: queue: "testing" command: - "tf/destroy.sh" + depends_on: "full_tests" allow_dependency_failure: true From 331a33e4a3cbec9283094b346fcf494a2728edbe Mon Sep 17 00:00:00 2001 From: Vasiliy Zaznobin <82812108+vasiliy-zaznobin@users.noreply.github.com> Date: Sat, 12 Mar 2022 18:21:28 +0300 Subject: [PATCH 21/73] #368 move sql schema to distinct file (#586) * Introduce proxy/db_scheme.py * Debug * Introduce run-db-creation.sh * Use entrypoint: proxy/run-db-creation.sh * Add dbcreation.log * Debug * Debug * Add dbcreation.log * Use from ..proxy * Debug * Debug * Introduce dbcreation_mode to run proxy * Change paths * Will run in dbcreation mode with SOLANA_URL and EVM_LOADER * Debug * Debug * Debug * SOLANA_URL: http://solana:8899 * Add logged_groups * Debug * Log schema * Use psql * Add postgresql-client-common in the docker image * Add postgresql-client in the docker image * Set network_mode: container:postgres * Remove networks * Use network_mode: service:postgres * Remove hostname * Remove hostname * ports: - "5432:5432" * networks: - net * Add ports: - "5432" * Add networks: - net * Try to use links: - postgres * Use export PGPASSWORD=${POSTGRES_PASSWORD} to set the password for psql Use psql -h ${POSTGRES_HOST} ... * Use run_dbcreation() * Correction after the prev merge * Correction after the prev merge * Lowercase in function names * Refactor to introduce scheme.sql * Debug * Debug * Debug * Add table airdrop_scheduled Add table constants * Revert some changes * Fixed: psql:proxy/db/scheme.sql:73: ERROR: syntax error at or near "{" LINE 1: ...IF NOT EXISTS neon_transaction_logs_block_hash ON {table_nam... * Use run_indexer(solana_url) * Use run_airdropper with 7 args * Remove creation.py * Remove creation.py * Remove dbcreation_mode * check it * check it * Catch "Operator has NO resources!" exception * Catch "Operator has NO resources!" exception * Get rid of create_table in sql_dict.py * Get rid of stuff from sql_dict * Get rid of crating table stuff from TrxReceiptsStorage * Roll back NEON_ERC20_MAX_AMOUNT NEON_ETH_MAX_AMOUNT * Spit and polish Co-authored-by: rozhkovdmitrii --- .buildkite/pipeline.yml | 1 + .buildkite/steps/deploy-test.sh | 1 + Dockerfile | 2 +- proxy/__main__.py | 9 +- proxy/common_neon/costs.py | 18 +-- proxy/db/scheme.sql | 148 +++++++++++++++++++++ proxy/docker-compose-test.yml | 28 ++++ proxy/indexer/accounts_db.py | 15 +-- proxy/indexer/airdropper.py | 28 +--- proxy/indexer/base_db.py | 11 +- proxy/indexer/blocks_db.py | 18 +-- proxy/indexer/logs_db.py | 24 +--- proxy/indexer/sql_dict.py | 20 +-- proxy/indexer/transactions_db.py | 53 +------- proxy/indexer/trx_receipts_storage.py | 13 +- proxy/run-dbcreation.sh | 16 +++ proxy/testing/test_neon_tx_sender.py | 64 +++++---- proxy/testing/test_trx_receipts_storage.py | 31 +++-- 18 files changed, 267 insertions(+), 233 deletions(-) create mode 100644 proxy/db/scheme.sql create mode 100755 proxy/run-dbcreation.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5ac8aa1a5..0a67fd002 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -16,6 +16,7 @@ steps: - "solana.log" - "measurements.log" - "evm_loader.log" + - "dbcreation.log" - "faucet.log" - "airdropper.log" - "indexer.log" diff --git a/.buildkite/steps/deploy-test.sh b/.buildkite/steps/deploy-test.sh index d2f386ed3..295e7a4d8 100755 --- a/.buildkite/steps/deploy-test.sh +++ b/.buildkite/steps/deploy-test.sh @@ -44,6 +44,7 @@ function cleanup_docker { if docker logs solana >solana.log 2>&1; then echo "solana logs saved"; fi if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi + if docker logs dbcreation >dbcreation.log 2>&1; then echo "dbcreation logs saved"; fi if docker logs faucet >faucet.log 2>&1; then echo "faucet logs saved"; fi if docker logs airdropper >airdropper.log 2>&1; then echo "airdropper logs saved"; fi if docker logs indexer >indexer.log 2>&1; then echo "indexer logs saved"; fi diff --git a/Dockerfile b/Dockerfile index b2308b668..a4ecea4a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /opt RUN apt update && \ DEBIAN_FRONTEND=noninteractive apt install -y git software-properties-common openssl curl parallel \ - ca-certificates python3-pip python3-venv && \ + ca-certificates python3-pip python3-venv postgresql-client && \ python3 -m venv venv && \ pip3 install --upgrade pip && \ /bin/bash -c "source venv/bin/activate" && \ diff --git a/proxy/__main__.py b/proxy/__main__.py index 41ad6dc43..bdee15917 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -16,11 +16,15 @@ from .indexer.indexer import run_indexer if __name__ == '__main__': + solana_url = os.environ['SOLANA_URL'] + evm_loader_id = os.environ['EVM_LOADER'] + print(f"Will run with SOLANA_URL={solana_url}; EVM_LOADER={evm_loader_id}") + airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True'] indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True'] + if airdropper_mode: print("Will run in airdropper mode") - solana_url = os.environ['SOLANA_URL'] pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT']) faucet_url = os.environ['FAUCET_URL'] wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'] @@ -40,9 +44,6 @@ max_conf) elif indexer_mode: print("Will run in indexer mode") - - solana_url = os.environ['SOLANA_URL'] - run_indexer(solana_url) else: entry_point() diff --git a/proxy/common_neon/costs.py b/proxy/common_neon/costs.py index 17ecf518c..66f461b88 100644 --- a/proxy/common_neon/costs.py +++ b/proxy/common_neon/costs.py @@ -6,23 +6,7 @@ class SQLCost(BaseDB): def __init__(self): - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - self._table_name = 'OPERATOR_COST' - return f""" - CREATE TABLE IF NOT EXISTS {self._table_name} ( - id SERIAL PRIMARY KEY, - hash char(64), - cost bigint, - used_gas bigint, - sender char(40), - to_address char(40) , - sig char(100), - status varchar(100), - reason varchar(100) - ); - """ + BaseDB.__init__(self, 'OPERATOR_COST') def insert(self, hash, cost, used_gas, sender, to_address, sig, status, reason): with self._conn.cursor() as cur: diff --git a/proxy/db/scheme.sql b/proxy/db/scheme.sql new file mode 100644 index 000000000..31675598a --- /dev/null +++ b/proxy/db/scheme.sql @@ -0,0 +1,148 @@ + CREATE TABLE IF NOT EXISTS constants ( + key TEXT UNIQUE, + value BYTEA + ); + + CREATE TABLE IF NOT EXISTS airdrop_scheduled ( + key TEXT UNIQUE, + value BYTEA + ); + + CREATE TABLE IF NOT EXISTS OPERATOR_COST ( + id SERIAL PRIMARY KEY, + hash char(64), + cost bigint, + used_gas bigint, + sender char(40), + to_address char(40) , + sig char(100), + status varchar(100), + reason varchar(100) + ); + + CREATE TABLE IF NOT EXISTS neon_accounts ( + neon_account CHAR(42), + pda_account VARCHAR(50), + code_account VARCHAR(50), + slot BIGINT, + code TEXT, + + UNIQUE(pda_account, code_account) + ); + + CREATE TABLE IF NOT EXISTS failed_airdrop_attempts ( + attempt_time BIGINT, + eth_address TEXT, + reason TEXT + ); + CREATE INDEX IF NOT EXISTS failed_attempt_time_idx ON failed_airdrop_attempts (attempt_time); + + CREATE TABLE IF NOT EXISTS airdrop_ready ( + eth_address TEXT UNIQUE, + scheduled_ts BIGINT, + finished_ts BIGINT, + duration INTEGER, + amount_galans INTEGER + ); + + CREATE TABLE IF NOT EXISTS solana_block ( + slot BIGINT, + hash CHAR(66), + + parent_hash CHAR(66), + blocktime BIGINT, + signatures BYTEA, + + UNIQUE(slot), + UNIQUE(hash) + ); + + CREATE TABLE IF NOT EXISTS neon_transaction_logs ( + address CHAR(42), + blockHash CHAR(66), + blockNumber BIGINT, + + transactionHash CHAR(66), + transactionLogIndex INT, + topic TEXT, + + json TEXT, + + UNIQUE(blockNumber, transactionHash, transactionLogIndex) + ); + CREATE INDEX IF NOT EXISTS neon_transaction_logs_block_hash ON neon_transaction_logs(blockHash); + CREATE INDEX IF NOT EXISTS neon_transaction_logs_address ON neon_transaction_logs(address); + CREATE INDEX IF NOT EXISTS neon_transaction_logs_topic ON neon_transaction_logs(topic); + + CREATE TABLE IF NOT EXISTS solana_neon_transactions ( + sol_sign CHAR(88), + neon_sign CHAR(66), + slot BIGINT, + idx INT, + + UNIQUE(sol_sign, neon_sign, idx), + UNIQUE(neon_sign, sol_sign, idx) + ); + + CREATE TABLE IF NOT EXISTS neon_transactions ( + neon_sign CHAR(66), + from_addr CHAR(42), + sol_sign CHAR(88), + slot BIGINT, + block_hash CHAR(66), + idx INT, + + nonce VARCHAR, + gas_price VARCHAR, + gas_limit VARCHAR, + value VARCHAR, + gas_used VARCHAR, + + to_addr CHAR(42), + contract CHAR(42), + + status CHAR(3), + + return_value TEXT, + + v TEXT, + r TEXT, + s TEXT, + + calldata TEXT, + logs BYTEA, + + UNIQUE(neon_sign), + UNIQUE(sol_sign, idx) + ); + + CREATE TABLE IF NOT EXISTS transaction_receipts ( + slot BIGINT, + signature VARCHAR(88), + trx BYTEA, + PRIMARY KEY (slot, signature) + ); + + CREATE TABLE IF NOT EXISTS constants ( + key TEXT UNIQUE, + value BYTEA + ) + + CREATE TABLE IF NOT EXISTS airdrop_scheduled ( + key TEXT UNIQUE, + value BYTEA + ) + + CREATE TABLE IF NOT EXISTS transaction_receipts ( + slot BIGINT, + signature VARCHAR(88), + trx BYTEA, + PRIMARY KEY (slot, signature) + ); + + CREATE TABLE IF NOT EXISTS test_storage ( + slot BIGINT, + signature VARCHAR(88), + trx BYTEA, + PRIMARY KEY (slot, signature) + ); diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index fe20142c3..a60207ee9 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -8,6 +8,8 @@ services: SOLANA_URL: http://solana:8899 RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug hostname: solana + ports: + - 127.0.0.1:8899:8899 expose: - "8899" - "9900" @@ -53,9 +55,29 @@ services: start_period: 5s expose: - "5432" + ports: + - "5432" networks: - net + dbcreation: + container_name: dbcreation + image: neonlabsorg/proxy:${REVISION} + environment: + SOLANA_URL: http://solana:8899 + POSTGRES_DB: neon-db + POSTGRES_USER: neon-proxy + POSTGRES_PASSWORD: neon-proxy-pass + POSTGRES_HOST: postgres + entrypoint: proxy/run-dbcreation.sh + networks: + - net + depends_on: + postgres: + condition: service_healthy + evm_loader: + condition: service_completed_successfully + proxy: container_name: proxy image: neonlabsorg/proxy:${REVISION} @@ -77,6 +99,8 @@ services: OPERATOR_GAS_ACCOUNTS: 0x8966Ef2ae7A109Fd0977F5151b4607dc42929fBD;0x619d670152103a972B67a45b9Be764FF11979E4E hostname: proxy depends_on: + dbcreation: + condition: service_completed_successfully postgres: condition: service_healthy evm_loader: @@ -143,6 +167,8 @@ services: depends_on: postgres: condition: service_healthy + dbcreation: + condition: service_completed_successfully faucet: condition: service_started @@ -161,6 +187,8 @@ services: condition: service_healthy evm_loader: condition: service_completed_successfully + dbcreation: + condition: service_completed_successfully networks: - net entrypoint: proxy/run-indexer.sh diff --git a/proxy/indexer/accounts_db.py b/proxy/indexer/accounts_db.py index 938791964..d0c5fd8d9 100644 --- a/proxy/indexer/accounts_db.py +++ b/proxy/indexer/accounts_db.py @@ -16,20 +16,7 @@ def __str__(self): class NeonAccountDB(BaseDB): def __init__(self): - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - self._table_name = 'neon_accounts' - return f""" - CREATE TABLE IF NOT EXISTS {self._table_name} ( - neon_account CHAR(42), - pda_account VARCHAR(50), - code_account VARCHAR(50), - slot BIGINT, - code TEXT, - - UNIQUE(pda_account, code_account) - );""" + BaseDB.__init__(self, 'neon_accounts') def set_acc_by_request(self, neon_account: str, pda_account: str, code_account: str, code: str): with self._conn.cursor() as cursor: diff --git a/proxy/indexer/airdropper.py b/proxy/indexer/airdropper.py index cf639e3af..08c3088a2 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/indexer/airdropper.py @@ -10,6 +10,7 @@ from datetime import datetime from decimal import Decimal from logged_groups import logged_group + from ..environment import NEON_PRICE_USD, EVM_LOADER_ID from ..common_neon.solana_interactor import SolanaInteractor @@ -20,18 +21,7 @@ class FailedAttempts(BaseDB): def __init__(self) -> None: - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - self._table_name = 'failed_airdrop_attempts' - return f''' - CREATE TABLE IF NOT EXISTS {self._table_name} ( - attempt_time BIGINT, - eth_address TEXT, - reason TEXT - ); - CREATE INDEX IF NOT EXISTS failed_attempt_time_idx ON {self._table_name} (attempt_time); - ''' + BaseDB.__init__(self, 'failed_airdrop_attempts') def airdrop_failed(self, eth_address, reason): with self._conn.cursor() as cur: @@ -43,19 +33,7 @@ def airdrop_failed(self, eth_address, reason): class AirdropReadySet(BaseDB): def __init__(self): - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - self._table_name = 'airdrop_ready' - return f''' - CREATE TABLE IF NOT EXISTS {self._table_name} ( - eth_address TEXT UNIQUE, - scheduled_ts BIGINT, - finished_ts BIGINT, - duration INTEGER, - amount_galans INTEGER - ) - ''' + BaseDB.__init__(self, 'airdrop_ready') def register_airdrop(self, eth_address: str, airdrop_info: dict): finished = int(datetime.now().timestamp()) diff --git a/proxy/indexer/base_db.py b/proxy/indexer/base_db.py index dc6d86fde..5a0ca085d 100644 --- a/proxy/indexer/base_db.py +++ b/proxy/indexer/base_db.py @@ -23,9 +23,9 @@ class DBQueryExpression(NamedTuple): @logged_group("neon.Indexer") class BaseDB: - _create_table_lock = multiprocessing.Lock() - def __init__(self): + def __init__(self, table_name): + self._table_name = table_name self._conn = psycopg2.connect( dbname=POSTGRES_DB, user=POSTGRES_USER, @@ -34,13 +34,6 @@ def __init__(self): ) self._conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - with self._create_table_lock: - cursor = self._conn.cursor() - cursor.execute(self._create_table_sql()) - - def _create_table_sql(self) -> str: - assert False, 'No script for the table' - def _build_expression(self, q: DBQuery) -> DBQueryExpression: return DBQueryExpression( diff --git a/proxy/indexer/blocks_db.py b/proxy/indexer/blocks_db.py index b96c18613..921060fa1 100644 --- a/proxy/indexer/blocks_db.py +++ b/proxy/indexer/blocks_db.py @@ -6,26 +6,10 @@ class SolanaBlocksDB(BaseDB): def __init__(self): - BaseDB.__init__(self) + BaseDB.__init__(self, 'solana_block') self._column_lst = ('slot', 'hash') self._full_column_lst = ('slot', 'hash', 'parent_hash', 'blocktime', 'signatures') - def _create_table_sql(self) -> str: - self._table_name = 'solana_block' - return f""" - CREATE TABLE IF NOT EXISTS {self._table_name} ( - slot BIGINT, - hash CHAR(66), - - parent_hash CHAR(66), - blocktime BIGINT, - signatures BYTEA, - - UNIQUE(slot), - UNIQUE(hash) - ); - """ - def _block_from_value(self, slot: Optional[int], values: []) -> SolanaBlockInfo: if not values: return SolanaBlockInfo(slot=slot) diff --git a/proxy/indexer/logs_db.py b/proxy/indexer/logs_db.py index 984b67019..ea8c510c0 100644 --- a/proxy/indexer/logs_db.py +++ b/proxy/indexer/logs_db.py @@ -1,31 +1,11 @@ import json + from ..indexer.base_db import BaseDB class LogsDB(BaseDB): def __init__(self): - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - self._table_name = 'neon_transaction_logs' - return f""" - CREATE TABLE IF NOT EXISTS {self._table_name} ( - address CHAR(42), - blockHash CHAR(66), - blockNumber BIGINT, - - transactionHash CHAR(66), - transactionLogIndex INT, - topic TEXT, - - json TEXT, - - UNIQUE(blockNumber, transactionHash, transactionLogIndex) - ); - CREATE INDEX IF NOT EXISTS {self._table_name}_block_hash ON {self._table_name}(blockHash); - CREATE INDEX IF NOT EXISTS {self._table_name}_address ON {self._table_name}(address); - CREATE INDEX IF NOT EXISTS {self._table_name}_topic ON {self._table_name}(topic); - """ + BaseDB.__init__(self, 'neon_transaction_logs') def push_logs(self, logs, block): rows = [] diff --git a/proxy/indexer/sql_dict.py b/proxy/indexer/sql_dict.py index 6da462c3c..8576ba4d1 100644 --- a/proxy/indexer/sql_dict.py +++ b/proxy/indexer/sql_dict.py @@ -6,24 +6,12 @@ class SQLDict(MutableMapping, BaseDB): """Serialize an object using pickle to a binary format accepted by SQLite.""" - def __init__(self, tablename='table', bin_key=False): - self.bin_key = bin_key + def __init__(self, tablename='table'): self.encode = encode self.decode = decode - self.key_encode = encode if self.bin_key else dummy - self.key_decode = decode if self.bin_key else dummy - self._table_name = tablename + ("_bin_key" if self.bin_key else "") - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - key_type = 'BYTEA' if self.bin_key else 'TEXT' - return f''' - CREATE TABLE IF NOT EXISTS - {self._table_name} ( - key {key_type} UNIQUE, - value BYTEA - ) - ''' + self.key_encode = dummy + self.key_decode = dummy + BaseDB.__init__(self, tablename) def __len__(self): with self._conn.cursor() as cur: diff --git a/proxy/indexer/transactions_db.py b/proxy/indexer/transactions_db.py index 24a7b5136..3f0609e96 100644 --- a/proxy/indexer/transactions_db.py +++ b/proxy/indexer/transactions_db.py @@ -7,20 +7,7 @@ class SolanaNeonTxsDB(BaseDB): def __init__(self): - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - self._table_name = 'solana_neon_transactions' - return f""" - CREATE TABLE IF NOT EXISTS {self._table_name} ( - sol_sign CHAR(88), - neon_sign CHAR(66), - slot BIGINT, - idx INT, - - UNIQUE(sol_sign, neon_sign, idx), - UNIQUE(neon_sign, sol_sign, idx) - );""" + BaseDB.__init__(self, 'solana_neon_transactions') def set_txs(self, neon_sign: str, used_ixs: [SolanaIxSignInfo]): @@ -38,48 +25,12 @@ def set_txs(self, neon_sign: str, used_ixs: [SolanaIxSignInfo]): class NeonTxsDB(BaseDB): def __init__(self): - BaseDB.__init__(self) + BaseDB.__init__(self, 'neon_transactions') self._column_lst = ('neon_sign', 'from_addr', 'sol_sign', 'slot', 'block_hash', 'idx', 'nonce', 'gas_price', 'gas_limit', 'to_addr', 'contract', 'value', 'calldata', 'v', 'r', 's', 'status', 'gas_used', 'return_value', 'logs') self._sol_neon_txs_db = SolanaNeonTxsDB() - def _create_table_sql(self) -> str: - self._table_name = 'neon_transactions' - return f""" - CREATE TABLE IF NOT EXISTS {self._table_name} ( - neon_sign CHAR(66), - from_addr CHAR(42), - sol_sign CHAR(88), - slot BIGINT, - block_hash CHAR(66), - idx INT, - - nonce VARCHAR, - gas_price VARCHAR, - gas_limit VARCHAR, - value VARCHAR, - gas_used VARCHAR, - - to_addr CHAR(42), - contract CHAR(42), - - status CHAR(3), - - return_value TEXT, - - v TEXT, - r TEXT, - s TEXT, - - calldata TEXT, - logs BYTEA, - - UNIQUE(neon_sign), - UNIQUE(sol_sign, idx) - ); - """ - def _tx_from_value(self, value) -> Optional[NeonTxFullInfo]: if not value: return None diff --git a/proxy/indexer/trx_receipts_storage.py b/proxy/indexer/trx_receipts_storage.py index 3c4354b75..e9a6afe73 100644 --- a/proxy/indexer/trx_receipts_storage.py +++ b/proxy/indexer/trx_receipts_storage.py @@ -4,18 +4,7 @@ class TrxReceiptsStorage(BaseDB): def __init__(self, table_name): - self._table_name = table_name - BaseDB.__init__(self) - - def _create_table_sql(self) -> str: - return f''' - CREATE TABLE IF NOT EXISTS {self._table_name} ( - slot BIGINT, - signature VARCHAR(88), - trx BYTEA, - PRIMARY KEY (slot, signature) - ); - ''' + BaseDB.__init__(self, table_name) def clear(self): with self._conn.cursor() as cur: diff --git a/proxy/run-dbcreation.sh b/proxy/run-dbcreation.sh new file mode 100755 index 000000000..fbef77700 --- /dev/null +++ b/proxy/run-dbcreation.sh @@ -0,0 +1,16 @@ +#!/bin/bash +COMPONENT="dbcreation" +echo "$(date "+%F %X.%3N") I $(basename "$0"):${LINENO} $$ ${COMPONENT}:StartScript {} Start ${COMPONENT} service" + +if [ -z "$EVM_LOADER" ]; then + echo "$(date "+%F %X.%3N") I $(basename "$0"):${LINENO} $$ ${COMPONENT}:StartScript {} Extracting EVM_LOADER address from keypair file..." + export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) + echo "$(date "+%F %X.%3N") I $(basename "$0"):${LINENO} $$ ${COMPONENT}:StartScript {} EVM_LOADER=$EVM_LOADER" +fi + +echo "$(date "+%F %X.%3N") I $(basename $0):${LINENO} $$ ${COMPONENT}:StartScript {} dbcreation" + +export PGPASSWORD=${POSTGRES_PASSWORD} +psql -h ${POSTGRES_HOST} ${POSTGRES_DB} ${POSTGRES_USER} -a -f proxy/db/scheme.sql +psql -h ${POSTGRES_HOST} ${POSTGRES_DB} ${POSTGRES_USER} --command "\\dt+ public.*" +psql -h ${POSTGRES_HOST} ${POSTGRES_DB} ${POSTGRES_USER} --command "\\d+ public.*" diff --git a/proxy/testing/test_neon_tx_sender.py b/proxy/testing/test_neon_tx_sender.py index 65114aa48..5e3b5adef 100644 --- a/proxy/testing/test_neon_tx_sender.py +++ b/proxy/testing/test_neon_tx_sender.py @@ -1,14 +1,16 @@ import os import unittest -from solana.rpc.api import Client as SolanaClient + +import logged_groups from unittest.mock import Mock -from proxy.common_neon.eth_proto import Trx as EthTrx -from proxy.common_neon.transaction_sender import NeonTxSender -from proxy.common_neon.solana_interactor import SolanaInteractor -from proxy.memdb.memdb import MemDB +from ..common_neon.eth_proto import Trx as EthTrx +from ..common_neon.transaction_sender import NeonTxSender +from ..common_neon.solana_interactor import SolanaInteractor +from ..memdb.memdb import MemDB +@logged_groups.logged_group("neon.TestCases") class TestNeonTxSender(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -16,19 +18,19 @@ def setUpClass(cls) -> None: def setUp(self) -> None: trx = EthTrx.fromString(bytearray.fromhex('f8678080843ade68b194f0dafe87532d4373453b2555c644390e1b99e84c8459682f0080820102a00193e1966a82c5597942370980fb78080901ca86eb3c1b25ec600b2760cfcc94a03efcc1169e161f9a148fd4586e0bcf880648ca74075bfa7a9acc8800614fc9ff')) - self.testee = NeonTxSender(MemDB(self.solana), self.solana, trx, 500) - self.testee._resource_list.free_resource_info() - self.testee._validate_pend_tx = Mock() - self.testee._validate_whitelist = Mock() - self.testee._validate_tx_count = Mock() - self.testee._validate_pend_tx.side_effect = [None] - self.testee._validate_whitelist.side_effect = [None] - self.testee._validate_tx_count.side_effect = [None] - self.testee._resource_list._min_operator_balance_to_warn = Mock() - self.testee._resource_list._min_operator_balance_to_err = Mock() + self.neon_tx_sender = NeonTxSender(MemDB(self.solana), self.solana, trx, 500) + self.neon_tx_sender._resource_list.free_resource_info() + self.neon_tx_sender._validate_pend_tx = Mock() + self.neon_tx_sender._validate_whitelist = Mock() + self.neon_tx_sender._validate_tx_count = Mock() + self.neon_tx_sender._validate_pend_tx.side_effect = [None] + self.neon_tx_sender._validate_whitelist.side_effect = [None] + self.neon_tx_sender._validate_tx_count.side_effect = [None] + self.neon_tx_sender._resource_list._min_operator_balance_to_warn = Mock() + self.neon_tx_sender._resource_list._min_operator_balance_to_err = Mock() def tearDown(self) -> None: - self.testee._resource_list.free_resource_info() + self.neon_tx_sender._resource_list.free_resource_info() # @unittest.skip("a.i.") def test_01_validate_execution_when_not_enough_sols(self): @@ -38,12 +40,12 @@ def test_01_validate_execution_when_not_enough_sols(self): then an error is returned to the client who requested the execution of the transaction and an error is written to the log. """ - self.testee._resource_list.reset() - self.testee._resource_list._min_operator_balance_to_warn.side_effect = [1_049_000_000 * 1_000_000_000 * 1_000_000_000 * 2, 1_000_000_000 * 2] - self.testee._resource_list._min_operator_balance_to_err.side_effect = [1_049_000_000 * 1_000_000_000 * 1_000_000_000, 1_000_000_000] + self.neon_tx_sender._resource_list.reset() + self.neon_tx_sender._resource_list._min_operator_balance_to_warn.side_effect = [1_049_000_000 * 1_000_000_000 * 1_000_000_000 * 2, 1_000_000_000 * 2] + self.neon_tx_sender._resource_list._min_operator_balance_to_err.side_effect = [1_049_000_000 * 1_000_000_000 * 1_000_000_000, 1_000_000_000] with self.assertLogs('neon', level='ERROR') as logs: - self.testee._validate_execution() + self.neon_tx_sender._validate_execution() print('logs.output:', str(logs.output)) self.assertRegex(str(logs.output), 'ERROR:neon.Proxy:Operator account [A-Za-z0-9]{40,}:[0-9]+ has NOT enough SOLs; balance = [0-9]+; min_operator_balance_to_err = 1049000000000000000000000000') @@ -54,13 +56,13 @@ def test_02_validate_warning_when_little_sols(self): the value of the variable MIN_OPERATOR_BALANCE_TO_WARN or less, then a warning is written to the log.: """ - self.testee._resource_list.reset() - self.testee._resource_list._min_operator_balance_to_warn.side_effect = [1_049_000_000 * 1_000_000_000 * 1_000_000_000, 1_000_000_000 * 2] - self.testee._resource_list._min_operator_balance_to_err.side_effect = [1_049_049_000, 1_000_000_000] + self.neon_tx_sender._resource_list.reset() + self.neon_tx_sender._resource_list._min_operator_balance_to_warn.side_effect = [1_049_000_000 * 1_000_000_000 * 1_000_000_000, 1_000_000_000 * 2] + self.neon_tx_sender._resource_list._min_operator_balance_to_err.side_effect = [1_049_049_000, 1_000_000_000] with self.assertLogs('neon', level='WARNING') as logs: # self.testee._resource_list.free_resource_info() - self.testee._validate_execution() + self.neon_tx_sender._validate_execution() print('logs.output:', str(logs.output)) self.assertRegex(str(logs.output), 'WARNING:neon.Proxy:Operator account [A-Za-z0-9]{40,}:[0-9]+ SOLs are running out; balance = [0-9]+; min_operator_balance_to_warn = 1049000000000000000000000000; min_operator_balance_to_err = 1049049000;') @@ -73,13 +75,17 @@ def test_03_validate_execution_when_not_enough_sols_for_all_operator_accounts(se who requested the execution of the transaction and an error is written to the log. """ - self.testee._resource_list.reset() - self.testee._resource_list._min_operator_balance_to_warn.return_value = 1_049_000_000 * 1_000_000_000 * 1_000_000_000 * 2 - self.testee._resource_list._min_operator_balance_to_err.return_value = 1_049_000_000 * 1_000_000_000 * 1_000_000_000 + self.neon_tx_sender._resource_list.reset() + self.neon_tx_sender._resource_list._min_operator_balance_to_warn.return_value = 1_049_000_000 * 1_000_000_000 * 1_000_000_000 * 2 + self.neon_tx_sender._resource_list._min_operator_balance_to_err.return_value = 1_049_000_000 * 1_000_000_000 * 1_000_000_000 with self.assertLogs('neon', level='ERROR') as logs: - # with self.assertRaises(RuntimeError): - self.testee._validate_execution() + try: + self.neon_tx_sender._validate_execution() + except RuntimeError: + # TODO: get rid of this eventual raising. Look at https://github.com/neonlabsorg/proxy-model.py/issues/629 + self.error("NeonTxSender has raised eventual (flaky) RuntimeError: `Operator has NO resources!` error") + print('logs.output:', str(logs.output)) self.assertRegex(str(logs.output), 'ERROR:neon.Proxy:Operator account [A-Za-z0-9]{40,}:[0-9]+ has NOT enough SOLs; balance = [0-9]+; min_operator_balance_to_err = 1049000000000000000000000000') diff --git a/proxy/testing/test_trx_receipts_storage.py b/proxy/testing/test_trx_receipts_storage.py index e14b71956..bd141516d 100644 --- a/proxy/testing/test_trx_receipts_storage.py +++ b/proxy/testing/test_trx_receipts_storage.py @@ -7,14 +7,13 @@ class TestTrxReceiptsStorage(TestCase): @classmethod def setUpClass(cls) -> None: - print("\n\nhttps://github.com/neonlabsorg/proxy-model.py/issues/421") - cls.testee = TrxReceiptsStorage('test_storage') + cls.trx_receipts_storage = TrxReceiptsStorage('test_storage') def create_signature(self): signature = b'' for i in range(0, 5): signature += randint(0, 255).to_bytes(1, byteorder='big') - return b58encode(signature).decode("utf-8") + return b58encode(signature).decode("utf-8") def create_slot_sig(self, max_slot): slot = randint(0, max_slot) @@ -24,9 +23,9 @@ def test_data_consistency(self): """ Test that data put into container is stored there """ - self.testee.clear() - self.assertEqual(self.testee.size(), 0) - self.assertEqual(self.testee.max_known_trx(), (0, None)) + self.trx_receipts_storage.clear() + self.assertEqual(self.trx_receipts_storage.size(), 0) + self.assertEqual(self.trx_receipts_storage.max_known_trx(), (0, None)) max_slot = 10 num_items = 100 @@ -34,20 +33,20 @@ def test_data_consistency(self): for _ in range(0, num_items): slot, signature = self.create_slot_sig(max_slot) trx = { 'slot': slot, 'signature': signature } - self.testee.add_trx(slot, signature, trx) + self.trx_receipts_storage.add_trx(slot, signature, trx) expected_items.append((slot, signature, trx)) - self.assertEqual(self.testee.max_known_trx()[0], max_slot) - self.assertEqual(self.testee.size(), num_items) + self.assertEqual(self.trx_receipts_storage.max_known_trx()[0], max_slot) + self.assertEqual(self.trx_receipts_storage.size(), num_items) for item in expected_items: - self.assertTrue(self.testee.contains(item[0], item[1])) + self.assertTrue(self.trx_receipts_storage.contains(item[0], item[1])) def test_query(self): """ Test get_trxs method workds as expected """ - self.testee.clear() - self.assertEqual(self.testee.size(), 0) + self.trx_receipts_storage.clear() + self.assertEqual(self.trx_receipts_storage.size(), 0) max_slot = 50 num_items = 100 @@ -55,17 +54,17 @@ def test_query(self): for _ in range(0, num_items): slot, signature = self.create_slot_sig(max_slot) trx = { 'slot': slot, 'signature': signature } - self.testee.add_trx(slot, signature, trx) + self.trx_receipts_storage.add_trx(slot, signature, trx) expected_items.append((slot, signature, trx)) start_slot = randint(0, 50) # query in ascending order - retrieved_trxs = [item for item in self.testee.get_trxs(start_slot, False)] + retrieved_trxs = [item for item in self.trx_receipts_storage.get_trxs(start_slot, False)] self.assertGreaterEqual(retrieved_trxs[0][0], start_slot) self.assertLessEqual(retrieved_trxs[-1][0], max_slot) # query in descending order - retrieved_trxs = [item for item in self.testee.get_trxs(start_slot, True)] + retrieved_trxs = [item for item in self.trx_receipts_storage.get_trxs(start_slot, True)] self.assertLessEqual(retrieved_trxs[0][0], max_slot) - self.assertGreaterEqual(retrieved_trxs[-1][0], start_slot) \ No newline at end of file + self.assertGreaterEqual(retrieved_trxs[-1][0], start_slot) From 5cbce09b1bce400f457377377b48891e7ad4a3c7 Mon Sep 17 00:00:00 2001 From: Rozhkov Dmitrii Date: Sat, 12 Mar 2022 22:43:10 +0500 Subject: [PATCH 22/73] #631 extract Airdropper class (#632) * Introduce proxy/db_scheme.py * Debug * Introduce run-db-creation.sh * Use entrypoint: proxy/run-db-creation.sh * Add dbcreation.log * Debug * Debug * Add dbcreation.log * Use from ..proxy * Debug * Debug * Introduce dbcreation_mode to run proxy * Change paths * Will run in dbcreation mode with SOLANA_URL and EVM_LOADER * Debug * Debug * Debug * SOLANA_URL: http://solana:8899 * Add logged_groups * Debug * Log schema * Use psql * Add postgresql-client-common in the docker image * Add postgresql-client in the docker image * Set network_mode: container:postgres * Remove networks * Use network_mode: service:postgres * Remove hostname * Remove hostname * ports: - "5432:5432" * networks: - net * Add ports: - "5432" * Add networks: - net * Try to use links: - postgres * Use export PGPASSWORD=${POSTGRES_PASSWORD} to set the password for psql Use psql -h ${POSTGRES_HOST} ... * Use run_dbcreation() * Correction after the prev merge * Correction after the prev merge * Lowercase in function names * Refactor to introduce scheme.sql * Debug * Debug * Debug * Add table airdrop_scheduled Add table constants * Revert some changes * Fixed: psql:proxy/db/scheme.sql:73: ERROR: syntax error at or near "{" LINE 1: ...IF NOT EXISTS neon_transaction_logs_block_hash ON {table_nam... * Use run_indexer(solana_url) * Use run_airdropper with 7 args * Remove creation.py * Remove creation.py * Remove dbcreation_mode * check it * check it * Catch "Operator has NO resources!" exception * Catch "Operator has NO resources!" exception * Get rid of create_table in sql_dict.py * Get rid of stuff from sql_dict * Get rid of crating table stuff from TrxReceiptsStorage * Roll back NEON_ERC20_MAX_AMOUNT NEON_ETH_MAX_AMOUNT * Spit and polish * AirdropperApp * Get rid off AIRDROPPER_MODE * Get rid off AIRDROPPER_MODE * Get rid off AIRDROPPER_MODE * spit and polish * spit and polish * spit and polish * spit and polish * spit and polish Co-authored-by: Vasiliy Zaznobin --- proxy/__main__.py | 24 +---------- proxy/airdropper/__init__.py | 2 + proxy/airdropper/__main__.py | 6 +++ proxy/{indexer => airdropper}/airdropper.py | 41 +++---------------- proxy/airdropper/airdropper_app.py | 45 +++++++++++++++++++++ proxy/indexer/__init__.py | 0 proxy/testing/test_airdropper.py | 2 +- run-airdropper.sh | 4 +- 8 files changed, 62 insertions(+), 62 deletions(-) create mode 100644 proxy/airdropper/__init__.py create mode 100644 proxy/airdropper/__main__.py rename proxy/{indexer => airdropper}/airdropper.py (89%) create mode 100644 proxy/airdropper/airdropper_app.py create mode 100644 proxy/indexer/__init__.py diff --git a/proxy/__main__.py b/proxy/__main__.py index bdee15917..8ddad90f9 100644 --- a/proxy/__main__.py +++ b/proxy/__main__.py @@ -9,10 +9,8 @@ :license: BSD, see LICENSE for more details. """ -from solana.publickey import PublicKey from .proxy import entry_point import os -from .indexer.airdropper import run_airdropper from .indexer.indexer import run_indexer if __name__ == '__main__': @@ -20,29 +18,9 @@ evm_loader_id = os.environ['EVM_LOADER'] print(f"Will run with SOLANA_URL={solana_url}; EVM_LOADER={evm_loader_id}") - airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True'] indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True'] - if airdropper_mode: - print("Will run in airdropper mode") - pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT']) - faucet_url = os.environ['FAUCET_URL'] - wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'] - if wrapper_whitelist != 'ANY': - wrapper_whitelist = wrapper_whitelist.split(',') - neon_decimals = int(os.environ.get('NEON_DECIMALS', '9')) - - pp_solana_url = os.environ.get('PP_SOLANA_URL', None) - max_conf = float(os.environ.get('MAX_CONFIDENCE_INTERVAL', 0.02)) - - run_airdropper(solana_url, - pyth_mapping_account, - faucet_url, - wrapper_whitelist, - neon_decimals, - pp_solana_url, - max_conf) - elif indexer_mode: + if indexer_mode: print("Will run in indexer mode") run_indexer(solana_url) else: diff --git a/proxy/airdropper/__init__.py b/proxy/airdropper/__init__.py new file mode 100644 index 000000000..4079bb6ea --- /dev/null +++ b/proxy/airdropper/__init__.py @@ -0,0 +1,2 @@ +from .airdropper import Airdropper, AIRDROP_AMOUNT_SOL, NEON_PRICE_USD +from .airdropper_app import AirdropperApp diff --git a/proxy/airdropper/__main__.py b/proxy/airdropper/__main__.py new file mode 100644 index 000000000..bb17d771f --- /dev/null +++ b/proxy/airdropper/__main__.py @@ -0,0 +1,6 @@ +from . import AirdropperApp + +if __name__ == '__main__': + airdropper_app = AirdropperApp() + exit_code = airdropper_app.run() + exit(exit_code) diff --git a/proxy/indexer/airdropper.py b/proxy/airdropper/airdropper.py similarity index 89% rename from proxy/indexer/airdropper.py rename to proxy/airdropper/airdropper.py index 08c3088a2..81f380106 100644 --- a/proxy/indexer/airdropper.py +++ b/proxy/airdropper/airdropper.py @@ -1,9 +1,4 @@ from solana.publickey import PublicKey -from proxy.indexer.indexer_base import IndexerBase -from proxy.indexer.pythnetwork import PythNetworkClient -from proxy.indexer.base_db import BaseDB -from proxy.indexer.utils import check_error -from proxy.indexer.sql_dict import SQLDict import requests import base58 import traceback @@ -13,6 +8,11 @@ from ..environment import NEON_PRICE_USD, EVM_LOADER_ID from ..common_neon.solana_interactor import SolanaInteractor +from ..indexer.indexer_base import IndexerBase +from ..indexer.pythnetwork import PythNetworkClient +from ..indexer.base_db import BaseDB +from ..indexer.utils import check_error +from ..indexer.sql_dict import SQLDict ACCOUNT_CREATION_PRICE_SOL = Decimal('0.00472692') @@ -271,34 +271,3 @@ def process_receipts(self): self.process_trx_airdropper_mode(trx) self.latest_processed_slot = max(self.latest_processed_slot, max_slot) self._constants['latest_processed_slot'] = self.latest_processed_slot - - -@logged_group("neon.Airdropper") -def run_airdropper(solana_url, - pyth_mapping_account: PublicKey, - faucet_url, - wrapper_whitelist = 'ANY', - neon_decimals = 9, - pp_solana_url = None, - max_conf = 0.1, *, logger): - logger.info(f"""Running indexer with params: - solana_url: {solana_url}, - evm_loader_id: {EVM_LOADER_ID}, - pyth.network mapping account: {pyth_mapping_account}, - faucet_url: {faucet_url}, - wrapper_whitelist: {wrapper_whitelist}, - NEON decimals: {neon_decimals}, - Price provider solana: {pp_solana_url}, - Max confidence interval: {max_conf}""") - - try: - airdropper = Airdropper(solana_url, - pyth_mapping_account, - faucet_url, - wrapper_whitelist, - neon_decimals, - pp_solana_url, - max_conf) - airdropper.run() - except Exception as err: - logger.error(f'Failed to start Airdropper: {err}') diff --git a/proxy/airdropper/airdropper_app.py b/proxy/airdropper/airdropper_app.py new file mode 100644 index 000000000..30d058a8b --- /dev/null +++ b/proxy/airdropper/airdropper_app.py @@ -0,0 +1,45 @@ +import os +from logged_groups import logged_group +from solana.publickey import PublicKey + +from ..environment import EVM_LOADER_ID + +from .airdropper import Airdropper + + +@logged_group("neon.Airdropper") +class AirdropperApp: + + def __init__(self): + self.info("Airdropper application is starting ...") + pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT']) + faucet_url = os.environ['FAUCET_URL'] + wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST'] + if wrapper_whitelist != 'ANY': + wrapper_whitelist = wrapper_whitelist.split(',') + neon_decimals = int(os.environ.get('NEON_DECIMALS', '9')) + + pp_solana_url = os.environ.get('PP_SOLANA_URL', None) + max_conf = float(os.environ.get('MAX_CONFIDENCE_INTERVAL', 0.02)) + solana_url = os.environ['SOLANA_URL'] + + self.info(f"""Construct Airdropper with params: + solana_url: {solana_url}, + evm_loader_id: {EVM_LOADER_ID}, + pyth.network mapping account: {pyth_mapping_account}, + faucet_url: {faucet_url}, + wrapper_whitelist: {wrapper_whitelist}, + NEON decimals: {neon_decimals}, + Price provider solana: {pp_solana_url}, + Max confidence interval: {max_conf}""") + + self._airdropper = Airdropper(solana_url, pyth_mapping_account, faucet_url, wrapper_whitelist, neon_decimals, + pp_solana_url, max_conf) + + def run(self) -> int: + try: + self._airdropper.run() + except Exception as err: + self.error(f'Failed to start Airdropper: {err}') + return 1 + return 0 diff --git a/proxy/indexer/__init__.py b/proxy/indexer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/proxy/testing/test_airdropper.py b/proxy/testing/test_airdropper.py index 264bcf803..a97aaa8a0 100644 --- a/proxy/testing/test_airdropper.py +++ b/proxy/testing/test_airdropper.py @@ -3,7 +3,7 @@ from solana.publickey import PublicKey from proxy.testing.mock_server import MockServer -from proxy.indexer.airdropper import Airdropper, AIRDROP_AMOUNT_SOL, NEON_PRICE_USD +from proxy.airdropper import Airdropper, AIRDROP_AMOUNT_SOL, NEON_PRICE_USD from proxy.indexer.sql_dict import SQLDict from proxy.common_neon.solana_interactor import SolanaInteractor import time diff --git a/run-airdropper.sh b/run-airdropper.sh index fcaf5716b..1636e0587 100755 --- a/run-airdropper.sh +++ b/run-airdropper.sh @@ -7,7 +7,7 @@ if [ -z "$EVM_LOADER" ]; then export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) echo "$(date "+%F %X.%3N") I $(basename "$0"):${LINENO} $$ ${COMPONENT}:StartScript {} EVM_LOADER=$EVM_LOADER" fi -export AIRDROPPER_MODE='true' + [[ -z "$FINALIZED" ]] && export FINALIZED="confirmed" -python3 -m proxy +python3 -m proxy.airdropper From 76b68aa7b7eb368eb5ff56f69ec51165530f90d5 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 08:16:37 +0300 Subject: [PATCH 23/73] debug --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1b398efee..23dcdf4a5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -73,5 +73,5 @@ steps: queue: "testing" command: - "tf/destroy.sh" - depends_on: "full_tests" - allow_dependency_failure: true +# depends_on: "full_tests" +# allow_dependency_failure: true From ecb4831b4e76dcff3bd279f2bb326917ef2efcbb Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 10:11:18 +0300 Subject: [PATCH 24/73] remove unused port --- proxy/docker-compose-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 77a460438..f13fb4a8e 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -8,8 +8,6 @@ services: SOLANA_URL: http://solana:8899 RUST_LOG: solana_runtime::system_instruction_processor=trace,solana_runtime::message_processor=debug,solana_bpf_loader=debug,solana_rbpf=debug hostname: solana - ports: - - 127.0.0.1:8899:8899 expose: - "8899" - "9900" From 92d207bd4dcad6c692d4d26fb6f361e543c28d89 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 10:49:49 +0300 Subject: [PATCH 25/73] wip --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4fb95eed7..a4ecea4a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG EVM_LOADER_REVISION=develop FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli -FROM neonlabsorg/evm_loader:${NEON_EVM_COMMIT} AS spl +FROM neonlabsorg/evm_loader:${EVM_LOADER_REVISION} AS spl FROM ubuntu:20.04 From 38edd9adaefb9e66e88ed156dd4f163115c31571 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 10:51:27 +0300 Subject: [PATCH 26/73] wip --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a4ecea4a4..86eac9bf8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG SOLANA_REVISION=v1.8.12-testnet -ARG EVM_LOADER_REVISION=develop +ARG EVM_LOADER_REVISION=latest FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli From c677de9027806b32be8508e929a91f1fdf27fdce Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 11:24:38 +0300 Subject: [PATCH 27/73] wip --- tf/output.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tf/output.tf b/tf/output.tf index 859b5d7bb..42fb8f370 100644 --- a/tf/output.tf +++ b/tf/output.tf @@ -6,3 +6,6 @@ output "proxy_ip" { value = aws_instance.proxy.public_ip } +output "branch" { + value = var.branch +} From 9e6af0645547af0893e4df66f3b785c960cb9b2f Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 11:25:56 +0300 Subject: [PATCH 28/73] wip --- tf/terraform.auto.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/terraform.auto.tfvars b/tf/terraform.auto.tfvars index 93f19e6fb..91fb576b2 100644 --- a/tf/terraform.auto.tfvars +++ b/tf/terraform.auto.tfvars @@ -1,4 +1,4 @@ -branch = "develop" +#branch = "develop" neon_evm_revision = "latest" proxy_model_revision = "latest" From 509d49b9e57e4ab3666509c2672a68728a844515 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 11:34:01 +0300 Subject: [PATCH 29/73] wip --- tf/proxy_init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 9261b26c7..3ce4b43e8 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -35,6 +35,7 @@ services: faucet: environment: - SOLANA_URL=$SOLANA_URL + - NEON_ETH_MAX_AMOUNT=50000 ports: - 3334:3333 airdropper: From c59129bcd60a4a414cac8aa1408ea6151d714ab6 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 12:04:49 +0300 Subject: [PATCH 30/73] wip --- tf/proxy_init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 3ce4b43e8..67312ec55 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -62,3 +62,5 @@ do done; docker-compose -f docker-compose-test.yml -f docker-compose-test.override.yml up -d $SERVICES + +docker rm -f solana From b2144cc5fec1341a3a60df386caf085cbff1fadd Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 12:25:11 +0300 Subject: [PATCH 31/73] change instance type --- tf/main.tf | 4 ++-- tf/terraform.auto.tfvars | 3 ++- tf/vars.tf | 6 +++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tf/main.tf b/tf/main.tf index 2f4c5ef1a..3ae1f660d 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -135,7 +135,7 @@ resource "aws_security_group" "test-stand-proxy" { resource "aws_instance" "solana" { - instance_type = var.instance_type + instance_type = var.solana_instance_type ami = var.ami key_name = data.aws_key_pair.ci-stands.key_name vpc_security_group_ids = [aws_security_group.test-stand-solana.id] @@ -156,7 +156,7 @@ resource "aws_instance" "solana" { } resource "aws_instance" "proxy" { - instance_type = var.instance_type + instance_type = var.proxy_instance_type ami = var.ami key_name = data.aws_key_pair.ci-stands.key_name vpc_security_group_ids = [aws_security_group.test-stand-proxy.id] diff --git a/tf/terraform.auto.tfvars b/tf/terraform.auto.tfvars index 91fb576b2..3ecd7f748 100644 --- a/tf/terraform.auto.tfvars +++ b/tf/terraform.auto.tfvars @@ -5,5 +5,6 @@ proxy_model_revision = "latest" aws_subnet = "subnet-19f77872" allow_list = ["172.31.0.0/16", "3.15.140.214/32", "18.219.70.113/32", "3.21.100.174/32", "3.137.181.30/32", "3.136.233.33/32", "142.132.171.62/32"] -instance_type = "t3.large" +solana_instance_type = "t3.large" +proxy_instance_type = "t3.xlarge" ami = "ami-0fb653ca2d3203ac1" diff --git a/tf/vars.tf b/tf/vars.tf index cd1865f5b..3210acccd 100644 --- a/tf/vars.tf +++ b/tf/vars.tf @@ -8,7 +8,11 @@ variable "allow_list" { type = list(string) } -variable "instance_type" { +variable "solana_instance_type" { + type = string +} + +variable "proxy_instance_type" { type = string } From f8a6c031ad1a6f79fa7642b6c4ff7203509af325 Mon Sep 17 00:00:00 2001 From: Anton Lisanin Date: Mon, 14 Mar 2022 14:00:57 +0300 Subject: [PATCH 32/73] Fix eth_estimageGas for Metamask transfer (#634) --- proxy/common_neon/estimate.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/proxy/common_neon/estimate.py b/proxy/common_neon/estimate.py index 354afd7a4..9da3f34c4 100644 --- a/proxy/common_neon/estimate.py +++ b/proxy/common_neon/estimate.py @@ -13,11 +13,20 @@ @logged_group("neon.Proxy") class GasEstimate: def __init__(self, request: dict, solana: SolanaInteractor): - self.sender = request.get('from', "0x0000000000000000000000000000000000000000")[2:] - self.contract = request.get('to', '0x')[2:] - self.value = request.get('value', '0x00') - self.data = request.get('data', '0x')[2:] + self.sender = request.get('from') or '0x0000000000000000000000000000000000000000' + if self.sender: + self.sender = self.sender[2:] + self.contract = request.get('to') or '' + if self.contract: + self.contract = self.contract[2:] + + self.data = request.get('data') or '' + if self.data: + self.data = self.data[2:] + + self.value = request.get('value') or '0x00' + self.solana = solana def execution_cost(self) -> int: @@ -76,10 +85,8 @@ def estimate(self): overhead = self.iterative_overhead_cost() gas = execution_cost + trx_size_cost + overhead + EXTRA_GAS - - # TODO: MM restriction. Uncomment ? - # if gas < 21000: - # gas = 21000 + if gas < 21000: + gas = 21000 self.debug(f'execution_cost: {execution_cost}, ' + f'trx_size_cost: {trx_size_cost}, ' + From f959e8219002bd3ba767c6b75fc7d37e2c42f443 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 14:16:00 +0300 Subject: [PATCH 33/73] wip --- tf/proxy_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 67312ec55..4f2f4b4a2 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -26,7 +26,7 @@ services: depends_on: [] networks: - net - command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh" + command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000 --owner /spl/bin/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" proxy: environment: - SOLANA_URL=$SOLANA_URL From c11abaa22ac28ca0fef0acc6aecaa2b649369ece Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 14:30:20 +0300 Subject: [PATCH 34/73] fix path --- tf/proxy_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 4f2f4b4a2..af6ad46ef 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -26,7 +26,7 @@ services: depends_on: [] networks: - net - command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000 --owner /spl/bin/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" + command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" proxy: environment: - SOLANA_URL=$SOLANA_URL From 9b0b67684f69c71ce1bd75b357aaaef676e17f90 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 15:20:06 +0300 Subject: [PATCH 35/73] wip --- tf/proxy_init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index af6ad46ef..766c47af9 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -47,6 +47,7 @@ services: EOF + SERVICES=$(docker-compose -f docker-compose-test.yml config --services | grep -v "solana") CHECK_COMMAND=`curl $SOLANA_URL -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getHealth"}'` From 5dd2ec9bfea70ea54d7f70fa7850fa9976f08fa6 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 15:29:21 +0300 Subject: [PATCH 36/73] wip --- tf/proxy_init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 766c47af9..82aa955e3 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -38,6 +38,7 @@ services: - NEON_ETH_MAX_AMOUNT=50000 ports: - 3334:3333 + entrypoint: /spl/bin/faucet --config /opt/proxy/faucet.conf run airdropper: environment: - SOLANA_URL=$SOLANA_URL From 3dba0fddf194e7c3906d0935870f3c5f1529808a Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Mon, 14 Mar 2022 15:41:29 +0300 Subject: [PATCH 37/73] wip --- tf/proxy_init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 82aa955e3..f23c13953 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -36,6 +36,8 @@ services: environment: - SOLANA_URL=$SOLANA_URL - NEON_ETH_MAX_AMOUNT=50000 + - EVM_LOADER=53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io + - NEON_TOKEN_MINT=HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU ports: - 3334:3333 entrypoint: /spl/bin/faucet --config /opt/proxy/faucet.conf run From 6a68eaec9aa3c444d049380ee3127d80398d99e6 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 08:43:25 +0300 Subject: [PATCH 38/73] artifacts --- .buildkite/pipeline.yml | 92 +++++++++++++++++++++-------------------- tf/destroy.sh | 12 ++++++ 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 65451398e..8481aa3dd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -26,50 +26,50 @@ steps: # - "indexer.log" - - label: ":coverage: full test suite (FTS)" - key: "full_tests" -# if: | -# (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || -# (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") - commands: - - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" - - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - - export SOLANA_URL="http://$$SOLANA_ADDR:8899" - - echo $$PROXY_URL - - echo $$FAUCET_URL - - echo $$SOLANA_URL - - echo Full test suite container name - $${FTS_CONTAINER_NAME} - - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up - - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) - - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ - - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log - - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f - - echo Full test passing - $${FTS_RESULT} - - echo Full test threshold - $${FTS_THRESHOLD} - - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} - - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} - artifact_paths: - - allure-reports.tar.gz - - fts_${BUILDKITE_BUILD_NUMBER}.log - env: - FTS_THRESHOLD: 1700 - FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} - FTS_IMAGE: neonlabsorg/full_test_suite:develop - agents: - queue: "testing" - artifact_paths: - - "proxy.log" - - "solana.log" - - "measurements.log" - - "evm_loader.log" - - "dbcreation.log" - - "faucet.log" - - "airdropper.log" - - "indexer.log" +# - label: ":coverage: full test suite (FTS)" +# key: "full_tests" +# # if: | +# # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || +# # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") +# commands: +# - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` +# - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` +# - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" +# - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" +# - export SOLANA_URL="http://$$SOLANA_ADDR:8899" +# - echo $$PROXY_URL +# - echo $$FAUCET_URL +# - echo $$SOLANA_URL +# - echo Full test suite container name - $${FTS_CONTAINER_NAME} +# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up +# - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) +# - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ +# - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log +# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f +# - echo Full test passing - $${FTS_RESULT} +# - echo Full test threshold - $${FTS_THRESHOLD} +# - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} +# - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} +# artifact_paths: +# - allure-reports.tar.gz +# - fts_${BUILDKITE_BUILD_NUMBER}.log +# env: +# FTS_THRESHOLD: 1700 +# FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} +# FTS_IMAGE: neonlabsorg/full_test_suite:develop +# agents: +# queue: "testing" +# artifact_paths: +# - "proxy.log" +# - "solana.log" +# - "measurements.log" +# - "evm_loader.log" +# - "dbcreation.log" +# - "faucet.log" +# - "airdropper.log" +# - "indexer.log" - - wait +# - wait - label: ":floppy_disk: publish image" command: ".buildkite/steps/publish-image.sh" @@ -82,5 +82,7 @@ steps: queue: "testing" command: - "tf/destroy.sh" -# depends_on: "full_tests" -# allow_dependency_failure: true + depends_on: "full_tests" + allow_dependency_failure: true + artifact_paths: + - "./tf/logs/solana.log" diff --git a/tf/destroy.sh b/tf/destroy.sh index da11f9923..200278b78 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -2,6 +2,18 @@ cd tf +### Receive artefacts +export SSH_KEY="~/.ssh/ci-stands" +export ARTIFACTS_LOGS="./logs" +mkdir -p $ARTIFACTS_LOGS + +# solana +export REMOTE_HOST=`buildkite-agent meta-data get "SOLANA_IP"` +ssh -i $SSH_KEY ubuntu@$REMOTE_HOST 'sudo docker logs solana > /tmp/solana.log' +scp -i $SSH_KEY ubuntu@$REMOTE_HOST:/tmp/solana.log $ARTIFACTS_LOGS + +# proxy + export TF_VAR_branch=$BUILDKITE_BRANCH export TFSTATE_BUCKET="nl-ci-stands" export TFSTATE_KEY="tests/test-$BUILDKITE_COMMIT" From 6bd5b62307eed008ba67989e4fa8b1dc7df21bca Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 08:48:13 +0300 Subject: [PATCH 39/73] enable required step --- .buildkite/pipeline.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8481aa3dd..784572f74 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -26,13 +26,13 @@ steps: # - "indexer.log" -# - label: ":coverage: full test suite (FTS)" -# key: "full_tests" -# # if: | -# # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || -# # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") -# commands: -# - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` + - label: ":coverage: full test suite (FTS)" + key: "full_tests" +# if: | +# (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || +# (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") + commands: + - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` # - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` # - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" # - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" From 8809fde45f9982bca11c44126c4a43a49aad5a26 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 08:50:51 +0300 Subject: [PATCH 40/73] wip --- .buildkite/pipeline.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 784572f74..4c2933270 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -26,13 +26,13 @@ steps: # - "indexer.log" - - label: ":coverage: full test suite (FTS)" - key: "full_tests" +# - label: ":coverage: full test suite (FTS)" +# key: "full_tests" # if: | # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") - commands: - - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` +# commands: +# - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` # - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` # - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" # - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" @@ -82,7 +82,7 @@ steps: queue: "testing" command: - "tf/destroy.sh" - depends_on: "full_tests" +# depends_on: "full_tests" allow_dependency_failure: true artifact_paths: - "./tf/logs/solana.log" From cc853d17a2c48ebc1164e906182b755beddfb5c3 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 09:00:49 +0300 Subject: [PATCH 41/73] wip --- tf/destroy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/destroy.sh b/tf/destroy.sh index 200278b78..5354fe629 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -9,6 +9,7 @@ mkdir -p $ARTIFACTS_LOGS # solana export REMOTE_HOST=`buildkite-agent meta-data get "SOLANA_IP"` +ssh-keyscan -H $REMOTE_HOST >> ~/.ssh/known_hosts ssh -i $SSH_KEY ubuntu@$REMOTE_HOST 'sudo docker logs solana > /tmp/solana.log' scp -i $SSH_KEY ubuntu@$REMOTE_HOST:/tmp/solana.log $ARTIFACTS_LOGS From 0045faa732e4c32a9a323cf9638d6b658434cf8b Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 09:21:43 +0300 Subject: [PATCH 42/73] proxy artefacts --- .buildkite/pipeline.yml | 2 +- tf/destroy.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4c2933270..2f65ca1dd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -85,4 +85,4 @@ steps: # depends_on: "full_tests" allow_dependency_failure: true artifact_paths: - - "./tf/logs/solana.log" + - "./tf/logs/*" diff --git a/tf/destroy.sh b/tf/destroy.sh index 5354fe629..1d1eb7a9a 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -14,7 +14,20 @@ ssh -i $SSH_KEY ubuntu@$REMOTE_HOST 'sudo docker logs solana > /tmp/solana.log' scp -i $SSH_KEY ubuntu@$REMOTE_HOST:/tmp/solana.log $ARTIFACTS_LOGS # proxy +export REMOTE_HOST=`buildkite-agent meta-data get "PROXY_IP"` +ssh-keyscan -H $REMOTE_HOST >> ~/.ssh/known_hosts +declare -a services=("evm_loader" "postgres" "dbcreation" "indexer" "proxy" "faucet" "airdropper") + +for service in "${services[@]}" +do + echo "$servce" + ssh -i $SSH_KEY ubuntu@$REMOTE_HOST "sudo docker logs $service > /tmp/$service.log" + scp -i $SSH_KEY ubuntu@$REMOTE_HOST:/tmp/$service.log $ARTIFACTS_LOGS +done + + +### Clean infrastructure by terraform export TF_VAR_branch=$BUILDKITE_BRANCH export TFSTATE_BUCKET="nl-ci-stands" export TFSTATE_KEY="tests/test-$BUILDKITE_COMMIT" @@ -25,5 +38,6 @@ export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-co terraform init $TF_BACKEND_CONFIG terraform destroy --auto-approve=true +# info buildkite-agent meta-data get "PROXY_IP" buildkite-agent meta-data get "SOLANA_IP" From 82847a37eb3828a761b11047a9d89d2f719060eb Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 10:44:45 +0300 Subject: [PATCH 43/73] redirect stderr --- tf/destroy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/destroy.sh b/tf/destroy.sh index 1d1eb7a9a..c48c7d366 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -10,7 +10,7 @@ mkdir -p $ARTIFACTS_LOGS # solana export REMOTE_HOST=`buildkite-agent meta-data get "SOLANA_IP"` ssh-keyscan -H $REMOTE_HOST >> ~/.ssh/known_hosts -ssh -i $SSH_KEY ubuntu@$REMOTE_HOST 'sudo docker logs solana > /tmp/solana.log' +ssh -i $SSH_KEY ubuntu@$REMOTE_HOST 'sudo docker logs solana > /tmp/solana.log 2>&1' scp -i $SSH_KEY ubuntu@$REMOTE_HOST:/tmp/solana.log $ARTIFACTS_LOGS # proxy @@ -21,7 +21,7 @@ declare -a services=("evm_loader" "postgres" "dbcreation" "indexer" "proxy" "fau for service in "${services[@]}" do echo "$servce" - ssh -i $SSH_KEY ubuntu@$REMOTE_HOST "sudo docker logs $service > /tmp/$service.log" + ssh -i $SSH_KEY ubuntu@$REMOTE_HOST "sudo docker logs $service > /tmp/$service.log 2>&1" scp -i $SSH_KEY ubuntu@$REMOTE_HOST:/tmp/$service.log $ARTIFACTS_LOGS done From 77429f576e0e00fef3c2b0bca10aa4caac0164b2 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 10:58:16 +0300 Subject: [PATCH 44/73] enable test step --- .buildkite/pipeline.yml | 82 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2f65ca1dd..1f8e7859e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -26,50 +26,50 @@ steps: # - "indexer.log" -# - label: ":coverage: full test suite (FTS)" -# key: "full_tests" + - label: ":coverage: full test suite (FTS)" + key: "full_tests" # if: | # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") -# commands: -# - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` -# - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` -# - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" -# - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" -# - export SOLANA_URL="http://$$SOLANA_ADDR:8899" -# - echo $$PROXY_URL -# - echo $$FAUCET_URL -# - echo $$SOLANA_URL -# - echo Full test suite container name - $${FTS_CONTAINER_NAME} -# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up -# - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) -# - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ -# - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log -# - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f -# - echo Full test passing - $${FTS_RESULT} -# - echo Full test threshold - $${FTS_THRESHOLD} -# - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} -# - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} -# artifact_paths: -# - allure-reports.tar.gz -# - fts_${BUILDKITE_BUILD_NUMBER}.log -# env: -# FTS_THRESHOLD: 1700 -# FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} -# FTS_IMAGE: neonlabsorg/full_test_suite:develop -# agents: -# queue: "testing" -# artifact_paths: -# - "proxy.log" -# - "solana.log" -# - "measurements.log" -# - "evm_loader.log" -# - "dbcreation.log" -# - "faucet.log" -# - "airdropper.log" -# - "indexer.log" + commands: + - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` + - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` + - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" + - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" + - export SOLANA_URL="http://$$SOLANA_ADDR:8899" + - echo $$PROXY_URL + - echo $$FAUCET_URL + - echo $$SOLANA_URL + - echo Full test suite container name - $${FTS_CONTAINER_NAME} + - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up + - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) + - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ + - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log + - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f + - echo Full test passing - $${FTS_RESULT} + - echo Full test threshold - $${FTS_THRESHOLD} + - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} + - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} + artifact_paths: + - allure-reports.tar.gz + - fts_${BUILDKITE_BUILD_NUMBER}.log + env: + FTS_THRESHOLD: 1700 + FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} + FTS_IMAGE: neonlabsorg/full_test_suite:develop + agents: + queue: "testing" + artifact_paths: + - "proxy.log" + - "solana.log" + - "measurements.log" + - "evm_loader.log" + - "dbcreation.log" + - "faucet.log" + - "airdropper.log" + - "indexer.log" -# - wait + - wait - label: ":floppy_disk: publish image" command: ".buildkite/steps/publish-image.sh" @@ -82,7 +82,7 @@ steps: queue: "testing" command: - "tf/destroy.sh" -# depends_on: "full_tests" + depends_on: "full_tests" allow_dependency_failure: true artifact_paths: - "./tf/logs/*" From 3938e6a20ac9d88b2ebd9a1103600cdcf46b9458 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 10:59:29 +0300 Subject: [PATCH 45/73] typo --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1f8e7859e..26b69ce4d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -69,7 +69,7 @@ steps: - "airdropper.log" - "indexer.log" - - wait + - wait - label: ":floppy_disk: publish image" command: ".buildkite/steps/publish-image.sh" From ea7b59fa79faff59737fa9173ebc5a1804074312 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 11:00:31 +0300 Subject: [PATCH 46/73] typo --- .buildkite/pipeline.yml | 78 ++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 26b69ce4d..f5c9203eb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -26,48 +26,48 @@ steps: # - "indexer.log" - - label: ":coverage: full test suite (FTS)" - key: "full_tests" + - label: ":coverage: full test suite (FTS)" + key: "full_tests" # if: | # (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || # (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") - commands: - - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" - - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - - export SOLANA_URL="http://$$SOLANA_ADDR:8899" - - echo $$PROXY_URL - - echo $$FAUCET_URL - - echo $$SOLANA_URL - - echo Full test suite container name - $${FTS_CONTAINER_NAME} - - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up - - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) - - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ - - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log - - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f - - echo Full test passing - $${FTS_RESULT} - - echo Full test threshold - $${FTS_THRESHOLD} - - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} - - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} - artifact_paths: - - allure-reports.tar.gz - - fts_${BUILDKITE_BUILD_NUMBER}.log - env: - FTS_THRESHOLD: 1700 - FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} - FTS_IMAGE: neonlabsorg/full_test_suite:develop - agents: - queue: "testing" - artifact_paths: - - "proxy.log" - - "solana.log" - - "measurements.log" - - "evm_loader.log" - - "dbcreation.log" - - "faucet.log" - - "airdropper.log" - - "indexer.log" + commands: + - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` + - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` + - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" + - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" + - export SOLANA_URL="http://$$SOLANA_ADDR:8899" + - echo $$PROXY_URL + - echo $$FAUCET_URL + - echo $$SOLANA_URL + - echo Full test suite container name - $${FTS_CONTAINER_NAME} + - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up + - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) + - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ + - docker logs $${FTS_CONTAINER_NAME} > ./$${FTS_CONTAINER_NAME}.log + - docker-compose -f docker-compose/docker-compose-full-test-suite.yml rm -f + - echo Full test passing - $${FTS_RESULT} + - echo Full test threshold - $${FTS_THRESHOLD} + - echo Check if $${FTS_RESULT} is greater or equeal $${FTS_THRESHOLD} + - test $${FTS_RESULT} -ge $${FTS_THRESHOLD} + artifact_paths: + - allure-reports.tar.gz + - fts_${BUILDKITE_BUILD_NUMBER}.log + env: + FTS_THRESHOLD: 1700 + FTS_CONTAINER_NAME: fts_${BUILDKITE_BUILD_NUMBER} + FTS_IMAGE: neonlabsorg/full_test_suite:develop + agents: + queue: "testing" + artifact_paths: + - "proxy.log" + - "solana.log" + - "measurements.log" + - "evm_loader.log" + - "dbcreation.log" + - "faucet.log" + - "airdropper.log" + - "indexer.log" - wait From e8c9696ac5b11e7512344ac4ee9620d04281359a Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 14:58:29 +0300 Subject: [PATCH 47/73] faucet --- tf/proxy_init.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index f23c13953..23e206ca4 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -33,6 +33,7 @@ services: ports: - 9091:9090 faucet: + image: neonlabsorg/faucet:latest environment: - SOLANA_URL=$SOLANA_URL - NEON_ETH_MAX_AMOUNT=50000 @@ -40,7 +41,7 @@ services: - NEON_TOKEN_MINT=HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU ports: - 3334:3333 - entrypoint: /spl/bin/faucet --config /opt/proxy/faucet.conf run + entrypoint: /opt/faucet/faucet --config /opt/proxy/faucet.conf run airdropper: environment: - SOLANA_URL=$SOLANA_URL From 3d30b36e59b0808876245a470aa41beb5906d4cb Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 15:26:22 +0300 Subject: [PATCH 48/73] wip --- tf/proxy_init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 23e206ca4..f97a154ee 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -35,6 +35,7 @@ services: faucet: image: neonlabsorg/faucet:latest environment: + - FAUCET_RPC_BIND=0.0.0.0 - SOLANA_URL=$SOLANA_URL - NEON_ETH_MAX_AMOUNT=50000 - EVM_LOADER=53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io From a76732afdafb7b30f9007c0c837a396f77febd94 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 15:37:26 +0300 Subject: [PATCH 49/73] debug --- .buildkite/pipeline.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f5c9203eb..eb9a03fd6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -77,12 +77,12 @@ steps: build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - - label: ":terraform: destroy infrastructure" - agents: - queue: "testing" - command: - - "tf/destroy.sh" - depends_on: "full_tests" - allow_dependency_failure: true - artifact_paths: - - "./tf/logs/*" + # - label: ":terraform: destroy infrastructure" + # agents: + # queue: "testing" + # command: + # - "tf/destroy.sh" + # depends_on: "full_tests" + # allow_dependency_failure: true + # artifact_paths: + # - "./tf/logs/*" From d61c342157fdc3b799b7edeb971b860bcaaecca5 Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Tue, 15 Mar 2022 17:20:29 +0400 Subject: [PATCH 50/73] Add implementation of neon_getSolanaTransactionByNeonTransaction. #639 --- proxy/indexer/indexer_db.py | 3 +++ proxy/indexer/transactions_db.py | 19 +++++++++++++++++++ proxy/memdb/memdb.py | 3 +++ proxy/plugin/solana_rest_api.py | 6 ++++++ 4 files changed, 31 insertions(+) diff --git a/proxy/indexer/indexer_db.py b/proxy/indexer/indexer_db.py index 143b77ecd..ab9de4c79 100644 --- a/proxy/indexer/indexer_db.py +++ b/proxy/indexer/indexer_db.py @@ -134,3 +134,6 @@ def get_contract_code(self, address) -> str: def fill_account_info_by_indexer(self, neon_account: str, pda_account: str, code_account: str, slot: int): self._account_db.set_acc_indexer(neon_account, pda_account, code_account, slot) + + def get_sol_sign_list_by_neon_sign(self, neon_sign: str) -> [str]: + return self._txs_db.get_sol_sign_list_by_neon_sign(neon_sign) diff --git a/proxy/indexer/transactions_db.py b/proxy/indexer/transactions_db.py index 3f0609e96..64199baf1 100644 --- a/proxy/indexer/transactions_db.py +++ b/proxy/indexer/transactions_db.py @@ -22,6 +22,22 @@ def set_txs(self, neon_sign: str, used_ixs: [SolanaIxSignInfo]): VALUES(%s, %s, %s, %s) ON CONFLICT DO NOTHING''', rows) + def get_sol_sign_list_by_neon_sign(self, neon_sign: str) -> [str]: + request = f''' + SELECT sol_sign + FROM {self._table_name} AS a + WHERE neon_sign = %s + ''' + + with self._conn.cursor() as cursor: + cursor.execute(request, [neon_sign]) + values = cursor.fetchall() + + if not values: + return [] + + return [v[0] for v in values] + class NeonTxsDB(BaseDB): def __init__(self): @@ -112,3 +128,6 @@ def get_tx_list_by_sol_sign(self, sol_sign_list: [str]) -> [NeonTxFullInfo]: return [] return [self._tx_from_value(v) for v in values if v is not None] + + def get_sol_sign_list_by_neon_sign(self, neon_sign: str) -> [str]: + return self._sol_neon_txs_db.get_sol_sign_list_by_neon_sign(neon_sign) diff --git a/proxy/memdb/memdb.py b/proxy/memdb/memdb.py index 1cdcb629d..3b059a56e 100644 --- a/proxy/memdb/memdb.py +++ b/proxy/memdb/memdb.py @@ -59,3 +59,6 @@ def get_logs(self, from_block, to_block, addresses, topics, block_hash): def get_contract_code(self, address: str) -> str: return self._db.get_contract_code(address) + + def get_sol_sign_list_by_neon_sign(self, neon_sign: str) -> [str]: + return self._db.get_sol_sign_list_by_neon_sign(neon_sign) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index bee8d4323..b185589e2 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -386,6 +386,12 @@ def eth_sendRawTransaction(self, rawTrx): # self.error(f"eth_sendRawTransaction type(err): {type(err}}, Exception: {err}") raise + def neon_getSolanaTransactionByNeonTransaction(self, neonTxId: str) -> [str]: + if not isinstance(neonTxId, str): + return [] + return self._db.get_sol_sign_list_by_neon_sign(neonTxId) + + class JsonEncoder(json.JSONEncoder): def default(self, obj): From 048830736b36c7042414bf5cd011355b3af65299 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 16:33:28 +0300 Subject: [PATCH 51/73] wip --- tf/proxy_init.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index f97a154ee..17f490422 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -36,10 +36,16 @@ services: image: neonlabsorg/faucet:latest environment: - FAUCET_RPC_BIND=0.0.0.0 + - FAUCET_RPC_PORT=3333 - SOLANA_URL=$SOLANA_URL - NEON_ETH_MAX_AMOUNT=50000 - EVM_LOADER=53DfF883gyixYNXnM7s5xhdeyV8mVk9T4i2hGV9vG9io - - NEON_TOKEN_MINT=HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU + - NEON_TOKEN_MINT=HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU + - FAUCET_RPC_ALLOWED_ORIGINS=["https://neonswap.live"] + - FAUCET_WEB3_ENABLE=false + - FAUCET_SOLANA_ENABLE=true + - NEON_TOKEN_MINT_DECIMALS=9 + - NEON_OPERATOR_KEYFILE=/opt/faucet/id.json ports: - 3334:3333 entrypoint: /opt/faucet/faucet --config /opt/proxy/faucet.conf run From 5f4f39e99e0c0a65740727198ca13929058d4c02 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Tue, 15 Mar 2022 17:10:22 +0300 Subject: [PATCH 52/73] Add SOLANA_COMMITMENT --- tf/proxy_init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 17f490422..8f8697c36 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -46,6 +46,7 @@ services: - FAUCET_SOLANA_ENABLE=true - NEON_TOKEN_MINT_DECIMALS=9 - NEON_OPERATOR_KEYFILE=/opt/faucet/id.json + - SOLANA_COMMITMENT=finalized ports: - 3334:3333 entrypoint: /opt/faucet/faucet --config /opt/proxy/faucet.conf run From cff095203fd197565fa1f3cdc0c85aab28cf4fe0 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 10:54:56 +0300 Subject: [PATCH 53/73] Enable cleaning step --- .buildkite/pipeline.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index eb9a03fd6..f5c9203eb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -77,12 +77,12 @@ steps: build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - # - label: ":terraform: destroy infrastructure" - # agents: - # queue: "testing" - # command: - # - "tf/destroy.sh" - # depends_on: "full_tests" - # allow_dependency_failure: true - # artifact_paths: - # - "./tf/logs/*" + - label: ":terraform: destroy infrastructure" + agents: + queue: "testing" + command: + - "tf/destroy.sh" + depends_on: "full_tests" + allow_dependency_failure: true + artifact_paths: + - "./tf/logs/*" From 86cecc703bc81696368c61eb8fa6846c035757c3 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 12:23:50 +0300 Subject: [PATCH 54/73] wip --- tf/proxy_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 8f8697c36..1e8148cc8 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -26,7 +26,7 @@ services: depends_on: [] networks: - net - command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" + command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" proxy: environment: - SOLANA_URL=$SOLANA_URL From e7ddf254c7757b39fccc911c2d285809372b0c8c Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 12:55:55 +0300 Subject: [PATCH 55/73] wip --- tf/proxy_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 1e8148cc8..801024fac 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -26,7 +26,7 @@ services: depends_on: [] networks: - net - command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" + command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 100000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" proxy: environment: - SOLANA_URL=$SOLANA_URL From bda6d6009ea664eb72f0a8e9c559cf9edacbc5e4 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 13:35:16 +0300 Subject: [PATCH 56/73] debug --- .buildkite/pipeline.yml | 18 +++++++++--------- tf/proxy_init.sh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f5c9203eb..eb9a03fd6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -77,12 +77,12 @@ steps: build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - - label: ":terraform: destroy infrastructure" - agents: - queue: "testing" - command: - - "tf/destroy.sh" - depends_on: "full_tests" - allow_dependency_failure: true - artifact_paths: - - "./tf/logs/*" + # - label: ":terraform: destroy infrastructure" + # agents: + # queue: "testing" + # command: + # - "tf/destroy.sh" + # depends_on: "full_tests" + # allow_dependency_failure: true + # artifact_paths: + # - "./tf/logs/*" diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 801024fac..8f8697c36 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -26,7 +26,7 @@ services: depends_on: [] networks: - net - command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 100000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" + command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh && /opt/spl-token create-account HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU && /opt/spl-token mint HPsV9Deocecw3GeZv1FkAPNCBRfuVyfw9MMwjwRe1xaU 1000000000 --owner /opt/evm_loader-keypair.json -- HX14J4Pp9CgSbWP13Dtpm8VLJpNxMYffLtRCRGsx7Edv" proxy: environment: - SOLANA_URL=$SOLANA_URL From cf7209201123e831b5b3eb495f3b0b82d1ce65cc Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 14:56:01 +0300 Subject: [PATCH 57/73] debug --- tf/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tf/build.sh b/tf/build.sh index c7098548b..97692a3ea 100644 --- a/tf/build.sh +++ b/tf/build.sh @@ -17,3 +17,6 @@ terraform output --json | jq -r '.solana_ip.value' | buildkite-agent meta-data s buildkite-agent meta-data get "PROXY_IP" buildkite-agent meta-data get "SOLANA_IP" + + +sleep 180 From 9fa4281c60fcd672eee3eb069e65ff64adf87698 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 15:09:44 +0300 Subject: [PATCH 58/73] testdrive --- .buildkite/pipeline.yml | 18 +++++++++--------- tf/terraform.auto.tfvars | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index eb9a03fd6..f5c9203eb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -77,12 +77,12 @@ steps: build.branch =~ /^(master|develop|^ci-.+|v[0-9]+\.[0-9]+\..+)$$/ && (build.env("NEON_EVM_BRANCH") == "develop" || build.env("NEON_EVM_BRANCH") == null) - # - label: ":terraform: destroy infrastructure" - # agents: - # queue: "testing" - # command: - # - "tf/destroy.sh" - # depends_on: "full_tests" - # allow_dependency_failure: true - # artifact_paths: - # - "./tf/logs/*" + - label: ":terraform: destroy infrastructure" + agents: + queue: "testing" + command: + - "tf/destroy.sh" + depends_on: "full_tests" + allow_dependency_failure: true + artifact_paths: + - "./tf/logs/*" diff --git a/tf/terraform.auto.tfvars b/tf/terraform.auto.tfvars index 3ecd7f748..541837f0c 100644 --- a/tf/terraform.auto.tfvars +++ b/tf/terraform.auto.tfvars @@ -5,6 +5,6 @@ proxy_model_revision = "latest" aws_subnet = "subnet-19f77872" allow_list = ["172.31.0.0/16", "3.15.140.214/32", "18.219.70.113/32", "3.21.100.174/32", "3.137.181.30/32", "3.136.233.33/32", "142.132.171.62/32"] -solana_instance_type = "t3.large" +solana_instance_type = "t3.xlarge" proxy_instance_type = "t3.xlarge" ami = "ami-0fb653ca2d3203ac1" From cf05a0f34ec144ba4d6e375cc9aef00b5f792f6b Mon Sep 17 00:00:00 2001 From: Andrew Falaleev Date: Wed, 16 Mar 2022 19:27:22 +0700 Subject: [PATCH 59/73] Last changes from v0.7.x to develop (#660) * Move CI operator keypairs to safety location * Update version to 0.7.1-dev --- Dockerfile | 2 +- proxy/deploy-test.sh | 2 ++ proxy/plugin/solana_rest_api.py | 2 +- proxy/run-test-proxy.sh | 1 + proxy/testing/test_neon_token.py | 30 +++++++++++++++--------------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index a4ecea4a4..752ddd711 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ COPY --from=spl /opt/solana_utils.py \ /spl/bin/ COPY --from=spl /opt/neon-cli /spl/bin/emulator -COPY proxy/operator-keypairs/* /root/.config/solana/ +COPY proxy/operator-keypairs/id.json /root/.config/solana/ COPY . /opt ARG PROXY_REVISION diff --git a/proxy/deploy-test.sh b/proxy/deploy-test.sh index bf0f14154..5958bdab3 100755 --- a/proxy/deploy-test.sh +++ b/proxy/deploy-test.sh @@ -4,6 +4,8 @@ set -xeuo pipefail echo "Deploy test..." solana config set -u $SOLANA_URL +ln -s /opt/proxy/operator-keypairs/id?*.json /root/.config/solana/ + solana address || solana-keygen new --no-passphrase export $(/spl/bin/neon-cli --commitment confirmed --url $SOLANA_URL --evm_loader "$EVM_LOADER" neon-elf-params) diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index b185589e2..f151d8171 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -42,7 +42,7 @@ modelInstanceLock = threading.Lock() modelInstance = None -NEON_PROXY_PKG_VERSION = '0.6.0-dev' +NEON_PROXY_PKG_VERSION = '0.7.1-dev' NEON_PROXY_REVISION = 'NEON_PROXY_REVISION_TO_BE_REPLACED' diff --git a/proxy/run-test-proxy.sh b/proxy/run-test-proxy.sh index d6f374e13..5a0ba54db 100755 --- a/proxy/run-test-proxy.sh +++ b/proxy/run-test-proxy.sh @@ -8,6 +8,7 @@ if [ -z "$SOLANA_URL" ]; then fi solana config set -u $SOLANA_URL +ln -s /opt/proxy/operator-keypairs/id?*.json /root/.config/solana/ echo "$(date "+%F %X.%3N") I $(basename "$0"):${LINENO} $$ ${COMPONENT}:StartScript {} Dumping evm_loader and extracting ELF parameters" export EVM_LOADER=$(solana address -k /spl/bin/evm_loader-keypair.json) diff --git a/proxy/testing/test_neon_token.py b/proxy/testing/test_neon_token.py index 30cb27d5d..dd4a3b523 100644 --- a/proxy/testing/test_neon_token.py +++ b/proxy/testing/test_neon_token.py @@ -62,7 +62,7 @@ def deploy_contract(self): artifacts = compile_source(NEON_TOKEN_CONTRACT) _, self.neon_token_iface = artifacts.popitem() - self.neon_contract = proxy.eth.contract(abi=self.neon_token_iface['abi'], + self.neon_contract = proxy.eth.contract(abi=self.neon_token_iface['abi'], bytecode=self.neon_token_iface['bin']) deployer = self.create_eth_account(self, 100) @@ -79,7 +79,7 @@ def deploy_contract(self): print(f'deploy status: {tx_deploy_receipt.status}') self.neon_token_address = tx_deploy_receipt.contractAddress print(f'NeonToken contract address is: {self.neon_token_address}') - self.neon_contract = proxy.eth.contract(address=self.neon_token_address, + self.neon_contract = proxy.eth.contract(address=self.neon_token_address, abi=self.neon_token_iface['abi']) def withdraw(self, source_acc: NeonAccount, dest_acc: SolanaAccount, withdraw_amount_alan: int): @@ -92,7 +92,7 @@ def withdraw(self, source_acc: NeonAccount, dest_acc: SolanaAccount, withdraw_am withdraw_tx_receipt = proxy.eth.wait_for_transaction_receipt(withdraw_tx_hash) print(f'withdraw_tx_receipt: {withdraw_tx_receipt}') print(f'deploy status: {withdraw_tx_receipt.status}') - + def test_success_withdraw_to_non_existing_account(self): """ Should succesfully withdraw NEON tokens to previously non-existing Associated Token Account @@ -102,7 +102,7 @@ def test_success_withdraw_to_non_existing_account(self): dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) print(f"Destination token account: {dest_token_acc}") - + withdraw_amount_alan = pow(10, 18) # 1 NEON withdraw_amount_galan = int(withdraw_amount_alan / 1_000_000_000) @@ -126,7 +126,7 @@ def test_success_withdraw_to_non_existing_account(self): destination_balance_after_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) print(f'Destination account balance after (Galan): {destination_balance_after_galan}') self.assertEqual(int(destination_balance_after_galan['result']['value']['amount']), withdraw_amount_galan) - + def test_success_withdraw_to_existing_account(self): """ Should succesfully withdraw NEON tokens to existing Associated Token Account @@ -138,17 +138,17 @@ def test_success_withdraw_to_existing_account(self): trx = Transaction() trx.add( create_associated_token_account( - dest_acc.public_key(), - dest_acc.public_key(), + dest_acc.public_key(), + dest_acc.public_key(), NEON_TOKEN_MINT ) ) opts = TxOpts(skip_preflight=True, skip_confirmation=False) solana.send_transaction(trx, dest_acc, opts=opts) - + dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) print(f"Destination token account: {dest_token_acc}") - + withdraw_amount_alan = 2_123_000_321_000_000_000 withdraw_amount_galan = int(withdraw_amount_alan / 1_000_000_000) @@ -174,7 +174,7 @@ def test_success_withdraw_to_existing_account(self): destination_balance_after_galan = int(resp['result']['value']['amount']) print(f'Destination account balance after (Galan): {destination_balance_after_galan}') self.assertEqual(destination_balance_after_galan, withdraw_amount_galan) - + def test_failed_withdraw_non_divisible_amount(self): """ Should fail withdrawal because amount not divised by 1 billion @@ -184,7 +184,7 @@ def test_failed_withdraw_non_divisible_amount(self): dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) print(f"Destination token account: {dest_token_acc}") - + withdraw_amount_alan = pow(10, 18) + 123 # NEONs # Check source balance @@ -209,7 +209,7 @@ def test_failed_withdraw_non_divisible_amount(self): destination_balance_after_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) print(f'Destination account balance after (Galan): {destination_balance_after_galan}') self.assertTrue(destination_balance_after_galan['error'] is not None) - + def test_failed_withdraw_insufficient_balance(self): """ Should fail withdrawal because of insufficient balance @@ -219,7 +219,7 @@ def test_failed_withdraw_insufficient_balance(self): dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) print(f"Destination token account: {dest_token_acc}") - + withdraw_amount_alan = 2 * pow(10, 18) # 2 NEONs # Check source balance @@ -244,7 +244,7 @@ def test_failed_withdraw_insufficient_balance(self): destination_balance_after_galan = self.spl_neon_token.get_balance(dest_token_acc, commitment=Confirmed) print(f'Destination account balance after (Galan): {destination_balance_after_galan}') self.assertTrue(destination_balance_after_galan['error'] is not None) - + def test_failed_withdraw_all_balance(self): """ Should fail withdrawal all balance @@ -254,7 +254,7 @@ def test_failed_withdraw_all_balance(self): dest_token_acc = get_associated_token_address(dest_acc.public_key(), NEON_TOKEN_MINT) print(f"Destination token account: {dest_token_acc}") - + withdraw_amount_alan = 1_000_000_000_000_000_000 # 1 NEON # Check source balance From ab84e08f9b6be10f8195961f820348f529eb8afd Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 15:27:44 +0300 Subject: [PATCH 60/73] solana tune --- tf/solana_init.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tf/solana_init.sh b/tf/solana_init.sh index 1349bbf20..046a75a70 100644 --- a/tf/solana_init.sh +++ b/tf/solana_init.sh @@ -5,6 +5,28 @@ sudo apt-get -y install ca-certificates curl gnupg lsb-release curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update + +sudo bash -c "cat >/etc/sysctl.d/20-solana-udp-buffers.conf </etc/sysctl.d/20-solana-mmaps.conf </etc/security/limits.d/90-solana-nofiles.conf < Date: Wed, 16 Mar 2022 18:26:54 +0300 Subject: [PATCH 61/73] debug --- tf/build.sh | 2 -- tf/destroy.sh | 4 ++-- tf/terraform.auto.tfvars | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tf/build.sh b/tf/build.sh index 97692a3ea..57b446898 100644 --- a/tf/build.sh +++ b/tf/build.sh @@ -18,5 +18,3 @@ terraform output --json | jq -r '.solana_ip.value' | buildkite-agent meta-data s buildkite-agent meta-data get "PROXY_IP" buildkite-agent meta-data get "SOLANA_IP" - -sleep 180 diff --git a/tf/destroy.sh b/tf/destroy.sh index c48c7d366..43e9b7649 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -35,8 +35,8 @@ export TFSTATE_REGION="us-east-2" export TF_VAR_neon_evm_revision=latest export TF_VAR_proxy_model_revision=latest export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" -terraform init $TF_BACKEND_CONFIG -terraform destroy --auto-approve=true +#terraform init $TF_BACKEND_CONFIG +#terraform destroy --auto-approve=true # info buildkite-agent meta-data get "PROXY_IP" diff --git a/tf/terraform.auto.tfvars b/tf/terraform.auto.tfvars index 541837f0c..09e7e6341 100644 --- a/tf/terraform.auto.tfvars +++ b/tf/terraform.auto.tfvars @@ -5,6 +5,6 @@ proxy_model_revision = "latest" aws_subnet = "subnet-19f77872" allow_list = ["172.31.0.0/16", "3.15.140.214/32", "18.219.70.113/32", "3.21.100.174/32", "3.137.181.30/32", "3.136.233.33/32", "142.132.171.62/32"] -solana_instance_type = "t3.xlarge" -proxy_instance_type = "t3.xlarge" +solana_instance_type = "t3.large" +proxy_instance_type = "t3.large" ami = "ami-0fb653ca2d3203ac1" From 6f0322cd38d272a8bb0a991a5caac5903d83ff96 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 18:47:22 +0300 Subject: [PATCH 62/73] wip --- tf/destroy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/destroy.sh b/tf/destroy.sh index 43e9b7649..48e50b2e5 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -38,6 +38,7 @@ export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-co #terraform init $TF_BACKEND_CONFIG #terraform destroy --auto-approve=true + # info buildkite-agent meta-data get "PROXY_IP" buildkite-agent meta-data get "SOLANA_IP" From fd0cb9959034773351ab5f4f7d0ec17571f61768 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 18:50:17 +0300 Subject: [PATCH 63/73] wip --- tf/destroy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/destroy.sh b/tf/destroy.sh index 48e50b2e5..8b3ac8977 100644 --- a/tf/destroy.sh +++ b/tf/destroy.sh @@ -35,7 +35,7 @@ export TFSTATE_REGION="us-east-2" export TF_VAR_neon_evm_revision=latest export TF_VAR_proxy_model_revision=latest export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" -#terraform init $TF_BACKEND_CONFIG +terraform init $TF_BACKEND_CONFIG #terraform destroy --auto-approve=true From fa3d60d43093728b88fcfd880f4ed614c9365f9e Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Wed, 16 Mar 2022 20:38:50 +0300 Subject: [PATCH 64/73] confirmed --- tf/proxy_init.sh | 2 +- tf/terraform.auto.tfvars | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 8f8697c36..3e1923f4b 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -46,7 +46,7 @@ services: - FAUCET_SOLANA_ENABLE=true - NEON_TOKEN_MINT_DECIMALS=9 - NEON_OPERATOR_KEYFILE=/opt/faucet/id.json - - SOLANA_COMMITMENT=finalized + - SOLANA_COMMITMENT=confirmed ports: - 3334:3333 entrypoint: /opt/faucet/faucet --config /opt/proxy/faucet.conf run diff --git a/tf/terraform.auto.tfvars b/tf/terraform.auto.tfvars index 09e7e6341..3ecd7f748 100644 --- a/tf/terraform.auto.tfvars +++ b/tf/terraform.auto.tfvars @@ -6,5 +6,5 @@ proxy_model_revision = "latest" aws_subnet = "subnet-19f77872" allow_list = ["172.31.0.0/16", "3.15.140.214/32", "18.219.70.113/32", "3.21.100.174/32", "3.137.181.30/32", "3.136.233.33/32", "142.132.171.62/32"] solana_instance_type = "t3.large" -proxy_instance_type = "t3.large" +proxy_instance_type = "t3.xlarge" ami = "ami-0fb653ca2d3203ac1" From 9b950fcff52dcfba67dfee616ed5864b6cd97a58 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Thu, 17 Mar 2022 09:40:45 +0300 Subject: [PATCH 65/73] test --- .buildkite/pipeline.yml | 2 +- tf/proxy_init.sh | 2 +- tf/solana_init.sh | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f5c9203eb..945569b7d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -36,7 +36,7 @@ steps: - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - - export SOLANA_URL="http://$$SOLANA_ADDR:8899" + - export SOLANA_URL="http://$$SOLANA_ADDR:8898" - echo $$PROXY_URL - echo $$FAUCET_URL - echo $$SOLANA_URL diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index 3e1923f4b..b9af5030a 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -12,7 +12,7 @@ cd /opt curl -O https://raw.githubusercontent.com/neonlabsorg/proxy-model.py/${branch}/proxy/docker-compose-test.yml export REVISION=${revision} -export SOLANA_URL=http:\/\/${solana_ip}:8899 +export SOLANA_URL=http:\/\/${solana_ip}:8898 cat > docker-compose-test.override.yml < Date: Thu, 17 Mar 2022 09:46:01 +0300 Subject: [PATCH 66/73] Revert "test" This reverts commit 9b950fcff52dcfba67dfee616ed5864b6cd97a58. --- .buildkite/pipeline.yml | 2 +- tf/proxy_init.sh | 2 +- tf/solana_init.sh | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 945569b7d..f5c9203eb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -36,7 +36,7 @@ steps: - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` - export PROXY_URL="http://$$PROXY_ADDR:9091/solana" - export FAUCET_URL="http://$$PROXY_ADDR:3334/request_neon" - - export SOLANA_URL="http://$$SOLANA_ADDR:8898" + - export SOLANA_URL="http://$$SOLANA_ADDR:8899" - echo $$PROXY_URL - echo $$FAUCET_URL - echo $$SOLANA_URL diff --git a/tf/proxy_init.sh b/tf/proxy_init.sh index b9af5030a..3e1923f4b 100644 --- a/tf/proxy_init.sh +++ b/tf/proxy_init.sh @@ -12,7 +12,7 @@ cd /opt curl -O https://raw.githubusercontent.com/neonlabsorg/proxy-model.py/${branch}/proxy/docker-compose-test.yml export REVISION=${revision} -export SOLANA_URL=http:\/\/${solana_ip}:8898 +export SOLANA_URL=http:\/\/${solana_ip}:8899 cat > docker-compose-test.override.yml < Date: Fri, 18 Mar 2022 12:12:09 +0400 Subject: [PATCH 67/73] Comment extra steps --- .buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f5c9203eb..f98b735e8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,6 +41,7 @@ steps: - echo $$FAUCET_URL - echo $$SOLANA_URL - echo Full test suite container name - $${FTS_CONTAINER_NAME} + - docker image rm -f ${FTS_IMAGE} - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ From 2f7f154e395bbe4cd28da0bd83ed0fec235d8634 Mon Sep 17 00:00:00 2001 From: rozhkovdmitrii Date: Fri, 18 Mar 2022 12:16:24 +0400 Subject: [PATCH 68/73] Comment extra steps --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f98b735e8..6a2955624 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,7 +41,7 @@ steps: - echo $$FAUCET_URL - echo $$SOLANA_URL - echo Full test suite container name - $${FTS_CONTAINER_NAME} - - docker image rm -f ${FTS_IMAGE} + - docker-compose -f docker-compose/docker-compose-full-test-suite.yml pull - docker-compose -f docker-compose/docker-compose-full-test-suite.yml up - FTS_RESULT=$(docker logs $${FTS_CONTAINER_NAME} | (grep -oP "(?<=Passing - )\d+" || echo 0)) - docker cp $${FTS_CONTAINER_NAME}:/opt/allure-reports.tar.gz ./ From 03ffaea1473a97bd914da9f1c083d67681e0fcf3 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 18 Mar 2022 13:08:16 +0300 Subject: [PATCH 69/73] refactoring --- .buildkite/pipeline.yml | 4 ++-- {tf => .buildkite/steps/tf}/build.sh | 9 +++++++-- {tf => .buildkite/steps/tf}/destroy.sh | 8 +++++--- {tf => .buildkite/steps/tf}/main.tf | 8 ++++---- {tf => .buildkite/steps/tf}/output.tf | 0 {tf => .buildkite/steps/tf}/proxy_init.sh | 20 ++++++++++++++++++- {tf => .buildkite/steps/tf}/solana_init.sh | 13 +++++++++++- .../steps/tf}/terraform.auto.tfvars | 0 {tf => .buildkite/steps/tf}/vars.tf | 0 9 files changed, 49 insertions(+), 13 deletions(-) rename {tf => .buildkite/steps/tf}/build.sh (90%) rename {tf => .buildkite/steps/tf}/destroy.sh (95%) rename {tf => .buildkite/steps/tf}/main.tf (95%) rename {tf => .buildkite/steps/tf}/output.tf (100%) rename {tf => .buildkite/steps/tf}/proxy_init.sh (90%) rename {tf => .buildkite/steps/tf}/solana_init.sh (90%) rename {tf => .buildkite/steps/tf}/terraform.auto.tfvars (100%) rename {tf => .buildkite/steps/tf}/vars.tf (100%) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6a2955624..1126e96b4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -8,7 +8,7 @@ steps: agents: queue: "testing" command: - - "tf/build.sh" + - ".buildkite/steps/tf/build.sh" - wait # - label: ":cop::skin-tone-2: deploy check" @@ -86,4 +86,4 @@ steps: depends_on: "full_tests" allow_dependency_failure: true artifact_paths: - - "./tf/logs/*" + - ".buildkite/steps/tf/logs/*" diff --git a/tf/build.sh b/.buildkite/steps/tf/build.sh similarity index 90% rename from tf/build.sh rename to .buildkite/steps/tf/build.sh index 57b446898..0a26c10f7 100644 --- a/tf/build.sh +++ b/.buildkite/steps/tf/build.sh @@ -1,7 +1,9 @@ #!/bin/bash -cd tf +cd .buildkite/steps/tf + +# Terraform part export TF_VAR_branch=$BUILDKITE_BRANCH export TFSTATE_BUCKET="nl-ci-stands" export TFSTATE_KEY="tests/test-$BUILDKITE_COMMIT" @@ -12,9 +14,12 @@ export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-co terraform init $TF_BACKEND_CONFIG terraform apply --auto-approve=true + +# Get IPs terraform output --json | jq -r '.proxy_ip.value' | buildkite-agent meta-data set "PROXY_IP" terraform output --json | jq -r '.solana_ip.value' | buildkite-agent meta-data set "SOLANA_IP" + +# Save IPs for next steps buildkite-agent meta-data get "PROXY_IP" buildkite-agent meta-data get "SOLANA_IP" - diff --git a/tf/destroy.sh b/.buildkite/steps/tf/destroy.sh similarity index 95% rename from tf/destroy.sh rename to .buildkite/steps/tf/destroy.sh index 8b3ac8977..eda18852a 100644 --- a/tf/destroy.sh +++ b/.buildkite/steps/tf/destroy.sh @@ -1,18 +1,21 @@ #!/bin/bash -cd tf +cd .buildkite/steps/tf + ### Receive artefacts export SSH_KEY="~/.ssh/ci-stands" export ARTIFACTS_LOGS="./logs" mkdir -p $ARTIFACTS_LOGS + # solana export REMOTE_HOST=`buildkite-agent meta-data get "SOLANA_IP"` ssh-keyscan -H $REMOTE_HOST >> ~/.ssh/known_hosts ssh -i $SSH_KEY ubuntu@$REMOTE_HOST 'sudo docker logs solana > /tmp/solana.log 2>&1' scp -i $SSH_KEY ubuntu@$REMOTE_HOST:/tmp/solana.log $ARTIFACTS_LOGS + # proxy export REMOTE_HOST=`buildkite-agent meta-data get "PROXY_IP"` ssh-keyscan -H $REMOTE_HOST >> ~/.ssh/known_hosts @@ -26,7 +29,6 @@ do done - ### Clean infrastructure by terraform export TF_VAR_branch=$BUILDKITE_BRANCH export TFSTATE_BUCKET="nl-ci-stands" @@ -36,7 +38,7 @@ export TF_VAR_neon_evm_revision=latest export TF_VAR_proxy_model_revision=latest export TF_BACKEND_CONFIG="-backend-config="bucket=${TFSTATE_BUCKET}" -backend-config="key=${TFSTATE_KEY}" -backend-config="region=${TFSTATE_REGION}"" terraform init $TF_BACKEND_CONFIG -#terraform destroy --auto-approve=true +terraform destroy --auto-approve=true # info diff --git a/tf/main.tf b/.buildkite/steps/tf/main.tf similarity index 95% rename from tf/main.tf rename to .buildkite/steps/tf/main.tf index 3ae1f660d..1c372cc32 100644 --- a/tf/main.tf +++ b/.buildkite/steps/tf/main.tf @@ -7,7 +7,7 @@ terraform { } backend "s3" { - + // Must be set from environment } } @@ -61,7 +61,7 @@ resource "aws_security_group" "test-stand-solana" { } ingress { - description = "allow incoming from ci test agent to SOLANA" + description = "allow incoming from world to SOLANA" from_port = 22 to_port = 22 protocol = "tcp" @@ -111,7 +111,7 @@ resource "aws_security_group" "test-stand-proxy" { } ingress { - description = "allow incoming from ci test agent to SOLANA" + description = "allow incoming from world to PROXY" from_port = 22 to_port = 22 protocol = "tcp" @@ -165,7 +165,7 @@ resource "aws_instance" "proxy" { device_name = "/dev/sda1" volume_size = 50 } - //user_data = data.template_file.proxy_init.rendered + tags = { Name = "${var.branch}-test-stand-proxy" purpose = "ci-oz-full-tests" diff --git a/tf/output.tf b/.buildkite/steps/tf/output.tf similarity index 100% rename from tf/output.tf rename to .buildkite/steps/tf/output.tf diff --git a/tf/proxy_init.sh b/.buildkite/steps/tf/proxy_init.sh similarity index 90% rename from tf/proxy_init.sh rename to .buildkite/steps/tf/proxy_init.sh index 3e1923f4b..fb9834fff 100644 --- a/tf/proxy_init.sh +++ b/.buildkite/steps/tf/proxy_init.sh @@ -1,4 +1,7 @@ #!/bin/bash + + +# Install docker sudo apt-get remove docker docker-engine docker.io containerd runc sudo apt-get update sudo apt-get -y install ca-certificates curl gnupg lsb-release @@ -6,14 +9,24 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo apt-get -y install docker-ce docker-ce-cli containerd.io + + +# Install docker-compose sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose + + +# Get docker-compose file cd /opt curl -O https://raw.githubusercontent.com/neonlabsorg/proxy-model.py/${branch}/proxy/docker-compose-test.yml + +# Set required environment variables export REVISION=${revision} export SOLANA_URL=http:\/\/${solana_ip}:8899 + +# Generate docker-compose override file cat > docker-compose-test.override.yml < /dev/null sudo apt-get update + +# Tune instance for Solana requirements(must be applied before start services) sudo bash -c "cat >/etc/sysctl.d/20-solana-udp-buffers.conf </etc/security/limits.d/90-solana-nofiles.conf < docker-compose-test.override.yml < Date: Fri, 18 Mar 2022 13:47:03 +0300 Subject: [PATCH 70/73] typo --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1126e96b4..0bb963ecf 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -82,7 +82,7 @@ steps: agents: queue: "testing" command: - - "tf/destroy.sh" + - ".buildkite/steps/tf/destroy.sh" depends_on: "full_tests" allow_dependency_failure: true artifact_paths: From b43719e06d649b8dd55f4a55e6d09228a2df8a71 Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 18 Mar 2022 16:05:35 +0300 Subject: [PATCH 71/73] enable basic logic --- .buildkite/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0bb963ecf..d693c8921 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -28,9 +28,9 @@ steps: - label: ":coverage: full test suite (FTS)" key: "full_tests" -# if: | -# (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || -# (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") + if: | + (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || + (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") commands: - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` From f118b79956957dcd47ebcdb2e9fc6ced380e27cd Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 18 Mar 2022 16:19:43 +0300 Subject: [PATCH 72/73] add dependences --- .buildkite/pipeline.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d693c8921..005429b9c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -5,6 +5,7 @@ steps: - ".buildkite/steps/upload-image.sh" - label: ":terraform: build infrastructure" + key: "create_infrastructure" agents: queue: "testing" command: @@ -83,7 +84,9 @@ steps: queue: "testing" command: - ".buildkite/steps/tf/destroy.sh" - depends_on: "full_tests" + depends_on: + - "full_tests" + - "create_infrastructure" allow_dependency_failure: true artifact_paths: - ".buildkite/steps/tf/logs/*" From d9c5aa561be61053ec075f97cf154674c1d677fd Mon Sep 17 00:00:00 2001 From: Andrey Voloshin Date: Fri, 18 Mar 2022 16:27:05 +0300 Subject: [PATCH 73/73] playground for if --- .buildkite/pipeline.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 005429b9c..60f52c5c9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,6 +6,9 @@ steps: - label: ":terraform: build infrastructure" key: "create_infrastructure" + if: &main_if | + (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || + (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") agents: queue: "testing" command: @@ -29,9 +32,7 @@ steps: - label: ":coverage: full test suite (FTS)" key: "full_tests" - if: | - (build.pull_request.base_branch == "develop" && !build.pull_request.draft) || - (build.source == "trigger_job" && build.env("NEON_EVM_FULL_TEST_SUITE") == "true") + if: *main_if commands: - PROXY_ADDR=`buildkite-agent meta-data get 'PROXY_IP'` - SOLANA_ADDR=`buildkite-agent meta-data get 'SOLANA_IP'` @@ -82,6 +83,7 @@ steps: - label: ":terraform: destroy infrastructure" agents: queue: "testing" + if: *main_if command: - ".buildkite/steps/tf/destroy.sh" depends_on: