diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index b35f132547..05ea846642 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -770,6 +770,7 @@ interface DBCollection { executeSQL(query: string, params: Array, options?: { query_name?: string, preferred_pool?: string }): Promise>; name: any; + schema: any; } type DBDoc = any; diff --git a/src/server/object_services/map_server.js b/src/server/object_services/map_server.js index d6ce2c8ac5..b3b3fb1d83 100644 --- a/src/server/object_services/map_server.js +++ b/src/server/object_services/map_server.js @@ -85,8 +85,8 @@ class GetMapping { if (!config.DEDUP_ENABLED) return; await Promise.all(Object.values(this.chunks_per_bucket).map(async chunks => { const bucket = chunks[0].bucket; - const dedup_keys = _.compact(_.map(chunks, - chunk => chunk.digest_b64 && Buffer.from(chunk.digest_b64, 'base64'))); + const dedup_keys = chunks.map(chunk => chunk.digest_b64).filter(Boolean); + if (!dedup_keys.length) return; dbg.log0('GetMapping.find_dups: found keys', dedup_keys.length); const dup_chunks_db = await MDStore.instance().find_chunks_by_dedup_key(bucket, dedup_keys); diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index 1e5212b16d..6e3558033b 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -13,6 +13,7 @@ const mime = require('mime-types'); const P = require('../../util/promise'); const dbg = require('../../util/debug_module')(__filename); const db_client = require('../../util/db_client'); +const { decode_json } = require('../../util/postgres_client.js'); const mongo_functions = require('../../util/mongo_functions'); const object_md_schema = require('./schemas/object_md_schema'); @@ -1537,27 +1538,34 @@ class MDStore { /** * @param {nb.Bucket} bucket - * @param {nb.DBBuffer[]} dedup_keys + * @param {string[]} dedup_keys * @returns {Promise} */ async find_chunks_by_dedup_key(bucket, dedup_keys) { - // TODO: This is temporary patch because of binary representation in MongoDB and PostgreSQL - /** @type {nb.ChunkSchemaDB[]} */ - const chunks = await this._chunks.find({ - system: bucket.system._id, - bucket: bucket._id, - dedup_key: { - $in: dedup_keys, - $exists: true - }, - deleted: null, - }, { - sort: { - _id: -1 // get newer chunks first - } - }); - await this.load_blocks_for_chunks(chunks); - return chunks; + if (!dedup_keys?.length) return []; + + const query = ` + SELECT * + FROM ${this._chunks.name} + WHERE + (data ->> 'system' = $1 + AND data ->> 'bucket' = $2 + AND (data ->> 'dedup_key' = ANY($3) + AND data ? 'dedup_key') + AND (data->'deleted' IS NULL OR data->'deleted' = 'null'::jsonb)) + ORDER BY _id DESC; + `; + const values = [`${bucket.system._id}`, `${bucket._id}`, dedup_keys]; + + try { + const res = await this._chunks.executeSQL(query, values); + const chunks = res.rows.map(row => decode_json(this._chunks.schema, row.data)); + await this.load_blocks_for_chunks(chunks); + return chunks; + } catch (err) { + dbg.error('Error while finding chunks by dedup_key. error is ', err); + throw err; + } } iterate_all_chunks_in_buckets(lower_marker, upper_marker, buckets, limit) { diff --git a/src/test/integration_tests/api/s3/test_bucket_replication.js b/src/test/integration_tests/api/s3/test_bucket_replication.js index 818e235e93..c1fe00062f 100644 --- a/src/test/integration_tests/api/s3/test_bucket_replication.js +++ b/src/test/integration_tests/api/s3/test_bucket_replication.js @@ -5,6 +5,7 @@ const mocha = require('mocha'); const assert = require('assert'); const _ = require('lodash'); const P = require('../../../../util/promise'); +const config = require('../../../../../config'); const coretest = require('../../../utils/coretest/coretest'); const { rpc_client, EMAIL } = coretest; //, PASSWORD, SYSTEM const util = require('util'); @@ -264,6 +265,9 @@ mocha.describe('replication collection tests', function() { }); mocha.describe('replication configuration bg worker tests', function() { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; + const self = this; // eslint-disable-line no-invalid-this self.timeout(60000); const bucket1 = 'bucket1-br-bg'; @@ -576,6 +580,9 @@ async function _list_objects_and_wait(s3_owner, bucket, expected_num_of_objects) } mocha.describe('Replication pagination test', function() { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; + const self = this; // eslint-disable-line no-invalid-this self.timeout(60000); const obj_amount = 11; diff --git a/src/test/integration_tests/api/s3/test_chunked_upload.js b/src/test/integration_tests/api/s3/test_chunked_upload.js index c70d2168f9..ed2430dc9a 100644 --- a/src/test/integration_tests/api/s3/test_chunked_upload.js +++ b/src/test/integration_tests/api/s3/test_chunked_upload.js @@ -24,6 +24,9 @@ const default_checksum_algorithm = ChecksumAlgorithm.SHA256; const non_chunked_upload_key = 'non_chunked_upload.txt'; mocha.describe('S3 basic chunked upload tests', async function() { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; + let s3; mocha.before(async () => { diff --git a/src/test/integration_tests/api/s3/test_lifecycle.js b/src/test/integration_tests/api/s3/test_lifecycle.js index 74d1b37455..2b9b0f90aa 100644 --- a/src/test/integration_tests/api/s3/test_lifecycle.js +++ b/src/test/integration_tests/api/s3/test_lifecycle.js @@ -821,6 +821,8 @@ mocha.describe('lifecycle', () => { }; mocha.it('should select rule with longest prefix', async () => { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; const rules = [ test_utils.generate_lifecycle_rule(10, 'short-prefix', 'test1/', [], undefined, undefined), test_utils.generate_lifecycle_rule(17, 'long-prefix', 'test1/logs/', [], undefined, undefined), @@ -834,6 +836,8 @@ mocha.describe('lifecycle', () => { }); mocha.it('should select rule with more tags when prefix is same', async () => { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; const rules = [ test_utils.generate_lifecycle_rule(5, 'one-tag', 'test2/', [{ Key: 'env', Value: 'prod' }], undefined, undefined), test_utils.generate_lifecycle_rule(9, 'two-tags', 'test2/', [ @@ -851,6 +855,8 @@ mocha.describe('lifecycle', () => { }); mocha.it('should select rule with narrower size span when prefix and tags are matching', async () => { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; const rules = [ test_utils.generate_lifecycle_rule(4, 'wide-range', 'test3/', [], 100, 10000), test_utils.generate_lifecycle_rule(6, 'narrow-range', 'test3/', [], 1000, 5000), @@ -865,6 +871,8 @@ mocha.describe('lifecycle', () => { }); mocha.it('should fallback to first matching rule if all filters are equal', async () => { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; const rules = [ test_utils.generate_lifecycle_rule(7, 'rule-a', 'test4/', [], 0, 10000), test_utils.generate_lifecycle_rule(11, 'rule-b', 'test4/', [], 0, 10000), diff --git a/src/test/integration_tests/api/s3/test_namespace_auth.js b/src/test/integration_tests/api/s3/test_namespace_auth.js index 343ba550ac..d6fb6b6b10 100644 --- a/src/test/integration_tests/api/s3/test_namespace_auth.js +++ b/src/test/integration_tests/api/s3/test_namespace_auth.js @@ -21,6 +21,8 @@ const FKEY = 'ns_auth_file'; const config = require('../../../../../config'); mocha.describe('Namespace Auth', function() { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; let s3; mocha.before(async function() { diff --git a/src/test/integration_tests/api/s3/test_notifications.js b/src/test/integration_tests/api/s3/test_notifications.js index 979c30450e..ec4caba508 100644 --- a/src/test/integration_tests/api/s3/test_notifications.js +++ b/src/test/integration_tests/api/s3/test_notifications.js @@ -57,6 +57,8 @@ let expect_test; // eslint-disable-next-line max-lines-per-function mocha.describe('notifications', function() { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; this.timeout(40000); // eslint-disable-line no-invalid-this let s3; diff --git a/src/test/integration_tests/api/s3/test_s3_bucket_policy.js b/src/test/integration_tests/api/s3/test_s3_bucket_policy.js index 43664b6d30..e895c1d559 100644 --- a/src/test/integration_tests/api/s3/test_s3_bucket_policy.js +++ b/src/test/integration_tests/api/s3/test_s3_bucket_policy.js @@ -175,6 +175,10 @@ async function setup() { }); } +// @ts-ignore +// Do not run below tests if DB is not PostgreSQL +if (config.DB_TYPE !== 'postgres') return; + /*eslint max-lines-per-function: ["error", 3000]*/ mocha.describe('s3_bucket_policy', function() { mocha.before(setup); diff --git a/src/test/integration_tests/api/s3/test_s3_bucket_policy_iam_user.js b/src/test/integration_tests/api/s3/test_s3_bucket_policy_iam_user.js index 35cdc255f5..9118559984 100644 --- a/src/test/integration_tests/api/s3/test_s3_bucket_policy_iam_user.js +++ b/src/test/integration_tests/api/s3/test_s3_bucket_policy_iam_user.js @@ -149,6 +149,9 @@ async function setup() { } mocha.describe('Integration between IAM and S3 bucket policy', async function() { + // Skip tests for other DB's + if (config.DB_TYPE !== 'postgres') return; + mocha.before(setup); mocha.after(async function() { this.timeout(60000); // eslint-disable-line no-invalid-this diff --git a/src/test/integration_tests/api/s3/test_s3_encryption.js b/src/test/integration_tests/api/s3/test_s3_encryption.js index 7a6925a516..47eb695e69 100644 --- a/src/test/integration_tests/api/s3/test_s3_encryption.js +++ b/src/test/integration_tests/api/s3/test_s3_encryption.js @@ -59,6 +59,8 @@ async function get_s3_instances() { } mocha.describe('Bucket Encryption Operations', async () => { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; const BKT = 'sloth-bucket-encryption'; let local_s3; diff --git a/src/test/integration_tests/api/s3/test_s3_ops.js b/src/test/integration_tests/api/s3/test_s3_ops.js index 5fe5c19032..5f2546d7ba 100644 --- a/src/test/integration_tests/api/s3/test_s3_ops.js +++ b/src/test/integration_tests/api/s3/test_s3_ops.js @@ -735,6 +735,8 @@ mocha.describe('s3_ops', function() { }); async function test_object_ops(bucket_name, bucket_type, caching, remote_endpoint_options, skip) { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; const is_azure_namespace = is_namespace_blob_bucket(bucket_type, remote_endpoint_options && remote_endpoint_options.endpoint_type); const is_azure_mock = is_namespace_blob_mock(bucket_type, remote_endpoint_options && remote_endpoint_options.endpoint_type); diff --git a/src/test/integration_tests/db/test_md_store.js b/src/test/integration_tests/db/test_md_store.js index 7e69c26df1..6d24ba24d5 100644 --- a/src/test/integration_tests/db/test_md_store.js +++ b/src/test/integration_tests/db/test_md_store.js @@ -380,6 +380,42 @@ mocha.describe('md_store', function() { return md_store.delete_chunks_by_ids(_.map(chunks, '_id')); }); + mocha.it('find_chunks_by_dedup_key()', async () => { + if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path + const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } }; + const chunk = { + _id: md_store.make_md_id(), + system: system_id, + bucket: bucket._id, + frags: [{ _id: md_store.make_md_id() }], + size: 10, + frag_size: 10, + dedup_key: Buffer.from('noobaa') + }; + await md_store.insert_chunks([chunk]); + const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, [Buffer.from('noobaa').toString('base64')]); + assert(Array.isArray(chunksArr)); + assert(chunksArr.length >= 1); + assert(chunksArr[0].frags[0]?._id?.toString() === chunk.frags[0]._id.toString()); + }); + + mocha.it('test find_chunks_by_dedup_key - dedup_key doesnt exist in DB', async () => { + if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path + const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } }; + const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, [Buffer.from('unknownkey').toString('base64')]); + assert(Array.isArray(chunksArr)); + assert(chunksArr.length === 0); + }); + + mocha.it('find_chunks_by_dedup_key empty dedup_key array passed', async () => { + if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path + const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } }; + + const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, []); + assert(Array.isArray(chunksArr)); + assert(chunksArr.length === 0); + }); + }); diff --git a/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js b/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js index bf314a0adc..02ce357528 100644 --- a/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js +++ b/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js @@ -113,6 +113,10 @@ class ReclaimerMock extends AgentBlocksReclaimer { } +// @ts-ignore +// Do not run below tests if DB is not PostgreSQL +if (config.DB_TYPE !== 'postgres') return; + mocha.describe('not mocked agent_blocks_reclaimer', function() { const object_io = new ObjectIO(); diff --git a/src/test/integration_tests/internal/test_encryption.js b/src/test/integration_tests/internal/test_encryption.js index 572e226d1d..e4ada4a0e2 100644 --- a/src/test/integration_tests/internal/test_encryption.js +++ b/src/test/integration_tests/internal/test_encryption.js @@ -31,6 +31,10 @@ const key_rotator = new KeyRotator({ name: 'kr'}); config.MIN_CHUNK_AGE_FOR_DEDUP = 0; +// @ts-ignore +// Do not run below tests if DB is not PostgreSQL +if (config.DB_TYPE !== 'postgres') return; + mocha.describe('Encryption tests', function() { const { rpc_client, EMAIL, SYSTEM } = coretest; let response_account; @@ -131,7 +135,7 @@ mocha.describe('Encryption tests', function() { })); }); - mocha.it('create accounts and compare acount access keys succefully', async function() { + mocha.it('create accounts and compare acount access keys succefully', async function() { this.timeout(600000); // eslint-disable-line no-invalid-this const db_system = await db_client.collection('systems').findOne({ name: SYSTEM }); const new_account_params = { @@ -141,7 +145,7 @@ mocha.describe('Encryption tests', function() { let i; for (i = 0; i < 20; i++) { response_account = await rpc_client.account.create_account({...new_account_params, - email: `email${i}`, name: `name${i}`}); + email: `email${i}`, name: `name${i}`}); accounts.push({email: `email${i}`, create_account_result: response_account, index: i}); const db_account = await db_client.collection('accounts').findOne({ email: `email${i}` }); const system_store_account = account_by_name(system_store.data.accounts, `email${i}`); @@ -213,7 +217,7 @@ mocha.describe('Encryption tests', function() { const system_store_account = account_by_name(system_store.data.accounts, cur_account.email); const db_ns_resource = await db_client.collection('namespace_resources').findOne({ name: namespace_resource_name }); const system_store_ns_resource = pool_by_name(system_store.data.namespace_resources, - namespace_resource_name); // system store data supposed to be decrypted + namespace_resource_name); // system store data supposed to be decrypted // check s3 creds key in db is encrypted const secrets = { db_secret: db_ns_resource.connection.secret_key, @@ -448,7 +452,7 @@ mocha.describe('Encryption tests', function() { name: BKT, }); }); - mocha.it('regenerate creds for all accounts (non coretest) succefully', async function() { + mocha.it('regenerate creds for all accounts (non coretest) succefully', async function() { this.timeout(600000); // eslint-disable-line no-invalid-this await P.all(_.map(accounts, async cur_account => { await rpc_client.account.generate_account_keys({ email: cur_account.email }); @@ -466,7 +470,7 @@ mocha.describe('Encryption tests', function() { })); }); // TODO: remove the comment - mocha.it('delete accounts succefully', async function() { + mocha.it('delete accounts succefully', async function() { this.timeout(600000); // eslint-disable-line no-invalid-this await P.all(_.map(accounts, async cur_account => { await rpc_client.account.delete_account({ email: cur_account.email}); @@ -479,7 +483,6 @@ mocha.describe('Encryption tests', function() { }); }); - /////////////// ROTATION & DISABLE & ENABLE TESTS ///////////////////////// mocha.describe('Rotation tests', function() { const { rpc_client, EMAIL, SYSTEM } = coretest; @@ -588,7 +591,7 @@ mocha.describe('Rotation tests', function() { await system_store.load(); const original_secrets = await get_account_secrets_from_system_store_and_db(accounts[0].email, 's3_creds'); await rpc_client.system.disable_master_key({entity: new SensitiveString(accounts[0].email), - entity_type: 'ACCOUNT'}); + entity_type: 'ACCOUNT'}); await system_store.load(); const system_store_account = account_by_name(system_store.data.accounts, accounts[0].email); const secrets = await get_account_secrets_from_system_store_and_db(accounts[0].email, 's3_creds'); @@ -611,7 +614,7 @@ mocha.describe('Rotation tests', function() { await system_store.load(); const original_secrets = await get_account_secrets_from_system_store_and_db(accounts[2].email, 's3_creds'); await rpc_client.system.disable_master_key({entity: new SensitiveString(accounts[2].email), - entity_type: 'ACCOUNT'}); + entity_type: 'ACCOUNT'}); await system_store.load(); const system_store_account = account_by_name(system_store.data.accounts, accounts[2].email); const secrets = await get_account_secrets_from_system_store_and_db(accounts[2].email, 's3_creds'); diff --git a/src/test/integration_tests/internal/test_map_builder.js b/src/test/integration_tests/internal/test_map_builder.js index 686ae60702..da9fc6f428 100644 --- a/src/test/integration_tests/internal/test_map_builder.js +++ b/src/test/integration_tests/internal/test_map_builder.js @@ -297,6 +297,9 @@ mocha.describe('map_builder', function() { }); mocha.it('does block movement across storage class', async function() { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; + this.timeout(600_000); // eslint-disable-line no-invalid-this const test_bucket_name = "block-movement-test-1"; diff --git a/src/test/integration_tests/internal/test_object_io.js b/src/test/integration_tests/internal/test_object_io.js index e29bdd0a6c..e4833fd5a3 100644 --- a/src/test/integration_tests/internal/test_object_io.js +++ b/src/test/integration_tests/internal/test_object_io.js @@ -12,6 +12,7 @@ const util = require('util'); const crypto = require('crypto'); const assert = require('assert'); const stream = require('stream'); +const config = require('../../../../config'); const ObjectIO = require('../../../sdk/object_io'); const { crypto_random_string } = require('../../../util/string_utils'); @@ -40,6 +41,9 @@ coretest.describe_mapper_test_case({ chunk_coder_config, }) => { + // Do not run below tests if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; + // TODO we need to create more nodes and pools to support all MAPPER_TEST_CASES if (data_placement !== 'SPREAD' || num_pools !== 1 || total_blocks > 10) return; // if (total_blocks > 10) return; diff --git a/src/test/integration_tests/internal/test_tiering_ttl_worker.js b/src/test/integration_tests/internal/test_tiering_ttl_worker.js index 896a9f2edd..0dd950aece 100644 --- a/src/test/integration_tests/internal/test_tiering_ttl_worker.js +++ b/src/test/integration_tests/internal/test_tiering_ttl_worker.js @@ -63,6 +63,9 @@ async function load_chunks(obj) { } mocha.describe('tiering_ttl_worker', function() { + // Skip test if DB is not PostgreSQL + if (config.DB_TYPE !== 'postgres') return; + const BUCKET_NAME = 'tiering-ttl-worker-test-bucket'; const default_ttl_ms = config.TIERING_TTL_MS; const ttl_worker = new TieringTTLWorker({ name: 'test_tiering_ttl_worker', client: rpc_client });