Skip to content

Generate optimized SQL query to improve performance of find_chunks_by_dedup_key method #9318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
karthikps97 wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Generate optimized SQL query to improve performance of find_chunks_by_dedup_key method #9318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3eaf530
Improve performance of find_chunks_by_dedup_key method
karthikps97 Nov 27, 2025
9295259
minor code improvements
karthikps97 Dec 9, 2025
612408d
refactoring code
karthikps97 Dec 17, 2025
8e11ad5
throw error from function
karthikps97 Dec 23, 2025
3be5ef6
Skip tests for MongoDB
karthikps97 Dec 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/sdk/nb.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,7 @@ interface DBCollection {

executeSQL<T>(query: string, params: Array<any>, options?: { query_name?: string, preferred_pool?: string }): Promise<sqlResult<T>>;
name: any;
schema: any;
}

type DBDoc = any;
Expand Down
4 changes: 2 additions & 2 deletions src/server/object_services/map_server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ class GetMapping {
if (!config.DEDUP_ENABLED) return;
await Promise.all(Object.values(this.chunks_per_bucket).map(async chunks => {
const bucket = chunks[0].bucket;
const dedup_keys = _.compact(_.map(chunks,
chunk => chunk.digest_b64 && Buffer.from(chunk.digest_b64, 'base64')));
const dedup_keys = chunks.map(chunk => chunk.digest_b64).filter(Boolean);

if (!dedup_keys.length) return;
dbg.log0('GetMapping.find_dups: found keys', dedup_keys.length);
const dup_chunks_db = await MDStore.instance().find_chunks_by_dedup_key(bucket, dedup_keys);
Expand Down
42 changes: 25 additions & 17 deletions src/server/object_services/md_store.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const mime = require('mime-types');
const P = require('../../util/promise');
const dbg = require('../../util/debug_module')(__filename);
const db_client = require('../../util/db_client');
const { decode_json } = require('../../util/postgres_client.js');

const mongo_functions = require('../../util/mongo_functions');
const object_md_schema = require('./schemas/object_md_schema');
Expand Down Expand Up @@ -1541,23 +1542,30 @@ class MDStore {
* @returns {Promise<nb.ChunkSchemaDB[]>}
*/
async find_chunks_by_dedup_key(bucket, dedup_keys) {
// TODO: This is temporary patch because of binary representation in MongoDB and PostgreSQL
/** @type {nb.ChunkSchemaDB[]} */
const chunks = await this._chunks.find({
system: bucket.system._id,
bucket: bucket._id,
dedup_key: {
$in: dedup_keys,
$exists: true
},
deleted: null,
}, {
sort: {
_id: -1 // get newer chunks first
}
});
await this.load_blocks_for_chunks(chunks);
return chunks;
if (!dedup_keys?.length) return [];

const query = `
SELECT *
FROM ${this._chunks.name}
WHERE
(data ->> 'system' = $1
AND data ->> 'bucket' = $2
AND (data ->> 'dedup_key' = ANY($3)
AND data ? 'dedup_key')
AND (data->'deleted' IS NULL OR data->'deleted' = 'null'::jsonb))
ORDER BY _id DESC;
`;
const values = [`${bucket.system._id}`, `${bucket._id}`, dedup_keys];

try {
const res = await this._chunks.executeSQL(query, values);
const chunks = res.rows.map(row => decode_json(this._chunks.schema, row.data));
await this.load_blocks_for_chunks(chunks);
return chunks;
} catch (err) {
dbg.error('Error while finding chunks by dedup_key. error is ', err);
throw err;
}
}

iterate_all_chunks_in_buckets(lower_marker, upper_marker, buckets, limit) {
Expand Down
7 changes: 7 additions & 0 deletions src/test/integration_tests/api/s3/test_bucket_replication.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const mocha = require('mocha');
const assert = require('assert');
const _ = require('lodash');
const P = require('../../../../util/promise');
const config = require('../../../../../config');
const coretest = require('../../../utils/coretest/coretest');
const { rpc_client, EMAIL } = coretest; //, PASSWORD, SYSTEM
const util = require('util');
Expand Down Expand Up @@ -264,6 +265,9 @@ mocha.describe('replication collection tests', function() {
});

mocha.describe('replication configuration bg worker tests', function() {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

const self = this; // eslint-disable-line no-invalid-this
self.timeout(60000);
const bucket1 = 'bucket1-br-bg';
Expand Down Expand Up @@ -576,6 +580,9 @@ async function _list_objects_and_wait(s3_owner, bucket, expected_num_of_objects)
}

mocha.describe('Replication pagination test', function() {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

const self = this; // eslint-disable-line no-invalid-this
self.timeout(60000);
const obj_amount = 11;
Expand Down
3 changes: 3 additions & 0 deletions src/test/integration_tests/api/s3/test_chunked_upload.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ const default_checksum_algorithm = ChecksumAlgorithm.SHA256;
const non_chunked_upload_key = 'non_chunked_upload.txt';

mocha.describe('S3 basic chunked upload tests', async function() {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

let s3;

mocha.before(async () => {
Expand Down
8 changes: 8 additions & 0 deletions src/test/integration_tests/api/s3/test_lifecycle.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,8 @@ mocha.describe('lifecycle', () => {
};

mocha.it('should select rule with longest prefix', async () => {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
const rules = [
test_utils.generate_lifecycle_rule(10, 'short-prefix', 'test1/', [], undefined, undefined),
test_utils.generate_lifecycle_rule(17, 'long-prefix', 'test1/logs/', [], undefined, undefined),
Expand All @@ -834,6 +836,8 @@ mocha.describe('lifecycle', () => {
});

mocha.it('should select rule with more tags when prefix is same', async () => {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
const rules = [
test_utils.generate_lifecycle_rule(5, 'one-tag', 'test2/', [{ Key: 'env', Value: 'prod' }], undefined, undefined),
test_utils.generate_lifecycle_rule(9, 'two-tags', 'test2/', [
Expand All @@ -851,6 +855,8 @@ mocha.describe('lifecycle', () => {
});

mocha.it('should select rule with narrower size span when prefix and tags are matching', async () => {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
const rules = [
test_utils.generate_lifecycle_rule(4, 'wide-range', 'test3/', [], 100, 10000),
test_utils.generate_lifecycle_rule(6, 'narrow-range', 'test3/', [], 1000, 5000),
Expand All @@ -865,6 +871,8 @@ mocha.describe('lifecycle', () => {
});

mocha.it('should fallback to first matching rule if all filters are equal', async () => {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
const rules = [
test_utils.generate_lifecycle_rule(7, 'rule-a', 'test4/', [], 0, 10000),
test_utils.generate_lifecycle_rule(11, 'rule-b', 'test4/', [], 0, 10000),
Expand Down
2 changes: 2 additions & 0 deletions src/test/integration_tests/api/s3/test_namespace_auth.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ const FKEY = 'ns_auth_file';
const config = require('../../../../../config');

mocha.describe('Namespace Auth', function() {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

let s3;
mocha.before(async function() {
Expand Down
2 changes: 2 additions & 0 deletions src/test/integration_tests/api/s3/test_notifications.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ let expect_test;

// eslint-disable-next-line max-lines-per-function
mocha.describe('notifications', function() {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

this.timeout(40000); // eslint-disable-line no-invalid-this
let s3;
Expand Down
4 changes: 4 additions & 0 deletions src/test/integration_tests/api/s3/test_s3_bucket_policy.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,10 @@ async function setup() {
});
}

// @ts-ignore
// Do not run below tests if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
Comment on lines +178 to +180
Copy link

@coderabbitai coderabbitai bot Dec 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Move the PostgreSQL guard inside the test suite for consistency and correctness.

The module-level return statement is technically invalid JavaScript syntax (flagged by static analysis) and creates inconsistency across the codebase. Other test files in this PR place the guard inside the test suite function (e.g., test_tiering_ttl_worker.js at line 67, test_object_io.js at line 45), which is both syntactically correct and follows mocha best practices.

🔎 Proposed fix 
-// @ts-ignore
-// Do not run below tests if DB is not PostgreSQL
-if (config.DB_TYPE !== 'postgres') return;
-
 /*eslint max-lines-per-function: ["error", 3000]*/
 mocha.describe('s3_bucket_policy', function() {
+    // Do not run below tests if DB is not PostgreSQL
+    if (config.DB_TYPE !== 'postgres') return;
+
     mocha.before(setup);
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
// @ts-ignore
// Do not run below tests if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
/*eslint max-lines-per-function: ["error", 3000]*/
mocha.describe('s3_bucket_policy', function() {
// Do not run below tests if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
mocha.before(setup);
🧰 Tools
🪛 Biome (2.1.2)

[error] 180-180: Illegal return statement outside of a function

(parse)

🤖 Prompt for AI Agents 
In src/test/integration_tests/api/s3/test_s3_bucket_policy.js around lines
178-180, the module-level PostgreSQL guard (module-level return) must be moved
inside the test suite; remove the top-level "if (config.DB_TYPE !== 'postgres')
return;" and instead place that same conditional as the first statement inside
the relevant describe(...) or top-level suite function so the suite exits early
when DB_TYPE !== 'postgres' (keeping the guard logic intact and avoiding invalid
top-level return syntax).


/*eslint max-lines-per-function: ["error", 3000]*/
mocha.describe('s3_bucket_policy', function() {
mocha.before(setup);
Expand Down
2 changes: 2 additions & 0 deletions src/test/integration_tests/api/s3/test_s3_encryption.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ async function get_s3_instances() {
}

mocha.describe('Bucket Encryption Operations', async () => {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

const BKT = 'sloth-bucket-encryption';
let local_s3;
Expand Down
2 changes: 2 additions & 0 deletions src/test/integration_tests/api/s3/test_s3_ops.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -735,6 +735,8 @@ mocha.describe('s3_ops', function() {
});

async function test_object_ops(bucket_name, bucket_type, caching, remote_endpoint_options, skip) {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

const is_azure_namespace = is_namespace_blob_bucket(bucket_type, remote_endpoint_options && remote_endpoint_options.endpoint_type);
const is_azure_mock = is_namespace_blob_mock(bucket_type, remote_endpoint_options && remote_endpoint_options.endpoint_type);
Expand Down
36 changes: 36 additions & 0 deletions src/test/integration_tests/db/test_md_store.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,42 @@ mocha.describe('md_store', function() {
return md_store.delete_chunks_by_ids(_.map(chunks, '_id'));
});

mocha.it('find_chunks_by_dedup_key()', async () => {
Copy link
Member

@dannyzaken dannyzaken Dec 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

worth adding more tests for different cases. e.g.: test that we return an empty array when there is no match, etc.

Copy link
Member Author

@karthikps97 karthikps97 Dec 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added one test for this scenario.

if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path
const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } };
const chunk = {
_id: md_store.make_md_id(),
system: system_id,
bucket: bucket._id,
frags: [{ _id: md_store.make_md_id() }],
size: 10,
frag_size: 10,
dedup_key: Buffer.from('noobaa')
};
await md_store.insert_chunks([chunk]);
const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, [Buffer.from('noobaa').toString('base64')]);
assert(Array.isArray(chunksArr));
assert(chunksArr.length >= 1);
assert(chunksArr[0].frags[0]?._id?.toString() === chunk.frags[0]._id.toString());
});

mocha.it('test find_chunks_by_dedup_key - dedup_key doesnt exist in DB', async () => {
if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path
const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } };
const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, [Buffer.from('unknownkey').toString('base64')]);
assert(Array.isArray(chunksArr));
assert(chunksArr.length === 0);
});

mocha.it('find_chunks_by_dedup_key empty dedup_key array passed', async () => {
Copy link
Member

@dannyzaken dannyzaken Dec 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why does this specific case need to be tested? We don't expect an empty array, IIRC

Copy link
Member Author

@karthikps97 karthikps97 Dec 23, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We return early from the function in case an empty dedup_keys array is passed. So, added this test case.

if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path
const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } };

const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, []);
assert(Array.isArray(chunksArr));
assert(chunksArr.length === 0);
});

});


Expand Down
4 changes: 4 additions & 0 deletions src/test/integration_tests/internal/test_agent_blocks_reclaimer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,10 @@ class ReclaimerMock extends AgentBlocksReclaimer {
}


// @ts-ignore
// Do not run below tests if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
Comment on lines +116 to +118
Copy link

@coderabbitai coderabbitai bot Dec 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Move the PostgreSQL guard inside the test suite for consistency and correctness.

The module-level return statement is technically invalid JavaScript syntax (flagged by static analysis) and creates inconsistency. The correct pattern is demonstrated in test_tiering_ttl_worker.js (lines 66-67 in this PR), where the guard is placed inside the test suite function.

🔎 Proposed fix 
-// @ts-ignore
-// Do not run below tests if DB is not PostgreSQL
-if (config.DB_TYPE !== 'postgres') return;
-
 mocha.describe('not mocked agent_blocks_reclaimer', function() {
+    // Do not run below tests if DB is not PostgreSQL
+    if (config.DB_TYPE !== 'postgres') return;
 
     const object_io = new ObjectIO();
🧰 Tools
🪛 Biome (2.1.2)

[error] 118-118: Illegal return statement outside of a function

(parse)

🤖 Prompt for AI Agents 
In src/test/integration_tests/internal/test_agent_blocks_reclaimer.js around
lines 116-118, there is a module-level early return checking config.DB_TYPE !==
'postgres' which is invalid and inconsistent; move that PostgreSQL guard inside
the test suite by removing the top-level return and instead place the same check
at the start of the describe/it block (or use describe.skip when DB_TYPE !==
'postgres') so the suite exits/gets skipped at runtime rather than using a
module-level return; ensure any TypeScript ignore comments are removed or
adjusted as needed.


mocha.describe('not mocked agent_blocks_reclaimer', function() {

const object_io = new ObjectIO();
Expand Down
19 changes: 11 additions & 8 deletions src/test/integration_tests/internal/test_encryption.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ const key_rotator = new KeyRotator({ name: 'kr'});

config.MIN_CHUNK_AGE_FOR_DEDUP = 0;

// @ts-ignore
// Do not run below tests if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;
Copy link
Member

@dannyzaken dannyzaken Dec 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@karthikps97 @liranmauda, why is this needed?

Copy link
Member Author

@karthikps97 karthikps97 Dec 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These tests are executed as part of both PostgreSQL tests and MongoDB tests. We are skipping this for the MongoDB run as it will fail since executeSQL is not implemented by mongo client.


Comment on lines +34 to +37
Copy link

@coderabbitai coderabbitai bot Dec 9, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Fix illegal top‑level return and use Mocha‑level skipping instead

The guard:

// @ts-ignore
// Do not run below tests if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

is at file scope, which is a syntax error in Node/JS (return is only allowed inside functions). This matches the Biome parse error on Line 36. As written, this file will fail to parse, so the tests will never run.

Suggest moving the DB-type guard into Mocha context and removing the // @ts-ignore, for example:

 config.MIN_CHUNK_AGE_FOR_DEDUP = 0;
-
-// @ts-ignore
-// Do not run below tests if DB is not PostgreSQL
-if (config.DB_TYPE !== 'postgres') return;
+// Run these tests only when DB is PostgreSQL
+const IS_POSTGRES = config.DB_TYPE === 'postgres';
@@
-mocha.describe('Encryption tests', function() {
+mocha.describe('Encryption tests', function() {
+    mocha.before(function() {
+        if (!IS_POSTGRES) this.skip();
+    });
@@
-mocha.describe('Rotation tests', function() {
+mocha.describe('Rotation tests', function() {
+    mocha.before(function() {
+        if (!IS_POSTGRES) this.skip();
+    });

This keeps the suite PostgreSQL-only, avoids top-level control flow, and gives Mocha proper “skipped” semantics instead of a parse error. You can then drop the // @ts-ignore entirely.

🧰 Tools
🪛 Biome (2.1.2)

[error] 36-36: Illegal return statement outside of a function

(parse)

🤖 Prompt for AI Agents 
In src/test/integration_tests/internal/test_encryption.js around lines 34–37,
remove the top-level "if (config.DB_TYPE !== 'postgres') return;" and the "//
@ts-ignore" (top-level return is illegal and causes parse errors); instead, keep
the test file but move the DB-type guard into the Mocha suite by placing a
before() hook (or the first statement inside the top-level describe) that checks
config.DB_TYPE !== 'postgres' and calls this.skip() (or uses the suite-level
skip mechanism) so the suite is properly skipped under Mocha when not running
against PostgreSQL.

Copy link
Contributor

@liranmauda liranmauda Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was a psedo-code
the actuall code should be something like:

mocha.describe('Encryption tests', function() {
  before(function () {
    if (config.DB_TYPE !== 'postgres') {
      this.skip(); 
    }
   ...
  });

On any top level describe.

Copy link

@coderabbitai coderabbitai bot Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@liranmauda, thanks for the clarification! I understand now—you want to use a before hook inside each top-level describe to skip the tests when the DB is not PostgreSQL.

Here's how to apply this pattern to both top-level describes in the file:

config.MIN_CHUNK_AGE_FOR_DEDUP = 0;

mocha.describe('Encryption tests', function() {
    mocha.before(function() {
        if (config.DB_TYPE !== 'postgres') {
            this.skip();
        }
    });

    const { rpc_client, EMAIL, SYSTEM } = coretest;
    let response_account;
    // ... rest of the tests
});

// ... (other code)

mocha.describe('Rotation tests', function() {
    mocha.before(function() {
        if (config.DB_TYPE !== 'postgres') {
            this.skip();
        }
    });

    const { rpc_client, EMAIL, SYSTEM } = coretest;
    let accounts;
    let buckets;
    // ... rest of the tests
});

This removes the illegal top-level return statement (and the // @ts-ignore) and properly skips both test suites when not running against PostgreSQL. Each suite will show as "skipped" in the test output rather than causing a parse error.

Copy link
Member Author

@karthikps97 karthikps97 Dec 9, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Skipping the test using before hook would still run all the "after" hooks. This would throw errors specifically in the unpopulate_system function call as the initilaization would not have taken place. I would have to repeat the same skip statements in each 'after' hook. Wouldn't a single return or conditional if be better?

Copy link

@coderabbitai coderabbitai bot Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like the humans are having a chat. I'll hop back into my burrow for now. If you need me again, just tag @coderabbitai in a new comment, and I'll come hopping out!

mocha.describe('Encryption tests', function() {
const { rpc_client, EMAIL, SYSTEM } = coretest;
let response_account;
Expand Down Expand Up @@ -131,7 +135,7 @@ mocha.describe('Encryption tests', function() {
}));
});

mocha.it('create accounts and compare acount access keys succefully', async function() {
mocha.it('create accounts and compare acount access keys succefully', async function() {
this.timeout(600000); // eslint-disable-line no-invalid-this
const db_system = await db_client.collection('systems').findOne({ name: SYSTEM });
const new_account_params = {
Expand All @@ -141,7 +145,7 @@ mocha.describe('Encryption tests', function() {
let i;
for (i = 0; i < 20; i++) {
response_account = await rpc_client.account.create_account({...new_account_params,
email: `email${i}`, name: `name${i}`});
email: `email${i}`, name: `name${i}`});
accounts.push({email: `email${i}`, create_account_result: response_account, index: i});
const db_account = await db_client.collection('accounts').findOne({ email: `email${i}` });
const system_store_account = account_by_name(system_store.data.accounts, `email${i}`);
Expand Down Expand Up @@ -213,7 +217,7 @@ mocha.describe('Encryption tests', function() {
const system_store_account = account_by_name(system_store.data.accounts, cur_account.email);
const db_ns_resource = await db_client.collection('namespace_resources').findOne({ name: namespace_resource_name });
const system_store_ns_resource = pool_by_name(system_store.data.namespace_resources,
namespace_resource_name); // system store data supposed to be decrypted
namespace_resource_name); // system store data supposed to be decrypted
// check s3 creds key in db is encrypted
const secrets = {
db_secret: db_ns_resource.connection.secret_key,
Expand Down Expand Up @@ -448,7 +452,7 @@ mocha.describe('Encryption tests', function() {
name: BKT,
});
});
mocha.it('regenerate creds for all accounts (non coretest) succefully', async function() {
mocha.it('regenerate creds for all accounts (non coretest) succefully', async function() {
this.timeout(600000); // eslint-disable-line no-invalid-this
await P.all(_.map(accounts, async cur_account => {
await rpc_client.account.generate_account_keys({ email: cur_account.email });
Expand All @@ -466,7 +470,7 @@ mocha.describe('Encryption tests', function() {
}));
});
// TODO: remove the comment
mocha.it('delete accounts succefully', async function() {
mocha.it('delete accounts succefully', async function() {
this.timeout(600000); // eslint-disable-line no-invalid-this
await P.all(_.map(accounts, async cur_account => {
await rpc_client.account.delete_account({ email: cur_account.email});
Expand All @@ -479,7 +483,6 @@ mocha.describe('Encryption tests', function() {
});
});


/////////////// ROTATION & DISABLE & ENABLE TESTS /////////////////////////
mocha.describe('Rotation tests', function() {
const { rpc_client, EMAIL, SYSTEM } = coretest;
Expand Down Expand Up @@ -588,7 +591,7 @@ mocha.describe('Rotation tests', function() {
await system_store.load();
const original_secrets = await get_account_secrets_from_system_store_and_db(accounts[0].email, 's3_creds');
await rpc_client.system.disable_master_key({entity: new SensitiveString(accounts[0].email),
entity_type: 'ACCOUNT'});
entity_type: 'ACCOUNT'});
await system_store.load();
const system_store_account = account_by_name(system_store.data.accounts, accounts[0].email);
const secrets = await get_account_secrets_from_system_store_and_db(accounts[0].email, 's3_creds');
Expand All @@ -611,7 +614,7 @@ mocha.describe('Rotation tests', function() {
await system_store.load();
const original_secrets = await get_account_secrets_from_system_store_and_db(accounts[2].email, 's3_creds');
await rpc_client.system.disable_master_key({entity: new SensitiveString(accounts[2].email),
entity_type: 'ACCOUNT'});
entity_type: 'ACCOUNT'});
await system_store.load();
const system_store_account = account_by_name(system_store.data.accounts, accounts[2].email);
const secrets = await get_account_secrets_from_system_store_and_db(accounts[2].email, 's3_creds');
Expand Down
3 changes: 3 additions & 0 deletions src/test/integration_tests/internal/test_map_builder.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,9 @@ mocha.describe('map_builder', function() {
});

mocha.it('does block movement across storage class', async function() {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

this.timeout(600_000); // eslint-disable-line no-invalid-this
const test_bucket_name = "block-movement-test-1";

Expand Down
4 changes: 4 additions & 0 deletions src/test/integration_tests/internal/test_object_io.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const util = require('util');
const crypto = require('crypto');
const assert = require('assert');
const stream = require('stream');
const config = require('../../../../config');

const ObjectIO = require('../../../sdk/object_io');
const { crypto_random_string } = require('../../../util/string_utils');
Expand Down Expand Up @@ -40,6 +41,9 @@ coretest.describe_mapper_test_case({
chunk_coder_config,
}) => {

// Do not run below tests if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

// TODO we need to create more nodes and pools to support all MAPPER_TEST_CASES
if (data_placement !== 'SPREAD' || num_pools !== 1 || total_blocks > 10) return;
// if (total_blocks > 10) return;
Expand Down
3 changes: 3 additions & 0 deletions src/test/integration_tests/internal/test_tiering_ttl_worker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ async function load_chunks(obj) {
}

mocha.describe('tiering_ttl_worker', function() {
// Skip test if DB is not PostgreSQL
if (config.DB_TYPE !== 'postgres') return;

const BUCKET_NAME = 'tiering-ttl-worker-test-bucket';
const default_ttl_ms = config.TIERING_TTL_MS;
const ttl_worker = new TieringTTLWorker({ name: 'test_tiering_ttl_worker', client: rpc_client });
Expand Down
Loading