Improve performance of find_chunks_by_dedup_key method

Signed-off-by: Karthik P S <[email protected]>

Author: karthikps97

src/sdk/nb.d.ts

@@ -770,6 +770,7 @@ interface DBCollection {
 
     executeSQL<T>(query: string, params: Array<any>, options?: { query_name?: string, preferred_pool?: string }): Promise<sqlResult<T>>;
     name: any;
+    schema: any;
 }
 
 type DBDoc = any;

src/server/object_services/map_server.js

@@ -85,8 +85,12 @@ class GetMapping {
         if (!config.DEDUP_ENABLED) return;
         await Promise.all(Object.values(this.chunks_per_bucket).map(async chunks => {
             const bucket = chunks[0].bucket;
-            const dedup_keys = _.compact(_.map(chunks,
-                chunk => chunk.digest_b64 && Buffer.from(chunk.digest_b64, 'base64')));
+            const dedup_keys = [];
+            chunks.forEach(chunk => {
+                if (chunk?.digest_b64) {
+                    dedup_keys.push(chunk.digest_b64);
+                }
+            });
             if (!dedup_keys.length) return;
             dbg.log0('GetMapping.find_dups: found keys', dedup_keys.length);
             const dup_chunks_db = await MDStore.instance().find_chunks_by_dedup_key(bucket, dedup_keys);

src/server/object_services/md_store.js

@@ -13,6 +13,7 @@ const mime = require('mime-types');
 const P = require('../../util/promise');
 const dbg = require('../../util/debug_module')(__filename);
 const db_client = require('../../util/db_client');
+const { decode_json } = require('../../util/postgres_client.js');
 
 const mongo_functions = require('../../util/mongo_functions');
 const object_md_schema = require('./schemas/object_md_schema');
@@ -1541,23 +1542,28 @@ class MDStore {
      * @returns {Promise<nb.ChunkSchemaDB[]>}
      */
     async find_chunks_by_dedup_key(bucket, dedup_keys) {
-        // TODO: This is temporary patch because of binary representation in MongoDB and PostgreSQL
-        /** @type {nb.ChunkSchemaDB[]} */
-        const chunks = await this._chunks.find({
-            system: bucket.system._id,
-            bucket: bucket._id,
-            dedup_key: {
-                $in: dedup_keys,
-                $exists: true
-            },
-            deleted: null,
-        }, {
-            sort: {
-                _id: -1 // get newer chunks first
-            }
-        });
-        await this.load_blocks_for_chunks(chunks);
-        return chunks;
+        const values = [];
+        let query = `SELECT * FROM ${this._chunks.name} WHERE (data ->> 'system' = $1 AND data ->> 'bucket' = $2`;
+        values.push(`${bucket.system._id}`, `${bucket._id}`);
+
+        if (dedup_keys.length) {
+            query += ` AND (data ->> 'dedup_key' = ANY($3) AND data ? 'dedup_key')`;
+            values.push(dedup_keys);
+        } else {
+            query += ` AND (FALSE AND data ? 'dedup_key')`;
+        }
+
+        query += ` AND (data->'deleted' IS NULL OR data->'deleted' = 'null'::jsonb)) ORDER BY _id DESC;`;
+
+        try {
+            const res = await this._chunks.executeSQL(query, values);
+            const chunks = res?.rows.map(row => decode_json(this._chunks.schema, row.data));
+            await this.load_blocks_for_chunks(chunks);
+            return chunks;
+        } catch (err) {
+            dbg.error('Error while finding chunks by dedup_key. error is ', err);
+            return [];
+        }
     }
 
     iterate_all_chunks_in_buckets(lower_marker, upper_marker, buckets, limit) {

src/test/integration_tests/db/test_md_store.js

@@ -380,6 +380,43 @@ mocha.describe('md_store', function() {
             return md_store.delete_chunks_by_ids(_.map(chunks, '_id'));
         });
 
+        mocha.it('find_chunks_by_dedup_key()', async () => {
+            if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path
+            const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } };
+            const chunk = {
+                _id: md_store.make_md_id(),
+                system: system_id,
+                bucket: bucket._id,
+                frags: [{ _id: md_store.make_md_id() }],
+                size: 10,
+                frag_size: 10,
+                dedup_key: Buffer.from('noobaa')
+            };
+            await md_store.insert_chunks([chunk]);
+            const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, [Buffer.from('noobaa').toString('base64')]);
+            assert(Array.isArray(chunksArr));
+            assert(chunksArr.length >= 1);
+            assert(chunksArr[0].frags[0]?._id?.toString() === chunk.frags[0]._id.toString());
+        });
+
+        mocha.it('find_chunks_by_dedup_key empty dedup_key array passed', async () => {
+            if (config.DB_TYPE !== 'postgres') return; // feature uses SQL path
+            const bucket = { _id: md_store.make_md_id(), system: { _id: system_id } };
+            const chunk = {
+                _id: md_store.make_md_id(),
+                system: system_id,
+                bucket: bucket._id,
+                frags: [{ _id: md_store.make_md_id() }],
+                size: 10,
+                frag_size: 10,
+                dedup_key: Buffer.from('noobaa')
+            };
+            await md_store.insert_chunks([chunk]);
+            const chunksArr = await md_store.find_chunks_by_dedup_key(bucket, []);
+            assert(Array.isArray(chunksArr));
+            assert(chunksArr.length === 0);
+        });
+
     });