From 86da86afca85928d70b5e1c48305708ae532cd09 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 11:07:22 -0700 Subject: [PATCH 1/9] convert client side encryption corpus test to TS --- ...t_side_encryption.prose.06.corpus.test.ts} | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) rename test/integration/client-side-encryption/{client_side_encryption.prose.06.corpus.test.js => client_side_encryption.prose.06.corpus.test.ts} (96%) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.js b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts similarity index 96% rename from test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.js rename to test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index ec5b7b35796..1f8ac1574be 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.js +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -1,16 +1,13 @@ -'use strict'; - // The corpus test exhaustively enumerates all ways to encrypt all BSON value types. Note, the test data includes BSON binary subtype 4 (or standard UUID), which MUST be decoded and encoded as subtype 4. Run the test as follows. -const fs = require('fs'); -const path = require('path'); -const BSON = require('bson'); -const { EJSON } = require('bson'); -const { expect } = require('chai'); -const { getEncryptExtraOptions } = require('../../tools/utils'); -const { installNodeDNSWorkaroundHooks } = require('../../tools/runner/hooks/configuration'); +import fs from 'fs'; +import path from 'path'; +import { EJSON } from 'bson'; +import { expect } from 'chai'; +import { getEncryptExtraOptions } from '../../tools/utils'; +import { installNodeDNSWorkaroundHooks } from '../../tools/runner/hooks/configuration'; // eslint-disable-next-line no-restricted-modules -const { ClientEncryption } = require('../../../src/client-side-encryption/client_encryption'); +import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; describe('Client Side Encryption Prose Corpus Test', function () { const metadata = { @@ -22,7 +19,9 @@ describe('Client Side Encryption Prose Corpus Test', function () { const corpusDir = path.resolve(__dirname, '../../spec/client-side-encryption/corpus'); function loadCorpusData(filename) { - return EJSON.parse(fs.readFileSync(path.resolve(corpusDir, filename)), { relaxed: false }); + return EJSON.parse(fs.readFileSync(path.resolve(corpusDir, filename), { encoding: 'utf8' }), { + relaxed: false + }); } const CSFLE_KMS_PROVIDERS = process.env.CSFLE_KMS_PROVIDERS; @@ -103,7 +102,7 @@ describe('Client Side Encryption Prose Corpus Test', function () { let client; - function assertion(clientEncryption, key, expected, actual) { + function assertion(clientEncryption, _key, expected, actual) { if (typeof expected === 'string') { expect(actual).to.equal(expected); return; @@ -235,11 +234,9 @@ describe('Client Side Encryption Prose Corpus Test', function () { return clientEncrypted.connect().then(() => { clientEncryption = new ClientEncryption(client, { - bson: BSON, keyVaultNamespace, kmsProviders, - tlsOptions, - extraOptions + tlsOptions }); }); }); @@ -402,7 +399,7 @@ describe('Client Side Encryption Prose Corpus Test', function () { // }); // }); - defineCorpusTests(corpusAll, corpusEncryptedExpectedAll); + // defineCorpusTests(corpusAll, corpusEncryptedExpectedAll); // 9. Repeat steps 1-8 with a local JSON schema. I.e. amend step 4 to configure the schema on ``client_encrypted`` with the ``schema_map`` option. defineCorpusTests(corpusAll, corpusEncryptedExpectedAll, true); From 7713fefb0c0a65c6fdc7a55d5cdcaf987a225ad0 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 11:10:40 -0700 Subject: [PATCH 2/9] convert before/after hooks to async and make before/after each hooks --- ...nt_side_encryption.prose.06.corpus.test.ts | 44 +++++++------------ 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index 1f8ac1574be..55c1b59f0cd 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -8,6 +8,7 @@ import { getEncryptExtraOptions } from '../../tools/utils'; import { installNodeDNSWorkaroundHooks } from '../../tools/runner/hooks/configuration'; // eslint-disable-next-line no-restricted-modules import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; +import { MongoClient } from '../../mongodb'; describe('Client Side Encryption Prose Corpus Test', function () { const metadata = { @@ -100,8 +101,6 @@ describe('Client Side Encryption Prose Corpus Test', function () { 'altname_kmip' ]); - let client; - function assertion(clientEncryption, _key, expected, actual) { if (typeof expected === 'string') { expect(actual).to.equal(expected); @@ -151,36 +150,27 @@ describe('Client Side Encryption Prose Corpus Test', function () { installNodeDNSWorkaroundHooks(); - before(function () { + let client: MongoClient; + + beforeEach(async function () { // 1. Create a MongoClient without encryption enabled (referred to as ``client``). client = this.configuration.newClient(); - return Promise.resolve() - .then(() => client.connect()) - .then(() => { - // 3. Using ``client``, drop the collection ``keyvault.datakeys``. Insert the documents `corpus/corpus-key-local.json <../corpus/corpus-key-local.json>`_ and `corpus/corpus-key-aws.json <../corpus/corpus-key-aws.json>`_. - const keyDb = client.db(keyVaultDbName); - return Promise.resolve() - .then(() => keyDb.dropCollection(keyVaultCollName)) - .catch(() => {}) - .then(() => keyDb.collection(keyVaultCollName)) - .then(keyColl => - keyColl.insertMany([ - corpusKeyLocal, - corpusKeyAws, - corpusKeyAzure, - corpusKeyGcp, - corpusKeyKmip - ]) - ); - }); + await client.connect(); + // 3. Using ``client``, drop the collection ``keyvault.datakeys``. Insert the documents `corpus/corpus-key-local.json <../corpus/corpus-key-local.json>`_ and `corpus/corpus-key-aws.json <../corpus/corpus-key-aws.json>`_. + const keyDb = client.db(keyVaultDbName); + await keyDb.dropCollection(keyVaultCollName); + const keyColl = keyDb.collection(keyVaultCollName); + await keyColl.insertMany([ + corpusKeyLocal, + corpusKeyAws, + corpusKeyAzure, + corpusKeyGcp, + corpusKeyKmip + ]); }); - after(function () { - if (client) { - return client.close(); - } - }); + afterEach(() => client?.close()); function defineCorpusTests(corpus, corpusEncryptedExpected, useClientSideSchema) { let clientEncrypted, clientEncryption; From 1aa246f73694726085ee97e46b5530290af4a969 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 11:13:11 -0700 Subject: [PATCH 3/9] convert inner before/after hooks to async --- ...nt_side_encryption.prose.06.corpus.test.ts | 106 ++++++++---------- 1 file changed, 48 insertions(+), 58 deletions(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index 55c1b59f0cd..350384ea405 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -172,64 +172,54 @@ describe('Client Side Encryption Prose Corpus Test', function () { afterEach(() => client?.close()); - function defineCorpusTests(corpus, corpusEncryptedExpected, useClientSideSchema) { - let clientEncrypted, clientEncryption; - beforeEach(function () { - return Promise.resolve() - .then(() => { - // 2. Using ``client``, drop and create the collection ``db.coll`` configured with the included JSON schema `corpus/corpus-schema.json <../corpus/corpus-schema.json>`_. - const dataDb = client.db(dataDbName); - return Promise.resolve() - .then(() => dataDb.dropCollection(dataCollName)) - .catch(() => {}) - .then(() => - dataDb.createCollection(dataCollName, { - validator: { $jsonSchema: corpusSchema } - }) - ); - }) - .then(() => { - // 4. Create the following: - // - A MongoClient configured with auto encryption (referred to as ``client_encrypted``) - // - A ``ClientEncryption`` object (referred to as ``client_encryption``) - // Configure both objects with ``aws`` and the ``local`` KMS providers as follows: - // .. code:: javascript - // { - // "aws": { }, - // "local": { "key": } - // } - // Where LOCAL_MASTERKEY is the following base64: - // .. code:: javascript - // Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk - // Configure both objects with ``keyVaultNamespace`` set to ``keyvault.datakeys``. - const tlsOptions = { - kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE - } - }; - const extraOptions = getEncryptExtraOptions(); - const autoEncryption = { - keyVaultNamespace, - kmsProviders, - tlsOptions, - extraOptions - }; - if (useClientSideSchema) { - autoEncryption.schemaMap = { - [dataNamespace]: corpusSchema - }; - } - clientEncrypted = this.configuration.newClient({}, { autoEncryption }); - - return clientEncrypted.connect().then(() => { - clientEncryption = new ClientEncryption(client, { - keyVaultNamespace, - kmsProviders, - tlsOptions - }); - }); - }); + function defineCorpusTests(corpus, corpusEncryptedExpected, useClientSideSchema: boolean) { + let clientEncrypted: MongoClient, clientEncryption: ClientEncryption; + beforeEach(async function () { + // 2. Using ``client``, drop and create the collection ``db.coll`` configured with the included JSON schema `corpus/corpus-schema.json <../corpus/corpus-schema.json>`_. + const dataDb = client.db(dataDbName); + await dataDb.dropCollection(dataCollName); + await dataDb.createCollection(dataCollName, { + validator: { $jsonSchema: corpusSchema } + }); + // 4. Create the following: + // - A MongoClient configured with auto encryption (referred to as ``client_encrypted``) + // - A ``ClientEncryption`` object (referred to as ``client_encryption``) + // Configure both objects with ``aws`` and the ``local`` KMS providers as follows: + // .. code:: javascript + // { + // "aws": { }, + // "local": { "key": } + // } + // Where LOCAL_MASTERKEY is the following base64: + // .. code:: javascript + // Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + // Configure both objects with ``keyVaultNamespace`` set to ``keyvault.datakeys``. + const tlsOptions = { + kmip: { + tlsCAFile: process.env.KMIP_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + } + }; + const extraOptions = getEncryptExtraOptions(); + const autoEncryption = { + keyVaultNamespace, + kmsProviders, + tlsOptions, + extraOptions + }; + if (useClientSideSchema) { + autoEncryption.schemaMap = { + [dataNamespace]: corpusSchema + }; + } + clientEncrypted = this.configuration.newClient({}, { autoEncryption }); + + await clientEncrypted.connect(); + clientEncryption = new ClientEncryption(client, { + keyVaultNamespace, + kmsProviders, + tlsOptions + }); }); afterEach(() => clientEncrypted.close()); From 02006b64bc1f1c7420e94e911cc914bff703e80d Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 11:26:46 -0700 Subject: [PATCH 4/9] sadf --- ...nt_side_encryption.prose.06.corpus.test.ts | 237 ++++++++---------- 1 file changed, 109 insertions(+), 128 deletions(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index 350384ea405..e58c450399a 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -1,7 +1,7 @@ // The corpus test exhaustively enumerates all ways to encrypt all BSON value types. Note, the test data includes BSON binary subtype 4 (or standard UUID), which MUST be decoded and encoded as subtype 4. Run the test as follows. -import fs from 'fs'; -import path from 'path'; +import * as fs from 'fs'; +import * as path from 'path'; import { EJSON } from 'bson'; import { expect } from 'chai'; import { getEncryptExtraOptions } from '../../tools/utils'; @@ -10,7 +10,7 @@ import { installNodeDNSWorkaroundHooks } from '../../tools/runner/hooks/configur import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; import { MongoClient } from '../../mongodb'; -describe('Client Side Encryption Prose Corpus Test', function () { +describe.only('Client Side Encryption Prose Corpus Test', function () { const metadata = { requires: { mongodb: '>=4.2.0', @@ -224,137 +224,118 @@ describe('Client Side Encryption Prose Corpus Test', function () { afterEach(() => clientEncrypted.close()); - function forEachP(list, fn) { - return list.reduce((p, item) => { - return p.then(() => fn(item)); - }, Promise.resolve()); - } - - it( - `should pass corpus ${useClientSideSchema ? 'with' : 'without'} client schema`, - metadata, - function () { - const corpusCopied = {}; - return Promise.resolve() - .then(() => { - // 5. Load `corpus/corpus.json <../corpus/corpus.json>`_ to a variable named ``corpus``. The corpus contains subdocuments with the following fields: - // - // - ``kms`` is either ``aws`` or ``local`` - // - ``type`` is a BSON type string `names coming from here `_) - // - ``algo`` is either ``rand`` or ``det`` for random or deterministic encryption - // - ``method`` is either ``auto``, for automatic encryption or ``explicit`` for explicit encryption - // - ``identifier`` is either ``id`` or ``altname`` for the key identifier - // - ``allowed`` is a boolean indicating whether the encryption for the given parameters is permitted. - // - ``value`` is the value to be tested. - // - // Create a new BSON document, named ``corpus_copied``. - // - // Iterate over each field of ``corpus``. - // - If the field name is ``_id``, ``altname_aws`` and ``altname_local``, copy the field to ``corpus_copied``. - // - If ``method`` is ``auto``, copy the field to ``corpus_copied``. - // - If ``method`` is ``explicit``, use ``client_encryption`` to explicitly encrypt the value. - // - Encrypt with the algorithm described by ``algo``. - // - If ``identifier`` is ``id`` - // - If ``kms`` is ``local`` set the key_id to the UUID with base64 value ``LOCALAAAAAAAAAAAAAAAAA==``. - // - If ``kms`` is ``aws`` set the key_id to the UUID with base64 value ``AWSAAAAAAAAAAAAAAAAAAA==``. - // - If ``identifier`` is ``altname`` - // - If ``kms`` is ``local`` set the key_alt_name to "local". - // - If ``kms`` is ``aws`` set the key_alt_name to "aws". - // If ``allowed`` is true, copy the field and encrypted value to ``corpus_copied``. - // If ``allowed`` is false. verify that an exception is thrown. Copy the unencrypted value to to ``corpus_copied``. - return forEachP(Object.keys(corpus), key => { - const field = corpus[key]; - if (copyOverValues.has(key)) { - corpusCopied[key] = field; - return; + for (let i = 0; i < 100; ++i) { + it( + `should pass corpus ${useClientSideSchema ? 'with' : 'without'} client schema ${i}`, + metadata, + async function () { + const corpusCopied = {}; + + // 5. Load `corpus/corpus.json <../corpus/corpus.json>`_ to a variable named ``corpus``. The corpus contains subdocuments with the following fields: + // + // - ``kms`` is either ``aws`` or ``local`` + // - ``type`` is a BSON type string `names coming from here `_) + // - ``algo`` is either ``rand`` or ``det`` for random or deterministic encryption + // - ``method`` is either ``auto``, for automatic encryption or ``explicit`` for explicit encryption + // - ``identifier`` is either ``id`` or ``altname`` for the key identifier + // - ``allowed`` is a boolean indicating whether the encryption for the given parameters is permitted. + // - ``value`` is the value to be tested. + // + // Create a new BSON document, named ``corpus_copied``. + // + // Iterate over each field of ``corpus``. + // - If the field name is ``_id``, ``altname_aws`` and ``altname_local``, copy the field to ``corpus_copied``. + // - If ``method`` is ``auto``, copy the field to ``corpus_copied``. + // - If ``method`` is ``explicit``, use ``client_encryption`` to explicitly encrypt the value. + // - Encrypt with the algorithm described by ``algo``. + // - If ``identifier`` is ``id`` + // - If ``kms`` is ``local`` set the key_id to the UUID with base64 value ``LOCALAAAAAAAAAAAAAAAAA==``. + // - If ``kms`` is ``aws`` set the key_id to the UUID with base64 value ``AWSAAAAAAAAAAAAAAAAAAA==``. + // - If ``identifier`` is ``altname`` + // - If ``kms`` is ``local`` set the key_alt_name to "local". + // - If ``kms`` is ``aws`` set the key_alt_name to "aws". + // If ``allowed`` is true, copy the field and encrypted value to ``corpus_copied``. + // If ``allowed`` is false. verify that an exception is thrown. Copy the unencrypted value to to ``corpus_copied``. + for (const key of Object.keys(corpus)) { + const field = corpus[key]; + if (copyOverValues.has(key)) { + corpusCopied[key] = field; + continue; + } + if (field.method === 'auto') { + corpusCopied[key] = Object.assign({}, field); + continue; + } + if (field.method === 'explicit') { + const encryptOptions = { + algorithm: algorithmMap.get(field.algo) + }; + if (field.identifier === 'id') { + encryptOptions.keyId = identifierMap.get(field.kms); + } else if (field.identifier === 'altname') { + encryptOptions.keyAltName = keyAltNameMap.get(field.kms); + } else { + throw new Error('Unexpected identifier: ' + field.identifier); } - if (field.method === 'auto') { - corpusCopied[key] = Object.assign({}, field); - return; - } - if (field.method === 'explicit') { - const encryptOptions = { - algorithm: algorithmMap.get(field.algo) - }; - if (field.identifier === 'id') { - encryptOptions.keyId = identifierMap.get(field.kms); - } else if (field.identifier === 'altname') { - encryptOptions.keyAltName = keyAltNameMap.get(field.kms); - } else { - throw new Error('Unexpected identifier: ' + field.identifier); - } - return Promise.resolve() - .then(() => clientEncryption.encrypt(field.value, encryptOptions)) - .then( - encryptedValue => { - if (field.allowed === true) { - corpusCopied[key] = Object.assign({}, field, { value: encryptedValue }); - } else { - throw new Error( - `Expected encryption to fail for case ${key} on value ${field.value}` - ); - } - }, - e => { - if (field.allowed === false) { - corpusCopied[key] = Object.assign({}, field); - } else { - throw e; - } - } + try { + const encryptedValue = await clientEncryption.encrypt(field.value, encryptOptions); + if (field.allowed === true) { + corpusCopied[key] = Object.assign({}, field, { value: encryptedValue }); + } else { + throw new Error( + `Expected encryption to fail for case ${key} on value ${field.value}` ); + } + } catch (error) { + if (field.allowed === false) { + corpusCopied[key] = Object.assign({}, field); + } else { + throw error; + } } - + } else { throw new Error('Unexpected method: ' + field.method); - }); - }) - .then(() => { - // 6. Using ``client_encrypted``, insert ``corpus_copied`` into ``db.coll``. - return clientEncrypted.db(dataDbName).collection(dataCollName).insertOne(corpusCopied); - }) - .then(() => { - // 7. Using ``client_encrypted``, find the inserted document from ``db.coll`` to a variable named ``corpus_decrypted``. - // Since it should have been automatically decrypted, assert the document exactly matches ``corpus``. - return clientEncrypted - .db(dataDbName) - .collection(dataCollName) - .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); - }) - .then(corpusDecrypted => { - expect(toComparableExtendedJSON(corpusDecrypted)).to.deep.equal( - toComparableExtendedJSON(corpus) + } + } + + // 6. Using ``client_encrypted``, insert ``corpus_copied`` into ``db.coll``. + await clientEncrypted.db(dataDbName).collection(dataCollName).insertOne(corpusCopied); + // 7. Using ``client_encrypted``, find the inserted document from ``db.coll`` to a variable named ``corpus_decrypted``. + // Since it should have been automatically decrypted, assert the document exactly matches ``corpus``. + const corpusDecrypted = await clientEncrypted + .db(dataDbName) + .collection(dataCollName) + .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); + expect(toComparableExtendedJSON(corpusDecrypted)).to.deep.equal( + toComparableExtendedJSON(corpus) + ); + + // 8. Load `corpus/corpus_encrypted.json <../corpus/corpus-encrypted.json>`_ to a variable named ``corpus_encrypted_expected``. + // Using ``client`` find the inserted document from ``db.coll`` to a variable named ``corpus_encrypted_actual``. + + // Iterate over each field of ``corpus_encrypted_expected`` and check the following: + + // - If the ``algo`` is ``det``, that the value equals the value of the corresponding field in ``corpus_encrypted_actual``. + // - If the ``algo`` is ``rand`` and ``allowed`` is true, that the value does not equal the value of the corresponding field in ``corpus_encrypted_actual``. + // - If ``allowed`` is true, decrypt the value with ``client_encryption``. Decrypt the value of the corresponding field of ``corpus_encrypted`` and validate that they are both equal. + // - If ``allowed`` is false, validate the value exactly equals the value of the corresponding field of ``corpus`` (neither was encrypted). + const corpusEncryptedActual = await client + .db(dataDbName) + .collection(dataCollName) + .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); + for (const key of Object.keys(corpusEncryptedExpected)) { + await assertion( + clientEncryption, + key, + corpusEncryptedExpected[key], + corpusEncryptedActual[key] ); - }) - .then(() => { - // 8. Load `corpus/corpus_encrypted.json <../corpus/corpus-encrypted.json>`_ to a variable named ``corpus_encrypted_expected``. - // Using ``client`` find the inserted document from ``db.coll`` to a variable named ``corpus_encrypted_actual``. - - // Iterate over each field of ``corpus_encrypted_expected`` and check the following: - - // - If the ``algo`` is ``det``, that the value equals the value of the corresponding field in ``corpus_encrypted_actual``. - // - If the ``algo`` is ``rand`` and ``allowed`` is true, that the value does not equal the value of the corresponding field in ``corpus_encrypted_actual``. - // - If ``allowed`` is true, decrypt the value with ``client_encryption``. Decrypt the value of the corresponding field of ``corpus_encrypted`` and validate that they are both equal. - // - If ``allowed`` is false, validate the value exactly equals the value of the corresponding field of ``corpus`` (neither was encrypted). - return client - .db(dataDbName) - .collection(dataCollName) - .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); - }) - .then(corpusEncryptedActual => { - return forEachP(Object.keys(corpusEncryptedExpected), key => { - return assertion( - clientEncryption, - key, - corpusEncryptedExpected[key], - corpusEncryptedActual[key] - ); - }); - }); - } - ); + } + } + ); + } } - // Note: You can uncomment the block below to run the corpus for each individial item // instead of running the entire corpus at once. It is significantly slower, // but gives you higher visibility into why the corpus may be failing @@ -379,7 +360,7 @@ describe('Client Side Encryption Prose Corpus Test', function () { // }); // }); - // defineCorpusTests(corpusAll, corpusEncryptedExpectedAll); + defineCorpusTests(corpusAll, corpusEncryptedExpectedAll, false); // 9. Repeat steps 1-8 with a local JSON schema. I.e. amend step 4 to configure the schema on ``client_encrypted`` with the ``schema_map`` option. defineCorpusTests(corpusAll, corpusEncryptedExpectedAll, true); From c83eb5767e39a7100406a07619c097c0b6c029c8 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 11:55:02 -0700 Subject: [PATCH 5/9] suppress NS not found errors --- .../client_side_encryption.prose.06.corpus.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index e58c450399a..563b99af934 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -159,7 +159,11 @@ describe.only('Client Side Encryption Prose Corpus Test', function () { await client.connect(); // 3. Using ``client``, drop the collection ``keyvault.datakeys``. Insert the documents `corpus/corpus-key-local.json <../corpus/corpus-key-local.json>`_ and `corpus/corpus-key-aws.json <../corpus/corpus-key-aws.json>`_. const keyDb = client.db(keyVaultDbName); - await keyDb.dropCollection(keyVaultCollName); + await keyDb.dropCollection(keyVaultCollName).catch((e: Error) => { + if (!/ns/i.test(e.message)) { + throw e; + } + }); const keyColl = keyDb.collection(keyVaultCollName); await keyColl.insertMany([ corpusKeyLocal, @@ -177,7 +181,11 @@ describe.only('Client Side Encryption Prose Corpus Test', function () { beforeEach(async function () { // 2. Using ``client``, drop and create the collection ``db.coll`` configured with the included JSON schema `corpus/corpus-schema.json <../corpus/corpus-schema.json>`_. const dataDb = client.db(dataDbName); - await dataDb.dropCollection(dataCollName); + await dataDb.dropCollection(dataCollName).catch((e: Error) => { + if (!/ns/i.test(e.message)) { + throw e; + } + }); await dataDb.createCollection(dataCollName, { validator: { $jsonSchema: corpusSchema } }); From 8db169d4f860dda7f2cdece0c8fb4aa21650b5c1 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 14:12:56 -0700 Subject: [PATCH 6/9] hopefully fix tests? --- ...nt_side_encryption.prose.06.corpus.test.ts | 238 +++++++++--------- 1 file changed, 118 insertions(+), 120 deletions(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index 563b99af934..b9ff1d3cfe2 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -8,7 +8,7 @@ import { getEncryptExtraOptions } from '../../tools/utils'; import { installNodeDNSWorkaroundHooks } from '../../tools/runner/hooks/configuration'; // eslint-disable-next-line no-restricted-modules import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; -import { MongoClient } from '../../mongodb'; +import { MongoClient, WriteConcern } from '../../mongodb'; describe.only('Client Side Encryption Prose Corpus Test', function () { const metadata = { @@ -101,7 +101,7 @@ describe.only('Client Side Encryption Prose Corpus Test', function () { 'altname_kmip' ]); - function assertion(clientEncryption, _key, expected, actual) { + async function assertion(clientEncryption, _key, expected, actual) { if (typeof expected === 'string') { expect(actual).to.equal(expected); return; @@ -129,18 +129,15 @@ describe.only('Client Side Encryption Prose Corpus Test', function () { } if (expected.allowed === true) { - return Promise.all([ + const [decryptedExpectedValue, decryptedActualValue] = await Promise.all([ clientEncryption.decrypt(expectedValue), clientEncryption.decrypt(actualValue) - ]).then(results => { - const decryptedExpectedValue = results[0]; - const decryptedActualValue = results[1]; + ]); - const decryptedExpectedJSON = toComparableExtendedJSON(decryptedExpectedValue); - const decryptedActualJSON = toComparableExtendedJSON(decryptedActualValue); + const decryptedExpectedJSON = toComparableExtendedJSON(decryptedExpectedValue); + const decryptedActualJSON = toComparableExtendedJSON(decryptedActualValue); - expect(decryptedActualJSON).to.deep.equal(decryptedExpectedJSON); - }); + expect(decryptedActualJSON).to.deep.equal(decryptedExpectedJSON); } else if (expected.allowed === false) { expect(actualJSON).to.deep.equal(expectedJSON); } else { @@ -165,13 +162,10 @@ describe.only('Client Side Encryption Prose Corpus Test', function () { } }); const keyColl = keyDb.collection(keyVaultCollName); - await keyColl.insertMany([ - corpusKeyLocal, - corpusKeyAws, - corpusKeyAzure, - corpusKeyGcp, - corpusKeyKmip - ]); + await keyColl.insertMany( + [corpusKeyLocal, corpusKeyAws, corpusKeyAzure, corpusKeyGcp, corpusKeyKmip], + { writeConcern: new WriteConcern('majority') } + ); }); afterEach(() => client?.close()); @@ -187,7 +181,8 @@ describe.only('Client Side Encryption Prose Corpus Test', function () { } }); await dataDb.createCollection(dataCollName, { - validator: { $jsonSchema: corpusSchema } + validator: { $jsonSchema: corpusSchema }, + writeConcern: new WriteConcern('majority') }); // 4. Create the following: // - A MongoClient configured with auto encryption (referred to as ``client_encrypted``) @@ -232,117 +227,120 @@ describe.only('Client Side Encryption Prose Corpus Test', function () { afterEach(() => clientEncrypted.close()); - for (let i = 0; i < 100; ++i) { - it( - `should pass corpus ${useClientSideSchema ? 'with' : 'without'} client schema ${i}`, - metadata, - async function () { - const corpusCopied = {}; - - // 5. Load `corpus/corpus.json <../corpus/corpus.json>`_ to a variable named ``corpus``. The corpus contains subdocuments with the following fields: - // - // - ``kms`` is either ``aws`` or ``local`` - // - ``type`` is a BSON type string `names coming from here `_) - // - ``algo`` is either ``rand`` or ``det`` for random or deterministic encryption - // - ``method`` is either ``auto``, for automatic encryption or ``explicit`` for explicit encryption - // - ``identifier`` is either ``id`` or ``altname`` for the key identifier - // - ``allowed`` is a boolean indicating whether the encryption for the given parameters is permitted. - // - ``value`` is the value to be tested. - // - // Create a new BSON document, named ``corpus_copied``. - // - // Iterate over each field of ``corpus``. - // - If the field name is ``_id``, ``altname_aws`` and ``altname_local``, copy the field to ``corpus_copied``. - // - If ``method`` is ``auto``, copy the field to ``corpus_copied``. - // - If ``method`` is ``explicit``, use ``client_encryption`` to explicitly encrypt the value. - // - Encrypt with the algorithm described by ``algo``. - // - If ``identifier`` is ``id`` - // - If ``kms`` is ``local`` set the key_id to the UUID with base64 value ``LOCALAAAAAAAAAAAAAAAAA==``. - // - If ``kms`` is ``aws`` set the key_id to the UUID with base64 value ``AWSAAAAAAAAAAAAAAAAAAA==``. - // - If ``identifier`` is ``altname`` - // - If ``kms`` is ``local`` set the key_alt_name to "local". - // - If ``kms`` is ``aws`` set the key_alt_name to "aws". - // If ``allowed`` is true, copy the field and encrypted value to ``corpus_copied``. - // If ``allowed`` is false. verify that an exception is thrown. Copy the unencrypted value to to ``corpus_copied``. - for (const key of Object.keys(corpus)) { - const field = corpus[key]; - if (copyOverValues.has(key)) { - corpusCopied[key] = field; - continue; - } - if (field.method === 'auto') { - corpusCopied[key] = Object.assign({}, field); - continue; + it( + `should pass corpus ${useClientSideSchema ? 'with' : 'without'} client schema`, + metadata, + async function () { + const corpusCopied = {}; + + // 5. Load `corpus/corpus.json <../corpus/corpus.json>`_ to a variable named ``corpus``. The corpus contains subdocuments with the following fields: + // + // - ``kms`` is either ``aws`` or ``local`` + // - ``type`` is a BSON type string `names coming from here `_) + // - ``algo`` is either ``rand`` or ``det`` for random or deterministic encryption + // - ``method`` is either ``auto``, for automatic encryption or ``explicit`` for explicit encryption + // - ``identifier`` is either ``id`` or ``altname`` for the key identifier + // - ``allowed`` is a boolean indicating whether the encryption for the given parameters is permitted. + // - ``value`` is the value to be tested. + // + // Create a new BSON document, named ``corpus_copied``. + // + // Iterate over each field of ``corpus``. + // - If the field name is ``_id``, ``altname_aws`` and ``altname_local``, copy the field to ``corpus_copied``. + // - If ``method`` is ``auto``, copy the field to ``corpus_copied``. + // - If ``method`` is ``explicit``, use ``client_encryption`` to explicitly encrypt the value. + // - Encrypt with the algorithm described by ``algo``. + // - If ``identifier`` is ``id`` + // - If ``kms`` is ``local`` set the key_id to the UUID with base64 value ``LOCALAAAAAAAAAAAAAAAAA==``. + // - If ``kms`` is ``aws`` set the key_id to the UUID with base64 value ``AWSAAAAAAAAAAAAAAAAAAA==``. + // - If ``identifier`` is ``altname`` + // - If ``kms`` is ``local`` set the key_alt_name to "local". + // - If ``kms`` is ``aws`` set the key_alt_name to "aws". + // If ``allowed`` is true, copy the field and encrypted value to ``corpus_copied``. + // If ``allowed`` is false. verify that an exception is thrown. Copy the unencrypted value to to ``corpus_copied``. + for (const key of Object.keys(corpus)) { + const field = corpus[key]; + if (copyOverValues.has(key)) { + corpusCopied[key] = field; + continue; + } + if (field.method === 'auto') { + corpusCopied[key] = Object.assign({}, field); + continue; + } + if (field.method === 'explicit') { + const encryptOptions = { + algorithm: algorithmMap.get(field.algo) + }; + if (field.identifier === 'id') { + encryptOptions.keyId = identifierMap.get(field.kms); + } else if (field.identifier === 'altname') { + encryptOptions.keyAltName = keyAltNameMap.get(field.kms); + } else { + throw new Error('Unexpected identifier: ' + field.identifier); } - if (field.method === 'explicit') { - const encryptOptions = { - algorithm: algorithmMap.get(field.algo) - }; - if (field.identifier === 'id') { - encryptOptions.keyId = identifierMap.get(field.kms); - } else if (field.identifier === 'altname') { - encryptOptions.keyAltName = keyAltNameMap.get(field.kms); + + try { + const encryptedValue = await clientEncryption.encrypt(field.value, encryptOptions); + if (field.allowed === true) { + corpusCopied[key] = Object.assign({}, field, { value: encryptedValue }); } else { - throw new Error('Unexpected identifier: ' + field.identifier); + throw new Error( + `Expected encryption to fail for case ${key} on value ${field.value}` + ); } - - try { - const encryptedValue = await clientEncryption.encrypt(field.value, encryptOptions); - if (field.allowed === true) { - corpusCopied[key] = Object.assign({}, field, { value: encryptedValue }); - } else { - throw new Error( - `Expected encryption to fail for case ${key} on value ${field.value}` - ); - } - } catch (error) { - if (field.allowed === false) { - corpusCopied[key] = Object.assign({}, field); - } else { - throw error; - } + } catch (error) { + if (field.allowed === false) { + corpusCopied[key] = Object.assign({}, field); + } else { + throw error; } - } else { - throw new Error('Unexpected method: ' + field.method); } + } else { + throw new Error('Unexpected method: ' + field.method); } + } - // 6. Using ``client_encrypted``, insert ``corpus_copied`` into ``db.coll``. - await clientEncrypted.db(dataDbName).collection(dataCollName).insertOne(corpusCopied); - // 7. Using ``client_encrypted``, find the inserted document from ``db.coll`` to a variable named ``corpus_decrypted``. - // Since it should have been automatically decrypted, assert the document exactly matches ``corpus``. - const corpusDecrypted = await clientEncrypted - .db(dataDbName) - .collection(dataCollName) - .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); - expect(toComparableExtendedJSON(corpusDecrypted)).to.deep.equal( - toComparableExtendedJSON(corpus) + // 6. Using ``client_encrypted``, insert ``corpus_copied`` into ``db.coll``. + await clientEncrypted + .db(dataDbName) + .collection(dataCollName) + .insertOne(corpusCopied, { + writeConcern: new WriteConcern('majority') + }); + // 7. Using ``client_encrypted``, find the inserted document from ``db.coll`` to a variable named ``corpus_decrypted``. + // Since it should have been automatically decrypted, assert the document exactly matches ``corpus``. + const corpusDecrypted = await clientEncrypted + .db(dataDbName) + .collection(dataCollName) + .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); + expect(toComparableExtendedJSON(corpusDecrypted)).to.deep.equal( + toComparableExtendedJSON(corpus) + ); + + // 8. Load `corpus/corpus_encrypted.json <../corpus/corpus-encrypted.json>`_ to a variable named ``corpus_encrypted_expected``. + // Using ``client`` find the inserted document from ``db.coll`` to a variable named ``corpus_encrypted_actual``. + + // Iterate over each field of ``corpus_encrypted_expected`` and check the following: + + // - If the ``algo`` is ``det``, that the value equals the value of the corresponding field in ``corpus_encrypted_actual``. + // - If the ``algo`` is ``rand`` and ``allowed`` is true, that the value does not equal the value of the corresponding field in ``corpus_encrypted_actual``. + // - If ``allowed`` is true, decrypt the value with ``client_encryption``. Decrypt the value of the corresponding field of ``corpus_encrypted`` and validate that they are both equal. + // - If ``allowed`` is false, validate the value exactly equals the value of the corresponding field of ``corpus`` (neither was encrypted). + const corpusEncryptedActual = await client + .db(dataDbName) + .collection(dataCollName) + .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); + for (const key of Object.keys(corpusEncryptedExpected)) { + await assertion( + clientEncryption, + key, + corpusEncryptedExpected[key], + corpusEncryptedActual[key] ); - - // 8. Load `corpus/corpus_encrypted.json <../corpus/corpus-encrypted.json>`_ to a variable named ``corpus_encrypted_expected``. - // Using ``client`` find the inserted document from ``db.coll`` to a variable named ``corpus_encrypted_actual``. - - // Iterate over each field of ``corpus_encrypted_expected`` and check the following: - - // - If the ``algo`` is ``det``, that the value equals the value of the corresponding field in ``corpus_encrypted_actual``. - // - If the ``algo`` is ``rand`` and ``allowed`` is true, that the value does not equal the value of the corresponding field in ``corpus_encrypted_actual``. - // - If ``allowed`` is true, decrypt the value with ``client_encryption``. Decrypt the value of the corresponding field of ``corpus_encrypted`` and validate that they are both equal. - // - If ``allowed`` is false, validate the value exactly equals the value of the corresponding field of ``corpus`` (neither was encrypted). - const corpusEncryptedActual = await client - .db(dataDbName) - .collection(dataCollName) - .findOne({ _id: corpusCopied._id }, { promoteLongs: false, promoteValues: false }); - for (const key of Object.keys(corpusEncryptedExpected)) { - await assertion( - clientEncryption, - key, - corpusEncryptedExpected[key], - corpusEncryptedActual[key] - ); - } } - ); - } + } + ); } // Note: You can uncomment the block below to run the corpus for each individial item // instead of running the entire corpus at once. It is significantly slower, From e884e86ad71adec31214ff08e38d795996247426 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 14:32:48 -0700 Subject: [PATCH 7/9] fix lint --- ...client_side_encryption.prose.06.corpus.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index b9ff1d3cfe2..7450406046b 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -1,16 +1,17 @@ // The corpus test exhaustively enumerates all ways to encrypt all BSON value types. Note, the test data includes BSON binary subtype 4 (or standard UUID), which MUST be decoded and encoded as subtype 4. Run the test as follows. -import * as fs from 'fs'; -import * as path from 'path'; import { EJSON } from 'bson'; import { expect } from 'chai'; -import { getEncryptExtraOptions } from '../../tools/utils'; -import { installNodeDNSWorkaroundHooks } from '../../tools/runner/hooks/configuration'; -// eslint-disable-next-line no-restricted-modules +import * as fs from 'fs'; +import * as path from 'path'; + +// eslint-disable-next-line @typescript-eslint/no-restricted-imports import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; -import { MongoClient, WriteConcern } from '../../mongodb'; +import { type MongoClient, WriteConcern } from '../../mongodb'; +import { installNodeDNSWorkaroundHooks } from '../../tools/runner/hooks/configuration'; +import { getEncryptExtraOptions } from '../../tools/utils'; -describe.only('Client Side Encryption Prose Corpus Test', function () { +describe('Client Side Encryption Prose Corpus Test', function () { const metadata = { requires: { mongodb: '>=4.2.0', From 8a436a439d2dc86f0a55a372c5bd788a5c50d43b Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 14:51:04 -0700 Subject: [PATCH 8/9] adjust name of file afte rTS conversion --- .../client-side-encryption/client_side_encryption.prose.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.test.js b/test/integration/client-side-encryption/client_side_encryption.prose.test.js index b25e9d35454..99b634ca4a2 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.test.js +++ b/test/integration/client-side-encryption/client_side_encryption.prose.test.js @@ -829,7 +829,7 @@ describe('Client Side Encryption Prose Tests', metadata, function () { describe('Corpus Test', function () { it('runs in a separate suite', () => { expect(() => - fs.statSync(path.resolve(__dirname, './client_side_encryption.prose.06.corpus.test.js')) + fs.statSync(path.resolve(__dirname, './client_side_encryption.prose.06.corpus.test.ts')) ).not.to.throw(); }); }); From d549f79b13e6af880bd46abbddf50b77f3e5b9e8 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 8 Mar 2024 15:37:15 -0700 Subject: [PATCH 9/9] drop collections with w: majority --- ...nt_side_encryption.prose.06.corpus.test.ts | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index 7450406046b..b6615267bfe 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -157,11 +157,13 @@ describe('Client Side Encryption Prose Corpus Test', function () { await client.connect(); // 3. Using ``client``, drop the collection ``keyvault.datakeys``. Insert the documents `corpus/corpus-key-local.json <../corpus/corpus-key-local.json>`_ and `corpus/corpus-key-aws.json <../corpus/corpus-key-aws.json>`_. const keyDb = client.db(keyVaultDbName); - await keyDb.dropCollection(keyVaultCollName).catch((e: Error) => { - if (!/ns/i.test(e.message)) { - throw e; - } - }); + await keyDb + .dropCollection(keyVaultCollName, { writeConcern: new WriteConcern('majority') }) + .catch((e: Error) => { + if (!/ns/i.test(e.message)) { + throw e; + } + }); const keyColl = keyDb.collection(keyVaultCollName); await keyColl.insertMany( [corpusKeyLocal, corpusKeyAws, corpusKeyAzure, corpusKeyGcp, corpusKeyKmip], @@ -176,11 +178,13 @@ describe('Client Side Encryption Prose Corpus Test', function () { beforeEach(async function () { // 2. Using ``client``, drop and create the collection ``db.coll`` configured with the included JSON schema `corpus/corpus-schema.json <../corpus/corpus-schema.json>`_. const dataDb = client.db(dataDbName); - await dataDb.dropCollection(dataCollName).catch((e: Error) => { - if (!/ns/i.test(e.message)) { - throw e; - } - }); + await dataDb + .dropCollection(dataCollName, { writeConcern: new WriteConcern('majority') }) + .catch((e: Error) => { + if (!/ns/i.test(e.message)) { + throw e; + } + }); await dataDb.createCollection(dataCollName, { validator: { $jsonSchema: corpusSchema }, writeConcern: new WriteConcern('majority')