Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
114 commits
Select commit Hold shift + click to select a range
d13b876
added console logs and stubbing files out
al-rosenthal Sep 30, 2022
f28c95c
Merge branch 'dev' into BHBC-1963
al-rosenthal Sep 30, 2022
b4c51dc
improved error codes
anissa-agahchen Sep 30, 2022
03df104
stubbed out occurrence service and repo
al-rosenthal Sep 30, 2022
083b524
S3 and file prep
al-rosenthal Oct 3, 2022
8c571c7
new functions to validation service, validation repo stubbed out
al-rosenthal Oct 3, 2022
6e3aba6
small POC of what the error handling could look like
anissa-agahchen Oct 3, 2022
b5dcbd1
submission service repo added
al-rosenthal Oct 3, 2022
9359309
finished stubing out functions
al-rosenthal Oct 4, 2022
40a9321
getting working started
al-rosenthal Oct 4, 2022
4d9f41a
further in process
al-rosenthal Oct 4, 2022
2a640af
happy path working
al-rosenthal Oct 4, 2022
d71cd91
more clean up, moved more code around
al-rosenthal Oct 5, 2022
1ec867e
clean up
al-rosenthal Oct 5, 2022
b33f74d
converted all error codes from hard-coded to coming from the submissi…
anissa-agahchen Oct 5, 2022
4d67960
updated transform, validate and process endpoints
al-rosenthal Oct 5, 2022
7a68e99
fixed view occurrence endpoint
al-rosenthal Oct 5, 2022
3cf404b
scrape occurrences is wired up with new service
al-rosenthal Oct 5, 2022
25ca83a
database migration to add more submission status and message types
anissa-agahchen Oct 5, 2022
63696e7
fix broken tests
anissa-agahchen Oct 5, 2022
ef8fc47
tweak
anissa-agahchen Oct 5, 2022
0047f9c
updated dwc validation endpoint
al-rosenthal Oct 5, 2022
b74d827
ran format and ling fix
al-rosenthal Oct 5, 2022
e091568
clenaed up endpoints for better connection handling
al-rosenthal Oct 6, 2022
2c0ebae
fixed dwc parsing
al-rosenthal Oct 6, 2022
5ee52d0
added comments to new service
al-rosenthal Oct 6, 2022
8df6607
started writing tests
al-rosenthal Oct 6, 2022
772f6e1
occurrence repo tests added
al-rosenthal Oct 6, 2022
87f87a2
ran lint and format fix
al-rosenthal Oct 6, 2022
a6bc8c0
added validation repo test
al-rosenthal Oct 6, 2022
adba47b
submission repo tests added
al-rosenthal Oct 6, 2022
99bda48
Merge branch 'BHBC-1963' into BHBC-1962
al-rosenthal Oct 6, 2022
bb4ed93
fixed import errors
al-rosenthal Oct 6, 2022
4b9cc95
stubbed out try catch on process
al-rosenthal Oct 11, 2022
2f90ec9
updated message types and submission types
anissa-agahchen Oct 11, 2022
d097788
updated exception throws
al-rosenthal Oct 11, 2022
d6bb2b8
conflict resolved
al-rosenthal Oct 11, 2022
44cb0a0
fixed errors
al-rosenthal Oct 11, 2022
1f16c7e
clean up errors
al-rosenthal Oct 11, 2022
a738382
tempalte scrape and upload wrapped in trycatch
al-rosenthal Oct 11, 2022
35f7b01
added more trycatches for process steps
al-rosenthal Oct 11, 2022
e77ed9c
added throws to s3 functions
al-rosenthal Oct 11, 2022
18a5d9c
fixing resent header issue
al-rosenthal Oct 11, 2022
1e735ab
wrapping prep work into function
al-rosenthal Oct 11, 2022
06f7df2
stubbing out error class
al-rosenthal Oct 11, 2022
604d3da
added new message objects to throw instead
al-rosenthal Oct 12, 2022
8c178df
updating validation results fucntion
al-rosenthal Oct 12, 2022
1854127
updated migrations and message throws for prep
al-rosenthal Oct 12, 2022
9127a30
updated migrations with new errors
al-rosenthal Oct 12, 2022
b71ee8f
updating enum
al-rosenthal Oct 12, 2022
fd6ed8e
replaced enum
al-rosenthal Oct 12, 2022
ee185cf
clean up
al-rosenthal Oct 12, 2022
6e8b4f3
all new errors added
al-rosenthal Oct 12, 2022
84f6254
fixing parent try catches
al-rosenthal Oct 12, 2022
7026a44
got the rejected status displaying
anissa-agahchen Oct 12, 2022
8922717
merge conflicts
anissa-agahchen Oct 12, 2022
ca88886
test setup
al-rosenthal Oct 12, 2022
4ba5528
updated endpoints with new error catching
al-rosenthal Oct 12, 2022
2fead93
all message types are accounted for in app
anissa-agahchen Oct 12, 2022
2ba18e9
front-end changes
anissa-agahchen Oct 12, 2022
138f46c
fixing up dwc endpoint
al-rosenthal Oct 12, 2022
8ae9d7a
pulled changes
al-rosenthal Oct 12, 2022
38af83b
submission status working
al-rosenthal Oct 12, 2022
9511ef2
ran lint fix
al-rosenthal Oct 12, 2022
693592f
updated more throws
al-rosenthal Oct 12, 2022
53ddc24
fixed content validation error
al-rosenthal Oct 13, 2022
7c40d49
fix front-end issue
anissa-agahchen Oct 13, 2022
2228046
fixed bug
anissa-agahchen Oct 13, 2022
0b1ef16
updated logic for transform state and verbage
al-rosenthal Oct 13, 2022
a8135ce
making tests compile
anissa-agahchen Oct 13, 2022
0448458
merge with dev
anissa-agahchen Oct 13, 2022
b576e4c
text change
anissa-agahchen Oct 13, 2022
0a53398
skip broken tests
anissa-agahchen Oct 13, 2022
ce73abf
tested parse XLSX
al-rosenthal Oct 13, 2022
5266216
more tests
al-rosenthal Oct 14, 2022
27ae885
testing template validation
al-rosenthal Oct 14, 2022
a5a86e2
update to user errors
anissa-agahchen Oct 14, 2022
7f1e2e4
improvements to use error
anissa-agahchen Oct 14, 2022
0d802f8
pulled changes
al-rosenthal Oct 14, 2022
9c233a3
fixed spelling mistakes
al-rosenthal Oct 14, 2022
2d161e4
validated persisted validation results
al-rosenthal Oct 14, 2022
9ac0c3c
fixing format, fixed spelling mistakes
al-rosenthal Oct 14, 2022
3218d64
get schema tested
al-rosenthal Oct 14, 2022
7b5e811
transformation schema tested
al-rosenthal Oct 14, 2022
91ce7d8
44% covered
al-rosenthal Oct 14, 2022
7777e07
prep dwc archive tested
al-rosenthal Oct 14, 2022
bd21df6
dwc/validate tests
anissa-agahchen Oct 14, 2022
a173872
merge conflicts
anissa-agahchen Oct 14, 2022
b6841bf
persist transformation tested
al-rosenthal Oct 14, 2022
35496c3
xlsx/validate
anissa-agahchen Oct 14, 2022
778c7e0
xlsx/process and xlsx/trasnform
anissa-agahchen Oct 15, 2022
5b1b922
merged changes
al-rosenthal Oct 15, 2022
b1845d6
more
al-rosenthal Oct 15, 2022
eaf32d7
65% coverage
al-rosenthal Oct 17, 2022
813303c
error-service tests
anissa-agahchen Oct 17, 2022
0ad22fe
validate file tested
al-rosenthal Oct 17, 2022
c08a7df
Merge branch 'BHBC-1962' of https://github.com/bcgov/biohubbc into BH…
al-rosenthal Oct 17, 2022
32395a5
finish coverage for repo files
KjartanE Oct 17, 2022
bfac311
Merge branch 'BHBC-1962' of https://github.com/bcgov/biohubbc into BH…
KjartanE Oct 17, 2022
512fbdc
pulled changes, added dwc process test
al-rosenthal Oct 17, 2022
5a499ec
process file tested
al-rosenthal Oct 17, 2022
5c30507
validate dwc tested
al-rosenthal Oct 17, 2022
2759fca
last function tested
al-rosenthal Oct 17, 2022
2ea15e0
s3key for biohub
anissa-agahchen Oct 17, 2022
f9bfd8a
merge conflicts
anissa-agahchen Oct 17, 2022
f819a94
removed some code smells
al-rosenthal Oct 17, 2022
b1ef792
Merge branch 'BHBC-1962' of https://github.com/bcgov/biohubbc into BH…
al-rosenthal Oct 17, 2022
cc41f76
fixed tests
anissa-agahchen Oct 17, 2022
ab6ba64
merge conflicts
anissa-agahchen Oct 17, 2022
aa31fa8
fixed more code smells
al-rosenthal Oct 17, 2022
db64126
remove coments
anissa-agahchen Oct 17, 2022
a8f8484
fixed various code smells
al-rosenthal Oct 17, 2022
29cc0de
fixed the last code smell
al-rosenthal Oct 17, 2022
8102a3d
ran lint fix
al-rosenthal Oct 17, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
last function tested
  • Loading branch information
al-rosenthal committed Oct 17, 2022
commit 2759fcaf012edfa6837411a0fc200355c0c44a6b
166 changes: 164 additions & 2 deletions api/src/services/validation-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ const buildFile = (fileName: string, customProps: { template_id?: number; csm_id
return new MediaFile(fileName, 'text/csv', buffer);
};

// 63% covered
describe('ValidationService', () => {
afterEach(() => {
sinon.restore();
Expand Down Expand Up @@ -862,7 +861,7 @@ describe('ValidationService', () => {
});
});

describe.only('validateDWC', () => {
describe('validateDWC', () => {
afterEach(() => {
sinon.restore();
});
Expand Down Expand Up @@ -922,6 +921,169 @@ describe('ValidationService', () => {
});
});

describe('templateScrapeAndUploadOccurrences', () => {
it('should run without issue', async () => {
const service = mockService();
const mockDWCArchive = new DWCArchive(new ArchiveFile("", "", Buffer.from([]), []))

const occurrence = sinon.stub(service.occurrenceService, 'getOccurrenceSubmission').resolves(mockOccurrenceSubmission);
const file = sinon.stub(FileUtils, 'getFileFromS3').resolves('file from s3' as any);
const archive = sinon.stub(service, 'prepDWCArchive').resolves(mockDWCArchive);
const scrape = sinon.stub(service.occurrenceService, 'scrapeAndUploadOccurrences').resolves();

await service.templateScrapeAndUploadOccurrences(1)

expect(occurrence).to.be.calledOnce;
expect(file).to.be.calledOnce;
expect(archive).to.be.calledOnce;
expect(scrape).to.be.calledOnce;
});

it('should throw Submission Error', async () => {
const service = mockService();

const occurrence = sinon.stub(service.occurrenceService, 'getOccurrenceSubmission').resolves(mockOccurrenceSubmission);
const file = sinon.stub(FileUtils, 'getFileFromS3').resolves('file from s3' as any);
const archive = sinon.stub(service, 'prepDWCArchive').throws(SubmissionErrorFromMessageType(SUBMISSION_MESSAGE_TYPE.INVALID_MEDIA))
const scrape = sinon.stub(service.occurrenceService, 'scrapeAndUploadOccurrences').resolves();

try {

await service.templateScrapeAndUploadOccurrences(1)

expect(occurrence).to.be.calledOnce;
expect(file).to.be.calledOnce;
expect(archive).to.be.calledOnce;
expect(scrape).not.to.be.calledOnce;
expect.fail()
} catch (error) {
expect(error instanceof SubmissionError).to.be.true;
}
})
});

describe('templateTransformation', () => {
it('should run without issue', async () => {
const file = buildFile("test file", {csm_id: 1, template_id: 1})
const xlsxCsv = new XLSXCSV(file);
const parser = new TransformationSchemaParser({})
const fileBuffer =
{
name: "",
buffer: Buffer.from([])
} as any

const service = mockService()

sinon.stub(FileUtils, 'getFileFromS3').resolves('file from s3' as any);

const getTransformation = sinon.stub(service, 'getTransformationSchema').resolves({});
const getRules = sinon.stub(service, 'getTransformationRules').resolves(parser);
const transform = sinon.stub(service, 'transformXLSX').resolves([fileBuffer]);
const persistResults = sinon.stub(service, 'persistTransformationResults').resolves();

await service.templateTransformation(1, xlsxCsv, "");

expect(getTransformation).to.be.calledOnce;
expect(getRules).to.be.calledOnce;
expect(transform).to.be.calledOnce;
expect(persistResults).to.be.calledOnce;
});

it('should Submission Error', async () => {
const file = buildFile("test file", {csm_id: 1, template_id: 1})
const xlsxCsv = new XLSXCSV(file);
const parser = new TransformationSchemaParser({})
const fileBuffer =
{
name: "",
buffer: Buffer.from([])
} as any

const service = mockService()

sinon.stub(FileUtils, 'getFileFromS3').resolves('file from s3' as any);

const getTransformation = sinon.stub(service, 'getTransformationSchema').resolves({});
const getRules = sinon.stub(service, 'getTransformationRules').resolves(parser);
const transform = sinon.stub(service, 'transformXLSX').resolves([fileBuffer]);
const persistResults = sinon.stub(service, 'persistTransformationResults').throws(SubmissionErrorFromMessageType(SUBMISSION_MESSAGE_TYPE.FAILED_UPLOAD_FILE_TO_S3))

try {

await service.templateTransformation(1, xlsxCsv, "");
expect(getTransformation).to.be.calledOnce;
expect(getRules).to.be.calledOnce;
expect(transform).to.be.calledOnce;
expect(persistResults).to.be.calledOnce;
expect.fail();
} catch (error) {
expect(error instanceof SubmissionError).to.be.true;
}
});
});

describe('validateXLSX', () => {
it('should return valid state object', async () => {
const service = mockService();
const xlsx = new XLSXCSV(buildFile("test file", {}));
const parser = new ValidationSchemaParser({});
const response = await service.validateXLSX(xlsx, parser)

expect(response.media_state.isValid).to.be.true;
expect(response.media_state.fileErrors).is.empty;
});

it('should throw Media is invalid error', async () => {
const service = mockService();
const mockMediaState = {
fileName: "test file",
isValid: false
} as IMediaState
const xlsx = new XLSXCSV(buildFile("test file", {}));
const parser = new ValidationSchemaParser({});

sinon.stub(XLSXCSV.prototype, 'isMediaValid').returns(mockMediaState)

try {
await service.validateXLSX(xlsx, parser)
expect.fail();
} catch (error) {
expect(error instanceof SubmissionError).to.be.true;
expect((error as SubmissionError).submissionMessages[0].type).to.be.eql(SUBMISSION_MESSAGE_TYPE.INVALID_MEDIA);
}
});

it('should return valid state object with content errors', async () => {
const service = mockService();
const mockState = {
fileName: "test",
isValid: false,
headerErrors: [{
errorCode: SUBMISSION_MESSAGE_TYPE.DUPLICATE_HEADER,
message: "Duplicate header found",
col: 1
}],
rowErrors: [
{
errorCode: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_FIELD,
message: "Missing required field",
col: "1",
row: 1
}
]
} as ICsvState;
const xlsx = new XLSXCSV(buildFile("test file", {}));
const parser = new ValidationSchemaParser({});
sinon.stub(XLSXCSV.prototype, 'isContentValid').returns([mockState])

const response = await service.validateXLSX(xlsx, parser)
expect(response.csv_state).is.not.empty;
expect(response.csv_state[0].headerErrors).is.not.empty;
expect(response.csv_state[0].rowErrors).is.not.empty;
});
});

describe('validateDWCArchive', () => {
afterEach(() => {
sinon.restore();
Expand Down