small POC of what the error handling could look like

api/src/errors/api-error.ts

@@ -75,7 +75,7 @@ export class ApiExecuteSQLError extends ApiError {
  * @class ApiBuildSQLError
  * @extends {ApiError}
  */
- export class ApiBuildSQLError extends ApiError {
+export class ApiBuildSQLError extends ApiError {
   constructor(message: string, errors?: (string | object)[]) {
     super(ApiErrorType.BUILD_SQL, message, errors);
   }

api/src/paths/codes.test.ts

@@ -4,7 +4,6 @@ import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 import * as db from '../database/db';
 import { HTTPError } from '../errors/http-error';
-
 import { CodeService } from '../services/code-service';
 import { getMockDBConnection } from '../__mocks__/db';
 import * as codes from './codes';

api/src/paths/codes.ts

@@ -2,7 +2,6 @@ import { RequestHandler } from 'express';
 import { Operation } from 'express-openapi';
 import { getAPIUserDBConnection } from '../database/db';
 import { HTTP500 } from '../errors/http-error';
-
 import { CodeService } from '../services/code-service';
 import { getLogger } from '../utils/logger';
 

api/src/paths/draft/{draftId}/get.test.ts

@@ -5,7 +5,6 @@ import sinonChai from 'sinon-chai';
 import SQL from 'sql-template-strings';
 import * as db from '../../../database/db';
 import { HTTPError } from '../../../errors/http-error';
-
 import draft_queries from '../../../queries/project/draft';
 import { getMockDBConnection } from '../../../__mocks__/db';
 import * as viewDraftProject from './get';

api/src/paths/drafts.test.ts

@@ -5,7 +5,6 @@ import sinonChai from 'sinon-chai';
 import SQL from 'sql-template-strings';
 import * as db from '../database/db';
 import { HTTPError } from '../errors/http-error';
-
 import draft_queries from '../queries/project/draft';
 import { getMockDBConnection } from '../__mocks__/db';
 import * as drafts from './drafts';

api/src/paths/drafts.ts

@@ -2,7 +2,6 @@ import { RequestHandler } from 'express';
 import { Operation } from 'express-openapi';
 import { getDBConnection } from '../database/db';
 import { HTTP400 } from '../errors/http-error';
-
 import { draftResponseObject } from '../openapi/schemas/draft';
 import { queries } from '../queries/queries';
 import { authorizeRequestHandler } from '../request-handlers/security/authorization';

api/src/paths/dwc/eml.test.ts

@@ -4,7 +4,6 @@ import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
 import * as db from '../../database/db';
 import { HTTPError } from '../../errors/http-error';
-
 import { EmlService } from '../../services/eml-service';
 import { getMockDBConnection, getRequestHandlerMocks } from '../../__mocks__/db';
 import { getProjectEml } from './eml';

api/src/paths/dwc/validate.ts

@@ -4,7 +4,9 @@ import { PROJECT_ROLE } from '../../constants/roles';
 import { getDBConnection, IDBConnection } from '../../database/db';
 import { HTTP400, HTTP500 } from '../../errors/http-error';
 import { queries } from '../../queries/queries';
+import { SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE } from '../../repositories/error-repository';
 import { authorizeRequestHandler } from '../../request-handlers/security/authorization';
+import { ErrorService } from '../../services/error-service';
 import { getFileFromS3 } from '../../utils/file-utils';
 import { getLogger } from '../../utils/logger';
 import { ICsvState, IHeaderError, IRowError } from '../../utils/media/csv/csv-file';
@@ -108,6 +110,15 @@ export const getValidateAPIDoc = (basicDescription: string, successDescription:
   };
 };
 
+//NOTES:
+//  Do we want a validation service, or an error service?
+// Currently, a failed validation is a submission status state
+// option 1: we keep it the way it is, and tailor the error message ... ie SQL, or other custom message
+// option 2: create a validation service, to group all validation related functions ... some reuse between dwc and xlsx validation
+// option 3: create an error-service, to manage all kinds of errors ... submission as a starting point
+// or some combination.
+// Both option 2 and 3 could help introduce more granular error messages and message types
+
 POST.apiDoc = {
   ...getValidateAPIDoc(
     'Validates a Darwin Core (DWC) Archive survey observation submission.',
@@ -148,8 +159,17 @@ export function getOccurrenceSubmission(): RequestHandler {
       req['occurrence_submission'] = response.rows[0];
 
       next();
-    } catch (error) {
+    } catch (error: any) {
       defaultLog.error({ label: 'getOccurrenceSubmission', message: 'error', error });
+
+      const errorService = new ErrorService(connection);
+
+      await errorService.insertSubmissionStatusAndMessage(
+        req['occurrence_submission'].occurrence_submission_id,
+        SUBMISSION_STATUS_TYPE.FAILED_VALIDATION,
+        SUBMISSION_MESSAGE_TYPE.ERROR,
+        error.message
+      );
       throw error;
     } finally {
       connection.release();

api/src/paths/logger.test.ts

@@ -1,7 +1,6 @@
 import { expect } from 'chai';
 import { describe } from 'mocha';
 import { HTTPError } from '../errors/http-error';
-
 import * as logger from './logger';
 
 describe('logger', () => {

api/src/paths/project/{projectId}/attachments/{attachmentId}/delete.ts

@@ -4,7 +4,6 @@ import { ATTACHMENT_TYPE } from '../../../../../constants/attachments';
 import { PROJECT_ROLE } from '../../../../../constants/roles';
 import { getDBConnection, IDBConnection } from '../../../../../database/db';
 import { HTTP400 } from '../../../../../errors/http-error';
-
 import { queries } from '../../../../../queries/queries';
 import { authorizeRequestHandler } from '../../../../../request-handlers/security/authorization';
 import { deleteFileFromS3 } from '../../../../../utils/file-utils';

api/src/paths/project/{projectId}/survey/{surveyId}/summary/submission/{summaryId}/view.test.ts

@@ -6,7 +6,6 @@ import sinonChai from 'sinon-chai';
 import SQL from 'sql-template-strings';
 import * as db from '../../../../../../../../database/db';
 import { HTTPError } from '../../../../../../../../errors/http-error';
-
 import survey_queries from '../../../../../../../../queries/survey';
 import * as file_utils from '../../../../../../../../utils/file-utils';
 import { MediaFile } from '../../../../../../../../utils/media/media-file';

api/src/paths/search.test.ts

@@ -6,7 +6,6 @@ import SQL from 'sql-template-strings';
 import { SYSTEM_ROLE } from '../constants/roles';
 import * as db from '../database/db';
 import { HTTPError } from '../errors/http-error';
-
 import search_queries from '../queries/search';
 import * as authorization from '../request-handlers/security/authorization';
 import { getMockDBConnection } from '../__mocks__/db';

api/src/repositories/error-repository.ts

@@ -0,0 +1,147 @@
+import SQL from 'sql-template-strings';
+import { ApiExecuteSQLError } from '../errors/api-error';
+import { BaseRepository } from './base-repository';
+
+export enum SUBMISSION_STATUS_TYPE {
+  'PUBLISHED' = 'Published',
+  'REJECTED' = 'Rejected',
+  'SYSTEM_ERROR' = 'System Error',
+  //Success
+  'OUT_DATED_RECORD' = 'Out Dated Record',
+  'INGESTED' = 'Ingested',
+  'UPLOADED' = 'Uploaded',
+  'VALIDATED' = 'Validated',
+  'SECURED' = 'Secured',
+  'EML_INGESTED' = 'EML Ingested',
+  'EML_TO_JSON' = 'EML To JSON',
+  'METADATA_TO_ES' = 'Metadata To ES',
+  'NORMALIZED' = 'Normalized',
+  'SPATIAL_TRANSFORM_UNSECURE' = 'Spatial Transform Unsecure',
+  'SPATIAL_TRANSFORM_SECURE' = 'Spatial Transform Secure',
+  //Failure
+  'FAILED_INGESTION' = 'Failed Ingestion',
+  'FAILED_UPLOAD' = 'Failed Upload',
+  'FAILED_VALIDATION' = 'Failed Validation',
+  'FAILED_SECURITY' = 'Failed Security',
+  'FAILED_EML_INGESTION' = 'Failed EML Ingestion',
+  'FAILED_EML_TO_JSON' = 'Failed EML To JSON',
+  'FAILED_METADATA_TO_ES' = 'Failed Metadata To ES',
+  'FAILED_NORMALIZATION' = 'Failed Normalization',
+  'FAILED_SPATIAL_TRANSFORM_UNSECURE' = 'Failed Spatial Transform Unsecure',
+  'FAILED_SPATIAL_TRANSFORM_SECURE' = 'Failed Spatial Transform Secure'
+}
+
+export enum SUBMISSION_MESSAGE_TYPE {
+  'NOTICE' = 'Notice',
+  'ERROR' = 'Error',
+  'WARNING' = 'Warning',
+  'DEBUG' = 'Debug'
+}
+
+/**
+ * A repository class for accessing permit data.
+ *
+ * @export
+ * @class PermitRepository
+ * @extends {BaseRepository}
+ */
+export class ErrorRepository extends BaseRepository {
+  /**
+   * Insert a new submission status record.
+   *
+   * @param {number} submissionId
+   * @param {SUBMISSION_STATUS_TYPE} submissionStatusType
+   * @return {*}  {Promise<{ submission_status_id: number; submission_status_type_id: number }>}
+   * @memberof SubmissionRepository
+   */
+  async insertSubmissionStatus(
+    submissionId: number,
+    submissionStatusType: SUBMISSION_STATUS_TYPE
+  ): Promise<{ submission_status_id: number; submission_status_type_id: number }> {
+    const sqlStatement = SQL`
+      INSERT INTO submission_status (
+        submission_id,
+        submission_status_type_id,
+        event_timestamp
+      ) VALUES (
+        ${submissionId},
+        (
+          SELECT
+            submission_status_type_id
+          FROM
+            submission_status_type
+          WHERE
+            name = ${submissionStatusType}
+        ),
+        now()
+      )
+      RETURNING
+        submission_status_id,
+        submission_status_type_id;
+    `;
+
+    const response = await this.connection.sql<{ submission_status_id: number; submission_status_type_id: number }>(
+      sqlStatement
+    );
+
+    if (response.rowCount !== 1) {
+      throw new ApiExecuteSQLError('Failed to insert submission status record', [
+        'SubmissionRepository->insertSubmissionStatus',
+        'rowCount was null or undefined, expected rowCount = 1'
+      ]);
+    }
+
+    return response.rows[0];
+  }
+
+  /**
+   * Insert a submission message record.
+   *
+   * @param {number} submissionStatusId
+   * @param {SUBMISSION_MESSAGE_TYPE} submissionMessageType
+   * @return {*}  {Promise<{ submission_message_id: number; submission_message_type_id: number }>}
+   * @memberof SubmissionRepository
+   */
+  async insertSubmissionMessage(
+    submissionStatusId: number,
+    submissionMessageType: SUBMISSION_MESSAGE_TYPE,
+    submissionMessage: string
+  ): Promise<{ submission_message_id: number; submission_message_type_id: number }> {
+    const sqlStatement = SQL`
+        INSERT INTO submission_message (
+          submission_status_id,
+          submission_message_type_id,
+          event_timestamp,
+          message
+        ) VALUES (
+          ${submissionStatusId},
+          (
+            SELECT
+              submission_message_type_id
+            FROM
+              submission_message_type
+            WHERE
+              name = ${submissionMessageType}
+          ),
+          now(),
+          ${submissionMessage}
+        )
+        RETURNING
+          submission_message_id,
+          submission_message_type_id;
+      `;
+
+    const response = await this.connection.sql<{ submission_message_id: number; submission_message_type_id: number }>(
+      sqlStatement
+    );
+
+    if (response.rowCount !== 1) {
+      throw new ApiExecuteSQLError('Failed to insert submission message record', [
+        'SubmissionRepository->insertSubmissionMessage',
+        'rowCount was null or undefined, expected rowCount = 1'
+      ]);
+    }
+
+    return response.rows[0];
+  }
+}

api/src/services/code-service.ts

@@ -1,7 +1,7 @@
 import { coordinator_agency, region, regional_offices } from '../constants/codes';
 import { queries } from '../queries/queries';
 import { getLogger } from '../utils/logger';
-import { DBService } from './service';
+import { DBService } from './db-service';
 
 const defaultLog = getLogger('queries/code-queries');
 

api/src/services/eml-service.ts

@@ -17,8 +17,8 @@ import {
 } from '../models/survey-view';
 import { getDbCharacterSystemMetaDataConstantSQL } from '../queries/codes/db-constant-queries';
 import { CodeService, IAllCodeSets } from './code-service';
+import { DBService } from './db-service';
 import { ProjectService } from './project-service';
-import { DBService } from './service';
 import { SurveyService } from './survey-service';
 import { TaxonomyService } from './taxonomy-service';
 

api/src/services/error-service.ts

@@ -0,0 +1,69 @@
+import { IDBConnection } from '../database/db';
+import { ErrorRepository, SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE } from '../repositories/error-repository';
+import { DBService } from './db-service';
+
+export class ErrorService extends DBService {
+  errorRepository: ErrorRepository;
+
+  constructor(connection: IDBConnection) {
+    super(connection);
+
+    this.errorRepository = new ErrorRepository(connection);
+  }
+
+  /**
+   * Inserts both the status and message of a submission
+   *
+   * @param {number} submissionId
+   * @param {SUBMISSION_STATUS_TYPE} submissionStatusType
+   * @param {SUBMISSION_MESSAGE_TYPE} submissionMessageType
+   * @param {string} submissionMessage
+   * @return {*}  {Promise<{
+   *     submission_status_id: number;
+   *     submission_message_id: number;
+   *   }>}
+   * @memberof SubmissionService
+   */
+  async insertSubmissionStatusAndMessage(
+    submissionId: number,
+    submissionStatusType: SUBMISSION_STATUS_TYPE,
+    submissionMessageType: SUBMISSION_MESSAGE_TYPE,
+    submissionMessage: string
+  ): Promise<{
+    submission_status_id: number;
+    submission_message_id: number;
+  }> {
+    const submission_status_id = (await this.errorRepository.insertSubmissionStatus(submissionId, submissionStatusType))
+      .submission_status_id;
+
+    const submission_message_id = (
+      await this.errorRepository.insertSubmissionMessage(submission_status_id, submissionMessageType, submissionMessage)
+    ).submission_message_id;
+
+    return {
+      submission_status_id,
+      submission_message_id
+    };
+  }
+
+  /**
+   * Insert a submission status record.
+   *
+   * @param {number} submissionId
+   * @param {SUBMISSION_STATUS_TYPE} submissionStatusType
+   * @return {*}  {Promise<{
+   *     submission_status_id: number;
+   *     submission_status_type_id: number;
+   *   }>}
+   * @memberof SubmissionService
+   */
+  async insertSubmissionStatus(
+    submissionId: number,
+    submissionStatusType: SUBMISSION_STATUS_TYPE
+  ): Promise<{
+    submission_status_id: number;
+    submission_status_type_id: number;
+  }> {
+    return this.errorRepository.insertSubmissionStatus(submissionId, submissionStatusType);
+  }
+}

api/src/services/permit-service.ts

@@ -2,7 +2,7 @@ import { SYSTEM_ROLE } from '../constants/roles';
 import { IDBConnection } from '../database/db';
 import { ApiGeneralError } from '../errors/api-error';
 import { IPermitModel, PermitRepository } from '../repositories/permit-repository';
-import { DBService } from './service';
+import { DBService } from './db-service';
 import { UserService } from './user-service';
 export class PermitService extends DBService {
   permitRepository: PermitRepository;

api/src/services/platform-service.ts

@@ -4,9 +4,9 @@ import FormData from 'form-data';
 import { URL } from 'url';
 import { HTTP400 } from '../errors/http-error';
 import { getFileFromS3 } from '../utils/file-utils';
+import { DBService } from './db-service';
 import { EmlService } from './eml-service';
 import { KeycloakService } from './keycloak-service';
-import { DBService } from './service';
 import { SurveyService } from './survey-service';
 
 export interface IDwCADataset {

api/src/services/project-service.ts

@@ -29,7 +29,7 @@ import { getSurveyAttachmentS3Keys } from '../paths/project/{projectId}/survey/{
 import { GET_ENTITIES, IUpdateProject } from '../paths/project/{projectId}/update';
 import { queries } from '../queries/queries';
 import { deleteFileFromS3 } from '../utils/file-utils';
-import { DBService } from './service';
+import { DBService } from './db-service';
 
 export class ProjectService extends DBService {
   /**