Feature/hanoak

README.md

@@ -2,7 +2,7 @@
 
 This is the TSO migration V2's node server.
 
-## To start the sever
+## To start the server
 
 1.  Run `npm i`
 2.  Install & start MongoDB

example.env

@@ -1,3 +1,5 @@
 PORT=
 APP_TOKEN_KEY=
-MONGODB_URI=
+MONGODB_URI=
+UPLOAD_BUCKET=
+AWS_REGION=

package.json

@@ -23,6 +23,8 @@
   },
   "homepage": "https://github.com/contentstack-expert-services/migration-v2-node-server#readme",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.511.0",
+    "@aws-sdk/s3-request-presigner": "^3.511.0",
     "@types/express": "^4.17.21",
     "axios": "^1.6.5",
     "cors": "^2.8.5",

src/config/index.ts

@@ -13,6 +13,9 @@ export type ConfigType = {
   PORT: string;
   APP_ENV: string;
   MONGODB_URI: string;
+  AWS_REGION: string;
+  UPLOAD_BUCKET: string;
+  UPLOAD_URL_EXPIRES: number;
   CS_API: {
     NA: string;
     EU: string;
@@ -27,5 +30,8 @@ export const config: ConfigType = {
   APP_ENV: process.env.NODE_ENV!,
   APP_TOKEN_KEY: process.env.APP_TOKEN_KEY!,
   MONGODB_URI: process.env.MONGODB_URI!,
+  AWS_REGION: process.env.AWS_REGION!,
+  UPLOAD_BUCKET: process.env.UPLOAD_BUCKET!,
+  UPLOAD_URL_EXPIRES: 60 * 30,
   ...(process.env.NODE_ENV === "production" ? prodConfig : devConfig),
 };

src/constants/index.ts

@@ -48,6 +48,7 @@ export const HTTP_TEXTS = {
   INVALID_CONTENT_TYPE: "Provide valid ContentType data",
   RESET_CONTENT_MAPPING:
     "ContentType has been successfully restored to its initial mapping",
+  UPLOAD_SUCCESS: "File uploaded successfully",
 };
 export const HTTP_RESPONSE_HEADERS = {
   "Access-Control-Allow-Origin": "*",
@@ -66,6 +67,7 @@ export const VALIDATION_ERRORS = {
   LENGTH_LIMIT: "$'s max limit reached.",
   STRING_REQUIRED: "Provided $ should be a string.",
   INVALID_REGION: "Provided region doesn't exists.",
+  FIELD_REQUIRED: "Field '$' is required.",
 };
 export const PROJECT_POPULATE_FIELDS = "content_mapper";
 export const CONTENT_TYPE_POPULATE_FIELDS =

src/controllers/projects.uploads.controller.ts

@@ -0,0 +1,22 @@
+import { Request, Response } from "express";
+import { uploadsService } from "../services/projects.uploads.service";
+
+const initializeUpload = async (req: Request, res: Response) => {
+  const resp = await uploadsService.initializeUpload(req);
+  res.status(resp.status).json(resp.data);
+};
+
+const getPreSignedUrls = async (req: Request, res: Response) => {
+  const resp = await uploadsService.getPreSignedUrls(req);
+  res.status(resp.status).json(resp.data);
+};
+const finalizeUpload = async (req: Request, res: Response) => {
+  const resp = await uploadsService.finalizeUpload(req);
+  res.status(resp.status).json(resp.data);
+};
+
+export const uploadController = {
+  initializeUpload,
+  getPreSignedUrls,
+  finalizeUpload,
+};

src/routes/projects.routes.ts

@@ -2,6 +2,7 @@ import express from "express";
 import { projectController } from "../controllers/projects.controller";
 import { asyncRouter } from "../utils/async-router.utils";
 import validator from "../validators";
+import uploadRouter from "./projects.uploads.routes";
 
 const router = express.Router({ mergeParams: true });
 
@@ -41,4 +42,7 @@ router.put(
 // Delete a project route
 router.delete("/:projectId", asyncRouter(projectController.deleteProject));
 
+// Multipart upload route
+router.use("/:projectId/uploads", uploadRouter);
+
 export default router;

src/routes/projects.uploads.routes.ts

@@ -0,0 +1,29 @@
+import express from "express";
+import { uploadController } from "../controllers/projects.uploads.controller";
+import { asyncRouter } from "../utils/async-router.utils";
+import validator from "../validators";
+
+const router = express.Router({ mergeParams: true });
+
+// Initialize multipart upload
+router.post(
+  "/initialize",
+  validator("initialize_upload"),
+  asyncRouter(uploadController.initializeUpload)
+);
+
+// GET pre-signed urls
+router.post(
+  "/pre-signed-urls",
+  validator("file_upload"),
+  asyncRouter(uploadController.getPreSignedUrls)
+);
+
+// Finalize multipart upload
+router.post(
+  "/finalize",
+  validator("file_upload"),
+  asyncRouter(uploadController.finalizeUpload)
+);
+
+export default router;

src/services/projects.service.ts

@@ -7,24 +7,11 @@ import {
   HTTP_TEXTS,
   HTTP_CODES,
 } from "../constants";
-import { MigrationQueryType } from "../models/types";
 import { config } from "../config";
-import { isValidObjectId, safePromise } from "../utils";
+import { safePromise } from "../utils";
 import getAuthtoken from "../utils/auth.utils";
 import https from "../utils/https.utils";
-
-const _getProject = async (projectId: string, query: MigrationQueryType) => {
-  if (!isValidObjectId(projectId))
-    throw new BadRequestError(HTTP_TEXTS.INVALID_ID.replace("$", "project"));
-
-  const project = await ProjectModel.findOne(query).select(
-    EXCLUDE_CONTENT_MAPPER
-  );
-
-  if (!project) throw new NotFoundError(HTTP_TEXTS.NO_PROJECT);
-
-  return project;
-};
+import getProjectUtil from "../utils/get-project.utils";
 
 const getAllProjects = async (req: Request) => {
   const orgId = req?.params?.orgId;
@@ -48,12 +35,16 @@ const getProject = async (req: Request) => {
   const decodedToken = req.body.token_payload;
   const { user_id = "", region = "" } = decodedToken;
   // Find the project based on both orgId and projectId, region, owner
-  const project = await _getProject(projectId, {
-    _id: projectId,
-    org_id: orgId,
-    region: region,
-    owner: user_id,
-  });
+  const project = await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: region,
+      owner: user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
 
   return project;
 };
@@ -97,12 +88,16 @@ const updateProject = async (req: Request) => {
   const { user_id = "", region = "" } = decodedToken;
 
   // Find the project based on both orgId and projectId
-  const project = await _getProject(projectId, {
-    _id: projectId,
-    org_id: orgId,
-    region: region,
-    owner: user_id,
-  });
+  const project = await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: region,
+      owner: user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
 
   // Update the project fields
   project.name = updateData?.name || project.name;
@@ -131,12 +126,16 @@ const updateLegacyCMS = async (req: Request) => {
   const { orgId, projectId } = req.params;
   const { token_payload, legacy_cms } = req.body;
 
-  const project = await _getProject(projectId, {
-    _id: projectId,
-    org_id: orgId,
-    region: token_payload?.region,
-    owner: token_payload?.user_id,
-  });
+  const project = await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: token_payload?.region,
+      owner: token_payload?.user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
 
   project.legacy_cms.cms = legacy_cms;
 
@@ -154,12 +153,16 @@ const updateFileFormat = async (req: Request) => {
   const { orgId, projectId } = req.params;
   const { token_payload, file_format } = req.body;
 
-  const project = await _getProject(projectId, {
-    _id: projectId,
-    org_id: orgId,
-    region: token_payload?.region,
-    owner: token_payload?.user_id,
-  });
+  const project = await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: token_payload?.region,
+      owner: token_payload?.user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
 
   project.legacy_cms.file_format = file_format;
 
@@ -177,12 +180,16 @@ const updateDestinationStack = async (req: Request) => {
   const { orgId, projectId } = req.params;
   const { token_payload, stack_api_key } = req.body;
 
-  const project = await _getProject(projectId, {
-    _id: projectId,
-    org_id: orgId,
-    region: token_payload?.region,
-    owner: token_payload?.user_id,
-  });
+  const project = await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: token_payload?.region,
+      owner: token_payload?.user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
 
   const authtoken = await getAuthtoken(
     token_payload?.region,

src/services/projects.uploads.service.ts

@@ -0,0 +1,93 @@
+import { Request } from "express";
+import { EXCLUDE_CONTENT_MAPPER, HTTP_TEXTS, HTTP_CODES } from "../constants";
+import { initialize, preSignedUrls, finalize } from "../utils/s3-uploads.utils";
+import getProjectUtil from "../utils/get-project.utils";
+
+const initializeUpload = async (req: Request) => {
+  const orgId = req?.params?.orgId;
+  const projectId = req?.params?.projectId;
+  const fileName = req.body.file_name;
+  const { user_id = "", region = "" } = req.body.token_payload;
+
+  // Find the project based on both orgId and projectId, region, owner
+  await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: region,
+      owner: user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
+
+  const result = await initialize(region, orgId, user_id, projectId, fileName);
+
+  return {
+    status: HTTP_CODES.OK,
+    data: {
+      file_id: result.UploadId,
+      file_key: result.Key,
+    },
+  };
+};
+
+const getPreSignedUrls = async (req: Request) => {
+  const orgId = req?.params?.orgId;
+  const projectId = req?.params?.projectId;
+  const { file_key, file_id, parts } = req.body;
+  const { user_id = "", region = "" } = req.body.token_payload;
+
+  // Find the project based on both orgId and projectId, region, owner
+  await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: region,
+      owner: user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
+
+  const result = await preSignedUrls(file_key, file_id, parts);
+
+  return {
+    status: HTTP_CODES.OK,
+    data: { parts: result },
+  };
+};
+
+const finalizeUpload = async (req: Request) => {
+  const orgId = req?.params?.orgId;
+  const projectId = req?.params?.projectId;
+  const { file_key, file_id, parts } = req.body;
+  const { user_id = "", region = "" } = req.body.token_payload;
+
+  // Find the project based on both orgId and projectId, region, owner
+  await getProjectUtil(
+    projectId,
+    {
+      _id: projectId,
+      org_id: orgId,
+      region: region,
+      owner: user_id,
+    },
+    EXCLUDE_CONTENT_MAPPER
+  );
+
+  await finalize(file_key, file_id, parts);
+
+  return {
+    status: HTTP_CODES.OK,
+    data: {
+      message: HTTP_TEXTS.UPLOAD_SUCCESS,
+    },
+  };
+};
+
+export const uploadsService = {
+  initializeUpload,
+  getPreSignedUrls,
+  finalizeUpload,
+};

src/utils/get-project.utils.ts

@@ -0,0 +1,28 @@
+import ProjectModel from "../models/project";
+import {
+  BadRequestError,
+  NotFoundError,
+  DatabaseError,
+} from "../utils/custom-errors.utils";
+import { HTTP_TEXTS } from "../constants";
+import { MigrationQueryType } from "../models/types";
+import { isValidObjectId } from "../utils";
+
+export default async (
+  projectId: string,
+  query: MigrationQueryType,
+  projections: string = ""
+) => {
+  try {
+    if (!isValidObjectId(projectId))
+      throw new BadRequestError(HTTP_TEXTS.INVALID_ID.replace("$", "project"));
+
+    const project = await ProjectModel.findOne(query).select(projections);
+
+    if (!project) throw new NotFoundError(HTTP_TEXTS.NO_PROJECT);
+
+    return project;
+  } catch (err) {
+    throw new DatabaseError(HTTP_TEXTS.SOMETHING_WENT_WRONG);
+  }
+};