Feature/rohit logger

.gitignore

@@ -355,4 +355,3 @@ package-lock.json
 ui/.env
 upload-api/sitecoreMigrationData
 upload-api/extracted_files
-locale-cli

api/.gitignore

@@ -361,4 +361,5 @@ package-lock.json
 !example.env
 
 database/
-/sitecoreMigrationData
+/sitecoreMigrationData
+/migration-data

api/package.json

@@ -35,6 +35,7 @@
     "express": "^4.18.2",
     "express-validator": "^7.0.1",
     "express-winston": "^4.2.0",
+    "fs-extra": "^11.2.0",
     "fs-readdir-recursive": "^1.1.0",
     "helmet": "^7.1.0",
     "html-to-json-parser": "^2.0.1",
@@ -50,6 +51,7 @@
   "devDependencies": {
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",
+    "@types/fs-extra": "^11.0.4",
     "@types/fs-readdir-recursive": "^1.1.3",
     "@types/jsdom": "^21.1.7",
     "@types/jsonwebtoken": "^9.0.5",

api/src/config/index.ts

@@ -10,6 +10,7 @@ dotenv.config({
   path: path.resolve(process.cwd(), `${process.env.NODE_ENV}.env`),
 });
 
+
 /**
  * Configuration type for the application.
  */

api/src/controllers/migration.controller.ts

@@ -21,7 +21,7 @@ const createTestStack = async (req: Request, res: Response): Promise<void> => {
  * @returns {Promise<void>} - A Promise that resolves when the stack is deleted.
  */
 const fieldMapping = async (req: Request, res: Response): Promise<void> => {
-  const resp = await migrationService.fieldMapping(req);
+  const resp = migrationService.fieldMapping(req);
   res.status(200).json(resp);
 };
 

api/src/services/migration.service.ts

@@ -1,5 +1,5 @@
 import { Request } from "express";
-import cliUtilities from '@contentstack/cli-utilities';
+// import cliUtilities from '@contentstack/cli-utilities';
 import { config } from "../config/index.js";
 import { safePromise, getLogMessage } from "../utils/index.js";
 import https from "../utils/https.utils.js";
@@ -10,12 +10,16 @@ import { HTTP_TEXTS, HTTP_CODES, CS_REGIONS } from "../constants/index.js";
 import { ExceptionFunction } from "../utils/custom-errors.utils.js";
 import { fieldAttacher } from "../utils/field-attacher.utils.js";
 import ProjectModelLowdb from "../models/project-lowdb.js";
-// import shell from 'shelljs'
-// import path from "path";
+import shell from 'shelljs'
+import path from "path";
 import AuthenticationModel from "../models/authentication.js";
 import { siteCoreService } from "./sitecore.service.js";
+import { fileURLToPath } from 'url';
+import { copyDirectory } from '../utils/index.js'
+
+
+
 
-// const importCmd: any = await import('@contentstack/cli-cm-import');
 
 /**
  * Creates a test stack.
@@ -198,65 +202,88 @@ const cliLogger = (child: any) => {
     console.info(`Error: Failed to install @contentstack/cli. Exit code: ${child.code}`);
     console.info(`stderr: ${child.stderr}`);
   } else {
-    console.info('Installation successful', child?.stdout);
+    console.info('Installation successful', child, child?.stdout);
   }
 };
 
-const runCli = async (rg: string, user_id: string) => {
+const runCli = async (rg: string, user_id: string, project: any) => {
   try {
     const regionPresent = CS_REGIONS?.find((item: string) => item === rg) ?? 'NA';
-    // const email = '[email protected]'
     await AuthenticationModel.read();
     const userData = AuthenticationModel.chain
       .get("users")
       .find({ region: regionPresent, user_id })
       .value();
-    if (userData?.authtoken) {
-
-      cliUtilities?.configHandler?.set('region', regionPresent);
-      cliUtilities?.configHandler?.set('authtoken', userData?.authtoken);
-      // shell.cd(path.resolve(process.cwd(), `../cli/packages/contentstack`));
-      // const pwd = shell.exec('pwd');
-      // cliLogger(pwd);
-      // const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
-      // cliLogger(region);
-      // const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${email}`)
-      // cliLogger(login);
-      // const exportData = shell.exec(`node bin/run cm:stacks:import  -k blt3e7d2a4135d8bfab -d "/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/data" --backup-dir="/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab"`);
-      // cliLogger(exportData);
-      // const cmd = [`-k ${userData?.authtoken}`, "-d /Users/umesh.more/Documents/ui-migration/migration-v2-node-server/api/sitecoreMigrationData", "--backup-dir=/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab", "--yes"]
-
-      // await importCmd.default.run(cmd);  // This will bypass the type issue
-      // shell.cd(path.resolve(process.cwd(), '..', 'locale-cli', 'packages', 'contentstack'));
-      // const pwd = shell.exec('pwd');
-      // cliLogger(pwd);
-      // const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
-      // cliLogger(region);
-      // const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${email}`)
-      // cliLogger(login);
-      // const exportData = shell.exec(`node bin/run cm:stacks:import  -k blt69235b992c3d99c6 -d "/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/api/sitecoreMigrationData" --backup-dir="/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/test"`);
-      // cliLogger(exportData);
+    if (userData?.authtoken && project?.destination_stack_id) {
+      // Manually define __filename and __dirname
+      const __filename = fileURLToPath(import.meta.url);
+      const dirPath = path.join(path.dirname(__filename), '..', '..');
+      const sourcePath = path.join(dirPath, 'sitecoreMigrationData', project?.destination_stack_id);
+      const backupPath = path.join(process.cwd(), 'migration-data', project?.destination_stack_id);
+      await copyDirectory(sourcePath, backupPath);
+      shell.cd(path.join(process.cwd(), '..', 'cli', 'packages', 'contentstack'));
+      const pwd = shell.exec('pwd');
+      cliLogger(pwd);
+      const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
+      cliLogger(region);
+      const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${userData?.email}`);
+      cliLogger(login);
+      const exportData = shell.exec(`node bin/run cm:stacks:import  -k ${project?.destination_stack_id} -d ${sourcePath} --backup-dir=${backupPath}  --yes`);
+      cliLogger(exportData);
     } else {
       console.info('user not found.')
     }
   } catch (er) {
-    console.info("🚀 ~ runCli ~ er:", er)
+    console.error("🚀 ~ runCli ~ er:", er)
   }
 }
 
 const fieldMapping = async (req: Request): Promise<any> => {
   const { orgId, projectId } = req?.params ?? {};
-  const contentTypes = await fieldAttacher({ orgId, projectId });
-  const packagePath = '/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/upload-api/extracted_files/package 45';
-  await siteCoreService?.createEntry({ packagePath, contentTypes });
-  await siteCoreService?.createLocale(req);
-  await siteCoreService?.createVersionFile();
   const { region, user_id } = req?.body?.token_payload ?? {};
-  await runCli(region, user_id);
+  const project = ProjectModelLowdb.chain
+    .get("projects")
+    .find({ id: projectId })
+    .value();
+  if (project?.extract_path && project?.destination_stack_id) {
+    const packagePath = project?.extract_path;
+    const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id });
+    await siteCoreService?.createEntry({ packagePath, contentTypes, destinationStackId: project?.destination_stack_id });
+    await siteCoreService?.createLocale(req, project?.destination_stack_id);
+    await siteCoreService?.createVersionFile(project?.destination_stack_id);
+    await runCli(region, user_id, project);
+  }
 }
 
 export const migrationService = {
   createTestStack,
   deleteTestStack,
   fieldMapping
 };
+
+
+
+
+
+// cliUtilities?.configHandler?.set('region', regionPresent);
+// cliUtilities?.configHandler?.set('authtoken', userData?.authtoken);
+// shell.cd(path.resolve(process.cwd(), `../cli/packages/contentstack`));
+// const pwd = shell.exec('pwd');
+// cliLogger(pwd);
+// const region = shell.exec(`node bin/run config:set:region ${regionPresent}`);
+// cliLogger(region);
+// const login = shell.exec(`node bin/run login -a ${userData?.authtoken}  -e ${email}`)
+// cliLogger(login);
+// const exportData = shell.exec(`node bin/run cm:stacks:import  -k blt3e7d2a4135d8bfab -d "/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/data" --backup-dir="/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab"`);
+// cliLogger(exportData);
+// const cmd = [`-k ${userData?.authtoken}`, "-d /Users/umesh.more/Documents/ui-migration/migration-v2-node-server/api/sitecoreMigrationData", "--backup-dir=/Users/umesh.more/Documents/ui-migration/migration-v2-node-server/migrations/blt3e7d2a4135d8bfab", "--yes"]
+// await importCmd.default.run(cmd);  // This will bypass the type issue
+// shell.cd(path.resolve(`${path?.dirname}`, '..', 'cli', 'packages', 'contentstack'));
+// const importCmd: any = await import('@contentstack/cli-cm-import');
+
+
+
+
+
+
+

api/src/services/sitecore.service.ts

@@ -6,9 +6,7 @@ import _ from 'lodash';
 import { LOCALE_MAPPER } from '../constants/index.js';
 import { entriesFieldCreator, unflatten } from '../utils/entries-field-creator.utils.js';
 import { orgService } from './org.service.js';
-const assetsSave = path.join('sitecoreMigrationData', 'assets');
-const entrySave = path.join('sitecoreMigrationData', 'entries');
-const localeSave = path.join('sitecoreMigrationData', 'locale');
+
 const append = "a";
 
 const idCorrector = ({ id }: any) => {
@@ -26,7 +24,7 @@ function startsWithNumber(str: string) {
 
 function getLastKey(path: string) {
   const keys = path?.split?.('.');
-  const lastKey = keys[keys.length - 1];
+  const lastKey = keys?.[keys?.length - 1];
   return lastKey;
 }
 
@@ -79,7 +77,8 @@ const uidCorrector = ({ uid }: any) => {
   return _.replace(uid, new RegExp("[ -]", "g"), '_')?.toLowerCase()
 }
 
-const cretaeAssets = async ({ packagePath }: any) => {
+const cretaeAssets = async ({ packagePath, baseDir }: any) => {
+  const assetsSave = path.join(baseDir, 'assets');
   const allAssetJSON: any = {};
   const folderName: any = path.join(packagePath, 'items', 'master', 'sitecore', 'media library');
   const entryPath = read?.(folderName);
@@ -163,9 +162,11 @@ const cretaeAssets = async ({ packagePath }: any) => {
   return allAssetJSON;
 }
 
-const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us' }: { packagePath: any; contentTypes: any; master_locale?: string }) => {
+const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us', destinationStackId }: { packagePath: any; contentTypes: any; master_locale?: string, destinationStackId: string }) => {
   try {
-    const allAssetJSON: any = await cretaeAssets({ packagePath });
+    const baseDir = path.join('sitecoreMigrationData', destinationStackId);
+    const entrySave = path.join(baseDir, 'entries');
+    const allAssetJSON: any = await cretaeAssets({ packagePath, baseDir });
     const folderName: any = path.join(packagePath, 'items', 'master', 'sitecore', 'content');
     const entriesData: any = [];
     if (fs.existsSync(folderName)) {
@@ -240,7 +241,9 @@ const createEntry = async ({ packagePath, contentTypes, master_locale = 'en-us'
   }
 }
 
-const createLocale = async (req: any) => {
+const createLocale = async (req: any, destinationStackId: string) => {
+  const baseDir = path.join('sitecoreMigrationData', destinationStackId);
+  const localeSave = path.join(baseDir, 'locale');
   const allLocalesResp = await orgService.getLocales(req)
   const masterLocale = Object?.keys?.(LOCALE_MAPPER?.masterLocale)?.[0];
   const msLocale: any = {};
@@ -280,8 +283,9 @@ const createLocale = async (req: any) => {
   })
 }
 
-const createVersionFile = async () => {
-  fs.writeFile(path?.join?.('sitecoreMigrationData', 'export-info.json'), JSON.stringify({
+const createVersionFile = async (destinationStackId: string) => {
+  const baseDir = path.join('sitecoreMigrationData', destinationStackId);
+  fs.writeFile(path?.join?.(baseDir, 'export-info.json'), JSON.stringify({
     "contentVersion": 2,
     "logsPath": ""
   }), (err) => {

api/src/utils/content-type-creator.utils.ts

@@ -1,7 +1,5 @@
 import fs from 'fs';
 import path from 'path';
-const contentSave = path.join('sitecoreMigrationData', 'content_types');
-const globalSave = path.join('sitecoreMigrationData', 'global_fields');
 interface Group {
   data_type: string;
   display_name?: string; // Assuming item?.contentstackField might be undefined
@@ -332,7 +330,7 @@ const convertToSchemaFormate = ({ field, advanced = true }: any) => {
   }
 }
 
-const saveContent = async (ct: any) => {
+const saveContent = async (ct: any, contentSave: string) => {
   try {
     // Check if the directory exists
     await fs.promises.access(contentSave).catch(async () => {
@@ -366,7 +364,7 @@ const saveContent = async (ct: any) => {
 }
 
 
-const writeGlobalField = async (schema: any) => {
+const writeGlobalField = async (schema: any, globalSave: string) => {
   const filePath = path.join(process.cwd(), globalSave, 'globalfields.json');
   try {
     await fs.promises.access(globalSave);
@@ -396,7 +394,7 @@ const writeGlobalField = async (schema: any) => {
   }
 };
 
-export const contenTypeMaker = async ({ contentType }: any) => {
+export const contenTypeMaker = async ({ contentType, destinationStackId }: any) => {
   const ct: ContentType = {
     title: contentType?.contentstackTitle,
     uid: contentType?.contentstackUid,
@@ -441,9 +439,11 @@ export const contenTypeMaker = async ({ contentType }: any) => {
   })
   if (ct?.uid) {
     if (contentType?.type === 'global_field') {
-      await writeGlobalField(ct);
+      const globalSave = path.join('sitecoreMigrationData', destinationStackId, 'global_fields');
+      await writeGlobalField(ct, globalSave);
     } else {
-      await saveContent(ct);
+      const contentSave = path.join('sitecoreMigrationData', destinationStackId, 'content_types');
+      await saveContent(ct, contentSave);
     }
   } else {
     console.info(contentType?.contentstackUid, 'missing')

api/src/utils/field-attacher.utils.ts

@@ -3,7 +3,7 @@ import ContentTypesMapperModelLowdb from "../models/contentTypesMapper-lowdb.js"
 import FieldMapperModel from "../models/FieldMapper.js";
 import { contenTypeMaker } from "./content-type-creator.utils.js";
 
-export const fieldAttacher = async ({ projectId, orgId }: any) => {
+export const fieldAttacher = async ({ projectId, orgId, destinationStackId }: any) => {
   await ProjectModelLowdb.read();
   const projectData = ProjectModelLowdb.chain.get("projects").find({
     id: projectId,
@@ -27,7 +27,7 @@ export const fieldAttacher = async ({ projectId, orgId }: any) => {
           return field;
         })
       }
-      await contenTypeMaker({ contentType })
+      await contenTypeMaker({ contentType, destinationStackId })
       contentTypes?.push?.(contentType);
     }
   }

api/src/utils/index.ts

@@ -1,9 +1,11 @@
+import fs from 'fs-extra';
 /**
  * Throws an error with a custom message and status code.
  * @param message - The error message.
  * @param statusCode - The HTTP status code associated with the error.
  * @throws {Error} - The error object with the specified message and status code.
  */
+
 export const throwError = (message: string, statusCode: number) => {
   throw Object.assign(new Error(message), { statusCode });
 };
@@ -37,6 +39,7 @@ export const safePromise = (promise: Promise<any>): Promise<any> =>
  * @param error - The error object. Optional.
  * @returns The log message object.
  */
+
 export const getLogMessage = (
   methodName: string,
   message: string,
@@ -50,3 +53,24 @@ export const getLogMessage = (
     ...(error && { error }),
   };
 };
+
+/*
+ * Recursively copies a directory from source to destination
+ * @param srcDir - Source directory path
+ * @param destDir - Destination directory path
+*/
+
+export async function copyDirectory(srcDir: string, destDir: string): Promise<void> {
+  try {
+    // Ensure the destination directory exists, if not, create it
+    await fs.ensureDir(destDir);
+
+    // Copy the source directory to the destination
+    await fs.copy(srcDir, destDir);
+
+    console.info(`Directory copied from ${srcDir} to ${destDir}`);
+
+  } catch (error) {
+    console.error(`Error copying directory: ${error}`);
+  }
+}

ui/src/components/TestMigration/index.tsx

@@ -79,7 +79,7 @@ const TestMigration = () => {
       <div className='content-block'>
         <div className='content-header'>Execution Logs</div>
         <div>
-          <LogViewer serverPath="http://localhost:5001" />
+          <LogViewer serverPath="http://localhost:5000" />
         </div>
       </div>
     </div>

upload-api/.env

@@ -1,2 +1,2 @@
 PORT=4002
-NODE_BACKEND_API =http://localhost:5001
+NODE_BACKEND_API =http://localhost:5000