From cbefd93e80c3943b11bc6f449f100c82a291bb85 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Wed, 21 May 2025 15:12:40 -0700 Subject: [PATCH 1/4] implement json parser for agent flow to allow dot notation and array access --- server/utils/agentFlows/executor.js | 118 +++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 4 deletions(-) diff --git a/server/utils/agentFlows/executor.js b/server/utils/agentFlows/executor.js index 5501bbfb4e8..56d0c79392b 100644 --- a/server/utils/agentFlows/executor.js +++ b/server/utils/agentFlows/executor.js @@ -6,6 +6,7 @@ const executeCode = require("./executors/code"); const executeLLMInstruction = require("./executors/llm-instruction"); const executeWebScraping = require("./executors/web-scraping"); const { Telemetry } = require("../../models/telemetry"); +const { safeJsonParse } = require("../http"); class FlowExecutor { constructor() { @@ -21,15 +22,124 @@ class FlowExecutor { this.logger = loggerFn || console.info; } + /** + * Resolves nested values from objects using dot notation and array indices + * Supports paths like "data.items[0].name" or "response.users[2].address.city" + * Returns empty string for invalid paths or errors + */ + getValueFromPath(obj, path) { + try { + if (!path || typeof path !== 'string') return ''; + + const parsedObj = typeof obj === 'string' + ? safeJsonParse(obj, obj) + : obj; + + // If no dots in path, return the whole object as a string + if (!path.includes('.')) { + if (typeof parsedObj === 'object') { + try { + return JSON.stringify(parsedObj, null, 0); + } catch (e) { + return ''; // fallback for stringify errors + } + } + return String(parsedObj); + } + + // Get path without the variable name prefix + const pathWithoutVar = path.split('.').slice(1).join('.'); + + // Parse path into segments, converting array indices to [array, index] pairs + const segments = []; + let currentSegment = ''; + let inBracket = false; + let bracketContent = ''; + + // Parse each character to handle both object properties and array indices + // Example path: items[0].name.details[1].type + // Becomes: ['items', ['array', 0], 'name', 'details', ['array', 1], 'type'] + for (let i = 0; i < pathWithoutVar.length; i++) { + const char = pathWithoutVar[i]; + + // Start of array index - push current property name if exists + if (char === '[' && !inBracket) { + inBracket = true; + if (currentSegment) { + segments.push(currentSegment); + currentSegment = ''; + } + continue; + } + + // End of array index - validate and convert to number + if (char === ']' && inBracket) { + inBracket = false; + if (/^\d+$/.test(bracketContent)) { + segments.push(['array', parseInt(bracketContent, 10)]); + } + bracketContent = ''; + continue; + } + + // Build up array index or property name + if (inBracket) { + bracketContent += char; + } else if (char === '.') { + if (currentSegment) { + segments.push(currentSegment); + currentSegment = ''; + } + } else { + currentSegment += char; + } + } + + if (currentSegment) { + segments.push(currentSegment); + } + + // Navigate through object using segments, handling both array and object access + let result = parsedObj; + for (const segment of segments) { + if (result === null || result === undefined) return ''; + + if (Array.isArray(segment)) { + const [type, index] = segment; + if (type === 'array') { + if (!Array.isArray(result) || index < 0 || index >= result.length) return ''; + result = result[index]; + } + } else { + result = result?.[segment]; + } + } + + // Convert final result to string, handling objects and error cases + if (result === null || result === undefined) return ''; + if (typeof result === 'object') { + try { + return JSON.stringify(result, null, 0); + } catch (e) { + return ''; // fallback for stringify errors + } + } + return String(result); + } catch (error) { + console.error('Error in getValueFromPath:', error); + return ''; + } + } + // Utility to replace variables in config replaceVariables(config) { const deepReplace = (obj) => { if (typeof obj === "string") { - return obj.replace(/\${([^}]+)}/g, (match, varName) => { - return this.variables[varName] !== undefined - ? this.variables[varName] - : match; + const result = obj.replace(/\${([^}]+)}/g, (match, varName) => { + if (!this.variables[varName.split('.')[0]]) return match; + return this.getValueFromPath(this.variables[varName.split('.')[0]], varName); }); + return result; } if (Array.isArray(obj)) { return obj.map((item) => deepReplace(item)); From e6a3de4d57350d899882754f98eb8edaa0841835 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Wed, 21 May 2025 15:12:59 -0700 Subject: [PATCH 2/4] lint --- server/utils/agentFlows/executor.js | 56 +++++++++++++++-------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/server/utils/agentFlows/executor.js b/server/utils/agentFlows/executor.js index 56d0c79392b..c9144173a98 100644 --- a/server/utils/agentFlows/executor.js +++ b/server/utils/agentFlows/executor.js @@ -29,32 +29,30 @@ class FlowExecutor { */ getValueFromPath(obj, path) { try { - if (!path || typeof path !== 'string') return ''; + if (!path || typeof path !== "string") return ""; - const parsedObj = typeof obj === 'string' - ? safeJsonParse(obj, obj) - : obj; + const parsedObj = typeof obj === "string" ? safeJsonParse(obj, obj) : obj; // If no dots in path, return the whole object as a string - if (!path.includes('.')) { - if (typeof parsedObj === 'object') { + if (!path.includes(".")) { + if (typeof parsedObj === "object") { try { return JSON.stringify(parsedObj, null, 0); } catch (e) { - return ''; // fallback for stringify errors + return ""; // fallback for stringify errors } } return String(parsedObj); } // Get path without the variable name prefix - const pathWithoutVar = path.split('.').slice(1).join('.'); + const pathWithoutVar = path.split(".").slice(1).join("."); // Parse path into segments, converting array indices to [array, index] pairs const segments = []; - let currentSegment = ''; + let currentSegment = ""; let inBracket = false; - let bracketContent = ''; + let bracketContent = ""; // Parse each character to handle both object properties and array indices // Example path: items[0].name.details[1].type @@ -63,32 +61,32 @@ class FlowExecutor { const char = pathWithoutVar[i]; // Start of array index - push current property name if exists - if (char === '[' && !inBracket) { + if (char === "[" && !inBracket) { inBracket = true; if (currentSegment) { segments.push(currentSegment); - currentSegment = ''; + currentSegment = ""; } continue; } // End of array index - validate and convert to number - if (char === ']' && inBracket) { + if (char === "]" && inBracket) { inBracket = false; if (/^\d+$/.test(bracketContent)) { - segments.push(['array', parseInt(bracketContent, 10)]); + segments.push(["array", parseInt(bracketContent, 10)]); } - bracketContent = ''; + bracketContent = ""; continue; } // Build up array index or property name if (inBracket) { bracketContent += char; - } else if (char === '.') { + } else if (char === ".") { if (currentSegment) { segments.push(currentSegment); - currentSegment = ''; + currentSegment = ""; } } else { currentSegment += char; @@ -102,12 +100,13 @@ class FlowExecutor { // Navigate through object using segments, handling both array and object access let result = parsedObj; for (const segment of segments) { - if (result === null || result === undefined) return ''; + if (result === null || result === undefined) return ""; if (Array.isArray(segment)) { const [type, index] = segment; - if (type === 'array') { - if (!Array.isArray(result) || index < 0 || index >= result.length) return ''; + if (type === "array") { + if (!Array.isArray(result) || index < 0 || index >= result.length) + return ""; result = result[index]; } } else { @@ -116,18 +115,18 @@ class FlowExecutor { } // Convert final result to string, handling objects and error cases - if (result === null || result === undefined) return ''; - if (typeof result === 'object') { + if (result === null || result === undefined) return ""; + if (typeof result === "object") { try { return JSON.stringify(result, null, 0); } catch (e) { - return ''; // fallback for stringify errors + return ""; // fallback for stringify errors } } return String(result); } catch (error) { - console.error('Error in getValueFromPath:', error); - return ''; + console.error("Error in getValueFromPath:", error); + return ""; } } @@ -136,8 +135,11 @@ class FlowExecutor { const deepReplace = (obj) => { if (typeof obj === "string") { const result = obj.replace(/\${([^}]+)}/g, (match, varName) => { - if (!this.variables[varName.split('.')[0]]) return match; - return this.getValueFromPath(this.variables[varName.split('.')[0]], varName); + if (!this.variables[varName.split(".")[0]]) return match; + return this.getValueFromPath( + this.variables[varName.split(".")[0]], + varName + ); }); return result; } From d52fc5fc9521f3f643e089d6cc42b9a0a202d42c Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Wed, 4 Jun 2025 17:50:20 -0700 Subject: [PATCH 3/4] patch parser for pathing on objects add tests for cases --- package.json | 6 +- server/.gitignore | 1 + .../utils/agentFlows/executor.test.js | 108 +++++++++++ server/utils/agentFlows/executor.js | 168 +++++++----------- .../agentFlows/executors/llm-instruction.js | 8 +- 5 files changed, 185 insertions(+), 106 deletions(-) create mode 100644 server/__tests__/utils/agentFlows/executor.test.js diff --git a/package.json b/package.json index 6cb64040661..04226ed4ff6 100644 --- a/package.json +++ b/package.json @@ -26,10 +26,12 @@ "prod:frontend": "cd frontend && yarn build", "generate:cloudformation": "node cloud-deployments/aws/cloudformation/generate.mjs", "generate::gcp_deployment": "node cloud-deployments/gcp/deployment/generate.mjs", - "verify:translations": "cd frontend/src/locales && node verifyTranslations.mjs" + "verify:translations": "cd frontend/src/locales && node verifyTranslations.mjs", + "test": "jest" }, "private": false, "devDependencies": { - "concurrently": "^9.1.2" + "concurrently": "^9.1.2", + "jest": "^29.7.0" } } \ No newline at end of file diff --git a/server/.gitignore b/server/.gitignore index 6fdabbc0dc4..45a0f0371da 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1,5 +1,6 @@ .env.production .env.development +.env.test storage/assets/* !storage/assets/anything-llm.png storage/documents/* diff --git a/server/__tests__/utils/agentFlows/executor.test.js b/server/__tests__/utils/agentFlows/executor.test.js new file mode 100644 index 00000000000..d9538f9035e --- /dev/null +++ b/server/__tests__/utils/agentFlows/executor.test.js @@ -0,0 +1,108 @@ +const { resolve } = require("path"); + +describe("FlowExecutor: getValueFromPath", () => { + let executor; + const originalEnv = process.env; + + beforeAll(async () => { + process.env = { ...originalEnv }; + process.env.NODE_ENV = "test"; + process.env.STORAGE_DIR = resolve(__dirname, "../../../storage"); + + const { FlowExecutor } = require("../../../utils/agentFlows/executor"); + executor = new FlowExecutor(); + await new Promise(resolve => setTimeout(resolve, 2500)); // wait for the rest of the stuff to be initialized + }); + + afterAll(() => { + process.env = originalEnv; + }); + + it("can handle invalid objects", () => { + expect(executor.getValueFromPath(null, "a.b.c")).toBe(""); + expect(executor.getValueFromPath(undefined, "a.b.c")).toBe(""); + expect(executor.getValueFromPath(1, "a.b.c")).toBe(""); + expect(executor.getValueFromPath("string", "a.b.c")).toBe(""); + expect(executor.getValueFromPath(true, "a.b.c")).toBe(""); + }); + + it("can handle invalid paths", () => { + const obj = { a: { b: { c: "answer" } } }; + expect(executor.getValueFromPath(obj, -1)).toBe(""); + expect(executor.getValueFromPath(obj, undefined)).toBe(""); + expect(executor.getValueFromPath(obj, [1, 2, 3])).toBe(""); + expect(executor.getValueFromPath(obj, () => { })).toBe(""); + }); + + it("should be able to resolve a value from a dot path at various levels", () => { + let obj = { + a: { + prop: "top-prop", + b: { + c: "answer", + num: 100, + arr: [1, 2, 3], + subarr: [ + { id: 1, name: "answer2" }, + { id: 2, name: "answer3" }, + { id: 3, name: "answer4" }, + ] + } + } + }; + expect(executor.getValueFromPath(obj, "a.prop")).toBe("top-prop"); + expect(executor.getValueFromPath(obj, "a.b.c")).toBe("answer"); + expect(executor.getValueFromPath(obj, "a.b.num")).toBe(100); + expect(executor.getValueFromPath(obj, "a.b.arr[0]")).toBe(1); + expect(executor.getValueFromPath(obj, "a.b.arr[1]")).toBe(2); + expect(executor.getValueFromPath(obj, "a.b.arr[2]")).toBe(3); + expect(executor.getValueFromPath(obj, "a.b.subarr[0].id")).toBe(1); + expect(executor.getValueFromPath(obj, "a.b.subarr[0].name")).toBe("answer2"); + expect(executor.getValueFromPath(obj, "a.b.subarr[1].id")).toBe(2); + expect(executor.getValueFromPath(obj, "a.b.subarr[2].name")).toBe("answer4"); + expect(executor.getValueFromPath(obj, "a.b.subarr[2].id")).toBe(3); + }); + + it("should return empty string if the path is invalid", () => { + const result = executor.getValueFromPath({}, "a.b.c"); + expect(result).toBe(""); + }); + + it("should return empty string if the object is invalid", () => { + const result = executor.getValueFromPath(null, "a.b.c"); + expect(result).toBe(""); + }); + + it("can return a stringified item if the path target is not an object or array", () => { + const obj = { a: { b: { c: "answer", numbers: [1, 2, 3] } } }; + expect(executor.getValueFromPath(obj, "a.b")).toEqual(JSON.stringify(obj.a.b)); + expect(executor.getValueFromPath(obj, "a.b.numbers")).toEqual(JSON.stringify(obj.a.b.numbers)); + expect(executor.getValueFromPath(obj, "a.b.c")).toBe("answer"); + }); + + it("can return a stringified object if the path target is an array", () => { + const obj = { a: { b: [1, 2, 3] } }; + expect(executor.getValueFromPath(obj, "a.b")).toEqual(JSON.stringify(obj.a.b)); + expect(executor.getValueFromPath(obj, "a.b[0]")).toBe(1); + expect(executor.getValueFromPath(obj, "a.b[1]")).toBe(2); + expect(executor.getValueFromPath(obj, "a.b[2]")).toBe(3); + }); + + it("can find a value by string key traversal", () => { + const obj = { + a: { + items: [ + { + 'my-long-key': [ + { id: 1, name: "answer1" }, + { id: 2, name: "answer2" }, + { id: 3, name: "answer3" }, + ] + }, + ], + } + }; + expect(executor.getValueFromPath(obj, "a.items[0]['my-long-key'][1].id")).toBe(2); + expect(executor.getValueFromPath(obj, "a.items[0]['my-long-key'][1].name")).toBe("answer2"); + }); +}); \ No newline at end of file diff --git a/server/utils/agentFlows/executor.js b/server/utils/agentFlows/executor.js index c9144173a98..ede25dc1794 100644 --- a/server/utils/agentFlows/executor.js +++ b/server/utils/agentFlows/executor.js @@ -25,127 +25,95 @@ class FlowExecutor { /** * Resolves nested values from objects using dot notation and array indices * Supports paths like "data.items[0].name" or "response.users[2].address.city" - * Returns empty string for invalid paths or errors + * Returns undefined for invalid paths or errors + * @param {Object|string} obj - The object to resolve the value from + * @param {string} path - The path to the value + * @returns {string} The resolved value */ - getValueFromPath(obj, path) { - try { - if (!path || typeof path !== "string") return ""; - - const parsedObj = typeof obj === "string" ? safeJsonParse(obj, obj) : obj; - - // If no dots in path, return the whole object as a string - if (!path.includes(".")) { - if (typeof parsedObj === "object") { - try { - return JSON.stringify(parsedObj, null, 0); - } catch (e) { - return ""; // fallback for stringify errors - } - } - return String(parsedObj); - } - - // Get path without the variable name prefix - const pathWithoutVar = path.split(".").slice(1).join("."); - - // Parse path into segments, converting array indices to [array, index] pairs - const segments = []; - let currentSegment = ""; - let inBracket = false; - let bracketContent = ""; - - // Parse each character to handle both object properties and array indices - // Example path: items[0].name.details[1].type - // Becomes: ['items', ['array', 0], 'name', 'details', ['array', 1], 'type'] - for (let i = 0; i < pathWithoutVar.length; i++) { - const char = pathWithoutVar[i]; - - // Start of array index - push current property name if exists - if (char === "[" && !inBracket) { - inBracket = true; - if (currentSegment) { - segments.push(currentSegment); - currentSegment = ""; - } - continue; - } + getValueFromPath(obj = {}, path = "") { + if (typeof obj === "string") obj = safeJsonParse(obj, {}); // If the object is a string, parse it as JSON + path = path.replace(/\s+/g, ""); // If the path has spaces (common after pasting) - remove them + + if ( + !obj || + !path || + typeof obj !== "object" || + Object.keys(obj).length === 0 || + typeof path !== "string" + ) + return ""; - // End of array index - validate and convert to number - if (char === "]" && inBracket) { - inBracket = false; - if (/^\d+$/.test(bracketContent)) { - segments.push(["array", parseInt(bracketContent, 10)]); - } - bracketContent = ""; - continue; + // First split by dots that are not inside brackets + const parts = []; + let currentPart = ""; + let inBrackets = false; + + for (let i = 0; i < path.length; i++) { + const char = path[i]; + if (char === "[") { + inBrackets = true; + if (currentPart) { + parts.push(currentPart); + currentPart = ""; } - - // Build up array index or property name - if (inBracket) { - bracketContent += char; - } else if (char === ".") { - if (currentSegment) { - segments.push(currentSegment); - currentSegment = ""; - } - } else { - currentSegment += char; + currentPart += char; + } else if (char === "]") { + inBrackets = false; + currentPart += char; + parts.push(currentPart); + currentPart = ""; + } else if (char === "." && !inBrackets) { + if (currentPart) { + parts.push(currentPart); + currentPart = ""; } + } else { + currentPart += char; } + } - if (currentSegment) { - segments.push(currentSegment); - } + if (currentPart) parts.push(currentPart); + let current = obj; + + for (const part of parts) { + if (current === null || typeof current !== "object") return undefined; - // Navigate through object using segments, handling both array and object access - let result = parsedObj; - for (const segment of segments) { - if (result === null || result === undefined) return ""; + // Handle bracket notation + if (part.startsWith("[") && part.endsWith("]")) { + const key = part.slice(1, -1); + const cleanKey = key.replace(/^['"]|['"]$/g, ""); - if (Array.isArray(segment)) { - const [type, index] = segment; - if (type === "array") { - if (!Array.isArray(result) || index < 0 || index >= result.length) - return ""; - result = result[index]; - } + if (!isNaN(cleanKey)) { + if (!Array.isArray(current)) return undefined; + current = current[parseInt(cleanKey)]; } else { - result = result?.[segment]; + if (!(cleanKey in current)) return undefined; + current = current[cleanKey]; } + } else { + // Handle dot notation + if (!(part in current)) return undefined; + current = current[part]; } - // Convert final result to string, handling objects and error cases - if (result === null || result === undefined) return ""; - if (typeof result === "object") { - try { - return JSON.stringify(result, null, 0); - } catch (e) { - return ""; // fallback for stringify errors - } - } - return String(result); - } catch (error) { - console.error("Error in getValueFromPath:", error); - return ""; + if (current === undefined || current === null) return undefined; } + + return typeof current === "object" ? JSON.stringify(current) : current; } // Utility to replace variables in config replaceVariables(config) { const deepReplace = (obj) => { if (typeof obj === "string") { - const result = obj.replace(/\${([^}]+)}/g, (match, varName) => { - if (!this.variables[varName.split(".")[0]]) return match; - return this.getValueFromPath( - this.variables[varName.split(".")[0]], - varName - ); + return obj.replace(/\${([^}]+)}/g, (match, varName) => { + const value = this.getValueFromPath(this.variables, varName); + return value !== undefined ? value : match; }); - return result; - } - if (Array.isArray(obj)) { - return obj.map((item) => deepReplace(item)); } + + if (Array.isArray(obj)) return obj.map((item) => deepReplace(item)); + if (obj && typeof obj === "object") { const result = {}; for (const [key, value] of Object.entries(obj)) { diff --git a/server/utils/agentFlows/executors/llm-instruction.js b/server/utils/agentFlows/executors/llm-instruction.js index c950d80fa99..b5000a89c10 100644 --- a/server/utils/agentFlows/executors/llm-instruction.js +++ b/server/utils/agentFlows/executors/llm-instruction.js @@ -1,5 +1,3 @@ -const AIbitat = require("../../agents/aibitat"); - /** * Execute an LLM instruction flow step * @param {Object} config Flow step configuration @@ -15,8 +13,10 @@ async function executeLLMInstruction(config, context) { introspect(`Processing data with LLM instruction...`); if (!variables[inputVariable]) { - logger(`Input variable ${inputVariable} not found`); - throw new Error(`Input variable ${inputVariable} not found`); + logger(`Input variable ${inputVariable} (${inputVariable}) not found`); + throw new Error( + `Input variable ${inputVariable} (${inputVariable}) not found` + ); } try { From 93022f3a16d68689f17754f109c6ec3aee26e2f4 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Thu, 5 Jun 2025 07:13:42 -0700 Subject: [PATCH 4/4] Move webscraping deps to closure update tests to not modify env since no longer needed do not modify paths with spaces - could be text key with spaces --- .../utils/agentFlows/executor.test.js | 19 ++----------------- server/utils/agentFlows/executor.js | 15 +++++++++++---- .../agentFlows/executors/web-scraping.js | 10 +++++----- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/server/__tests__/utils/agentFlows/executor.test.js b/server/__tests__/utils/agentFlows/executor.test.js index d9538f9035e..e62c8b42db0 100644 --- a/server/__tests__/utils/agentFlows/executor.test.js +++ b/server/__tests__/utils/agentFlows/executor.test.js @@ -1,22 +1,7 @@ -const { resolve } = require("path"); +const { FlowExecutor } = require("../../../utils/agentFlows/executor"); describe("FlowExecutor: getValueFromPath", () => { - let executor; - const originalEnv = process.env; - - beforeAll(async () => { - process.env = { ...originalEnv }; - process.env.NODE_ENV = "test"; - process.env.STORAGE_DIR = resolve(__dirname, "../../../storage"); - - const { FlowExecutor } = require("../../../utils/agentFlows/executor"); - executor = new FlowExecutor(); - await new Promise(resolve => setTimeout(resolve, 2500)); // wait for the rest of the stuff to be initialized - }); - - afterAll(() => { - process.env = originalEnv; - }); + const executor = new FlowExecutor(); it("can handle invalid objects", () => { expect(executor.getValueFromPath(null, "a.b.c")).toBe(""); diff --git a/server/utils/agentFlows/executor.js b/server/utils/agentFlows/executor.js index ede25dc1794..5b9b4c8ecc9 100644 --- a/server/utils/agentFlows/executor.js +++ b/server/utils/agentFlows/executor.js @@ -31,8 +31,7 @@ class FlowExecutor { * @returns {string} The resolved value */ getValueFromPath(obj = {}, path = "") { - if (typeof obj === "string") obj = safeJsonParse(obj, {}); // If the object is a string, parse it as JSON - path = path.replace(/\s+/g, ""); // If the path has spaces (common after pasting) - remove them + if (typeof obj === "string") obj = safeJsonParse(obj, {}); if ( !obj || @@ -102,7 +101,11 @@ class FlowExecutor { return typeof current === "object" ? JSON.stringify(current) : current; } - // Utility to replace variables in config + /** + * Replaces variables in the config with their values + * @param {Object} config - The config to replace variables in + * @returns {Object} The config with variables replaced + */ replaceVariables(config) { const deepReplace = (obj) => { if (typeof obj === "string") { @@ -127,7 +130,11 @@ class FlowExecutor { return deepReplace(config); } - // Main execution method + /** + * Executes a single step of the flow + * @param {Object} step - The step to execute + * @returns {Promise} The result of the step + */ async executeStep(step) { const config = this.replaceVariables(step.config); let result; diff --git a/server/utils/agentFlows/executors/web-scraping.js b/server/utils/agentFlows/executors/web-scraping.js index e54b95fe6f7..e10770f82e9 100644 --- a/server/utils/agentFlows/executors/web-scraping.js +++ b/server/utils/agentFlows/executors/web-scraping.js @@ -1,8 +1,3 @@ -const { CollectorApi } = require("../../collectorApi"); -const { TokenManager } = require("../../helpers/tiktoken"); -const Provider = require("../../agents/aibitat/providers/ai-provider"); -const { summarizeContent } = require("../../agents/aibitat/utils/summarize"); - /** * Execute a web scraping flow step * @param {Object} config Flow step configuration @@ -10,6 +5,11 @@ const { summarizeContent } = require("../../agents/aibitat/utils/summarize"); * @returns {Promise} Scraped content */ async function executeWebScraping(config, context) { + const { CollectorApi } = require("../../collectorApi"); + const { TokenManager } = require("../../helpers/tiktoken"); + const Provider = require("../../agents/aibitat/providers/ai-provider"); + const { summarizeContent } = require("../../agents/aibitat/utils/summarize"); + const { url, captureAs = "text", enableSummarization = true } = config; const { introspect, logger, aibitat } = context; logger(