diff --git a/.changeset/calm-owls-research.md b/.changeset/calm-owls-research.md new file mode 100644 index 0000000000..94e7bcd121 --- /dev/null +++ b/.changeset/calm-owls-research.md @@ -0,0 +1,7 @@ +--- +"@flow-state-dev/patterns": minor +--- + +Add an installable `research()` kit that plans multi-step web research, uses search/fetch/crawl tools, retries isolated task failures, and returns a provenance-checked cited report with its typed task results (FIX-2822). + +Retrieval provenance and attempt counts reset for each run. Set `maxAttemptsPerTask: 1` or `maxTotalRetries: 0` to disable task retries. diff --git a/goals/kits/research-returns-a-cited-report/fixture/fsdev.config.mts b/goals/kits/research-returns-a-cited-report/fixture/fsdev.config.mts new file mode 100644 index 0000000000..9d359245b4 --- /dev/null +++ b/goals/kits/research-returns-a-cited-report/fixture/fsdev.config.mts @@ -0,0 +1,48 @@ +import { createFlowState, inMemoryStores } from "@flow-state-dev/engine"; +import { handler } from "@flow-state-dev/core"; +import { research } from "@flow-state-dev/patterns/research"; +import { createMockModelResolver, mockGenerator } from "@flow-state-dev/testing"; +import { z } from "zod"; + +const url = "https://fixture.example/research"; +let fetchAttempts = 0; +const search = handler({ name: "search", inputSchema: z.any(), execute: (input) => ({ query: input.query, results: [{ title: "Fixture", url, snippet: "Evidence", source: "tavily" }] }) }); +const fetch = handler({ + name: "fetch", inputSchema: z.any(), + execute: (input) => { + fetchAttempts += 1; + if (fetchAttempts === 1 || process.env.GOAL_CONTROL === "no-rescue") throw new Error("injected flaky fetch"); + return { url: input.url, title: "Fixture", markdown: "The fixture documents the result.", metadata: { statusCode: 200, contentType: "text/html", wordCount: 6 }, source: "builtin" }; + }, +}); +const crawl = handler({ name: "crawl", inputSchema: z.any(), execute: (input) => ({ + rootUrl: input.url, + pages: [{ url: input.url, title: "Fixture section", markdown: "More fixture evidence.", metadata: { statusCode: 200, contentType: "text/html", wordCount: 3 }, source: "builtin" }], + totalPages: 1, crawlDepth: 1, source: "builtin", +}) }); + +const invented = process.env.GOAL_CONTROL === "invent-citation"; +const models = createMockModelResolver({ + generators: { + "research-planner": mockGenerator({ name: "planner", script: [{ structuredOutput: { tasks: [{ id: "evidence", title: null, goal: "Find and read the fixture evidence", context: null }] } }] }), + "research-executor": mockGenerator({ name: "executor", script: [ + { toolCalls: [{ toolCallId: "s1", toolName: "search", args: { query: "fixture evidence", maxResults: 5, topic: "general" } }, { toolCallId: "f1", toolName: "fetch", args: { url } }] }, + { toolCalls: [{ toolCallId: "s2", toolName: "search", args: { query: "fixture evidence", maxResults: 5, topic: "general" } }, { toolCallId: "f2", toolName: "fetch", args: { url } }, { toolCallId: "c2", toolName: "crawl", args: { url, maxPages: 2, maxDepth: 1 } }] }, + { structuredOutput: { summary: "The fixture documents the result.", success: true, reason: "", sources: [{ title: "Fixture", url: process.env.GOAL_CONTROL === "invent-executor-source" ? "https://invented.example/executor" : url, method: "fetch" }] } }, + ] }), + "research-evaluate-llm": mockGenerator({ name: "evaluator", script: [{ structuredOutput: { decision: "complete", reasoning: "enough evidence" } }] }), + "research-synthesizer": mockGenerator({ name: "synthesizer", script: [{ structuredOutput: { + question: "What does the fixture document?", summary: "The fixture documents the result.", + findings: [{ claim: "The fixture documents the result.", citations: [invented ? "https://invented.example/report" : url] }], + sources: [{ title: "Fixture", url: invented ? "https://invented.example/report" : url }], + } }] }), + }, + policy: "error", +}); + +export default createFlowState({ + kits: [research({ model: "fixture/model", toolOverrides: { search, fetch, crawl }, maxAttemptsPerTask: 2 })], + root: import.meta.url, + stores: { default: { primary: inMemoryStores() } }, + modelResolver: models, +}); diff --git a/goals/kits/research-returns-a-cited-report/goal.md b/goals/kits/research-returns-a-cited-report/goal.md new file mode 100644 index 0000000000..cecad58416 --- /dev/null +++ b/goals/kits/research-returns-a-cited-report/goal.md @@ -0,0 +1,12 @@ +# kits › research returns a cited report + +**Outcome:** An app installs `research()` and receives a cited report from a planned run that searches, fetches, crawls, and survives one flaky fetch. +**Input:** The fixture app asks “What does the fixture document?”. Its search, page and crawl are deterministic retrieval fixtures; the first fetch throws and the retry succeeds. +**Signal:** `fsdev run` returns a report with a non-empty finding and citation list, every citation occurs in the returned retrieved-source list, and the preserved task state has one completed task and its typed result. +**Anti-game:** The check grades the running app's captured action output, not the kit factory. The first fetch always fails, so a pass proves task rescue. Every graded list is floored with `tell.over`. +**Model:** Scripted generators drive planning, tool calls, evaluation and synthesis; no vendor key or live web is used. +**Run:** `pnpm tsx goals/kits/research-returns-a-cited-report/run.mts` +**Controls:** +- `GOAL_CONTROL=invent-citation` makes synthesis cite an unobserved URL. Citation validation must fail the run. +- `GOAL_CONTROL=invent-executor-source` makes the executor claim an unobserved URL. Runtime retrieval provenance must keep it out of a valid report, so the goal must fail. +- `GOAL_CONTROL=no-rescue` makes every fetch attempt fail. The task cannot complete, so the goal must fail. diff --git a/goals/kits/research-returns-a-cited-report/run.mts b/goals/kits/research-returns-a-cited-report/run.mts new file mode 100644 index 0000000000..00462888e1 --- /dev/null +++ b/goals/kits/research-returns-a-cited-report/run.mts @@ -0,0 +1,41 @@ +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { goalTmpDir, readCapture, REPO_ROOT, runGoal } from "../../lib/index.mts"; + +const here = fileURLToPath(new URL(".", import.meta.url)); +const capture = join(goalTmpDir("research-returns-a-cited-report"), "run.json"); +const fsdev = join(REPO_ROOT, "packages/fsdev/bin/fsdev.ts"); +const tsx = join(REPO_ROOT, "packages/fsdev/node_modules/.bin/tsx"); +const config = join(here, "fixture/fsdev.config.mts"); + +await runGoal((failures, tell) => { + const ran = spawnSync(tsx, [fsdev, "run", "research", "run", "--input", JSON.stringify({ question: "What does the fixture document?" }), "--config", config, "--capture", capture], { + cwd: join(here, "fixture"), env: process.env, encoding: "utf8", + }); + if (ran.status !== 0) { + failures.push(`run: research app failed (exit ${ran.status}): ${`${ran.stderr}${ran.stdout}`.slice(-800)}`); + return { failures, evidence: "the running app did not return a report" }; + } + const output = readCapture(capture).result.output as { + summary?: string; findings?: Array<{ claim: string; citations: string[] }>; sources?: Array<{ url: string }>; + retrieval?: { urls?: string[]; attempts?: Record }; + tasks?: Array<{ status: string; result?: { summary?: string; sources?: Array<{ url: string }> }; error?: string }>; completedSteps?: number; + }; + const findings = tell.over("report:findings", output.findings ?? []); + const sources = tell.over("report:sources", output.sources ?? []); + const tasks = tell.over("state:tasks", output.tasks ?? []); + const citations = tell.over("report:citations", findings.flatMap((finding) => finding.citations)); + const retrieved = tell.over("retrieval:urls", output.retrieval?.urls ?? []); + if (!citations.every((citation) => sources.some((source) => source.url === citation))) failures.push(`report:citations · citations are not all retrieved sources: ${JSON.stringify(citations)}`); + if (!citations.every((citation) => retrieved.includes(citation))) failures.push(`retrieval:urls · citation was not observed in a tool result: ${JSON.stringify({ citations, retrieved })}`); + const task = tasks[0]; + if (output.completedSteps !== 1 || task?.status !== "completed" || task.result?.summary !== output.summary || task.result?.sources?.[0]?.url !== citations[0]) { + failures.push(`state:tasks · typed task evidence did not feed the report: ${JSON.stringify(output)}`); + } + const attempts = output.retrieval?.attempts ?? {}; + if (attempts.search !== 2 || attempts.fetch !== 2 || attempts.crawl !== 1 || task?.error !== "injected flaky fetch") { + failures.push(`retry:attempts · expected search=2, fetch=2, crawl=1 and retained first error, got ${JSON.stringify({ attempts, error: task?.error })}`); + } + return { failures, evidence: `report ${JSON.stringify(findings)}\nretrieval ${JSON.stringify(output.retrieval)}\ntasks ${JSON.stringify(tasks)}` }; +}); diff --git a/packages/patterns/package.json b/packages/patterns/package.json index 8f68143699..a66f373abe 100644 --- a/packages/patterns/package.json +++ b/packages/patterns/package.json @@ -36,6 +36,10 @@ "types": "./dist/plan-and-execute/index.d.ts", "default": "./src/plan-and-execute/index.ts" }, + "./research": { + "types": "./dist/research/index.d.ts", + "default": "./src/research/index.ts" + }, "./supervisor": { "types": "./dist/supervisor/index.d.ts", "default": "./src/supervisor/index.ts" @@ -83,6 +87,10 @@ "types": "./dist/plan-and-execute/index.d.ts", "default": "./dist/plan-and-execute/index.js" }, + "./research": { + "types": "./dist/research/index.d.ts", + "default": "./dist/research/index.js" + }, "./supervisor": { "types": "./dist/supervisor/index.d.ts", "default": "./dist/supervisor/index.js" @@ -120,6 +128,7 @@ "dependencies": { "@flow-state-dev/core": "workspace:*", "@flow-state-dev/orchestration": "workspace:*", + "@flow-state-dev/tools": "workspace:*", "zod": "^3.24.1" }, "peerDependencies": { diff --git a/packages/patterns/src/research/index.ts b/packages/patterns/src/research/index.ts new file mode 100644 index 0000000000..14b824d300 --- /dev/null +++ b/packages/patterns/src/research/index.ts @@ -0,0 +1,225 @@ +/** Installable deep research built from plan-and-execute and retrieval tools. */ +import { defineFlow, defineKit, generator, handler, sequencer } from "@flow-state-dev/core"; +import type { BlockDefinition } from "@flow-state-dev/core/types"; +import { crawl, fetch, search } from "@flow-state-dev/tools"; +import type { CrawlConfig, FetchConfig, SearchConfig } from "@flow-state-dev/tools"; +import { z } from "zod"; +import { planAndExecute } from "../plan-and-execute"; + +const sourceSchema = z.object({ + title: z.string().default(""), + url: z.string().url(), + method: z.enum(["search", "fetch", "crawl"]).optional(), +}); +const researchSessionStateSchema = z.object({ + researchRetrievedUrls: z.array(z.string()).default([]), + researchToolAttempts: z.record(z.number()).default({}), +}); +/** Typed evidence passed from a research task to evaluation and synthesis. */ +export const researchTaskResultSchema = z.object({ + summary: z.string(), success: z.boolean(), reason: z.string().default(""), sources: z.array(sourceSchema).default([]), +}); +const taskResultSchema = researchTaskResultSchema; + +/** One evidence-backed finding in a research report. */ +export const researchFindingSchema = z.object({ claim: z.string(), citations: z.array(z.string().url()).min(1) }); + +/** Public result returned by the research flow. */ +export const researchReportSchema = z.object({ + question: z.string(), + summary: z.string(), + findings: z.array(researchFindingSchema), + sources: z.array(sourceSchema.omit({ method: true })), + retrieval: z.object({ urls: z.array(z.string()), attempts: z.record(z.number()) }), + tasks: z.array(z.object({ + id: z.string(), goal: z.string(), + status: z.enum(["pending", "in-progress", "completed", "failed", "skipped"]), + result: taskResultSchema.optional(), error: z.string().optional(), + })), + completedSteps: z.number(), + totalSteps: z.number(), +}); +/** A cited report plus the typed plan state and observed retrieval provenance. */ +export type ResearchReport = z.infer; + +const reportDraftSchema = researchReportSchema.pick({ question: true, summary: true, findings: true, sources: true }); +const planOutputSchema = z.object({ + goal: z.string(), status: z.string(), tasks: researchReportSchema.shape.tasks, + completedSteps: z.number(), totalSteps: z.number(), +}); + +/** Refuse a report that cites a URL no retrieval tool actually returned. */ +export function validateResearchReport>( + report: T, + retrievedUrls: ReadonlySet, +): T { + const urls = [...report.findings.flatMap((finding) => finding.citations), ...report.sources.map((source) => source.url)]; + for (const url of urls) { + if (!retrievedUrls.has(url)) throw new Error(`Research report citation was not retrieved: ${url}`); + } + const listed = new Set(report.sources.map((source) => source.url)); + for (const url of report.findings.flatMap((finding) => finding.citations)) { + if (!listed.has(url)) throw new Error(`Research report citation is missing from sources: ${url}`); + } + return report; +} + +const researchConfigSchema = z.object({ + model: z.string().optional(), + models: z.object({ + planner: z.string().optional(), executor: z.string().optional(), synthesizer: z.string().optional(), + }).optional(), + search: z.custom().optional(), fetch: z.custom().optional(), crawl: z.custom().optional(), + maxTasks: z.number().int().positive().default(12), + /** Total attempts per task, including the first (default 2). Set 1 to disable retries. */ + maxAttemptsPerTask: z.number().int().min(1).default(2), + maxIterations: z.number().int().positive().default(3), + /** + * Shared retry budget across all tasks (default 6); 0 disables retries. + * A retry needs both this budget and a remaining per-task attempt. + * With maxAttemptsPerTask: 1, a positive budget is valid but unused. + */ + maxTotalRetries: z.number().int().nonnegative().default(6), + /** Advanced overrides for apps that own their retrieval blocks. */ + toolOverrides: z.object({ + search: z.custom>().optional(), + fetch: z.custom>().optional(), + crawl: z.custom>().optional(), + }).optional(), +}).strict().refine((config) => config.model !== undefined || ( + config.models?.planner !== undefined && config.models.executor !== undefined && config.models.synthesizer !== undefined +), { + message: "provide model or role-specific models", +}); + +type ResearchConfig = z.output; +function modelFor(config: ResearchConfig, role: "planner" | "executor" | "synthesizer"): string { + return config.models?.[role] ?? config.model ?? "openai/gpt-6-luna"; +} + +function retrievedUrls(method: "search" | "fetch" | "crawl", output: unknown): string[] { + if (output === null || typeof output !== "object") return []; + const value = output as Record; + if (method === "search") { + return Array.isArray(value.results) + ? value.results.flatMap((result) => typeof result === "object" && result !== null && typeof (result as { url?: unknown }).url === "string" ? [(result as { url: string }).url] : []) + : []; + } + if (method === "fetch") return typeof value.url === "string" ? [value.url] : []; + return Array.isArray(value.pages) + ? value.pages.flatMap((page) => typeof page === "object" && page !== null && typeof (page as { url?: unknown }).url === "string" ? [(page as { url: string }).url] : []) + : []; +} + +function observableTool(method: "search" | "fetch" | "crawl", tool: BlockDefinition) { + const countAttempt = handler({ + name: `research-${method}-attempt`, inputSchema: z.unknown(), sessionStateSchema: researchSessionStateSchema, + execute: async (_input, ctx) => { + const attempts = ctx.session.state.researchToolAttempts; + await ctx.session.patchState({ researchToolAttempts: { ...attempts, [method]: (attempts[method] ?? 0) + 1 } }); + }, + }); + const recordResult = handler({ + name: `research-${method}-record`, inputSchema: z.unknown(), sessionStateSchema: researchSessionStateSchema, + execute: async (output, ctx) => { + const urls = [...new Set([...ctx.session.state.researchRetrievedUrls, ...retrievedUrls(method, output)])]; + await ctx.session.patchState({ researchRetrievedUrls: urls }); + }, + }); + return sequencer({ name: method }).tap(countAttempt).step(tool).tap(recordResult); +} + +function createExecutor(config: ResearchConfig) { + return generator({ + name: "research-executor", + model: modelFor(config, "executor"), + inputSchema: z.object({ + stepId: z.string(), goal: z.string(), context: z.string().optional(), dependencyResults: z.record(z.unknown()).optional(), + }), + outputSchema: taskResultSchema, + tools: [ + observableTool("search", config.toolOverrides?.search ?? search(config.search)), + observableTool("fetch", config.toolOverrides?.fetch ?? fetch(config.fetch)), + observableTool("crawl", config.toolOverrides?.crawl ?? crawl(config.crawl)), + ], + prompt: [ + "Execute one step in a deep-research plan.", + "Use search to discover sources, fetch to read a page, and crawl when evidence spans multiple pages on one site.", + "Record only URLs returned by those tools in sources, with the matching retrieval method. Never invent a URL.", + "Set success false only when the step cannot produce useful evidence; thrown tool errors are retried by the task board.", + ], + user: (input) => JSON.stringify(input), + itemVisibility: { client: true, history: false }, + }); +} + +function createSynthesizer(config: ResearchConfig) { + const stateSchema = z.object({ plan: planOutputSchema.nullable().default(null) }); + const capturePlan = handler({ + name: "research-capture-plan", inputSchema: planOutputSchema, sequencerStateSchema: stateSchema, + execute: async (input, ctx) => { + await ctx.sequencer!.patchState({ plan: input }); + }, + }); + const draft = generator({ + name: "research-synthesizer", model: modelFor(config, "synthesizer"), inputSchema: planOutputSchema, + outputSchema: reportDraftSchema, + prompt: [ + "Write a cited research report from the completed task results.", + "Every substantive finding must cite source URLs present in those results. Never cite any other URL.", + "Preserve the original question verbatim.", + ], + user: (input) => JSON.stringify(input), itemVisibility: { client: true, history: true }, + }); + const validate = handler({ + name: "research-validate-report", inputSchema: reportDraftSchema, outputSchema: researchReportSchema, + sequencerStateSchema: stateSchema, sessionStateSchema: researchSessionStateSchema, + execute: (report, ctx) => { + const plan = ctx.sequencer!.state.plan; + if (plan === null) throw new Error("Research synthesis is missing its plan state"); + const urls = ctx.session.state.researchRetrievedUrls; + for (const task of plan.tasks) { + if (task.result === undefined) continue; + for (const source of task.result.sources) { + if (!urls.includes(source.url)) throw new Error(`Research task source was not retrieved: ${source.url}`); + } + } + validateResearchReport(report, new Set(urls)); + return { + ...report, retrieval: { urls, attempts: ctx.session.state.researchToolAttempts }, + tasks: plan.tasks, completedSteps: plan.completedSteps, totalSteps: plan.totalSteps, + }; + }, + }); + return sequencer({ name: "research-cited-synthesis", inputSchema: planOutputSchema, stateSchema }) + .tap(capturePlan).step(draft).step(validate); +} + +/** Install a planned, retryable deep-research flow. */ +export const research = defineKit({ + name: "research", + config: researchConfigSchema, + install: ({ config }) => { + const researchBlock = planAndExecute({ + name: "research", model: modelFor(config, "planner"), stepExecutor: createExecutor(config), + synthesizer: createSynthesizer(config), outputSchema: researchReportSchema, enableReplanning: true, + maxIterations: config.maxIterations, maxAttemptsPerTask: config.maxAttemptsPerTask, + maxTotalRetries: config.maxTotalRetries, maxTotalTasks: config.maxTasks, maxEnqueuedTasks: config.maxTasks, + instructions: `Plan at most ${config.maxTasks} focused research tasks. Prefer primary sources and make dependencies explicit.`, + }); + const inputSchema = z.object({ question: z.string().min(1) }); + const resetRetrieval = handler({ + name: "research-reset-retrieval", inputSchema, sessionStateSchema: researchSessionStateSchema, + execute: async (_input, ctx) => { + await ctx.session.patchState({ researchRetrievedUrls: [], researchToolAttempts: {} }); + }, + }); + const run = sequencer({ name: "research-run", inputSchema }) + .tap(resetRetrieval) + .map((input) => ({ goal: input.question })).step(researchBlock); + return { flows: [defineFlow({ + kind: "research", requireUser: true, + actions: { run: { inputSchema, block: run, userMessage: (input) => input.question } }, + })()] }; + }, +}); diff --git a/packages/patterns/test/research.test.ts b/packages/patterns/test/research.test.ts new file mode 100644 index 0000000000..019bffb1bf --- /dev/null +++ b/packages/patterns/test/research.test.ts @@ -0,0 +1,188 @@ +import { handler } from "@flow-state-dev/core"; +import type { BlockDefinition } from "@flow-state-dev/core/types"; +import { mockGenerator, testBlock } from "@flow-state-dev/testing"; +import { describe, expect, it } from "vitest"; +import { research, validateResearchReport } from "../src/research"; + +describe("research report citations", () => { + it("rejects citations that no completed research task retrieved", () => { + expect(() => + validateResearchReport( + { + question: "What changed?", + summary: "A sourced answer.", + findings: [{ claim: "The change shipped.", citations: ["https://invented.example/report"] }], + sources: [{ title: "Invented", url: "https://invented.example/report" }], + }, + new Set(["https://fixture.example/source"]), + ), + ).toThrow(/not retrieved/); + }); + + it("returns a report when every finding cites retrieved evidence", () => { + const report = { + question: "What changed?", + summary: "A sourced answer.", + findings: [{ claim: "The change shipped.", citations: ["https://fixture.example/source"] }], + sources: [{ title: "Fixture", url: "https://fixture.example/source" }], + }; + + expect(validateResearchReport(report, new Set(["https://fixture.example/source"]))).toEqual(report); + }); +}); + +describe("research configuration", () => { + it("allows a single attempt with either a zero or unused positive retry budget", () => { + for (const maxTotalRetries of [0, 6]) { + expect(() => research({ model: "test", maxAttemptsPerTask: 1, maxTotalRetries })).not.toThrow(); + } + }); + + it("requires a fallback model or all three role-specific models", () => { + expect(() => research({})).toThrow(/provide model or role-specific models/); + for (const models of [ + { planner: "test" }, { executor: "test" }, { synthesizer: "test" }, + { planner: "test", executor: "test" }, { executor: "test", synthesizer: "test" }, + { planner: "test", synthesizer: "test" }, + ]) { + expect(() => research({ models })).toThrow(/provide model or role-specific models/); + expect(() => research({ model: "fallback", models })).not.toThrow(); + } + expect(() => research({ models: { planner: "p", executor: "e", synthesizer: "s" } })).not.toThrow(); + }); +}); + +// Walk the kit's declared block graph to exercise its blocks without exporting internals. +async function researchRun(toolOverrides = {}) { + const kit = research({ model: "test", toolOverrides }); + const installed = await kit.definition.install({ config: kit.config, folder: undefined, slots: {} }); + return installed.flows[0]!.actions.run!.block; +} + +function findBlock(root: BlockDefinition, name: string): BlockDefinition { + const pending = [root]; + const visited = new Set>(); + while (pending.length > 0) { + const block = pending.pop()!; + if (visited.has(block)) continue; + visited.add(block); + if (block.name === name) return block; + pending.push(...(block.childBlocks ?? []), ...(block.staticTools ?? [])); + } + throw new Error(`Missing research block: ${name}`); +} + +const sourceUrl = "https://fixture.example/source"; +const draft = { question: "What changed?", summary: "An answer", findings: [], sources: [] }; + +function runGenerators(retrieve = false) { + return { + "research-planner": mockGenerator({ name: "planner", script: [{ structuredOutput: { + tasks: [{ id: "s1", title: null, goal: "Find evidence", context: null }], + } }] }), + "research-executor": mockGenerator({ name: "executor", script: [ + ...(!retrieve ? [] : [{ toolCalls: [{ toolCallId: "fetch-1", toolName: "fetch", args: {} }] }]), + { structuredOutput: { summary: "Evidence", success: true, reason: "", sources: [] } }, + ] }), + "research-evaluate-llm": mockGenerator({ name: "evaluator", script: [{ structuredOutput: { decision: "complete", reasoning: "Done" } }] }), + "research-synthesizer": mockGenerator({ name: "synthesizer", script: [{ structuredOutput: draft }] }), + }; +} + +describe("research run provenance", () => { + it("clears the first run's URLs and attempt counts before a second question", async () => { + const run = await researchRun({ fetch: handler({ name: "fixture-fetch", execute: () => ({ url: sourceUrl }) }) }); + const first = await testBlock(run, { + input: { question: "First question" }, + session: { state: { researchRetrievedUrls: [], researchToolAttempts: {} } }, generators: runGenerators(true), unmockedGeneratorPolicy: "error", + }); + expect(first.error).toBeNull(); + expect(first.output).toEqual(expect.objectContaining({ retrieval: { urls: [sourceUrl], attempts: { fetch: 1 } } })); + const second = await testBlock(run, { + input: { question: "Second question" }, session: { state: first.state.session }, + generators: runGenerators(), unmockedGeneratorPolicy: "error", + }); + expect(second.error).toBeNull(); + expect(second.output).toEqual(expect.objectContaining({ retrieval: { urls: [], attempts: {} } })); + + const staleCitation = await testBlock(run, { + input: { question: "Second question" }, session: { state: first.state.session }, + generators: { + ...runGenerators(), + "research-synthesizer": mockGenerator({ name: "stale-synthesizer", script: [{ structuredOutput: { + ...draft, findings: [{ claim: "Old evidence", citations: [sourceUrl] }], sources: [{ title: "Old", url: sourceUrl }], + } }] }), + }, + unmockedGeneratorPolicy: "error", + }); + expect(staleCitation.error?.message).toContain(`Research report citation was not retrieved: ${sourceUrl}`); + }); +}); + +describe("research retrieval tools", () => { + it.each([ + ["search", { results: [{ url: sourceUrl }, null, {}, { url: 4 }, { url: sourceUrl }] }, [sourceUrl]], + ["fetch", { url: sourceUrl }, [sourceUrl]], + ["crawl", { pages: [{ url: sourceUrl }, null, {}, { url: false }, { url: sourceUrl }] }, [sourceUrl]], + ["search", { results: "invalid" }, []], + ["fetch", { url: 4 }, []], + ["crawl", { pages: "invalid" }, []], + ["search", null, []], + ["fetch", "invalid", []], + ["crawl", {}, []], + ] as const)("records only returned URLs for %s: %j", async (method, output, urls) => { + const run = await researchRun({ [method]: handler({ name: "fixture-tool", execute: () => output }) }); + const result = await testBlock(findBlock(run, method), { + input: {}, session: { state: { researchRetrievedUrls: [], researchToolAttempts: { [method]: 1 } } }, + }); + expect(result.error).toBeNull(); + expect(result.output).toEqual(output); + expect(result.state.session).toMatchObject({ + researchRetrievedUrls: urls, researchToolAttempts: { [method]: 2 }, + }); + }); + + it("counts failed attempts without recording a URL, then records a successful retry once", async () => { + let attempts = 0; + const run = await researchRun({ fetch: handler({ name: "flaky-fetch", execute: () => { + if (++attempts === 1) throw new Error("Temporary retrieval failure"); + return { url: sourceUrl }; + } }) }); + const tool = findBlock(run, "fetch"); + const failed = await testBlock(tool, { + input: {}, session: { state: { researchRetrievedUrls: [], researchToolAttempts: { search: 3 } } }, + }); + expect(failed.error?.message).toContain("Temporary retrieval failure"); + expect(failed.state.session).toMatchObject({ researchRetrievedUrls: [], researchToolAttempts: { search: 3, fetch: 1 } }); + const retried = await testBlock(tool, { input: {}, session: { state: failed.state.session } }); + expect(retried.error).toBeNull(); + expect(retried.state.session).toMatchObject({ researchRetrievedUrls: [sourceUrl], researchToolAttempts: { search: 3, fetch: 2 } }); + const repeated = await testBlock(tool, { input: {}, session: { state: retried.state.session } }); + expect(repeated.error).toBeNull(); + expect(repeated.state.session).toMatchObject({ researchRetrievedUrls: [sourceUrl], researchToolAttempts: { search: 3, fetch: 3 } }); + }); +}); + +describe("research executor provenance", () => { + it.each([true, false])("accepts a task source only when a tool retrieved it (retrieved: %s)", async (retrieved) => { + const run = await researchRun(); + const synthesis = findBlock(run, "research-cited-synthesis"); + const task = { + id: "s1", goal: "Find evidence", status: "completed", + result: { summary: "Evidence", success: true, reason: "", sources: [{ title: "Evidence", url: sourceUrl }] }, + }; + const result = await testBlock(synthesis, { + input: { goal: draft.question, status: "completed", tasks: [task], completedSteps: 1, totalSteps: 1 }, + session: { state: { researchRetrievedUrls: retrieved ? [sourceUrl] : [], researchToolAttempts: { fetch: 1 } } }, + // The report itself has no citations: executor sources must be checked independently. + generators: { "research-synthesizer": mockGenerator({ name: "synth", script: [{ structuredOutput: draft }] }) }, + unmockedGeneratorPolicy: "error", + }); + if (retrieved) { + expect(result.error).toBeNull(); + expect(result.output).toMatchObject({ tasks: [task], retrieval: { urls: [sourceUrl], attempts: { fetch: 1 } } }); + } else { + expect(result.error?.message).toContain(`Research task source was not retrieved: ${sourceUrl}`); + } + }); +}); diff --git a/packages/patterns/tsconfig.json b/packages/patterns/tsconfig.json index ad576a3303..c79da581d8 100644 --- a/packages/patterns/tsconfig.json +++ b/packages/patterns/tsconfig.json @@ -9,6 +9,7 @@ "references": [ { "path": "../core" }, { "path": "../orchestration" }, + { "path": "../tools" }, { "path": "../testing" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9af630ff01..a9df716f04 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1981,6 +1981,9 @@ importers: '@flow-state-dev/orchestration': specifier: workspace:* version: link:../orchestration + '@flow-state-dev/tools': + specifier: workspace:* + version: link:../tools zod: specifier: ^3.24.1 version: 3.25.76