fix: refine field extraction logic and update test cases

- Enhanced the streamingExtractFinalValue function to accurately determine field boundaries, preventing content overlap between fields.
- Updated the parseMissedFieldsFromFullContent function to process content line by line for improved field extraction precision.
- Adjusted test cases in extract.test.ts to reflect changes in field handling and ensure correct functionality.
- Added new words to the project dictionary for better spell checking support.

Author: dosco

.cspell/project-words.txt

@@ -142,4 +142,5 @@ configtestagent
 openrouter
 anns
 annos
-mundo
+mundo
+fmap

src/ax/dsp/extract.test.ts

@@ -168,8 +168,8 @@ Model Answer 2: false`;
     const content = `Plan: First, I will check the weather in San Francisco to determine if it's a nice day for outdoor seating. Then, I will search for restaurants in San Francisco that offer sushi, Chinese, or Indian cuisine, prioritizing outdoor seating if the weather is good. Finally, I will suggest a suitable restaurant.
 getCurrentWeather location: San Francisco
 findRestaurants location: San Francisco
-findRestaurants cuisine: sushi, chinese, indian
 findRestaurants outdoor: true
+findRestaurants cuisine: sushi, chinese, indian
 findRestaurants priceRange: $$-$$$
 `;
 

src/ax/dsp/extract.ts

@@ -271,7 +271,25 @@ export const streamingExtractFinalValue = (
   const forceFinalize = options?.forceFinalize ?? false;
 
   if (xstate.currField) {
-    const val = content.substring(xstate.s).trim();
+    let endIndex = content.length;
+
+    // Look for the next field boundary to avoid including content from other fields
+    const outputFields = sig.getOutputFields();
+    for (const otherField of outputFields) {
+      if (otherField.name === xstate.currField.name) {
+        continue;
+      }
+
+      // Look for the next field title after current position
+      const nextFieldPattern = `\n${otherField.title}:`;
+      const nextFieldIndex = content.indexOf(nextFieldPattern, xstate.s);
+
+      if (nextFieldIndex !== -1 && nextFieldIndex < endIndex) {
+        endIndex = nextFieldIndex;
+      }
+    }
+
+    const val = content.substring(xstate.s, endIndex).trim();
     const parsedValue = validateAndParseFieldValue(xstate.currField, val);
     if (parsedValue !== undefined) {
       values[xstate.currField.name] = parsedValue;
@@ -296,7 +314,7 @@ export const streamingExtractFinalValue = (
   }
 
   // Check for optional fields that might have been missed by streaming parser
-  parseOptionalFieldsFromFullContent(sig, values, content);
+  parseMissedFieldsFromFullContent(sig, values, content);
 
   // Check all previous required fields before processing current field
   // In streaming scenarios (non-strict), defer missing-required enforcement until end
@@ -321,57 +339,48 @@ export const streamingExtractFinalValue = (
   }
 };
 
-// Helper function to parse optional fields from full content that streaming parser might have missed
-const parseOptionalFieldsFromFullContent = (
+// Helper function to parse missed fields from full content that streaming parser might have missed
+const parseMissedFieldsFromFullContent = (
   sig: Readonly<AxSignature>,
   values: Record<string, unknown>,
   content: string
 ) => {
   const outputFields = sig.getOutputFields();
 
+  // Process content line by line for more precise field extraction
+  const lines = content.split('\n');
+
   for (const field of outputFields) {
-    // Skip if field is not optional or already found
-    if (!field.isOptional || field.name in values) {
+    // Skip if field is already found
+    if (field.name in values) {
       continue;
     }
 
-    // Look for field.title pattern in content
+    // Look for field.title pattern in each line
     const prefix = `${field.title}:`;
-    const fieldIndex = content.indexOf(prefix);
-
-    if (fieldIndex === -1) {
-      continue;
-    }
 
-    // Extract content after the field prefix
-    const startIndex = fieldIndex + prefix.length;
-    let endIndex = content.length;
-
-    // Find the end of this field's content by looking for the next field or end of content
-    for (const otherField of outputFields) {
-      if (otherField.name === field.name) {
-        continue;
-      }
-
-      const otherPrefix = `${otherField.title}:`;
-      const otherFieldIndex = content.indexOf(otherPrefix, startIndex);
-
-      if (otherFieldIndex !== -1 && otherFieldIndex < endIndex) {
-        endIndex = otherFieldIndex;
-      }
-    }
-
-    // Extract and validate the field value
-    const fieldValue = content.substring(startIndex, endIndex).trim();
-
-    if (fieldValue) {
-      try {
-        const parsedValue = validateAndParseFieldValue(field, fieldValue);
-        if (parsedValue !== undefined) {
-          values[field.name] = parsedValue;
+    for (const line of lines) {
+      const trimmedLine = line.trim();
+      if (trimmedLine.startsWith(prefix)) {
+        // Extract the value after the colon
+        const fieldValue = trimmedLine.substring(prefix.length).trim();
+
+        if (fieldValue) {
+          try {
+            const parsedValue = validateAndParseFieldValue(field, fieldValue);
+            if (parsedValue !== undefined) {
+              values[field.name] = parsedValue;
+              break; // Found the field, stop looking
+            }
+          } catch (e) {
+            // Only ignore validation errors for optional fields
+            if (!field.isOptional) {
+              throw e;
+            }
+            // Ignore validation errors for optional fields in this fallback parser
+          }
         }
-      } catch {
-        // Ignore validation errors for optional fields in this fallback parser
+        break; // Found the field marker, stop looking even if value was empty
       }
     }
   }

src/ax/dsp/functions.ts

@@ -185,7 +185,7 @@ export class AxFunctionProcessor {
     } as const;
   };
 
-  public execute = async <MODEL>(
+  public executeWithDetails = async <MODEL>(
     func: Readonly<AxChatResponseFunctionCall>,
     options?: Readonly<
       AxProgramForwardOptions<MODEL> & {
@@ -222,6 +222,19 @@ export class AxFunctionProcessor {
       throw e;
     }
   };
+
+  public execute = async <MODEL>(
+    func: Readonly<AxChatResponseFunctionCall>,
+    options?: Readonly<
+      AxProgramForwardOptions<MODEL> & {
+        traceId?: string;
+        stopFunctionNames?: readonly string[];
+      }
+    >
+  ): Promise<string> => {
+    const result = await this.executeWithDetails<MODEL>(func, options);
+    return result.formatted;
+  };
 }
 
 export type AxInputFunctionType = (
@@ -328,7 +341,7 @@ export const processFunctions = async ({
 
     if (!tracer) {
       return funcProc
-        .execute(func, {
+        .executeWithDetails(func, {
           sessionId,
           ai,
           functionResultFormatter,
@@ -427,7 +440,7 @@ export const processFunctions = async ({
             rawResult,
             parsedArgs,
           }: { formatted: string; rawResult: unknown; parsedArgs: unknown } =
-            await funcProc.execute(func, {
+            await funcProc.executeWithDetails(func, {
               sessionId,
               ai,
               functionResultFormatter,