Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
updates
  • Loading branch information
sebastiancrossa committed Jul 10, 2025
commit 635fcddecaeeff5b1e0ed12452316530c4eb83b4
11 changes: 11 additions & 0 deletions eslint.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,17 @@ export default [
sourceType: 'module',
project: './tsconfig.json',
},
globals: {
process: 'readonly',
console: 'readonly',
Buffer: 'readonly',
__dirname: 'readonly',
__filename: 'readonly',
fetch: 'readonly',
Response: 'readonly',
Request: 'readonly',
Headers: 'readonly',
},
},
plugins: {
'@typescript-eslint': tsPlugin,
Expand Down
14 changes: 10 additions & 4 deletions examples/04-langchain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
ZeroEvalCallbackHandler,
setGlobalCallbackHandler,
} from "zeroeval/langchain";
import { init } from "zeroeval"

import { ChatOpenAI } from "@langchain/openai";
import { ChatPromptTemplate } from "@langchain/core/prompts";
Expand All @@ -13,6 +14,11 @@ import { z } from "zod";

setGlobalCallbackHandler(new ZeroEvalCallbackHandler());

init({
apiKey: "sk_ze_UKGIwfckKDmIMlRt3F5r-8GYGARl7hO46W1XhLR7618",
debug: true,
})

interface AgentState {
messages: BaseMessage[];
}
Expand Down Expand Up @@ -106,7 +112,7 @@ async function main() {
const structuredReport = await structuredModel.invoke(
`Based on this weather information: "${weatherInfo}", generate a detailed weather report.`
);

console.log("\nStructured Weather Report:");
console.log(JSON.stringify(structuredReport, null, 2));

Expand All @@ -132,17 +138,17 @@ async function main() {
const structuredReport2 = await structuredModel.invoke(
`Based on this weather information: "${weatherInfo2}", generate a detailed weather report.`
);

console.log("\nStructured Weather Report for NY:");
console.log(JSON.stringify(structuredReport2, null, 2));

// Example 3: Direct structured output without tool calling
console.log("\n\nDirect structured output example...\n");

const directStructuredResult = await structuredModel.invoke(
"Generate a weather report for London, UK. Make it rainy and cold, around 45°F."
);

console.log("Direct Structured Output:");
console.log(JSON.stringify(directStructuredResult, null, 2));

Expand Down
8 changes: 4 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 14 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,19 @@
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"typesVersions": {
"*": {
".": [
"./dist/index.d.ts"
],
"langchain": [
"./dist/langchain.d.ts"
],
"*": [
"./dist/*"
]
}
},
"exports": {
".": {
"import": "./dist/index.js",
Expand Down Expand Up @@ -112,7 +125,7 @@
},
"devDependencies": {
"@eslint/js": "^9.30.1",
"@types/node": "^20.19.4",
"@types/node": "^20.19.6",
"@typescript-eslint/eslint-plugin": "^8.36.0",
"@typescript-eslint/parser": "^8.36.0",
"@vitest/coverage-v8": "^3.2.4",
Expand Down
2 changes: 1 addition & 1 deletion src/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { tracer } from './observability/Tracer';
import { Span } from './observability/Span';
import type { Span } from './observability/Span';

/** Return the current active Span (or undefined). */
export function getCurrentSpan(): Span | undefined {
Expand Down
36 changes: 36 additions & 0 deletions src/init.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { tracer } from './observability/Tracer';
import { Logger, getLogger } from './observability/logger';

const logger = getLogger('zeroeval');

export interface InitOptions {
apiKey?: string;
Expand All @@ -7,6 +10,7 @@ export interface InitOptions {
maxSpans?: number;
collectCodeDetails?: boolean;
integrations?: Record<string, boolean>;
debug?: boolean;
}

// Track whether init has been called
Expand All @@ -32,8 +36,40 @@ export function init(opts: InitOptions = {}): void {
maxSpans,
collectCodeDetails,
integrations,
debug,
} = opts;

// Check if debug mode is enabled via param or env var
const isDebugMode =
debug || process.env.ZEROEVAL_DEBUG?.toLowerCase() === 'true';

// Enable debug mode
if (isDebugMode) {
process.env.ZEROEVAL_DEBUG = 'true';
Logger.setDebugMode(true);

// Log all configuration values as the first log message
const maskedApiKey = Logger.maskApiKey(
apiKey || process.env.ZEROEVAL_API_KEY
);
const finalApiUrl =
apiUrl || process.env.ZEROEVAL_API_URL || 'https://api.zeroeval.com';

logger.debug('ZeroEval SDK Configuration:');
logger.debug(` API Key: ${maskedApiKey}`);
logger.debug(` API URL: ${finalApiUrl}`);
logger.debug(` Debug Mode: ${isDebugMode}`);
logger.debug(` Flush Interval: ${flushInterval ?? '10s (default)'}`);
logger.debug(` Max Spans: ${maxSpans ?? '100 (default)'}`);
logger.debug(
` Collect Code Details: ${collectCodeDetails ?? 'true (default)'}`
);

logger.info('SDK initialized in debug mode.');
} else {
Logger.setDebugMode(false);
}

if (apiKey) process.env.ZEROEVAL_API_KEY = apiKey;
if (apiUrl) process.env.ZEROEVAL_API_URL = apiUrl;

Expand Down
68 changes: 61 additions & 7 deletions src/observability/Tracer.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import { AsyncLocalStorage } from 'async_hooks';
import { randomUUID } from 'crypto';
import { Span } from './Span';
import { SpanWriter, BackendSpanWriter } from './writer';
import type { SpanWriter } from './writer';
import { BackendSpanWriter } from './writer';
import { setInterval } from 'timers';
import { discoverIntegrations } from './integrations/utils';
import type { Integration } from './integrations/base';
import { getLogger } from './logger';

const logger = getLogger('zeroeval.tracer');

interface ConfigureOptions {
flushInterval?: number;
Expand All @@ -30,6 +34,11 @@ export class Tracer {
private _shuttingDown = false;

constructor() {
logger.debug('Initializing tracer...');
logger.debug(
`Tracer config: flush_interval=${this._flushIntervalMs}ms, max_spans=${this._maxSpans}`
);

// schedule periodic flush
setInterval(() => {
if (Date.now() - this._lastFlush >= this._flushIntervalMs) {
Expand All @@ -54,10 +63,17 @@ export class Tracer {

/* CONFIG ----------------------------------------------------------------*/
configure(opts: ConfigureOptions = {}) {
if (opts.flushInterval !== undefined)
if (opts.flushInterval !== undefined) {
this._flushIntervalMs = opts.flushInterval * 1000;
if (opts.maxSpans !== undefined) this._maxSpans = opts.maxSpans;
// Other options ignored for now (collectCodeDetails, integrations)
logger.info(
`Tracer flush_interval configured to ${opts.flushInterval}s.`
);
}
if (opts.maxSpans !== undefined) {
this._maxSpans = opts.maxSpans;
logger.info(`Tracer max_spans configured to ${opts.maxSpans}.`);
}
logger.debug(`Tracer configuration updated:`, opts);
}

/* ACTIVE SPAN -----------------------------------------------------------*/
Expand All @@ -76,6 +92,8 @@ export class Tracer {
tags?: Record<string, string>;
} = {}
): Span {
logger.debug(`Starting span: ${name}`);

const parent = this.currentSpan();
const span = new Span(name, parent?.traceId);

Expand All @@ -85,10 +103,14 @@ export class Tracer {
span.sessionName = parent.sessionName;
// inherit tags
span.tags = { ...parent.tags, ...opts.tags };
logger.debug(`Span ${name} inherits from parent ${parent.name}`);
} else {
span.sessionId = opts.sessionId ?? randomUUID();
span.sessionName = opts.sessionName;
span.tags = { ...opts.tags };
logger.debug(
`Span ${name} is a root span with session ${span.sessionId}`
);
}

Object.assign(span.attributes, opts.attributes);
Expand All @@ -107,6 +129,8 @@ export class Tracer {
endSpan(span: Span): void {
if (!span.endTime) span.end();

logger.debug(`Ending span: ${span.name} (duration: ${span.durationMs}ms)`);

// pop stack
const stack = als.getStore();
if (stack && stack[stack.length - 1] === span) {
Expand All @@ -124,14 +148,25 @@ export class Tracer {
const ordered = traceBucket.sort((a) => (a.parentId ? 1 : -1));
delete this._traceBuckets[span.traceId];
this._buffer.push(...ordered);

logger.debug(
`Trace ${span.traceId} complete with ${ordered.length} spans`
);
}

// flush if buffer full
if (this._buffer.length >= this._maxSpans) this.flush();
if (this._buffer.length >= this._maxSpans) {
logger.debug(
`Buffer full (${this._buffer.length} spans), triggering flush`
);
this.flush();
}
}

/* TAG HELPERS -----------------------------------------------------------*/
addTraceTags(traceId: string, tags: Record<string, string>): void {
logger.debug(`Adding trace tags to ${traceId}:`, tags);

// update buckets
for (const span of this._traceBuckets[traceId] ?? [])
Object.assign(span.tags, tags);
Expand All @@ -142,6 +177,8 @@ export class Tracer {
}

addSessionTags(sessionId: string, tags: Record<string, string>): void {
logger.debug(`Adding session tags to ${sessionId}:`, tags);

const all = [...Object.values(this._traceBuckets).flat(), ...this._buffer];
all
.filter((s) => s.sessionId === sessionId)
Expand All @@ -155,31 +192,48 @@ export class Tracer {
/* FLUSH -----------------------------------------------------------------*/
flush(): void {
if (this._buffer.length === 0) return;

logger.info(`Flushing ${this._buffer.length} spans to backend`);

this._lastFlush = Date.now();
this._writer.write(this._buffer.splice(0));
}

private async _setupAvailableIntegrations(): Promise<void> {
logger.info('Checking for available integrations...');

const available = await discoverIntegrations();

for (const [key, Ctor] of Object.entries(available)) {
try {
const inst = new Ctor();
if ((Ctor as any).isAvailable?.() !== false) {
logger.info(`Setting up integration: ${key}`);
inst.setup();
this._integrations[key] = inst;
logger.info(`✅ Successfully set up integration: ${key}`);
}
} catch (err) {
// eslint-disable-next-line no-console
console.warn(`[ZeroEval] Failed to setup integration ${key}`, err);
logger.error(`❌ Failed to setup integration ${key}:`, err);
}
}

if (Object.keys(this._integrations).length > 0) {
logger.info(
`Active integrations: ${Object.keys(this._integrations).join(', ')}`
);
} else {
logger.info('No active integrations found.');
}
}

/** Flush remaining spans and teardown integrations */
shutdown(): void {
if (this._shuttingDown) return;
this._shuttingDown = true;

logger.info('Shutting down tracer...');

try {
this.flush();
} catch (_) {}
Expand Down
2 changes: 1 addition & 1 deletion src/observability/integrations/langchain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export class LangChainIntegration extends Integration {
for (const method of methods) {
if (typeof Runnable.prototype[method] !== 'function') continue;
this.patchMethod(
Runnable.prototype as any,
Runnable.prototype,
method as any,
(orig: AnyFn): AnyFn => {
const isAsync = method.toString().startsWith('a');
Expand Down
Loading