Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/guides/aeo-auditor.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,15 @@ Settings, all in Vercel → project → Settings → Environment Variables (the
| `AEO_GATE_MODE` | `off` / `observe` / `enforce` |
| `AEO_UNLOCK_COOKIE_SECRET` | Signs the per-audit unlock cookie. Without it, `enforce` refuses new audits (fails closed). |
| `AEO_DAILY_AUDIT_CAP` | Audits per UTC day. Without it, `enforce` refuses new audits. |
| `AEO_DAILY_AI_CALL_CAP` | Paid AI-engine calls per UTC day, all visitors together; always on, default 2000. An audit reserves 41, a custom-question run 4 per question. Staff are counted but not refused. Past it, new audits and custom questions get "Free audits have reached their limit for today" (`AI_DAILY_LIMIT` in the logs) until 00:00 UTC. If the count can't be read, nothing runs. |
| `AEO_PREVIEW_DB_ISOLATED` | `true` only when a preview has its own database. Otherwise `enforce` refuses audits on previews. |
| `AEO_AUDITS_PAUSED` | `true` stops new audits and custom questions in every mode. |
| `AEO_PUBLIC_ACCESS` | `staff_only` refuses every public audit. Never set it on production. |
| `AEO_HUBSPOT_SYNC` | `off` stores leads without syncing them to HubSpot (for test environments). |
| `NEXT_PUBLIC_AEO_GUIDE` | `on` shows the first-visit guide and the Tutorials button on the report. Needs a redeploy. |

Refusal codes in the logs: `PAUSED`, `CONFIG_MISSING`, `PREVIEW_SHARED_DB`, `STAFF_ONLY`,
`DAILY_CAP`, `REPORT_LOCKED`, `UNLOCK_NOT_READY`.
`DAILY_CAP`, `REPORT_LOCKED`, `UNLOCK_NOT_READY`, `AI_DAILY_LIMIT`.

## Checking a setup
- `npm run gate:check`, in a shell with the environment loaded, prints each name as
Expand Down
1 change: 1 addition & 0 deletions scripts/gate-check.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const NAMES = [
'AEO_DAILY_AUDIT_CAP', 'AEO_UNLOCK_COOKIE_SECRET', 'AEO_STAFF_EMAILS', 'AEO_HUBSPOT_SYNC',
'NEXT_PUBLIC_SUPABASE_URL', 'SUPABASE_SERVICE_ROLE_KEY', 'ANTHROPIC_API_KEY',
'INNGEST_EVENT_KEY', 'INTERNAL_CRAWLER_SECRET', 'HUBSPOT_PRIVATE_APP_TOKEN', 'VERCEL_ENV',
'AEO_DAILY_AI_CALL_CAP',
];

export function report(env) {
Expand Down
36 changes: 36 additions & 0 deletions src/app/api/audit/[auditId]/custom-prompts/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ const { db, probes } = vi.hoisted(() => ({

vi.mock('@/lib/auditor/ai-probe', () => probes);

const { reserveMock } = vi.hoisted(() => ({ reserveMock: vi.fn() }));
vi.mock('@/lib/aiBudget', async (orig) => ({
...(await orig<typeof import('@/lib/aiBudget')>()),
reserveAiCalls: reserveMock,
}));

type Filter = { col: string; op: 'eq' | 'gte'; val: unknown };

function query(table: string) {
Expand Down Expand Up @@ -156,6 +162,8 @@ beforeEach(() => {
}
process.env.NEXT_PUBLIC_SUPABASE_URL = 'https://example.supabase.co';
process.env.SUPABASE_SERVICE_ROLE_KEY = 'service-role-key';
reserveMock.mockReset();
reserveMock.mockResolvedValue(null);
delete process.env.CUSTOM_PROMPTS_GLOBAL_MAX;
delete process.env.AEO_GATE_MODE;
delete process.env.AEO_UNLOCK_COOKIE_SECRET;
Expand Down Expand Up @@ -350,3 +358,31 @@ describe('POST /api/audit/[auditId]/custom-prompts', () => {
expect(engineCalls()).toBe(0);
});
});

describe('custom prompts and the daily AI-call ceiling', () => {
it('reserves four engines × the number of questions before any engine runs', async () => {
addAudit(AUDIT_ID);
const res = await call(AUDIT_ID, undefined, ['one', 'two']);
expect(res.status).toBe(200);
expect(reserveMock).toHaveBeenCalledWith(8);
});

it('refuses with a plain message and calls no engine once the day is spent', async () => {
reserveMock.mockResolvedValue({
status: 503,
body: { error: 'Free audits have reached their limit for today. Please try again tomorrow.', code: 'AI_DAILY_LIMIT' },
});
addAudit(AUDIT_ID);
const res = await call(AUDIT_ID);
expect(res.status).toBe(503);
expect((await res.json()).code).toBe('AI_DAILY_LIMIT');
expect(engineCalls()).toBe(0);
});

it('stored results cost nothing from the day', async () => {
addAudit(AUDIT_ID);
db.results.set(AUDIT_ID, { raw_findings: { customProbe: { prompts: ['x'] } } });
await call(AUDIT_ID);
expect(reserveMock).not.toHaveBeenCalled();
});
});
8 changes: 8 additions & 0 deletions src/app/api/audit/[auditId]/custom-prompts/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ import {
import { checkRateLimit, getClientIp } from '@/lib/rateLimit';
import { countTodaysAudits, reportAccess, runGate, unlockCookieName } from '@/lib/gate';
import { staffList } from '@/lib/staff';
import { reserveAiCalls } from '@/lib/aiBudget';

/** Paid engines each question goes to. */
const ENGINES = 4;

export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
Expand Down Expand Up @@ -153,6 +157,10 @@ export async function POST(
return NextResponse.json({ error: 'Try again later' }, { status: 503 });
}

// Last check before spending: the daily ceiling for all visitors together.
const overBudget = await reserveAiCalls(ENGINES * validPrompts.length);
if (overBudget) return NextResponse.json(overBudget.body, { status: overBudget.status });

const domain = requestData.url as string;
const companyName = domain
.replace(/^https?:\/\//, '')
Expand Down
44 changes: 44 additions & 0 deletions src/app/api/audit/start/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ vi.mock('@/lib/staff', async (orig) => ({
staffEmail: staffMock,
}));

const { reserveMock } = vi.hoisted(() => ({ reserveMock: vi.fn() }));
vi.mock('@/lib/aiBudget', async (orig) => ({
...(await orig<typeof import('@/lib/aiBudget')>()),
reserveAiCalls: reserveMock,
}));

vi.mock('@/lib/rateLimit', () => ({
checkRateLimit: rateLimitMock,
/* getClientIp moved into this module so both rate-limited routes bucket callers
Expand Down Expand Up @@ -62,6 +68,8 @@ beforeEach(() => {
rateLimitMock.mockReset();
supabaseInsertMock.mockReset();
rateLimitMock.mockResolvedValue({ limited: false });
reserveMock.mockReset();
reserveMock.mockResolvedValue(null);
supabaseInsertMock.mockReturnValue({ data: { id: 'audit-123' }, error: null });
process.env.NEXT_PUBLIC_SUPABASE_URL = 'https://example.supabase.co';
process.env.SUPABASE_SERVICE_ROLE_KEY = 'service-role-key';
Expand Down Expand Up @@ -128,6 +136,42 @@ describe('POST /api/audit/start', () => {
});
});

describe('POST /api/audit/start daily AI-call ceiling', () => {
it('reserves a whole audit\'s worth of paid calls before creating the audit', async () => {
const { POST } = await import('./route');
const res = await POST(makeRequest({ url: 'https://example.com' }));
expect(res.status).toBe(201);
expect(reserveMock).toHaveBeenCalledWith(41, { staff: false });
});

it('staff are recorded but not refused by the ceiling', async () => {
staffMock.mockResolvedValueOnce('staff@example.com');
const { POST } = await import('./route');
await POST(makeRequest({ url: 'https://example.com' }));
expect(reserveMock).toHaveBeenCalledWith(41, { staff: true });
});

it('refuses with a plain message and creates nothing once the day is spent', async () => {
reserveMock.mockResolvedValue({
status: 503,
body: { error: 'Free audits have reached their limit for today. Please try again later.', code: 'AI_DAILY_LIMIT' },
});
const { POST } = await import('./route');
const res = await POST(makeRequest({ url: 'https://example.com' }));
expect(res.status).toBe(503);
expect((await res.json()).code).toBe('AI_DAILY_LIMIT');
expect(supabaseInsertMock).not.toHaveBeenCalled();
expect(fetch).not.toHaveBeenCalled();
});

it('an invalid request spends nothing from the day', async () => {
const { POST } = await import('./route');
await POST(makeRequest({ url: 'http://10.0.0.1/admin' }));
await POST(makeRequest({ url: 'not-a-url' }));
expect(reserveMock).not.toHaveBeenCalled();
});
});

describe('POST /api/audit/start daily cap', () => {
const GATE_KEYS = ['AEO_GATE_MODE', 'AEO_DAILY_AUDIT_CAP', 'VERCEL_ENV', 'ANTHROPIC_API_KEY', 'AEO_UNLOCK_COOKIE_SECRET'];
beforeEach(() => {
Expand Down
5 changes: 5 additions & 0 deletions src/app/api/audit/start/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { isBlockedHost } from '@/lib/safeFetch';
import { inngest } from '@/lib/inngest/client';
import { countTodaysAudits, pauseRefusal, runGate } from '@/lib/gate';
import { staffEmail, staffList } from '@/lib/staff';
import { AUDIT_AI_CALLS, reserveAiCalls } from '@/lib/aiBudget';

export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
Expand Down Expand Up @@ -107,6 +108,10 @@ export async function POST(request: NextRequest): Promise<Response> {

const normalizedUrl = normalizeUrl(parsed.data.url);

// Last check before an audit exists: the daily ceiling on paid AI calls, all visitors.
const overBudget = await reserveAiCalls(AUDIT_AI_CALLS, { staff });
if (overBudget) return NextResponse.json(overBudget.body, { status: overBudget.status });

let auditId: string;
try {
const { data, error } = await getSupabase()
Expand Down
143 changes: 143 additions & 0 deletions src/lib/aiBudget.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';

const { db } = vi.hoisted(() => ({
db: { rows: [] as Array<{ ip: string; created_at: string }>, failInsert: false, failCount: false, inserts: 0 },
}));

vi.mock('@supabase/supabase-js', () => ({
createClient: () => ({
from: () => ({
insert: async (rows: Array<{ ip: string; created_at: string }> | { ip: string; created_at: string }) => {
if (db.failInsert) return { error: { message: 'down' } };
db.inserts += 1;
db.rows.push(...(Array.isArray(rows) ? rows : [rows]));
return { error: null };
},
select: () => ({
like: (_c: string, pattern: string) => ({
gte: (_c2: string, since: string) => ({
limit: async (n: number) => db.failCount
? { data: null, error: { message: 'down' } }
: {
data: db.rows
.filter(r => r.ip.startsWith(pattern.replace(/%$/, '')) && r.created_at >= since)
.slice(0, n)
.map(r => ({ ip: r.ip })),
error: null,
},
}),
}),
}),
}),
}),
}));

beforeEach(() => {
vi.resetModules();
db.rows = [];
db.failInsert = false;
db.failCount = false;
db.inserts = 0;
delete process.env.AEO_DAILY_AI_CALL_CAP;
process.env.NEXT_PUBLIC_SUPABASE_URL = 'https://db.test';
process.env.SUPABASE_SERVICE_ROLE_KEY = 'k';
vi.spyOn(console, 'error').mockImplementation(() => {});
vi.spyOn(console, 'log').mockImplementation(() => {});
});
afterEach(() => { vi.restoreAllMocks(); delete process.env.AEO_DAILY_AI_CALL_CAP; });

describe('daily ceiling on paid AI-engine calls, all visitors together', () => {
it('allows calls up to the ceiling and refuses past it, with a plain message', async () => {
process.env.AEO_DAILY_AI_CALL_CAP = '100';
const { reserveAiCalls } = await import('./aiBudget');
expect(await reserveAiCalls(41)).toBeNull();
expect(await reserveAiCalls(41)).toBeNull();
const refused = await reserveAiCalls(41);
expect(refused?.status).toBe(503);
expect(refused?.body.code).toBe('AI_DAILY_LIMIT');
expect(refused?.body.error).toMatch(/today/i);
expect(refused?.body.error).not.toMatch(/AI_|cap|budget|tomorrow/i);
});

it('a request that fits exactly is allowed', async () => {
process.env.AEO_DAILY_AI_CALL_CAP = '12';
const { reserveAiCalls } = await import('./aiBudget');
expect(await reserveAiCalls(12)).toBeNull();
expect(await reserveAiCalls(1)).not.toBeNull();
});

it('one reservation is one row, whatever its size', async () => {
const { reserveAiCalls } = await import('./aiBudget');
await reserveAiCalls(41);
expect(db.rows).toHaveLength(1);
});

it('counts only today (UTC)', async () => {
process.env.AEO_DAILY_AI_CALL_CAP = '10';
const { reserveAiCalls } = await import('./aiBudget');
db.rows.push({ ip: 'ai-calls:10', created_at: '2020-01-01T00:00:00.000Z' });
expect(await reserveAiCalls(10)).toBeNull();
});

it('other limiter keys never count', async () => {
process.env.AEO_DAILY_AI_CALL_CAP = '10';
const { reserveAiCalls } = await import('./aiBudget');
const now = new Date().toISOString();
db.rows.push({ ip: '203.0.113.7', created_at: now }, { ip: 'custom-global', created_at: now });
expect(await reserveAiCalls(10)).toBeNull();
});

it('a row never counts for more than the largest real reservation', async () => {
process.env.AEO_DAILY_AI_CALL_CAP = '100';
const { reserveAiCalls } = await import('./aiBudget');
db.rows.push({ ip: 'ai-calls:999999', created_at: new Date().toISOString() });
expect(await reserveAiCalls(41)).toBeNull();
});

it('staff are recorded but never refused', async () => {
process.env.AEO_DAILY_AI_CALL_CAP = '10';
const { reserveAiCalls } = await import('./aiBudget');
expect(await reserveAiCalls(10)).toBeNull();
expect(await reserveAiCalls(41, { staff: true })).toBeNull();
expect(db.rows).toHaveLength(2);
expect(await reserveAiCalls(1)).not.toBeNull();
});

it('has a default ceiling when none is set, and ignores unusable values', async () => {
const { dailyAiCallCap, DEFAULT_DAILY_AI_CALLS } = await import('./aiBudget');
expect(dailyAiCallCap()).toBe(DEFAULT_DAILY_AI_CALLS);
expect(DEFAULT_DAILY_AI_CALLS).toBeGreaterThan(0);
for (const v of ['0', '-5', 'abc', '1e3', '1.5', '']) {
process.env.AEO_DAILY_AI_CALL_CAP = v;
expect(dailyAiCallCap()).toBe(DEFAULT_DAILY_AI_CALLS);
}
process.env.AEO_DAILY_AI_CALL_CAP = '250';
expect(dailyAiCallCap()).toBe(250);
});

it('fails closed when the reservation cannot be written', async () => {
db.failInsert = true;
const { reserveAiCalls } = await import('./aiBudget');
expect((await reserveAiCalls(1))?.status).toBe(503);
});

it('fails closed when the count cannot be read', async () => {
db.failCount = true;
const { reserveAiCalls } = await import('./aiBudget');
expect((await reserveAiCalls(1))?.status).toBe(503);
});

it('fails closed when there are more reservations today than one read returns', async () => {
process.env.AEO_DAILY_AI_CALL_CAP = '100000';
const { reserveAiCalls, MAX_ROWS_READ } = await import('./aiBudget');
const now = new Date().toISOString();
db.rows.push(...Array.from({ length: MAX_ROWS_READ }, () => ({ ip: 'ai-calls:1', created_at: now })));
expect((await reserveAiCalls(1))?.status).toBe(503);
});

it('a whole audit is costed at its worst case', async () => {
const { AUDIT_AI_CALLS } = await import('./aiBudget');
// One call to write the questions, then four engines × ten questions.
expect(AUDIT_AI_CALLS).toBe(41);
});
});
Loading
Loading