Skip to content

Move llm-api endpoints to vercel serverless #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mdroidian merged 17 commits into from
Apr 10, 2025
Merged

Move llm-api endpoints to vercel serverless #102

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
4df05ea
testing gemini
sid597 Mar 7, 2025
0bd5ee8
move endgoint to website
sid597 Mar 29, 2025
e61d221
open ai endpoint
sid597 Mar 30, 2025
331cd10
added anthropic endpoint
sid597 Mar 30, 2025
b3d8257
pass env vars
sid597 Mar 31, 2025
ed6c5b2
add cors handdling and options
sid597 Mar 31, 2025
487f3e2
.
sid597 Mar 31, 2025
2a1471d
using centralised cors middleware
sid597 Apr 2, 2025
259ff33
only adding bypass cookie
sid597 Apr 2, 2025
dd5a687
use right key
sid597 Apr 2, 2025
cc96752
remove the bypass token requirement
sid597 Apr 2, 2025
a5f7276
sanitize, fix routes
sid597 Apr 3, 2025
413988b
remove server action config
sid597 Apr 3, 2025
2b99736
DRY
sid597 Apr 3, 2025
d529174
remove unused
sid597 Apr 3, 2025
c2a2c91
address review
sid597 Apr 4, 2025
04c1c6c
adress review
sid597 Apr 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
DRY
  • Loading branch information
@sid597
sid597 committed Apr 3, 2025
commit 2b99736b0ecc0f4b72ed6fb01be1fc2115a1fa45
141 changes: 8 additions & 133 deletions apps/website/app/api/llm/anthropic/chat/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,143 +1,18 @@
import { NextRequest } from "next/server";
import cors from "../../../../../lib/cors";

type Message = {
role: string;
content: string;
};

type Settings = {
model: string;
maxTokens: number;
temperature: number;
};

type AnthropicUsage = {
input_tokens: number;
output_tokens: number;
};

type RequestBody = {
documents: Message[];
passphrase?: string;
settings: Settings;
};

const CONTENT_TYPE_JSON = "application/json";
const CONTENT_TYPE_TEXT = "text/plain";
const ANTHROPIC_API_VERSION = "2023-06-01";
import {
handleLLMRequest,
handleOptionsRequest,
} from "../../../../../lib/llm/handler";
import { anthropicConfig } from "../../../../../lib/llm/providers";

export const runtime = "nodejs";
export const preferredRegion = "auto";
export const maxDuration = 300;

export async function POST(request: NextRequest): Promise<Response> {
try {
const requestData: RequestBody = await request.json();
const { documents: messages, settings } = requestData;
const { model, maxTokens, temperature } = settings;

const apiKey = process.env.ANTHROPIC_API_KEY;

if (!apiKey) {
console.error("ANTHROPIC_API_KEY environment variable is not set");
return cors(
request,
new Response(
JSON.stringify({
error:
"API key not configured. Please set the ANTHROPIC_API_KEY environment variable in your Vercel project settings.",
}),
{
status: 500,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}

const url = "https://api.anthropic.com/v1/messages";

const body = JSON.stringify({
model: model,
max_tokens: maxTokens,
messages: messages,
temperature: temperature,
});

const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": CONTENT_TYPE_JSON,
"x-api-key": apiKey,
"anthropic-version": ANTHROPIC_API_VERSION,
},
body,
});

const responseData = await response.json();

if (!response.ok) {
console.error("Anthropic API error:", responseData);
return cors(
request,
new Response(
JSON.stringify({
error: `Anthropic API error: ${responseData.error?.message || "Unknown error"}`,
}),
{
status: response.status,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}

const replyText = responseData.content?.[0]?.text;

if (!replyText) {
console.error(
"Invalid response format from Anthropic API:",
responseData,
);
return cors(
request,
new Response(
JSON.stringify({
error:
"Invalid response format from Anthropic API. Check server logs for details.",
}),
{
status: 500,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}

return cors(
request,
new Response(replyText, {
headers: { "Content-Type": CONTENT_TYPE_TEXT },
}),
);
} catch (error) {
console.error("Error processing request:", error);
return cors(
request,
new Response(
JSON.stringify({
error: `Internal Server Error: ${error instanceof Error ? error.message : "Unknown error"}`,
}),
{
status: 500,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}
return handleLLMRequest(request, anthropicConfig);
}

export async function OPTIONS(request: NextRequest) {
return cors(request, new Response(null, { status: 204 }));
export async function OPTIONS(request: NextRequest): Promise<Response> {
return handleOptionsRequest(request);
}
150 changes: 8 additions & 142 deletions apps/website/app/api/llm/gemini/chat/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,152 +1,18 @@
import { NextRequest } from "next/server";
import cors from "../../../../../lib/cors";

type Message = {
role: string;
content: string;
};

type Settings = {
model: string;
maxTokens: number;
temperature: number;
safetySettings?: Array<{
category: string;
threshold: string;
}>;
};

type RequestBody = {
documents: Message[];
passphrase?: string;
settings: Settings;
};

type GeminiMessage = {
role: string;
parts: Array<{
text: string;
}>;
};

const CONTENT_TYPE_JSON = "application/json";
const CONTENT_TYPE_TEXT = "text/plain";
import {
handleLLMRequest,
handleOptionsRequest,
} from "../../../../../lib/llm/handler";
import { geminiConfig } from "../../../../../lib/llm/providers";

export const runtime = "nodejs";
export const preferredRegion = "auto";
export const maxDuration = 300;

function convertToGeminiFormat(messages: Message[]): GeminiMessage[] {
return messages.map((msg) => ({
role: msg.role === "user" ? "user" : "model",
parts: [{ text: msg.content }],
}));
}

export async function POST(request: NextRequest): Promise<Response> {
try {
const requestData: RequestBody = await request.json();
const { documents: messages, settings } = requestData;
const { model, maxTokens, temperature } = settings;

const apiKey = process.env.GEMINI_API_KEY;

if (!apiKey) {
console.error("GEMINI_API_KEY environment variable is not set");
return cors(
request,
new Response(
JSON.stringify({
error:
"API key not configured. Please set the GEMINI_API_KEY environment variable in your Vercel project settings.",
}),
{
status: 500,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}

const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`;

const body = JSON.stringify({
contents: convertToGeminiFormat(messages),
generationConfig: {
maxOutputTokens: maxTokens,
temperature: temperature,
},
safetySettings: settings.safetySettings,
});

const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": CONTENT_TYPE_JSON,
},
body,
});

const responseData = await response.json();

if (!response.ok) {
console.error("Gemini API error:", responseData);
return cors(
request,
new Response(
JSON.stringify({
error: `Gemini API error: ${responseData.error?.message || "Unknown error"}`,
}),
{
status: response.status,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}

const replyText = responseData.candidates?.[0]?.content?.parts?.[0]?.text;

if (!replyText) {
console.error("Invalid response format from Gemini API:", responseData);
return cors(
request,
new Response(
JSON.stringify({
error:
"Invalid response format from Gemini API. Check server logs for details.",
}),
{
status: 500,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}

return cors(
request,
new Response(replyText, {
headers: { "Content-Type": CONTENT_TYPE_TEXT },
}),
);
} catch (error) {
console.error("Error processing request:", error);
return cors(
request,
new Response(
JSON.stringify({
error: `Internal Server Error: ${error instanceof Error ? error.message : "Unknown error"}`,
}),
{
status: 500,
headers: { "Content-Type": CONTENT_TYPE_JSON },
},
),
);
}
return handleLLMRequest(request, geminiConfig);
}

export async function OPTIONS(request: NextRequest) {
return cors(request, new Response(null, { status: 204 }));
export async function OPTIONS(request: NextRequest): Promise<Response> {
return handleOptionsRequest(request);
}
Loading