94 lines
3.2 KiB
TypeScript
94 lines
3.2 KiB
TypeScript
import { SSESession } from '../source/sse-session/index.ts';
|
|
|
|
// Because this is in a library, and we don't want to add the node types to this as it is intended to be used in a browser
|
|
// we will just declare the process object here locally so we don't get type errors from this script
|
|
declare const process: {
|
|
env: Record<string, string>;
|
|
stdout: {
|
|
write: (data: string) => void;
|
|
};
|
|
};
|
|
|
|
// Recommended URL: https://openrouter.ai/api/v1/chat/completions
|
|
// Recommended Model: nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free
|
|
// Exmaple Command: API_KEY="your-api-key" URL="https://openrouter.ai/api/v1/chat/completions" MODEL="ibm-granite/granite-4.1-8b" PROMPT="Hello, Tell me a joke about robots?" npx tsx ./sandbox/sandbox-llm.ts
|
|
|
|
// Read the Environemt Variables for url, model, prompt and api key
|
|
const url = process.env.URL ?? 'https://openrouter.ai/api/v1/chat/completions';
|
|
const model = process.env.MODEL ?? 'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free';
|
|
const prompt = process.env.PROMPT ?? 'Hello, Tell me a joke about robots?';
|
|
|
|
const apiKey = process.env.API_KEY ?? '';
|
|
|
|
// Throw an error if the api key is not set
|
|
if (!apiKey) {
|
|
throw new Error('API key is required');
|
|
}
|
|
|
|
// Create a function to get the auth header
|
|
const getAuthHeader = (): string => {
|
|
return `Bearer ${apiKey}`;
|
|
};
|
|
|
|
// Create our sse session
|
|
const sseSession = new SSESession(url, {
|
|
// LLMs use requests
|
|
method: 'POST',
|
|
|
|
// Create the body of the request
|
|
body: JSON.stringify({
|
|
model: model,
|
|
messages: [
|
|
{
|
|
role: 'user',
|
|
content: prompt,
|
|
},
|
|
],
|
|
stream: true,
|
|
}),
|
|
|
|
// Create a custom request handler to set the headers
|
|
onRequest: async (requestInit): Promise<RequestInit> => {
|
|
requestInit.headers ??= {} as HeadersInit;
|
|
|
|
// Handle typescript annoyances
|
|
const headers = requestInit.headers as Record<string, string>;
|
|
|
|
// Set our headers - We could also do this using the `headers` property in the SSESession constructor
|
|
// Doing it here to demonstrate dynamic headers, for example a signed timestamp could be used to authenticate the request.
|
|
headers.Authorization = getAuthHeader();
|
|
headers['Content-Type'] = 'application/json';
|
|
|
|
return requestInit;
|
|
},
|
|
});
|
|
|
|
// Connect to the SSESession
|
|
await sseSession.connect();
|
|
|
|
// Loop over the message chunks using `for await`
|
|
for await (const message of sseSession.messages) {
|
|
// Handle `[DONE]` (this may be specific to OpenRouter)
|
|
if (message.data === '[DONE]') {
|
|
continue;
|
|
}
|
|
|
|
// First, we will parse the event to JSON
|
|
const responseJson = JSON.parse(message.data);
|
|
|
|
// Then we will grab the relavent part of the response (we want to first grab the choices array)
|
|
const choices = responseJson.choices;
|
|
|
|
// Then we will grab the first choice
|
|
const firstChoice = choices[0];
|
|
|
|
// Then we will grab the next chunk of text
|
|
const messageContent = firstChoice.delta?.content;
|
|
|
|
// Then we will append the text to the console
|
|
process.stdout.write(messageContent || '');
|
|
}
|
|
|
|
// Just a terminal/node thing. If we dont put a new line, the console will overwrite the text with the `cwd` or next command input
|
|
process.stdout.write('\n');
|