107 lines
5.1 KiB
JavaScript
107 lines
5.1 KiB
JavaScript
import { stream, streamSimple } from "@earendil-works/pi-ai/compat";
|
|
import { LlamaClient, llamaInferenceUrl, normalizeLlamaServerUrl } from "./client.js";
|
|
export const LLAMA_PROVIDER_ID = "llama.cpp";
|
|
export const DEFAULT_LLAMA_SERVER_URL = "http://127.0.0.1:8080";
|
|
function credentialServerUrl(credential) {
|
|
const value = credential?.env?.LLAMA_BASE_URL;
|
|
return typeof value === "string" && value.trim() ? normalizeLlamaServerUrl(value) : undefined;
|
|
}
|
|
async function resolveServerUrl(ctx, credential) {
|
|
const configured = credentialServerUrl(credential) ?? (await ctx.env("LLAMA_BASE_URL"))?.trim();
|
|
return configured ? normalizeLlamaServerUrl(configured) : undefined;
|
|
}
|
|
function toPiModel(model, serverUrl) {
|
|
const reportedContextWindow = model.meta?.n_ctx ?? model.meta?.n_ctx_train;
|
|
const contextWindow = reportedContextWindow && reportedContextWindow > 0 ? reportedContextWindow : 128000;
|
|
return {
|
|
id: model.id,
|
|
name: model.id,
|
|
api: "openai-completions",
|
|
provider: LLAMA_PROVIDER_ID,
|
|
baseUrl: llamaInferenceUrl(serverUrl),
|
|
reasoning: false,
|
|
input: model.architecture?.input_modalities?.includes("image") ? ["text", "image"] : ["text"],
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow,
|
|
maxTokens: contextWindow,
|
|
compat: {
|
|
supportsStore: false,
|
|
supportsDeveloperRole: false,
|
|
supportsReasoningEffort: false,
|
|
supportsUsageInStreaming: true,
|
|
supportsStrictMode: false,
|
|
maxTokensField: "max_tokens",
|
|
},
|
|
};
|
|
}
|
|
export function createLlamaProvider() {
|
|
let models = [];
|
|
const setCatalog = (catalog, serverUrl) => {
|
|
models = catalog.filter((model) => model.status.value === "loaded").map((model) => toPiModel(model, serverUrl));
|
|
};
|
|
const provider = {
|
|
id: LLAMA_PROVIDER_ID,
|
|
name: "llama.cpp",
|
|
baseUrl: llamaInferenceUrl(DEFAULT_LLAMA_SERVER_URL),
|
|
auth: {
|
|
apiKey: {
|
|
name: "llama.cpp server",
|
|
login: async (interaction) => {
|
|
const enteredUrl = await interaction.prompt({
|
|
type: "text",
|
|
message: "llama.cpp server URL",
|
|
placeholder: process.env.LLAMA_BASE_URL ?? DEFAULT_LLAMA_SERVER_URL,
|
|
});
|
|
const serverUrl = normalizeLlamaServerUrl(enteredUrl.trim() || process.env.LLAMA_BASE_URL || DEFAULT_LLAMA_SERVER_URL);
|
|
const apiKey = (await interaction.prompt({
|
|
type: "secret",
|
|
message: "API key (optional)",
|
|
})).trim();
|
|
await new LlamaClient(serverUrl, apiKey || undefined).list({ signal: interaction.signal });
|
|
return {
|
|
type: "api_key",
|
|
key: apiKey || undefined,
|
|
env: { LLAMA_BASE_URL: serverUrl },
|
|
};
|
|
},
|
|
check: async ({ ctx, credential }) => {
|
|
const serverUrl = await resolveServerUrl(ctx, credential);
|
|
return serverUrl
|
|
? { type: "api_key", source: credential ? "stored credential" : "LLAMA_BASE_URL" }
|
|
: undefined;
|
|
},
|
|
resolve: async ({ ctx, credential }) => {
|
|
const serverUrl = await resolveServerUrl(ctx, credential);
|
|
if (!serverUrl)
|
|
return undefined;
|
|
const apiKey = credential?.key ?? (await ctx.env("LLAMA_API_KEY")) ?? "local";
|
|
return {
|
|
auth: { apiKey, baseUrl: llamaInferenceUrl(serverUrl) },
|
|
env: { ...credential?.env, LLAMA_BASE_URL: serverUrl },
|
|
source: credential ? "stored credential" : "LLAMA_BASE_URL",
|
|
};
|
|
},
|
|
},
|
|
},
|
|
getModels: () => models,
|
|
refreshModels: async (context) => {
|
|
const stored = await context.store.read();
|
|
if (stored) {
|
|
models = stored.models.filter((model) => model.provider === LLAMA_PROVIDER_ID && model.api === "openai-completions");
|
|
}
|
|
if (!context.allowNetwork || context.signal?.aborted || context.credential?.type !== "api_key")
|
|
return;
|
|
const serverUrl = credentialServerUrl(context.credential);
|
|
if (!serverUrl)
|
|
return;
|
|
const catalog = await new LlamaClient(serverUrl, context.credential.key).list({ signal: context.signal });
|
|
setCatalog(catalog, serverUrl);
|
|
if (!context.signal?.aborted)
|
|
await context.store.write({ models, checkedAt: Date.now() });
|
|
},
|
|
stream: (model, context, options) => stream(model, context, options),
|
|
streamSimple: (model, context, options) => streamSimple(model, context, options),
|
|
};
|
|
return { provider, setCatalog };
|
|
}
|
|
//# sourceMappingURL=provider.js.map
|