feat: add model availability monitoring for OpenCode fallback chain
- Add background job that checks model availability every 5 hours - Automatically removes unavailable models from fallback chain - Adds unavailable models section in admin UI with blur styling - Allows admins to re-add models when they become available again - Extends model schema with available, lastChecked, unavailableSince fields - Adds API endpoints: GET /api/admin/models/availability, POST /api/admin/models/:id/readd
This commit is contained in:
southseact-3d
126
chat/server.js
126
chat/server.js
@@ -1527,6 +1527,8 @@ let adminModelIndex = new Map();
|
||||
// Simple model structure - just two lists
|
||||
let opencodeModels = []; // Models from OpenCode - order determines fallback chain
|
||||
let publicModels = []; // Public-facing models (completely separate)
|
||||
const MODEL_AVAILABILITY_CHECK_INTERVAL_MS = 5 * 60 * 60 * 1000; // 5 hours
|
||||
let modelAvailabilityCheckTimer = null;
|
||||
let openrouterSettings = {
|
||||
primaryModel: OPENROUTER_MODEL_PRIMARY,
|
||||
backupModel1: OPENROUTER_MODEL_BACKUP_1,
|
||||
@@ -5792,6 +5794,9 @@ async function loadAdminModelStore() {
|
||||
tier: normalizeTier(m.tier),
|
||||
supportsMedia: m.supportsMedia ?? false,
|
||||
multiplier: m.multiplier || getTierMultiplier(m.tier),
|
||||
available: m.available !== false,
|
||||
lastChecked: m.lastChecked || null,
|
||||
unavailableSince: m.unavailableSince || null,
|
||||
})).filter((m) => !!m.name);
|
||||
|
||||
// Validate public models
|
||||
@@ -5833,6 +5838,71 @@ async function persistAdminModels() {
|
||||
refreshAdminModelIndex();
|
||||
}
|
||||
|
||||
async function checkSingleModelAvailability(modelName) {
|
||||
try {
|
||||
const availableModels = await listModels('opencode');
|
||||
const isAvailable = availableModels.some((m) => {
|
||||
const mName = m.name || m.id || m;
|
||||
return mName === modelName || mName.endsWith('/' + modelName) || modelName.endsWith('/' + mName);
|
||||
});
|
||||
return isAvailable;
|
||||
} catch (error) {
|
||||
log('Error checking model availability', { model: modelName, error: String(error) });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function runModelAvailabilityCheck() {
|
||||
if (!opencodeModels.length) return;
|
||||
|
||||
log('Running model availability check', { modelCount: opencodeModels.length });
|
||||
|
||||
const now = new Date().toISOString();
|
||||
let hasChanges = false;
|
||||
|
||||
for (const model of opencodeModels) {
|
||||
const isAvailable = await checkSingleModelAvailability(model.name);
|
||||
|
||||
if (isAvailable === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
model.lastChecked = now;
|
||||
|
||||
if (!isAvailable && model.available !== false) {
|
||||
model.available = false;
|
||||
model.unavailableSince = now;
|
||||
log('Model marked as unavailable', { model: model.name, label: model.label });
|
||||
hasChanges = true;
|
||||
} else if (isAvailable && model.available === false) {
|
||||
log('Model is available again but keeping unavailable status', { model: model.name, label: model.label });
|
||||
}
|
||||
}
|
||||
|
||||
if (hasChanges) {
|
||||
await persistAdminModels();
|
||||
log('Model availability changes persisted');
|
||||
}
|
||||
}
|
||||
|
||||
function startModelAvailabilityCheck() {
|
||||
if (modelAvailabilityCheckTimer) {
|
||||
clearInterval(modelAvailabilityCheckTimer);
|
||||
}
|
||||
|
||||
runModelAvailabilityCheck().catch((err) => {
|
||||
log('Initial model availability check failed', { error: String(err) });
|
||||
});
|
||||
|
||||
modelAvailabilityCheckTimer = setInterval(() => {
|
||||
runModelAvailabilityCheck().catch((err) => {
|
||||
log('Model availability check failed', { error: String(err) });
|
||||
});
|
||||
}, MODEL_AVAILABILITY_CHECK_INTERVAL_MS);
|
||||
|
||||
log('Model availability check scheduled', { intervalMs: MODEL_AVAILABILITY_CHECK_INTERVAL_MS });
|
||||
}
|
||||
|
||||
async function loadOpenRouterSettings() {
|
||||
try {
|
||||
await ensureStateFile();
|
||||
@@ -10240,7 +10310,10 @@ function buildOpencodeAttemptChain(cli, preferredModel) {
|
||||
// Build chain from opencodeModels order only
|
||||
// The order in the list determines fallback priority
|
||||
// Never use the public model's Model ID - opencode models are independent
|
||||
opencodeModels.forEach((m, idx) => {
|
||||
// Filter out unavailable models
|
||||
const availableModels = opencodeModels.filter((m) => m.available !== false);
|
||||
|
||||
availableModels.forEach((m, idx) => {
|
||||
const modelToUse = m.name;
|
||||
const key = `${modelToUse}`;
|
||||
if (seen.has(key)) return;
|
||||
@@ -16009,6 +16082,51 @@ async function handleAdminModelDelete(req, res, id) {
|
||||
});
|
||||
}
|
||||
|
||||
async function handleAdminModelsAvailability(req, res) {
|
||||
const session = requireAdminAuth(req, res);
|
||||
if (!session) return;
|
||||
|
||||
sendJson(res, 200, {
|
||||
opencodeModels: opencodeModels.map((m) => ({
|
||||
id: m.id,
|
||||
name: m.name,
|
||||
label: m.label,
|
||||
available: m.available !== false,
|
||||
lastChecked: m.lastChecked,
|
||||
unavailableSince: m.unavailableSince,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
async function handleAdminModelReadd(req, res, id) {
|
||||
const session = requireAdminAuth(req, res);
|
||||
if (!session) return;
|
||||
|
||||
const model = opencodeModels.find((m) => m.id === id);
|
||||
if (!model) {
|
||||
return sendJson(res, 404, { error: 'Model not found' });
|
||||
}
|
||||
|
||||
const isAvailable = await checkSingleModelAvailability(model.name);
|
||||
if (!isAvailable) {
|
||||
return sendJson(res, 400, { error: 'Model is not available from OpenCode' });
|
||||
}
|
||||
|
||||
model.available = true;
|
||||
model.unavailableSince = null;
|
||||
model.lastChecked = new Date().toISOString();
|
||||
|
||||
await persistAdminModels();
|
||||
|
||||
log('Model re-added to fallback chain', { model: model.name, label: model.label });
|
||||
|
||||
sendJson(res, 200, {
|
||||
ok: true,
|
||||
opencodeModels,
|
||||
publicModels,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleAdminOpenRouterSettingsGet(req, res) {
|
||||
const session = requireAdminAuth(req, res);
|
||||
if (!session) return;
|
||||
@@ -18906,6 +19024,7 @@ async function routeInternal(req, res, url, pathname) {
|
||||
if (req.method === 'GET' && pathname === '/api/admin/models') return handleAdminModelsList(req, res);
|
||||
if (req.method === 'POST' && pathname === '/api/admin/models') return handleAdminModelUpsert(req, res);
|
||||
if (req.method === 'POST' && pathname === '/api/admin/models/reorder') return handleAdminModelsReorder(req, res);
|
||||
if (req.method === 'GET' && pathname === '/api/admin/models/availability') return handleAdminModelsAvailability(req, res);
|
||||
if (req.method === 'GET' && pathname === '/api/admin/openrouter-settings') return handleAdminOpenRouterSettingsGet(req, res);
|
||||
if (req.method === 'POST' && pathname === '/api/admin/openrouter-settings') return handleAdminOpenRouterSettingsPost(req, res);
|
||||
if (req.method === 'GET' && pathname === '/api/admin/mistral-settings') return handleAdminMistralSettingsGet(req, res);
|
||||
@@ -18937,6 +19056,8 @@ async function routeInternal(req, res, url, pathname) {
|
||||
if (req.method === 'POST' && pathname === '/api/admin/ollama-test') return handleAdminOllamaTest(req, res);
|
||||
const adminDeleteMatch = pathname.match(/^\/api\/admin\/models\/([a-f0-9\-]+)$/i);
|
||||
if (req.method === 'DELETE' && adminDeleteMatch) return handleAdminModelDelete(req, res, adminDeleteMatch[1]);
|
||||
const adminReaddMatch = pathname.match(/^\/api\/admin\/models\/([a-f0-9\-]+)\/readd$/i);
|
||||
if (req.method === 'POST' && adminReaddMatch) return handleAdminModelReadd(req, res, adminReaddMatch[1]);
|
||||
if (req.method === 'GET' && pathname === '/api/models') {
|
||||
const cliParam = url.searchParams.get('cli');
|
||||
return handleModels(req, res, cliParam);
|
||||
@@ -19413,6 +19534,9 @@ async function bootstrap() {
|
||||
|
||||
// Start periodic resource monitoring for analytics
|
||||
startPeriodicMonitoring();
|
||||
|
||||
// Start model availability check for OpenCode models
|
||||
startModelAvailabilityCheck();
|
||||
|
||||
server = http.createServer((req, res) => { route(req, res); });
|
||||
|
||||
|
||||
Reference in New Issue
Block a user