From a92797d3a75cd071ddd935c39528df7a3b6ec382 Mon Sep 17 00:00:00 2001 From: southseact-3d Date: Thu, 19 Feb 2026 19:55:48 +0000 Subject: [PATCH] feat: add model availability monitoring for OpenCode fallback chain - Add background job that checks model availability every 5 hours - Automatically removes unavailable models from fallback chain - Adds unavailable models section in admin UI with blur styling - Allows admins to re-add models when they become available again - Extends model schema with available, lastChecked, unavailableSince fields - Adds API endpoints: GET /api/admin/models/availability, POST /api/admin/models/:id/readd --- chat/public/admin.html | 10 +++ chat/public/admin.js | 137 ++++++++++++++++++++++++++++++++++++++--- chat/public/styles.css | 25 ++++++++ chat/server.js | 126 ++++++++++++++++++++++++++++++++++++- 4 files changed, 288 insertions(+), 10 deletions(-) diff --git a/chat/public/admin.html b/chat/public/admin.html index b41211b..4b95a71 100644 --- a/chat/public/admin.html +++ b/chat/public/admin.html @@ -160,6 +160,16 @@
+ + +
diff --git a/chat/public/admin.js b/chat/public/admin.js index 530337c..b866863 100644 --- a/chat/public/admin.js +++ b/chat/public/admin.js @@ -32,6 +32,10 @@ reloadOpencodeModels: document.getElementById('reload-opencode-models'), opencodeModelsList: document.getElementById('opencode-models-list'), opencodeModelsCount: document.getElementById('opencode-models-count'), + // Unavailable Models + unavailableModelsSection: document.querySelector('.unavailable-models-section'), + unavailableModelsList: document.getElementById('unavailable-models-list'), + unavailableModelsCount: document.getElementById('unavailable-models-count'), // Public Models (user-facing selection) publicModelForm: document.getElementById('public-model-form'), publicModelLabel: document.getElementById('public-model-label'), @@ -602,17 +606,22 @@ function renderOpencodeModels() { if (!el.opencodeModelsList) return; el.opencodeModelsList.innerHTML = ''; - if (el.opencodeModelsCount) el.opencodeModelsCount.textContent = state.opencodeModels.length.toString(); - if (!state.opencodeModels.length) { + const availableModels = state.opencodeModels.filter((m) => m.available !== false); + const unavailableModels = state.opencodeModels.filter((m) => m.available === false); + + if (el.opencodeModelsCount) el.opencodeModelsCount.textContent = availableModels.length.toString(); + + if (!availableModels.length) { const empty = document.createElement('div'); empty.className = 'muted'; empty.textContent = 'No OpenCode models configured. Add models to enable fallback chain for build execution.'; el.opencodeModelsList.appendChild(empty); + renderUnavailableModels(unavailableModels); return; } - state.opencodeModels.forEach((m, idx) => { + availableModels.forEach((m, idx) => { const row = document.createElement('div'); row.className = 'provider-row slim'; @@ -672,10 +681,11 @@ upBtn.title = 'Move up'; upBtn.disabled = idx === 0; upBtn.addEventListener('click', async () => { - const next = [...state.opencodeModels]; + const next = [...availableModels]; const [item] = next.splice(idx, 1); next.splice(Math.max(0, idx - 1), 0, item); - await persistOpencodeModelsOrder(next); + const fullNext = [...next, ...unavailableModels]; + await persistOpencodeModelsOrder(fullNext); }); headerActions.appendChild(upBtn); @@ -684,12 +694,13 @@ downBtn.className = 'ghost'; downBtn.textContent = '↓'; downBtn.title = 'Move down'; - downBtn.disabled = idx === state.opencodeModels.length - 1; + downBtn.disabled = idx === availableModels.length - 1; downBtn.addEventListener('click', async () => { - const next = [...state.opencodeModels]; + const next = [...availableModels]; const [item] = next.splice(idx, 1); - next.splice(Math.min(state.opencodeModels.length, idx + 1), 0, item); - await persistOpencodeModelsOrder(next); + next.splice(Math.min(availableModels.length, idx + 1), 0, item); + const fullNext = [...next, ...unavailableModels]; + await persistOpencodeModelsOrder(fullNext); }); headerActions.appendChild(downBtn); @@ -782,6 +793,114 @@ row.appendChild(header); el.opencodeModelsList.appendChild(row); }); + + renderUnavailableModels(unavailableModels); + } + + // Render unavailable models list with re-add option + function renderUnavailableModels(models) { + if (!el.unavailableModelsList || !el.unavailableModelsSection) return; + + if (!models.length) { + el.unavailableModelsSection.style.display = 'none'; + return; + } + + el.unavailableModelsSection.style.display = 'block'; + el.unavailableModelsList.innerHTML = ''; + + if (el.unavailableModelsCount) { + el.unavailableModelsCount.textContent = models.length.toString(); + } + + models.forEach((m) => { + const row = document.createElement('div'); + row.className = 'provider-row slim unavailable-model'; + + const header = document.createElement('div'); + header.className = 'provider-row-header'; + + const info = document.createElement('div'); + info.className = 'model-chip'; + + // Unavailable badge + const unavailableBadge = document.createElement('span'); + unavailableBadge.className = 'pill'; + unavailableBadge.style.background = 'var(--danger)'; + unavailableBadge.textContent = 'Unavailable'; + info.appendChild(unavailableBadge); + + if (m.icon) { + const img = document.createElement('img'); + img.src = m.icon; + img.alt = ''; + img.style.filter = 'grayscale(100%)'; + info.appendChild(img); + } + + const label = document.createElement('span'); + label.textContent = m.label || m.name; + label.style.opacity = '0.6'; + info.appendChild(label); + + const namePill = document.createElement('span'); + namePill.className = 'pill'; + namePill.textContent = m.name; + namePill.style.opacity = '0.6'; + info.appendChild(namePill); + + if (m.unavailableSince) { + const sincePill = document.createElement('span'); + sincePill.className = 'pill'; + sincePill.textContent = `Since ${new Date(m.unavailableSince).toLocaleDateString()}`; + info.appendChild(sincePill); + } + + header.appendChild(info); + + const headerActions = document.createElement('div'); + headerActions.className = 'provider-row-actions'; + + // Re-add button + const readdBtn = document.createElement('button'); + readdBtn.className = 'primary'; + readdBtn.textContent = 'Re-add'; + readdBtn.addEventListener('click', async () => { + readdBtn.disabled = true; + readdBtn.textContent = 'Checking...'; + try { + await api(`/api/admin/models/${m.id}/readd`, { method: 'POST' }); + await loadConfigured(); + setStatus('Model re-added to fallback chain'); + setTimeout(() => setStatus(''), 2000); + } catch (err) { + setStatus(err.message, true); + readdBtn.disabled = false; + readdBtn.textContent = 'Re-add'; + } + }); + headerActions.appendChild(readdBtn); + + // Delete button + const delBtn = document.createElement('button'); + delBtn.className = 'ghost'; + delBtn.textContent = 'Delete'; + delBtn.addEventListener('click', async () => { + delBtn.disabled = true; + try { + await api(`/api/admin/models/${m.id}?type=opencode`, { method: 'DELETE' }); + await loadConfigured(); + } catch (err) { + setStatus(err.message, true); + } + delBtn.disabled = false; + }); + headerActions.appendChild(delBtn); + + header.appendChild(headerActions); + row.appendChild(header); + el.unavailableModelsList.appendChild(row); + }); } // Render Public models list with up/down ordering diff --git a/chat/public/styles.css b/chat/public/styles.css index 5d02418..b9c4711 100644 --- a/chat/public/styles.css +++ b/chat/public/styles.css @@ -1161,4 +1161,29 @@ textarea:focus { font-size: 14px; font-weight: 600; letter-spacing: 0.02em; +} + +/* Unavailable models styling */ +.unavailable-models-section { + background: rgba(176, 0, 32, 0.02); + border-color: rgba(176, 0, 32, 0.15); +} + +.unavailable-model { + opacity: 0.7; + filter: blur(0.5px); + background: #fafafa; +} + +.unavailable-model:hover { + opacity: 1; + filter: none; +} + +.unavailable-model .model-chip { + filter: grayscale(30%); +} + +.unavailable-model img { + filter: grayscale(100%); } \ No newline at end of file diff --git a/chat/server.js b/chat/server.js index b831cca..70b3378 100644 --- a/chat/server.js +++ b/chat/server.js @@ -1527,6 +1527,8 @@ let adminModelIndex = new Map(); // Simple model structure - just two lists let opencodeModels = []; // Models from OpenCode - order determines fallback chain let publicModels = []; // Public-facing models (completely separate) +const MODEL_AVAILABILITY_CHECK_INTERVAL_MS = 5 * 60 * 60 * 1000; // 5 hours +let modelAvailabilityCheckTimer = null; let openrouterSettings = { primaryModel: OPENROUTER_MODEL_PRIMARY, backupModel1: OPENROUTER_MODEL_BACKUP_1, @@ -5792,6 +5794,9 @@ async function loadAdminModelStore() { tier: normalizeTier(m.tier), supportsMedia: m.supportsMedia ?? false, multiplier: m.multiplier || getTierMultiplier(m.tier), + available: m.available !== false, + lastChecked: m.lastChecked || null, + unavailableSince: m.unavailableSince || null, })).filter((m) => !!m.name); // Validate public models @@ -5833,6 +5838,71 @@ async function persistAdminModels() { refreshAdminModelIndex(); } +async function checkSingleModelAvailability(modelName) { + try { + const availableModels = await listModels('opencode'); + const isAvailable = availableModels.some((m) => { + const mName = m.name || m.id || m; + return mName === modelName || mName.endsWith('/' + modelName) || modelName.endsWith('/' + mName); + }); + return isAvailable; + } catch (error) { + log('Error checking model availability', { model: modelName, error: String(error) }); + return null; + } +} + +async function runModelAvailabilityCheck() { + if (!opencodeModels.length) return; + + log('Running model availability check', { modelCount: opencodeModels.length }); + + const now = new Date().toISOString(); + let hasChanges = false; + + for (const model of opencodeModels) { + const isAvailable = await checkSingleModelAvailability(model.name); + + if (isAvailable === null) { + continue; + } + + model.lastChecked = now; + + if (!isAvailable && model.available !== false) { + model.available = false; + model.unavailableSince = now; + log('Model marked as unavailable', { model: model.name, label: model.label }); + hasChanges = true; + } else if (isAvailable && model.available === false) { + log('Model is available again but keeping unavailable status', { model: model.name, label: model.label }); + } + } + + if (hasChanges) { + await persistAdminModels(); + log('Model availability changes persisted'); + } +} + +function startModelAvailabilityCheck() { + if (modelAvailabilityCheckTimer) { + clearInterval(modelAvailabilityCheckTimer); + } + + runModelAvailabilityCheck().catch((err) => { + log('Initial model availability check failed', { error: String(err) }); + }); + + modelAvailabilityCheckTimer = setInterval(() => { + runModelAvailabilityCheck().catch((err) => { + log('Model availability check failed', { error: String(err) }); + }); + }, MODEL_AVAILABILITY_CHECK_INTERVAL_MS); + + log('Model availability check scheduled', { intervalMs: MODEL_AVAILABILITY_CHECK_INTERVAL_MS }); +} + async function loadOpenRouterSettings() { try { await ensureStateFile(); @@ -10240,7 +10310,10 @@ function buildOpencodeAttemptChain(cli, preferredModel) { // Build chain from opencodeModels order only // The order in the list determines fallback priority // Never use the public model's Model ID - opencode models are independent - opencodeModels.forEach((m, idx) => { + // Filter out unavailable models + const availableModels = opencodeModels.filter((m) => m.available !== false); + + availableModels.forEach((m, idx) => { const modelToUse = m.name; const key = `${modelToUse}`; if (seen.has(key)) return; @@ -16009,6 +16082,51 @@ async function handleAdminModelDelete(req, res, id) { }); } +async function handleAdminModelsAvailability(req, res) { + const session = requireAdminAuth(req, res); + if (!session) return; + + sendJson(res, 200, { + opencodeModels: opencodeModels.map((m) => ({ + id: m.id, + name: m.name, + label: m.label, + available: m.available !== false, + lastChecked: m.lastChecked, + unavailableSince: m.unavailableSince, + })), + }); +} + +async function handleAdminModelReadd(req, res, id) { + const session = requireAdminAuth(req, res); + if (!session) return; + + const model = opencodeModels.find((m) => m.id === id); + if (!model) { + return sendJson(res, 404, { error: 'Model not found' }); + } + + const isAvailable = await checkSingleModelAvailability(model.name); + if (!isAvailable) { + return sendJson(res, 400, { error: 'Model is not available from OpenCode' }); + } + + model.available = true; + model.unavailableSince = null; + model.lastChecked = new Date().toISOString(); + + await persistAdminModels(); + + log('Model re-added to fallback chain', { model: model.name, label: model.label }); + + sendJson(res, 200, { + ok: true, + opencodeModels, + publicModels, + }); +} + async function handleAdminOpenRouterSettingsGet(req, res) { const session = requireAdminAuth(req, res); if (!session) return; @@ -18906,6 +19024,7 @@ async function routeInternal(req, res, url, pathname) { if (req.method === 'GET' && pathname === '/api/admin/models') return handleAdminModelsList(req, res); if (req.method === 'POST' && pathname === '/api/admin/models') return handleAdminModelUpsert(req, res); if (req.method === 'POST' && pathname === '/api/admin/models/reorder') return handleAdminModelsReorder(req, res); + if (req.method === 'GET' && pathname === '/api/admin/models/availability') return handleAdminModelsAvailability(req, res); if (req.method === 'GET' && pathname === '/api/admin/openrouter-settings') return handleAdminOpenRouterSettingsGet(req, res); if (req.method === 'POST' && pathname === '/api/admin/openrouter-settings') return handleAdminOpenRouterSettingsPost(req, res); if (req.method === 'GET' && pathname === '/api/admin/mistral-settings') return handleAdminMistralSettingsGet(req, res); @@ -18937,6 +19056,8 @@ async function routeInternal(req, res, url, pathname) { if (req.method === 'POST' && pathname === '/api/admin/ollama-test') return handleAdminOllamaTest(req, res); const adminDeleteMatch = pathname.match(/^\/api\/admin\/models\/([a-f0-9\-]+)$/i); if (req.method === 'DELETE' && adminDeleteMatch) return handleAdminModelDelete(req, res, adminDeleteMatch[1]); + const adminReaddMatch = pathname.match(/^\/api\/admin\/models\/([a-f0-9\-]+)\/readd$/i); + if (req.method === 'POST' && adminReaddMatch) return handleAdminModelReadd(req, res, adminReaddMatch[1]); if (req.method === 'GET' && pathname === '/api/models') { const cliParam = url.searchParams.get('cli'); return handleModels(req, res, cliParam); @@ -19413,6 +19534,9 @@ async function bootstrap() { // Start periodic resource monitoring for analytics startPeriodicMonitoring(); + + // Start model availability check for OpenCode models + startModelAvailabilityCheck(); server = http.createServer((req, res) => { route(req, res); });