feat: add model availability monitoring for OpenCode fallback chain

- Add background job that checks model availability every 5 hours - Automatically removes unavailable models from fallback chain - Adds unavailable models section in admin UI with blur styling - Allows admins to re-add models when they become available again - Extends model schema with available, lastChecked, unavailableSince fields - Adds API endpoints: GET /api/admin/models/availability, POST /api/admin/models/:id/readd
2026-02-19 19:55:48 +00:00
parent 5df7ef1c8d
commit a92797d3a7
4 changed files with 288 additions and 10 deletions
--- a/chat/public/admin.html
+++ b/chat/public/admin.html
@@ -160,6 +160,16 @@
      <div id="opencode-models-list" class="admin-list"></div>
    </div>

+    <!-- Unavailable Models Section -->
+    <div class="admin-card unavailable-models-section" style="margin-top: 16px; display: none;">
+      <header>
+        <h3>Unavailable Models</h3>
+        <div class="pill" id="unavailable-models-count">0</div>
+      </header>
+      <p class="muted" style="margin-top:0;">These models were removed from the fallback chain because they are no longer available from OpenCode. You can re-add them if they become available again.</p>
+      <div id="unavailable-models-list" class="admin-list"></div>
+    </div>
+
    <!-- Public Models List with Ordering -->
    <div class="admin-card" style="margin-top: 16px;">
      <header>
--- a/chat/public/admin.js
+++ b/chat/public/admin.js
@@ -32,6 +32,10 @@
    reloadOpencodeModels: document.getElementById('reload-opencode-models'),
    opencodeModelsList: document.getElementById('opencode-models-list'),
    opencodeModelsCount: document.getElementById('opencode-models-count'),
+    // Unavailable Models
+    unavailableModelsSection: document.querySelector('.unavailable-models-section'),
+    unavailableModelsList: document.getElementById('unavailable-models-list'),
+    unavailableModelsCount: document.getElementById('unavailable-models-count'),
    // Public Models (user-facing selection)
    publicModelForm: document.getElementById('public-model-form'),
    publicModelLabel: document.getElementById('public-model-label'),
@@ -602,17 +606,22 @@
  function renderOpencodeModels() {
    if (!el.opencodeModelsList) return;
    el.opencodeModelsList.innerHTML = '';
-    if (el.opencodeModelsCount) el.opencodeModelsCount.textContent = state.opencodeModels.length.toString();
    
-    if (!state.opencodeModels.length) {
+    const availableModels = state.opencodeModels.filter((m) => m.available !== false);
+    const unavailableModels = state.opencodeModels.filter((m) => m.available === false);
+    
+    if (el.opencodeModelsCount) el.opencodeModelsCount.textContent = availableModels.length.toString();
+    
+    if (!availableModels.length) {
      const empty = document.createElement('div');
      empty.className = 'muted';
      empty.textContent = 'No OpenCode models configured. Add models to enable fallback chain for build execution.';
      el.opencodeModelsList.appendChild(empty);
+      renderUnavailableModels(unavailableModels);
      return;
    }

-    state.opencodeModels.forEach((m, idx) => {
+    availableModels.forEach((m, idx) => {
      const row = document.createElement('div');
      row.className = 'provider-row slim';

@@ -672,10 +681,11 @@
      upBtn.title = 'Move up';
      upBtn.disabled = idx === 0;
      upBtn.addEventListener('click', async () => {
-        const next = [...state.opencodeModels];
+        const next = [...availableModels];
        const [item] = next.splice(idx, 1);
        next.splice(Math.max(0, idx - 1), 0, item);
-        await persistOpencodeModelsOrder(next);
+        const fullNext = [...next, ...unavailableModels];
+        await persistOpencodeModelsOrder(fullNext);
      });
      headerActions.appendChild(upBtn);

@@ -684,12 +694,13 @@
      downBtn.className = 'ghost';
      downBtn.textContent = '↓';
      downBtn.title = 'Move down';
-      downBtn.disabled = idx === state.opencodeModels.length - 1;
+      downBtn.disabled = idx === availableModels.length - 1;
      downBtn.addEventListener('click', async () => {
-        const next = [...state.opencodeModels];
+        const next = [...availableModels];
        const [item] = next.splice(idx, 1);
-        next.splice(Math.min(state.opencodeModels.length, idx + 1), 0, item);
-        await persistOpencodeModelsOrder(next);
+        next.splice(Math.min(availableModels.length, idx + 1), 0, item);
+        const fullNext = [...next, ...unavailableModels];
+        await persistOpencodeModelsOrder(fullNext);
      });
      headerActions.appendChild(downBtn);

@@ -782,6 +793,114 @@
      row.appendChild(header);
      el.opencodeModelsList.appendChild(row);
    });
+    
+    renderUnavailableModels(unavailableModels);
+  }
+
+  // Render unavailable models list with re-add option
+  function renderUnavailableModels(models) {
+    if (!el.unavailableModelsList || !el.unavailableModelsSection) return;
+    
+    if (!models.length) {
+      el.unavailableModelsSection.style.display = 'none';
+      return;
+    }
+    
+    el.unavailableModelsSection.style.display = 'block';
+    el.unavailableModelsList.innerHTML = '';
+    
+    if (el.unavailableModelsCount) {
+      el.unavailableModelsCount.textContent = models.length.toString();
+    }
+    
+    models.forEach((m) => {
+      const row = document.createElement('div');
+      row.className = 'provider-row slim unavailable-model';
+
+      const header = document.createElement('div');
+      header.className = 'provider-row-header';
+      
+      const info = document.createElement('div');
+      info.className = 'model-chip';
+      
+      // Unavailable badge
+      const unavailableBadge = document.createElement('span');
+      unavailableBadge.className = 'pill';
+      unavailableBadge.style.background = 'var(--danger)';
+      unavailableBadge.textContent = 'Unavailable';
+      info.appendChild(unavailableBadge);
+      
+      if (m.icon) {
+        const img = document.createElement('img');
+        img.src = m.icon;
+        img.alt = '';
+        img.style.filter = 'grayscale(100%)';
+        info.appendChild(img);
+      }
+      
+      const label = document.createElement('span');
+      label.textContent = m.label || m.name;
+      label.style.opacity = '0.6';
+      info.appendChild(label);
+      
+      const namePill = document.createElement('span');
+      namePill.className = 'pill';
+      namePill.textContent = m.name;
+      namePill.style.opacity = '0.6';
+      info.appendChild(namePill);
+      
+      if (m.unavailableSince) {
+        const sincePill = document.createElement('span');
+        sincePill.className = 'pill';
+        sincePill.textContent = `Since ${new Date(m.unavailableSince).toLocaleDateString()}`;
+        info.appendChild(sincePill);
+      }
+      
+      header.appendChild(info);
+
+      const headerActions = document.createElement('div');
+      headerActions.className = 'provider-row-actions';
+
+      // Re-add button
+      const readdBtn = document.createElement('button');
+      readdBtn.className = 'primary';
+      readdBtn.textContent = 'Re-add';
+      readdBtn.addEventListener('click', async () => {
+        readdBtn.disabled = true;
+        readdBtn.textContent = 'Checking...';
+        try {
+          await api(`/api/admin/models/${m.id}/readd`, { method: 'POST' });
+          await loadConfigured();
+          setStatus('Model re-added to fallback chain');
+          setTimeout(() => setStatus(''), 2000);
+        } catch (err) {
+          setStatus(err.message, true);
+          readdBtn.disabled = false;
+          readdBtn.textContent = 'Re-add';
+        }
+      });
+      headerActions.appendChild(readdBtn);
+
+      // Delete button
+      const delBtn = document.createElement('button');
+      delBtn.className = 'ghost';
+      delBtn.textContent = 'Delete';
+      delBtn.addEventListener('click', async () => {
+        delBtn.disabled = true;
+        try {
+          await api(`/api/admin/models/${m.id}?type=opencode`, { method: 'DELETE' });
+          await loadConfigured();
+        } catch (err) {
+          setStatus(err.message, true);
+        }
+        delBtn.disabled = false;
+      });
+      headerActions.appendChild(delBtn);
+
+      header.appendChild(headerActions);
+      row.appendChild(header);
+      el.unavailableModelsList.appendChild(row);
+    });
  }

  // Render Public models list with up/down ordering
--- a/chat/public/styles.css
+++ b/chat/public/styles.css
@@ -1162,3 +1162,28 @@ textarea:focus {
  font-weight: 600;
  letter-spacing: 0.02em;
 }
+
+/* Unavailable models styling */
+.unavailable-models-section {
+  background: rgba(176, 0, 32, 0.02);
+  border-color: rgba(176, 0, 32, 0.15);
+}
+
+.unavailable-model {
+  opacity: 0.7;
+  filter: blur(0.5px);
+  background: #fafafa;
+}
+
+.unavailable-model:hover {
+  opacity: 1;
+  filter: none;
+}
+
+.unavailable-model .model-chip {
+  filter: grayscale(30%);
+}
+
+.unavailable-model img {
+  filter: grayscale(100%);
+}
--- a/chat/server.js
+++ b/chat/server.js
@@ -1527,6 +1527,8 @@ let adminModelIndex = new Map();
 // Simple model structure - just two lists
 let opencodeModels = []; // Models from OpenCode - order determines fallback chain
 let publicModels = []; // Public-facing models (completely separate)
+const MODEL_AVAILABILITY_CHECK_INTERVAL_MS = 5 * 60 * 60 * 1000; // 5 hours
+let modelAvailabilityCheckTimer = null;
 let openrouterSettings = {
  primaryModel: OPENROUTER_MODEL_PRIMARY,
  backupModel1: OPENROUTER_MODEL_BACKUP_1,
@@ -5792,6 +5794,9 @@ async function loadAdminModelStore() {
      tier: normalizeTier(m.tier),
      supportsMedia: m.supportsMedia ?? false,
      multiplier: m.multiplier || getTierMultiplier(m.tier),
+      available: m.available !== false,
+      lastChecked: m.lastChecked || null,
+      unavailableSince: m.unavailableSince || null,
    })).filter((m) => !!m.name);
    
    // Validate public models
@@ -5833,6 +5838,71 @@ async function persistAdminModels() {
  refreshAdminModelIndex();
 }

+async function checkSingleModelAvailability(modelName) {
+  try {
+    const availableModels = await listModels('opencode');
+    const isAvailable = availableModels.some((m) => {
+      const mName = m.name || m.id || m;
+      return mName === modelName || mName.endsWith('/' + modelName) || modelName.endsWith('/' + mName);
+    });
+    return isAvailable;
+  } catch (error) {
+    log('Error checking model availability', { model: modelName, error: String(error) });
+    return null;
+  }
+}
+
+async function runModelAvailabilityCheck() {
+  if (!opencodeModels.length) return;
+  
+  log('Running model availability check', { modelCount: opencodeModels.length });
+  
+  const now = new Date().toISOString();
+  let hasChanges = false;
+  
+  for (const model of opencodeModels) {
+    const isAvailable = await checkSingleModelAvailability(model.name);
+    
+    if (isAvailable === null) {
+      continue;
+    }
+    
+    model.lastChecked = now;
+    
+    if (!isAvailable && model.available !== false) {
+      model.available = false;
+      model.unavailableSince = now;
+      log('Model marked as unavailable', { model: model.name, label: model.label });
+      hasChanges = true;
+    } else if (isAvailable && model.available === false) {
+      log('Model is available again but keeping unavailable status', { model: model.name, label: model.label });
+    }
+  }
+  
+  if (hasChanges) {
+    await persistAdminModels();
+    log('Model availability changes persisted');
+  }
+}
+
+function startModelAvailabilityCheck() {
+  if (modelAvailabilityCheckTimer) {
+    clearInterval(modelAvailabilityCheckTimer);
+  }
+  
+  runModelAvailabilityCheck().catch((err) => {
+    log('Initial model availability check failed', { error: String(err) });
+  });
+  
+  modelAvailabilityCheckTimer = setInterval(() => {
+    runModelAvailabilityCheck().catch((err) => {
+      log('Model availability check failed', { error: String(err) });
+    });
+  }, MODEL_AVAILABILITY_CHECK_INTERVAL_MS);
+  
+  log('Model availability check scheduled', { intervalMs: MODEL_AVAILABILITY_CHECK_INTERVAL_MS });
+}
+
 async function loadOpenRouterSettings() {
  try {
    await ensureStateFile();
@@ -10240,7 +10310,10 @@ function buildOpencodeAttemptChain(cli, preferredModel) {
  // Build chain from opencodeModels order only
  // The order in the list determines fallback priority
  // Never use the public model's Model ID - opencode models are independent
-  opencodeModels.forEach((m, idx) => {
+  // Filter out unavailable models
+  const availableModels = opencodeModels.filter((m) => m.available !== false);
+  
+  availableModels.forEach((m, idx) => {
    const modelToUse = m.name;
    const key = `${modelToUse}`;
    if (seen.has(key)) return;
@@ -16009,6 +16082,51 @@ async function handleAdminModelDelete(req, res, id) {
  });
 }

+async function handleAdminModelsAvailability(req, res) {
+  const session = requireAdminAuth(req, res);
+  if (!session) return;
+  
+  sendJson(res, 200, {
+    opencodeModels: opencodeModels.map((m) => ({
+      id: m.id,
+      name: m.name,
+      label: m.label,
+      available: m.available !== false,
+      lastChecked: m.lastChecked,
+      unavailableSince: m.unavailableSince,
+    })),
+  });
+}
+
+async function handleAdminModelReadd(req, res, id) {
+  const session = requireAdminAuth(req, res);
+  if (!session) return;
+  
+  const model = opencodeModels.find((m) => m.id === id);
+  if (!model) {
+    return sendJson(res, 404, { error: 'Model not found' });
+  }
+  
+  const isAvailable = await checkSingleModelAvailability(model.name);
+  if (!isAvailable) {
+    return sendJson(res, 400, { error: 'Model is not available from OpenCode' });
+  }
+  
+  model.available = true;
+  model.unavailableSince = null;
+  model.lastChecked = new Date().toISOString();
+  
+  await persistAdminModels();
+  
+  log('Model re-added to fallback chain', { model: model.name, label: model.label });
+  
+  sendJson(res, 200, {
+    ok: true,
+    opencodeModels,
+    publicModels,
+  });
+}
+
 async function handleAdminOpenRouterSettingsGet(req, res) {
  const session = requireAdminAuth(req, res);
  if (!session) return;
@@ -18906,6 +19024,7 @@ async function routeInternal(req, res, url, pathname) {
  if (req.method === 'GET' && pathname === '/api/admin/models') return handleAdminModelsList(req, res);
  if (req.method === 'POST' && pathname === '/api/admin/models') return handleAdminModelUpsert(req, res);
  if (req.method === 'POST' && pathname === '/api/admin/models/reorder') return handleAdminModelsReorder(req, res);
+  if (req.method === 'GET' && pathname === '/api/admin/models/availability') return handleAdminModelsAvailability(req, res);
  if (req.method === 'GET' && pathname === '/api/admin/openrouter-settings') return handleAdminOpenRouterSettingsGet(req, res);
  if (req.method === 'POST' && pathname === '/api/admin/openrouter-settings') return handleAdminOpenRouterSettingsPost(req, res);
  if (req.method === 'GET' && pathname === '/api/admin/mistral-settings') return handleAdminMistralSettingsGet(req, res);
@@ -18937,6 +19056,8 @@ async function routeInternal(req, res, url, pathname) {
  if (req.method === 'POST' && pathname === '/api/admin/ollama-test') return handleAdminOllamaTest(req, res);
  const adminDeleteMatch = pathname.match(/^\/api\/admin\/models\/([a-f0-9\-]+)$/i);
  if (req.method === 'DELETE' && adminDeleteMatch) return handleAdminModelDelete(req, res, adminDeleteMatch[1]);
+  const adminReaddMatch = pathname.match(/^\/api\/admin\/models\/([a-f0-9\-]+)\/readd$/i);
+  if (req.method === 'POST' && adminReaddMatch) return handleAdminModelReadd(req, res, adminReaddMatch[1]);
  if (req.method === 'GET' && pathname === '/api/models') {
    const cliParam = url.searchParams.get('cli');
    return handleModels(req, res, cliParam);
@@ -19414,6 +19535,9 @@ async function bootstrap() {
  // Start periodic resource monitoring for analytics
  startPeriodicMonitoring();
  
+  // Start model availability check for OpenCode models
+  startModelAvailabilityCheck();
+
  server = http.createServer((req, res) => { route(req, res); });

  // Disable Node's built-in request timeouts so long-running SSE streams / long LLM "thinking" phases