Add database migration scripts and configuration files

- Add verify-migration.js script for testing database migrations - Add database config module for centralized configuration - Add chutes.txt prompt for system responses - Update database implementation and testing documentation - Add database migration and setup scripts - Update session system and LLM tool configuration - Update deployment checklist and environment example - Update Dockerfile and docker-compose configuration
2026-02-20 12:38:43 +00:00
parent a92797d3a7
commit cb95a916ae
19 changed files with 1104 additions and 143 deletions
--- a/chat/scripts/init-database.js
+++ b/chat/scripts/init-database.js
@@ -7,9 +7,9 @@
 const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
+const { DATA_ROOT, DB_PATH, KEY_FILE } = require('../src/database/config');

-const DATA_ROOT = process.env.CHAT_DATA_ROOT || '/home/web/data/.data';
-const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
+const DATABASE_PATH = DB_PATH;
 const USE_JSON_DATABASE = process.env.USE_JSON_DATABASE === '1' || process.env.USE_JSON_DATABASE === 'true';

 async function initializeDatabase() {
@@ -33,7 +33,15 @@ async function initializeDatabase() {
  
  if (dbExists) {
    console.log('✅ Database already exists:', DATABASE_PATH);
-    
+
+    if (!process.env.DATABASE_ENCRYPTION_KEY && KEY_FILE && fs.existsSync(KEY_FILE)) {
+      const persistedKey = fs.readFileSync(KEY_FILE, 'utf8').trim();
+      if (persistedKey) {
+        process.env.DATABASE_ENCRYPTION_KEY = persistedKey;
+        console.log('✅ Loaded encryption key from file');
+      }
+    }
+
    // Verify encryption key is set
    if (!process.env.DATABASE_ENCRYPTION_KEY) {
      console.error('❌ DATABASE_ENCRYPTION_KEY not set!');
@@ -48,6 +56,14 @@ async function initializeDatabase() {
  console.log('🔧 Database not found, setting up new database...');
  
  // Generate encryption key if not provided
+  if (!process.env.DATABASE_ENCRYPTION_KEY && KEY_FILE && fs.existsSync(KEY_FILE)) {
+    const persistedKey = fs.readFileSync(KEY_FILE, 'utf8').trim();
+    if (persistedKey) {
+      process.env.DATABASE_ENCRYPTION_KEY = persistedKey;
+      console.log('✅ Loaded encryption key from file');
+    }
+  }
+
  if (!process.env.DATABASE_ENCRYPTION_KEY) {
    const generatedKey = crypto.randomBytes(32).toString('hex');
    process.env.DATABASE_ENCRYPTION_KEY = generatedKey;
@@ -57,7 +73,7 @@ async function initializeDatabase() {
    console.log('⚠️  Add this to your environment configuration to persist it!');
    
    // Save to a file for persistence
-    const keyFile = path.join(dataDir, '.encryption_key');
+    const keyFile = KEY_FILE || path.join(dataDir, '.encryption_key');
    fs.writeFileSync(keyFile, generatedKey, { mode: 0o600 });
    console.log('⚠️  Saved to:', keyFile);
  }
@@ -86,8 +102,8 @@ async function initializeDatabase() {
  }
  
  // Check if there are JSON files to migrate
-  const usersFile = path.join(DATA_ROOT, 'users.json');
-  const sessionsFile = path.join(DATA_ROOT, 'user-sessions.json');
+  const usersFile = path.join(path.join(DATA_ROOT, '.opencode-chat'), 'users.json');
+  const sessionsFile = path.join(path.join(DATA_ROOT, '.opencode-chat'), 'user-sessions.json');
  
  const hasJsonData = fs.existsSync(usersFile) || fs.existsSync(sessionsFile);
  
--- a/chat/scripts/migrate-to-database.js
+++ b/chat/scripts/migrate-to-database.js
@@ -6,16 +6,16 @@
 const fs = require('fs');
 const path = require('path');
 const { initDatabase, getDatabase, closeDatabase } = require('../src/database/connection');
-const { initEncryption } = require('../src/utils/encryption');
-const userRepo = require('../src/repositories/userRepository');
+const { initEncryption, encrypt } = require('../src/utils/encryption');
+const { STATE_DIR, DB_PATH, KEY_FILE } = require('../src/database/config');

-const DATA_ROOT = process.env.CHAT_DATA_ROOT || path.join(__dirname, '..', '.data');
-const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
+const DATABASE_PATH = DB_PATH;
 const DATABASE_ENCRYPTION_KEY = process.env.DATABASE_ENCRYPTION_KEY;
+const DATABASE_USE_SQLCIPHER = process.env.DATABASE_USE_SQLCIPHER !== '0' && process.env.DATABASE_USE_SQLCIPHER !== 'false';

-const USERS_FILE = path.join(DATA_ROOT, 'users.json');
-const SESSIONS_FILE = path.join(DATA_ROOT, 'user-sessions.json');
-const AFFILIATES_FILE = path.join(DATA_ROOT, 'affiliates.json');
+const USERS_FILE = path.join(STATE_DIR, 'users.json');
+const SESSIONS_FILE = path.join(STATE_DIR, 'user-sessions.json');
+const AFFILIATES_FILE = path.join(STATE_DIR, 'affiliates.json');

 async function loadJsonFile(filePath, defaultValue = []) {
  try {
@@ -44,30 +44,88 @@ async function migrateUsers() {
  let success = 0;
  let failed = 0;
  
+  const db = getDatabase();
+  const stmt = db.prepare(`
+    INSERT INTO users (
+      id, email, email_encrypted, name, name_encrypted, password_hash,
+      providers, email_verified, verification_token, verification_expires_at,
+      reset_token, reset_expires_at, plan, billing_status, billing_email,
+      payment_method_last4, subscription_renews_at, referred_by_affiliate_code,
+      affiliate_attribution_at, affiliate_payouts, two_factor_secret,
+      two_factor_enabled, created_at, updated_at, last_login_at, data
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      email = excluded.email,
+      email_encrypted = excluded.email_encrypted,
+      name = excluded.name,
+      name_encrypted = excluded.name_encrypted,
+      password_hash = excluded.password_hash,
+      providers = excluded.providers,
+      email_verified = excluded.email_verified,
+      verification_token = excluded.verification_token,
+      verification_expires_at = excluded.verification_expires_at,
+      reset_token = excluded.reset_token,
+      reset_expires_at = excluded.reset_expires_at,
+      plan = excluded.plan,
+      billing_status = excluded.billing_status,
+      billing_email = excluded.billing_email,
+      payment_method_last4 = excluded.payment_method_last4,
+      subscription_renews_at = excluded.subscription_renews_at,
+      referred_by_affiliate_code = excluded.referred_by_affiliate_code,
+      affiliate_attribution_at = excluded.affiliate_attribution_at,
+      affiliate_payouts = excluded.affiliate_payouts,
+      two_factor_secret = excluded.two_factor_secret,
+      two_factor_enabled = excluded.two_factor_enabled,
+      created_at = excluded.created_at,
+      updated_at = excluded.updated_at,
+      last_login_at = excluded.last_login_at,
+      data = excluded.data
+  `);
+
  for (const user of users) {
    try {
-      // Check if user already exists
-      const existing = userRepo.getUserById(user.id);
-      if (existing) {
-        console.log(`  Skipping existing user: ${user.email}`);
-        success++;
+      const normalizedEmail = (user.email || '').trim().toLowerCase();
+      if (!normalizedEmail) {
+        console.log('  Skipping user without email');
+        failed++;
        continue;
      }
-      
-      // Create user in database
-      userRepo.createUser({
-        id: user.id,
-        email: user.email,
-        name: user.name || null,
-        passwordHash: user.passwordHash || user.password_hash,
-        providers: user.providers || [],
-        emailVerified: user.emailVerified,
-        verificationToken: user.verificationToken || null,
-        verificationExpiresAt: user.verificationExpiresAt || null,
-        plan: user.plan || 'hobby',
-        billingStatus: user.billingStatus || 'active',
-        billingEmail: user.billingEmail || user.email
-      });
+      const passwordHash = user.passwordHash || user.password_hash || user.password || '';
+      const providers = Array.isArray(user.providers) ? user.providers : [];
+      const affiliatePayouts = Array.isArray(user.affiliatePayouts) ? user.affiliatePayouts : [];
+      const dataPayload = JSON.stringify({ ...user, email: normalizedEmail || user.email || '' });
+      const emailEncrypted = normalizedEmail ? encrypt(normalizedEmail) : '';
+      const nameEncrypted = user.name ? encrypt(user.name) : null;
+      const twoFactorEncrypted = user.twoFactorSecret ? encrypt(user.twoFactorSecret) : null;
+
+      stmt.run(
+        user.id,
+        normalizedEmail,
+        emailEncrypted,
+        user.name || null,
+        nameEncrypted,
+        passwordHash,
+        JSON.stringify(providers),
+        user.emailVerified ? 1 : 0,
+        user.verificationToken || null,
+        user.verificationExpiresAt || null,
+        user.resetToken || null,
+        user.resetExpiresAt || null,
+        user.plan || 'hobby',
+        user.billingStatus || 'active',
+        user.billingEmail || user.email || '',
+        user.paymentMethodLast4 || '',
+        user.subscriptionRenewsAt || null,
+        user.referredByAffiliateCode || null,
+        user.affiliateAttributionAt || null,
+        JSON.stringify(affiliatePayouts),
+        twoFactorEncrypted,
+        user.twoFactorEnabled ? 1 : 0,
+        user.createdAt || Date.now(),
+        user.updatedAt || user.createdAt || Date.now(),
+        user.lastLoginAt || null,
+        dataPayload
+      );
      
      console.log(`  ✓ Migrated user: ${user.email}`);
      success++;
@@ -99,8 +157,19 @@ async function migrateSessions() {
  let expired = 0;
  
  const db = getDatabase();
-  const sessionRepo = require('../src/repositories/sessionRepository');
-  
+  const userExistsStmt = db.prepare('SELECT id FROM users WHERE id = ?');
+  const sessionStmt = db.prepare(`
+    INSERT INTO sessions (
+      id, user_id, token, refresh_token_hash, device_fingerprint,
+      ip_address, user_agent, expires_at, created_at, last_accessed_at
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      user_id = excluded.user_id,
+      token = excluded.token,
+      expires_at = excluded.expires_at,
+      last_accessed_at = excluded.last_accessed_at
+  `);
+
  for (const [token, session] of Object.entries(sessions)) {
    try {
      // Skip expired sessions
@@ -110,25 +179,25 @@ async function migrateSessions() {
      }
      
      // Check if user exists
-      const user = userRepo.getUserById(session.userId);
+      const user = userExistsStmt.get(session.userId);
      if (!user) {
        console.log(`  Skipping session for non-existent user: ${session.userId}`);
        failed++;
        continue;
      }
      
-      // Create session in database
-      sessionRepo.createSession({
-        id: session.id || require('crypto').randomUUID(),
-        userId: session.userId,
-        token: token,
-        deviceFingerprint: session.deviceFingerprint || null,
-        ipAddress: session.ipAddress || null,
-        userAgent: session.userAgent || null,
-        expiresAt: session.expiresAt,
-        createdAt: session.createdAt || now,
-        lastAccessedAt: session.lastAccessedAt || now
-      });
+      sessionStmt.run(
+        token,
+        session.userId,
+        token,
+        null,
+        session.deviceFingerprint || null,
+        session.ipAddress || null,
+        session.userAgent || null,
+        session.expiresAt,
+        session.createdAt || now,
+        session.lastAccessedAt || now
+      );
      
      success++;
    } catch (error) {
@@ -156,38 +225,53 @@ async function migrateAffiliates() {
  let failed = 0;
  
  const db = getDatabase();
-  
+  const stmt = db.prepare(`
+    INSERT INTO affiliate_accounts (
+      id, email, name, password_hash, codes, earnings, commission_rate,
+      created_at, updated_at, last_login_at, last_payout_at,
+      email_verified, verification_token, verification_expires_at
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(id) DO UPDATE SET
+      email = excluded.email,
+      name = excluded.name,
+      password_hash = excluded.password_hash,
+      codes = excluded.codes,
+      earnings = excluded.earnings,
+      commission_rate = excluded.commission_rate,
+      updated_at = excluded.updated_at,
+      last_login_at = excluded.last_login_at,
+      last_payout_at = excluded.last_payout_at,
+      email_verified = excluded.email_verified,
+      verification_token = excluded.verification_token,
+      verification_expires_at = excluded.verification_expires_at
+  `);
+
  for (const affiliate of affiliates) {
    try {
-      // Check if user exists
-      const user = userRepo.getUserById(affiliate.userId);
-      if (!user) {
-        console.log(`  Skipping affiliate for non-existent user: ${affiliate.userId}`);
+      if (!affiliate.email) {
+        console.log('  Skipping affiliate without email');
        failed++;
        continue;
      }
-      
-      // Insert affiliate
-      const stmt = db.prepare(`
-        INSERT INTO affiliates (
-          id, user_id, codes, earnings, commission_rate,
-          total_referrals, total_earnings_cents, created_at, updated_at
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-      `);
-      
+      const affiliateEmail = (affiliate.email || '').trim().toLowerCase();
      stmt.run(
        affiliate.id || require('crypto').randomUUID(),
-        affiliate.userId,
+        affiliateEmail,
+        affiliate.name || '',
+        affiliate.password || affiliate.passwordHash || '',
        JSON.stringify(affiliate.codes || []),
        JSON.stringify(affiliate.earnings || []),
        affiliate.commissionRate || 0.15,
-        affiliate.totalReferrals || 0,
-        affiliate.totalEarningsCents || 0,
-        affiliate.createdAt || Date.now(),
-        affiliate.updatedAt || Date.now()
+        affiliate.createdAt || new Date().toISOString(),
+        affiliate.updatedAt || affiliate.createdAt || new Date().toISOString(),
+        affiliate.lastLoginAt || null,
+        affiliate.lastPayoutAt || null,
+        affiliate.emailVerified ? 1 : 0,
+        affiliate.verificationToken || null,
+        affiliate.verificationExpiresAt || null
      );
      
-      console.log(`  ✓ Migrated affiliate for user: ${user.email}`);
+      console.log(`  ✓ Migrated affiliate: ${affiliate.email}`);
      success++;
    } catch (error) {
      console.error(`  ✗ Failed to migrate affiliate:`, error.message);
@@ -229,7 +313,7 @@ async function createBackup() {

 async function runMigration() {
  console.log('🔄 Starting database migration...');
-  console.log('  Source: JSON files in', DATA_ROOT);
+  console.log('  Source: JSON files in', STATE_DIR);
  console.log('  Target: Database at', DATABASE_PATH);
  
  // Check if database exists
@@ -239,13 +323,21 @@ async function runMigration() {
  }
  
  // Initialize encryption
-  if (!DATABASE_ENCRYPTION_KEY) {
+  let encryptionKey = DATABASE_ENCRYPTION_KEY;
+  if (!encryptionKey && KEY_FILE && fs.existsSync(KEY_FILE)) {
+    encryptionKey = fs.readFileSync(KEY_FILE, 'utf8').trim();
+    if (encryptionKey) {
+      process.env.DATABASE_ENCRYPTION_KEY = encryptionKey;
+      console.log('✅ Loaded encryption key from file');
+    }
+  }
+  if (!encryptionKey) {
    console.error('❌ DATABASE_ENCRYPTION_KEY not set');
    process.exit(1);
  }
  
  try {
-    initEncryption(DATABASE_ENCRYPTION_KEY);
+    initEncryption(encryptionKey);
    console.log('✅ Encryption initialized');
  } catch (error) {
    console.error('❌ Failed to initialize encryption:', error.message);
@@ -254,7 +346,12 @@ async function runMigration() {
  
  // Initialize database
  try {
-    initDatabase(DATABASE_PATH, { verbose: false });
+    initDatabase(DATABASE_PATH, {
+      verbose: false,
+      sqlcipherKey: DATABASE_USE_SQLCIPHER ? encryptionKey : null,
+      cipherCompatibility: process.env.DATABASE_CIPHER_COMPAT || 4,
+      kdfIter: process.env.DATABASE_KDF_ITER || 64000
+    });
    console.log('✅ Database connected');
  } catch (error) {
    console.error('❌ Failed to connect to database:', error.message);
--- a/chat/scripts/setup-database.js
+++ b/chat/scripts/setup-database.js
@@ -8,12 +8,13 @@ const fs = require('fs');
 const path = require('path');
 const { initDatabase, getDatabase, closeDatabase } = require('../src/database/connection');
 const { initEncryption } = require('../src/utils/encryption');
+const { DB_PATH, KEY_FILE } = require('../src/database/config');
 const crypto = require('crypto');

-const DATA_ROOT = process.env.CHAT_DATA_ROOT || path.join(__dirname, '..', '.data');
-const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
+const DATABASE_PATH = DB_PATH;
 const DATABASE_ENCRYPTION_KEY = process.env.DATABASE_ENCRYPTION_KEY;
 const WAL_MODE = process.env.DATABASE_WAL_MODE !== '0' && process.env.DATABASE_WAL_MODE !== 'false';
+const DATABASE_USE_SQLCIPHER = process.env.DATABASE_USE_SQLCIPHER !== '0' && process.env.DATABASE_USE_SQLCIPHER !== 'false';

 async function setupDatabase() {
  console.log('🔧 Setting up database...');
@@ -26,14 +27,35 @@ async function setupDatabase() {
    console.log('  Created data directory:', dataDir);
  }
  
+  let encryptionKey = DATABASE_ENCRYPTION_KEY;
+
  // Check if encryption key is provided
-  if (!DATABASE_ENCRYPTION_KEY) {
+  if (!encryptionKey) {
+    if (fs.existsSync(KEY_FILE)) {
+      encryptionKey = fs.readFileSync(KEY_FILE, 'utf8').trim();
+      if (encryptionKey) {
+        process.env.DATABASE_ENCRYPTION_KEY = encryptionKey;
+        console.log('✅ Loaded encryption key from file');
+      }
+    }
+  }
+
+  if (!encryptionKey) {
    console.warn('⚠️  WARNING: No DATABASE_ENCRYPTION_KEY found!');
    console.warn('⚠️  Generating a random key for this session (not persistent).');
    console.warn('⚠️  For production, set DATABASE_ENCRYPTION_KEY environment variable.');
    console.warn('⚠️  Generate one with: openssl rand -hex 32');
    const generatedKey = crypto.randomBytes(32).toString('hex');
    process.env.DATABASE_ENCRYPTION_KEY = generatedKey;
+    encryptionKey = generatedKey;
+    try {
+      const keyDir = path.dirname(KEY_FILE);
+      if (!fs.existsSync(keyDir)) fs.mkdirSync(keyDir, { recursive: true });
+      fs.writeFileSync(KEY_FILE, generatedKey, { mode: 0o600 });
+      console.log('⚠️  Saved generated key to:', KEY_FILE);
+    } catch (err) {
+      console.warn('⚠️  Failed to persist encryption key:', err.message);
+    }
    console.log('✅ Generated temporary encryption key');
  } else {
    console.log('✅ Using encryption key from environment');
@@ -41,7 +63,7 @@ async function setupDatabase() {
  
  // Initialize encryption
  try {
-    initEncryption(process.env.DATABASE_ENCRYPTION_KEY);
+    initEncryption(encryptionKey);
    console.log('✅ Encryption initialized');
  } catch (error) {
    console.error('❌ Failed to initialize encryption:', error.message);
@@ -52,7 +74,10 @@ async function setupDatabase() {
  try {
    initDatabase(DATABASE_PATH, {
      verbose: false,
-      walMode: WAL_MODE
+      walMode: WAL_MODE,
+      sqlcipherKey: DATABASE_USE_SQLCIPHER ? encryptionKey : null,
+      cipherCompatibility: process.env.DATABASE_CIPHER_COMPAT || 4,
+      kdfIter: process.env.DATABASE_KDF_ITER || 64000
    });
    console.log('✅ Database initialized');
  } catch (error) {
@@ -70,7 +95,15 @@ async function setupDatabase() {
    // Execute the entire schema as one block
    // SQLite can handle multiple statements with exec()
    db.exec(schema);
-    
+
+    // Add missing columns if this is an upgraded database
+    const userColumns = db.prepare('PRAGMA table_info(users)').all();
+    const userColumnNames = new Set(userColumns.map((c) => c.name));
+    if (!userColumnNames.has('data')) {
+      db.exec('ALTER TABLE users ADD COLUMN data TEXT');
+      console.log('✅ Added users.data column');
+    }
+
    console.log('✅ Database schema created');
  } catch (error) {
    console.error('❌ Failed to create schema:', error.message);
--- a/chat/scripts/verify-migration.js
+++ b/chat/scripts/verify-migration.js
@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+/**
+ * Verify JSON -> Database migration counts
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { initDatabase, getDatabase, closeDatabase } = require('../src/database/connection');
+const { initEncryption } = require('../src/utils/encryption');
+const { STATE_DIR, DB_PATH, KEY_FILE } = require('../src/database/config');
+
+function loadJson(filePath, fallback) {
+  try {
+    const raw = fs.readFileSync(filePath, 'utf8');
+    return JSON.parse(raw || JSON.stringify(fallback));
+  } catch (_) {
+    return fallback;
+  }
+}
+
+function resolveKey() {
+  if (process.env.DATABASE_ENCRYPTION_KEY) return process.env.DATABASE_ENCRYPTION_KEY.trim();
+  if (KEY_FILE && fs.existsSync(KEY_FILE)) {
+    const key = fs.readFileSync(KEY_FILE, 'utf8').trim();
+    if (key) {
+      process.env.DATABASE_ENCRYPTION_KEY = key;
+      return key;
+    }
+  }
+  return '';
+}
+
+function main() {
+  const key = resolveKey();
+  if (!key) {
+    console.error('❌ DATABASE_ENCRYPTION_KEY not set (and no key file found).');
+    process.exit(1);
+  }
+
+  initEncryption(key);
+  const useSqlcipher = process.env.DATABASE_USE_SQLCIPHER !== '0' && process.env.DATABASE_USE_SQLCIPHER !== 'false';
+  initDatabase(DB_PATH, {
+    sqlcipherKey: useSqlcipher ? key : null,
+    cipherCompatibility: process.env.DATABASE_CIPHER_COMPAT || 4,
+    kdfIter: process.env.DATABASE_KDF_ITER || 64000
+  });
+
+  const db = getDatabase();
+  const now = Date.now();
+
+  const usersJson = loadJson(path.join(STATE_DIR, 'users.json'), []);
+  const sessionsJson = loadJson(path.join(STATE_DIR, 'user-sessions.json'), {});
+  const affiliatesJson = loadJson(path.join(STATE_DIR, 'affiliates.json'), []);
+
+  const usersDb = db.prepare('SELECT COUNT(*) as count FROM users').get().count;
+  const sessionsDb = db.prepare('SELECT COUNT(*) as count FROM sessions WHERE expires_at > ?').get(now).count;
+  const affiliatesDb = db.prepare('SELECT COUNT(*) as count FROM affiliate_accounts').get().count;
+
+  const activeJsonSessions = Object.values(sessionsJson || {}).filter((session) => session?.expiresAt && session.expiresAt > now).length;
+
+  console.log('Migration verification:');
+  console.log(`  Users: JSON=${Array.isArray(usersJson) ? usersJson.length : 0} DB=${usersDb}`);
+  console.log(`  Sessions (active): JSON=${activeJsonSessions} DB=${sessionsDb}`);
+  console.log(`  Affiliates: JSON=${Array.isArray(affiliatesJson) ? affiliatesJson.length : 0} DB=${affiliatesDb}`);
+
+  closeDatabase();
+}
+
+main();