Add database migration scripts and configuration files

- Add verify-migration.js script for testing database migrations
- Add database config module for centralized configuration
- Add chutes.txt prompt for system responses
- Update database implementation and testing documentation
- Add database migration and setup scripts
- Update session system and LLM tool configuration
- Update deployment checklist and environment example
- Update Dockerfile and docker-compose configuration

chat/scripts/setup-database.js

@@ -8,12 +8,13 @@ const fs = require('fs');
 const path = require('path');
 const { initDatabase, getDatabase, closeDatabase } = require('../src/database/connection');
 const { initEncryption } = require('../src/utils/encryption');
+const { DB_PATH, KEY_FILE } = require('../src/database/config');
 const crypto = require('crypto');
 
-const DATA_ROOT = process.env.CHAT_DATA_ROOT || path.join(__dirname, '..', '.data');
-const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
+const DATABASE_PATH = DB_PATH;
 const DATABASE_ENCRYPTION_KEY = process.env.DATABASE_ENCRYPTION_KEY;
 const WAL_MODE = process.env.DATABASE_WAL_MODE !== '0' && process.env.DATABASE_WAL_MODE !== 'false';
+const DATABASE_USE_SQLCIPHER = process.env.DATABASE_USE_SQLCIPHER !== '0' && process.env.DATABASE_USE_SQLCIPHER !== 'false';
 
 async function setupDatabase() {
   console.log('🔧 Setting up database...');
@@ -26,14 +27,35 @@ async function setupDatabase() {
     console.log('  Created data directory:', dataDir);
   }
   
+  let encryptionKey = DATABASE_ENCRYPTION_KEY;
+
   // Check if encryption key is provided
-  if (!DATABASE_ENCRYPTION_KEY) {
+  if (!encryptionKey) {
+    if (fs.existsSync(KEY_FILE)) {
+      encryptionKey = fs.readFileSync(KEY_FILE, 'utf8').trim();
+      if (encryptionKey) {
+        process.env.DATABASE_ENCRYPTION_KEY = encryptionKey;
+        console.log('✅ Loaded encryption key from file');
+      }
+    }
+  }
+
+  if (!encryptionKey) {
     console.warn('⚠️  WARNING: No DATABASE_ENCRYPTION_KEY found!');
     console.warn('⚠️  Generating a random key for this session (not persistent).');
     console.warn('⚠️  For production, set DATABASE_ENCRYPTION_KEY environment variable.');
     console.warn('⚠️  Generate one with: openssl rand -hex 32');
     const generatedKey = crypto.randomBytes(32).toString('hex');
     process.env.DATABASE_ENCRYPTION_KEY = generatedKey;
+    encryptionKey = generatedKey;
+    try {
+      const keyDir = path.dirname(KEY_FILE);
+      if (!fs.existsSync(keyDir)) fs.mkdirSync(keyDir, { recursive: true });
+      fs.writeFileSync(KEY_FILE, generatedKey, { mode: 0o600 });
+      console.log('⚠️  Saved generated key to:', KEY_FILE);
+    } catch (err) {
+      console.warn('⚠️  Failed to persist encryption key:', err.message);
+    }
     console.log('✅ Generated temporary encryption key');
   } else {
     console.log('✅ Using encryption key from environment');
@@ -41,7 +63,7 @@ async function setupDatabase() {
   
   // Initialize encryption
   try {
-    initEncryption(process.env.DATABASE_ENCRYPTION_KEY);
+    initEncryption(encryptionKey);
     console.log('✅ Encryption initialized');
   } catch (error) {
     console.error('❌ Failed to initialize encryption:', error.message);
@@ -52,7 +74,10 @@ async function setupDatabase() {
   try {
     initDatabase(DATABASE_PATH, {
       verbose: false,
-      walMode: WAL_MODE
+      walMode: WAL_MODE,
+      sqlcipherKey: DATABASE_USE_SQLCIPHER ? encryptionKey : null,
+      cipherCompatibility: process.env.DATABASE_CIPHER_COMPAT || 4,
+      kdfIter: process.env.DATABASE_KDF_ITER || 64000
     });
     console.log('✅ Database initialized');
   } catch (error) {
@@ -70,7 +95,15 @@ async function setupDatabase() {
     // Execute the entire schema as one block
     // SQLite can handle multiple statements with exec()
     db.exec(schema);
-    
+
+    // Add missing columns if this is an upgraded database
+    const userColumns = db.prepare('PRAGMA table_info(users)').all();
+    const userColumnNames = new Set(userColumns.map((c) => c.name));
+    if (!userColumnNames.has('data')) {
+      db.exec('ALTER TABLE users ADD COLUMN data TEXT');
+      console.log('✅ Added users.data column');
+    }
+
     console.log('✅ Database schema created');
   } catch (error) {
     console.error('❌ Failed to create schema:', error.message);