Implement Phase 1.2: Database with encryption at rest and core infrastructure
Co-authored-by: southseact-3d <217551146+southseact-3d@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
114
chat/scripts/init-database.js
Executable file
114
chat/scripts/init-database.js
Executable file
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Database initialization script for container startup
|
||||
* Automatically sets up database on first run or when database doesn't exist
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const DATA_ROOT = process.env.CHAT_DATA_ROOT || '/home/web/data/.data';
|
||||
const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
|
||||
const USE_JSON_DATABASE = process.env.USE_JSON_DATABASE === '1' || process.env.USE_JSON_DATABASE === 'true';
|
||||
|
||||
async function initializeDatabase() {
|
||||
// Skip if using JSON mode
|
||||
if (USE_JSON_DATABASE) {
|
||||
console.log('📁 Using JSON database mode (backward compatibility)');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('🔍 Checking database status...');
|
||||
|
||||
// Ensure data directory exists
|
||||
const dataDir = path.dirname(DATABASE_PATH);
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
console.log('📁 Creating data directory:', dataDir);
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Check if database exists
|
||||
const dbExists = fs.existsSync(DATABASE_PATH);
|
||||
|
||||
if (dbExists) {
|
||||
console.log('✅ Database already exists:', DATABASE_PATH);
|
||||
|
||||
// Verify encryption key is set
|
||||
if (!process.env.DATABASE_ENCRYPTION_KEY) {
|
||||
console.error('❌ DATABASE_ENCRYPTION_KEY not set!');
|
||||
console.error(' Database exists but encryption key is missing.');
|
||||
console.error(' Set DATABASE_ENCRYPTION_KEY to the key used when creating the database.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('🔧 Database not found, setting up new database...');
|
||||
|
||||
// Generate encryption key if not provided
|
||||
if (!process.env.DATABASE_ENCRYPTION_KEY) {
|
||||
const generatedKey = crypto.randomBytes(32).toString('hex');
|
||||
process.env.DATABASE_ENCRYPTION_KEY = generatedKey;
|
||||
|
||||
console.log('⚠️ Generated new encryption key (save this!)');
|
||||
console.log('⚠️ DATABASE_ENCRYPTION_KEY=' + generatedKey);
|
||||
console.log('⚠️ Add this to your environment configuration to persist it!');
|
||||
|
||||
// Save to a file for persistence
|
||||
const keyFile = path.join(dataDir, '.encryption_key');
|
||||
fs.writeFileSync(keyFile, generatedKey, { mode: 0o600 });
|
||||
console.log('⚠️ Saved to:', keyFile);
|
||||
}
|
||||
|
||||
// Generate JWT secret if not provided
|
||||
if (!process.env.JWT_SECRET && !process.env.SESSION_SECRET) {
|
||||
const jwtSecret = crypto.randomBytes(32).toString('hex');
|
||||
process.env.JWT_SECRET = jwtSecret;
|
||||
|
||||
console.log('⚠️ Generated new JWT secret (save this!)');
|
||||
console.log('⚠️ JWT_SECRET=' + jwtSecret);
|
||||
|
||||
// Save to a file for persistence
|
||||
const jwtFile = path.join(dataDir, '.jwt_secret');
|
||||
fs.writeFileSync(jwtFile, jwtSecret, { mode: 0o600 });
|
||||
console.log('⚠️ Saved to:', jwtFile);
|
||||
}
|
||||
|
||||
// Run setup script
|
||||
try {
|
||||
const setupScript = require('./setup-database.js');
|
||||
console.log('✅ Database setup complete');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to setup database:', error.message);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Check if there are JSON files to migrate
|
||||
const usersFile = path.join(DATA_ROOT, 'users.json');
|
||||
const sessionsFile = path.join(DATA_ROOT, 'user-sessions.json');
|
||||
|
||||
const hasJsonData = fs.existsSync(usersFile) || fs.existsSync(sessionsFile);
|
||||
|
||||
if (hasJsonData) {
|
||||
console.log('📦 Found existing JSON data files');
|
||||
console.log(' To migrate data, run: node scripts/migrate-to-database.js');
|
||||
console.log(' Or set USE_JSON_DATABASE=1 to continue using JSON files');
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-initialize if called directly
|
||||
if (require.main === module) {
|
||||
initializeDatabase()
|
||||
.then(() => {
|
||||
console.log('✅ Database initialization complete');
|
||||
process.exit(0);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('❌ Database initialization failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { initializeDatabase };
|
||||
310
chat/scripts/migrate-to-database.js
Executable file
310
chat/scripts/migrate-to-database.js
Executable file
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Migration script - Migrate data from JSON files to database
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { initDatabase, getDatabase, closeDatabase } = require('../src/database/connection');
|
||||
const { initEncryption } = require('../src/utils/encryption');
|
||||
const userRepo = require('../src/repositories/userRepository');
|
||||
|
||||
const DATA_ROOT = process.env.CHAT_DATA_ROOT || path.join(__dirname, '..', '.data');
|
||||
const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
|
||||
const DATABASE_ENCRYPTION_KEY = process.env.DATABASE_ENCRYPTION_KEY;
|
||||
|
||||
const USERS_FILE = path.join(DATA_ROOT, 'users.json');
|
||||
const SESSIONS_FILE = path.join(DATA_ROOT, 'user-sessions.json');
|
||||
const AFFILIATES_FILE = path.join(DATA_ROOT, 'affiliates.json');
|
||||
|
||||
async function loadJsonFile(filePath, defaultValue = []) {
|
||||
try {
|
||||
const data = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(data);
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.log(` File not found: ${filePath}, using default`);
|
||||
return defaultValue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateUsers() {
|
||||
console.log('\n📦 Migrating users...');
|
||||
|
||||
const users = await loadJsonFile(USERS_FILE, []);
|
||||
console.log(` Found ${users.length} users in JSON`);
|
||||
|
||||
if (users.length === 0) {
|
||||
console.log(' No users to migrate');
|
||||
return { success: 0, failed: 0 };
|
||||
}
|
||||
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const user of users) {
|
||||
try {
|
||||
// Check if user already exists
|
||||
const existing = userRepo.getUserById(user.id);
|
||||
if (existing) {
|
||||
console.log(` Skipping existing user: ${user.email}`);
|
||||
success++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create user in database
|
||||
userRepo.createUser({
|
||||
id: user.id,
|
||||
email: user.email,
|
||||
name: user.name || null,
|
||||
passwordHash: user.passwordHash || user.password_hash,
|
||||
providers: user.providers || [],
|
||||
emailVerified: user.emailVerified,
|
||||
verificationToken: user.verificationToken || null,
|
||||
verificationExpiresAt: user.verificationExpiresAt || null,
|
||||
plan: user.plan || 'hobby',
|
||||
billingStatus: user.billingStatus || 'active',
|
||||
billingEmail: user.billingEmail || user.email
|
||||
});
|
||||
|
||||
console.log(` ✓ Migrated user: ${user.email}`);
|
||||
success++;
|
||||
} catch (error) {
|
||||
console.error(` ✗ Failed to migrate user ${user.email}:`, error.message);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Completed: ${success} success, ${failed} failed`);
|
||||
return { success, failed };
|
||||
}
|
||||
|
||||
async function migrateSessions() {
|
||||
console.log('\n📦 Migrating sessions...');
|
||||
|
||||
const sessions = await loadJsonFile(SESSIONS_FILE, {});
|
||||
const sessionCount = Object.keys(sessions).length;
|
||||
console.log(` Found ${sessionCount} sessions in JSON`);
|
||||
|
||||
if (sessionCount === 0) {
|
||||
console.log(' No sessions to migrate');
|
||||
return { success: 0, failed: 0, expired: 0 };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
let expired = 0;
|
||||
|
||||
const db = getDatabase();
|
||||
const sessionRepo = require('../src/repositories/sessionRepository');
|
||||
|
||||
for (const [token, session] of Object.entries(sessions)) {
|
||||
try {
|
||||
// Skip expired sessions
|
||||
if (session.expiresAt && session.expiresAt <= now) {
|
||||
expired++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if user exists
|
||||
const user = userRepo.getUserById(session.userId);
|
||||
if (!user) {
|
||||
console.log(` Skipping session for non-existent user: ${session.userId}`);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create session in database
|
||||
sessionRepo.createSession({
|
||||
id: session.id || require('crypto').randomUUID(),
|
||||
userId: session.userId,
|
||||
token: token,
|
||||
deviceFingerprint: session.deviceFingerprint || null,
|
||||
ipAddress: session.ipAddress || null,
|
||||
userAgent: session.userAgent || null,
|
||||
expiresAt: session.expiresAt,
|
||||
createdAt: session.createdAt || now,
|
||||
lastAccessedAt: session.lastAccessedAt || now
|
||||
});
|
||||
|
||||
success++;
|
||||
} catch (error) {
|
||||
console.error(` ✗ Failed to migrate session:`, error.message);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Completed: ${success} success, ${failed} failed, ${expired} expired`);
|
||||
return { success, failed, expired };
|
||||
}
|
||||
|
||||
async function migrateAffiliates() {
|
||||
console.log('\n📦 Migrating affiliates...');
|
||||
|
||||
const affiliates = await loadJsonFile(AFFILIATES_FILE, []);
|
||||
console.log(` Found ${affiliates.length} affiliates in JSON`);
|
||||
|
||||
if (affiliates.length === 0) {
|
||||
console.log(' No affiliates to migrate');
|
||||
return { success: 0, failed: 0 };
|
||||
}
|
||||
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
|
||||
const db = getDatabase();
|
||||
|
||||
for (const affiliate of affiliates) {
|
||||
try {
|
||||
// Check if user exists
|
||||
const user = userRepo.getUserById(affiliate.userId);
|
||||
if (!user) {
|
||||
console.log(` Skipping affiliate for non-existent user: ${affiliate.userId}`);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Insert affiliate
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO affiliates (
|
||||
id, user_id, codes, earnings, commission_rate,
|
||||
total_referrals, total_earnings_cents, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
stmt.run(
|
||||
affiliate.id || require('crypto').randomUUID(),
|
||||
affiliate.userId,
|
||||
JSON.stringify(affiliate.codes || []),
|
||||
JSON.stringify(affiliate.earnings || []),
|
||||
affiliate.commissionRate || 0.15,
|
||||
affiliate.totalReferrals || 0,
|
||||
affiliate.totalEarningsCents || 0,
|
||||
affiliate.createdAt || Date.now(),
|
||||
affiliate.updatedAt || Date.now()
|
||||
);
|
||||
|
||||
console.log(` ✓ Migrated affiliate for user: ${user.email}`);
|
||||
success++;
|
||||
} catch (error) {
|
||||
console.error(` ✗ Failed to migrate affiliate:`, error.message);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Completed: ${success} success, ${failed} failed`);
|
||||
return { success, failed };
|
||||
}
|
||||
|
||||
async function createBackup() {
|
||||
console.log('\n💾 Creating backup of JSON files...');
|
||||
|
||||
const backupDir = path.join(DATA_ROOT, `migration_backup_${Date.now()}`);
|
||||
|
||||
if (!fs.existsSync(backupDir)) {
|
||||
fs.mkdirSync(backupDir, { recursive: true });
|
||||
}
|
||||
|
||||
const files = [USERS_FILE, SESSIONS_FILE, AFFILIATES_FILE];
|
||||
let backedUp = 0;
|
||||
|
||||
for (const file of files) {
|
||||
if (fs.existsSync(file)) {
|
||||
const fileName = path.basename(file);
|
||||
const backupPath = path.join(backupDir, fileName);
|
||||
fs.copyFileSync(file, backupPath);
|
||||
console.log(` ✓ Backed up: ${fileName}`);
|
||||
backedUp++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Created backup in: ${backupDir}`);
|
||||
console.log(` Backed up ${backedUp} files`);
|
||||
|
||||
return backupDir;
|
||||
}
|
||||
|
||||
async function runMigration() {
|
||||
console.log('🔄 Starting database migration...');
|
||||
console.log(' Source: JSON files in', DATA_ROOT);
|
||||
console.log(' Target: Database at', DATABASE_PATH);
|
||||
|
||||
// Check if database exists
|
||||
if (!fs.existsSync(DATABASE_PATH)) {
|
||||
console.error('❌ Database not found. Please run setup-database.js first.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Initialize encryption
|
||||
if (!DATABASE_ENCRYPTION_KEY) {
|
||||
console.error('❌ DATABASE_ENCRYPTION_KEY not set');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
initEncryption(DATABASE_ENCRYPTION_KEY);
|
||||
console.log('✅ Encryption initialized');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to initialize encryption:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Initialize database
|
||||
try {
|
||||
initDatabase(DATABASE_PATH, { verbose: false });
|
||||
console.log('✅ Database connected');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to connect to database:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Create backup
|
||||
const backupDir = await createBackup();
|
||||
|
||||
// Run migrations
|
||||
const results = {
|
||||
users: await migrateUsers(),
|
||||
sessions: await migrateSessions(),
|
||||
affiliates: await migrateAffiliates()
|
||||
};
|
||||
|
||||
// Close database
|
||||
closeDatabase();
|
||||
|
||||
// Print summary
|
||||
console.log('\n📊 Migration Summary:');
|
||||
console.log(' Users:');
|
||||
console.log(` ✓ Success: ${results.users.success}`);
|
||||
console.log(` ✗ Failed: ${results.users.failed}`);
|
||||
console.log(' Sessions:');
|
||||
console.log(` ✓ Success: ${results.sessions.success}`);
|
||||
console.log(` ✗ Failed: ${results.sessions.failed}`);
|
||||
console.log(` ⏰ Expired: ${results.sessions.expired}`);
|
||||
console.log(' Affiliates:');
|
||||
console.log(` ✓ Success: ${results.affiliates.success}`);
|
||||
console.log(` ✗ Failed: ${results.affiliates.failed}`);
|
||||
|
||||
const totalSuccess = results.users.success + results.sessions.success + results.affiliates.success;
|
||||
const totalFailed = results.users.failed + results.sessions.failed + results.affiliates.failed;
|
||||
|
||||
console.log('\n Total:');
|
||||
console.log(` ✓ Success: ${totalSuccess}`);
|
||||
console.log(` ✗ Failed: ${totalFailed}`);
|
||||
|
||||
console.log('\n✅ Migration complete!');
|
||||
console.log(` Backup created in: ${backupDir}`);
|
||||
console.log('\nNext steps:');
|
||||
console.log(' 1. Verify migration: node scripts/verify-migration.js');
|
||||
console.log(' 2. Test the application with: USE_JSON_DATABASE=1 npm start');
|
||||
console.log(' 3. Switch to database mode: unset USE_JSON_DATABASE && npm start');
|
||||
}
|
||||
|
||||
// Run migration
|
||||
runMigration().catch(error => {
|
||||
console.error('❌ Migration failed:', error);
|
||||
closeDatabase();
|
||||
process.exit(1);
|
||||
});
|
||||
122
chat/scripts/setup-database.js
Executable file
122
chat/scripts/setup-database.js
Executable file
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Setup database script
|
||||
* Initializes the SQLite database with the schema
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { initDatabase, getDatabase, closeDatabase } = require('../src/database/connection');
|
||||
const { initEncryption } = require('../src/utils/encryption');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const DATA_ROOT = process.env.CHAT_DATA_ROOT || path.join(__dirname, '..', '.data');
|
||||
const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
|
||||
const DATABASE_ENCRYPTION_KEY = process.env.DATABASE_ENCRYPTION_KEY;
|
||||
const WAL_MODE = process.env.DATABASE_WAL_MODE !== '0' && process.env.DATABASE_WAL_MODE !== 'false';
|
||||
|
||||
async function setupDatabase() {
|
||||
console.log('🔧 Setting up database...');
|
||||
console.log(' Database path:', DATABASE_PATH);
|
||||
|
||||
// Ensure data directory exists
|
||||
const dataDir = path.dirname(DATABASE_PATH);
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
console.log(' Created data directory:', dataDir);
|
||||
}
|
||||
|
||||
// Check if encryption key is provided
|
||||
if (!DATABASE_ENCRYPTION_KEY) {
|
||||
console.warn('⚠️ WARNING: No DATABASE_ENCRYPTION_KEY found!');
|
||||
console.warn('⚠️ Generating a random key for this session (not persistent).');
|
||||
console.warn('⚠️ For production, set DATABASE_ENCRYPTION_KEY environment variable.');
|
||||
console.warn('⚠️ Generate one with: openssl rand -hex 32');
|
||||
const generatedKey = crypto.randomBytes(32).toString('hex');
|
||||
process.env.DATABASE_ENCRYPTION_KEY = generatedKey;
|
||||
console.log('✅ Generated temporary encryption key');
|
||||
} else {
|
||||
console.log('✅ Using encryption key from environment');
|
||||
}
|
||||
|
||||
// Initialize encryption
|
||||
try {
|
||||
initEncryption(process.env.DATABASE_ENCRYPTION_KEY);
|
||||
console.log('✅ Encryption initialized');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to initialize encryption:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Initialize database
|
||||
try {
|
||||
initDatabase(DATABASE_PATH, {
|
||||
verbose: false,
|
||||
walMode: WAL_MODE
|
||||
});
|
||||
console.log('✅ Database initialized');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to initialize database:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Load and execute schema
|
||||
try {
|
||||
const schemaPath = path.join(__dirname, '..', 'src', 'database', 'schema.sql');
|
||||
const schema = fs.readFileSync(schemaPath, 'utf8');
|
||||
|
||||
const db = getDatabase();
|
||||
|
||||
// Split by semicolon and execute each statement
|
||||
const statements = schema
|
||||
.split(';')
|
||||
.map(s => s.trim())
|
||||
.filter(s => s.length > 0 && !s.startsWith('--'));
|
||||
|
||||
for (const statement of statements) {
|
||||
db.exec(statement);
|
||||
}
|
||||
|
||||
console.log('✅ Database schema created');
|
||||
console.log(` Executed ${statements.length} SQL statements`);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to create schema:', error.message);
|
||||
closeDatabase();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Verify tables
|
||||
try {
|
||||
const db = getDatabase();
|
||||
const tables = db.prepare(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type='table' AND name NOT LIKE 'sqlite_%'
|
||||
ORDER BY name
|
||||
`).all();
|
||||
|
||||
console.log('✅ Database tables created:');
|
||||
tables.forEach(table => {
|
||||
console.log(` - ${table.name}`);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to verify tables:', error.message);
|
||||
}
|
||||
|
||||
// Close database
|
||||
closeDatabase();
|
||||
|
||||
console.log('');
|
||||
console.log('✅ Database setup complete!');
|
||||
console.log('');
|
||||
console.log('Next steps:');
|
||||
console.log(' 1. Run migration: node scripts/migrate-to-database.js');
|
||||
console.log(' 2. Verify migration: node scripts/verify-migration.js');
|
||||
console.log(' 3. Switch to database mode: unset USE_JSON_DATABASE');
|
||||
console.log(' 4. Start server: npm start');
|
||||
}
|
||||
|
||||
// Run setup
|
||||
setupDatabase().catch(error => {
|
||||
console.error('❌ Setup failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user