Add database migration scripts and configuration files
- Add verify-migration.js script for testing database migrations - Add database config module for centralized configuration - Add chutes.txt prompt for system responses - Update database implementation and testing documentation - Add database migration and setup scripts - Update session system and LLM tool configuration - Update deployment checklist and environment example - Update Dockerfile and docker-compose configuration
This commit is contained in:
southseact-3d
@@ -6,16 +6,16 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { initDatabase, getDatabase, closeDatabase } = require('../src/database/connection');
|
||||
const { initEncryption } = require('../src/utils/encryption');
|
||||
const userRepo = require('../src/repositories/userRepository');
|
||||
const { initEncryption, encrypt } = require('../src/utils/encryption');
|
||||
const { STATE_DIR, DB_PATH, KEY_FILE } = require('../src/database/config');
|
||||
|
||||
const DATA_ROOT = process.env.CHAT_DATA_ROOT || path.join(__dirname, '..', '.data');
|
||||
const DATABASE_PATH = process.env.DATABASE_PATH || path.join(DATA_ROOT, 'shopify_ai.db');
|
||||
const DATABASE_PATH = DB_PATH;
|
||||
const DATABASE_ENCRYPTION_KEY = process.env.DATABASE_ENCRYPTION_KEY;
|
||||
const DATABASE_USE_SQLCIPHER = process.env.DATABASE_USE_SQLCIPHER !== '0' && process.env.DATABASE_USE_SQLCIPHER !== 'false';
|
||||
|
||||
const USERS_FILE = path.join(DATA_ROOT, 'users.json');
|
||||
const SESSIONS_FILE = path.join(DATA_ROOT, 'user-sessions.json');
|
||||
const AFFILIATES_FILE = path.join(DATA_ROOT, 'affiliates.json');
|
||||
const USERS_FILE = path.join(STATE_DIR, 'users.json');
|
||||
const SESSIONS_FILE = path.join(STATE_DIR, 'user-sessions.json');
|
||||
const AFFILIATES_FILE = path.join(STATE_DIR, 'affiliates.json');
|
||||
|
||||
async function loadJsonFile(filePath, defaultValue = []) {
|
||||
try {
|
||||
@@ -44,30 +44,88 @@ async function migrateUsers() {
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO users (
|
||||
id, email, email_encrypted, name, name_encrypted, password_hash,
|
||||
providers, email_verified, verification_token, verification_expires_at,
|
||||
reset_token, reset_expires_at, plan, billing_status, billing_email,
|
||||
payment_method_last4, subscription_renews_at, referred_by_affiliate_code,
|
||||
affiliate_attribution_at, affiliate_payouts, two_factor_secret,
|
||||
two_factor_enabled, created_at, updated_at, last_login_at, data
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
email = excluded.email,
|
||||
email_encrypted = excluded.email_encrypted,
|
||||
name = excluded.name,
|
||||
name_encrypted = excluded.name_encrypted,
|
||||
password_hash = excluded.password_hash,
|
||||
providers = excluded.providers,
|
||||
email_verified = excluded.email_verified,
|
||||
verification_token = excluded.verification_token,
|
||||
verification_expires_at = excluded.verification_expires_at,
|
||||
reset_token = excluded.reset_token,
|
||||
reset_expires_at = excluded.reset_expires_at,
|
||||
plan = excluded.plan,
|
||||
billing_status = excluded.billing_status,
|
||||
billing_email = excluded.billing_email,
|
||||
payment_method_last4 = excluded.payment_method_last4,
|
||||
subscription_renews_at = excluded.subscription_renews_at,
|
||||
referred_by_affiliate_code = excluded.referred_by_affiliate_code,
|
||||
affiliate_attribution_at = excluded.affiliate_attribution_at,
|
||||
affiliate_payouts = excluded.affiliate_payouts,
|
||||
two_factor_secret = excluded.two_factor_secret,
|
||||
two_factor_enabled = excluded.two_factor_enabled,
|
||||
created_at = excluded.created_at,
|
||||
updated_at = excluded.updated_at,
|
||||
last_login_at = excluded.last_login_at,
|
||||
data = excluded.data
|
||||
`);
|
||||
|
||||
for (const user of users) {
|
||||
try {
|
||||
// Check if user already exists
|
||||
const existing = userRepo.getUserById(user.id);
|
||||
if (existing) {
|
||||
console.log(` Skipping existing user: ${user.email}`);
|
||||
success++;
|
||||
const normalizedEmail = (user.email || '').trim().toLowerCase();
|
||||
if (!normalizedEmail) {
|
||||
console.log(' Skipping user without email');
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create user in database
|
||||
userRepo.createUser({
|
||||
id: user.id,
|
||||
email: user.email,
|
||||
name: user.name || null,
|
||||
passwordHash: user.passwordHash || user.password_hash,
|
||||
providers: user.providers || [],
|
||||
emailVerified: user.emailVerified,
|
||||
verificationToken: user.verificationToken || null,
|
||||
verificationExpiresAt: user.verificationExpiresAt || null,
|
||||
plan: user.plan || 'hobby',
|
||||
billingStatus: user.billingStatus || 'active',
|
||||
billingEmail: user.billingEmail || user.email
|
||||
});
|
||||
const passwordHash = user.passwordHash || user.password_hash || user.password || '';
|
||||
const providers = Array.isArray(user.providers) ? user.providers : [];
|
||||
const affiliatePayouts = Array.isArray(user.affiliatePayouts) ? user.affiliatePayouts : [];
|
||||
const dataPayload = JSON.stringify({ ...user, email: normalizedEmail || user.email || '' });
|
||||
const emailEncrypted = normalizedEmail ? encrypt(normalizedEmail) : '';
|
||||
const nameEncrypted = user.name ? encrypt(user.name) : null;
|
||||
const twoFactorEncrypted = user.twoFactorSecret ? encrypt(user.twoFactorSecret) : null;
|
||||
|
||||
stmt.run(
|
||||
user.id,
|
||||
normalizedEmail,
|
||||
emailEncrypted,
|
||||
user.name || null,
|
||||
nameEncrypted,
|
||||
passwordHash,
|
||||
JSON.stringify(providers),
|
||||
user.emailVerified ? 1 : 0,
|
||||
user.verificationToken || null,
|
||||
user.verificationExpiresAt || null,
|
||||
user.resetToken || null,
|
||||
user.resetExpiresAt || null,
|
||||
user.plan || 'hobby',
|
||||
user.billingStatus || 'active',
|
||||
user.billingEmail || user.email || '',
|
||||
user.paymentMethodLast4 || '',
|
||||
user.subscriptionRenewsAt || null,
|
||||
user.referredByAffiliateCode || null,
|
||||
user.affiliateAttributionAt || null,
|
||||
JSON.stringify(affiliatePayouts),
|
||||
twoFactorEncrypted,
|
||||
user.twoFactorEnabled ? 1 : 0,
|
||||
user.createdAt || Date.now(),
|
||||
user.updatedAt || user.createdAt || Date.now(),
|
||||
user.lastLoginAt || null,
|
||||
dataPayload
|
||||
);
|
||||
|
||||
console.log(` ✓ Migrated user: ${user.email}`);
|
||||
success++;
|
||||
@@ -99,8 +157,19 @@ async function migrateSessions() {
|
||||
let expired = 0;
|
||||
|
||||
const db = getDatabase();
|
||||
const sessionRepo = require('../src/repositories/sessionRepository');
|
||||
|
||||
const userExistsStmt = db.prepare('SELECT id FROM users WHERE id = ?');
|
||||
const sessionStmt = db.prepare(`
|
||||
INSERT INTO sessions (
|
||||
id, user_id, token, refresh_token_hash, device_fingerprint,
|
||||
ip_address, user_agent, expires_at, created_at, last_accessed_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
user_id = excluded.user_id,
|
||||
token = excluded.token,
|
||||
expires_at = excluded.expires_at,
|
||||
last_accessed_at = excluded.last_accessed_at
|
||||
`);
|
||||
|
||||
for (const [token, session] of Object.entries(sessions)) {
|
||||
try {
|
||||
// Skip expired sessions
|
||||
@@ -110,25 +179,25 @@ async function migrateSessions() {
|
||||
}
|
||||
|
||||
// Check if user exists
|
||||
const user = userRepo.getUserById(session.userId);
|
||||
const user = userExistsStmt.get(session.userId);
|
||||
if (!user) {
|
||||
console.log(` Skipping session for non-existent user: ${session.userId}`);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create session in database
|
||||
sessionRepo.createSession({
|
||||
id: session.id || require('crypto').randomUUID(),
|
||||
userId: session.userId,
|
||||
token: token,
|
||||
deviceFingerprint: session.deviceFingerprint || null,
|
||||
ipAddress: session.ipAddress || null,
|
||||
userAgent: session.userAgent || null,
|
||||
expiresAt: session.expiresAt,
|
||||
createdAt: session.createdAt || now,
|
||||
lastAccessedAt: session.lastAccessedAt || now
|
||||
});
|
||||
sessionStmt.run(
|
||||
token,
|
||||
session.userId,
|
||||
token,
|
||||
null,
|
||||
session.deviceFingerprint || null,
|
||||
session.ipAddress || null,
|
||||
session.userAgent || null,
|
||||
session.expiresAt,
|
||||
session.createdAt || now,
|
||||
session.lastAccessedAt || now
|
||||
);
|
||||
|
||||
success++;
|
||||
} catch (error) {
|
||||
@@ -156,38 +225,53 @@ async function migrateAffiliates() {
|
||||
let failed = 0;
|
||||
|
||||
const db = getDatabase();
|
||||
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO affiliate_accounts (
|
||||
id, email, name, password_hash, codes, earnings, commission_rate,
|
||||
created_at, updated_at, last_login_at, last_payout_at,
|
||||
email_verified, verification_token, verification_expires_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
email = excluded.email,
|
||||
name = excluded.name,
|
||||
password_hash = excluded.password_hash,
|
||||
codes = excluded.codes,
|
||||
earnings = excluded.earnings,
|
||||
commission_rate = excluded.commission_rate,
|
||||
updated_at = excluded.updated_at,
|
||||
last_login_at = excluded.last_login_at,
|
||||
last_payout_at = excluded.last_payout_at,
|
||||
email_verified = excluded.email_verified,
|
||||
verification_token = excluded.verification_token,
|
||||
verification_expires_at = excluded.verification_expires_at
|
||||
`);
|
||||
|
||||
for (const affiliate of affiliates) {
|
||||
try {
|
||||
// Check if user exists
|
||||
const user = userRepo.getUserById(affiliate.userId);
|
||||
if (!user) {
|
||||
console.log(` Skipping affiliate for non-existent user: ${affiliate.userId}`);
|
||||
if (!affiliate.email) {
|
||||
console.log(' Skipping affiliate without email');
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Insert affiliate
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO affiliates (
|
||||
id, user_id, codes, earnings, commission_rate,
|
||||
total_referrals, total_earnings_cents, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const affiliateEmail = (affiliate.email || '').trim().toLowerCase();
|
||||
stmt.run(
|
||||
affiliate.id || require('crypto').randomUUID(),
|
||||
affiliate.userId,
|
||||
affiliateEmail,
|
||||
affiliate.name || '',
|
||||
affiliate.password || affiliate.passwordHash || '',
|
||||
JSON.stringify(affiliate.codes || []),
|
||||
JSON.stringify(affiliate.earnings || []),
|
||||
affiliate.commissionRate || 0.15,
|
||||
affiliate.totalReferrals || 0,
|
||||
affiliate.totalEarningsCents || 0,
|
||||
affiliate.createdAt || Date.now(),
|
||||
affiliate.updatedAt || Date.now()
|
||||
affiliate.createdAt || new Date().toISOString(),
|
||||
affiliate.updatedAt || affiliate.createdAt || new Date().toISOString(),
|
||||
affiliate.lastLoginAt || null,
|
||||
affiliate.lastPayoutAt || null,
|
||||
affiliate.emailVerified ? 1 : 0,
|
||||
affiliate.verificationToken || null,
|
||||
affiliate.verificationExpiresAt || null
|
||||
);
|
||||
|
||||
console.log(` ✓ Migrated affiliate for user: ${user.email}`);
|
||||
console.log(` ✓ Migrated affiliate: ${affiliate.email}`);
|
||||
success++;
|
||||
} catch (error) {
|
||||
console.error(` ✗ Failed to migrate affiliate:`, error.message);
|
||||
@@ -229,7 +313,7 @@ async function createBackup() {
|
||||
|
||||
async function runMigration() {
|
||||
console.log('🔄 Starting database migration...');
|
||||
console.log(' Source: JSON files in', DATA_ROOT);
|
||||
console.log(' Source: JSON files in', STATE_DIR);
|
||||
console.log(' Target: Database at', DATABASE_PATH);
|
||||
|
||||
// Check if database exists
|
||||
@@ -239,13 +323,21 @@ async function runMigration() {
|
||||
}
|
||||
|
||||
// Initialize encryption
|
||||
if (!DATABASE_ENCRYPTION_KEY) {
|
||||
let encryptionKey = DATABASE_ENCRYPTION_KEY;
|
||||
if (!encryptionKey && KEY_FILE && fs.existsSync(KEY_FILE)) {
|
||||
encryptionKey = fs.readFileSync(KEY_FILE, 'utf8').trim();
|
||||
if (encryptionKey) {
|
||||
process.env.DATABASE_ENCRYPTION_KEY = encryptionKey;
|
||||
console.log('✅ Loaded encryption key from file');
|
||||
}
|
||||
}
|
||||
if (!encryptionKey) {
|
||||
console.error('❌ DATABASE_ENCRYPTION_KEY not set');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
initEncryption(DATABASE_ENCRYPTION_KEY);
|
||||
initEncryption(encryptionKey);
|
||||
console.log('✅ Encryption initialized');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to initialize encryption:', error.message);
|
||||
@@ -254,7 +346,12 @@ async function runMigration() {
|
||||
|
||||
// Initialize database
|
||||
try {
|
||||
initDatabase(DATABASE_PATH, { verbose: false });
|
||||
initDatabase(DATABASE_PATH, {
|
||||
verbose: false,
|
||||
sqlcipherKey: DATABASE_USE_SQLCIPHER ? encryptionKey : null,
|
||||
cipherCompatibility: process.env.DATABASE_CIPHER_COMPAT || 4,
|
||||
kdfIter: process.env.DATABASE_KDF_ITER || 64000
|
||||
});
|
||||
console.log('✅ Database connected');
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to connect to database:', error.message);
|
||||
|
||||
Reference in New Issue
Block a user