Add database migration scripts and configuration files

- Add verify-migration.js script for testing database migrations
- Add database config module for centralized configuration
- Add chutes.txt prompt for system responses
- Update database implementation and testing documentation
- Add database migration and setup scripts
- Update session system and LLM tool configuration
- Update deployment checklist and environment example
- Update Dockerfile and docker-compose configuration
This commit is contained in:
southseact-3d
2026-02-20 12:38:43 +00:00
parent a92797d3a7
commit cb95a916ae
19 changed files with 1104 additions and 143 deletions

View File

@@ -18,6 +18,9 @@ const security = require('./security');
const { createExternalWpTester, getExternalTestingConfig } = require('./external-wp-testing');
const blogSystem = require('./blog-system');
const versionManager = require('./src/utils/versionManager');
const { DATA_ROOT, STATE_DIR, DB_PATH, KEY_FILE } = require('./src/database/config');
const { initDatabase, getDatabase, closeDatabase } = require('./src/database/connection');
const { initEncryption, encrypt } = require('./src/utils/encryption');
let sharp = null;
try {
@@ -100,8 +103,11 @@ function detectDockerLimits(baseMemoryBytes, baseCpuCores, repoRoot) {
const PORT = Number(process.env.CHAT_PORT || 4000);
const HOST = process.env.CHAT_HOST || '0.0.0.0';
const DATA_ROOT = process.env.CHAT_DATA_ROOT || path.join(process.cwd(), '.data');
const STATE_DIR = path.join(DATA_ROOT, '.opencode-chat');
const USE_JSON_DATABASE = process.env.USE_JSON_DATABASE === '1' || process.env.USE_JSON_DATABASE === 'true';
const DATABASE_WAL_MODE = process.env.DATABASE_WAL_MODE !== '0' && process.env.DATABASE_WAL_MODE !== 'false';
const DATABASE_USE_SQLCIPHER = process.env.DATABASE_USE_SQLCIPHER !== '0' && process.env.DATABASE_USE_SQLCIPHER !== 'false';
const DATABASE_CIPHER_COMPAT = process.env.DATABASE_CIPHER_COMPAT || 4;
const DATABASE_KDF_ITER = process.env.DATABASE_KDF_ITER || 64000;
const STATE_FILE = path.join(STATE_DIR, 'sessions.json');
const WORKSPACES_ROOT = path.join(DATA_ROOT, 'apps');
const STATIC_ROOT = path.join(__dirname, 'public');
@@ -1574,6 +1580,8 @@ const apiRateLimit = new Map(); // { userId: { requests, windowStart } }
const csrfTokens = new Map(); // { token: { userId, expiresAt } }
let usersDb = []; // In-memory user database cache
let invoicesDb = []; // In-memory invoice database cache
let databaseEnabled = false;
let databaseFallbackReason = '';
let mailTransport = null;
function summarizeMailConfig() {
@@ -2154,9 +2162,197 @@ function requireAdminAuth(req, res) {
return session;
}
async function loadDatabaseKey() {
if (process.env.DATABASE_ENCRYPTION_KEY) {
return process.env.DATABASE_ENCRYPTION_KEY.trim();
}
try {
if (KEY_FILE && fsSync.existsSync(KEY_FILE)) {
const key = fsSync.readFileSync(KEY_FILE, 'utf8').trim();
if (key) {
process.env.DATABASE_ENCRYPTION_KEY = key;
return key;
}
}
} catch (_) { }
return '';
}
async function initializeDatabaseLayer() {
if (USE_JSON_DATABASE) {
databaseEnabled = false;
databaseFallbackReason = 'USE_JSON_DATABASE';
log('Database disabled via USE_JSON_DATABASE, using JSON storage');
return;
}
const dbExists = fsSync.existsSync(DB_PATH);
const key = await loadDatabaseKey();
if (!key) {
databaseEnabled = false;
databaseFallbackReason = dbExists ? 'missing_key' : 'no_key';
if (dbExists) {
log('DATABASE_ENCRYPTION_KEY missing; falling back to JSON mode', { DB_PATH, KEY_FILE });
} else {
log('No database key and no database found; using JSON mode', { DB_PATH });
}
return;
}
try {
initEncryption(key);
} catch (error) {
databaseEnabled = false;
databaseFallbackReason = 'encryption_init_failed';
log('Failed to initialize encryption; falling back to JSON mode', { error: String(error) });
return;
}
try {
initDatabase(DB_PATH, {
walMode: DATABASE_WAL_MODE,
sqlcipherKey: DATABASE_USE_SQLCIPHER ? key : null,
cipherCompatibility: DATABASE_CIPHER_COMPAT,
kdfIter: DATABASE_KDF_ITER
});
ensureDatabaseSchema();
databaseEnabled = true;
databaseFallbackReason = '';
log('Database mode enabled', { dbPath: DB_PATH, sqlcipher: DATABASE_USE_SQLCIPHER });
} catch (error) {
closeDatabase();
databaseEnabled = false;
databaseFallbackReason = 'db_init_failed';
log('Failed to initialize database; falling back to JSON mode', { error: String(error) });
}
}
function ensureDatabaseSchema() {
const db = getDatabase();
if (!db) return;
try {
const userColumns = db.prepare('PRAGMA table_info(users)').all();
const userColumnNames = new Set(userColumns.map((c) => c.name));
if (!userColumnNames.has('data')) {
db.exec('ALTER TABLE users ADD COLUMN data TEXT');
log('Added users.data column');
}
db.exec(`
CREATE TABLE IF NOT EXISTS affiliate_accounts (
id TEXT PRIMARY KEY,
email TEXT UNIQUE NOT NULL,
name TEXT,
password_hash TEXT NOT NULL,
codes TEXT NOT NULL DEFAULT '[]',
earnings TEXT NOT NULL DEFAULT '[]',
commission_rate REAL NOT NULL DEFAULT 0.15,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
last_login_at TEXT,
last_payout_at TEXT,
email_verified INTEGER DEFAULT 0,
verification_token TEXT,
verification_expires_at TEXT
);
`);
db.exec('CREATE INDEX IF NOT EXISTS idx_affiliate_accounts_email ON affiliate_accounts(email);');
} catch (error) {
log('Failed to ensure database schema', { error: String(error) });
}
}
// User authentication functions
async function loadUsersDb() {
let shouldPersist = false;
if (databaseEnabled) {
try {
const db = getDatabase();
if (!db) throw new Error('Database not initialized');
const rows = db.prepare('SELECT * FROM users').all();
usersDb = rows.map((row) => {
let payload = {};
if (row?.data) {
try {
payload = JSON.parse(row.data);
} catch (_) {
payload = {};
}
}
const user = { ...payload };
user.id = row.id;
user.email = row.email || user.email || '';
user.password = row.password_hash || user.password || '';
if (row.providers) {
try {
user.providers = JSON.parse(row.providers);
} catch (_) {
user.providers = Array.isArray(user.providers) ? user.providers : [];
}
} else {
user.providers = Array.isArray(user.providers) ? user.providers : [];
}
user.emailVerified = typeof user.emailVerified === 'boolean' ? user.emailVerified : Boolean(row.email_verified);
user.verificationToken = row.verification_token || user.verificationToken || '';
user.verificationExpiresAt = row.verification_expires_at || user.verificationExpiresAt || null;
user.resetToken = row.reset_token || user.resetToken || '';
user.resetExpiresAt = row.reset_expires_at || user.resetExpiresAt || null;
user.plan = row.plan || user.plan || null;
user.billingStatus = row.billing_status || user.billingStatus || DEFAULT_BILLING_STATUS;
user.billingEmail = row.billing_email || user.billingEmail || user.email || '';
user.paymentMethodLast4 = row.payment_method_last4 || user.paymentMethodLast4 || '';
user.subscriptionRenewsAt = row.subscription_renews_at || user.subscriptionRenewsAt || null;
user.referredByAffiliateCode = sanitizeAffiliateCode(row.referred_by_affiliate_code || user.referredByAffiliateCode);
user.affiliateAttributionAt = row.affiliate_attribution_at || user.affiliateAttributionAt || null;
if (row.affiliate_payouts) {
try {
user.affiliatePayouts = JSON.parse(row.affiliate_payouts);
} catch (_) {
user.affiliatePayouts = Array.isArray(user.affiliatePayouts) ? user.affiliatePayouts : [];
}
} else {
user.affiliatePayouts = Array.isArray(user.affiliatePayouts) ? user.affiliatePayouts : [];
}
user.twoFactorSecret = row.two_factor_secret || user.twoFactorSecret || null;
user.twoFactorEnabled = typeof user.twoFactorEnabled === 'boolean' ? user.twoFactorEnabled : Boolean(row.two_factor_enabled);
user.createdAt = row.created_at || user.createdAt || null;
user.updatedAt = row.updated_at || user.updatedAt || null;
user.lastLoginAt = row.last_login_at || user.lastLoginAt || null;
const verification = normalizeVerificationState(user);
if (verification.shouldPersist) shouldPersist = true;
const normalizedPlan = normalizePlanSelection(user?.plan) || DEFAULT_PLAN;
return {
...user,
emailVerified: verification.verified,
verificationToken: verification.verificationToken,
verificationExpiresAt: verification.verificationExpiresAt,
plan: normalizedPlan,
billingStatus: user?.billingStatus || DEFAULT_BILLING_STATUS,
billingEmail: user?.billingEmail || user?.email || '',
paymentMethodLast4: user?.paymentMethodLast4 || '',
subscriptionRenewsAt: user?.subscriptionRenewsAt || null,
referredByAffiliateCode: sanitizeAffiliateCode(user?.referredByAffiliateCode),
affiliateAttributionAt: user?.affiliateAttributionAt || null,
affiliatePayouts: Array.isArray(user?.affiliatePayouts)
? user.affiliatePayouts.map((p) => normalizePlanSelection(p)).filter(Boolean)
: [],
};
});
if (shouldPersist) {
await persistUsersDb();
}
log('Loaded users database', { count: usersDb.length, mode: 'database' });
return;
} catch (error) {
log('Failed to load users database from DB, falling back to JSON', { error: String(error) });
closeDatabase();
databaseEnabled = false;
databaseFallbackReason = 'load_failed';
}
}
try {
await ensureStateFile();
await fs.access(USERS_DB_FILE);
@@ -2198,12 +2394,136 @@ async function loadUsersDb() {
}
async function persistUsersDb() {
if (databaseEnabled) {
try {
const db = getDatabase();
if (!db) throw new Error('Database not initialized');
const now = new Date().toISOString();
const stmt = db.prepare(`
INSERT INTO users (
id, email, email_encrypted, name, name_encrypted, password_hash,
providers, email_verified, verification_token, verification_expires_at,
reset_token, reset_expires_at, plan, billing_status, billing_email,
payment_method_last4, subscription_renews_at, referred_by_affiliate_code,
affiliate_attribution_at, affiliate_payouts, two_factor_secret,
two_factor_enabled, created_at, updated_at, last_login_at, data
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
email = excluded.email,
email_encrypted = excluded.email_encrypted,
name = excluded.name,
name_encrypted = excluded.name_encrypted,
password_hash = excluded.password_hash,
providers = excluded.providers,
email_verified = excluded.email_verified,
verification_token = excluded.verification_token,
verification_expires_at = excluded.verification_expires_at,
reset_token = excluded.reset_token,
reset_expires_at = excluded.reset_expires_at,
plan = excluded.plan,
billing_status = excluded.billing_status,
billing_email = excluded.billing_email,
payment_method_last4 = excluded.payment_method_last4,
subscription_renews_at = excluded.subscription_renews_at,
referred_by_affiliate_code = excluded.referred_by_affiliate_code,
affiliate_attribution_at = excluded.affiliate_attribution_at,
affiliate_payouts = excluded.affiliate_payouts,
two_factor_secret = excluded.two_factor_secret,
two_factor_enabled = excluded.two_factor_enabled,
created_at = excluded.created_at,
updated_at = excluded.updated_at,
last_login_at = excluded.last_login_at,
data = excluded.data
`);
const ids = [];
db.transaction(() => {
for (const user of usersDb) {
if (!user || !user.id) continue;
ids.push(user.id);
const normalizedEmail = (user.email || '').trim().toLowerCase();
const passwordHash = user.passwordHash || user.password || '';
const referredCode = sanitizeAffiliateCode(user.referredByAffiliateCode);
const providers = Array.isArray(user.providers) ? user.providers : [];
const affiliatePayouts = Array.isArray(user.affiliatePayouts) ? user.affiliatePayouts : [];
const dataPayload = JSON.stringify({ ...user, email: normalizedEmail || user.email || '' });
const emailEncrypted = normalizedEmail ? encrypt(normalizedEmail) : '';
const nameEncrypted = user.name ? encrypt(user.name) : null;
const twoFactorEncrypted = user.twoFactorSecret ? encrypt(user.twoFactorSecret) : null;
stmt.run(
user.id,
normalizedEmail,
emailEncrypted,
user.name || null,
nameEncrypted,
passwordHash,
JSON.stringify(providers),
user.emailVerified ? 1 : 0,
user.verificationToken || null,
user.verificationExpiresAt || null,
user.resetToken || null,
user.resetExpiresAt || null,
user.plan || 'hobby',
user.billingStatus || DEFAULT_BILLING_STATUS,
user.billingEmail || user.email || '',
user.paymentMethodLast4 || '',
user.subscriptionRenewsAt || null,
referredCode || null,
user.affiliateAttributionAt || null,
JSON.stringify(affiliatePayouts),
twoFactorEncrypted,
user.twoFactorEnabled ? 1 : 0,
user.createdAt || now,
user.updatedAt || user.createdAt || now,
user.lastLoginAt || null,
dataPayload
);
}
if (ids.length === 0) {
db.prepare('DELETE FROM users').run();
} else if (ids.length <= 900) {
const placeholders = ids.map(() => '?').join(',');
db.prepare(`DELETE FROM users WHERE id NOT IN (${placeholders})`).run(...ids);
} else {
log('Skipping users cleanup (too many rows for NOT IN)', { count: ids.length });
}
})();
} catch (error) {
log('Failed to persist users database (DB)', { error: String(error) });
}
return;
}
await ensureStateFile();
const payload = JSON.stringify(usersDb, null, 2);
await safeWriteFile(USERS_DB_FILE, payload);
}
async function loadUserSessions() {
userSessions.clear();
if (databaseEnabled) {
try {
const db = getDatabase();
if (!db) throw new Error('Database not initialized');
const now = Date.now();
const rows = db.prepare('SELECT token, user_id, expires_at FROM sessions WHERE expires_at > ?').all(now);
for (const row of rows) {
if (row?.token && row?.user_id) {
userSessions.set(row.token, { userId: row.user_id, expiresAt: row.expires_at });
}
}
log('Loaded user sessions', { count: userSessions.size, mode: 'database' });
return;
} catch (error) {
log('Failed to load user sessions from DB, falling back to JSON', { error: String(error) });
closeDatabase();
databaseEnabled = false;
databaseFallbackReason = 'load_failed';
}
}
try {
await ensureStateFile();
const raw = await fs.readFile(USER_SESSIONS_FILE, 'utf8').catch(() => null);
@@ -2223,6 +2543,57 @@ async function loadUserSessions() {
}
async function persistUserSessions() {
if (databaseEnabled) {
try {
const db = getDatabase();
if (!db) throw new Error('Database not initialized');
const now = Date.now();
const tokens = [];
const stmt = db.prepare(`
INSERT INTO sessions (
id, user_id, token, refresh_token_hash, device_fingerprint,
ip_address, user_agent, expires_at, created_at, last_accessed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
user_id = excluded.user_id,
token = excluded.token,
expires_at = excluded.expires_at,
last_accessed_at = excluded.last_accessed_at
`);
db.transaction(() => {
db.prepare('DELETE FROM sessions WHERE expires_at <= ?').run(now);
for (const [token, session] of userSessions.entries()) {
if (!session.expiresAt || session.expiresAt <= now) continue;
tokens.push(token);
stmt.run(
token,
session.userId,
token,
null,
null,
null,
null,
session.expiresAt,
now,
now
);
}
if (tokens.length === 0) {
db.prepare('DELETE FROM sessions').run();
} else if (tokens.length <= 900) {
const placeholders = tokens.map(() => '?').join(',');
db.prepare(`DELETE FROM sessions WHERE token NOT IN (${placeholders})`).run(...tokens);
} else {
log('Skipping session cleanup (too many rows for NOT IN)', { count: tokens.length });
}
})();
} catch (error) {
log('Failed to persist user sessions (DB)', { error: String(error) });
}
return;
}
await ensureStateFile();
const now = Date.now();
const sessions = {};
@@ -2240,6 +2611,72 @@ function sanitizeAffiliateCode(code) {
}
async function loadAffiliatesDb() {
if (databaseEnabled) {
try {
const db = getDatabase();
if (!db) throw new Error('Database not initialized');
const rows = db.prepare('SELECT * FROM affiliate_accounts').all();
affiliatesDb = rows.map((row) => ({
id: row.id,
email: (row.email || '').trim().toLowerCase(),
name: row.name || row.email,
password: row.password_hash || '',
createdAt: row.created_at || new Date().toISOString(),
lastLoginAt: row.last_login_at || null,
commissionRate: Number.isFinite(row.commission_rate) ? row.commission_rate : AFFILIATE_COMMISSION_RATE,
codes: (() => {
try {
const parsed = JSON.parse(row.codes || '[]');
return Array.isArray(parsed) ? parsed : [];
} catch (_) {
return [];
}
})(),
earnings: (() => {
try {
const parsed = JSON.parse(row.earnings || '[]');
return Array.isArray(parsed) ? parsed : [];
} catch (_) {
return [];
}
})(),
lastPayoutAt: row.last_payout_at || null,
emailVerified: Boolean(row.email_verified),
verificationToken: row.verification_token || '',
verificationExpiresAt: row.verification_expires_at || null,
updatedAt: row.updated_at || row.created_at || new Date().toISOString(),
}));
let changed = false;
for (const affiliate of affiliatesDb) {
affiliate.codes = Array.isArray(affiliate?.codes) && affiliate.codes.length
? affiliate.codes.map((c) => ({
code: sanitizeAffiliateCode(c.code || c.id || c.slug || ''),
label: c.label || 'Tracking link',
createdAt: c.createdAt || affiliate.createdAt || new Date().toISOString(),
})).filter((c) => c.code)
: [];
affiliate.earnings = Array.isArray(affiliate?.earnings) ? affiliate.earnings : [];
if (!affiliate.codes || !affiliate.codes.length) {
const fallbackCode = generateTrackingCode();
affiliate.codes = [{
code: fallbackCode,
label: 'Default link',
createdAt: affiliate.createdAt || new Date().toISOString(),
}];
changed = true;
}
}
if (changed) await persistAffiliatesDb();
log('Loaded affiliates database', { count: affiliatesDb.length, mode: 'database' });
return;
} catch (error) {
log('Failed to load affiliates database from DB, falling back to JSON', { error: String(error) });
closeDatabase();
databaseEnabled = false;
databaseFallbackReason = 'load_failed';
}
}
try {
await ensureStateFile();
let raw = '[]';
@@ -2278,6 +2715,69 @@ async function loadAffiliatesDb() {
}
async function persistAffiliatesDb() {
if (databaseEnabled) {
try {
const db = getDatabase();
if (!db) throw new Error('Database not initialized');
const now = new Date().toISOString();
const stmt = db.prepare(`
INSERT INTO affiliate_accounts (
id, email, name, password_hash, codes, earnings, commission_rate,
created_at, updated_at, last_login_at, last_payout_at,
email_verified, verification_token, verification_expires_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
email = excluded.email,
name = excluded.name,
password_hash = excluded.password_hash,
codes = excluded.codes,
earnings = excluded.earnings,
commission_rate = excluded.commission_rate,
updated_at = excluded.updated_at,
last_login_at = excluded.last_login_at,
last_payout_at = excluded.last_payout_at,
email_verified = excluded.email_verified,
verification_token = excluded.verification_token,
verification_expires_at = excluded.verification_expires_at
`);
const ids = [];
db.transaction(() => {
for (const affiliate of affiliatesDb) {
if (!affiliate || !affiliate.id) continue;
ids.push(affiliate.id);
stmt.run(
affiliate.id,
(affiliate.email || '').trim().toLowerCase(),
affiliate.name || '',
affiliate.password || affiliate.passwordHash || '',
JSON.stringify(Array.isArray(affiliate.codes) ? affiliate.codes : []),
JSON.stringify(Array.isArray(affiliate.earnings) ? affiliate.earnings : []),
Number.isFinite(affiliate.commissionRate) ? affiliate.commissionRate : AFFILIATE_COMMISSION_RATE,
affiliate.createdAt || now,
affiliate.updatedAt || affiliate.createdAt || now,
affiliate.lastLoginAt || null,
affiliate.lastPayoutAt || null,
affiliate.emailVerified ? 1 : 0,
affiliate.verificationToken || null,
affiliate.verificationExpiresAt || null
);
}
if (ids.length === 0) {
db.prepare('DELETE FROM affiliate_accounts').run();
} else if (ids.length <= 900) {
const placeholders = ids.map(() => '?').join(',');
db.prepare(`DELETE FROM affiliate_accounts WHERE id NOT IN (${placeholders})`).run(...ids);
} else {
log('Skipping affiliates cleanup (too many rows for NOT IN)', { count: ids.length });
}
})();
} catch (error) {
log('Failed to persist affiliates database (DB)', { error: String(error) });
}
return;
}
await ensureStateFile();
const payload = JSON.stringify(affiliatesDb, null, 2);
await safeWriteFile(AFFILIATES_FILE, payload);
@@ -19421,6 +19921,7 @@ async function bootstrap() {
await loadSubscriptionSessions();
await loadPendingSubscriptions();
await loadInvoicesDb();
await initializeDatabaseLayer();
await loadUsersDb(); // Load user authentication database
await loadUserSessions(); // Load user sessions
await loadAffiliatesDb();