rem last build change
This commit is contained in:
southseact-3d
963
external-wp-testing-plan.md
Normal file
963
external-wp-testing-plan.md
Normal file
@@ -0,0 +1,963 @@
|
||||
# PluginCompass External WordPress Testing System
|
||||
|
||||
## Executive Summary
|
||||
|
||||
A production-ready testing system that uses an externally hosted WordPress site with WP-CLI to verify plugin functionality. Supports high concurrency (20+ sessions) through multisite subsite isolation, with two testing modes (automated WP-CLI commands and visual browser testing).
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ PluginCompass CLI Users │
|
||||
│ (20+ Concurrent Sessions) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Test Orchestrator Layer │
|
||||
│ - Session Queue Manager (prevents conflicts) │
|
||||
│ - Subsite Provisioning (auto-creates test sites) │
|
||||
│ - Resource Cleanup (automatic teardown) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ External WordPress Multisite │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ Session 1 │ │ Session 2 │ │ Session N │ │
|
||||
│ │ Subsite │ │ Subsite │ │ Subsite │ │
|
||||
│ │ /test-abc │ │ /test-def │ │ /test-xyz │ │
|
||||
│ └──────────────┘ └──────────────┘ └──────────────┘ │
|
||||
│ │
|
||||
│ Shared: Core WP, Plugins, Themes (isolated per subsite) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Core Components
|
||||
|
||||
### 1. Test Orchestrator (`test_orchestrator.js`)
|
||||
|
||||
Central controller managing all test sessions.
|
||||
|
||||
**Responsibilities:**
|
||||
- Queue incoming test requests
|
||||
- Provision subsites automatically
|
||||
- Manage plugin installations
|
||||
- Coordinate cleanup
|
||||
- Prevent cross-session contamination
|
||||
|
||||
**Configuration:**
|
||||
```javascript
|
||||
const TEST_CONFIG = {
|
||||
// External WP Site Connection
|
||||
wpHost: process.env.TEST_WP_HOST || 'testsite.example.com',
|
||||
wpSshUser: process.env.TEST_WP_SSH_USER || 'wordpress',
|
||||
wpSshKey: process.env.TEST_WP_SSH_KEY || '~/.ssh/wp-test-key',
|
||||
wpPath: process.env.TEST_WP_PATH || '/var/www/html',
|
||||
|
||||
// Multisite Settings
|
||||
enableMultisite: true,
|
||||
subsitePrefix: 'test',
|
||||
subsiteDomain: 'testsite.example.com',
|
||||
|
||||
// Concurrency
|
||||
maxConcurrentTests: 20,
|
||||
queueTimeout: 300000, // 5 minutes
|
||||
testTimeout: 600000, // 10 minutes
|
||||
|
||||
// Cleanup
|
||||
autoCleanup: true,
|
||||
cleanupDelay: 3600000, // 1 hour after test completion
|
||||
|
||||
// Testing Modes
|
||||
enableVisualTesting: process.env.ENABLE_VISUAL_TESTING === 'true',
|
||||
visualTestBrowser: 'chromium', // chromium, firefox, webkit
|
||||
};
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. New Tool: `test_plugin_external_wp`
|
||||
|
||||
**Tool Name:** `test_plugin_external_wp`
|
||||
|
||||
**Description:** Deploys and tests plugin on external WordPress site with full isolation. Automatically provisions subsite, installs dependencies, runs verification tests, and provides detailed results.
|
||||
|
||||
**Parameters:**
|
||||
```json
|
||||
{
|
||||
"name": "test_plugin_external_wp",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"plugin_path": {
|
||||
"type": "string",
|
||||
"description": "Local path to generated plugin directory"
|
||||
},
|
||||
"test_mode": {
|
||||
"type": "string",
|
||||
"enum": ["cli", "visual", "both"],
|
||||
"default": "cli",
|
||||
"description": "Testing mode: cli (WP-CLI commands), visual (browser automation), or both"
|
||||
},
|
||||
"required_plugins": {
|
||||
"type": "array",
|
||||
"description": "Other plugins that must be installed for this plugin to work",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"plugin_slug": { "type": "string" },
|
||||
"version": { "type": "string" },
|
||||
"source": {
|
||||
"type": "string",
|
||||
"enum": ["wordpress.org", "url", "local"],
|
||||
"default": "wordpress.org"
|
||||
},
|
||||
"source_url": { "type": "string" },
|
||||
"activate": { "type": "boolean", "default": true }
|
||||
},
|
||||
"required": ["plugin_slug"]
|
||||
}
|
||||
},
|
||||
"test_scenarios": {
|
||||
"type": "array",
|
||||
"description": "Specific test scenarios to verify",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": { "type": "string" },
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["endpoint", "admin_page", "ajax", "shortcode", "hook", "visual", "custom"]
|
||||
},
|
||||
"url": { "type": "string" },
|
||||
"selector": { "type": "string" },
|
||||
"expected_text": { "type": "string" },
|
||||
"wp_cli_command": { "type": "string" },
|
||||
"assertions": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"status_code": { "type": "number" },
|
||||
"contains": { "type": "array", "items": { "type": "string" } },
|
||||
"not_contains": { "type": "array", "items": { "type": "string" } },
|
||||
"wp_cli_success": { "type": "boolean" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["name", "type"]
|
||||
}
|
||||
},
|
||||
"visual_tests": {
|
||||
"type": "array",
|
||||
"description": "Visual testing scenarios (only used when test_mode is 'visual' or 'both')",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": { "type": "string" },
|
||||
"url": { "type": "string" },
|
||||
"actions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["goto", "click", "fill", "wait", "screenshot", "evaluate"]
|
||||
},
|
||||
"selector": { "type": "string" },
|
||||
"value": { "type": "string" },
|
||||
"timeout": { "type": "number" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"assertions": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"visible": { "type": "array", "items": { "type": "string" } },
|
||||
"hidden": { "type": "array", "items": { "type": "string" } },
|
||||
"text_contains": { "type": "array", "items": { "type": "string" } },
|
||||
"no_console_errors": { "type": "boolean" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"timeout": {
|
||||
"type": "number",
|
||||
"default": 300,
|
||||
"description": "Test timeout in seconds"
|
||||
},
|
||||
"auto_fix": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"description": "Automatically retry and fix on test failure"
|
||||
},
|
||||
"max_fix_attempts": {
|
||||
"type": "number",
|
||||
"default": 3,
|
||||
"description": "Maximum auto-fix attempts before giving up"
|
||||
}
|
||||
},
|
||||
"required": ["plugin_path"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Return Value:**
|
||||
```javascript
|
||||
{
|
||||
"ok": true,
|
||||
"session_id": "sess_abc123",
|
||||
"subsite_url": "https://testsite.example.com/test-abc123",
|
||||
"test_results": {
|
||||
"mode": "both",
|
||||
"cli_tests": {
|
||||
"passed": 5,
|
||||
"failed": 0,
|
||||
"results": [
|
||||
{
|
||||
"name": "Plugin activates without errors",
|
||||
"status": "passed",
|
||||
"command": "wp plugin activate test-plugin",
|
||||
"output": "Success: Activated 1 of 1 plugins.",
|
||||
"duration": 2340
|
||||
},
|
||||
{
|
||||
"name": "Custom login endpoint responds",
|
||||
"status": "passed",
|
||||
"command": "wp eval 'echo wp_remote_retrieve_body(wp_remote_get(home_url(\"/custom-login\")));'",
|
||||
"assertions": {
|
||||
"contains": ["<form", "login"],
|
||||
"status_code": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"visual_tests": {
|
||||
"passed": 3,
|
||||
"failed": 0,
|
||||
"screenshots": [
|
||||
{
|
||||
"name": "login-page",
|
||||
"path": "/results/sess_abc123/login-page.png",
|
||||
"url": "https://testsite.example.com/test-abc123/custom-login"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"installed_plugins": [
|
||||
{ "slug": "woocommerce", "version": "8.0.0", "status": "active" },
|
||||
{ "slug": "test-plugin", "version": "1.0.0", "status": "active" }
|
||||
],
|
||||
"errors": [],
|
||||
"warnings": [],
|
||||
"duration": 45230,
|
||||
"cleanup_scheduled": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Testing Modes
|
||||
|
||||
### Mode 1: WP-CLI Automated Testing
|
||||
|
||||
**Use Case:** Fast, deterministic testing without browser overhead
|
||||
|
||||
**How it works:**
|
||||
1. Deploy plugin to subsite
|
||||
2. Run WP-CLI commands to verify functionality
|
||||
3. Check HTTP endpoints via `wp eval` with `wp_remote_get()`
|
||||
4. Verify database state changes
|
||||
5. Check error logs
|
||||
|
||||
**Example Test Scenarios:**
|
||||
```javascript
|
||||
[
|
||||
{
|
||||
"name": "Plugin activates cleanly",
|
||||
"type": "custom",
|
||||
"wp_cli_command": "wp plugin activate {plugin_slug} --skip-plugins",
|
||||
"assertions": {
|
||||
"wp_cli_success": true,
|
||||
"not_contains": ["Fatal error", "Parse error", "Warning"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Custom endpoint accessible",
|
||||
"type": "endpoint",
|
||||
"url": "/custom-login",
|
||||
"wp_cli_command": "wp eval 'echo wp_remote_retrieve_response_code(wp_remote_get(home_url(\"/custom-login\")));'",
|
||||
"assertions": {
|
||||
"status_code": 200,
|
||||
"contains": ["<form", "name=\"log\""]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Rewrite rules flushed",
|
||||
"type": "custom",
|
||||
"wp_cli_command": "wp rewrite list --match=/custom-login --format=json",
|
||||
"assertions": {
|
||||
"wp_cli_success": true,
|
||||
"contains": ["custom-login"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "No PHP errors in logs",
|
||||
"type": "custom",
|
||||
"wp_cli_command": "tail -n 50 /var/log/wp-errors.log | grep -i 'test-plugin' || echo 'No errors found'",
|
||||
"assertions": {
|
||||
"not_contains": ["Fatal error", "Parse error", "test-plugin"]
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Fast (5-30 seconds per test)
|
||||
- No browser dependencies
|
||||
- Perfect for API/endpoint testing
|
||||
- Can check database state directly
|
||||
- Scales to 100+ concurrent tests
|
||||
|
||||
---
|
||||
|
||||
### Mode 2: Visual Browser Testing
|
||||
|
||||
**Use Case:** UI verification, JavaScript functionality, styling checks
|
||||
|
||||
**How it works:**
|
||||
1. Deploy plugin to subsite
|
||||
2. Launch headless browser (Playwright)
|
||||
3. Navigate to specified URLs
|
||||
4. Perform user interactions (click, type, submit)
|
||||
5. Capture screenshots
|
||||
6. Verify visual elements and JavaScript execution
|
||||
|
||||
**Example Visual Tests:**
|
||||
```javascript
|
||||
[
|
||||
{
|
||||
"name": "Login form renders correctly",
|
||||
"url": "/custom-login",
|
||||
"actions": [
|
||||
{ "action": "goto", "url": "/custom-login" },
|
||||
{ "action": "screenshot", "name": "initial-load" },
|
||||
{ "action": "wait", "selector": "form" }
|
||||
],
|
||||
"assertions": {
|
||||
"visible": ["input[name='log']", "input[name='pwd']", ".submit-button"],
|
||||
"no_console_errors": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Add to cart button works",
|
||||
"url": "/shop",
|
||||
"actions": [
|
||||
{ "action": "goto", "url": "/shop" },
|
||||
{ "action": "click", "selector": "[data-product_id='123']" },
|
||||
{ "action": "wait", "timeout": 2000 },
|
||||
{ "action": "screenshot", "name": "after-click" }
|
||||
],
|
||||
"assertions": {
|
||||
"visible": [".cart-count"],
|
||||
"text_contains": ["1 item"],
|
||||
"no_console_errors": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Admin settings page loads",
|
||||
"url": "/wp-admin/admin.php?page=my-plugin",
|
||||
"actions": [
|
||||
{
|
||||
"action": "evaluate",
|
||||
"value": "document.querySelector('#user_login').value = 'admin'; document.querySelector('#user_pass').value = 'password'; document.querySelector('#wp-submit').click();"
|
||||
},
|
||||
{ "action": "wait", "url": "/wp-admin/admin.php?page=my-plugin" },
|
||||
{ "action": "screenshot", "name": "admin-page" }
|
||||
],
|
||||
"assertions": {
|
||||
"visible": [".wrap h1", "form"],
|
||||
"no_console_errors": true
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
- Catches visual/styling issues
|
||||
- Verifies JavaScript execution
|
||||
- Tests actual user workflows
|
||||
- Screenshots provide proof
|
||||
- Finds timing/race condition issues
|
||||
|
||||
---
|
||||
|
||||
## Session Isolation Strategy
|
||||
|
||||
### High Concurrency Solution: Multisite Subsites
|
||||
|
||||
**Why Multisite:**
|
||||
- Each session gets completely isolated WordPress instance
|
||||
- Shared core (efficient) but isolated content/options
|
||||
- Can run 20+ tests simultaneously without conflicts
|
||||
- Automatic cleanup by deleting subsite
|
||||
|
||||
**Setup Requirements:**
|
||||
1. Convert hosted WP to multisite (one-time)
|
||||
2. Enable subdomain or subdirectory mode
|
||||
3. Configure wildcard DNS or subdirectory rewrite rules
|
||||
|
||||
**Subsite Provisioning:**
|
||||
```javascript
|
||||
class SubsiteProvisioner {
|
||||
async createTestSubsite(sessionId) {
|
||||
const slug = `test-${sessionId.substring(0, 8)}`;
|
||||
const title = `Test Environment ${sessionId}`;
|
||||
const adminEmail = 'test@example.com';
|
||||
|
||||
// Create subsite via WP-CLI
|
||||
await this.runWpCli(`
|
||||
wp site create
|
||||
--slug=${slug}
|
||||
--title="${title}"
|
||||
--email=${adminEmail}
|
||||
`);
|
||||
|
||||
// Configure subsite
|
||||
await this.runWpCli(`
|
||||
wp site switch --url=https://${this.config.subsiteDomain}/${slug}
|
||||
wp option update blogname "Test Site"
|
||||
wp option update permalink_structure "/%postname%/"
|
||||
wp rewrite flush
|
||||
`);
|
||||
|
||||
return {
|
||||
slug,
|
||||
url: `https://${this.config.subsiteDomain}/${slug}`,
|
||||
adminUrl: `https://${this.config.subsiteDomain}/${slug}/wp-admin`
|
||||
};
|
||||
}
|
||||
|
||||
async deleteTestSubsite(slug) {
|
||||
await this.runWpCli(`wp site delete --slug=${slug} --yes`);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dependency Management
|
||||
|
||||
### Installing Required Plugins
|
||||
|
||||
The AI can specify dependencies that must be present:
|
||||
|
||||
```javascript
|
||||
{
|
||||
"required_plugins": [
|
||||
{
|
||||
"plugin_slug": "woocommerce",
|
||||
"version": ">=8.0",
|
||||
"source": "wordpress.org",
|
||||
"activate": true
|
||||
},
|
||||
{
|
||||
"plugin_slug": "advanced-custom-fields",
|
||||
"version": "6.2.0",
|
||||
"source": "wordpress.org",
|
||||
"activate": true
|
||||
},
|
||||
{
|
||||
"plugin_slug": "custom-premium-plugin",
|
||||
"source": "url",
|
||||
"source_url": "https://example.com/plugins/custom.zip",
|
||||
"activate": true
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Installation Process:**
|
||||
```javascript
|
||||
class PluginInstaller {
|
||||
async installPlugin(subsiteUrl, pluginSpec) {
|
||||
const { plugin_slug, source, source_url, activate } = pluginSpec;
|
||||
|
||||
switch (source) {
|
||||
case 'wordpress.org':
|
||||
await this.runWpCli(`
|
||||
wp --url=${subsiteUrl} plugin install ${plugin_slug} --activate
|
||||
`);
|
||||
break;
|
||||
|
||||
case 'url':
|
||||
// Download and install from URL
|
||||
await this.runWpCli(`
|
||||
wp --url=${subsiteUrl} plugin install ${source_url} --activate
|
||||
`);
|
||||
break;
|
||||
|
||||
case 'local':
|
||||
// Copy from local path to subsite
|
||||
const targetPath = `${this.config.wpPath}/wp-content/plugins/${plugin_slug}`;
|
||||
await this.ssh.copyFile(source_url, targetPath);
|
||||
if (activate) {
|
||||
await this.runWpCli(`
|
||||
wp --url=${subsiteUrl} plugin activate ${plugin_slug}
|
||||
`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Queue Management for High Concurrency
|
||||
|
||||
### Session Queue System
|
||||
|
||||
**Challenge:** Even with multisite, some operations (like subsite creation) can have race conditions.
|
||||
|
||||
**Solution:** Queue system with parallel execution lanes
|
||||
|
||||
```javascript
|
||||
class TestQueue {
|
||||
constructor(maxConcurrent = 20) {
|
||||
this.maxConcurrent = maxConcurrent;
|
||||
this.activeSessions = new Map();
|
||||
this.pendingQueue = [];
|
||||
this.lanes = new Array(maxConcurrent).fill(null); // Execution lanes
|
||||
}
|
||||
|
||||
async enqueue(testRequest) {
|
||||
const sessionId = this.generateSessionId();
|
||||
|
||||
// Find available lane
|
||||
const laneIndex = this.findAvailableLane();
|
||||
|
||||
if (laneIndex === -1) {
|
||||
// All lanes busy, add to pending
|
||||
return new Promise((resolve) => {
|
||||
this.pendingQueue.push({
|
||||
sessionId,
|
||||
request: testRequest,
|
||||
resolve,
|
||||
enqueuedAt: Date.now()
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Execute immediately in available lane
|
||||
return this.executeInLane(laneIndex, sessionId, testRequest);
|
||||
}
|
||||
|
||||
async executeInLane(laneIndex, sessionId, request) {
|
||||
this.lanes[laneIndex] = sessionId;
|
||||
|
||||
try {
|
||||
const result = await this.runTest(sessionId, request);
|
||||
return result;
|
||||
} finally {
|
||||
this.lanes[laneIndex] = null;
|
||||
this.processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
processQueue() {
|
||||
if (this.pendingQueue.length === 0) return;
|
||||
|
||||
const laneIndex = this.findAvailableLane();
|
||||
if (laneIndex === -1) return;
|
||||
|
||||
const next = this.pendingQueue.shift();
|
||||
|
||||
// Check for timeout
|
||||
if (Date.now() - next.enqueuedAt > TEST_CONFIG.queueTimeout) {
|
||||
next.resolve({
|
||||
ok: false,
|
||||
error: 'Test queue timeout - too many concurrent tests'
|
||||
});
|
||||
this.processQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
this.executeInLane(laneIndex, next.sessionId, next.request)
|
||||
.then(next.resolve)
|
||||
.catch(error => next.resolve({ ok: false, error: error.message }));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Complete Workflow
|
||||
|
||||
### Step-by-Step Process
|
||||
|
||||
```
|
||||
User Request
|
||||
↓
|
||||
AI Generates Plugin Code
|
||||
↓
|
||||
AI Calls test_plugin_external_wp
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 1. QUEUE & PREPARE │
|
||||
│ - Add to test queue │
|
||||
│ - Wait for available lane │
|
||||
│ - Generate session ID │
|
||||
└─────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 2. PROVISION SUBSITE │
|
||||
│ - Create multisite subsite │
|
||||
│ - Configure permalinks │
|
||||
│ - Set up test database tables │
|
||||
│ Duration: 3-5 seconds │
|
||||
└─────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 3. INSTALL DEPENDENCIES │
|
||||
│ - Install WordPress (if needed) │
|
||||
│ - Install WooCommerce │
|
||||
│ - Install other required plugins │
|
||||
│ - Activate all │
|
||||
│ Duration: 10-30 seconds │
|
||||
└─────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 4. DEPLOY TEST PLUGIN │
|
||||
│ - Copy plugin files via SFTP │
|
||||
│ - Activate plugin │
|
||||
│ - Check for activation errors │
|
||||
│ Duration: 2-5 seconds │
|
||||
└─────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 5. RUN TESTS │
|
||||
│ CLI Mode: │
|
||||
│ - WP-CLI verification commands │
|
||||
│ - HTTP endpoint checks │
|
||||
│ - Database state validation │
|
||||
│ - Error log scanning │
|
||||
│ │
|
||||
│ Visual Mode (if enabled): │
|
||||
│ - Launch headless browser │
|
||||
│ - Navigate to test URLs │
|
||||
│ - Execute user interactions │
|
||||
│ - Capture screenshots │
|
||||
│ - Check JavaScript console │
|
||||
│ Duration: 10-60 seconds │
|
||||
└─────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 6. EVALUATE & AUTO-FIX (if needed) │
|
||||
│ IF tests failed AND auto_fix=true:│
|
||||
│ - Analyze failure patterns │
|
||||
│ - Generate fix suggestions │
|
||||
│ - Modify plugin code │
|
||||
│ - Re-deploy and re-test │
|
||||
│ - Max 3 attempts │
|
||||
└─────────────────────────────────────┘
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 7. RETURN RESULTS │
|
||||
│ - Compile test results │
|
||||
│ - Include screenshots (visual) │
|
||||
│ - Provide fix recommendations │
|
||||
│ - Schedule cleanup │
|
||||
└─────────────────────────────────────┘
|
||||
↓
|
||||
AI Receives Results
|
||||
↓
|
||||
IF passed → Complete task
|
||||
IF failed → Fix issues and re-call tool
|
||||
↓
|
||||
┌─────────────────────────────────────┐
|
||||
│ 8. CLEANUP (async, after delay) │
|
||||
│ - Delete subsite │
|
||||
│ - Remove plugin files │
|
||||
│ - Clear test data │
|
||||
│ - Free up lane │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Error Handling & Auto-Fix
|
||||
|
||||
### Common Failures & Fixes
|
||||
|
||||
```javascript
|
||||
const AUTO_FIX_PATTERNS = [
|
||||
{
|
||||
pattern: /Rewrite rules not working/i,
|
||||
check: async (subsite) => {
|
||||
const rules = await runWpCli(`wp --url=${subsite} rewrite list`);
|
||||
return !rules.includes('custom-login');
|
||||
},
|
||||
fix: async (pluginPath) => {
|
||||
// Add flush_rewrite_rules() to activation hook
|
||||
await editFile(pluginPath + '/includes/activation.php',
|
||||
'function activate_plugin() {',
|
||||
'function activate_plugin() {\n flush_rewrite_rules();'
|
||||
);
|
||||
}
|
||||
},
|
||||
{
|
||||
pattern: /Undefined function.*wc_/i,
|
||||
check: async (subsite) => {
|
||||
const logs = await runWpCli(`tail -n 20 /var/log/wp-errors.log`);
|
||||
return logs.includes('Call to undefined function wc_');
|
||||
},
|
||||
fix: async (pluginPath) => {
|
||||
// Add WooCommerce dependency check
|
||||
await editFile(pluginPath + '/main.php',
|
||||
'<?php',
|
||||
'<?php\n// Check WooCommerce is active\nif (!in_array(\'woocommerce/woocommerce.php\', apply_filters(\'active_plugins\', get_option(\'active_plugins\')))) {\n return;\n}'
|
||||
);
|
||||
}
|
||||
},
|
||||
{
|
||||
pattern: /CSS not loading/i,
|
||||
check: async (subsite) => {
|
||||
const html = await runWpCli(`wp --url=${subsite} eval 'echo wp_remote_retrieve_body(wp_remote_get(home_url("/custom-login")));'`);
|
||||
return !html.includes('custom-login.css');
|
||||
},
|
||||
fix: async (pluginPath) => {
|
||||
// Ensure CSS is enqueued
|
||||
await editFile(pluginPath + '/includes/enqueue.php',
|
||||
'',
|
||||
'wp_enqueue_style(\'custom-login-css\', plugins_url(\'css/custom-login.css\', __FILE__));'
|
||||
);
|
||||
}
|
||||
}
|
||||
];
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration & Environment Variables
|
||||
|
||||
### Required Environment Variables
|
||||
|
||||
```bash
|
||||
# WordPress Test Site Connection
|
||||
TEST_WP_HOST=testsite.example.com
|
||||
TEST_WP_SSH_USER=wordpress
|
||||
TEST_WP_SSH_KEY_PATH=/path/to/ssh/key
|
||||
TEST_WP_PATH=/var/www/html
|
||||
|
||||
# Multisite Configuration
|
||||
TEST_WP_MULTISITE=true
|
||||
TEST_WP_SUBSITE_DOMAIN=testsite.example.com
|
||||
|
||||
# Testing Configuration
|
||||
TEST_ENABLE_VISUAL=true
|
||||
TEST_VISUAL_BROWSER=chromium
|
||||
TEST_MAX_CONCURRENT=20
|
||||
TEST_TIMEOUT=300
|
||||
TEST_AUTO_CLEANUP=true
|
||||
TEST_CLEANUP_DELAY=3600
|
||||
|
||||
# Optional: Browser Testing (for visual mode)
|
||||
TEST_BROWSER_HEADLESS=true
|
||||
TEST_BROWSER_TIMEOUT=30000
|
||||
TEST_SCREENSHOT_PATH=/var/results/screenshots
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Core Infrastructure (Week 1)
|
||||
- [ ] Set up multisite WordPress
|
||||
- [ ] Implement SSH/SFTP connection layer
|
||||
- [ ] Create `test_plugin_external_wp` tool
|
||||
- [ ] Build subsite provisioning system
|
||||
- [ ] Basic CLI test mode
|
||||
|
||||
### Phase 2: Testing Modes (Week 2)
|
||||
- [ ] Implement CLI test scenarios
|
||||
- [ ] Add visual testing with Playwright
|
||||
- [ ] Create queue management system
|
||||
- [ ] Build dependency installer
|
||||
|
||||
### Phase 3: Auto-Fix & Polish (Week 3)
|
||||
- [ ] Implement auto-fix patterns
|
||||
- [ ] Add retry logic
|
||||
- [ ] Error classification
|
||||
- [ ] Result formatting for AI
|
||||
|
||||
### Phase 4: Production Ready (Week 4)
|
||||
- [ ] Concurrent session testing
|
||||
- [ ] Performance optimization
|
||||
- [ ] Documentation
|
||||
- [ ] Monitoring & alerting
|
||||
|
||||
---
|
||||
|
||||
## Example Usage in AI Workflow
|
||||
|
||||
### Example 1: Custom Login Plugin
|
||||
|
||||
**User:** "Create a plugin that adds a custom login page at /vendor-login"
|
||||
|
||||
**AI Actions:**
|
||||
1. Generates plugin code
|
||||
2. Calls `test_plugin_external_wp`:
|
||||
|
||||
```javascript
|
||||
{
|
||||
"plugin_path": "./vendor-login-plugin",
|
||||
"test_mode": "both",
|
||||
"required_plugins": [],
|
||||
"test_scenarios": [
|
||||
{
|
||||
"name": "Endpoint accessible",
|
||||
"type": "endpoint",
|
||||
"url": "/vendor-login",
|
||||
"assertions": {
|
||||
"status_code": 200,
|
||||
"contains": ["<form", "login"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Rewrite rule exists",
|
||||
"type": "custom",
|
||||
"wp_cli_command": "wp rewrite list --match=/vendor-login",
|
||||
"assertions": {
|
||||
"contains": ["vendor-login"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"visual_tests": [
|
||||
{
|
||||
"name": "Login form renders",
|
||||
"url": "/vendor-login",
|
||||
"actions": [
|
||||
{ "action": "goto", "url": "/vendor-login" },
|
||||
{ "action": "screenshot", "name": "login-page" }
|
||||
],
|
||||
"assertions": {
|
||||
"visible": ["input[name='log']", "input[type='submit']"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Result:**
|
||||
- Subsite created: `https://testsite.example.com/test-a3f7d2`
|
||||
- Plugin installed and activated
|
||||
- Tests run (5 CLI, 3 visual)
|
||||
- Screenshot: `login-page.png` showing working form
|
||||
- AI confirms: "Plugin tested and verified - custom login page working"
|
||||
|
||||
---
|
||||
|
||||
### Example 2: WooCommerce Payment Gateway
|
||||
|
||||
**User:** "Create a WooCommerce payment gateway"
|
||||
|
||||
**AI Actions:**
|
||||
```javascript
|
||||
{
|
||||
"plugin_path": "./custom-gateway",
|
||||
"test_mode": "both",
|
||||
"required_plugins": [
|
||||
{ "plugin_slug": "woocommerce", "activate": true }
|
||||
],
|
||||
"test_scenarios": [
|
||||
{
|
||||
"name": "Gateway registered",
|
||||
"type": "custom",
|
||||
"wp_cli_command": "wp eval 'var_dump(array_keys(WC()->payment_gateways->payment_gateways()));'",
|
||||
"assertions": {
|
||||
"contains": ["custom_gateway"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Checkout loads without errors",
|
||||
"type": "endpoint",
|
||||
"url": "/checkout",
|
||||
"assertions": {
|
||||
"status_code": 200,
|
||||
"not_contains": ["Fatal error", "Warning"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"visual_tests": [
|
||||
{
|
||||
"name": "Gateway appears in checkout",
|
||||
"url": "/checkout",
|
||||
"actions": [
|
||||
{ "action": "goto", "url": "/product/test-product" },
|
||||
{ "action": "click", "selector": ".add_to_cart_button" },
|
||||
{ "action": "wait", "timeout": 2000 },
|
||||
{ "action": "goto", "url": "/checkout" },
|
||||
{ "action": "screenshot", "name": "checkout-gateway" }
|
||||
],
|
||||
"assertions": {
|
||||
"visible": ["#payment_method_custom_gateway"],
|
||||
"no_console_errors": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Monitoring & Maintenance
|
||||
|
||||
### Health Checks
|
||||
|
||||
```javascript
|
||||
// Periodic health check
|
||||
async function healthCheck() {
|
||||
const checks = {
|
||||
ssh: await checkSshConnection(),
|
||||
wp_cli: await checkWpCli(),
|
||||
multisite: await checkMultisiteEnabled(),
|
||||
disk_space: await checkDiskSpace(),
|
||||
queue_length: testQueue.pendingQueue.length
|
||||
};
|
||||
|
||||
if (Object.values(checks).some(c => !c.ok)) {
|
||||
await alertAdmin(checks);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Metrics to Track
|
||||
- Test queue length and wait times
|
||||
- Average test duration per mode
|
||||
- Failure rate by plugin type
|
||||
- Auto-fix success rate
|
||||
- Resource usage (disk, memory)
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
This system provides:
|
||||
|
||||
1. **Full Isolation:** Each test runs in its own multisite subsite
|
||||
2. **High Concurrency:** Queue system handles 20+ simultaneous tests
|
||||
3. **Two Testing Modes:**
|
||||
- CLI mode: Fast, deterministic WP-CLI testing
|
||||
- Visual mode: Real browser automation with screenshots
|
||||
4. **Automatic Setup:** Provisions subsite, installs dependencies, deploys plugin
|
||||
5. **Auto-Fix:** Attempts to fix common failures automatically
|
||||
6. **Proof of Work:** Returns screenshots, logs, and detailed test results
|
||||
7. **Cleanup:** Automatic teardown after test completion
|
||||
|
||||
**Result:** Users receive plugins that are proven to work on a real WordPress environment, with objective evidence of functionality.
|
||||
Reference in New Issue
Block a user