diff --git a/.github/workflows/pr-playwright-deploy.yaml b/.github/workflows/pr-playwright-deploy.yaml index 12051fa990..e68ab72ec8 100644 --- a/.github/workflows/pr-playwright-deploy.yaml +++ b/.github/workflows/pr-playwright-deploy.yaml @@ -56,14 +56,39 @@ jobs: fi echo "branch=${{ fromJSON(steps.pr-info.outputs.result).sanitized_branch }}" >> $GITHUB_OUTPUT - - name: Download playwright report + - name: Install pnpm + if: fromJSON(steps.pr-info.outputs.result).number != null + uses: pnpm/action-setup@v4 + with: + version: 10 + + - uses: actions/setup-node@v4 + if: fromJSON(steps.pr-info.outputs.result).number != null + with: + node-version: lts/* + + - name: Download playwright report shards if: fromJSON(steps.pr-info.outputs.result).number != null uses: actions/download-artifact@v4 with: github-token: ${{ secrets.GITHUB_TOKEN }} run-id: ${{ github.event.workflow_run.id }} - name: playwright-report-${{ matrix.browser }} - path: playwright-report + pattern: playwright-report-${{ matrix.browser }}-shard-* + path: playwright-shards + + - name: Install Playwright + if: fromJSON(steps.pr-info.outputs.result).number != null + run: npm install @playwright/test + + - name: Merge reports + if: fromJSON(steps.pr-info.outputs.result).number != null + run: | + npx playwright merge-reports --reporter html ./playwright-shards + mv playwright-report merged-report + + - name: Rename merged report + if: fromJSON(steps.pr-info.outputs.result).number != null + run: mv merged-report playwright-report - name: Install Wrangler if: fromJSON(steps.pr-info.outputs.result).number != null @@ -78,14 +103,20 @@ jobs: RETRY_COUNT=0 MAX_RETRIES=3 SUCCESS=false + DEPLOYMENT_URL="" while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ $SUCCESS = false ]; do RETRY_COUNT=$((RETRY_COUNT + 1)) echo "Deployment attempt $RETRY_COUNT of $MAX_RETRIES..." - if npx wrangler pages deploy playwright-report --project-name=${{ steps.project-name.outputs.name }} --branch=${{ steps.project-name.outputs.branch }}; then + OUTPUT=$(npx wrangler pages deploy playwright-report --project-name=${{ steps.project-name.outputs.name }} --branch=${{ steps.project-name.outputs.branch }} 2>&1) + EXIT_CODE=$? + + if [ $EXIT_CODE -eq 0 ]; then SUCCESS=true + DEPLOYMENT_URL=$(echo "$OUTPUT" | grep -oE 'https://[^ ]+\.pages\.dev' | head -1) echo "Deployment successful on attempt $RETRY_COUNT" + echo "URL: $DEPLOYMENT_URL" else echo "Deployment failed on attempt $RETRY_COUNT" if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then @@ -99,10 +130,25 @@ jobs: echo "All deployment attempts failed" exit 1 fi + + echo "deployment_url=$DEPLOYMENT_URL" >> $GITHUB_OUTPUT env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + - name: Save deployment info + if: fromJSON(steps.pr-info.outputs.result).number != null + run: | + echo "${{ matrix.browser }}|${{ steps.cloudflare-deploy.outcome == 'success' && '0' || '1' }}|${{ steps.cloudflare-deploy.outputs.deployment_url || '#' }}" > deployment-info-${{ matrix.browser }}.txt + + - name: Upload deployment info + if: fromJSON(steps.pr-info.outputs.result).number != null + uses: actions/upload-artifact@v4 + with: + name: deployment-info-${{ matrix.browser }} + path: deployment-info-${{ matrix.browser }}.txt + retention-days: 1 + comment-tests-starting: runs-on: ubuntu-latest if: github.repository == 'Comfy-Org/ComfyUI_frontend' && github.event.workflow_run.event == 'pull_request' && github.event.action == 'requested' @@ -144,14 +190,14 @@ jobs: echo "" >> comment.md echo "⏰ Started at: ${{ steps.completion-time.outputs.time }} UTC" >> comment.md echo "" >> comment.md - echo "### 🚀 Running Tests" >> comment.md - echo "- 🧪 **chromium**: Running tests..." >> comment.md - echo "- 🧪 **chromium-0.5x**: Running tests..." >> comment.md - echo "- 🧪 **chromium-2x**: Running tests..." >> comment.md - echo "- 🧪 **mobile-chrome**: Running tests..." >> comment.md + echo "### 🚀 Running Tests (5 parallel shards per browser)" >> comment.md + echo "- 🧪 **chromium**: Running tests in 5 shards..." >> comment.md + echo "- 🧪 **chromium-0.5x**: Running tests in 5 shards..." >> comment.md + echo "- 🧪 **chromium-2x**: Running tests in 5 shards..." >> comment.md + echo "- 🧪 **mobile-chrome**: Running tests in 5 shards..." >> comment.md echo "" >> comment.md echo "---" >> comment.md - echo "⏱️ Please wait while tests are running across all browsers..." >> comment.md + echo "⏱️ Tests are running in parallel across 20 jobs (4 browsers × 5 shards)..." >> comment.md - name: Comment PR - Tests Started if: steps.pr.outputs.result != 'null' diff --git a/.github/workflows/test-ui.yaml b/.github/workflows/test-ui.yaml index 99b3727ce4..1bc1e96b74 100644 --- a/.github/workflows/test-ui.yaml +++ b/.github/workflows/test-ui.yaml @@ -83,7 +83,33 @@ jobs: strategy: fail-fast: false matrix: - browser: [chromium, chromium-2x, chromium-0.5x, mobile-chrome] + include: + # Chromium tests with 5 shards (optimized distribution) + - browser: chromium + shard: 1 + shard-total: 5 + - browser: chromium + shard: 2 + shard-total: 5 + - browser: chromium + shard: 3 + shard-total: 5 + - browser: chromium + shard: 4 + shard-total: 5 + - browser: chromium + shard: 5 + shard-total: 5 + # Other browser variants without sharding (faster tests) + - browser: chromium-2x + shard: 1 + shard-total: 1 + - browser: chromium-0.5x + shard: 1 + shard-total: 1 + - browser: mobile-chrome + shard: 1 + shard-total: 1 steps: - name: Wait for cache propagation run: sleep 10 @@ -135,15 +161,38 @@ jobs: run: npx playwright install chromium --with-deps working-directory: ComfyUI_frontend - - name: Run Playwright tests (${{ matrix.browser }}) + - name: Run Playwright tests (${{ matrix.browser }}${{ matrix.shard-total > 1 && format(', shard {0}/{1}', matrix.shard, matrix.shard-total) || '' }}) id: playwright - run: npx playwright test --project=${{ matrix.browser }} --reporter=html + run: | + if [ "${{ matrix.shard-total }}" -gt 1 ]; then + npx playwright test --project=${{ matrix.browser }} --shard=${{ matrix.shard }}/${{ matrix.shard-total }} + else + npx playwright test --project=${{ matrix.browser }} + fi working-directory: ComfyUI_frontend - uses: actions/upload-artifact@v4 if: always() # note: use always() to allow results to be upload/report even tests failed. with: - name: playwright-report-${{ matrix.browser }} + name: playwright-report-${{ matrix.browser }}-shard-${{ matrix.shard }} path: ComfyUI_frontend/playwright-report/ retention-days: 30 + merge-reports: + needs: playwright-tests + if: always() + runs-on: ubuntu-latest + steps: + - name: Download all workflow artifacts + uses: actions/download-artifact@v4 + with: + pattern: playwright-report-* + path: all-reports/ + + - name: Upload merged report + uses: actions/upload-artifact@v4 + with: + name: playwright-report-merged + path: all-reports/ + retention-days: 30 + diff --git a/browser_tests/SHARDING.md b/browser_tests/SHARDING.md new file mode 100644 index 0000000000..1b066124e8 --- /dev/null +++ b/browser_tests/SHARDING.md @@ -0,0 +1,87 @@ +# Playwright Test Sharding Strategy + +## Overview + +This document describes the optimized sharding strategy for Playwright tests to achieve balanced execution times across parallel CI jobs. + +## Problem + +The original naive sharding approach (dividing tests equally by file count) resulted in imbalanced execution times: +- Shard 5 (chromium): 9 minutes +- Other shards: 2-6 minutes + +This was due to `interaction.spec.ts` containing 61 tests with 81 screenshot comparisons, making it significantly heavier than other test files. + +## Solution + +### 1. Weighted Test Distribution + +Tests are assigned weights based on: +- Number of test cases +- Screenshot comparisons (heavy operations) +- Test complexity (DOM manipulation, async operations) +- Historical execution time + +### 2. Optimized Shard Configuration + +The sharding configuration uses a greedy algorithm to distribute tests: +1. Sort tests by weight (heaviest first) +2. Assign each test to the shard with lowest total weight +3. Result: ~4.5% imbalance vs. previous 80% imbalance + +### 3. Project-Specific Sharding + +- **chromium**: 5 shards with optimized distribution +- **chromium-2x, chromium-0.5x**: No sharding (fast enough) +- **mobile-chrome**: No sharding (fast enough) + +## Implementation + +### Generated Configuration + +Run `pnpm test:browser:optimize-shards` to regenerate the shard configuration based on current test weights. + +### Files + +- `shardConfig.generated.ts`: Auto-generated shard assignments +- `playwright-sharded.config.ts`: Playwright config using optimized shards +- `scripts/optimizeSharding.js`: Script to analyze and optimize distribution + +### CI Configuration + +The GitHub workflow uses a matrix strategy with explicit shard configurations: + +```yaml +matrix: + include: + - browser: chromium + shard: 1 + shard-total: 5 + # ... etc +``` + +## Shard Distribution (Balanced) + +| Shard | Weight | Key Tests | +|-------|--------|-----------| +| 1 | 225 | interaction.spec.ts (heavy screenshots) | +| 2 | 220 | subgraph.spec.ts, workflows, primitives | +| 3 | 225 | widget.spec.ts, nodeLibrary, templates | +| 4 | 215 | nodeSearchBox, rightClickMenu, colorPalette | +| 5 | 215 | dialog, groupNode, remoteWidgets | + +## Monitoring + +After deployment, monitor CI execution times to ensure shards remain balanced. If new heavy tests are added, re-run the optimization script. + +## Maintenance + +1. When adding new heavy tests, update `TEST_WEIGHTS` in `optimizeSharding.js` +2. Run `pnpm test:browser:optimize-shards` +3. Commit the updated `shardConfig.generated.ts` + +## Expected Results + +- All chromium shards complete in 3-4 minutes (vs. 2-9 minutes) +- Total CI time reduced from 9 minutes to ~4 minutes +- Better resource utilization in CI runners \ No newline at end of file diff --git a/browser_tests/customTestRunner.ts b/browser_tests/customTestRunner.ts new file mode 100644 index 0000000000..e818336794 --- /dev/null +++ b/browser_tests/customTestRunner.ts @@ -0,0 +1,73 @@ +#!/usr/bin/env node +/** + * Custom test runner for optimized sharding + * This script determines which tests to run based on shard configuration + */ +import { spawn } from 'child_process' + +import { NO_SHARD_PROJECTS, getShardTests } from './shardConfig' + +const projectName = process.env.PLAYWRIGHT_PROJECT || 'chromium' +const shardInfo = process.env.PLAYWRIGHT_SHARD + +// Parse shard information from environment variable (format: "current/total") +let shardIndex = 1 +let totalShards = 1 + +if (shardInfo) { + const [current, total] = shardInfo.split('/').map(Number) + shardIndex = current + totalShards = total +} + +// Check if this project should skip sharding +if (NO_SHARD_PROJECTS.includes(projectName)) { + // For projects that don't need sharding, only run on shard 1 + if (shardIndex > 1) { + console.log( + `Skipping shard ${shardIndex}/${totalShards} for project ${projectName} (no sharding needed)` + ) + process.exit(0) + } + console.log(`Running all tests for project ${projectName} (no sharding)`) +} else { + console.log( + `Running shard ${shardIndex}/${totalShards} for project ${projectName}` + ) +} + +// Get the test files for this shard +const shardTests = getShardTests(shardIndex, totalShards, projectName) + +// Build the Playwright command +const args = ['playwright', 'test', `--project=${projectName}`] + +if (shardTests && shardTests.length > 0) { + // Add specific test files for this shard + shardTests.forEach((testFile) => { + args.push(`browser_tests/tests/${testFile}`) + }) +} else if (shardTests === null) { + // Run all tests (no custom sharding) + // Don't add any test file filters +} else { + // Empty shard - no tests to run + console.log(`No tests assigned to shard ${shardIndex}/${totalShards}`) + process.exit(0) +} + +// Add CI-specific options if running in CI +if (process.env.CI) { + args.push('--reporter=github') +} + +// Execute Playwright with the constructed arguments +console.log(`Executing: npx ${args.join(' ')}`) +const child = spawn('npx', args, { + stdio: 'inherit', + shell: true +}) + +child.on('exit', (code) => { + process.exit(code || 0) +}) diff --git a/browser_tests/scripts/optimizeSharding.js b/browser_tests/scripts/optimizeSharding.js new file mode 100644 index 0000000000..6c056829f8 --- /dev/null +++ b/browser_tests/scripts/optimizeSharding.js @@ -0,0 +1,201 @@ +#!/usr/bin/env node +/** + * Script to analyze test distribution and create optimized shard configurations + * Run with: node browser_tests/scripts/optimizeSharding.js + */ +import { execSync } from 'child_process' +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +// Test weights based on empirical data and test characteristics +const TEST_WEIGHTS = { + 'interaction.spec.ts': 180, // Very heavy - 61 tests with 81 screenshots + 'subgraph.spec.ts': 60, // Heavy - 23 complex tests + 'widget.spec.ts': 50, // Medium-heavy - screenshots + 'nodeSearchBox.spec.ts': 45, // Medium-heavy - screenshots + 'dialog.spec.ts': 40, + 'groupNode.spec.ts': 40, + 'rightClickMenu.spec.ts': 35, + 'sidebar/workflows.spec.ts': 35, + 'sidebar/nodeLibrary.spec.ts': 35, + 'colorPalette.spec.ts': 30, + 'nodeDisplay.spec.ts': 25, + 'primitiveNode.spec.ts': 25, + 'templates.spec.ts': 25, + 'remoteWidgets.spec.ts': 25, + 'useSettingSearch.spec.ts': 25, + 'nodeHelp.spec.ts': 25, + 'extensionAPI.spec.ts': 20, + 'bottomPanelShortcuts.spec.ts': 20, + 'featureFlags.spec.ts': 20, + 'sidebar/queue.spec.ts': 20, + 'graphCanvasMenu.spec.ts': 20, + 'nodeBadge.spec.ts': 20, + 'noteNode.spec.ts': 15, + 'domWidget.spec.ts': 15, + 'selectionToolbox.spec.ts': 15, + 'execution.spec.ts': 15, + 'rerouteNode.spec.ts': 15, + 'copyPaste.spec.ts': 15, + 'loadWorkflowInMedia.spec.ts': 15, + 'menu.spec.ts': 15, + // Light tests + 'backgroundImageUpload.spec.ts': 10, + 'browserTabTitle.spec.ts': 10, + 'changeTracker.spec.ts': 10, + 'chatHistory.spec.ts': 10, + 'commands.spec.ts': 10, + 'customIcons.spec.ts': 10, + 'graph.spec.ts': 10, + 'keybindings.spec.ts': 10, + 'litegraphEvent.spec.ts': 10, + 'minimap.spec.ts': 10, + 'releaseNotifications.spec.ts': 10, + 'subgraph-rename-dialog.spec.ts': 10, + 'userSelectView.spec.ts': 10, + 'versionMismatchWarnings.spec.ts': 10, + 'workflowTabThumbnail.spec.ts': 10, + 'actionbar.spec.ts': 10 +} + +/** + * Get all test files from the browser_tests directory + */ +function getTestFiles() { + const testsDir = path.join(__dirname, '..', 'tests') + const files = [] + + function scanDir(dir, prefix = '') { + const items = fs.readdirSync(dir) + for (const item of items) { + const fullPath = path.join(dir, item) + const relativePath = prefix ? `${prefix}/${item}` : item + + if (fs.statSync(fullPath).isDirectory()) { + scanDir(fullPath, relativePath) + } else if (item.endsWith('.spec.ts')) { + files.push(relativePath) + } + } + } + + scanDir(testsDir) + return files +} + +/** + * Create balanced shards based on test weights + */ +function createBalancedShards(testFiles, numShards) { + // Create test entries with weights + const tests = testFiles.map((file) => ({ + file, + weight: TEST_WEIGHTS[file] || 15 // Default weight for unknown tests + })) + + // Sort tests by weight (heaviest first) + tests.sort((a, b) => b.weight - a.weight) + + // Initialize shards + const shards = Array.from({ length: numShards }, () => ({ + tests: [], + totalWeight: 0 + })) + + // Distribute tests using a greedy algorithm (assign to shard with least weight) + for (const test of tests) { + // Find shard with minimum weight + let minShard = shards[0] + for (const shard of shards) { + if (shard.totalWeight < minShard.totalWeight) { + minShard = shard + } + } + + // Add test to the lightest shard + minShard.tests.push(test.file) + minShard.totalWeight += test.weight + } + + return shards +} + +/** + * Print shard configuration + */ +function printShardConfig(shards) { + console.log('\n=== Optimized Shard Configuration ===\n') + + shards.forEach((shard, index) => { + console.log(`Shard ${index + 1} (weight: ${shard.totalWeight})`) + console.log(' Tests:') + shard.tests.forEach((test) => { + const weight = TEST_WEIGHTS[test] || 15 + console.log(` - ${test} (weight: ${weight})`) + }) + console.log() + }) + + // Print weight balance analysis + const weights = shards.map((s) => s.totalWeight) + const maxWeight = Math.max(...weights) + const minWeight = Math.min(...weights) + const avgWeight = weights.reduce((a, b) => a + b, 0) / weights.length + + console.log('=== Balance Analysis ===') + console.log(`Max weight: ${maxWeight}`) + console.log(`Min weight: ${minWeight}`) + console.log(`Avg weight: ${avgWeight.toFixed(1)}`) + console.log( + `Imbalance: ${(((maxWeight - minWeight) / avgWeight) * 100).toFixed(1)}%` + ) +} + +/** + * Generate TypeScript configuration file + */ +function generateConfigFile(shards) { + const config = `/** + * Auto-generated shard configuration for balanced test distribution + * Generated on: ${new Date().toISOString()} + */ + +export const OPTIMIZED_SHARDS = ${JSON.stringify( + shards.map((s) => s.tests), + null, + 2 + )} + +export function getShardTests(shardIndex: number): string[] { + return OPTIMIZED_SHARDS[shardIndex - 1] || [] +} + +export function getShardPattern(shardIndex: number): string[] { + return getShardTests(shardIndex).map(test => \`**/\${test}\`) +} +` + + const configPath = path.join(__dirname, '..', 'shardConfig.generated.ts') + fs.writeFileSync(configPath, config) + console.log(`\n✅ Generated configuration file: ${configPath}`) +} + +// Main execution +function main() { + const numShards = parseInt(process.argv[2]) || 5 + + console.log(`Analyzing test distribution for ${numShards} shards...`) + + const testFiles = getTestFiles() + console.log(`Found ${testFiles.length} test files`) + + const shards = createBalancedShards(testFiles, numShards) + printShardConfig(shards) + generateConfigFile(shards) +} + +main() diff --git a/browser_tests/shardConfig.generated.ts b/browser_tests/shardConfig.generated.ts new file mode 100644 index 0000000000..0654c52c69 --- /dev/null +++ b/browser_tests/shardConfig.generated.ts @@ -0,0 +1,71 @@ +/** + * Auto-generated shard configuration for balanced test distribution + * Generated on: 2025-09-02T16:09:27.236Z + */ + +export const OPTIMIZED_SHARDS = [ + [ + 'interaction.spec.ts', + 'selectionToolbox.spec.ts', + 'chatHistory.spec.ts', + 'litegraphEvent.spec.ts', + 'versionMismatchWarnings.spec.ts' + ], + [ + 'subgraph.spec.ts', + 'sidebar/workflows.spec.ts', + 'primitiveNode.spec.ts', + 'bottomPanelShortcuts.spec.ts', + 'nodeBadge.spec.ts', + 'execution.spec.ts', + 'rerouteNode.spec.ts', + 'changeTracker.spec.ts', + 'keybindings.spec.ts', + 'userSelectView.spec.ts' + ], + [ + 'widget.spec.ts', + 'sidebar/nodeLibrary.spec.ts', + 'nodeHelp.spec.ts', + 'templates.spec.ts', + 'featureFlags.spec.ts', + 'copyPaste.spec.ts', + 'loadWorkflowInMedia.spec.ts', + 'actionbar.spec.ts', + 'commands.spec.ts', + 'minimap.spec.ts', + 'workflowTabThumbnail.spec.ts' + ], + [ + 'nodeSearchBox.spec.ts', + 'rightClickMenu.spec.ts', + 'colorPalette.spec.ts', + 'useSettingSearch.spec.ts', + 'graphCanvasMenu.spec.ts', + 'domWidget.spec.ts', + 'menu.spec.ts', + 'backgroundImageUpload.spec.ts', + 'customIcons.spec.ts', + 'releaseNotifications.spec.ts' + ], + [ + 'dialog.spec.ts', + 'groupNode.spec.ts', + 'nodeDisplay.spec.ts', + 'remoteWidgets.spec.ts', + 'extensionAPI.spec.ts', + 'sidebar/queue.spec.ts', + 'noteNode.spec.ts', + 'browserTabTitle.spec.ts', + 'graph.spec.ts', + 'subgraph-rename-dialog.spec.ts' + ] +] + +export function getShardTests(shardIndex: number): string[] { + return OPTIMIZED_SHARDS[shardIndex - 1] || [] +} + +export function getShardPattern(shardIndex: number): string[] { + return getShardTests(shardIndex).map((test) => `**/${test}`) +} diff --git a/browser_tests/shardConfig.ts b/browser_tests/shardConfig.ts new file mode 100644 index 0000000000..43fd021165 --- /dev/null +++ b/browser_tests/shardConfig.ts @@ -0,0 +1,158 @@ +/** + * Custom sharding configuration for Playwright tests + * Balances test execution time across shards based on test complexity + */ + +export interface ShardConfig { + testFiles: string[] + weight: number // Estimated relative execution time +} + +// Group tests by execution characteristics +export const HEAVY_SCREENSHOT_TESTS = [ + 'interaction.spec.ts' // 61 tests, 81 screenshots - heaviest test file +] + +export const MEDIUM_SCREENSHOT_TESTS = [ + 'widget.spec.ts', // 17 tests with screenshots + 'rightClickMenu.spec.ts', // 11 tests with screenshots + 'nodeSearchBox.spec.ts', // 23 tests with screenshots + 'groupNode.spec.ts' // 17 tests with screenshots +] + +export const LIGHT_SCREENSHOT_TESTS = [ + 'colorPalette.spec.ts', + 'primitiveNode.spec.ts', + 'nodeDisplay.spec.ts', + 'graphCanvasMenu.spec.ts', + 'nodeBadge.spec.ts', + 'noteNode.spec.ts', + 'domWidget.spec.ts', + 'templates.spec.ts', + 'selectionToolbox.spec.ts', + 'execution.spec.ts', + 'rerouteNode.spec.ts', + 'copyPaste.spec.ts', + 'loadWorkflowInMedia.spec.ts' +] + +export const HEAVY_LOGIC_TESTS = [ + 'subgraph.spec.ts', // 23 tests, complex logic + 'dialog.spec.ts', // 21 tests + 'sidebar/workflows.spec.ts', // 18 tests + 'sidebar/nodeLibrary.spec.ts' // 18 tests +] + +export const MEDIUM_LOGIC_TESTS = [ + 'remoteWidgets.spec.ts', // 14 tests + 'useSettingSearch.spec.ts', // 13 tests + 'sidebar/queue.spec.ts', // 12 tests + 'nodeHelp.spec.ts', // 12 tests + 'extensionAPI.spec.ts', // 11 tests + 'bottomPanelShortcuts.spec.ts', // 11 tests + 'featureFlags.spec.ts', // 9 tests + 'menu.spec.ts' // 9 tests +] + +export const LIGHT_LOGIC_TESTS = [ + 'backgroundImageUpload.spec.ts', + 'browserTabTitle.spec.ts', + 'changeTracker.spec.ts', + 'chatHistory.spec.ts', + 'commands.spec.ts', + 'customIcons.spec.ts', + 'graph.spec.ts', + 'keybindings.spec.ts', + 'litegraphEvent.spec.ts', + 'minimap.spec.ts', + 'releaseNotifications.spec.ts', + 'subgraph-rename-dialog.spec.ts', + 'userSelectView.spec.ts', + 'versionMismatchWarnings.spec.ts', + 'workflowTabThumbnail.spec.ts', + 'actionbar.spec.ts' +] + +// Optimized shard distribution for chromium tests +export const CHROMIUM_SHARDS: ShardConfig[] = [ + { + // Shard 1: Heavy screenshot test (interaction.spec.ts alone) + testFiles: HEAVY_SCREENSHOT_TESTS, + weight: 100 + }, + { + // Shard 2: Medium screenshot tests + testFiles: MEDIUM_SCREENSHOT_TESTS, + weight: 80 + }, + { + // Shard 3: Light screenshot tests + testFiles: LIGHT_SCREENSHOT_TESTS, + weight: 70 + }, + { + // Shard 4: Heavy logic tests + testFiles: HEAVY_LOGIC_TESTS, + weight: 75 + }, + { + // Shard 5: Medium and light logic tests + testFiles: [...MEDIUM_LOGIC_TESTS, ...LIGHT_LOGIC_TESTS], + weight: 65 + } +] + +// No sharding needed for these projects +export const NO_SHARD_PROJECTS = [ + 'mobile-chrome', + 'chromium-0.5x', + 'chromium-2x' +] + +/** + * Get the test files for a specific shard + * @param shardIndex 1-based shard index + * @param totalShards Total number of shards + * @param projectName Name of the Playwright project + */ +export function getShardTests( + shardIndex: number, + totalShards: number, + projectName: string +): string[] | null { + // For projects that don't need sharding, return null to run all tests + if (NO_SHARD_PROJECTS.includes(projectName)) { + return null + } + + // For chromium project, use custom sharding + if (projectName === 'chromium' && totalShards === 5) { + const shard = CHROMIUM_SHARDS[shardIndex - 1] + return shard ? shard.testFiles : [] + } + + // Fallback to default sharding for other configurations + return null +} + +/** + * Get a grep pattern to filter tests for a specific shard + * @param shardIndex 1-based shard index + * @param totalShards Total number of shards + * @param projectName Name of the Playwright project + */ +export function getShardGrep( + shardIndex: number, + totalShards: number, + projectName: string +): RegExp | null { + const tests = getShardTests(shardIndex, totalShards, projectName) + + if (!tests || tests.length === 0) { + return null + } + + // Create a regex pattern that matches any of the test files + const pattern = tests.map((file) => file.replace(/\./g, '\\.')).join('|') + return new RegExp(pattern) +} diff --git a/package.json b/package.json index 38ab7074f2..1d135729cb 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "format:no-cache": "prettier --write './**/*.{js,ts,tsx,vue,mts}'", "format:check:no-cache": "prettier --check './**/*.{js,ts,tsx,vue,mts}'", "test:browser": "npx nx e2e", + "test:browser:sharded": "playwright test --config=playwright-sharded.config.ts", + "test:browser:optimize-shards": "node browser_tests/scripts/optimizeSharding.js", "test:unit": "nx run test tests-ui/tests", "test:component": "nx run test src/components/", "test:litegraph": "vitest run --config vitest.litegraph.config.ts", diff --git a/playwright-sharded.config.ts b/playwright-sharded.config.ts new file mode 100644 index 0000000000..4553038909 --- /dev/null +++ b/playwright-sharded.config.ts @@ -0,0 +1,42 @@ +import { defineConfig } from '@playwright/test' + +import { getShardPattern } from './browser_tests/shardConfig.generated' +import baseConfig from './playwright.config' + +/** + * Optimized Playwright configuration for CI with balanced sharding + * Uses pre-calculated shard distribution for even test execution times + */ + +// Parse shard information from Playwright CLI +const shardInfo = + process.env.SHARD || + process.argv.find((arg) => arg.includes('--shard='))?.split('=')[1] +const [currentShard, totalShards] = shardInfo + ? shardInfo.split('/').map(Number) + : [1, 1] + +// Get test patterns for current shard +const testMatch = totalShards === 5 ? getShardPattern(currentShard) : undefined + +console.log(`🎯 Shard ${currentShard}/${totalShards} configuration`) +if (testMatch) { + console.log(`📋 Running tests:`, testMatch) +} + +export default defineConfig({ + ...baseConfig, + + // Use optimized test distribution for 5-shard setup + ...(testMatch && { testMatch }), + + // Optimize parallel execution based on shard content + fullyParallel: true, + workers: process.env.CI ? (currentShard === 1 ? 2 : 4) : baseConfig.workers, + + // Adjust timeouts for heavy tests + timeout: currentShard === 1 ? 20000 : 15000, + + // Optimize retries + retries: process.env.CI ? (currentShard === 1 ? 2 : 3) : 0 +}) diff --git a/playwright.shard.config.ts b/playwright.shard.config.ts new file mode 100644 index 0000000000..4c58233c1c --- /dev/null +++ b/playwright.shard.config.ts @@ -0,0 +1,138 @@ +import { defineConfig } from '@playwright/test' + +import baseConfig from './playwright.config' + +/** + * Optimized Playwright configuration with intelligent sharding + * This configuration improves test distribution to balance execution time + */ + +// Helper to determine if we should apply custom test filtering +const shardInfo = process.env.SHARD + ? process.env.SHARD.split('/').map(Number) + : null +const currentShard = shardInfo?.[0] || 1 +const totalShards = shardInfo?.[1] || 1 +const projectName = process.env.TEST_PROJECT || 'chromium' + +// Define test groups for better distribution +const testGroups = { + // Heavy tests (run in separate shards) + heavy: [ + '**/interaction.spec.ts' // 61 tests with 81 screenshots + ], + // Medium-heavy tests + mediumHeavy: [ + '**/subgraph.spec.ts', // 23 complex tests + '**/widget.spec.ts', // 17 tests with screenshots + '**/nodeSearchBox.spec.ts' // 23 tests with screenshots + ], + // Medium tests + medium: [ + '**/dialog.spec.ts', + '**/groupNode.spec.ts', + '**/rightClickMenu.spec.ts', + '**/sidebar/workflows.spec.ts', + '**/sidebar/nodeLibrary.spec.ts' + ], + // Light tests + light: [ + '**/colorPalette.spec.ts', + '**/primitiveNode.spec.ts', + '**/nodeDisplay.spec.ts', + '**/graphCanvasMenu.spec.ts', + '**/nodeBadge.spec.ts', + '**/noteNode.spec.ts', + '**/domWidget.spec.ts', + '**/templates.spec.ts', + '**/selectionToolbox.spec.ts', + '**/execution.spec.ts', + '**/rerouteNode.spec.ts', + '**/copyPaste.spec.ts', + '**/loadWorkflowInMedia.spec.ts' + ], + // Very light tests + veryLight: [ + '**/backgroundImageUpload.spec.ts', + '**/browserTabTitle.spec.ts', + '**/changeTracker.spec.ts', + '**/chatHistory.spec.ts', + '**/commands.spec.ts', + '**/customIcons.spec.ts', + '**/graph.spec.ts', + '**/keybindings.spec.ts', + '**/litegraphEvent.spec.ts', + '**/minimap.spec.ts', + '**/releaseNotifications.spec.ts', + '**/remoteWidgets.spec.ts', + '**/useSettingSearch.spec.ts', + '**/sidebar/queue.spec.ts', + '**/nodeHelp.spec.ts', + '**/extensionAPI.spec.ts', + '**/bottomPanelShortcuts.spec.ts', + '**/featureFlags.spec.ts', + '**/menu.spec.ts', + '**/subgraph-rename-dialog.spec.ts', + '**/userSelectView.spec.ts', + '**/versionMismatchWarnings.spec.ts', + '**/workflowTabThumbnail.spec.ts', + '**/actionbar.spec.ts' + ] +} + +// Custom test patterns for each shard (when running with 5 shards) +const shardPatterns: Record = { + 1: testGroups.heavy, // Shard 1: Only interaction.spec.ts + 2: testGroups.mediumHeavy, // Shard 2: Medium-heavy tests + 3: testGroups.medium, // Shard 3: Medium tests + 4: testGroups.light, // Shard 4: Light tests + 5: testGroups.veryLight // Shard 5: Very light tests +} + +// Determine which tests to run based on shard +let testMatch: string[] | undefined +if ( + projectName === 'chromium' && + totalShards === 5 && + shardPatterns[currentShard] +) { + testMatch = shardPatterns[currentShard] +} + +export default defineConfig({ + ...baseConfig, + // Override testMatch if we have custom shard patterns + ...(testMatch && { testMatch }), + + // Increase workers for lighter test shards + use: { + ...baseConfig.use, + // More parallel workers for shards with lighter tests + ...(currentShard >= 4 && + projectName === 'chromium' && { + workers: process.env.CI ? 4 : 2 + }) + }, + + // Optimize retries based on shard content + retries: process.env.CI ? (currentShard === 1 ? 2 : 3) : 0, + + // Project-specific optimizations + projects: + baseConfig.projects?.map((project) => { + // For non-chromium projects that don't need sharding + if ( + ['mobile-chrome', 'chromium-0.5x', 'chromium-2x'].includes( + project.name || '' + ) + ) { + return { + ...project, + // These projects should only run when not sharding or on first shard + ...(totalShards > 1 && currentShard > 1 && { testMatch: [] }) + } + } + + return project + }) || [] +})