From 9935f75cc91f47e7b1096aa4f7e4cfe7ea8afc92 Mon Sep 17 00:00:00 2001 From: midzelis Date: Wed, 25 Mar 2026 02:23:44 +0000 Subject: [PATCH] chore(ci): add unified test report PR comment Change-Id: I1cee5c74dcff06215bf8f75b307a2d296a6a6964 --- .github/scripts/write-test-summary.mjs | 395 +++++++++++++++++++++++++ .github/workflows/test.yml | 153 +++++++++- e2e/docker-compose.yml | 1 + e2e/playwright.config.ts | 6 +- 4 files changed, 547 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/write-test-summary.mjs diff --git a/.github/scripts/write-test-summary.mjs b/.github/scripts/write-test-summary.mjs new file mode 100644 index 0000000000..7503bcb699 --- /dev/null +++ b/.github/scripts/write-test-summary.mjs @@ -0,0 +1,395 @@ +import { readFileSync, appendFileSync, readdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; + +const { values } = parseArgs({ + options: { + json: { type: "string" }, + name: { type: "string" }, + framework: { type: "string", default: "vitest" }, + coverage: { type: "string" }, + "pr-comment": { type: "boolean", default: false }, + "artifacts-dir": { type: "string" }, + }, +}); + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return undefined; + } +} + +function formatDuration(milliseconds) { + if (milliseconds < 1000) { + return `${milliseconds}ms`; + } + const seconds = milliseconds / 1000; + if (seconds < 60) { + return `${seconds.toFixed(1)}s`; + } + const minutes = Math.floor(seconds / 60); + const remainingSeconds = (seconds % 60).toFixed(0); + return `${minutes}m ${remainingSeconds}s`; +} + +function parseVitestResults(data) { + const startTime = data.startTime ?? 0; + const endTime = Math.max( + ...(data.testResults ?? []).map((r) => r.endTime ?? 0), + startTime, + ); + + return { + total: data.numTotalTests ?? 0, + passed: data.numPassedTests ?? 0, + failed: data.numFailedTests ?? 0, + skipped: data.numPendingTests ?? 0, + flaky: 0, + duration: endTime - startTime, + success: data.success ?? false, + }; +} + +function parsePlaywrightResults(data) { + const stats = data.stats ?? {}; + const passed = stats.expected ?? 0; + const failed = stats.unexpected ?? 0; + const flaky = stats.flaky ?? 0; + const skipped = stats.skipped ?? 0; + + return { + total: passed + failed + flaky + skipped, + passed, + failed, + skipped, + flaky, + duration: stats.duration ?? 0, + success: failed === 0, + }; +} + +function parseCoverageSummary(data) { + const total = data.total ?? {}; + + const files = []; + for (const [filePath, entry] of Object.entries(data)) { + if (filePath === "total") { + continue; + } + files.push({ + file: filePath.replace(/^.*?\/src\//, "src/"), + lines: entry.lines?.pct ?? 0, + branches: entry.branches?.pct ?? 0, + functions: entry.functions?.pct ?? 0, + statements: entry.statements?.pct ?? 0, + }); + } + files.sort((a, b) => a.lines - b.lines); + + return { + lines: total.lines?.pct ?? 0, + branches: total.branches?.pct ?? 0, + functions: total.functions?.pct ?? 0, + statements: total.statements?.pct ?? 0, + files, + }; +} + +function buildMarkdown(name, results, coverage) { + const statusIcon = + results.failed > 0 + ? "\u274c" + : results.flaky > 0 + ? "\u26a0\ufe0f" + : "\u2705"; + const lines = []; + + lines.push(`### ${statusIcon} ${name}`); + lines.push(""); + lines.push("| Metric | Value |"); + lines.push("|--------|-------|"); + lines.push(`| Total | ${results.total} |`); + lines.push(`| Passed | ${results.passed} |`); + lines.push(`| Failed | ${results.failed} |`); + lines.push(`| Skipped | ${results.skipped} |`); + if (results.flaky > 0) { + lines.push(`| Flaky | ${results.flaky} |`); + } + lines.push(`| Duration | ${formatDuration(results.duration)} |`); + lines.push(""); + + if (coverage) { + lines.push("#### Coverage"); + lines.push(""); + lines.push("| Metric | Coverage |"); + lines.push("|--------|----------|"); + lines.push(`| Lines | ${coverage.lines}% |`); + lines.push(`| Branches | ${coverage.branches}% |`); + lines.push(`| Functions | ${coverage.functions}% |`); + lines.push(`| Statements | ${coverage.statements}% |`); + lines.push(""); + + if (coverage.files?.length > 0) { + lines.push("
"); + lines.push( + `File coverage (${coverage.files.length} files)`, + ); + lines.push(""); + lines.push("| File | Lines | Branches | Functions |"); + lines.push("|------|-------|----------|-----------|"); + for (const file of coverage.files) { + lines.push( + `| ${file.file} | ${file.lines}% | ${file.branches}% | ${file.functions}% |`, + ); + } + lines.push(""); + lines.push("
"); + lines.push(""); + } + } + + return lines.join("\n"); +} + +const ARTIFACT_CONFIGS = [ + { + pattern: "report-server-unit", + name: "Server Unit Tests", + framework: "vitest", + testFile: "test-results.json", + coverageFile: "coverage/coverage-summary.json", + }, + { + pattern: "report-web-unit", + name: "Web Unit Tests", + framework: "vitest", + testFile: "test-results.json", + coverageFile: "coverage/coverage-summary.json", + }, + { + pattern: "report-server-medium", + name: "Server Medium Tests", + framework: "vitest", + testFile: "test-results-medium.json", + coverageFile: "coverage/coverage-summary.json", + }, + { + pattern: "report-cli-unit", + name: "CLI Unit Tests", + framework: "vitest", + testFile: "test-results.json", + coverageFile: undefined, + }, + { + pattern: "report-cli-unit-win", + name: "CLI Unit Tests (Windows)", + framework: "vitest", + testFile: "test-results.json", + coverageFile: undefined, + }, + { + pattern: "report-e2e-server-cli-", + name: "E2E Server & CLI", + framework: "vitest", + testFile: "test-results.json", + coverageFile: undefined, + }, + { + pattern: "report-e2e-server-maintenance-", + name: "E2E Server Maintenance", + framework: "vitest", + testFile: "test-results.json", + coverageFile: undefined, + }, + { + pattern: "report-e2e-web-", + name: "E2E Web", + framework: "playwright", + testFile: "test-results.json", + coverageFile: undefined, + }, + { + pattern: "report-e2e-web-ui-", + name: "E2E Web UI", + framework: "playwright", + testFile: "test-results.json", + coverageFile: undefined, + }, + { + pattern: "report-e2e-web-maintenance-", + name: "E2E Web Maintenance", + framework: "playwright", + testFile: "test-results.json", + coverageFile: undefined, + }, +]; + +function getStatusIcon(results) { + if (results.failed > 0) { + return "\u274c"; + } + if (results.flaky > 0) { + return "\u26a0\ufe0f"; + } + return "\u2705"; +} + +function discoverArtifacts(artifactsDir) { + const dirs = readdirSync(artifactsDir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + + const suites = []; + + for (const dir of dirs) { + const config = ARTIFACT_CONFIGS.find((c) => dir.startsWith(c.pattern)); + if (!config) { + continue; + } + + const suffix = dir.slice(config.pattern.length); + const displayName = suffix ? `${config.name} (${suffix})` : config.name; + + const testFilePath = join(artifactsDir, dir, config.testFile); + const testData = readJson(testFilePath); + if (!testData) { + continue; + } + + const results = + config.framework === "playwright" + ? parsePlaywrightResults(testData) + : parseVitestResults(testData); + + let coverage; + if (config.coverageFile) { + const coveragePath = join(artifactsDir, dir, config.coverageFile); + const coverageData = readJson(coveragePath); + if (coverageData) { + coverage = parseCoverageSummary(coverageData); + } + } + + suites.push({ name: displayName, results, coverage }); + } + + return suites; +} + +function buildPrComment(suites) { + if (suites.length === 0) { + return "## Test Report\n\nNo test results found.\n"; + } + + const lines = []; + const totalFailed = suites.reduce((s, r) => s + r.results.failed, 0); + const totalFlaky = suites.reduce((s, r) => s + r.results.flaky, 0); + const overallIcon = + totalFailed > 0 ? "\u274c" : totalFlaky > 0 ? "\u26a0\ufe0f" : "\u2705"; + + lines.push(`## ${overallIcon} Test Report`); + lines.push(""); + lines.push("| Suite | Tests | Passed | Failed | Skipped | Duration |"); + lines.push("|-------|------:|-------:|-------:|--------:|---------:|"); + + for (const suite of suites) { + const { results } = suite; + const icon = getStatusIcon(results); + const flaky = results.flaky > 0 ? ` (${results.flaky} flaky)` : ""; + lines.push( + `| ${icon} ${suite.name} | ${results.total} | ${results.passed} | ${results.failed}${flaky} | ${results.skipped} | ${formatDuration(results.duration)} |`, + ); + } + lines.push(""); + + const suitesWithCoverage = suites.filter((s) => s.coverage); + if (suitesWithCoverage.length > 0) { + lines.push("### Coverage"); + lines.push(""); + lines.push("| Suite | Lines | Branches | Functions | Statements |"); + lines.push("|-------|------:|---------:|----------:|-----------:|"); + + for (const suite of suitesWithCoverage) { + const c = suite.coverage; + lines.push( + `| ${suite.name} | ${c.lines}% | ${c.branches}% | ${c.functions}% | ${c.statements}% |`, + ); + } + lines.push(""); + + const allFiles = suitesWithCoverage.flatMap( + (s) => + s.coverage.files?.map((f) => ({ + ...f, + suite: s.name, + })) ?? [], + ); + + if (allFiles.length > 0) { + lines.push("
"); + lines.push(`File coverage (${allFiles.length} files)`); + lines.push(""); + + for (const suite of suitesWithCoverage) { + if (!suite.coverage.files?.length) { + continue; + } + lines.push(`#### ${suite.name}`); + lines.push(""); + lines.push("| File | Lines | Branches | Functions |"); + lines.push("|------|------:|---------:|----------:|"); + for (const file of suite.coverage.files) { + lines.push( + `| ${file.file} | ${file.lines}% | ${file.branches}% | ${file.functions}% |`, + ); + } + lines.push(""); + } + + lines.push("
"); + lines.push(""); + } + } + + return lines.join("\n"); +} + +if (values["pr-comment"]) { + const artifactsDir = values["artifacts-dir"]; + if (!artifactsDir || !existsSync(artifactsDir)) { + console.error(`Artifacts directory not found: ${artifactsDir}`); + process.exit(1); + } + + const suites = discoverArtifacts(artifactsDir); + const markdown = buildPrComment(suites); + process.stdout.write(markdown); +} else { + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (!summaryFile) { + console.error("GITHUB_STEP_SUMMARY is not set"); + process.exit(1); + } + + const testData = readJson(values.json); + if (!testData) { + const fallback = `### \u26a0\ufe0f ${values.name}\n\nNo test results found at \`${values.json}\`\n\n`; + appendFileSync(summaryFile, fallback); + process.exit(0); + } + + const results = + values.framework === "playwright" + ? parsePlaywrightResults(testData) + : parseVitestResults(testData); + + const coverageData = values.coverage ? readJson(values.coverage) : undefined; + const coverage = coverageData + ? parseCoverageSummary(coverageData) + : undefined; + + const markdown = buildMarkdown(values.name, results, coverage); + appendFileSync(summaryFile, markdown); +} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2de238535..93bebbb939 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,8 +94,20 @@ jobs: run: pnpm check if: ${{ !cancelled() }} - name: Run small tests & coverage - run: pnpm test + run: pnpm test --reporter=default --reporter=json --outputFile test-results.json --coverage --coverage.reporter=text --coverage.reporter=json-summary --coverage.reporter=json if: ${{ !cancelled() }} + - name: Write test summary + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json test-results.json --name "Server Unit Tests" --framework vitest --coverage coverage/coverage-summary.json + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-server-unit + path: | + server/test-results.json + server/coverage/coverage-summary.json + retention-days: 1 cli-unit-tests: name: Unit Test CLI needs: pre-job @@ -141,8 +153,18 @@ jobs: run: pnpm check if: ${{ !cancelled() }} - name: Run unit tests & coverage - run: pnpm test + run: pnpm test --reporter=default --reporter=json --outputFile test-results.json if: ${{ !cancelled() }} + - name: Write test summary + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json test-results.json --name "CLI Unit Tests" --framework vitest + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-cli-unit + path: cli/test-results.json + retention-days: 1 cli-unit-tests-win: name: Unit Test CLI (Windows) needs: pre-job @@ -183,8 +205,18 @@ jobs: run: pnpm check if: ${{ !cancelled() }} - name: Run unit tests & coverage - run: pnpm test + run: pnpm test --reporter=default --reporter=json --outputFile test-results.json if: ${{ !cancelled() }} + - name: Write test summary + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json test-results.json --name "CLI Unit Tests (Windows)" --framework vitest + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-cli-unit-win + path: cli/test-results.json + retention-days: 1 web-lint: name: Lint Web needs: pre-job @@ -268,8 +300,20 @@ jobs: run: pnpm check:typescript if: ${{ !cancelled() }} - name: Run unit tests & coverage - run: pnpm test + run: pnpm test --reporter=default --reporter=json --outputFile test-results.json --coverage --coverage.reporter=text --coverage.reporter=json-summary --coverage.reporter=json if: ${{ !cancelled() }} + - name: Write test summary + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json test-results.json --name "Web Unit Tests" --framework vitest --coverage coverage/coverage-summary.json + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-web-unit + path: | + web/test-results.json + web/coverage/coverage-summary.json + retention-days: 1 i18n-tests: name: Test i18n needs: pre-job @@ -395,8 +439,20 @@ jobs: - name: Run pnpm install run: SHARP_IGNORE_GLOBAL_LIBVIPS=true pnpm install --frozen-lockfile - name: Run medium tests - run: pnpm test:medium + run: pnpm test:medium --reporter=default --reporter=json --outputFile test-results-medium.json --coverage --coverage.reporter=text --coverage.reporter=json-summary --coverage.reporter=json if: ${{ !cancelled() }} + - name: Write test summary + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json test-results-medium.json --name "Server Medium Tests" --framework vitest --coverage coverage/coverage-summary.json + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-server-medium + path: | + server/test-results-medium.json + server/coverage/coverage-summary.json + retention-days: 1 e2e-tests-server-cli: name: End-to-End Tests (Server & CLI) needs: pre-job @@ -438,8 +494,18 @@ jobs: run: docker compose up -d --renew-anon-volumes --force-recreate --remove-orphans --wait --wait-timeout 300 if: ${{ !cancelled() }} - name: Run e2e tests (api & cli) - run: docker compose --profile test run --rm e2e-runner pnpm test + run: docker compose --profile test run --rm e2e-runner pnpm test --reporter=default --reporter=json --outputFile playwright-report/test-results.json if: ${{ !cancelled() }} + - name: Write test summary + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json playwright-report/test-results.json --name "E2E Server & CLI Tests (${{ matrix.runner }})" --framework vitest + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-e2e-server-cli-${{ matrix.runner }} + path: e2e/playwright-report/test-results.json + retention-days: 1 - name: Capture Docker logs if: always() run: docker compose logs --no-color > docker-compose-logs.txt @@ -490,8 +556,18 @@ jobs: run: docker compose up -d --renew-anon-volumes --force-recreate --remove-orphans --wait --wait-timeout 300 if: ${{ !cancelled() }} - name: Run e2e tests (maintenance) - run: docker compose --profile test run --rm e2e-runner pnpm test:maintenance + run: docker compose --profile test run --rm e2e-runner pnpm test:maintenance --reporter=default --reporter=json --outputFile playwright-report/test-results.json if: ${{ !cancelled() }} + - name: Write test summary + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json playwright-report/test-results.json --name "E2E Server Maintenance Tests (${{ matrix.runner }})" --framework vitest + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-e2e-server-maintenance-${{ matrix.runner }} + path: e2e/playwright-report/test-results.json + retention-days: 1 - name: Capture Docker logs if: always() run: docker compose logs --no-color > docker-compose-logs.txt @@ -544,6 +620,16 @@ jobs: - name: Run e2e tests (web) run: docker compose --profile test run --rm e2e-runner pnpm test:web if: ${{ !cancelled() }} + - name: Write test summary (web) + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json playwright-report/test-results.json --name "E2E Web Tests (${{ matrix.runner }})" --framework playwright + - name: Upload test results (web) + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-e2e-web-${{ matrix.runner }} + path: e2e/playwright-report/test-results.json + retention-days: 1 - name: Archive e2e test (web) results uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: success() || failure() @@ -602,6 +688,16 @@ jobs: - name: Run ui tests (web) run: docker compose --profile test run --rm e2e-runner pnpm test:web:ui if: ${{ !cancelled() }} + - name: Write test summary (ui) + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json playwright-report/test-results.json --name "E2E Web UI Tests (${{ matrix.runner }})" --framework playwright + - name: Upload test results (ui) + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-e2e-web-ui-${{ matrix.runner }} + path: e2e/playwright-report/test-results.json + retention-days: 1 - name: Archive ui test (web) results uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: success() || failure() @@ -660,6 +756,16 @@ jobs: - name: Run maintenance tests run: docker compose --profile test run --rm e2e-runner pnpm test:web:maintenance if: ${{ !cancelled() }} + - name: Write test summary (maintenance) + if: always() + run: node ${{ github.workspace }}/.github/scripts/write-test-summary.mjs --json playwright-report/test-results.json --name "E2E Web Maintenance Tests (${{ matrix.runner }})" --framework playwright + - name: Upload test results (maintenance) + if: ${{ !cancelled() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: report-e2e-web-maintenance-${{ matrix.runner }} + path: e2e/playwright-report/test-results.json + retention-days: 1 - name: Archive maintenance tests (web) results uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: success() || failure() @@ -692,6 +798,39 @@ jobs: - uses: immich-app/devtools/actions/success-check@68f10eb389bb02a3cf9d1156111964c549eb421b # 0.0.4 with: needs: ${{ toJSON(needs) }} + test-report: + name: PR Test Report + if: github.event_name == 'pull_request' && always() + needs: + - server-unit-tests + - web-unit-tests + - server-medium-tests + - cli-unit-tests + - cli-unit-tests-win + - e2e-tests-server-cli + - e2e-tests-server-maintenance + - e2e-tests-web + - e2e-tests-web-ui + - e2e-tests-web-maintenance + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: 'report-*' + path: artifacts + - name: Generate unified report + run: node .github/scripts/write-test-summary.mjs --pr-comment --artifacts-dir artifacts > pr-comment.md + - name: Post PR comment + uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3.0.2 + with: + header: test-report + path: pr-comment.md mobile-unit-tests: name: Unit Test Mobile needs: pre-job diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index c4eb4b8d9f..b47679a80e 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -84,6 +84,7 @@ services: volumes: - ./test-assets:/app/e2e/test-assets - ./playwright-report:/app/e2e/playwright-report + - ./blob-report:/app/e2e/blob-report - /var/run/docker.sock:/var/run/docker.sock depends_on: immich-server: diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 2e9b920b6e..814208e624 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -20,7 +20,11 @@ const config: PlaywrightTestConfig = { fullyParallel: false, forbidOnly: !!process.env.CI, retries: process.env.CI ? 4 : 0, - reporter: 'html', + reporter: [ + ['html'], + ['json', { outputFile: 'playwright-report/test-results.json' }], + ...(process.env.CI ? [['blob', { outputDir: 'blob-report' }] as const] : []), + ], use: { baseURL: playwriteBaseUrl, trace: 'on-first-retry',