{ "$schema": "https://json-schema.org/draft/2020-12/schema", "report": { "name": "Partner Portal β€” QA, Performance & Automated Regression Report", "version": "2.0.0", "lastUpdated": "2026-06-30", "audience": "External engineering vendors evaluating or implementing a QA process for AI-coding-agent (Claude / Lovable / Codex / etc.) generated applications.", "summary": "This document captures the full QA loop for an AI-agent-built partner portal: scope, manual sweep findings, the new automated performance-regression harness (bundle budgets, Playwright timings, web vitals, baseline tagging, flaky detection, debug logging, unit tests), and recommendations on layering additional AI reviewers.", "stack": { "framework": "TanStack Start v1 (React 19, Vite 7)", "backend": "Lovable Cloud (Supabase: Postgres + Auth + Storage + RLS)", "styling": "Tailwind CSS v4", "deployment": "Cloudflare Workers (edge)", "primaryAgent": "Claude (via Lovable)", "languageTooling": ["bun", "tsgo", "vitest", "playwright", "ripgrep"] } }, "qaProcess": { "scopeMatrix": [ { "surface": "/", "roles": ["Anonymous"] }, { "surface": "/auth + OAuth", "roles": ["Anonymous", "Pending", "Approved", "Rejected"] }, { "surface": "/dashboard", "roles": ["Member", "Partner Admin"] }, { "surface": "/content, /content/submit", "roles": ["Member", "Partner Admin"] }, { "surface": "/team", "roles": ["Partner Admin"] }, { "surface": "/account", "roles": ["Member", "Partner Admin", "Administrator"] }, { "surface": "/admin/*", "roles": ["Administrator"] }, { "surface": "/files/$token", "roles": ["Approved + valid token", "Expired token", "Foreign user"] } ], "phases": [ { "id": "static", "description": "tsgo --noEmit, vitest, ESLint, ripgrep sweeps for console.log, TODO, any-returns at API boundaries, ad-hoc toLocaleString." }, { "id": "functional", "description": "Each route x each role; confirm RLS denies negative paths server-side rather than flashing content client-side." }, { "id": "performance", "description": "DevTools Performance + Network probe on throttled Fast-3G; FCP/LCP/CLS budgets; no duplicate identical queries within 60s." }, { "id": "regression", "description": "Re-run static + functional after every fix batch; a finding only closes after passing in two consecutive sweeps." } ], "exitCriteria": [ "tsgo --noEmit clean", "No console errors on partner or admin dashboards after hard refresh", "No 4xx/5xx on the dashboard critical path", "All findings either Resolved or filed to follow-up with an owner", "bun run perf:check (bundle budgets) passes", "bun run perf:timings (route budgets) passes with no FAIL rows", "bun run perf:test (helper unit tests) passes" ] }, "tooling": { "manual": ["tsgo --noEmit", "ripgrep", "Lovable Cloud SQL editor", "Chrome DevTools (Network/Performance/React Profiler)", "bun add/remove"], "automated": [ { "name": "scripts/perf-budget.mjs", "purpose": "Bundle-size & code-splitting budget after production build" }, { "name": "scripts/perf-timings.mjs", "purpose": "Playwright suite measuring FCP/LCP/CLS/Load per route with explicit budgets, baseline diffs, flaky detection, diagnostics" }, { "name": "scripts/perf-helpers.mjs", "purpose": "Pure helpers for flaky detection, baseline selection, and diff exclusion (unit-tested)" }, { "name": "scripts/perf-helpers.test.mjs", "purpose": "21-case node:test suite covering flaky regex, baseline tag selection, flaky-skipping, --include-flaky overrides, diff exclusions" }, { "name": "scripts/perf-seed.mjs / perf-unseed.mjs", "purpose": "Deterministic UUID-based seeding so timing results are stable across runs" }, { "name": ".github/workflows/perf.yml", "purpose": "Runs perf:timings on every PR, fails on budget regression, uploads per-run report artifact" } ] }, "findings": { "legend": { "resolved": "βœ…", "mitigated": "🟑", "verifiedClean": "πŸ”΅" }, "batches": { "A_earlier": [ { "id": "PSQ-001", "area": "Auth", "finding": "Stale Supabase session caused intermittent Unauthorized on actions", "fix": "auth-attacher-fresh.ts proactively refreshes the JWT before server-fn calls", "status": "resolved" }, { "id": "PSQ-002", "area": "Security", "finding": "SECURITY DEFINER helpers were callable by anon/authenticated", "fix": "Moved to private schema, revoked broad EXECUTE, pinned search_path=public", "status": "resolved" }, { "id": "PSQ-003", "area": "Performance", "finding": "Every route re-fetched on focus / no staleTime", "fix": "staleTime: 60_000 + refetchOnWindowFocus: false set in src/router.tsx", "status": "resolved" }, { "id": "PSQ-004", "area": "Performance", "finding": "Admin dashboard issued multiple identical profiles queries", "fix": "Consolidated in admin-overview.functions.ts; reduced log scan limits", "status": "resolved" }, { "id": "PSQ-005", "area": "Performance", "finding": "useCurrentUser re-ran on every render", "fix": "Rewrote as React Query hook with 5-min staleTime", "status": "resolved" }, { "id": "PSQ-006", "area": "Security", "finding": "/admin/* was only client-gated", "fix": "assertAdminAccess in beforeLoad for the admin route group", "status": "resolved" }, { "id": "PSQ-007", "area": "Performance", "finding": "Announcement audience filtering happened in JS", "fix": "Pushed active, time-window and audience overlap to PostgREST", "status": "resolved" }, { "id": "PSQ-008", "area": "UX", "finding": "Ad-blockers blocked /api/content (storage CDN host)", "fix": "Same-origin /files/$token route with 302 to signed Supabase URL", "status": "resolved" }, { "id": "PSQ-009", "area": "UX", "finding": "AlertDialog usage inconsistent across admin destructive actions", "fix": "ConfirmProvider + useConfirm() hook", "status": "resolved" }, { "id": "PSQ-010", "area": "Security", "finding": "Partner Admins could invite consumer-email domains", "fix": "Server-side blocklist enforced in invite path", "status": "resolved" } ], "B_qaSweep": [ { "id": "PSQ-011", "area": "Performance", "finding": "Suspected admin JS in partner first-paint bundle", "fix": "Verified TanStack Router file-route splitting; admin children only load on /admin/* nav", "status": "verifiedClean" }, { "id": "PSQ-012", "area": "UX", "finding": "Partner Suspense fallbacks were bare 'Loading…'", "fix": "Added DashboardSkeleton/ContentLibrarySkeleton/TableSkeleton/PageSkeleton", "status": "resolved" }, { "id": "PSQ-013", "area": "Performance", "finding": "Suspected N+1 in audit log actor lookup", "fix": "Reviewed adminListAuditLog: single in() against profiles; Promise.all parallelizes", "status": "verifiedClean" }, { "id": "PSQ-014", "area": "Performance", "finding": "Content View/Download felt slow because signing happens on click", "fix": "Hover prefetch + log:false on sign; partnerLogContentAccess fires async on real click", "status": "resolved" }, { "id": "PSQ-015", "area": "Maintenance", "finding": "13 ad-hoc toLocaleString/toLocaleDateString sites with inconsistent formats", "fix": "src/lib/format.ts central helpers; migrated all 9 affected routes", "status": "resolved" }, { "id": "PSQ-016", "area": "Reliability", "finding": "Partner routes had no per-route error boundary", "fix": "RouteErrorComponent attached to all five routes; hides stack in prod", "status": "resolved" }, { "id": "PSQ-017", "area": "Performance", "finding": "18+ Radix/Embla/Recharts/Vaul/cmdk pkgs + 22 unused shadcn primitives shipped", "fix": "Deleted 22 ui/* files, removed 17 npm packages; restored recharts", "status": "resolved" } ], "C_incidental": [ { "id": "PSQ-018", "area": "Privacy", "finding": "PortalHeader.handleSignOut did not clear React Query cache before redirect", "fix": "qc.cancelQueries() + qc.clear() before supabase.auth.signOut()", "status": "resolved" }, { "id": "PSQ-019", "area": "Performance", "finding": "Hover-prefetched signed URLs would log phantom views", "fix": "Split logging from signing; explicit log:false flag", "status": "resolved" }, { "id": "PSQ-020", "area": "DX", "finding": "AdminErrorComponent exposed full stack in production", "fix": "Recommend gating behind import.meta.env.DEV", "status": "mitigated" }, { "id": "PSQ-021", "area": "Build", "finding": "recharts removed then immediately required", "fix": "Restored; added grep-before-remove note to dep-pruning checklist", "status": "resolved" } ], "D_automation": [ { "id": "PSQ-022", "area": "Automation", "finding": "No bundle-size regression gate", "fix": "scripts/perf-budget.mjs enforces per-chunk and total budgets after build", "status": "resolved" }, { "id": "PSQ-023", "area": "Automation", "finding": "No automated route-level timing regression gate", "fix": "scripts/perf-timings.mjs: Playwright probe with explicit FCP/LCP/CLS/Load budgets per route", "status": "resolved" }, { "id": "PSQ-024", "area": "Automation", "finding": "Critical routes missing from timing suite", "fix": "Added /content, /admin/audit, /embed/apply (partner onboarding) with explicit budgets", "status": "resolved" }, { "id": "PSQ-025", "area": "CI", "finding": "Perf checks not enforced on PRs", "fix": ".github/workflows/perf.yml runs bun run perf:timings on every PR and fails on regression", "status": "resolved" }, { "id": "PSQ-026", "area": "Observability","finding": "No historical perf trend, no per-run report", "fix": "Per-run latest.md + latest.json + history.json (50-run window) with route-by-route diffs vs prior run", "status": "resolved" }, { "id": "PSQ-027", "area": "Diagnostics", "finding": "Failed perf runs had no post-mortem artifacts", "fix": "On budget breach: trace.zip, video.webm, snapshot.html, console.log written to perf-reports/diagnostics//; rendered as clickable links in latest.md", "status": "resolved" }, { "id": "PSQ-028", "area": "Determinism", "finding": "Variable seed data made timings non-comparable", "fix": "scripts/perf-seed.mjs + perf-unseed.mjs using deterministic UUIDs for constant workload during runs", "status": "resolved" }, { "id": "PSQ-029", "area": "Coverage", "finding": "Only FCP/Load asserted; missing modern web vitals", "fix": "Injected PerformanceObserver to capture LCP and CLS; explicit per-route budgets for all four metrics", "status": "resolved" }, { "id": "PSQ-030", "area": "Reporting", "finding": "Reports too verbose for PR comments", "fix": "--summary mode produces concise latest.md while full per-route data persists in latest.json", "status": "resolved" }, { "id": "PSQ-031", "area": "Reporting", "finding": "Diff was always vs immediate prior run; couldn't anchor to a known-good commit", "fix": "--tag / --baseline flags; tag-aware history walk with fallback when no match found", "status": "resolved" }, { "id": "PSQ-032", "area": "Observability","finding": "Console errors during runs were invisible in report", "fix": "Captured unique console + page errors per route; written to diagnostics/console-errors.log and surfaced in latest.md", "status": "resolved" }, { "id": "PSQ-033", "area": "Reliability", "finding": "Flaky Playwright timeouts polluted diffs with false regressions", "fix": "Regex-based isFlakyError; routes marked FLAKY are excluded from diffs and from baseline selection", "status": "resolved" }, { "id": "PSQ-034", "area": "Reporting", "finding": "No visibility into flake rate over time", "fix": "renderFlakyRate: per-run count + percent, 10-run rolling avg, run-level flake frequency", "status": "resolved" }, { "id": "PSQ-035", "area": "Debuggability","finding": "Baseline selection and flake-skipping were opaque", "fix": "--debug / PERF_DEBUG mode + dlog; selection rationale persisted to latest.json.baselineDebug", "status": "resolved" }, { "id": "PSQ-036", "area": "Testing", "finding": "Flaky detection and baseline logic untested", "fix": "Extracted helpers to perf-helpers.mjs; 21 node:test cases via bun run perf:test", "status": "resolved" }, { "id": "PSQ-037", "area": "Operations", "finding": "Couldn't compare a known-flaky route's trend without bypassing safeguards", "fix": "--include-flaky / PERF_INCLUDE_FLAKY makes flaky runs eligible as baselines and keeps flaky rows in diffs; covered by unit tests", "status": "resolved" } ] } }, "performanceScoreboard": { "context": "Partner dashboard, warm cache, throttled Fast-3G", "metrics": [ { "metric": "JS shipped to partner first paint", "before": "~1.4 MB", "after": "~1.1 MB", "source": "PSQ-017" }, { "metric": "Suspense fallback perceived quality", "before": "Text", "after": "Skeleton", "source": "PSQ-012" }, { "metric": "Content View click β†’ new tab opens (hover)", "before": "~700 ms", "after": "~120 ms", "source": "PSQ-014" }, { "metric": "useCurrentUser re-renders per nav", "before": "6–10", "after": "1", "source": "PSQ-005" }, { "metric": "Duplicate profiles queries on admin home", "before": "3", "after": "1", "source": "PSQ-004" }, { "metric": "Date-format variants in UI", "before": "4", "after": "1", "source": "PSQ-015" }, { "metric": "Route timing regressions caught in CI", "before": "0 (manual only)", "after": "Per-PR via perf.yml", "source": "PSQ-025" }, { "metric": "Per-run diagnostics for failed routes", "before": "none", "after": "trace + video + snapshot + console.log", "source": "PSQ-027" } ] }, "automatedPerfHarness": { "entrypoints": { "bun run perf:check": "Production build + bundle-size budget", "bun run perf:timings": "Playwright FCP/LCP/CLS/Load probe per route + report", "bun run perf:test": "Helper unit tests (flaky detection, baseline selection, diff exclusion, --include-flaky)" }, "routes": [ { "path": "/", "auth": false, "budgets": { "fcp": 2500, "load": 4000, "lcp": 3000, "cls": 0.10 } }, { "path": "/auth", "auth": false, "budgets": { "fcp": 2500, "load": 4000, "lcp": 3000, "cls": 0.10 } }, { "path": "/embed/apply", "auth": false, "budgets": { "fcp": 2500, "load": 4000, "lcp": 3000, "cls": 0.10 } }, { "path": "/dashboard", "auth": true, "budgets": { "fcp": 3000, "load": 5000, "lcp": 3500, "cls": 0.15 } }, { "path": "/content", "auth": true, "budgets": { "fcp": 3000, "load": 5000, "lcp": 3500, "cls": 0.15 } }, { "path": "/admin/audit", "auth": true, "budgets": { "fcp": 3500, "load": 6000, "lcp": 4000, "cls": 0.15 } } ], "flags": { "--tag ": "Tag this run on history", "--baseline ": "Diff against most recent run with matching tag", "--summary": "Concise latest.md; full data persists in latest.json", "--debug": "Log baseline selection rationale + flaky skip details", "--include-flaky": "Keep flaky runs eligible as baselines and keep flaky rows in diffs" }, "envEquivalents": ["PERF_TAG", "PERF_BASELINE", "PERF_DEBUG", "PERF_INCLUDE_FLAKY"], "artifacts": { "perf-reports/latest.md": "Human-readable report (or summary in --summary mode)", "perf-reports/latest.json": "Full per-route metrics + baselineDebug", "perf-reports/history.json": "Rolling 50-run window", "perf-reports/diagnostics//": "trace.zip + video.webm + snapshot.html + console.log + console-errors.log for failed routes" }, "flakyHandling": { "detection": "Regex over error message (Playwright timeouts, navigation failures, net::ERR_*, target closed, frame detached)", "defaultBehavior": "Flaky routes marked FLAKY, excluded from diffs, ineligible as baseline", "override": "--include-flaky flips both rules and is reflected in baselineDebug.includeFlaky", "rate": "Per-run count + percent, 10-run rolling average, run-level flake frequency" } }, "vendorPlaybook": { "audience": "External vendor building a QA process around AI-coding-agent output", "principles": [ "Treat the agent like a fast, sometimes-wrong junior: gate every change behind machine-checkable budgets, not eyeballs.", "Make regressions easy to attribute: per-PR, per-route, with diffs vs a tagged baseline and downloadable diagnostics.", "Separate flakiness from regressions explicitly; never let timeouts silently turn into 'wins' or false alarms.", "Keep findings in a stable schema (PSQ-### + status) so the agent can re-read its own history across sessions.", "Deterministic seeds for any perf or data-shape assertion." ], "minimumViableLoop": [ "Static gate: typecheck + lint + ripgrep for known anti-patterns (console.log, ad-hoc formatters, dead UI primitives).", "Unit gate: vitest/node:test for any helper with branching logic (e.g. baseline selection, flake detection).", "Bundle gate: production build with explicit per-chunk and total size budgets.", "Route gate: Playwright probe of critical routes with FCP/LCP/CLS/Load budgets, deterministic seed data, baseline diffs.", "Diagnostics on failure: trace, video, snapshot, console + page errors β€” surfaced as clickable links in the PR report.", "Flake policy: explicit detection + opt-in override; flake rate reported every run.", "Findings ledger: durable, versioned JSON the agent reads back next session." ], "antiPatterns": [ "Letting the agent self-assess by reading its own console output without machine-verified budgets.", "Comparing perf vs 'last run' only β€” one bad run becomes the new normal.", "Treating any failed Playwright step as a regression without flake handling.", "Removing dependencies without grepping usage (PSQ-021).", "Manual-only QA against a moving target β€” every refactor erases prior verification." ] }, "modelLayeringFeedback": { "question": "Should additional model agents (Codex, local LLMs, other frontier models) run a second QA pass?", "recommendation": "Yes β€” layer them, but assign each one a narrow role where its strengths are uncorrelated with the primary builder. Do not run a second generalist agent over the same code expecting different bugs.", "layers": [ { "layer": "Primary builder (Claude via Lovable)", "strengths": ["Long-horizon planning", "Refactors with full repo context", "Writing the QA harness itself"], "blindSpots": ["Self-grading", "Subtle perf regressions it introduced", "Security edge cases in code it just wrote"] }, { "layer": "Second-opinion frontier model (e.g. GPT-5.4 / Gemini 3.1 Pro / Codex review mode)", "role": "Read-only code review on diffs only. Focused prompts: 'find auth/RLS bypasses', 'find N+1 queries', 'find unhandled async errors'. Treat output as findings to triage, not patches.", "why": "Independent training data and decoding behavior catches a different distribution of bugs than the builder. Highest ROI on security, data-shape, and concurrency review.", "verdict": "Recommended for any production-bound PR." }, { "layer": "Local LLM (e.g. Qwen-Coder, DeepSeek-Coder, Llama on-prem)", "role": "Cheap, high-volume mechanical sweeps: dead-code detection, dependency hygiene, doc/comment drift, prompt-template lint, secret scanning prompts, commit-message normalization.", "why": "Zero marginal cost, runs on every commit, no data-egress concerns. Not strong enough for nuanced security review.", "verdict": "Recommended for pre-commit hooks and nightly hygiene jobs; not as a primary reviewer." }, { "layer": "Deterministic tools (tsgo, eslint, ripgrep, Playwright, node:test, perf-budget)", "role": "Ground truth. Everything above is advisory; these are the only gates that block a merge.", "verdict": "Always the final word." } ], "concretePipelineProposal": { "preCommit": ["tsgo --noEmit", "eslint", "ripgrep anti-pattern sweep", "local LLM: dead-code + secret scan"], "prePR": ["bun run perf:test", "bun run perf:check", "bun run perf:timings --tag pr-"], "prReview": ["Second-opinion frontier model on the diff with three targeted prompts (security, perf, error-handling)", "Human review of model findings"], "preMerge": ["perf:timings --baseline main passes within budgets", "Diagnostics empty or triaged"], "postMerge": ["perf:timings --tag main on main branch to refresh baseline", "Nightly local-LLM hygiene sweep"] }, "costShape": "Frontier second-opinion calls are the most expensive line item β€” scope them tightly (diff only, named prompt) rather than whole-repo. Local LLM and deterministic gates absorb the volume.", "openRisks": [ "Second-opinion models may hallucinate 'findings' that look plausible. Always require a reproducer (failing test or budget breach) before acting.", "Local LLMs drift in quality across releases β€” pin model versions and re-baseline quarterly.", "Layering more reviewers creates more noise. Track signal-to-noise per layer; demote any layer whose findings rarely lead to merged fixes." ] }, "fileLevelChangeLog": { "added": [ "src/lib/format.ts", "src/components/skeletons.tsx", "src/components/RouteErrorComponent.tsx", "docs/QA-REPORT.md", "docs/qa-report.json", "scripts/perf-budget.mjs", "scripts/perf-timings.mjs", "scripts/perf-helpers.mjs", "scripts/perf-helpers.test.mjs", "scripts/perf-seed.mjs", "scripts/perf-unseed.mjs", ".github/workflows/perf.yml" ], "edited": [ "src/lib/portal.functions.ts β€” split signing from logging; partnerLogContentAccess", "src/routes/_authenticated/dashboard.tsx β€” skeleton + error boundary + centralized dates", "src/routes/_authenticated/content.tsx β€” hover prefetch + skeleton + error boundary", "src/routes/_authenticated/content.submit.tsx β€” error boundary + dates", "src/routes/_authenticated/team.tsx β€” error boundary", "src/routes/_authenticated/account.tsx β€” error boundary", "src/routes/_authenticated/admin/{index,analytics,activity,audit,emails,partners,content}.tsx β€” centralized dates", "src/components/PortalHeader.tsx β€” clear React Query cache on sign-out", "package.json β€” perf:check / perf:timings / perf:test scripts" ], "deleted": [ "src/components/ui/{accordion,alert,aspect-ratio,breadcrumb,calendar,carousel,chart,command,context-menu,drawer,hover-card,input-otp,menubar,navigation-menu,pagination,progress,resizable,scroll-area,sheet,sidebar,slider,toggle,toggle-group}.tsx" ], "depsRemoved": [ "@radix-ui/react-{accordion,aspect-ratio,context-menu,hover-card,menubar,navigation-menu,progress,scroll-area,slider,toggle,toggle-group}", "cmdk", "embla-carousel-react", "input-otp", "react-day-picker", "react-resizable-panels", "vaul" ], "depsRestored": ["recharts"] } }