feat: store failure screenshots using artifacts API (#9588)

Author: macarie

docs/api/advanced/artifacts.md

@@ -39,7 +39,9 @@ function recordArtifact<Artifact extends TestArtifact>(task: Test, artifact: Art
 
 The `recordArtifact` function records an artifact during test execution and returns it. It expects a [task](/api/advanced/runner#tasks) as the first parameter and an object assignable to [`TestArtifact`](#testartifact) as the second.
 
-This function has to be used within a test, and the test has to still be running. Recording after test completion will throw an error.
+::: info
+Artifacts must be recorded before the task is reported. Any artifacts recorded after that will not be included in the task.
+:::
 
 When an artifact is recorded on a test, it emits an `onTestArtifactRecord` runner event and a [`onTestCaseArtifactRecord` reporter event](/api/advanced/reporters#ontestcaseartifactrecord). To retrieve recorded artifacts from a test case, use the [`artifacts()`](/api/advanced/test-case#artifacts) method.
 

packages/browser/src/client/tester/runner.ts

@@ -18,7 +18,7 @@ import type { VitestBrowserClientMocker } from './mocker'
 import type { CommandsManager } from './tester-utils'
 import { globalChannel, onCancel } from '@vitest/browser/client'
 import { getTestName } from '@vitest/runner/utils'
-import { BenchmarkRunner, TestRunner } from 'vitest'
+import { BenchmarkRunner, recordArtifact, TestRunner } from 'vitest'
 import { page, userEvent } from 'vitest/browser'
 import {
  DecodedMap,
@@ -175,7 +175,10 @@ export function createBrowserRunner(
  console.error('[vitest] Failed to take a screenshot', err)
  })
  if (screenshot) {
- task.meta.failScreenshotPath = screenshot
+ await recordArtifact(task, {
+ type: 'internal:failureScreenshot',
+ attachments: [{ contentType: 'image/png', path: screenshot, originalPath: screenshot }],
+ } as const)
  }
  }
  }

packages/browser/src/node/plugin.ts

@@ -1,14 +1,13 @@
-import type { Stats } from 'node:fs'
 import type { HtmlTagDescriptor } from 'vite'
 import type { Plugin } from 'vitest/config'
 import type { Vitest } from 'vitest/node'
 import type { ParentBrowserProject } from './projectParent'
-import { createReadStream, lstatSync, readFileSync } from 'node:fs'
+import { createReadStream, readFileSync } from 'node:fs'
 import { createRequire } from 'node:module'
 import { dynamicImportPlugin } from '@vitest/mocker/node'
 import { toArray } from '@vitest/utils/helpers'
 import MagicString from 'magic-string'
-import { basename, dirname, extname, join, resolve } from 'pathe'
+import { basename, dirname, join, resolve } from 'pathe'
 import sirv from 'sirv'
 import { coverageConfigDefaults } from 'vitest/config'
 import {
@@ -97,61 +96,6 @@ export default (parentServer: ParentBrowserProject, base = '/'): Plugin[] => {
  )
  }
 
- const uiEnabled = parentServer.config.browser.ui
-
- if (uiEnabled) {
- // eslint-disable-next-line prefer-arrow-callback
- server.middlewares.use(`${base}__screenshot-error`, function vitestBrowserScreenshotError(req, res) {
- if (!req.url) {
- res.statusCode = 404
- res.end()
- return
- }
-
- const url = new URL(req.url, 'http://localhost')
- const id = url.searchParams.get('id')
- if (!id) {
- res.statusCode = 404
- res.end()
- return
- }
-
- const task = parentServer.vitest.state.idMap.get(id)
- const file = task?.meta.failScreenshotPath
- if (!file) {
- res.statusCode = 404
- res.end()
- return
- }
-
- let stat: Stats | undefined
- try {
- stat = lstatSync(file)
- }
- catch {
- }
-
- if (!stat?.isFile()) {
- res.statusCode = 404
- res.end()
- return
- }
-
- const ext = extname(file)
- const buffer = readFileSync(file)
- res.setHeader(
- 'Cache-Control',
- 'public,max-age=0,must-revalidate',
- )
- res.setHeader('Content-Length', buffer.length)
- res.setHeader('Content-Type', ext === 'jpeg' || ext === 'jpg'
- ? 'image/jpeg'
- : ext === 'webp'
- ? 'image/webp'
- : 'image/png')
- res.end(buffer)
- })
- }
  server.middlewares.use((req, res, next) => {
  // 9000 mega head move
  // Vite always caches optimized dependencies, but users might mock

packages/runner/src/artifact.ts

@@ -13,12 +13,13 @@ import { findTestFileStackTrace } from './utils/collect'
  *
  * Vitest automatically injects the source location where the artifact was created and manages any attachments you include.
  *
+ * **Note:** artifacts must be recorded before the task is reported. Any artifacts recorded after that will not be included in the task.
+ *
  * @param task - The test task context, typically accessed via `this.task` in custom matchers or `context.task` in tests
  * @param artifact - The artifact to record. Must extend {@linkcode TestArtifactBase}
  *
  * @returns A promise that resolves to the recorded artifact with location injected
  *
- * @throws {Error} If called after the test has finished running
  * @throws {Error} If the test runner doesn't support artifacts
  *
  * @example
@@ -40,10 +41,6 @@ import { findTestFileStackTrace } from './utils/collect'
 export async function recordArtifact<Artifact extends TestArtifact>(task: Test, artifact: Artifact): Promise<Artifact> {
  const runner = getRunner()
 
- if (task.result && task.result.state !== 'run') {
- throw new Error(`Cannot record a test artifact outside of the test run. The test "${task.name}" finished running with the "${task.result.state}" state already.`)
- }
-
  const stack = findTestFileStackTrace(
  task.file.filepath,
  new Error('STACK_TRACE').stack!,

packages/runner/src/types.ts

@@ -14,6 +14,7 @@ export type {
  AroundEachListener,
  BeforeAllListener,
  BeforeEachListener,
+ FailureScreenshotArtifact,
  File,
  Fixture,
  FixtureFn,

packages/runner/src/types/tasks.ts

@@ -1345,6 +1345,23 @@ export interface VisualRegressionArtifact extends TestArtifactBase {
  attachments: VisualRegressionArtifactAttachment[]
 }
 
+interface FailureScreenshotArtifactAttachment extends TestAttachment {
+ path: string
+ /** Original file system path to the screenshot, before attachment resolution */
+ originalPath: string
+ body?: undefined
+}
+
+/**
+ * @experimental
+ *
+ * Artifact type for failure screenshots.
+ */
+export interface FailureScreenshotArtifact extends TestArtifactBase {
+ type: 'internal:failureScreenshot'
+ attachments: [FailureScreenshotArtifactAttachment] | []
+}
+
 /**
  * @experimental
  * @advanced
@@ -1426,4 +1443,8 @@ export interface TestArtifactRegistry {}
  *
  * This type automatically includes all artifacts registered via {@link TestArtifactRegistry}.
  */
-export type TestArtifact = TestAnnotationArtifact | VisualRegressionArtifact | TestArtifactRegistry[keyof TestArtifactRegistry]
+export type TestArtifact
+ = | FailureScreenshotArtifact
+ | TestAnnotationArtifact
+ | VisualRegressionArtifact
+ | TestArtifactRegistry[keyof TestArtifactRegistry]

packages/ui/client/components/FailureScreenshot.vue

@@ -0,0 +1,70 @@
+<script setup lang="ts">
+import type { RunnerTask } from 'vitest'
+import { computed, ref } from 'vue'
+import { getAttachmentUrl } from '~/composables/attachments'
+import { isReport } from '~/constants'
+import IconButton from './IconButton.vue'
+import Modal from './Modal.vue'
+import ScreenshotError from './views/ScreenshotError.vue'
+
+const { task } = defineProps<{
+ task: RunnerTask
+}>()
+
+const showScreenshot = ref(false)
+const artifact = computed(() => {
+ if (task.type === 'test') {
+ const artifact = task.artifacts.find(artifact => artifact.type === 'internal:failureScreenshot')
+
+ if (artifact !== undefined) {
+ return artifact
+ }
+ }
+
+ return null
+})
+const screenshotUrl = computed(() =>
+ artifact.value && artifact.value.attachments.length && getAttachmentUrl(artifact.value.attachments[0]),
+)
+
+function openScreenshot() {
+ if (artifact.value === null || artifact.value.attachments.length === 0) {
+ return
+ }
+
+ const filePath = artifact.value.attachments[0].originalPath
+
+ fetch(`/__open-in-editor?file=${encodeURIComponent(filePath)}`)
+}
+</script>
+
+<template>
+ <template v-if="screenshotUrl">
+ <div flex="~ gap-2 items-center">
+ <IconButton
+ v-tooltip.bottom="'View screenshot error'"
+ class="!op-100"
+ icon="i-carbon:image"
+ title="View screenshot error"
+ @click="showScreenshot = true"
+ />
+ <!-- in a report there is no dev server to handle the action -->
+ <IconButton
+ v-if="!isReport"
+ v-tooltip.bottom="'Open screenshot error in editor'"
+ class="!op-100"
+ icon="i-carbon:image-reference"
+ title="Open screenshot error in editor"
+ @click="openScreenshot"
+ />
+ </div>
+ <Modal :key="screenshotUrl" v-model="showScreenshot" direction="right">
+ <ScreenshotError
+ :file="task.file.filepath"
+ :name="task.name"
+ :url="screenshotUrl"
+ @close="showScreenshot = false"
+ />
+ </Modal>
+ </template>
+</template>

packages/ui/client/components/artifacts/Artifacts.vue

@@ -0,0 +1,78 @@
+<script setup lang="ts">
+import type { RunnerTestCase, TestArtifact } from 'vitest'
+import type { Component } from 'vue'
+import { computed } from 'vue'
+import { getLocationString, openLocation } from '~/composables/location'
+import VisualRegression from './visual-regression/VisualRegression.vue'
+
+const { test } = defineProps<{ test: RunnerTestCase }>()
+
+interface HandledArtifact { artifact: TestArtifact; component: Component; props: object }
+
+type ComponentProps<T> = T extends new(...args: any) => { $props: infer P } ? NonNullable<P>
+ : T extends (props: infer P, ...args: any) => any ? P
+ : object
+
+const handledArtifacts = computed<readonly HandledArtifact[]>(() => {
+ const handledArtifacts: HandledArtifact[] = []
+
+ for (const artifact of test.artifacts) {
+ switch (artifact.type) {
+ case 'internal:toMatchScreenshot': {
+ if (artifact.kind === 'visual-regression') {
+ handledArtifacts.push({
+ artifact,
+ component: VisualRegression,
+ props: { regression: artifact } satisfies ComponentProps<typeof VisualRegression>,
+ })
+ }
+
+ continue
+ }
+ }
+ }
+
+ return handledArtifacts
+})
+</script>
+
+<template>
+ <template v-if="handledArtifacts.length">
+ <h1 m-2>
+ Test Artifacts
+ </h1>
+ <div
+ v-for="{ artifact, component, props }, index of handledArtifacts"
+ :key="artifact.type + index"
+ bg="yellow-500/10"
+ text="yellow-500 sm"
+ p="x3 y2"
+ m-2
+ rounded
+ role="note"
+ >
+ <div flex="~ gap-2 items-center justify-between" overflow-hidden>
+ <div>
+ <span
+ v-if="artifact.location && artifact.location.file === test.file.filepath"
+ v-tooltip.bottom="'Open in Editor'"
+ title="Open in Editor"
+ class="flex gap-1 text-yellow-500/80 cursor-pointer"
+ ws-nowrap
+ @click="openLocation(test, artifact.location)"
+ >
+ {{ getLocationString(artifact.location) }}
+ </span>
+ <span
+ v-else-if="artifact.location && artifact.location.file !== test.file.filepath"
+ class="flex gap-1 text-yellow-500/80"
+ ws-nowrap
+ >
+ {{ getLocationString(artifact.location) }}
+ </span>
+ </div>
+ </div>
+ <component :is="component" v-bind="props" />
+ </div>
+ </template>
+</template>