diff --git a/docs/src/test-api/class-teststepinfo.md b/docs/src/test-api/class-teststepinfo.md index a453d212c820d..f4bba6232b6b6 100644 --- a/docs/src/test-api/class-teststepinfo.md +++ b/docs/src/test-api/class-teststepinfo.md @@ -16,6 +16,70 @@ test('basic test', async ({ page, browserName }, TestStepInfo) => { }); ``` +## async method: TestStepInfo.attach +* since: v1.51 + +Attach a value or a file from disk to the current test step. Some reporters show test step attachments. Either [`option: path`] or [`option: body`] must be specified, but not both. Calling this method will attribute the attachment to the step, as opposed to [`method: TestInfo.attach`] which stores all attachments at the test level. + +For example, you can attach a screenshot to the test step: + +```js +import { test, expect } from '@playwright/test'; + +test('basic test', async ({ page }) => { + await page.goto('https://playwright.dev'); + await test.step('check page rendering', async step => { + const screenshot = await page.screenshot(); + await step.attach('screenshot', { body: screenshot, contentType: 'image/png' }); + }); +}); +``` + +Or you can attach files returned by your APIs: + +```js +import { test, expect } from '@playwright/test'; +import { download } from './my-custom-helpers'; + +test('basic test', async ({}) => { + await test.step('check download behavior', async step => { + const tmpPath = await download('a'); + await step.attach('downloaded', { path: tmpPath }); + }); +}); +``` + +:::note +[`method: TestStepInfo.attach`] automatically takes care of copying attached files to a +location that is accessible to reporters. You can safely remove the attachment +after awaiting the attach call. +::: + +### param: TestStepInfo.attach.name +* since: v1.51 +- `name` <[string]> + +Attachment name. The name will also be sanitized and used as the prefix of file name +when saving to disk. + +### option: TestStepInfo.attach.body +* since: v1.51 +- `body` <[string]|[Buffer]> + +Attachment body. Mutually exclusive with [`option: path`]. + +### option: TestStepInfo.attach.contentType +* since: v1.51 +- `contentType` <[string]> + +Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. If omitted, content type is inferred based on the [`option: path`], or defaults to `text/plain` for [string] attachments and `application/octet-stream` for [Buffer] attachments. + +### option: TestStepInfo.attach.path +* since: v1.51 +- `path` <[string]> + +Path on the filesystem to the attached file. Mutually exclusive with [`option: body`]. + ## method: TestStepInfo.skip#1 * since: v1.51 diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts index bc8c9ad0e473e..68e52c46c88a4 100644 --- a/packages/playwright/src/matchers/expect.ts +++ b/packages/playwright/src/matchers/expect.ts @@ -49,8 +49,9 @@ import { toHaveValues, toPass } from './matchers'; +import type { ExpectMatcherStateInternal } from './matchers'; import { toMatchSnapshot, toHaveScreenshot, toHaveScreenshotStepTitle } from './toMatchSnapshot'; -import type { Expect, ExpectMatcherState } from '../../types/test'; +import type { Expect } from '../../types/test'; import { currentTestInfo } from '../common/globals'; import { filteredStackTrace, trimLongString } from '../util'; import { @@ -61,6 +62,7 @@ import { } from '../common/expectBundle'; import { zones } from 'playwright-core/lib/utils'; import { TestInfoImpl } from '../worker/testInfo'; +import type { TestStepInfoImpl } from '../worker/testInfo'; import { ExpectError, isJestError } from './matcherHint'; import { toMatchAriaSnapshot } from './toMatchAriaSnapshot'; @@ -195,6 +197,7 @@ function createExpect(info: ExpectMetaInfo, prefix: string[], userMatchers: Reco type MatcherCallContext = { expectInfo: ExpectMetaInfo; testInfo: TestInfoImpl | null; + step?: TestStepInfoImpl; }; let matcherCallContext: MatcherCallContext | undefined; @@ -211,10 +214,6 @@ function takeMatcherCallContext(): MatcherCallContext { } } -type ExpectMatcherStateInternal = ExpectMatcherState & { - _context: MatcherCallContext | undefined; -}; - const defaultExpectTimeout = 5000; function wrapPlaywrightMatcherToPassNiceThis(matcher: any) { @@ -227,7 +226,7 @@ function wrapPlaywrightMatcherToPassNiceThis(matcher: any) { promise, utils, timeout, - _context: context, + _stepInfo: context.step, }; (newThis as any).equals = throwUnsupportedExpectMatcherError; return matcher.call(newThis, ...args); @@ -376,6 +375,7 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler { }; try { + setMatcherCallContext({ expectInfo: this._info, testInfo, step: step.info }); const callback = () => matcher.call(target, ...args); const result = zones.run('stepZone', step, callback); if (result instanceof Promise) diff --git a/packages/playwright/src/matchers/matchers.ts b/packages/playwright/src/matchers/matchers.ts index e426cf994890c..be960dd424b6f 100644 --- a/packages/playwright/src/matchers/matchers.ts +++ b/packages/playwright/src/matchers/matchers.ts @@ -24,10 +24,13 @@ import { toMatchText } from './toMatchText'; import { isRegExp, isString, isTextualMimeType, pollAgainstDeadline, serializeExpectedTextValues } from 'playwright-core/lib/utils'; import { currentTestInfo } from '../common/globals'; import { TestInfoImpl } from '../worker/testInfo'; +import type { TestStepInfoImpl } from '../worker/testInfo'; import type { ExpectMatcherState } from '../../types/test'; import { takeFirst } from '../common/config'; import { toHaveURL as toHaveURLExternal } from './toHaveURL'; +export type ExpectMatcherStateInternal = ExpectMatcherState & { _stepInfo?: TestStepInfoImpl }; + export interface LocatorEx extends Locator { _expect(expression: string, options: FrameExpectParams): Promise<{ matches: boolean, received?: any, log?: string[], timedOut?: boolean }>; } diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index 193b57058f47a..09ca66ab3ca5f 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -29,8 +29,8 @@ import { colors } from 'playwright-core/lib/utilsBundle'; import fs from 'fs'; import path from 'path'; import { mime } from 'playwright-core/lib/utilsBundle'; -import type { TestInfoImpl } from '../worker/testInfo'; -import type { ExpectMatcherState } from '../../types/test'; +import type { TestInfoImpl, TestStepInfoImpl } from '../worker/testInfo'; +import type { ExpectMatcherStateInternal } from './matchers'; import { matcherHint, type MatcherResult } from './matcherHint'; import type { FullProjectInternal } from '../common/config'; @@ -221,13 +221,13 @@ class SnapshotHelper { return this.createMatcherResult(message, true); } - handleMissing(actual: Buffer | string): ImageMatcherResult { + handleMissing(actual: Buffer | string, step: TestStepInfoImpl | undefined): ImageMatcherResult { const isWriteMissingMode = this.updateSnapshots !== 'none'; if (isWriteMissingMode) writeFileSync(this.expectedPath, actual); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); + step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); writeFileSync(this.actualPath, actual); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); + step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', writing actual.' : '.'}`; if (this.updateSnapshots === 'all' || this.updateSnapshots === 'changed') { /* eslint-disable no-console */ @@ -249,28 +249,29 @@ class SnapshotHelper { diff: Buffer | string | undefined, header: string, diffError: string, - log: string[] | undefined): ImageMatcherResult { + log: string[] | undefined, + step: TestStepInfoImpl | undefined): ImageMatcherResult { const output = [`${header}${indent(diffError, ' ')}`]; if (expected !== undefined) { // Copy the expectation inside the `test-results/` folder for backwards compatibility, // so that one can upload `test-results/` directory and have all the data inside. writeFileSync(this.legacyExpectedPath, expected); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); + step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); output.push(`\nExpected: ${colors.yellow(this.expectedPath)}`); } if (previous !== undefined) { writeFileSync(this.previousPath, previous); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-previous'), contentType: this.mimeType, path: this.previousPath }); + step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-previous'), contentType: this.mimeType, path: this.previousPath }); output.push(`Previous: ${colors.yellow(this.previousPath)}`); } if (actual !== undefined) { writeFileSync(this.actualPath, actual); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); + step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); output.push(`Received: ${colors.yellow(this.actualPath)}`); } if (diff !== undefined) { writeFileSync(this.diffPath, diff); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-diff'), contentType: this.mimeType, path: this.diffPath }); + step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-diff'), contentType: this.mimeType, path: this.diffPath }); output.push(` Diff: ${colors.yellow(this.diffPath)}`); } @@ -288,7 +289,7 @@ class SnapshotHelper { } export function toMatchSnapshot( - this: ExpectMatcherState, + this: ExpectMatcherStateInternal, received: Buffer | string, nameOrOptions: NameOrSegments | { name?: NameOrSegments } & ImageComparatorOptions = {}, optOptions: ImageComparatorOptions = {} @@ -315,7 +316,7 @@ export function toMatchSnapshot( } if (!fs.existsSync(helper.expectedPath)) - return helper.handleMissing(received); + return helper.handleMissing(received, this._stepInfo); const expected = fs.readFileSync(helper.expectedPath); @@ -344,7 +345,7 @@ export function toMatchSnapshot( const receiver = isString(received) ? 'string' : 'Buffer'; const header = matcherHint(this, undefined, 'toMatchSnapshot', receiver, undefined, undefined); - return helper.handleDifferent(received, expected, undefined, result.diff, header, result.errorMessage, undefined); + return helper.handleDifferent(received, expected, undefined, result.diff, header, result.errorMessage, undefined, this._stepInfo); } export function toHaveScreenshotStepTitle( @@ -360,7 +361,7 @@ export function toHaveScreenshotStepTitle( } export async function toHaveScreenshot( - this: ExpectMatcherState, + this: ExpectMatcherStateInternal, pageOrLocator: Page | Locator, nameOrOptions: NameOrSegments | { name?: NameOrSegments } & ToHaveScreenshotOptions = {}, optOptions: ToHaveScreenshotOptions = {} @@ -425,11 +426,11 @@ export async function toHaveScreenshot( // This can be due to e.g. spinning animation, so we want to show it as a diff. if (errorMessage) { const header = matcherHint(this, locator, 'toHaveScreenshot', receiver, undefined, undefined, timedOut ? timeout : undefined); - return helper.handleDifferent(actual, undefined, previous, diff, header, errorMessage, log); + return helper.handleDifferent(actual, undefined, previous, diff, header, errorMessage, log, this._stepInfo); } // We successfully generated new screenshot. - return helper.handleMissing(actual!); + return helper.handleMissing(actual!, this._stepInfo); } // General case: @@ -460,7 +461,7 @@ export async function toHaveScreenshot( return writeFiles(); const header = matcherHint(this, undefined, 'toHaveScreenshot', receiver, undefined, undefined, timedOut ? timeout : undefined); - return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, header, errorMessage, log); + return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, header, errorMessage, log, this._stepInfo); } function writeFileSync(aPath: string, content: Buffer | string) { diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 09ce7a0dddb05..740f7579bdaaf 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -270,7 +270,7 @@ export class TestInfoImpl implements TestInfo { ...data, steps: [], attachmentIndices, - info: new TestStepInfoImpl(), + info: new TestStepInfoImpl(this, stepId), complete: result => { if (step.endWallTime) return; @@ -417,7 +417,7 @@ export class TestInfoImpl implements TestInfo { step.complete({}); } - private _attach(attachment: TestInfo['attachments'][0], stepId: string | undefined) { + _attach(attachment: TestInfo['attachments'][0], stepId: string | undefined) { const index = this._attachmentsPush(attachment) - 1; if (stepId) { this._stepMap.get(stepId)!.attachmentIndices.push(index); @@ -510,6 +510,14 @@ export class TestInfoImpl implements TestInfo { export class TestStepInfoImpl implements TestStepInfo { annotations: Annotation[] = []; + private _testInfo: TestInfoImpl; + private _stepId: string; + + constructor(testInfo: TestInfoImpl, stepId: string) { + this._testInfo = testInfo; + this._stepId = stepId; + } + async _runStepBody(skip: boolean, body: (step: TestStepInfo) => T | Promise) { if (skip) { this.annotations.push({ type: 'skip' }); @@ -524,6 +532,14 @@ export class TestStepInfoImpl implements TestStepInfo { } } + _attachToStep(attachment: TestInfo['attachments'][0]): void { + this._testInfo._attach(attachment, this._stepId); + } + + async attach(name: string, options?: { body?: string | Buffer; contentType?: string; path?: string; }): Promise { + this._attachToStep(await normalizeAndSaveAttachment(this._testInfo.outputPath(), name, options)); + } + skip(...args: unknown[]) { // skip(); // skip(condition: boolean, description: string); diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 5fc76a9168d8c..ef839e44cbe86 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -9575,6 +9575,72 @@ export interface TestInfoError { * */ export interface TestStepInfo { + /** + * Attach a value or a file from disk to the current test step. Some reporters show test step attachments. Either + * [`path`](https://playwright.dev/docs/api/class-teststepinfo#test-step-info-attach-option-path) or + * [`body`](https://playwright.dev/docs/api/class-teststepinfo#test-step-info-attach-option-body) must be specified, + * but not both. Calling this method will attribute the attachment to the step, as opposed to + * [testInfo.attach(name[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach) which stores + * all attachments at the test level. + * + * For example, you can attach a screenshot to the test step: + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('basic test', async ({ page }) => { + * await page.goto('https://playwright.dev'); + * await test.step('check page rendering', async step => { + * const screenshot = await page.screenshot(); + * await step.attach('screenshot', { body: screenshot, contentType: 'image/png' }); + * }); + * }); + * ``` + * + * Or you can attach files returned by your APIs: + * + * ```js + * import { test, expect } from '@playwright/test'; + * import { download } from './my-custom-helpers'; + * + * test('basic test', async ({}) => { + * await test.step('check download behavior', async step => { + * const tmpPath = await download('a'); + * await step.attach('downloaded', { path: tmpPath }); + * }); + * }); + * ``` + * + * **NOTE** + * [testStepInfo.attach(name[, options])](https://playwright.dev/docs/api/class-teststepinfo#test-step-info-attach) + * automatically takes care of copying attached files to a location that is accessible to reporters. You can safely + * remove the attachment after awaiting the attach call. + * + * @param name Attachment name. The name will also be sanitized and used as the prefix of file name when saving to disk. + * @param options + */ + attach(name: string, options?: { + /** + * Attachment body. Mutually exclusive with + * [`path`](https://playwright.dev/docs/api/class-teststepinfo#test-step-info-attach-option-path). + */ + body?: string|Buffer; + + /** + * Content type of this attachment to properly present in the report, for example `'application/json'` or + * `'image/png'`. If omitted, content type is inferred based on the + * [`path`](https://playwright.dev/docs/api/class-teststepinfo#test-step-info-attach-option-path), or defaults to + * `text/plain` for [string] attachments and `application/octet-stream` for [Buffer] attachments. + */ + contentType?: string; + + /** + * Path on the filesystem to the attached file. Mutually exclusive with + * [`body`](https://playwright.dev/docs/api/class-teststepinfo#test-step-info-attach-option-body). + */ + path?: string; + }): Promise; + /** * Unconditionally skip the currently running step. Test step is immediately aborted. This is similar to * [test.step.skip(title, body[, options])](https://playwright.dev/docs/api/class-test#test-step-skip). diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index b05ff8ad55c26..fdb026d66c452 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -1019,6 +1019,27 @@ for (const useIntermediateMergeReport of [true, false] as const) { await expect(attachment).toBeInViewport(); }); + test('step.attach have links', async ({ runInlineTest, page, showReport }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('passing test', async ({ page }, testInfo) => { + await test.step('step', async (step) => { + await step.attach('text attachment', { body: 'content', contentType: 'text/plain' }); + }) + }); + `, + }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' }); + expect(result.exitCode).toBe(0); + + await showReport(); + await page.getByRole('link', { name: 'passing test' }).click(); + + await page.getByLabel('step').getByTitle('reveal attachment').click(); + await page.getByText('text attachment', { exact: true }).click(); + await expect(page.locator('.attachment-body')).toHaveText('content'); + }); + test('should highlight textual diff', async ({ runInlineTest, showReport, page }) => { const result = await runInlineTest({ 'helper.ts': ` diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 67a8250086190..b2a68bfeb0776 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -735,6 +735,43 @@ test('step attachments are referentially equal to result attachments', async ({ ]); }); +test('step.attach attachments are reported on right steps', async ({ runInlineTest }) => { + class TestReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + console.log('%%%', JSON.stringify({ + title: step.title, + attachments: step.attachments.map(a => ({ ...a, body: a.body.toString('utf8') })), + })); + } + } + const result = await runInlineTest({ + 'reporter.ts': `module.exports = ${TestReporter.toString()}`, + 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test.beforeAll(async () => { + await test.step('step in beforeAll', async (step) => { + await step.attach('attachment1', { body: 'content1' }); + }); + }); + test('test', async () => { + await test.step('step', async (step) => { + await step.attach('attachment2', { body: 'content2' }); + }); + }); + `, + }, { 'reporter': '', 'workers': 1 }); + + const steps = result.outputLines.map(line => JSON.parse(line)); + expect(steps).toEqual([ + { title: 'step in beforeAll', attachments: [{ body: 'content1', contentType: 'text/plain', name: 'attachment1' }] }, + { title: 'beforeAll hook', attachments: [] }, + { title: 'Before Hooks', attachments: [] }, + { title: 'step', attachments: [{ body: 'content2', contentType: 'text/plain', name: 'attachment2' }] }, + { title: 'After Hooks', attachments: [] }, + ]); +}); + test('attachments are reported in onStepEnd', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/14364' } }, async ({ runInlineTest }) => { class TestReporter implements Reporter { onStepEnd(test: TestCase, result: TestResult, step: TestStep) {