fix: add retry 2 for failing e2e tests

Author: jsjoeio

ci/dev/test-e2e.sh

@@ -4,8 +4,7 @@ set -euo pipefail
 main() {
   cd "$(dirname "$0")/../.."
   cd test
-  # TODO@jsjoeio remove the test-match
-  PASSWORD=e45432jklfdsab CODE_SERVER_ADDRESS=http://localhost:8080 yarn folio --config=config.ts --test-match login.test.ts --reporter=list
+  PASSWORD=e45432jklfdsab CODE_SERVER_ADDRESS=http://localhost:8080 yarn folio --config=config.ts --reporter=list
 }
 
 main "$@"

test/config.ts

@@ -51,10 +51,7 @@ globalSetup(async () => {
 const config: Config = {
   testDir: path.join(__dirname, "e2e"), // Search for tests in this directory.
   timeout: 30000, // Each test is given 30 seconds.
-}
-
-if (process.env.CI) {
-  config.retries = 2 // Retry failing tests 2 times
+  retries: 2, // Retry failing tests 2 times
 }
 
 setConfig(config)

test/e2e/openHelpAbout.test.ts

@@ -2,35 +2,45 @@ import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
 
 test.describe("Open Help > About", () => {
-  test.beforeEach(async ({ page }) => {
-    // Create a new context with the saved storage state
-    // so we don't have to logged in
-    // TODO@jsjoeio reset context and use storageState
+  // Create a new context with the saved storage state
+  // so we don't have to logged in
+  const options: any = {}
+  // TODO@jsjoeio
+  // Fix this once https://github.com/microsoft/playwright-test/issues/240
+  // is fixed
+  if (STORAGE) {
     const storageState = JSON.parse(STORAGE) || {}
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  })
+    options.contextOptions = {
+      storageState,
+    }
+  }
 
-  test("should see a 'Help' then 'About' button in the Application Menu that opens a dialog", async ({ page }) => {
-    // Make sure the editor actually loaded
-    expect(await page.isVisible("div.monaco-workbench"))
+  test(
+    "should see a 'Help' then 'About' button in the Application Menu that opens a dialog",
+    options,
+    async ({ page }) => {
+      await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+      // Make sure the editor actually loaded
+      expect(await page.isVisible("div.monaco-workbench"))
 
-    // Click the Application menu
-    await page.click("[aria-label='Application Menu']")
-    // See the Help button
-    const helpButton = "a.action-menu-item span[aria-label='Help']"
-    expect(await page.isVisible(helpButton))
+      // Click the Application menu
+      await page.click("[aria-label='Application Menu']")
+      // See the Help button
+      const helpButton = "a.action-menu-item span[aria-label='Help']"
+      expect(await page.isVisible(helpButton))
 
-    // Hover the helpButton
-    await page.hover(helpButton)
+      // Hover the helpButton
+      await page.hover(helpButton)
 
-    // see the About button and click it
-    const aboutButton = "a.action-menu-item span[aria-label='About']"
-    expect(await page.isVisible(aboutButton))
-    // NOTE: it won't work unless you hover it first
-    await page.hover(aboutButton)
-    await page.click(aboutButton)
+      // see the About button and click it
+      const aboutButton = "a.action-menu-item span[aria-label='About']"
+      expect(await page.isVisible(aboutButton))
+      // NOTE: it won't work unless you hover it first
+      await page.hover(aboutButton)
+      await page.click(aboutButton)
 
-    const codeServerText = "text=code-server"
-    expect(await page.isVisible(codeServerText))
-  })
+      const codeServerText = "text=code-server"
+      expect(await page.isVisible(codeServerText))
+    },
+  )
 })