Writing First Test with Playwright

Let’s see how we can write a simple test that would use Playwright to open DuckDuckGo, search for a Mercury element, ensure that search results are loaded, include the Wikipedia article about the element, and check that its symbol is presented correctly on the page.

Install Dependencies

Make sure you have Alumnium installed in your project.

build.gradle

plugins {
    id 'java'
}

java {
    toolchain {
        languageVersion = JavaLanguageVersion.of(21)
    }
}

repositories {
    mavenCentral()
}

dependencies {
    testImplementation 'ai.alumnium:alumnium:0.21.0'
    testRuntimeOnly    'ai.alumnium:alumnium-cli-darwin-arm64:0.21.0' // adjust as needed
    testImplementation 'com.microsoft.playwright:playwright:1.60.0'
    testImplementation 'org.junit.jupiter:junit-jupiter:5.11.4'
    testRuntimeOnly    'org.junit.platform:junit-platform-launcher'
}

tasks.withType(Test).configureEach {
    useJUnitPlatform()
}

Installing dependencies

$ pip install alumnium pytest-playwright
$ playwright install chromium

Installing dependencies

$ npm install alumnium

Now initialize Playwright in your project if you haven’t done it yet. Leave all the options as default and remove example test after initialization.

Initializing Playwright

$ npm init playwright
$ rm tests/example.spec.ts

Setup Browser

Start by creating a test file for your tests and instantiating a browser instance:

src/test/java/SearchTest.java

import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false); // DuckDuckGo blocks headless browsers
    }
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
  }
}

Now run the test. You should see a Chrome browser window with DuckDuckGo opened.

Running test

$ ./gradlew test --info
...
BUILD SUCCESSFUL in 7s

test_search.py

from playwright.sync_api import Page


def test_search(page: Page):
    page.goto("https://duckduckgo.com")

Now run the test. You should see a Chrome browser window with DuckDuckGo opened.

Running test

$ pytest --quiet --headed
...
1 passed in 1.32s

tests/search.spec.ts

import { test, expect } from "@playwright/test";

test("should search", async ({ page }) => {
  await page.goto("https://duckduckgo.com");
});

Now run the test. You should see a Chrome browser window with DuckDuckGo opened.

Running test

$ npx playwright test --project chromium --headed

Running 1 test using 1 worker
  1 passed (4.0s)

Setup Alumnium

Now let’s add the code that would instantiate Alumnium using the browser. We can use a test fixture to ensure proper setup and teardown of the Alumnium instance:

src/test/java/SearchTest.java

import ai.alumnium.Alumni;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false);
    }
  }

  private Alumni al;

  @BeforeEach
  void setUp(Page page) {
    al = new Alumni(page);
  }

  @AfterEach
  void tearDown() {
    al.quit();
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
  }
}

Alumni shuts down both the AI session and the wrapped driver when quit() is called, so an explicit driver.quit() is no longer needed.

The test should still work fine, let’s re-run it to make sure:

Running test

$ ./gradlew test --info
...
BUILD SUCCESSFUL in 8s

test_search.py

from alumnium import Alumni
from playwright.sync_api import Page
from pytest import fixture


@fixture
def al(page: Page):
    al = Alumni(page)
    yield al
    al.quit()


def test_search(al: Alumni, page: Page):
    page.goto("https://duckduckgo.com")

The test should still work fine, let’s re-run it to make sure:

Running test

$ pytest --quiet --headed
...
1 passed in 7.81s

tests/index.ts

import { test as base } from "@playwright/test";
import { Alumni } from "alumnium";

export const test = base.extend<{ al: Alumni }>({
  al: async ({ page }, use) => {
    const al = new Alumni(page);
    await use(al);
    await al.quit();
  },
});

export { expect } from "@playwright/test";

tests/search.spec.ts

import { test, expect } from "@playwright/test";
import { test, expect } from ".";

test("should search", async ({ page, al }) => {
  await page.goto("https://duckduckgo.com");
});

The test should still work fine, let’s re-run it to make sure:

Running test

$ npx playwright test --project chromium --headed

Running 1 test using 1 worker
  1 passed (8.2s)

Note

If you are getting errors about missing API keys, make sure you configured Alumnium and try again!

Add Actions

Now let’s add some actions that Alumnium should do on the page. Our test needs to search for a Mercury element, so let’s use this exact command:

src/test/java/SearchTest.java

import ai.alumnium.Alumni;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false);
    }
  }

  private Alumni al;

  @BeforeEach
  void setUp(Page page) {
    al = new Alumni(page);
  }

  @AfterEach
  void tearDown() {
    al.quit();
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
    al.act("search for 'Mercury element' and press Enter");
  }
}

The Java client uses act because do is a reserved word in the language.

Run the test, you should now see “Mercury element” typed in the search box and the page with results loaded.

Running test

$ ./gradlew test --info
...
BUILD SUCCESSFUL in 21s

test_search.py

from alumnium import Alumni
from playwright.sync_api import Page
from pytest import fixture


@fixture
def al(page: Page):
    al = Alumni(page)
    yield al
    al.quit()


def test_search(al: Alumni, page: Page):
    page.goto("https://duckduckgo.com")
    al.do("search for 'Mercury element' and press Enter")

Run the test, you should now see “Mercury element” typed in the search box and the page with results loaded.

Running test

$ pytest --quiet --headed
...
1 passed in 11.34s

tests/search.spec.ts

import { test, expect } from ".";

test("should search", async ({ page, al }) => {
  await page.goto("https://duckduckgo.com");
  await al.do("search for 'Mercury element' and press Enter");
});

Run the test, you should now see “Mercury element” typed in the search box and the page with results loaded.

Running test

$ npx playwright test --project chromium --headed
...
  1 passed (8.6s)

Add Verifications

The next step is to add some verifications that Alumnium should check on the page. We are going to add two of them, one that checks that the page title contains the search keyword and one to see if the Wikipedia article is present in the results.

Common wisdom says to never trust a test you haven’t seen fail. Let’s add the first verification and see it fail!

src/test/java/SearchTest.java

import ai.alumnium.Alumni;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false);
    }
  }

  private Alumni al;

  @BeforeEach
  void setUp(Page page) {
    al = new Alumni(page);
  }

  @AfterEach
  void tearDown() {
    al.quit();
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
    al.act("search for 'Mercury element' and press Enter");
    al.check("page title contains Aluminium word");
  }
}

Now let’s run our test:

Running test

$ ./gradlew test --info
...
java.lang.AssertionError: The requested information is whether the page title contains the word 'Aluminium'. The page title is 'Mercury element at DuckDuckGo', which does not include the word 'Aluminium'. Therefore, the statement is false.
...
BUILD FAILED in 27s

test_search.py

from alumnium import Alumni
from playwright.sync_api import Page
from pytest import fixture


@fixture
def al(page: Page):
    al = Alumni(page)
    yield al
    al.quit()


def test_search(al: Alumni, page: Page):
    page.goto("https://duckduckgo.com")
    al.do("search for 'Mercury element' and press Enter")
    al.check("page title contains Aluminium word")

Now let’s run our test:

Running test

$ pytest --quiet --headed
F
...
E       AssertionError: The requested information is whether the page title contains the word 'Aluminium'. The page title is 'Mercury element at DuckDuckGo', which does not include the word 'Aluminium'. Therefore, the statement is false.
...
1 failed in 25.21s

tests/search.spec.ts

import { test, expect } from ".";

test("should search", async ({ page, al }) => {
  await page.goto("https://duckduckgo.com");
  await al.do("search for 'Mercury element' and press Enter");
  await al.check("page title contains Aluminium word");
});

Now let’s run our test:

Running test

$ npx playwright test --project chromium --headed
...
Error: The requested information is whether the page title contains the word 'Aluminium'. The page title is 'Mercury element at DuckDuckGo', which does not include 'Aluminium'. Therefore, the statement is false.
...
  1 failed

Our test failed as we expected and provided a meaningful explanation of what went wrong.

Let’s fix the first check and add another one to fail again:

src/test/java/SearchTest.java

import ai.alumnium.Alumni;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false);
    }
  }

  private Alumni al;

  @BeforeEach
  void setUp(Page page) {
    al = new Alumni(page);
  }

  @AfterEach
  void tearDown() {
    al.quit();
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
    al.act("search for 'Mercury element' and press Enter");
    al.check("page title contains Aluminium word");
    al.check("page title contains Mercury word");
    al.check("search results do not contain Wikipedia articles");
  }
}

Time to re-run the test:

Running test

$ ./gradlew test --info
...
java.lang.AssertionError: The search results do contain links to Wikipedia articles, specifically mentioning 'Mercury (element) - Wikipedia' and providing a link to the Wikipedia page for Mercury. Therefore, the statement is false.
...
BUILD FAILED in 37s

test_search.py

from alumnium import Alumni
from playwright.sync_api import Page
from pytest import fixture


@fixture
def al(page: Page):
    al = Alumni(page)
    yield al
    al.quit()


def test_search(al: Alumni, page: Page):
    page.goto("https://duckduckgo.com")
    al.do("search for 'Mercury element' and press Enter")
    al.check("page title contains Aluminium word")
    al.check("page title contains Mercury word")
    al.check("search results do not contain Wikipedia articles")

Time to re-run the test:

Running test

$ pytest --quiet --headed
F
...
E       AssertionError: The search results do contain links to Wikipedia articles, specifically mentioning 'Mercury (element) - Wikipedia' and providing a link to the Wikipedia page for Mercury. Therefore, the statement is false.
...
1 failed in 37.28s

tests/search.spec.ts

import { test, expect } from ".";

test("should search", async ({ page, al }) => {
  await page.goto("https://duckduckgo.com");
  await al.do("search for 'Mercury element' and press Enter");
  await al.check("page title contains Aluminium word");
  await al.check("page title contains Mercury word");
  await al.check("search results do not contain Wikipedia articles");
});

Time to re-run the test:

Running test

$ npx playwright test --project chromium --headed
...
Error: The search results do contain Wikipedia articles, as indicated by the presence of links to Wikipedia in the accessibility tree. For example, there is a link with the text 'Mercury (element) - Wikipedia' and another link labeled 'More at Wikipedia'. Therefore, the statement is false.
...
  1 failed

Ok, the test failed as we wanted it to, let’s fix it:

src/test/java/SearchTest.java

import ai.alumnium.Alumni;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false);
    }
  }

  private Alumni al;

  @BeforeEach
  void setUp(Page page) {
    al = new Alumni(page);
  }

  @AfterEach
  void tearDown() {
    al.quit();
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
    al.act("search for 'Mercury element' and press Enter");
    al.check("page title contains Mercury word");
    al.check("search results do not contain Wikipedia articles");
    al.check("search results contain Wikipedia articles");
  }
}

Now, re-run to make sure it passes:

Running test

$ ./gradlew test --info
...
BUILD SUCCESSFUL in 44s

test_search.py

from alumnium import Alumni
from playwright.sync_api import Page
from pytest import fixture


@fixture
def al(page: Page):
    al = Alumni(page)
    yield al
    al.quit()


def test_search(al: Alumni, page: Page):
    page.goto("https://duckduckgo.com")
    al.do("search for 'Mercury element' and press Enter")
    al.check("page title contains Mercury word")
    al.check("search results do not contain Wikipedia articles")
    al.check("search results contain Wikipedia articles")

Now, re-run to make sure it passes:

Running test

$ pytest --quiet --headed
...
1 passed in 38.24s

tests/search.spec.ts

import { test, expect } from ".";

test("should search", async ({ page, al }) => {
  await page.goto("https://duckduckgo.com");
  await al.do("search for 'Mercury element' and press Enter");
  await al.check("page title contains Mercury word");
  await al.check("search results do not contain Wikipedia articles");
  await al.check("search results contain Wikipedia articles");
});

Now, re-run to make sure it passes:

Running test

$ npx playwright test --project chromium --headed
...
  1 passed (14.7s)

Add Data Retrieval

Finally, let’s verify some data that Alumnium can get from the page. We are going to ensure the Mercury element card shows its symbol correctly.

Let’s add a failing verification first:

src/test/java/SearchTest.java

import ai.alumnium.Alumni;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false);
    }
  }

  private Alumni al;

  @BeforeEach
  void setUp(Page page) {
    al = new Alumni(page);
  }

  @AfterEach
  void tearDown() {
    al.quit();
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
    al.act("search for 'Mercury element' and press Enter");
    al.check("page title contains Mercury word");
    al.check("search results contain Wikipedia articles");
    String symbol = (String) al.get("chemical symbol");
    Assertions.assertEquals("Al", symbol);
  }
}

al.get() returns Object and Alumnium supports the following types — String, Long, Double, Boolean, or a List of those. A simple cast on the call site is enough.

Time to re-run the test:

Running test

$ ./gradlew test --info
...
org.opentest4j.AssertionFailedError: expected: <Al> but was: <Hg>
...
BUILD FAILED in 34s

test_search.py

from alumnium import Alumni
from playwright.sync_api import Page
from pytest import fixture


@fixture
def al(page: Page):
    al = Alumni(page)
    yield al
    al.quit()


def test_search(al: Alumni, page: Page):
    page.goto("https://duckduckgo.com")
    al.do("search for 'Mercury element' and press Enter")
    al.check("page title contains Mercury word")
    al.check("search results contain Wikipedia articles")
    symbol = al.get("chemical symbol")
    assert symbol == "Al"

Time to re-run the test:

Running test

$ pytest --quiet --headed
...
E       AssertionError: assert 'Hg' == 'Al'
E
E         - Al
E         + Hg
...
1 failed in 49.14s

tests/search.spec.ts

import { test, expect } from ".";

test("should search", async ({ page, al }) => {
  await page.goto("https://duckduckgo.com");
  await al.do("search for 'Mercury element' and press Enter");
  await al.check("page title contains Mercury word");
  await al.check("search results contain Wikipedia articles");
  const symbol = await al.get("chemical symbol");
  expect(symbol).toBe("Al");
});

Time to re-run the test:

Running test

$ npx playwright test --project chromium --headed
...
    Expected: "Al"
    Received: "Hg"
...
  1 failed

The test failed as expected, let’s fix it and re-run to make sure it’s passing:

src/test/java/SearchTest.java

import ai.alumnium.Alumni;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.junit.Options;
import com.microsoft.playwright.junit.OptionsFactory;
import com.microsoft.playwright.junit.UsePlaywright;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

@UsePlaywright(SearchTest.CustomOptions.class)
class SearchTest {
  public static class CustomOptions implements OptionsFactory {
    @Override
    public Options getOptions() {
      return new Options().setHeadless(false);
    }
  }

  private Alumni al;

  @BeforeEach
  void setUp(Page page) {
    al = new Alumni(page);
  }

  @AfterEach
  void tearDown() {
    al.quit();
  }

  @Test
  void shouldSearch(Page page) {
    page.navigate("https://duckduckgo.com");
    al.act("search for 'Mercury element' and press Enter");
    al.check("page title contains Mercury word");
    al.check("search results contain Wikipedia articles");
    String symbol = (String) al.get("chemical symbol");
    Assertions.assertEquals("Al", symbol);
    Assertions.assertEquals("Hg", symbol);
  }
}

Running test

$ ./gradlew test --info
...
BUILD SUCCESSFUL in 29s

test_search.py

from alumnium import Alumni
from playwright.sync_api import Page
from pytest import fixture


@fixture
def al(page: Page):
    al = Alumni(page)
    yield al
    al.quit()


def test_search(al: Alumni, page: Page):
    page.goto("https://duckduckgo.com")
    al.do("search for 'Mercury element' and press Enter")
    al.check("page title contains Mercury word")
    al.check("search results contain Wikipedia articles")
    symbol = al.get("chemical symbol")
    assert symbol == "Al"
    assert symbol == "Hg"

Running test

$ pytest --quiet --headed
...
1 passed in 64.50s (0:01:04)

tests/search.spec.ts

import { test, expect } from ".";

test("should search", async ({ page, al }) => {
  await page.goto("https://duckduckgo.com");
  await al.do("search for 'Mercury element' and press Enter");
  await al.check("page title contains Mercury word");
  await al.check("search results contain Wikipedia articles");
  const symbol = await al.get("chemical symbol");
  expect(symbol).toBe("Al");
  expect(symbol).toBe("Hg");
});

Running test

$ npx playwright test --project chromium --headed
...
  1 passed (19.5s)

Congratulations, we have completed our first test!