Run the SDK live tests against the test seeder

tools/sdk-live-tests/run.sh builds the server with unsafe_test_mode and runs
the live tests of the SDK's e2e/wasm suite. For SDK versions whose tests log in
to one given account, each user test vector with a master password is seeded
with its vault on a fresh server; for versions that seed their own accounts,
the tests get the seeder's address. Every vector runs, then a summary.

The workflow runs it by hand, and on changes to the seeder or the runner.

.github/workflows/sdk-live-tests.yml

@@ -0,0 +1,74 @@
+name: SDK live tests
+permissions: {}
+
+# Runs the live-server end-to-end tests of bitwarden/sdk-internal against this branch, see tools/sdk-live-tests/run.sh.
+# It builds the SDK too, so it only runs by hand and on changes to the test seeder or the runner.
+on:
+  workflow_dispatch:
+    inputs:
+      sdk_ref:
+        description: "Branch, tag or commit of bitwarden/sdk-internal to test with"
+        required: true
+        default: "main"
+  pull_request:
+    paths:
+      - ".github/workflows/sdk-live-tests.yml"
+      - "src/api/test_seeder.rs"
+      - "tools/sdk-live-tests/**"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  sdk-live-tests:
+    name: SDK live tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - name: "Install dependencies Ubuntu"
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential libssl-dev pkg-config
+
+      - name: "Checkout"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: "Checkout sdk-internal"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: bitwarden/sdk-internal
+          ref: ${{ inputs.sdk_ref || 'main' }}
+          path: sdk-internal
+          persist-credentials: false
+
+      # Each checkout pins its own toolchain in rust-toolchain.toml
+      - name: "Install toolchains"
+        run: |
+          rustup toolchain install
+          cd sdk-internal
+          rustup toolchain install
+          rustup target add wasm32-unknown-unknown
+          rustup component add rust-src
+
+      - name: "Cache Rust builds"
+        uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+        with:
+          workspaces: |
+            . -> target
+            sdk-internal -> target
+
+      - name: "Setup Node"
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: sdk-internal/.nvmrc
+
+      - name: "Run the SDK live tests"
+        run: tools/sdk-live-tests/run.sh --sdk-dir sdk-internal
+
+      - name: "Upload logs"
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: sdk-live-tests-logs
+          path: target/sdk-live-tests/**/*.log

tools/sdk-live-tests/run.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+# Runs the live-server end-to-end tests of bitwarden/sdk-internal (e2e/wasm) against vaultwarden.
+#
+# Usage: tools/sdk-live-tests/run.sh --sdk-dir <path> [--skip-sdk-build] [--mode vectors|seeder] [-- <vitest args>]
+#
+# The server is built with `unsafe_test_mode`, whose test seeder (src/api/test_seeder.rs) creates the test accounts.
+# - vectors: for SDK versions whose live tests log in to one given account. Every user test vector with a master
+#   password gets a fresh server, is seeded there with its vault, and the tests run on it (they rotate its keys).
+# - seeder: for SDK versions whose tests seed their own accounts through BW_LIVE_SEEDER_URL. One server, one run.
+# The mode follows what the SDK's e2e/wasm/live/config.ts reads, unless --mode says otherwise. Arguments after `--` go
+# to vitest, e.g. `-- tests/user-crypto-management/rotate-user-keys.live.test.ts` to run one file.
+#
+# Every vector runs even after a failure; a summary follows, and the exit code is 1 if any failed. The server logs are
+# in <target dir>/sdk-live-tests/. BW_LIVE_CLIENT_VERSION defaults to 2026.10.1, the minimum for V2 accounts.
+#
+# Needs: the SDK's Rust toolchain with the wasm32-unknown-unknown target and rust-src, Node.js >= 22.12, curl, python3.
+set -euo pipefail
+
+VW_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
+TARGET_DIR="${CARGO_TARGET_DIR:-${VW_DIR}/target}"
+OUT_DIR="${TARGET_DIR}/sdk-live-tests"
+PORT=8099
+URL="http://127.0.0.1:${PORT}"
+SEEDER_URL="${URL}/test/seeder"
+SEEDER_USERNAME="sdk-live-tests"
+SEEDER_PASSWORD="$(python3 -c 'import secrets; print(secrets.token_urlsafe(24))')"
+SKIP=3 # seed_vector.py's exit code for a vector the tests can't use
+export BW_LIVE_CLIENT_VERSION="${BW_LIVE_CLIENT_VERSION:-2026.10.1}"
+
+SDK_DIR=""
+BUILD_SDK=1
+MODE=""
+VITEST_ARGS=()
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sdk-dir) SDK_DIR="$(cd "$2" && pwd)"; shift 2 ;;
+    --skip-sdk-build) BUILD_SDK=0; shift ;;
+    --mode) MODE="$2"; shift 2 ;;
+    --) shift; VITEST_ARGS=("$@"); break ;;
+    *) echo "Unknown argument: $1" >&2; exit 2 ;;
+  esac
+done
+[[ -n "${SDK_DIR}" ]] || { echo "--sdk-dir is required" >&2; exit 2; }
+TESTS_DIR="${SDK_DIR}/e2e/wasm"
+[[ -f "${TESTS_DIR}/live/config.ts" ]] || { echo "${TESTS_DIR}/live/config.ts not found: no live tests in this SDK" >&2; exit 2; }
+if [[ -z "${MODE}" ]]; then
+  if grep -q BW_LIVE_SEEDER_URL "${TESTS_DIR}/live/config.ts"; then MODE=seeder; else MODE=vectors; fi
+fi
+[[ "${MODE}" == vectors || "${MODE}" == seeder ]] || { echo "--mode must be vectors or seeder" >&2; exit 2; }
+echo "Mode: ${MODE}, client version ${BW_LIVE_CLIENT_VERSION}"
+
+(cd "${VW_DIR}" && cargo build --features sqlite,unsafe_test_mode)
+if [[ ${BUILD_SDK} -eq 1 ]]; then
+  bash "${SDK_DIR}/crates/bitwarden-wasm-internal/build.sh"
+  (cd "${TESTS_DIR}" && npm ci)
+fi
+
+server=""
+trap 'if [[ -n "${server}" ]]; then kill "${server}" 2>/dev/null || true; fi' EXIT
+
+# Starts a server on an empty database in $OUT_DIR/$1
+start_server() {
+  local data_dir="${OUT_DIR}/$1"
+  rm -rf "${data_dir}" && mkdir -p "${data_dir}"
+  if curl -s -o /dev/null "${URL}/alive"; then
+    echo "Something already answers on port ${PORT}" >&2
+    return 1
+  fi
+  ENV_FILE=/dev/null DATA_FOLDER="${data_dir}" DATABASE_URL="sqlite://${data_dir}/db.sqlite3" \
+    ROCKET_ADDRESS=127.0.0.1 ROCKET_PORT="${PORT}" DOMAIN="${URL}" WEB_VAULT_ENABLED=false \
+    SIGNUPS_ALLOWED=true SIGNUPS_VERIFY=false LOGIN_RATELIMIT_MAX_BURST=1000 \
+    TEST_SEEDER_USERNAME="${SEEDER_USERNAME}" TEST_SEEDER_PASSWORD="${SEEDER_PASSWORD}" \
+    "${TARGET_DIR}/debug/vaultwarden" > "${data_dir}/vaultwarden.log" 2>&1 &
+  server=$!
+  local i
+  for i in $(seq 1 60); do
+    if ! kill -0 "${server}" 2>/dev/null; then break; fi
+    if curl -sf -o /dev/null "${URL}/alive"; then return 0; fi
+    sleep 1
+  done
+  echo "The server didn't start; the end of ${data_dir}/vaultwarden.log:" >&2
+  tail -50 "${data_dir}/vaultwarden.log" >&2
+  stop_server
+  return 1
+}
+
+stop_server() {
+  if [[ -n "${server}" ]]; then
+    kill "${server}" 2>/dev/null || true
+    wait "${server}" 2>/dev/null || true
+    server=""
+  fi
+}
+
+run_tests() {
+  (cd "${TESTS_DIR}" && BW_LIVE_SERVER_URL="${URL}" "$@" npm run test:live -- "${VITEST_ARGS[@]}")
+}
+
+results=()
+failed=0
+if [[ "${MODE}" == seeder ]]; then
+  if start_server seeder && run_tests env BW_LIVE_SEEDER_URL="${SEEDER_URL}" \
+    BW_LIVE_SEEDER_USERNAME="${SEEDER_USERNAME}" BW_LIVE_SEEDER_PASSWORD="${SEEDER_PASSWORD}"; then
+    results+=("seeder: pass")
+  else
+    results+=("seeder: FAIL")
+    failed=1
+  fi
+  stop_server
+else
+  for vector_file in "${SDK_DIR}"/test-vectors/users/*.json; do
+    vector="$(basename "${vector_file}" .json)"
+    echo "=== ${vector}"
+    if ! start_server "${vector}"; then
+      results+=("${vector}: FAIL (server didn't start)")
+      failed=1
+      continue
+    fi
+    status=0
+    credentials="$(python3 -I "${VW_DIR}/tools/sdk-live-tests/seed_vector.py" "${vector_file}" "${SEEDER_URL}" \
+      "${SEEDER_USERNAME}" "${SEEDER_PASSWORD}")" || status=$?
+    if [[ ${status} -eq ${SKIP} ]]; then
+      results+=("${vector}: skip (${credentials})")
+    elif [[ ${status} -ne 0 ]]; then
+      results+=("${vector}: FAIL (seeding)")
+      failed=1
+    elif run_tests env BW_LIVE_EMAIL="$(sed -n 1p <<< "${credentials}")" \
+      BW_LIVE_PASSWORD="$(sed -n 2p <<< "${credentials}")"; then
+      results+=("${vector}: pass")
+    else
+      results+=("${vector}: FAIL")
+      failed=1
+    fi
+    stop_server
+  done
+fi
+
+echo
+echo "=== Summary (logs in ${OUT_DIR})"
+printf '%s\n' "${results[@]}"
+exit "${failed}"

tools/sdk-live-tests/seed_vector.py

@@ -0,0 +1,59 @@
+"""Seeds one of the SDK's user test vectors, vault included, through vaultwarden's test seeder.
+
+Usage: seed_vector.py <vector.json> <seeder url> <seeder username> <seeder password>
+
+Prints the account's email and password, one per line. Exits 3, printing why, for a vector the tests can't log in
+with (no master password), and 1 if the seeder refuses it.
+"""
+
+import base64
+import json
+import sys
+import urllib.error
+import urllib.request
+import uuid
+
+SKIP = 3
+
+
+def main() -> int:
+    vector_file, seeder_url, username, password = sys.argv[1:5]
+    with open(vector_file, encoding="utf-8") as f:
+        vector = json.load(f)
+
+    has_master_password = any("masterPasswordUnlock" in method for method in vector.get("unlockMethods", []))
+    if not has_master_password or "password" not in vector.get("account", {}):
+        print("no master password")
+        return SKIP
+
+    # Without attachments: the rotation test compares the vault before and after, and our sync gives each attachment
+    # a link with a fresh download token every time (upstream's is the same on every sync). A vector's legacy
+    # attachments, without a key of their own, would also make the SDK refuse to rotate at all.
+    for cipher in vector.get("vault", {}).get("ciphers", []):
+        cipher.get("encrypted", {}).pop("attachments", None)
+
+    credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
+    request = urllib.request.Request(
+        f"{seeder_url}/seed",
+        data=json.dumps({"template": "TestVectorUserScene", "arguments": vector}).encode(),
+        method="POST",
+        headers={
+            "Authorization": f"Basic {credentials}",
+            "Content-Type": "application/json",
+            "X-Play-Id": f"sdk-live-tests-{uuid.uuid4()}",
+        },
+    )
+    try:
+        with urllib.request.urlopen(request) as response:
+            result = json.load(response)["result"]
+    except urllib.error.HTTPError as e:
+        print(f"The seeder refused {vector_file}: {e.code} {e.read().decode(errors='replace')}", file=sys.stderr)
+        return 1
+
+    print(result["email"])
+    print(vector["account"]["password"])
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())