diff --git a/.github/workflows/sdk-live-tests.yml b/.github/workflows/sdk-live-tests.yml new file mode 100644 index 00000000..45b37c7e --- /dev/null +++ b/.github/workflows/sdk-live-tests.yml @@ -0,0 +1,74 @@ +name: SDK live tests +permissions: {} + +# Runs the live-server end-to-end tests of bitwarden/sdk-internal against this branch, see tools/sdk-live-tests/run.sh. +# It builds the SDK too, so it only runs by hand and on changes to the test seeder or the runner. +on: + workflow_dispatch: + inputs: + sdk_ref: + description: "Branch, tag or commit of bitwarden/sdk-internal to test with" + required: true + default: "main" + pull_request: + paths: + - ".github/workflows/sdk-live-tests.yml" + - "src/api/test_seeder.rs" + - "tools/sdk-live-tests/**" + +defaults: + run: + shell: bash + +jobs: + sdk-live-tests: + name: SDK live tests + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: "Install dependencies Ubuntu" + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential libssl-dev pkg-config + + - name: "Checkout" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: "Checkout sdk-internal" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: bitwarden/sdk-internal + ref: ${{ inputs.sdk_ref || 'main' }} + path: sdk-internal + persist-credentials: false + + # Each checkout pins its own toolchain in rust-toolchain.toml + - name: "Install toolchains" + run: | + rustup toolchain install + cd sdk-internal + rustup toolchain install + rustup target add wasm32-unknown-unknown + rustup component add rust-src + + - name: "Cache Rust builds" + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: | + . -> target + sdk-internal -> target + + - name: "Setup Node" + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: sdk-internal/.nvmrc + + - name: "Run the SDK live tests" + run: tools/sdk-live-tests/run.sh --sdk-dir sdk-internal + + - name: "Upload logs" + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sdk-live-tests-logs + path: target/sdk-live-tests/**/*.log diff --git a/tools/sdk-live-tests/run.sh b/tools/sdk-live-tests/run.sh new file mode 100755 index 00000000..d5d9456f --- /dev/null +++ b/tools/sdk-live-tests/run.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# Runs the live-server end-to-end tests of bitwarden/sdk-internal (e2e/wasm) against vaultwarden. +# +# Usage: tools/sdk-live-tests/run.sh --sdk-dir [--skip-sdk-build] [--mode vectors|seeder] [-- ] +# +# The server is built with `unsafe_test_mode`, whose test seeder (src/api/test_seeder.rs) creates the test accounts. +# - vectors: for SDK versions whose live tests log in to one given account. Every user test vector with a master +# password gets a fresh server, is seeded there with its vault, and the tests run on it (they rotate its keys). +# - seeder: for SDK versions whose tests seed their own accounts through BW_LIVE_SEEDER_URL. One server, one run. +# The mode follows what the SDK's e2e/wasm/live/config.ts reads, unless --mode says otherwise. Arguments after `--` go +# to vitest, e.g. `-- tests/user-crypto-management/rotate-user-keys.live.test.ts` to run one file. +# +# Every vector runs even after a failure; a summary follows, and the exit code is 1 if any failed. The server logs are +# in /sdk-live-tests/. BW_LIVE_CLIENT_VERSION defaults to 2026.10.1, the minimum for V2 accounts. +# +# Needs: the SDK's Rust toolchain with the wasm32-unknown-unknown target and rust-src, Node.js >= 22.12, curl, python3. +set -euo pipefail + +VW_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +TARGET_DIR="${CARGO_TARGET_DIR:-${VW_DIR}/target}" +OUT_DIR="${TARGET_DIR}/sdk-live-tests" +PORT=8099 +URL="http://127.0.0.1:${PORT}" +SEEDER_URL="${URL}/test/seeder" +SEEDER_USERNAME="sdk-live-tests" +SEEDER_PASSWORD="$(python3 -c 'import secrets; print(secrets.token_urlsafe(24))')" +SKIP=3 # seed_vector.py's exit code for a vector the tests can't use +export BW_LIVE_CLIENT_VERSION="${BW_LIVE_CLIENT_VERSION:-2026.10.1}" + +SDK_DIR="" +BUILD_SDK=1 +MODE="" +VITEST_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --sdk-dir) SDK_DIR="$(cd "$2" && pwd)"; shift 2 ;; + --skip-sdk-build) BUILD_SDK=0; shift ;; + --mode) MODE="$2"; shift 2 ;; + --) shift; VITEST_ARGS=("$@"); break ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done +[[ -n "${SDK_DIR}" ]] || { echo "--sdk-dir is required" >&2; exit 2; } +TESTS_DIR="${SDK_DIR}/e2e/wasm" +[[ -f "${TESTS_DIR}/live/config.ts" ]] || { echo "${TESTS_DIR}/live/config.ts not found: no live tests in this SDK" >&2; exit 2; } +if [[ -z "${MODE}" ]]; then + if grep -q BW_LIVE_SEEDER_URL "${TESTS_DIR}/live/config.ts"; then MODE=seeder; else MODE=vectors; fi +fi +[[ "${MODE}" == vectors || "${MODE}" == seeder ]] || { echo "--mode must be vectors or seeder" >&2; exit 2; } +echo "Mode: ${MODE}, client version ${BW_LIVE_CLIENT_VERSION}" + +(cd "${VW_DIR}" && cargo build --features sqlite,unsafe_test_mode) +if [[ ${BUILD_SDK} -eq 1 ]]; then + bash "${SDK_DIR}/crates/bitwarden-wasm-internal/build.sh" + (cd "${TESTS_DIR}" && npm ci) +fi + +server="" +trap 'if [[ -n "${server}" ]]; then kill "${server}" 2>/dev/null || true; fi' EXIT + +# Starts a server on an empty database in $OUT_DIR/$1 +start_server() { + local data_dir="${OUT_DIR}/$1" + rm -rf "${data_dir}" && mkdir -p "${data_dir}" + if curl -s -o /dev/null "${URL}/alive"; then + echo "Something already answers on port ${PORT}" >&2 + return 1 + fi + ENV_FILE=/dev/null DATA_FOLDER="${data_dir}" DATABASE_URL="sqlite://${data_dir}/db.sqlite3" \ + ROCKET_ADDRESS=127.0.0.1 ROCKET_PORT="${PORT}" DOMAIN="${URL}" WEB_VAULT_ENABLED=false \ + SIGNUPS_ALLOWED=true SIGNUPS_VERIFY=false LOGIN_RATELIMIT_MAX_BURST=1000 \ + TEST_SEEDER_USERNAME="${SEEDER_USERNAME}" TEST_SEEDER_PASSWORD="${SEEDER_PASSWORD}" \ + "${TARGET_DIR}/debug/vaultwarden" > "${data_dir}/vaultwarden.log" 2>&1 & + server=$! + local i + for i in $(seq 1 60); do + if ! kill -0 "${server}" 2>/dev/null; then break; fi + if curl -sf -o /dev/null "${URL}/alive"; then return 0; fi + sleep 1 + done + echo "The server didn't start; the end of ${data_dir}/vaultwarden.log:" >&2 + tail -50 "${data_dir}/vaultwarden.log" >&2 + stop_server + return 1 +} + +stop_server() { + if [[ -n "${server}" ]]; then + kill "${server}" 2>/dev/null || true + wait "${server}" 2>/dev/null || true + server="" + fi +} + +run_tests() { + (cd "${TESTS_DIR}" && BW_LIVE_SERVER_URL="${URL}" "$@" npm run test:live -- "${VITEST_ARGS[@]}") +} + +results=() +failed=0 +if [[ "${MODE}" == seeder ]]; then + if start_server seeder && run_tests env BW_LIVE_SEEDER_URL="${SEEDER_URL}" \ + BW_LIVE_SEEDER_USERNAME="${SEEDER_USERNAME}" BW_LIVE_SEEDER_PASSWORD="${SEEDER_PASSWORD}"; then + results+=("seeder: pass") + else + results+=("seeder: FAIL") + failed=1 + fi + stop_server +else + for vector_file in "${SDK_DIR}"/test-vectors/users/*.json; do + vector="$(basename "${vector_file}" .json)" + echo "=== ${vector}" + if ! start_server "${vector}"; then + results+=("${vector}: FAIL (server didn't start)") + failed=1 + continue + fi + status=0 + credentials="$(python3 -I "${VW_DIR}/tools/sdk-live-tests/seed_vector.py" "${vector_file}" "${SEEDER_URL}" \ + "${SEEDER_USERNAME}" "${SEEDER_PASSWORD}")" || status=$? + if [[ ${status} -eq ${SKIP} ]]; then + results+=("${vector}: skip (${credentials})") + elif [[ ${status} -ne 0 ]]; then + results+=("${vector}: FAIL (seeding)") + failed=1 + elif run_tests env BW_LIVE_EMAIL="$(sed -n 1p <<< "${credentials}")" \ + BW_LIVE_PASSWORD="$(sed -n 2p <<< "${credentials}")"; then + results+=("${vector}: pass") + else + results+=("${vector}: FAIL") + failed=1 + fi + stop_server + done +fi + +echo +echo "=== Summary (logs in ${OUT_DIR})" +printf '%s\n' "${results[@]}" +exit "${failed}" diff --git a/tools/sdk-live-tests/seed_vector.py b/tools/sdk-live-tests/seed_vector.py new file mode 100644 index 00000000..85db6f07 --- /dev/null +++ b/tools/sdk-live-tests/seed_vector.py @@ -0,0 +1,59 @@ +"""Seeds one of the SDK's user test vectors, vault included, through vaultwarden's test seeder. + +Usage: seed_vector.py + +Prints the account's email and password, one per line. Exits 3, printing why, for a vector the tests can't log in +with (no master password), and 1 if the seeder refuses it. +""" + +import base64 +import json +import sys +import urllib.error +import urllib.request +import uuid + +SKIP = 3 + + +def main() -> int: + vector_file, seeder_url, username, password = sys.argv[1:5] + with open(vector_file, encoding="utf-8") as f: + vector = json.load(f) + + has_master_password = any("masterPasswordUnlock" in method for method in vector.get("unlockMethods", [])) + if not has_master_password or "password" not in vector.get("account", {}): + print("no master password") + return SKIP + + # Without attachments: the rotation test compares the vault before and after, and our sync gives each attachment + # a link with a fresh download token every time (upstream's is the same on every sync). A vector's legacy + # attachments, without a key of their own, would also make the SDK refuse to rotate at all. + for cipher in vector.get("vault", {}).get("ciphers", []): + cipher.get("encrypted", {}).pop("attachments", None) + + credentials = base64.b64encode(f"{username}:{password}".encode()).decode() + request = urllib.request.Request( + f"{seeder_url}/seed", + data=json.dumps({"template": "TestVectorUserScene", "arguments": vector}).encode(), + method="POST", + headers={ + "Authorization": f"Basic {credentials}", + "Content-Type": "application/json", + "X-Play-Id": f"sdk-live-tests-{uuid.uuid4()}", + }, + ) + try: + with urllib.request.urlopen(request) as response: + result = json.load(response)["result"] + except urllib.error.HTTPError as e: + print(f"The seeder refused {vector_file}: {e.code} {e.read().decode(errors='replace')}", file=sys.stderr) + return 1 + + print(result["email"]) + print(vector["account"]["password"]) + return 0 + + +if __name__ == "__main__": + sys.exit(main())