chore: add agent-run dashboard flow checks and Magellan explorer

.github/workflows/flow-checks.yml

@@ -0,0 +1,152 @@
+# Dashboard flow checks: can a user still finish each core product flow?
+# See e2e/flows/README.md.
+#
+# Replays every e2e/flows/*/flow.spec.ts against a freshly booted stack. When a
+# flow fails, a Claude Code agent re-walks that flow's story and says whether
+# the UI only moved (healed), the flow broke, or the PR changed it on purpose.
+# The verdicts go into one PR comment. This check never blocks a merge yet.
+name: Flow checks
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - client/dashboard/**
+      - e2e/flows/**
+      - e2e/flows.config.ts
+      - .mise-tasks/test/flows.sh
+      - .mise-tasks/flows/**
+      - .github/workflows/flow-checks.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: flow-checks-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  flow-checks:
+    name: flow-checks
+    # Booting the stack needs ATLAS_TOKEN and healing needs ANTHROPIC_API_KEY;
+    # a fork PR gets neither.
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    timeout-minutes: 60
+    permissions:
+      contents: read
+      pull-requests: write # posts the verdict comment
+    env:
+      GOMAXPROCS: 8
+      ATLAS_TOKEN: ${{ secrets.ATLAS_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1
+        with:
+          # The heal agent reads `git diff origin/main...HEAD` to tell an
+          # intended flow change from a broken one.
+          fetch-depth: 0
+
+      - name: Setup Mise
+        uses: jdx/mise-action@9149ea85001c7435d5a66bb127d6a1b6227cb0a5 # v5.0.0
+        with:
+          minimum_release_age: 1d
+          install: true
+          cache: true
+          cache_key_prefix: mise-blacksmith-v1
+          env: false
+
+      - name: Restore Go module cache
+        uses: ./.github/workflows/composite/go-mod-cache # zizmor: ignore[self-repository]
+
+      # Same answers as wake-stack-e2e.yml: zero must not prompt on a runner.
+      - name: Answer zero's questions before it asks them
+        run: |
+          cat > mise.local.toml <<'TOML'
+          [env]
+          CLICKHOUSE_MIGRATION_ENGINE = "golang-migrate"
+          TOML
+
+      - name: Give the seed someone to be
+        run: |
+          git config user.email "flow-checks@example.com"
+          git config user.name "Flow Checks"
+
+      - name: Boot the stack
+        run: ./zero --agent
+
+      - name: Replay flows
+        id: replay
+        continue-on-error: true
+        run: mise run test:flows
+
+      - name: Install Claude Code
+        if: steps.replay.outcome == 'failure'
+        run: curl -fsSL https://claude.ai/install.sh | bash
+
+      # One agent per failing flow. Each writes e2e/.flows/<flow>-verdict.json.
+      - name: Heal failing flows
+        if: steps.replay.outcome == 'failure'
+        continue-on-error: true
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          failed=$(jq -r '[.. | objects | select(.specs?) | .specs[]
+            | select(.ok == false) | .file | split("/")[0]] | unique[]' \
+            e2e/.flows/results.json)
+          for flow in $failed; do
+            PATH="$HOME/.local/bin:$PATH" mise run flows:heal "$flow" &
+          done
+          wait
+
+      - name: Post verdicts
+        if: always() && github.event_name == 'pull_request'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ github.event.pull_request.number }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPLAY: ${{ steps.replay.outcome }}
+        run: |
+          marker='<!-- flow-checks -->'
+          {
+            echo "$marker"
+            echo "### Flow checks"
+            if [ "$REPLAY" = success ]; then
+              echo "Every flow replayed cleanly."
+            else
+              echo "| Flow | Verdict | Step | Summary |"
+              echo "| --- | --- | --- | --- |"
+              for f in e2e/.flows/*-verdict.json; do
+                [ -f "$f" ] || continue
+                jq -r '"| \(.flow) | **\(.verdict)** | \(.step // "-") | \(.summary) |"' "$f" ||
+                  echo "| $(basename "$f" -verdict.json) | unknown | - | the agent did not return a verdict |"
+              done
+              echo
+              echo "_healed_: UI moved, flow still works; the new script is in the run artifacts."
+              echo "_broken_: a user can no longer finish the flow."
+              echo "_changed_: this PR changes the flow on purpose; update its story.md."
+            fi
+            echo
+            echo "[Run, traces and screenshots]($RUN_URL). This check does not block merging."
+          } > comment.md
+          id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" --paginate \
+            --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1)
+          if [ -n "$id" ]; then
+            gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$id" -F body=@comment.md
+          else
+            gh pr comment "$PR" --body-file comment.md
+          fi
+
+      # Healed scripts, transcripts, traces and screenshots.
+      - name: Upload flow artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: flow-checks
+          path: |
+            e2e/.flows
+            e2e/flows/*/flow.spec.ts
+          retention-days: 7
+          if-no-files-found: ignore

.github/workflows/magellan.yml

@@ -0,0 +1,106 @@
+# Magellan: a nightly agent that explores the production dashboard as a bot
+# user in a dedicated integration org, and files the bugs it can prove as
+# Linear Triage issues. See e2e/magellan/README.md.
+#
+# The repo is public, so this workflow uploads no artifacts and logs no
+# findings: screenshots and detail go to Linear only.
+#
+# Setup, in the `magellan` environment:
+# - vars:    MAGELLAN_SITE_URL (e.g. https://app.speakeasy.com), MAGELLAN_ORG
+# - secrets: MAGELLAN_SESSION (the bot's `gram_session`), ANTHROPIC_API_KEY,
+#            MAGELLAN_LINEAR_API_KEY
+name: Magellan
+
+on:
+  workflow_dispatch:
+    inputs:
+      since:
+        description: Focus on dashboard changes merged since (git date)
+        default: 24 hours ago
+      file:
+        description: File Linear issues (otherwise dry run)
+        type: boolean
+        default: true
+  schedule:
+    # Explore on weeknights, after the day's merges.
+    - cron: "0 3 * * 2-6"
+    # Keep the bot's session alive. It expires after 72h without use, and
+    # the weekend gap alone is about that long.
+    - cron: "0 15 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: magellan-${{ github.event.schedule || 'dispatch' }}
+  cancel-in-progress: false
+
+jobs:
+  keepalive:
+    name: keepalive
+    if: github.event.schedule == '0 15 * * *'
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    environment: magellan
+    timeout-minutes: 5
+    steps:
+      # Any authenticated call slides the session's 72h expiry. A failure
+      # here means the session is gone and someone has to capture a new one.
+      - name: Refresh the bot session
+        env:
+          SITE_URL: ${{ vars.MAGELLAN_SITE_URL }}
+          SESSION: ${{ secrets.MAGELLAN_SESSION }}
+        run: |
+          status=$(curl -s -o /dev/null -w '%{http_code}' \
+            -H "Gram-Session: $SESSION" "${SITE_URL%/}/rpc/auth.info")
+          if [ "$status" != 200 ]; then
+            echo "::error::The Magellan bot session is no longer valid (auth.info returned $status). Capture a new one; see e2e/magellan/README.md."
+            exit 1
+          fi
+
+  explore:
+    name: explore
+    if: github.event_name == 'workflow_dispatch' || github.event.schedule == '0 3 * * 2-6'
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    environment: magellan
+    timeout-minutes: 75
+    steps:
+      - name: Checkout
+        uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1
+        with:
+          fetch-depth: 0
+
+      - name: Setup Mise
+        uses: jdx/mise-action@9149ea85001c7435d5a66bb127d6a1b6227cb0a5 # v5.0.0
+        with:
+          minimum_release_age: 1d
+          install: true
+          cache: true
+          cache_key_prefix: mise-blacksmith-v1
+          env: false
+
+      - name: Install the browser
+        run: mise run playwright:install-browser
+
+      - name: Install Claude Code
+        run: curl -fsSL https://claude.ai/install.sh | bash
+
+      - name: Explore
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          MAGELLAN_SITE_URL: ${{ vars.MAGELLAN_SITE_URL }}
+          MAGELLAN_ORG: ${{ vars.MAGELLAN_ORG }}
+          MAGELLAN_SESSION: ${{ secrets.MAGELLAN_SESSION }}
+          SINCE: ${{ inputs.since || '24 hours ago' }}
+        run: PATH="$HOME/.local/bin:$PATH" mise run magellan:explore --since "$SINCE"
+
+      - name: File findings
+        env:
+          LINEAR_API_KEY: ${{ secrets.MAGELLAN_LINEAR_API_KEY }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          FILE: ${{ github.event_name == 'schedule' || inputs.file }}
+        run: |
+          if [ "$FILE" = true ]; then
+            mise run magellan:file --file --run-url "$RUN_URL"
+          else
+            mise run magellan:file --run-url "$RUN_URL"
+          fi

.gitignore

@@ -106,6 +106,10 @@ __pycache__/
 
 # Traces, videos and screenshots from the e2e suite (mise run test:wake-stack)
 e2e/.playwright
+# Flow check results, traces and heal-agent transcripts (mise run test:flows)
+e2e/.flows
+# Magellan explorer runs: findings, screenshots, transcripts (mise run magellan:explore)
+e2e/.magellan
 
 # Riptide artifacts (cloud-synced)
 .humanlayer/tasks/

.mise-tasks/flows/heal.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+#MISE description="Have a Claude Code agent re-walk a flow story, then heal its script or report the flow broken"
+#MISE dir="{{ config_root }}"
+#MISE depends=["ensure-stack"]
+#USAGE arg "<flow>" help="Directory name under e2e/flows/, e.g. catalog-connect"
+
+set -euo pipefail
+
+flow="${usage_flow:?}"
+if [ ! -f "e2e/flows/${flow}/story.md" ]; then
+    echo "No story at e2e/flows/${flow}/story.md" >&2
+    exit 1
+fi
+
+mkdir -p e2e/.flows
+verdict_file="e2e/.flows/${flow}-verdict.json"
+
+# The agent's last line is the verdict JSON (see e2e/flows/HEAL.md). Keep the
+# full transcript for whoever reads the CI artifacts.
+sed "s/{{FLOW}}/${flow}/g" e2e/flows/HEAL.md |
+    claude -p \
+        --permission-mode acceptEdits \
+        --allowedTools "Read,Write,Edit,Glob,Grep,Bash(mise run playwright:*),Bash(mise run test:flows:*),Bash(git diff:*),Bash(git log:*),Bash(aube run -F @gram/e2e type-check)" \
+        --max-turns 80 |
+    tee "e2e/.flows/${flow}-transcript.txt" |
+    tail -n 1 >"$verdict_file"
+
+cat "$verdict_file"

.mise-tasks/magellan/explore.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+#MISE description="Magellan: let a Claude Code agent explore the dashboard and record bugs it can prove"
+#MISE dir="{{ config_root }}"
+#USAGE flag "--since <since>" default="24 hours ago" help="Focus on dashboard changes merged to main since this git date"
+#USAGE flag "--budget <minutes>" default="45" help="Rough time budget for the agent"
+
+# Two targets:
+#
+# - Local (default): this worktree's stack, logged in through dev-idp.
+# - Remote: set MAGELLAN_SITE_URL, MAGELLAN_ORG and MAGELLAN_SESSION (a
+#   `gram_session` token for a dedicated bot user, captured from a real login).
+#   No login can be scripted against WorkOS, so the session is injected into
+#   the browser instead. See e2e/magellan/README.md.
+
+set -euo pipefail
+
+run_dir="e2e/.magellan/$(date -u +%Y%m%dT%H%M%SZ)"
+mkdir -p "$run_dir"
+touch "$run_dir/findings.jsonl"
+
+if [ -n "${MAGELLAN_SITE_URL:-}" ]; then
+    : "${MAGELLAN_ORG:?MAGELLAN_ORG is required with MAGELLAN_SITE_URL}"
+    : "${MAGELLAN_SESSION:?MAGELLAN_SESSION is required with MAGELLAN_SITE_URL}"
+    site_url="${MAGELLAN_SITE_URL%/}"
+    org="$MAGELLAN_ORG"
+    pw="playwright-cli -s=magellan"
+    allowed_pw="Bash(playwright-cli -s=magellan:*)"
+
+    # Outside the run directory, which gets uploaded: this file is a login.
+    state_file="$(mktemp -t magellan-state.XXXXXX)"
+    trap 'rm -f "$state_file"' EXIT
+    host="$(node -e 'console.log(new URL(process.argv[1]).hostname)' "$site_url")"
+    node -e '
+      const [host, token, file] = process.argv.slice(1);
+      const cookie = { name: "gram_session", value: token, domain: host, path: "/",
+        expires: Math.floor(Date.now() / 1000) + 86400, httpOnly: true, secure: true, sameSite: "Lax" };
+      require("node:fs").writeFileSync(file, JSON.stringify({ cookies: [cookie], origins: [] }));
+    ' "$host" "$MAGELLAN_SESSION" "$state_file"
+    login="You are already logged in as a bot user. Start with \`$pw open\`, \`$pw state-load $state_file\`, then \`$pw goto $site_url/$org\`. Never log out: it ends the bot's session for good. Do not read the state file."
+else
+    mise run ensure-stack
+    site_url="$GRAM_SITE_URL"
+    org="${MAGELLAN_ORG:-speakeasy}"
+    pw="mise run playwright -s=magellan"
+    allowed_pw="Bash(mise run playwright:*)"
+    login="Log in with the \"Log in\" button."
+fi
+
+# What changed recently is where new bugs are most likely. The agent maps
+# these files to routes itself.
+git fetch --quiet origin main || true
+{
+    echo "# Dashboard changes on main since ${usage_since:?}"
+    echo
+    # Tests and the generated SDK change no page a user sees.
+    git log origin/main --since="$usage_since" --format='## %h %s' --name-only -- \
+        client/dashboard/src ':!client/dashboard/src/sdk' ':!*.test.ts' ':!*.test.tsx' |
+        grep -v '^$' || echo "(none — roam instead)"
+} >"$run_dir/focus.md"
+
+# `#` delimits sed below, so none of the substituted values may contain one.
+sed -e "s#{{RUN_DIR}}#${run_dir}#g" \
+    -e "s#{{BUDGET}}#${usage_budget:?}#g" \
+    -e "s#{{SITE_URL}}#${site_url}#g" \
+    -e "s#{{ORG}}#${org}#g" \
+    -e "s#{{PW}}#${pw}#g" \
+    -e "s#{{LOGIN}}#${login}#g" \
+    e2e/magellan/EXPLORE.md |
+    claude -p \
+        --permission-mode acceptEdits \
+        --allowedTools "Read,Write,Edit,Glob,Grep,${allowed_pw},Bash(git log:*),Bash(git diff:*),Bash(date:*)" \
+        --max-turns 200 \
+        >"$run_dir/transcript.txt"
+
+echo "$run_dir"
+wc -l <"$run_dir/findings.jsonl" | xargs echo "findings:"