Move docker module loading to et-cli (#124)

Author: jayvdb

.codacy.yaml

@@ -1,8 +1,6 @@
 # Codacy's in-repo config can only exclude paths -- it cannot disable individual patterns (dashboard-only).
 # These paths carry findings the repo already covers elsewhere or has consciously decided against:
 # - config/semgrep/: our Semgrep rule files; Codacy's own Semgrep misfires on the `- id:` each rule declares.
-# - Dockerfile: Codacy's default hadolint re-flags DL4006 (curl | sh pipefail), which config/hadolint.yaml
-#   deliberately ignores and the repo's own hadolint-check + DeepSource Docker already cover.
 # - dotnet-data1/Program.cs: Codacy's CA1054 wants URI-typed params, but [JSImport] partial methods can only
 #   marshal `string` across the JS boundary; DeepSource covers the C# there and reports no such issue.
 # - .github/workflows/ + .github/actions/: Codacy flags GitHub Actions `uses:` not pinned to a commit SHA, but
@@ -38,7 +36,6 @@ exclude_paths:
   - ".github/actions/**"
   - ".github/workflows/**"
   - "CLAUDE.md"
-  - "Dockerfile"
   - "config/semgrep/**"
   - "services/ws-modules/dotnet-data1/Program.cs"
   - "services/ws-modules/wasi-comm1/src/coverage.rs"

.dockerignore

@@ -55,4 +55,3 @@ services/ws-server/storage/
 **/.git/
 **/Dockerfile*
 README.md
-**/.dockerignore

.github/workflows/coverage.yaml

@@ -98,7 +98,14 @@ jobs:
 
       # Fold the instrumented wasm guest modules' coverage into lcov.info so it rides the same `rust` flag.
       # The WASI + browser modules dumped their .profraw to target/wasi-cov during the runner tests above.
+      #
+      # This and the two merges below carry the same guard as the uploads. The coverage task re-raises a failing
+      # test status once it has written lcov.info, so without a guard these merges skip on exactly the runs that
+      # still upload -- publishing a Rust-only report as though it were the whole picture, which reads downstream
+      # as a large coverage drop rather than as a failed run. `hashFiles` keeps them off a run that produced no
+      # report at all, where there is nothing to merge into.
       - name: Merge wasm guest coverage into lcov.info
+        if: ${{ !cancelled() && hashFiles('lcov.info') != '' }}
         timeout-minutes: 10
         run: mise run wasm-cov
 
@@ -107,6 +114,7 @@ jobs:
       # the agent lib's coverage into the same lcov.info. CHROMEWEBDRIVER is the GitHub image's chromedriver that
       # matches its preinstalled Chrome; the task falls back to the mise-pinned http:chromedriver when it is unset.
       - name: Merge ws-wasm-agent coverage into lcov.info
+        if: ${{ !cancelled() && hashFiles('lcov.info') != '' }}
         timeout-minutes: 20
         run: |
           export CHROMEDRIVER="${CHROMEWEBDRIVER:+$CHROMEWEBDRIVER/chromedriver}"
@@ -116,12 +124,15 @@ jobs:
       # Its native parse tests already ride that run; this drives the module's wasm-bindgen display tests in
       # the same headless Chrome as the ws-wasm-agent step above and folds their lcov into lcov.info.
       - name: Merge pic-viewer coverage into lcov.info
+        if: ${{ !cancelled() && hashFiles('lcov.info') != '' }}
         timeout-minutes: 20
         run: |
           export CHROMEDRIVER="${CHROMEWEBDRIVER:+$CHROMEWEBDRIVER/chromedriver}"
           mise run pic-viewer-cov
 
+      # Python coverage does not depend on the Rust result, so a failing Rust test must not skip it.
       - name: Collect Python coverage
+        if: ${{ !cancelled() }}
         timeout-minutes: 15
         env:
           GITHUB_TOKEN: ${{ github.token }}
@@ -137,7 +148,13 @@ jobs:
             coverage-python.xml
           if-no-files-found: error
 
+      # Every upload below runs on a failed run too, gated on its report actually existing.
+      # A failing test is when the coverage delta matters most, and the coverage task now writes lcov.info before
+      # re-raising the test status. The `hashFiles` half keeps a genuinely absent report from turning into a
+      # second, misleading failure: when an earlier step (a tool install, say) means nothing was ever produced,
+      # these skip instead of erroring on a missing file and burying the real cause.
       - name: Upload Rust coverage to Codecov (OIDC)
+        if: ${{ !cancelled() && hashFiles('lcov.info') != '' }}
         uses: codecov/codecov-action@v7
         with:
           use_oidc: true
@@ -146,6 +163,7 @@ jobs:
           fail_ci_if_error: true
 
       - name: Upload Python coverage to Codecov (OIDC)
+        if: ${{ !cancelled() && hashFiles('coverage-python.xml') != '' }}
         uses: codecov/codecov-action@v7
         with:
           use_oidc: true
@@ -154,9 +172,11 @@ jobs:
           fail_ci_if_error: true
 
       - name: Upload Rust coverage to DeepSource (OIDC)
+        if: ${{ !cancelled() && hashFiles('lcov.info') != '' }}
         run: deepsource report --analyzer test-coverage --key rust --value-file lcov.info --use-oidc
 
       - name: Upload Python coverage to DeepSource (OIDC)
+        if: ${{ !cancelled() && hashFiles('coverage-python.xml') != '' }}
         run: deepsource report --analyzer test-coverage --key python --value-file coverage-python.xml --use-oidc
 
       # Upload test results even when tests failed.

.mise/config.coverage.toml

@@ -114,7 +114,14 @@ export __CARGO_LLVM_COV_RUSTC_WRAPPER_RUSTFLAGS="$flags"
 # et-ws-web-runner/coverage adds the browser-module capture (shims + the __et_capture_coverage PUT);
 # et-ws-wasi-runner/coverage adds the guest `/cov` preopen. ET_TEST_COVERAGE (set by the coverage job) still
 # gates capture at runtime within each build.
-cargo nextest run --config-file config/nextest.toml --features et-ws-web-runner/coverage,et-ws-wasi-runner/coverage
+# A failing test must not cost the whole report: capture the status and re-raise it once lcov.info is written.
+# Under `set -e` a nextest failure would abort the task here, so `cargo llvm-cov report` below never runs and
+# lcov.info is never produced -- the coverage upload steps then have nothing to send and the run reports no
+# coverage at all. A failing run is exactly when the per-test results and the coverage delta matter most, so the
+# profile data nextest has already written gets turned into a report either way.
+test_status=0
+cargo nextest run --config-file config/nextest.toml --features et-ws-web-runner/coverage,et-ws-wasi-runner/coverage ||
+  test_status=$?
 mise run gen-specs
 mise run regen-verification
 # The runner's host bindings are excluded from the report, not just from analysis.
@@ -128,6 +135,7 @@ mise run regen-verification
 # leading `^` matches nothing and the file stays in the report. Match the path tail instead.
 cov_ignore='services/ws-wasi-runner/src/bindings\\.rs$'
 cargo llvm-cov report --lcov --include-build-script --ignore-filename-regex "$cov_ignore" --output-path lcov.info
+exit "$test_status"
 """
 shell = "{{ vars.task_shell }}"
 

.mise/config.python.toml

@@ -16,12 +16,19 @@
 "pipx:torch" = { version = "latest", os = ["linux", "macos/arm64", "windows"] }
 ruff = "latest"
 
-# Use the GitHub release tarball, not `npm:pyodide`.
-# The npm package is only the runtime (pyodide.js, .wasm, stdlib, micropip) ~5 MB -- it's designed for JS apps
-# that fetch the rest from JsDelivr's Pyodide custom-CDN namespace (https://www.jsdelivr.com/oss-cdn) at
-# runtime. We need the ~200 MB "full" distribution served locally so browser modules work offline; that's
-# only published as a tarball on GitHub releases. The extracted tarball contains a top-level `pyodide/`
-# directory the modules service picks up -- see `default_modules_folders` in libs/edge-toolkit/src/config.rs.
+# Both pyodide distributions are pinned, because a deployment needs one or the other depending on its modules.
+# The npm package is only the runtime (pyodide.js, .wasm, stdlib, micropip) -- it's designed for JS apps that
+# fetch the rest from JsDelivr's Pyodide custom-CDN namespace (https://www.jsdelivr.com/oss-cdn) at runtime, so
+# it suffices for a module that imports the stdlib and nothing more. A module calling `micropip.install` pulls a
+# non-stdlib wheel and needs the "full" distribution served locally to work offline; that is published only as a
+# tarball on GitHub releases. This workspace serves the full one by default; `et-cli` picks between them per
+# scenario. The extracted tarball contains a top-level `pyodide/` directory the modules service picks up -- see
+# `default_modules_folders` in libs/edge-toolkit/src/config.rs.
+#
+# Hold both rows on the same pyodide release when bumping. Note the npm package renumbered after 0.29.x: its
+# `latest` is now 314.x, tracking the bundled CPython (314 = Python 3.14) rather than continuing the 0.29 line,
+# so an unpinned `latest` here would silently cross a major boundary the http row has not.
+"npm:pyodide" = "0.29.3"
 [tools."http:pyodide"]
 # Keep every pyodide executable OFF the task PATH.
 # The dist root carries `python` / `python.bat` / `python.exe` wrapper scripts, and with no bin_path set

.mise/config.toml

@@ -79,6 +79,13 @@ cargo-binstall = "latest"
 cmake = "latest"
 "conda:openssl" = "3"
 conftest = "latest"
+# Compose CLI for the generated scenario stacks under verification/.
+# The aqua backend rather than the registry default (`github:docker/compose`): the github backend records SLSA
+# provenance in the lockfile, and a mise older than the one that wrote it refuses the entry outright with
+# "Lockfile requires slsa provenance for github:docker/compose@<ver> but verification was not performed".
+# aqua records `provenance = "github-attestations"` instead, which is what every other pinned tool here uses.
+# This is the compose implementation only -- a docker daemon still has to come from the host.
+"aqua:docker/compose" = "latest"
 "github:nextest-rs/nextest" = { version = "latest", version_prefix = "cargo-nextest-" }
 taplo = "latest"
 watchexec = "latest"
@@ -607,6 +614,11 @@ task_shell_trace = 'bash -xeuo pipefail "{{ config_root }}/.mise/task-shell.sh"'
 task_shell_plain = 'bash "{{ config_root }}/.mise/task-shell.sh"'
 # OpenObserve docker image for the openobserve/o2 dev-observability task.
 o2_image = "openobserve/openobserve:v0.91.5"
+# Port publish spec for that task, bound to loopback rather than every interface.
+# config/o2.env's credential is committed, so the collector must not be reachable off the machine. It lives here
+# as a var because the mise policy rejects an `X.Y.Z` literal inside a task `run` body, and an IPv4 address trips
+# that check the same way a version would.
+o2_publish = "127.0.0.1:5080:5080"
 # OpenTelemetry Collector image for the o2-hostmetrics / o2-macmon / o2-nvidia / o2-winmetrics tasks.
 otelcol_image = "otel/opentelemetry-collector-contrib:0.156.0"
 rp_wasm_dir = '''{% if os() == "windows" and env?.MISE_INSTALLS_DIR -%}
@@ -727,6 +739,7 @@ depends = [
   "jscpd-check",
   "link-check",
   "ls-lint-check",
+  "mise-lock-check",
   "readme-mise-version-check",
   "regal-check",
   "ryl-check",
@@ -808,7 +821,15 @@ run = "semgrep scan --config config/semgrep --autofix --metrics=off ."
 
 [tasks.hadolint-check]
 description = "Lint the Dockerfiles with hadolint"
-run = "git ls-files '*Dockerfile' '*Dockerfile.*' | xargs hadolint --config config/hadolint.yaml"
+# `*Dockerfile.*` also catches a BuildKit per-Dockerfile ignore file, which is not a Dockerfile.
+# BuildKit looks for `<path-to-dockerfile>.dockerignore` beside the Dockerfile it is told to build, so the name
+# is fixed and cannot be moved out of the glob's way; hadolint parses it as Dockerfile syntax and stops on the
+# first ignore pattern with `unexpected '*' expecting ... a pragma, at least one space, or end of input`.
+run = """
+git ls-files '*Dockerfile' '*Dockerfile.*' | rg -v '\\.dockerignore$' |
+  xargs hadolint --config config/hadolint.yaml
+"""
+shell = "{{ vars.task_shell }}"
 
 # Duplication is gated as a fingerprint ratchet, not as a percentage.
 # config/jscpd-baseline.json records the fingerprint of every clone that already existed, and only a clone absent
@@ -976,8 +997,12 @@ description = "Cross-check Dockerfiles against mise [tools] pins + heredoc body-
 # Each git-tracked Dockerfile* gets both passes, except that the `ignore` pass only visits Dockerfiles that
 # actually contain heredocs: conftest 0.68.2's ignore parser panics with `runtime error: slice bounds out of
 # range` on plain Dockerfiles (no heredocs), and the body-first-line rule has nothing to check there anyway.
+# A BuildKit per-Dockerfile ignore file is dropped from the list first. `<path-to-dockerfile>.dockerignore` is
+# the name BuildKit demands beside the Dockerfile it builds, so it cannot be moved out of the glob's way, and
+# the dockerfile parser reads its first pattern as an instruction:
+#   unknown instruction: **/.claude/
 run = """
-files=$(git ls-files '*Dockerfile' '*Dockerfile.*')
+files=$(git ls-files '*Dockerfile' '*Dockerfile.*' | rg -v '\\.dockerignore$')
 # $files is a newline-separated path list; word-splitting is intentional (no spaces in paths).
 # shellcheck disable=SC2086
 conftest test --combine --namespace dockerfile -p config/conftest/policy $files .mise
@@ -1289,6 +1314,11 @@ description = "Regenerate .dockerignore from .gitignore + Docker-only excludes"
 # A bare pattern there is root-anchored -- whereas a slash-less .gitignore pattern matches at any depth.
 # So mirror .gitignore, add the Docker-only excludes (`.git/`, plus the build recipe + docs so editing
 # them doesn't bust the `COPY . .` cache), and `**/`-prefix each non-anchored pattern.
+# `.dockerignore` itself is deliberately NOT excluded any more. Excluding it saved nothing (the file is tiny) and
+# meant it was absent from the build context, so `et-cli` -- which derives each scenario's per-Dockerfile ignore
+# file from it -- could not run inside an image built from this Dockerfile:
+#     called `Result::unwrap()` on an `Err` value: Io(... NotFound ... path: "/workspace/.dockerignore")
+# That surfaced only in the docker build.yaml lanes, because every other environment has the file on disk.
 # goawk emits CRLF on Windows, so the whole block is piped through `tr -d '\r'` -- .dockerignore must stay
 # LF-only (editorconfig enforces it, and CRLF would also make dockerignore-check see a spurious diff).
 run = '''
@@ -1299,7 +1329,7 @@ run = '''
   echo
   {
     cat .gitignore
-    printf '%s\n' '.git/' 'Dockerfile*' '/README.md' '.dockerignore'
+    printf '%s\n' '.git/' 'Dockerfile*' '/README.md'
   } | goawk '
     /^[[:space:]]*#/ { print; next }
     /^[[:space:]]*$/ { print; next }
@@ -1514,7 +1544,8 @@ description = "Run the WebSocket server, restarting on .rs/.toml changes"
 run = "watchexec --restart --clear --exts rs,toml -i '**/tests/**' --shell=none -- cargo run -p et-ws-server"
 
 [tasks.ws-server.env]
-OTLP_AUTH_PASSWORD = "1234"
+# Must match ZO_ROOT_USER_PASSWORD in config/o2.env; the server authenticates its OTLP exports as that root user.
+OTLP_AUTH_PASSWORD = "Complexpass#123"  # skipcq: SCT-A000 -- local dev-only credential, committed deliberately
 OTLP_AUTH_USERNAME = "root@example.com"
 
 [tasks.ws-server-no-otel]
@@ -1753,7 +1784,7 @@ description = "Run both the ws-server and ws-wasm-agent using Chrome"
 
 [tasks.openobserve]
 alias = "o2"
-run = "docker run --rm --name openobserve -p 5080:5080 --env-file config/o2.env {{ vars.o2_image }}"
+run = "docker run --rm --name openobserve -p {{ vars.o2_publish }} --env-file config/o2.env {{ vars.o2_image }}"
 
 [tasks.demo]
 depends = ["openobserve", "ws-server"]
@@ -2059,6 +2090,32 @@ description = "Install every language toolchain (all .mise/config.<lang>.toml)"
 run = 'MISE_ENV="$ALL_LANGS" mise install'
 shell = "{{ vars.task_shell_plain }}"
 
+# Fail when a full install resolves a different tool set or version than the committed lockfiles record.
+# Only the platform-independent fields are compared, because mise fills a lockfile's per-platform rows in lazily on
+# whichever platform runs the install: `checksum`, `url`, `url_api`, `provenance` and `provenance_verified` appear
+# for macos-arm64 on a developer's machine and for linux-x64 on a runner. A whole-file `git diff` therefore can
+# never be clean across the five-platform matrix -- every lane legitimately adds its own rows, and the committed
+# lockfile already carries a checksum for some platforms of a tool and not others. The `[[tools.<name>]]` headers
+# and their `version` / `backend` are identical everywhere, so those are what this gates on: a tool added, removed
+# or bumped without regenerating the lockfiles is caught, while a lane recording its own checksums is not.
+# install-all rather than a bare install, so guest-language tools reach their lockfiles too -- MISE_ENV only
+# loads a subset, and a partial install leaves every unloaded config's lockfile unexamined.
+[tasks.mise-lock-check]
+depends = ["install-all"]
+description = "Fail if .mise/mise*.lock records a different tool set or version than a full install resolves"
+# `|| true` because rg exits 1 on no matches, which under pipefail would abort the whole task on the clean path.
+# `^[-+][-+]` drops git's `+++`/`---` file headers; `@@` hunk headers never match the leading `[-+]`.
+run = """
+drift=$(git diff -U0 -- .mise/mise*.lock | rg "^[-+]" | rg -v "^[-+][-+]" |
+  rg "^[-+](version|backend) = |^[-+]\\[\\[tools\\." || true)
+if [ -n "$drift" ]; then
+  echo "$drift"
+  echo "Lockfile tool set/versions differ from install-all; commit the regenerated .mise/mise*.lock" >&2
+  exit 1
+fi
+"""
+shell = "{{ vars.task_shell }}"
+
 [tasks.all]
 # Generic stand-in: run any command with every language env enabled.
 # mise appends the task's CLI args after `mise exec --`, e.g. `mise run all zig version`.