Move the staging commit from tags to the revision annotation

.github/workflows/reusable.yml

@@ -240,10 +240,12 @@ jobs:
           # the runner image's standalone LLVM, not the clang bundled with Visual Studio
           _windows_llvm = "C:/Program Files/LLVM/bin/"
 
-          # the generator drives the arm64 cross toolchain, so no msvc-dev-cmd environment
+          # the generator drives the arm64 cross toolchain, so no msvc-dev-cmd environment;
+          # the image has Visual Studio 2026 only, which carries the v143 toolset as well
           _windows_11_armv8_msvs = {
               **_windows_2022_x86_64_msvs,
               "runner": "windows-11-arm",
+              "generator": "Visual Studio 18 2026",
               "target-arch": "armv8",
               "setup-msvc-toolchain": False,
           }

.github/workflows/staging-cleanup.yml

@@ -103,7 +103,7 @@ jobs:
                   continue
               cutoff = now - window
 
-              # every index is tagged (sha + short) and tags/list is OCI's only enumeration; two
+              # every index is tagged (by its run) and tags/list is OCI's only enumeration; two
               # tags share one index digest, so resolve each tag and collapse to unique digests
               digests = {}
               for tag in oras(["repo", "tags", path], f"listing tags for {path} failed").split():

.github/workflows/staging-push.yml

@@ -7,8 +7,8 @@ name: Staging Push
 # children: unreachable by name until the index lands, so a set publishes atomically, and the
 # registry refuses an index with a dangling reference.
 #
-#   ghcr.io/opendaq/nightly/staging/opendaq : <sha>, <short-sha>, latest
-#     └─ sha256:index…
+#   ghcr.io/opendaq/nightly/staging/opendaq : run-<run id>-<run attempt>, latest
+#     └─ sha256:index…  revision=<commit>  com.opendaq.dependencies.<name>.revision=<commit>
 #        ├─ sha256:a1…  title=opendaq-x86_64-linux-gcc-11-release.tar.gz
 #        │              com.opendaq: platform=linux    settings.arch=x86_64  settings.compiler=gcc   settings.compiler.version=11
 #        └─ sha256:b2…  title=opendaq-x86_64-windows-msvc-143-release.tar.gz
@@ -17,6 +17,11 @@ name: Staging Push
 # A child is named by its file; the com.opendaq.* annotations are optional, and buy a consumer a
 # resolve by settings rather than by file name.
 #
+# The run tag is unique to its index and says nothing more: one commit built against two cores
+# is two indexes, and a tag naming the commit could point at one of them only. What a package was
+# built from is in its annotations: the commit itself, and the commits of its dependencies from
+# its metadata, which the index repeats.
+#
 # NEVER prune "untagged versions" of these packages -- the live children are untagged too.
 # Pruning goes through the index only.
 on:
@@ -29,7 +34,7 @@ on:
             {
               "path": "ghcr.io/<org>/path/to/staging",   // the package, in full
               "artifacts": ["staging-*"],                // globs; each match becomes a child of its index
-              "tags": ["latest", "v1.2.3"]               // extra tags; <sha> and <short-sha> are automatic
+              "tags": ["latest", "v1.2.3"]               // extra tags; run-<run id>-<run attempt> is automatic
             },
             {                                            // one entry per kind: a tarball and a .deb are
               "path": "ghcr.io/<org>/path/to/package",   // variants of nothing, so they never share
@@ -61,28 +66,22 @@ jobs:
           TOKEN: ${{ github.token }}
           ACTOR: ${{ github.actor }}
           SHA: ${{ github.sha }}
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
           REPO: ${{ github.repository }}
           SERVER_URL: ${{ github.server_url }}
         run: |
           import json, os, glob, subprocess, sys
           from datetime import datetime, timezone
 
-          stagings = json.loads(os.environ["STAGINGS"])
-          token    = os.environ["TOKEN"]
-          actor    = os.environ["ACTOR"]
-          sha      = os.environ["SHA"]
-          short    = sha[:7]
-          repo     = os.environ["REPO"]
-          server   = os.environ["SERVER_URL"]
-          created  = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")   # push time; staging-cleanup ages by this
-
           PACKAGE_TYPE = "application/vnd.opendaq.package.v1"
           INDEX_TYPE   = "application/vnd.opendaq.index.v1"   # what is inside: see the descriptors
+          DEPENDENCIES = "com.opendaq.dependencies."
 
           # The registry UI offers `docker pull` for anything it hosts, so both say what they are
           # and how to actually get a package out.
           INDEX_DESC = """Package manifest.
-          1) oras manifest fetch {base}:<commit>
+          1) oras manifest fetch {base}:<tag>
           2) pick an entry of manifests[] by its com.opendaq.* annotations
           3) oras pull {base}@<entry.digest>"""
 
@@ -101,112 +100,92 @@ jobs:
               ".dmg":    "application/octet-stream",
           }
 
-          def run(cmd, err, **kw):
-              print("+ " + " ".join(cmd), flush=True)
-              r = subprocess.run(cmd, text=True, capture_output=True, **kw)
-              if r.stdout: print(r.stdout, end="")
-              if r.stderr: print(r.stderr, end="")
-              if r.returncode != 0:
-                  sys.exit(f"::error::{err}")
-              return r
-
-          hosts = {staging["path"].rstrip("/").split("/")[0].lower() for staging in stagings}
-          for host in sorted(hosts):
-              run(["oras", "login", host, "-u", actor, "--password-stdin"],
-                  f"login failed for {host}", input=token)
-
-          published = []
+          def oras(*args, cwd=None, input=None):
+              """Runs oras: its output; a failure fails the job."""
+              print("+ oras " + " ".join(args), flush=True)
+              run = subprocess.run(["oras", *args], capture_output=True, text=True, cwd=cwd, input=input)
+              if run.stdout + run.stderr:
+                  print((run.stdout + run.stderr).rstrip("\n"))
+              if run.returncode != 0:
+                  sys.exit(f"::error::oras {args[0]} failed")
+              return run.stdout
+
+          def flatten(meta, prefix="com.opendaq"):
+              """staging-meta.json as annotations: nested keys joined by dots, empty values left out."""
+              out = {}
+              for key, value in meta.items():
+                  if isinstance(value, dict):
+                      out.update(flatten(value, f"{prefix}.{key}"))
+                  elif value not in (None, ""):
+                      out[f"{prefix}.{key}"] = str(value)
+              return out
+
+          sha     = os.environ["SHA"]
+          source  = f"{os.environ['SERVER_URL']}/{os.environ['REPO']}"
+          run_tag = f"run-{os.environ['RUN_ID']}-{os.environ['RUN_ATTEMPT']}"           # unique to this publish
+          created = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")   # staging-cleanup ages by this
+
+          stagings = json.loads(os.environ["STAGINGS"])
+          for host in {s["path"].split("/")[0].lower() for s in stagings}:
+              oras("login", host, "-u", os.environ["ACTOR"], "--password-stdin", input=os.environ["TOKEN"])
+
+          summary = ["### Published packages", ""]
           for staging in stagings:
-              base = staging["path"].rstrip("/").lower()   # OCI repositories are lowercase
-              tags = list(dict.fromkeys([sha, short] + staging.get("tags", [])))
-              dirs = sorted({d for g in staging["artifacts"]
-                               for d in glob.glob(f".artifacts/{g}") if os.path.isdir(d)})
+              base   = staging["path"].rstrip("/").lower()   # OCI repositories are lowercase
+              tags   = list(dict.fromkeys([run_tag] + staging.get("tags", [])))
+              layout = os.path.abspath(".oci/" + base.replace("/", "_"))
+              dirs   = sorted({d for g in staging["artifacts"] for d in glob.glob(f".artifacts/{g}") if os.path.isdir(d)})
               if not dirs:
                   sys.exit(f"::error::no artifacts matched {staging['artifacts']}")
 
-              layout_dir = os.path.abspath(".oci/" + base.replace("/", "_"))
-              digest_ann = {}
-              names      = []
-
-              # each artifact becomes an annotated manifest in a local layout
+              # each artifact -- one payload and, optionally, staging-meta.json -- becomes an
+              # annotated manifest in a local layout
+              packages = {}   # digest -> annotations
               for adir in dirs:
-                  artifact = os.path.basename(adir)
-
-                  # an artifact holds one payload plus, optionally, staging-meta.json
                   files = [f for f in sorted(os.listdir(adir))
                            if f != "staging-meta.json" and os.path.isfile(os.path.join(adir, f))]
                   if len(files) != 1:
-                      sys.exit(f"::error::expected one payload in {artifact}, found {len(files)}: {files}")
-                  fname = files[0]
-                  names.append(fname)
-
-                  meta_file = os.path.join(adir, "staging-meta.json")
-                  meta      = json.load(open(meta_file)) if os.path.exists(meta_file) else None
-
-                  def flatten(obj, prefix, out):
-                      for k, v in obj.items():
-                          key = f"{prefix}.{k}"
-                          if isinstance(v, dict):
-                              flatten(v, key, out)
-                          elif v not in (None, ""):
-                              out[key] = str(v)
-                      return out
-
-                  ann = flatten(meta, "com.opendaq", {}) if meta else {}
-
-                  ext   = ".tar.gz" if fname.endswith(".tar.gz") else os.path.splitext(fname)[1].lower()
-                  media = MEDIA.get(ext, "application/octet-stream")
-
-                  ann["org.opencontainers.image.title"]       = fname
-                  ann["org.opencontainers.image.description"] = PACKAGE_DESC.format(base=base)
-
-                  args = []
-                  for k, v in ann.items():
-                      args += ["-a", f"{k}={v}"]
-
-                  r = run(["oras", "push", "--oci-layout", f"{layout_dir}:{artifact}",
-                           "--artifact-type", PACKAGE_TYPE, f"{fname}:{media}", *args,
-                           "--format", "go-template={{.digest}}"],
-                          f"failed to build the manifest for {fname}", cwd=adir)
-                  digest_ann[r.stdout.strip()] = ann
-
-              # the index: oras leaves descriptors bare, so carry the annotations ourselves
-              run(["oras", "manifest", "index", "create", "--oci-layout", f"{layout_dir}:tmp",
-                   *digest_ann, "-o", "skeleton.json"],
-                  f"failed to assemble the index for {base}")
-
-              # Annotations only, no `platform`: it cannot express the compiler, so gcc-11 and
-              # gcc-14 would share one and a docker-style client would silently pick either.
-              index = json.load(open("skeleton.json"))
+                      sys.exit(f"::error::expected one payload in {adir}, found {files}")
+                  name = files[0]
+                  meta = os.path.join(adir, "staging-meta.json")
+                  ann  = {**(flatten(json.load(open(meta))) if os.path.exists(meta) else {}),
+                          "org.opencontainers.image.title":       name,
+                          "org.opencontainers.image.description": PACKAGE_DESC.format(base=base),
+                          "org.opencontainers.image.revision":    sha,   # a package names its build too
+                          "org.opencontainers.image.source":      source}
+                  ext  = ".tar.gz" if name.endswith(".tar.gz") else os.path.splitext(name)[1].lower()
+                  digest = oras("push", "--oci-layout", f"{layout}:{os.path.basename(adir)}",
+                                "--artifact-type", PACKAGE_TYPE, f"{name}:{MEDIA.get(ext, 'application/octet-stream')}",
+                                *[arg for key, value in ann.items() for arg in ("-a", f"{key}={value}")],
+                                "--format", "go-template={{.digest}}", cwd=adir).strip()
+                  packages[digest] = ann
+
+              # what the index was built against: its packages agree on it, or it is no one build
+              deps = [{k: v for k, v in ann.items() if k.startswith(DEPENDENCIES)} for ann in packages.values()]
+              if any(d != deps[0] for d in deps):
+                  sys.exit(f"::error::the packages of {base} differ in their {DEPENDENCIES}* annotations")
+
+              # the index: oras leaves descriptors bare, so they carry the annotations of their
+              # packages; no `platform`, which cannot tell gcc-11 from gcc-14
+              oras("manifest", "index", "create", "--oci-layout", f"{layout}:tmp", *packages, "-o", "index.json")
+              index = json.load(open("index.json"))
               for desc in index["manifests"]:
-                  desc["annotations"] = digest_ann[desc["digest"]]
+                  desc["annotations"] = packages[desc["digest"]]
               index["artifactType"] = INDEX_TYPE
-
-              index["annotations"] = {
-                  "org.opencontainers.image.created":     created,
-                  "org.opencontainers.image.description": INDEX_DESC.format(base=base),
-                  "org.opencontainers.image.revision":    sha,
-                  "org.opencontainers.image.source":      f"{server}/{repo}",
-              }
+              index["annotations"]  = {"org.opencontainers.image.created":     created,
+                                       "org.opencontainers.image.description": INDEX_DESC.format(base=base),
+                                       "org.opencontainers.image.revision":    sha,
+                                       "org.opencontainers.image.source":      source,
+                                       **deps[0]}
               json.dump(index, open("index.json", "w"), separators=(",", ":"))
-              run(["oras", "manifest", "push", "--oci-layout", f"{layout_dir}:index", "index.json",
-                   "--media-type", "application/vnd.oci.image.index.v1+json"],
-                  f"failed to store the index for {base}")
-
-              # one copy publishes index + children; the children land untagged
-              run(["oras", "cp", "--from-oci-layout", f"{layout_dir}:index",
-                   base + ":" + ",".join(tags)],
-                  f"failed to publish {base}")
-
-              published.append((base, tags, names))
-              print(f"published {base} ({len(names)} packages)")
-
-          gs = os.environ.get("GITHUB_STEP_SUMMARY")
-          if gs and published:
-              with open(gs, "a") as f:
-                  f.write("### Published packages\n\n")
-                  for base, tags, names in published:
-                      f.write(f"- `{base}`\n")
-                      f.write(f"  - tags: {', '.join('`' + t + '`' for t in tags)}\n")
-                      for name in names:
-                          f.write(f"  - `{name}`\n")
+              oras("manifest", "push", "--oci-layout", f"{layout}:index", "index.json",
+                   "--media-type", "application/vnd.oci.image.index.v1+json")
+
+              # one copy publishes the index with its packages; the packages land untagged
+              oras("cp", "--from-oci-layout", f"{layout}:index", f"{base}:{','.join(tags)}")
+
+              summary += [f"- `{base}`: {', '.join(f'`{t}`' for t in tags)}",
+                          *[f"  - `{ann['org.opencontainers.image.title']}`" for ann in packages.values()]]
+
+          with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
+              f.write("\n".join(summary) + "\n")