diff --git a/.github/workflows/reusable.yml b/.github/workflows/reusable.yml index 67145ab..8b132e7 100644 --- a/.github/workflows/reusable.yml +++ b/.github/workflows/reusable.yml @@ -240,10 +240,12 @@ jobs: # the runner image's standalone LLVM, not the clang bundled with Visual Studio _windows_llvm = "C:/Program Files/LLVM/bin/" - # the generator drives the arm64 cross toolchain, so no msvc-dev-cmd environment + # the generator drives the arm64 cross toolchain, so no msvc-dev-cmd environment; + # the image has Visual Studio 2026 only, which carries the v143 toolset as well _windows_11_armv8_msvs = { **_windows_2022_x86_64_msvs, "runner": "windows-11-arm", + "generator": "Visual Studio 18 2026", "target-arch": "armv8", "setup-msvc-toolchain": False, } diff --git a/.github/workflows/staging-cleanup.yml b/.github/workflows/staging-cleanup.yml index 9ae8645..fce5245 100644 --- a/.github/workflows/staging-cleanup.yml +++ b/.github/workflows/staging-cleanup.yml @@ -103,7 +103,7 @@ jobs: continue cutoff = now - window - # every index is tagged (sha + short) and tags/list is OCI's only enumeration; two + # every index is tagged (by its run) and tags/list is OCI's only enumeration; two # tags share one index digest, so resolve each tag and collapse to unique digests digests = {} for tag in oras(["repo", "tags", path], f"listing tags for {path} failed").split(): diff --git a/.github/workflows/staging-push.yml b/.github/workflows/staging-push.yml index fc79db5..671dfd1 100644 --- a/.github/workflows/staging-push.yml +++ b/.github/workflows/staging-push.yml @@ -7,8 +7,8 @@ name: Staging Push # children: unreachable by name until the index lands, so a set publishes atomically, and the # registry refuses an index with a dangling reference. # -# ghcr.io/opendaq/nightly/staging/opendaq : , , latest -# └─ sha256:index… +# ghcr.io/opendaq/nightly/staging/opendaq : run--, latest +# └─ sha256:index… revision= com.opendaq.dependencies..revision= # ├─ sha256:a1… title=opendaq-x86_64-linux-gcc-11-release.tar.gz # │ com.opendaq: platform=linux settings.arch=x86_64 settings.compiler=gcc settings.compiler.version=11 # └─ sha256:b2… title=opendaq-x86_64-windows-msvc-143-release.tar.gz @@ -17,6 +17,11 @@ name: Staging Push # A child is named by its file; the com.opendaq.* annotations are optional, and buy a consumer a # resolve by settings rather than by file name. # +# The run tag is unique to its index and says nothing more: one commit built against two cores +# is two indexes, and a tag naming the commit could point at one of them only. What a package was +# built from is in its annotations: the commit itself, and the commits of its dependencies from +# its metadata, which the index repeats. +# # NEVER prune "untagged versions" of these packages -- the live children are untagged too. # Pruning goes through the index only. on: @@ -29,7 +34,7 @@ on: { "path": "ghcr.io//path/to/staging", // the package, in full "artifacts": ["staging-*"], // globs; each match becomes a child of its index - "tags": ["latest", "v1.2.3"] // extra tags; and are automatic + "tags": ["latest", "v1.2.3"] // extra tags; run-- is automatic }, { // one entry per kind: a tarball and a .deb are "path": "ghcr.io//path/to/package", // variants of nothing, so they never share @@ -61,28 +66,22 @@ jobs: TOKEN: ${{ github.token }} ACTOR: ${{ github.actor }} SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} REPO: ${{ github.repository }} SERVER_URL: ${{ github.server_url }} run: | import json, os, glob, subprocess, sys from datetime import datetime, timezone - stagings = json.loads(os.environ["STAGINGS"]) - token = os.environ["TOKEN"] - actor = os.environ["ACTOR"] - sha = os.environ["SHA"] - short = sha[:7] - repo = os.environ["REPO"] - server = os.environ["SERVER_URL"] - created = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") # push time; staging-cleanup ages by this - PACKAGE_TYPE = "application/vnd.opendaq.package.v1" INDEX_TYPE = "application/vnd.opendaq.index.v1" # what is inside: see the descriptors + DEPENDENCIES = "com.opendaq.dependencies." # The registry UI offers `docker pull` for anything it hosts, so both say what they are # and how to actually get a package out. INDEX_DESC = """Package manifest. - 1) oras manifest fetch {base}: + 1) oras manifest fetch {base}: 2) pick an entry of manifests[] by its com.opendaq.* annotations 3) oras pull {base}@""" @@ -101,112 +100,92 @@ jobs: ".dmg": "application/octet-stream", } - def run(cmd, err, **kw): - print("+ " + " ".join(cmd), flush=True) - r = subprocess.run(cmd, text=True, capture_output=True, **kw) - if r.stdout: print(r.stdout, end="") - if r.stderr: print(r.stderr, end="") - if r.returncode != 0: - sys.exit(f"::error::{err}") - return r - - hosts = {staging["path"].rstrip("/").split("/")[0].lower() for staging in stagings} - for host in sorted(hosts): - run(["oras", "login", host, "-u", actor, "--password-stdin"], - f"login failed for {host}", input=token) - - published = [] + def oras(*args, cwd=None, input=None): + """Runs oras: its output; a failure fails the job.""" + print("+ oras " + " ".join(args), flush=True) + run = subprocess.run(["oras", *args], capture_output=True, text=True, cwd=cwd, input=input) + if run.stdout + run.stderr: + print((run.stdout + run.stderr).rstrip("\n")) + if run.returncode != 0: + sys.exit(f"::error::oras {args[0]} failed") + return run.stdout + + def flatten(meta, prefix="com.opendaq"): + """staging-meta.json as annotations: nested keys joined by dots, empty values left out.""" + out = {} + for key, value in meta.items(): + if isinstance(value, dict): + out.update(flatten(value, f"{prefix}.{key}")) + elif value not in (None, ""): + out[f"{prefix}.{key}"] = str(value) + return out + + sha = os.environ["SHA"] + source = f"{os.environ['SERVER_URL']}/{os.environ['REPO']}" + run_tag = f"run-{os.environ['RUN_ID']}-{os.environ['RUN_ATTEMPT']}" # unique to this publish + created = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") # staging-cleanup ages by this + + stagings = json.loads(os.environ["STAGINGS"]) + for host in {s["path"].split("/")[0].lower() for s in stagings}: + oras("login", host, "-u", os.environ["ACTOR"], "--password-stdin", input=os.environ["TOKEN"]) + + summary = ["### Published packages", ""] for staging in stagings: - base = staging["path"].rstrip("/").lower() # OCI repositories are lowercase - tags = list(dict.fromkeys([sha, short] + staging.get("tags", []))) - dirs = sorted({d for g in staging["artifacts"] - for d in glob.glob(f".artifacts/{g}") if os.path.isdir(d)}) + base = staging["path"].rstrip("/").lower() # OCI repositories are lowercase + tags = list(dict.fromkeys([run_tag] + staging.get("tags", []))) + layout = os.path.abspath(".oci/" + base.replace("/", "_")) + dirs = sorted({d for g in staging["artifacts"] for d in glob.glob(f".artifacts/{g}") if os.path.isdir(d)}) if not dirs: sys.exit(f"::error::no artifacts matched {staging['artifacts']}") - layout_dir = os.path.abspath(".oci/" + base.replace("/", "_")) - digest_ann = {} - names = [] - - # each artifact becomes an annotated manifest in a local layout + # each artifact -- one payload and, optionally, staging-meta.json -- becomes an + # annotated manifest in a local layout + packages = {} # digest -> annotations for adir in dirs: - artifact = os.path.basename(adir) - - # an artifact holds one payload plus, optionally, staging-meta.json files = [f for f in sorted(os.listdir(adir)) if f != "staging-meta.json" and os.path.isfile(os.path.join(adir, f))] if len(files) != 1: - sys.exit(f"::error::expected one payload in {artifact}, found {len(files)}: {files}") - fname = files[0] - names.append(fname) - - meta_file = os.path.join(adir, "staging-meta.json") - meta = json.load(open(meta_file)) if os.path.exists(meta_file) else None - - def flatten(obj, prefix, out): - for k, v in obj.items(): - key = f"{prefix}.{k}" - if isinstance(v, dict): - flatten(v, key, out) - elif v not in (None, ""): - out[key] = str(v) - return out - - ann = flatten(meta, "com.opendaq", {}) if meta else {} - - ext = ".tar.gz" if fname.endswith(".tar.gz") else os.path.splitext(fname)[1].lower() - media = MEDIA.get(ext, "application/octet-stream") - - ann["org.opencontainers.image.title"] = fname - ann["org.opencontainers.image.description"] = PACKAGE_DESC.format(base=base) - - args = [] - for k, v in ann.items(): - args += ["-a", f"{k}={v}"] - - r = run(["oras", "push", "--oci-layout", f"{layout_dir}:{artifact}", - "--artifact-type", PACKAGE_TYPE, f"{fname}:{media}", *args, - "--format", "go-template={{.digest}}"], - f"failed to build the manifest for {fname}", cwd=adir) - digest_ann[r.stdout.strip()] = ann - - # the index: oras leaves descriptors bare, so carry the annotations ourselves - run(["oras", "manifest", "index", "create", "--oci-layout", f"{layout_dir}:tmp", - *digest_ann, "-o", "skeleton.json"], - f"failed to assemble the index for {base}") - - # Annotations only, no `platform`: it cannot express the compiler, so gcc-11 and - # gcc-14 would share one and a docker-style client would silently pick either. - index = json.load(open("skeleton.json")) + sys.exit(f"::error::expected one payload in {adir}, found {files}") + name = files[0] + meta = os.path.join(adir, "staging-meta.json") + ann = {**(flatten(json.load(open(meta))) if os.path.exists(meta) else {}), + "org.opencontainers.image.title": name, + "org.opencontainers.image.description": PACKAGE_DESC.format(base=base), + "org.opencontainers.image.revision": sha, # a package names its build too + "org.opencontainers.image.source": source} + ext = ".tar.gz" if name.endswith(".tar.gz") else os.path.splitext(name)[1].lower() + digest = oras("push", "--oci-layout", f"{layout}:{os.path.basename(adir)}", + "--artifact-type", PACKAGE_TYPE, f"{name}:{MEDIA.get(ext, 'application/octet-stream')}", + *[arg for key, value in ann.items() for arg in ("-a", f"{key}={value}")], + "--format", "go-template={{.digest}}", cwd=adir).strip() + packages[digest] = ann + + # what the index was built against: its packages agree on it, or it is no one build + deps = [{k: v for k, v in ann.items() if k.startswith(DEPENDENCIES)} for ann in packages.values()] + if any(d != deps[0] for d in deps): + sys.exit(f"::error::the packages of {base} differ in their {DEPENDENCIES}* annotations") + + # the index: oras leaves descriptors bare, so they carry the annotations of their + # packages; no `platform`, which cannot tell gcc-11 from gcc-14 + oras("manifest", "index", "create", "--oci-layout", f"{layout}:tmp", *packages, "-o", "index.json") + index = json.load(open("index.json")) for desc in index["manifests"]: - desc["annotations"] = digest_ann[desc["digest"]] + desc["annotations"] = packages[desc["digest"]] index["artifactType"] = INDEX_TYPE - - index["annotations"] = { - "org.opencontainers.image.created": created, - "org.opencontainers.image.description": INDEX_DESC.format(base=base), - "org.opencontainers.image.revision": sha, - "org.opencontainers.image.source": f"{server}/{repo}", - } + index["annotations"] = {"org.opencontainers.image.created": created, + "org.opencontainers.image.description": INDEX_DESC.format(base=base), + "org.opencontainers.image.revision": sha, + "org.opencontainers.image.source": source, + **deps[0]} json.dump(index, open("index.json", "w"), separators=(",", ":")) - run(["oras", "manifest", "push", "--oci-layout", f"{layout_dir}:index", "index.json", - "--media-type", "application/vnd.oci.image.index.v1+json"], - f"failed to store the index for {base}") - - # one copy publishes index + children; the children land untagged - run(["oras", "cp", "--from-oci-layout", f"{layout_dir}:index", - base + ":" + ",".join(tags)], - f"failed to publish {base}") - - published.append((base, tags, names)) - print(f"published {base} ({len(names)} packages)") - - gs = os.environ.get("GITHUB_STEP_SUMMARY") - if gs and published: - with open(gs, "a") as f: - f.write("### Published packages\n\n") - for base, tags, names in published: - f.write(f"- `{base}`\n") - f.write(f" - tags: {', '.join('`' + t + '`' for t in tags)}\n") - for name in names: - f.write(f" - `{name}`\n") + oras("manifest", "push", "--oci-layout", f"{layout}:index", "index.json", + "--media-type", "application/vnd.oci.image.index.v1+json") + + # one copy publishes the index with its packages; the packages land untagged + oras("cp", "--from-oci-layout", f"{layout}:index", f"{base}:{','.join(tags)}") + + summary += [f"- `{base}`: {', '.join(f'`{t}`' for t in tags)}", + *[f" - `{ann['org.opencontainers.image.title']}`" for ann in packages.values()]] + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write("\n".join(summary) + "\n") diff --git a/.github/workflows/test-staging.yml b/.github/workflows/test-staging.yml index 03df187..d23e0bd 100644 --- a/.github/workflows/test-staging.yml +++ b/.github/workflows/test-staging.yml @@ -51,7 +51,7 @@ jobs: "match-jobs": ["*"], "stagings": [ { "name": "core", "variables": { "CPACK_OPENDAQ_META_PACKAGE_NAME": "hello-core", "CPACK_ARCHIVE_COMPONENT_INSTALL": "ON", "CPACK_COMPONENTS_ALL": "core" } }, - { "name": "module", "variables": { "CPACK_OPENDAQ_META_PACKAGE_NAME": "hello-module", "CPACK_ARCHIVE_COMPONENT_INSTALL": "ON", "CPACK_COMPONENTS_ALL": "module" } } + { "name": "module", "variables": { "CPACK_OPENDAQ_META_PACKAGE_NAME": "hello-module", "CPACK_ARCHIVE_COMPONENT_INSTALL": "ON", "CPACK_COMPONENTS_ALL": "module", "CPACK_OPENDAQ_META_FIELDS": "dependencies/hello-core/revision=${{ github.sha }};dependencies/test/revision=test-revision" } } ] }, { "match-jobs": ["ubuntu-*", "manylinux*"], "installers": [ @@ -87,13 +87,13 @@ jobs: with: staging-pull: >- [ - { "artifact": "dep-core-ubuntu-22.04-x86_64-ninja-gcc-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "${{ github.sha }}", "annotations": { "com.opendaq.custom.platform": "linux", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "dep-core-ubuntu-22.04-armv8-ninja-gcc-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "${{ github.sha }}", "annotations": { "com.opendaq.custom.platform": "linux", "com.opendaq.settings.arch": "armv8" } }] }, - { "artifact": "dep-core-manylinux_2_28-x86_64-ninja-gcc-14-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "${{ github.sha }}", "annotations": { "com.opendaq.custom.platform": "manylinux_2_28", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "dep-core-windows-2022-x86_64-msvs-v143-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "${{ github.sha }}", "annotations": { "com.opendaq.custom.platform": "windows", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "dep-core-windows-2022-x86-msvs-v143-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "${{ github.sha }}", "annotations": { "com.opendaq.custom.platform": "windows", "com.opendaq.settings.arch": "x86" } }] }, - { "artifact": "dep-core-macos-26-x86_64-ninja-appleclang-21-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "${{ github.sha }}", "annotations": { "com.opendaq.custom.platform": "macos", "com.opendaq.settings.arch": "x86_64" } }] }, - { "artifact": "dep-core-macos-26-armv8-ninja-appleclang-21-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "${{ github.sha }}", "annotations": { "com.opendaq.custom.platform": "macos", "com.opendaq.settings.arch": "armv8" } }] } + { "artifact": "dep-core-ubuntu-22.04-x86_64-ninja-gcc-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "run-${{ github.run_id }}-${{ github.run_attempt }}", "annotations": { "com.opendaq.custom.platform": "linux", "com.opendaq.settings.arch": "x86_64" } }] }, + { "artifact": "dep-core-ubuntu-22.04-armv8-ninja-gcc-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "run-${{ github.run_id }}-${{ github.run_attempt }}", "annotations": { "com.opendaq.custom.platform": "linux", "com.opendaq.settings.arch": "armv8" } }] }, + { "artifact": "dep-core-manylinux_2_28-x86_64-ninja-gcc-14-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "run-${{ github.run_id }}-${{ github.run_attempt }}", "annotations": { "com.opendaq.custom.platform": "manylinux_2_28", "com.opendaq.settings.arch": "x86_64" } }] }, + { "artifact": "dep-core-windows-2022-x86_64-msvs-v143-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "run-${{ github.run_id }}-${{ github.run_attempt }}", "annotations": { "com.opendaq.custom.platform": "windows", "com.opendaq.settings.arch": "x86_64" } }] }, + { "artifact": "dep-core-windows-2022-x86-msvs-v143-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "run-${{ github.run_id }}-${{ github.run_attempt }}", "annotations": { "com.opendaq.custom.platform": "windows", "com.opendaq.settings.arch": "x86" } }] }, + { "artifact": "dep-core-macos-26-x86_64-ninja-appleclang-21-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "run-${{ github.run_id }}-${{ github.run_attempt }}", "annotations": { "com.opendaq.custom.platform": "macos", "com.opendaq.settings.arch": "x86_64" } }] }, + { "artifact": "dep-core-macos-26-armv8-ninja-appleclang-21-release", "packages": [{ "path": "ghcr.io/${{ github.repository_owner }}/test/staging/hello-core", "tag": "run-${{ github.run_id }}-${{ github.run_attempt }}", "annotations": { "com.opendaq.custom.platform": "macos", "com.opendaq.settings.arch": "armv8" } }] } ] include-jobs: >- [ @@ -192,14 +192,14 @@ jobs: shell: python3 {0} env: OWNER: ${{ github.repository_owner }} - SHA: ${{ github.sha }} + RUN_TAG: run-${{ github.run_id }}-${{ github.run_attempt }} TOKEN: ${{ github.token }} ACTOR: ${{ github.actor }} run: | import os, subprocess, sys owner = os.environ["OWNER"].lower() - sha = os.environ["SHA"] + tag = os.environ["RUN_TAG"] subprocess.run(["oras", "login", "ghcr.io", "-u", os.environ["ACTOR"], "--password-stdin"], input=os.environ["TOKEN"], text=True, check=True) @@ -207,7 +207,7 @@ jobs: gone = [] for kind in ("staging", "installer"): for name in ("hello-core", "hello-module"): - ref = f"ghcr.io/{owner}/test/{kind}/{name}:{sha}" + ref = f"ghcr.io/{owner}/test/{kind}/{name}:{tag}" if subprocess.run(["oras", "manifest", "fetch", "--descriptor", ref], capture_output=True).returncode != 0: gone.append(ref) @@ -219,3 +219,45 @@ jobs: + ", ".join(gone)) print("ok: everything this run published is still there") + + # An index and each of its packages name the commit they were built from, and those of the + # module the core they were built against -- the packages from their metadata, the index + # repeating it. + - name: Assert the indexes say what they were built from + shell: python3 {0} + env: + OWNER: ${{ github.repository_owner }} + RUN_TAG: run-${{ github.run_id }}-${{ github.run_attempt }} + SHA: ${{ github.sha }} + run: | + import json, os, subprocess, sys + + owner = os.environ["OWNER"].lower() + tag = os.environ["RUN_TAG"] + sha = os.environ["SHA"] + + REVISION = "org.opencontainers.image.revision" + DEPENDENCIES = "com.opendaq.dependencies." + + expected = { + "hello-core": {REVISION: sha}, + "hello-module": {REVISION: sha, DEPENDENCIES + "hello-core.revision": sha, DEPENDENCIES + "test.revision": "test-revision"}, + } + + def named(annotations): + return {k: v for k, v in annotations.items() if k == REVISION or k.startswith(DEPENDENCIES)} + + wrong = [] + for name, want in expected.items(): + ref = f"ghcr.io/{owner}/test/staging/{name}:{tag}" + index = json.loads(subprocess.run(["oras", "manifest", "fetch", ref], check=True, + capture_output=True, text=True).stdout) + print(f"{ref}: {json.dumps(named(index.get('annotations', {})), sort_keys=True)}") + if named(index.get("annotations", {})) != want: + wrong.append(f"{name}: the index is not {json.dumps(want, sort_keys=True)}") + for package in index["manifests"]: + if named(package.get("annotations", {})) != want: + wrong.append(f"{name}: package {package['digest']} is not {json.dumps(want, sort_keys=True)}") + + if wrong: + sys.exit("::error::annotations are off -- " + "; ".join(wrong)) diff --git a/cmake/StagingBootstrap.cmake b/cmake/StagingBootstrap.cmake index 5995fba..d1b8729 100644 --- a/cmake/StagingBootstrap.cmake +++ b/cmake/StagingBootstrap.cmake @@ -9,7 +9,7 @@ if (NOT COMMAND opendaq_write_metadata) FetchContent_Declare( opendaq-cmake-utils GIT_REPOSITORY https://github.com/openDAQ/opendaq-cmake-utils.git - GIT_TAG v1.0.3 + GIT_TAG v1.1.3 SOURCE_SUBDIR _download_only_ ) FetchContent_MakeAvailable(opendaq-cmake-utils)