[DRAFT] All changes for pyiceberg-core wheel license attribution

.github/workflows/bindings_python_ci.yml

@@ -24,7 +24,6 @@ on:
     paths:
       - '**'                                       # Include all files and directories in the repository by default.
       - '!.github/ISSUE_TEMPLATE/**'               # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
-      - '!dev/release/**'
       - '!website/**'
       - '!.asf.yml'
       - '!.gitattributes'
@@ -67,15 +66,49 @@ jobs:
         run: |
           uvx ruff check .
 
+  wheel-license-files:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Setup Rust toolchain
+        uses: $/.github/actions/setup-builder
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: 3.12
+      - name: Install cargo-about
+        uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
+        with:
+          # Keep the version in sync with EXPECTED_CARGO_ABOUT_VERSION in
+          # dev/release/licenses.py.
+          tool: cargo-about@0.8.4
+      - name: Generate per-wheel LICENSE, NOTICE and THIRD-PARTY-LICENSES
+        shell: bash
+        run: python3 ./dev/release/licenses.py wheel generate bindings/python
+      # License files for all wheels are uploaded as a single artifact, specific per-wheel files taken later.
+      - name: Upload generated license files
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: wheel-license-files
+          path: bindings/python/licenses
+          if-no-files-found: error
+
   test:
     runs-on: ${{ matrix.os }}
+    needs: [wheel-license-files]
     strategy:
       max-parallel: 15
       matrix:
-        os:
-          - ubuntu-latest
-          - macos-latest
-          - windows-latest
+        include:
+          - os: macos-latest
+            # Uses runner's target rather than a cross-architect target as in release workflows.
+            # No point building x86 wheel if we do not test it.
+            maturin_target: aarch64-apple-darwin
+          - os: windows-latest
+            maturin_target: x86_64-pc-windows-msvc
+          - os: ubuntu-latest
+            maturin_target: x86_64-unknown-linux-gnu
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -93,11 +126,29 @@ jobs:
         with:
           key: bindings-python
           save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      - name: Download generated license files
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: wheel-license-files
+          path: bindings/python/licenses
+      # Must run before maturin, maturin will bundle these in to the wheels.
+      - name: Stage license files for wheel
+        shell: bash
+        run: |
+          python3 ./dev/release/licenses.py wheel stage bindings/python \
+            --maturin-target "${{ matrix.maturin_target }}"
       - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
         with:
+          target: ${{ matrix.maturin_target }}
           working-directory: "bindings/python"
           command: build
           args: --out dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one
+      # Additional verifications - missing files aren't flagged by Maturin.
+      - name: Verify wheel bundles third-party licenses
+        shell: bash
+        run: |
+          python3 ./dev/release/licenses.py wheel verify bindings/python/dist \
+            --maturin-target "${{ matrix.maturin_target }}"
       - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: "0.9.3"

.github/workflows/ci.yml

@@ -61,6 +61,10 @@ jobs:
         with:
           tool: taplo@0.9.3,cargo-machete@0.7.0,cargo-deny@0.19.9
 
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: 3.12
+
       - name: Check License Header
         uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0
 
@@ -80,7 +84,7 @@ jobs:
         run: make check-license-notice
 
       - name: Check dependency licenses
-        run: dev/release/dependencies.sh check
+        run: python3 ./dev/release/licenses.py dependencies check
 
       # Trusted publishing cannot create a crate (https://crates.io/docs/trusted-publishing),
       # so a new publishable crate must be reserved on crates.io before it is merged.

.github/workflows/release_python.yml

@@ -1,3 +1,3 @@
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -136,22 +136,55 @@
           name: wheels-sdist
           path: bindings/python/dist
 
+  wheel-license-files:
+    runs-on: ubuntu-latest
+    needs: [validate-release-tag]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Setup Rust toolchain
+        uses: $/.github/actions/setup-builder
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: 3.12
+      - name: Install cargo-about
+        uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
+        with:
+          # Keep the version in sync with EXPECTED_CARGO_ABOUT_VERSION in
+          # dev/release/licenses.py.
+          tool: cargo-about@0.8.4
+      - name: Generate per-wheel LICENSE, NOTICE and THIRD-PARTY-LICENSES
+        shell: bash
+        run: python3 ./dev/release/licenses.py wheel generate bindings/python
+      # License files for all wheels are uploaded as a single artifact, specific per-wheel files taken later.
+      - name: Upload generated license files
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: wheel-license-files
+          path: bindings/python/licenses
+          if-no-files-found: error
+
   wheels:
+    needs: [validate-release-tag, wheel-license-files]
     runs-on: "${{ matrix.os }}"
-    needs: [validate-release-tag]
     strategy:
       max-parallel: 15
       matrix:
         include:
-          - { os: windows-latest }
-          - { os: macos-latest, target: "universal2-apple-darwin" }
-          - { os: ubuntu-latest, target: "x86_64" }
-          - {
-              os: ubuntu-latest,
-              target: "aarch64",
-              manylinux: "manylinux_2_28",
-            }
-          - { os: ubuntu-latest, target: "armv7l" }
+          - os: macos-latest
+            maturin_target: universal2-apple-darwin
+          - os: windows-latest
+            maturin_target: x86_64-pc-windows-msvc
+          - os: ubuntu-latest
+            maturin_target: x86_64-unknown-linux-gnu
+          - os: ubuntu-latest
+            maturin_target: armv7-unknown-linux-gnueabihf
+            cross: true
+          - os: ubuntu-latest
+            maturin_target: aarch64-unknown-linux-gnu
+            cross: true
+            manylinux: manylinux_2_28
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -183,30 +216,46 @@
         uses: $/.github/actions/setup-builder
         with:
           rust-version: ${{ steps.get-msrv.outputs.msrv }}
+      - name: Download generated license files
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: wheel-license-files
+          path: bindings/python/licenses
+      # Must execute ahead of Maturin, Maturin will bundle them in the wheel.
+      - name: Stage the license files for this wheel
+        shell: bash
+        run: |
+          python3 ./dev/release/licenses.py wheel stage bindings/python \
+            --maturin-target "${{ matrix.maturin_target }}"
       - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
         with:
-          target: ${{ matrix.target }}
+          target: ${{ matrix.maturin_target }}
           manylinux: ${{ matrix.manylinux || 'auto' }}
           working-directory: "bindings/python"
           command: build
           args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one
+      - name: Verify wheel bundles third-party licenses
+        shell: bash
+        run: |
+          python3 ./dev/release/licenses.py wheel verify bindings/python/dist \
+            --maturin-target "${{ matrix.maturin_target }}"
       - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: "0.9.3"
           enable-cache: true
       # Verify the wheel is abi3-compatible and installable without building from source.
-      # Skipped for cross-compiled targets since they share the same maturin config;
-      # if abi3 is broken, the native target build will catch it.
+      # Cross-compiled wheels cannot be executed by the runner that built them.
+      # If abi3 is broken, the native target build will catch it.
       - name: Verify wheel installs with no build fallback
-        if: matrix.target != 'aarch64' && matrix.target != 'armv7l'
+        if: ${{ !matrix.cross }}
         working-directory: "bindings/python"
         shell: bash
         run: |
           uv pip install --no-build --reinstall --system --no-index --find-links dist/ pyiceberg-core
       - name: Upload wheels
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: wheels-${{ matrix.os }}-${{ matrix.target }}
+          name: wheels-${{ matrix.os }}-${{ matrix.maturin_target }}
           path: bindings/python/dist
 
   pypi-publish:

.github/workflows/release_python_nightly.yml

@@ -1,3 +1,3 @@
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -69,22 +69,56 @@
           name: wheels-sdist
           path: bindings/python/dist
 
-  wheels:
+  wheel-license-files:
     needs: set-version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Setup Rust toolchain
+        uses: $/.github/actions/setup-builder
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: 3.12
+      #
+      - name: Install cargo-about
+        uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
+        with:
+          # Keep the version in sync with EXPECTED_CARGO_ABOUT_VERSION in
+          # dev/release/licenses.py.
+          tool: cargo-about@0.8.4
+      - name: Generate per-wheel LICENSE, NOTICE and THIRD-PARTY-LICENSES
+        shell: bash
+        run: python3 ./dev/release/licenses.py wheel generate bindings/python
+      # License files for all wheels are uploaded as a single artifact, specific per-wheel files taken later.
+      - name: Upload generated license files
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: wheel-license-files
+          path: bindings/python/licenses
+          if-no-files-found: error
+
+  wheels:
+    needs: [set-version, wheel-license-files]
     runs-on: "${{ matrix.os }}"
     strategy:
       max-parallel: 15
       matrix:
         include:
-          - { os: windows-latest }
-          - { os: macos-latest, target: "universal2-apple-darwin" }
-          - { os: ubuntu-latest, target: "x86_64" }
-          - {
-              os: ubuntu-latest,
-              target: "aarch64",
-              manylinux: "manylinux_2_28",
-            }
-          - { os: ubuntu-latest, target: "armv7l" }
+          - os: macos-latest
+            maturin_target: universal2-apple-darwin
+          - os: windows-latest
+            maturin_target: x86_64-pc-windows-msvc
+          - os: ubuntu-latest
+            maturin_target: x86_64-unknown-linux-gnu
+          - os: ubuntu-latest
+            maturin_target: armv7-unknown-linux-gnueabihf
+            cross: true
+          - os: ubuntu-latest
+            maturin_target: aarch64-unknown-linux-gnu
+            cross: true
+            manylinux: manylinux_2_28
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -116,23 +150,42 @@
         with:
           rust-version: ${{ steps.get-msrv.outputs.msrv }}
 
+      - name: Download generated license files
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: wheel-license-files
+          path: bindings/python/licenses
+
+      # Must execute ahead of Maturin, Maturin will bundle them in the wheel.
+      - name: Stage the license files for this wheel
+        shell: bash
+        run: |
+          python3 ./dev/release/licenses.py wheel stage bindings/python \
+            --maturin-target "${{ matrix.maturin_target }}"
+
       - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
         with:
-          target: ${{ matrix.target }}
+          target: ${{ matrix.maturin_target }}
           manylinux: ${{ matrix.manylinux || 'auto' }}
           working-directory: "bindings/python"
           command: build
           args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one
 
+      - name: Verify wheel bundles third-party licenses
+        shell: bash
+        run: |
+          python3 ./dev/release/licenses.py wheel verify bindings/python/dist \
+            --maturin-target "${{ matrix.maturin_target }}"
+
       - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: "0.9.3"
           enable-cache: true
       # Verify the wheel is abi3-compatible and installable without building from source.
-      # Skipped for cross-compiled targets since they share the same maturin config;
-      # if abi3 is broken, the native target build will catch it.
+      # Cross-compiled wheels cannot be executed by the runner that built them.
+      # If abi3 is broken, the native target build will catch it.
       - name: Verify wheel installs with no build fallback
-        if: matrix.target != 'aarch64' && matrix.target != 'armv7l'
+        if: ${{ !matrix.cross }}
         working-directory: "bindings/python"
         shell: bash
         run: |
@@ -141,7 +194,7 @@
       - name: Upload wheels
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: wheels-${{ matrix.os }}-${{ matrix.target }}
+          name: wheels-${{ matrix.os }}-${{ matrix.maturin_target }}
           path: bindings/python/dist
 
   testpypi-publish:

.gitignore

@@ -26,3 +26,7 @@ dist/*
 *.pyc
 *.whl
 *.tar.gz
+
+# Generated at build time by dev/release/licenses.py, never checked in.
+/bindings/python/licenses/
+/bindings/python/THIRD-PARTY-LICENSES

.licenserc.yaml

@@ -27,6 +27,10 @@ header:
     - "**/*.json"
     # Generated content by mdbook
     - "website/book"
+    # Generated content by cargo-about; contains third-party license texts
+    - "**/THIRD-PARTY-LICENSES"
+    # Per-wheel license files
+    - "bindings/python/licenses/"
     # Generated content by cargo-public-api
     - "**/public-api.txt"
     # Generated content by trybuild

Makefile

@@ -94,7 +94,7 @@ install-cargo-deny:
 	cargo install --locked cargo-deny@0.19.9
 
 check-dependency-licenses: install-cargo-deny
-	bash ./dev/release/dependencies.sh check
+	python3 ./dev/release/licenses.py dependencies check
 
 check: check-fmt check-clippy check-toml cargo-machete check-license-notice check-dependency-licenses