Rebuild Comet and MaRaCluster with zlib 1.3.2 and expat 2.8.5; remove X!Tandem

.gitattributes

@@ -0,0 +1,6 @@
+# The build recipes run in bash on Windows too, so they keep LF line ends there.
+# The patches go to git apply byte for byte: some of the files they change use CRLF.
+recipes/**/*.sh text eol=lf
+recipes/**/*.py text eol=lf
+recipes/versions.env text eol=lf
+recipes/**/*.patch -text

.github/workflows/rebuild-engines.yml

@@ -0,0 +1,303 @@
+name: Rebuild engines
+
+# Rebuilds the engines whose upstream binaries bundle an outdated zlib or expat
+# from the sources pinned in recipes/versions.env (see recipes/README.md), with
+# the build procedure of their upstream release builds, checks the zlib and expat
+# versions they carry, and compares their results with those of the binaries
+# they replace. The binaries are uploaded in this repository's folder layout.
+
+on:
+  push:
+    branches: [master]  # other branches build in their pull requests
+    paths:
+      - 'recipes/**'
+      - '.github/workflows/rebuild-engines.yml'
+  pull_request:
+    paths:
+      - 'recipes/**'
+      - '.github/workflows/rebuild-engines.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  comet:
+    name: Comet ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # the runners and compilers of Comet's own release builds
+          - platform: Linux/x86_64
+            runner: ubuntu-24.04
+            container: ubuntu:22.04
+          - platform: Linux/aarch64
+            runner: ubuntu-24.04-arm
+            container: ubuntu:22.04
+          - platform: MacOS/arm64
+            runner: macos-14
+            macos_min: '14.0'
+          - platform: MacOS/x86_64
+            runner: macos-15-intel
+            macos_min: '13.0'
+          - platform: Windows/x86_64
+            runner: windows-2022
+    runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}
+    timeout-minutes: 60
+    steps:
+      - name: Install the build tools
+        if: runner.os == 'Linux'
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends ca-certificates curl git g++ make xz-utils python3 binutils
+
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up the work folder
+        run: |
+          {
+            if [ "$RUNNER_OS" = Windows ]; then
+              echo "WORK=D:/w"
+              echo "PYTHON=python"
+            else
+              echo "WORK=$RUNNER_TEMP/w"
+            fi
+            grep -E '^[A-Z0-9_]+=' recipes/versions.env
+          } >> "$GITHUB_ENV"
+          # for the git commands of the later steps (in a container, the checkout has another owner)
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Fetch the sources
+        run: recipes/fetch-sources.sh "$WORK" zlib expat comet openms-testdata
+
+      - name: Swap MSToolkit's zlib and expat
+        run: recipes/comet/prepare.sh "$WORK/comet" "$WORK"
+
+      - name: Build (make)
+        if: runner.os != 'Windows'
+        working-directory: ${{ env.WORK }}/comet
+        env:
+          MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_min }}
+        # GITHUB_SHA: Comet shows the first seven characters of the commit it was built from
+        run: |
+          make GITHUB_SHA="$COMET_COMMIT"
+          echo "BINARY=$PWD/comet.exe" >> "$GITHUB_ENV"
+          echo "LIBRARIES=$PWD/MSToolkit/libmstoolkitlite.a" >> "$GITHUB_ENV"
+
+      # MSToolkit's RAW reader #imports the type library of Thermo's MSFileReader,
+      # whose DLLs need the Visual C++ 2010 runtime: the windows-2019 image of
+      # Comet's release builds had it, windows-2022 has not.
+      - name: Install the Visual C++ 2010 runtime (Windows)
+        if: runner.os == 'Windows'
+        run: choco install vcredist2010 --no-progress -y
+
+      - name: Unpack Thermo's MSFileReader (Windows)
+        if: runner.os == 'Windows'
+        working-directory: ${{ env.WORK }}/comet
+        run: unzip -q -P iacceptthelicense extern/msfreader.zip -d dlls
+
+      - name: Register Thermo's MSFileReader (Windows)
+        if: runner.os == 'Windows'
+        working-directory: ${{ env.WORK }}/comet
+        shell: cmd
+        run: regsvr32 /S dlls\XRawfile2_x64.dll
+
+      - uses: microsoft/setup-msbuild@v2
+        if: runner.os == 'Windows'
+
+      - name: Build (MSBuild)
+        if: runner.os == 'Windows'
+        working-directory: ${{ env.WORK }}/comet
+        shell: cmd  # Git Bash does not find msbuild
+        run: msbuild Comet.sln -t:Comet -p:Configuration=Release -p:Platform=x64 -p:PlatformToolset=v142 -tv:Current
+
+      - name: Locate the binary (Windows)
+        if: runner.os == 'Windows'
+        working-directory: ${{ env.WORK }}/comet
+        run: |
+          echo "BINARY=$PWD/x64/Release/comet.exe" >> "$GITHUB_ENV"
+          echo "LIBRARIES=$(find "$PWD" -name MSToolkitLite.lib | tr '\n' ' ')" >> "$GITHUB_ENV"
+
+      - name: Check the zlib and expat versions
+        run: |
+          # shellcheck disable=SC2086 # LIBRARIES lists several files
+          recipes/check-bundled-libs.sh "$BINARY" $LIBRARIES
+
+      - name: Compare with the binary it replaces
+        run: |
+          "$BINARY" -p > /dev/null && rm comet.params.new
+          git fetch -q --depth 1 origin "$BASELINE_COMMIT"
+          mkdir -p "$WORK/baseline"
+          git cat-file blob "FETCH_HEAD:${{ matrix.platform }}/Comet/comet.exe" > "$WORK/baseline/comet.exe"
+          chmod +x "$WORK/baseline/comet.exe"
+          recipes/compare/comet.sh "$WORK/baseline/comet.exe" "$BINARY" "$WORK/compare" "$WORK/openms"
+
+      - name: Stage the binary
+        run: |
+          mkdir -p "$WORK/out/${{ matrix.platform }}/Comet"
+          cp "$BINARY" "$WORK/out/${{ matrix.platform }}/Comet/comet.exe"
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: comet-${{ runner.os }}-${{ runner.arch }}
+          path: ${{ env.WORK }}/out/
+          if-no-files-found: error
+
+  maracluster:
+    name: MaRaCluster ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # the runners of MaRaCluster's rel-1-04-1 release build
+          - platform: Linux/x86_64
+            runner: ubuntu-24.04
+            container: ubuntu:22.04
+          - platform: MacOS/arm64
+            runner: macos-14
+          - platform: Windows/x86_64
+            runner: windows-2022
+    runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}
+    timeout-minutes: 300
+    steps:
+      - name: Install the build tools
+        if: runner.os == 'Linux'
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends ca-certificates curl git python3 binutils sudo wget rsync g++ make cmake lbzip2
+
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up the work folder
+        run: |
+          {
+            if [ "$RUNNER_OS" = Windows ]; then
+              echo "WORK=D:/w"  # short paths for ProteoWizard's build
+              echo 'WORK_WIN=D:\w'
+              echo "PYTHON=python"
+              echo "PWIZ_SOURCE=pwiz-with-vendor-apis"
+            else
+              echo "WORK=$RUNNER_TEMP/w"
+              echo "PWIZ_SOURCE=pwiz"
+            fi
+            grep -E '^[A-Z0-9_]+=' recipes/versions.env
+          } >> "$GITHUB_ENV"
+          # for the git commands of the later steps (in a container, the checkout has another owner)
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Fetch the sources
+        run: recipes/fetch-sources.sh "$WORK" zlib maracluster "$PWIZ_SOURCE" openms-testdata
+
+      - name: Put the pinned ProteoWizard and zlib in place
+        run: recipes/maracluster/prepare.sh "$WORK/maracluster" "$WORK/pwiz" "$WORK" "$WORK/build"
+
+      # MaRaCluster's own builders, as its release build ran them (without the GUI).
+      # JAVA_HOME has to be empty, or ProteoWizard adds its Java bindings, which
+      # break its "libraries" target (MaRaCluster's CI clears it on macOS too).
+      - name: Build (admin/builders/tarball64_build.sh)
+        if: runner.os == 'Linux'
+        env:
+          JAVA_HOME: ''
+        run: |
+          "$WORK/maracluster/admin/builders/tarball64_build.sh" -s "$WORK" -r "$WORK/release" -b "$WORK/build" -g
+          echo "BINARY=$WORK/build/maracluster/src/maracluster" >> "$GITHUB_ENV"
+          echo "LIBRARIES=$WORK/build/tools/lib/libzlib.a" >> "$GITHUB_ENV"
+
+      - name: Build (admin/builders/osx64_build.sh)
+        if: runner.os == 'macOS'
+        env:
+          JAVA_HOME: ''
+        run: |
+          "$WORK/maracluster/admin/builders/osx64_build.sh" -s "$WORK" -r "$WORK/release" -b "$WORK/build" -g
+          echo "BINARY=$WORK/build/maracluster/src/maracluster" >> "$GITHUB_ENV"
+          echo "LIBRARIES=$WORK/build/tools/lib/libzlib.a" >> "$GITHUB_ENV"
+
+      - name: Build (admin/builders/nativew64_build.bat)
+        if: runner.os == 'Windows'
+        shell: cmd
+        run: |
+          CALL %WORK_WIN%\maracluster\admin\builders\nativew64_build.bat -s %WORK_WIN% -r %WORK_WIN%\release -b %WORK_WIN%\build -g
+          IF NOT EXIST %WORK_WIN%\build\maracluster\src\Release\maracluster.exe EXIT /B 1
+          ECHO BINARY=%WORK%/build/maracluster/src/Release/maracluster.exe>> %GITHUB_ENV%
+          ECHO LIBRARIES=%WORK%/build/tools/proteowizard/lib/zlib.lib>> %GITHUB_ENV%
+
+      - name: Show ProteoWizard's build log
+        if: failure()
+        run: |
+          for log in "$WORK"/build/tools/pwiz_installation.log "$WORK"/build/tools/proteowizard/pwiz_installation.log; do
+            if [ -f "$log" ]; then echo "== $log"; tail -n 200 "$log"; fi
+          done
+
+      - name: Check the zlib and expat versions
+        run: |
+          # shellcheck disable=SC2086 # LIBRARIES lists several files
+          recipes/check-bundled-libs.sh "$BINARY" $LIBRARIES
+
+      - name: Compare with the binary it replaces
+        env:
+          REPEAT: 25
+        run: |
+          name=maracluster
+          [ "$RUNNER_OS" = Windows ] && name=maracluster.exe
+          git fetch -q --depth 1 origin "$BASELINE_COMMIT"
+          mkdir -p "$WORK/baseline"
+          git cat-file blob "FETCH_HEAD:${{ matrix.platform }}/MaRaCluster/$name" > "$WORK/baseline/$name"
+          chmod +x "$WORK/baseline/$name"
+          recipes/compare/maracluster.sh "$WORK/baseline/$name" "$BINARY" "$WORK/compare" "$WORK/openms"
+
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: maracluster-diagnostics-${{ runner.os }}-${{ runner.arch }}
+          path: |
+            ${{ env.WORK }}/compare/out/**/*.log
+            ${{ env.BINARY }}
+          if-no-files-found: ignore
+
+      - name: Stage the binary
+        run: |
+          mkdir -p "$WORK/out/${{ matrix.platform }}/MaRaCluster"
+          cp "$BINARY" "$WORK/out/${{ matrix.platform }}/MaRaCluster/"
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: maracluster-${{ runner.os }}-${{ runner.arch }}
+          path: ${{ env.WORK }}/out/
+          if-no-files-found: error
+
+  engines:
+    name: All engines
+    needs: [comet, maracluster]
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - uses: actions/download-artifact@v8
+        with:
+          path: engines
+          merge-multiple: true
+
+      - name: Check the zlib and expat versions
+        run: |
+          find engines -type f | sort
+          find engines -type f -print0 | sort -z | xargs -0 recipes/check-bundled-libs.sh
+          (cd engines && find . -type f -print0 | sort -z | xargs -0 sha256sum)
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: engines
+          path: engines/
+          if-no-files-found: error

Licenses.txt

@@ -66,12 +66,6 @@ SIRIUS:
   - License:
   https://www.gnu.org/licenses/agpl-3.0
 
-X!Tandem:
-  - URL and source code:
-  http://www.thegpm.org/tandem/
-  - License:
-  Artistic License (Perl), https://spdx.org/licenses/Artistic-1.0-Perl.html
-
 Sage:
   - URL and source code:
   https://github.com/lazear/sage

Linux/aarch64/Comet/LICENSE-expat.txt

@@ -0,0 +1,21 @@
+Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper
+Copyright (c) 2001-2025 Expat maintainers
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Linux/aarch64/Comet/README.md

@@ -1,23 +1,11 @@
-<img src="https://uwpr.github.io/Comet/images/cometlogo_1_small.png" align="right">
-
-# Comet MS/MS
-
-Comet is an open source tandem mass spectrometry (MS/MS) sequence database search tool written primarily in C/C++. The original Comet repository lived on [SourceForge](https://sourceforge.net/projects/comet-ms/) since 2012. It was migrated to GitHub on September 2021.
-
-The project website [can be found here](https://uwpr.github.io/Comet/). This includes release notes and search parameters documentation.
-
-To compile on linux and macOS:
-
-- Type 'make'.  This will generate a binary "comet.exe".
-
-To compile with Microsoft Visual Studio:
-
-- Use build tools v142 with Microsoft Visual Studio 2019.
-
-- First install [MSFileReader from Thermo Fischer Scientific](https://uwpr.github.io/Comet/notes/20220228_rawfile.html).
-
-- Load "Comet.sln" in Visual Studio.
-
-- Set the build to "Release" and "x64".
-
-- Right-mouse-click on the "Comet" project and choose "Build". This should generate a binary "Comet.exe" in x64/Release.
+# Comet 2025.01 rev. 1
+
+`comet.exe` is Comet 2025.01 rev. 1 (UWPR/Comet v2025.01.1, commit 4181df6),
+built by OpenMS from Comet's source with zlib 1.3.2 and expat 2.8.5 in place of
+the zlib 1.2.11 and expat 2.2.9 that Comet's MSToolkit bundles, which have known
+vulnerabilities. Otherwise it is built as Comet's own release binaries are; the
+recipe is in https://github.com/OpenMS/THIRDPARTY/tree/master/recipes.
+
+- Comet: https://github.com/UWPR/Comet, Apache License 2.0 (LICENSE)
+- expat: https://libexpat.github.io, MIT License (LICENSE-expat.txt)
+- zlib: https://zlib.net, zlib License