Add 64-bit ID build to the Ubuntu matrix, publish/upload it, document it

Adds an avx2_int64 leg to build_ubuntu (configures with -DMMSEQS_INT64_IDS=1,
drops -DENABLE_WERROR until the 64-bit build is vetted warning-clean), publishes
it as mmseqs-linux-AVX2-64bit via a per-leg ARTIFACT name (no collision with the
32-bit AVX2 build), and wires upload_artifacts to sign + POST
mmseqs-linux-avx2-64bit.tar.gz to mmseqs.com. README lists the new download.

Author: martin-steinegger

README.md

@@ -37,6 +37,8 @@ When compiling from source, pass `-DMMSEQS_INT64_IDS=1` to CMake to build MMseqs
     wget https://mmseqs.com/latest/mmseqs-linux-gpu.tar.gz; tar xvfz mmseqs-linux-gpu.tar.gz; export PATH=$(pwd)/mmseqs/bin/:$PATH
     # static build with AVX2 (fastest)
     wget https://mmseqs.com/latest/mmseqs-linux-avx2.tar.gz; tar xvfz mmseqs-linux-avx2.tar.gz; export PATH=$(pwd)/mmseqs/bin/:$PATH
+    # static build with AVX2 (fastest) with > 2^32 sequences support (>4B)
+    wget https://mmseqs.com/latest/mmseqs-linux-avx2-64bit.tar.gz; tar xvfz mmseqs-linux-avx2-64bit.tar.gz; export PATH=$(pwd)/mmseqs/bin/:$PATH
     # static build with SSE4.1
     wget https://mmseqs.com/latest/mmseqs-linux-sse41.tar.gz; tar xvfz mmseqs-linux-sse41.tar.gz; export PATH=$(pwd)/mmseqs/bin/:$PATH
     # static build with SSE2 (slowest, for very old systems)

azure-pipelines.yml

@@ -74,16 +74,26 @@ jobs:
           STATIC: 1
           MPI: 0
           BUILD_TYPE: Release
+          ARTIFACT: 'mmseqs-linux-AVX2'
         sse41:
           SIMD: 'SSE4_1'
           STATIC: 1
           MPI: 0
           BUILD_TYPE: Release
+          ARTIFACT: 'mmseqs-linux-SSE4_1'
         sse2:
           SIMD: 'SSE2'
           STATIC: 1
           MPI: 0
           BUILD_TYPE: Release
+          ARTIFACT: 'mmseqs-linux-SSE2'
+        avx2_int64:
+          SIMD: 'AVX2'
+          STATIC: 1
+          MPI: 0
+          BUILD_TYPE: Release
+          INT64: 1
+          ARTIFACT: 'mmseqs-linux-AVX2-64bit'
         avx2_mpi:
           SIMD: 'AVX2'
           STATIC: 0
@@ -112,14 +122,20 @@ jobs:
       - script: |
           mkdir build && cd build
           export CC=gcc-11 ; export CXX=g++-11
+          EXTRA=""
+          WERROR="-DENABLE_WERROR=1"
+          if [ "${INT64}" = "1" ]; then
+            EXTRA="-DMMSEQS_INT64_IDS=1"
+            WERROR=""   # 64-bit-ID build not yet vetted warning-clean under -Werror
+          fi
           if [ "${STATIC}" -eq "1" ]; then
             LIBGOMP=/usr/lib/gcc/x86_64-linux-gnu/11
             cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DFORCE_STATIC_DEPS=1 -DHAVE_TESTS=1 \
               -DOpenMP_C_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_C_LIB_NAMES=gomp -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_CXX_LIB_NAMES=gomp -DOpenMP_gomp_LIBRARY=${LIBGOMP}/libgomp.a \
-              -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
+              ${WERROR} ${EXTRA} -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
           else
             cmake -DHAVE_SANITIZER=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DHAVE_TESTS=1 \
-              -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
+              ${WERROR} ${EXTRA} -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
           fi
 
           make -j $(nproc --all) VERBOSE=1
@@ -133,42 +149,7 @@ jobs:
         condition: eq(variables['STATIC'], 1)
         inputs:
           targetPath: $(Build.SourcesDirectory)/build/src/mmseqs
-          artifactName: mmseqs-linux-$(SIMD)
-
-  - job: build_ubuntu_int64
-    displayName: Ubuntu MMseqs2 64-bit IDs
-    pool:
-      vmImage: 'Ubuntu-22.04'
-    timeoutInMinutes: 120
-    steps:
-      - checkout: self
-        submodules: true
-      - script: |
-          sudo apt-get update
-          sudo apt-get install -y gcc-11 g++-11 libgcc-11-dev rustc cargo build-essential
-        retryCountOnTaskFailure: "2"
-        displayName: Install newer G++
-      - script: |
-          mkdir build && cd build
-          export CC=gcc-11 ; export CXX=g++-11
-          LIBGOMP=/usr/lib/gcc/x86_64-linux-gnu/11
-          # NOTE: -DENABLE_WERROR is intentionally omitted; the 64-bit-ID build has
-          # not yet been vetted warning-clean under -Werror. Add it once it is.
-          cmake -DCMAKE_BUILD_TYPE=Release -DFORCE_STATIC_DEPS=1 -DHAVE_TESTS=1 \
-            -DMMSEQS_INT64_IDS=1 \
-            -DOpenMP_C_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_C_LIB_NAMES=gomp -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_CXX_LIB_NAMES=gomp -DOpenMP_gomp_LIBRARY=${LIBGOMP}/libgomp.a \
-            -DHAVE_AVX2=1 -DHAVE_MPI=0 ..
-          make -j $(nproc --all) VERBOSE=1
-        displayName: Build MMseqs2 (64-bit IDs)
-      - script: |
-          export TTY=0
-          ${BUILD_SOURCESDIRECTORY}/util/regression/run_regression.sh ${BUILD_SOURCESDIRECTORY}/build/src/mmseqs ${BUILD_SOURCESDIRECTORY}/regression
-        displayName: Run Regression Suite
-        condition: eq(variables['regression'], 1)
-      - task: PublishPipelineArtifact@0
-        inputs:
-          targetPath: $(Build.SourcesDirectory)/build/src/mmseqs
-          artifactName: mmseqs-linux-avx2-64bit
+          artifactName: $(ARTIFACT)
 
   - job: build_ubuntu_gpu
     displayName: Ubuntu MMseqs2 GPU
@@ -461,6 +442,18 @@ jobs:
           archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-arm64.tar.gz
           includeRootFolder: true
           archiveType: tar
+      - task: DownloadPipelineArtifact@1
+        inputs:
+          artifactName: mmseqs-linux-AVX2-64bit
+          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
+      - script:
+          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
+      - task: ArchiveFiles@2
+        inputs:
+          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
+          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-avx2-64bit.tar.gz
+          includeRootFolder: true
+          archiveType: tar
       - script:
           rm "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
       # - task: DownloadPipelineArtifact@1
@@ -496,6 +489,7 @@ jobs:
             mmseqs-linux-sse2.tar.gz \
             mmseqs-linux-sse41.tar.gz \
             mmseqs-linux-avx2.tar.gz \
+            mmseqs-linux-avx2-64bit.tar.gz \
             mmseqs-linux-gpu.tar.gz \
             mmseqs-linux-gpu-cuda13.tar.gz \
             mmseqs-linux-gpu-arm64.tar.gz \
@@ -507,6 +501,7 @@ jobs:
             -F file[]=@mmseqs-linux-sse2.tar.gz -F signature[]=@mmseqs-linux-sse2.tar.gz.sig \
             -F file[]=@mmseqs-linux-sse41.tar.gz -F signature[]=@mmseqs-linux-sse41.tar.gz.sig \
             -F file[]=@mmseqs-linux-avx2.tar.gz -F signature[]=@mmseqs-linux-avx2.tar.gz.sig \
+            -F file[]=@mmseqs-linux-avx2-64bit.tar.gz -F signature[]=@mmseqs-linux-avx2-64bit.tar.gz.sig \
             -F file[]=@mmseqs-linux-gpu.tar.gz -F signature[]=@mmseqs-linux-gpu.tar.gz.sig \
             -F file[]=@mmseqs-linux-gpu-cuda13.tar.gz -F signature[]=@mmseqs-linux-gpu-cuda13.tar.gz.sig \
             -F file[]=@mmseqs-linux-gpu-arm64.tar.gz -F signature[]=@mmseqs-linux-gpu-arm64.tar.gz.sig \