vgteam · adamnovak · Feb 4, 2021 · Oct 5, 2020 · Jun 25, 2020 · Jun 25, 2020
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -81,16 +81,49 @@ build-job:
     - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 >  ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
     # Connect to the Kubernetes-based builder "buildkit"
     # See vgci/buildkit-deployment.yml
-    - docker buildx create --use --name=buildkit --platform=linux/amd64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64"
+    - docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64"
+    # Prune down build cache to make space
+    - docker buildx prune --keep-storage 80G
     # Connect so we can upload our images
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
     - make include/vg_git_version.hpp
-    - docker buildx build --push --build-arg THREADS=8 -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile .
+    - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile .
+    # Also run the tests
+    - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target test -f Dockerfile .
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
-
-# We define a second follow-on phase to run the tests
+
+# The arm container build takes like 90 minutes, so we don't want to run it
+# before the main test phase where the other long tests live.
+# They also don't pass yet.
+arm-build-job:
+  stage: test
+  only:
+    - /^arm/
+  before_script:
+    # Don't bother starting the Docker daemon or installing much
+    - which docker || (sudo apt-get -q -y update && sudo apt-get -q -y install --no-upgrade docker.io)
+    # Get buildx
+    - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 >  ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
+    # Connect to the Kubernetes-based builder "buildkit"
+    # See vgci/buildkit-deployment.yml
+    - docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64"
+    # Prune down build cache to make space
+    - docker buildx prune --keep-storage 80G
+    # Connect so we can upload our images
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+  script:
+    - make include/vg_git_version.hpp
+    # Build the arm container (emulated!)
+    - docker buildx build --platform=linux/arm64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile .
+    # Also run the tests (emulated!)
+    # But don't fail if they fail yet, because they don't yet actually work.
+    - docker buildx build --platform=linux/arm64 --build-arg THREADS=8 --target test -f Dockerfile . || true
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+
+# We also run the toil-vg/pytest-based tests
 # Note that WE ONLY RUN TESTS LISTED IN vgci/test-list.txt
 test-job:
   stage: test

diff --git a/.gitmodules b/.gitmodules
@@ -54,7 +54,7 @@
 	url = https://github.com/benedictpaten/sonLib.git
 [submodule "deps/fermi-lite"]
 	path = deps/fermi-lite
-    url = https://github.com/edawson/fermi-lite.git
+    url = https://github.com/vgteam/fermi-lite.git
 [submodule "deps/gbwt"]
 	path = deps/gbwt
 	url = https://github.com/jltsiren/gbwt.git
@@ -116,6 +116,9 @@
 [submodule "vgteam_bbhash"]
 	path = deps/BBHash
 	url = https://github.com/vgteam/BBHash.git
+[submodule "src/simde"]
+	path = src/simde
+	url = https://github.com/nemequ/simde-no-tests
 [submodule "doc/wiki"]
 	path = doc/wiki
 	url = https://github.com/vgteam/vg.wiki.git

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Multi-container Dockerfile for build and run containers for vg
-FROM ubuntu:18.04 AS base
+FROM ubuntu:20.04 AS base
 MAINTAINER vgteam
 
 RUN echo base > /stage.txt
@@ -12,6 +12,7 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 
 FROM base AS build
 ARG THREADS=8
+ARG TARGETARCH
 
 RUN echo build > /stage.txt
 
@@ -36,28 +37,35 @@ RUN apt-get -qq -y update && apt-get -qq -y upgrade && apt-get -qq -y install \
     libcairo2-dev libpixman-1-dev libffi-dev libcairo-dev libprotobuf-dev libboost-all-dev
 ###DEPS_END###
 
-# Pre-build non-package dependencies
+# Prepare to build submodule dependencies
 COPY source_me.sh /vg/source_me.sh
 COPY deps /vg/deps
-# To increase portability of the docker image, set the target CPU architecture to
-# Nehalem (2008) rather than auto-detecting the build machine's CPU.
-# This has no AVX1, AVX2, or PCLMUL, but it does have SSE4.2.
-# UCSC has a Nehalem machine that we want to support.
-RUN sed -i s/march=native/march=nehalem/ deps/sdsl-lite/CMakeLists.txt
+# To increase portability of the docker image, when building for amd64, set the
+# target CPU architecture to Nehalem (2008) rather than auto-detecting the
+# build machine's CPU. This has no AVX1, AVX2, or PCLMUL, but it does have
+# SSE4.2. UCSC has a Nehalem machine that we want to support.
+RUN if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then sed -i s/march=native/march=nehalem/ deps/sdsl-lite/CMakeLists.txt; fi
+# Clear any CMake caches in case we are building from someone's checkout
+RUN find . -name CMakeCache.txt | xargs rm -f
+# Build the dependencies
 COPY Makefile /vg/Makefile
-RUN . ./source_me.sh && CXXFLAGS=" -march=nehalem " make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) deps
+RUN . ./source_me.sh && CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) deps
 
-# Bring in the rest of the build tree that we need
+# Bring in the sources, which we need in order to build
 COPY src /vg/src
-COPY test /vg/test
-COPY doc /vg/doc
-COPY scripts /vg/scripts
+
+# Build all the object files for vg, but don't link.
+# Also pass the arch here
+RUN . ./source_me.sh && CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) objs
+
 # Bring in any includes we pre-made, like the git version
 COPY include /vg/include
 
-# Do the build. Trim down the resulting binary but make sure to include enough debug info for profiling.
-# Also pass the arch here
-RUN . ./source_me.sh && CXXFLAGS=" -march=nehalem " make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) static && strip -d bin/vg
+# Do the final build and link, knowing the version. Trim down the resulting binary but make sure to include enough debug info for profiling.
+RUN . ./source_me.sh && CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) static && strip -d bin/vg
+
+# Ship the scripts
+COPY scripts /vg/scripts
 
 ENV PATH /vg/bin:$PATH
 
@@ -69,8 +77,16 @@ RUN echo test > /stage.txt
 
 # Fail if any non-portable instructions were used
 RUN /bin/bash -e -c 'if objdump -d /vg/bin/vg | grep vperm2i128 ; then exit 1 ; else exit 0 ; fi'
+
+# Bring in the tests and docs, which have doctests
+COPY test /vg/test
+COPY doc /vg/doc
+# We test the README so bring it along.
+COPY README.md /vg/
+
 # Run tests in the middle so the final container that gets tagged is the run container.
-RUN export OMP_NUM_THREADS=$((THREADS < $(nproc) ? THREADS : $(nproc))) make test
+# Tests may not actually be run by smart builders like buildkit.
+RUN /bin/bash -e -c "export OMP_NUM_THREADS=$((THREADS < $(nproc) ? THREADS : $(nproc))); make test"
 
 
 ############################################################################################
@@ -98,7 +114,6 @@ RUN ls -lah /vg && \
     fontconfig-config \
     awscli \
     binutils \
-    libssl1.0.0 \
     libpython2.7 \
     libperl-dev \
     libelf1 \
@@ -116,7 +131,7 @@ COPY --from=build /vg/bin/vg /vg/bin/
 
 COPY --from=build /vg/scripts/* /vg/scripts/
 # Make sure we have the flame graph scripts so we can do self-profiling
-COPY deps/FlameGraph /vg/deps/FlameGraph
+COPY --from=build /vg/deps/FlameGraph /vg/deps/FlameGraph
 
 ENV PATH /vg/bin:$PATH
 

diff --git a/Makefile b/Makefile
@@ -17,7 +17,8 @@ CWD:=$(shell pwd)
 CXX ?= g++
 PKG_CONFIG ?= pkg-config
 
-EXE:=vg
+SFX :=
+EXE:=vg$(SFX)
 
 all: $(BIN_DIR)/$(EXE)
 
@@ -181,9 +182,11 @@ else
     # We get OpenMP the normal way, using whatever the compiler knows about
     CXXFLAGS += -fopenmp
 
-    # We care about building for SSE4.2 only and not AVX, to have vaguely portable binaries
-    CXXFLAGS += -msse4.2
-
+    ifeq ($(shell arch), x86_64)
+        # We care about building for SSE4.2 only and not AVX, to have vaguely portable binaries
+        CXXFLAGS += -msse4.2
+    endif
+
     # Note shared libraries are so files
     SHARED_SUFFIX = so
     # Define options to start static linking of libraries on GNU ld.
@@ -355,10 +358,21 @@ ifneq ($(shell uname -s),Darwin)
 	LD_LIB_FLAGS += -ljemalloc
 endif
 
-.PHONY: clean get-deps deps test set-path static static-docker docs man .pre-build .check-environment .check-git .no-git 
+.PHONY: clean get-deps deps test set-path objs static static-docker docs man .pre-build .check-environment .check-git .no-git 
+
+# Aggregate all libvg deps, and exe deps other than libvg
+LIBVG_DEPS = $(OBJ) $(ALGORITHMS_OBJ) $(IO_OBJ) $(DEP_OBJ) $(DEPS)
+EXE_DEPS = $(OBJ_DIR)/main.o $(UNITTEST_OBJ) $(SUBCOMMAND_OBJ) $(CONFIGURATION_OBJ) $(DEPS) $(LINK_DEPS)
+
+# We have a target we can build to do everything but link the library and executable
+objs: $(LIBVG_DEPS) $(EXE_DEPS)
+
+$(LIB_DIR)/libvg.a: $(LIBVG_DEPS)
+	rm -f $@
+	ar rs $@ $(OBJ) $(ALGORITHMS_OBJ) $(IO_OBJ) $(DEP_OBJ)
 
 # For a normal dynamic build we remove the static build marker
-$(BIN_DIR)/$(EXE): $(OBJ_DIR)/main.o $(LIB_DIR)/libvg.a $(UNITTEST_OBJ) $(SUBCOMMAND_OBJ) $(CONFIGURATION_OBJ) $(DEPS) $(LINK_DEPS)
+$(BIN_DIR)/$(EXE): $(LIB_DIR)/libvg.a $(EXE_DEPS)
 	-rm -f $(LIB_DIR)/vg_is_static
 	. ./source_me.sh && $(CXX) $(LDFLAGS) $(INCLUDE_FLAGS) $(CPPFLAGS) $(CXXFLAGS) -o $(BIN_DIR)/$(EXE) $(OBJ_DIR)/main.o $(UNITTEST_OBJ) $(SUBCOMMAND_OBJ) $(CONFIGURATION_OBJ) -lvg $(LD_LIB_DIR_FLAGS) $(LD_LIB_FLAGS) $(START_STATIC) $(LD_STATIC_LIB_FLAGS) $(END_STATIC) $(LD_STATIC_LIB_DEPS) 
 # We keep a file that we touch on the last static build.
@@ -379,10 +393,6 @@ static-docker: static scripts/*
 	strip -d $(BIN_DIR)/$(EXE)
 	DOCKER_BUILDKIT=1 docker build . -f Dockerfile.static -t vg
 
-$(LIB_DIR)/libvg.a: $(OBJ) $(ALGORITHMS_OBJ) $(IO_OBJ) $(DEP_OBJ) $(DEPS)
-	rm -f $@
-	ar rs $@ $(OBJ) $(ALGORITHMS_OBJ) $(IO_OBJ) $(DEP_OBJ)
-
 # We have system-level deps to install
 # We want the One True Place for them to be in the Dockerfile.
 get-deps:

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 <!-- !test program bash -eo pipefail -->
 # vg
 
-[![Join the chat at https://gitter.im/vgteam/vg](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/vgteam/vg?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Latest Release](https://img.shields.io/github/release/vgteam/vg.svg)](https://github.com/vgteam/vg/releases/latest) [![Build Status](https://travis-ci.org/vgteam/vg.svg?branch=master)](https://travis-ci.org/vgteam/vg) [![Performance Report](https://img.shields.io/badge/performance-report-brightgreen.svg)](https://vg-data.s3.amazonaws.com/vg_ci/vgci_reports/branch/master/index.html) 
+[![Join the chat at https://gitter.im/vgteam/vg](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/vgteam/vg?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Latest Release](https://img.shields.io/github/release/vgteam/vg.svg)](https://github.com/vgteam/vg/releases/latest) [![Performance Report](https://img.shields.io/badge/performance-report-brightgreen.svg)](https://vg-data.s3.amazonaws.com/vg_ci/vgci_reports/branch/master/index.html) 
 [![Doxygen API Documentation](https://img.shields.io/badge/doxygen-docs-brightgreen.svg)](https://vgteam.github.io/vg/)
 
 ## variation graph data structures, interchange formats, alignment, genotyping, and variant calling methods
@@ -70,7 +70,7 @@ At present, you will need GCC version 4.9 or greater, with support for C++14, to
 
 Other libraries may be required. Please report any build difficulties.
 
-Note that a 64-bit OS is required. Ubuntu 18.04 should work. You will also need a CPU that supports SSE 4.2 to run VG; you can check this with `cat /proc/cpuinfo | grep sse4_2`.
+Note that a 64-bit OS is required. Ubuntu 18.04 should work.
 
 When you are ready, build with `. ./source_me.sh && make`, and run with `./bin/vg`.
 

diff --git a/deps/fermi-lite b/deps/fermi-lite
diff --git a/deps/gssw b/deps/gssw
diff --git a/deps/ssw b/deps/ssw
diff --git a/src/crash.cpp b/src/crash.cpp
@@ -168,7 +168,7 @@ void emit_stacktrace(int signalNumber, siginfo_t *signalInfo, void *signalContex
         *out << "Caught signal " << signalNumber << " raised at address " << ip << endl;
         // Do our own tracing because backtrace doesn't really work on all platforms.
         stacktrace_manually(*out, signalNumber, ip, bp);
-    #else
+    #elif __x86_64__
         // Linux 64 bit does it this way
         ip = (void*)context->uc_mcontext.gregs[REG_RIP];
         bp = (void**)context->uc_mcontext.gregs[REG_RBP];

diff --git a/src/preflight.cpp b/src/preflight.cpp
@@ -2,14 +2,18 @@
 
 #include <iostream>
 #include <cstdlib>
+
+#ifdef __x86_64__
 #include <cpuid.h>
+#endif
 
 namespace vg {
 
 using namespace std;
 
 void preflight_check() {
 
+#ifdef __x86_64__
     // We assume we are on x86_64 on POSIX (and not Windows).
     // We use the method of dlib's dlib/simd/simd_check.h
 
@@ -27,6 +31,8 @@ void preflight_check() {
             << "Please use a system with SSE4.2 support." << endl;
         exit(1);
     }
+#endif
+    // If not on x86_64, we are probably on ARM and using fake SSE anyway.
 
 }
 

diff --git a/src/preflight.hpp b/src/preflight.hpp
@@ -11,10 +11,10 @@
 
 namespace vg {
 
-/// Define a macro to tell things to be built for every architecture, if possible.
+/// Define a macro to tell things to be built for every X86_64 architecture, if possible.
 /// This *doesn't* work on Mac with GNU GCC and Apple libc++, because functions
 /// for x86-64 can't use std::endl, so we exclude that combination.
-#if (!defined(__GNUC__) || !defined(_LIBCPP_VERSION) || !defined(__APPLE__))
+#if defined(__x86_64__) && (!defined(__GNUC__) || !defined(_LIBCPP_VERSION) || !defined(__APPLE__))
     #define VG_PREFLIGHT_EVERYWHERE __attribute__((__target__("arch=x86-64")))
 #else
     #define VG_PREFLIGHT_EVERYWHERE

diff --git a/src/simde b/src/simde
diff --git a/src/ssw_aligner.cpp b/src/ssw_aligner.cpp
@@ -23,7 +23,13 @@ Alignment SSWAligner::align(const string& query, const string& ref) {
                                           gap_extension);
     StripedSmithWaterman::Filter filter;
     StripedSmithWaterman::Alignment alignment;
-    aligner.Align(query.c_str(), ref.c_str(), ref.size(), filter, &alignment);
+
+    // We need to send out own mask length, recommended to be half the sequence length and at least 15.
+    int32_t mask_len = min(max((size_t) 15, query.size()), (size_t) numeric_limits<int32_t>::max());
+
+    assert(ref.size() <= numeric_limits<int>::max());
+
+    aligner.Align(query.c_str(), ref.c_str(), (int) ref.size(), filter, &alignment, mask_len);
     return ssw_to_vg(alignment, query, ref);
 }
+0 −53		src/example.cpp
+0 −401		src/ssw_cpp.cpp
+0 −213		src/ssw_cpp.h
+3 −0		.gitmodules
+97 −70		README.md
+400 −0		demo/new.txt
+400 −0		demo/old.txt
+0 −0		demo/output.sam
+3 −0		demo/pRead.fa
+3 −0		demo/pRef.fa
+2 −0		demo/query.fa
+8 −0		demo/query.fastq
+2 −0		demo/query1.fa
+2 −0		demo/query2.fa
+2 −0		demo/r1.fa
+4 −0		demo/r1_query.fq
+2 −0		demo/target.fa
+8 −0		demo/target.fastq
+2 −0		demo/target1.fa
+2 −0		demo/target2.fa
+4 −4		src/Makefile
+5 −2		src/example.cpp
+19 −42		src/main.c
+1 −1		src/pyssw.py
+1 −0		src/simde
+132 −27		src/ssw.c
+37 −35		src/ssw.h
+10 −10		src/ssw_cpp.cpp
+12 −2		src/ssw_cpp.h
+ −		src/ssw_lib.pyc