Cross-platform: referenceless CRAM, macOS readFilesCommand fix, big-endian support

README.md

@@ -24,8 +24,9 @@ https://github.com/birdingman0626/STAR/issues
 
 HARDWARE/SOFTWARE REQUIREMENTS
 ==============================
-  * x86-64 compatible processors
+  * x86-64 compatible processors, or ARM64/AArch64 (incl. Apple Silicon)
   * 64 bit Linux, Mac OS X, or Windows
+  * little-endian or big-endian hosts (big-endian, e.g. s390x/ppc64, is compile-supported but not part of CI)
 
 MANUAL
 ======
@@ -331,6 +332,11 @@ FORK CHANGES
   * Pure C++ implementation using cpp-httplib + nlohmann/json; no Node.js required
   * Child-process execution model keeps the server stable across run failures
 
+### Output Formats & Portability (new)
+  * **Referenceless CRAM output** (`--outSAMtype CRAM Unsorted|SortedByCoordinate`): STAR produces its normal BAM via the proven output path, then transcodes it to CRAM at finalization using bundled HTSlib (`source/cramOutput.cpp`). Uses `CRAM_OPT_NO_REF`, so **no external reference FASTA is required**. CRAM is typically ~10–25% smaller than BAM on full-quality data (the gain comes from CRAM's rANS/quality/read-name codecs, not reference compression). On conversion failure the original BAM is kept and the run continues. Applies to the main `Aligned.*` outputs; the transcriptome BAM (`--quantMode TranscriptomeSAM`) stays BAM for RSEM/Salmon compatibility. Selectable in the Web UI.
+  * **ARM64 / Apple Silicon**: native arm64 builds; AVX2 auto-disabled on ARM (CMake) and `-march=armv8-a+simd` selected in the Makefile. macOS native build target added: `make STARforMac CXX=clang++` (links libomp dynamically).
+  * **Big-endian support** (`source/byteOrder.h`): the genome, suffix array and packed arrays are accessed as a little-endian byte stream regardless of host byte order, fixing the "next index is smaller than previous" failure on big-endian hosts (s390x, ppc64). Guarded so little-endian builds keep the native single-instruction load (zero performance/behavior change); only known big-endian compiles take the portable byte-wise path. Ported from the patch in upstream issue #2690. *Compile-validated only — no big-endian runner in CI.*
+
 ### Performance Optimizations
   * MSVC compiler: `/O2 /Ob2 /Oi /GL` with `/LTCG` link-time optimization (Windows)
   * SRW locks replacing CRITICAL_SECTION (faster mutex, Windows)
@@ -350,6 +356,7 @@ FORK CHANGES
     - Better exon-pair selection for chimeric junctions (overlapping cross-reference exon, not just first/last)
     - Trim stitched transcripts to the junction-relevant side before rescoring
     - Fix cross-mate `roStart` computation (`a2.Lread` instead of `a1.Lread` on negative strand)
+  * macOS: spawn `readFilesCommand` via `posix_spawnp` instead of `vfork()`+`execlp()`+`exit()`, fixing "Failed spawning readFilesCommand" with gzipped input on macOS (upstream issue #2663). Avoids the undefined behavior of calling `exit()` in a `vfork` child. POSIX-only path; the Windows `system()`-based path is unchanged.
 
 ### Project Quality
   * C++17 standard (upgraded from C++11)

source/CMakeLists.txt

@@ -383,6 +383,7 @@ set(STAR_SOURCES
     serviceFuns.cpp
     GlobalVariables.cpp
     BAMoutput.cpp
+    cramOutput.cpp
     BAMfunctions.cpp
     ReadAlign_alignBAM.cpp
     BAMbinSortByCoordinate.cpp

source/Genome_genomeGenerate.cpp

@@ -28,33 +28,32 @@ uint globalL;
 
 inline int funCompareSuffixes ( const void *a, const void *b){
 
-    uint *ga=(uint*)((globalG-7LLU)+(*((uint*)a)));
-    uint *gb=(uint*)((globalG-7LLU)+(*((uint*)b)));
+    const char *ga=(globalG-7LLU)+(*((uint*)a));
+    const char *gb=(globalG-7LLU)+(*((uint*)b));
 
     uint jj=0;
     int  ii=0;
     uint va=0,vb=0;
-    uint8 *va1, *vb1;
 
     while (jj < globalL) {
-        va=*(ga-jj);
-        vb=*(gb-jj);
+        va=loadUintLE(ga-8*jj); // little-endian 8-byte load (native on LE)
+        vb=loadUintLE(gb-8*jj);
 
         #define has5(v) ((((v)^0x0505050505050505) - 0x0101010101010101) & ~((v)^0x0505050505050505) & 0x8080808080808080)
 
         if (has5(va) && has5(vb))
         {//there is 5 in the sequence - only compare bytes before 5
-            va1=(uint8*) &va;
-            vb1=(uint8*) &vb;
             for (ii=7;ii>=0;ii--)
-            {
-                if (va1[ii]>vb1[ii])
+            {//extract byte ii via shift: portable regardless of host byte order
+                uchar va1=(uchar)((va>>(8*ii))&0xffLLU);
+                uchar vb1=(uchar)((vb>>(8*ii))&0xffLLU);
+                if (va1>vb1)
                 {
                     return 1;
-                } else if (va1[ii]<vb1[ii])
+                } else if (va1<vb1)
                 {
                     return -1;
-                } else if (va1[ii]==5)
+                } else if (va1==5)
                 {//va=vb at the end of chr
                     if ( *((uint*)a) > *((uint*)b) )
                     {//anti-stable order,since indexes are sorted in the reverse order

source/Makefile

@@ -11,18 +11,46 @@ CXXFLAGSextra ?=
 # user may define the compiler
 CXX ?= g++
 
+# ---- Platform / architecture detection -----------------------------------
+UNAME_S := $(shell uname -s)
+UNAME_M := $(shell uname -m)
+
+# ---- OpenMP flags --------------------------------------------------------
+# GNU/Linux: -fopenmp links libgomp automatically.
+# macOS + Apple clang: needs -Xpreprocessor -fopenmp at compile time, plus an
+# explicit libomp include path and -lomp at link time (no bundled OpenMP).
+ifeq ($(UNAME_S),Darwin)
+    LIBOMP ?= $(shell brew --prefix libomp 2>/dev/null)
+    ifeq ($(LIBOMP),)
+        ifneq ($(wildcard /opt/homebrew/opt/libomp/include/omp.h),)
+            LIBOMP := /opt/homebrew/opt/libomp
+        else
+            LIBOMP := /usr/local/opt/libomp
+        endif
+    endif
+    OPENMP_CXXFLAGS := -Xpreprocessor -fopenmp -I$(LIBOMP)/include
+    OPENMP_LDFLAGS  := -L$(LIBOMP)/lib -lomp
+else
+    OPENMP_CXXFLAGS := -fopenmp
+    OPENMP_LDFLAGS  :=
+endif
+
 # pre-defined flags
 LDFLAGS_shared := -pthread -Lhtslib -Bstatic -lhts -Bdynamic -lz -lparasail
 LDFLAGS_static := -static -static-libgcc -pthread -Lhtslib -lhts -lz -lparasail
-LDFLAGS_Mac :=-pthread -lz htslib/libhts.a -lparasail
-LDFLAGS_Mac_static :=-pthread -lz -static-libgcc htslib/libhts.a -lparasail
+# macOS: Apple clang has no -static-libgcc and no fully static binary; "static"
+# here means htslib is linked as a static archive. libomp is linked via OPENMP_LDFLAGS.
+LDFLAGS_Mac :=-pthread -lz htslib/libhts.a -lparasail $(OPENMP_LDFLAGS)
+LDFLAGS_Mac_static :=-pthread -lz htslib/libhts.a -lparasail $(OPENMP_LDFLAGS)
+LDFLAGS_Mac_dynamic :=-pthread -lz htslib/libhts.a -lparasail $(OPENMP_LDFLAGS)
 LDFLAGS_gdb := $(LDFLAGS_shared)
 
 DATE_FMT = --iso-8601=seconds
 ifdef SOURCE_DATE_EPOCH
     BUILD_DATE ?= $(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u -r "$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u "$(DATE_FMT)")
 else
-    BUILD_DATE ?= $(shell date "$(DATE_FMT)")
+    # GNU date supports --iso-8601; BSD/macOS date does not, so fall back to a portable format.
+    BUILD_DATE ?= $(shell date "$(DATE_FMT)" 2>/dev/null || date -u "+%Y-%m-%dT%H:%M:%S%z")
 endif
 
 BUILD_PLACE ?= $(HOSTNAME):$(shell pwd)
@@ -42,10 +70,17 @@ GIT_BRANCH_COMMIT_DIFF := -D'GIT_BRANCH_COMMIT_DIFF="$(GIT_BRANCH_COMMIT_DIFF)"'
 # Defaults, can be overridden by make arguments or environment
 CXXFLAGS ?= -pipe -Wall -Wextra
 CFLAGS ?= -pipe -Wall -Wextra -O3
-CXXFLAGS_SIMD ?= -mavx2
+# SIMD flags: AVX2 on x86_64, NEON on ARM64/AArch64 (incl. Apple Silicon).
+ifeq ($(UNAME_M),x86_64)
+    CXXFLAGS_SIMD ?= -mavx2
+else ifeq ($(UNAME_M),amd64)
+    CXXFLAGS_SIMD ?= -mavx2
+else
+    CXXFLAGS_SIMD ?= -march=armv8-a+simd
+endif
 
 # Unconditionally set essential flags and optimization options
-CXXFLAGS_common := -std=c++17 -fopenmp $(COMPTIMEPLACE) $(GIT_BRANCH_COMMIT_DIFF)
+CXXFLAGS_common := -std=c++17 $(OPENMP_CXXFLAGS) $(COMPTIMEPLACE) $(GIT_BRANCH_COMMIT_DIFF)
 CXXFLAGS_main := -O3 $(CXXFLAGS_common)
 CXXFLAGS_gdb := -O0 -g3 $(CXXFLAGS_common)
 
@@ -91,7 +126,7 @@ OBJECTS = systemFunctions.o funPrimaryAlignMark.o \
 	sjdbLoadFromFiles.o sjdbLoadFromStream.o sjdbPrepare.o sjdbBuildIndex.o sjdbInsertJunctions.o mapThreadsSpawn.o \
 	Parameters_readFilesInit.o Parameters_openReadsFiles.o Parameters_closeReadsFiles.o Parameters_readSAMheader.o \
 	bam_cat.o serviceFuns.o GlobalVariables.o \
-	BAMoutput.o BAMfunctions.o ReadAlign_alignBAM.o BAMbinSortByCoordinate.o signalFromBAM.o bamRemoveDuplicates.o BAMbinSortUnmapped.o
+	BAMoutput.o cramOutput.o BAMfunctions.o ReadAlign_alignBAM.o BAMbinSortByCoordinate.o signalFromBAM.o bamRemoveDuplicates.o BAMbinSortUnmapped.o
 
 SOURCES := $(wildcard *.cpp) $(wildcard *.c)
 
@@ -125,8 +160,6 @@ ifneq ($(MAKECMDGOALS),clean)
 ifneq ($(MAKECMDGOALS),CLEAN)
 ifneq ($(MAKECMDGOALS),CLEAN)
 ifneq ($(MAKECMDGOALS),clean_solo)
-ifneq ($(MAKECMDGOALS),STARforMac)
-ifneq ($(MAKECMDGOALS),STARforMacGDB)
 Depend.list: $(SOURCES) parametersDefault.xxd htslib
 	echo $(SOURCES)
 	'rm' -f ./Depend.list
@@ -136,8 +169,6 @@ endif
 endif
 endif
 endif
-endif
-endif
 
 htslib : htslib/libhts.a
 
@@ -197,6 +228,14 @@ STARforMacStatic : LDFLAGS := $(LDFLAGSextra) $(LDFLAGS_Mac_static) $(LDFLAGS)
 STARforMacStatic : Depend.list parametersDefault.xxd $(OBJECTS)
 	$(CXX) -o STAR $(CXXFLAGS) $(OBJECTS) $(LDFLAGS)
 
+# Native macOS build (Apple clang). Links htslib statically and libomp
+# dynamically. On Apple Silicon this produces an arm64-native binary; on Intel
+# Macs an x86_64 binary. Build with:  make STARforMac CXX=clang++
+STARforMac : CXXFLAGS := $(CXXFLAGSextra) $(CXXFLAGS_main) -D'COMPILE_FOR_MAC' $(CXXFLAGS)
+STARforMac : LDFLAGS := $(LDFLAGSextra) $(LDFLAGS_Mac_dynamic) $(LDFLAGS)
+STARforMac : Depend.list parametersDefault.xxd $(OBJECTS)
+	$(CXX) -o STAR $(CXXFLAGS) $(OBJECTS) $(LDFLAGS)
+
 STARlongForMacStatic : CXXFLAGS := -D'COMPILE_FOR_LONG_READS' $(CXXFLAGSextra) $(CXXFLAGS_main) -D'COMPILE_FOR_MAC' $(CXXFLAGS)
 STARlongForMacStatic : LDFLAGS := $(LDFLAGSextra) $(LDFLAGS_Mac_static) $(LDFLAGS)
 STARlongForMacStatic : Depend.list parametersDefault.xxd $(OBJECTS)

source/PackedArray.cpp

@@ -20,8 +20,9 @@ void PackedArray::writePacked( uint jj, uint x) {
    uint S=b%8LLU;
 
    x = x << S;
-   uint* a1 = (uint*) (charArray+B);
-   *a1 = ( (*a1) & ~(bitRecMask<<S) ) | x;
+   uint a1 = loadUintLE(charArray+B); // little-endian byte stream (native load on LE)
+   a1 = ( a1 & ~(bitRecMask<<S) ) | x;
+   storeUintLE(charArray+B, a1);
 };
 
 void PackedArray::pointArray(char* pointerCharIn) {

source/PackedArray.h

@@ -2,6 +2,7 @@
 #define PACKEDARRAY_DEF
 
 #include "IncludeDefine.h"
+#include "byteOrder.h"
 
 class PackedArray {
     private:
@@ -26,7 +27,7 @@ inline uint PackedArray::operator [] (uint ii) {
    uint B=b/8;
    uint S=b%8;
 
-   uint a1 = *((uint*) (charArray+B));
+   uint a1 = loadUintLE(charArray+B); // little-endian byte stream (native load on LE)
    a1 = (a1>>S) & bitRecMask; // bitmask instead of double-shift (upstream PR #791)
    return a1;
 };

source/Parameters.h

@@ -163,6 +163,7 @@ class Parameters {
 
         //SAM output
         string outBAMfileCoordName, outBAMfileUnsortedName, outQuantBAMfileName;
+        string outCRAMfileCoordName, outCRAMfileUnsortedName;//referenceless CRAM outputs (--outSAMtype CRAM)
         string samHeader, samHeaderHD, samHeaderSortedCoord, samHeaderExtra;
         string outSAMmode,  outSAMorder, outSAMprimaryFlag;
         vector<string> outSAMattributes, outSAMheaderHD, outSAMheaderPG;
@@ -184,6 +185,7 @@ class Parameters {
         int outBAMcompression;
         vector <string> outSAMtype;
         bool outBAMunsorted, outBAMcoord, outSAMbool;
+        bool outCRAMbool;//true when --outSAMtype CRAM: produce BAM then transcode to referenceless CRAM
         uint32 outBAMcoordNbins;
         uint32 outBAMsortingBinsN;//user-defined number of bins for sorting
         string outBAMsortTmpDir;

source/Parameters_openReadsFiles.cpp

@@ -5,6 +5,12 @@
 
 #ifdef _WIN32
 #include "wincompat.h"
+#else
+#include <spawn.h>
+#include <unistd.h>
+// environ is declared in <unistd.h> on glibc but not on macOS; declare it
+// explicitly so posix_spawnp can pass the current environment on every platform.
+extern char **environ;
 #endif
 
 // Large read buffer for ifstream to reduce system call overhead
@@ -132,24 +138,31 @@ void Parameters::openReadsFiles()
 
             readFilesCommandPID[imate]=0;
 
+            // Run the generated readsCommand script through a shell using
+            // posix_spawnp. Invoking it explicitly as "<shell> <script>" means
+            // the script does not need a #! shebang line (none is written by
+            // default, i.e. when --sysShell is "-") and we do not rely on
+            // execvp's ENOEXEC -> /bin/sh fallback, which posix_spawn does not
+            // provide. This fixes "Failed spawning readFilesCommand" on macOS
+            // (upstream issue #2663). posix_spawnp also avoids the undefined
+            // behavior of the previous vfork()+exit(0): calling exit() in a
+            // vfork child flushes the parent's stdio buffers and runs atexit
+            // handlers in the shared address space, and reported success even
+            // when exec failed.
             ostringstream errOut;
-            pid_t PID=vfork();
-            switch (PID) {
-                case -1:
-                    errOut << "EXITING: because of fatal EXECUTION error: Failed vforking readFilesCommand\n";
-                    errOut << errno << ": " << strerror(errno) << "\n";
-                    exitWithError(errOut.str(), std::cerr, inOut->logMain, EXIT_CODE_PARAMETER, *this);
-                    break;
-
-                case 0:
-                    //this is the child
-                    execlp(readsCommandFileName.at(imate).c_str(), readsCommandFileName.at(imate).c_str(), (char*) NULL);
-                    exit(0);
-
-                default:
-                    //this is the father, record PID of the children
-                    readFilesCommandPID[imate]=PID;
+            string readsCommandShell = (sysShell!="-" ? sysShell : "/bin/sh");
+            char* spawnArgv[] = { const_cast<char*>(readsCommandShell.c_str()),
+                                  const_cast<char*>(readsCommandFileName.at(imate).c_str()),
+                                  (char*) NULL };
+            pid_t PID=0;
+            int spawnErr = posix_spawnp(&PID, readsCommandShell.c_str(), NULL, NULL, spawnArgv, environ);
+            if (spawnErr != 0) {
+                errOut << "EXITING: because of fatal EXECUTION error: Failed spawning readFilesCommand: "
+                       << readsCommandShell << " " << readsCommandFileName.at(imate) << "\n";
+                errOut << spawnErr << ": " << strerror(spawnErr) << "\n";
+                exitWithError(errOut.str(), std::cerr, inOut->logMain, EXIT_CODE_PARAMETER, *this);
             };
+            readFilesCommandPID[imate]=PID; //record PID of the child
 
             inOut->readIn[imate].open(readFilesInTmp.at(imate).c_str());
 #endif

source/Parameters_runtimeSetup.cpp

@@ -155,12 +155,22 @@ void Parameters::inputParameters_runtimeSetup() {
     outSAMbool=false;
     outBAMunsorted=false;
     outBAMcoord=false;
+    outCRAMbool=false;
     if (runMode=="alignReads" && outSAMmode != "None") {//open SAM file and write header
-        if (outSAMtype.at(0)=="BAM") {
+        if (outSAMtype.at(0)=="BAM" || outSAMtype.at(0)=="CRAM") {
+            //CRAM reuses the proven BAM output path and is transcoded to referenceless
+            //CRAM at finalization (see bamToCramReferenceless / STAR.cpp)
+            outCRAMbool = (outSAMtype.at(0)=="CRAM");
+            if (outCRAMbool && outStd!="Log") {
+                ostringstream errOut;
+                errOut <<"EXITING because of fatal PARAMETER error: --outSAMtype CRAM is not compatible with --outStd "<<outStd<<"\n";
+                errOut <<"SOLUTION: re-run STAR with --outStd Log (the default) when using --outSAMtype CRAM\n";
+                exitWithError(errOut.str(), std::cerr, inOut->logMain, EXIT_CODE_PARAMETER, *this);
+            };
             if (outSAMtype.size()<2) {
                 ostringstream errOut;
-                errOut <<"EXITING because of fatal PARAMETER error: missing BAM option\n";
-                errOut <<"SOLUTION: re-run STAR with one of the allowed values of --outSAMtype BAM Unsorted OR SortedByCoordinate OR both\n";
+                errOut <<"EXITING because of fatal PARAMETER error: missing "<<outSAMtype.at(0)<<" option\n";
+                errOut <<"SOLUTION: re-run STAR with one of the allowed values of --outSAMtype "<<outSAMtype.at(0)<<" Unsorted OR SortedByCoordinate OR both\n";
                 exitWithError(errOut.str(), std::cerr, inOut->logMain, EXIT_CODE_PARAMETER, *this);
             };
             for (uint32 ii=1; ii<outSAMtype.size(); ii++) {
@@ -171,7 +181,7 @@ void Parameters::inputParameters_runtimeSetup() {
                 } else {
                     ostringstream errOut;
                     errOut <<"EXITING because of fatal input ERROR: unknown value for the word " <<ii+1<<" of outSAMtype: "<< outSAMtype.at(ii) <<"\n";
-                    errOut <<"SOLUTION: re-run STAR with one of the allowed values of --outSAMtype BAM Unsorted or SortedByCoordinate or both\n";
+                    errOut <<"SOLUTION: re-run STAR with one of the allowed values of --outSAMtype "<<outSAMtype.at(0)<<" Unsorted or SortedByCoordinate or both\n";
                     exitWithError(errOut.str(), std::cerr, inOut->logMain, EXIT_CODE_PARAMETER, *this);
                 };
             };
@@ -182,6 +192,8 @@ void Parameters::inputParameters_runtimeSetup() {
                 } else {
                     outBAMfileUnsortedName=outFileNamePrefix + "Aligned.out.bam";
                 };
+                if (outCRAMbool)
+                    outCRAMfileUnsortedName=outFileNamePrefix + "Aligned.out.cram";
                 inOut->outBAMfileUnsorted = bgzf_open(outBAMfileUnsortedName.c_str(),("w"+to_string((long long) outBAMcompression)).c_str());
             };
             if (outBAMcoord) {
@@ -190,6 +202,8 @@ void Parameters::inputParameters_runtimeSetup() {
                 } else {
                     outBAMfileCoordName=outFileNamePrefix + "Aligned.sortedByCoord.out.bam";
                 };
+                if (outCRAMbool)
+                    outCRAMfileCoordName=outFileNamePrefix + "Aligned.sortedByCoord.out.cram";
                 inOut->outBAMfileCoord = bgzf_open(outBAMfileCoordName.c_str(),("w"+to_string((long long) outBAMcompression)).c_str());
                 if (outBAMsortingThreadN==0) {
                     outBAMsortingThreadNactual=min(6, runThreadN);
@@ -223,7 +237,7 @@ void Parameters::inputParameters_runtimeSetup() {
         } else {
             ostringstream errOut;
             errOut <<"EXITING because of fatal input ERROR: unknown value for the first word of outSAMtype: "<< outSAMtype.at(0) <<"\n";
-            errOut <<"SOLUTION: re-run STAR with one of the allowed values of outSAMtype: BAM or SAM \n"<<flush;
+            errOut <<"SOLUTION: re-run STAR with one of the allowed values of outSAMtype: BAM, CRAM, or SAM \n"<<flush;
             exitWithError(errOut.str(), std::cerr, inOut->logMain, EXIT_CODE_PARAMETER, *this);
         };
     };