Export of raw parameters as numpy array, plus some minor fixes

README.md

@@ -31,11 +31,11 @@ When doing computations on the GPU, the available memory limits the size of the
 
 You can calculate the memory requirements in bytes for L columns and N rows using the following formula:
 
-	4*(4*(L*L*21*21 + L*20) + 23*N*L + N + L*L) + 2*N*L + 1024
+	4*(4*(L*L*21*21 + L*20) + 23*N*L + N) + 2*N*L
 
 For the padded version:
 
-	4*(4*(L*L*32*21 + L*20) + 23*N*L + N + L*L) + 2*N*L + 1024
+	4*(4*(L*L*32*21 + L*20) + 34*N*L + N) + 2*N*L
 
 ## Installation
 We recommend compiling CCMpred on the machine that should run the computations so that it can be optimized for the appropriate CPU/GPU architecture.

include/evaluate_cuda_kernels.h

@@ -6,7 +6,7 @@
 #include "evaluate_cuda.h"
 #include "ccmpred.h"
 
-#define CHECK_ERR(err) {if (cudaSuccess != (err)) { printf("CUDA error No. %d in %s at line %d\n", (err), __FILE__, __LINE__); exit(EXIT_FAILURE); } }
+#define CHECK_ERR(err) {int e = (err); if (cudaSuccess != e) { printf("CUDA error No. %d in %s at line %d\n", e, __FILE__, __LINE__); exit(EXIT_FAILURE); } }
 
 #ifdef __cplusplus
 extern "C" {

include/io.h

@@ -20,6 +20,15 @@ void write_matrix(FILE *out, conjugrad_float_t *mat, int ncol, int nrow);
  */
 void write_raw(FILE *out, conjugrad_float_t *x, int ncol);
 
+/**
+ * Write out raw MRF data to a file descriptor as a numpy array
+ *
+ * @param[in] out The file handle to write to
+ * @param[in] x The parameters of the MRF to write out
+ * @param[in] ncol The number of columns in the underlying MSA
+ */
+void write_raw_numpy(FILE *out, conjugrad_float_t *x, int ncol);
+
 /**
  * Read initial weights from a file.
  *

src/ccmpred.c

@@ -198,6 +198,7 @@ void usage(char* exename, int long_usage) {
 		printf("\n");
 		printf("\t-i INIFILE\tRead initial weights from INIFILE\n");
 		printf("\t-r RAWFILE\tStore raw prediction matrix in RAWFILE\n");
+		printf("\t-y NUMPYFILE\tStore raw prediction matrix in binary numpy array NUMPYFILE\n");
 
 #ifdef MSGPACK
 		printf("\t-b BRAWFLE\tStore raw prediction matrix in msgpack format in BRAWFLE\n");
@@ -219,6 +220,7 @@ void usage(char* exename, int long_usage) {
 int main(int argc, char **argv)
 {
 	char *rawfilename = NULL;
+	char *numpyfilename = NULL;
 	int numiter = 250;
 	int use_apc = 1;
 	int use_normalization = 0;
@@ -233,7 +235,7 @@ int main(int argc, char **argv)
 	char *optstr;
 	char *old_optstr = malloc(1);
 	old_optstr[0] = 0;
-	optstr = concat("r:i:n:w:k:e:l:ARh?", old_optstr);
+	optstr = concat("y:r:i:n:w:k:e:l:ARh?", old_optstr);
 	free(old_optstr);
 
 #ifdef OPENMP
@@ -293,6 +295,9 @@ int main(int argc, char **argv)
 			case 'r':
 				rawfilename = thisOpt->argument;
 				break;
+		    case 'y':
+		        numpyfilename = thisOpt->argument;
+		        break;
 			case 'i':
 				initfilename = thisOpt->argument;
 				break;
@@ -683,6 +688,19 @@ int main(int argc, char **argv)
 		write_raw(rawfile, x, ncol);
 	}
 
+	FILE *numpyfile = NULL;
+	if (numpyfilename != NULL) {
+	    printf("Writing numpy array %s\n", numpyfilename);
+	    numpyfile = fopen(numpyfilename, "w");
+
+	    if (numpyfile == NULL) {
+	        printf("Cannot open %s for writing!\n\n", numpyfilename);
+	        return 5;
+	    }
+
+	    write_raw_numpy(numpyfile, x, ncol);
+	}
+
 #ifdef MSGPACK
 
 	FILE *msgpackfile = NULL;
@@ -727,6 +745,10 @@ int main(int argc, char **argv)
 		json_object_set(meta_results, "rawfile", meta_file_from_path(rawfilename));
 	}
 
+    if(numpyfilename != NULL) {
+        json_object_set(meta_results, "numpyfile", meta_file_from_path(numpyfilename));
+    }
+
 #ifdef MSGPACK
 	if(msgpackfilename != NULL) {
 		json_object_set(meta_results, "msgpackfile", meta_file_from_path(msgpackfilename));
@@ -744,6 +766,10 @@ int main(int argc, char **argv)
 		fclose(rawfile);
 	}
 
+    if(numpyfile != NULL) {
+        fclose(numpyfile);
+    }
+
 #ifdef MSGPACK
 	if(msgpackfile != NULL) {
 		write_raw_msgpack(msgpackfile, x, ncol, meta);

src/io.c

@@ -59,6 +59,62 @@ void write_raw(FILE *out, conjugrad_float_t *x, int ncol) {
 	}
 }
 
+void write_raw_numpy(FILE *out, conjugrad_float_t *x, int ncol) {
+
+    int nsingle = ncol * (N_ALPHA - 1);
+    int nsingle_padded = nsingle + N_ALPHA_PAD - (nsingle % N_ALPHA_PAD);
+
+    conjugrad_float_t *x1 = x;
+    //conjugrad_float_t *x2 = &x[nsingle];
+    conjugrad_float_t *x2 = &x[nsingle_padded];
+    (void)x2;
+
+    // Write magic code
+    char magic[6] = {'\x93', '\x4E', '\x55', '\x4D', '\x50', '\x59'};
+    fwrite(magic, 1, 6, out);
+
+    // Write version information
+    char version[2] = {0x01, 0x00};
+    fwrite(version, 1, 2, out);
+
+    int digits = 0;
+    int num = ncol;
+    while (num != 0) {
+        digits++;
+        num /= 10;
+    }
+
+    // Determine padded header size
+    char header[200];
+    int dict_len = snprintf(header, 200, "{'descr': '<f4', 'fortran_order': False, 'shape': (%d, %d, 441), }", ncol, ncol);
+    int temp_len = 6 + 2 + 2 + dict_len + 1; // magic + version + header_length + dict + 0x0A;
+    short padding = 0;
+    short r = temp_len % 64;
+    if (r != 0)
+        padding = 64 - r;
+    short header_length = dict_len + padding + 1;
+
+    // Write header length, the header itself, padding and terminating character
+    fwrite(&header_length, 2, 1, out);
+    fwrite(header, 1, dict_len, out);
+    char pad = '\x20';
+    for (int i=0 ; i<padding ; ++i)
+        fwrite(&pad, 1, 1, out);
+    char terminator = '\x0A';
+    fwrite(&terminator, 1, 1, out);
+
+    // Write parameters in C-contiguous order
+    for(int i = 0; i < ncol; i++) {
+        for(int j = 0; j < ncol; j++) {
+            for(int a = 0; a < N_ALPHA; a++) {
+                for(int b = 0; b < N_ALPHA; b++) {
+                    fwrite(&W(b,j,a,i), 4, 1, out);
+                }
+            }
+        }
+    }
+}
+
 void read_raw(char *filename, userdata *ud, conjugrad_float_t *x) {
 	FILE *f = fopen(filename, "r");
 	char *line = (char*) malloc(sizeof(char) * 8192);

src/meta.c

@@ -1,5 +1,6 @@
 #include <time.h>
 #include <stdlib.h>
+#include <libgen.h>
 
 #ifdef UUID
 #include <uuid/uuid.h>