fileloader.py

from optparse import OptionParser
import sys
import numpy as np
import os

def main(argv):
    srcdir = ''
    outputfile = ''
    
    parser = OptionParser("usage: %prog <-d> srcdir <-o> outputfile")
    
    parser.add_option("-d", "--srcdir", dest="srcdir",
                  help="Path to directory for fileload")
    parser.add_option("-o", "--outfile", dest="outfile", default="fileloader_out",
                  help="Filename for output in numpy format")

    (opts, args) = parser.parse_args(args=None, values=None)
    
    #print opts
    
    if len(args)==0:
        args=["."]
    if opts.srcdir==None :
        print "No directory specified. Run -h flag!"
        sys.exit()
    
    srcdir = opts.srcdir
    outputfile = opts.outfile + ".txt"
    
    print 'The source directory is  "'+ srcdir+'"'
    print 'The output filename is"'+ outputfile+'"'


    '''Now, generate a list of all files'''
    filelist = np.empty([0,1], dtype='string')
    for root, dirs, files in os.walk(srcdir):
        #..\\.\\110413_lechner\\
        dirs[:]=''
        for file in files:
            if file.endswith(".chi"):# and file.startswith("bi_cp_s"):
                filelist=np.vstack((filelist,os.path.join(root, file)))
                #print(os.path.join(root, file))
    
    phi = np.genfromtxt(filelist[0][0],usecols=(0),skip_header=4 ,dtype='float32')
    phi_oldsize=phi.size
    print 'Filelist was generated! Loading a total of ',filelist.size, 'files.'
    
    '''Now read in the actual data'''
    imported_data = np.empty([filelist.size,phi.size])
    for i in range(filelist.size):
        try:
            imported_data[i,:]=np.genfromtxt(filelist[i][0],usecols=(1),skip_header=4 ,dtype='float32')
        except:
            print "something went wrong in ",filelist[i][0]
        if (i%(filelist.size/10)==0):
            print "Loaded %d %%" % (float(i)/filelist.size *100.)
            
    print "We now imported a total of ", imported_data.shape[0], " files."
    
    outfile =os.path.join(srcdir,outputfile)
    #outfile = srcdir + outputfile
    print 'The data is saved in "'+ outfile+'"'
    np.savetxt(outfile,imported_data)
    print "Done!"
    

if __name__ == "__main__":
   main(sys.argv[1:])