diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 1dbc687..0000000 --- a/.gitignore +++ /dev/null @@ -1,62 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -#Ipython Notebook -.ipynb_checkpoints diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 4cf4055..8da9fe1 --- a/README.md +++ b/README.md @@ -1,2 +1,25 @@ -# h5boss -Exploratory tools for reformatting BOSS spectra as hdf5 files +# H5Boss +H5Boss is an exploratory python tool for managing BOSS spectra data, SDSS-II. + +Boss is originally maintained as millions of fits file in thousands of different folders. Accessing and analyzing them are inefficient in terms of I/O bandwidth and programming productivity. In h5boss, we developed functions to support: + + 1. Reformatting: Preserve the fits file structure and specture hierarchicy using HDF5 + 2. Subsetting: Support subset/add/update operation, to extract selected ‘plate/mjd/fiber’ and save in one HDF5 file + +Currently, h5boss is implemented in both python and c version, in which the python version is actively maintained and supported. The c version is mainly for I/O sensitive users/applications. + +# Tutorial +1. download repo +2. cd h5boss_py/docs/ +3. make html +4. open build/html/index.html + +# Demo with h5boss_py +1. cd h5boss_py/demo +2. subset -h +3. add -h +4. update -h + +# Demo with h5boss_c +1. cd h5boss_c +2. make diff --git a/h5boss/io.py b/h5boss/io.py deleted file mode 100644 index 618b83f..0000000 --- a/h5boss/io.py +++ /dev/null @@ -1,121 +0,0 @@ -import os.path -import numpy as np -from astropy.io import fits -from astropy.table import Table - -def load_coadds(platefile, zbestfile=None, run1d=None): - ''' - Document ... - ''' - #- Load spPlate data - fx = fits.open(platefile, memmap=False) - header = fx[0].header - c0 = header['COEFF0'] - c1 = header['COEFF1'] - nwave = header['NAXIS1'] - nfiber = header['NAXIS2'] - wave = (10**(c0 + c1*np.arange(nwave))).astype(np.float32) - flux = fx[0].data - ivar = fx[1].data - and_mask = fx[2].data - or_mask = fx[3].data - wavedisp = fx[4].data - sky = fx[6].data - fx.close() - - if run1d is None: - run1d = header['RUN2D'] #- default run1d == run2d - - #- Get best fit model from zbest file - if zbestfile is None: - zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d)) - - model = fits.getdata(zbestfile, 2) - - coadds = list() - for i in range(nfiber): - sp = Table() - sp['WAVE'] = wave #- repeat ! - sp['FLUX'] = flux[i] - sp['IVAR'] = ivar[i] - sp['AND_MASK'] = and_mask[i] - sp['OR_MASK'] = or_mask[i] - sp['WAVEDISP'] = wavedisp[i] - sp['SKY'] = sky[i] - sp['MODEL'] = model[i] - sp.meta = header - - #- TODO: Add units, comments to each column - - coadds.append(sp) - - return coadds - -def load_frame(framefile, cframefile=None, flatfile=None): - """ - Document ... - """ - if cframefile is None: - cframefile = framefile.replace('spFrame', 'spCFrame') - if cframefile.endswith('.gz'): - cframefile = cframefile[:-3] - - #- Load framefile and get original dimensions - eflux = fits.getdata(framefile, 0) - nfiber, npix = eflux.shape - - #- Load spCFrame file; trim arrays back to original size - fx = fits.open(cframefile, memmap=False) - header = fx[0].header - flux = fx[0].data[:, 0:npix] - ivar = fx[1].data[:, 0:npix] - mask = fx[2].data[:, 0:npix] - wave = (10**fx[3].data[:, 0:npix]).astype(np.float32) - wavedisp = fx[4].data[:, 0:npix] - sky = fx[6].data[:, 0:npix] - x = fx[7].data[:, 0:npix] - superflat = fx[8].data[:, 0:npix] - - #- Load fiberflat spFlat[0] - if flatfile is None: - flatfile = header['FLATFILE'].replace('sdR', 'spFlat') - flatfile = flatfile.replace('.fit', '.fits.gz') - filedir, basename = os.path.split(os.path.abspath(cframefile)) - flatfile = os.path.join(filedir, flatfile) - - fiberflat = fits.getdata(flatfile, 0) - - #- Calculate calibration vector: flux = electrons * calib - electrons = eflux * fiberflat * superflat - ii = np.where(electrons != 0.0) - calib = np.zeros(flux.shape) - calib[ii] = flux[ii] / electrons[ii] - - fx.close() - - #- Assemble spectra tables - spectra = list() - for i in range(nfiber): - sp = Table() - sp['WAVE'] = wave[i] - sp['FLUX'] = flux[i] - sp['IVAR'] = ivar[i] - sp['MASK'] = mask[i] - sp['WAVEDISP'] = wavedisp[i] - sp['SKY'] = sky[i] - sp['X'] = x[i] - sp['CALIB'] = calib[i].astype(np.float32) - sp.meta = header - - #- TODO: Add units, comments to each column - - spectra.append(sp) - - return spectra - - - - - - - diff --git a/h5boss/select.py b/h5boss/select.py deleted file mode 100644 index fc0e292..0000000 --- a/h5boss/select.py +++ /dev/null @@ -1,43 +0,0 @@ -import numpy as np -import h5py - -def select(infiles, outfile, plates, mjds, fibers): - ''' - Select a set of (plates,mjds,fibers) from a set of input files - - Args: - infiles : list of input filenames, or single filename - outfile : output file to write the selection - plates : list of plates - mjds : list of plates - fibers : list of fibers - ''' - plates = np.asarray(plates) - mjds = np.asarray(mjds) - fibers = np.asarray(fibers) - - if not isinstance(infiles, (list, tuple)): - infiles = [infiles,] - - hx = h5py.File(outfile) - - for infile in infiles: - fx = h5py.File(infile, mode='r') - for plate in fx.keys(): - for mjd in fx[plate].keys(): - ii = (plates == int(plate)) & (mjds == int(mjd)) - xfibers = fibers[ii] - for fiber in xfibers: - id = '{}/{}/{}'.format(plate, mjd, fiber) - # hx[id] = fx[id].copy() - hx.create_group(id) - fx.copy(id, hx[id]) - - for name in ['plugmap', 'zbest', 'zline', - 'photo/match', 'photo/matchflux', 'photo/matchpos']: - id = '{}/{}/{}'.format(plate, mjd, name) - catalog = fx[id] - jj = np.in1d(catalog['FIBERID'], xfibers) - hx[id] = fx[id][jj].copy() - - hx.close() diff --git a/h5boss_c/Makefile b/h5boss_c/Makefile new file mode 100755 index 0000000..1f1cc35 --- /dev/null +++ b/h5boss_c/Makefile @@ -0,0 +1,16 @@ +CC=cc +CFLAGS=-g -c -Wall +LDFLAGS= +SOURCES=subset.c parse_node.c compound_copy.c +OBJECTS=$(SOURCES:.c=.o) +EXECUTABLE=subset.exe + +all: $(SOURCES) $(EXECUTABLE) + +$(EXECUTABLE): $(OBJECTS) + $(CC) $(LDFLAGS) $(OBJECTS) -g -o $@ + +.c.o: + $(CC) $(CFLAGS) $< -o $@ +clean: + rm *.o subset.exe diff --git a/h5boss_c/compound_copy.c b/h5boss_c/compound_copy.c new file mode 100755 index 0000000..ec487c6 --- /dev/null +++ b/h5boss_c/compound_copy.c @@ -0,0 +1,1187 @@ +#include +#include"compound_copy.h" +#include"parse_node.h" +#include +#include +#include +#include +#include//this is needed for using the hdf5 table api +size_t COADD_REC_SIZE=sizeof(struct COADD); +size_t EXPOSURE_REC_SIZE=sizeof(struct EXPOSURE); +size_t CATALOG_REC_SIZE[]={sizeof(struct PLUGMAP),sizeof(struct ZBEST), sizeof(struct ZLINE), + sizeof(struct MATCH), sizeof(struct MATCHFLUX), sizeof(struct MATCHPOS)}; +#define NFIELDS (hsize_t) 8 +#define NFIELDS_PLUGMAP (hsize_t) 35 +#define NFIELDS_ZBEST (hsize_t) 73 +#define NFIELDS_ZLINE (hsize_t) 19 +#define NFIELDS_MATCH (hsize_t) 24 +#define NFIELDS_MATCHFLUX (hsize_t) 140 +#define NFIELDS_MATCHPOS (hsize_t) 140 +#define TABLE_COADD_NAME "coadd" +#define TABLE_EXPOSURE_B_NAME "b" +#define TABLE_EXPOSURE_R_NAME "r" +const char * FIBERID="FIBERID"; +const char * catalog_array[] = { + "plugmap", + "zbest", + "zline", + "match", + "matchflux", + "matchpos" +}; +#define n_catalog (sizeof(catalog_array)/sizeof(const char *)) +const char photo[6]="/photo"; +size_t coadd_sizes[NFIELDS] = { + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(float), + sizeof(float) +}; +size_t coadd_offsets[NFIELDS] = { + HOFFSET( COADD, WAVE ), + HOFFSET( COADD, FLUX ), + HOFFSET( COADD, IVAR ), + HOFFSET( COADD, AND_MASK ), + HOFFSET( COADD, OR_MASK ), + HOFFSET( COADD, WAVEDISP ), + HOFFSET( COADD, SKY ), + HOFFSET( COADD, MODEL ), +}; +size_t exposure_offsets[NFIELDS] = { + HOFFSET( EXPOSURE, WAVE ), + HOFFSET( EXPOSURE, FLUX ), + HOFFSET( EXPOSURE, IVAR ), + HOFFSET( EXPOSURE, MASK ), + HOFFSET( EXPOSURE, WAVEDISP ), + HOFFSET( EXPOSURE, SKY ), + HOFFSET( EXPOSURE, X), + HOFFSET( EXPOSURE, CALIB ), +}; +size_t exposure_sizes[NFIELDS] ={ + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float) +}; +size_t plugmap_sizes[NFIELDS_PLUGMAP] = { + sizeof(int)*5, + sizeof(char)*6, + sizeof(double), + sizeof(double), + sizeof(float)*5, + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(char)*16, + sizeof(double), + sizeof(double), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(char)*15, + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(int64_t), + sizeof(int64_t), + sizeof(int64_t), + sizeof(int64_t), + sizeof(int), + sizeof(char)*4, + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(float)*5, + sizeof(float)*5, + sizeof(int)*5, + sizeof(float) +}; +size_t plugmap_offsets[NFIELDS_PLUGMAP] = { + HOFFSET(PLUGMAP,OBJID), + HOFFSET(PLUGMAP,HOLETYPE), + HOFFSET(PLUGMAP,RA), + HOFFSET(PLUGMAP,DEC), + HOFFSET(PLUGMAP,MAG), + HOFFSET(PLUGMAP,STARL), + HOFFSET(PLUGMAP,EXPL), + HOFFSET(PLUGMAP,DEVAUCL), + HOFFSET(PLUGMAP,OBJTYPE), + HOFFSET(PLUGMAP,XFOCAL), + HOFFSET(PLUGMAP,YFOCAL), + HOFFSET(PLUGMAP,SPECTROGRAPHID), + HOFFSET(PLUGMAP,FIBERID), + HOFFSET(PLUGMAP,THROUGHPUT), + HOFFSET(PLUGMAP,PRIMTARGET), + HOFFSET(PLUGMAP,SECTARGET), + HOFFSET(PLUGMAP,OFFSETID), + HOFFSET(PLUGMAP,SCI_EXPTIME), + HOFFSET(PLUGMAP,SOURCETYPE), + HOFFSET(PLUGMAP,LAMBDA_EFF), + HOFFSET(PLUGMAP,ZOFFSET), + HOFFSET(PLUGMAP,BLUEFIBER), + HOFFSET(PLUGMAP,BOSS_TARGET1), + HOFFSET(PLUGMAP,BOSS_TARGET2), + HOFFSET(PLUGMAP,ANCILLARY_TARGET1), + HOFFSET(PLUGMAP,ANCILLARY_TARGET2), + HOFFSET(PLUGMAP,RUN), + HOFFSET(PLUGMAP,RERUN), + HOFFSET(PLUGMAP,CAMCOL), + HOFFSET(PLUGMAP,FIELD), + HOFFSET(PLUGMAP,ID), + HOFFSET(PLUGMAP,CALIBFLUX), + HOFFSET(PLUGMAP,CALIBFLUX_IVAR), + HOFFSET(PLUGMAP,CALIB_STATUS), + HOFFSET(PLUGMAP,SFD_EBV) +}; +size_t zbest_sizes[NFIELDS_ZBEST] = { + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(char)*6, + sizeof(char)*6, + sizeof(int)*5, + sizeof(char)*16, + sizeof(double), + sizeof(double), + sizeof(char)*6, + sizeof(char)*21, + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(float), + sizeof(char)*24, + sizeof(int)*10, + sizeof(int), + sizeof(float)*10, + sizeof(float)*100, + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(float)*5, + sizeof(float), + sizeof(float), + sizeof(float)*10, + sizeof(float)*10, + sizeof(float)*10, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(char)*16, + sizeof(char)*8, + sizeof(char)*12, + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(int), + sizeof(char)*6, + sizeof(char)*19, + sizeof(float), + sizeof(float)*35, + sizeof(int64_t) +}; +size_t zbest_offsets[NFIELDS_ZBEST] = { + HOFFSET ( ZBEST,PLATE ), + HOFFSET ( ZBEST,TILE ), + HOFFSET ( ZBEST,MJD ), + HOFFSET ( ZBEST,FIBERID ), + HOFFSET ( ZBEST,RUN2D ), + HOFFSET ( ZBEST,RUN1D ), + HOFFSET ( ZBEST,OBJID ), + HOFFSET ( ZBEST,OBJTYPE ), + HOFFSET ( ZBEST,PLUG_RA ), + HOFFSET ( ZBEST,PLUG_DEC ), + HOFFSET ( ZBEST,CLASS ), + HOFFSET ( ZBEST,SUBCLASS ), + HOFFSET ( ZBEST,Z ), + HOFFSET ( ZBEST,Z_ERR ), + HOFFSET ( ZBEST,RCHI2 ), + HOFFSET ( ZBEST,DOF ), + HOFFSET ( ZBEST,RCHI2DIFF ), + HOFFSET ( ZBEST,TFILE ), + HOFFSET ( ZBEST,TCOLUMN ), + HOFFSET ( ZBEST,NPOLY ), + HOFFSET ( ZBEST,THETA ), + HOFFSET ( ZBEST,THETA_COVAR ), + HOFFSET ( ZBEST,VDISP ), + HOFFSET ( ZBEST,VDISP_ERR ), + HOFFSET ( ZBEST,VDISPZ ), + HOFFSET ( ZBEST,VDISPZ_ERR ), + HOFFSET ( ZBEST,VDISPCHI2 ), + HOFFSET ( ZBEST,VDISPNPIX ), + HOFFSET ( ZBEST,VDISPDOF ), + HOFFSET ( ZBEST,WAVEMIN ), + HOFFSET ( ZBEST,WAVEMAX ), + HOFFSET ( ZBEST,WCOVERAGE ), + HOFFSET ( ZBEST,ZWARNING ), + HOFFSET ( ZBEST,SN_MEDIAN ), + HOFFSET ( ZBEST,SN_MEDIAN_ALL ), + HOFFSET ( ZBEST,CHI68P ), + HOFFSET ( ZBEST,FRACNSIGMA ), + HOFFSET ( ZBEST,FRACNSIGHI ), + HOFFSET ( ZBEST,FRACNSIGLO ), + HOFFSET ( ZBEST,SPECTROFLUX ), + HOFFSET ( ZBEST,SPECTROFLUX_IVAR ), + HOFFSET ( ZBEST,SPECTROSYNFLUX ), + HOFFSET ( ZBEST,SPECTROSYNFLUX_IVAR ), + HOFFSET ( ZBEST,SPECTROSKYFLUX ), + HOFFSET ( ZBEST,ANYANDMASK ), + HOFFSET ( ZBEST,ANYORMASK ), + HOFFSET ( ZBEST,SPEC1_G ), + HOFFSET ( ZBEST,SPEC1_R ), + HOFFSET ( ZBEST,SPEC1_I ), + HOFFSET ( ZBEST,SPEC2_G ), + HOFFSET ( ZBEST,SPEC2_R ), + HOFFSET ( ZBEST,SPEC2_I ), + HOFFSET ( ZBEST,ELODIE_FILENAME ), + HOFFSET ( ZBEST,ELODIE_OBJECT ), + HOFFSET ( ZBEST,ELODIE_SPTYPE ), + HOFFSET ( ZBEST,ELODIE_BV ), + HOFFSET ( ZBEST,ELODIE_TEFF ), + HOFFSET ( ZBEST,ELODIE_LOGG ), + HOFFSET ( ZBEST,ELODIE_FEH ), + HOFFSET ( ZBEST,ELODIE_Z ), + HOFFSET ( ZBEST,ELODIE_Z_ERR ), + HOFFSET ( ZBEST,ELODIE_Z_MODELERR ), + HOFFSET ( ZBEST,ELODIE_RCHI2 ), + HOFFSET ( ZBEST,ELODIE_DOF ), + HOFFSET ( ZBEST,Z_NOQSO ), + HOFFSET ( ZBEST,Z_ERR_NOQSO ), + HOFFSET ( ZBEST,ZNUM_NOQSO ), + HOFFSET ( ZBEST,ZWARNING_NOQSO ), + HOFFSET ( ZBEST,CLASS_NOQSO ), + HOFFSET ( ZBEST,SUBCLASS_NOQSO ), + HOFFSET ( ZBEST,RCHI2DIFF_NOQSO ), + HOFFSET ( ZBEST,VDISP_LNL ), + HOFFSET ( ZBEST,SPECOBJID ) +}; +size_t zline_sizes[NFIELDS_ZLINE] = { + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(char)*13, + sizeof(double), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(float) +}; +size_t zline_offsets[NFIELDS_ZLINE] ={ + HOFFSET ( ZLINE,PLATE ), + HOFFSET ( ZLINE,MJD ), + HOFFSET ( ZLINE,FIBERID ), + HOFFSET ( ZLINE,LINENAME ), + HOFFSET ( ZLINE,LINEWAVE ), + HOFFSET ( ZLINE,LINEZ ), + HOFFSET ( ZLINE,LINEZ_ERR ), + HOFFSET ( ZLINE,LINESIGMA ), + HOFFSET ( ZLINE,LINESIGMA_ERR ), + HOFFSET ( ZLINE,LINEAREA ), + HOFFSET ( ZLINE,LINEAREA_ERR ), + HOFFSET ( ZLINE,LINEEW ), + HOFFSET ( ZLINE,LINEEW_ERR ), + HOFFSET ( ZLINE,LINECONTLEVEL ), + HOFFSET ( ZLINE,LINECONTLEVEL_ERR ), + HOFFSET ( ZLINE,LINENPIXLEFT ), + HOFFSET ( ZLINE,LINENPIXRIGHT ), + HOFFSET ( ZLINE,LINEDOF ), + HOFFSET ( ZLINE,LINECHI2 ) +}; +size_t match_sizes[NFIELDS_MATCH] ={ + sizeof(char)*19, + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(char)*3, + sizeof(int), + sizeof(int), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(char)*19, + sizeof(int), + sizeof(double), + sizeof(double), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(int), + sizeof(int), + sizeof(int16_t) +}; +size_t match_offsets[NFIELDS_MATCH] = { + HOFFSET ( MATCH,FLUX_OBJID ), + HOFFSET ( MATCH,RUN ), + HOFFSET ( MATCH,CAMCOL ), + HOFFSET ( MATCH,FIELD ), + HOFFSET ( MATCH,FLUX_ID ), + HOFFSET ( MATCH,THING_ID ), + HOFFSET ( MATCH,RERUN ), + HOFFSET ( MATCH,NOBSERVE ), + HOFFSET ( MATCH,NDETECT ), + HOFFSET ( MATCH,FLUX_RA ), + HOFFSET ( MATCH,FLUX_DEC ), + HOFFSET ( MATCH,MATCH_RA ), + HOFFSET ( MATCH,MATCH_DEC ), + HOFFSET ( MATCH,ORIG_OBJID ), + HOFFSET ( MATCH,ORIG_ID ), + HOFFSET ( MATCH,ORIG_RA ), + HOFFSET ( MATCH,ORIG_DEC ), + HOFFSET ( MATCH,APERFLUX3_R ), + HOFFSET ( MATCH,FIBERFLUX_R ), + HOFFSET ( MATCH,APERFLUX3_R_TOTAL ), + HOFFSET ( MATCH,APERFLUX3_R_MATCH ), + HOFFSET ( MATCH,FLUXMATCH_PARENT ), + HOFFSET ( MATCH,FLUXMATCH_STATUS ), + HOFFSET ( MATCH,FIBERID ) +}; +size_t matchflux_sizes[NFIELDS_MATCHFLUX]={ + sizeof(char)*19, + sizeof(char)*19, + sizeof(char)*19, + sizeof(uint8_t), + sizeof(uint8_t), + sizeof(uint8_t), + sizeof(int16_t), + sizeof(char)*3, + sizeof(uint8_t), + sizeof(int16_t), + sizeof(int16_t), + sizeof(int16_t), + sizeof(int16_t), + sizeof(int), + sizeof(float), + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(int)*5, + sizeof(int)*5, + sizeof(int)*5, + sizeof(float)*5, + sizeof(int)*5, + sizeof(float)*75, + sizeof(float)*75, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(int)*5, + sizeof(float)*5, + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(int), + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*40, + sizeof(float)*40, + sizeof(int)*5, + sizeof(int)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(double)*5, + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(int16_t) +}; +size_t matchflux_offsets[NFIELDS_MATCHFLUX]={ + HOFFSET ( MATCHFLUX,OBJID ), + HOFFSET ( MATCHFLUX,PARENTID ), + HOFFSET ( MATCHFLUX,FIELDID ), + HOFFSET ( MATCHFLUX,SKYVERSION ), + HOFFSET ( MATCHFLUX,MODE ), + HOFFSET ( MATCHFLUX,CLEAN ), + HOFFSET ( MATCHFLUX,RUN ), + HOFFSET ( MATCHFLUX,RERUN ), + HOFFSET ( MATCHFLUX,CAMCOL ), + HOFFSET ( MATCHFLUX,FIELD ), + HOFFSET ( MATCHFLUX,ID ), + HOFFSET ( MATCHFLUX,PARENT ), + HOFFSET ( MATCHFLUX,NCHILD ), + HOFFSET ( MATCHFLUX,OBJC_TYPE ), + HOFFSET ( MATCHFLUX,OBJC_PROB_PSF ), + HOFFSET ( MATCHFLUX,OBJC_FLAGS ), + HOFFSET ( MATCHFLUX,OBJC_FLAGS2 ), + HOFFSET ( MATCHFLUX,OBJC_ROWC ), + HOFFSET ( MATCHFLUX,OBJC_ROWCERR ), + HOFFSET ( MATCHFLUX,OBJC_COLC ), + HOFFSET ( MATCHFLUX,OBJC_COLCERR ), + HOFFSET ( MATCHFLUX,ROWVDEG ), + HOFFSET ( MATCHFLUX,ROWVDEGERR ), + HOFFSET ( MATCHFLUX,COLVDEG ), + HOFFSET ( MATCHFLUX,COLVDEGERR ), + HOFFSET ( MATCHFLUX,ROWC ), + HOFFSET ( MATCHFLUX,ROWCERR ), + HOFFSET ( MATCHFLUX,COLC ), + HOFFSET ( MATCHFLUX,COLCERR ), + HOFFSET ( MATCHFLUX,PETROTHETA ), + HOFFSET ( MATCHFLUX,PETROTHETAERR ), + HOFFSET ( MATCHFLUX,PETROTH50 ), + HOFFSET ( MATCHFLUX,PETROTH50ERR ), + HOFFSET ( MATCHFLUX,PETROTH90 ), + HOFFSET ( MATCHFLUX,PETROTH90ERR ), + HOFFSET ( MATCHFLUX,Q ), + HOFFSET ( MATCHFLUX,QERR ), + HOFFSET ( MATCHFLUX,U ), + HOFFSET ( MATCHFLUX,UERR ), + HOFFSET ( MATCHFLUX,M_E1 ), + HOFFSET ( MATCHFLUX,M_E2 ), + HOFFSET ( MATCHFLUX,M_E1E1ERR ), + HOFFSET ( MATCHFLUX,M_E1E2ERR ), + HOFFSET ( MATCHFLUX,M_E2E2ERR ), + HOFFSET ( MATCHFLUX,M_RR_CC ), + HOFFSET ( MATCHFLUX,M_RR_CCERR ), + HOFFSET ( MATCHFLUX,M_CR4 ), + HOFFSET ( MATCHFLUX,M_E1_PSF ), + HOFFSET ( MATCHFLUX,M_E2_PSF ), + HOFFSET ( MATCHFLUX,M_RR_CC_PSF ), + HOFFSET ( MATCHFLUX,M_CR4_PSF ), + HOFFSET ( MATCHFLUX,THETA_DEV ), + HOFFSET ( MATCHFLUX,THETA_DEVERR ), + HOFFSET ( MATCHFLUX,AB_DEV ), + HOFFSET ( MATCHFLUX,AB_DEVERR ), + HOFFSET ( MATCHFLUX,THETA_EXP ), + HOFFSET ( MATCHFLUX,THETA_EXPERR ), + HOFFSET ( MATCHFLUX,AB_EXP ), + HOFFSET ( MATCHFLUX,AB_EXPERR ), + HOFFSET ( MATCHFLUX,FRACDEV ), + HOFFSET ( MATCHFLUX,FLAGS ), + HOFFSET ( MATCHFLUX,FLAGS2 ), + HOFFSET ( MATCHFLUX,TYPE ), + HOFFSET ( MATCHFLUX,PROB_PSF ), + HOFFSET ( MATCHFLUX,NPROF ), + HOFFSET ( MATCHFLUX,PROFMEAN_NMGY ), + HOFFSET ( MATCHFLUX,PROFERR_NMGY ), + HOFFSET ( MATCHFLUX,STAR_LNL ), + HOFFSET ( MATCHFLUX,EXP_LNL ), + HOFFSET ( MATCHFLUX,DEV_LNL ), + HOFFSET ( MATCHFLUX,PSP_STATUS ), + HOFFSET ( MATCHFLUX,PIXSCALE ), + HOFFSET ( MATCHFLUX,RA ), + HOFFSET ( MATCHFLUX,DEC ), + HOFFSET ( MATCHFLUX,CX ), + HOFFSET ( MATCHFLUX,CY ), + HOFFSET ( MATCHFLUX,CZ ), + HOFFSET ( MATCHFLUX,RAERR ), + HOFFSET ( MATCHFLUX,DECERR ), + HOFFSET ( MATCHFLUX,L ), + HOFFSET ( MATCHFLUX,B ), + HOFFSET ( MATCHFLUX,OFFSETRA ), + HOFFSET ( MATCHFLUX,OFFSETDEC ), + HOFFSET ( MATCHFLUX,PSF_FWHM ), + HOFFSET ( MATCHFLUX,MJD ), + HOFFSET ( MATCHFLUX,AIRMASS ), + HOFFSET ( MATCHFLUX,PHI_OFFSET ), + HOFFSET ( MATCHFLUX,PHI_DEV_DEG ), + HOFFSET ( MATCHFLUX,PHI_EXP_DEG ), + HOFFSET ( MATCHFLUX,EXTINCTION ), + HOFFSET ( MATCHFLUX,SKYFLUX ), + HOFFSET ( MATCHFLUX,SKYFLUX_IVAR ), + HOFFSET ( MATCHFLUX,PSFFLUX ), + HOFFSET ( MATCHFLUX,PSFFLUX_IVAR ), + HOFFSET ( MATCHFLUX,PSFMAG ), + HOFFSET ( MATCHFLUX,PSFMAGERR ), + HOFFSET ( MATCHFLUX,FIBERFLUX ), + HOFFSET ( MATCHFLUX,FIBERFLUX_IVAR ), + HOFFSET ( MATCHFLUX,FIBERMAG ), + HOFFSET ( MATCHFLUX,FIBERMAGERR ), + HOFFSET ( MATCHFLUX,FIBER2FLUX ), + HOFFSET ( MATCHFLUX,FIBER2FLUX_IVAR ), + HOFFSET ( MATCHFLUX,FIBER2MAG ), + HOFFSET ( MATCHFLUX,FIBER2MAGERR ), + HOFFSET ( MATCHFLUX,CMODELFLUX ), + HOFFSET ( MATCHFLUX,CMODELFLUX_IVAR ), + HOFFSET ( MATCHFLUX,CMODELMAG ), + HOFFSET ( MATCHFLUX,CMODELMAGERR ), + HOFFSET ( MATCHFLUX,MODELFLUX ), + HOFFSET ( MATCHFLUX,MODELFLUX_IVAR ), + HOFFSET ( MATCHFLUX,MODELMAG ), + HOFFSET ( MATCHFLUX,MODELMAGERR ), + HOFFSET ( MATCHFLUX,PETROFLUX ), + HOFFSET ( MATCHFLUX,PETROFLUX_IVAR ), + HOFFSET ( MATCHFLUX,PETROMAG ), + HOFFSET ( MATCHFLUX,PETROMAGERR ), + HOFFSET ( MATCHFLUX,DEVFLUX ), + HOFFSET ( MATCHFLUX,DEVFLUX_IVAR ), + HOFFSET ( MATCHFLUX,DEVMAG ), + HOFFSET ( MATCHFLUX,DEVMAGERR ), + HOFFSET ( MATCHFLUX,EXPFLUX ), + HOFFSET ( MATCHFLUX,EXPFLUX_IVAR ), + HOFFSET ( MATCHFLUX,EXPMAG ), + HOFFSET ( MATCHFLUX,EXPMAGERR ), + HOFFSET ( MATCHFLUX,APERFLUX ), + HOFFSET ( MATCHFLUX,APERFLUX_IVAR ), + HOFFSET ( MATCHFLUX,CLOUDCAM ), + HOFFSET ( MATCHFLUX,CALIB_STATUS ), + HOFFSET ( MATCHFLUX,NMGYPERCOUNT ), + HOFFSET ( MATCHFLUX,NMGYPERCOUNT_IVAR ), + HOFFSET ( MATCHFLUX,TAI ), + HOFFSET ( MATCHFLUX,RESOLVE_STATUS ), + HOFFSET ( MATCHFLUX,THING_ID ), + HOFFSET ( MATCHFLUX,IFIELD ), + HOFFSET ( MATCHFLUX,BALKAN_ID ), + HOFFSET ( MATCHFLUX,NOBSERVE ), + HOFFSET ( MATCHFLUX,NDETECT ), + HOFFSET ( MATCHFLUX,NEDGE ), + HOFFSET ( MATCHFLUX,SCORE ), + HOFFSET ( MATCHFLUX,FIBERID ) +}; +size_t matchpos_sizes[NFIELDS_MATCHPOS] ={ + sizeof(char)*19, + sizeof(char)*19, + sizeof(char)*19, + sizeof(uint8_t), + sizeof(uint8_t), + sizeof(uint8_t), + sizeof(int16_t), + sizeof(char)*3, + sizeof(uint8_t), + sizeof(int16_t), + sizeof(int16_t), + sizeof(int16_t), + sizeof(int16_t), + sizeof(int), + sizeof(float), + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float), + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(int)*5, + sizeof(int)*5, + sizeof(int)*5, + sizeof(float)*5, + sizeof(int)*5, + sizeof(float)*75, + sizeof(float)*75, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(int)*5, + sizeof(float)*5, + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(double), + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(int), + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(float)*40, + sizeof(float)*40, + sizeof(int)*5, + sizeof(int)*5, + sizeof(float)*5, + sizeof(float)*5, + sizeof(double)*5, + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(int), + sizeof(float), + sizeof(int16_t) +}; +size_t matchpos_offsets[NFIELDS_MATCHPOS] ={ + HOFFSET ( MATCHPOS,OBJID ), + HOFFSET ( MATCHPOS,PARENTID ), + HOFFSET ( MATCHPOS,FIELDID ), + HOFFSET ( MATCHPOS,SKYVERSION ), + HOFFSET ( MATCHPOS,MODE ), + HOFFSET ( MATCHPOS,CLEAN ), + HOFFSET ( MATCHPOS,RUN ), + HOFFSET ( MATCHPOS,RERUN ), + HOFFSET ( MATCHPOS,CAMCOL ), + HOFFSET ( MATCHPOS,FIELD ), + HOFFSET ( MATCHPOS,ID ), + HOFFSET ( MATCHPOS,PARENT ), + HOFFSET ( MATCHPOS,NCHILD ), + HOFFSET ( MATCHPOS,OBJC_TYPE ), + HOFFSET ( MATCHPOS,OBJC_PROB_PSF ), + HOFFSET ( MATCHPOS,OBJC_FLAGS ), + HOFFSET ( MATCHPOS,OBJC_FLAGS2 ), + HOFFSET ( MATCHPOS,OBJC_ROWC ), + HOFFSET ( MATCHPOS,OBJC_ROWCERR ), + HOFFSET ( MATCHPOS,OBJC_COLC ), + HOFFSET ( MATCHPOS,OBJC_COLCERR ), + HOFFSET ( MATCHPOS,ROWVDEG ), + HOFFSET ( MATCHPOS,ROWVDEGERR ), + HOFFSET ( MATCHPOS,COLVDEG ), + HOFFSET ( MATCHPOS,COLVDEGERR ), + HOFFSET ( MATCHPOS,ROWC ), + HOFFSET ( MATCHPOS,ROWCERR ), + HOFFSET ( MATCHPOS,COLC ), + HOFFSET ( MATCHPOS,COLCERR ), + HOFFSET ( MATCHPOS,PETROTHETA ), + HOFFSET ( MATCHPOS,PETROTHETAERR ), + HOFFSET ( MATCHPOS,PETROTH50 ), + HOFFSET ( MATCHPOS,PETROTH50ERR ), + HOFFSET ( MATCHPOS,PETROTH90 ), + HOFFSET ( MATCHPOS,PETROTH90ERR ), + HOFFSET ( MATCHPOS,Q ), + HOFFSET ( MATCHPOS,QERR ), + HOFFSET ( MATCHPOS,U ), + HOFFSET ( MATCHPOS,UERR ), + HOFFSET ( MATCHPOS,M_E1 ), + HOFFSET ( MATCHPOS,M_E2 ), + HOFFSET ( MATCHPOS,M_E1E1ERR ), + HOFFSET ( MATCHPOS,M_E1E2ERR ), + HOFFSET ( MATCHPOS,M_E2E2ERR ), + HOFFSET ( MATCHPOS,M_RR_CC ), + HOFFSET ( MATCHPOS,M_RR_CCERR ), + HOFFSET ( MATCHPOS,M_CR4 ), + HOFFSET ( MATCHPOS,M_E1_PSF ), + HOFFSET ( MATCHPOS,M_E2_PSF ), + HOFFSET ( MATCHPOS,M_RR_CC_PSF ), + HOFFSET ( MATCHPOS,M_CR4_PSF ), + HOFFSET ( MATCHPOS,THETA_DEV ), + HOFFSET ( MATCHPOS,THETA_DEVERR ), + HOFFSET ( MATCHPOS,AB_DEV ), + HOFFSET ( MATCHPOS,AB_DEVERR ), + HOFFSET ( MATCHPOS,THETA_EXP ), + HOFFSET ( MATCHPOS,THETA_EXPERR ), + HOFFSET ( MATCHPOS,AB_EXP ), + HOFFSET ( MATCHPOS,AB_EXPERR ), + HOFFSET ( MATCHPOS,FRACDEV ), + HOFFSET ( MATCHPOS,FLAGS ), + HOFFSET ( MATCHPOS,FLAGS2 ), + HOFFSET ( MATCHPOS,TYPE ), + HOFFSET ( MATCHPOS,PROB_PSF ), + HOFFSET ( MATCHPOS,NPROF ), + HOFFSET ( MATCHPOS,PROFMEAN_NMGY ), + HOFFSET ( MATCHPOS,PROFERR_NMGY ), + HOFFSET ( MATCHPOS,STAR_LNL ), + HOFFSET ( MATCHPOS,EXP_LNL ), + HOFFSET ( MATCHPOS,DEV_LNL ), + HOFFSET ( MATCHPOS,PSP_STATUS ), + HOFFSET ( MATCHPOS,PIXSCALE ), + HOFFSET ( MATCHPOS,RA ), + HOFFSET ( MATCHPOS,DEC ), + HOFFSET ( MATCHPOS,CX ), + HOFFSET ( MATCHPOS,CY ), + HOFFSET ( MATCHPOS,CZ ), + HOFFSET ( MATCHPOS,RAERR ), + HOFFSET ( MATCHPOS,DECERR ), + HOFFSET ( MATCHPOS,L ), + HOFFSET ( MATCHPOS,B ), + HOFFSET ( MATCHPOS,OFFSETRA ), + HOFFSET ( MATCHPOS,OFFSETDEC ), + HOFFSET ( MATCHPOS,PSF_FWHM ), + HOFFSET ( MATCHPOS,MJD ), + HOFFSET ( MATCHPOS,AIRMASS ), + HOFFSET ( MATCHPOS,PHI_OFFSET ), + HOFFSET ( MATCHPOS,PHI_DEV_DEG ), + HOFFSET ( MATCHPOS,PHI_EXP_DEG ), + HOFFSET ( MATCHPOS,EXTINCTION ), + HOFFSET ( MATCHPOS,SKYFLUX ), + HOFFSET ( MATCHPOS,SKYFLUX_IVAR ), + HOFFSET ( MATCHPOS,PSFFLUX ), + HOFFSET ( MATCHPOS,PSFFLUX_IVAR ), + HOFFSET ( MATCHPOS,PSFMAG ), + HOFFSET ( MATCHPOS,PSFMAGERR ), + HOFFSET ( MATCHPOS,FIBERFLUX ), + HOFFSET ( MATCHPOS,FIBERFLUX_IVAR ), + HOFFSET ( MATCHPOS,FIBERMAG ), + HOFFSET ( MATCHPOS,FIBERMAGERR ), + HOFFSET ( MATCHPOS,FIBER2FLUX ), + HOFFSET ( MATCHPOS,FIBER2FLUX_IVAR ), + HOFFSET ( MATCHPOS,FIBER2MAG ), + HOFFSET ( MATCHPOS,FIBER2MAGERR ), + HOFFSET ( MATCHPOS,CMODELFLUX ), + HOFFSET ( MATCHPOS,CMODELFLUX_IVAR ), + HOFFSET ( MATCHPOS,CMODELMAG ), + HOFFSET ( MATCHPOS,CMODELMAGERR ), + HOFFSET ( MATCHPOS,MODELFLUX ), + HOFFSET ( MATCHPOS,MODELFLUX_IVAR ), + HOFFSET ( MATCHPOS,MODELMAG ), + HOFFSET ( MATCHPOS,MODELMAGERR ), + HOFFSET ( MATCHPOS,PETROFLUX ), + HOFFSET ( MATCHPOS,PETROFLUX_IVAR ), + HOFFSET ( MATCHPOS,PETROMAG ), + HOFFSET ( MATCHPOS,PETROMAGERR ), + HOFFSET ( MATCHPOS,DEVFLUX ), + HOFFSET ( MATCHPOS,DEVFLUX_IVAR ), + HOFFSET ( MATCHPOS,DEVMAG ), + HOFFSET ( MATCHPOS,DEVMAGERR ), + HOFFSET ( MATCHPOS,EXPFLUX ), + HOFFSET ( MATCHPOS,EXPFLUX_IVAR ), + HOFFSET ( MATCHPOS,EXPMAG ), + HOFFSET ( MATCHPOS,EXPMAGERR ), + HOFFSET ( MATCHPOS,APERFLUX ), + HOFFSET ( MATCHPOS,APERFLUX_IVAR ), + HOFFSET ( MATCHPOS,CLOUDCAM ), + HOFFSET ( MATCHPOS,CALIB_STATUS ), + HOFFSET ( MATCHPOS,NMGYPERCOUNT ), + HOFFSET ( MATCHPOS,NMGYPERCOUNT_IVAR ), + HOFFSET ( MATCHPOS,TAI ), + HOFFSET ( MATCHPOS,RESOLVE_STATUS ), + HOFFSET ( MATCHPOS,THING_ID ), + HOFFSET ( MATCHPOS,IFIELD ), + HOFFSET ( MATCHPOS,BALKAN_ID ), + HOFFSET ( MATCHPOS,NOBSERVE ), + HOFFSET ( MATCHPOS,NDETECT ), + HOFFSET ( MATCHPOS,NEDGE ), + HOFFSET ( MATCHPOS,SCORE ), + HOFFSET ( MATCHPOS,FIBERID ) +}; +size_t * catalog_field_sizes[]={ + plugmap_sizes, + zbest_sizes, + zline_sizes, + match_sizes, + matchflux_sizes, + matchpos_sizes +}; +size_t * catalog_field_offsets []={ + plugmap_offsets, + zbest_offsets, + zline_offsets, + match_offsets, + matchflux_offsets, + matchpos_offsets +}; +void print_record_cad(hsize_t nrecords,COADD * data){ + hsize_t i; + for (i=0;i=0) H5Gclose(igroup); + //if(ifile>=0) H5Fclose(ifile); +} + +void compound_read_catalog(struct Catalog * cl, hid_t dst_file, int it, int write,int rank){ + hid_t ifile=-1; + hid_t igroup=-1; + hid_t fapl_id=-1; + herr_t ierr=-1; + hsize_t nfields, nrecords; + size_t * field_sizes; + size_t * field_offsets; + size_t record_size=0; + char photo_gp[20]; + strncpy(photo_gp,cl->plate_mjd[it],sizeof(cl->plate_mjd[it])-1); + photo_gp[sizeof(cl->plate_mjd[it]-1)]='\0'; + strcat(photo_gp,photo); + //open the source file by rank + ifile= H5Fopen(cl->filepath[it],H5F_ACC_RDONLY,H5P_DEFAULT); + if(ifile<0) printf("srcfile '%s' open error in rank %d\n",cl->filepath[it],rank); + /* Groups Table_Name Index in catalog_array + plate/mjd plugmap 0 + plate/mjd zbest 1 + plate/mjd zline 2 + plate/mjd/photo match 3 + plate/mjd/photo matchflux 4 + plate/mjd/photo matchpos 5 + */ + + int icat; + void * data_catalog=NULL; // buffer for storing the catalog records + hsize_t * data_field=NULL; // buffer for storing the catalog fiberid field + + //loop through the 6 catalog tables + for (icat=0;icatplate_mjd[it],H5P_DEFAULT); + } + else{ //tables: match, matchflux,matchpos + igroup = H5Gopen(ifile,photo_gp,H5P_DEFAULT); + } + if(igroup<0) printf("group '%s' or '%s' open error rank: %d\n",cl->plate_mjd[it], photo_gp,rank); + + herr_t ierr=H5TBget_table_info (igroup, catalog_array[icat], &nfields, &nrecords); + if(ierr<0) printf("ierr of get table info:%d in rank %d\n",ierr,rank); + data_field=malloc(sizeof(hsize_t)*nrecords); + herr_t ifield_err=H5TBread_fields_name( igroup, catalog_array[icat], FIBERID, 0, + nrecords, record_size, field_offsets, field_sizes, data_field); + //search the data_field to get the local offsets list + int len_offset=0; + hsize_t * offsetlist=malloc(sizeof(hsize_t )* nrecords);//nrecords will be larger or equal than len_offset; + get_catalog_offset(offsetlist, &len_offset, cl->fiberid[it], nrecords, data_field); + //compare len_offset with (cl->fiber_llength[it]) + assert (len_offset==(cl->fiber_llength[it])); + int ifd; + data_catalog=malloc(record_size*len_offset); + for(ifd=0;ifdplate_mjd[it],catalog_array[icat], + len_offset,cl->fiber_gstart[it],record_size, field_offsets, field_sizes,data_catalog); + } + } + + if(data_catalog!=NULL) free(data_catalog); + if(data_field!=NULL) free(data_field); + H5Gclose(igroup); + H5Fclose(ifile); +} + +//void compound_write(char * dst_file,char * grp, const char * table_name, hsize_t nrecords,size_t type_size, const size_t * field_offsets, const size_t * field_sizes,void * data){ +void compound_write_catalog(hid_t dst_file,char * grp, const char * table_name, hsize_t nrecords, + hsize_t start, size_t type_size, const size_t * field_offsets, const size_t * field_sizes,void * data){ + herr_t ierr=-1; + hid_t igroup=-1; + hid_t ifile=-1; + //ifile= H5Fopen(dst_file,H5F_ACC_RDWR,H5P_DEFAULT); + //if(ifile<0) printf("file open error:%s",dst_file); + ifile=dst_file; + igroup= H5Gopen(ifile,grp,H5P_DEFAULT); + ierr=H5TBwrite_records ( igroup, table_name, start, nrecords, type_size, field_offsets, field_sizes, data); + if(ierr<0) printf("write records error\n"); + if(igroup>=0) H5Gclose(igroup); + //if(ifile>=0) H5Fclose(ifile); +} +void get_catalog_offset(hsize_t * offsetlist, hsize_t * len_offset, hsize_t fiberid, hsize_t nrecords, hsize_t * data_field){ + int id=0; + int jd=0; + len_offset=0; + if(offsetlist==NULL || data_field==NULL) { + printf("offsetlist or data_field is NULL\n"); + exit(EXIT_FAILURE); + } + for (id=0;id +#include +#include +typedef struct COADD +{ + float WAVE; + float FLUX; + float IVAR; + int AND_MASK; + int OR_MASK; + float WAVEDISP; + float SKY; + float MODEL; +}COADD; + +typedef struct EXPOSURE +{ + float WAVE; + float FLUX; + float IVAR; + int MASK; + float WAVEDISP; + float SKY; + float X; + float CALIB; +}EXPOSURE; +typedef struct ZLINE{ + int PLATE; + int MJD; + int FIBERID; + char LINENAME[13]; + double LINEWAVE; + float LINEZ; + float LINEZ_ERR; + float LINESIGMA; + float LINESIGMA_ERR; + float LINEAREA; + float LINEAREA_ERR; + float LINEEW; + float LINEEW_ERR; + float LINECONTLEVEL; + float LINECONTLEVEL_ERR; + int LINENPIXLEFT; + int LINENPIXRIGHT; + float LINEDOF; + float LINECHI2; +}ZLINE; +typedef struct MATCH{ + char FLUX_OBJID[19]; + int RUN; + int CAMCOL; + int FIELD; + int FLUX_ID; + int THING_ID; + char RERUN[3]; + int NOBSERVE; + int NDETECT; + double FLUX_RA; + double FLUX_DEC; + double MATCH_RA; + double MATCH_DEC; + char ORIG_OBJID[19]; + int ORIG_ID; + double ORIG_RA; + double ORIG_DEC; + float APERFLUX3_R; + float FIBERFLUX_R; + float APERFLUX3_R_TOTAL; + float APERFLUX3_R_MATCH; + int FLUXMATCH_PARENT; + int FLUXMATCH_STATUS; + int FIBERID; +}MATCH; +typedef struct MATCHFLUX{ + char OBJID[19]; + char PARENTID[19]; + char FIELDID[19]; + unsigned SKYVERSION; + unsigned MODE; + unsigned CLEAN; + int RUN; + char RERUN[3]; + unsigned CAMCOL; + int FIELD; + int ID; + int PARENT; + int NCHILD; + int OBJC_TYPE; + float OBJC_PROB_PSF; + int OBJC_FLAGS; + int OBJC_FLAGS2; + float OBJC_ROWC; + float OBJC_ROWCERR; + float OBJC_COLC; + float OBJC_COLCERR; + float ROWVDEG; + float ROWVDEGERR; + float COLVDEG; + float COLVDEGERR; + float ROWC[5]; + float ROWCERR[5]; + float COLC[5]; + float COLCERR[5]; + float PETROTHETA[5]; + float PETROTHETAERR[5]; + float PETROTH50[5]; + float PETROTH50ERR[5]; + float PETROTH90[5]; + float PETROTH90ERR[5]; + float Q[5]; + float QERR[5]; + float U[5]; + float UERR[5]; + float M_E1[5]; + float M_E2[5]; + float M_E1E1ERR[5]; + float M_E1E2ERR[5]; + float M_E2E2ERR[5]; + float M_RR_CC[5]; + float M_RR_CCERR[5]; + float M_CR4[5]; + float M_E1_PSF[5]; + float M_E2_PSF[5]; + float M_RR_CC_PSF[5]; + float M_CR4_PSF[5]; + float THETA_DEV[5]; + float THETA_DEVERR[5]; + float AB_DEV[5]; + float AB_DEVERR[5]; + float THETA_EXP[5]; + float THETA_EXPERR[5]; + float AB_EXP[5]; + float AB_EXPERR[5]; + float FRACDEV[5]; + int FLAGS[5]; + int FLAGS2[5]; + int TYPE[5]; + float PROB_PSF[5]; + int NPROF[5]; + float PROFMEAN_NMGY[75]; + float PROFERR_NMGY[75]; + float STAR_LNL[5]; + float EXP_LNL[5]; + float DEV_LNL[5]; + int PSP_STATUS[5]; + float PIXSCALE[5]; + double RA; + double DEC; + double CX; + double CY; + double CZ; + double RAERR; + double DECERR; + double L; + double B; + float OFFSETRA[5]; + float OFFSETDEC[5]; + float PSF_FWHM[5]; + int MJD; + float AIRMASS[5]; + float PHI_OFFSET[5]; + float PHI_DEV_DEG[5]; + float PHI_EXP_DEG[5]; + float EXTINCTION[5]; + float SKYFLUX[5]; + float SKYFLUX_IVAR[5]; + float PSFFLUX[5]; + float PSFFLUX_IVAR[5]; + float PSFMAG[5]; + float PSFMAGERR[5]; + float FIBERFLUX[5]; + float FIBERFLUX_IVAR[5]; + float FIBERMAG[5]; + float FIBERMAGERR[5]; + float FIBER2FLUX[5]; + float FIBER2FLUX_IVAR[5]; + float FIBER2MAG[5]; + float FIBER2MAGERR[5]; + float CMODELFLUX[5]; + float CMODELFLUX_IVAR[5]; + float CMODELMAG[5]; + float CMODELMAGERR[5]; + float MODELFLUX[5]; + float MODELFLUX_IVAR[5]; + float MODELMAG[5]; + float MODELMAGERR[5]; + float PETROFLUX[5]; + float PETROFLUX_IVAR[5]; + float PETROMAG[5]; + float PETROMAGERR[5]; + float DEVFLUX[5]; + float DEVFLUX_IVAR[5]; + float DEVMAG[5]; + float DEVMAGERR[5]; + float EXPFLUX[5]; + float EXPFLUX_IVAR[5]; + float EXPMAG[5]; + float EXPMAGERR[5]; + float APERFLUX[40]; + float APERFLUX_IVAR[40]; + int CLOUDCAM[5]; + int CALIB_STATUS[5]; + float NMGYPERCOUNT[5]; + float NMGYPERCOUNT_IVAR[5]; + double TAI[5]; + int RESOLVE_STATUS; + int THING_ID; + int IFIELD; + int BALKAN_ID; + int NOBSERVE; + int NDETECT; + int NEDGE; + float SCORE; + int FIBERID; +}MATCHFLUX; +typedef struct MATCHPOS{ + char OBJID[19]; + char PARENTID[19]; + char FIELDID[19]; + unsigned SKYVERSION; + unsigned MODE; + unsigned CLEAN; + int RUN; + char RERUN[3]; + unsigned CAMCOL; + int FIELD; + int ID; + int PARENT; + int NCHILD; + int OBJC_TYPE; + float OBJC_PROB_PSF; + int OBJC_FLAGS; + int OBJC_FLAGS2; + float OBJC_ROWC; + float OBJC_ROWCERR; + float OBJC_COLC; + float OBJC_COLCERR; + float ROWVDEG; + float ROWVDEGERR; + float COLVDEG; + float COLVDEGERR; + float ROWC[5]; + float ROWCERR[5]; + float COLC[5]; + float COLCERR[5]; + float PETROTHETA[5]; + float PETROTHETAERR[5]; + float PETROTH50[5]; + float PETROTH50ERR[5]; + float PETROTH90[5]; + float PETROTH90ERR[5]; + float Q[5]; + float QERR[5]; + float U[5]; + float UERR[5]; + float M_E1[5]; + float M_E2[5]; + float M_E1E1ERR[5]; + float M_E1E2ERR[5]; + float M_E2E2ERR[5]; + float M_RR_CC[5]; + float M_RR_CCERR[5]; + float M_CR4[5]; + float M_E1_PSF[5]; + float M_E2_PSF[5]; + float M_RR_CC_PSF[5]; + float M_CR4_PSF[5]; + float THETA_DEV[5]; + float THETA_DEVERR[5]; + float AB_DEV[5]; + float AB_DEVERR[5]; + float THETA_EXP[5]; + float THETA_EXPERR[5]; + float AB_EXP[5]; + float AB_EXPERR[5]; + float FRACDEV[5]; + int FLAGS[5]; + int FLAGS2[5]; + int TYPE[5]; + float PROB_PSF[5]; + int NPROF[5]; + float PROFMEAN_NMGY[75]; + float PROFERR_NMGY[75]; + float STAR_LNL[5]; + float EXP_LNL[5]; + float DEV_LNL[5]; + int PSP_STATUS[5]; + float PIXSCALE[5]; + double RA; + double DEC; + double CX; + double CY; + double CZ; + double RAERR; + double DECERR; + double L; + double B; + float OFFSETRA[5]; + float OFFSETDEC[5]; + float PSF_FWHM[5]; + int MJD; + float AIRMASS[5]; + float PHI_OFFSET[5]; + float PHI_DEV_DEG[5]; + float PHI_EXP_DEG[5]; + float EXTINCTION[5]; + float SKYFLUX[5]; + float SKYFLUX_IVAR[5]; + float PSFFLUX[5]; + float PSFFLUX_IVAR[5]; + float PSFMAG[5]; + float PSFMAGERR[5]; + float FIBERFLUX[5]; + float FIBERFLUX_IVAR[5]; + float FIBERMAG[5]; + float FIBERMAGERR[5]; + float FIBER2FLUX[5]; + float FIBER2FLUX_IVAR[5]; + float FIBER2MAG[5]; + float FIBER2MAGERR[5]; + float CMODELFLUX[5]; + float CMODELFLUX_IVAR[5]; + float CMODELMAG[5]; + float CMODELMAGERR[5]; + float MODELFLUX[5]; + float MODELFLUX_IVAR[5]; + float MODELMAG[5]; + float MODELMAGERR[5]; + float PETROFLUX[5]; + float PETROFLUX_IVAR[5]; + float PETROMAG[5]; + float PETROMAGERR[5]; + float DEVFLUX[5]; + float DEVFLUX_IVAR[5]; + float DEVMAG[5]; + float DEVMAGERR[5]; + float EXPFLUX[5]; + float EXPFLUX_IVAR[5]; + float EXPMAG[5]; + float EXPMAGERR[5]; + float APERFLUX[40]; + float APERFLUX_IVAR[40]; + int CLOUDCAM[5]; + int CALIB_STATUS[5]; + float NMGYPERCOUNT[5]; + float NMGYPERCOUNT_IVAR[5]; + double TAI[5]; + int RESOLVE_STATUS; + int THING_ID; + int IFIELD; + int BALKAN_ID; + int NOBSERVE; + int NDETECT; + int NEDGE; + float SCORE; + int FIBERID; + +}MATCHPOS; +typedef struct ZBEST{ + int PLATE; + int TILE; + int MJD; + int FIBERID; + char RUN2D[6]; + char RUN1D[6]; + int OBJID[5]; + char OBJTYPE[16]; + double PLUG_RA; + double PLUG_DEC; + char CLASS[6]; + char SUBCLASS[21]; + float Z; + float Z_ERR; + float RCHI2; + int DOF; + float RCHI2DIFF; + char TFILE[24]; + int TCOLUMN[10]; + int NPOLY; + float THETA[10]; + float THETA_COVAR[100]; + float VDISP; + float VDISP_ERR; + float VDISPZ; + float VDISPZ_ERR; + float VDISPCHI2; + float VDISPNPIX; + int VDISPDOF; + float WAVEMIN; + float WAVEMAX; + float WCOVERAGE; + int ZWARNING; + float SN_MEDIAN[5]; + float SN_MEDIAN_ALL; + float CHI68P; + float FRACNSIGMA[10]; + float FRACNSIGHI[10]; + float FRACNSIGLO[10]; + float SPECTROFLUX[5]; + float SPECTROFLUX_IVAR[5]; + float SPECTROSYNFLUX[5]; + float SPECTROSYNFLUX_IVAR[5]; + float SPECTROSKYFLUX[5]; + int ANYANDMASK; + int ANYORMASK; + float SPEC1_G; + float SPEC1_R; + float SPEC1_I; + float SPEC2_G; + float SPEC2_R; + float SPEC2_I; + char ELODIE_FILENAME[16]; + char ELODIE_OBJECT[8]; + char ELODIE_SPTYPE[12]; + float ELODIE_BV; + float ELODIE_TEFF; + float ELODIE_LOGG; + float ELODIE_FEH; + float ELODIE_Z; + float ELODIE_Z_ERR; + float ELODIE_Z_MODELERR; + float ELODIE_RCHI2; + int ELODIE_DOF; + float Z_NOQSO; + float Z_ERR_NOQSO; + int ZNUM_NOQSO; + int ZWARNING_NOQSO; + char CLASS_NOQSO[6]; + char SUBCLASS_NOQSO[19]; + float RCHI2DIFF_NOQSO; + float VDISP_LNL[35]; + long SPECOBJID; +}ZBEST; +typedef struct PLUGMAP{ + int OBJID[5]; + char HOLETYPE[6]; + double RA; + double DEC; + float MAG[5]; + float STARL; + float EXPL; + float DEVAUCL; + char OBJTYPE[16]; + double XFOCAL; + double YFOCAL; + int SPECTROGRAPHID; + int FIBERID; + int THROUGHPUT; + int PRIMTARGET; + int SECTARGET; + int OFFSETID; + float SCI_EXPTIME; + char SOURCETYPE[15]; + float LAMBDA_EFF; + float ZOFFSET; + int BLUEFIBER; + long BOSS_TARGET1; + long BOSS_TARGET2; + long ANCILLARY_TARGET1; + long ANCILLARY_TARGET2; + int RUN; + char RERUN[4]; + int CAMCOL; + int FIELD; + int ID; + float CALIBFLUX[5]; + float CALIBFLUX_IVAR[5]; + int CALIB_STATUS[5]; + float SFD_EBV; +}PLUGMAP; +//typedef enum { false; true } bool; +void compound_read(char * src_file, hid_t dst_file, char * path_table, int write,int rank); +void compound_write(hid_t dst_file,char * grp, const char * table_name, hsize_t nrecords, size_t type_size, const size_t * field_offsets, const size_t * field_sizes, void * data); +//void compound_read(char * src_file;char * dst_file; char * path_table; bool write); +//void compound_write(char * dst_file;char * grp; const char * table_name; hsize_t nrecords;size_t type_size; const size_t * field_offsets; const size_t * field_sizes;void * data); +void print_record_cad(hsize_t nrecords, COADD * data); +void print_record_exp(hsize_t nrecords, EXPOSURE * data); +#endif diff --git a/h5boss_c/h5read.c b/h5boss_c/h5read.c new file mode 100755 index 0000000..92ed62e --- /dev/null +++ b/h5boss_c/h5read.c @@ -0,0 +1,175 @@ +#include "stdlib.h" +#include "hdf5.h" +#include "getopt.h" +#include + + +#define NAME_MAX 255 +char filename[NAME_MAX]; +char DATASETNAME[NAME_MAX]; +char cb_buffer_size[NAME_MAX]; +char cb_nodes[NAME_MAX]; +int NDIMS=2; +int main(int argc, char **argv){ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info; + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + int c; + opterr = 0; + strncpy(filename, "./fake_xyz_default.h5", NAME_MAX); + //strncpy(cb_buffer_size,"16777216", NAME_MAX); + //strncpy(cb_nodes, "16", NAME_MAX); + int col=1;//collective write + //input args: f: inputfilename, b: collective_buffersize, n: collective_buffernodes, k:iscollective, v:datasetname + while ((c = getopt (argc, argv, "f:b:c:k:v:")) != -1) + switch (c) + { + case 'f': + strncpy(filename, optarg, NAME_MAX); + break; + case 'b': + strncpy(cb_buffer_size,optarg, NAME_MAX); + break; + case 'c': + strncpy(cb_nodes, optarg, NAME_MAX); + break; + case 'k': + col = strtol(optarg, NULL, 10); + case 'v': + strncpy(DATASETNAME, optarg, NAME_MAX); + default: + break; + } + + MPI_Info_create(&info); + //MPI_Info_set(info, "cb_buffer_size", cb_buffer_size); + //MPI_Info_set(info, "cb_nodes", cb_nodes); + + + //Open file/dataset + hid_t fapl,file,dataset; + fapl = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl, comm, info); + file= H5Fopen(filename, H5F_ACC_RDONLY, fapl); + H5Pclose(fapl); + if(mpi_rank==0) { + if(file<0){ + printf("File %s open error\n",filename); + return 0; + } + else { + printf("File %s open ok\n",filename); + } + } + dataset= H5Dopen(file, DATASETNAME,H5P_DEFAULT); + if(dataset <0 && mpi_rank==0) {printf("Data %s open error\n",DATASETNAME); return 0;} + + + hid_t dataspace; + size_t size; /* size of data*/ + int i, status_n,rank; + dataspace = H5Dget_space(dataset); /* dataspace handle */ + rank = H5Sget_simple_extent_ndims(dataspace); + hsize_t dims_out[rank]; + status_n = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); + if(mpi_rank==0){ + for(i=0;i +#include +#include +#include +#include +int main(int argc, char ** argv){ + if(argc!=4) { + printf("usage: %s csvfile output 1:readwrite(0:readonly)\n",argv[0]); + exit(EXIT_FAILURE); + } + int j; + time_t begin,end,final; + const char sep=':'; + begin = time(NULL); + bool write=false; + long x = strtol(argv[3], NULL, 10); + if(x==1) write=true; + struct Fiber * dl=dataset_list(argv[1],sep); + end= time(NULL); + for(j=0;jcount;j++){ + printf("%d: %s, %s\n",j,dl->keys[j],dl->values[j]); + compound_read(dl->values[j],argv[2], dl->keys[j], write); + } + final=time(NULL); + printf("parse csv:%.2f, read/write:%.2f\n",difftime(end,begin),difftime(final,end)); + if(dl!=NULL) free(dl); + return 0; +} diff --git a/h5boss_c/mt.c b/h5boss_c/mt.c new file mode 100755 index 0000000..48b771a --- /dev/null +++ b/h5boss_c/mt.c @@ -0,0 +1,101 @@ +/* +*HDF ETL Tooset +*This file is a HDF5 matcher +*Date: Mar 14 2016 +*Author: +*Jialin Liu, jalnliu@lbl.gov +*Input: a list of plate/mjd/fiber +*Output: One hdf5 file +*HDF5 version: cray-hdf5/1.8.16 +*/ + +#include +#include +#include +#include +#include +#include "getopt.h" +int file_exist (char *filename) +{ + struct stat buffer; + return (stat (filename, &buffer) == 0); +} + +//void matching ( int * mt, char * ipmf, char * ihdf){ + +//} +#define NAME_MAX 255 +#define BUF 20 + +int count(char * in){ + FILE * file=fopen(in,"r"); + int lines = 0; + int c; + int last = '\n'; + while (EOF != (c = fgetc(file))) { + if (c == '\n' && last != '\n') { + ++lines; + } + last = c; + } + rewind(file); + fclose(file); + return lines; +} +void toarray(char ** line, char * file){ + FILE * plist=NULL; + int i=0; + plist = fopen(file,"r"); + while(fgets(line[i],BUF, plist)){ + line[i][strlen(line[i]-1)]='\0'; + i++; + } + rewind(plist); + fclose(plist); +} + +char ipmf[NAME_MAX]; +char ihdf [NAME_MAX]; +char ohdf [NAME_MAX]; + +int main(int argc, char ** argv) +{ + int c=0; + strncpy(ipmf, "./pmf.txt",NAME_MAX); + strncpy(ihdf,"./hdf.txt",NAME_MAX); + strncpy(ohdf, "./o.h5", NAME_MAX); + while ((c = getopt (argc, argv, "i:j:o:")) != -1) + switch (c) + { + case 'i': + strncpy(ipmf, optarg, NAME_MAX); + break; + case 'j': + strncpy(ihdf, optarg,NAME_MAX); + break; + case 'o': + strncpy(ohdf, optarg, NAME_MAX); + default: + break; + } + + if(argc<3) {printf("input args\n"); return 0;} + if(!file_exist(ipmf)||!file_exist(ihdf)) {printf("inputs not exist\n");return 0;} + //toarray(ipmflist,ipmf); + int line_ipmf=count(ipmf); + printf("lines in %s is %d\n",ipmf,line_ipmf); + char ** ipmflist=(char **)malloc(sizeof(char *)*line_ipmf); + toarray(ipmflist,ipmf); + + int line_ihdf=count(ihdf); + printf("lines in %s is %d\n",ihdf,line_ihdf); + char ** ihdflist=(char **)malloc(sizeof(char *)*line_ihdf); + + int * mt=(int *) malloc(line_ipmf*sizeof(int)); + //match(mt, line_ipmf,line_ihdf); + if(ipmflist) free(ipmflist); + if(ihdflist) free(ihdflist); + if(mt) free(mt); + + return 0; +} diff --git a/h5boss_c/parse_node.c b/h5boss_c/parse_node.c new file mode 100755 index 0000000..2c49534 --- /dev/null +++ b/h5boss_c/parse_node.c @@ -0,0 +1,225 @@ +#include "parse_node.h" +#include +#include +#include +#include +#include +//structure for maintaining key value pair +int bufi=0; +//split a string by delimiter, return a char ** +char** str_split(char* a_str, const char a_delim) +{ + char** result = NULL; + size_t count = 0; + char* tmp = a_str; + char* last_comma = NULL; + char delim[2]; + delim[0] = a_delim; + delim[1] = 0; + /* Count how many elements will be extracted. */ + while (*tmp) + { + if (a_delim == *tmp) + { + count++; + last_comma = tmp; + } + tmp++; + } + /* Add space for trailing token. */ + count += last_comma < (a_str + strlen(a_str) - 1); + /* Add space for terminating null string so caller + knows where the list of returned strings ends. */ + count++; + result = malloc(sizeof(char*) * count); + if (result) + { + size_t idx = 0; + char* token = strtok(a_str, delim); + while (token) + { + assert(idx < count); + *(result + idx++) = strdup(token); + token = strtok(0, delim); + } + assert(idx == count - 1); + *(result + idx) = 0; + } + return result; +} +char ** path_split(char* path) { + char **token=(char **)malloc(2*sizeof(char *)); + char * dict=strdup(path); + char * basec=strdup(path); + char * d=strdup(dirname(dict)); + char * b=strdup(basename(basec)); + token[0]=d; + token[1]=b; + if(dict!=NULL) free(dict); + if(basec!=NULL) free(basec); + return token; +} +char ** parse_nodes(char * file,int numline){ + char ** buf=NULL; + buf=(char **)malloc(sizeof(char *)*(numline+1)); + if(buf==NULL){ + printf("buf allocate error\n"); + exit(0); + } + FILE * fp; + char * line = (char *)malloc(sizeof(char)*200); + if(line==NULL){ + printf("line allocate error\n"); + exit(0); + } + size_t len = 0; + size_t read; + fp = fopen(file,"r"); + if (fp == NULL) + exit(EXIT_FAILURE); + while ((read = getline(&line,&len,fp))!= -1 ){ + //printf("line of length %zu:\n",read); + //printf("%s",line); + if('\n'!=line[0]){ + buf[bufi] = (char *)malloc(strlen(line)+1); + line[strcspn(line,"\n")]=0; + strcpy(buf[bufi],line); + bufi++; + } + } + fclose(fp); + if (line!=NULL) + free(line); + return buf; +} +struct Fiber * dataset_list (char * file,const char sep,int numline){ + struct Fiber * dl= malloc(sizeof(struct Fiber)); + + char ** lines=NULL; + lines=parse_nodes(file,numline); + if(lines==NULL){ + printf("lines parsing error\n"); + exit(0); + } + char ** dl_keys=NULL; + char ** dl_values=NULL; + dl_keys=(char **)malloc(sizeof(char *)*numline); + if (dl_keys==NULL) { + printf("dl_keys allocation error\n"); + exit(0); + } + dl_values=(char **)malloc(sizeof(char *)*numline); + if (dl_values==NULL) { + printf("dl_values allocation error\n"); + exit(0); + } + dl->count=bufi; + int i; + for (i=0;ikeys=dl_keys; + dl->values=dl_values; + bufi=0; + if(lines!=NULL) free(lines); + return dl; +} +struct Catalog * catalog_list (char * file,const char sep,int numline){ + struct Catalog * dl= malloc(sizeof(struct Catalog)); + if(dl==NULL){ + printf("catalog dl mem err\n"); + exit(0); + } + char ** lines=parse_nodes(file,numline); + if(lines==NULL){ + printf("lines parsing error\n"); + exit(0); + } + char ** dl_plate_mjd=NULL; + hsize_t * dl_fiber_id=NULL; + char ** dl_filepath=NULL; + hsize_t * dl_fiber_offset=NULL; + hsize_t * dl_fiber_local_length=NULL; + dl_plate_mjd=(char **)malloc(sizeof(char *)*(numline+1)); + if (dl_plate_mjd==NULL) { + printf("dl_platemjd allocation error\n"); + exit(0); + } + dl_fiber_id=(hsize_t *)malloc(sizeof(hsize_t)*(numline+1)); + if (dl_fiber_id==NULL) { + printf("dl_fiber_id allocation error\n"); + exit(0); + } + dl_filepath=(char **)malloc(sizeof(char *)*(numline+1)); + if (dl_filepath==NULL) { + printf("dl_filepath allocation error\n"); + exit(0); + } + dl_fiber_offset=(hsize_t *)malloc(sizeof(hsize_t)*(numline+1)); + if (dl_fiber_offset==NULL) { + printf("dl_fiber_offset allocation error\n"); + exit(0); + } + dl_fiber_local_length=(hsize_t *)malloc(sizeof(hsize_t)*(numline+1)); + if (dl_fiber_local_length==NULL) { + printf("dl_fiber_local_length allocation error\n"); + exit(0); + } + dl->count=bufi; + int i; + for (i=0;iplate_mjd=dl_plate_mjd; + dl->fiberid=dl_fiber_id; + dl->filepath=dl_filepath; + dl->fiber_gstart=dl_fiber_offset; + dl->fiber_llength=dl_fiber_local_length; + bufi=0; + return dl; +} +/* +void main(int argc, char ** argv){ + if(argc!=2) { + printf("usage: %s filename\n",argv[0]); + exit(EXIT_FAILURE); + } + int j; + const char sep=':'; + struct Fiber * dl=dataset_list(argv[1],sep); + //parse_nodes(argv[1]); + for(j=0;jcount;j++){ + printf("parsed line %d: %s, %s",j,dl->keys[j],dl->values[j]); + } + exit(EXIT_SUCCESS); +} +*/ diff --git a/h5boss_c/parse_node.h b/h5boss_c/parse_node.h new file mode 100755 index 0000000..1e469c5 --- /dev/null +++ b/h5boss_c/parse_node.h @@ -0,0 +1,25 @@ +#ifndef PARSE_NODES_H_ +#define PARSE_NODES_H_ +#include +struct Fiber{ + int count; // number of counts, e.g., 188000 + char ** keys; // a list of fiber dataset name, e.g., 5732/56326/936/coadd + char ** values;// a list of file path, /global/cscratch1/sd/jialin/h5boss/5732-56326.hdf5 +}; +struct Catalog{ + int count; // number of catalog rows, e.g., 19800 + char ** plate_mjd;// a list of plate/mjd pair, e.g., 3690/55182 + hsize_t * fiberid; // a list of fiber original offset, e.g., 0 + char ** filepath; // a list of file path, e.g., global/cscratch1/sd/jialin/h5boss/3690-55182.hdf5 + hsize_t * fiber_gstart; // a list of fiber new offsets, e.g., 1 + hsize_t * fiber_llength; +}; +//split a string by delimiter, return a char ** +char ** str_split(char* a_str, const char a_delim); +char ** parse_nodes(char * file,int numline); +struct Fiber * dataset_list (char * file,const char sep,int numline); +struct Catalog * catalog_list (char * file, const char sep, int numline); +char ** path_split(char* path); + + +#endif diff --git a/h5boss_c/strstr.c b/h5boss_c/strstr.c new file mode 100755 index 0000000..f87dbfe --- /dev/null +++ b/h5boss_c/strstr.c @@ -0,0 +1,11 @@ +#include +#include +#include +void main(){ +char *path ="ab/cde/fg.out"; +printf("orig:%s\n",path); +char * pp=strdup(path); +char * ff=strdup(path); +printf("path:%s\n",dirname(pp)); +printf("bname:%s\n",basename(ff)); +} diff --git a/h5boss_c/submit.sh b/h5boss_c/submit.sh new file mode 100755 index 0000000..adb782e --- /dev/null +++ b/h5boss_c/submit.sh @@ -0,0 +1,8 @@ +#!/bin/bash +#SBATCH -p debug +#SBATCH -N 10 +#SBATCH -t 00:15:00 +#SBATCH -J ch5boss +#SBATCH -e %j.err +#SBATCH -o %j.out +srun -n 300 ./subset.exe -f /global/cscratch1/sd/jialin/bosslover/scaling-test/ost72/1k_sep9.1058am.h5 -m 2970602_nodes1k_fiber.txt -n 9619 -l 2970602_nodes1k_catalog.txt -k 981 diff --git a/h5boss_c/subselect.c b/h5boss_c/subselect.c new file mode 100755 index 0000000..6ddead4 --- /dev/null +++ b/h5boss_c/subselect.c @@ -0,0 +1,244 @@ +/* +*HDF ETL Tooset +*This file is a HDF5 subselecter +*Date: Mar 14 2016 +*Author: +*Jialin Liu, jalnliu@lbl.gov +*Input: a list of plate/mjd/fiber +*Output: One hdf5 file +*HDF5 version: cray-hdf5/1.8.16 +*/ +#include +#include +#include +#include +#include + +const char *pPath=NULL; +const char * ext=".h5"; +const char *DATASET=NULL; +char * concat_filepath(int id){ + char fid_str[5]; + sprintf(fid_str,"%d",id); + char * newfile; + newfile=malloc(strlen(pPath)+strlen(ext)+strlen(fid_str)+1); + strcpy(newfile,pPath); + strcat(newfile,fid_str); + strcat(newfile,ext); + return newfile; +} +int file_exist (char *filename) +{ + struct stat buffer; + return (stat (filename, &buffer) == 0); +} + +int [] matching (char * ipmf, char * ihdf){ + + +} +int count(char * in){ + File * input=fopen(in, "r"); + int ch, number_of_lines=0; + do + { + ch = fgetc(input); + if(ch == '\n') + number_of_lines++; + } while (ch != EOF); + + if(ch != '\n' && number_of_lines != 0) + number_of_lines++; + return number_of_lines; + input.close(); +} + +void toarray(char ** list, char * file){ + FILE * plist=NULL; + int i=0; + plist = fopen(file,"r"); + while(fgets(line[i],BUF, plist)){ + line[i][strlen(line[i]-1)]='\0'; + i++; + } +} +void print_usage(){ + + printf("\n\nWarning:Arguments Incomplete\n\n********Arguments List********\n"); + printf("* ./subselect\n*"); + printf("* output\t(e.g., o.h5)\n"); + printf("* pmf-list\t(e.g., pmf.txt contains a list of plate/mjd/fiber)\n"); + printf("* hdf-list\t(e.g., hdf.txt contains a list of hdf file list)\n"); + printf("******************************\n"); + printf("Sample Command: ./subselect pmf.txt hdf.txt $SCRATCH/o.h5\n\n\n"); + return 1; +} + +int main(int argc, char ** argv) +{ + int mpi_size,mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info; + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + if(argc<6){ + print_usage(); + return 1; + } + int c; + char * plate,mjd,fiber;//we will query all files based on plate/mjd/fiber infor, and then get all spectrum in one file. + char * ipmf = NULL; + char * ihdf = NULL; + char * ohdf = NULL; + while ((c = getopt (argc, argv, "i:j:o:")) != -1) + switch (c) + { + case 'i': + strncpy(ipmf, optarg, NAME_MAX); + break; + case 'j': + strncpy(ihdf, optarg,NAME_MAX); + break; + case 'o': + strncpy(ohdf, optarg, NAME_MAX); + default: + break; + } + if(!file_exist(ipmf)||!file_exist(ihdf)) return 0; + + int line_ipmf=count(ipmf); + char ** ipmflist=(char **)malloc(sizeof(char *)*line_ipmf) + toarray(ipmflist,ipmf); + + int line_ihdf=count(ihdf); + char ** ihdflist=(char **)malloc(sizeof(char *)*line_ihdf); + + int * mt=(int) malloc(line_ipmf*sizeof(int)); + match(mt, line_ipmf,line_ihdf); + // using one rank to read the input: ipmf, ihdf + if(mpi_rank==0){ + + + } + + hsize_t ndims,ncols,nrows,ex_nrows; + int total=atoi(argv[1]), startid=atoi(argv[2]); + char *saveFilePath=NULL; + saveFilePath=argv[3]; + pPath=argv[4]; + DATASET=argv[5]; + printf("Files: Combining %d Files(Extracting One Dataset: %s) from %s to %s\n\n",total,DATASET,pPath,saveFilePath); + ndims=2; + //create datasets + hid_t ex_file, ex_dataset,file_space,mem_space; + hid_t plist = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_layout(plist, H5D_CHUNKED); + hsize_t chunk_dims[2]; + chunk_dims[0]=1024; + chunk_dims[1]=11; + H5Pset_chunk(plist, ndims, chunk_dims); + hsize_t dims[ndims]; + dims[0]=0; + dims[1]=0; + hsize_t maxdims[2]; + maxdims[0] = H5S_UNLIMITED; + maxdims[1] = H5S_UNLIMITED; + file_space=H5Screate_simple(ndims,dims,maxdims); + ex_file=H5Fcreate(saveFilePath, H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT); + ex_dataset=H5Dcreate(ex_file,DATASET, H5T_NATIVE_FLOAT,file_space,H5P_DEFAULT,plist,H5P_DEFAULT); + mem_space = H5Screate_simple(ndims,dims,NULL); + hsize_t start[ndims],count[ndims]; + start[0]=dims[0]; + hid_t err_stack=0, nextFid,nextDid; + //float buffer[nrows][ncols]; + herr_t status; + int err_read=0,err_write=0,i=0,update_last=0; + hsize_t acdims[ndims]; + acdims[0]=dims[0]; + acdims[1]=dims[1]; + // disable error info + //H5Eset_auto(err_stack, NULL, NULL); + //retrieve new data dims, ndims + hsize_t new_ndims=0; + int nonpath=0; + int skip_read=0; + hid_t new_dataspace; + hsize_t dim_before=0; + int firstread=0; + for(i=0;i +#include +#include +#include +#define NAME_MAX 255 +char filename[NAME_MAX]; +char csvfile[NAME_MAX]; +char catalog_csvfile[NAME_MAX]; +int main(int argc, char **argv){ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info; + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + //printf("argc:%d\n",argc); + //if (argc != 7){ + if (mpi_rank==0) + printf("usage: %s -f output -m csv -l number_lines -c catalog -k number_catalog\n",argv[0]); + // printf("only got %d\n",argc); + // return 0; + //} + int c; + int numline; + int catalog_numline; + opterr = 0; + strncpy(filename, "fiber.h5", NAME_MAX); + strncpy(csvfile, "fiberlist.txt",NAME_MAX); + strncpy(catalog_csvfile, "cataloglist.txt",NAME_MAX); + /***input arguments****/ + //f: output, m:csv + while ((c = getopt (argc, argv, "f:m:l:n:k:")) != -1) + switch (c) + { + case 'f': + strncpy(filename, optarg, NAME_MAX); + break; + case 'm': + strncpy(csvfile, optarg, NAME_MAX); + break; + case 'l': + strncpy(catalog_csvfile, optarg, NAME_MAX); + break; + case 'n': + numline=strtol(optarg, NULL, 10); + break; + case 'k': + catalog_numline=strtol(optarg, NULL, 10); + break; + default: + break; + } + MPI_Info_create(&info); + //Open file/dataset + if (mpi_rank==0){ + printf("input:%s\n",filename); + printf("number of fiber lines:%d\n",numline); + + printf("fiber csv:%s\n",csvfile); + printf("number of catalog lines:%d\n",catalog_numline); + printf("catalog csv:%s\n",catalog_csvfile); + } + hid_t fapl,file; + file=-1; + fapl = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl, comm, info); +// hbool_t is_collective=true; +// H5Pset_all_coll_metadata_ops(fapl, is_collective); + file= H5Fopen(filename, H5F_ACC_RDWR, fapl); + //if(mpi_rank==0) printf("output file hanlde= %lld / %0llx\n", (long long)file, (unsigned long long)file); + if(file<0) printf("filename '%s' open error in rank %d\n",filename,mpi_rank); + //else printf("filename '%s' open correctly rank %d\n",filename, mpi_rank); +//H5Fclose(file); +//MPI_Finalize(); +//exit(0); + + //file = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); + + H5Pclose(fapl); + if(mpi_rank==0) { + if(file<0){ + printf("File %s open error\n",filename); + return 0; + } + else { + printf("File %s open ok\n",filename); + } + } + + const char sep=':'; + struct Fiber * dl_fiber=NULL; + struct Catalog *dl_catalog=NULL; + dl_fiber=dataset_list(csvfile,sep,numline); + if(dl_fiber==NULL) { + printf("dl_fiber memorry allocation error\n"); + exit(0); + + } + //dl_catalog=catalog_list(catalog_csvfile,sep,catalog_numline); + /*if(dl_catalog==NULL) { + printf("dl_catalog memorry allocation error, rank:%d\n",mpi_rank); + exit(0); + } + */ + int total_nodes=dl_fiber->count; + if(mpi_rank==0){ + printf("dl_fiber count:%d,pre-allocate:%d\n",dl_fiber->count,numline); + // printf("dl_catalog count:%d,pre-allocate:%d\n",dl_catalog->count,catalog_numline); + } + int step = total_nodes /mpi_size +1; + int rank_start=mpi_rank*step; + int rank_end=rank_start+step; + if(mpi_rank == mpi_size-1){ + rank_end=total_nodes; + if(rank_start>total_nodes){ + rank_start=total_nodes; + } + } + int i; + double t0=MPI_Wtime(); + //read fiber + for(i=rank_start;ivalues[i],file, dl_fiber->keys[i],1,mpi_rank); + //compound_read_fiber(dl_fiber,file,i,1,mpi_rank);//rewirte into this interface + } + + + //read catalog + /* for(i=rank_start;i +#include +#include +#define NAME_MAX 255 +char filename[NAME_MAX]; +char groupname[NAME_MAX]; +int main(int argc, char ** argv){ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info; + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + strncpy(filename, "/global/cscratch1/sd/jialin/h5boss/6054-56089.hdf5", NAME_MAX); + strncpy(groupname,"6054/56089/575/exposures/144631",NAME_MAX); + hid_t ifile= H5Fopen(filename,H5F_ACC_RDONLY,H5P_DEFAULT); + if(ifile<0) printf("file '%s' open error in rank %d\n",filename,mpi_rank); + herr_t igroup= H5Gopen(ifile,groupname,H5P_DEFAULT); + if(igroup<0) printf("group '%d' open error in rank %d\n",igroup,mpi_rank); + return 0; +} + diff --git a/LICENSE b/h5boss_py/LICENSE old mode 100644 new mode 100755 similarity index 100% rename from LICENSE rename to h5boss_py/LICENSE diff --git a/h5boss_py/README.md b/h5boss_py/README.md new file mode 100755 index 0000000..4cf4055 --- /dev/null +++ b/h5boss_py/README.md @@ -0,0 +1,2 @@ +# h5boss +Exploratory tools for reformatting BOSS spectra as hdf5 files diff --git a/h5boss_py/demo/notebook/boss2hdf5_v2.ipynb b/h5boss_py/demo/notebook/boss2hdf5_v2.ipynb new file mode 100755 index 0000000..465f501 --- /dev/null +++ b/h5boss_py/demo/notebook/boss2hdf5_v2.ipynb @@ -0,0 +1,760 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import os.path\n", + "import numpy as np\n", + "from astropy.io import fits\n", + "from astropy.table import Table\n", + "\n", + "def load_coadds(platefile, zbestfile=None, run1d=None):\n", + " '''\n", + " Document ...\n", + " '''\n", + " #- Load spPlate data\n", + " fx = fits.open(platefile, memmap=False)\n", + " header = fx[0].header\n", + " c0 = header['COEFF0']\n", + " c1 = header['COEFF1']\n", + " nwave = header['NAXIS1']\n", + " nfiber = header['NAXIS2']\n", + " wave = (10**(c0 + c1*np.arange(nwave))).astype(np.float32)\n", + " flux = fx[0].data\n", + " ivar = fx[1].data\n", + " and_mask = fx[2].data\n", + " or_mask = fx[3].data\n", + " wavedisp = fx[4].data\n", + " sky = fx[6].data\n", + " fx.close()\n", + "\n", + " if run1d is None:\n", + " run1d = header['RUN2D'] #- default run1d == run2d\n", + " \n", + " #- Get best fit model from zbest file\n", + " if zbestfile is None:\n", + " zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d))\n", + "\n", + " model = fits.getdata(zbestfile, 2)\n", + "\n", + " coadds = list()\n", + " for i in range(nfiber):\n", + " sp = Table()\n", + " sp['WAVE'] = wave #- repeat !\n", + " sp['FLUX'] = flux[i]\n", + " sp['IVAR'] = ivar[i]\n", + " sp['AND_MASK'] = and_mask[i]\n", + " sp['OR_MASK'] = or_mask[i]\n", + " sp['WAVEDISP'] = wavedisp[i]\n", + " sp['SKY'] = sky[i]\n", + " sp['MODEL'] = model[i]\n", + " sp.meta = header\n", + " \n", + " #- TODO: Add units, comments to each column\n", + " \n", + " coadds.append(sp)\n", + " \n", + " return coadds\n", + "\n", + "def load_frame(framefile, cframefile=None, flatfile=None):\n", + " \"\"\"\n", + " Document ...\n", + " \"\"\"\n", + " if cframefile is None:\n", + " cframefile = framefile.replace('spFrame', 'spCFrame')\n", + " if cframefile.endswith('.gz'):\n", + " cframefile = cframefile[:-3]\n", + " import os.path\n", + " if os.path.exists(framefile)==False:\n", + " exit()\n", + " #- Load framefile and get original dimensions\n", + " eflux = fits.getdata(framefile, 0)\n", + " nfiber, npix = eflux.shape\n", + " if os.path.exists(cframefile)==False:\n", + " exit() \n", + " #- Load spCFrame file; trim arrays back to original size\n", + " fx = fits.open(cframefile, memmap=False)\n", + " header = fx[0].header\n", + " flux = fx[0].data[:, 0:npix]\n", + " ivar = fx[1].data[:, 0:npix]\n", + " mask = fx[2].data[:, 0:npix]\n", + " wave = (10**fx[3].data[:, 0:npix]).astype(np.float32)\n", + " wavedisp = fx[4].data[:, 0:npix]\n", + " sky = fx[6].data[:, 0:npix]\n", + " x = fx[7].data[:, 0:npix]\n", + " superflat = fx[8].data[:, 0:npix]\n", + " \n", + " #- Load fiberflat spFlat[0]\n", + " if flatfile is None:\n", + " flatfile = header['FLATFILE'].replace('sdR', 'spFlat')\n", + " flatfile = flatfile.replace('.fit', '.fits.gz')\n", + " filedir, basename = os.path.split(os.path.abspath(cframefile))\n", + " flatfile = os.path.join(filedir, flatfile)\n", + "\n", + " if os.path.exists(flatfile)==False:\n", + " exit()\n", + " fiberflat = fits.getdata(flatfile, 0)\n", + " \n", + " #- Calculate calibration vector: flux = electrons * calib\n", + " electrons = eflux * fiberflat * superflat\n", + " ii = np.where(electrons != 0.0)\n", + " calib = np.zeros(flux.shape)\n", + " calib[ii] = flux[ii] / electrons[ii]\n", + " \n", + " fx.close()\n", + " \n", + " #- Assemble spectra tables\n", + " spectra = list()\n", + " for i in range(nfiber):\n", + " sp = Table()\n", + " sp['WAVE'] = wave[i]\n", + " sp['FLUX'] = flux[i]\n", + " sp['IVAR'] = ivar[i]\n", + " sp['MASK'] = mask[i]\n", + " sp['WAVEDISP'] = wavedisp[i]\n", + " sp['SKY'] = sky[i]\n", + " sp['X'] = x[i]\n", + " sp['CALIB'] = calib[i].astype(np.float32)\n", + " sp.meta = header\n", + " \n", + " #- TODO: Add units, comments to each column\n", + " \n", + " spectra.append(sp)\n", + " \n", + " return spectra\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ImportError", + "evalue": "No module named 'h5boss'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m\u001b[0m", + "\u001b[1;31mImportError\u001b[0mTraceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 12\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mastropy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mio\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mfits\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mastropy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtable\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mTable\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mh5boss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mio\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 15\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mImportError\u001b[0m: No module named 'h5boss'" + ] + } + ], + "source": [ + "\"\"\"\n", + "Create an HDF5 file from BOSS data\n", + "\n", + "TODO:\n", + " - include comments in meta/attrs\n", + " - platelist quantities\n", + "\"\"\"\n", + "\n", + "from __future__ import division, print_function\n", + "import sys, os\n", + "import numpy as np\n", + "from astropy.io import fits\n", + "from astropy.table import Table\n", + "import h5boss.io\n", + "import time\n", + "\n", + "def serial_convert(platefile,hdf5output):\n", + " platefile=platefile[0]\n", + " hdf5output=str(hdf5output)\n", + " filedir = os.path.split(os.path.abspath(platefile))[0]\n", + " hdr = fits.getheader(platefile) \n", + " plate = hdr['PLATEID']\n", + " mjd = hdr['MJD']\n", + " tstart=time.time()\n", + " #--- Plugmap ---\n", + " print('plugmap')\n", + " plugmap = Table.read(platefile, 5)\n", + " dataname = '{}/{}/plugmap'.format(plate, mjd)\n", + " plugmap.write(hdf5output, path=dataname, append=True)\n", + "\n", + " #--- zbest ---\n", + " print('zbest')\n", + " run1d = hdr['RUN2D'] #- default run1d == run2d\n", + " zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d))\n", + " zbest = Table.read(zbestfile, 1)\n", + " dataname = '{}/{}/zbest'.format(plate, mjd)\n", + " zbest.write(hdf5output, path=dataname, append=True)\n", + " nfiber = len(zbest)\n", + "\n", + " #--- zall (skip) ---\n", + " pass\n", + "\n", + " #--- zline ---\n", + " print('zline')\n", + " zlinefile = zbestfile.replace('spZbest-', 'spZline-')\n", + " zline = Table.read(zlinefile, 1)\n", + " dataname = '{}/{}/zline'.format(plate, mjd)\n", + " zline.write(hdf5output, path=dataname, append=True)\n", + "\n", + " #--- photometric matches ---\n", + " print('photo')\n", + " photomatchfile = platefile.replace('spPlate-', 'photoMatchPlate-')\n", + " photomatch = Table.read(photomatchfile, 1)\n", + " photomatch['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16)\n", + " dataname = '{}/{}/photo/match'.format(plate, mjd)\n", + " photomatch.write(hdf5output, path=dataname, append=True)\n", + "\n", + " photoposfile = platefile.replace('spPlate-', 'photoPosPlate-')\n", + " photopos = Table.read(photoposfile, 1)\n", + " photopos['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16)\n", + " dataname = '{}/{}/photo/matchpos'.format(plate, mjd)\n", + " photopos.write(hdf5output, path=dataname, append=True)\n", + "\n", + " photofluxfile = platefile.replace('spPlate-', 'photoPlate-')\n", + " photoflux = Table.read(photofluxfile, 1)\n", + " photoflux['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16)\n", + " dataname = '{}/{}/photo/matchflux'.format(plate, mjd)\n", + " photoflux.write(hdf5output, path=dataname, append=True)\n", + "\n", + " #--- Coadd ---\n", + " print('loading coadds')\n", + " coadds = h5boss.io.load_coadds(platefile)\n", + "\n", + " print('writing coadds')\n", + " for i, cx in enumerate(coadds):\n", + " dataname = '{}/{}/{}/coadd'.format(plate, mjd, i+1)\n", + " cx.write(hdf5output, path=dataname, append=True)\n", + "\n", + " #--- Individual exposures ---\n", + " #- Parse spPlancomb to get exposures that were used\n", + " print('parsing planfile')\n", + " planfile = platefile.replace('spPlate-', 'spPlancomb-').replace('.fits', '.par')\n", + " framefiles = list()\n", + " for line in open(planfile):\n", + " if line.startswith('SPEXP '):\n", + " tmp = line.split()\n", + " tmp = [x+'.gz' for x in tmp[7:-1]]\n", + " framefiles.extend(tmp)\n", + "\n", + " print('individual exposures')\n", + " for filename in framefiles:\n", + " print(filename)\n", + " frame = h5boss.io.load_frame(filedir+'/'+filename)\n", + " if ('spFrame-b1' in filename) or ('spFrame-r1' in filename):\n", + " offset = 0\n", + " elif ('spFrame-b2' in filename) or ('spFrame-r2' in filename):\n", + " offset = 500\n", + " else:\n", + " print('huh?', filename)\n", + " sys.exit(1)\n", + "\n", + " for i, fx in enumerate(frame):\n", + " br = fx.meta['CAMERAS'][0]\n", + " expid = fx.meta['EXPOSURE']\n", + " fiber = offset+i+1\n", + " dataname = '{}/{}/{}/exposures/{}/{}'.format(plate, mjd, fiber, expid, br)\n", + " fx.write(hdf5output, path=dataname, append=True)\n", + "\n", + "\n", + " tend=time.time()-tstart\n", + " print ('time',tend) \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Rewrite the boss2hdf5 converter " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['/global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0/5290/spPlate-5290-55862.fits']" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#sections in serial convert\n", + "from __future__ import division, print_function\n", + "import sys, os\n", + "import numpy as np\n", + "from astropy.io import fits\n", + "from astropy.table import Table\n", + "import time\n", + "datapath = \"/global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0/\"\n", + "def listfiles():\n", + " ldir=os.listdir(datapath)\n", + " lldir=[fn for fn in ldir if fn.isdigit()]\n", + " return lldir\n", + "plateslist=listfiles()\n", + "platepath_for_current_rank = datapath+plateslist[0]\n", + "def findseed(x):\n", + " fitsfiles = [os.path.join(root, name)\n", + " for root, dirs, files in os.walk(x)\n", + " for name in files\n", + " if name.startswith(\"spPlate\") and name.endswith(\".fits\")]\n", + " return fitsfiles\n", + "fitspath_name_for_current_rank = findseed(platepath_for_current_rank)\n", + "fitspath_name_for_current_rank" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#sections in load coadd\n", + "platefile=fitspath_name_for_current_rank[0]\n", + "filedir = os.path.split(os.path.abspath(platefile))[0]\n", + "hdr = fits.getheader(platefile) \n", + "plate = hdr['PLATEID']\n", + "mjd = hdr['MJD']\n", + "#- Load spPlate data\n", + "fx = fits.open(platefile, memmap=False)\n", + "header = fx[0].header\n", + "c0 = header['COEFF0']\n", + "c1 = header['COEFF1']\n", + "nwave = header['NAXIS1']\n", + "nfiber = header['NAXIS2']\n", + "wave = (10**(c0 + c1*np.arange(nwave))).astype(np.float32)\n", + "flux = fx[0].data\n", + "ivar = fx[1].data\n", + "and_mask = fx[2].data\n", + "or_mask = fx[3].data\n", + "wavedisp = fx[4].data\n", + "sky = fx[6].data\n", + "fx.close()\n", + "run1d = header['RUN2D'] #- default run1d == run2d\n", + " \n", + "#- Get best fit model from zbest file\n", + "zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d))\n", + "model = fits.getdata(zbestfile, 2)\n", + "coadds=(wave,flux,ivar,and_mask,or_mask,wavedisp,sky,model,header)\n", + "hdf5output=\"h5boss_v2_15.h5\"\n", + "dat=(\"wave\",\"flux\",\"ivar\",\"and_mask\",\"or_mask\",\"wavedisp\",\"sky\",\"model\")\n", + "#list(header.cards)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#copy each wavelength dataset into the hdf5 file\n", + "import h5py\n", + "outx=h5py.File(hdf5output,'w')\n", + "for i in range(0,8):\n", + " id = '{}/{}/{}'.format(plate, mjd, dat[i])\n", + " dx=outx.create_dataset(id,data=coadds[i])\n", + " temptb=Table()\n", + " temptb.meta=coadds[8]\n", + " #temptb.write(hdf5output,id,append=True)\n", + " #dx.attrs.__setitem__(dat[i], coadds[8])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "parsing planfile\n", + "individual exposures\n" + ] + } + ], + "source": [ + "#--- Individual exposures ---\n", + "#- Parse spPlancomb to get exposures that were used\n", + "print('parsing planfile')\n", + "planfile = platefile.replace('spPlate-', 'spPlancomb-').replace('.fits', '.par')\n", + "framefiles = list()\n", + "for line in open(planfile):\n", + " if line.startswith('SPEXP '):\n", + " tmp = line.split()\n", + " tmp = [x+'.gz' for x in tmp[7:-1]]\n", + " framefiles.extend(tmp)\n", + "\n", + "print('individual exposures')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#sections in load exposures\n", + "def load_frame(framefile, cframefile=None, flatfile=None):\n", + " \"\"\"\n", + " Document ...\n", + " \"\"\"\n", + " if cframefile is None:\n", + " cframefile = framefile.replace('spFrame', 'spCFrame')\n", + " if cframefile.endswith('.gz'):\n", + " cframefile = cframefile[:-3]\n", + " import os.path\n", + " if os.path.exists(framefile)==False:\n", + " exit()\n", + " #- Load framefile and get original dimensions\n", + " eflux = fits.getdata(framefile, 0)\n", + " nfiber, npix = eflux.shape\n", + " if os.path.exists(cframefile)==False:\n", + " exit() \n", + " #- Load spCFrame file; trim arrays back to original size\n", + " fx = fits.open(cframefile, memmap=False)\n", + " header = fx[0].header # this means the exposureid is same for flux, ivar, etc\n", + " expid=header['EXPOSURE']\n", + " flux = fx[0].data[:, 0:npix]\n", + " ivar = fx[1].data[:, 0:npix]\n", + " mask = fx[2].data[:, 0:npix]\n", + " wave = (10**fx[3].data[:, 0:npix]).astype(np.float32)\n", + " wavedisp = fx[4].data[:, 0:npix]\n", + " sky = fx[6].data[:, 0:npix]\n", + " x = fx[7].data[:, 0:npix]\n", + " superflat = fx[8].data[:, 0:npix]\n", + " \n", + " #- Load fiberflat spFlat[0]\n", + " if flatfile is None:\n", + " flatfile = header['FLATFILE'].replace('sdR', 'spFlat')\n", + " flatfile = flatfile.replace('.fit', '.fits.gz')\n", + " filedir, basename = os.path.split(os.path.abspath(cframefile))\n", + " flatfile = os.path.join(filedir, flatfile)\n", + "\n", + " if os.path.exists(flatfile)==False:\n", + " exit()\n", + " fiberflat = fits.getdata(flatfile, 0)\n", + " \n", + " #- Calculate calibration vector: flux = electrons * calib\n", + " electrons = eflux * fiberflat * superflat\n", + " ii = np.where(electrons != 0.0)\n", + " calib = np.zeros(flux.shape)\n", + " calib[ii] = flux[ii] / electrons[ii]\n", + " \n", + " fx.close()\n", + " exposure=(wave,flux,ivar,mask,wavedisp,sky,x,calib.astype(np.float32), header,expid) \n", + " return exposure" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "expdat=(\"wave\",\"flux\",\"ivar\",\"mask\",\"wavedisp\",\"sky\",\"x\",\"calib\")\n", + "def dump2h5(frame1,frame2,expid,hdf5output,br):\n", + " import h5py\n", + " print ('dump:',expid)\n", + "\n", + " for i in range(0,len(expdat)):\n", + " try:\n", + " outx=h5py.File(hdf5output,'w')\n", + " except Exception as e:\n", + " print (\"file open error\")\n", + " pass\n", + " id = '{}/{}/exposures/{}/{}/{}'.format(plate, mjd, expid,br,expdat[i])\n", + " try:\n", + " dset=np.append(frame1[i],frame2[i])\n", + " print (\"id:%s\"%id)\n", + " print (\"shape:\",dset.shape)\n", + " except Exception as e:\n", + " print (\"append error\")\n", + " pass\n", + " try:\n", + " dx=outx.create_dataset(id,data=dset)\n", + " except Exception as e:\n", + " print ('error in frame dump')\n", + " pass\n", + " #outx.flush()\n", + " outx.close() \n", + "def single_dump2h5(frame1,expid,hdf5output,br):\n", + " import h5py\n", + " print (\"single dump:\",expid)\n", + "\n", + " for i in range(0,len(expdat)):\n", + " try:\n", + " outx=h5py.File(hdf5output,'w')\n", + " except Exception as e:\n", + " print (\"file open error\")\n", + " pass\n", + " id = '{}/{}/exposures/{}/{}/{}'.format(plate, mjd, expid,br,expdat[i])\n", + " dset=frame1[i]\n", + " try:\n", + " dx=outx.create_dataset(id,data=dset)\n", + " except Exception as e:\n", + " print ('error in frame dump')\n", + " pass\n", + " outx.flush()\n", + " outx.close() " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "40\n" + ] + } + ], + "source": [ + "# initialize the empty arrays, \n", + "# b: wave, flux, ivar, mask, wavedisp, sky, x, calib, header\n", + "# r: wave, flux, ivar, mask, wavedisp, sky, x, calib, header\n", + "b1=0\n", + "r1=0\n", + "print (len(framefiles))\n", + "frameb1=list()\n", + "framer1=list()\n", + "frameb2=list()\n", + "framer2=list()\n", + "filename=framefiles[0]\n", + "if ('spFrame-b1' in filename):\n", + " frame=load_frame(filedir+'/'+filename)\n", + "#for filename in framefiles:\n", + "# offset = 0 #fiber 0-499\n", + "# if ('spFrame-b1' in filename):\n", + "# try:\n", + "# frame=load_frame(filedir+'/'+filename)\n", + "# frameb1.append(frame)\n", + "# print (filename, frame[9])\n", + "# except Exception as e:\n", + "# print (\"File not found\")\n", + "# pass\n", + " #if ('spFrame-r1' in filename):\n", + " # try:\n", + " # frame=load_frame(filedir+'/'+filename)\n", + " # framer1.append(frame)\n", + " # print (filename, frame[9])\n", + " # except Exception as e:\n", + " # print (\"File not found\")\n", + " # pass\n", + " \n", + "\n", + "#combine b1 and b2\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(500, 4112)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "frame[0].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "for filename in framefiles:\n", + " offset = 500 #fiber 500-999\n", + " if ('spFrame-b2' in filename):\n", + " try:\n", + " frame=load_frame(filedir+'/'+filename)\n", + " frameb2.append(frame)\n", + " print (filename, frame[9])\n", + " except Exception as e:\n", + " print (\"File not found\")\n", + " pass \n", + " if ('spFrame-r2' in filename):\n", + " try:\n", + " frame=load_frame(filedir+'/'+filename)\n", + " framer2.append(frame)\n", + " print (filename, frame[9])\n", + " except Exception as e:\n", + " print (\"File not found\")\n", + " pass \n", + "\n", + "print (\"frameb1:%d\"%(len(frameb1))) \n", + "print (\"frameb2:%d\"%(len(frameb2))) \n", + "print (\"framer1:%d\"%(len(framer1))) \n", + "print (\"framer2:%d\"%(len(framer2))) " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'framefiles' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[1;31m\u001b[0m", + "\u001b[1;31mNameError\u001b[0mTraceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mframefiles\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;31mNameError\u001b[0m: name 'framefiles' is not defined" + ] + } + ], + "source": [ + "framefiles" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "for i in range(0,len(frameb1)):\n", + " expidb1=frameb1[i][9]\n", + " hit=0\n", + " for j in range(0,len(frameb2)):\n", + " expidb2=frameb2[j][9]\n", + " if expidb1==expidb2:\n", + " #combine frameb1 and frameb2\n", + " dump2h5(frameb1[i],frameb2[j],expidb1,hdf5output,'b')\n", + " frameb2.remove(frameb2[j])\n", + " hit=1\n", + " break\n", + " if hit==0:\n", + " single_dump2h5(frameb1[i],expidb1,hdf5output,'b')\n", + "for i in range(0,len(frameb2)):\n", + " expidb2=frameb2[i][9]\n", + " single_dump2h5(frameb2[i],expidb2,hdf5output,'b')\n", + "\n", + "#combine r1 and r2\n", + "for i in range(0,len(framer1)):\n", + " expidr1=framer1[i][9]\n", + " hit=0\n", + " for j in range(0,len(framer2)):\n", + " expidr2=framer2[j][9]\n", + " if expidr1==expidr2:\n", + " #combine frameb1 and frameb2\n", + " dump2h5(framer1[i],framer2[j],expidr1,hdf5output,'r')\n", + " framer2.remove(framer2[j])\n", + " hit=1\n", + " break\n", + " if hit==0:\n", + " single_dump2h5(framer1[i],expidr1,hdf5output,'r')\n", + "for i in range(0,len(framer2)):\n", + " expidr2=framer2[i][9]\n", + " single_dump2h5(framer2[i],expidr2,hdf5output,'r')" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "'spFrame-b1' in 'spCFrame-b1-00135668.fits'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "id5290/55862/exposures/135671/b/calib\n", + "id5290/55862/exposures/135671/r/calib\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/h5boss_py/demo/notebook/h5boss-test.ipynb b/h5boss_py/demo/notebook/h5boss-test.ipynb new file mode 100755 index 0000000..94a758d --- /dev/null +++ b/h5boss_py/demo/notebook/h5boss-test.ipynb @@ -0,0 +1,2594 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 81, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3687-55269.hdf5 7456-56727.hdf5 7457-56746.hdf5\r\n" + ] + } + ], + "source": [ + "ls *.hdf5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fiber Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "length:1000\n", + "min:148448.00\n", + "max:148448.00\n", + "median:148448.00\n", + "mean:148448.00\n" + ] + } + ], + "source": [ + "import h5py\n", + "import numpy as np\n", + "testf=\"7456-56727.hdf5\"\n", + "def fibersta(testf):\n", + " fx1=h5py.File(testf,\"r\")\n", + " p=fx1.keys()[0]\n", + " m=fx1[str(p)].keys()[0]\n", + " coad_size=[]\n", + " pm=str(p)+'/'+str(m)\n", + " fs=fx1[pm].keys()\n", + " for i in range(0,len(fs)):\n", + " pmfi=pm+'/'+str(fs[i])+'/'+'coadd'\n", + " try:\n", + " #print pmfi\n", + " coad_size.append(fx1[pmfi].value.nbytes)\n", + " except:\n", + " pass\n", + " #print coad_size[i]\n", + " return coad_size\n", + "cs1=fibersta(testf)\n", + "#print cs1\n", + "print \"length:%d\"%len(cs1)\n", + "print \"min:%.2f\"%np.amin(cs1)\n", + "print \"max:%.2f\"%np.amax(cs1)\n", + "print \"median:%.2f\"%np.median(cs1)\n", + "print \"mean:%.2f\"%np.mean(cs1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "# Catalog Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "length:6\n", + "min:147000.00\n", + "max:3012000.00\n", + "median:1991500.00\n", + "mean:1738500.00\n" + ] + } + ], + "source": [ + "meta=['plugmap', 'zbest', 'zline', 'photo/match', 'photo/matchflux', 'photo/matchpos']\n", + "testf=\"3687-55269.hdf5\"\n", + "def catasta(testf):\n", + " fx1=h5py.File(testf,\"r\")\n", + " p=fx1.keys()[0]\n", + " m=fx1[str(p)].keys()[0]\n", + " cata_size=[]\n", + " pm=str(p)+'/'+str(m)\n", + " fs=fx1[pm].keys()\n", + " for i in range(0,len(meta)):\n", + " pmfi=pm+'/'+meta[i]\n", + " try:\n", + " #print pmfi\n", + " cata_size.append(fx1[pmfi].value.nbytes)\n", + " except:\n", + " pass\n", + " #print coad_size[i]\n", + " return cata_size\n", + "cs1=catasta(testf)\n", + "#print cs1\n", + "print \"length:%d\"%len(cs1)\n", + "print \"min:%.2f\"%np.amin(cs1)\n", + "print \"max:%.2f\"%np.amax(cs1)\n", + "print \"median:%.2f\"%np.median(cs1)\n", + "print \"mean:%.2f\"%np.mean(cs1)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[u'3687']\n", + "[u'55269']\n" + ] + } + ], + "source": [ + "print fx1.keys()\n", + "print fx1['3687'].keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "parent_id='3687/55269'" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[u'1',\n", + " u'10',\n", + " u'100',\n", + " u'1000',\n", + " u'101',\n", + " u'102',\n", + " u'103',\n", + " u'104',\n", + " u'105',\n", + " u'106',\n", + " u'107',\n", + " u'108',\n", + " u'109',\n", + " u'11',\n", + " u'110',\n", + " u'111',\n", + " u'112',\n", + " u'113',\n", + " u'114',\n", + " u'115',\n", + " u'116',\n", + " u'117',\n", + " u'118',\n", + " u'119',\n", + " u'12',\n", + " u'120',\n", + " u'121',\n", + " u'122',\n", + " u'123',\n", + " u'124',\n", + " u'125',\n", + " u'126',\n", + " u'127',\n", + " u'128',\n", + " u'129',\n", + " u'13',\n", + " u'130',\n", + " u'131',\n", + " u'132',\n", + " u'133',\n", + " u'134',\n", + " u'135',\n", + " u'136',\n", + " u'137',\n", + " u'138',\n", + " u'139',\n", + " u'14',\n", + " u'140',\n", + " u'141',\n", + " u'142',\n", + " u'143',\n", + " u'144',\n", + " u'145',\n", + " u'146',\n", + " u'147',\n", + " u'148',\n", + " u'149',\n", + " u'15',\n", + " u'150',\n", + " u'151',\n", + " u'152',\n", + " u'153',\n", + " u'154',\n", + " u'155',\n", + " u'156',\n", + " u'157',\n", + " u'158',\n", + " u'159',\n", + " u'16',\n", + " u'160',\n", + " u'161',\n", + " u'162',\n", + " u'163',\n", + " u'164',\n", + " u'165',\n", + " u'166',\n", + " u'167',\n", + " u'168',\n", + " u'169',\n", + " u'17',\n", + " u'170',\n", + " u'171',\n", + " u'172',\n", + " u'173',\n", + " u'174',\n", + " u'175',\n", + " u'176',\n", + " u'177',\n", + " u'178',\n", + " u'179',\n", + " u'18',\n", + " u'180',\n", + " u'181',\n", + " u'182',\n", + " u'183',\n", + " u'184',\n", + " u'185',\n", + " u'186',\n", + " u'187',\n", + " u'188',\n", + " u'189',\n", + " u'19',\n", + " u'190',\n", + " u'191',\n", + " u'192',\n", + " u'193',\n", + " u'194',\n", + " u'195',\n", + " u'196',\n", + " u'197',\n", + " u'198',\n", + " u'199',\n", + " u'2',\n", + " u'20',\n", + " u'200',\n", + " u'201',\n", + " u'202',\n", + " u'203',\n", + " u'204',\n", + " u'205',\n", + " u'206',\n", + " u'207',\n", + " u'208',\n", + " u'209',\n", + " u'21',\n", + " u'210',\n", + " u'211',\n", + " u'212',\n", + " u'213',\n", + " u'214',\n", + " u'215',\n", + " u'216',\n", + " u'217',\n", + " u'218',\n", + " u'219',\n", + " u'22',\n", + " u'220',\n", + " u'221',\n", + " u'222',\n", + " u'223',\n", + " u'224',\n", + " u'225',\n", + " u'226',\n", + " u'227',\n", + " u'228',\n", + " u'229',\n", + " u'23',\n", + " u'230',\n", + " u'231',\n", + " u'232',\n", + " u'233',\n", + " u'234',\n", + " u'235',\n", + " u'236',\n", + " u'237',\n", + " u'238',\n", + " u'239',\n", + " u'24',\n", + " u'240',\n", + " u'241',\n", + " u'242',\n", + " u'243',\n", + " u'244',\n", + " u'245',\n", + " u'246',\n", + " u'247',\n", + " u'248',\n", + " u'249',\n", + " u'25',\n", + " u'250',\n", + " u'251',\n", + " u'252',\n", + " u'253',\n", + " u'254',\n", + " u'255',\n", + " u'256',\n", + " u'257',\n", + " u'258',\n", + " u'259',\n", + " u'26',\n", + " u'260',\n", + " u'261',\n", + " u'262',\n", + " u'263',\n", + " u'264',\n", + " u'265',\n", + " u'266',\n", + " u'267',\n", + " u'268',\n", + " u'269',\n", + " u'27',\n", + " u'270',\n", + " u'271',\n", + " u'272',\n", + " u'273',\n", + " u'274',\n", + " u'275',\n", + " u'276',\n", + " u'277',\n", + " u'278',\n", + " u'279',\n", + " u'28',\n", + " u'280',\n", + " u'281',\n", + " u'282',\n", + " u'283',\n", + " u'284',\n", + " u'285',\n", + " u'286',\n", + " u'287',\n", + " u'288',\n", + " u'289',\n", + " u'29',\n", + " u'290',\n", + " u'291',\n", + " u'292',\n", + " u'293',\n", + " u'294',\n", + " u'295',\n", + " u'296',\n", + " u'297',\n", + " u'298',\n", + " u'299',\n", + " u'3',\n", + " u'30',\n", + " u'300',\n", + " u'301',\n", + " u'302',\n", + " u'303',\n", + " u'304',\n", + " u'305',\n", + " u'306',\n", + " u'307',\n", + " u'308',\n", + " u'309',\n", + " u'31',\n", + " u'310',\n", + " u'311',\n", + " u'312',\n", + " u'313',\n", + " u'314',\n", + " u'315',\n", + " u'316',\n", + " u'317',\n", + " u'318',\n", + " u'319',\n", + " u'32',\n", + " u'320',\n", + " u'321',\n", + " u'322',\n", + " u'323',\n", + " u'324',\n", + " u'325',\n", + " u'326',\n", + " u'327',\n", + " u'328',\n", + " u'329',\n", + " u'33',\n", + " u'330',\n", + " u'331',\n", + " u'332',\n", + " u'333',\n", + " u'334',\n", + " u'335',\n", + " u'336',\n", + " u'337',\n", + " u'338',\n", + " u'339',\n", + " u'34',\n", + " u'340',\n", + " u'341',\n", + " u'342',\n", + " u'343',\n", + " u'344',\n", + " u'345',\n", + " u'346',\n", + " u'347',\n", + " u'348',\n", + " u'349',\n", + " u'35',\n", + " u'350',\n", + " u'351',\n", + " u'352',\n", + " u'353',\n", + " u'354',\n", + " u'355',\n", + " u'356',\n", + " u'357',\n", + " u'358',\n", + " u'359',\n", + " u'36',\n", + " u'360',\n", + " u'361',\n", + " u'362',\n", + " u'363',\n", + " u'364',\n", + " u'365',\n", + " u'366',\n", + " u'367',\n", + " u'368',\n", + " u'369',\n", + " u'37',\n", + " u'370',\n", + " u'371',\n", + " u'372',\n", + " u'373',\n", + " u'374',\n", + " u'375',\n", + " u'376',\n", + " u'377',\n", + " u'378',\n", + " u'379',\n", + " u'38',\n", + " u'380',\n", + " u'381',\n", + " u'382',\n", + " u'383',\n", + " u'384',\n", + " u'385',\n", + " u'386',\n", + " u'387',\n", + " u'388',\n", + " u'389',\n", + " u'39',\n", + " u'390',\n", + " u'391',\n", + " u'392',\n", + " u'393',\n", + " u'394',\n", + " u'395',\n", + " u'396',\n", + " u'397',\n", + " u'398',\n", + " u'399',\n", + " u'4',\n", + " u'40',\n", + " u'400',\n", + " u'401',\n", + " u'402',\n", + " u'403',\n", + " u'404',\n", + " u'405',\n", + " u'406',\n", + " u'407',\n", + " u'408',\n", + " u'409',\n", + " u'41',\n", + " u'410',\n", + " u'411',\n", + " u'412',\n", + " u'413',\n", + " u'414',\n", + " u'415',\n", + " u'416',\n", + " u'417',\n", + " u'418',\n", + " u'419',\n", + " u'42',\n", + " u'420',\n", + " u'421',\n", + " u'422',\n", + " u'423',\n", + " u'424',\n", + " u'425',\n", + " u'426',\n", + " u'427',\n", + " u'428',\n", + " u'429',\n", + " u'43',\n", + " u'430',\n", + " u'431',\n", + " u'432',\n", + " u'433',\n", + " u'434',\n", + " u'435',\n", + " u'436',\n", + " u'437',\n", + " u'438',\n", + " u'439',\n", + " u'44',\n", + " u'440',\n", + " u'441',\n", + " u'442',\n", + " u'443',\n", + " u'444',\n", + " u'445',\n", + " u'446',\n", + " u'447',\n", + " u'448',\n", + " u'449',\n", + " u'45',\n", + " u'450',\n", + " u'451',\n", + " u'452',\n", + " u'453',\n", + " u'454',\n", + " u'455',\n", + " u'456',\n", + " u'457',\n", + " u'458',\n", + " u'459',\n", + " u'46',\n", + " u'460',\n", + " u'461',\n", + " u'462',\n", + " u'463',\n", + " u'464',\n", + " u'465',\n", + " u'466',\n", + " u'467',\n", + " u'468',\n", + " u'469',\n", + " u'47',\n", + " u'470',\n", + " u'471',\n", + " u'472',\n", + " u'473',\n", + " u'474',\n", + " u'475',\n", + " u'476',\n", + " u'477',\n", + " u'478',\n", + " u'479',\n", + " u'48',\n", + " u'480',\n", + " u'481',\n", + " u'482',\n", + " u'483',\n", + " u'484',\n", + " u'485',\n", + " u'486',\n", + " u'487',\n", + " u'488',\n", + " u'489',\n", + " u'49',\n", + " u'490',\n", + " u'491',\n", + " u'492',\n", + " u'493',\n", + " u'494',\n", + " u'495',\n", + " u'496',\n", + " u'497',\n", + " u'498',\n", + " u'499',\n", + " u'5',\n", + " u'50',\n", + " u'500',\n", + " u'501',\n", + " u'502',\n", + " u'503',\n", + " u'504',\n", + " u'505',\n", + " u'506',\n", + " u'507',\n", + " u'508',\n", + " u'509',\n", + " u'51',\n", + " u'510',\n", + " u'511',\n", + " u'512',\n", + " u'513',\n", + " u'514',\n", + " u'515',\n", + " u'516',\n", + " u'517',\n", + " u'518',\n", + " u'519',\n", + " u'52',\n", + " u'520',\n", + " u'521',\n", + " u'522',\n", + " u'523',\n", + " u'524',\n", + " u'525',\n", + " u'526',\n", + " u'527',\n", + " u'528',\n", + " u'529',\n", + " u'53',\n", + " u'530',\n", + " u'531',\n", + " u'532',\n", + " u'533',\n", + " u'534',\n", + " u'535',\n", + " u'536',\n", + " u'537',\n", + " u'538',\n", + " u'539',\n", + " u'54',\n", + " u'540',\n", + " u'541',\n", + " u'542',\n", + " u'543',\n", + " u'544',\n", + " u'545',\n", + " u'546',\n", + " u'547',\n", + " u'548',\n", + " u'549',\n", + " u'55',\n", + " u'550',\n", + " u'551',\n", + " u'552',\n", + " u'553',\n", + " u'554',\n", + " u'555',\n", + " u'556',\n", + " u'557',\n", + " u'558',\n", + " u'559',\n", + " u'56',\n", + " u'560',\n", + " u'561',\n", + " u'562',\n", + " u'563',\n", + " u'564',\n", + " u'565',\n", + " u'566',\n", + " u'567',\n", + " u'568',\n", + " u'569',\n", + " u'57',\n", + " u'570',\n", + " u'571',\n", + " u'572',\n", + " u'573',\n", + " u'574',\n", + " u'575',\n", + " u'576',\n", + " u'577',\n", + " u'578',\n", + " u'579',\n", + " u'58',\n", + " u'580',\n", + " u'581',\n", + " u'582',\n", + " u'583',\n", + " u'584',\n", + " u'585',\n", + " u'586',\n", + " u'587',\n", + " u'588',\n", + " u'589',\n", + " u'59',\n", + " u'590',\n", + " u'591',\n", + " u'592',\n", + " u'593',\n", + " u'594',\n", + " u'595',\n", + " u'596',\n", + " u'597',\n", + " u'598',\n", + " u'599',\n", + " u'6',\n", + " u'60',\n", + " u'600',\n", + " u'601',\n", + " u'602',\n", + " u'603',\n", + " u'604',\n", + " u'605',\n", + " u'606',\n", + " u'607',\n", + " u'608',\n", + " u'609',\n", + " u'61',\n", + " u'610',\n", + " u'611',\n", + " u'612',\n", + " u'613',\n", + " u'614',\n", + " u'615',\n", + " u'616',\n", + " u'617',\n", + " u'618',\n", + " u'619',\n", + " u'62',\n", + " u'620',\n", + " u'621',\n", + " u'622',\n", + " u'623',\n", + " u'624',\n", + " u'625',\n", + " u'626',\n", + " u'627',\n", + " u'628',\n", + " u'629',\n", + " u'63',\n", + " u'630',\n", + " u'631',\n", + " u'632',\n", + " u'633',\n", + " u'634',\n", + " u'635',\n", + " u'636',\n", + " u'637',\n", + " u'638',\n", + " u'639',\n", + " u'64',\n", + " u'640',\n", + " u'641',\n", + " u'642',\n", + " u'643',\n", + " u'644',\n", + " u'645',\n", + " u'646',\n", + " u'647',\n", + " u'648',\n", + " u'649',\n", + " u'65',\n", + " u'650',\n", + " u'651',\n", + " u'652',\n", + " u'653',\n", + " u'654',\n", + " u'655',\n", + " u'656',\n", + " u'657',\n", + " u'658',\n", + " u'659',\n", + " u'66',\n", + " u'660',\n", + " u'661',\n", + " u'662',\n", + " u'663',\n", + " u'664',\n", + " u'665',\n", + " u'666',\n", + " u'667',\n", + " u'668',\n", + " u'669',\n", + " u'67',\n", + " u'670',\n", + " u'671',\n", + " u'672',\n", + " u'673',\n", + " u'674',\n", + " u'675',\n", + " u'676',\n", + " u'677',\n", + " u'678',\n", + " u'679',\n", + " u'68',\n", + " u'680',\n", + " u'681',\n", + " u'682',\n", + " u'683',\n", + " u'684',\n", + " u'685',\n", + " u'686',\n", + " u'687',\n", + " u'688',\n", + " u'689',\n", + " u'69',\n", + " u'690',\n", + " u'691',\n", + " u'692',\n", + " u'693',\n", + " u'694',\n", + " u'695',\n", + " u'696',\n", + " u'697',\n", + " u'698',\n", + " u'699',\n", + " u'7',\n", + " u'70',\n", + " u'700',\n", + " u'701',\n", + " u'702',\n", + " u'703',\n", + " u'704',\n", + " u'705',\n", + " u'706',\n", + " u'707',\n", + " u'708',\n", + " u'709',\n", + " u'71',\n", + " u'710',\n", + " u'711',\n", + " u'712',\n", + " u'713',\n", + " u'714',\n", + " u'715',\n", + " u'716',\n", + " u'717',\n", + " u'718',\n", + " u'719',\n", + " u'72',\n", + " u'720',\n", + " u'721',\n", + " u'722',\n", + " u'723',\n", + " u'724',\n", + " u'725',\n", + " u'726',\n", + " u'727',\n", + " u'728',\n", + " u'729',\n", + " u'73',\n", + " u'730',\n", + " u'731',\n", + " u'732',\n", + " u'733',\n", + " u'734',\n", + " u'735',\n", + " u'736',\n", + " u'737',\n", + " u'738',\n", + " u'739',\n", + " u'74',\n", + " u'740',\n", + " u'741',\n", + " u'742',\n", + " u'743',\n", + " u'744',\n", + " u'745',\n", + " u'746',\n", + " u'747',\n", + " u'748',\n", + " u'749',\n", + " u'75',\n", + " u'750',\n", + " u'751',\n", + " u'752',\n", + " u'753',\n", + " u'754',\n", + " u'755',\n", + " u'756',\n", + " u'757',\n", + " u'758',\n", + " u'759',\n", + " u'76',\n", + " u'760',\n", + " u'761',\n", + " u'762',\n", + " u'763',\n", + " u'764',\n", + " u'765',\n", + " u'766',\n", + " u'767',\n", + " u'768',\n", + " u'769',\n", + " u'77',\n", + " u'770',\n", + " u'771',\n", + " u'772',\n", + " u'773',\n", + " u'774',\n", + " u'775',\n", + " u'776',\n", + " u'777',\n", + " u'778',\n", + " u'779',\n", + " u'78',\n", + " u'780',\n", + " u'781',\n", + " u'782',\n", + " u'783',\n", + " u'784',\n", + " u'785',\n", + " u'786',\n", + " u'787',\n", + " u'788',\n", + " u'789',\n", + " u'79',\n", + " u'790',\n", + " u'791',\n", + " u'792',\n", + " u'793',\n", + " u'794',\n", + " u'795',\n", + " u'796',\n", + " u'797',\n", + " u'798',\n", + " u'799',\n", + " u'8',\n", + " u'80',\n", + " u'800',\n", + " u'801',\n", + " u'802',\n", + " u'803',\n", + " u'804',\n", + " u'805',\n", + " u'806',\n", + " u'807',\n", + " u'808',\n", + " u'809',\n", + " u'81',\n", + " u'810',\n", + " u'811',\n", + " u'812',\n", + " u'813',\n", + " u'814',\n", + " u'815',\n", + " u'816',\n", + " u'817',\n", + " u'818',\n", + " u'819',\n", + " u'82',\n", + " u'820',\n", + " u'821',\n", + " u'822',\n", + " u'823',\n", + " u'824',\n", + " u'825',\n", + " u'826',\n", + " u'827',\n", + " u'828',\n", + " u'829',\n", + " u'83',\n", + " u'830',\n", + " u'831',\n", + " u'832',\n", + " u'833',\n", + " u'834',\n", + " u'835',\n", + " u'836',\n", + " u'837',\n", + " u'838',\n", + " u'839',\n", + " u'84',\n", + " u'840',\n", + " u'841',\n", + " u'842',\n", + " u'843',\n", + " u'844',\n", + " u'845',\n", + " u'846',\n", + " u'847',\n", + " u'848',\n", + " u'849',\n", + " u'85',\n", + " u'850',\n", + " u'851',\n", + " u'852',\n", + " u'853',\n", + " u'854',\n", + " u'855',\n", + " u'856',\n", + " u'857',\n", + " u'858',\n", + " u'859',\n", + " u'86',\n", + " u'860',\n", + " u'861',\n", + " u'862',\n", + " u'863',\n", + " u'864',\n", + " u'865',\n", + " u'866',\n", + " u'867',\n", + " u'868',\n", + " u'869',\n", + " u'87',\n", + " u'870',\n", + " u'871',\n", + " u'872',\n", + " u'873',\n", + " u'874',\n", + " u'875',\n", + " u'876',\n", + " u'877',\n", + " u'878',\n", + " u'879',\n", + " u'88',\n", + " u'880',\n", + " u'881',\n", + " u'882',\n", + " u'883',\n", + " u'884',\n", + " u'885',\n", + " u'886',\n", + " u'887',\n", + " u'888',\n", + " u'889',\n", + " u'89',\n", + " u'890',\n", + " u'891',\n", + " u'892',\n", + " u'893',\n", + " u'894',\n", + " u'895',\n", + " u'896',\n", + " u'897',\n", + " u'898',\n", + " u'899',\n", + " u'9',\n", + " u'90',\n", + " u'900',\n", + " u'901',\n", + " u'902',\n", + " u'903',\n", + " u'904',\n", + " u'905',\n", + " u'906',\n", + " u'907',\n", + " u'908',\n", + " u'909',\n", + " u'91',\n", + " u'910',\n", + " u'911',\n", + " u'912',\n", + " u'913',\n", + " u'914',\n", + " u'915',\n", + " u'916',\n", + " u'917',\n", + " u'918',\n", + " u'919',\n", + " u'92',\n", + " u'920',\n", + " u'921',\n", + " u'922',\n", + " u'923',\n", + " u'924',\n", + " u'925',\n", + " u'926',\n", + " u'927',\n", + " u'928',\n", + " u'929',\n", + " u'93',\n", + " u'930',\n", + " u'931',\n", + " u'932',\n", + " u'933',\n", + " u'934',\n", + " u'935',\n", + " u'936',\n", + " u'937',\n", + " u'938',\n", + " u'939',\n", + " u'94',\n", + " u'940',\n", + " u'941',\n", + " u'942',\n", + " u'943',\n", + " u'944',\n", + " u'945',\n", + " u'946',\n", + " u'947',\n", + " u'948',\n", + " u'949',\n", + " u'95',\n", + " u'950',\n", + " u'951',\n", + " u'952',\n", + " u'953',\n", + " u'954',\n", + " u'955',\n", + " u'956',\n", + " u'957',\n", + " u'958',\n", + " u'959',\n", + " u'96',\n", + " u'960',\n", + " u'961',\n", + " u'962',\n", + " u'963',\n", + " u'964',\n", + " u'965',\n", + " u'966',\n", + " u'967',\n", + " u'968',\n", + " u'969',\n", + " u'97',\n", + " u'970',\n", + " u'971',\n", + " u'972',\n", + " u'973',\n", + " u'974',\n", + " u'975',\n", + " u'976',\n", + " u'977',\n", + " u'978',\n", + " u'979',\n", + " u'98',\n", + " u'980',\n", + " u'981',\n", + " u'982',\n", + " u'983',\n", + " u'984',\n", + " u'985',\n", + " u'986',\n", + " u'987',\n", + " u'988',\n", + " u'989',\n", + " u'99',\n", + " u'990',\n", + " u'991',\n", + " u'992',\n", + " u'993',\n", + " u'994',\n", + " u'995',\n", + " u'996',\n", + " u'997',\n", + " u'998',\n", + " u'999',\n", + " ...]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "arr=fx1[parent_id].keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "id=parent_id+'/plugmap'\n", + "sfiber=parent_id+'/129'\n", + "coaid=sfiber+\"/coadd\"\n", + "expid=sfiber+\"/exposures\"" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[u'coadd', u'exposures']" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d1=fx1[id]\n", + "#print fx1[sfiber].keys()\n", + "#print fx1[sfiber]\n", + "#print fx1[coaid]\n", + "#print fx1[expid].keys()\n", + "d_coad=fx1[coaid][:]\n", + "d_coad\n", + "for i in fx1[expid].keys():\n", + " xb=expid+'/'+i+'/b'\n", + " xr=expid+'/'+i+'/r'\n", + " d_xb=fx1[xb][:]\n", + " #print 'd_xb:',d_xb\n", + " d_xr=fx1[xr][:]\n", + " #print 'd_xr:',d_xr\n", + "fx1[sfiber].keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d1" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "meta=[]\n", + "path=[]\n", + "cur={}\n", + "def find_id(name):\n", + " cur_node=name.encode('ascii','ignore')\n", + " node=pid+cur_node\n", + " node_t=str(type(fx1[node]))\n", + " if 'group' in node_t:\n", + " node_t='g'\n", + " else:\n", + " #node_t='d-'+str(fx1[node].shape)+'-'+str(fx1[node].dtype)\n", + " node_t=fx1[node].dtype\n", + " path.append(node)\n", + " meta.append(node_t)\n", + " cur[node]=node_t\n", + "#d1.visit(find_id)\n", + "#'g'==cur['3687/55269/1/exposures/111770/b']\n" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "ddtype1" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#ffx=h5py.File('gtest7.h5','w')\n", + "#dtype1=[('WAVE', '" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fxdbin1=h5py.File('dbin4.h5','a')\n", + "fxdbin1" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "for id in fx1['3687/55269'].keys():\n", + " cid='3687/55269/'+id\n", + " fxdbin1.create_group(cid)\n", + " fx1.copy(cid,fxdbin1[cid])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "ls" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Fit the I/O time" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "fiber_number=[10.0,100.0,1000.0,10000.0]\n", + "fiber_copy_time=[6.26,81.97,877.53,7498.72]\n", + "catalog_copy_time=[3.61,37.35,414.17,927.33]\n", + "read_fiber_col_time=[2.67,29.86,291.27,799.41]\n", + "catalog_rowcopy_time=[0.9,7.19,120.07,124.77]\n", + "total_cost=[114.53,233.48,1421.72,8607.17]\n", + "fiber_number=np.array(fiber_number,dtype=float)\n", + "fiber_copy_time=np.array(fiber_copy_time,dtype=float)\n", + "catalog_copy_time=np.array(catalog_copy_time,dtype=float)\n", + "read_fiber_col_time=np.array(read_fiber_col_time,dtype=float)\n", + "catalog_rowcopy_time=np.array(catalog_rowcopy_time,dtype=float)\n", + "fiber_copy=np.polyfit(fiber_copy_time,fiber_number,1)\n", + "catalog_copy=np.polyfit(catalog_copy_time,fiber_number,1)" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ 1.33934963 -56.72454817]\n", + "[ 10.54151859 -865.80694843]\n" + ] + } + ], + "source": [ + "print fiber_copy\n", + "print catalog_copy" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "from scipy.optimize import curve_fit\n", + "def func(x,a,b):\n", + " return a*x+b\n", + "popt,pcov = curve_fit(func,fiber_number,fiber_copy_time)\n", + "popt_cata,pcov_cata=curve_fit(func,fiber_number,catalog_copy_time)" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 77, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEZCAYAAABFFVgWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmczdX/wPHXe+yDsY19J2Tta0kpZVSUCBVSFFHZl2ih\nLCOVNkVKVNaEpPppsS8TlS070yLL2BWyj2XM+/fH5zNc0yx3mDt3lvfz8bgPn3s+y3nfe8c99yyf\nc0RVMcYYY3whwN8BGGOMSb+skDHGGOMzVsgYY4zxGStkjDHG+IwVMsYYY3zGChljjDE+Y4VMBiEi\nJUXkpIiI+3yZiHTyd1wpSUTmisjj/o4jNRKRaBEp56e8K4rIBhE5ISI949hfSESWu/vfFpGBIvKx\nu6+0G7t9l6VSmf0dgEleIrIbKAREAQIoUFFV9wJBfgwtRYnIUKC8qj4Rk6aq9/sxpNTOnzfMvQAs\nVdWa8ex/BvhbVfPEs99u9kvFrPRPfxRoqqpBqprb/feQLzOMqR2ZNC1ZPkMRyXQNp5UGtiWyP/za\nIvLeNcZuEmGFTPr0ny+MeJoVbhCR1W4zxDciktfj+FtF5GcR+ddtymjgsW+ZiLwqIj+JyBmgbBz5\nlRCRr0TkbxH5R0Ted9NFRAaJyG4ROSQik0UkKFaMT4vIfvfR391XWETOiEg+jzxqudfPFCvve4GX\ngEdE5JSIbPCIu5O73cGN/133Nf4lIvXc9D1ubE94XDOriLwjIhEiclBExopItjhed1b3elU80oJF\n5Kz7bwER+c495qiI/Bjvh+i8F11E5E8ROSYiH3jsGyoin8X3+bqvdbj7GZ4SkTkikl9Eprmf92oR\nKRUry6YissN9T9+KFUsnEQl3Y57nea6bb3cR+RP4M57X0lxEtrqvY6mIVHLTlwANgQ/Fac69IdZ5\nk4AOwIvu/rtiv3acv/fOsf9m3PNFRAa4n+8/IjIz5u/c4z3rJCIRwJL4PgtzHVTVHunoAewC7ooj\nvTRwCQhwny8D9gKVgRzAbOAzd19x4Ahwr/v8bvd5AY9zdwM34vxQyRQrrwBgI/AOkB3ICtzm7uuE\n80VUGggEvgKmesQYDXzunlcN+Dvm9QDfA1088nkXGB3P+zA05roeacuATu52B+AC8ATOl9RwIAIY\nA2QBGgEngUD3+PeA/wPyADmBOcBr8eT9KTDc43l3YK67/TowNuZ9A25P4LOMBr4FcgMl3feicVyv\nL57P90+gjHv+NuB3nC/0AGAKMCFWXkvc11cC+MPjvWrhXquie+5LwM+xzl3gnpstjtdRETgN3OW+\n5ueB7UDm2J9LPO/DJOCVuD5bL/5m+gC/AEXdz/UjYHqscyfj/B/4T+z2SIbvJH8HYI9k/kCdQuYk\ncMx9fO2mx/Ul9LrHeZWBc+4X7gvAlFjXnQ887nFuaAIx3Aocjskr1r7FQFeP5xVxvuwDPP7TV/DY\n/ybwibvdBvjJ3Q4ADgJ14onBm0LmD4991dz3J9gj7QhQw90+DZT12FcP2BlP3ncDf3k8/wlo524P\nA77B6S9K7LOMBup5PP8CeCGu1xfP5zvQY/87wA8ez5sB62Pl1cjjeTdgkbs9F3jSY18AcAYo6XFu\ngwRexyBgpsdzAfYBd8b+XOI535tCJr6/mXCgoce+orH+3i4Bpf3xfzWjPKy5LH1qoar53cdDCRy3\n12M7AueXXjDOf742btPGMRH5F7gdKBLPubGVBCJUNTqOfcXcvDzzzQwUdp8rzheQ5/5i7vYcoLKI\nlAYaA8dV9dcE4kjMYY/tSABVPRIrLZeIFMSpda2LeU+AeUCBeK67DMghIje7sd6EUwsCeAvYASx0\nm3BeTEKMZ4FcXryuuM6NjON57GvF976XBkZ7vPajOJ9T8XjOje2qz1ydb/u9sc6/Hgn9zZQGvvGI\nPRy4yJW/N0g4dnOdbHRZ+uRtJ25Jj+3SOP/5juB8AUxV1S4JnJvQiJ69QCkRCYijoDng5hU738Nu\nPOL+G9O2X8o9B1U9LyKzgMdxmuo82+WTEl9SHcH5gq+qqgcTO1hVo904H8N5Xd+r6hl33xngOeA5\nt99mmYisUdVlSYzpDE7BF6NoEs+PS0ngN3e7NO77jvN5vqqqMxI4N6H3+wBOTTF2Xsn55R7n3wyw\nB6eWtDL2Ce4PALDRaT5lNZmMJXbh015EbhSRQJxmnC/dX5nTgAdEpLGIBIhIdhFpICLF/nPFuK3B\nacp6Q0QCRSSbiNzm7psBPCsiZUQkF/AaTlOKZ2E0WERyiEhV4Elgpse+z4COwAMkXMgcBsqIJGnk\nW5zHuu/JJ8Aot1aDiBQXkcYJXGsG8AhOQTP9cgYiTUWkvPv0FM5Q87hqfInZCNwpzv1PeYAB13CN\n2J4XkbwiUhLozZX3fRzwUsxgBhHJIyKtknDdWTiDChqKSGYReQ6nafY/X/zXSIj/b2Y88HrMQAUR\nKSgizWOda3zICpn0J6FfZRpr+zOcDuADOJ3zfQBUdR9OZ+9LwD84zQ/PceXvJcFffm6B8QBQAeeX\n5F6c/hSAiW6+y3Gajc7ifKF5+hH4C1gEvKWql0f9qOovOF/K69W59yc+X+J8gRwVkZgmtcR+scbe\n7/l8gBvTKhE5DizE6U+K+0Kqa3BqG0VxmtZiVAAWi8gp4GfgQ1WNb4RZvPGo6mKcPprNwFrgu0TO\nTYziNEeuA9a715vo5vV/wBvATPe1bwbu8zYvVf0TaA98gPP31BR4QFWjvIzVm/3x/c2Mdl/XQhE5\ngTMIoG4Srm2uk7idYb7LQORZoDPOF8MWnF8ZOXH+g5TGGaXURlVPuMcPxBmBFAX0UdWFbnotnFEg\n2XFG6vT1aeAmxbnNFzuBLPH058QctwT4XFUnplhwxphr4tOajNu80guopao1cPqAHsX5VbhYVSsB\nS4GB7vFVcH7xVgaaAGM9mjs+AjqrakWgojj3Qpj0J8HmCxG5GaiJ8yPFGJPKpURzWSYgp4hkxhmL\nvh+nKWaKu38K0NLdbo7TPh+lqrtxxtLXFZEiQG5VXeseN9XjHJO+xFu1FpHJOM1UfWI60o0xqZtP\nR5ep6gERGYnTLn8WWKiqi0WksKoedo85JCKF3FOKc3Vn4H43LYqrR6LsI/mGP5pUQlUjcH6UxLe/\nY8pFY4xJDr5uLsuLU2spjTNuPaeItCPhDlZjjDHphK/vk7kH567oYwAi8g1wG3A4pjbjNoX97R6/\nn6vv3SjhpsWX/h8iYgWWMcZcA1VN9iHdvu6T2QPc6t5nITjTbYTjzMfU0T2mA84QQ9z0tuJMMlgW\nuAFYo84swidEpK57nSc8zvkPf0+jkFoeQ4cO9XsMqeVh74W9F/ZeJPzwFV/3yawRkdnABpy7ujcA\nH+NM2DdLnBlxI3DvoVDVcPdO6ZipH7rrlVffg6uHMM/3ZezGGGOun8+nlVHVYTh3k3s6htOUFtfx\nI4ARcaSvA6one4DGGGN8xu74T8dCQkL8HUKqYe/FFfZeXGHvhe/5/I7/lCYimt5ekzHG+JqIoD7o\n+M8wszCXKVOGiIiIxA80aV7p0qXZvXu3v8MwxpCBajJuKe2HiExKs8/amKTzVU3G+mSMMcb4jBUy\nxhhjfMYKGWOMMT5jhUwaNmLECJ555plkPzYxAQEB7Ny5M1muZYzxr127ImjfPvatjMnHOv5TicmT\nJ/Puu++yY8cO8uTJQ8uWLRkxYgR58uTxd2j/kSlTJrZv3065cuX+sy8kJITVq1eTNWtWRIQKFSrQ\nqlUrnn32WbJmzerV9QMCAvjrr7/ivL43UvtnbUxqsWtXBI0ajWHH0SfheDXr+PeFmFK8YcOhtG8/\njF27kj7M+XqvMXLkSAYOHMjIkSM5efIkq1atIiIigkaNGhEVFRXnOZcuXUpynMkloS9wEWHs2LGc\nOHGCgwcPMnLkSGbOnMn999/v9fWvrFNnjPGl54Z8yI5qf0PHpr7LxN+TsvlgkjeNS1zpO3fu1vLl\n+yucVlCF01q+fH/duXN3nNeIy/Ve4+TJk5orVy6dPXv2VemnT5/WggUL6qRJk1RVNTQ0VFu1aqXt\n27fXPHny6IQJEzQ0NFTbt29/+ZwpU6Zo6dKlNTg4WIcPH65lypTRJUuWXD4/5tjdu3eriOiUKVO0\nVKlSWrBgQX3ttdcuX2fNmjVar149zZs3rxYrVkx79uypFy9evLxfRHTHjh1xvp6QkBCdMGHCVWl7\n9uzRwMBA/eGHHxK9/p133qkiojlz5tTcuXPrrFmz9N9//9VmzZppwYIFNX/+/NqsWTPdv39/vO9p\nfH8DxhhHdHS0DpszSeX5nMp9fZSsJ2P+3yT7d3KGrskMHjyZHTuGATndlJzs2DGMwYMnp9g1fvnl\nF86fP8+DDz54VXrOnDm5//77WbRo0eW0b7/9ljZt2nD8+HEee+wx4Mqv/vDwcHr06MGMGTM4ePAg\nJ06c4MCBA1ddM3YN4eeff2b79u0sXryYV155hT/++ANwmsNGjRrFsWPHWLlyJUuXLmXs2LFeviP/\nVbJkSerUqcOKFSsSvf6PP/4IwJYtWzh58iStW7cmOjqaTp06sXfvXvbs2UNgYCA9e/a85niMycjW\nRfxOqSENGb7gA6psaAfzX4MLuX2WX4YuZPbvj+ZK4RAjJ59/Ho0IXj0+/zzuaxw4EO1VDEeOHCE4\nOJiAgP9+FEWLFuXIkSOXn9erV48HHngAgOzZs1917FdffUXz5s2pV68emTNn5pVXXkkwXxEhNDSU\nrFmzUqNGDW666SY2bdoEQK1atahbty4iQqlSpXjmmWcuf/lfq2LFinHs2DGvr68eTXL58+fnwQcf\nJFu2bOTMmZOBAwdedzzGZDSRF8/RauwQ6n50B8VOPMyewav57uOXKF9+KOC71cwzdCFTvHgA/31z\nz9CuXQDqNn4l9mjXLu5rFCvm3VsbHBzMkSNHiI7+b6F08OBBgoODLz8vWbLkf46JceDAgav258iR\ngwIFCiSYd+HChS9vBwYGcvr0aQC2b9/OAw88QNGiRcmbNy8vv/zyVYXdtdi/fz/58+e/putHRkbS\npUsXypQpQ968eWnQoAHHjx+3zn1jvDT1p8UED67OwvXhzLp7I6vf70XRIpkoW7Y0ixb1ol27d3yW\nd4YuZIYP7xirFD9D+fJDGT68Y4pdo169emTLlo2vv/76qvTTp08zb9487rnnyooICXWIFy1alH37\n9l1+HhkZydGjR717EbF069aNypUrs2PHDo4fP85rr712XV/oe/fuZd26ddx5553XdP2RI0eyfft2\n1q5dy/Hjx1m+fDmQ8AAEYwzsOXaYm4a148lvnqJN0CiOjJ3Nw42KX3VM2bKlmTZtqM9iyDATZMYl\nphQfPPgdDhyIplixAIYP70XZsqVT7BpBQUEMGTKEXr16kTt3bu6++2727dtHjx49KFWqFO3bt/fq\nOq1ataJevXqsWrWK2rVrExoamuDxCX1Bnzp1iqCgIAIDA/n999/56KOPKFSokFdxeIqMjGTNmjX0\n69ePW2+9lSZNmnh1/SJFirBz587LQ5hPnTpFjhw5CAoK4tixY4m+NmMyumiNpv/0TxmzdRBljz/J\n1v7bqHxD7Gb9FOKL0QT+fJCE0WWpycSJE7VatWoaGBioRYoU0W7duunx48cv7w8NDdXHH3/8qnNi\np8WMFgsODtZXX31VS5QooT/99NN/jt29e7cGBATopUuXLp/bsGHDy6PCli9frjfeeKPmzp1b77zz\nTh06dKjecccdl48NCAhIcHRZjhw5NCgoSIOCgrRWrVo6YsQIPX/+/OVjErv++PHjtWjRopovXz79\n8ssv9eDBgxoSEqK5cuXSSpUq6ccff/yf+D2l9s/aGF9aunWzBr94m2btfquOnrnJ6/Pw0egyuxkz\nnTpz5gx58+blr7/+onRp72tm6UFG+6yNATh17iwPjX6FJccm0jjLcL4c8DS5c3nfI2KzMJtEff/9\n90RGRnLmzBn69+9PjRo1MlwBY0xGNOqHuQQPrcqm3XsJe2QL81/tkqQCxpd8GoWIVBSRDSKy3v33\nhIj0FpF8IrJQRP4QkQUiksfjnIEisl1EfhORxh7ptURks4j8KSKjfBl3WjVnzhyKFStGiRIl2LFj\nBzNnzvR3SMYYH/pt3wFuGNia55b0pnfZ8Rwe+zl31iqc+IkpKMWay0QkANgH3AL0BI6q6lsi8iKQ\nT1UHiEgV4HPgZqAEsBiooKoqIquBnqq6VkTmAqNVdUEc+VhzWQZnn7VJ76IuXaLzuI/4bO8wakZ1\n5dsXXqJ4oRzXdc30sPzyPcAOVd0rIi2ABm76FCAMGAA0B2aqahSwW0S2A3VFJALIrapr3XOmAi2B\n/xQyxhiTnn29cgMdZz9D9IVAprdYTtt7Kvs7pASlZCHzCDDd3S6sqocBVPWQiMSMXy0OrPQ4Z7+b\nFoVTC4qxz003xpgM4fDxUzzw7hB+PT+dtoXfYMqzHcmSJfVPJpsihYyIZMGppbzoJsVuy7C2DWOM\nicegaf/HG5t6UzLqbjb13Eb18sGJn5RKpFRNpgmwTlVj5g45LCKFVfWwiBQB/nbT9wOec6eUcNPi\nS4+T5816ISEhhISEXG/8xhiT4lb9tocHP+nFUf5gRL2pPN86JNmuHRYWRlhYWLJdLz4p0vEvIjOA\n+ao6xX3+JnBMVd+Mp+P/FpzmsEVc6fhfBfQG1gI/AO+r6vw48rKO/wzOPmuT1kWej6LNu+/zw4nX\naZC9D//33AvkyZXNp3n6quPf54WMiAQCEUA5VT3lpuUHZuHUTiKANqp63N03EOgMXAT6qOpCN702\nMBnIDsxV1T7x5JduCpm9e/dStWpVTpw44bOFvJ588klKliyZ6KzNaUla/KyNifHJ3DX0XtSFQAow\ns/1HNKpdIUXyTbM3Y6rqWVUtGFPAuGnHVPUeVa2kqo1jChh33whVvUFVK8cUMG76OlWtrqoV4itg\n0qoyZcoQGBhIUFAQuXPnJigoiEOHDlGyZElOnjx5uYBp2LAhEydOvOrcgIAAdu7c6bPYDh06xFNP\nPUWxYsXIkycPVapUYdiwYURGRvosT2Myop37T1D5uR50/bEFT1V5jn9GLkqxAsaXUsctoRmciPDD\nDz9w8uRJTp06xcmTJylSpIjX5/rKv//+S7169Th//jyrV6/mxIkTLFq0iBMnTrBjx44kX8+fS0Yb\nk1pduqR0HTOLCqOqkDlrFDv7hzPm6XYEBKT+kWPesEImlYireSciIoKAgACio6MZNGgQK1asoGfP\nngQFBdG7d28aNGiAqlKjRg2CgoL48ssvAWd6mZo1a5IvXz7q16/Pli1bLl9zw4YN1K5dmzx58tC2\nbVvOnTsXb0wjR44kKCiIzz777PJaNcWLF+fdd9+lWrVqV8UXw7O2NWXKFOrXr0+/fv0oWLAggwcP\nJl++fISHh18+/siRIwQGBl5eTyah2I1Jb+au3Elw3/uZGjGcT++dxZbXx1O6UD5/h5W8fDHrpj8f\npMFZmMuUKaNLliz5T3rs2ZJDQkIuz5QcQ0R0586dl5+vX79eCxUqpGvXrtXo6GidOnWqlilTRi9c\nuKAXLlzQ0qVL6+jRozUqKkpnz56tWbJk0cGDB8cZ16233qqhoaHxxh3XbM6eMU6ePFkzZ86sH374\noV66dEkjIyO1c+fOOmjQoMvHf/jhh9qkSZNEY0+K1PxZG6Oqeuz4Ba0/cITKiwX0wXfe0MjzSfsb\n9wV8NAtzhl5PxpMMS56qqQ69tg7nli1bkjmz83GEhIT8ZxGzBPP0qAV98skndO3alTp16gDw+OOP\n89prr7Fq1SoAoqKi6N27NwAPP/wwN998c7zXPXr0KEWLFk3ya/FUvHhxunfvDjhLRj/66KN06dKF\n4cOHAzB9+nS6deuWaOx33HHHdcVhTGqgCiOm/czQdV0onK00q59ey803lPV3WD5lhYzrWguH5DJn\nzhwaNmx43deJiIhg6tSpjBkzBnAKoIsXL3LgwAHA+dL3lNAszQUKFODgwYPXFU/sJaMbNmxIZGQk\na9eupVChQmzatImWLVt6Fbsxadn6347R8oMXOZhrLoNuH82QVg/7tE81tbA+mVTCszYSH2/+IEuW\nLMnLL7/MsWPHOHbsGP/++y+nT5/mkUceoWjRouzff/U9rHv27In3Wvfccw/ffPNNvPtz5nRW2jt7\n9uzltEOHDiUYc0BAAG3atGH69OnMmDGDZs2aXb5OQrEbk1adO6e0Hj6NOpOqUqJIdg68HM7Q1q0y\nRAEDVsikep6FT+HChf8zXDlmqeIYTz/9NOPGjWPNmjWAs3jZ3LlzOXPmDPXq1SNz5syMGTOGqKgo\nvv7668vHxaVfv36cPHmSDh06XC6M9u/fT//+/dm6dSvBwcEUL16cadOmER0dzcSJE70adfboo4/y\nxRdfMH36dB577DGvYjcmLZry/Z8UeLYRC0+P5OvWc/hl8BgKBuVJ/MR0xAqZVCChXzSe+/r06cOX\nX35JgQIF6Nu3LwBDhw7liSeeIH/+/MyePZvatWvzySef0LNnT/Lnz0/FihWZMmUKAFmyZOHrr79m\n0qRJFChQgC+//JKHH3443rzz5cvHL7/8QpYsWbjlllvIkycPjRo1Im/evNxwww2A04/y1ltvERwc\nzG+//cbtt9+e6OutW7cuOXPm5ODBgzRp0uRyekKxG5OWROw/T41ew+j08220rdWUoyPW0vLmuv4O\nyy9s+WWT7thnbfzl0iXo9/4yPozoRrnclfmu+/tUKloy8RNTgfSwnowxxqRbi37+h0cnPcfpgst4\n9/4x9G7cwt8hpQpWyBhjzHU49m80rV6bxI+ZXqJR1fZ82T2c3Nly+TusVMMKGWOMuQaq8PbkcAav\n7kqeAudY8sR8QirV9HdYqY4VMsYYk0SbtkXS8t1X2Vv4Y/reFcqbrbqSKSCTv8NKlayQMcYYL0VG\nQucRC/jidHdqlK/Dj902USpfMX+HlapZIWOMMV6YNucQXb95FoqvZnKbD3n81iaJn2QSLmREpB7Q\nHrgDKApEAltxVqacpqonfB5hMildunSGucM2o0toqhxjkmrP3mhaDB/P5gJDaHXbU0x6cgKBWQL9\nHVaaEe99MiIyDzgAzAF+Bf7GWZWyItAQeAB4V1W/TZlQvRPffTLGGJMUUVEw4L3NjN7RhSJFAvim\n03jqlKrm77B8JsWXXxaRYFU9kkhQiR6T0qyQMcZcryUrztB2XCgnykxhyO2v8VKTzgRI+p4gJcVv\nxowpPEQkJxCpqtEiUhG4EZinqhdTWwFjjDHX4+hReHTodyzN3ot6de5g9tNbKZyrkL/DStO8KZqX\nA9lFpDiwEHgcmOxtBiKSR0S+FJHfRGSbiNwiIvlEZKGI/CEiC0Qkj8fxA0Vku3t8Y4/0WiKyWUT+\nFJFR3r9EY4xJWHQ0vPPxPoo/+zBr8vfjqw6fsuLZz6yASQbeFDKiqmeBh4CxqtoaqJqEPEYDc1W1\nMnAT8DswAFisqpWApcBAABGpArQBKgNNgLFypbf+I6CzqlYEKorIvUmIwRhj4rRx8yVuaDeagRH/\n44km1Tg0ZAstqt/j77DSDW+GMIs7yqwd0NlN8+quIxEJAu5Q1Y4AqhoFnBCRFkAD97ApQBhOwdMc\nmOket1tEtgN1RSQCyK2qa91zpgItgQXexGGMMbGdPg3dhq9jxulnKFc9iI2df6Jq4Rv9HVa6401N\npi9OTeMbVd0mIuWAZV5evyxwREQmich6EflYRAKBwqp6GEBVDwExddLiwF6P8/e7acWBfR7p+9w0\nY4xJElX4fPZJinbqw+ysTXnvkT78MXCpFTA+kmhNRlV/BH70eL4T6J2E69cCeqjqryLyHk6NJfbw\nr2QdDhYaGnp5OyQkhJCQkOS8vDEmjdq5U2k99Bs2F+tDo7sa89kT2ygQWMDfYflFWFgYYWFhPs8n\noSHM35HAl7+qNk/04iKFgZWqWs59Xh+nkCkPhKjqYREpAixT1coiMsC5tL7pHj8fGApExBzjprcF\nGqhqtzjytCHMxpirXLgAL78dwejtPclbdgcz2o3j7hvu9HdYqYqvhjAn1Fz2DjAS2IVzp/8n7uM0\nkPgau4DbJLbXHfoMcDewDfgW6OimdcC54RM3va2IZBWRssANwBq3Se2EiNR1BwI84XGOMcbEa9GS\ni5R45G1GR9am94O3sm/QRitgUlCiK2OKyK+qWiextATOvwn4FMgC7ASexBk4MAsoiVNLaaOqx93j\nB+IMMLgI9FHVhW56bZyh09lxRqv1iSc/q8kYYzh8GDoMWsnSwC5ULVWULzuO5YYC5f0dVqqV4nf8\ne2T8G9DU7YvBrWHEDElOdayQMSZju3QJRo07zqBlA8lUdQ5jmo2kY522NndhIvy5/PKzQJiI7AQE\nKA10Se5AjDHmeq1bp7QZPpN9VfrTsnkLxrcJJ2/2vP4OK0NLtCYDICLZcKaTAfhdVc/7NKrrYDUZ\nYzKeEyeg15AdfHG6O4XKH+KL9uO5rdSt/g4rTfFbc5mb+W1AGTxqPqo6NbmDSQ5WyBiTcajCtBkX\n6DH9bS7Ueo8Bd7zIy3f1JUumLP4OLc3xW3OZiHyGM+R4I3DJTVacu+6NMcYv/vwTHhu4nK1lulK7\nSTmmt1tH6by2llBq402fTB2gilUPjDGpQWQkDB5xlA/+eJ7sNy1k6kOjaV31IevYT6W8mVZmK1DE\n14EYY0xi5s1TSreYwgfRVWnXKjd7XgynTbWHrYBJxbypyQQD4SKyBrjc4e/NHf/GGJMc9u+HTi/8\nzvKgbpRodJK57b6nTjGvbtUzfuZNIRPq6yCMMSYuUVHw7vvnCF0yAm7+kOF3D6bv7T3IHODNV5dJ\nDbwdXVYYuNl9ukZV//ZpVNfBRpcZkz6sXAnthyzhYK1u1K9YnYmtR1MiqIS/w0q3/HnHfxvgbZw1\nXwS4A3heVWcndzDJwQoZY9K2Y8egz0t/89Wp/uSssoKJrT7ggUrN/B1WuufPQmYT0Cim9iIiBXFW\ntbwpuYNJDlbIGJM2qcKkydH0nTqBqDtf5qmbOzCicSg5s+b0d2gZgj+nlQmI1Tx2FO9GpRljjFe2\nboUnntucFOCpAAAgAElEQVTK9kpdKdMyimmPLOKmIqnyd6xJIm8KmfkisgCY4T5/BJjnu5CMMRnF\nmTMw+JWzjPttOJlu+5Q37xtO15ufIUDsd2x64W3H/0NAfffpClX9xqdRXQdrLjMmbZgzB55+ax6R\nDXtwz4238FHL9yiSy27J8xd/9smUBQ6q6jn3eQ6gsKruTu5gkoMVMsakbrt3wzP9D7Aqb19yVVzH\npIfHcu8N9/o7rAzPHytjxvgSiPZ4fslNM8YYr124AK+9folqT37IzzVuotsjFdjRb6sVMOmcN30y\nmVX1QswTVb0gIll9GJMxJp0JC4NOL23k2O3PULlNNqa0+pEqBav4OyyTArwpZP4Rkeaq+i2AiLQA\njvg2LGNMevD339Dn+dN8f2YomZpO450mr9Op5pPWsZ+BeNMnUx74HCiOM8X/PuAJVf3L9+ElnfXJ\nGON/0dHw8cfw4qQ56H29aVYthNH3v0PBnAX9HZqJh18XLXMDyAWgqqeTlIHIbuAETr/ORVWtKyL5\ngC9wlnLeDbRR1RPu8QOBTkAU0EdVF7rptYDJQHZgrqr2jSc/K2SM8aMNG+DJvnvZW60Xucv9xqSH\nxtGwbEN/h2US4beOfxEpLCITgC9V9bSIVBGRzknIIxoIUdWaqlrXTRuAM2tAJWApMNDNqwrQBqgM\nNAHGypU5vD8COqtqRaCiiFhvoTGpyMmT0KtPFHe+8B477qlJr1Y1+aPPZitgMjhvGkYnAwuAYu7z\nP4E4axHxkDjyaQFMcbenAC3d7ebATFWNcodIbwfqikgRILeqrnWPm+pxjjHGj1Thiy+g/J1rmZ7r\nZmo+8j3ruv9CaMOhZMuczd/hGT/zppAJVtVZuMOYVTWKK8swe0OBRSKyVkSectMKq+ph93qHgEJu\nenFgr8e5+9204jh9QTH2uWnGGD/avh3uuv8E3X/oSfQjDzCqTT9+7LyYigUq+js0k0p4M7rsjIgU\nwCksEJFbcfpYvHW7qh50J9ZcKCJ/xFzLQ7J2ooSGhl7eDgkJISQkJDkvb0yGd+4cjHhDeXf+bDLd\n/ywP39SEtxuHkz9Hfn+HZrwUFhZGWFiYz/PxZnRZLWAMUA1nKeaCQCtV3ZzkzESGAqeBp3D6aQ67\nTWHLVLWyiAwAVFXfdI+fDwwFImKOcdPbAg1UtVsceVjHvzE+tGABPPPiLs7f3YM8pfYw4cFx1C9V\nP/ETTarmt45/VV0PNABuA7oAVb0tYEQkMGZUmojkBBoDW4BvgY7uYR2AOe72t0BbEcnqTmdzA84i\naYeAEyJS1x0I8ITHOcaYFLB/P7Rqc5FHP3yD461vpk/LO9jSc70VMCZBiTaXiUhrYL6qbhORQUAt\nEXnVLXwSUxj4RkTUzetzVV0oIr8Cs0SkE04tpQ2AqoaLyCwgHLgIdPeolvTg6iHM85P0So0x1yQq\nCj74AIZ++gvZWnWhTrkSjHtgDeXylfN3aCYN8Ka5bLOq1hCR+sBw4B1giKrekhIBJpU1lxmTfFat\ngqd7H+NozQFcKv8DY5q+R+sqrblyZ4FJL/w5QWbMSLKmwCeq+gNgc5cZk44dOwbPdFGavPA5e5tX\npWXzLPzZO5w2VdtYAWOSxJuazPc4Q4kbAbWASJx+klS5bJ3VZIy5dqowdSr0f307OVp1J2+xf/i0\nxXhuKZEqGy5MMvLnejKBwH3AFlXdLiJFgeox072kNlbIGHNttm2Drj3Os7PoW5ypPpohDV+i9y29\nyRzgzZ0OJq1L8UJGRHIlNk+ZN8ekNCtkjEmaM2dg+HD4aH4YOVp1pW75Snxw/xhK5Snl79BMCvJH\nn8wcERkpIne6w49jAiknIp1FZAFODccYk0Z9+y3cWOsIsy50JFf7xxnX6g2+fXSOFTAm2STYXCYi\n9wPtgNuBfDgzI/8B/ABMcO9fSVWsJmNM4iIioFdvZc2FyZy/YwAdaz3GKw1fIXe23P4OzfiJ36f6\nTyuskDEmfhcuwHvvwYhPfyNf+64UKHKWj5uPp1bRWv4OzfiZP4cwG2PSgR9/hJvqRPLJjkEEdL6D\n/k1as/rpVVbAGJ+yYSPGpHN//w3PPw9z/1hE5ke7Ub98TUbdt4niQTaRufE9ay4zJp2KjoZPPoGX\nRxwiuH0/zhVYydhmH3J/hfv9HZpJhfzaXCYi9UXkSXe7oDt5pTEmldq4EerdFs1bS8YT3aU6LRqU\nZFuPrVbAmBTnzQSZQ4E6QCVgEpAFmIYz4swYk4qcPAlDhsBnCzeT7/GuFC4E/9dsKdULV/d3aCaD\n8qYm8yDOsshnAFT1AGDjHI1JRVRh1iy4scYZlmZ6AelwNy807shPnX6yAsb4lTcd/xdUVd3p+vG8\nMdMY439//QU9esDv+j3arSfVK9zOosZbKZyrsL9DM8arQmaWiIwH8orI00An4BPfhmWMScy5c/Dm\nmzBq4n5KPt2HrEGb+LTpJzQq38jfoRlzmVejy0SkEc6qlgIsUNVFvg7sWtnoMpMRLFoE3XpcImfI\nh+wt9wo9b+nOwPoDyZElh79DM2mU3+/4F5EgPGo+qnosuYNJDlbImPTswAHo1w+Wb19PrrZdKFow\nkHFNx1G5YGV/h2bSOL8NYRaRLiJyCNgM/Aqsc/81xqSQqCh4/32oXucUOyv2JaptE15q1IOwDmFW\nwJhUzZs+meeAaqp6xNfBGGP+a80a6NJVuVj+G7L27UPVSvcwt9E2ggOD/R2aMYnyZgjzDuDs9WQi\nIgEisl5EvnWf5xORhSLyh4gsEJE8HscOFJHtIvKbiDT2SK8lIptF5E8RGXU98RiTFvz7L3TtCs3a\nRSCPtuBSyMvMbDONSS0mWQFj0gxvCpmBwC8iMl5E3o95JDGfPkC4x/MBwGJVrQQsdfNARKoAbYDK\nQBNgrFxZUPwjoLOqVgQqisi9SYzBmDQhZgnkytUusjnXO1x6qjYP3VKXjV020qBMA3+HZ0ySeNNc\nNh6nINgCRCc1AxEpAdwPvAb0c5NbADH/W6YAYTgFT3NgpqpGAbtFZDtQV0QigNyqutY9ZyrQEliQ\n1HiMSc3Cw6F7dziUeRW5+3chZ8FCrLp/JRUKVPB3aMZcE28KmSyq2i/xw+L1HvA8kMcjrbCqHgZQ\n1UMiUshNLw6s9Dhuv5sWBezzSN/nphuTLpw96yyB/PHU41Tp9RInsnzDyEYjebTao1ypzBuT9nhT\nyMwTkWeA74DzMYneDGEWkabAYVXdKCIhCRyarGOOQ0NDL2+HhIQQEpJQ1sb413ffOatUFm88iyx9\nn6Vq5eZ8e3c4+XLk83doJh0LCwsjLCzM5/kkep+MiOyKI1lVtVyiFxd5HWiPUxPJgTPn2Tc4E26G\nqOphESkCLFPVyiIywL32m+7584GhQETMMW56W6CBqnaLI0+7T8akCXv2QO/esGnPToI7dOdc1v2M\nbzae20re5u/QTAbkt/tkVLVsHI9ECxj33JdUtZR7fFtgqao+jlMr6uge1gGY425/C7QVkazucgI3\nAGtU9RBwQkTqugMBnvA4x5g05eJFeOstqFnnAqdrvc6ptnVpXecu1j+z3goYk+7E21wmInep6lIR\neSiu/ar69XXk+wbOnGidcGopbdxrhovILJyRaBeB7h7Vkh7AZCA7MFdV519H/sb4xYoV0K0b5Kyy\ngvwvdSVboTKsbbKWsvlsiSaTPsXbXCYioaoaKiKT4titqtrJt6FdG2suM6nRP//ACy/AghVHqdzr\nRf64NJ9R943i4coPW8e+SRV81VyWUMf/ZgBVfTK5MzUmo4iOhgkT4OVBSq2On3GpywtUqdCar+/a\nRp7seRK/gDFpXEI1mfWqWiuF47luVpMxqcWmTU7TWGTgH2R9sDsXM//L+Gbjubn4zf4OzZj/8FvH\nvzEmaU6dcmZKvue+cxR4OJS9jW/n0doPsObpNVbAmAwnoeayG0VkcxzpgtMnU8NHMRmTJqnCV19B\n375Q7YGl5BnQjcxFqrDhvg2UzFPS3+EZ4xcJFTK7gAdSKhBj0rIdO6BnT9j1999UHdSf384tZ8x9\nY2heqbm/QzPGrxIqZC6oakSKRWJMGnT+vLME8uj3o2nYfyLHMr9Es3JP8FXINnJlzeXv8Izxu4QK\nmZ9TLApj0qDFi53JLEvW3kb5V7uyN/MFFjZbyP+K/M/foRmTani9/HJaYaPLjK8dPOh07P+y9ix1\nn3uVsFOfMCxkGF1qdyFTQCZ/h2fMNbHRZcb42aVLMGYM1KgB0eXmk6lndQKCd7C562a639zdChhj\n4mA1GWO8sHats0pl1gIHydf2WX4/tYaxTcdy3w33+Ts0Y5KF32oyIrJORHqIiM07bjKc48edfpcH\nWlyi2pNj+atRDf5Xuhxbu2+1AsYYL3jTXPYIUAxYKyIzReRescmWTDqnCtOmQeXK8E+mjRQfchs7\nAqezrMMyXr/7dQKzBPo7RGPSBK+by0QkAGgGfARcAiYBo71ZvCwlWXOZuV6//+7UXo6eOk21nqEs\n+nsqr9/9Op1qdiJArBvTpE9+7fgXkRrASOBt4CugNXASWJrcARnjL2fPwssvwx13QPn7v+X4Y1XJ\nFPQ3W7tv5alaT1kBY8w18GZlzHXAcWAC8JWqnvfY97WqxrnejL9YTcZcix9+gF69oNpte7l4T292\nnNrGR00/4u5yd/s7NGNShK9qMt4UMuVUdWdyZ+wrVsiYpNi7F/r0gS3borjnpQ/48vCr9KzbkwH1\nB5A9c3Z/h2dMivHHejIxTojI+0B9QIGfgFdU9WhyB2NMSrl4EUaNcqaEebjXr+Rq/Ay/k4efO/1M\npeBK/g7PmHTDm0bmmcA/wMNAK3f7C18GZYwv/fQT1KoFC8JOcv8HvZmTsxnP1uvL0ieWWgFjTDLz\npiZTVFWHezx/VUQe8VVAxvjKkSPuEsgLlbbDvuKLE30pneNetnXfRoHAAv4Oz5h0yZuazEIRaSsi\nAe6jDbDAm4uLSDYRWS0iG0Rki4gMddPzichCEflDRBaISB6PcwaKyHYR+U1EGnuk1xKRzSLyp4iM\nSuoLNRlXdDR8+ilUrQrk202115sx79wQZjw8gwktJlgBY4wPedPxfwrICUS7SQHAGXdbVTUokfMD\nVfWsiGTCmdm5N07T21FVfUtEXgTyqeoAEakCfA7cDJQAFgMVVFVFZDXQU1XXishcnHt0/lPYWce/\n8bR5s7ME8sXoi9zW7z2m7XqLfvX68dxtz5E1U1Z/h2dMquG3jn9VzX09GajqWXczm5ufAi2ABm76\nFCAMGAA0B2aqahSwW0S2A3VFJALIrapr3XOmAi3xskZlMp5TpyA0FD77DJ4cspJ5mbrw2/mirH5q\nNeXzl/d3eMZkGN7ejNlcRN5xH82SkoHbxLYBOAQscguKwqp6GEBVDwGF3MOLA3s9Tt/vphUH9nmk\n73PTjLlKzBLIVarAgX//5b4Pu/DZ+Yd56Y6XmN9uvhUwxqSwRGsyIvIGTvPV525SHxG5XVUHepOB\nqkYDNUUkCPhGRKri1GauOiwJMScqNDT08nZISAghISHJeXmTSu3c6SyBvDtC6fjuDCbseY6WgS0J\n7xFO3ux5/R2eMalKWFgYYWFhPs/Hmz6ZzcD/3MICt29lg6rWSHJmIoOBs8BTQIiqHhaRIsAyVa0s\nIgNw+nnedI+fDwwFImKOcdPbAg1UtVsceVifTAZz/jy8/bZz38uT/f9iQ7Hu/BN5mPHNxnNriVv9\nHZ4xaYK/Fy3z/BmYJ96jYhGR4JiRYyKSA2gE/AZ8C3R0D+sAzHG3vwXaikhWESkL3ACscZvUTohI\nXXcG6Cc8zjEZ2JIlcNNNsOrX83SYOJxJmW7lvgqN+fXpX62AMSYV8OY+mRHABhFZBghwJ04nvTeK\nAlPcGZwDgC9Uda6IrAJmiUgnnFpKGwBVDReRWUA4cBHo7lEt6QFMBrIDc1V1vpcxmHTo0CHo3x9+\n/hmeee1HPjveFTlzA+ueWUfpvKX9HZ4xxuXVVP8iUhSnXwau1CxSJWsuS98uXYJx45yRY489dYRj\ntZ8nbO9iRt83mgdvfBBb6siYa5PizWXu4mStAFT1oKp+q6rfAvVFpFFyB2JMYn79FW65Bb6YpfSd\nMpmZBapSIFcewruH81Dlh6yAMSYVircmIyI/Ay1V9Z9Y6cHAd6paLwXiSzKryaQ/x4/DoEHO0OQ+\nw39nfuaunL5wmvHNxlO7WG1/h2dMuuCPjv9ssQsYAFU9gjMDgDE+pQqff+7c83Iu6hztJwzhnWP1\neajyQ6x+arUVMMakAQl1/AeJSGb37vvLRCQLkMO3YZmM7vffoUcPOHYMBnyymDE7u3HT6ZvY1HUT\nxYPsPlxj0oqEajJfA5+IyOVai4jkAsa5+4xJdpGRTtNY/frQsNlhbny5He9uf4r37n2P2W1mWwFj\nTBqTUCEzCDgMRIjIOncZ5l0468kMSongTMYyd64zU/Kf26N5/ouPGR1VjZJ5irOt+zaaVUzSbEbG\nmFTCmzv+c+DcFAnwl6pG+jyq62Ad/2nP3r3Qt68zY/Jzb21h6r9didZoxjUdx01FbvJ3eMZkCH67\n419VI1V1i/tI1QWMSVsuXoSRI6FmTahU/QzNx7zIoL/u4vEaj/Nzp5+tgDEmHfDmjn9jkt3PPzvr\nvBQpAiNmz+X1TT2od6YeW7ptoUiuIv4OzxiTTBJsLnPnCSuhqnvjPSiVseay1O3oUXjxRZg3Dwa/\ndYAlWfqw4dAGxjYdS+PyjRO/gDHGJ/zSXOZ+W89N7kxNxhMdDRMnOh37gbku8eyMDxi0vwaVgiux\npdsWK2CMSae8aS5bLyI3e6xKaUySbNniLoF8Ed6dsZ73/urCxh05WP7kcqoUrOLv8IwxPuTN6LLf\ncUaXRQBncGZi1mtZTyYlWHNZ6nH6tDOR5dSp8PKwU+wsM4SZ26Yz4u4RdPxfRwLE25UmjDG+5qvm\nMm9qMvcmd6YmfVOFb75xhiU3bAhvffd/DPmlN3edv4ut3bZSMGdBf4dojEkh3k71Xx+ooKqTRKQg\nkEtVd/k8umtgNRn/2rXLWQJ51y4IfW8Pn//bi9+P/M64puNoWLahv8MzxsTDb/fJiMhQ4EVgoJuU\nBZiW3IGYtO38eXjtNbj5ZritfhQdxo+k+6Za1C5am81dN1sBY0wG5U1z2YNATWA9gKoeEJHcPo3K\npCnLljkd+xUrwqfz1jBsfRcK7CrAys4rqVCggr/DM8b4kTeFzAVVVRFRAM8JM03GdviwswTyihUw\n4r0T/JztJbot/5p3Gr3DY9Ufs0XEjDGJN5cBs0RkPJBXRJ4GFgOf+DYsk5pdugRjx0K1alCsuPLK\n11/w/O4qXIy+yLbu22hXo50VMMYYwPuO/0ZAzN1yC1V1kVcXFykBTAUKA9HAJ6r6vojkA74ASgO7\ngTaqesI9ZyDQCYgC+qjqQje9FjAZyA7MVdW+8eRpHf8+tG4ddO0KOXLAwLd28v6OHuw9sZfxzcZz\ne6nb/R2eMeYa+a3j37UFWAEsd7e9FQX0U9WqQD2gh4jcCAwAFqtqJWAp7qACEakCtAEqA02AsXLl\nJ/FHQGdVrQhUFBEbWp2CTpyAXr2gaVPo2uMC9702gseX16VB6Qas77LeChhjTJy8GV32FLAGeAho\nBawSkU7eXFxVD6nqRnf7NPAbUAJoAUxxD5sCtHS3mwMzVTVKVXcD24G6IlIEyO0x68BUj3OMD6nC\njBnOEsgXLsDExT/x3tlarNiznLVPr2VA/QFkzZTV32EaY1Ipbzr+nwdqqupRABEpAPwCTExKRiJS\nBvgfsAoorKqHwSmIRKSQe1hxYKXHafvdtChgn0f6Pjfd+NAffzhLIB85AhNnHGP28Rd5euFc3rv3\nPVpXaW39LsaYRHlTyBwFTnk8P+Wmec1dtnk2Th/L6ZiRah6StRMlNDT08nZISAghISHJefl0LzIS\nRoxwOvdfflnJH/I5HZY8R6sqrQjvHk6e7Hn8HaIx5jqFhYURFhbm83zi7fgXkX7u5v+A6sAcnMKg\nBbBZVTt6lYFIZuB7YJ6qjnbTfgNCVPWw2xS2TFUri8gAnHnR3nSPmw8MxZk3bZmqVnbT2wINVLVb\nHPlZx/91mDfPuWO/dm3oFfonw37tztHIo4xvNp66xev6OzxjjI/4o+M/t/vYAfwfV2obc4CkTCkz\nEQiPKWBc3wId3e0O7jVj0tuKSFYRKYszMecaVT0EnBCRuu5AgCc8zjHJYN8+aNXK6dx/b8x5qnYf\nxoPf38b9Fe5n7dNrrYAxxlwTr4YwX/PFRW7nyog0dR8v4QwkmAWUxKmltFHV4+45A4HOwEWuHsJc\nm6uHMPeJJ0+rySRBVBS8/z68/rrT/1Lv0WX0WdyVysGVeb/J+5TKU8rfIRpjUoCvajIJNZeNUtW+\nIvIdcfSZqGrz5A4mOVgh471ffnGmgylUCIaP/IePdj7Hsl3LeL/J+7S80QbvGZOR+GOq/8/cf99J\n7kyNfx09CgMGwNy58PY70ZytNInmCwbSvkZ7tnXfRu5sNjWdMSZ5JFTI/AOgqj+mUCzGx6KjYcoU\nGDgQ2rSBr1eE8/yPXTm37hwL2i+gZtGa/g7RGJPOJNTx/38xGyLyVQrEYnxo61Zo0AA++gi+/i6S\n3C1epunsO2lTtQ0rO6+0AsYY4xMJ1WQ82+bK+ToQ4xunT8Mrr8DkyTBsGJS+ewGPz+/urPPSbTPF\nchfzd4jGmHQsoZqMxrNt0gBV+L//g6pV4eBBWLL6EMsLPUqPeV0Z02QMs1rPsgLGGONzCdVkbhKR\nkzg1mhzuNu5zVdUgn0dnrsnu3c79Ln/9BRMnRfNn7vHcNXsInWt2ZkLzCQRmCfR3iMaYDMKn98n4\nQ0YewnzhArzzDrz7rrOYWKP2m+i5oAuZAjIxruk4qheu7u8QjTGplL+n+jepXFgY3HQTrFwJP/5y\nhmO1n6fJzHvoXLMzK55cYQWMMcYvvJkg06Rihw/Dc8/B8uUwejQE3PgdTef2on6p+mzttpXCuQr7\nO0RjTAZmNZk06tIlZzhy9epQtCgsXLWPqecfov+ifnza/FOmPTTNChhjjN9ZIZMGrV8Pt90G06fD\noiWXKP7QaG7/7H9UK1SNLd22cE+5e/wdojHGANZclqacOAGDB8OsWc56L1Ub/UqnuV3InTU3P3X6\niRuDb/R3iMYYcxWryaQBqjBzprMEcmQkrNpwko1F+vDAzKb0qtuLZR2WWQFjjEmVrCaTyv35pzMF\n/99/w6xZyqF8X3PHzL40KteIbd23ERwY7O8QjTEmXlbIpFLnzjlNYh9+CC+/DA88vpu+C3uyY+MO\nPn/oc+4sfae/QzTGmERZc1kqtGABVKsG4eGwdv1Fouq+za0T63BriVvZ2GWjFTDGmDTDajKpyP79\n8OyzsG4dfPAB5K22khbfd6FIriKsemoVN+S/wd8hGmNMklhNJhWIioJRo+B//4Mbb4QVv/7LnEtd\neWjWQwysP5AF7RdYAWOMSZN8WsiIyAQROSwimz3S8onIQhH5Q0QWiEgej30DRWS7iPwmIo090muJ\nyGYR+VNERvky5pS2ahXUqQPffw8rViiVW8+gzsSqAIR3D+fR6o8ikuzTCRljTIrw6QSZIlIfOA1M\nVdUabtqbwFFVfUtEXgTyqeoAEakCfA7cDJQAFgMVVFVFZDXQU1XXishcYLSqLognzzQxQeaxY84K\nld9/70xqeXPjHXSf241Dpw8xvtl46pWs5+8QjTEZSJqcIFNVfwL+jZXcApjibk8BWrrbzYGZqhql\nqruB7UBdESkC5FbVte5xUz3OSXNUnSWQq1SBrFlh45YL7Cz+GrdOuIVG5Rqx7pl1VsAYY9INf3T8\nF1LVwwCqekhECrnpxYGVHsftd9OigH0e6fvc9DRn2zbo1s25ofKHH+BM8HIazOhKuXzl+PWZXymT\nt4y/QzTGmGSVGjr+U3/b1nU6cwZefBFCQqBtW/hh6VE+3NeJx756jOENh/Pdo99ZAWOMSZf8UZM5\nLCKFVfWw2xT2t5u+HyjpcVwJNy2+9HiFhoZe3g4JCSEkJOT6o/bCrl0RDB48mf37oylePIDhwzuy\neXNpeveGO+6AzZuVhYenUmP8i7Sp2obwHuEEZbMFRo0xKS8sLIywsDCf5+PzlTFFpAzwnapWd5+/\nCRxT1Tfj6fi/Bac5bBFXOv5XAb2BtcAPwPuqOj+e/PzS8b9rVwSNGo1hx45hQE7gDIGBQylcuBef\nflqaYjV+p9sP3Th5/iTjm42nTrE6KR6jMcbEx1cd/z6tyYjIdCAEKCAie4ChwBvAlyLSCYgA2gCo\nariIzALCgYtAd4/SogcwGcgOzI2vgPGnwYMnexQwADk5e3YYdW4dQZgEMHbiWAbfOZgedXuQOcDu\ngTXGZAw+/bZT1cfi2RXngieqOgIYEUf6OiBVrx+8f380VwoYV9lVfF/yI6L+bsDGrhspEVTCL7EZ\nY4y/2E/qZBIVFQCcAXJCzr/h3n5Qajl1j9/L149M93d4xhjjFz7vk0lpKd0nExkJffvCvHkRROv7\n7C9UFu5+BTY9StkIZcm8/pQtWzrF4jHGmGuRJm/GTO/Cw6FuXTh1CmaFnaLIwOUUaPQGdX57kHYF\n81sBY4zJ8Kwmcw1UYeJEGDAAXhlxlogyw5mw4VOGhQyjS+0u/H97dx5tVVnGcfz7A8UURUXDAQsH\nVJbmWOCUw8K6oJSUpUlWjjnjQKlYmg26lNTUHHIMzVQ0LUHUHFJ0pakYFy8CKjihXIVcCs6C8PTH\nfvFur5d7L3AO556zf5+17mLv9+x99vs+nHuf/e69z/t27tS5rMc3Myu1qny6rBa98w4cdRRMmgRn\njbqHEc8cS785/Wg4qoH1Vluv0tUzM+tQ3JNZAk89lX1jf6e6Rj7Y7UQmzv4vl+99OQN6DyjL8czM\nlhffk6mgiGy+l70GLWD3ky/jnxttw+Zf7M2koyc5wZiZtcI9mTa8+SYccgi89FE9nQcfSbeuK3HF\noFTPZc8AAAknSURBVCvYsseWJTuGmVmluSdTAY88Atv0fY/XtxnG7LoBHL/zkTx88MNOMGZm7eQb\n/y1YsADOPhsuvGc0Kx4xlC367MHddc/Qo2uPtnc2M7NP+XJZM42N8L1DX2X6pkPptslUrh78J/pv\n1L+ENTQz63h8uWw5uPOuT9j84Atp2Gk7jtl3OyYf97QTjJnZMvDlMmDePDj09PHc+tERfGXQmtx8\n4KNsvvbmla6WmVnVK3ySefrZudSN+CVze97GxYPP46idf4RU8h6jmVkhFTbJRATDrrmNPz5/Iv02\n25uxJ0xhrVW6V7paZmY1pZBJZkrjSwy85FhmffQKV+41isPrdq10lczMalKhbvzPXzCfYbePYOtL\n+9L93V1pPLPeCcbMrIwK05N5dMZj7H/DkfzvhZ6cvdOTnHL4xvjWi5lZedV8knn7w7cZdvdwbqkf\ny9oT/sDE8/dniy2cXczMloeqSjKSBgIXkV3muzYiRrS03UsvvcLpZ4ykfn4DL/S+nxWnf5cha0/m\n0tvXYOWVl2uVzcwKrWruyUjqBFwKDAC2BIZI6tPStnvs+1tu6vQIU7u/yLzr72TVh3tw+s/nFi7B\njBs3rtJV6DAciyaORRPHovyqJskA/YBpEfFKRMwHRgGDW9pwRt0dMH0QXPUUzNyDWbN+wxlnXLc8\n69oh+BeoiWPRxLFo4liUXzVdLusJvJpbf40s8XzelfUw98u5gq40Ni4sY9XMzKwl1dSTab+5azUr\neJ/116/NppqZdWRVMwqzpB2BX0fEwLQ+HIjmN/8lVUeDzMw6mHKMwlxNSaYz8BywJ/A68CQwJCKm\nVrRiZma2WFVzTyYiFkg6DriPpkeYnWDMzDqwqunJmJlZ9amZu+GSBkp6VtLzkk6tdH3KQdIGkh6U\nNFnSJEnHp/I1Jd0n6TlJ90paPbfPaZKmSZoqqS5Xvr2khhSviyrRnmUlqZOkCZLGpPVCxgFA0uqS\n/pbaN1nSDkWMh6STJD2T2nCjpC5FioOkayXNktSQKytZ+1M8R6V9/iMp/xhvyyKi6n/IkuV0oBew\nIjAR6FPpepWhnesC26blVcnuUfUBRgCnpPJTgXPT8hZAPdll0Q1TjBb1Xp8A+qblu4EBlW7fUsTj\nJOCvwJi0Xsg4pLpfBxySllcAVi9aPID1gReBLmn9FuCgIsUB+DqwLdCQKytZ+4GjgcvT8g+AUW3V\nqVZ6Mu3+omY1i4g3ImJiWn4PmApsQNbW69Nm1wPfScv7kH0IPomIl4FpQD9J6wKrRcT4tN1fcvtU\nBUkbAHsD1+SKCxcHAEndgF0jYiRAaudcihmPzkBXSSsAKwMzKVAcIuLfwNvNikvZ/vx73Ub2IFar\naiXJtPRFzZ4VqstyIWlDsjOWx4F1ImIWZIkI6JE2ax6XmamsJ1mMFqnGeF0InAzkbyoWMQ4AGwFv\nShqZLh9eJWkVChaPiGgELgBmkLVpbkQ8QMHi0IIeJWz/p/tExAJgjqRWZ3uslSRTKJJWJTuLOCH1\naJo/vVHTT3NIGgTMSr261p7rr+k45KwAbA9cFhHbA+8Dwyne52INsjPtXmSXzrpKOpCCxaEdStn+\nNr9XUytJZiaQvwG1QSqrOekywG3ADRExOhXPkrROen1dYHYqnwl8Kbf7orgsrrxa7ALsI+lF4Gag\nv6QbgDcKFodFXgNejYin0vrtZEmnaJ+LbwAvRsRb6Sz7H8DOFC8OzZWy/Z++lr672C0i3mrt4LWS\nZMYDvSX1ktQFOAAYU+E6lcufgSkRcXGubAxwcFo+CBidKz8gPRGyEdAbeDJ1medK6idJwE9y+3R4\nEfGLiPhyRGxM9n/9YET8GLiTAsVhkXQp5FVJm6WiPYHJFOxzQXaZbEdJX0j13xOYQvHiID7bwyhl\n+8ek9wDYD3iwzdpU+mmIEj5VMZDsaatpwPBK16dMbdwFWED29Fw9MCG1uzvwQGr/fcAauX1OI3tq\nZCpQlyv/KjApxeviSrdtGWKyO01PlxU5DtuQnWxNBP5O9nRZ4eIBnJna1EB2g3rFIsUBuAloBD4m\nS7qHAGuWqv3ASsCtqfxxYMO26uQvY5qZWdnUyuUyMzPrgJxkzMysbJxkzMysbJxkzMysbJxkzMys\nbJxkzMysbJxkrOpJWijpvNz6zyT9qkTvPVLSvqV4rzaO831JUyT9q1l5L0kfpDHJ6tO/K0j6tqRT\nlmcdzZZG1cyMadaKj4F9JZ0TbQxxsTxJ6hzZ8CbtcRhweEQ81sJr0yMbkyzvzvSzTJawjmZLzD0Z\nqwWfAFcBw5q/0PwsX9K76d/dJY2TdIek6ZLOkfRDSU9IejoNs7HINyWNVzYp3qC0fydJv0/bT5T0\n09z7PiJpNNnQLs3rMyRNBtUg6ZxUdgbZPCDXShrRQvs+NwihpIMkXVKKOkpaRdLY1FNqkLRfq9E2\nWwLuyVgtCOAyYNJi/kg333aRrckmfZtDNtnV1RGxg7IZR4fSlLR6RURfSb2BhyRtQjZ+05y0fRfg\nUUn3pe23A7aMiBn5A0taDzg3vT4HuF/SPhHxO0n9gWERUd9CnTeRNCEtPxoRQ1toy1LXMSXhmRHx\nrVTP1dqIoVm7OclYTYiI9yRdD5wAfNjO3cZHxGwASS+QjesE2ZhNe+S2uzUdY3rarg9QB2yVO+vv\nBmwKzCcbZPAzCSbpCzy06JKepBuB3WgazHVxw6a3dLmsuWWp4yTg/NSzuiuyia/MSsJJxmrJxWSD\nho7MlX1CuiycRpTtknvt49zywtz6Qj77u5HvMSitCxgaEffnKyBpd7L5XBanzfk3ltJS1zEipkna\nnmym0bMkPRARZ5WpnlYwvidjtUAAEfE22Rn9YbnXXga+lpYHk43Ku6T2U2YTslkonwPuBY5RNr8P\nkjZVNhtla54EdpPUXdlcHEOAce04fnsS01LXMV3G+zAibgLOI5uLxqwk3JOxWpA/i78AODZXdjUw\nWlI92R/dxfUyWhuOfAZZglgNODIi5km6BtgQmJB6SLNpYx74iHhD0nCaEsvYiBjbjuO3NVR6LGMd\ntwLOk7QQmAcc3cbxzNrNQ/2bmVnZ+HKZmZmVjZOMmZmVjZOMmZmVjZOMmZmVjZOMmZmVjZOMmZmV\njZOMmZmVjZOMmZmVzf8BsH31HqS8w/EAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import sympy as sym\n", + "%matplotlib inline\n", + "plt.plot(fiber_number,fiber_copy_time,'-o',label='Original Data')\n", + "plt.plot(fiber_number,func(fiber_number,*popt),label=\"Fitted Curve\")\n", + "plt.legend(loc='upper left')\n", + "xs = sym.Symbol('x') \n", + "#tex = sym.latex(func(fiber_number,*popt)).replace('$', '')\n", + "#plt.title(r'$f(x)= %s$' %(tex),fontsize=16)\n", + "plt.title(\"Fiber copy time vs number of fiber\")\n", + "plt.xlabel(r\"Number of Fibers\")\n", + "plt.ylabel(r\"Fiber Copy Time (seconds)\")" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEZCAYAAABFFVgWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFX3wPHvDU1a6L2DVHlBqiAICcWCiPoTAQUUsAPW\nUBWliQgI+r6KDSUgvYiKgkrvVYrSe5MmvZeU8/vjTmCJSdiUzWyS83mePOzOzs6c3Q17MvfcYkQE\npZRSyhcC3A5AKaVU6qVJRimllM9oklFKKeUzmmSUUkr5jCYZpZRSPqNJRimllM9oklFxMsYsNMZ0\ncjuO5GCM6W2M+drtOPyRm78Hxpg7jDE/G2POGmOmxLJPqDHmtDFmlTGmvjFmm8dj+4wxjZIvYuUp\nvdsBqPgzxjwNvAlUAM4DG4EPRGS5F8+NBO4Ukb2+jdK/GWMaAuNFpFjUNhEZ7GJIKnYtgXxALolh\nYJ8xpj7QGCgsIledzRWTMT4VB72SSWGMMW8BI4D3gfxAcWAk8IiXh9DRt5ZB3wtXGGNMPJ9SAtgZ\nU4JxlAT2eyQYnzDGpPPl8VMtEdGfFPIDBAIXgP+LY59awArgDHAY+BRI7zy2GIgELmKvgJ4EcgI/\nA/8Ap5zbRTyOtxDo5Nw2QB9gP3AMGAMEeuz7jPPYCWe/fUCjWOK8Axju7H8GWAJkch5rAWwGTgML\ngAoez9sH9AK2OPGOBjI6j20CHvbYN70TS9Vo584CXAbCnffzPFAQ6AuMc/Yp4bxXHYCDzrleAmoC\nfzqxfRrtuJ2Arc6+vwLFY3nts4HO0bZtBB5zbn8MHAfOOeeqFMtxFgIDgGXOa/gNyO081hA4FG3/\nG5+H81qnAuOc5/4JlHXe2+PAAaBptHN9AKx24voByOnxeB1gufNZbgAaRnvu+06cl4DSMbyWCs5+\nZ5zP8RFnez/gGnDdibNjDO/5FSDMebxv9Ncew+/Mt1G/M87jzZ2Yzzgx/ifac3s4788VIMDt74GU\n9uN6APoTjw8LHnD+s8X6iw5UB2pjE0Jx5z/Wax6PRwKlPO7nBh4HMgFZgSnADx6PeyaZTsBO7Bdw\nFuB74DvnsUrYL+y62C/3Yc6XQ2xJZiQ2gRR0Yq0DZADKYZNgIyAd0B3Yxc1EuQ/4CyiMTZDLgAHO\nY92ByR7neBT4M5bzNwQORtvW1+P1RCWZz4GMQBPnS2YGkMc5/3HgPo9z7XTiDwDeBpbHcu72wDKP\n+5WwSSsDcD+wFsjuPFYeKBDLcRY6700Z5/NbiG02je31RU8yl53XFQCMBfYCvZ33/Xlgb7RzHcI2\nQ2UGpnMzIRcBTgIPOPcbO/fzeDx3PzaRBADposWV3nkdPZ3bwdiEUTb65xLL+/AssCS2z/Y2vzPV\nnM+xJvb3sL2zfwaP5653npvJ7e+AlPjjegD6E48PC54GjsTzOa8D33vcjySGvyQ9Hr8bOOVx3zPJ\nzANe9nisHDaRBADvAhM8HstMLEnG+c98Gagcw2N9uDVRGOBvoIFzfx/wgsfjDwG7nNuFsH9lZ3Pu\nTwO6xfI6vUkyEUBBj8dPAk963J+Ok8CxVycdPR4LwP7VXiyGc2fDJuRizv33gW+c28HAduAewNzm\ns10IvO1x/xVgdhyvL3qS+d3jsebYL3bjEWMkzpUqHgnMuV8RuOp8Pj2AsdHO9RvQ3uO5/eJ4HfWj\n/14DE4H3on8usTzfmyQT2+/M50D/aMfbzs0/HvYBzybk/6v+2B+tyaQsp4C8xphYPzdjTFmnJ85R\nY8xZYBCQN479MxtjvjLG7Hf2XwzkjKXdvDC2GSXKAexfngWcxw5FPSAiV5x4Y5IX+5d3TJ0PbjmH\n2P/ph7B/LUf5O1oMhZ19j2KbbJ4wxuTAfplMiCUGb/3jcfsK9q9ez/vZnNslgP86PZxOY1+7RIsb\nJ86L2KTUxtn0VFScIrIQ+Ax7pXfcGPOlMSZb9GN4OOZx+7JHPN6I/lpOOu931H2iHe+Qx+0D2Cuv\nvNjX3irqtRtjzgD1sFepMT03usIxPH6AGN67RIjxdwYbe0i02It6PB79uSqeNMmkLCuxVwePxbHP\nF8A2oIyI5ATewf61GZsQbFt8LWf/Bs72mJ5zBPufMkoJbF3jOHAU+5/TPtmYzNhmpZicxP4VXMaL\ncwAU49b/6MU8bpdwnhPlO2yTx5PACifxxERi2Z5Qh4CXRCS385NLRLKJyKpY9p8EPG2MqYNthll4\nIzCRz0SkJrYZrTy2GTC+LmGbNIEbRet8CTiOp+jvexj2szyEvdLwfO3ZRWSYx/5xvd9Hoh0bbFPv\n4UTG6ym235lDwKAYPjfPrtJJ/buSpmiSSUFEJKqwOdIY86hzFZLeGPOQMeZDZ7fswHkRuWyMqYBt\nQvF0DCjtcT879q/W88aY3NhCa2wmAW8aY0o6f10PwjZtRWKbjh4xxtQxxmSI6zjOX8ujgRHGmELG\nmACP500FHjbGBDuvrRs2Ia30OEQXY0wRJ963gckej/2IrUu9hk04sTkO5DHGBMaxT3x6QX0JvG2M\nqQRgjMlhjGkZx/6zsV92A7B1MJzn1TTG1DbGpMd+LlexzVbxtRO4w/ndSI9thsyYgON4ameMqWCM\nyQL0B6Y5n+V47Gd/v/NZ3mGMaWiMKRz34W5YDVw2xvRwPvMgbPPdpETG6ym235lRwMvGmNoAxpis\nxphmxpisSXjuNE2TTAojIiOAt7BfGv9gez51xn65AnQD2hpjzgNfcesXMNgv/++cpoGW2J5MWbB/\nka7AfvndckqP26OxvZGWAHuwzTOvOXFtBV7FfmEewbbv/4O98opJN2wvorXYpqUPsR0adgLtsE1G\nJ4CHsT2Nwj2eOxGYA+zGFowHebw/V7EdEkphi/QxEpEd2C+xvc57UTCm3by9LyI/Oq9hstPs+Bfw\nYBznv+7E19h5PVECsV98p7H1gJPYThQxHiaO45/H/l58i70KvED8m30k2u1x2A4CR7AJ63XnXH9j\nOz68jf3MDmA/34AYjhNTrGHYLvjNsK/3M2w9Z1c8443rdcT4OyMi64AXgM+cZs6d2BqP53NVIkQV\n+XxzcGO+xf5FclxEqjjbcmG/iEpge5y0EpFzzmO9sT2YwoHXRWSOs706trvsHdjC5hs+C1olCecv\nwbPYgZ8Hbrd/PI67D3hORBbEsc+72J5JzyTVeZVSCePrK5lQbLdbT72AeSJSHtuFtTeA08zQCttr\n5SHgc4/i8xfYL5ZyQDljTPRjKj9gjGnuNOFlxY6B+SspE4yXMeQGnsNexSmlXObTJCMiy7ADnDw9\nir3kxvk3qojdAtu+Hy4i+7GXtLWdZozsIrLW2e874i58K/c8im1K+Rtb1G8T9+4JEuultzHmeWzz\n4SzxYoodpZTvuTF3WX4ROQ4gIseMMfmd7UW4tbh72NkWzq1tyX+TtF0bVRIRkRew7du+PEfpOB77\nBvjGl+dXSsWPPxT+tbCmlFKplBtXMseNMQVE5LjTFBY12O0wt/ZlL+psi217jIwxmrSUUioBRCS+\nk5feVnJcyRhuHW8wEzvpINiugj95bG9jjMlojCkF3AmsEZFjwDln7IDBTsL4E3FwexoFf/np27ev\n6zH4y4++F/pe6HsR94+v+PRKxhgzEQjCDno7iB1I+CEwzVkA6QC2RxkistUYMxU7i20YdpbaqFfe\nhVu7MP/my7iVUkolDZ8mGRF5OpaHmsSy/2DgXwtHiR0w9Z8kDE0ppVQy8IfCv/KRoKAgt0PwG/pe\n3KTvxU36XvieT0f8u8EYI6ntNSmllK8ZYxAfFP7d6F3mipIlS3LgQLIOPlcuKVGiBPv373c7DKUU\naehKxsnSLkSkkpt+1krFn6+uZLQmo5RSymc0ySillPKZNFOTUUop9W8RkREs2BfryhmJplcyKdjg\nwYN58cUXk3zf2wkICGDv3r1JciyllHvm7Z1Hja9r0H9xf5+dQwv/fmLMmDGMGDGCPXv2kCNHDh57\n7DEGDx5Mjhw53A7tX9KlS8euXbsoXfrfEyIHBQWxevVqMmbMiDGGsmXL0rJlS958800yZvRu9d+A\ngAB2794d4/G94e+ftVJu2/LPFrrP7c6u07sY0mQIj1d4nICAAC38+8K+fQdo164/wcF9adeuP/v2\nxb+bc2KPMXz4cHr37s3w4cM5f/48q1at4sCBAzRt2pTw8PAYnxMRERHvOJNKXF/gxhg+//xzzp07\nx9GjRxk+fDiTJ0+mWbNmXh//5lp1SqmkdOziMV76+SWCxwbzQJkH2NJ5C9XuqEH79gN8d1K3J2Xz\nwSRvEpOYtu/du1/KlAkRuCggAhelTJkQ2bt3f4zHiElij3H+/HnJli2bTJ8+/ZbtFy9elHz58klo\naKiIiPTr109atmwp7dq1kxw5csi3334r/fr1k3bt2t14ztixY6VEiRKSN29eGThwoJQsWVLmz59/\n4/lR++7fv1+MMTJ27FgpXry45MuXTwYNGnTjOGvWrJG6detKzpw5pXDhwtK1a1cJCwu78bgxRvbs\n2RPj6wkKCpJvv/32lm0HDx6ULFmyyKxZs257/AYNGogxRrJmzSrZs2eXqVOnypkzZ6R58+aSL18+\nyZ07tzRv3lwOHz4c63sa2++AUmnVpeuXZMCiAZJnSB7p9ns3OX35tIhE//5CxAffyWn6Subdd8ew\nZ09/IKuzJSt79vTn3XfHJNsxVqxYwbVr13j88cdv2Z41a1aaNWvG3Llzb2ybOXMmrVq14uzZszz9\ntJ0WLuqv/q1bt9KlSxcmTZrE0aNHOXfuHEeOHLnlmNGvEJYvX86uXbuYN28eAwYMYMeOHYBtDvvk\nk084ffo0K1euZMGCBXz++edeviP/VqxYMWrWrMnSpUtve/zFixcDsGnTJs6fP8+TTz5JZGQknTp1\n4tChQxw8eJAsWbLQtWvXBMejVFoRERnBmI1jKPdpObac2MKaF9Yw7P5h5MqcC4C3347+/ZX00nSS\nOXw4kn+/uVmZMCESY/DqZ8KEmI9x5EikVzGcPHmSvHnzEhDw74+iUKFCnDx58sb9unXr8sgjjwBw\nxx133LLv999/T4sWLahbty7p06dnwIC4L3+NMfTr14+MGTNSpUoVqlatyp9//glA9erVqV27NsYY\nihcvzosvvnjjyz+hChcuzOnTp70+vng0yeXOnZvHH3+cTJkykTVrVnr37p3oeJRK7ebvnU/NUTUZ\ntX4U01tNZ3LLyZTOVRoRWLcOunaF6dNj+v5KWmk6yRQpEgBcirb1Em3bBiBO49ftftq2jfkYhQt7\n99bmzZuXkydPEhn576R09OhR8ubNe+N+sWLF/rVPlCNHjtzyeObMmcmTJ0+c5y5QoMCN21myZOHi\nxYsA7Nq1i0ceeYRChQqRM2dO3nnnnVuSXUIcPnyY3LlzJ+j4V65c4aWXXqJkyZLkzJmThg0bcvbs\nWS3uKxWDLf9s4eGJD/PSLy/R574+LOu4jDpF63DiBHzyCdx9N7RsCfnzw8MPx/T9lbTSdJIZOLAD\nZcr05eabfIkyZfoycGCHZDtG3bp1yZQpEzNmzLhl+8WLF/n1119p0uTmqghxFcQLFSrE33//feP+\nlStXOHXqlHcvIppXXnmFihUrsmfPHs6ePcugQYMS9YV+6NAh1q1bR4MGDRJ0/OHDh7Nr1y7Wrl3L\n2bNnWbJkCRB3BwSl0prjF4/z8i8vEzw2mKalm7K1y1YeLfcEs2YZ/u//oGxZWL/eJpo9e+C99+Dj\nj6N/fyW9NJ1kSpUqwdy5r9K27UcEB/elbduPmDv3VUqVKpFsxwgMDOS9997j1Vdf5ffffyc8PJz9\n+/fTunVrihcvTrt27bw6TsuWLfn5559ZtWoVYWFh9OvXL8794/qCvnDhAoGBgWTJkoXt27fzxRdf\neBVDdFeuXGHx4sU89thj1KlTh4ceesir4xcsWPCWcTgXLlwgc+bMBAYGcvr06du+NqXSksthl3l/\nyfvc9fldZM2QlR1dd/Bgzjd49+2MFCsGH3wADz0EBw7Ad99BcDBEtc57fn/5jC96E7j5Qzx6l/mT\n0aNHS+XKlSVLlixSsGBBeeWVV+Ts2bM3Hu/Xr5+0b9/+ludE3xbVWyxv3rzy/vvvS9GiRWXZsmX/\n2nf//v0SEBAgERERN54bHBx8o1fYkiVLpEKFCpI9e3Zp0KCB9O3bV+67774b+wYEBMTZuyxz5swS\nGBgogYGBUr16dRk8eLBcu3btxj63O/5XX30lhQoVkly5csm0adPk6NGjEhQUJNmyZZPy5cvL119/\n/a/4Pfn7Z61UUoiIjJAxG8ZI0RFFpdW0VrLxwB756iuROnVEChUS6dFDZNs274+Hj3qX6WDMVOrS\npUvkzJmT3bt3U6KE91dmqUFa+6xV2rNg3wJC5oSQOX1mns47nNXT6/Lzz9C4MXTqBA88AOnjOWmY\nr2Zh1iSTivzyyy80btyYyMhIQkJCWLt2LevWrXM7rGSXFj5rlTZtO7GN7nO7s+noNu65OIQ1Y54g\nR6ChUydo2xY8+gnFmy5apm7rp59+on379gDUrFmTyZMnuxyRUiopHL94nD7z+zF10/cU2Nmbi799\nT4HWmZjxPVSrZodT+Cu9klGpjn7WKrW4dP0y3aZ/wpgdI5CNz1Avog8vPZObFi0g2lC5RNMrGaWU\nSiOOHI3krTHj+f5MHzKdvIfOZVbz5sgyFC3qdmTxp0lGKaX8wPXrMGsWDJu2kDW5QsidIxP/azCZ\nlx++16+bw25Hk4xSSrlo0yYIDYWxs7dBkx6ku2sLoQ8OoV31lqliRnKtyahURz9r5e/OnIFJk2xy\n+fvMPxR+uh/7skyjT8PedKnVhUzpMyV7TFqTUUqpFCwiAubPt4nl11+hyUNXqPzKJ+w7OZwGVdoz\np8F28mSJe77BlChNTyvj7w4dOkRgYKBP/yrv2LEj7733ns+Or1Rat3s39OkDpUrB22/DvfUiGTx7\nHGtql+dCtnWsen4VHz/4capMMKBJxi+ULFmSLFmyEBgYSPbs2QkMDOTYsWMUK1aM8+fP32iXDQ4O\nZvTo0bc8NyAg4JZ5vpLasWPHeP755ylcuDA5cuSgUqVK9O/fnytXrvjsnEqldBcvwpgx0LAh3Hsv\nXLoEv/wCH01fxJiMtRi7dSSTnpjE9FbTuTP3nW6H61OaZPyAMYZZs2Zx/vx5Lly4wPnz5ylYsKDX\nz/WVM2fOULduXa5du8bq1as5d+4cc+fO5dy5c+zZsyfex3NzyWilfE0Eli2D556DYsXg++/hjTfg\n77/hpXe28+7WR+n4U0d63NuDlc+tpF7xem6HnCw0yfiJmJrEDhw4QEBAAJGRkfTp04elS5fStWtX\nAgMDee2112jYsCEiQpUqVQgMDGTatGmAnV6mWrVq5MqVi/r167Np06Ybx9ywYQM1atQgR44ctGnT\nhqtXr8Ya0/DhwwkMDGTcuHE31qopUqQII0aMoHLlyrfEF8Xzamvs2LHUr1+ft956i3z58vHuu++S\nK1cutm7demP/kydPkiVLlhvrycQVu1L+6PBhGDwYypeHF16AChVg61b4+Weof/8J3pzbhftC76NB\n8QZs77Kd1pVbp4peY17zxaybbv6QAmdhLlmypMyfP/9f26PPlhwUFHRjpuQoxhjZu3fvjfvr16+X\n/Pnzy9q1ayUyMlK+++47KVmypFy/fl2uX78uJUqUkP/+978SHh4u06dPlwwZMsi7774bY1x16tSR\nfv36xRp3TLM5e8Y4ZswYSZ8+vYwcOVIiIiLkypUr8txzz0mfPn1u7D9y5Eh56KGHbht7fPjzZ61S\nh6tXRaZOFXnoIZFcuURefFFk5UqRyEj7+OXrl2Xw0sGSZ0geef3X1+XkpZPuBuwFfDQLs/Yuc5j+\nSfOXhfRNWJH+scceI70zbWpQUNC/FjGL85weV0GjRo3i5ZdfpmbNmgC0b9+eQYMGsWrVKgDCw8N5\n7bXXAHjiiSeoVatWrMc9deoUhQoVivdr8VSkSBE6d+4M2CWjn3rqKV566SUGDhwIwMSJE3nllVdu\nG/t9992XqDiUSgobNtjeYRMnQtWq0LEjTJ8OWbLYxyMlkol/TeLtBW9Ts3BNVj63krJ5yrobtMs0\nyTgSmhySyk8//URwcHCij3PgwAG+++47Pv30U8AmoLCwMI4cOQLYL31PcS0DkCdPHo4ePZqoeKIv\nGR0cHMyVK1dYu3Yt+fPn588//+Sxxx7zKnal3HDypE0qo0fD2bPQoQOsXWt7i3lavH8xIXNCSBeQ\njgn/N4H6xeu7Eq+/0STjJzyvRmLjTTtusWLFeOedd+jdu/e/HluyZAmHDx++ZdvBgwe5886Ye7c0\nadKEH374gb59+8b4eNasWQG4fPky2bJlA2xvtLhiDggIoFWrVkycOJECBQrQvHnzG8eJK3alklN4\nOPz+u71qmTcPmjeHESMgKOjmqpJRdpzcQc95Pfnz+J8MbjyYVne1IsBouTuKvhN+zjP5FChQ4F/d\nlaMvVfzCCy/w5ZdfsmbNGsAuXjZ79mwuXbpE3bp1SZ8+PZ9++inh4eHMmDHjxn4xeeuttzh//jzP\nPvssBw8eBODw4cOEhISwefNm8ubNS5EiRRg/fjyRkZGMHj3aq15nTz31FFOmTGHixIk8/fTTXsWu\nVHLYsQN69YISJWDgQLj/frts8fjx0KjRrQnmxKUTdJ3dlfqh9alXrB7bumyjTeU2mmCi0XfDD8R1\nheL52Ouvv860adPIkycPb7zxBgB9+/blmWeeIXfu3EyfPp0aNWowatQounbtSu7cuSlXrhxjx44F\nIEOGDMyYMYPQ0FDy5MnDtGnTeOKJJ2I9d65cuVixYgUZMmTgnnvuIUeOHDRt2pScOXPeuPoZNWoU\nQ4cOJW/evGzbto169W7fLbN27dpkzZqVo0eP8tBDD93YHlfsSvnK+fPwzTdQr569UomMhLlzYdUq\nePFFyJHj1v2vhl9lyLIhVBxZkQATwLYu2+herzt3pE/iufdTCa/nLjPGZAWuikiSDHYwxrwJPAdE\nApuAjkBWYApQAtgPtBKRc87+vYFOQDjwuojMieW4EtNr0vms0g79rNXtREbCkiW2Oeynn+xVSseO\n8OCDkCFDLM+RSCZvnszb89+meqHqfNjkQ8rlKZe8gftQsi+/bIwJANoAbYFawDUgE3ASmAV8JSK7\nE3RSYwoDy4AKInLdGDMFmA1UAk6JyFBjTE8gl4j0MsZUAiY4cRQF5gFlY8ommmSUftYqNgcPwtix\nNrlky2YTS9u2kD9/3M9bcmAJ3eZ0A2D4/cO5r0Tq6+3oxgSZC7Ff5r2BzSIS6QSSGwgGhhhjfhCR\n8Qk8dzogqzEmEsgMHHbO1dB5fCywCOgFtAAmi0g4sN8YswuoDaxO4LmVUmnElSvw44+2d9iGDdC6\nNUydCjVq3H7Z4p2ndtJzXk82HN3A4MaDaV25tdZc4imuJNNERMKibxSR08D3wPfGmFguLOMmIkeM\nMcOBg8BlYI6IzDPGFBCR484+x4wxUX9fFAFWehzisLNNKaX+RcR2Mw4NtQmlZk14/nl49FHvli0+\nefkk/Rf1Z9LmSfSo14NJT0zSmksCxZpkohKMMaYM8LeIXDPGBAFVgO9E5GxMScgbxpicwKPY2ss5\nYJoxpi0QvY0jQW0e/fr1u3E7KCiIoKCghBxGKZXCHD9ue4KFhsLVq7Y5bONGO5eYN66GX+V/q//H\nsBXDaHNXG7Z33U7eLHl9G7RLFi1axKJFi3x+ntsW/o0xG4GaQEls3eQn4C4RaZbgkxrTEnhARF5w\n7rcH6gCNgCAROW6MKQgsFJGKxphe2CkPhjj7/wb0FZF/NZdpTUbpZ522hIXB7Nk2sSxaBI8/bpPL\nfffdvjksSqREMmXzFHrP7021QtX4sPGHlM9b3qdx+xs3Fy2LFJFwY8zjwKci8qkxZkMiz3sQqGOM\nuQPboaAxsBa4CHQAhgDPYhMawExggjHmY2wz2Z1A7AM8lFKp3pYtNrGMHw9ly9rEMm4cZM8ev+Ms\nPbCUkDkhCMLYx8bSsGTD2z9Jec2bJBNmjHkK+6X/iLMtQbWYKCKyxhgzHdgAhDn/fg1kB6YaYzoB\nB4BWzv5bjTFTga3O/p1jvFyJQ4kSJdLWzKdpWFxT5aiU7ezZm8sWHzkCzzxjuyKXS0BP4l2ndtFz\nXk/WHV3H4MaDdSClj3jTXFYJeBlYKSKTjDGlsONXhiRHgPEVW3OZUiplioy8uWzx7Nl2FH6nTtC0\nKaRLF//jnbx8koGLBzJh0wS639ud1+55jcwZMid94ClMso+TSak0ySiVOuzda1eXHDsW8uSxieWp\np+zthLgafpXP1nzGkOVDaH1Xa/o27Eu+rPmSNOaULNlrMsaYTcTRu0tEqiR1MEqptO3SJbuiZGgo\nbN5sB0rOnGmn1U8oEWHKFlvUr1qgKss6LktzRX03xVWTae7828X5d5zzbzsS2LVYKaWiE4GVK21i\nmT7dziHWtaud+ThTpsQde9nBZYTMCSEiMoLQR0MJKhmUJDEr73lTk9kgItWibVsvItV9GlkCaXOZ\nUinDkSO2N1hoqL3fsSO0bw+FCyf+2LtP76bnvJ6sPbyWDxp/wNP/eVqL+rfhq+Yyb951Y4yp53Hn\nXi+fp5RSt7h+3TaHPfwwVK4Mu3fb6V62bYOePROfYE5dPsUbv71BnW/qUKtwLXZ03UG7Ku00wbjI\nmy7MzwGjjTE5AAOcwc6GrJRSXtm48eayxZUr2yL+1KngrFeXaNfCr/HZms/4cPmHtKrUiq1dtpI/\n621mvVTJ4rZJRkTWAVWdJEPU1PtKKRWXU6dsUgkNtbc7dIDVq6F06aQ7h4gwdctUes/vTeX8lVna\ncSkV8lZIuhOoRPOmJpMJeAI7rcyNpCQiA3waWQJpTUYp90REwJw5NrHMmWObxTp2/Peqkklh+cHl\nhMwJISwyjI+afkRwqeCkPUEa4+a0Mj9hJ7Fch50CRimlbrFz580xLUWL2sTy9deQM2fSn2vP6T30\nnNeTNYfXMKjRINpWaas1Fz/mTZIpKiIP+jwSpVSKcuECTJtmC/e7d0O7dvbq5a67fHO+01dOM3Dx\nQMb9NY5BHkb2AAAgAElEQVSQuiGMe3ycjtRPAbxJMiuMMf8RkU0+j0Yp5ddEYOlSm1h++gkaNoTu\n3aFZs9iXLU6sa+HXGLl2JIOXDebJSk9qUT+F8aYmsxU76/E+bHOZwU6775cj/rUmo1TSO3TINoWN\nGWMX/erUyV653G7Z4sQQEaZvnU6v+b2olK8SQ5sMpWK+ir47YRrnZk3moaQ+qVLK/129apctDg2F\nP/6wyxZPmmRXmfT1hOYrDq2g25xuXA2/yqhHRtGoVCPfnlD5jFcTZBpjqgL3OXeXisifPo0qEfRK\nRqmEE4F162ximTwZatSwRfzHHoPMyVD+2HN6D73n92bl3ysZ1GiQDqRMRq6N+DfGvA5MAPI7P+ON\nMa8mdSBKKfecOAEffwxVqtgrloIFYcMGW8h/6infJ5jTV07z1u9vcc8391C1QFV2dN3BM1Wf0QST\nCnhTk/kLqCsil5z7WbFry2hNRqkULCwMfv3VXrUsXAiPPmprLffdl/RjWmJzLfwan6/9nMHLBvNE\nxSfoF9SPAtkKJM/J1S3crMkYIMLjfoSzTSmVAm3denPZ4tKlbWIZOxYCA5MvBhHh+23f03NeTyrm\nrciiDouolK9S8gWgko03SSYUWG2M+cG5/xjwre9CUkoltXPnbI0lNBQOHoRnn4VFi6C8C8uqrDy0\nkpA5IVwJv8LXzb+mcenGyR+ESjbeFv6rA/Wdu0tFZINPo0oEbS5TyoqMtM1goaHwyy92ueKOHe3y\nxem9+fMyie09s5fe83uz4tAK3g9+n/ZV22vNxY+4tvyyMaYOsEVELjj3A4GKIrI6qYNJCppkVFq3\nf78dzzJmDOTKZRPL009D3rzuxHPmyhkGLR3EmI1jeKPOG7xV9y2yZMjiTjAqVm6uJ/MFcNHj/kVn\nm1LKT1y+bGssjRpBrVpw+jT88IPtIfbaa+4kmOsR1/lk1SeU/6w8F69fZHPnzfRp0EcTTBrjVeHf\n89JARCKNMS5cbCulPInYqfNHj7bLFtepA6+8Ai1aJH7Z4sTFJczYNoOe83pSPm95Fj67kLvy+2hC\nM+X3vEkWe40xr3Hz6qUzsNd3ISml4nL06M1liyMibO+wTZugSBG3I4PVf68mZE4IF65f4MvmX9Kk\ndBO3Q1Iu86Ymkx/4H9AIEGA+8IaI/OP78OJPazIqNbp+3RbvQ0Nh2TJ44glba7n3Xt9P8eKNfWf2\n0Xt+b5YdXMb7jd6nfZX2pAtI53ZYKh5cK/ynNJpkVGry1182sUyYAJUq2cTSsmXSLVucWGeunOGD\npR8weuNo3rjHFvWzZvST4FS8uDYY0xhTDttUVkBEKhtjqgAtROT9pA5GKWWL9pMm2VrLiRN2TMvK\nlVCmjNuR3XQ94jpfrP2CD5Z9wGPlH2NL5y0UzFbQ7bCUH/KmuWwx0B34SkSqOds2i0jlZIgv3vRK\nRqVEEREwd669avn9d3joIVtradQI0vlRq5OI8MP2H+g5rydlc5dlaNOhVM7vl18FKp7cnFYmi4is\nMbc2/IYndSBKpUW7d9vE8t13dlLKTp3gyy/t+BZ/s+bwGkLmhHD+2nk+b/Y5Tcs0dTsklQJ4k2RO\nGmPKYIv+GGNaAkd9GpVSqdjFi3bZ4tBQ2LHDLv71669Q2U8vCPaf3U/v+b1ZemApA4MH8kzVZ7So\nr7zmTXNZaeBr4F7gDHaFzHYist/n0SWANpcpfyRie4WFhtpBkg0a2CJ+s2aQMaPb0cXs7NWzfLD0\nA77d8C2v3/M6IXVDtKifirneu8yZ4j8ganoZf6VJRvmTv/+2TWGhoTaZdOxor1wK+nGN/HrEdb76\n4yveX/o+Lcq1YEDwAAplL+R2WMrH3Oxd9jp2JuYLwChnssxeIjInqYNRKjW4ehVmzrS9w9asgVat\n7JQvtWv7x5iW2IgIP27/kZ7zelImdxnmtZ/Hfwr8x+2wVArnTU2mk4j81xjzAJAHaA+MAzTJKOUQ\ngfXrby5bfPfdtoj/ww/Js2xxYq09vJaQOSGcvXqWz5p9xv1l7nc7JJVKeLtoGUAz4DsR2WKMP/89\nplTyOXHCDpQMDYXz521z2Lp1UKKE25F5Z//Z/bw9/20WH1jMgKABdLi7gxb1VZLyJsmsM8bMAUoB\nvY0x2YFI34allP8KD4fffrOJZf58OyHlJ59Aw4bJt2xxYp29epbBSwfzzYZveK32a3z9yNdky5jN\n7bBUKuRNknkOuBvYKyKXjTF5gI6JPbExJgfwDVAZm7Q6ATuBKUAJYD/QSkTOOfv3dvYJB17XmpBK\nbtu33xzTUqqUvWoZPRpy5HA7Mu+FRYTx1bqvGLhkII+Ue4RNr2yicPbCboelUrFYe5cZY0rG1U3Z\naTIrIiJ/J+jExowBFotIqLN0QFbgbeCUiAw1xvQEcolIL2NMJWACUAsoCswDysbUjUx7l6mkdP48\nTJlik8mBA9C+PXToABUruh1Z/IgIM3fMpMe8HpTMWZJhTYdRpUAVt8NSfiTZuzAbY6ZhFzX7CVgH\nnADuAO4EgoHGQF8RmRvvk9rVNTeISJlo27cDDUXkuDGmILBIRCoYY3oBIiJDnP1+BfrFtDqnJhmV\nWJGRsHixTSw//wyNG9urlgcfdGfZ4sT648gfhMwJ4fSV03zU9CMeuPMBt0NSfijZuzCLyJPOFURb\nbDNVIeAysA2YDQwSkasJPG8p7EwCoUBV4A/gDewknMed8x9zlhkAKAKs9Hj+YWebUknmwIGbyxYH\nBtrEMmIE5MvndmQJc+DsAd5Z8A4L9i1gQPAAOt7dUYv6KtnF+XeZiGwF3vHReasDXUTkD2PMx0Av\nnKlrPEPwwbmVuuHKFZgxw9ZaNm6Ep56C77+HatX8e0xLXM5dPcfgZYMZtX4Ur9Z+lS+bf6lFfeUa\nty7+/wYOicgfzv3vsUnmuDGmgEdzWdTCaIeBYh7PL+psi1G/fv1u3A4KCiIoKCjpIlcpnogdJBka\nClOnwj33wIsv2l5id9zhdnQJFxYRxtfrvmbgkoE8XPZh/nr5L4oE6gW/itmiRYtYtGiRz8/j2qJl\nzhICL4jITmNMXyCL89BpERkSS+H/Hmwz2Vy08K/i6dgxO/I+NNSuNNmxIzzzDBQt6nZkiSMi/Lzz\nZ3rM7UGxHMX4qOlHVC1Y1e2wVArj5lT/vvIaMMEYkwHYi+0WnQ6YaozpBBwAWoFttjPGTAW2AmFA\nZ80kyhthYTBrli3iL10Kjz9up9KvXz/lNod5+uPIH3Sb042Tl0/yyYOf8ECZB9Cx0sqfeDMLs8EW\n/0uLyABjTHGgoIisSY4A40uvZBTApk03ly0uX95O8dKyJWRLJaWJg+cO8s6Cd5i/dz79g/rTsVpH\n0gekwK5vym+4eSXzOXawZCNgAHaizO+xY1aU8htnzthli0ND4ehRO55l+XK48063I0s656+d58Nl\nH/LVuq/oUqsLO7ruIHum7G6HpVSsvEky94hIdWPMBgAROWOM8dMVMFRaExFhp3YJDbULfz34ILz/\nPjRp4l/LFidWWEQYo9aPYsDiATQr20yL+irF8CbJhBlj0nFzZcx86NxlymV79tjxLGPHQv78tog/\nciTkzu12ZElLRPhl5y/0mNeDItmL8Fu737i74N1uh6WU17xJMv8DfgDyG2MGAS2BPj6NSqkYXLoE\n06fbIv62bdC2LfzyC1RJpbOjrDuyjm5zu/HPpX8Ycf8IHrzzQS3qqxTHqy7MxpgK2GlkDDBfRLb5\nOrCE0sJ/6iICK1bYxDJjhu0V1rEjNG/uv8sWJ9ahc4d4Z8E7zN07l/5B/elUrZMW9ZXPud2F+Tiw\n1Nk/szGmuoisT+pglIpy+LCd7XjMGDt9fqdOsHUrFErFqwCfv3aeIcuG8OW6L+lcszM7u+7Uor5K\n8bxZfnkg0AHYw81pXgTb20ypJHPtml22ODQUVq2CJ5+0NZd77kkdY1piEx4Zzqh1o+i/uD8P3vkg\nf778J0UDU/gIUaUc3lzJtALKiMh1Xwej0qYNG2ximTTJ1lc6drS1lyxZbv/clExEmLVrFt3ndqdw\n9sL82vZXqhWq5nZYSiUpb5LMZiAnN+cRUyrRTp6EiRNtreXsWTumZc0auxhYWrDh6AZC5oRw7OIx\nPmr6Ec3KNtOivkqVvBnxXxO7psxm4FrUdhFp4dvQEkYL//4rPBzmzLGJZd48W7zv2BGCg1POssWJ\ndejcIfos7MOcPXPo17Afz1V/Tov6yi+4WfgfCwwBNqHjY1QC7Nhhm8PGjYNixWxi+fbblLVscWJd\nuHaBIcuH8MUfX/BKzVfY0XUHgZkC3Q5LKZ/zJslcFpH/+TwSlaqcP2+n0Q8Nhb17oV07mDsXKlVy\nO7LkFR4Zzjfrv6H/4v7cX+Z+Nr60kWI5it3+iUqlEt40l43ANpPN5NbmMr/swqzNZe6JjIQlS2xi\n+eknaNTo5rLFGTK4HV3yEhFm75pN97ndKZitIB/d/xHVC1V3OyylYuWr5jJvkszCGDaLiPhlF2ZN\nMsnv4EHb1Tg01M5y3LGjHY2fP//tn5sabTy2kW5zunH4wmGGNR3Gw2Uf1qK+8nuuJZmURpNM8rhy\nBX780Rbx16+HNm1scqlRI3WPaYnL3+f/ps+CPvy2+zf6NuzLCzVe0KK+SjGSvfBvjGknIuONMW/F\n9LiIjEjqYJR/E4E//rCJZepUqFkTnnsOfv45ZS9bnFgXrl1g6PKhfP7H57xc42V2vrpTi/pKOeL6\nMytqKJzOa5HGHT9+c9niq1ftFcvGjbanWFoWHhnOt+u/pd/ifjQt3ZQNL22geI7iboellF+JK8lk\nBBCR/skUi/IjYWEwe7ZNLIsW2WWLP/8c7rsv7TaHRRERft39K93ndid/1vzMenqWFvWVikWsNRlj\nzHoRSXH/c7QmkzhbttjEMn48lC1rr1qefBKy6/UsYIv63ed259C5QwxrOozm5ZprUV+lCm7PwqxS\nsbNnYfJkW2s5cgSeecZ2RS5Xzu3I/Mfh84fps7APv+76lfcavscL1V8gQ7o01i9bqQSIK8lUMcac\nj2G7wXZh1spmChYZCQsW2MQyezbcfz8MGABNm6auZYsT6+L1iwxdPpSRa0fyUo2X2NF1BznuSENT\nFSiVSHElmU0iolPCpjJ7995ctjhPHtsc9umn9ra6KTwynNANofRd1JfGpRtrUV+pBNLmsjTg0iX4\n/ntba9m82Q6UnDkTqlZ1OzL/IyL8tvs3us/tTt4seZn51ExqFq7pdlhKpVhxJZlpyRaFSnIisHKl\nTSzTp0O9etC1q535OFMmt6PzT38e+5Puc7tz4NwBhjUdxiPlHtGivlKJpCP+U5kjR+xsx6Gh9n7H\njtC+PRQu7G5c/uzIhSP0WdCHWbtm8V6D93ixxota1FdpjvYuU7G6ft2Ouh89GlasgJYt7e26dXVM\nS1wuXr/IsOXD+GztZ7xQ/QV2dt2pRX2lkthtk4wxJp2IRCRHMCp+/vzTJpOJE6FyZXvVMnUqZM3q\ndmT+LSIygtCNtqgfXDKY9S+up0TOEm6HpVSq5M2VzC5jzPdAqIhs9XVAKm6nTtmkEhpqbz/7LKxe\nDaVLux1ZyvD77t/pNrcbuTPn5sfWP1KrSC23Q1IqVfNmqv/sQBugIxAAjAYmi0hMY2hclxprMhER\ndtni0FD778MP26uWRo3SzrLFibXp+Ca6ze3G/rP7GdpkKC3Kt9CivlIe/GKqf2NMQ2AikBOYDgwU\nkd1JHVRipKYks3OnHdPy3XdQpIhNLG3aQM6cbkeWchy5cIT3Fr7Hzzt/5t0G7/JSjZe0qK9UDFwr\n/Btj0gEPY69kSgLDgQnAfcBsQCcfSUIXLsC0abbWsnu3Xbb499/hrrvcjixluXT9EsNWDOPTNZ/y\nfLXn2dF1Bznv0OysVHLzqiYDLASGicgKj+3TjTENfBNW2iICS5faxPLTT9CwIXTvDs2apb1lixMr\nIjKCMRvH8N6i92hYoiHrXlxHyZwl3Q5LqTTLm5pMNhG5mEzxJFpKai47dMhO7zJmjF30q2NHe+VS\noIDbkaVMc/bModucbuS4IwfD7x9O7SK13Q5JqRTDzXEy+Y0xk4C6QCSwEnhTRPYmdTBpwdWrdtni\n0FC7ymTr1jBpkl1lUuvQCbP5n810m9ONPWf2MLTJUB6r8JgW9ZXyE94kmYnASOBx534bYBJwj6+C\nSqn27TvAu++O4fDhSIoUCWDgwA6UKlUCEVi3ziaWyZOhRg171fLjj5A5s9tRp1xHLxzlvYXvMXPn\nTPrc14eXar5ExnQZ3Q5LKeXBmySTRUTGedwfb4zp7quAUqp9+w7QtOmn7NnTH8gKXGL58r48/fSr\nzJxZgsuXoUMH2LABiutkvoly6folPlrxEf9b8z+eq/acFvWV8mPe1GSGAGeAyYAArYFcwDAAETmd\n4JMbEwD8AfwtIi2MMbmAKUAJYD/QSkTOOfv2BjoB4cDrIjInlmO6UpNp164/EyZ0wyaYKJcoVeoj\nRo/uS4MGOqYlsSIiIxj751jeXfguDUo0YHDjwVrUVyqJuFmTaeX8+1K07W2wSScxY81fB7YCUQug\n9QLmichQY0xPoDfQyxhTyYmjIlAUmGeMKetPFf7DhyO5NcEAZKVkyUiCglwIKJWZu2cu3eZ2I3vG\n7MxoNYN7imprrVIpwW2TjIiU8sWJjTFFgWbAIOAtZ/OjQEPn9lhgETbxtMDOMhAO7DfG7AJqA6t9\nEVtCFCkSAFwi+pVM4cJ6+ZIYm//ZTPe53dl9ejdDmgzh8QqPa1FfqRTktt+AxpgMxpjXjDHTnZ+u\nxpikGL3xMdAdezUUpYCIHAcQkWNAfmd7EeCQx36HnW1+Y+DADhQv3hebaAAuUaZMXwYO7OBaTCnZ\nsYvHePHnF2k0thEP3fkQWzpv4f8q/p8mGKVSGG+ay74AMgCfO/fbO9ueT+hJjTEPA8dFZKMxJiiO\nXf2mOex2SpUqwZNPvsrMmR9RtGgkhQsHMHDgq5QqpbP7xsel65cYsXIEn6z+hE53d2JH1x3kypzL\n7bCUUgnkTZKpJSKeC/UuMMb8mcjz1gNaGGOaAZmB7MaYccAxY0wBETlujCkI/OPsfxgo5vH8os62\nGPXr1+/G7aCgIIKSoSgiArNmlWDMmL7ce6/PT5fqRERGMO6vcfRZ0If6xeuz9oW1lM6lU0sr5SuL\nFi1i0aJFPj+PN73L1gNPisge535pYLqIVE+SAOykmyFO77KhwCkRGeIU/nOJSFThfwJ2bE4RYC4Q\nY+Hfrd5l69fbxcL27NFBlfE1b+88us3pRtaMWRl+/3DqFK3jdkhKpTlu9i7rDiw0xuwFDLZ7ccek\nDsTxITDVGNMJOIDTs01EthpjpmJ7ooUBnf2pZxnA+PHQtq0mmPjY8s8Wus/tzs5TOxnSZIjWXJRK\nhbya6t8Ykwko79zdISLXfBpVIrhxJRMRAcWKwYIFUKFCsp46RTp28Rh9F/blh+0/8PZ9b9O5Vmcd\nqa+Uy5L9SsYY0w6bhMY5SeUvZ3t7Y0yEiExM6mBSqgUL7HovmmDidjnsMiNWjuDjVR/ToWoHLeor\nlQbE1Vz2KtA4hu0zgCXYOc0UtqmsXTu3o/BfkRLJuD/H0WdhH+4tdq8W9ZVKQ+JKMhlimuJfRC4l\n0TiZVOHyZZg5E4YMcTsS/zR/73y6ze1G5vSZmdpyKnWL1XU7JKVUMooryWQ2xmQVkUueG40x2QFt\nQHfMnAn33AMFC7odiX/ZemIrPeb2YNvJbQxpMoQnKj6hRX2l0qC4Rvx/i1398sZoQmNMSexEmd/6\nNqyUQ5vKbnX84nFe/uVlgsYE0bhUY7Z23krLSi01wSiVRsV6JSMiHxljLgJLjDHZnM0XgQ9F5Itk\nic7PnTgBy5bZNWLSusthl/l45cd8vOpjnqn6DNu7bid35txuh6WUclmc42RE5EvgS6eJDBG5kCxR\npRBTp8LDD0O2bLffN7WKlEjG/zWedxa8Q92idVn9/GrK5C7jdlhKKT/hzWBMTS6xGD8e3nvP7Sjc\ns2DfArrN6Uam9JmY0nIK9xbT+XSUUrfyajBmSpJcgzF374Z69eDwYUjvVapOPbad2EaPeT3Y8s8W\nhjQZojUXpVIBXw3G1MVOEmjiRGjdOm0lmH8u/UPnWZ1pMKYBwSWD2dZlG0/e9aQmGKVUrG77FWmM\n+b8YNp8DNonIPzE8luqJ2Kay8ePdjiR5XAm7wserPmbEyhG0r9KeHV13aFFfKeUVb/4Ofw6oCyx0\n7gcB64BSxpgBIjLOR7H5rbVr7b+1arkbh69FSiQT/prAOwveoXaR2qx6fhV35r7T7bCUUimIN0km\nPVAxasVKY0wB4DvstPtLgDSXZKLGxqTmVqJF+xcRMieEDAEZmPTEJOoVr+d2SEqpFMibJFMsKsE4\n/nG2nTbGhPkoLr8VFgZTpsDy5W5H4hvbT26nx9webPpnEx82/pBWd7XSmotSKsG8STKLjDG/ANOc\n+y2dbVmBsz6LzE/NmwelS8OdqazV6J9L/9B/UX+mbp1Kr3q9mPbkNDKlz+R2WEqpFM6bJNMF+D+g\nvnN/LPC900842FeB+avUNo3MlbArfLLqE4avHE67Ku3Y3mU7ebLkcTsspVQq4e2iZQWA2oAAa/y5\nV5kvx8lcvAhFi8KuXZAvn09OkWwiJZKJmybyzoJ3qFm4Jh82/pCyecq6HZZSyiWuLb9sjGkFDAMW\nYZdf/tQY011Epid1MP7uxx+hfv2Un2AW719MyJwQ0gWkY8L/TaB+8fq3f5JSSiWAN81l7wC1oq5e\njDH5gHlAmksy48dDhw5uR5FwO07uoMe8Hvx1/C8GNx5M67taa1FfKeVT3oz4D4jWPHbKy+elKseO\nwerV0KKF25HE34lLJ+g6uyv1Q+tTv1h9tnXZRpvKbTTBKKV8zpsrmd+MMb8Dk5z7rYHZvgvJP02Z\nYhNMlixuR+K9q+FX+e+q/zJsxTDa/qct27psI2+WvG6HpZRKQ26bZESkuzHmCSBqNN7XIvKDb8Py\nP+PHwwcfuB2FdyIlkkmbJvH2grepUagGK59bqUV9pZQrdBZmL+zYAcHBcOgQpEuXpIdOcksOLCFk\nTggGw/D7h3NfifvcDkkplQIke+8yY8wFbJflfz0EiIgEJnUw/mrCBGjTxr8TzM5TO+kxtwcbj220\nRf3KrQkwaa50ppTyM3Etv5w9OQPxV1EzLk/30750Jy6dYMDiAUzaPIke9XowueVk7kh/h9thKaUU\n4OXKmADGmPzAjW8vETnok4j8zMqVcMcdUK2a25Hc6mr4Vf63+n8MWzGMpyo/xfau27Wor5TyO94M\nxmwBDAcKYyfHLAFsA+7ybWj+wd9mXI6USKZsnkLv+b2pVqgayzstp1yecm6HpZRSMfLmSmYgUAeY\nJyLVjDHBQCqavSt216/DtGk3149x29IDSwmZE4IgjH1sLA1LNnQ7JKWUipM3SSZMRE4ZYwKMMQEi\nstAY84nPI/MDv/8OFSpAyZLuxrHz1E56zevFuqPrGNx4MG0qt9GivlIqRfAmyZw1xmTDLlA2wRjz\nD3DJt2H5B7dnXD55+SQDFg9g4qaJdL+3OxOfmKhFfaVUinLbcTLOujFXsFPJtAVyAONF5LTvw4u/\npBonc/48FCsG+/ZB7mRezv5q+FU+Xf0pQ1cMpfVdrenbsC/5sqbwWTmVUn7NtVmYgfdEpCcQiV1L\nBmPMEKBnUgfjT2bMsAMwkzPBiAhTttiiftUCVVnWcRnl85ZPvgCUUiqJeXMls15Eqkfb9peIVPFp\nZAmUVFcyTZrAyy9Dy5ZJEJQXlh1cRsicECIiI/jo/o8IKhmUPCdWSincGfH/CtAZKG2M+cvjoexA\nKl3h3jp8GNavh+bNfX+uXad20Wt+L/448gcfNPqAp/7zlBb1lVKpRlzNZROBX4HBQC+P7Rf8tR6T\nVCZPhscft4MwfeXU5VMMXDKQ8X+Np9u93Rj/+HgyZ8jsuxMqpZQLYv2TWUTOich+EXlKRA5gi/8C\nZDPGFE+2CF3gy15l18Kv8dGKj6gwsgJhEWFs7bKVXvV7aYJRSqVKt22XMcY8YozZBewDFgP7sVc4\nCWaMKWqMWWCM2WKM2WSMec3ZnssYM8cYs8MY87sxJofHc3obY3YZY7YZY+5PzPnjsmULnDwJDZN4\nnKOIMGXzFCqOrMjSg0tZ2nEpIx8eSf6s+ZP2REop5Ue86V32Pkk/4j8ceEtENjpjcNYZY+YAHZ3z\nDDXG9AR6A72MMZWAVkBFoCgwzxhTNsnn9MfOuPzUUxCQhGWR5QeXEzInhLDIML5t8S3BpYKT7uBK\nKeXHXBnxLyLHgGPO7YvGmG3Y5PEoEHUNMRZYhK0HtQAmi0g4sN+5sqoNrE5MHNFFRtok8/PPSXO8\n3ad302teL9YcXsOgRoNoW6WtFvWVUmmKN9940Uf8/5ckHPFvjCkJ3A2sAgqIyHG4kYii2pKKAIc8\nnnbY2Zakli2DHDmgSiI7Z5+6fIo3f3uTOt/UoUahGuzouoP2VdtrglFKpTneXMk8ii36v8nNEf8D\nkuLkTvKaDrzuXNFEb/5KUHNYv379btwOCgoiKCjIq+cltuB/LfwaI9eOZPCywTxZ6Um2dtmqNRel\nlF9atGgRixYt8vl5Yh2MaYy5E3tlsTza9vrAURHZk6gTG5Me+AX4VUT+62zbBgSJyHFjTEFgoYhU\nNMb0wq7GOcTZ7zegr4j8q7ksoYMxr12DwoVh40Y7nUx8iAjTtk6j17xeVM5fmSFNhlAxX8V4x6CU\nUm7x1WDMuNpvPgHOx7D9nPNYYo0GtkYlGMdMoINz+1ngJ4/tbYwxGY0xpYA7gTVJEMMNs2fbZrL4\nJpgVh1Zw7+h7+XDZh3zT4htmPjVTE4xSSjniai4rICKbom8UkU1OHSXBjDH1sE1vm4wxG7DNYm8D\nQ4CpxphOwAFsjzJEZKsxZiqwFQgDOid1z7L4NpXtOb2HXvN7servVQxqNIh2VdppzUUppaKJq7ls\nl+/o6UAAAAvvSURBVIiUjeWx3SJyp08jS6CENJedPQslSsCBA5AzZ9z7nr5ymveXvM/YP8fyVp23\neLPum2TJkCURESullPvcaC77wxjzQgyBPA+sS+pA3DR9OjRtGneCuRZ+jY9XfkyFzypwJewKWztv\n5Z0G72iCUUqpOMTVXPYG8IMxpi03k0pNICPwuK8DS07jx8Mbb8T8mIgwfet0es3vRcW8FVnUYRGV\n8lVK3gCVUiqF8maq/2CgsnN3i4gs8HlUiRDf5rKDB6F6dTvzcqZMtz628tBKQuaEcDnsMsPvH07j\n0o2TOFqllPIPri1aJiILgYVJfWJ/MWkSPPHErQlm75m99J7fmxWHVvB+8Pu0q9KOdAHp3AtSKaVS\nqDTdHUoExo272avszJUzhPweQq1RtfhP/v+wo+sOnr37WU0wSimVQGk6yfz1F1y8CLXqXOeTVZ9Q\n/rPyXLx+kS2dt9CnQR8t6iulVCLdtiaT0sSnJtO9h7A34wz+zN+T8nnLM7TJUO7Kf5ePI1RKKf/j\nWk0mtVp+YBX/uxBCyfIX+bL5lzQp3cTtkJRSKtVJc1cy+87so/f83szftYwsqweyd8YzWnNRSqV5\nbgzGTFXOXDlD9zndqTmqJnflu4sHd+/g9QYdNcEopZQPpfokcz3iOv9d9V/Kf1aec9fOsaXzFrrV\nfpdZP2SlTRu3o1NKqdQt1dZkRIQftv9Az3k9uTP3nSx4dgGV89sxpdOmQY0admp/pZRSvpMqk8zq\nv1fTbW43zl09x8hmI7m/zP23PJ7YxcmUUkp5J1UW/gsPL8zA4IE8W/XfAylPnYIyZex0MoGBLgWp\nlFJ+Rrswx8POrjvJmjFrjI9NmwYPPqgJRimlkkOqLPzHlmBAm8qUUio5pcrmsthe0759cM89dsbl\nDBmSOTCllPJjOk4mCUycCE8+qQlGKaWSS5pJMiLaVKaUUsktzSSZ9evh+nWoU8ftSJRSKu1IM0lm\nwgR7FWOSvMVRKaVUbNJE4T88HIoVg8WLoVw5lwJTSik/poX/RFiwwCYZTTBKKZW80kSS0YK/Ukq5\nI9U3l126BEWLwvbtUKCAi4Ep9f/t3X2wVVUZx/HvD5FCfINxlJAEutAwOZZRvkyWOprKZGHjxCQ5\npY41jRU5aik2Q+TgjJo66Zj94UtAopHZC1drRjBlmiyFBghSUl5VIMBGcdJ8v09/rHW92+t95xzO\nPXv/PjMMe6+z9zlrPVzus9fe66xlNoj5dtkAtbamEWVOMGZme1/pk4xvlZmZNU6pb5c9/zxMmpSm\nkRnR/XRmZmaV51mY+2Hz5meYPXs+y5e3MXLkEHbtOp8JE8Y1ulpmZpVTyp5MS8tlbNx4FTACeIWW\nljksXTrTicbMrBv16smUMsnAy6QE0+4Vzj33BhYunNOoapmZDWoeXdYvnR/AjGD79raG1MTMrMpK\nmmReec/+mDElbaqZ2SBWyt+8o0fPoSPRpGcyc+ee37gKmZlVVCmfyZx88hZee20+w4e3MWbMEObO\n9egyM7Oe+ME/IGkqcBOpB3ZnRFzXxTExalSwdSsMH77Xq2hm1pQq/+Bf0hDgp8AZwJHADEmTuzr2\nkEOuYseOZ/Zm9QalZcuWNboKg4Zj0cGx6OBY1F/TJBngWGB9RDwTEW8Ci4Czujrw6ae/x2mn3cLm\nzdVONP4P1MGx6OBYdHAs6q+ZkszhwHOF/a25rAsj2LjxKmbPnl//WpmZWbeaKcn0k78bY2bWaE3z\n4F/S8cCPImJq3p8FROeH/+kb/2Zm1l+VHl0maR/gKeBU4N/AcmBGRKxraMXMzKxbTTMLc0S8Lek7\nwBI6hjA7wZiZDWJN05MxM7PmU5oH/5KmSvqXpKclXdHo+tSDpLGSHpb0hKS1kr6by0dKWiLpKUkP\nSjqocM6VktZLWifp9EL5FElrcrxuakR79pSkIZJWSmrN+5WMA4CkgyT9OrfvCUnHVTEeki6R9M/c\nhrslDatSHCTdKWmnpDWFspq1P8dzUT7nb5KO6LVSEdH0f0jJcgMwDtgXWA1MbnS96tDO0cDReXt/\n0jOqycB1wOW5/Arg2rz9EWAV6bbo+Byj9t7r48AxefuPwBmNbt8A4nEJsBBozfuVjEOu+3zggrw9\nFDioavEAxgCbgGF5/1fAeVWKA/Bp4GhgTaGsZu0HLgJ+lre/DCzqrU5l6cn0+YuazSwidkTE6rz9\nMrAOGEtq64J82ALgi3l7GumH4K2I2AKsB46VNBo4ICJW5ON+UTinKUgaC3wOuKNQXLk4AEg6EPhM\nRMwDyO18iWrGYx9ghKShwHBgGxWKQ0T8BXixU3Et2198r/tIA7F6VJYk048vapaDpPGkK5bHgMMi\nYiekRAQcmg/rHJdtuexwUozaNWO8fgJ8Hyg+VKxiHAAmAP+RNC/fPrxN0n5ULB4RsR24EXiW1KaX\nIuIhKhaHLhxaw/a/c05EvA3sljSqpw8vS5KpFEn7k64iLs49ms6jN0o9mkPSmcDO3KvraVx/qeNQ\nMBSYAtwaEVNI61zMono/FweTrrTHkW6djZB0LhWLQx/Usv29fq+mLElmG1B8ADU2l5VOvg1wH3BX\nRCzOxTslHZZfHw3syuXbgA8WTm+PS3flzeIEYJqkTcAvgVMk3QXsqFgc2m0FnouIv+f935CSTtV+\nLj4LbIqIF/JV9u+AT1G9OHRWy/a/81r+7uKBEfFCTx9eliSzApgoaZykYcA5QGuD61QvPweejIib\nC2WtwPl5+zxgcaH8nDwiZAIwEVieu8wvSTpWkoCvFc4Z9CLiBxFxRER8iPRv/XBEfBW4nwrFoV2+\nFfKcpA/nolOBJ6jYzwXpNtnxkt6f638q8CTVi4N4dw+jlu1vze8BMB14uNfaNHo0RA1HVUwljbZa\nD8xqdH3q1MYTgLdJo+dWAStzu0cBD+X2LwEOLpxzJWnUyDrg9EL5J4C1OV43N7ptexCTk+gYXVbl\nOHyMdLG1GvgtaXRZ5eIBzMltWkN6QL1vleIA3ANsB14nJd0LgJG1aj/wPuDeXP4YML63OvnLmGZm\nVjdluV1mZmaDkJOMmZnVjZOMmZnVjZOMmZnVjZOMmZnVjZOMmZnVjZOMNT1JbZKuL+xfJumHNXrv\neZLOrsV79fI5X5L0pKQ/dSofJ+l/eU6yVfnvoZK+IOnyvVlHs4FompUxzXrwOnC2pGuilyku9iZJ\n+0Sa3qQvLgS+HhF/7eK1DZHmJCu6P//ZI/2so1m/uSdjZfAWcBtwaecXOl/lS/pv/vskScsk/V7S\nBknXSPqKpMcl/SNPs9HuNEkrlBbFOzOfP0TSj/PxqyV9o/C+f5a0mDS1S+f6zMiLQa2RdE0um01a\nB+ROSdd10b73TEIo6TxJt9SijpL2k/RA7imtkTS9x2ib9YN7MlYGAdwKrO3ml3TnY9t9lLTo227S\nYle3R8RxSiuOzqQjaY2LiGMkTQQekdRCmr9pdz5+GPCopCX5+I8DR0bEs8UPlvQB4Nr8+m5gqaRp\nETFX0inApRGxqos6t0hambcfjYiZXbRlwHXMSXhbRHw+1/OAXmJo1mdOMlYKEfGypAXAxcCrfTxt\nRUTsApC0kTSvE6Q5m04uHHdv/owN+bjJwOnAUYWr/gOBScCbpEkG35VgsmOAR9pv6Um6GziRjslc\nu5s2vavbZZ3tSR3XAjfkntUfIi18ZVYTTjJWJjeTJg2dVyh7i3xbOM8oO6zw2uuF7bbCfhvv/r9R\n7DEo7wuYGRFLixWQdBJpPZfu9Lr+xgANuI4RsV7SFNJKo1dLeigirq5TPa1i/EzGykAAEfEi6Yr+\nwsJrW4BP5u2zSLPy9td0JS2kVSifAh4EvqW0vg+SJimtRtmT5cCJkkYprcUxA1jWh8/vS2IacB3z\nbbxXI+Ie4HrSWjRmNeGejJVB8Sr+RuDbhbLbgcWSVpF+6XbXy+hpOvJnSQniAOCbEfGGpDuA8cDK\n3EPaRS/rwEfEDkmz6EgsD0TEA334/N6mSo89rONRwPWS2oA3gIt6+TyzPvNU/2ZmVje+XWZmZnXj\nJGNmZnXjJGNmZnXjJGNmZnXjJGNmZnXjJGNmZnXjJGNmZnXjJGNmZnXzf0PwwQEXAUxxAAAAAElF\nTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(fiber_number,catalog_copy_time,'-o',label='Original Data')\n", + "plt.plot(fiber_number,func(fiber_number,*popt_cata),label=\"Fitted Curve\")\n", + "plt.legend(loc='upper left')\n", + "plt.title(\"Catalog copy time vs number of fiber\")\n", + "plt.xlabel(r\"Number of Fibers\")\n", + "plt.ylabel(r\"Catalog Copy Time (seconds)\")" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "all-ost1\r\n" + ] + } + ], + "source": [ + "ls all-ost1" + ] + }, + { + "cell_type": "code", + "execution_count": 123, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "text_file = open(\"all-ost1\", \"r\")\n", + "lines = text_file.read().splitlines()" + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "19160" + ] + }, + "execution_count": 124, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(lines)" + ] + }, + { + "cell_type": "code", + "execution_count": 143, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "myarray = map(int, lines)\n", + "myarray = np.asarray(myarray)\n", + "#print lines\n", + "xr=range(0,96)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 148, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([205, 201, 200, 208, 210, 199, 207, 214, 210, 200, 205, 195, 209,\n", + " 208, 207, 218, 203, 199, 217, 200, 207, 201, 198, 215, 206, 210,\n", + " 203, 196, 209, 208, 209, 219, 203, 0, 210, 208, 221, 206, 203,\n", + " 208, 204, 197, 208, 207, 215, 0, 207, 203, 205, 199, 193, 207,\n", + " 209, 202, 193, 220, 206, 204, 195, 203, 218, 202, 202, 216, 208,\n", + " 0, 205, 196, 221, 212, 209, 203, 207, 208, 192, 204, 204, 207,\n", + " 197, 221, 213, 212, 196, 205, 202, 200, 208, 218, 209, 206, 197,\n", + " 207, 200, 203, 420]),\n", + " array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,\n", + " 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,\n", + " 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,\n", + " 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67,\n", + " 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84,\n", + " 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95]))" + ] + }, + "execution_count": 148, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.histogram(myarray,bins=xr)" + ] + }, + { + "cell_type": "code", + "execution_count": 152, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([ 0, 0, 0, ..., 95, 95, 95])" + ] + }, + "execution_count": 152, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "myarray" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/global/u1/j/jialin/h5boss-util/h5boss_py/demo/pmf-list\n" + ] + } + ], + "source": [ + "cd pmf-list/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# sorting pmf" + ] + }, + { + "cell_type": "code", + "execution_count": 215, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "filename='pmf500k'\n", + "dc=pd.read_csv(filename,sep=' ',dtype=str,header=0)\n", + "ds=dc.sort_values(by='plates' ,ascending=True)\n", + "nan_rows = ds[ds.isnull().T.any().T]\n", + "dc.drop(dc.index[nan_rows.index], inplace=True)\n", + "ds=dc.sort_values(by='plates' ,ascending=True)\n", + "ds.to_csv(filename+'-sorted.csv',index=False,sep=' ',header='plates mjds fibers')\n", + "ds=pd.read_csv(filename+'-sorted.csv',sep=' ',dtype=int,header=0)\n", + "ps=pd.Series(ds['plates'])\n", + "vc=ps.value_counts(sort=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 216, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3662 252\n", + "6682 248\n", + "6873 248\n", + "6265 246\n", + "4804 244\n", + "3787 244\n", + "4180 242\n", + "5949 242\n", + "3784 242\n", + "5135 242\n", + "4081 241\n", + "4773 240\n", + "4654 239\n", + "7454 239\n", + "7385 239\n", + "5002 239\n", + "4778 238\n", + "3773 238\n", + "7383 237\n", + "6783 237\n", + "4693 237\n", + "4235 237\n", + "5857 237\n", + "6787 237\n", + "4238 237\n", + "6831 237\n", + "4576 236\n", + "6474 236\n", + "5441 236\n", + "6258 236\n", + " ... \n", + "36936187 1\n", + "5834 1\n", + "57299 1\n", + "44552 1\n", + "45136 1\n", + "63639 1\n", + "136 1\n", + "33805 1\n", + "42226583 1\n", + "969 1\n", + "404482 1\n", + "4167 1\n", + "36574543 1\n", + "396038 1\n", + "71236508 1\n", + "594404 1\n", + "397 1\n", + "44953 1\n", + "75490 1\n", + "76002 1\n", + "543 1\n", + "68 1\n", + "6447368 1\n", + "4743699 1\n", + "139 1\n", + "711 1\n", + "3767016 1\n", + "69 1\n", + "679509 1\n", + "179 1\n", + "Name: plates, dtype: int64\n" + ] + } + ], + "source": [ + "print vc\n", + "ds=ds[ds.mjds<60000]\n", + "ds=ds[ds.mjds>1000]\n", + "ds=ds[ds.plates<8000]\n", + "ds=ds[ds.plates>1000]\n", + "ds=ds[ds.fibers<1001]\n", + "ds=ds[ds.fibers>0]\n", + "\n", + "#dds=dds[dds.mjds>20000 ]\n", + "#dds=dds[dds.plates>100 ]\n", + "#dds=dds[dds.plates>9000 ]\n", + "#print len(dc)\n", + "#print len(ddc)\n", + "ds.to_csv(filename+'-sorted.csv',index=False,sep=' ',header='plates mjds fibers',na_rep='NA')\n", + "#ddss=dds[dds.plates/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/h5boss.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/h5boss.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/h5boss" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/h5boss" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/h5boss_py/docs/images/fmt1.png b/h5boss_py/docs/images/fmt1.png new file mode 100755 index 0000000..91ac7db Binary files /dev/null and b/h5boss_py/docs/images/fmt1.png differ diff --git a/h5boss_py/docs/images/fmt2.png b/h5boss_py/docs/images/fmt2.png new file mode 100755 index 0000000..ac29280 Binary files /dev/null and b/h5boss_py/docs/images/fmt2.png differ diff --git a/h5boss_py/docs/make.bat b/h5boss_py/docs/make.bat new file mode 100755 index 0000000..1c26492 --- /dev/null +++ b/h5boss_py/docs/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +set I18NSPHINXOPTS=%SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 2> nul +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\h5boss.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\h5boss.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/h5boss_py/docs/readme b/h5boss_py/docs/readme new file mode 100644 index 0000000..e88e042 --- /dev/null +++ b/h5boss_py/docs/readme @@ -0,0 +1,22 @@ +To build the documentation for your python code, e.g., h5boss: + +1. Install python packages: Sphinx, numpydoc + +2. Follow Sphinx's tutorial and run a quickstart + +To edit and publish your website locally + +1. Edit web page file in source/ + e.g., install.rst + +2. Export Sphinx's binary to $PATH + e.g., export PATH=$PATH:/somepath/Sphinx/bin + +3. Export sphinx and numpydoc's libraries in $PYTHONPATH + e.g., export PYTHONPATH=$PYTHONPATH:/somepath/Sphinx/lib64/python2.6/site-packages/:/somepath/numpydoc/lib/python2.7/site-packages/ + +4. run `make html` at the same level of source/ + +5. launch website locally: open build/index.rst in any browser + +For anyother questions, email jalnliu@lbl.gov diff --git a/h5boss_py/docs/source/_static/base.h5 b/h5boss_py/docs/source/_static/base.h5 new file mode 100755 index 0000000..6fbe77c Binary files /dev/null and b/h5boss_py/docs/source/_static/base.h5 differ diff --git a/h5boss_py/docs/source/_static/input_sample.txt b/h5boss_py/docs/source/_static/input_sample.txt new file mode 100755 index 0000000..34b42ba --- /dev/null +++ b/h5boss_py/docs/source/_static/input_sample.txt @@ -0,0 +1,2393 @@ +/global/cscratch1/sd/jialin/h5boss/3665-55247.hdf5 +/global/cscratch1/sd/jialin/h5boss/3666-55159.hdf5 +/global/cscratch1/sd/jialin/h5boss/3667-55186.hdf5 +/global/cscratch1/sd/jialin/h5boss/3668-55478.hdf5 +/global/cscratch1/sd/jialin/h5boss/3669-55481.hdf5 +/global/cscratch1/sd/jialin/h5boss/3670-55480.hdf5 +/global/cscratch1/sd/jialin/h5boss/3671-55483.hdf5 +/global/cscratch1/sd/jialin/h5boss/3672-55484.hdf5 +/global/cscratch1/sd/jialin/h5boss/3673-55160.hdf5 +/global/cscratch1/sd/jialin/h5boss/3674-55153.hdf5 +/global/cscratch1/sd/jialin/h5boss/3675-55154.hdf5 +/global/cscratch1/sd/jialin/h5boss/3676-55186.hdf5 +/global/cscratch1/sd/jialin/h5boss/3677-55205.hdf5 +/global/cscratch1/sd/jialin/h5boss/3678-55208.hdf5 +/global/cscratch1/sd/jialin/h5boss/3679-55158.hdf5 +/global/cscratch1/sd/jialin/h5boss/3680-55159.hdf5 +/global/cscratch1/sd/jialin/h5boss/3681-55243.hdf5 +/global/cscratch1/sd/jialin/h5boss/3682-55244.hdf5 +/global/cscratch1/sd/jialin/h5boss/3683-55178.hdf5 +/global/cscratch1/sd/jialin/h5boss/3684-55246.hdf5 +/global/cscratch1/sd/jialin/h5boss/3685-55247.hdf5 +/global/cscratch1/sd/jialin/h5boss/3686-55268.hdf5 +/global/cscratch1/sd/jialin/h5boss/3687-55269.hdf5 +/global/cscratch1/sd/jialin/h5boss/3688-55273.hdf5 +/global/cscratch1/sd/jialin/h5boss/3689-55180.hdf5 +/global/cscratch1/sd/jialin/h5boss/3690-55182.hdf5 +/global/cscratch1/sd/jialin/h5boss/3691-55274.hdf5 +/global/cscratch1/sd/jialin/h5boss/3693-55208.hdf5 +/global/cscratch1/sd/jialin/h5boss/3694-55209.hdf5 +/global/cscratch1/sd/jialin/h5boss/3695-55212.hdf5 +/global/cscratch1/sd/jialin/h5boss/3696-55277.hdf5 +/global/cscratch1/sd/jialin/h5boss/3697-55290.hdf5 +/global/cscratch1/sd/jialin/h5boss/3698-55501.hdf5 +/global/cscratch1/sd/jialin/h5boss/3699-55517.hdf5 +/global/cscratch1/sd/jialin/h5boss/3700-55542.hdf5 +/global/cscratch1/sd/jialin/h5boss/3701-55540.hdf5 +/global/cscratch1/sd/jialin/h5boss/3735-55209.hdf5 +/global/cscratch1/sd/jialin/h5boss/3736-55214.hdf5 +/global/cscratch1/sd/jialin/h5boss/3744-55209.hdf5 +/global/cscratch1/sd/jialin/h5boss/3745-55234.hdf5 +/global/cscratch1/sd/jialin/h5boss/3751-55234.hdf5 +/global/cscratch1/sd/jialin/h5boss/3752-55236.hdf5 +/global/cscratch1/sd/jialin/h5boss/3753-55486.hdf5 +/global/cscratch1/sd/jialin/h5boss/3754-55488.hdf5 +/global/cscratch1/sd/jialin/h5boss/3755-55504.hdf5 +/global/cscratch1/sd/jialin/h5boss/3756-55505.hdf5 +/global/cscratch1/sd/jialin/h5boss/3757-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/3758-55506.hdf5 +/global/cscratch1/sd/jialin/h5boss/3759-55236.hdf5 +/global/cscratch1/sd/jialin/h5boss/3760-55268.hdf5 +/global/cscratch1/sd/jialin/h5boss/3761-55272.hdf5 +/global/cscratch1/sd/jialin/h5boss/3762-55507.hdf5 +/global/cscratch1/sd/jialin/h5boss/3763-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/3764-55514.hdf5 +/global/cscratch1/sd/jialin/h5boss/3765-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/3766-55213.hdf5 +/global/cscratch1/sd/jialin/h5boss/3767-55214.hdf5 +/global/cscratch1/sd/jialin/h5boss/3768-55564.hdf5 +/global/cscratch1/sd/jialin/h5boss/3769-55240.hdf5 +/global/cscratch1/sd/jialin/h5boss/3770-55234.hdf5 +/global/cscratch1/sd/jialin/h5boss/3771-55293.hdf5 +/global/cscratch1/sd/jialin/h5boss/3772-55277.hdf5 +/global/cscratch1/sd/jialin/h5boss/3773-55240.hdf5 +/global/cscratch1/sd/jialin/h5boss/3774-55244.hdf5 +/global/cscratch1/sd/jialin/h5boss/3775-55207.hdf5 +/global/cscratch1/sd/jialin/h5boss/3776-55209.hdf5 +/global/cscratch1/sd/jialin/h5boss/3777-55210.hdf5 +/global/cscratch1/sd/jialin/h5boss/3778-55213.hdf5 +/global/cscratch1/sd/jialin/h5boss/3779-55222.hdf5 +/global/cscratch1/sd/jialin/h5boss/3780-55240.hdf5 +/global/cscratch1/sd/jialin/h5boss/3781-55243.hdf5 +/global/cscratch1/sd/jialin/h5boss/3782-55244.hdf5 +/global/cscratch1/sd/jialin/h5boss/3783-55246.hdf5 +/global/cscratch1/sd/jialin/h5boss/3784-55269.hdf5 +/global/cscratch1/sd/jialin/h5boss/3785-55273.hdf5 +/global/cscratch1/sd/jialin/h5boss/3786-55563.hdf5 +/global/cscratch1/sd/jialin/h5boss/3787-55565.hdf5 +/global/cscratch1/sd/jialin/h5boss/3788-55246.hdf5 +/global/cscratch1/sd/jialin/h5boss/3789-55269.hdf5 +/global/cscratch1/sd/jialin/h5boss/3790-55208.hdf5 +/global/cscratch1/sd/jialin/h5boss/3791-55501.hdf5 +/global/cscratch1/sd/jialin/h5boss/3792-55212.hdf5 +/global/cscratch1/sd/jialin/h5boss/3793-55214.hdf5 +/global/cscratch1/sd/jialin/h5boss/3794-55241.hdf5 +/global/cscratch1/sd/jialin/h5boss/3800-55486.hdf5 +/global/cscratch1/sd/jialin/h5boss/3801-55509.hdf5 +/global/cscratch1/sd/jialin/h5boss/3802-55528.hdf5 +/global/cscratch1/sd/jialin/h5boss/3803-55513.hdf5 +/global/cscratch1/sd/jialin/h5boss/3804-55267.hdf5 +/global/cscratch1/sd/jialin/h5boss/3805-55269.hdf5 +/global/cscratch1/sd/jialin/h5boss/3806-55505.hdf5 +/global/cscratch1/sd/jialin/h5boss/3807-55511.hdf5 +/global/cscratch1/sd/jialin/h5boss/3808-55513.hdf5 +/global/cscratch1/sd/jialin/h5boss/3809-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/3810-56015.hdf5 +/global/cscratch1/sd/jialin/h5boss/3811-55538.hdf5 +/global/cscratch1/sd/jialin/h5boss/3812-55513.hdf5 +/global/cscratch1/sd/jialin/h5boss/3813-55532.hdf5 +/global/cscratch1/sd/jialin/h5boss/3814-55535.hdf5 +/global/cscratch1/sd/jialin/h5boss/3815-55537.hdf5 +/global/cscratch1/sd/jialin/h5boss/3816-55272.hdf5 +/global/cscratch1/sd/jialin/h5boss/3817-55277.hdf5 +/global/cscratch1/sd/jialin/h5boss/3818-55532.hdf5 +/global/cscratch1/sd/jialin/h5boss/3819-55540.hdf5 +/global/cscratch1/sd/jialin/h5boss/3820-55542.hdf5 +/global/cscratch1/sd/jialin/h5boss/3821-55535.hdf5 +/global/cscratch1/sd/jialin/h5boss/3822-55544.hdf5 +/global/cscratch1/sd/jialin/h5boss/3823-55534.hdf5 +/global/cscratch1/sd/jialin/h5boss/3825-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/3826-55563.hdf5 +/global/cscratch1/sd/jialin/h5boss/3827-55565.hdf5 +/global/cscratch1/sd/jialin/h5boss/3828-55539.hdf5 +/global/cscratch1/sd/jialin/h5boss/3829-55300.hdf5 +/global/cscratch1/sd/jialin/h5boss/3830-55574.hdf5 +/global/cscratch1/sd/jialin/h5boss/3831-55543.hdf5 +/global/cscratch1/sd/jialin/h5boss/3832-55289.hdf5 +/global/cscratch1/sd/jialin/h5boss/3833-55290.hdf5 +/global/cscratch1/sd/jialin/h5boss/3834-56014.hdf5 +/global/cscratch1/sd/jialin/h5boss/3835-55570.hdf5 +/global/cscratch1/sd/jialin/h5boss/3836-55302.hdf5 +/global/cscratch1/sd/jialin/h5boss/3837-55572.hdf5 +/global/cscratch1/sd/jialin/h5boss/3838-55588.hdf5 +/global/cscratch1/sd/jialin/h5boss/3839-55575.hdf5 +/global/cscratch1/sd/jialin/h5boss/3840-55574.hdf5 +/global/cscratch1/sd/jialin/h5boss/3841-55295.hdf5 +/global/cscratch1/sd/jialin/h5boss/3842-55565.hdf5 +/global/cscratch1/sd/jialin/h5boss/3843-55278.hdf5 +/global/cscratch1/sd/jialin/h5boss/3844-55321.hdf5 +/global/cscratch1/sd/jialin/h5boss/3845-55323.hdf5 +/global/cscratch1/sd/jialin/h5boss/3846-55327.hdf5 +/global/cscratch1/sd/jialin/h5boss/3847-55298.hdf5 +/global/cscratch1/sd/jialin/h5boss/3848-55647.hdf5 +/global/cscratch1/sd/jialin/h5boss/3849-55274.hdf5 +/global/cscratch1/sd/jialin/h5boss/3850-55293.hdf5 +/global/cscratch1/sd/jialin/h5boss/3851-55298.hdf5 +/global/cscratch1/sd/jialin/h5boss/3852-55243.hdf5 +/global/cscratch1/sd/jialin/h5boss/3853-55268.hdf5 +/global/cscratch1/sd/jialin/h5boss/3854-55247.hdf5 +/global/cscratch1/sd/jialin/h5boss/3855-55268.hdf5 +/global/cscratch1/sd/jialin/h5boss/3856-55269.hdf5 +/global/cscratch1/sd/jialin/h5boss/3857-55272.hdf5 +/global/cscratch1/sd/jialin/h5boss/3858-55273.hdf5 +/global/cscratch1/sd/jialin/h5boss/3859-55246.hdf5 +/global/cscratch1/sd/jialin/h5boss/3860-55269.hdf5 +/global/cscratch1/sd/jialin/h5boss/3861-55274.hdf5 +/global/cscratch1/sd/jialin/h5boss/3862-55276.hdf5 +/global/cscratch1/sd/jialin/h5boss/3863-55280.hdf5 +/global/cscratch1/sd/jialin/h5boss/3864-55649.hdf5 +/global/cscratch1/sd/jialin/h5boss/3865-55272.hdf5 +/global/cscratch1/sd/jialin/h5boss/3866-55623.hdf5 +/global/cscratch1/sd/jialin/h5boss/3867-55652.hdf5 +/global/cscratch1/sd/jialin/h5boss/3868-55360.hdf5 +/global/cscratch1/sd/jialin/h5boss/3869-55273.hdf5 +/global/cscratch1/sd/jialin/h5boss/3870-55273.hdf5 +/global/cscratch1/sd/jialin/h5boss/3871-55307.hdf5 +/global/cscratch1/sd/jialin/h5boss/3872-55382.hdf5 +/global/cscratch1/sd/jialin/h5boss/3873-55277.hdf5 +/global/cscratch1/sd/jialin/h5boss/3874-55280.hdf5 +/global/cscratch1/sd/jialin/h5boss/3875-55364.hdf5 +/global/cscratch1/sd/jialin/h5boss/3876-55245.hdf5 +/global/cscratch1/sd/jialin/h5boss/3877-55365.hdf5 +/global/cscratch1/sd/jialin/h5boss/3878-55361.hdf5 +/global/cscratch1/sd/jialin/h5boss/3879-55244.hdf5 +/global/cscratch1/sd/jialin/h5boss/3922-55333.hdf5 +/global/cscratch1/sd/jialin/h5boss/3923-55325.hdf5 +/global/cscratch1/sd/jialin/h5boss/3924-55332.hdf5 +/global/cscratch1/sd/jialin/h5boss/3925-55338.hdf5 +/global/cscratch1/sd/jialin/h5boss/3926-55327.hdf5 +/global/cscratch1/sd/jialin/h5boss/3927-55333.hdf5 +/global/cscratch1/sd/jialin/h5boss/3928-55331.hdf5 +/global/cscratch1/sd/jialin/h5boss/3929-55335.hdf5 +/global/cscratch1/sd/jialin/h5boss/3930-55332.hdf5 +/global/cscratch1/sd/jialin/h5boss/3931-55350.hdf5 +/global/cscratch1/sd/jialin/h5boss/3932-55337.hdf5 +/global/cscratch1/sd/jialin/h5boss/3933-55335.hdf5 +/global/cscratch1/sd/jialin/h5boss/3934-55336.hdf5 +/global/cscratch1/sd/jialin/h5boss/3935-55326.hdf5 +/global/cscratch1/sd/jialin/h5boss/3936-55302.hdf5 +/global/cscratch1/sd/jialin/h5boss/3937-55352.hdf5 +/global/cscratch1/sd/jialin/h5boss/3938-55328.hdf5 +/global/cscratch1/sd/jialin/h5boss/3939-55633.hdf5 +/global/cscratch1/sd/jialin/h5boss/3940-55327.hdf5 +/global/cscratch1/sd/jialin/h5boss/3941-55321.hdf5 +/global/cscratch1/sd/jialin/h5boss/3942-55338.hdf5 +/global/cscratch1/sd/jialin/h5boss/3943-55336.hdf5 +/global/cscratch1/sd/jialin/h5boss/3944-55660.hdf5 +/global/cscratch1/sd/jialin/h5boss/3945-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/3946-55298.hdf5 +/global/cscratch1/sd/jialin/h5boss/3947-55332.hdf5 +/global/cscratch1/sd/jialin/h5boss/3948-55331.hdf5 +/global/cscratch1/sd/jialin/h5boss/3949-55650.hdf5 +/global/cscratch1/sd/jialin/h5boss/3950-55680.hdf5 +/global/cscratch1/sd/jialin/h5boss/3951-55681.hdf5 +/global/cscratch1/sd/jialin/h5boss/3952-55330.hdf5 +/global/cscratch1/sd/jialin/h5boss/3953-55322.hdf5 +/global/cscratch1/sd/jialin/h5boss/3954-55680.hdf5 +/global/cscratch1/sd/jialin/h5boss/3955-55678.hdf5 +/global/cscratch1/sd/jialin/h5boss/3956-55656.hdf5 +/global/cscratch1/sd/jialin/h5boss/3957-55664.hdf5 +/global/cscratch1/sd/jialin/h5boss/3958-55329.hdf5 +/global/cscratch1/sd/jialin/h5boss/3959-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/3960-55663.hdf5 +/global/cscratch1/sd/jialin/h5boss/3961-55654.hdf5 +/global/cscratch1/sd/jialin/h5boss/3962-55660.hdf5 +/global/cscratch1/sd/jialin/h5boss/3963-55659.hdf5 +/global/cscratch1/sd/jialin/h5boss/3964-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/3965-55302.hdf5 +/global/cscratch1/sd/jialin/h5boss/3966-55571.hdf5 +/global/cscratch1/sd/jialin/h5boss/3967-55294.hdf5 +/global/cscratch1/sd/jialin/h5boss/3968-55590.hdf5 +/global/cscratch1/sd/jialin/h5boss/3969-55307.hdf5 +/global/cscratch1/sd/jialin/h5boss/3970-55296.hdf5 +/global/cscratch1/sd/jialin/h5boss/3971-55322.hdf5 +/global/cscratch1/sd/jialin/h5boss/3972-55297.hdf5 +/global/cscratch1/sd/jialin/h5boss/3973-55323.hdf5 +/global/cscratch1/sd/jialin/h5boss/3974-55320.hdf5 +/global/cscratch1/sd/jialin/h5boss/3975-55321.hdf5 +/global/cscratch1/sd/jialin/h5boss/3976-55298.hdf5 +/global/cscratch1/sd/jialin/h5boss/3977-55335.hdf5 +/global/cscratch1/sd/jialin/h5boss/3978-55330.hdf5 +/global/cscratch1/sd/jialin/h5boss/3979-55597.hdf5 +/global/cscratch1/sd/jialin/h5boss/3980-55331.hdf5 +/global/cscratch1/sd/jialin/h5boss/3981-55603.hdf5 +/global/cscratch1/sd/jialin/h5boss/3982-55332.hdf5 +/global/cscratch1/sd/jialin/h5boss/3983-55603.hdf5 +/global/cscratch1/sd/jialin/h5boss/3984-55333.hdf5 +/global/cscratch1/sd/jialin/h5boss/3985-55320.hdf5 +/global/cscratch1/sd/jialin/h5boss/3986-55329.hdf5 +/global/cscratch1/sd/jialin/h5boss/3987-55590.hdf5 +/global/cscratch1/sd/jialin/h5boss/4004-55321.hdf5 +/global/cscratch1/sd/jialin/h5boss/4005-55325.hdf5 +/global/cscratch1/sd/jialin/h5boss/4006-55328.hdf5 +/global/cscratch1/sd/jialin/h5boss/4007-55327.hdf5 +/global/cscratch1/sd/jialin/h5boss/4008-55356.hdf5 +/global/cscratch1/sd/jialin/h5boss/4009-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/4010-55321.hdf5 +/global/cscratch1/sd/jialin/h5boss/4011-55635.hdf5 +/global/cscratch1/sd/jialin/h5boss/4012-55327.hdf5 +/global/cscratch1/sd/jialin/h5boss/4013-55629.hdf5 +/global/cscratch1/sd/jialin/h5boss/4014-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4015-55624.hdf5 +/global/cscratch1/sd/jialin/h5boss/4016-55632.hdf5 +/global/cscratch1/sd/jialin/h5boss/4017-55329.hdf5 +/global/cscratch1/sd/jialin/h5boss/4018-55622.hdf5 +/global/cscratch1/sd/jialin/h5boss/4019-55363.hdf5 +/global/cscratch1/sd/jialin/h5boss/4020-55332.hdf5 +/global/cscratch1/sd/jialin/h5boss/4021-55620.hdf5 +/global/cscratch1/sd/jialin/h5boss/4022-55352.hdf5 +/global/cscratch1/sd/jialin/h5boss/4023-55328.hdf5 +/global/cscratch1/sd/jialin/h5boss/4024-55646.hdf5 +/global/cscratch1/sd/jialin/h5boss/4025-55350.hdf5 +/global/cscratch1/sd/jialin/h5boss/4026-55325.hdf5 +/global/cscratch1/sd/jialin/h5boss/4027-55629.hdf5 +/global/cscratch1/sd/jialin/h5boss/4028-55621.hdf5 +/global/cscratch1/sd/jialin/h5boss/4029-55618.hdf5 +/global/cscratch1/sd/jialin/h5boss/4030-55634.hdf5 +/global/cscratch1/sd/jialin/h5boss/4031-55604.hdf5 +/global/cscratch1/sd/jialin/h5boss/4032-55333.hdf5 +/global/cscratch1/sd/jialin/h5boss/4033-55332.hdf5 +/global/cscratch1/sd/jialin/h5boss/4034-55635.hdf5 +/global/cscratch1/sd/jialin/h5boss/4035-55383.hdf5 +/global/cscratch1/sd/jialin/h5boss/4036-55330.hdf5 +/global/cscratch1/sd/jialin/h5boss/4037-55631.hdf5 +/global/cscratch1/sd/jialin/h5boss/4038-55363.hdf5 +/global/cscratch1/sd/jialin/h5boss/4039-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/4040-55605.hdf5 +/global/cscratch1/sd/jialin/h5boss/4041-55361.hdf5 +/global/cscratch1/sd/jialin/h5boss/4042-55626.hdf5 +/global/cscratch1/sd/jialin/h5boss/4043-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4044-55359.hdf5 +/global/cscratch1/sd/jialin/h5boss/4045-55622.hdf5 +/global/cscratch1/sd/jialin/h5boss/4046-55605.hdf5 +/global/cscratch1/sd/jialin/h5boss/4047-55352.hdf5 +/global/cscratch1/sd/jialin/h5boss/4048-55602.hdf5 +/global/cscratch1/sd/jialin/h5boss/4049-55591.hdf5 +/global/cscratch1/sd/jialin/h5boss/4050-55599.hdf5 +/global/cscratch1/sd/jialin/h5boss/4051-55337.hdf5 +/global/cscratch1/sd/jialin/h5boss/4052-55589.hdf5 +/global/cscratch1/sd/jialin/h5boss/4053-55591.hdf5 +/global/cscratch1/sd/jialin/h5boss/4054-55358.hdf5 +/global/cscratch1/sd/jialin/h5boss/4055-55359.hdf5 +/global/cscratch1/sd/jialin/h5boss/4056-55357.hdf5 +/global/cscratch1/sd/jialin/h5boss/4057-55357.hdf5 +/global/cscratch1/sd/jialin/h5boss/4058-55358.hdf5 +/global/cscratch1/sd/jialin/h5boss/4059-55360.hdf5 +/global/cscratch1/sd/jialin/h5boss/4060-55359.hdf5 +/global/cscratch1/sd/jialin/h5boss/4061-55362.hdf5 +/global/cscratch1/sd/jialin/h5boss/4062-55383.hdf5 +/global/cscratch1/sd/jialin/h5boss/4063-55364.hdf5 +/global/cscratch1/sd/jialin/h5boss/4064-55366.hdf5 +/global/cscratch1/sd/jialin/h5boss/4065-55368.hdf5 +/global/cscratch1/sd/jialin/h5boss/4066-55444.hdf5 +/global/cscratch1/sd/jialin/h5boss/4067-55361.hdf5 +/global/cscratch1/sd/jialin/h5boss/4068-55445.hdf5 +/global/cscratch1/sd/jialin/h5boss/4069-55365.hdf5 +/global/cscratch1/sd/jialin/h5boss/4070-55681.hdf5 +/global/cscratch1/sd/jialin/h5boss/4071-55660.hdf5 +/global/cscratch1/sd/jialin/h5boss/4072-55362.hdf5 +/global/cscratch1/sd/jialin/h5boss/4073-55663.hdf5 +/global/cscratch1/sd/jialin/h5boss/4074-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/4075-55352.hdf5 +/global/cscratch1/sd/jialin/h5boss/4077-55361.hdf5 +/global/cscratch1/sd/jialin/h5boss/4078-55358.hdf5 +/global/cscratch1/sd/jialin/h5boss/4079-55363.hdf5 +/global/cscratch1/sd/jialin/h5boss/4080-55368.hdf5 +/global/cscratch1/sd/jialin/h5boss/4081-55365.hdf5 +/global/cscratch1/sd/jialin/h5boss/4082-55367.hdf5 +/global/cscratch1/sd/jialin/h5boss/4083-55443.hdf5 +/global/cscratch1/sd/jialin/h5boss/4084-55447.hdf5 +/global/cscratch1/sd/jialin/h5boss/4085-55452.hdf5 +/global/cscratch1/sd/jialin/h5boss/4086-55446.hdf5 +/global/cscratch1/sd/jialin/h5boss/4087-55449.hdf5 +/global/cscratch1/sd/jialin/h5boss/4088-55451.hdf5 +/global/cscratch1/sd/jialin/h5boss/4089-55470.hdf5 +/global/cscratch1/sd/jialin/h5boss/4090-55500.hdf5 +/global/cscratch1/sd/jialin/h5boss/4091-55498.hdf5 +/global/cscratch1/sd/jialin/h5boss/4092-55477.hdf5 +/global/cscratch1/sd/jialin/h5boss/4093-55475.hdf5 +/global/cscratch1/sd/jialin/h5boss/4094-55481.hdf5 +/global/cscratch1/sd/jialin/h5boss/4095-55497.hdf5 +/global/cscratch1/sd/jialin/h5boss/4096-55501.hdf5 +/global/cscratch1/sd/jialin/h5boss/4097-55506.hdf5 +/global/cscratch1/sd/jialin/h5boss/4174-55659.hdf5 +/global/cscratch1/sd/jialin/h5boss/4175-55680.hdf5 +/global/cscratch1/sd/jialin/h5boss/4176-55682.hdf5 +/global/cscratch1/sd/jialin/h5boss/4177-55688.hdf5 +/global/cscratch1/sd/jialin/h5boss/4178-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4179-55684.hdf5 +/global/cscratch1/sd/jialin/h5boss/4180-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/4181-55685.hdf5 +/global/cscratch1/sd/jialin/h5boss/4182-55446.hdf5 +/global/cscratch1/sd/jialin/h5boss/4183-55447.hdf5 +/global/cscratch1/sd/jialin/h5boss/4184-55450.hdf5 +/global/cscratch1/sd/jialin/h5boss/4185-55469.hdf5 +/global/cscratch1/sd/jialin/h5boss/4186-55691.hdf5 +/global/cscratch1/sd/jialin/h5boss/4187-55693.hdf5 +/global/cscratch1/sd/jialin/h5boss/4188-55684.hdf5 +/global/cscratch1/sd/jialin/h5boss/4189-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/4190-55686.hdf5 +/global/cscratch1/sd/jialin/h5boss/4191-55444.hdf5 +/global/cscratch1/sd/jialin/h5boss/4192-55469.hdf5 +/global/cscratch1/sd/jialin/h5boss/4193-55476.hdf5 +/global/cscratch1/sd/jialin/h5boss/4194-55450.hdf5 +/global/cscratch1/sd/jialin/h5boss/4195-55452.hdf5 +/global/cscratch1/sd/jialin/h5boss/4196-55478.hdf5 +/global/cscratch1/sd/jialin/h5boss/4197-55479.hdf5 +/global/cscratch1/sd/jialin/h5boss/4198-55480.hdf5 +/global/cscratch1/sd/jialin/h5boss/4199-55481.hdf5 +/global/cscratch1/sd/jialin/h5boss/4200-55499.hdf5 +/global/cscratch1/sd/jialin/h5boss/4201-55443.hdf5 +/global/cscratch1/sd/jialin/h5boss/4202-55445.hdf5 +/global/cscratch1/sd/jialin/h5boss/4203-55447.hdf5 +/global/cscratch1/sd/jialin/h5boss/4204-55470.hdf5 +/global/cscratch1/sd/jialin/h5boss/4205-55454.hdf5 +/global/cscratch1/sd/jialin/h5boss/4206-55471.hdf5 +/global/cscratch1/sd/jialin/h5boss/4207-55475.hdf5 +/global/cscratch1/sd/jialin/h5boss/4208-55476.hdf5 +/global/cscratch1/sd/jialin/h5boss/4209-55478.hdf5 +/global/cscratch1/sd/jialin/h5boss/4210-55444.hdf5 +/global/cscratch1/sd/jialin/h5boss/4211-55446.hdf5 +/global/cscratch1/sd/jialin/h5boss/4212-55447.hdf5 +/global/cscratch1/sd/jialin/h5boss/4213-55449.hdf5 +/global/cscratch1/sd/jialin/h5boss/4214-55451.hdf5 +/global/cscratch1/sd/jialin/h5boss/4215-55471.hdf5 +/global/cscratch1/sd/jialin/h5boss/4216-55477.hdf5 +/global/cscratch1/sd/jialin/h5boss/4217-55478.hdf5 +/global/cscratch1/sd/jialin/h5boss/4218-55479.hdf5 +/global/cscratch1/sd/jialin/h5boss/4219-55480.hdf5 +/global/cscratch1/sd/jialin/h5boss/4220-55447.hdf5 +/global/cscratch1/sd/jialin/h5boss/4221-55443.hdf5 +/global/cscratch1/sd/jialin/h5boss/4222-55444.hdf5 +/global/cscratch1/sd/jialin/h5boss/4223-55451.hdf5 +/global/cscratch1/sd/jialin/h5boss/4224-55481.hdf5 +/global/cscratch1/sd/jialin/h5boss/4225-55455.hdf5 +/global/cscratch1/sd/jialin/h5boss/4226-55475.hdf5 +/global/cscratch1/sd/jialin/h5boss/4227-55481.hdf5 +/global/cscratch1/sd/jialin/h5boss/4228-55484.hdf5 +/global/cscratch1/sd/jialin/h5boss/4229-55501.hdf5 +/global/cscratch1/sd/jialin/h5boss/4230-55483.hdf5 +/global/cscratch1/sd/jialin/h5boss/4231-55444.hdf5 +/global/cscratch1/sd/jialin/h5boss/4232-55447.hdf5 +/global/cscratch1/sd/jialin/h5boss/4233-55449.hdf5 +/global/cscratch1/sd/jialin/h5boss/4234-55478.hdf5 +/global/cscratch1/sd/jialin/h5boss/4235-55451.hdf5 +/global/cscratch1/sd/jialin/h5boss/4236-55479.hdf5 +/global/cscratch1/sd/jialin/h5boss/4237-55478.hdf5 +/global/cscratch1/sd/jialin/h5boss/4238-55455.hdf5 +/global/cscratch1/sd/jialin/h5boss/4239-55458.hdf5 +/global/cscratch1/sd/jialin/h5boss/4240-55455.hdf5 +/global/cscratch1/sd/jialin/h5boss/4241-55450.hdf5 +/global/cscratch1/sd/jialin/h5boss/4242-55476.hdf5 +/global/cscratch1/sd/jialin/h5boss/4256-55477.hdf5 +/global/cscratch1/sd/jialin/h5boss/4257-55480.hdf5 +/global/cscratch1/sd/jialin/h5boss/4258-55481.hdf5 +/global/cscratch1/sd/jialin/h5boss/4259-55501.hdf5 +/global/cscratch1/sd/jialin/h5boss/4260-55478.hdf5 +/global/cscratch1/sd/jialin/h5boss/4261-55503.hdf5 +/global/cscratch1/sd/jialin/h5boss/4262-55482.hdf5 +/global/cscratch1/sd/jialin/h5boss/4263-55484.hdf5 +/global/cscratch1/sd/jialin/h5boss/4264-55506.hdf5 +/global/cscratch1/sd/jialin/h5boss/4265-55505.hdf5 +/global/cscratch1/sd/jialin/h5boss/4266-55486.hdf5 +/global/cscratch1/sd/jialin/h5boss/4267-55484.hdf5 +/global/cscratch1/sd/jialin/h5boss/4268-55483.hdf5 +/global/cscratch1/sd/jialin/h5boss/4269-55502.hdf5 +/global/cscratch1/sd/jialin/h5boss/4270-55511.hdf5 +/global/cscratch1/sd/jialin/h5boss/4271-55507.hdf5 +/global/cscratch1/sd/jialin/h5boss/4272-55509.hdf5 +/global/cscratch1/sd/jialin/h5boss/4273-55506.hdf5 +/global/cscratch1/sd/jialin/h5boss/4274-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/4275-55501.hdf5 +/global/cscratch1/sd/jialin/h5boss/4276-55505.hdf5 +/global/cscratch1/sd/jialin/h5boss/4277-55506.hdf5 +/global/cscratch1/sd/jialin/h5boss/4278-55505.hdf5 +/global/cscratch1/sd/jialin/h5boss/4279-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/4280-55503.hdf5 +/global/cscratch1/sd/jialin/h5boss/4281-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/4282-55507.hdf5 +/global/cscratch1/sd/jialin/h5boss/4283-55864.hdf5 +/global/cscratch1/sd/jialin/h5boss/4284-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/4285-55881.hdf5 +/global/cscratch1/sd/jialin/h5boss/4286-55499.hdf5 +/global/cscratch1/sd/jialin/h5boss/4287-55483.hdf5 +/global/cscratch1/sd/jialin/h5boss/4288-55501.hdf5 +/global/cscratch1/sd/jialin/h5boss/4289-55856.hdf5 +/global/cscratch1/sd/jialin/h5boss/4290-55527.hdf5 +/global/cscratch1/sd/jialin/h5boss/4291-55525.hdf5 +/global/cscratch1/sd/jialin/h5boss/4292-55532.hdf5 +/global/cscratch1/sd/jialin/h5boss/4293-55509.hdf5 +/global/cscratch1/sd/jialin/h5boss/4294-55500.hdf5 +/global/cscratch1/sd/jialin/h5boss/4295-55858.hdf5 +/global/cscratch1/sd/jialin/h5boss/4296-55499.hdf5 +/global/cscratch1/sd/jialin/h5boss/4297-55806.hdf5 +/global/cscratch1/sd/jialin/h5boss/4298-55511.hdf5 +/global/cscratch1/sd/jialin/h5boss/4299-55827.hdf5 +/global/cscratch1/sd/jialin/h5boss/4300-55528.hdf5 +/global/cscratch1/sd/jialin/h5boss/4301-55810.hdf5 +/global/cscratch1/sd/jialin/h5boss/4302-55531.hdf5 +/global/cscratch1/sd/jialin/h5boss/4303-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/4304-55506.hdf5 +/global/cscratch1/sd/jialin/h5boss/4305-55509.hdf5 +/global/cscratch1/sd/jialin/h5boss/4306-55584.hdf5 +/global/cscratch1/sd/jialin/h5boss/4307-55531.hdf5 +/global/cscratch1/sd/jialin/h5boss/4308-55565.hdf5 +/global/cscratch1/sd/jialin/h5boss/4309-55528.hdf5 +/global/cscratch1/sd/jialin/h5boss/4310-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/4311-55506.hdf5 +/global/cscratch1/sd/jialin/h5boss/4312-55511.hdf5 +/global/cscratch1/sd/jialin/h5boss/4313-55857.hdf5 +/global/cscratch1/sd/jialin/h5boss/4314-55855.hdf5 +/global/cscratch1/sd/jialin/h5boss/4315-55503.hdf5 +/global/cscratch1/sd/jialin/h5boss/4316-55505.hdf5 +/global/cscratch1/sd/jialin/h5boss/4317-55480.hdf5 +/global/cscratch1/sd/jialin/h5boss/4318-55508.hdf5 +/global/cscratch1/sd/jialin/h5boss/4319-55507.hdf5 +/global/cscratch1/sd/jialin/h5boss/4320-55894.hdf5 +/global/cscratch1/sd/jialin/h5boss/4321-55504.hdf5 +/global/cscratch1/sd/jialin/h5boss/4322-55503.hdf5 +/global/cscratch1/sd/jialin/h5boss/4323-55884.hdf5 +/global/cscratch1/sd/jialin/h5boss/4341-55538.hdf5 +/global/cscratch1/sd/jialin/h5boss/4342-55531.hdf5 +/global/cscratch1/sd/jialin/h5boss/4343-55540.hdf5 +/global/cscratch1/sd/jialin/h5boss/4344-55557.hdf5 +/global/cscratch1/sd/jialin/h5boss/4345-55569.hdf5 +/global/cscratch1/sd/jialin/h5boss/4346-56209.hdf5 +/global/cscratch1/sd/jialin/h5boss/4347-55830.hdf5 +/global/cscratch1/sd/jialin/h5boss/4348-55559.hdf5 +/global/cscratch1/sd/jialin/h5boss/4349-55803.hdf5 +/global/cscratch1/sd/jialin/h5boss/4350-55556.hdf5 +/global/cscratch1/sd/jialin/h5boss/4351-55538.hdf5 +/global/cscratch1/sd/jialin/h5boss/4352-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/4353-55532.hdf5 +/global/cscratch1/sd/jialin/h5boss/4354-55810.hdf5 +/global/cscratch1/sd/jialin/h5boss/4355-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/4356-55829.hdf5 +/global/cscratch1/sd/jialin/h5boss/4357-55829.hdf5 +/global/cscratch1/sd/jialin/h5boss/4358-55833.hdf5 +/global/cscratch1/sd/jialin/h5boss/4359-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/4360-55539.hdf5 +/global/cscratch1/sd/jialin/h5boss/4361-55831.hdf5 +/global/cscratch1/sd/jialin/h5boss/4362-55828.hdf5 +/global/cscratch1/sd/jialin/h5boss/4363-55537.hdf5 +/global/cscratch1/sd/jialin/h5boss/4364-55855.hdf5 +/global/cscratch1/sd/jialin/h5boss/4365-55539.hdf5 +/global/cscratch1/sd/jialin/h5boss/4366-55536.hdf5 +/global/cscratch1/sd/jialin/h5boss/4367-55566.hdf5 +/global/cscratch1/sd/jialin/h5boss/4368-55559.hdf5 +/global/cscratch1/sd/jialin/h5boss/4369-55827.hdf5 +/global/cscratch1/sd/jialin/h5boss/4370-55534.hdf5 +/global/cscratch1/sd/jialin/h5boss/4371-55830.hdf5 +/global/cscratch1/sd/jialin/h5boss/4372-55541.hdf5 +/global/cscratch1/sd/jialin/h5boss/4373-55811.hdf5 +/global/cscratch1/sd/jialin/h5boss/4374-55883.hdf5 +/global/cscratch1/sd/jialin/h5boss/4375-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/4376-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/4377-55828.hdf5 +/global/cscratch1/sd/jialin/h5boss/4378-55853.hdf5 +/global/cscratch1/sd/jialin/h5boss/4379-55881.hdf5 +/global/cscratch1/sd/jialin/h5boss/4380-55857.hdf5 +/global/cscratch1/sd/jialin/h5boss/4381-55824.hdf5 +/global/cscratch1/sd/jialin/h5boss/4382-55742.hdf5 +/global/cscratch1/sd/jialin/h5boss/4383-55881.hdf5 +/global/cscratch1/sd/jialin/h5boss/4384-56105.hdf5 +/global/cscratch1/sd/jialin/h5boss/4385-55752.hdf5 +/global/cscratch1/sd/jialin/h5boss/4386-55540.hdf5 +/global/cscratch1/sd/jialin/h5boss/4387-55534.hdf5 +/global/cscratch1/sd/jialin/h5boss/4388-55536.hdf5 +/global/cscratch1/sd/jialin/h5boss/4389-55539.hdf5 +/global/cscratch1/sd/jialin/h5boss/4390-55948.hdf5 +/global/cscratch1/sd/jialin/h5boss/4391-55866.hdf5 +/global/cscratch1/sd/jialin/h5boss/4392-55833.hdf5 +/global/cscratch1/sd/jialin/h5boss/4393-55944.hdf5 +/global/cscratch1/sd/jialin/h5boss/4394-55924.hdf5 +/global/cscratch1/sd/jialin/h5boss/4395-55828.hdf5 +/global/cscratch1/sd/jialin/h5boss/4396-55829.hdf5 +/global/cscratch1/sd/jialin/h5boss/4397-55921.hdf5 +/global/cscratch1/sd/jialin/h5boss/4398-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/4399-55811.hdf5 +/global/cscratch1/sd/jialin/h5boss/4400-55509.hdf5 +/global/cscratch1/sd/jialin/h5boss/4401-55510.hdf5 +/global/cscratch1/sd/jialin/h5boss/4402-55514.hdf5 +/global/cscratch1/sd/jialin/h5boss/4403-55536.hdf5 +/global/cscratch1/sd/jialin/h5boss/4404-55513.hdf5 +/global/cscratch1/sd/jialin/h5boss/4405-55854.hdf5 +/global/cscratch1/sd/jialin/h5boss/4406-55858.hdf5 +/global/cscratch1/sd/jialin/h5boss/4407-55884.hdf5 +/global/cscratch1/sd/jialin/h5boss/4408-55888.hdf5 +/global/cscratch1/sd/jialin/h5boss/4409-55883.hdf5 +/global/cscratch1/sd/jialin/h5boss/4410-56187.hdf5 +/global/cscratch1/sd/jialin/h5boss/4411-56164.hdf5 +/global/cscratch1/sd/jialin/h5boss/4412-55912.hdf5 +/global/cscratch1/sd/jialin/h5boss/4413-55894.hdf5 +/global/cscratch1/sd/jialin/h5boss/4414-55882.hdf5 +/global/cscratch1/sd/jialin/h5boss/4415-55831.hdf5 +/global/cscratch1/sd/jialin/h5boss/4416-55828.hdf5 +/global/cscratch1/sd/jialin/h5boss/4417-55829.hdf5 +/global/cscratch1/sd/jialin/h5boss/4418-55862.hdf5 +/global/cscratch1/sd/jialin/h5boss/4419-55867.hdf5 +/global/cscratch1/sd/jialin/h5boss/4420-55883.hdf5 +/global/cscratch1/sd/jialin/h5boss/4421-55858.hdf5 +/global/cscratch1/sd/jialin/h5boss/4422-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/4423-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/4424-55532.hdf5 +/global/cscratch1/sd/jialin/h5boss/4425-55864.hdf5 +/global/cscratch1/sd/jialin/h5boss/4426-56105.hdf5 +/global/cscratch1/sd/jialin/h5boss/4427-56107.hdf5 +/global/cscratch1/sd/jialin/h5boss/4428-56189.hdf5 +/global/cscratch1/sd/jialin/h5boss/4440-55539.hdf5 +/global/cscratch1/sd/jialin/h5boss/4441-55532.hdf5 +/global/cscratch1/sd/jialin/h5boss/4442-55532.hdf5 +/global/cscratch1/sd/jialin/h5boss/4443-55539.hdf5 +/global/cscratch1/sd/jialin/h5boss/4444-55538.hdf5 +/global/cscratch1/sd/jialin/h5boss/4445-55869.hdf5 +/global/cscratch1/sd/jialin/h5boss/4446-55589.hdf5 +/global/cscratch1/sd/jialin/h5boss/4447-55542.hdf5 +/global/cscratch1/sd/jialin/h5boss/4448-55538.hdf5 +/global/cscratch1/sd/jialin/h5boss/4449-55544.hdf5 +/global/cscratch1/sd/jialin/h5boss/4450-55540.hdf5 +/global/cscratch1/sd/jialin/h5boss/4451-55537.hdf5 +/global/cscratch1/sd/jialin/h5boss/4452-55536.hdf5 +/global/cscratch1/sd/jialin/h5boss/4453-55535.hdf5 +/global/cscratch1/sd/jialin/h5boss/4454-55536.hdf5 +/global/cscratch1/sd/jialin/h5boss/4455-55539.hdf5 +/global/cscratch1/sd/jialin/h5boss/4456-55537.hdf5 +/global/cscratch1/sd/jialin/h5boss/4457-55858.hdf5 +/global/cscratch1/sd/jialin/h5boss/4458-55536.hdf5 +/global/cscratch1/sd/jialin/h5boss/4459-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/4460-55533.hdf5 +/global/cscratch1/sd/jialin/h5boss/4461-55888.hdf5 +/global/cscratch1/sd/jialin/h5boss/4462-55600.hdf5 +/global/cscratch1/sd/jialin/h5boss/4463-55868.hdf5 +/global/cscratch1/sd/jialin/h5boss/4464-55866.hdf5 +/global/cscratch1/sd/jialin/h5boss/4465-55858.hdf5 +/global/cscratch1/sd/jialin/h5boss/4466-55857.hdf5 +/global/cscratch1/sd/jialin/h5boss/4467-55894.hdf5 +/global/cscratch1/sd/jialin/h5boss/4468-55894.hdf5 +/global/cscratch1/sd/jialin/h5boss/4469-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/4470-55587.hdf5 +/global/cscratch1/sd/jialin/h5boss/4471-55617.hdf5 +/global/cscratch1/sd/jialin/h5boss/4472-55893.hdf5 +/global/cscratch1/sd/jialin/h5boss/4473-55589.hdf5 +/global/cscratch1/sd/jialin/h5boss/4478-55600.hdf5 +/global/cscratch1/sd/jialin/h5boss/4479-55592.hdf5 +/global/cscratch1/sd/jialin/h5boss/4480-55591.hdf5 +/global/cscratch1/sd/jialin/h5boss/4481-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4482-55617.hdf5 +/global/cscratch1/sd/jialin/h5boss/4483-55587.hdf5 +/global/cscratch1/sd/jialin/h5boss/4484-55565.hdf5 +/global/cscratch1/sd/jialin/h5boss/4485-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/4486-55588.hdf5 +/global/cscratch1/sd/jialin/h5boss/4487-55866.hdf5 +/global/cscratch1/sd/jialin/h5boss/4488-55571.hdf5 +/global/cscratch1/sd/jialin/h5boss/4489-55545.hdf5 +/global/cscratch1/sd/jialin/h5boss/4490-55629.hdf5 +/global/cscratch1/sd/jialin/h5boss/4491-55570.hdf5 +/global/cscratch1/sd/jialin/h5boss/4492-55565.hdf5 +/global/cscratch1/sd/jialin/h5boss/4493-55585.hdf5 +/global/cscratch1/sd/jialin/h5boss/4494-55569.hdf5 +/global/cscratch1/sd/jialin/h5boss/4495-55566.hdf5 +/global/cscratch1/sd/jialin/h5boss/4496-55544.hdf5 +/global/cscratch1/sd/jialin/h5boss/4497-55564.hdf5 +/global/cscratch1/sd/jialin/h5boss/4498-55615.hdf5 +/global/cscratch1/sd/jialin/h5boss/4499-55572.hdf5 +/global/cscratch1/sd/jialin/h5boss/4500-55543.hdf5 +/global/cscratch1/sd/jialin/h5boss/4501-55590.hdf5 +/global/cscratch1/sd/jialin/h5boss/4502-55569.hdf5 +/global/cscratch1/sd/jialin/h5boss/4503-55563.hdf5 +/global/cscratch1/sd/jialin/h5boss/4504-55571.hdf5 +/global/cscratch1/sd/jialin/h5boss/4505-55603.hdf5 +/global/cscratch1/sd/jialin/h5boss/4506-55568.hdf5 +/global/cscratch1/sd/jialin/h5boss/4507-55564.hdf5 +/global/cscratch1/sd/jialin/h5boss/4508-55600.hdf5 +/global/cscratch1/sd/jialin/h5boss/4509-55574.hdf5 +/global/cscratch1/sd/jialin/h5boss/4510-55559.hdf5 +/global/cscratch1/sd/jialin/h5boss/4511-55602.hdf5 +/global/cscratch1/sd/jialin/h5boss/4526-55559.hdf5 +/global/cscratch1/sd/jialin/h5boss/4527-55590.hdf5 +/global/cscratch1/sd/jialin/h5boss/4528-55559.hdf5 +/global/cscratch1/sd/jialin/h5boss/4529-55563.hdf5 +/global/cscratch1/sd/jialin/h5boss/4530-55564.hdf5 +/global/cscratch1/sd/jialin/h5boss/4531-55563.hdf5 +/global/cscratch1/sd/jialin/h5boss/4532-55559.hdf5 +/global/cscratch1/sd/jialin/h5boss/4533-55855.hdf5 +/global/cscratch1/sd/jialin/h5boss/4534-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/4535-55860.hdf5 +/global/cscratch1/sd/jialin/h5boss/4536-55857.hdf5 +/global/cscratch1/sd/jialin/h5boss/4537-55806.hdf5 +/global/cscratch1/sd/jialin/h5boss/4538-55860.hdf5 +/global/cscratch1/sd/jialin/h5boss/4539-55865.hdf5 +/global/cscratch1/sd/jialin/h5boss/4540-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/4541-55882.hdf5 +/global/cscratch1/sd/jialin/h5boss/4542-55833.hdf5 +/global/cscratch1/sd/jialin/h5boss/4543-55888.hdf5 +/global/cscratch1/sd/jialin/h5boss/4544-55855.hdf5 +/global/cscratch1/sd/jialin/h5boss/4545-55567.hdf5 +/global/cscratch1/sd/jialin/h5boss/4546-55835.hdf5 +/global/cscratch1/sd/jialin/h5boss/4547-55893.hdf5 +/global/cscratch1/sd/jialin/h5boss/4548-55565.hdf5 +/global/cscratch1/sd/jialin/h5boss/4549-55556.hdf5 +/global/cscratch1/sd/jialin/h5boss/4550-55894.hdf5 +/global/cscratch1/sd/jialin/h5boss/4551-55569.hdf5 +/global/cscratch1/sd/jialin/h5boss/4552-55884.hdf5 +/global/cscratch1/sd/jialin/h5boss/4553-55584.hdf5 +/global/cscratch1/sd/jialin/h5boss/4554-56193.hdf5 +/global/cscratch1/sd/jialin/h5boss/4555-56189.hdf5 +/global/cscratch1/sd/jialin/h5boss/4556-55912.hdf5 +/global/cscratch1/sd/jialin/h5boss/4557-55588.hdf5 +/global/cscratch1/sd/jialin/h5boss/4558-55569.hdf5 +/global/cscratch1/sd/jialin/h5boss/4559-55597.hdf5 +/global/cscratch1/sd/jialin/h5boss/4560-55575.hdf5 +/global/cscratch1/sd/jialin/h5boss/4561-55614.hdf5 +/global/cscratch1/sd/jialin/h5boss/4562-55570.hdf5 +/global/cscratch1/sd/jialin/h5boss/4563-55617.hdf5 +/global/cscratch1/sd/jialin/h5boss/4564-55570.hdf5 +/global/cscratch1/sd/jialin/h5boss/4565-55591.hdf5 +/global/cscratch1/sd/jialin/h5boss/4566-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4567-55589.hdf5 +/global/cscratch1/sd/jialin/h5boss/4568-55600.hdf5 +/global/cscratch1/sd/jialin/h5boss/4569-55631.hdf5 +/global/cscratch1/sd/jialin/h5boss/4570-55623.hdf5 +/global/cscratch1/sd/jialin/h5boss/4571-55629.hdf5 +/global/cscratch1/sd/jialin/h5boss/4572-55622.hdf5 +/global/cscratch1/sd/jialin/h5boss/4573-55587.hdf5 +/global/cscratch1/sd/jialin/h5boss/4574-55621.hdf5 +/global/cscratch1/sd/jialin/h5boss/4575-55590.hdf5 +/global/cscratch1/sd/jialin/h5boss/4576-55592.hdf5 +/global/cscratch1/sd/jialin/h5boss/4601-55589.hdf5 +/global/cscratch1/sd/jialin/h5boss/4602-55644.hdf5 +/global/cscratch1/sd/jialin/h5boss/4603-55999.hdf5 +/global/cscratch1/sd/jialin/h5boss/4604-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/4605-55971.hdf5 +/global/cscratch1/sd/jialin/h5boss/4606-56223.hdf5 +/global/cscratch1/sd/jialin/h5boss/4607-56248.hdf5 +/global/cscratch1/sd/jialin/h5boss/4608-55973.hdf5 +/global/cscratch1/sd/jialin/h5boss/4609-56251.hdf5 +/global/cscratch1/sd/jialin/h5boss/4610-55621.hdf5 +/global/cscratch1/sd/jialin/h5boss/4611-55603.hdf5 +/global/cscratch1/sd/jialin/h5boss/4612-55590.hdf5 +/global/cscratch1/sd/jialin/h5boss/4613-55591.hdf5 +/global/cscratch1/sd/jialin/h5boss/4614-55604.hdf5 +/global/cscratch1/sd/jialin/h5boss/4615-55618.hdf5 +/global/cscratch1/sd/jialin/h5boss/4616-55617.hdf5 +/global/cscratch1/sd/jialin/h5boss/4617-55617.hdf5 +/global/cscratch1/sd/jialin/h5boss/4618-55600.hdf5 +/global/cscratch1/sd/jialin/h5boss/4619-55599.hdf5 +/global/cscratch1/sd/jialin/h5boss/4620-55652.hdf5 +/global/cscratch1/sd/jialin/h5boss/4621-55603.hdf5 +/global/cscratch1/sd/jialin/h5boss/4622-55629.hdf5 +/global/cscratch1/sd/jialin/h5boss/4623-55621.hdf5 +/global/cscratch1/sd/jialin/h5boss/4624-55654.hdf5 +/global/cscratch1/sd/jialin/h5boss/4625-55632.hdf5 +/global/cscratch1/sd/jialin/h5boss/4626-55647.hdf5 +/global/cscratch1/sd/jialin/h5boss/4627-55626.hdf5 +/global/cscratch1/sd/jialin/h5boss/4628-55646.hdf5 +/global/cscratch1/sd/jialin/h5boss/4629-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4630-55623.hdf5 +/global/cscratch1/sd/jialin/h5boss/4631-55617.hdf5 +/global/cscratch1/sd/jialin/h5boss/4632-55644.hdf5 +/global/cscratch1/sd/jialin/h5boss/4633-55620.hdf5 +/global/cscratch1/sd/jialin/h5boss/4634-55626.hdf5 +/global/cscratch1/sd/jialin/h5boss/4635-55615.hdf5 +/global/cscratch1/sd/jialin/h5boss/4636-55945.hdf5 +/global/cscratch1/sd/jialin/h5boss/4637-55616.hdf5 +/global/cscratch1/sd/jialin/h5boss/4638-55956.hdf5 +/global/cscratch1/sd/jialin/h5boss/4639-55944.hdf5 +/global/cscratch1/sd/jialin/h5boss/4640-55927.hdf5 +/global/cscratch1/sd/jialin/h5boss/4641-55947.hdf5 +/global/cscratch1/sd/jialin/h5boss/4642-55926.hdf5 +/global/cscratch1/sd/jialin/h5boss/4643-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/4644-55922.hdf5 +/global/cscratch1/sd/jialin/h5boss/4645-55623.hdf5 +/global/cscratch1/sd/jialin/h5boss/4646-55622.hdf5 +/global/cscratch1/sd/jialin/h5boss/4647-55621.hdf5 +/global/cscratch1/sd/jialin/h5boss/4648-55673.hdf5 +/global/cscratch1/sd/jialin/h5boss/4649-55621.hdf5 +/global/cscratch1/sd/jialin/h5boss/4650-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/4651-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/4652-55672.hdf5 +/global/cscratch1/sd/jialin/h5boss/4653-55622.hdf5 +/global/cscratch1/sd/jialin/h5boss/4654-55659.hdf5 +/global/cscratch1/sd/jialin/h5boss/4655-55620.hdf5 +/global/cscratch1/sd/jialin/h5boss/4656-55828.hdf5 +/global/cscratch1/sd/jialin/h5boss/4657-55591.hdf5 +/global/cscratch1/sd/jialin/h5boss/4658-55592.hdf5 +/global/cscratch1/sd/jialin/h5boss/4659-55587.hdf5 +/global/cscratch1/sd/jialin/h5boss/4660-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/4661-55614.hdf5 +/global/cscratch1/sd/jialin/h5boss/4662-55590.hdf5 +/global/cscratch1/sd/jialin/h5boss/4663-55893.hdf5 +/global/cscratch1/sd/jialin/h5boss/4664-56192.hdf5 +/global/cscratch1/sd/jialin/h5boss/4665-56209.hdf5 +/global/cscratch1/sd/jialin/h5boss/4666-55832.hdf5 +/global/cscratch1/sd/jialin/h5boss/4667-55868.hdf5 +/global/cscratch1/sd/jialin/h5boss/4668-56211.hdf5 +/global/cscratch1/sd/jialin/h5boss/4669-55831.hdf5 +/global/cscratch1/sd/jialin/h5boss/4683-55924.hdf5 +/global/cscratch1/sd/jialin/h5boss/4684-56226.hdf5 +/global/cscratch1/sd/jialin/h5boss/4685-55657.hdf5 +/global/cscratch1/sd/jialin/h5boss/4686-56013.hdf5 +/global/cscratch1/sd/jialin/h5boss/4687-56338.hdf5 +/global/cscratch1/sd/jialin/h5boss/4688-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/4689-55656.hdf5 +/global/cscratch1/sd/jialin/h5boss/4690-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4691-55651.hdf5 +/global/cscratch1/sd/jialin/h5boss/4692-55644.hdf5 +/global/cscratch1/sd/jialin/h5boss/4693-55632.hdf5 +/global/cscratch1/sd/jialin/h5boss/4694-55649.hdf5 +/global/cscratch1/sd/jialin/h5boss/4695-55957.hdf5 +/global/cscratch1/sd/jialin/h5boss/4696-56354.hdf5 +/global/cscratch1/sd/jialin/h5boss/4697-55660.hdf5 +/global/cscratch1/sd/jialin/h5boss/4698-55623.hdf5 +/global/cscratch1/sd/jialin/h5boss/4699-55684.hdf5 +/global/cscratch1/sd/jialin/h5boss/4700-55709.hdf5 +/global/cscratch1/sd/jialin/h5boss/4701-55709.hdf5 +/global/cscratch1/sd/jialin/h5boss/4702-55618.hdf5 +/global/cscratch1/sd/jialin/h5boss/4703-55617.hdf5 +/global/cscratch1/sd/jialin/h5boss/4704-55681.hdf5 +/global/cscratch1/sd/jialin/h5boss/4705-55705.hdf5 +/global/cscratch1/sd/jialin/h5boss/4706-55705.hdf5 +/global/cscratch1/sd/jialin/h5boss/4707-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4708-55704.hdf5 +/global/cscratch1/sd/jialin/h5boss/4709-55720.hdf5 +/global/cscratch1/sd/jialin/h5boss/4710-55707.hdf5 +/global/cscratch1/sd/jialin/h5boss/4711-55737.hdf5 +/global/cscratch1/sd/jialin/h5boss/4712-55738.hdf5 +/global/cscratch1/sd/jialin/h5boss/4713-56044.hdf5 +/global/cscratch1/sd/jialin/h5boss/4714-56041.hdf5 +/global/cscratch1/sd/jialin/h5boss/4715-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4716-55738.hdf5 +/global/cscratch1/sd/jialin/h5boss/4717-55742.hdf5 +/global/cscratch1/sd/jialin/h5boss/4718-55750.hdf5 +/global/cscratch1/sd/jialin/h5boss/4719-55736.hdf5 +/global/cscratch1/sd/jialin/h5boss/4720-55691.hdf5 +/global/cscratch1/sd/jialin/h5boss/4721-55709.hdf5 +/global/cscratch1/sd/jialin/h5boss/4722-55735.hdf5 +/global/cscratch1/sd/jialin/h5boss/4723-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/4724-55742.hdf5 +/global/cscratch1/sd/jialin/h5boss/4725-55711.hdf5 +/global/cscratch1/sd/jialin/h5boss/4726-55712.hdf5 +/global/cscratch1/sd/jialin/h5boss/4727-55693.hdf5 +/global/cscratch1/sd/jialin/h5boss/4729-55626.hdf5 +/global/cscratch1/sd/jialin/h5boss/4730-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4731-55656.hdf5 +/global/cscratch1/sd/jialin/h5boss/4732-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/4733-55649.hdf5 +/global/cscratch1/sd/jialin/h5boss/4734-55646.hdf5 +/global/cscratch1/sd/jialin/h5boss/4735-55651.hdf5 +/global/cscratch1/sd/jialin/h5boss/4736-55631.hdf5 +/global/cscratch1/sd/jialin/h5boss/4737-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4738-55650.hdf5 +/global/cscratch1/sd/jialin/h5boss/4739-55644.hdf5 +/global/cscratch1/sd/jialin/h5boss/4740-55651.hdf5 +/global/cscratch1/sd/jialin/h5boss/4741-55704.hdf5 +/global/cscratch1/sd/jialin/h5boss/4742-55660.hdf5 +/global/cscratch1/sd/jialin/h5boss/4743-55645.hdf5 +/global/cscratch1/sd/jialin/h5boss/4745-55892.hdf5 +/global/cscratch1/sd/jialin/h5boss/4746-55632.hdf5 +/global/cscratch1/sd/jialin/h5boss/4747-55652.hdf5 +/global/cscratch1/sd/jialin/h5boss/4748-55631.hdf5 +/global/cscratch1/sd/jialin/h5boss/4749-55633.hdf5 +/global/cscratch1/sd/jialin/h5boss/4750-55630.hdf5 +/global/cscratch1/sd/jialin/h5boss/4751-55646.hdf5 +/global/cscratch1/sd/jialin/h5boss/4752-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4753-55674.hdf5 +/global/cscratch1/sd/jialin/h5boss/4754-55649.hdf5 +/global/cscratch1/sd/jialin/h5boss/4755-55660.hdf5 +/global/cscratch1/sd/jialin/h5boss/4756-55631.hdf5 +/global/cscratch1/sd/jialin/h5boss/4757-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4758-55682.hdf5 +/global/cscratch1/sd/jialin/h5boss/4759-55649.hdf5 +/global/cscratch1/sd/jialin/h5boss/4760-55656.hdf5 +/global/cscratch1/sd/jialin/h5boss/4761-55633.hdf5 +/global/cscratch1/sd/jialin/h5boss/4762-55888.hdf5 +/global/cscratch1/sd/jialin/h5boss/4763-55869.hdf5 +/global/cscratch1/sd/jialin/h5boss/4764-55646.hdf5 +/global/cscratch1/sd/jialin/h5boss/4765-55674.hdf5 +/global/cscratch1/sd/jialin/h5boss/4766-55677.hdf5 +/global/cscratch1/sd/jialin/h5boss/4767-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/4768-55944.hdf5 +/global/cscratch1/sd/jialin/h5boss/4769-55931.hdf5 +/global/cscratch1/sd/jialin/h5boss/4770-55928.hdf5 +/global/cscratch1/sd/jialin/h5boss/4771-55925.hdf5 +/global/cscratch1/sd/jialin/h5boss/4772-55654.hdf5 +/global/cscratch1/sd/jialin/h5boss/4773-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/4774-55659.hdf5 +/global/cscratch1/sd/jialin/h5boss/4775-55708.hdf5 +/global/cscratch1/sd/jialin/h5boss/4776-55652.hdf5 +/global/cscratch1/sd/jialin/h5boss/4777-55707.hdf5 +/global/cscratch1/sd/jialin/h5boss/4778-55706.hdf5 +/global/cscratch1/sd/jialin/h5boss/4779-55656.hdf5 +/global/cscratch1/sd/jialin/h5boss/4780-55682.hdf5 +/global/cscratch1/sd/jialin/h5boss/4781-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4782-55654.hdf5 +/global/cscratch1/sd/jialin/h5boss/4783-55652.hdf5 +/global/cscratch1/sd/jialin/h5boss/4784-55677.hdf5 +/global/cscratch1/sd/jialin/h5boss/4785-55659.hdf5 +/global/cscratch1/sd/jialin/h5boss/4786-55651.hdf5 +/global/cscratch1/sd/jialin/h5boss/4787-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/4788-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/4789-55923.hdf5 +/global/cscratch1/sd/jialin/h5boss/4790-55888.hdf5 +/global/cscratch1/sd/jialin/h5boss/4791-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/4792-55925.hdf5 +/global/cscratch1/sd/jialin/h5boss/4793-55648.hdf5 +/global/cscratch1/sd/jialin/h5boss/4794-55647.hdf5 +/global/cscratch1/sd/jialin/h5boss/4795-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/4796-55924.hdf5 +/global/cscratch1/sd/jialin/h5boss/4797-55662.hdf5 +/global/cscratch1/sd/jialin/h5boss/4798-55672.hdf5 +/global/cscratch1/sd/jialin/h5boss/4799-55656.hdf5 +/global/cscratch1/sd/jialin/h5boss/4800-55674.hdf5 +/global/cscratch1/sd/jialin/h5boss/4801-55653.hdf5 +/global/cscratch1/sd/jialin/h5boss/4802-55652.hdf5 +/global/cscratch1/sd/jialin/h5boss/4803-55734.hdf5 +/global/cscratch1/sd/jialin/h5boss/4804-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/4805-55715.hdf5 +/global/cscratch1/sd/jialin/h5boss/4806-55688.hdf5 +/global/cscratch1/sd/jialin/h5boss/4807-55687.hdf5 +/global/cscratch1/sd/jialin/h5boss/4808-55705.hdf5 +/global/cscratch1/sd/jialin/h5boss/4831-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/4832-55680.hdf5 +/global/cscratch1/sd/jialin/h5boss/4833-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/4834-55685.hdf5 +/global/cscratch1/sd/jialin/h5boss/4835-55688.hdf5 +/global/cscratch1/sd/jialin/h5boss/4836-55689.hdf5 +/global/cscratch1/sd/jialin/h5boss/4837-55707.hdf5 +/global/cscratch1/sd/jialin/h5boss/4838-55686.hdf5 +/global/cscratch1/sd/jialin/h5boss/4839-55703.hdf5 +/global/cscratch1/sd/jialin/h5boss/4840-55690.hdf5 +/global/cscratch1/sd/jialin/h5boss/4841-55895.hdf5 +/global/cscratch1/sd/jialin/h5boss/4842-55867.hdf5 +/global/cscratch1/sd/jialin/h5boss/4843-55860.hdf5 +/global/cscratch1/sd/jialin/h5boss/4847-55931.hdf5 +/global/cscratch1/sd/jialin/h5boss/4848-55955.hdf5 +/global/cscratch1/sd/jialin/h5boss/4849-55945.hdf5 +/global/cscratch1/sd/jialin/h5boss/4850-55929.hdf5 +/global/cscratch1/sd/jialin/h5boss/4851-55680.hdf5 +/global/cscratch1/sd/jialin/h5boss/4852-55689.hdf5 +/global/cscratch1/sd/jialin/h5boss/4853-55928.hdf5 +/global/cscratch1/sd/jialin/h5boss/4854-55685.hdf5 +/global/cscratch1/sd/jialin/h5boss/4855-55926.hdf5 +/global/cscratch1/sd/jialin/h5boss/4856-55712.hdf5 +/global/cscratch1/sd/jialin/h5boss/4857-55711.hdf5 +/global/cscratch1/sd/jialin/h5boss/4858-55686.hdf5 +/global/cscratch1/sd/jialin/h5boss/4859-55684.hdf5 +/global/cscratch1/sd/jialin/h5boss/4860-55691.hdf5 +/global/cscratch1/sd/jialin/h5boss/4861-55710.hdf5 +/global/cscratch1/sd/jialin/h5boss/4862-55685.hdf5 +/global/cscratch1/sd/jialin/h5boss/4863-55688.hdf5 +/global/cscratch1/sd/jialin/h5boss/4864-55680.hdf5 +/global/cscratch1/sd/jialin/h5boss/4865-55713.hdf5 +/global/cscratch1/sd/jialin/h5boss/4866-55895.hdf5 +/global/cscratch1/sd/jialin/h5boss/4867-55924.hdf5 +/global/cscratch1/sd/jialin/h5boss/4868-55895.hdf5 +/global/cscratch1/sd/jialin/h5boss/4869-55896.hdf5 +/global/cscratch1/sd/jialin/h5boss/4870-55923.hdf5 +/global/cscratch1/sd/jialin/h5boss/4871-55928.hdf5 +/global/cscratch1/sd/jialin/h5boss/4872-55944.hdf5 +/global/cscratch1/sd/jialin/h5boss/4873-55926.hdf5 +/global/cscratch1/sd/jialin/h5boss/4874-55673.hdf5 +/global/cscratch1/sd/jialin/h5boss/4875-55677.hdf5 +/global/cscratch1/sd/jialin/h5boss/4876-55679.hdf5 +/global/cscratch1/sd/jialin/h5boss/4877-55707.hdf5 +/global/cscratch1/sd/jialin/h5boss/4878-55710.hdf5 +/global/cscratch1/sd/jialin/h5boss/4879-55706.hdf5 +/global/cscratch1/sd/jialin/h5boss/4881-55733.hdf5 +/global/cscratch1/sd/jialin/h5boss/4882-55721.hdf5 +/global/cscratch1/sd/jialin/h5boss/4885-55735.hdf5 +/global/cscratch1/sd/jialin/h5boss/4886-55737.hdf5 +/global/cscratch1/sd/jialin/h5boss/4889-55709.hdf5 +/global/cscratch1/sd/jialin/h5boss/4890-55741.hdf5 +/global/cscratch1/sd/jialin/h5boss/4891-55736.hdf5 +/global/cscratch1/sd/jialin/h5boss/4892-55738.hdf5 +/global/cscratch1/sd/jialin/h5boss/4893-55709.hdf5 +/global/cscratch1/sd/jialin/h5boss/4895-55708.hdf5 +/global/cscratch1/sd/jialin/h5boss/4896-55712.hdf5 +/global/cscratch1/sd/jialin/h5boss/4900-55739.hdf5 +/global/cscratch1/sd/jialin/h5boss/4901-55711.hdf5 +/global/cscratch1/sd/jialin/h5boss/4903-55927.hdf5 +/global/cscratch1/sd/jialin/h5boss/4953-55749.hdf5 +/global/cscratch1/sd/jialin/h5boss/4955-55750.hdf5 +/global/cscratch1/sd/jialin/h5boss/4956-55737.hdf5 +/global/cscratch1/sd/jialin/h5boss/4960-55747.hdf5 +/global/cscratch1/sd/jialin/h5boss/4961-55719.hdf5 +/global/cscratch1/sd/jialin/h5boss/4964-55749.hdf5 +/global/cscratch1/sd/jialin/h5boss/4965-55721.hdf5 +/global/cscratch1/sd/jialin/h5boss/4966-55712.hdf5 +/global/cscratch1/sd/jialin/h5boss/4969-56041.hdf5 +/global/cscratch1/sd/jialin/h5boss/4970-56039.hdf5 +/global/cscratch1/sd/jialin/h5boss/4971-55747.hdf5 +/global/cscratch1/sd/jialin/h5boss/4973-56042.hdf5 +/global/cscratch1/sd/jialin/h5boss/4974-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/4975-56037.hdf5 +/global/cscratch1/sd/jialin/h5boss/4976-56046.hdf5 +/global/cscratch1/sd/jialin/h5boss/4977-56047.hdf5 +/global/cscratch1/sd/jialin/h5boss/4978-56036.hdf5 +/global/cscratch1/sd/jialin/h5boss/4979-56045.hdf5 +/global/cscratch1/sd/jialin/h5boss/4980-56048.hdf5 +/global/cscratch1/sd/jialin/h5boss/4983-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/4984-55827.hdf5 +/global/cscratch1/sd/jialin/h5boss/4987-55746.hdf5 +/global/cscratch1/sd/jialin/h5boss/4988-55825.hdf5 +/global/cscratch1/sd/jialin/h5boss/4989-55743.hdf5 +/global/cscratch1/sd/jialin/h5boss/4990-55743.hdf5 +/global/cscratch1/sd/jialin/h5boss/4992-55723.hdf5 +/global/cscratch1/sd/jialin/h5boss/4993-55738.hdf5 +/global/cscratch1/sd/jialin/h5boss/4994-55739.hdf5 +/global/cscratch1/sd/jialin/h5boss/4995-55739.hdf5 +/global/cscratch1/sd/jialin/h5boss/4996-55720.hdf5 +/global/cscratch1/sd/jialin/h5boss/4997-55738.hdf5 +/global/cscratch1/sd/jialin/h5boss/4998-55722.hdf5 +/global/cscratch1/sd/jialin/h5boss/4999-55721.hdf5 +/global/cscratch1/sd/jialin/h5boss/5000-55715.hdf5 +/global/cscratch1/sd/jialin/h5boss/5001-55719.hdf5 +/global/cscratch1/sd/jialin/h5boss/5002-55710.hdf5 +/global/cscratch1/sd/jialin/h5boss/5003-55715.hdf5 +/global/cscratch1/sd/jialin/h5boss/5004-55711.hdf5 +/global/cscratch1/sd/jialin/h5boss/5005-55751.hdf5 +/global/cscratch1/sd/jialin/h5boss/5006-55706.hdf5 +/global/cscratch1/sd/jialin/h5boss/5007-55710.hdf5 +/global/cscratch1/sd/jialin/h5boss/5008-55744.hdf5 +/global/cscratch1/sd/jialin/h5boss/5009-55707.hdf5 +/global/cscratch1/sd/jialin/h5boss/5010-55748.hdf5 +/global/cscratch1/sd/jialin/h5boss/5011-55739.hdf5 +/global/cscratch1/sd/jialin/h5boss/5013-55723.hdf5 +/global/cscratch1/sd/jialin/h5boss/5014-55717.hdf5 +/global/cscratch1/sd/jialin/h5boss/5015-55707.hdf5 +/global/cscratch1/sd/jialin/h5boss/5016-55709.hdf5 +/global/cscratch1/sd/jialin/h5boss/5017-55715.hdf5 +/global/cscratch1/sd/jialin/h5boss/5018-55734.hdf5 +/global/cscratch1/sd/jialin/h5boss/5020-55852.hdf5 +/global/cscratch1/sd/jialin/h5boss/5021-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/5022-55827.hdf5 +/global/cscratch1/sd/jialin/h5boss/5023-55858.hdf5 +/global/cscratch1/sd/jialin/h5boss/5024-55854.hdf5 +/global/cscratch1/sd/jialin/h5boss/5025-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/5026-55855.hdf5 +/global/cscratch1/sd/jialin/h5boss/5027-56218.hdf5 +/global/cscratch1/sd/jialin/h5boss/5028-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/5029-55750.hdf5 +/global/cscratch1/sd/jialin/h5boss/5030-55830.hdf5 +/global/cscratch1/sd/jialin/h5boss/5031-56209.hdf5 +/global/cscratch1/sd/jialin/h5boss/5032-56208.hdf5 +/global/cscratch1/sd/jialin/h5boss/5033-56244.hdf5 +/global/cscratch1/sd/jialin/h5boss/5034-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/5035-55829.hdf5 +/global/cscratch1/sd/jialin/h5boss/5036-55751.hdf5 +/global/cscratch1/sd/jialin/h5boss/5037-55823.hdf5 +/global/cscratch1/sd/jialin/h5boss/5038-56235.hdf5 +/global/cscratch1/sd/jialin/h5boss/5039-56220.hdf5 +/global/cscratch1/sd/jialin/h5boss/5040-56243.hdf5 +/global/cscratch1/sd/jialin/h5boss/5041-55749.hdf5 +/global/cscratch1/sd/jialin/h5boss/5042-55856.hdf5 +/global/cscratch1/sd/jialin/h5boss/5043-56187.hdf5 +/global/cscratch1/sd/jialin/h5boss/5044-56186.hdf5 +/global/cscratch1/sd/jialin/h5boss/5045-56181.hdf5 +/global/cscratch1/sd/jialin/h5boss/5046-56243.hdf5 +/global/cscratch1/sd/jialin/h5boss/5047-55833.hdf5 +/global/cscratch1/sd/jialin/h5boss/5048-56218.hdf5 +/global/cscratch1/sd/jialin/h5boss/5049-56103.hdf5 +/global/cscratch1/sd/jialin/h5boss/5050-56215.hdf5 +/global/cscratch1/sd/jialin/h5boss/5051-56214.hdf5 +/global/cscratch1/sd/jialin/h5boss/5052-55857.hdf5 +/global/cscratch1/sd/jialin/h5boss/5053-56213.hdf5 +/global/cscratch1/sd/jialin/h5boss/5054-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/5055-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/5056-55921.hdf5 +/global/cscratch1/sd/jialin/h5boss/5057-56209.hdf5 +/global/cscratch1/sd/jialin/h5boss/5058-56208.hdf5 +/global/cscratch1/sd/jialin/h5boss/5059-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/5060-56181.hdf5 +/global/cscratch1/sd/jialin/h5boss/5061-55806.hdf5 +/global/cscratch1/sd/jialin/h5boss/5062-55803.hdf5 +/global/cscratch1/sd/jialin/h5boss/5063-55831.hdf5 +/global/cscratch1/sd/jialin/h5boss/5064-55864.hdf5 +/global/cscratch1/sd/jialin/h5boss/5065-55739.hdf5 +/global/cscratch1/sd/jialin/h5boss/5066-55749.hdf5 +/global/cscratch1/sd/jialin/h5boss/5067-55751.hdf5 +/global/cscratch1/sd/jialin/h5boss/5068-55749.hdf5 +/global/cscratch1/sd/jialin/h5boss/5069-56211.hdf5 +/global/cscratch1/sd/jialin/h5boss/5106-55833.hdf5 +/global/cscratch1/sd/jialin/h5boss/5107-55940.hdf5 +/global/cscratch1/sd/jialin/h5boss/5108-55888.hdf5 +/global/cscratch1/sd/jialin/h5boss/5109-55892.hdf5 +/global/cscratch1/sd/jialin/h5boss/5110-55913.hdf5 +/global/cscratch1/sd/jialin/h5boss/5111-55840.hdf5 +/global/cscratch1/sd/jialin/h5boss/5112-55895.hdf5 +/global/cscratch1/sd/jialin/h5boss/5113-55924.hdf5 +/global/cscratch1/sd/jialin/h5boss/5114-55883.hdf5 +/global/cscratch1/sd/jialin/h5boss/5115-55885.hdf5 +/global/cscratch1/sd/jialin/h5boss/5116-55857.hdf5 +/global/cscratch1/sd/jialin/h5boss/5117-55925.hdf5 +/global/cscratch1/sd/jialin/h5boss/5118-55830.hdf5 +/global/cscratch1/sd/jialin/h5boss/5119-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/5120-55895.hdf5 +/global/cscratch1/sd/jialin/h5boss/5121-55856.hdf5 +/global/cscratch1/sd/jialin/h5boss/5122-55841.hdf5 +/global/cscratch1/sd/jialin/h5boss/5123-55841.hdf5 +/global/cscratch1/sd/jialin/h5boss/5124-55894.hdf5 +/global/cscratch1/sd/jialin/h5boss/5125-55863.hdf5 +/global/cscratch1/sd/jialin/h5boss/5126-55923.hdf5 +/global/cscratch1/sd/jialin/h5boss/5127-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/5128-55912.hdf5 +/global/cscratch1/sd/jialin/h5boss/5129-55864.hdf5 +/global/cscratch1/sd/jialin/h5boss/5130-55835.hdf5 +/global/cscratch1/sd/jialin/h5boss/5131-55835.hdf5 +/global/cscratch1/sd/jialin/h5boss/5132-55858.hdf5 +/global/cscratch1/sd/jialin/h5boss/5133-55860.hdf5 +/global/cscratch1/sd/jialin/h5boss/5134-55868.hdf5 +/global/cscratch1/sd/jialin/h5boss/5135-55862.hdf5 +/global/cscratch1/sd/jialin/h5boss/5136-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/5137-55833.hdf5 +/global/cscratch1/sd/jialin/h5boss/5138-55830.hdf5 +/global/cscratch1/sd/jialin/h5boss/5139-55866.hdf5 +/global/cscratch1/sd/jialin/h5boss/5140-55836.hdf5 +/global/cscratch1/sd/jialin/h5boss/5141-55746.hdf5 +/global/cscratch1/sd/jialin/h5boss/5142-55825.hdf5 +/global/cscratch1/sd/jialin/h5boss/5143-55828.hdf5 +/global/cscratch1/sd/jialin/h5boss/5144-55829.hdf5 +/global/cscratch1/sd/jialin/h5boss/5145-55835.hdf5 +/global/cscratch1/sd/jialin/h5boss/5146-55831.hdf5 +/global/cscratch1/sd/jialin/h5boss/5147-55854.hdf5 +/global/cscratch1/sd/jialin/h5boss/5148-56220.hdf5 +/global/cscratch1/sd/jialin/h5boss/5149-55944.hdf5 +/global/cscratch1/sd/jialin/h5boss/5150-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/5151-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/5152-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5153-56577.hdf5 +/global/cscratch1/sd/jialin/h5boss/5154-55864.hdf5 +/global/cscratch1/sd/jialin/h5boss/5155-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/5156-55925.hdf5 +/global/cscratch1/sd/jialin/h5boss/5157-55923.hdf5 +/global/cscratch1/sd/jialin/h5boss/5158-55913.hdf5 +/global/cscratch1/sd/jialin/h5boss/5159-55856.hdf5 +/global/cscratch1/sd/jialin/h5boss/5160-55895.hdf5 +/global/cscratch1/sd/jialin/h5boss/5161-55968.hdf5 +/global/cscratch1/sd/jialin/h5boss/5162-55893.hdf5 +/global/cscratch1/sd/jialin/h5boss/5163-55889.hdf5 +/global/cscratch1/sd/jialin/h5boss/5164-56067.hdf5 +/global/cscratch1/sd/jialin/h5boss/5165-56063.hdf5 +/global/cscratch1/sd/jialin/h5boss/5166-56065.hdf5 +/global/cscratch1/sd/jialin/h5boss/5167-56066.hdf5 +/global/cscratch1/sd/jialin/h5boss/5168-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5169-56045.hdf5 +/global/cscratch1/sd/jialin/h5boss/5170-56063.hdf5 +/global/cscratch1/sd/jialin/h5boss/5171-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/5172-56065.hdf5 +/global/cscratch1/sd/jialin/h5boss/5173-56046.hdf5 +/global/cscratch1/sd/jialin/h5boss/5174-56047.hdf5 +/global/cscratch1/sd/jialin/h5boss/5175-55955.hdf5 +/global/cscratch1/sd/jialin/h5boss/5176-56221.hdf5 +/global/cscratch1/sd/jialin/h5boss/5177-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/5178-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/5179-55957.hdf5 +/global/cscratch1/sd/jialin/h5boss/5180-55947.hdf5 +/global/cscratch1/sd/jialin/h5boss/5181-56298.hdf5 +/global/cscratch1/sd/jialin/h5boss/5182-56341.hdf5 +/global/cscratch1/sd/jialin/h5boss/5183-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5184-56352.hdf5 +/global/cscratch1/sd/jialin/h5boss/5185-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5186-55987.hdf5 +/global/cscratch1/sd/jialin/h5boss/5187-56074.hdf5 +/global/cscratch1/sd/jialin/h5boss/5188-55803.hdf5 +/global/cscratch1/sd/jialin/h5boss/5189-56074.hdf5 +/global/cscratch1/sd/jialin/h5boss/5190-56077.hdf5 +/global/cscratch1/sd/jialin/h5boss/5191-56065.hdf5 +/global/cscratch1/sd/jialin/h5boss/5192-56066.hdf5 +/global/cscratch1/sd/jialin/h5boss/5193-56066.hdf5 +/global/cscratch1/sd/jialin/h5boss/5194-56062.hdf5 +/global/cscratch1/sd/jialin/h5boss/5195-56048.hdf5 +/global/cscratch1/sd/jialin/h5boss/5196-55831.hdf5 +/global/cscratch1/sd/jialin/h5boss/5197-55828.hdf5 +/global/cscratch1/sd/jialin/h5boss/5198-55823.hdf5 +/global/cscratch1/sd/jialin/h5boss/5199-56067.hdf5 +/global/cscratch1/sd/jialin/h5boss/5200-56091.hdf5 +/global/cscratch1/sd/jialin/h5boss/5201-55832.hdf5 +/global/cscratch1/sd/jialin/h5boss/5202-55824.hdf5 +/global/cscratch1/sd/jialin/h5boss/5203-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5204-56036.hdf5 +/global/cscratch1/sd/jialin/h5boss/5205-56040.hdf5 +/global/cscratch1/sd/jialin/h5boss/5206-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5207-56018.hdf5 +/global/cscratch1/sd/jialin/h5boss/5208-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/5209-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5210-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5211-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5212-56016.hdf5 +/global/cscratch1/sd/jialin/h5boss/5284-55866.hdf5 +/global/cscratch1/sd/jialin/h5boss/5285-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/5286-56225.hdf5 +/global/cscratch1/sd/jialin/h5boss/5287-55952.hdf5 +/global/cscratch1/sd/jialin/h5boss/5288-55865.hdf5 +/global/cscratch1/sd/jialin/h5boss/5289-55893.hdf5 +/global/cscratch1/sd/jialin/h5boss/5290-55862.hdf5 +/global/cscratch1/sd/jialin/h5boss/5291-55947.hdf5 +/global/cscratch1/sd/jialin/h5boss/5292-55926.hdf5 +/global/cscratch1/sd/jialin/h5boss/5293-55953.hdf5 +/global/cscratch1/sd/jialin/h5boss/5294-55922.hdf5 +/global/cscratch1/sd/jialin/h5boss/5295-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5296-55922.hdf5 +/global/cscratch1/sd/jialin/h5boss/5297-55913.hdf5 +/global/cscratch1/sd/jialin/h5boss/5298-55979.hdf5 +/global/cscratch1/sd/jialin/h5boss/5299-55927.hdf5 +/global/cscratch1/sd/jialin/h5boss/5300-55982.hdf5 +/global/cscratch1/sd/jialin/h5boss/5301-55987.hdf5 +/global/cscratch1/sd/jialin/h5boss/5302-55896.hdf5 +/global/cscratch1/sd/jialin/h5boss/5303-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5304-55925.hdf5 +/global/cscratch1/sd/jialin/h5boss/5305-55984.hdf5 +/global/cscratch1/sd/jialin/h5boss/5306-55926.hdf5 +/global/cscratch1/sd/jialin/h5boss/5307-55998.hdf5 +/global/cscratch1/sd/jialin/h5boss/5308-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5309-55929.hdf5 +/global/cscratch1/sd/jialin/h5boss/5310-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/5311-55956.hdf5 +/global/cscratch1/sd/jialin/h5boss/5312-56001.hdf5 +/global/cscratch1/sd/jialin/h5boss/5313-55973.hdf5 +/global/cscratch1/sd/jialin/h5boss/5314-55952.hdf5 +/global/cscratch1/sd/jialin/h5boss/5315-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5316-55955.hdf5 +/global/cscratch1/sd/jialin/h5boss/5317-56000.hdf5 +/global/cscratch1/sd/jialin/h5boss/5318-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/5319-55953.hdf5 +/global/cscratch1/sd/jialin/h5boss/5320-55999.hdf5 +/global/cscratch1/sd/jialin/h5boss/5321-55945.hdf5 +/global/cscratch1/sd/jialin/h5boss/5322-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5323-55957.hdf5 +/global/cscratch1/sd/jialin/h5boss/5324-55947.hdf5 +/global/cscratch1/sd/jialin/h5boss/5325-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5326-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5327-55979.hdf5 +/global/cscratch1/sd/jialin/h5boss/5328-55982.hdf5 +/global/cscratch1/sd/jialin/h5boss/5329-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/5330-56000.hdf5 +/global/cscratch1/sd/jialin/h5boss/5331-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5332-55981.hdf5 +/global/cscratch1/sd/jialin/h5boss/5333-56001.hdf5 +/global/cscratch1/sd/jialin/h5boss/5334-55928.hdf5 +/global/cscratch1/sd/jialin/h5boss/5335-56013.hdf5 +/global/cscratch1/sd/jialin/h5boss/5336-55957.hdf5 +/global/cscratch1/sd/jialin/h5boss/5337-55987.hdf5 +/global/cscratch1/sd/jialin/h5boss/5338-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5339-55922.hdf5 +/global/cscratch1/sd/jialin/h5boss/5340-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5341-55931.hdf5 +/global/cscratch1/sd/jialin/h5boss/5342-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5343-55999.hdf5 +/global/cscratch1/sd/jialin/h5boss/5344-55924.hdf5 +/global/cscratch1/sd/jialin/h5boss/5345-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5346-55955.hdf5 +/global/cscratch1/sd/jialin/h5boss/5347-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/5348-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5349-55929.hdf5 +/global/cscratch1/sd/jialin/h5boss/5350-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5351-55952.hdf5 +/global/cscratch1/sd/jialin/h5boss/5352-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/5353-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5354-55927.hdf5 +/global/cscratch1/sd/jialin/h5boss/5355-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5356-55979.hdf5 +/global/cscratch1/sd/jialin/h5boss/5357-55956.hdf5 +/global/cscratch1/sd/jialin/h5boss/5358-56016.hdf5 +/global/cscratch1/sd/jialin/h5boss/5359-55953.hdf5 +/global/cscratch1/sd/jialin/h5boss/5360-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/5361-55973.hdf5 +/global/cscratch1/sd/jialin/h5boss/5362-56017.hdf5 +/global/cscratch1/sd/jialin/h5boss/5363-55956.hdf5 +/global/cscratch1/sd/jialin/h5boss/5364-56000.hdf5 +/global/cscratch1/sd/jialin/h5boss/5365-55945.hdf5 +/global/cscratch1/sd/jialin/h5boss/5366-55958.hdf5 +/global/cscratch1/sd/jialin/h5boss/5367-55986.hdf5 +/global/cscratch1/sd/jialin/h5boss/5368-56001.hdf5 +/global/cscratch1/sd/jialin/h5boss/5369-56272.hdf5 +/global/cscratch1/sd/jialin/h5boss/5370-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5371-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5372-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5373-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5374-55947.hdf5 +/global/cscratch1/sd/jialin/h5boss/5375-55973.hdf5 +/global/cscratch1/sd/jialin/h5boss/5376-55987.hdf5 +/global/cscratch1/sd/jialin/h5boss/5377-55957.hdf5 +/global/cscratch1/sd/jialin/h5boss/5378-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5379-55986.hdf5 +/global/cscratch1/sd/jialin/h5boss/5380-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5381-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5382-55982.hdf5 +/global/cscratch1/sd/jialin/h5boss/5383-56013.hdf5 +/global/cscratch1/sd/jialin/h5boss/5384-55984.hdf5 +/global/cscratch1/sd/jialin/h5boss/5385-55960.hdf5 +/global/cscratch1/sd/jialin/h5boss/5386-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/5387-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5388-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/5389-55953.hdf5 +/global/cscratch1/sd/jialin/h5boss/5390-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5391-56000.hdf5 +/global/cscratch1/sd/jialin/h5boss/5392-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5393-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/5394-56001.hdf5 +/global/cscratch1/sd/jialin/h5boss/5395-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5396-55947.hdf5 +/global/cscratch1/sd/jialin/h5boss/5397-55944.hdf5 +/global/cscratch1/sd/jialin/h5boss/5398-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5399-55956.hdf5 +/global/cscratch1/sd/jialin/h5boss/5400-56308.hdf5 +/global/cscratch1/sd/jialin/h5boss/5401-55949.hdf5 +/global/cscratch1/sd/jialin/h5boss/5402-55927.hdf5 +/global/cscratch1/sd/jialin/h5boss/5403-55979.hdf5 +/global/cscratch1/sd/jialin/h5boss/5404-56013.hdf5 +/global/cscratch1/sd/jialin/h5boss/5405-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5406-55955.hdf5 +/global/cscratch1/sd/jialin/h5boss/5407-55926.hdf5 +/global/cscratch1/sd/jialin/h5boss/5408-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/5409-55957.hdf5 +/global/cscratch1/sd/jialin/h5boss/5410-56016.hdf5 +/global/cscratch1/sd/jialin/h5boss/5411-55953.hdf5 +/global/cscratch1/sd/jialin/h5boss/5412-55981.hdf5 +/global/cscratch1/sd/jialin/h5boss/5413-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5414-56014.hdf5 +/global/cscratch1/sd/jialin/h5boss/5415-55982.hdf5 +/global/cscratch1/sd/jialin/h5boss/5416-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5417-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5418-56037.hdf5 +/global/cscratch1/sd/jialin/h5boss/5419-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/5420-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5421-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5422-55986.hdf5 +/global/cscratch1/sd/jialin/h5boss/5423-55958.hdf5 +/global/cscratch1/sd/jialin/h5boss/5424-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5425-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5426-55987.hdf5 +/global/cscratch1/sd/jialin/h5boss/5427-56001.hdf5 +/global/cscratch1/sd/jialin/h5boss/5428-56029.hdf5 +/global/cscratch1/sd/jialin/h5boss/5429-55979.hdf5 +/global/cscratch1/sd/jialin/h5boss/5430-56000.hdf5 +/global/cscratch1/sd/jialin/h5boss/5431-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5432-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/5433-55981.hdf5 +/global/cscratch1/sd/jialin/h5boss/5434-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5435-55982.hdf5 +/global/cscratch1/sd/jialin/h5boss/5436-56015.hdf5 +/global/cscratch1/sd/jialin/h5boss/5437-55973.hdf5 +/global/cscratch1/sd/jialin/h5boss/5438-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5439-56045.hdf5 +/global/cscratch1/sd/jialin/h5boss/5440-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/5441-56017.hdf5 +/global/cscratch1/sd/jialin/h5boss/5442-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5443-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5444-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/5445-55987.hdf5 +/global/cscratch1/sd/jialin/h5boss/5446-56014.hdf5 +/global/cscratch1/sd/jialin/h5boss/5447-55958.hdf5 +/global/cscratch1/sd/jialin/h5boss/5448-56013.hdf5 +/global/cscratch1/sd/jialin/h5boss/5449-56030.hdf5 +/global/cscratch1/sd/jialin/h5boss/5450-55986.hdf5 +/global/cscratch1/sd/jialin/h5boss/5451-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5452-55957.hdf5 +/global/cscratch1/sd/jialin/h5boss/5453-56001.hdf5 +/global/cscratch1/sd/jialin/h5boss/5454-56015.hdf5 +/global/cscratch1/sd/jialin/h5boss/5455-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5456-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5457-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5458-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5459-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5460-56000.hdf5 +/global/cscratch1/sd/jialin/h5boss/5461-56018.hdf5 +/global/cscratch1/sd/jialin/h5boss/5462-55978.hdf5 +/global/cscratch1/sd/jialin/h5boss/5463-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5464-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5465-55988.hdf5 +/global/cscratch1/sd/jialin/h5boss/5466-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5467-55973.hdf5 +/global/cscratch1/sd/jialin/h5boss/5468-56014.hdf5 +/global/cscratch1/sd/jialin/h5boss/5469-56037.hdf5 +/global/cscratch1/sd/jialin/h5boss/5470-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/5471-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5472-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5473-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5474-56036.hdf5 +/global/cscratch1/sd/jialin/h5boss/5475-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5476-55986.hdf5 +/global/cscratch1/sd/jialin/h5boss/5477-56030.hdf5 +/global/cscratch1/sd/jialin/h5boss/5478-56014.hdf5 +/global/cscratch1/sd/jialin/h5boss/5479-56016.hdf5 +/global/cscratch1/sd/jialin/h5boss/5480-56001.hdf5 +/global/cscratch1/sd/jialin/h5boss/5481-55983.hdf5 +/global/cscratch1/sd/jialin/h5boss/5482-56017.hdf5 +/global/cscratch1/sd/jialin/h5boss/5483-56017.hdf5 +/global/cscratch1/sd/jialin/h5boss/5484-56039.hdf5 +/global/cscratch1/sd/jialin/h5boss/5485-56063.hdf5 +/global/cscratch1/sd/jialin/h5boss/5486-56030.hdf5 +/global/cscratch1/sd/jialin/h5boss/5487-55982.hdf5 +/global/cscratch1/sd/jialin/h5boss/5488-56013.hdf5 +/global/cscratch1/sd/jialin/h5boss/5489-55990.hdf5 +/global/cscratch1/sd/jialin/h5boss/5490-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5491-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5492-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5493-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5494-55857.hdf5 +/global/cscratch1/sd/jialin/h5boss/5495-55896.hdf5 +/global/cscratch1/sd/jialin/h5boss/5496-55888.hdf5 +/global/cscratch1/sd/jialin/h5boss/5648-55923.hdf5 +/global/cscratch1/sd/jialin/h5boss/5649-55912.hdf5 +/global/cscratch1/sd/jialin/h5boss/5656-55940.hdf5 +/global/cscratch1/sd/jialin/h5boss/5693-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/5694-56213.hdf5 +/global/cscratch1/sd/jialin/h5boss/5695-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/5696-55946.hdf5 +/global/cscratch1/sd/jialin/h5boss/5697-55944.hdf5 +/global/cscratch1/sd/jialin/h5boss/5698-55947.hdf5 +/global/cscratch1/sd/jialin/h5boss/5699-55953.hdf5 +/global/cscratch1/sd/jialin/h5boss/5700-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/5701-55949.hdf5 +/global/cscratch1/sd/jialin/h5boss/5702-56221.hdf5 +/global/cscratch1/sd/jialin/h5boss/5703-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/5704-56187.hdf5 +/global/cscratch1/sd/jialin/h5boss/5705-55973.hdf5 +/global/cscratch1/sd/jialin/h5boss/5706-56165.hdf5 +/global/cscratch1/sd/jialin/h5boss/5707-56217.hdf5 +/global/cscratch1/sd/jialin/h5boss/5708-56605.hdf5 +/global/cscratch1/sd/jialin/h5boss/5709-56571.hdf5 +/global/cscratch1/sd/jialin/h5boss/5710-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/5712-56602.hdf5 +/global/cscratch1/sd/jialin/h5boss/5713-56686.hdf5 +/global/cscratch1/sd/jialin/h5boss/5714-56660.hdf5 +/global/cscratch1/sd/jialin/h5boss/5715-56657.hdf5 +/global/cscratch1/sd/jialin/h5boss/5716-56662.hdf5 +/global/cscratch1/sd/jialin/h5boss/5718-56693.hdf5 +/global/cscratch1/sd/jialin/h5boss/5719-56014.hdf5 +/global/cscratch1/sd/jialin/h5boss/5720-56602.hdf5 +/global/cscratch1/sd/jialin/h5boss/5722-56008.hdf5 +/global/cscratch1/sd/jialin/h5boss/5723-56597.hdf5 +/global/cscratch1/sd/jialin/h5boss/5724-56364.hdf5 +/global/cscratch1/sd/jialin/h5boss/5725-56625.hdf5 +/global/cscratch1/sd/jialin/h5boss/5726-56626.hdf5 +/global/cscratch1/sd/jialin/h5boss/5728-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/5729-56598.hdf5 +/global/cscratch1/sd/jialin/h5boss/5730-56607.hdf5 +/global/cscratch1/sd/jialin/h5boss/5731-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/5732-56326.hdf5 +/global/cscratch1/sd/jialin/h5boss/5733-56575.hdf5 +/global/cscratch1/sd/jialin/h5boss/5735-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5736-55984.hdf5 +/global/cscratch1/sd/jialin/h5boss/5737-56381.hdf5 +/global/cscratch1/sd/jialin/h5boss/5738-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5739-55976.hdf5 +/global/cscratch1/sd/jialin/h5boss/5740-55998.hdf5 +/global/cscratch1/sd/jialin/h5boss/5741-55980.hdf5 +/global/cscratch1/sd/jialin/h5boss/5742-56636.hdf5 +/global/cscratch1/sd/jialin/h5boss/5743-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5745-55998.hdf5 +/global/cscratch1/sd/jialin/h5boss/5766-56248.hdf5 +/global/cscratch1/sd/jialin/h5boss/5767-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/5768-56017.hdf5 +/global/cscratch1/sd/jialin/h5boss/5769-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/5770-56014.hdf5 +/global/cscratch1/sd/jialin/h5boss/5771-56011.hdf5 +/global/cscratch1/sd/jialin/h5boss/5772-56003.hdf5 +/global/cscratch1/sd/jialin/h5boss/5773-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5774-56002.hdf5 +/global/cscratch1/sd/jialin/h5boss/5775-56009.hdf5 +/global/cscratch1/sd/jialin/h5boss/5776-56010.hdf5 +/global/cscratch1/sd/jialin/h5boss/5777-56280.hdf5 +/global/cscratch1/sd/jialin/h5boss/5778-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/5779-56338.hdf5 +/global/cscratch1/sd/jialin/h5boss/5780-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/5781-56337.hdf5 +/global/cscratch1/sd/jialin/h5boss/5782-56272.hdf5 +/global/cscratch1/sd/jialin/h5boss/5783-56017.hdf5 +/global/cscratch1/sd/jialin/h5boss/5784-56029.hdf5 +/global/cscratch1/sd/jialin/h5boss/5785-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/5786-56251.hdf5 +/global/cscratch1/sd/jialin/h5boss/5787-56254.hdf5 +/global/cscratch1/sd/jialin/h5boss/5788-56255.hdf5 +/global/cscratch1/sd/jialin/h5boss/5789-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/5790-56271.hdf5 +/global/cscratch1/sd/jialin/h5boss/5791-56255.hdf5 +/global/cscratch1/sd/jialin/h5boss/5792-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/5793-56337.hdf5 +/global/cscratch1/sd/jialin/h5boss/5794-56282.hdf5 +/global/cscratch1/sd/jialin/h5boss/5795-56298.hdf5 +/global/cscratch1/sd/jialin/h5boss/5796-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/5797-56273.hdf5 +/global/cscratch1/sd/jialin/h5boss/5798-56326.hdf5 +/global/cscratch1/sd/jialin/h5boss/5799-56325.hdf5 +/global/cscratch1/sd/jialin/h5boss/5800-56279.hdf5 +/global/cscratch1/sd/jialin/h5boss/5801-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/5802-56309.hdf5 +/global/cscratch1/sd/jialin/h5boss/5803-56272.hdf5 +/global/cscratch1/sd/jialin/h5boss/5804-56304.hdf5 +/global/cscratch1/sd/jialin/h5boss/5805-56336.hdf5 +/global/cscratch1/sd/jialin/h5boss/5806-56310.hdf5 +/global/cscratch1/sd/jialin/h5boss/5807-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/5808-56325.hdf5 +/global/cscratch1/sd/jialin/h5boss/5809-56353.hdf5 +/global/cscratch1/sd/jialin/h5boss/5810-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/5811-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/5812-56354.hdf5 +/global/cscratch1/sd/jialin/h5boss/5813-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/5846-56017.hdf5 +/global/cscratch1/sd/jialin/h5boss/5847-56037.hdf5 +/global/cscratch1/sd/jialin/h5boss/5848-56029.hdf5 +/global/cscratch1/sd/jialin/h5boss/5849-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5850-56042.hdf5 +/global/cscratch1/sd/jialin/h5boss/5851-56075.hdf5 +/global/cscratch1/sd/jialin/h5boss/5852-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5853-56045.hdf5 +/global/cscratch1/sd/jialin/h5boss/5854-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5855-56088.hdf5 +/global/cscratch1/sd/jialin/h5boss/5856-56090.hdf5 +/global/cscratch1/sd/jialin/h5boss/5857-56092.hdf5 +/global/cscratch1/sd/jialin/h5boss/5858-56063.hdf5 +/global/cscratch1/sd/jialin/h5boss/5859-56065.hdf5 +/global/cscratch1/sd/jialin/h5boss/5860-56046.hdf5 +/global/cscratch1/sd/jialin/h5boss/5861-56069.hdf5 +/global/cscratch1/sd/jialin/h5boss/5862-56045.hdf5 +/global/cscratch1/sd/jialin/h5boss/5863-56042.hdf5 +/global/cscratch1/sd/jialin/h5boss/5864-56047.hdf5 +/global/cscratch1/sd/jialin/h5boss/5865-56067.hdf5 +/global/cscratch1/sd/jialin/h5boss/5866-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5867-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5868-56045.hdf5 +/global/cscratch1/sd/jialin/h5boss/5869-56064.hdf5 +/global/cscratch1/sd/jialin/h5boss/5870-56065.hdf5 +/global/cscratch1/sd/jialin/h5boss/5871-56041.hdf5 +/global/cscratch1/sd/jialin/h5boss/5872-56027.hdf5 +/global/cscratch1/sd/jialin/h5boss/5873-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5874-56039.hdf5 +/global/cscratch1/sd/jialin/h5boss/5875-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/5876-56042.hdf5 +/global/cscratch1/sd/jialin/h5boss/5877-56041.hdf5 +/global/cscratch1/sd/jialin/h5boss/5878-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5879-56047.hdf5 +/global/cscratch1/sd/jialin/h5boss/5880-56042.hdf5 +/global/cscratch1/sd/jialin/h5boss/5881-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/5882-56029.hdf5 +/global/cscratch1/sd/jialin/h5boss/5883-56033.hdf5 +/global/cscratch1/sd/jialin/h5boss/5884-56046.hdf5 +/global/cscratch1/sd/jialin/h5boss/5885-56036.hdf5 +/global/cscratch1/sd/jialin/h5boss/5886-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5887-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5888-56041.hdf5 +/global/cscratch1/sd/jialin/h5boss/5889-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/5890-56037.hdf5 +/global/cscratch1/sd/jialin/h5boss/5891-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5892-56035.hdf5 +/global/cscratch1/sd/jialin/h5boss/5893-56034.hdf5 +/global/cscratch1/sd/jialin/h5boss/5894-56039.hdf5 +/global/cscratch1/sd/jialin/h5boss/5895-56046.hdf5 +/global/cscratch1/sd/jialin/h5boss/5896-56047.hdf5 +/global/cscratch1/sd/jialin/h5boss/5897-56042.hdf5 +/global/cscratch1/sd/jialin/h5boss/5898-56045.hdf5 +/global/cscratch1/sd/jialin/h5boss/5899-56038.hdf5 +/global/cscratch1/sd/jialin/h5boss/5900-56043.hdf5 +/global/cscratch1/sd/jialin/h5boss/5901-56039.hdf5 +/global/cscratch1/sd/jialin/h5boss/5902-56042.hdf5 +/global/cscratch1/sd/jialin/h5boss/5903-56064.hdf5 +/global/cscratch1/sd/jialin/h5boss/5904-56046.hdf5 +/global/cscratch1/sd/jialin/h5boss/5905-56065.hdf5 +/global/cscratch1/sd/jialin/h5boss/5941-56193.hdf5 +/global/cscratch1/sd/jialin/h5boss/5942-56210.hdf5 +/global/cscratch1/sd/jialin/h5boss/5943-56218.hdf5 +/global/cscratch1/sd/jialin/h5boss/5944-56220.hdf5 +/global/cscratch1/sd/jialin/h5boss/5945-56213.hdf5 +/global/cscratch1/sd/jialin/h5boss/5946-56101.hdf5 +/global/cscratch1/sd/jialin/h5boss/5947-56093.hdf5 +/global/cscratch1/sd/jialin/h5boss/5948-56107.hdf5 +/global/cscratch1/sd/jialin/h5boss/5949-56096.hdf5 +/global/cscratch1/sd/jialin/h5boss/5950-56211.hdf5 +/global/cscratch1/sd/jialin/h5boss/5951-56102.hdf5 +/global/cscratch1/sd/jialin/h5boss/5952-56453.hdf5 +/global/cscratch1/sd/jialin/h5boss/5953-56092.hdf5 +/global/cscratch1/sd/jialin/h5boss/5954-56462.hdf5 +/global/cscratch1/sd/jialin/h5boss/5955-56456.hdf5 +/global/cscratch1/sd/jialin/h5boss/5956-56104.hdf5 +/global/cscratch1/sd/jialin/h5boss/5957-56210.hdf5 +/global/cscratch1/sd/jialin/h5boss/5958-56213.hdf5 +/global/cscratch1/sd/jialin/h5boss/5959-56214.hdf5 +/global/cscratch1/sd/jialin/h5boss/5960-56097.hdf5 +/global/cscratch1/sd/jialin/h5boss/5961-56460.hdf5 +/global/cscratch1/sd/jialin/h5boss/5962-56265.hdf5 +/global/cscratch1/sd/jialin/h5boss/5963-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/5964-56098.hdf5 +/global/cscratch1/sd/jialin/h5boss/5965-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/5966-56101.hdf5 +/global/cscratch1/sd/jialin/h5boss/5967-56063.hdf5 +/global/cscratch1/sd/jialin/h5boss/5968-56356.hdf5 +/global/cscratch1/sd/jialin/h5boss/5969-56069.hdf5 +/global/cscratch1/sd/jialin/h5boss/5970-56046.hdf5 +/global/cscratch1/sd/jialin/h5boss/5971-56064.hdf5 +/global/cscratch1/sd/jialin/h5boss/5972-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/5973-56067.hdf5 +/global/cscratch1/sd/jialin/h5boss/5974-56314.hdf5 +/global/cscratch1/sd/jialin/h5boss/5975-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/5976-56356.hdf5 +/global/cscratch1/sd/jialin/h5boss/5977-56098.hdf5 +/global/cscratch1/sd/jialin/h5boss/5978-56073.hdf5 +/global/cscratch1/sd/jialin/h5boss/5979-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/5980-56338.hdf5 +/global/cscratch1/sd/jialin/h5boss/5981-56340.hdf5 +/global/cscratch1/sd/jialin/h5boss/5982-56074.hdf5 +/global/cscratch1/sd/jialin/h5boss/5983-56310.hdf5 +/global/cscratch1/sd/jialin/h5boss/5984-56337.hdf5 +/global/cscratch1/sd/jialin/h5boss/5985-56089.hdf5 +/global/cscratch1/sd/jialin/h5boss/5986-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/5987-56339.hdf5 +/global/cscratch1/sd/jialin/h5boss/5988-56072.hdf5 +/global/cscratch1/sd/jialin/h5boss/5989-56312.hdf5 +/global/cscratch1/sd/jialin/h5boss/5990-56066.hdf5 +/global/cscratch1/sd/jialin/h5boss/5991-56076.hdf5 +/global/cscratch1/sd/jialin/h5boss/5992-56066.hdf5 +/global/cscratch1/sd/jialin/h5boss/5993-56070.hdf5 +/global/cscratch1/sd/jialin/h5boss/5994-56101.hdf5 +/global/cscratch1/sd/jialin/h5boss/5995-56093.hdf5 +/global/cscratch1/sd/jialin/h5boss/5996-56096.hdf5 +/global/cscratch1/sd/jialin/h5boss/5997-56309.hdf5 +/global/cscratch1/sd/jialin/h5boss/5998-56087.hdf5 +/global/cscratch1/sd/jialin/h5boss/5999-56089.hdf5 +/global/cscratch1/sd/jialin/h5boss/6000-56102.hdf5 +/global/cscratch1/sd/jialin/h5boss/6001-56072.hdf5 +/global/cscratch1/sd/jialin/h5boss/6002-56104.hdf5 +/global/cscratch1/sd/jialin/h5boss/6003-56311.hdf5 +/global/cscratch1/sd/jialin/h5boss/6004-56066.hdf5 +/global/cscratch1/sd/jialin/h5boss/6005-56090.hdf5 +/global/cscratch1/sd/jialin/h5boss/6006-56105.hdf5 +/global/cscratch1/sd/jialin/h5boss/6007-56066.hdf5 +/global/cscratch1/sd/jialin/h5boss/6008-56093.hdf5 +/global/cscratch1/sd/jialin/h5boss/6009-56313.hdf5 +/global/cscratch1/sd/jialin/h5boss/6010-56097.hdf5 +/global/cscratch1/sd/jialin/h5boss/6011-56103.hdf5 +/global/cscratch1/sd/jialin/h5boss/6012-56101.hdf5 +/global/cscratch1/sd/jialin/h5boss/6013-56074.hdf5 +/global/cscratch1/sd/jialin/h5boss/6014-56072.hdf5 +/global/cscratch1/sd/jialin/h5boss/6015-56096.hdf5 +/global/cscratch1/sd/jialin/h5boss/6016-56073.hdf5 +/global/cscratch1/sd/jialin/h5boss/6017-56075.hdf5 +/global/cscratch1/sd/jialin/h5boss/6018-56067.hdf5 +/global/cscratch1/sd/jialin/h5boss/6019-56074.hdf5 +/global/cscratch1/sd/jialin/h5boss/6020-56087.hdf5 +/global/cscratch1/sd/jialin/h5boss/6021-56067.hdf5 +/global/cscratch1/sd/jialin/h5boss/6022-56076.hdf5 +/global/cscratch1/sd/jialin/h5boss/6023-56069.hdf5 +/global/cscratch1/sd/jialin/h5boss/6024-56088.hdf5 +/global/cscratch1/sd/jialin/h5boss/6025-56098.hdf5 +/global/cscratch1/sd/jialin/h5boss/6026-56071.hdf5 +/global/cscratch1/sd/jialin/h5boss/6027-56103.hdf5 +/global/cscratch1/sd/jialin/h5boss/6028-56102.hdf5 +/global/cscratch1/sd/jialin/h5boss/6029-56099.hdf5 +/global/cscratch1/sd/jialin/h5boss/6030-56097.hdf5 +/global/cscratch1/sd/jialin/h5boss/6031-56091.hdf5 +/global/cscratch1/sd/jialin/h5boss/6032-56067.hdf5 +/global/cscratch1/sd/jialin/h5boss/6033-56069.hdf5 +/global/cscratch1/sd/jialin/h5boss/6034-56103.hdf5 +/global/cscratch1/sd/jialin/h5boss/6035-56076.hdf5 +/global/cscratch1/sd/jialin/h5boss/6036-56093.hdf5 +/global/cscratch1/sd/jialin/h5boss/6037-56160.hdf5 +/global/cscratch1/sd/jialin/h5boss/6038-56090.hdf5 +/global/cscratch1/sd/jialin/h5boss/6039-56099.hdf5 +/global/cscratch1/sd/jialin/h5boss/6040-56101.hdf5 +/global/cscratch1/sd/jialin/h5boss/6041-56102.hdf5 +/global/cscratch1/sd/jialin/h5boss/6042-56101.hdf5 +/global/cscratch1/sd/jialin/h5boss/6043-56096.hdf5 +/global/cscratch1/sd/jialin/h5boss/6044-56090.hdf5 +/global/cscratch1/sd/jialin/h5boss/6045-56072.hdf5 +/global/cscratch1/sd/jialin/h5boss/6046-56096.hdf5 +/global/cscratch1/sd/jialin/h5boss/6047-56098.hdf5 +/global/cscratch1/sd/jialin/h5boss/6048-56072.hdf5 +/global/cscratch1/sd/jialin/h5boss/6049-56091.hdf5 +/global/cscratch1/sd/jialin/h5boss/6050-56089.hdf5 +/global/cscratch1/sd/jialin/h5boss/6051-56093.hdf5 +/global/cscratch1/sd/jialin/h5boss/6052-56092.hdf5 +/global/cscratch1/sd/jialin/h5boss/6053-56091.hdf5 +/global/cscratch1/sd/jialin/h5boss/6054-56089.hdf5 +/global/cscratch1/sd/jialin/h5boss/6055-56102.hdf5 +/global/cscratch1/sd/jialin/h5boss/6056-56092.hdf5 +/global/cscratch1/sd/jialin/h5boss/6057-56099.hdf5 +/global/cscratch1/sd/jialin/h5boss/6058-56090.hdf5 +/global/cscratch1/sd/jialin/h5boss/6059-56093.hdf5 +/global/cscratch1/sd/jialin/h5boss/6060-56074.hdf5 +/global/cscratch1/sd/jialin/h5boss/6061-56076.hdf5 +/global/cscratch1/sd/jialin/h5boss/6062-56091.hdf5 +/global/cscratch1/sd/jialin/h5boss/6063-56077.hdf5 +/global/cscratch1/sd/jialin/h5boss/6064-56097.hdf5 +/global/cscratch1/sd/jialin/h5boss/6110-56279.hdf5 +/global/cscratch1/sd/jialin/h5boss/6111-56270.hdf5 +/global/cscratch1/sd/jialin/h5boss/6112-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/6113-56219.hdf5 +/global/cscratch1/sd/jialin/h5boss/6114-56209.hdf5 +/global/cscratch1/sd/jialin/h5boss/6115-56237.hdf5 +/global/cscratch1/sd/jialin/h5boss/6116-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/6117-56272.hdf5 +/global/cscratch1/sd/jialin/h5boss/6118-56189.hdf5 +/global/cscratch1/sd/jialin/h5boss/6119-56181.hdf5 +/global/cscratch1/sd/jialin/h5boss/6120-56206.hdf5 +/global/cscratch1/sd/jialin/h5boss/6121-56187.hdf5 +/global/cscratch1/sd/jialin/h5boss/6122-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/6123-56217.hdf5 +/global/cscratch1/sd/jialin/h5boss/6124-56211.hdf5 +/global/cscratch1/sd/jialin/h5boss/6125-56273.hdf5 +/global/cscratch1/sd/jialin/h5boss/6126-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/6127-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/6128-56236.hdf5 +/global/cscratch1/sd/jialin/h5boss/6129-56301.hdf5 +/global/cscratch1/sd/jialin/h5boss/6130-56214.hdf5 +/global/cscratch1/sd/jialin/h5boss/6131-56211.hdf5 +/global/cscratch1/sd/jialin/h5boss/6132-56267.hdf5 +/global/cscratch1/sd/jialin/h5boss/6133-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/6134-56244.hdf5 +/global/cscratch1/sd/jialin/h5boss/6135-56192.hdf5 +/global/cscratch1/sd/jialin/h5boss/6136-56206.hdf5 +/global/cscratch1/sd/jialin/h5boss/6137-56270.hdf5 +/global/cscratch1/sd/jialin/h5boss/6138-56598.hdf5 +/global/cscratch1/sd/jialin/h5boss/6139-56192.hdf5 +/global/cscratch1/sd/jialin/h5boss/6140-56189.hdf5 +/global/cscratch1/sd/jialin/h5boss/6141-56210.hdf5 +/global/cscratch1/sd/jialin/h5boss/6142-56219.hdf5 +/global/cscratch1/sd/jialin/h5boss/6143-56267.hdf5 +/global/cscratch1/sd/jialin/h5boss/6144-56237.hdf5 +/global/cscratch1/sd/jialin/h5boss/6145-56266.hdf5 +/global/cscratch1/sd/jialin/h5boss/6146-56265.hdf5 +/global/cscratch1/sd/jialin/h5boss/6147-56239.hdf5 +/global/cscratch1/sd/jialin/h5boss/6148-56209.hdf5 +/global/cscratch1/sd/jialin/h5boss/6149-56194.hdf5 +/global/cscratch1/sd/jialin/h5boss/6150-56187.hdf5 +/global/cscratch1/sd/jialin/h5boss/6151-56265.hdf5 +/global/cscratch1/sd/jialin/h5boss/6152-56164.hdf5 +/global/cscratch1/sd/jialin/h5boss/6153-56164.hdf5 +/global/cscratch1/sd/jialin/h5boss/6154-56237.hdf5 +/global/cscratch1/sd/jialin/h5boss/6155-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/6156-56239.hdf5 +/global/cscratch1/sd/jialin/h5boss/6157-56238.hdf5 +/global/cscratch1/sd/jialin/h5boss/6158-56239.hdf5 +/global/cscratch1/sd/jialin/h5boss/6159-56214.hdf5 +/global/cscratch1/sd/jialin/h5boss/6160-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/6161-56238.hdf5 +/global/cscratch1/sd/jialin/h5boss/6162-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/6163-56219.hdf5 +/global/cscratch1/sd/jialin/h5boss/6164-56181.hdf5 +/global/cscratch1/sd/jialin/h5boss/6165-56208.hdf5 +/global/cscratch1/sd/jialin/h5boss/6166-56181.hdf5 +/global/cscratch1/sd/jialin/h5boss/6167-56189.hdf5 +/global/cscratch1/sd/jialin/h5boss/6168-56187.hdf5 +/global/cscratch1/sd/jialin/h5boss/6169-56193.hdf5 +/global/cscratch1/sd/jialin/h5boss/6170-56240.hdf5 +/global/cscratch1/sd/jialin/h5boss/6171-56311.hdf5 +/global/cscratch1/sd/jialin/h5boss/6172-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/6173-56238.hdf5 +/global/cscratch1/sd/jialin/h5boss/6174-56243.hdf5 +/global/cscratch1/sd/jialin/h5boss/6175-56265.hdf5 +/global/cscratch1/sd/jialin/h5boss/6176-56264.hdf5 +/global/cscratch1/sd/jialin/h5boss/6177-56268.hdf5 +/global/cscratch1/sd/jialin/h5boss/6178-56213.hdf5 +/global/cscratch1/sd/jialin/h5boss/6179-56221.hdf5 +/global/cscratch1/sd/jialin/h5boss/6180-56301.hdf5 +/global/cscratch1/sd/jialin/h5boss/6181-56239.hdf5 +/global/cscratch1/sd/jialin/h5boss/6182-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/6183-56210.hdf5 +/global/cscratch1/sd/jialin/h5boss/6184-56267.hdf5 +/global/cscratch1/sd/jialin/h5boss/6185-56217.hdf5 +/global/cscratch1/sd/jialin/h5boss/6186-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/6187-56244.hdf5 +/global/cscratch1/sd/jialin/h5boss/6188-56208.hdf5 +/global/cscratch1/sd/jialin/h5boss/6189-56237.hdf5 +/global/cscratch1/sd/jialin/h5boss/6190-56210.hdf5 +/global/cscratch1/sd/jialin/h5boss/6191-56248.hdf5 +/global/cscratch1/sd/jialin/h5boss/6192-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/6193-56237.hdf5 +/global/cscratch1/sd/jialin/h5boss/6194-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/6195-56220.hdf5 +/global/cscratch1/sd/jialin/h5boss/6196-56189.hdf5 +/global/cscratch1/sd/jialin/h5boss/6197-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/6198-56211.hdf5 +/global/cscratch1/sd/jialin/h5boss/6199-56220.hdf5 +/global/cscratch1/sd/jialin/h5boss/6200-56217.hdf5 +/global/cscratch1/sd/jialin/h5boss/6201-56186.hdf5 +/global/cscratch1/sd/jialin/h5boss/6202-56266.hdf5 +/global/cscratch1/sd/jialin/h5boss/6203-56266.hdf5 +/global/cscratch1/sd/jialin/h5boss/6204-56220.hdf5 +/global/cscratch1/sd/jialin/h5boss/6205-56187.hdf5 +/global/cscratch1/sd/jialin/h5boss/6206-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/6207-56239.hdf5 +/global/cscratch1/sd/jialin/h5boss/6252-56248.hdf5 +/global/cscratch1/sd/jialin/h5boss/6253-56574.hdf5 +/global/cscratch1/sd/jialin/h5boss/6254-56268.hdf5 +/global/cscratch1/sd/jialin/h5boss/6255-56240.hdf5 +/global/cscratch1/sd/jialin/h5boss/6256-56323.hdf5 +/global/cscratch1/sd/jialin/h5boss/6257-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/6258-56238.hdf5 +/global/cscratch1/sd/jialin/h5boss/6259-56565.hdf5 +/global/cscratch1/sd/jialin/h5boss/6260-56568.hdf5 +/global/cscratch1/sd/jialin/h5boss/6261-56219.hdf5 +/global/cscratch1/sd/jialin/h5boss/6262-56267.hdf5 +/global/cscratch1/sd/jialin/h5boss/6263-56279.hdf5 +/global/cscratch1/sd/jialin/h5boss/6264-56223.hdf5 +/global/cscratch1/sd/jialin/h5boss/6265-56248.hdf5 +/global/cscratch1/sd/jialin/h5boss/6266-56223.hdf5 +/global/cscratch1/sd/jialin/h5boss/6267-56279.hdf5 +/global/cscratch1/sd/jialin/h5boss/6268-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/6269-56246.hdf5 +/global/cscratch1/sd/jialin/h5boss/6270-56273.hdf5 +/global/cscratch1/sd/jialin/h5boss/6271-56304.hdf5 +/global/cscratch1/sd/jialin/h5boss/6272-56325.hdf5 +/global/cscratch1/sd/jialin/h5boss/6273-56243.hdf5 +/global/cscratch1/sd/jialin/h5boss/6274-56550.hdf5 +/global/cscratch1/sd/jialin/h5boss/6275-56245.hdf5 +/global/cscratch1/sd/jialin/h5boss/6276-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/6277-56273.hdf5 +/global/cscratch1/sd/jialin/h5boss/6278-56266.hdf5 +/global/cscratch1/sd/jialin/h5boss/6279-56243.hdf5 +/global/cscratch1/sd/jialin/h5boss/6280-56570.hdf5 +/global/cscratch1/sd/jialin/h5boss/6281-56295.hdf5 +/global/cscratch1/sd/jialin/h5boss/6282-56572.hdf5 +/global/cscratch1/sd/jialin/h5boss/6283-56239.hdf5 +/global/cscratch1/sd/jialin/h5boss/6284-56310.hdf5 +/global/cscratch1/sd/jialin/h5boss/6285-56248.hdf5 +/global/cscratch1/sd/jialin/h5boss/6286-56301.hdf5 +/global/cscratch1/sd/jialin/h5boss/6287-56221.hdf5 +/global/cscratch1/sd/jialin/h5boss/6288-56192.hdf5 +/global/cscratch1/sd/jialin/h5boss/6289-56559.hdf5 +/global/cscratch1/sd/jialin/h5boss/6290-56238.hdf5 +/global/cscratch1/sd/jialin/h5boss/6291-56209.hdf5 +/global/cscratch1/sd/jialin/h5boss/6292-56566.hdf5 +/global/cscratch1/sd/jialin/h5boss/6293-56561.hdf5 +/global/cscratch1/sd/jialin/h5boss/6294-56482.hdf5 +/global/cscratch1/sd/jialin/h5boss/6295-56536.hdf5 +/global/cscratch1/sd/jialin/h5boss/6296-56219.hdf5 +/global/cscratch1/sd/jialin/h5boss/6297-56218.hdf5 +/global/cscratch1/sd/jialin/h5boss/6298-56208.hdf5 +/global/cscratch1/sd/jialin/h5boss/6299-56478.hdf5 +/global/cscratch1/sd/jialin/h5boss/6300-56449.hdf5 +/global/cscratch1/sd/jialin/h5boss/6301-56543.hdf5 +/global/cscratch1/sd/jialin/h5boss/6302-56565.hdf5 +/global/cscratch1/sd/jialin/h5boss/6303-56539.hdf5 +/global/cscratch1/sd/jialin/h5boss/6304-56570.hdf5 +/global/cscratch1/sd/jialin/h5boss/6305-56563.hdf5 +/global/cscratch1/sd/jialin/h5boss/6306-56487.hdf5 +/global/cscratch1/sd/jialin/h5boss/6307-56205.hdf5 +/global/cscratch1/sd/jialin/h5boss/6308-56215.hdf5 +/global/cscratch1/sd/jialin/h5boss/6309-56485.hdf5 +/global/cscratch1/sd/jialin/h5boss/6310-56455.hdf5 +/global/cscratch1/sd/jialin/h5boss/6311-56447.hdf5 +/global/cscratch1/sd/jialin/h5boss/6312-56487.hdf5 +/global/cscratch1/sd/jialin/h5boss/6313-56460.hdf5 +/global/cscratch1/sd/jialin/h5boss/6314-56191.hdf5 +/global/cscratch1/sd/jialin/h5boss/6315-56181.hdf5 +/global/cscratch1/sd/jialin/h5boss/6316-56483.hdf5 +/global/cscratch1/sd/jialin/h5boss/6317-56448.hdf5 +/global/cscratch1/sd/jialin/h5boss/6318-56186.hdf5 +/global/cscratch1/sd/jialin/h5boss/6319-56452.hdf5 +/global/cscratch1/sd/jialin/h5boss/6320-56453.hdf5 +/global/cscratch1/sd/jialin/h5boss/6321-56449.hdf5 +/global/cscratch1/sd/jialin/h5boss/6322-56190.hdf5 +/global/cscratch1/sd/jialin/h5boss/6323-56189.hdf5 +/global/cscratch1/sd/jialin/h5boss/6369-56217.hdf5 +/global/cscratch1/sd/jialin/h5boss/6370-56238.hdf5 +/global/cscratch1/sd/jialin/h5boss/6372-56341.hdf5 +/global/cscratch1/sd/jialin/h5boss/6373-56298.hdf5 +/global/cscratch1/sd/jialin/h5boss/6374-56326.hdf5 +/global/cscratch1/sd/jialin/h5boss/6375-56323.hdf5 +/global/cscratch1/sd/jialin/h5boss/6376-56269.hdf5 +/global/cscratch1/sd/jialin/h5boss/6377-56243.hdf5 +/global/cscratch1/sd/jialin/h5boss/6378-56330.hdf5 +/global/cscratch1/sd/jialin/h5boss/6379-56340.hdf5 +/global/cscratch1/sd/jialin/h5boss/6380-56340.hdf5 +/global/cscratch1/sd/jialin/h5boss/6381-56353.hdf5 +/global/cscratch1/sd/jialin/h5boss/6382-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6383-56362.hdf5 +/global/cscratch1/sd/jialin/h5boss/6384-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/6385-56356.hdf5 +/global/cscratch1/sd/jialin/h5boss/6386-56325.hdf5 +/global/cscratch1/sd/jialin/h5boss/6387-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/6388-56309.hdf5 +/global/cscratch1/sd/jialin/h5boss/6389-56270.hdf5 +/global/cscratch1/sd/jialin/h5boss/6390-56244.hdf5 +/global/cscratch1/sd/jialin/h5boss/6391-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/6392-56331.hdf5 +/global/cscratch1/sd/jialin/h5boss/6393-56341.hdf5 +/global/cscratch1/sd/jialin/h5boss/6394-56330.hdf5 +/global/cscratch1/sd/jialin/h5boss/6395-56354.hdf5 +/global/cscratch1/sd/jialin/h5boss/6396-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6397-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6398-56356.hdf5 +/global/cscratch1/sd/jialin/h5boss/6402-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/6403-56357.hdf5 +/global/cscratch1/sd/jialin/h5boss/6404-56330.hdf5 +/global/cscratch1/sd/jialin/h5boss/6405-56342.hdf5 +/global/cscratch1/sd/jialin/h5boss/6406-56330.hdf5 +/global/cscratch1/sd/jialin/h5boss/6407-56311.hdf5 +/global/cscratch1/sd/jialin/h5boss/6408-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/6409-56306.hdf5 +/global/cscratch1/sd/jialin/h5boss/6410-56336.hdf5 +/global/cscratch1/sd/jialin/h5boss/6411-56331.hdf5 +/global/cscratch1/sd/jialin/h5boss/6412-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6413-56336.hdf5 +/global/cscratch1/sd/jialin/h5boss/6414-56306.hdf5 +/global/cscratch1/sd/jialin/h5boss/6415-56310.hdf5 +/global/cscratch1/sd/jialin/h5boss/6416-56312.hdf5 +/global/cscratch1/sd/jialin/h5boss/6417-56308.hdf5 +/global/cscratch1/sd/jialin/h5boss/6418-56354.hdf5 +/global/cscratch1/sd/jialin/h5boss/6419-56331.hdf5 +/global/cscratch1/sd/jialin/h5boss/6420-56304.hdf5 +/global/cscratch1/sd/jialin/h5boss/6421-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/6422-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/6423-56313.hdf5 +/global/cscratch1/sd/jialin/h5boss/6424-56272.hdf5 +/global/cscratch1/sd/jialin/h5boss/6425-56298.hdf5 +/global/cscratch1/sd/jialin/h5boss/6426-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/6427-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/6428-56279.hdf5 +/global/cscratch1/sd/jialin/h5boss/6429-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/6430-56299.hdf5 +/global/cscratch1/sd/jialin/h5boss/6431-56311.hdf5 +/global/cscratch1/sd/jialin/h5boss/6432-56309.hdf5 +/global/cscratch1/sd/jialin/h5boss/6433-56339.hdf5 +/global/cscratch1/sd/jialin/h5boss/6434-56362.hdf5 +/global/cscratch1/sd/jialin/h5boss/6435-56341.hdf5 +/global/cscratch1/sd/jialin/h5boss/6436-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/6437-56340.hdf5 +/global/cscratch1/sd/jialin/h5boss/6438-56367.hdf5 +/global/cscratch1/sd/jialin/h5boss/6439-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6440-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6441-56364.hdf5 +/global/cscratch1/sd/jialin/h5boss/6442-56369.hdf5 +/global/cscratch1/sd/jialin/h5boss/6443-56367.hdf5 +/global/cscratch1/sd/jialin/h5boss/6444-56339.hdf5 +/global/cscratch1/sd/jialin/h5boss/6445-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6446-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/6447-56362.hdf5 +/global/cscratch1/sd/jialin/h5boss/6448-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6449-56356.hdf5 +/global/cscratch1/sd/jialin/h5boss/6450-56331.hdf5 +/global/cscratch1/sd/jialin/h5boss/6451-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6452-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6453-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6454-56336.hdf5 +/global/cscratch1/sd/jialin/h5boss/6455-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6456-56339.hdf5 +/global/cscratch1/sd/jialin/h5boss/6457-56330.hdf5 +/global/cscratch1/sd/jialin/h5boss/6458-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/6459-56273.hdf5 +/global/cscratch1/sd/jialin/h5boss/6460-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/6461-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/6462-56326.hdf5 +/global/cscratch1/sd/jialin/h5boss/6463-56340.hdf5 +/global/cscratch1/sd/jialin/h5boss/6464-56309.hdf5 +/global/cscratch1/sd/jialin/h5boss/6465-56279.hdf5 +/global/cscratch1/sd/jialin/h5boss/6466-56310.hdf5 +/global/cscratch1/sd/jialin/h5boss/6467-56270.hdf5 +/global/cscratch1/sd/jialin/h5boss/6468-56311.hdf5 +/global/cscratch1/sd/jialin/h5boss/6469-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/6470-56334.hdf5 +/global/cscratch1/sd/jialin/h5boss/6471-56309.hdf5 +/global/cscratch1/sd/jialin/h5boss/6472-56362.hdf5 +/global/cscratch1/sd/jialin/h5boss/6473-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/6474-56362.hdf5 +/global/cscratch1/sd/jialin/h5boss/6475-56337.hdf5 +/global/cscratch1/sd/jialin/h5boss/6476-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6477-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6478-56341.hdf5 +/global/cscratch1/sd/jialin/h5boss/6479-56364.hdf5 +/global/cscratch1/sd/jialin/h5boss/6480-56340.hdf5 +/global/cscratch1/sd/jialin/h5boss/6481-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/6482-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6483-56341.hdf5 +/global/cscratch1/sd/jialin/h5boss/6484-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6485-56342.hdf5 +/global/cscratch1/sd/jialin/h5boss/6486-56328.hdf5 +/global/cscratch1/sd/jialin/h5boss/6487-56362.hdf5 +/global/cscratch1/sd/jialin/h5boss/6488-56364.hdf5 +/global/cscratch1/sd/jialin/h5boss/6489-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/6490-56330.hdf5 +/global/cscratch1/sd/jialin/h5boss/6491-56337.hdf5 +/global/cscratch1/sd/jialin/h5boss/6492-56362.hdf5 +/global/cscratch1/sd/jialin/h5boss/6493-56338.hdf5 +/global/cscratch1/sd/jialin/h5boss/6494-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/6495-56339.hdf5 +/global/cscratch1/sd/jialin/h5boss/6496-56363.hdf5 +/global/cscratch1/sd/jialin/h5boss/6497-56329.hdf5 +/global/cscratch1/sd/jialin/h5boss/6498-56565.hdf5 +/global/cscratch1/sd/jialin/h5boss/6499-56571.hdf5 +/global/cscratch1/sd/jialin/h5boss/6500-56541.hdf5 +/global/cscratch1/sd/jialin/h5boss/6501-56563.hdf5 +/global/cscratch1/sd/jialin/h5boss/6502-56544.hdf5 +/global/cscratch1/sd/jialin/h5boss/6503-56564.hdf5 +/global/cscratch1/sd/jialin/h5boss/6504-56540.hdf5 +/global/cscratch1/sd/jialin/h5boss/6505-56560.hdf5 +/global/cscratch1/sd/jialin/h5boss/6506-56564.hdf5 +/global/cscratch1/sd/jialin/h5boss/6507-56478.hdf5 +/global/cscratch1/sd/jialin/h5boss/6508-56535.hdf5 +/global/cscratch1/sd/jialin/h5boss/6509-56486.hdf5 +/global/cscratch1/sd/jialin/h5boss/6510-56486.hdf5 +/global/cscratch1/sd/jialin/h5boss/6511-56540.hdf5 +/global/cscratch1/sd/jialin/h5boss/6512-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/6513-56543.hdf5 +/global/cscratch1/sd/jialin/h5boss/6514-56487.hdf5 +/global/cscratch1/sd/jialin/h5boss/6515-56536.hdf5 +/global/cscratch1/sd/jialin/h5boss/6516-56571.hdf5 +/global/cscratch1/sd/jialin/h5boss/6517-56563.hdf5 +/global/cscratch1/sd/jialin/h5boss/6518-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/6519-56566.hdf5 +/global/cscratch1/sd/jialin/h5boss/6520-56541.hdf5 +/global/cscratch1/sd/jialin/h5boss/6521-56537.hdf5 +/global/cscratch1/sd/jialin/h5boss/6526-56543.hdf5 +/global/cscratch1/sd/jialin/h5boss/6529-56535.hdf5 +/global/cscratch1/sd/jialin/h5boss/6530-56537.hdf5 +/global/cscratch1/sd/jialin/h5boss/6581-56540.hdf5 +/global/cscratch1/sd/jialin/h5boss/6582-56274.hdf5 +/global/cscratch1/sd/jialin/h5boss/6583-56566.hdf5 +/global/cscratch1/sd/jialin/h5boss/6584-56568.hdf5 +/global/cscratch1/sd/jialin/h5boss/6585-56479.hdf5 +/global/cscratch1/sd/jialin/h5boss/6586-56485.hdf5 +/global/cscratch1/sd/jialin/h5boss/6587-56537.hdf5 +/global/cscratch1/sd/jialin/h5boss/6588-56536.hdf5 +/global/cscratch1/sd/jialin/h5boss/6589-56536.hdf5 +/global/cscratch1/sd/jialin/h5boss/6590-56273.hdf5 +/global/cscratch1/sd/jialin/h5boss/6591-56535.hdf5 +/global/cscratch1/sd/jialin/h5boss/6592-56535.hdf5 +/global/cscratch1/sd/jialin/h5boss/6593-56270.hdf5 +/global/cscratch1/sd/jialin/h5boss/6594-56272.hdf5 +/global/cscratch1/sd/jialin/h5boss/6595-56569.hdf5 +/global/cscratch1/sd/jialin/h5boss/6596-56331.hdf5 +/global/cscratch1/sd/jialin/h5boss/6597-56304.hdf5 +/global/cscratch1/sd/jialin/h5boss/6598-56574.hdf5 +/global/cscratch1/sd/jialin/h5boss/6599-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/6600-56566.hdf5 +/global/cscratch1/sd/jialin/h5boss/6601-56335.hdf5 +/global/cscratch1/sd/jialin/h5boss/6602-56571.hdf5 +/global/cscratch1/sd/jialin/h5boss/6603-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/6604-56337.hdf5 +/global/cscratch1/sd/jialin/h5boss/6605-56565.hdf5 +/global/cscratch1/sd/jialin/h5boss/6606-56304.hdf5 +/global/cscratch1/sd/jialin/h5boss/6617-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6618-56401.hdf5 +/global/cscratch1/sd/jialin/h5boss/6619-56371.hdf5 +/global/cscratch1/sd/jialin/h5boss/6620-56368.hdf5 +/global/cscratch1/sd/jialin/h5boss/6621-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6622-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6623-56367.hdf5 +/global/cscratch1/sd/jialin/h5boss/6624-56385.hdf5 +/global/cscratch1/sd/jialin/h5boss/6625-56386.hdf5 +/global/cscratch1/sd/jialin/h5boss/6626-56330.hdf5 +/global/cscratch1/sd/jialin/h5boss/6627-56369.hdf5 +/global/cscratch1/sd/jialin/h5boss/6628-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6629-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6630-56358.hdf5 +/global/cscratch1/sd/jialin/h5boss/6631-56364.hdf5 +/global/cscratch1/sd/jialin/h5boss/6632-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6633-56369.hdf5 +/global/cscratch1/sd/jialin/h5boss/6634-56367.hdf5 +/global/cscratch1/sd/jialin/h5boss/6635-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6636-56367.hdf5 +/global/cscratch1/sd/jialin/h5boss/6637-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6638-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6639-56385.hdf5 +/global/cscratch1/sd/jialin/h5boss/6640-56385.hdf5 +/global/cscratch1/sd/jialin/h5boss/6641-56383.hdf5 +/global/cscratch1/sd/jialin/h5boss/6642-56396.hdf5 +/global/cscratch1/sd/jialin/h5boss/6643-56386.hdf5 +/global/cscratch1/sd/jialin/h5boss/6644-56384.hdf5 +/global/cscratch1/sd/jialin/h5boss/6645-56398.hdf5 +/global/cscratch1/sd/jialin/h5boss/6646-56389.hdf5 +/global/cscratch1/sd/jialin/h5boss/6647-56390.hdf5 +/global/cscratch1/sd/jialin/h5boss/6648-56383.hdf5 +/global/cscratch1/sd/jialin/h5boss/6649-56364.hdf5 +/global/cscratch1/sd/jialin/h5boss/6656-56624.hdf5 +/global/cscratch1/sd/jialin/h5boss/6657-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6658-56606.hdf5 +/global/cscratch1/sd/jialin/h5boss/6659-56607.hdf5 +/global/cscratch1/sd/jialin/h5boss/6660-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6661-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6662-56364.hdf5 +/global/cscratch1/sd/jialin/h5boss/6663-56338.hdf5 +/global/cscratch1/sd/jialin/h5boss/6664-56383.hdf5 +/global/cscratch1/sd/jialin/h5boss/6665-56390.hdf5 +/global/cscratch1/sd/jialin/h5boss/6666-56371.hdf5 +/global/cscratch1/sd/jialin/h5boss/6667-56412.hdf5 +/global/cscratch1/sd/jialin/h5boss/6668-56605.hdf5 +/global/cscratch1/sd/jialin/h5boss/6669-56413.hdf5 +/global/cscratch1/sd/jialin/h5boss/6670-56389.hdf5 +/global/cscratch1/sd/jialin/h5boss/6671-56388.hdf5 +/global/cscratch1/sd/jialin/h5boss/6672-56386.hdf5 +/global/cscratch1/sd/jialin/h5boss/6673-56419.hdf5 +/global/cscratch1/sd/jialin/h5boss/6674-56416.hdf5 +/global/cscratch1/sd/jialin/h5boss/6675-56415.hdf5 +/global/cscratch1/sd/jialin/h5boss/6676-56389.hdf5 +/global/cscratch1/sd/jialin/h5boss/6677-56385.hdf5 +/global/cscratch1/sd/jialin/h5boss/6678-56401.hdf5 +/global/cscratch1/sd/jialin/h5boss/6679-56401.hdf5 +/global/cscratch1/sd/jialin/h5boss/6680-56420.hdf5 +/global/cscratch1/sd/jialin/h5boss/6681-56419.hdf5 +/global/cscratch1/sd/jialin/h5boss/6682-56390.hdf5 +/global/cscratch1/sd/jialin/h5boss/6683-56416.hdf5 +/global/cscratch1/sd/jialin/h5boss/6684-56412.hdf5 +/global/cscratch1/sd/jialin/h5boss/6685-56412.hdf5 +/global/cscratch1/sd/jialin/h5boss/6686-56389.hdf5 +/global/cscratch1/sd/jialin/h5boss/6687-56602.hdf5 +/global/cscratch1/sd/jialin/h5boss/6688-56412.hdf5 +/global/cscratch1/sd/jialin/h5boss/6689-56396.hdf5 +/global/cscratch1/sd/jialin/h5boss/6690-56416.hdf5 +/global/cscratch1/sd/jialin/h5boss/6691-56413.hdf5 +/global/cscratch1/sd/jialin/h5boss/6692-56390.hdf5 +/global/cscratch1/sd/jialin/h5boss/6693-56384.hdf5 +/global/cscratch1/sd/jialin/h5boss/6694-56386.hdf5 +/global/cscratch1/sd/jialin/h5boss/6695-56628.hdf5 +/global/cscratch1/sd/jialin/h5boss/6696-56398.hdf5 +/global/cscratch1/sd/jialin/h5boss/6697-56419.hdf5 +/global/cscratch1/sd/jialin/h5boss/6698-56637.hdf5 +/global/cscratch1/sd/jialin/h5boss/6699-56411.hdf5 +/global/cscratch1/sd/jialin/h5boss/6700-56384.hdf5 +/global/cscratch1/sd/jialin/h5boss/6701-56367.hdf5 +/global/cscratch1/sd/jialin/h5boss/6702-56381.hdf5 +/global/cscratch1/sd/jialin/h5boss/6703-56636.hdf5 +/global/cscratch1/sd/jialin/h5boss/6704-56388.hdf5 +/global/cscratch1/sd/jialin/h5boss/6705-56636.hdf5 +/global/cscratch1/sd/jialin/h5boss/6706-56385.hdf5 +/global/cscratch1/sd/jialin/h5boss/6707-56383.hdf5 +/global/cscratch1/sd/jialin/h5boss/6708-56452.hdf5 +/global/cscratch1/sd/jialin/h5boss/6709-56429.hdf5 +/global/cscratch1/sd/jialin/h5boss/6710-56416.hdf5 +/global/cscratch1/sd/jialin/h5boss/6711-56388.hdf5 +/global/cscratch1/sd/jialin/h5boss/6712-56421.hdf5 +/global/cscratch1/sd/jialin/h5boss/6713-56402.hdf5 +/global/cscratch1/sd/jialin/h5boss/6714-56447.hdf5 +/global/cscratch1/sd/jialin/h5boss/6715-56449.hdf5 +/global/cscratch1/sd/jialin/h5boss/6716-56401.hdf5 +/global/cscratch1/sd/jialin/h5boss/6717-56397.hdf5 +/global/cscratch1/sd/jialin/h5boss/6718-56398.hdf5 +/global/cscratch1/sd/jialin/h5boss/6719-56390.hdf5 +/global/cscratch1/sd/jialin/h5boss/6720-56390.hdf5 +/global/cscratch1/sd/jialin/h5boss/6721-56398.hdf5 +/global/cscratch1/sd/jialin/h5boss/6722-56431.hdf5 +/global/cscratch1/sd/jialin/h5boss/6723-56428.hdf5 +/global/cscratch1/sd/jialin/h5boss/6724-56416.hdf5 +/global/cscratch1/sd/jialin/h5boss/6725-56390.hdf5 +/global/cscratch1/sd/jialin/h5boss/6726-56394.hdf5 +/global/cscratch1/sd/jialin/h5boss/6727-56369.hdf5 +/global/cscratch1/sd/jialin/h5boss/6728-56426.hdf5 +/global/cscratch1/sd/jialin/h5boss/6729-56419.hdf5 +/global/cscratch1/sd/jialin/h5boss/6730-56425.hdf5 +/global/cscratch1/sd/jialin/h5boss/6731-56385.hdf5 +/global/cscratch1/sd/jialin/h5boss/6732-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6733-56369.hdf5 +/global/cscratch1/sd/jialin/h5boss/6734-56386.hdf5 +/global/cscratch1/sd/jialin/h5boss/6735-56397.hdf5 +/global/cscratch1/sd/jialin/h5boss/6736-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6737-56365.hdf5 +/global/cscratch1/sd/jialin/h5boss/6738-56424.hdf5 +/global/cscratch1/sd/jialin/h5boss/6739-56393.hdf5 +/global/cscratch1/sd/jialin/h5boss/6740-56401.hdf5 +/global/cscratch1/sd/jialin/h5boss/6741-56394.hdf5 +/global/cscratch1/sd/jialin/h5boss/6742-56415.hdf5 +/global/cscratch1/sd/jialin/h5boss/6743-56385.hdf5 +/global/cscratch1/sd/jialin/h5boss/6744-56399.hdf5 +/global/cscratch1/sd/jialin/h5boss/6745-56425.hdf5 +/global/cscratch1/sd/jialin/h5boss/6746-56386.hdf5 +/global/cscratch1/sd/jialin/h5boss/6747-56388.hdf5 +/global/cscratch1/sd/jialin/h5boss/6748-56371.hdf5 +/global/cscratch1/sd/jialin/h5boss/6749-56370.hdf5 +/global/cscratch1/sd/jialin/h5boss/6750-56367.hdf5 +/global/cscratch1/sd/jialin/h5boss/6751-56368.hdf5 +/global/cscratch1/sd/jialin/h5boss/6752-56366.hdf5 +/global/cscratch1/sd/jialin/h5boss/6753-56399.hdf5 +/global/cscratch1/sd/jialin/h5boss/6754-56414.hdf5 +/global/cscratch1/sd/jialin/h5boss/6755-56413.hdf5 +/global/cscratch1/sd/jialin/h5boss/6756-56426.hdf5 +/global/cscratch1/sd/jialin/h5boss/6757-56416.hdf5 +/global/cscratch1/sd/jialin/h5boss/6758-56415.hdf5 +/global/cscratch1/sd/jialin/h5boss/6759-56416.hdf5 +/global/cscratch1/sd/jialin/h5boss/6760-56425.hdf5 +/global/cscratch1/sd/jialin/h5boss/6780-56605.hdf5 +/global/cscratch1/sd/jialin/h5boss/6781-56575.hdf5 +/global/cscratch1/sd/jialin/h5boss/6782-56602.hdf5 +/global/cscratch1/sd/jialin/h5boss/6783-56284.hdf5 +/global/cscratch1/sd/jialin/h5boss/6785-56487.hdf5 +/global/cscratch1/sd/jialin/h5boss/6786-56448.hdf5 +/global/cscratch1/sd/jialin/h5boss/6787-56447.hdf5 +/global/cscratch1/sd/jialin/h5boss/6788-56428.hdf5 +/global/cscratch1/sd/jialin/h5boss/6789-56401.hdf5 +/global/cscratch1/sd/jialin/h5boss/6790-56430.hdf5 +/global/cscratch1/sd/jialin/h5boss/6791-56429.hdf5 +/global/cscratch1/sd/jialin/h5boss/6792-56432.hdf5 +/global/cscratch1/sd/jialin/h5boss/6793-56430.hdf5 +/global/cscratch1/sd/jialin/h5boss/6794-56448.hdf5 +/global/cscratch1/sd/jialin/h5boss/6795-56425.hdf5 +/global/cscratch1/sd/jialin/h5boss/6796-56453.hdf5 +/global/cscratch1/sd/jialin/h5boss/6797-56426.hdf5 +/global/cscratch1/sd/jialin/h5boss/6798-56485.hdf5 +/global/cscratch1/sd/jialin/h5boss/6799-56478.hdf5 +/global/cscratch1/sd/jialin/h5boss/6800-56399.hdf5 +/global/cscratch1/sd/jialin/h5boss/6801-56487.hdf5 +/global/cscratch1/sd/jialin/h5boss/6802-56456.hdf5 +/global/cscratch1/sd/jialin/h5boss/6803-56402.hdf5 +/global/cscratch1/sd/jialin/h5boss/6804-56447.hdf5 +/global/cscratch1/sd/jialin/h5boss/6805-56442.hdf5 +/global/cscratch1/sd/jialin/h5boss/6806-56419.hdf5 +/global/cscratch1/sd/jialin/h5boss/6807-56429.hdf5 +/global/cscratch1/sd/jialin/h5boss/6808-56443.hdf5 +/global/cscratch1/sd/jialin/h5boss/6809-56454.hdf5 +/global/cscratch1/sd/jialin/h5boss/6810-56447.hdf5 +/global/cscratch1/sd/jialin/h5boss/6813-56419.hdf5 +/global/cscratch1/sd/jialin/h5boss/6814-56487.hdf5 +/global/cscratch1/sd/jialin/h5boss/6815-56419.hdf5 +/global/cscratch1/sd/jialin/h5boss/6816-56444.hdf5 +/global/cscratch1/sd/jialin/h5boss/6817-56455.hdf5 +/global/cscratch1/sd/jialin/h5boss/6821-56396.hdf5 +/global/cscratch1/sd/jialin/h5boss/6822-56711.hdf5 +/global/cscratch1/sd/jialin/h5boss/6823-56687.hdf5 +/global/cscratch1/sd/jialin/h5boss/6825-56717.hdf5 +/global/cscratch1/sd/jialin/h5boss/6826-56449.hdf5 +/global/cscratch1/sd/jialin/h5boss/6827-56428.hdf5 +/global/cscratch1/sd/jialin/h5boss/6828-56430.hdf5 +/global/cscratch1/sd/jialin/h5boss/6829-56453.hdf5 +/global/cscratch1/sd/jialin/h5boss/6830-56429.hdf5 +/global/cscratch1/sd/jialin/h5boss/6831-56426.hdf5 +/global/cscratch1/sd/jialin/h5boss/6832-56426.hdf5 +/global/cscratch1/sd/jialin/h5boss/6833-56413.hdf5 +/global/cscratch1/sd/jialin/h5boss/6834-56478.hdf5 +/global/cscratch1/sd/jialin/h5boss/6835-56420.hdf5 +/global/cscratch1/sd/jialin/h5boss/6836-56443.hdf5 +/global/cscratch1/sd/jialin/h5boss/6837-56442.hdf5 +/global/cscratch1/sd/jialin/h5boss/6838-56429.hdf5 +/global/cscratch1/sd/jialin/h5boss/6839-56425.hdf5 +/global/cscratch1/sd/jialin/h5boss/6840-56685.hdf5 +/global/cscratch1/sd/jialin/h5boss/6872-56540.hdf5 +/global/cscratch1/sd/jialin/h5boss/6873-56541.hdf5 +/global/cscratch1/sd/jialin/h5boss/6874-56550.hdf5 +/global/cscratch1/sd/jialin/h5boss/6877-56544.hdf5 +/global/cscratch1/sd/jialin/h5boss/6879-56539.hdf5 +/global/cscratch1/sd/jialin/h5boss/6880-56543.hdf5 +/global/cscratch1/sd/jialin/h5boss/6881-56540.hdf5 +/global/cscratch1/sd/jialin/h5boss/6882-56541.hdf5 +/global/cscratch1/sd/jialin/h5boss/6931-56388.hdf5 +/global/cscratch1/sd/jialin/h5boss/6932-56397.hdf5 +/global/cscratch1/sd/jialin/h5boss/6933-56398.hdf5 +/global/cscratch1/sd/jialin/h5boss/6963-56724.hdf5 +/global/cscratch1/sd/jialin/h5boss/6964-56748.hdf5 +/global/cscratch1/sd/jialin/h5boss/6966-56748.hdf5 +/global/cscratch1/sd/jialin/h5boss/6967-56447.hdf5 +/global/cscratch1/sd/jialin/h5boss/6968-56443.hdf5 +/global/cscratch1/sd/jialin/h5boss/6969-56420.hdf5 +/global/cscratch1/sd/jialin/h5boss/6970-56444.hdf5 +/global/cscratch1/sd/jialin/h5boss/6972-56426.hdf5 +/global/cscratch1/sd/jialin/h5boss/6973-56741.hdf5 +/global/cscratch1/sd/jialin/h5boss/6974-56442.hdf5 +/global/cscratch1/sd/jialin/h5boss/6975-56720.hdf5 +/global/cscratch1/sd/jialin/h5boss/6976-56448.hdf5 +/global/cscratch1/sd/jialin/h5boss/6981-56443.hdf5 +/global/cscratch1/sd/jialin/h5boss/6982-56444.hdf5 +/global/cscratch1/sd/jialin/h5boss/6983-56447.hdf5 +/global/cscratch1/sd/jialin/h5boss/6985-56472.hdf5 +/global/cscratch1/sd/jialin/h5boss/6986-56717.hdf5 +/global/cscratch1/sd/jialin/h5boss/7027-56448.hdf5 +/global/cscratch1/sd/jialin/h5boss/7028-56449.hdf5 +/global/cscratch1/sd/jialin/h5boss/7029-56455.hdf5 +/global/cscratch1/sd/jialin/h5boss/7030-56448.hdf5 +/global/cscratch1/sd/jialin/h5boss/7031-56449.hdf5 +/global/cscratch1/sd/jialin/h5boss/7032-56471.hdf5 +/global/cscratch1/sd/jialin/h5boss/7033-56565.hdf5 +/global/cscratch1/sd/jialin/h5boss/7034-56564.hdf5 +/global/cscratch1/sd/jialin/h5boss/7035-56568.hdf5 +/global/cscratch1/sd/jialin/h5boss/7036-56569.hdf5 +/global/cscratch1/sd/jialin/h5boss/7037-56570.hdf5 +/global/cscratch1/sd/jialin/h5boss/7038-56571.hdf5 +/global/cscratch1/sd/jialin/h5boss/7039-56572.hdf5 +/global/cscratch1/sd/jialin/h5boss/7040-56574.hdf5 +/global/cscratch1/sd/jialin/h5boss/7041-56569.hdf5 +/global/cscratch1/sd/jialin/h5boss/7042-56573.hdf5 +/global/cscratch1/sd/jialin/h5boss/7043-56576.hdf5 +/global/cscratch1/sd/jialin/h5boss/7044-56591.hdf5 +/global/cscratch1/sd/jialin/h5boss/7045-56577.hdf5 +/global/cscratch1/sd/jialin/h5boss/7046-56568.hdf5 +/global/cscratch1/sd/jialin/h5boss/7047-56572.hdf5 +/global/cscratch1/sd/jialin/h5boss/7048-56575.hdf5 +/global/cscratch1/sd/jialin/h5boss/7049-56570.hdf5 +/global/cscratch1/sd/jialin/h5boss/7050-56573.hdf5 +/global/cscratch1/sd/jialin/h5boss/7051-56574.hdf5 +/global/cscratch1/sd/jialin/h5boss/7052-56539.hdf5 +/global/cscratch1/sd/jialin/h5boss/7053-56564.hdf5 +/global/cscratch1/sd/jialin/h5boss/7054-56575.hdf5 +/global/cscratch1/sd/jialin/h5boss/7055-56576.hdf5 +/global/cscratch1/sd/jialin/h5boss/7056-56577.hdf5 +/global/cscratch1/sd/jialin/h5boss/7057-56593.hdf5 +/global/cscratch1/sd/jialin/h5boss/7058-56593.hdf5 +/global/cscratch1/sd/jialin/h5boss/7059-56592.hdf5 +/global/cscratch1/sd/jialin/h5boss/7081-56684.hdf5 +/global/cscratch1/sd/jialin/h5boss/7082-56713.hdf5 +/global/cscratch1/sd/jialin/h5boss/7083-56710.hdf5 +/global/cscratch1/sd/jialin/h5boss/7084-56624.hdf5 +/global/cscratch1/sd/jialin/h5boss/7085-56625.hdf5 +/global/cscratch1/sd/jialin/h5boss/7086-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/7087-56637.hdf5 +/global/cscratch1/sd/jialin/h5boss/7088-56657.hdf5 +/global/cscratch1/sd/jialin/h5boss/7089-56661.hdf5 +/global/cscratch1/sd/jialin/h5boss/7090-56659.hdf5 +/global/cscratch1/sd/jialin/h5boss/7091-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/7092-56683.hdf5 +/global/cscratch1/sd/jialin/h5boss/7093-56657.hdf5 +/global/cscratch1/sd/jialin/h5boss/7094-56660.hdf5 +/global/cscratch1/sd/jialin/h5boss/7095-56625.hdf5 +/global/cscratch1/sd/jialin/h5boss/7096-56683.hdf5 +/global/cscratch1/sd/jialin/h5boss/7097-56667.hdf5 +/global/cscratch1/sd/jialin/h5boss/7098-56661.hdf5 +/global/cscratch1/sd/jialin/h5boss/7099-56666.hdf5 +/global/cscratch1/sd/jialin/h5boss/7100-56636.hdf5 +/global/cscratch1/sd/jialin/h5boss/7101-56634.hdf5 +/global/cscratch1/sd/jialin/h5boss/7102-56666.hdf5 +/global/cscratch1/sd/jialin/h5boss/7103-56661.hdf5 +/global/cscratch1/sd/jialin/h5boss/7104-56711.hdf5 +/global/cscratch1/sd/jialin/h5boss/7105-56740.hdf5 +/global/cscratch1/sd/jialin/h5boss/7106-56663.hdf5 +/global/cscratch1/sd/jialin/h5boss/7107-56740.hdf5 +/global/cscratch1/sd/jialin/h5boss/7108-56686.hdf5 +/global/cscratch1/sd/jialin/h5boss/7109-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/7110-56746.hdf5 +/global/cscratch1/sd/jialin/h5boss/7111-56741.hdf5 +/global/cscratch1/sd/jialin/h5boss/7112-56666.hdf5 +/global/cscratch1/sd/jialin/h5boss/7113-56711.hdf5 +/global/cscratch1/sd/jialin/h5boss/7114-56748.hdf5 +/global/cscratch1/sd/jialin/h5boss/7115-56667.hdf5 +/global/cscratch1/sd/jialin/h5boss/7116-56681.hdf5 +/global/cscratch1/sd/jialin/h5boss/7117-56685.hdf5 +/global/cscratch1/sd/jialin/h5boss/7118-56686.hdf5 +/global/cscratch1/sd/jialin/h5boss/7120-56720.hdf5 +/global/cscratch1/sd/jialin/h5boss/7123-56726.hdf5 +/global/cscratch1/sd/jialin/h5boss/7124-56720.hdf5 +/global/cscratch1/sd/jialin/h5boss/7126-56568.hdf5 +/global/cscratch1/sd/jialin/h5boss/7127-56566.hdf5 +/global/cscratch1/sd/jialin/h5boss/7128-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/7129-56569.hdf5 +/global/cscratch1/sd/jialin/h5boss/7130-56568.hdf5 +/global/cscratch1/sd/jialin/h5boss/7131-56570.hdf5 +/global/cscratch1/sd/jialin/h5boss/7132-56565.hdf5 +/global/cscratch1/sd/jialin/h5boss/7133-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/7134-56566.hdf5 +/global/cscratch1/sd/jialin/h5boss/7135-56564.hdf5 +/global/cscratch1/sd/jialin/h5boss/7136-56570.hdf5 +/global/cscratch1/sd/jialin/h5boss/7137-56566.hdf5 +/global/cscratch1/sd/jialin/h5boss/7138-56572.hdf5 +/global/cscratch1/sd/jialin/h5boss/7139-56568.hdf5 +/global/cscratch1/sd/jialin/h5boss/7140-56569.hdf5 +/global/cscratch1/sd/jialin/h5boss/7141-56565.hdf5 +/global/cscratch1/sd/jialin/h5boss/7142-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/7143-56572.hdf5 +/global/cscratch1/sd/jialin/h5boss/7144-56564.hdf5 +/global/cscratch1/sd/jialin/h5boss/7145-56567.hdf5 +/global/cscratch1/sd/jialin/h5boss/7146-56573.hdf5 +/global/cscratch1/sd/jialin/h5boss/7147-56574.hdf5 +/global/cscratch1/sd/jialin/h5boss/7148-56591.hdf5 +/global/cscratch1/sd/jialin/h5boss/7149-56593.hdf5 +/global/cscratch1/sd/jialin/h5boss/7150-56597.hdf5 +/global/cscratch1/sd/jialin/h5boss/7151-56598.hdf5 +/global/cscratch1/sd/jialin/h5boss/7152-56660.hdf5 +/global/cscratch1/sd/jialin/h5boss/7155-56629.hdf5 +/global/cscratch1/sd/jialin/h5boss/7159-56603.hdf5 +/global/cscratch1/sd/jialin/h5boss/7160-56599.hdf5 +/global/cscratch1/sd/jialin/h5boss/7161-56625.hdf5 +/global/cscratch1/sd/jialin/h5boss/7162-56605.hdf5 +/global/cscratch1/sd/jialin/h5boss/7163-56593.hdf5 +/global/cscratch1/sd/jialin/h5boss/7164-56597.hdf5 +/global/cscratch1/sd/jialin/h5boss/7165-56603.hdf5 +/global/cscratch1/sd/jialin/h5boss/7166-56602.hdf5 +/global/cscratch1/sd/jialin/h5boss/7167-56604.hdf5 +/global/cscratch1/sd/jialin/h5boss/7168-56625.hdf5 +/global/cscratch1/sd/jialin/h5boss/7169-56628.hdf5 +/global/cscratch1/sd/jialin/h5boss/7181-56662.hdf5 +/global/cscratch1/sd/jialin/h5boss/7182-56665.hdf5 +/global/cscratch1/sd/jialin/h5boss/7183-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/7184-56629.hdf5 +/global/cscratch1/sd/jialin/h5boss/7235-56603.hdf5 +/global/cscratch1/sd/jialin/h5boss/7236-56605.hdf5 +/global/cscratch1/sd/jialin/h5boss/7237-56662.hdf5 +/global/cscratch1/sd/jialin/h5boss/7238-56660.hdf5 +/global/cscratch1/sd/jialin/h5boss/7239-56693.hdf5 +/global/cscratch1/sd/jialin/h5boss/7240-56665.hdf5 +/global/cscratch1/sd/jialin/h5boss/7241-56606.hdf5 +/global/cscratch1/sd/jialin/h5boss/7242-56628.hdf5 +/global/cscratch1/sd/jialin/h5boss/7243-56629.hdf5 +/global/cscratch1/sd/jialin/h5boss/7245-56636.hdf5 +/global/cscratch1/sd/jialin/h5boss/7246-56625.hdf5 +/global/cscratch1/sd/jialin/h5boss/7247-56626.hdf5 +/global/cscratch1/sd/jialin/h5boss/7248-56630.hdf5 +/global/cscratch1/sd/jialin/h5boss/7253-56598.hdf5 +/global/cscratch1/sd/jialin/h5boss/7254-56625.hdf5 +/global/cscratch1/sd/jialin/h5boss/7255-56597.hdf5 +/global/cscratch1/sd/jialin/h5boss/7256-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/7257-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/7258-56605.hdf5 +/global/cscratch1/sd/jialin/h5boss/7259-56603.hdf5 +/global/cscratch1/sd/jialin/h5boss/7260-56659.hdf5 +/global/cscratch1/sd/jialin/h5boss/7261-56603.hdf5 +/global/cscratch1/sd/jialin/h5boss/7262-56682.hdf5 +/global/cscratch1/sd/jialin/h5boss/7277-56748.hdf5 +/global/cscratch1/sd/jialin/h5boss/7280-56709.hdf5 +/global/cscratch1/sd/jialin/h5boss/7282-56660.hdf5 +/global/cscratch1/sd/jialin/h5boss/7284-56683.hdf5 +/global/cscratch1/sd/jialin/h5boss/7292-56709.hdf5 +/global/cscratch1/sd/jialin/h5boss/7293-56741.hdf5 +/global/cscratch1/sd/jialin/h5boss/7294-56739.hdf5 +/global/cscratch1/sd/jialin/h5boss/7298-56661.hdf5 +/global/cscratch1/sd/jialin/h5boss/7299-56686.hdf5 +/global/cscratch1/sd/jialin/h5boss/7300-56707.hdf5 +/global/cscratch1/sd/jialin/h5boss/7301-56746.hdf5 +/global/cscratch1/sd/jialin/h5boss/7304-56745.hdf5 +/global/cscratch1/sd/jialin/h5boss/7306-56663.hdf5 +/global/cscratch1/sd/jialin/h5boss/7307-56720.hdf5 +/global/cscratch1/sd/jialin/h5boss/7308-56709.hdf5 +/global/cscratch1/sd/jialin/h5boss/7310-56693.hdf5 +/global/cscratch1/sd/jialin/h5boss/7315-56685.hdf5 +/global/cscratch1/sd/jialin/h5boss/7316-56710.hdf5 +/global/cscratch1/sd/jialin/h5boss/7318-56738.hdf5 +/global/cscratch1/sd/jialin/h5boss/7320-56722.hdf5 +/global/cscratch1/sd/jialin/h5boss/7325-56717.hdf5 +/global/cscratch1/sd/jialin/h5boss/7326-56710.hdf5 +/global/cscratch1/sd/jialin/h5boss/7327-56715.hdf5 +/global/cscratch1/sd/jialin/h5boss/7328-56715.hdf5 +/global/cscratch1/sd/jialin/h5boss/7329-56719.hdf5 +/global/cscratch1/sd/jialin/h5boss/7330-56684.hdf5 +/global/cscratch1/sd/jialin/h5boss/7331-56658.hdf5 +/global/cscratch1/sd/jialin/h5boss/7332-56683.hdf5 +/global/cscratch1/sd/jialin/h5boss/7333-56686.hdf5 +/global/cscratch1/sd/jialin/h5boss/7334-56656.hdf5 +/global/cscratch1/sd/jialin/h5boss/7336-56656.hdf5 +/global/cscratch1/sd/jialin/h5boss/7337-56662.hdf5 +/global/cscratch1/sd/jialin/h5boss/7338-56669.hdf5 +/global/cscratch1/sd/jialin/h5boss/7339-56686.hdf5 +/global/cscratch1/sd/jialin/h5boss/7340-56726.hdf5 +/global/cscratch1/sd/jialin/h5boss/7374-56751.hdf5 +/global/cscratch1/sd/jialin/h5boss/7376-56749.hdf5 +/global/cscratch1/sd/jialin/h5boss/7377-56741.hdf5 +/global/cscratch1/sd/jialin/h5boss/7378-56715.hdf5 +/global/cscratch1/sd/jialin/h5boss/7379-56713.hdf5 +/global/cscratch1/sd/jialin/h5boss/7380-56753.hdf5 +/global/cscratch1/sd/jialin/h5boss/7381-56717.hdf5 +/global/cscratch1/sd/jialin/h5boss/7383-56749.hdf5 +/global/cscratch1/sd/jialin/h5boss/7384-56715.hdf5 +/global/cscratch1/sd/jialin/h5boss/7385-56710.hdf5 +/global/cscratch1/sd/jialin/h5boss/7393-56749.hdf5 +/global/cscratch1/sd/jialin/h5boss/7414-56748.hdf5 +/global/cscratch1/sd/jialin/h5boss/7417-56753.hdf5 +/global/cscratch1/sd/jialin/h5boss/7420-56753.hdf5 +/global/cscratch1/sd/jialin/h5boss/7427-56751.hdf5 +/global/cscratch1/sd/jialin/h5boss/7429-56740.hdf5 +/global/cscratch1/sd/jialin/h5boss/7445-56720.hdf5 +/global/cscratch1/sd/jialin/h5boss/7446-56722.hdf5 +/global/cscratch1/sd/jialin/h5boss/7447-56746.hdf5 +/global/cscratch1/sd/jialin/h5boss/7448-56739.hdf5 +/global/cscratch1/sd/jialin/h5boss/7449-56740.hdf5 +/global/cscratch1/sd/jialin/h5boss/7450-56722.hdf5 +/global/cscratch1/sd/jialin/h5boss/7451-56739.hdf5 +/global/cscratch1/sd/jialin/h5boss/7452-56745.hdf5 +/global/cscratch1/sd/jialin/h5boss/7453-56749.hdf5 +/global/cscratch1/sd/jialin/h5boss/7454-56751.hdf5 +/global/cscratch1/sd/jialin/h5boss/7456-56727.hdf5 +/global/cscratch1/sd/jialin/h5boss/7457-56746.hdf5 diff --git a/h5boss_py/docs/source/_static/pmf_add.txt b/h5boss_py/docs/source/_static/pmf_add.txt new file mode 100755 index 0000000..e8e3d74 --- /dev/null +++ b/h5boss_py/docs/source/_static/pmf_add.txt @@ -0,0 +1,11 @@ +plates mjds fibers +6697 56419 796 +4697 55660 190 +4191 55444 636 +7109 56658 751 +7377 56741 443 +4021 55620 32 +6932 56397 358 +4494 55569 506 +4277 55506 292 +6753 56399 289 diff --git a/h5boss_py/docs/source/_static/pmf_sample.txt b/h5boss_py/docs/source/_static/pmf_sample.txt new file mode 100755 index 0000000..f64a614 --- /dev/null +++ b/h5boss_py/docs/source/_static/pmf_sample.txt @@ -0,0 +1,33 @@ +plates mjds fibers +4562 55570 622 +4479 55592 543 +7294 56739 242 +4224 55481 134 +6973 56741 656 +6035 56076 873 +5706 56165 399 +6731 56385 263 +4638 55956 13 +4895 55708 510 +5067 55751 822 +7084 56624 396 +6675 56415 42 +6124 56211 553 +5067 55751 386 +4778 55706 650 +5009 55707 586 +7093 56657 649 +6276 56269 559 +6603 56567 798 +7240 56665 352 +4372 55541 854 +6515 56536 42 +4646 55622 885 +7039 56572 792 +7184 56629 962 +4049 55591 443 +3928 55331 553 +5204 56036 724 +6735 56397 844 +5427 56001 273 +6975 56720 760 diff --git a/h5boss_py/docs/source/_static/pmf_update.txt b/h5boss_py/docs/source/_static/pmf_update.txt new file mode 100755 index 0000000..2fb5b56 --- /dev/null +++ b/h5boss_py/docs/source/_static/pmf_update.txt @@ -0,0 +1,12 @@ +plates mjds fibers +6697 56419 796 +4697 55660 190 +4191 55444 636 +7109 56658 751 +7377 56741 443 +4021 55620 32 +4318 55508 614 +6932 56397 358 +4494 55569 506 +4277 55506 292 +6753 56399 289 diff --git a/h5boss_py/docs/source/add.rst b/h5boss_py/docs/source/add.rst new file mode 100755 index 0000000..0769b49 --- /dev/null +++ b/h5boss_py/docs/source/add.rst @@ -0,0 +1,73 @@ +.. _add: +Add +======== +Given a plate/mjd/fiber query list. This `add` command will update the base hdf5 accordingly. Basically, add will + * Compare the existing plate/mjd/fibers in the base hdf5 file with this query, + * Search the new plate/mjd/fiber, which are not existed in the base hdf5 file, from the input hdf5 files + * Add the founded plate/mjd/fiber into the base file + +The base file is supposed to be augmented with a few new plate/mjd/fibers. + +Usage: +------ + +.. highlight:: c + +add -h + +usage:: + + add [-h] base input pmf + +positional arguments:: + + base Pre-subseted HDF5 file + + input HDF5 input files list + + pmf Plate/mjd/fiber list + +optional arguments:: + + -h, --help show this help message and exit + +Example: +-------- +.. highlight:: c + +Prepare input:: + + cat input_sample.txt + + /global/cscratch1/sd/jialin/h5boss/3665-55247.hdf5 + + ... + + cat pmf_add.txt + + plates mjds fibers + + 3665 55247 65 + + 3665 55247 390 + + ... + +Download: :download:`input_sample.txt <_static/input_sample.txt>`, :download:`pmf_add.txt <_static/pmf_add.txt>`, :download:`base.h5 <_static/base.h5>` + +Execute command:: + + add base.h5 input_sample.txt pmf_add.txt + +Output:: + + -Source file open: 2.28 + -Fiber query time: 0.00 + -Fiber copy time: 0.21 + -Catalog copy time: 0.84 + -Group create time: 0.00 + -File close time: 0.04 + Selection Time: 7.43 seconds + Done selection + + diff --git a/h5boss_py/docs/source/boss2fits.rst b/h5boss_py/docs/source/boss2fits.rst new file mode 100755 index 0000000..80fa33f --- /dev/null +++ b/h5boss_py/docs/source/boss2fits.rst @@ -0,0 +1,7 @@ +boss2fits module +================ + +.. automodule:: boss2fits + :members: + :undoc-members: + :show-inheritance: diff --git a/h5boss_py/docs/source/boss2hdf5-parallel.rst b/h5boss_py/docs/source/boss2hdf5-parallel.rst new file mode 100755 index 0000000..a43244a --- /dev/null +++ b/h5boss_py/docs/source/boss2hdf5-parallel.rst @@ -0,0 +1,7 @@ +boss2hdf5-parallel module +========================= + +.. automodule:: boss2hdf5-parallel + :members: + :undoc-members: + :show-inheritance: diff --git a/h5boss_py/docs/source/boss2hdf5_v1.rst b/h5boss_py/docs/source/boss2hdf5_v1.rst new file mode 100755 index 0000000..42d0398 --- /dev/null +++ b/h5boss_py/docs/source/boss2hdf5_v1.rst @@ -0,0 +1,7 @@ +boss2hdf5_v1 module +=================== + +.. automodule:: boss2hdf5_v1 + :members: + :undoc-members: + :show-inheritance: diff --git a/h5boss_py/docs/source/boss2hdf5_v2.rst b/h5boss_py/docs/source/boss2hdf5_v2.rst new file mode 100755 index 0000000..46e24b3 --- /dev/null +++ b/h5boss_py/docs/source/boss2hdf5_v2.rst @@ -0,0 +1,7 @@ +boss2hdf5_v2 module +=================== + +.. automodule:: boss2hdf5_v2 + :members: + :undoc-members: + :show-inheritance: diff --git a/h5boss_py/docs/source/conf.py b/h5boss_py/docs/source/conf.py new file mode 100755 index 0000000..861769e --- /dev/null +++ b/h5boss_py/docs/source/conf.py @@ -0,0 +1,283 @@ +# -*- coding: utf-8 -*- +# +# h5boss documentation build configuration file, created by +# sphinx-quickstart on Tue Oct 11 12:04:08 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../..')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc','numpydoc'] +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'h5boss' +copyright = u'2016, Jialin Liu' +author = u'Jialin Liu' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = '0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'bizstyle' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'h5bossdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'h5boss.tex', u'h5boss Documentation', + u'Jialin Liu', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'h5boss', u'h5boss Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'h5boss', u'h5boss Documentation', + author, 'h5boss', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/h5boss_py/docs/source/convert.rst b/h5boss_py/docs/source/convert.rst new file mode 100755 index 0000000..0cbe46c --- /dev/null +++ b/h5boss_py/docs/source/convert.rst @@ -0,0 +1,75 @@ +.. _convert: +Convert +======== +Convert the boss data from `Fits `_ to `HDF5 `_ format + +At NERSC, BOSS data are stored on global projecta file system:: + +/global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0/ + +H5Boss offeres two alternative hdf5 formats, v1 and v2. The two formats are essentially designed to ease the file management and speedup the I/O. The two formats differ in the way how the fibers are organized, etc. More details can be found at :ref:`Design ` + +We have converted all Fits files into HDF5 formats, which has largely reduced the number of files from ~1 million to 2500. Follow the steps below to convert all Fits file of plate `4444` and mjd `55538` into a single HDF5 file, named `4444-55538_v1.h5`. + +Usage: +------ + +.. highlight:: c + +Usage:: + + boss2hdf5_v1 [options] or boss2hdf5_v2 [options] + +Options:: + + -h, --help show this help message and exit + + -i INPUT, --input=INPUT input spPlate file + + -o OUTPUT, --output=OUTPUT output hdf5 file + + + +Example: +-------- +.. highlight:: c + +Execute command:: + + export BOSS_DIR=/global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0/ + boss2hdf5_v1 -i $BOSS_DIR/4444/spPlate-4444-55538.fits -o $SCRATCH/4444-55538_v1.h5 + +This command not only converts the specified fits file, i.e., spPlate-4444-55538.fits, but also scan other related fits file in the same directory, $BOSS_DIR/4444. The detailed conversion can be found at :ref:`Design Fits2hdf ` + +Output:: + + Running Conversion: + plugmap + zbest + zline + photo + loading coadds + writing coadds + parsing planfile + copying individual exposure + spFrame-b1-00123585.fits.gz + spFrame-b2-00123585.fits.gz + spFrame-r1-00123585.fits.gz + spFrame-r2-00123585.fits.gz + spFrame-b1-00123586.fits.gz + spFrame-b2-00123586.fits.gz + spFrame-r1-00123586.fits.gz + spFrame-r2-00123586.fits.gz + spFrame-b1-00123587.fits.gz + spFrame-b2-00123587.fits.gz + spFrame-r1-00123587.fits.gz + spFrame-r2-00123587.fits.gz + spFrame-b1-00123588.fits.gz + spFrame-b2-00123588.fits.gz + spFrame-r1-00123588.fits.gz + spFrame-r2-00123588.fits.gz + spFrame-b1-00123589.fits.gz + spFrame-b2-00123589.fits.gz + spFrame-r1-00123589.fits.gz + spFrame-r2-00123589.fits.gz + Conversion time: 1521.86 seconds diff --git a/h5boss_py/docs/source/csubset.rst b/h5boss_py/docs/source/csubset.rst new file mode 100755 index 0000000..7e23285 --- /dev/null +++ b/h5boss_py/docs/source/csubset.rst @@ -0,0 +1,19 @@ +.. _csubset: +Parallel Subset +=============== + +Usage: +------ +.. highlight:: c + +usage:: + +Example: +-------- +.. highlight:: c + +download: :download:`input <_static/input_sample.txt>`, :download:`pmf <_static/pmf_sample.txt>` + +execute command:: + + diff --git a/h5boss_py/docs/source/fits2hdf.rst b/h5boss_py/docs/source/fits2hdf.rst new file mode 100755 index 0000000..36972d9 --- /dev/null +++ b/h5boss_py/docs/source/fits2hdf.rst @@ -0,0 +1,186 @@ +.. _fits2hdf: + +Convertion +========== +This section shows the mapping between Fits and HDF5 formats, and following with the design of two H5Boss :ref:`Formats, ` + +* The :ref:`mappingdiag` helps to visualize the mapping between Fits and HDF5. +* The :ref:`mappingtab` contains more detailed information of the relationship between the Boss-Fits structure and the Boss-HDF5 hierarchy. +* The :ref:`examplecode` helps to understand 'what is the difference in reading a'Flux' dataset in the two formats', etc. + +.. _mappingdiag: + +Mapping Diagram +=============== + +.. figure:: images/fits2fmt1.png + + +.. _mappingtab: + +Mapping Table +============= + +The conversion :ref:`code, ` took all files in a PLATE folder and convert into a single HDF5 file. Note that the BOSS data on NERSC project file system is organized as: + +__ highlight:: c + +**Fits Folder Top Level**:: + + user@edison12: pwd + /global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0 + user@edison12: ls + 3520 4032 4458 4848 5338 5897 6375 6758 + 3523 4033 4459 4849 5339 5898 6376 6759 + 3536 4034 4460 4850 5340 5899 6377 6760 + 3537 4035 4461 4851 5341 5900 6378 6780 + 3538 4036 4462 4852 5342 5901 6379 6781 + 3540 4037 4463 4853 5343 5902 6380 6782 + 3548 4038 4464 4854 5344 5903 6381 6783 + ... + ... + +Then inside one folder, which is named afte the **PLATE** number, e.g., plate 4857: + +**Fits Folder Second Level**:: + + 4857/ + photoMatchPlate-4857-55711.fits spFluxcalib-r2-00131416.fits.gz + photoPlate-4857-55711.fits spFluxcalib-r2-00131417.fits.gz + photoPosPlate-4857-55711.fits spFluxcalib-r2-00131418.fits.gz + redux-4857-55711 spFluxcalib-r2-00131419.fits.gz + redux-4857-55711.o22925 spFluxcorr-b1-00131415.fits.gz + ... ... + spCFrame-b1-00131415.fits spFluxcorr-b2-00131415.fits.gz + spCFrame-b1-00131416.fits spFluxcorr-b2-00131416.fits.gz + spCFrame-r2-00131415.fits spFluxdistort-4857-55711.fits + spCFrame-r2-00131416.fits spFluxdistort-4857-55711.ps + spCFrame-r2-00131417.fits spFrame-b1-00131415.fits.gz + spCFrame-r2-00131418.fits spFrame-b1-00131416.fits.gz + spFluxcalib-b2-00131417.fits.gz spFrame-r2-00131418.fits.gz + spFluxcalib-b2-00131418.fits.gz spFrame-r2-00131419.fits.gz + spFluxcalib-r1-00131416.fits.gz spPlate-4857-55711.fits + spFluxcalib-r1-00131417.fits.gz spSN2d-4857-55711.ps + spFluxcalib-r1-00131418.fits.gz spSN2d-4857-55711.ps.orig + spFluxcalib-r1-00131419.fits.gz spectro_redux_v5_7_0_4857.sha1sum + spFluxcalib-r2-00131415.fits.gz v5_7_0 + +After the conversion, the whole folder 4857 becomes a **single** HDF5 file, in which, there are 1000 fiber groups and a few catalog datasets. + +**HDF5 Folder Top Level**:: + + user@edison12: pwd + $SCRATCH/h5boss/v5_7_0 + user@edison12: ls + 3520.h5 4032.h5 4458.h5 4848.h5 5338.h5 5897.h5 6375.h5 6758.h5 + 3523.h5 4033.h5 4459.h5 4849.h5 5339.h5 5898.h5 6376.h5 6759.h5 + 3536.h5 4034.h5 4460.h5 4850.h5 5340.h5 5899.h5 6377.h5 6760.h5 + 3537.h5 4035.h5 4461.h5 4851.h5 5341.h5 5900.h5 6378.h5 6780.h5 + 3538.h5 4036.h5 4462.h5 4852.h5 5342.h5 5901.h5 6379.h5 6781.h5 + 3540.h5 4037.h5 4463.h5 4853.h5 5343.h5 5902.h5 6380.h5 6782.h5 + 3548.h5 4038.h5 4464.h5 4854.h5 5344.h5 5903.h5 6381.h5 6783.h5 + ... + ... +For the catalog, the conversion simply reads one HDU in a fits file and writes into a table dataset in HDF5. For example, as shown in the following catalog mapping table, the HDU1 in 'photoMatchPlate-pppp-mmmmm.fits' becomes a HDF5 dataset within the 'photo' group, where the higher level groups are 'plate' and 'mjd'. + + +Catalog: + +=============================== ======== =============== ============ +Fits File Fits HDU HDF5 Group HDF5 Dataset +=============================== ======== =============== ============ +photoMatchPlate-pppp-mmmmm.fits HDU 1 plate/mjd/photo match +photoPosPlate-pppp-mmmmm.fits HDU 1 plate/mjd/photo matchpos +photoPlate-pppp-mmmmm.fits HDU 1 plate/mjd/photo matchflux +v5_7_0/spZbest-pppp-mmmmm.fits HDU 1 plate/mjd zbest +v5_7_0/spZline-pppp-mmmmm.fits HDU 1 plate/mjd zline +spPlate-pppp-mmmmm.fits HDU 5 plate/mjd plugmap +=============================== ======== =============== ============ + + +In each of the 1000 fiber groups, the fiber number is used as the group name in HDF5, e.g., 1, which is under 'plate/mjd'. In a fiber group, there is a 'coadd' dataset, which is a 4000*8 2D array,(the number 4000 varies in different plates). The number 8 refers to the total number of wavelengths that're converted, i.e., **Flux, Ivar, and_mask, or_mask, wavedisp, wave, sky and model**. These wavelengths are from different HDUs of the 'spPlate-pppp-mmmmm.fits' file. For example, the 'Flux' is from the Primary HDU. In fits, this primary HDU is a 1000 by 4000 2D table, in HDF5 file, this 2D table is split into 1000 fiber groups, where each fiber group only has the wavelength of one fiber. Similar convertion was conducted on the 'Exposures', which is from 'spCFrame-[br][12]-[0-9]{8}.fits' file. Special attention needs to be paid on the column 'wave' in coadd, and columns 'wave' and 'clib' in the b/r dataset, as noted below the table. + +Spectra: + +== ================================ ================ ===================================== ================================== +Id Fits File Fits HDU HDF5 Group HDF5 Dataset(ColumnID) ColumnName +== ================================ ================ ===================================== ================================== +1 spPlate-pppp-mmmmm.fits HDU 0 plate/mjd/[1-1000] coadd(col2) FLUX +2 spPlate-pppp-mmmmm.fits HDU 1 IVAR plate/mjd/[1-1000] coadd(col3) IVAR +3 spPlate-pppp-mmmmm.fits HDU 2 ANDMASK plate/mjd/[1-1000] coadd(col4) AND_MASK +4 spPlate-pppp-mmmmm.fits HDU 3 ORMASK plate/mjd/[1-1000] coadd(col5) OR_MASK +5 spPlate-pppp-mmmmm.fits HDU 4 WAVEDISP plate/mjd/[1-1000] coadd(col6) WAVEDISP +6 spPlate-pppp-mmmmm.fits HDU 5 PLUGMAP plate/mjd/[1-1000] coadd(col1)* WAVE +7 spPlate-pppp-mmmmm.fits HDU 6 SKY plate/mjd/[1-1000] coadd(col7) SKY +8 plate/mjd/[1-1000] coadd(col8) MODEL +9 spCFrame-[br][12]-[0-9]{8}.fits HDU 0 plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col1) FLUX +10 spCFrame-[br][12]-[0-9]{8}.fits HDU 1 IVAR plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col2) IVAR +11 spCFrame-[br][12]-[0-9]{8}.fits HDU 2 MASK plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col3) MASK +12 spCFrame-[br][12]-[0-9]{8}.fits HDU 3 WAVELENGTH plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col4)* WAVE +13 spCFrame-[br][12]-[0-9]{8}.fits HDU 4 WAVEDISP plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col5) WAVEDISP +14 spCFrame-[br][12]-[0-9]{8}.fits HDU 6 SKY plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col6) SKY +15 spCFrame-[br][12]-[0-9]{8}.fits HDU 7 X plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col7) X +16 spCFrame-[br][12]-[0-9]{8}.fits HDU 8 SUPERFLAT plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col8)* CLIB +17 spFlat-[br][12]-[0-9]{8}.fits.gz HDU 0 FIBERFLAT plate/mjd/[1-1000]/exposures/[0-9]{8} b/r(col8)* CLIB +== ================================ ================ ===================================== ================================== + + +Notes: The convertion simply copy the data in fits file and reorginze in HDF5 file, a few exceptions where the data need additional calculation can be better understood by reading the code. Here are just brief descriptions: + +.. highlight:: c + +line 8, This `wave` column is obtained with the following python code:: + + header = fits.open(platefile)[0].header + c0 = header['COEFF0'] + c1 = header['COEFF1'] + nwave = header['NAXIS1'] + wave = (10**(c0 + c1*np.arange(nwave))) + +line 12, wavelength is log based, so the conversion code calculates the reverse, i.e., 'WAVE'=10^wavelength + +line 16 and 17, the 'CLIB' is calculated with the following python code:: + + electrons = eflux * fiberflat * superflat + calib = flux / electrons + +.. highlight:: c + +.. _examplecode: + +Example Codes +============= + +The sample codes for reading same data from Fits versus from the converted HDF5 file: + +**Read plate: 4857, mjd: 55711, fiber: 4, FLUX** + +Read Flux from Fits:: + + dfits = fitsio.FITS('spPlate-4857-55711.fits') + dflux = dfits[0][3:4,:] + +Read Flux from HDF5:: + + dh5 = h5py.File('4857-55711.h5') + dflux = dh5['4857/55711/4/coadd']['FLUX'] + +Read Multiple HDUs from Fits:: + + dfits = fitsio.FITS('spPlate-4857-55711.fits') + dflux = dfits[0][3:4,:] + dwave = dfits[1][3:4,:] + +Read Multiple HDUs from HDF5:: + + dh5 = h5py.File('4857-55711.h5') + dflux_wave = dh5['4857/55711/4/coadd'][('FLUX','IVAR')] + +Read All HDUs from Fits:: + + for i in range(0,6): + dall[i] = dfits[i][3:4,:] + +Read All HDUs from HDF5:: + + dall = dh5['4857/55711/4/coadd'][()] diff --git a/h5boss_py/docs/source/h5boss.rst b/h5boss_py/docs/source/h5boss.rst new file mode 100755 index 0000000..4b14a51 --- /dev/null +++ b/h5boss_py/docs/source/h5boss.rst @@ -0,0 +1,94 @@ +h5boss package +============== + +Submodules +---------- + +h5boss.boss2hdf5 module +----------------------- + +.. automodule:: h5boss.boss2hdf5 + :members: + :undoc-members: + :show-inheritance: + +h5boss.h5map module +------------------- + +.. automodule:: h5boss.h5map + :members: + :undoc-members: + :show-inheritance: + +h5boss.io module +---------------- + +.. automodule:: h5boss.io + :members: + :undoc-members: + :show-inheritance: + +h5boss.remove module +-------------------- + +.. automodule:: h5boss.remove + :members: + :undoc-members: + :show-inheritance: + +h5boss.select module +-------------------- + +.. automodule:: h5boss.select + :members: + :undoc-members: + :show-inheritance: + +h5boss.select_add module +------------------------ + +.. automodule:: h5boss.select_add + :members: + :undoc-members: + :show-inheritance: + +h5boss.select_update module +--------------------------- + +.. automodule:: h5boss.select_update + :members: + :undoc-members: + :show-inheritance: + +h5boss.selectmpi_v1 module +-------------------------- + +.. automodule:: h5boss.selectmpi_v1 + :members: + :undoc-members: + :show-inheritance: + +h5boss.selectmpi_v2 module +-------------------------- + +.. automodule:: h5boss.selectmpi_v2 + :members: + :undoc-members: + :show-inheritance: + +h5boss.sql module +----------------- + +.. automodule:: h5boss.sql + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: h5boss + :members: + :undoc-members: + :show-inheritance: diff --git a/h5boss_py/docs/source/h5bossdev.rst b/h5boss_py/docs/source/h5bossdev.rst new file mode 100755 index 0000000..2899081 --- /dev/null +++ b/h5boss_py/docs/source/h5bossdev.rst @@ -0,0 +1,29 @@ +.. _h5bossdev: + +Missing Features/Funcionts +========================== + +This page documents the missing features/functions in various versions of the h5boss package. Further development: `Github:H5boss-dev `_. + +H5boss python **serial** package for format version **1**:: + + This version is complete + Support all features: convert/subset/add/update + +H5boss python **parallel** package for format version **1**:: + + Support convert/subset in parallel + Not support add/update + +H5boss python **serial** package for format version **2** :: + + Support convert + +H5boss python **parallel** package for format version **2** :: + + Support convert/subset(fiber only) + Not support add/update and catalog subset + +H5boss c **parallel** package for format version **1** :: + + Support subset (fiber only), and requires a pre-created template diff --git a/h5boss_py/docs/source/h5bossfmts.rst b/h5boss_py/docs/source/h5bossfmts.rst new file mode 100755 index 0000000..ab80bbb --- /dev/null +++ b/h5boss_py/docs/source/h5bossfmts.rst @@ -0,0 +1,25 @@ +.. _h5bossfmts: +Two H5Boss HDF5 Formats +----------------------- + +:ref:`myfigure-v1` + +:ref:`myfigure-v2` + +[:ref:`Fits to Format v1 `] + +.. _myfigure-v1: + +.. figure:: images/fmt1.png + :alt: H5Boss format 1 + :align: left + + Format v1 + +.. _myfigure-v2: + +.. figure:: images/fmt2.png + :alt: H5Boss format 2 + :align: left + + Format v2 diff --git a/h5boss_py/docs/source/images/fits2fmt1.png b/h5boss_py/docs/source/images/fits2fmt1.png new file mode 100755 index 0000000..bf8240b Binary files /dev/null and b/h5boss_py/docs/source/images/fits2fmt1.png differ diff --git a/h5boss_py/docs/source/images/fmt1.png b/h5boss_py/docs/source/images/fmt1.png new file mode 100755 index 0000000..0cf3254 Binary files /dev/null and b/h5boss_py/docs/source/images/fmt1.png differ diff --git a/h5boss_py/docs/source/images/fmt2.png b/h5boss_py/docs/source/images/fmt2.png new file mode 100755 index 0000000..f9d3596 Binary files /dev/null and b/h5boss_py/docs/source/images/fmt2.png differ diff --git a/h5boss_py/docs/source/index.rst b/h5boss_py/docs/source/index.rst new file mode 100755 index 0000000..d230bed --- /dev/null +++ b/h5boss_py/docs/source/index.rst @@ -0,0 +1,54 @@ +.. h5boss documentation master file, created by + sphinx-quickstart on Tue Oct 11 12:04:08 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome! +======== + +H5Boss is an exploratory python tool for managing BOSS spectra data, SDSS-II. + +Boss is originally maintained as millions of fits file in thousands of different folders. Accessing and analyzing them are inefficient in terms of I/O bandwidth and programming productivity. In h5boss, we developed functions to support: + +* Reformatting: Preserve the fits file structure and spectrum hierarchicy using HDF5 +* Subsetting: Support subset/add/update operation, to extract selected 'plate/mjd/fiber' and save in one HDF5 file + +Currently, h5boss is implemented in both python and c version, in which the python version is actively maintained and supported. The c version is mainly for I/O sensitive users/applications. + +Quickstart: +=========== + +* :ref:`Installation ` + +Formats: +======== + +* :ref:`Fits->Hdf5 ` +* :ref:`Design ` + +Commands: +============= + +* :ref:`Convert ` +* :ref:`Subset ` +* :ref:`Add ` +* :ref:`Update ` + + +Modules: +======== + +* :ref:`H5boss ` + +Missing Features: +================= + +* :ref:`H5bossdev ` + +Indices: +======== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/h5boss_py/docs/source/install.rst b/h5boss_py/docs/source/install.rst new file mode 100755 index 0000000..18cfc5b --- /dev/null +++ b/h5boss_py/docs/source/install.rst @@ -0,0 +1,16 @@ +.. _install: + +Installation +================= + +Download:: + + git clone https://github.com/valiantljk/h5boss.git + +Configure:: + + cd h5boss/ + export PYTHONPATH=`pwd`/h5boss_py:$PYTHONPATH + export PATH=`pwd`/h5boss_py/script:$PATH + module load python/2.7-anaconda (on Edison/Cori, You need astropy, etc) + diff --git a/h5boss_py/docs/source/modules.rst b/h5boss_py/docs/source/modules.rst new file mode 100755 index 0000000..628274c --- /dev/null +++ b/h5boss_py/docs/source/modules.rst @@ -0,0 +1,9 @@ +.. _modules: + +H5Boss +====== + +.. toctree:: + :maxdepth: 4 + + h5boss diff --git a/h5boss_py/docs/source/practice.rst b/h5boss_py/docs/source/practice.rst new file mode 100755 index 0000000..1239f4a --- /dev/null +++ b/h5boss_py/docs/source/practice.rst @@ -0,0 +1,7 @@ +.. _practice: +Practice +======== + + + + diff --git a/h5boss_py/docs/source/psubset.rst b/h5boss_py/docs/source/psubset.rst new file mode 100755 index 0000000..55ca9c2 --- /dev/null +++ b/h5boss_py/docs/source/psubset.rst @@ -0,0 +1,69 @@ +.. _psubset: +Parallel Subset +=============== + +Usage: +------ +.. highlight:: c + +usage:: + + subset_mpi [-h] [--template TEMPLATE] [--mpi MPI] [--fiber FIBER] + + [--catalog CATALOG] [--datamap DATAMAP] + + input output pmf + +positional arguments:: + + input HDF5 input list + + output HDF5 output + + pmf Plate/mjd/fiber list + +optional arguments:: + + -h, --help show this help message and exit + + --template TEMPLATE Create template only,yes/no/all + + --mpi MPI using mpi yes/no + + --fiber FIBER specify fiber csv output + + --catalog CATALOG specify catalog csv output + + --datamap DATAMAP specify datamap pickle file + + +Example: +-------- +.. highlight:: c + +prepare input:: + + >cat input_sample.txt + + /global/cscratch1/sd/jialin/h5boss/3665-55247.hdf5 + + ... + + >cat pmf_sample.txt + + plates mjds fibers + + 3665 55247 65 + + 3665 55247 390 + + ... + +download: :download:`input <_static/input_sample.txt>`, :download:`pmf <_static/pmf_sample.txt>` + +execute command:: + + >subset_mpi input_sample.txt output.h5 pmf_sample.txt + + + diff --git a/h5boss_py/docs/source/subset.rst b/h5boss_py/docs/source/subset.rst new file mode 100755 index 0000000..868314c --- /dev/null +++ b/h5boss_py/docs/source/subset.rst @@ -0,0 +1,77 @@ +.. _subset: +Subset +======== +The `subset` function will find the specified plates/mjds/fibers from source files and copy into a single shared output. The source files are the BOSS data in HDF5 format. The user needs to specify the query as `plate mjd fiber`. The output file has a same structure with the source files. + +Usage: +------ +.. highlight:: c + + +usage:: + + subset [-h] input output pmf + +positional arguments:: + + input HDF5 input list + + output HDF5 output + + pmf Plate/mjd/fiber list in csv + +optional arguments:: + + -h, --help show this help message and exit + +Example: +-------- +.. highlight:: c + +Prepare input:: + + cat input_sample.txt + + /global/cscratch1/sd/jialin/h5boss/3665-55247.hdf5 + + ... + + cat pmf_sample.txt + + plates mjds fibers + + 4562 55570 622 + + 4479 55592 543 + + 7294 56739 242 + + ... + +Download: :download:`input_sample.txt <_static/input_sample.txt>`, :download:`pmf_sample.txt <_static/pmf_sample.txt>` + +Execute command:: + + subset input_sample.txt output.h5 pmf_sample.txt + +Output:: + + Output: output.h5 + Running selection... + -Source file open: 2.04 + -Fiber query time: 0.00 + -Fiber copy time: 11.35 + -Catalog copy time: 6.70 + -Group create time: 0.01 + -File close time: 0.04 + Selection Time: 22.34 seconds + Done selection + Total selection Cost 22.34 + + +For detailed performance evaluation, please read our Techincal Report(To be released by Nov.1 2016). + +High Performance Parallel Version: +------------------------ +* :ref:`mpi4py ` +* :ref:`C-MPI ` diff --git a/h5boss_py/docs/source/subset_mpi_v1.rst b/h5boss_py/docs/source/subset_mpi_v1.rst new file mode 100755 index 0000000..966146f --- /dev/null +++ b/h5boss_py/docs/source/subset_mpi_v1.rst @@ -0,0 +1,7 @@ +subset_mpi_v1 module +==================== + +.. automodule:: subset_mpi_v1 + :members: + :undoc-members: + :show-inheritance: diff --git a/h5boss_py/docs/source/subset_mpi_v2.rst b/h5boss_py/docs/source/subset_mpi_v2.rst new file mode 100755 index 0000000..45322f1 --- /dev/null +++ b/h5boss_py/docs/source/subset_mpi_v2.rst @@ -0,0 +1,7 @@ +subset_mpi_v2 module +==================== + +.. automodule:: subset_mpi_v2 + :members: + :undoc-members: + :show-inheritance: diff --git a/h5boss_py/docs/source/update.rst b/h5boss_py/docs/source/update.rst new file mode 100755 index 0000000..3d3ecb5 --- /dev/null +++ b/h5boss_py/docs/source/update.rst @@ -0,0 +1,86 @@ +.. _update: +Update +======== +Given a plate/mjd/fiber query list. This `update` command will update the base hdf5 accordingly. Basically, `update` will + * Compare the existing plate/mjd/fibers in the base hdf5 file with this query, + * Search the new plate/mjd/fiber, which are not existed in the base hdf5 file, from the input hdf5 files + * Add the founded plate/mjd/fiber into the base file + * Remove the existing plate/mjd/fiber from the base file, which are not found in the query list. + * [optional] Repacking the final output to make the file contiguous on storage + +Usage: +------ +.. highlight:: c + +update -h + +usage:: + + update [-h] base input pmf + +positional arguments:: + + base Pre-subseted HDF5 file + + input HDF5 input files list + + pmf Plate/mjd/fiber list + +optional arguments:: + + -h, --help show this help message and exit + --repack, REPACK repack after changing the file, yes or no +Example: +-------- +.. highlight:: c + +Prepare input:: + + cat input_sample.txt + + /global/cscratch1/sd/jialin/h5boss/3665-55247.hdf5 + + ... + + cat pmf_update.txt + + plates mjds fibers + + 6697 56419 796 + + 4697 55660 190 + + 4191 55444 636 + + ... + +Download: :download:`input_sample.txt <_static/input_sample.txt>`, :download:`pmf_update.txt <_static/pmf_update.txt>`, :download:`base.h5 <_static/base.h5>` + +Execute command:: + + update base.h5 input_sample.txt pmf_update.txt --repack=yes + +Output:: + + Will repack the file for better storage layout afterwards. + Query: Plates/Mjds/Fibers: 11 + Input: 2393 hdf5 files + Output: base.h5 + Running Updating: + Query time: 0.00 seconds + plates/mjds/fibers to be added: 10 + Running selection: + -Source file open: 1.93 + -Fiber query time: 0.00 + -Fiber copy time: 0.20 + -Catalog copy time: 0.83 + -Group create time: 0.00 + -File close time: 0.04 + Selection Time: 5.15 seconds + plates/mjds/fibers to be removed: 6 + Running removing: + Removed 6 plates/mjds/fibers,Skipped 0 + Remvoing Time: 0.02 seconds + Running repacking: + Repacking Time: 2.33 seconds + Updating complete: 8.40 seconds diff --git a/h5boss/__init__.py b/h5boss_py/h5boss/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from h5boss/__init__.py rename to h5boss_py/h5boss/__init__.py diff --git a/h5boss_py/h5boss/boss2hdf5_v1.py b/h5boss_py/h5boss/boss2hdf5_v1.py new file mode 100755 index 0000000..eb544cf --- /dev/null +++ b/h5boss_py/h5boss/boss2hdf5_v1.py @@ -0,0 +1,109 @@ +import sys, os +import numpy as np +from astropy.io import fits +from astropy.table import Table +import h5boss.io +import time + +def serial_convert(platefile,hdf5output): + platefile=platefile[0] + if not os.path.isfile(platefile): + print("%s not existing"%platefile) + sys.exit(0) + hdf5output=str(hdf5output) + print ("Output:%s"%hdf5output) + print ("Running Conversion:") + filedir = os.path.split(os.path.abspath(platefile))[0] + hdr = fits.getheader(platefile) + plate = hdr['PLATEID'] + mjd = hdr['MJD'] + tstart=time.time() + #--- Plugmap --- + print('plugmap') + plugmap = Table.read(platefile, 5) + dataname = '{}/{}/plugmap'.format(plate, mjd) + plugmap.write(hdf5output, path=dataname, append=True) + + #--- zbest --- + print('zbest') + run1d = hdr['RUN2D'] #- default run1d == run2d + zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d)) + zbest = Table.read(zbestfile, 1) + dataname = '{}/{}/zbest'.format(plate, mjd) + zbest.write(hdf5output, path=dataname, append=True) + nfiber = len(zbest) + + #--- zall (skip) --- + pass + + #--- zline --- + print('zline') + zlinefile = zbestfile.replace('spZbest-', 'spZline-') + zline = Table.read(zlinefile, 1) + dataname = '{}/{}/zline'.format(plate, mjd) + zline.write(hdf5output, path=dataname, append=True) + + #--- photometric matches --- + print('photo') + photomatchfile = platefile.replace('spPlate-', 'photoMatchPlate-') + photomatch = Table.read(photomatchfile, 1) + photomatch['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) + dataname = '{}/{}/photo/match'.format(plate, mjd) + photomatch.write(hdf5output, path=dataname, append=True) + + photoposfile = platefile.replace('spPlate-', 'photoPosPlate-') + photopos = Table.read(photoposfile, 1) + photopos['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) + dataname = '{}/{}/photo/matchpos'.format(plate, mjd) + photopos.write(hdf5output, path=dataname, append=True) + + photofluxfile = platefile.replace('spPlate-', 'photoPlate-') + photoflux = Table.read(photofluxfile, 1) + photoflux['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) + dataname = '{}/{}/photo/matchflux'.format(plate, mjd) + photoflux.write(hdf5output, path=dataname, append=True) + + #--- Coadd --- + print('loading coadds') + coadds = h5boss.io.load_coadds(platefile) + + print('writing coadds') + for i, cx in enumerate(coadds): + dataname = '{}/{}/{}/coadd'.format(plate, mjd, i+1) + cx.write(hdf5output, path=dataname, append=True) +# h5boss.io.write_coadds_vstack(platefile, plate,mjd,hdf5output) + + #--- Individual exposures --- + #- Parse spPlancomb to get exposures that were used + print('parsing planfile') + planfile = platefile.replace('spPlate-', 'spPlancomb-').replace('.fits', '.par') + framefiles = list() + for line in open(planfile): + if line.startswith('SPEXP '): + tmp = line.split() + tmp = [x+'.gz' for x in tmp[7:-1]] + framefiles.extend(tmp) + + print('copying individual exposure') + for filename in framefiles: + print(filename) + frame = h5boss.io.load_frame(filedir+'/'+filename) + if ('spFrame-b1' in filename) or ('spFrame-r1' in filename): + offset = 0 + elif ('spFrame-b2' in filename) or ('spFrame-r2' in filename): + offset = 500 + else: + print('huh?', filename) + sys.exit(1) + + for i, fx in enumerate(frame): + br = fx.meta['CAMERAS'][0] + expid = fx.meta['EXPOSURE'] + fiber = offset+i+1 + dataname = '{}/{}/{}/exposures/{}/{}'.format(plate, mjd, fiber, expid, br) + fx.write(hdf5output, path=dataname, append=True) +# print('writing exposures') +# h5boss.io.write_frame_vstack(filedir,framefiles,plate,mjd,hdf5output) + + tend=time.time()-tstart + print ('Conversion time: %.2f seconds'%tend) diff --git a/h5boss_py/h5boss/boss2hdf5_v2.py b/h5boss_py/h5boss/boss2hdf5_v2.py new file mode 100755 index 0000000..ce8d599 --- /dev/null +++ b/h5boss_py/h5boss/boss2hdf5_v2.py @@ -0,0 +1,87 @@ +import sys, os +import numpy as np +from astropy.io import fits +from astropy.table import Table +import h5boss.io +import time + +def serial_convert(platefile,hdf5output): + platefile=platefile[0] + if not os.path.isfile(platefile): + print("%s not existing"%platefile) + sys.exit(0) + hdf5output=str(hdf5output) + print ("Output: %s"%hdf5output) + print ("Running Conversion:") + filedir = os.path.split(os.path.abspath(platefile))[0] + hdr = fits.getheader(platefile) + plate = hdr['PLATEID'] + mjd = hdr['MJD'] + tstart=time.time() + #--- Plugmap --- + print('plugmap') + plugmap = Table.read(platefile, 5) + dataname = '{}/{}/plugmap'.format(plate, mjd) + plugmap.write(hdf5output, path=dataname, append=True) + + #--- zbest --- + print('zbest') + run1d = hdr['RUN2D'] #- default run1d == run2d + zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d)) + zbest = Table.read(zbestfile, 1) + dataname = '{}/{}/zbest'.format(plate, mjd) + zbest.write(hdf5output, path=dataname, append=True) + nfiber = len(zbest) + + #--- zall (skip) --- + pass + + #--- zline --- + print('zline') + zlinefile = zbestfile.replace('spZbest-', 'spZline-') + zline = Table.read(zlinefile, 1) + dataname = '{}/{}/zline'.format(plate, mjd) + zline.write(hdf5output, path=dataname, append=True) + + #--- photometric matches --- + print('photo') + photomatchfile = platefile.replace('spPlate-', 'photoMatchPlate-') + photomatch = Table.read(photomatchfile, 1) + photomatch['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) + dataname = '{}/{}/photo/match'.format(plate, mjd) + photomatch.write(hdf5output, path=dataname, append=True) + + photoposfile = platefile.replace('spPlate-', 'photoPosPlate-') + photopos = Table.read(photoposfile, 1) + photopos['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) + dataname = '{}/{}/photo/matchpos'.format(plate, mjd) + photopos.write(hdf5output, path=dataname, append=True) + + photofluxfile = platefile.replace('spPlate-', 'photoPlate-') + photoflux = Table.read(photofluxfile, 1) + photoflux['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) + dataname = '{}/{}/photo/matchflux'.format(plate, mjd) + photoflux.write(hdf5output, path=dataname, append=True) + + #--- Coadd --- + print('loading coadds') + + print('writing coadds') + h5boss.io.write_coadds_vstack(platefile, plate,mjd,hdf5output) + + #--- Individual exposures --- + #- Parse spPlancomb to get exposures that were used + print('parsing planfile') + planfile = platefile.replace('spPlate-', 'spPlancomb-').replace('.fits', '.par') + framefiles = list() + for line in open(planfile): + if line.startswith('SPEXP '): + tmp = line.split() + tmp = [x+'.gz' for x in tmp[7:-1]] + framefiles.extend(tmp) + + print('copying individual exposure') + h5boss.io.write_frame_vstack(filedir,framefiles,plate,mjd,hdf5output) + + tend=time.time()-tstart + print ('Conversion time: %.2f seconds'%tend) diff --git a/h5boss_py/h5boss/h5map.py b/h5boss_py/h5boss/h5map.py new file mode 100755 index 0000000..ad83082 --- /dev/null +++ b/h5boss_py/h5boss/h5map.py @@ -0,0 +1,147 @@ +import numpy as np +import h5py +import time +import sys +import os +import csv +import traceback +from collections import defaultdict +fx="" +pid="" +inputfile="" +fiberdatalink={} +cataloglink={} +meta=['plugmap', 'zbest', 'zline', + 'photo/match', 'photo/matchflux', 'photo/matchpos'] + +def _traverse_fibernode(name): + ''' + para : node name in a hdf5 group + purpose: Find a dataset node, which should be an endpoint in its group hierarchy +unique_datsetpath datasettype datasetshape filepath plate mjd fiber + return : (key,value)->(path_to_dataset, (dataset type, shape, filename, plate, mjd, fiber)) + ''' + global fx,pid,fiberdatalink,inputfile + try: + cur_node=name.encode('ascii','ignore') + node=pid+'/'+cur_node + p=pid.split('/')[0] + m=pid.split('/')[1] + f=pid.split('/')[2] + node_t=str(type(fx[node])) + if 'dataset' in node_t: + node_t=fx[node].dtype + node_sp=fx[node].shape + fiberdatalink[node]=(node_t,node_sp,inputfile,p,m,f) + except Exception as e: + traceback.print_exc() + pass +#node_type is used in ../script/subset_mpi.py, which is to create single shared file +def map_fiber(infile): + ''' + para : filename, plate, mjd, fiber + return: (key, value)->(datapath, (dtype, shape, filename, plate, mjd, fiber)) + python dict's updating can ensure that the key is unique, i.e., datapath: plate/mjd/fiber/../dataset is unique + ''' + global pid,fiberdatalink, cataloglink, fx, inputfile + inputfile=infile + try: + fx = h5py.File(infile, mode='r') + for plate in fx.keys(): + for mjd in fx[plate].keys(): + spid= '{}/{}'.format(plate, mjd) + for fib in fx[spid].keys(): + if (fib.isdigit()): + pid = '{}/{}/{}'.format(plate, mjd, fib) + fx[pid].visit(_traverse_fibernode) + #for im in meta: + # mnode=spid+'/'+im + # mnode_t=fx[mnode].dtype + # mnode_sp=fx[mnode].shape + # fiberdatalink[mnode]=(mnode_t,mnode_sp,infile) + fx.close() + except Exception as e: + print (pid) + traceback.print_exc() + print (pid,infile) + pass + return (fiberdatalink) +def map_pmf(infile): + ''' + para : filename, plate, mjd, fiber + return: (key, value)->(plates/mjd/fiber/, filename) + python dict's updating can ensure that the key is unique, i.e., plate/mjd/fiber/../dataset is unique + ''' + pmf={} + try: + fx = h5py.File(infile, mode='r') + for plate in fx.keys(): + for mjd in fx[plate].keys(): + spid= '{}/{}'.format(plate, mjd) + for fib in fx[spid].keys(): + if fib.isdigit(): + pid = '{}/{}/{}'.format(plate, mjd, fib) + pmf[pid]=infile + fx.close() + except Exception as e: + print (pid) + traceback.print_exc() + print (pid,infile) + pass + return (pmf) +def type_map(infile): + ''' + para : filename + return: type and shape for each object, returned as tuple, dmap[0] is coadds, dmap[1] is exposure + plate/mjd/coadds: 8 datasets + plate/mjd/exposure/exposureid/b(r): 8 datasets + # (key, value)->(plates/mjd/, (filename, fiberlist, fiberoffsetlist)) + ''' + coadds_map={} + exposures_map={} + with h5py.File(infile,'r') as fx: + try: + p=fx.keys()[0] + m=fx[p].keys()[0] + pm=p+'/'+m + coad_name=pm+'/coadds' + expo_name=pm+'/exposures' + coad=fx[coad_name].keys() + subexpo_name=expo_name+'/'+fx[expo_name].keys()[0]+'/b' + expo=fx[subexpo_name].keys() + for icoad in coad: + try: + icoad_name=coad_name+'/'+icoad + coadds_map[icoad]=(fx[icoad_name].dtype,fx[icoad_name].shape) + except Exception as e: + print (icoad) + for iexpo in expo: + try: + iexpo_name=subexpo_name+'/'+iexpo + exposures_map[iexpo]=(fx[iexpo_name].dtype,fx[iexpo_name].shape) + except Exception as e: + print (iexpo) + except Exception as e: + print (infile) + traceback.print_exc() + dmap=(coadds_map,exposures_map) + return dmap + +def coadd_map(fname_list): + coadmap={} + for ifile in fname_list: + try: + f=h5py.File(ifile,'r') + p=f.keys()[0] + m=f[p].keys()[0] + pm=p+'/'+m + pmc=pm+'/coadds' + dsets=f[pmc].keys() + if dsets[0]!='wave': + dsize=f[pmc+'/'+dsets[0]].shape[1] + else: + dsize=f[pmc+'/'+dsets[1]].shape[0] + coadmap[pm]=dsize + except Exception as e: + pass + return coadmap diff --git a/h5boss_py/h5boss/io.py b/h5boss_py/h5boss/io.py new file mode 100755 index 0000000..05e2524 --- /dev/null +++ b/h5boss_py/h5boss/io.py @@ -0,0 +1,395 @@ +import traceback +import os.path +import numpy as np +from astropy.io import fits +from astropy.table import Table +import h5py +expdat=("wave","flux","ivar","mask","wavedisp","sky","x","calib") #exposures +dat=("wave","flux","ivar","and_mask","or_mask","wavedisp","sky","model") # wavelength +def load_coadds(platefile, zbestfile=None, run1d=None): + ''' + Document ... + ''' + #- Load spPlate data + fx = fits.open(platefile, memmap=False) + header = fx[0].header + c0 = header['COEFF0'] + c1 = header['COEFF1'] + nwave = header['NAXIS1'] + nfiber = header['NAXIS2'] + wave = (10**(c0 + c1*np.arange(nwave))).astype(np.float32) + flux = fx[0].data + ivar = fx[1].data + and_mask = fx[2].data + or_mask = fx[3].data + wavedisp = fx[4].data + sky = fx[6].data + fx.close() + + if run1d is None: + run1d = header['RUN2D'] #- default run1d == run2d + + #- Get best fit model from zbest file + if zbestfile is None: + zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d)) + + model = fits.getdata(zbestfile, 2) + + coadds = list() + for i in range(nfiber): + sp = Table() + sp['WAVE'] = wave #- repeat ! + sp['FLUX'] = flux[i] + sp['IVAR'] = ivar[i] + sp['AND_MASK'] = and_mask[i] + sp['OR_MASK'] = or_mask[i] + sp['WAVEDISP'] = wavedisp[i] + sp['SKY'] = sky[i] + sp['MODEL'] = model[i] + sp.meta = header + + #- TODO: Add units, comments to each column + + coadds.append(sp) + + return coadds + +def load_frame(framefile, cframefile=None, flatfile=None): + """ + Document ... + """ + if cframefile is None: + cframefile = framefile.replace('spFrame', 'spCFrame') + if cframefile.endswith('.gz'): + cframefile = cframefile[:-3] + import os.path + if os.path.exists(framefile)==False: + print("v1,framefile:%s not exist"%framefile) + exit() + #- Load framefile and get original dimensions + eflux = fits.getdata(framefile, 0) + nfiber, npix = eflux.shape + if os.path.exists(cframefile)==False: + print("v1,cframefile:%s not exist"%cframefile) + #exit() + spectra=list() + return spectra + #- Load spCFrame file; trim arrays back to original size + fx = fits.open(cframefile, memmap=False) + header = fx[0].header + flux = fx[0].data[:, 0:npix] + ivar = fx[1].data[:, 0:npix] + mask = fx[2].data[:, 0:npix] + wave = (10**fx[3].data[:, 0:npix]).astype(np.float32) + wavedisp = fx[4].data[:, 0:npix] + sky = fx[6].data[:, 0:npix] + x = fx[7].data[:, 0:npix] + superflat = fx[8].data[:, 0:npix] + + #- Load fiberflat spFlat[0] + if flatfile is None: + flatfile = header['FLATFILE'].replace('sdR', 'spFlat') + flatfile = flatfile.replace('.fit', '.fits.gz') + filedir, basename = os.path.split(os.path.abspath(cframefile)) + flatfile = os.path.join(filedir, flatfile) + + if os.path.exists(flatfile)==False: + #exit() + spectra=list() + return spectra + fiberflat = fits.getdata(flatfile, 0) + + #- Calculate calibration vector: flux = electrons * calib + electrons = eflux * fiberflat * superflat + ii = np.where(electrons != 0.0) + calib = np.zeros(flux.shape) + calib[ii] = flux[ii] / electrons[ii] + + fx.close() + + #- Assemble spectra tables + spectra = list() + for i in range(nfiber): + sp = Table() + sp['WAVE'] = wave[i] + sp['FLUX'] = flux[i] + sp['IVAR'] = ivar[i] + sp['MASK'] = mask[i] + sp['WAVEDISP'] = wavedisp[i] + sp['SKY'] = sky[i] + sp['X'] = x[i] + sp['CALIB'] = calib[i].astype(np.float32) + sp.meta = header + + #- TODO: Add units, comments to each column + + spectra.append(sp) + + return spectra + + +##Newly Added functions for converting into the stacked format. +def load_coadds_vstack(platefile,zbestfile=None, run1d=None): + ''' + Document ... + ''' + + #- Load spPlate data + fx = fits.open(platefile, memmap=False) + header = fx[0].header + c0 = header['COEFF0'] + c1 = header['COEFF1'] + nwave = header['NAXIS1'] + nfiber = header['NAXIS2'] + wave = (10**(c0 + c1*np.arange(nwave))).astype(np.float32) + flux = fx[0].data + ivar = fx[1].data + and_mask = fx[2].data + or_mask = fx[3].data + wavedisp = fx[4].data + sky = fx[6].data + fx.close() + run1d = header['RUN2D'] #- default run1d == run2d + + #- Get best fit model from zbest file + zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d)) + model = fits.getdata(zbestfile, 2) + coadds=(wave,flux,ivar,and_mask,or_mask,wavedisp,sky,model,header) + return coadds +#copy each wavelength dataset into the hdf5 file +def write_coadds_vstack(platefile, plate,mjd,hdf5output,zbestfile=None, run1d=None): + coadds=load_coadds_vstack(platefile, zbestfile=None, run1d=None) + global dat + outx=h5py.File(hdf5output,'a') + for i in range(0,8): + id = '{}/{}/coadds/{}'.format(plate, mjd, dat[i]) + print(id) + try: + dx=outx.create_dataset(id,data=coadds[i]) + except Exception as e: + pass + #temptb.write(hdf5output,id,append=True) #TODO: add table.meta to hdf5 attributes + #dx.attrs.__setitem__(dat[i], coadds[8]) + #print(outx['7094/56660'].keys()) + outx.close() +#sections in load exposures +def load_frame_vstack(framefile,cframefile=None,flatfile=None): + """ + Document ... + """ + if cframefile is None: + cframefile = framefile.replace('spFrame', 'spCFrame') + if cframefile.endswith('.gz'): + cframefile = cframefile[:-3] + import os.path + if os.path.exists(framefile)==False: + print ("v2,framefile:%s not exist"%(framefile)) + exposure=tuple() + return exposure + #exit() + #- Load framefile and get original dimensions + eflux = fits.getdata(framefile, 0) + nfiber, npix = eflux.shape + if os.path.exists(cframefile)==False: + print ("v2,cframefile:%s not exist"%(cframefile)) + exposure=tuple() + return exposure + #exit() + #- Load spCFrame file; trim arrays back to original size + fx = fits.open(cframefile, memmap=False) + header = fx[0].header # this means the exposureid is same for flux, ivar, etc + expid=header['EXPOSURE'] + flux = fx[0].data[:, 0:npix] + ivar = fx[1].data[:, 0:npix] + mask = fx[2].data[:, 0:npix] + wave = (10**fx[3].data[:, 0:npix]).astype(np.float32) + wavedisp = fx[4].data[:, 0:npix] + sky = fx[6].data[:, 0:npix] + x = fx[7].data[:, 0:npix] + superflat = fx[8].data[:, 0:npix] + + #- Load fiberflat spFlat[0] + if flatfile is None: + flatfile = header['FLATFILE'].replace('sdR', 'spFlat') + flatfile = flatfile.replace('.fit', '.fits.gz') + filedir, basename = os.path.split(os.path.abspath(cframefile)) + flatfile = os.path.join(filedir, flatfile) + + if os.path.exists(flatfile)==False: + print("%s not exist"%(flatfile)) + exit() + fiberflat = fits.getdata(flatfile, 0) + + #- Calculate calibration vector: flux = electrons * calib + electrons = eflux * fiberflat * superflat + ii = np.where(electrons != 0.0) + calib = np.zeros(flux.shape) + calib[ii] = flux[ii] / electrons[ii] + + fx.close() + exposure=(wave,flux,ivar,mask,wavedisp,sky,x,calib.astype(np.float32), header,expid) + return exposure + +def combine_write_frame(frame1,frame2,expid,plate,mjd,hdf5output,br): + #print ('dump:',expid) + global expdat + try: + outx=h5py.File(hdf5output,'a') + #print(outx) + except Exception as e: + print ("file open error") + traceback.print_exc() + pass + #print("len expdat:%d"%(len(expdat))) + for i in range(0,len(expdat)): + id = '{}/{}/exposures/{}/{}/{}'.format(plate, mjd, expid,br,expdat[i]) + #print (id) + try: + dset=np.append(frame1[i],frame2[i],axis=0) + #print ("id:%s"%id) + #print ("shape:",dset.shape) + except Exception as e: + print ("append error") + trackback.print_exc() + pass + try: + dx=outx.create_dataset(id,data=dset) + #print(dx) + except Exception as e: + print ('error in frame dump') + trackback.print_exc() + pass + #outx.flush() + try: + print("done writing exposure %d, br:%s"%(expid,br)) + outx.close() + except Exception as e: + print ("file close error") + traceback.print_exc() +def write_frame(frame1,expid,plate,mjd,hdf5output,br): + #print ("single dump:",expid) + global expdat + try: + outx=h5py.File(hdf5output,'a') + print (outx) + except Exception as e: + print ("file open error") + traceback.print_exc() + pass + print("len expdat:%d"%(len(expdat))) + for i in range(0,len(expdat)): + id = '{}/{}/exposures/{}/{}/{}'.format(plate, mjd, expid,br,expdat[i]) + #print(id) + dset=frame1[i] + try: + #print ("creating dataset in frames") + dx=outx.create_dataset(id,data=dset) + #print(dx) + except Exception as e: + print ('error in frame dump') + trackback.print_exc() + pass + try: + print("done writing exposure %d, br:%s"%(expid,br)) + outx.close() + except Exception as e: + print ("file close error") + trackback.print_exc() +def write_frame_vstack(filedir,framefiles,plate,mjd,hdf5output, cframefile=None, flatfile=None): + frameb1=list() + framer1=list() + frameb2=list() + framer2=list() + ii=0 + #print("length of framefiles:%d"%(len(framefiles))) + for filename in framefiles: + #print ("iteration:%d"%(ii)) + ii=ii+1 + offset = 0 #fiber 0-499 + if ('spFrame-b1' in filename): + try: + frame=load_frame_vstack(filedir+'/'+filename) + if len(frame)!=0: + frameb1.append(frame) + #print (filename, frame[9]) + except Exception as e: + print ("File not found") + traceback.print_exc() + pass + if ('spFrame-r1' in filename): + try: + frame=load_frame_vstack(filedir+'/'+filename) + if len(frame)!=0: + framer1.append(frame) + #print (filename, frame[9]) + except Exception as e: + print ("File not found") + traceback.print_exc() + pass + print ("finish loading b1, r1") + for filename in framefiles: + offset = 500 #fiber 500-999 + if ('spFrame-b2' in filename): + try: + frame=load_frame_vstack(filedir+'/'+filename) + if len(frame)!=0: + frameb2.append(frame) + #print (filename, frame[9]) + except Exception as e: + print ("File not found") + pass + if ('spFrame-r2' in filename): + try: + frame=load_frame_vstack(filedir+'/'+filename) + if len(frame)!=0: + framer2.append(frame) + #print (filename, frame[9]) + except Exception as e: + print ("File not found") + pass + print("finish loading b2,r2") + #combine b1 and b2 + print ("frameb1:%d"%(len(frameb1))) + for i in range(0,len(frameb1)): + expidb1=frameb1[i][9] + #print("expidb1:%d"%expidb1) + hit=0 + for j in range(0,len(frameb2)): + expidb2=frameb2[j][9] + #print("expidb2:%d"%expidb2) + if expidb1==expidb2: + #combine frameb1 and frameb2 + combine_write_frame(frameb1[i],frameb2[j],expidb1,plate,mjd,hdf5output,'b') + try: + frameb2.remove(frameb2[j]) + except Exception as e: + traceback.print_exc() + hit=1 + break + if hit==0: + #print ("hit is 0") + write_frame(frameb1[i],expidb1,plate,mjd,hdf5output,'b') + for i in range(0,len(frameb2)): + #print("leftover frameb2:%d"%(frameb2)) + expidb2=frameb2[i][9] + write_frame(frameb2[i],expidb2,plate,mjd,hdf5output,'b') + + #combine r1 and r2 + print ("framer1:%d"%(len(framer1))) + for i in range(0,len(framer1)): + expidr1=framer1[i][9] + hit=0 + for j in range(0,len(framer2)): + expidr2=framer2[j][9] + if expidr1==expidr2: + #combine frameb1 and frameb2 + combine_write_frame(framer1[i],framer2[j],expidr1,plate,mjd,hdf5output,'r') + framer2.remove(framer2[j]) + hit=1 + break + if hit==0: + write_frame(framer1[i],expidr1,plate,mjd,hdf5output,'r') + for i in range(0,len(framer2)): + #print("leftover:%d"%(len(framer2))) + expidr2=framer2[i][9] + write_frame(framer2[i],expidr2,plate,mjd,hdf5output,'r') diff --git a/h5boss_py/h5boss/remove.py b/h5boss_py/h5boss/remove.py new file mode 100755 index 0000000..e3bd604 --- /dev/null +++ b/h5boss_py/h5boss/remove.py @@ -0,0 +1,55 @@ +import numpy as np +import h5py +import time +import sys,os +import commands +from h5boss.select import select +def remove(infile, plates, mjds, fibers,repack=None): + ''' + remove the additional (plates, mjds, fibers) from the pre-existing file + + Args: + infile : pre-existing subset file + plates : list of plates + mjds : list of plates + fibers : list of fibers + ''' + meta=['plugmap', 'zbest', 'zline', + 'photo/match', 'photo/matchflux', 'photo/matchpos'] + tstart=time.time() + try: + fx=h5py.File(infile,'a') + except Exception, e: + print ('File open error in removing') + try: + k=0 + j=0 + for i in range(0,len(plates)): + a=str(plates[i])+'/'+str(mjds[i])+'/'+str(fibers[i]) + if a in fx: + fx.__delitem__(a) + k+=1 + else: + j+=1 + #TODO:remove entries in photo, plugmap, etc + print ('Removed %d plates/mjds/fibers,Skipped %d'%(k,j)) + fx.close() + except Exception, e: + traceback.print_exc() + print ('Error in removing') + tend=time.time()-tstart + print ("Remvoing Time: %.2f seconds"%tend) + if repack=="yes": + print ("Running repacking:") + repack_start=time.time() + #run hdf5 repack utility from + cmd_moduleload = "module load cray-hdf5 >/dev/null" + oufile=infile.split('.')[0]+"_repack"+".h5" + cmd_repack = "h5repack %s %s 2>/dev/null"%(infile,oufile) + try: + commands.getstatusoutput(cmd_moduleload) + commands.getstatusoutput(cmd_repack) + except Exception,e: + print ('Repack error') + repack_end=time.time()-repack_start + print ("Repacking Time: %.2f seconds"%repack_end) diff --git a/h5boss_py/h5boss/select.py b/h5boss_py/h5boss/select.py new file mode 100755 index 0000000..9822682 --- /dev/null +++ b/h5boss_py/h5boss/select.py @@ -0,0 +1,209 @@ +import numpy as np +import h5py +import time +import traceback +from os.path import basename +import os,sys +''' + select: Query a list of (plates,mjds,fibers) from source files and produce a single shared output + sub_select: call select() but produce one output per (plate, mjd,fiber) query, + all those subfiles will be managed by a master file + kv_nodetype: return a key,value list, where key is the dataset link, and value is its type. + traverse_node: used in kv_nodetype for traversing the hdf5 group hierarchicy and reach the endpoint, i.e., dataset +''' + +fx="" +fiberdatalink={} +pid="" +cata_create=0.0 +cata_resize=0.0 +get_cata=0.0 +dest_cata_read=0.0 +src_cata_read=0.0 +meta=['plugmap', 'zbest', 'zline', 'photo/match', 'photo/matchflux', 'photo/matchpos'] + +def traverse_node(name): + global fx,pid,fiberdatalink + try: + cur_node=name.encode('ascii','ignore') + node=pid+'/'+cur_node + node_t=str(type(fx[node])) + if 'dataset' in node_t: + # this means we find a dataset node, which must be an endpoint in its group hierarchy, + #we don't need to record the group information, as the path to the dataset already contains path to the groups. + node_t=fx[node].dtype + fiberdatalink[node]=node_t + except Exception as e: + traceback.print_exc() + pass + +def kv_nodetype(infile,plates,mjds,fibers): + global pid,fiberdatalink,fx + try: + fx = h5py.File(infile, mode='r') + # (kev,value) dictionary for caching (plates/mjd/fiber/../dataset, dataset type) + # python dict's updating can ensure that the key is unique, i.e., plate/mjd/fiber/../dataset is unique + fiberlink={} + for plate in fx.keys(): + for mjd in fx[plate].keys(): + ii = (plates == plate) & (mjds == mjd) + xfibers = fibers[ii] + if np.any(ii): # fiber is found + for fiber in xfibers:#for each fiber node, recursively visit its members and record the + #fiberlink={id:infile} + pid = '{}/{}/{}'.format(plate, mjd, fiber) + fx[pid].visit(traverse_node) + fx.close() + except Exception as e: + print (pid) + traceback.print_exc() + print (pid,infile) + pass + return (fiberdatalink) +def sub_select(infile,plates,mjds,fibers,masterfile,rank,id): + master_dir=os.path.dirname(os.path.realpath(masterfile))+'/'+os.path.basename(masterfile).split('.')[0] + + slavefile=master_dir+'/'+str(rank)+'_'+str(id)+'.h5' + try: + select(infile, slavefile, plates, mjds, fibers) + except Exception as e: + print ("Error in slave file:%s") + traceback.print_exc() + +def fiber_copy(xfibers,fx,hx,parent_id, plate, mjd): +#Step 1: copy a fiber object, count the time: data_copy + for fiber in xfibers: + id = '{}/{}/{}'.format(plate, mjd, fiber) + if id not in hx: + try: + fx.copy(id, hx[parent_id]) + except Exception as e: + pass + +def catalog_copy(xfibers,fx,hx,plate,mjd): + global cata_create, meta,cata_resize,dest_cata_read,get_cata,src_cata_read + for name in meta: + id = '{}/{}/{}'.format(plate, mjd, name) + try: + get_cata_start=time.time() + yfib=xfibers.astype(np.int32) + # copy all catalog metadata + temp_cata_fiber_all=fx[id] + #temp_cata_fiber_col=fx[id]['FIBERID'] + src_cata_read+=time.time()-get_cata_start + #jj = np.in1d(temp_cata_fiber_col,yfib) + jj = np.in1d(temp_cata_fiber_all['FIBERID'],yfib) + get_cata+=time.time()-get_cata_start + cataid_start_time=time.time() + if id not in hx: + cata_create_start=time.time() + #dset=hx.create_dataset(id,maxshape=(None,),dtype=fx[id][jj].dtype,data=fx[id][jj]) + dset=hx.create_dataset(id,maxshape=(None,),dtype=temp_cata_fiber_all[jj].dtype,data=temp_cata_fiber_all[jj]) + cata_create+=time.time()-cata_create_start + else: + print("cata id in outputfile") + if fx[id][jj]['FIBERID'] not in hx[id]['FIBERID']: + print("fiberid in fx but not in output file") + cata_read_start=time.time() + dset=hx[id] + dest_cata_read+=time.time()-cata_read_start + cata_resize_start=time.time() + dset.resize(len(dset)+1) + dset[len(dset)-1]=fx[id][jj] + dset.close() + cata_resize+=cata_resize_start + except Exception as e: + print("catalog %s add error"%id) + traceback.print_exc() + pass + +def select(infiles, outfile, plates, mjds,fibers): +#def select(infiles, outfile, pmflist): + ''' + Select a set of (plates,mjds,fibers) from a set of input files + + Args: + infiles : list of input filenames, or single filename + outfile : output file to write the selection + plates : list of plates + mjds : list of plates + fibers : list of fibers + ''' + if not isinstance(infiles, (list, tuple)): + infiles = [infiles,] + assert (len(plates)!=0) + assert (len(plates)==len(mjds)) + assert (len(plates)==len(fibers)) + plates = np.asarray(plates) + mjds = np.asarray(mjds) + fibers = np.asarray(fibers) + global meta, cata_create, dest_cata_read, cata_resize, get_cata,src_cata_read + cata_copy=0.0 + fopentime=0.0 + data_copy=0.0 + group_create=0.0 + query_time=0.0 + tstart=time.time() + try: + hx = h5py.File(outfile,'a') + except Exception as e: + print ("output file open error") + traceback.print_exc() + sys.exit(0) + file_open=time.time()-tstart + for infile in infiles: + fopen_start=time.time() + try: + fx = h5py.File(infile, mode='r') + except Exception as e: + print ("File open error: ",infile) + continue + ttopen=time.time()-fopen_start + fopentime+=ttopen + for plate in fx.keys(): + tquery=time.time() + iplate = (plates == plate) + if not np.any(iplate): + continue + query_time+=time.time()-tquery + for mjd in fx[plate].keys(): + tquery=time.time() + ii = (plates == plate) & (mjds == mjd) + xfibers = fibers[ii] + parent_id='{}/{}'.format(plate, mjd) + query_time+=time.time()-tquery + if not np.any(ii): + continue + else: + #create new group in output file + group_start=time.time() + if parent_id not in hx: + hx.create_group(parent_id) + group_create+=time.time()-group_start + #Step 1: copy a fiber object, count the time: data_copy + data_copy_start=time.time() + fiber_copy(xfibers,fx,hx,parent_id, plate, mjd) + data_copy+=time.time()-data_copy_start + #Step 2: copy a catalog row from catalog table, + catalog_start=time.time() + catalog_copy(xfibers,fx,hx,plate,mjd) + cata_copy+=time.time()-catalog_start + fx.close() + close_start=time.time() + try: + hx.close() + except: + pass + file_close=time.time()-close_start + tend=time.time()-tstart + #in case of parallel output + print ('-Source file open: %.2f'%(file_open+fopentime)) + print ('-Fiber query time: %.2f'%query_time) + print ('-Fiber copy time: %.2f'%(data_copy)) + print ('-Catalog copy time: %.2f'%(cata_copy)) + print ('-Group create time: %.2f'%(group_create)) + print ('-File close time: %.2f'%(file_close)) + print ('Selection Time: %.2f seconds'%tend) + +if __name__ == '__main__': + select(infiles, outfile, plates, mjds,fibers) diff --git a/h5boss_py/h5boss/select_add.py b/h5boss_py/h5boss/select_add.py new file mode 100755 index 0000000..7ca5f92 --- /dev/null +++ b/h5boss_py/h5boss/select_add.py @@ -0,0 +1,28 @@ +import numpy as np +import h5py +import time +from h5boss.sql import sql +from h5boss.select import select +import traceback +import os,sys +def select_add(infiles, infile, pmflist): + ''' + add the missing (plates,mjds,fibers) from a set of input files + to the pre-existing subset file + Args: + infiles : list of input filenames, or single filename + infile : pre-existing subset file + pmflist : list of [plate, mjd, fiber] + ''' + + to_add, to_del = sql(infile,pmflist) + if len(to_add)==0: + print ("nothing to be added") + sys.exit() + print "TO_ADD (Fibers are listed in the query, and not found in the pre-existing file: %d\n"%len(to_add) + print (to_add) + #split the `to_add` set into `plate` `mjd` `fiber` lists + plates=[i.split(' ')[0] for i in to_add] + mjds=[i.split(' ')[1] for i in to_add] + fibers=[i.split(' ')[2] for i in to_add] + select(infiles,infile,plates,mjds,fibers) diff --git a/h5boss_py/h5boss/select_update.py b/h5boss_py/h5boss/select_update.py new file mode 100755 index 0000000..0543b94 --- /dev/null +++ b/h5boss_py/h5boss/select_update.py @@ -0,0 +1,51 @@ +import numpy as np +import h5py +import time +import traceback +from h5boss.sql import sql +from h5boss.select import select +from h5boss.remove import remove +def select_update(infiles, infile, pmflist,repack=None): + ''' + add the missing (plates,mjds,fibers) from a set of input files + to the pre-existing subset file + remove the additional (plates, mjds, fibers) from the pre-existing file + + Args: + infiles : list of input filenames, or single filename + infile : pre-existing subset file + plates : list of plates + mjds : list of plates + fibers : list of fibers + ''' + to_add=[] + to_del=[] + try: + to_add, to_del = sql(infile,pmflist) + except Exception, e: + print ('Metadata query error') + traceback.print_exc() + + add_plates=[i.split(' ')[0] for i in to_add] + add_mjds=[i.split(' ')[1] for i in to_add] + add_fibers=[i.split(' ')[2] for i in to_add] + del_plates=[i.split(' ')[0] for i in to_del] + del_mjds=[i.split(' ')[1] for i in to_del] + del_fibers=[i.split(' ')[2] for i in to_del] + + try: + print ('plates/mjds/fibers to be added: %d'%len(add_plates)) + if len(add_plates)>0: + print ("Running selection:") + select(infiles,infile,add_plates,add_mjds,add_fibers) + except Exception, e: + print ('Error in adding new pmf to the file:%s'%infile) + + try: + print ('plates/mjds/fibers to be removed: %d'%len(del_plates)) + if len(del_plates)>0: + print ("Running removing:") + remove(infile, del_plates,del_mjds,del_fibers,repack) + except Exception, e: + print ('Error in removing pmf in the file:%s'%infile) + diff --git a/h5boss_py/h5boss/selectmpi_v1.py b/h5boss_py/h5boss/selectmpi_v1.py new file mode 100755 index 0000000..1e7ee07 --- /dev/null +++ b/h5boss_py/h5boss/selectmpi_v1.py @@ -0,0 +1,199 @@ +import numpy as np +import h5py +import time,os +import traceback +from h5boss.select import select +catalog_meta=['plugmap', 'zbest', 'zline', + 'match', 'matchflux', 'matchpos'] +meta=['plugmap', 'zbest', 'zline', + 'photo/match', 'photo/matchflux', 'photo/matchpos'] +kk=1 +## Global variables: +## Data structure of fiberdatalink: {key, value_pair()} ---> {path_dataset, (datatype,datashape,filename)} +## For example fiberdatalink['3665/52273/360/coadd']= (V32, $SCRATCH/h5boss/3665-52273.h5) +## Aug 3 2016 +## Jialin Liu, jalnliu@lbl.gov +# create_slavefile is used in ../script/subset_mpi-sf.py, which creates multiple sub files +def create_slavefile(infile,plates,mjds,fibers,masterfile,rank,id): + master_dir=os.path.dirname(os.path.realpath(masterfile))+'/'+os.path.basename(masterfile).split('.')[0] + + slavefile=master_dir+'/'+str(rank)+'_'+str(id)+'.h5' + try: + select(infile, slavefile, plates, mjds, fibers) + except Exception as e: + print ("Error in slave file:%s") + traceback.print_exc() + pass +def add_dic(dict1,dict2,datatype): + for item in dict2: + if item not in dict1: + dict1[item] = dict2[item] + return dict1 +def add_numpy(dict1, dict2, datatype): + if(dict2.size==0): + return dict1 + if(dict1.size==0): + return dict2 + dict1=np.append(dict1,dict2,axis=0) + return dict1 +def create_template(outfile, global_dict,choice,rank): + if choice=='fiber': + create_fiber_template(outfile,global_dict,rank) + elif choice=='catalog': + create_catlog_template(outfile,global_dict) +def create_fiber_template(outfile,global_dict,rank): +#use one process to create the template + try: + #creating the file with 'latest' formating, rather than earliest, will utilize the most state-of-art optimizing, e.g., indexing, and optimal heap storage inside hdf5 file + hx = h5py.File(outfile,'a',libver="latest") + #hx = h5py.File(outfile,'a') + except Exception as e: + print ("rank:%d, Output file open error:%s"%(rank,outfile)) + traceback.print_exc() + try:#Set the allocate time as early. --Quincey Koziol + for key,value in global_dict.items(): + if key.split('/')[-1] not in catalog_meta: + _fiber_template(hx,key,value) +# else: +# _catalog_template(hx,key,value) + except Exception as e: + traceback.print_exc() + + try: + hx.flush() + hx.close() + except Exception as e: + print("hx close error in %d"%rank) + traceback.print_exc() + + +def create_catlog_template(outfile,global_dict): +#use one process to create the template + try: + hx = h5py.File(outfile,'a') + except Exception as e: + print ("Output file creat error:%s"%outfile) + traceback.print_exc() + try:#Set the allocate time as early. --Quincey Koziol + catalog_dict=global_dict[0] #dict: plate/mjd, number of fiber + catalog_types=global_dict[1] #dict: meta, (type, shape) + for key,value in catalog_dict.items(): + # key=plate/mjd + # value= number of fibers + # catalog_types:types of each catalog table + _catalog_template(hx,key,value,catalog_types) + except Exception as e: + traceback.print_exc() + pass + try: + hx.flush() + hx.close() + except Exception as e: + print("hx close error in rank0") + traceback.print_exc() + pass + +def _fiber_template(hx,key,value): + space=h5py.h5s.create_simple(value[1]) + plist=h5py.h5p.create(h5py.h5p.DATASET_CREATE) + plist.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY) + tid=h5py.h5t.py_create(value[0], False) + try:#create intermediate groups + hx.create_group(os.path.dirname(key)) + except Exception as e: + pass #groups existed, so pass it + try: + h5py.h5d.create(hx.id,key,tid,space,plist)#create dataset with property list:early allocate + except Exception as e: + #print("dataset create error: %s"%key) + #traceback.print_exc() + pass +def _catalog_template(hx,key,value,catalog_types): + value=int(value) + item_shape=(value,1) + space=h5py.h5s.create_simple(item_shape) + plist=h5py.h5p.create(h5py.h5p.DATASET_CREATE) + plist.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY) + plist.set_layout(h5py.h5d.CHUNKED) + ichunk=1 + chunk_shape=(ichunk,1) #Quincey suggests to optimize the ik in H5Pset_istore_k + # for controling the btree for indexing chunked datasets + plist.set_chunk(chunk_shape) + #catalog_types: [plate/mjd/im],( dtype,dshape) + for ktype, vtype in catalog_types.items(): + ikey=key+'/'+ktype + tid=h5py.h5t.py_create(vtype[0], False) + max_shape=int(vtype[1][0]) + #print(value,vtype) + assert(value<=max_shape),"value:%d,maxshape:%d"%(value,max_shape) # number of fibers is larger than the maximum catalog size + try:#create intermediate groups + hx.create_group(os.path.dirname(ikey)) + except Exception as e: + #traceback.print_exc() + pass #groups existed, so pass it + try: + h5py.h5d.create(hx.id,ikey,tid,space,plist)#create dataset with property list:early allocate + except Exception as e: + print("dataset create error: %s\n"%ikey) + traceback.print_exc() + pass + +def overwrite_template(hx, data_dict,choice): + #Read/Write all dataset into final file, + #each rank handles one fiber_dict, which contains multiple fiber_item + if choice=='fiber': + try: + for key, value in data_dict.items(): + if key.split('/')[-1] not in catalog_meta: + _copy_fiber(hx,key,value) + except Exception as e: + print ("Data read/write error key:%s file:%s"%(key,value[2])) + traceback.print_exc() + pass + elif choice=='catalog': + try: + for i in range(0,len(data_dict)): + pm=data_dict[i][0] + # key: pm, value: (fiberid, global_offset, infile) + values_off=data_dict[i][1] + for i in range(0,len(values_off)): + _copy_catalog(hx,pm,values_off[i]) + except Exception as e: + print ("Data read/write error key:%s file:%s"%(key,value[2])) + traceback.print_exc() + pass + +def _copy_fiber(hx,key,value): + try: + subfx=h5py.File(value[2],'r') + subdx=subfx[key].value + subfx.close() + except Exception as e: + traceback.print_exc() + print ("read subfile %s error"%value[2]) + pass + try: + dx=hx[str(key)] + dx[:]=subdx #overwrite the existing template data + except Exception as e: + traceback.print_exc() + print ("overwrite error") + pass + +def _copy_catalog(hx,key,values_off): + try: + fx=h5py.File(values_off[1],'r') + plate=key.split('/')[0] + mjd=key.split('/')[1] + fiber_id=values_off[0] + global_offset=values_off[2] + for name in meta: + id = '{}/{}/{}'.format(plate,mjd,name) + offset=int(global_offset) + maxoff=hx[id].shape[0]-1 + if offset>maxoff: + offset=maxoff + hx[id][offset]=fx[id][int(fiber_id)-1] + except Exception as e: + traceback.print_exc() + print ("catacopy:%s error:%d"%(id,int(fiber_id)-1)) diff --git a/h5boss_py/h5boss/selectmpi_v2.py b/h5boss_py/h5boss/selectmpi_v2.py new file mode 100755 index 0000000..2a0db58 --- /dev/null +++ b/h5boss_py/h5boss/selectmpi_v2.py @@ -0,0 +1,273 @@ +import numpy as np +import h5py +import time,os +import traceback +from h5boss.select import select +catalog_meta=['plugmap', 'zbest', 'zline', + 'match', 'matchflux', 'matchpos'] +meta=['plugmap', 'zbest', 'zline', + 'photo/match', 'photo/matchflux', 'photo/matchpos'] +kk=1 +exposure_dat=("wave","flux","ivar","mask","wavedisp","sky","x","calib") #exposures +coadd_dat=("wave","flux","ivar","and_mask","or_mask","wavedisp","sky","model") # wavelength +## Global variables: +## Data structure of fiberdatalink: {key, value_pair()} ---> {path_dataset, (datatype,datashape,filename)} +## For example fiberdatalink['3665/52273/360/coadd']= (V32, $SCRATCH/h5boss/3665-52273.h5) +## Aug 3 2016 +## Jialin Liu, jalnliu@lbl.gov +# create_slavefile is used in ../script/subset_mpi-sf.py, which creates multiple sub files +def create_slavefile(infile,plates,mjds,fibers,masterfile,rank,id): + master_dir=os.path.dirname(os.path.realpath(masterfile))+'/'+os.path.basename(masterfile).split('.')[0] + + slavefile=master_dir+'/'+str(rank)+'_'+str(id)+'.h5' + try: + select(infile, slavefile, plates, mjds, fibers) + except Exception as e: + print ("Error in slave file:%s") + traceback.print_exc() + pass +def add_dic(dict1,dict2,datatype): + for item in dict2: + if item not in dict1: + dict1[item] = dict2[item] + else: + dict1[item][1]=dict1[item][1]+dict2[item][1] + dict1[item][2]=dict2[item][2]+dict2[item][2] + return dict1 +def add_numpy(dict1, dict2, datatype): + if(dict2.size==0): + return dict1 + if(dict1.size==0): + return dict2 + dict1=np.append(dict1,dict2,axis=0) + return dict1 +def create_template(outfile, global_dict,dmap,choice,rank): + if choice=='fiber': + create_fiber_template(outfile,global_dict,dmap,rank) + elif choice=='catalog': + create_catlog_template(outfile,global_dict) + +def create_fiber_template(outfile,global_dict,dmap,rank): +#use one process to create the template + try: + hx = h5py.File(outfile,'a',libver='latest') + except Exception as e: + print ("rank:%d, Output file open error:%s"%(rank,outfile)) + traceback.print_exc() + try:#Set the allocate time as early. --Quincey Koziol + all_coadd_info=(dmap[0][0],dmap[1])#(type, size_dic) + all_exp_info=(dmap[0][1],dmap[2],dmap[3])#(type, sizeb,sizer) + for key,value in global_dict.items(): + fiberlength=len(value[1]) + if fiberlength>0: + inter_grp=key + if (inter_grp.split('/')[-1]=="coadds"): + _fiber_template(hx,inter_grp,fiberlength,all_coadd_info) + else: + _fiber_template(hx,inter_grp,fiberlength,all_exp_info) +# else: +# _catalog_template(hx,key,value) + except Exception as e: + traceback.print_exc() + + try: + hx.flush() + hx.close() + except Exception as e: + print("hx close error in %d"%rank) + traceback.print_exc() +def create_catlog_template(outfile,global_dict): +#use one process to create the template + try: + hx = h5py.File(outfile,'a',libver='latest') + except Exception as e: + print ("Output file creat error:%s"%outfile) + traceback.print_exc() + try:#Set the allocate time as early. --Quincey Koziol + catalog_dict=global_dict[0] #dict: plate/mjd, number of fiber + catalog_types=global_dict[1] #dict: meta, (type, shape) + for key,value in catalog_dict.items(): + # key=plate/mjd + # value= number of fibers + # catalog_types:types of each catalog table + _catalog_template(hx,key,value,catalog_types) + except Exception as e: + traceback.print_exc() + pass + try: + hx.flush() + hx.close() + except Exception as e: + print("hx close error in rank0") + traceback.print_exc() + pass +def _fiber_template(hx,inter_grp,fiberlength,d_info): + for dset in range(0,len(exposure_dat)): + try: + #cur_dset_name=inter_grp+'/'+dset + #cur_dset_type=d_info[0][dset][0] + pm=inter_grp.split('/')[0]+'/'+inter_grp.split('/')[1] + if inter_grp.split('/')[2]=="coadds": + dg="coadds" + cur_dset_shape=d_info[1][pm] + cur_dset_name=inter_grp+'/'+coadd_dat[dset] + cur_dset_type=d_info[0][coadd_dat[dset]][0] + elif inter_grp.split('/')[-1]=="b": + dg="b" + cur_dset_shape=d_info[1] + cur_dset_name=inter_grp+'/'+exposure_dat[dset] + cur_dset_type=d_info[0][exposure_dat[dset]][0] + else: + dg="r" + cur_dset_shape=d_info[2] + cur_dset_name=inter_grp+'/'+exposure_dat[dset] + cur_dset_type=d_info[0][exposure_dat[dset]][0] + #space=(fiberlength,cur_dset_shape[1]) + space=(fiberlength,cur_dset_shape) + if dset=='wave' and dg=="coadds": + space=(cur_dset_shape,) + spaceid=h5py.h5s.create_simple(space) + plist=h5py.h5p.create(h5py.h5p.DATASET_CREATE) + plist.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY) + tid=h5py.h5t.py_create(cur_dset_type, False) + except Exception as e: + traceback.print_exc() + print("error in fiber template create:",dset) + pass + try:#create intermediate groups + hx.create_group(inter_grp) + except Exception as e: + pass #groups existed, so pass it + try: + h5py.h5d.create(hx.id,cur_dset_name,tid,spaceid,plist)#create dataset with property list:early allocate + except Exception as e: + print("dataset create error: %s"%cur_dset_name) + traceback.print_exc() + pass +def _catalog_template(hx,key,value,catalog_types): + value=int(value) + item_shape=(value,1) + space=h5py.h5s.create_simple(item_shape) + plist=h5py.h5p.create(h5py.h5p.DATASET_CREATE) + plist.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY) + plist.set_layout(h5py.h5d.CHUNKED) + ichunk=1 + chunk_shape=(ichunk,1) #Quincey suggests to optimize the ik in H5Pset_istore_k + # for controling the btree for indexing chunked datasets + plist.set_chunk(chunk_shape) + #catalog_types: [plate/mjd/im],( dtype,dshape) + for ktype, vtype in catalog_types.items(): + ikey=key+'/'+ktype + tid=h5py.h5t.py_create(vtype[0], False) + max_shape=int(vtype[1][0]) + #print(value,vtype) + assert(value<=max_shape),"value:%d,maxshape:%d"%(value,max_shape) # number of fibers is larger than the maximum catalog size + try:#create intermediate groups + hx.create_group(os.path.dirname(ikey)) + except Exception as e: + #traceback.print_exc() + pass #groups existed, so pass it + try: + h5py.h5d.create(hx.id,ikey,tid,space,plist)#create dataset with property list:early allocate + except Exception as e: + print("dataset create error: %s\n"%ikey) + traceback.print_exc() + pass + +def overwrite_template(hx, data_dict,choice): + #Read/Write all dataset into final file, + #each rank handles one fiber_dict, which contains multiple fiber_item + if choice=='fiber': + try: + for key, value in data_dict.items(): + _copy_fiber(hx,key,value) + except Exception as e: + print ("Data read/write error key:%s file:%s"%(key,value[2])) + traceback.print_exc() + pass + elif choice=='catalog': + try: + for i in range(0,len(data_dict)): + pm=data_dict[i][0] + # key: pm, value: (fiberid, global_offset, infile) + values_off=data_dict[i][1] + for i in range(0,len(values_off)): + _copy_catalog(hx,pm,values_off[i]) + except Exception as e: + print ("Data read/write error key:%s file:%s"%(key,value[2])) + traceback.print_exc() + pass +def _copy_fiber(hx,key,value): # key is the inter_group, value has filename, fiberlist, fiberoffsetlist + try: + subfx=h5py.File(value[0],'r') + if key.split('/')[2]=="coadds": # copy datasets in coadds group + for icoad in coadd_dat: + idset=key+'/'+icoad + dx=hx[idset] + try: + if icoad!='wave': # wave dataset only has 1 dimensional, and all fiber should entirly copy it. + # may replace this for loop with a signle I/O call: subdx=subfx[idset][value[2]] then dx=subdx +# for ifiber in range(0,len(value[2])): +# fiber_off=value[2][ifiber] +# if fiber_off < subfx[idset].shape[0]: +# subdx=subfx[idset][fiber_off] +# else: +# subdx=[0]*subfx[idset].shape[1] +# dx[ifiber]=subdx + vlist=value[2] + vlist.sort() + try: + dx=subfx[idset][vlist] + except Exception as e: + print ("vlist has %s,needs to be fixed"%vlist) + pass + else: + subdx=subfx[idset] + dx=subdx + except Exception as e: + print ("infile:%s,group:%s,dataset:%s,shape:%s. outdt:%s,otshape:%s"%(value[0],key,icoad,subdx.shape,idset,dx.shape)) + elif key.split('/')[-1]=="b" or key.split('/')[-1]=="r": # copy datasets in exposures groups + for iexp in exposure_dat: + idset=key+'/'+iexp + dx=hx[idset] + vlist=value[2] + vlist.sort() + try: + dx=subfx[idset][vlist] + except Exception as e: + print("vlist has %s, needs to be fixed"%vlist) + pass + # may replace this for loop with a signle I/O call: subdx=subfx[idset][value[2]] then dx=subdx +# for ifiber in range(0,len(value[2])): +# fiber_off=value[2][ifiber] +# if fiber_off< subfx[idset].shape[0]: +# subdx=subfx[idset][fiber_off] +# else: +# subdx=[0]*subfx[idset].shape[1] + #print ("group:%s dataset:%s ifiber:%d"%(key,iexp,ifiber)) +# dx[ifiber]=subdx + else: + print ("wrong inter group found%s"%key) + subfx.close() + except Exception as e: + traceback.print_exc() + print ("read subfile %s error at key:%s value:%s"%(value[0],key,value)) + pass + +def _copy_catalog(hx,key,values_off): + try: + fx=h5py.File(values_off[1],'r') + plate=key.split('/')[0] + mjd=key.split('/')[1] + fiber_id=values_off[0] + global_offset=values_off[2] + for name in meta: + id = '{}/{}/{}'.format(plate,mjd,name) + offset=int(global_offset) + maxoff=hx[id].shape[0]-1 + if offset>maxoff: + offset=maxoff + hx[id][offset]=fx[id][int(fiber_id)-1] + except Exception as e: + traceback.print_exc() + print ("catacopy:%s error:%d"%(id,int(fiber_id)-1)) diff --git a/h5boss_py/h5boss/sql.py b/h5boss_py/h5boss/sql.py new file mode 100755 index 0000000..6bd27f7 --- /dev/null +++ b/h5boss_py/h5boss/sql.py @@ -0,0 +1,379 @@ +import numpy as np +import h5py +import time +import sys +import os +import csv +import traceback +from collections import defaultdict +fx="" +pid="" +inputfile="" +fiberdatalink={} +cataloglink={} +meta=['plugmap', 'zbest', 'zline', + 'photo/match', 'photo/matchflux', 'photo/matchpos'] + +def sql(infile, pmflist): + ''' + check (plates,mjds,fibers) in the source hdf5 file + return the matched/missed pmf + + Args: + infile : input file + pmflist : list of [plate, mjd, fiber] + ''' + tstart=time.time() + if (len(pmflist)==0): + print('input query is empty') + sys.exit(0) + + inx=h5py.File(infile,'r') + in_pmf=[] + todel_pmf=[] + try: + for pid in inx.keys(): + for mid in inx[pid].keys(): + for fid in inx[pid+'/'+mid].keys(): + if fid.isdigit(): + a='{} {} {}'.format(pid, mid, fid) + #print (a) + if a in pmflist: + in_pmf.append(a) + else: + todel_pmf.append(a) + except Exception as e: + pass + toadd_pmf=[] + try: + toadd_pmf=list(set(pmflist)-set(in_pmf)) + except Exception as e: + print ("set operation error") + print ("pmflist:",pmflist) + print ("set of pmflist:",set(pmflist)) + print ("in_pmf:",in_pmf) + print ("set of in_pmf:",set(in_pmf)) + tend=time.time()-tstart + print ('Query time: %.2f seconds'%tend) + return (toadd_pmf, todel_pmf) + +def parse_csv(input,output,pmflist,rank): + ''' + Check the input/output and pmflist + return plates, mjds, fibers as separate numpy arrays + Args: + input: HDF5 files list, i.e., source data + output: HDF5 file, to be created or updated + pmflist: Plates/mjds/fibers numbers to be quried + ''' + # check output file and its path + if os.path.exists(output): + if rank==0: + print ("The output file %s is existed, your job is going to overwrite it or update it"%output) + elif os.access(os.path.dirname(output),os.W_OK): + if rank==0: + print ("The output file %s is not existed, your job will create a new file"%output) + else: + if rank==0: + print ("The output file's path does not exist, job exits now") + sys.exit() + + # parse plates/mjds/fibers + plates=[] + mjds=[] + fibers=[] + try: + df = list_csv(pmflist) + plates = df['plates'] + mjds = df['mjds'] + fibers = df['fibers'] + except Exception as e: + print("pmflist csv read error or not exist:%s"%e,pmflist) + traceback.print_exc() + print("Note: 1st row of csv should start with 'plates mjds fibers'") + if len(plates)==0: + print ("No query is found, plate is empty") + sys.exit() + + try: + with open(input,'rt') as f: + reader = csv.reader(f) + hdfsource = list(reader) + hdfsource = [x for sublist in hdfsource for x in sublist] + except Exception as e: + print ("HDF5 inputlist csv read error or not exist: %s"%e,input) + + if(len(hdfsource)==0): + print("HDF5 source is empty") + sys.exit(0) + + plates = np.asarray(plates) + mjds = np.asarray(mjds) + fibers = np.asarray(fibers) + + return (plates,mjds,fibers,hdfsource) + + +def list_csv(x): + ''' + Return a array[list], where each list is a column + Args: + csv file + ''' + columns = defaultdict(list) # each value in each column is appended to a list + try: + with open(x) as f: + reader = csv.DictReader(f,delimiter=' ') # read rows into a dictionary format + for row in reader: # read a row as {column1: value1, column2: value2,...} + for (k,v) in row.items(): # go over each column name and value + columns[k].append(v) # append the value into the appropriate list + # based on column name k + except Exception as e: + print ("read pmf csv error") + traceback.print_exc() + sys.exit() + return columns + +def _traverse_fibernode(name): + ''' + para : node name in a hdf5 group + purpose: Find a dataset node, which should be an endpoint in its group hierarchy + return : (key,value)->(path_to_dataset, (dataset type, shape, filename)) + ''' + global fx,pid,fiberdatalink,inputfile + try: + cur_node=name.encode('ascii','ignore') + node=pid+'/'+cur_node + node_t=str(type(fx[node])) + if 'dataset' in node_t: + node_t=fx[node].dtype + node_sp=fx[node].shape + fiberdatalink[node]=(node_t,node_sp,inputfile) + except Exception as e: + traceback.print_exc() + pass + +def dedup(dict1):# might be costly, as the python tuple is inmutable, + #Deduplication, dict(plate/mjd/../coadds, ifile, fiberlists, fiberoffsetlists) + #TODO: Optimize this stupid: currently: pop out each key, dedup, sort, then re-insert it in the dictionary. + for idict in dict1: + v1=dict1[idict][0] + v2=dict1[idict][1] + v3=dict1[idict][2] + v2_len=len(v2) + v3_len=len(v3) + v2=list(set(v2)) + v2.sort() + v3=list(set(v3)) + v3.sort() + dedup_v2_len=len(v2) + dedup_v3_len=len(v3) + dict1.pop(idict,None) + dict1[idict]=(v1,v2,v3) + if v2_len!=dedup_v2_len or v3_len!=dedup_v3_len: + print("key:%s, v2:%d,dv2:%d; v3:%d,dv3:%d"%(idict,v2_len,dedup_v2_len,v3_len,dedup_v3_len)) + return dict1 +#node_type is used in ../script/subset_mpi.py, which is to create single shared file +def get_fiberlink_v1(infile,plates,mjds,fibers): + ''' + para : filename, plate, mjd, fiber + return: (key, value)->(plates/mjd/fiber/../dataset, (type,shape,filename)) + python dict's updating can ensure that the key is unique, i.e., plate/mjd/fiber/../dataset is unique + ''' + global pid,fiberdatalink, cataloglink, fx, inputfile + inputfile=infile + try: + fx = h5py.File(infile, mode='r') + for plate in fx.keys(): + for mjd in fx[plate].keys(): + ii = (plates == plate) & (mjds == mjd) + spid= '{}/{}'.format(plate, mjd) + xfibers = fibers[ii] + if np.any(ii): # fiber is found + for fiber in xfibers:#for each fiber node, recursively visit its members and record the + #fiberlink={id:infile} + pid = '{}/{}/{}'.format(plate, mjd, fiber) + fx[pid].visit(_traverse_fibernode) + #for im in meta: + # mnode=spid+'/'+im + # mnode_t=fx[mnode].dtype + # mnode_sp=fx[mnode].shape + # fiberdatalink[mnode]=(mnode_t,mnode_sp,infile) + fx.close() + except Exception as e: + print (pid) + traceback.print_exc() + print (pid,infile) + pass + return (fiberdatalink) +def get_fiberlink_v2(infile,plates,mjds,fibers): + ''' + para : filename, plate, mjd, fiber + return: fiberdatalink: + (key, value)->(plates/mjd/coadds, (filename, fiberlist, fiberoffsetlist)) + (key, value)->(plates/mjd/exposures/?id/b(r), (filename, fiberlist, fiberoffsetlist)) + + ''' + #global pid,fiberdatalink, cataloglink, fx, inputfile + #inputfile=infile + fiberdatalink={} + import os.path + if not os.path.isfile(infile): + return fiberdatalink + try: + fx = h5py.File(infile, mode='r') + for plate in fx.keys(): + for mjd in fx[plate].keys(): + ii = (plates == plate) & (mjds == mjd) + spid= '{}/{}'.format(plate, mjd) + # get the fiber column + #TODO: try the fx[spid+'/plugmap'][()]['FIBERID'] or fx[spid+'/plugmap'][:]['FIBERID'] + data_value=fx[spid+'/plugmap'].value['FIBERID'] + xfibers = fibers[ii] + fiber_list=data_value.tolist() + if np.any(ii): # plate and mjd are matching + for fiber in xfibers: + # return spid, + fiber=int(fiber) + if fiber not in fiber_list: + #print("fiber not found,e..g, %s"%(fiber_list.dtype)) + continue + try: + fiber_offset=fiber_list.index(fiber) # this may triger error if fiber not found + #update k,v store + #print ("fiber_offset:%d"%(fiber_offset)) + spid_coad=spid+'/coadds' + if spid_coad not in fiberdatalink: # when coadds is added, exposures will be also added + #print ("spid_coad:%s not in fiberdata, fiber now is %d"%(spid_coad,fiber)) + fiberlist=list() + fiberlist.append(fiber) + offsetlist=list() + offsetlist.append(fiber_offset) + #spid_coad=spid+'/coadds' + fiberdatalink[spid_coad]=(infile,fiberlist,offsetlist) + spid_expo=spid+'/exposures' + #try: + # print("fx[%s].keys:%s"%(spid_expo,fx[spid_expo].keys())) + #except Exception as e: + # print("get key error in infile:%s, fx[%s]"%(infile,spid_expo)) + # pass + for expid in fx[spid_expo].keys(): + expid_name=spid_expo+'/'+expid+'/b' + #print("expid:%s"%expid) + fiberlist=list() + fiberlist.append(fiber) + offsetlist=list() + offsetlist.append(fiber_offset) + fiberdatalink[expid_name]=(infile,fiberlist,offsetlist) + expid_name=spid_expo+'/'+expid+'/r' + fiberlist=list() + fiberlist.append(fiber) + offsetlist=list() + offsetlist.append(fiber_offset) + fiberdatalink[expid_name]=(infile,fiberlist,offsetlist) + #print ("BEFORE0:fiberdatalink[%s][1]:%s"%(expid_name,fiberdatalink[expid_name][1])) + else: + #print ("spid_coad:%s in fiberdata, fiber now is %d"%(spid_coad,fiber)) + #print ("before:",fiberdatalink[spid_coad]) + fiberdatalink[spid_coad][1].append(fiber) # update fiberlist + #print ("after:",fiberdatalink[spid_coad]) + fiberdatalink[spid_coad][2].append(fiber_offset) # update offsetlist + spid_expo=spid+'/exposures' + for expid in fx[spid_expo].keys(): + expid_name=spid_expo+'/'+expid+'/b' + expid_name_r=spid_expo+'/'+expid+'/r' + #print ("BEFORE1:fiberdatalink[%s][1]:%s"%(expid_name,fiberdatalink[expid_name][1])) + #print ("BEFORE1:fiberdatalink[%s][1]:%s"%(expid_name_r,fiberdatalink[expid_name_r][1])) + fiberdatalink[expid_name][1].append(fiber) + fiberdatalink[expid_name][2].append(fiber_offset) + #print ("AFTER1:fiberdatalink[%s][1]:%s"%(expid_name_r,fiberdatalink[expid_name_r][1])) + expid_name=spid_expo+'/'+expid+'/r' + #print ("BEFORE2:fiberdatalink[%s][1]:%s"%(expid_name,fiberdatalink[expid_name][1])) + fiberdatalink[expid_name][1].append(fiber) + fiberdatalink[expid_name][2].append(fiber_offset) + #print("expid_name:%s,fiber:%d"%(expid_name,fiber)) + #print("fiber",fiber) + #print("fiber_offset",fiber_offset) + #print ("AFTER2:fiberdatalink[%s][1]:%s"%(expid_name,fiberdatalink[expid_name][1])) + except Exception as e: + #print("fiber kv update error:file:%s,spid:%s"%(infile,spid)) + #traceback.print_exc() + pass # fiber not existing + fx.close() + except Exception as e: + #print (spid) + traceback.print_exc() + #print (spid,infile) + pass + return (fiberdatalink) + +def get_catalogtypes(infile): + ''' + para:hdf5 file + return: dict: meta, (type, shape) + ''' + catalog_types={} + try: + fx = h5py.File(infile, mode='r') + plate=fx.keys()[0] + mjd=fx[plate].keys()[0] + for im in meta: + mnode=plate+'/'+mjd+'/'+im + #print (mnode) + mnode_t=fx[mnode].dtype + mnode_sp=fx[mnode].shape + #print (mnode_t) + catalog_types[im]=(mnode_t,mnode_sp) + except Exception as e: + print ("file:",infile) + traceback.print_exc() + pass + return catalog_types + +def count_unique(global_dict): + ''' + para: dict: (plates/mjd/fiber/../dataset, (type,shape,filename)) + return: dict: (plates/mjd, num_fibers) + ''' + count_fiber={} + for key, value in global_dict.items(): + dname=key.split('/')[-1] + if dname=='coadd': + plate=key.split('/')[0] + mjd=key.split('/')[1] + temp_key=plate+'/'+mjd + #print (temp_key) + if temp_key in count_fiber: + pre_count=int(count_fiber[temp_key]) + count_fiber[temp_key]=pre_count+1 + else: + count_fiber[temp_key]=int(1) + return count_fiber + + +def fiber_union(fiber_dict1, fiber_dict2,interkey_set): + if len(interkey_set)==0: + return fiber_dict1 + for ikey in interkey_set: + fiber_dict1[ikey][1]+=fiber_dict2[ikey][1] + fiber_dict1[ikey][2]+=fiber_dict2[ikey][2] + #TODO: need to remove duplication,Sep 23. 2016 + return fiber_dict1 + + +def locate_fiber_in_catalog(global_dict): + revised_dict={}# key: pm, value: (fiberid, global_offset, infile) + for key,value in global_dict.items(): + if key.split('/')[-1]=='coadd': + pm=key.split('/')[0]+'/'+key.split('/')[1] + if pm in revised_dict.keys(): + latest=len(revised_dict[pm]) + new_value=(key.split('/')[2],value[2],latest) + revised_dict[pm].append(new_value) + else: + latest=0 + new_value=(key.split('/')[2],value[2],latest) + revised_dict.setdefault(pm, []) + revised_dict[pm].append(new_value) + return revised_dict + diff --git a/h5boss_py/script/add b/h5boss_py/script/add new file mode 100755 index 0000000..d2196f5 --- /dev/null +++ b/h5boss_py/script/add @@ -0,0 +1,77 @@ +#!/usr/bin/env python +""" +Create an HDF5 file from BOSS data + +TODO: + - include comments in meta/attrs + - platelist quantities +""" +from __future__ import division, print_function +#from __future__ import absolute_import +from h5boss.select import select +from h5boss.select_add import select_add +import sys,os +import time +import optparse +import csv +import traceback + +import argparse +parser = argparse.ArgumentParser(prog='add',description='Compare base with pmf, add the new plates/mjds/fibers founded in both (pmf,input) into the pre-selected HDF5 base file.') +parser.add_argument("base", help="Pre-selected HDF5 file") +parser.add_argument("input", help="HDF5 input list") +parser.add_argument("pmf", help="Plate/mjd/fiber list in csv") + +opts=parser.parse_args() + + + +#parser = optparse.OptionParser(usage = "%prog [options]") +#parser.add_option("-b", "--base", type=str, help="base file") +#parser.add_option("-i", "--input", type=str, help="files to be quried") +#parser.add_option("-x", "--xls", type=str,help="plate/mjd/fiber excel file") +#opts, args = parser.parse_args() + +pmflist = opts.pmf +infiles = opts.input +outfile = opts.base +tstart=time.time() +import pandas as pd +try: + df = pd.read_csv(pmflist,delimiter=' ',names=["plates","mjds","fibers"],index_col=None,dtype=str) + plates = map(str,df['plates'].values.tolist()) + mjds = map(str,df['mjds'].values.tolist()) + fibers = map(str,df['fibers'].values.tolist()) +except Exception, e: + print("pmf csv read error or not exist:%s"%e,pmflist) + print("e.g., 1st row of csv should be 'plates mjds fibers'") + +try: + with open(infiles,'rb') as f: + reader = csv.reader(f) + infile = list(reader) + +except Exception, e: + print ("CSV file open error or not exist: %s"%e,infiles) + #traceback.print_exc() + +infile = [x for sublist in infile for x in sublist] +if(len(plates)==0 or len(infile)==0): + print("pmf or input is empty") + sys.exit(0) + +print ("Input: %d files:"%len(infile),infile[0],"...",infile[-1]) +print ("Output: ", outfile) +print ("Running selection...") + +try: + dflist=pd.read_csv(pmflist,delimiter='\n',index_col=None,dtype=str) + listdf=list(dflist.values.flatten()) + select_add(infile, outfile, listdf) + +except Exception, e: + print ("Error in select:") + traceback.print_exc() + + +print ("Done selection") diff --git a/scripts/boss2fits b/h5boss_py/script/boss2fits similarity index 100% rename from scripts/boss2fits rename to h5boss_py/script/boss2fits diff --git a/h5boss_py/script/boss2hdf5-parallel b/h5boss_py/script/boss2hdf5-parallel new file mode 100755 index 0000000..151c020 --- /dev/null +++ b/h5boss_py/script/boss2hdf5-parallel @@ -0,0 +1,57 @@ +import os +from h5boss import boss2hdf5 +import traceback +import argparse +from mpi4py import MPI + +datapath = "/global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0/" +#outputpath= "/global/cscratch1/sd/jialin/h5boss-bp/" +outputpath= "/global/cscratch1/sd/jialin/h5boss_v2/" + +def listfiles(): + ldir=os.listdir(datapath) + lldir=[fn for fn in ldir if fn.isdigit()] + return lldir + +def findseed(x): + fitsfiles = [os.path.join(root, name) + for root, dirs, files in os.walk(x) + for name in files + if name.startswith("spPlate") and name.endswith(".fits")] + return fitsfiles + +def parallel_convert(): + parser = argparse.ArgumentParser(prog='convert') + parser.add_argument("--input", type=str, help="input fits directory") + parser.add_argument("--output", type=str, help="output hdf5") + + opts = parser.parse_args() + global datapath + global outputpath + if opts.input: + datapath = opts.input + + if opts.output: + outputpath = opts.output + + rank = MPI.COMM_WORLD.Get_rank() + nproc = MPI.COMM_WORLD.Get_size() + plateslist=listfiles() + if (nproc>len(plateslist)): + nproc=len(plateslist) + plateslist_for_each_process = [fname for fname in plateslist[:nproc]] + if(rank==0): + print "nproc:%d\n"%nproc + print "\nrank :%d\n"%rank + if(rank0): + hdf5file=fitspath_name_for_current_rank[0].split('/')[-1].replace('spPlate-','',1).replace('fits','hdf5',1) + try: + boss2hdf5.serial_convert(fitspath_name_for_current_rank,outputpath+hdf5file) + except Exception, e: + print "Error:%s"%e, fitspath_name_for_current_rank + traceback.print_exc() +if __name__ == '__main__': + parallel_convert() diff --git a/h5boss_py/script/boss2hdf5_v1 b/h5boss_py/script/boss2hdf5_v1 new file mode 100755 index 0000000..88eb8a8 --- /dev/null +++ b/h5boss_py/script/boss2hdf5_v1 @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +""" +Create an HDF5 file from BOSS data + +TODO: + - include comments in meta/attrs + - platelist quantities +""" + +from __future__ import division, print_function + +import sys, os +import numpy as np +from astropy.io import fits +from astropy.table import Table +import h5boss.boss2hdf5_v1 +import time +import optparse +import h5py +parser = optparse.OptionParser(usage = "%prog [options]") +parser.add_option("-i", "--input", type=str, help="input spPlate file") +parser.add_option("-o", "--output", type=str, help="output hdf5 file") + +opts, args = parser.parse_args() +if len(sys.argv)<=1: + print ("boss2hdf5 -help") + exit(1) + +platefile = opts.input +hdf5output = opts.output +platefiles=list() +platefiles.append(platefile) +h5boss.boss2hdf5_v1.serial_convert(platefiles,hdf5output) diff --git a/h5boss_py/script/boss2hdf5_v2 b/h5boss_py/script/boss2hdf5_v2 new file mode 100755 index 0000000..fb94f81 --- /dev/null +++ b/h5boss_py/script/boss2hdf5_v2 @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +""" +Create an HDF5 file from BOSS data + +TODO: + - include comments in meta/attrs + - platelist quantities +""" + +from __future__ import division, print_function + +import sys, os +import numpy as np +from astropy.io import fits +from astropy.table import Table +import h5boss.boss2hdf5_v2 +import time +import optparse +import h5py +parser = optparse.OptionParser(usage = "%prog [options]") +parser.add_option("-i", "--input", type=str, help="input spPlate file") +parser.add_option("-o", "--output", type=str, help="outputput hdf5 file") + +opts, args = parser.parse_args() +if len(sys.argv)<=1: + print ("boss2hdf5 -help") + exit(1) + +platefile = opts.input +filedir = os.path.split(os.path.abspath(platefile))[0] +hdf5output = opts.output +platefiles=list() +platefiles.append(platefile) +h5boss.boss2hdf5_v2.serial_convert(platefiles,hdf5output) diff --git a/h5boss_py/script/subset b/h5boss_py/script/subset new file mode 100755 index 0000000..f6a61ba --- /dev/null +++ b/h5boss_py/script/subset @@ -0,0 +1,76 @@ +#!/usr/bin/env python +""" +Create an HDF5 file from BOSS data + +TODO: + - include comments in meta/attrs + - platelist quantities +""" +from __future__ import division, print_function +#from __future__ import absolute_import +#from h5boss.selectmpi import select +#from h5boss.select_c import select +from h5boss.select import select +#from h5boss.select_idx import select +import sys,os +import time +import optparse +import csv +import traceback +import pandas as pd +import numpy as np +import optparse +import argparse +import commands +parser = argparse.ArgumentParser(prog='subset') +parser.add_argument("input", help="HDF5 input list") +parser.add_argument("output", help="HDF5 output") +parser.add_argument("pmf", help="Plate/mjd/fiber list in csv") +opts=parser.parse_args() +pmflist = opts.pmf +infiles = opts.input +outfile = opts.output +nproc=1 +tstart=time.time() +plates=[] +mjds=[] +fibers=[] +try: + df = pd.read_csv(pmflist,delimiter=' ',names=["plates","mjds","fibers"],index_col=None,dtype=str) + df = df.sort(['plates'],ascending=[1]) + plates = list(map(str,df['plates'].values)) + plates_array = np.asarray(plates) + plates_uni_array = np.unique(plates_array) + print ("number of unique plates is %d"%plates_uni_array.size) + mjds = list(map(str,df['mjds'].values)) + fibers = list(map(str,df['fibers'].values)) +except Exception as e: + print("pmf csv read error or not exist:%s"%e,pmflist) + print("e.g., 1st row of csv should be 'plates mjds fibers'") +infile=[] +try: + with open(infiles,'rt') as f: + reader = csv.reader(f) + infile = list(reader) + infile = [x for sublist in infile for x in sublist] +except Exception as e: + print ("input filelist csv read error or not exist: %s"%e,infiles) + +print ("Before running selection, parsing costs %.2f"%(time.time()-tstart)) +if(len(plates)==0 or len(infile)==0): + print("pmf or input is empty") + sys.exit(0) + +print ("Input: %d files:"%len(infile),infile[0],"...",infile[-1]) +print ("Output: ", outfile) +print ("Running selection...") +tstart=time.time() +try: + #pmflist=df.values.tolist() + #select(infile, outfile, pmflist) + select(infile,outfile,plates,mjds,fibers) +except Exception as e: + print ("Error in select:") + traceback.print_exc() +print ("Done selection") +print ("Total selection Cost %.2f"%(time.time()-tstart)) diff --git a/h5boss_py/script/subset_mpi_v1 b/h5boss_py/script/subset_mpi_v1 new file mode 100755 index 0000000..1ebed98 --- /dev/null +++ b/h5boss_py/script/subset_mpi_v1 @@ -0,0 +1,196 @@ +#!/usr/bin/env python +""" +Create an HDF5 file from BOSS data + +TODO: + - include comments in meta/attrs + - platelist quantities +""" +from __future__ import division, print_function +#from __future__ import absolute_import +from mpi4py import MPI +import h5py +from h5boss.sql import parse_csv +from h5boss.sql import get_fiberlink_v1 +from h5boss.sql import get_catalogtypes +from h5boss.sql import count_unique +from h5boss.sql import locate_fiber_in_catalog +from h5boss.selectmpi_v1 import add_dic +from h5boss.selectmpi_v1 import add_numpy +from h5boss.selectmpi_v1 import create_template +from h5boss.selectmpi_v1 import overwrite_template +from time import gmtime, strftime +import datetime +import sys,os +import time +import optparse +import csv +import traceback +#import pandas as pd +import numpy as np +import optparse +import argparse +import datetime +from collections import defaultdict +import cPickle as pickle +def parallel_select(): + ''' + Select a set of (plates,mjds,fibers) from the realesed BOSS data in HDF5 formats. + + Args: + input: HDF5 files list, i.e., source data, [csv file] + output: HDF5 file, to be created or updated + pmf: Plates/mjds/fibers numbers to be quried, [csv file] + + ''' + parser = argparse.ArgumentParser(prog='subset_mpi') + parser.add_argument("input", help="HDF5 input list") + parser.add_argument("output", help="HDF5 output") + parser.add_argument("pmf", help="Plate/mjd/fiber list") + parser.add_argument("--template", help="Create template only,yes/no/all") + parser.add_argument("--mpi", help="using mpi yes/no") + parser.add_argument("--fiber", help="specify fiber csv output") + parser.add_argument("--catalog", help="specify catalog csv output") + parser.add_argument("--datamap", help="specify datamap pickle file") + opts=parser.parse_args() + + infiles = opts.input + outfile = opts.output + pmflist = opts.pmf + fiberout = "fibercsv" + catalogout = "catalogcsv" + if opts.fiber: + fiberout = opts.fiber + if opts.catalog: + catalogout = opts.catalog + catalog_meta=['plugmap', 'zbest', 'zline', + 'match', 'matchflux', 'matchpos'] + meta=['plugmap', 'zbest', 'zline', + 'photo/match', 'photo/matchflux', 'photo/matchpos'] + if opts.template is None or opts.template=="no": + template=0 + elif opts.template and opts.template=="yes": + template=1 + elif opts.template and opts.template=="all": + template=2 + if opts.mpi is None or opts.mpi=="no": + #starts seirial processing + print ("Try the subset.py or subset command") + sys.exit() + elif opts.mpi and opts.mpi=="yes": + comm =MPI.COMM_WORLD + nproc = comm.Get_size() + rank = comm.Get_rank() + tstartcsv=MPI.Wtime() + (plates,mjds,fibers,hdfsource) = parse_csv(infiles, outfile, pmflist,rank) + tstart=MPI.Wtime() + if rank==0: + print ("Number of processes %d"%nproc) + print ("parse csv time: %.2f"%(tstart-tstartcsv)) + total_files=len(hdfsource) + #distribute the workload evenly to each process + step=int(total_files / nproc) + rank_start =int( rank * step) + rank_end = int(rank_start + step) + if(rank==nproc-1): + rank_end=total_files # adjust the last rank's range + if rank_start>total_files: + rank_start=total_files + range_files=hdfsource[rank_start:rank_end] + #print ("rank:%d,file:%d"%(rank,len(range_files))) + if rank==0: + sample_file=range_files[0] + fiber_dict={} + for i in range(0,len(range_files)): + fiber_item = get_fiberlink_v1(range_files[i],plates,mjds,fibers) + if len(fiber_item)>0: + fiber_dict.update(fiber_item) + tend=MPI.Wtime() + if rank==0: + print ("Get metadata of fiber ojbect time: %.2f"%(tend-tstart)) + #rank0 create all, then close an reopen.-Quincey Koziol + counterop = MPI.Op.Create(add_dic, commute=True) #define reduce operation + global_fiber={}#(key, value)->(plates/mjd/fiber/../dataset, (type,shape,filename), unordered + fiber_item_length=len(fiber_dict) + fiber_dict_tmp=fiber_dict + global_fiber= comm.allreduce(fiber_dict_tmp, op=counterop) + treduce=MPI.Wtime() + #print ("rank: ",rank,fiber_dict_tmp_numpy) + if rank==0: + print ("Allreduce %d fiber meta:kv(dataset, type): %.2f"%(len(global_fiber),(treduce-tend))) + #print (global_fiber) # expect: key(plate/mjd), value(filename, fiberlist, fiberoffsetlist) + #TODO:remove duplication of fiberlist and fiberoffsetlist in global_fiber +############# CREATE TEMPLATE WITH ONE PROCESS ############ + #Create the template using 1 process + if rank==0 and (template==1 or template==2): + try: + ##can not parallel create metadata is really painful. + pickle.dump(global_fiber,open("global_fiber1k_1","wb")) + #sys.exit() + create_template(outfile,global_fiber,'fiber',rank) +# catalog_number=count_unique(global_fiber) #(plates/mjd, num_fibers) +# print ('number of unique fibers:%d '%len(catalog_number)) +# catalog_types=get_catalogtypes(sample_file) # dict: meta, (type, shape) +# global_catalog=(catalog_number,catalog_types) +# create_template(outfile,global_catalog,'catalog',rank) + except Exception as e: + traceback.print_exc() + tcreated=MPI.Wtime() + #if rank==0: + #TODO: rewrite this to get the catalog and fiber in one shot + copy_global_catalog=global_fiber + revised_dict=locate_fiber_in_catalog(copy_global_catalog) + #now revised_dict has: p/m, fiber_id, infile, global_offset + copy_revised_dict=revised_dict.items() + total_unique_fiber=len(copy_revised_dict) + #distribute the workload evenly to each process + step=int(total_unique_fiber / nproc)+1 + rank_start =int( rank * step) + rank_end = int(rank_start + step) + if(rank==nproc-1): + rank_end=total_unique_fiber # adjust the last rank's range + if rank_start>total_unique_fiber: + rank_start=total_unique_fiber + catalog_dict=copy_revised_dict[rank_start:rank_end] +# twritecsv_start=MPI.Wtime() + if rank==0 and (template==1 or template==2): + print ("Template creation time: %.2f"%(tcreated-treduce)) +# with open(fiberout, 'a') as f: +# f.writelines('{}:{}\n'.format(k,v[2]) for k, v in global_fiber.items()) +# f.write('\n') +# with open(catalogout, 'a') as f: +# for k,v in revised_dict.items(): +# for iv in v: +# f.writelines('{}:{}:{}:{}\n'.format(k,iv[0],iv[1],iv[2])) + #f.write('\n') +# twritecsv_end=MPI.Wtime() +# if rank==0: +# print ("count unique fiber and global offset time:%.2f"%(twritecsv_start-tcreated)) +# print ("write fiber/catalog csv time: %.2f"%(twritecsv_end-twritecsv_start)) +############# OVERWRITE THE TEMPLATE WITH ACTUAL DATA ############ + if template ==0 or template==2: + try: + hx = h5py.File(outfile,'a',driver='mpio', comm=MPI.COMM_WORLD) ## collectively open file + hx.atomic=False + except Exception as e: + traceback.print_exc() + topen=MPI.Wtime() + tclose=topen + + if template==0 or template==2: + fiber_copyts=MPI.Wtime() + overwrite_template(hx,fiber_dict,'fiber') + fiber_copyte=MPI.Wtime() + print ("rank:%d\tlength:%d\tcost:%.2f"%(rank,fiber_item_length,fiber_copyte-fiber_copyts)) +# #for each fiber, find the catalog, then copy it +# catalog_copyts=MPI.Wtime() +# overwrite_template(hx,catalog_dict,'catalog') +# catalog_copyte=MPI.Wtime() + comm.Barrier() + tclose_s=MPI.Wtime() + hx.close() + tclose=MPI.Wtime() + if rank==0: + print ("File open: %.2f\nFiber copy: %.2f\nFile close: %.2f\nTotal Cost: %.2f"%(topen-tcreated,fiber_copyte-fiber_copyts,tclose-tclose_s,tclose-tstart)) +if __name__=='__main__': + parallel_select() diff --git a/h5boss_py/script/subset_mpi_v2 b/h5boss_py/script/subset_mpi_v2 new file mode 100755 index 0000000..2ff8cf8 --- /dev/null +++ b/h5boss_py/script/subset_mpi_v2 @@ -0,0 +1,214 @@ +#!/usr/bin/env python +""" +Create an HDF5 file from BOSS data + +TODO: + - include comments in meta/attrs + - platelist quantities +""" +from __future__ import division, print_function +#from __future__ import absolute_import +from mpi4py import MPI +import h5py +from h5boss.sql import parse_csv +from h5boss.sql import get_fiberlink_v2 +from h5boss.sql import get_catalogtypes +from h5boss.sql import count_unique +from h5boss.sql import locate_fiber_in_catalog +from h5boss.sql import dedup +from h5boss.selectmpi_v2 import add_dic +from h5boss.selectmpi_v2 import add_numpy +from h5boss.selectmpi_v2 import create_template +from h5boss.selectmpi_v2 import overwrite_template +from h5boss.h5map import type_map +from h5boss.h5map import coadd_map +from time import gmtime, strftime +import datetime +import sys,os +import time +import optparse +import csv +import traceback +#import pandas as pd +import numpy as np +import optparse +import argparse +import datetime +import cPickle as pickle +from collections import defaultdict +def parallel_select(): + ''' + Select a set of (plates,mjds,fibers) from the realesed BOSS data in HDF5 formats. + + Args: + input: HDF5 files list, i.e., source data, [csv file] + output: HDF5 file, to be created or updated + pmf: Plates/mjds/fibers numbers to be quried, [csv file] + + ''' + parser = argparse.ArgumentParser(prog='subset_mpi') + parser.add_argument("input", help="HDF5 input list") + parser.add_argument("output", help="HDF5 output") + parser.add_argument("pmf", help="Plate/mjd/fiber list") + parser.add_argument("--template", help="Create template only,yes/no/all") + parser.add_argument("--mpi", help="using mpi yes/no") + parser.add_argument("--fiber", help="specify fiber csv output") + parser.add_argument("--catalog", help="specify catalog csv output") + parser.add_argument("--datamap", help="specify datamap pickle file") + + opts=parser.parse_args() + + infiles = opts.input + outfile = opts.output + pmflist = opts.pmf + pkfile = "datamappk" + fiberout = "fibercsv" + catalogout = "catalogcsv" + pk_exist=0 + if opts.fiber: + fiberout = opts.fiber + if opts.catalog: + catalogout = opts.catalog + if opts.datamap: + pkfile = opts.datamap + pk_exist=1 + catalog_meta=['plugmap', 'zbest', 'zline', + 'match', 'matchflux', 'matchpos'] + meta=['plugmap', 'zbest', 'zline', + 'photo/match', 'photo/matchflux', 'photo/matchpos'] + if opts.template is None or opts.template=="no": + template=0 + elif opts.template and opts.template=="yes": + template=1 + elif opts.template and opts.template=="all": + template=2 + if opts.mpi is None or opts.mpi=="no": + #starts seirial processing + print ("Try the subset.py or subset command") + sys.exit() + elif opts.mpi and opts.mpi=="yes": + comm =MPI.COMM_WORLD + nproc = comm.Get_size() + rank = comm.Get_rank() + tstartcsv=MPI.Wtime() + (plates,mjds,fibers,hdfsource) = parse_csv(infiles, outfile, pmflist,rank) + tstart=MPI.Wtime() + if rank==0: + print ("Number of processes %d"%nproc) + print ("parse csv time: %.2f"%(tstart-tstartcsv)) + total_files=len(hdfsource) + #distribute the workload evenly to each process + step=int(total_files / nproc)+1 + rank_start =int( rank * step) + rank_end = int(rank_start + step) + if(rank==nproc-1): + rank_end=total_files # adjust the last rank's range + if rank_start>total_files: + rank_start=total_files + range_files=hdfsource[rank_start:rank_end] + #print ("rank:%d,file:%d"%(rank,len(range_files))) + if rank==0: + sample_file=range_files[0] + fiber_dict={} + for i in range(0,len(range_files)): + fiber_item = get_fiberlink(range_files[i],plates,mjds,fibers) + #print ("p:%dm:%df:%d"%(len(plates),len(mjds),len(fibers))) + #print("len(fiber_item):%d"%(len(fiber_item))) + if len(fiber_item)>0: + inter_keys=fiber_dict.viewkeys() & fiber_item.viewkeys() + if len(inter_keys)==0: + fiber_dict.update(fiber_item) + else: + fiber_dict=fiber_union(fiber_dict,fiber_item,inter_keys) + tend=MPI.Wtime() + #print ("rank:%d,fiber_dict:%d"%(rank,len(fiber_dict))) + #rank0 create all, then close an reopen.-Quincey Koziol + counterop = MPI.Op.Create(add_dic, commute=True) #define reduce operation + global_fiber={}#(key, value)->(plates/mjd/fiber/../dataset, (type,shape,filename), unordered + fiber_item_length=len(fiber_dict) + fiber_dict_tmp=fiber_dict + global_fiber= comm.allreduce(fiber_dict_tmp, op=counterop) + treduce=MPI.Wtime() + #print ("rank: ",rank,fiber_dict_tmp_numpy) + if rank==0: + print ("Get fiber metadata costs %.2f"%(tend-tstart)) + print ("Allreduce dics costs %.2f"%((treduce-tend))) + print ("length of global_fiber: %d"%(len(global_fiber))) + # remove duplication of fiberlist and fiberoffsetlist in global_fiber + if rank==0: + tpickle_start=MPI.Wtime() + dup_gf=pickle.dumps(global_fiber) + dup_size=sys.getsizeof(dup_gf) + dup_len=len(global_fiber) + global_fiber=dedup(global_fiber) + dedup_gf=pickle.dumps(global_fiber) + dedup_size=sys.getsizeof(dedup_gf) + dedup_len=len(global_fiber) + tpickle_end=MPI.Wtime() + print("Pickling and dedup cost %.2f seconds"%(tpickle_end-tpickle_start)) + print("Before dedup: %d bytes. After dedup:%d bytes"%(dup_size,dedup_size)) + print("Before dedup: %d pm. After dedup:%d pm"%(dup_len,dedup_len)) + # get datamap,i.e., type and shape of each dataset in coadds and exposures. + if rank==0: + tdmap_start=MPI.Wtime() + if (pk_exist==0): + coaddmap1=coadd_map(hdfsource) + typemap1=type_map(sample_file) + expb_size=4112 + expr_size=4128 + datamap1=(typemap1,coaddmap1,expb_size,expr_size) + pickle.dump(datamap1,open(pkfile,"wb")) + else: + try: + datamap1=pickle.load(open(pkfile,"rb")) + except Exception as e: + print ("loading datamap pickle file:%s error"%pkfile) + sys.exit() + tdmap_end=MPI.Wtime() + if (pk_exist==0): + print("datamap not exists, scanning all files cost %.2f seconds"%(tdmap_end-tdmap_start)) + else: + print("datamap exists, loading cost %.2f seconds"%(tdmap_end-tdmap_start)) + #Create the template using 1 process + if rank==0 and (template==1 or template==2): + temp_start=MPI.Wtime() + try: + create_template(outfile,global_fiber,datamap1,'fiber',rank) + except Exception as e: + traceback.print_exc() + temp_end=MPI.Wtime() + print("template creation cost %.2f"%(temp_end-temp_start)) + tcreated=MPI.Wtime() +############# OVERWRITE THE TEMPLATE WITH ACTUAL DATA ############ + if template ==0 or template==2: + try: + hx = h5py.File(outfile,'a',driver='mpio', comm=MPI.COMM_WORLD) ## collectively open file + hx.atomic=False + except Exception as e: + traceback.print_exc() + topen=MPI.Wtime() + tclose=topen + fiber_copyte=topen + fiber_copyts=topen + catalog_copyts=topen + catalog_copyte=topen + if template==0 or template==2: + fiber_copyts=MPI.Wtime() + overwrite_template(hx,fiber_dict,'fiber') + fiber_copyte=MPI.Wtime() + print("rank:%d\tlength:%d\tcost:%.2f"%(rank,fiber_item_length,fiber_copyte-fiber_copyts)) + #for each fiber, find the catalog, then copy it + #catalog_copyts=MPI.Wtime() + #overwrite_template(hx,catalog_dict,'catalog') + #catalog_copyte=MPI.Wtime() + catalog_copyts=0 + catalog_copyte=0 + comm.Barrier() + tclose_s=MPI.Wtime() + hx.close() + tclose=MPI.Wtime() + #print("rank:%d,fiber cost:%.2f"%(rank,fiber_copyte-fiber_copyts)) + if rank==0: + print ("Overview after template creation:\n1.SSF File open: %.2f\nFiber copy total: %.2f\nCatalog copy total: %.2f\n SSF File close: %.2f\nTotal Cost(including everything): %.2f"%(topen-tcreated,fiber_copyte-fiber_copyts,catalog_copyte-catalog_copyts,tclose-tclose_s,tclose-tstart)) +if __name__=='__main__': + parallel_select() diff --git a/h5boss_py/script/update b/h5boss_py/script/update new file mode 100755 index 0000000..4a8e812 --- /dev/null +++ b/h5boss_py/script/update @@ -0,0 +1,74 @@ +#!/usr/bin/env python +""" +Create an HDF5 file from BOSS data + +TODO: + - include comments in meta/attrs + - platelist quantities +""" +from __future__ import division, print_function +#from __future__ import absolute_import +from h5boss.select import select +from h5boss.select_add import select_add +from h5boss.select_update import select_update +import sys,os +import time +import optparse +import csv +import traceback + + +import argparse + +parser = argparse.ArgumentParser(prog='update',description='Compare base with pmf, add the new plates/mjds/fibers founded in both (pmf,input) into the pre-selected HDF5 base file, then remove the plates/mjds/fibers in pre-selected HDF5 base file that are not found in pmf') +parser.add_argument("base", help="Pre-selected HDF5 file") +parser.add_argument("input", help="HDF5 input list") +parser.add_argument("pmf", help="Plate/mjd/fiber list in csv") +parser.add_argument("--repack", help="repack after changing the file, yes or no") +opts=parser.parse_args() + +pmflist = opts.pmf +infiles = opts.input +outfile = opts.base +repack="no" +if opts.repack=="yes": + print ("Will repack the file for better storage layout afterwards.") + repack="yes" +tstart=time.time() +import pandas as pd +try: + dflist=pd.read_csv(pmflist,delimiter='\n',index_col=None,dtype=str) +except Exception, e: + print("pmf csv read error or not exist:%s"%e,pmflist) + print("Note that the 1st row of csv should be 'plates mjds fibers'") + sys.exit(0) +try: + with open(infiles,'rb') as f: + reader = csv.reader(f) + infile = list(reader) +except Exception, e: + print ("CSV file open error or not exist: %s"%e,infiles) + sys.exit(0) + +infile = [x for sublist in infile for x in sublist] + +if(len(infile)==0): + print("source files list is empty") + sys.exit(0) +if not os.path.isfile(outfile): + print("base file not exists") + sys.exit(0) +print ("Query: Plates/Mjds/Fibers: %d"%len(dflist.index)) +#print ("Input: %d files:"%len(infile),infile[0],"...",infile[-1]) +print ("Input: %d hdf5 files"%len(infile)) +print ("Output: %s "%outfile) +print ("Running Updating:") + +try: + listdf=list(dflist.values.flatten()) + select_update(infile, outfile, listdf,repack) +except Exception, e: + print ("Error in select:") + traceback.print_exc() +tend=time.time()-tstart +print ("Updating complete: %.2f seconds"%tend) diff --git a/h5boss_py/setup.py b/h5boss_py/setup.py new file mode 100755 index 0000000..c483896 --- /dev/null +++ b/h5boss_py/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup + +setup(name='h5boss', + version='0.1', + description='boss data manager', + url='git@github.com:valiantljk/h5boss.git', + author='Jialin Liu, Stephen Bailey', + author_email='jalnliu@lbl.gov, stephenbailey@lbl.gov', + license='BSD', + packages=['h5boss'], + zip_safe=False) + + + + + diff --git a/scripts/boss2hdf5 b/scripts/boss2hdf5 deleted file mode 100755 index d81dec9..0000000 --- a/scripts/boss2hdf5 +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python - -""" -Create an HDF5 file from BOSS data - -TODO: - - include comments in meta/attrs - - platelist quantities -""" - -from __future__ import division, print_function - -import sys, os -import numpy as np -from astropy.io import fits -from astropy.table import Table -import h5boss.io - -import optparse - -parser = optparse.OptionParser(usage = "%prog [options]") -parser.add_option("-i", "--input", type=str, help="input spPlate file") -parser.add_option("-o", "--output", type=str, help="input hdf5 file") -parser.add_option("--test", action="store_true", help="write subset for faster testing") -# parser.add_option("-v", "--verbose", action="store_true", help="more verbose output") - -opts, args = parser.parse_args() - -platefile = opts.input -filedir = os.path.split(os.path.abspath(platefile))[0] - -hdr = fits.getheader(platefile) -plate = hdr['PLATEID'] -mjd = hdr['MJD'] - -#--- Plugmap --- -print('plugmap') -plugmap = Table.read(platefile, 5) -dataname = '{}/{}/plugmap'.format(plate, mjd) -plugmap.write(opts.output, path=dataname, append=True) - -#--- zbest --- -print('zbest') -run1d = hdr['RUN2D'] #- default run1d == run2d -zbestfile = platefile.replace('spPlate', '{}/spZbest'.format(run1d)) -zbest = Table.read(zbestfile, 1) -dataname = '{}/{}/zbest'.format(plate, mjd) -zbest.write(opts.output, path=dataname, append=True) -nfiber = len(zbest) - -#--- zall (skip) --- -pass - -#--- zline --- -print('zline') -zlinefile = zbestfile.replace('spZbest-', 'spZline-') -zline = Table.read(zlinefile, 1) -dataname = '{}/{}/zline'.format(plate, mjd) -zline.write(opts.output, path=dataname, append=True) - -#--- photometric matches --- -print('photo') -photomatchfile = platefile.replace('spPlate-', 'photoMatchPlate-') -photomatch = Table.read(photomatchfile, 1) -photomatch['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) -dataname = '{}/{}/photo/match'.format(plate, mjd) -photomatch.write(opts.output, path=dataname, append=True) - -photoposfile = platefile.replace('spPlate-', 'photoPosPlate-') -photopos = Table.read(photoposfile, 1) -photopos['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) -dataname = '{}/{}/photo/matchpos'.format(plate, mjd) -photopos.write(opts.output, path=dataname, append=True) - -photofluxfile = platefile.replace('spPlate-', 'photoPlate-') -photoflux = Table.read(photofluxfile, 1) -photoflux['FIBERID'] = np.arange(1, nfiber+1, dtype=np.int16) -dataname = '{}/{}/photo/matchflux'.format(plate, mjd) -photoflux.write(opts.output, path=dataname, append=True) - -#--- Coadd --- -print('loading coadds') -coadds = h5boss.io.load_coadds(platefile) - -print('writing coadds') -for i, cx in enumerate(coadds): - if opts.test and i%100 != 0: continue #- TESTING - dataname = '{}/{}/{}/coadd'.format(plate, mjd, i+1) - cx.write(opts.output, path=dataname, append=True) - -#--- Individual exposures --- -#- Parse spPlancomb to get exposures that were used -print('parsing planfile') -planfile = platefile.replace('spPlate-', 'spPlancomb-').replace('.fits', '.par') -framefiles = list() -for line in open(planfile): - if line.startswith('SPEXP '): - tmp = line.split() - tmp = [x+'.gz' for x in tmp[7:-1]] - framefiles.extend(tmp) - -print('individual exposures') -for filename in framefiles: - print(filename) - frame = h5boss.io.load_frame(filedir+'/'+filename) - if ('spFrame-b1' in filename) or ('spFrame-r1' in filename): - offset = 0 - elif ('spFrame-b2' in filename) or ('spFrame-r2' in filename): - offset = 500 - else: - print('huh?', filename) - sys.exit(1) - - for i, fx in enumerate(frame): - if opts.test and i%100 != 0: continue - br = fx.meta['CAMERAS'][0] - expid = fx.meta['EXPOSURE'] - fiber = offset+i+1 - dataname = '{}/{}/{}/exposures/{}/{}'.format(plate, mjd, fiber, expid, br) - fx.write(opts.output, path=dataname, append=True) - - -