h5boss version 2

.classpath

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/Python 2.7.10 (/anaconda/bin/python2.7)"/>
+	<classpathentry exported="true" kind="con" path="org.eclipse.jdt.USER_LIBRARY/h5boss"/>
+	<classpathentry kind="output" path="out/production/h5boss-util"/>
+</classpath>

.gitignore

@@ -1,11 +1,45 @@
+*.err
+*.out
+h5boss_py/demo/*.txt
+h5boss_c/*.txt
+h5boss_c/checkendian
+h5boss_c/checkendian.c
+h5boss_c/iniaa
+h5boss_c/iniaa.c
+h5boss_c/str
+h5boss_c/strtoltest
+h5boss_c/strtoltest.c
+h5boss_py/demo/2979928_10k.out-rank
+h5boss_py/demo/2979975_10k.out-rank
+h5boss_py/demo/2979984_10k.out-rank
+h5boss_py/h5boss/boss2hdf5_v2.ipynb
+h5boss_py/demo/log
+h5boss_py/demo/drop_cache/drop_file_from_page_cache
+h5boss_py/demo/drop_cache/is_file_in_page_cache
+h5boss_py/demo/drop_cache/pagesize
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
-
+*.hdf5
+*.h5
 # C extensions
 *.so
-
+h5boss_c/nodes10k.txt
+h5boss_c/nodes10k_fiber.txt
+h5boss_c/testh5g
+h5boss_c/*.exe
+h5boss_c/*.o
+h5boss_py/demo/drop_file_from_page_cache
+h5boss_py/demo/input-full-cori-output
+h5boss_py/demo/is_file_in_page_cache
+h5boss_py/demo/nodes.txt
+h5boss_py/demo/nodes10k.txt
+h5boss_py/demo/nodes10k_fiber.txt
+h5boss_py/demo/nodes1k.txt
+h5boss_py/demo/pagesize
+h5boss_py/demo/test1.sh
+h5boss_py/demo/pmf-list/*
 # Distribution / packaging
 .Python
 env/

README.md

@@ -1,2 +1,23 @@
-# h5boss
-Exploratory tools for reformatting BOSS spectra as hdf5 files
+# H5Boss
+Exploratory tools for managing BOSS spectra. 
+
+Boss is originally maintained as millions of fits file in thousands of different folders. Accessing and analyzing them are inefficient in terms of I/O bandwidth and programming productivity. In h5boss, we focus on:
+
+1. Reformatting: Preserve the fits file structure and specture hierarchicy using HDF5
+2. Object I/O: Design object interface for accessing the files as pmf indexed object
+3. Query Caching: Develop transparent cache for restoring analysis workflow and reducing metadata overhead
+4. Subset Selection: Support subset selection, moving various interested fiber object and catalog together and save in one file
+4. Data mover: Design API for moving data through various storage tiers. 
+
+Currently, h5boss is implemented in both python and c version, in which the python version is actively maintained and supported. The c version is mainly for I/O sensitive users/applications. 
+
+#Demo with h5boss_py
+1. source cori-setup
+2. cd h5boss_py/demo
+2. subset -h
+3. add -h
+4. update -h
+
+#Demo with h5boss_c
+1. cd h5boss_c
+2. make 

cori-bb

@@ -0,0 +1,5 @@
+#!/bin/bash
+module load python
+module load h5py-parallel
+export PYTHONPATH=`pwd`/h5boss_py:$PYTHONPATH
+export PATH=`pwd`/h5boss_py/scripts:$PATH

cori-conda-serial

@@ -0,0 +1,9 @@
+#!/bin/bash
+export PYTHONPATH=`pwd`/h5boss_py:$PYTHONPATH
+export PATH=`pwd`/h5boss_py/scripts:$PATH
+export basedir=/global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0
+export PYTHONPATH=/global/homes/j/jialin/anaconda2/lib/python2.7/site-packages:$PYTHONPATH
+export HDF5_PATH=/global/homes/j/jialin/packages/serial-hdf5/hdf5-1.8.16/hdf5path
+export H5PY_PATH=/global/homes/j/jialin/packages/h5py-2.6.0/h5pypath
+export PYTHONPATH=$H5PY_PATH/lib/python2.7/site-packages:$PYTHONPATH
+export PATH=/global/homes/j/jialin/anaconda2/bin:$PATH

cori-setup

@@ -0,0 +1,5 @@
+#!/bin/bash
+module load python
+module load h5py-parallel
+export PYTHONPATH=`pwd`/h5boss_py:$PYTHONPATH
+export PATH=`pwd`/h5boss_py/scripts:$PATH

cori-setup-serial

@@ -0,0 +1,6 @@
+#!/bin/bash
+#module load python/3.4-anaconda
+module load python/2.7-anaconda
+export PYTHONPATH=`pwd`/h5boss_py:$PYTHONPATH
+export PATH=`pwd`/h5boss_py/scripts:$PATH
+export basedir=/global/projecta/projectdirs/sdss/data/sdss/dr12/boss/spectro/redux/v5_7_0

h5boss-util.eml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<component inherit-compiler-output="true" jdk="Python 2.7.10 (/anaconda/bin/python2.7)" jdk_type="Python SDK">
+	<output-test url="file://$MODULE_DIR$/out/test/h5boss-util"/>
+	<exclude-output/>
+	<contentEntry url="file://$MODULE_DIR$"/>
+	<levels>
+		<level name="h5boss" value="project"/>
+	</levels>
+</component>