Merge pull request #100 from pacificclimate/issue/96

Create generate_manifest script and python module

Author: eyvorchuk

mm_cataloguer/generate_manifest.py

@@ -0,0 +1,21 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from modelmeta import DataFile, DataFileVariable, Ensemble, EnsembleDataFileVariables
+import sys
+
+def list_filepaths(session, ensembles, since, outfile):
+    if ensembles == ["all"]:
+        ensembles = ["all_files"]
+    print(session.query(DataFile.filename).join(DataFileVariable)
+                                          .join(EnsembleDataFileVariables)
+                                          .join(Ensemble)
+                                          .filter(Ensemble.name.in_(ensembles))
+                                          .filter(DataFile.index_time >= since).all(),
+         file=outfile)
+
+def generate_manifest(dsn, ensembles, since, outfile):
+    engine = create_engine(dsn)
+    session = sessionmaker(bind=engine)()
+    if outfile != sys.stdout:
+        outfile = open(outfile, "w")
+    list_filepaths(session, ensembles, since, outfile) 

scripts/generate_manifest

@@ -0,0 +1,21 @@
+#! python
+from argparse import ArgumentParser
+import sys
+from dateutil.parser import parse
+
+from mm_cataloguer.generate_manifest import generate_manifest
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(
+        description='Generate manifest of files requested from database')
+    parser.add_argument("-c", "--connection_string", help="DSN for modelmeta database")
+    parser.add_argument("-e", "--ensembles", nargs="+", default=["all"], 
+                        help="Ensembles from which files should be listed")
+    parser.add_argument("-s", "--since", help="Date after which files should be listed. "
+                                              "Date is parsed using dateutil.parser.parse")
+    parser.add_argument("-o", "--outfile", default=sys.stdout, 
+                        help="Path to file that should contain output")
+    args = parser.parse_args()
+    since = parse(args.since)
+    generate_manifest(args.connection_string, args.ensembles, since, args.outfile)

setup.py

@@ -49,6 +49,7 @@
         scripts/index_netcdf
         scripts/associate_ensemble
         scripts/ncwms_configurator
+        scripts/generate_manifest
     '''.split(),
     classifiers=['Development Status :: 5 - Production/Stable',
                  'Environment :: Console',