[WIP] Port from STORM to peewee and add python3 support

.github/workflows/main.yml

@@ -0,0 +1,22 @@
+name: SAMADhi
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [3.7, 3.8, 3.9]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install tox and SAMADhi
+        run: pip install tox
+      - name: Run tox
+        run: tox -e py

.gitignore

@@ -10,6 +10,7 @@ python/__init__.py
 dist
 build
 eggs
+.eggs
 parts
 bin
 var
@@ -26,6 +27,7 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+.pytest_cache
 
 # Translations
 *.mo

.pre-commit-config.yaml

@@ -0,0 +1,85 @@
+exclude: '(^tests/data/|^html|^data)'
+repos:
+
+- repo: https://github.com/psf/black
+  rev: 21.9b0
+  hooks:
+    - id: black
+      language_version: python3
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.0.1
+  hooks:
+  - id: check-added-large-files
+  - id: check-case-conflict
+  - id: check-merge-conflict
+  - id: trailing-whitespace
+    exclude: '^ext/jetclasses.patch$'
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+  - id: check-yaml
+  - id: check-ast
+  - id: fix-byte-order-marker
+  #- id: check-builtin-literals
+  - id: check-toml
+  - id: debug-statements
+
+- repo: https://github.com/pre-commit/pygrep-hooks
+  rev: v1.9.0
+  hooks:
+  - id: python-check-blanket-noqa
+  - id: python-check-blanket-type-ignore
+  - id: python-no-log-warn
+  - id: python-no-eval
+  - id: python-use-type-annotations
+  - id: rst-backticks
+  - id: rst-directive-colons
+  - id: rst-inline-touching-normal
+
+- repo: https://github.com/PyCQA/isort
+  rev: 5.9.3
+  hooks:
+  - id: isort
+    exclude: '^examples/df_nano.py$'
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v2.29.0
+  hooks:
+  - id: pyupgrade
+    args: ["--py36-plus"]
+
+- repo: https://github.com/asottile/setup-cfg-fmt
+  rev: v1.18.0
+  hooks:
+  - id: setup-cfg-fmt
+
+- repo: https://github.com/asottile/yesqa
+  rev: v1.2.3
+  hooks:
+  - id: yesqa
+    exclude: docs/conf.py
+    additional_dependencies: &flake8_dependencies
+      - flake8-bugbear
+      - flake8-print
+
+- repo: https://github.com/pycqa/flake8
+  rev: 3.9.2
+  hooks:
+  - id: flake8
+    exclude: docs/conf.py
+    additional_dependencies: *flake8_dependencies
+    args: ['--ignore=E501,W503']
+
+- repo: local
+  hooks:
+  - id: disallow-caps
+    name: Disallow improper capitalization
+    language: pygrep
+    entry: PyBind|Numpy|Cmake|CCache|Github|PyTest
+    exclude: .pre-commit-config.yaml
+
+- repo: https://github.com/mgedmin/check-manifest
+  rev: "0.47"
+  hooks:
+  - id: check-manifest
+    stages: [manual]

.travis.yml

@@ -5,4 +5,5 @@ python:
     - "2.7"
     - "3.6"
     - "3.7"
-script: python --version
+install: pip install tox-travis
+script: tox

README.md

@@ -7,7 +7,7 @@ Samādhi in Hinduism, Buddhism, Jainism, Sikhism and yogic schools is a higher l
 
 This project is to develop a database to keep track of samples used by our group for CMS data analysis, and of (groups of) analysis results.
 
-A python interface is provided via the STORM package.
+A python interface is provided via the [peewee](http://docs.peewee-orm.com/en/latest/) package.
 
 Setup inside a CMSSW project area:
 ```
@@ -21,13 +21,13 @@ source installdeps_cmssw.sh   ## only on first use
 scram b
 ```
 
-Standalone setup on ingrid:
+For standalone use the python interface can be installed with setuptools or pip,
+e.g. in a [virtual environment](https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments) with
+```bash
+python -m venv samadhi_env
+source samadhi_env/bin/activate
+pip install git+https://github.com/cp3-llbb/SAMADhi.git
 ```
-source setup_standalone.sh    ## in every new shell
-```
-this will create an install tree and symlink if needed, and otherwise only set some environment variables.
-The python installation used can be customized with the `--python` option (e.g. `--python=/nfs/soft/python/python-2.7.5-sl6_amd64_gcc44/bin/python` on `ingrid-ui2`),
-and the install tree location can be set with the `--install` option.
 
 
 To start the xataface interface in a docker image:

documentation/SAMADhi_examples.py

@@ -1,61 +1,80 @@
-from cp3_llbb.SAMADhi import SAMADhi
+from cp3_llbb.SAMADhi.SAMADhi import SAMADhiDB, Sample
+
 
 # Example method to generate a dictionary relating PAT name and luminosity
-# This version is optimized and only load the needed columns. 
+# This version is optimized and only load the needed columns.
 # We also do an implicit join between dataset and sample.
-def getPATlumi(sampletype=u"mc"): # can be u"mc", u"data", u"%"
-  dbstore = SAMADhi.DbStore()
-  pattuples = dbstore.find(SAMADhi.Sample,SAMADhi.Dataset.dataset_id==SAMADhi.Sample.source_dataset_id,
-                           (SAMADhi.Sample.sampletype==u"PAT") & (SAMADhi.Dataset.datatype.like(sampletype)))
-  luminosities = pattuples.values(SAMADhi.Sample.name, SAMADhi.Sample.luminosity)
-  dictionary = {}
-  for name,lumi in luminosities:
-    dictionary[name]=lumi
-  return dictionary
+def getPATlumi(sampletype="mc"):  # can be "mc", "data", "%"
+    with SAMADhiDB() as db:
+        return {
+            smp.name: smp.luminosity
+            for smp in Sample.select(Sample.name, Sample.luminosity).where(
+                (Sample.sampletype == "PAT") & (Sample.source_dataset.datatype % sampletype)
+            )
+        }
+
 
 # Example method to access a PAT based on the path and access results and dataset
-def getPAT(path=u"%"):
-  dbstore = SAMADhi.DbStore()
-  pattuples = dbstore.find(SAMADhi.Sample,(SAMADhi.Sample.sampletype==u"PAT") & (SAMADhi.Sample.path.like(path)))
-  for pattuple in pattuples:
-    print pattuple
-    print "results obtained from that sample:"
-    for res in pattuple.results:
-      print res
-    print "source dataset:"
-    print pattuple.source_dataset
+def getPAT(path="%"):
+    with SAMADhiDB() as db:
+        for pattuple in Sample.select().where((Sample.sampletype == "PAT") & (Sample.path % path)):
+            print(pattuple)
+            print("results obtained from that sample:")
+            for res in pattuple.results:
+                print(res)
+            print("source dataset:")
+            print(pattuple.source_dataset)
+
 
 # Example to access the weight of an event
 def getWeights(dataset, run, event):
-  dbstore = SAMADhi.DbStore()
-  event = dbstore.find(SAMADhi.Event,(SAMADhi.Event.run_number==run) & (SAMADhi.Event.event_number==event) & (SAMADhi.Event.dataset_id==dataset))
-  theEvent = event.one()
-  for w in theEvent.weights:
-    print "weight for process %s (version %d): %g+/-%g"%(w.process.name,w.version,w.value,w.uncertainty)
+    dbstore = SAMADhi.DbStore()
+    event = dbstore.find(
+        SAMADhi.Event,
+        (SAMADhi.Event.run_number == run)
+        & (SAMADhi.Event.event_number == event)
+        & (SAMADhi.Event.dataset_id == dataset),
+    )
+    theEvent = event.one()
+    for w in theEvent.weights:
+        print(
+            "weight for process %s (version %d): %g+/-%g"
+            % (w.process.name, w.version, w.value, w.uncertainty)
+        )
+
 
 # Get a single event weight
 # Note that I think that the getWeights above will be faster than n times this method.
 def getWeight(dataset, run, event, process, version=None):
-  dbstore = SAMADhi.DbStore()
-  weight = dbstore.find(SAMADhi.Weight, SAMADhi.Weight.event_id==SAMADhi.Event.event_id, 
-                        (SAMADhi.Event.run_number==run) & (SAMADhi.Event.event_number==event) & (SAMADhi.Event.dataset_id==dataset) &
-                        (SAMADhi.Weight.madweight_process==process))
-  if version is None: # take the most recent
-    w = weight.order_by(SAMADhi.Weight.version).last()
-  else:
-    w = weight.find(SAMADhi.Weight.version==version).one()
-  return (w.value, w.uncertainty)
+    dbstore = SAMADhi.DbStore()
+    weight = dbstore.find(
+        SAMADhi.Weight,
+        SAMADhi.Weight.event_id == SAMADhi.Event.event_id,
+        (SAMADhi.Event.run_number == run)
+        & (SAMADhi.Event.event_number == event)
+        & (SAMADhi.Event.dataset_id == dataset)
+        & (SAMADhi.Weight.madweight_process == process),
+    )
+    if version is None:  # take the most recent
+        w = weight.order_by(SAMADhi.Weight.version).last()
+    else:
+        w = weight.find(SAMADhi.Weight.version == version).one()
+    return (w.value, w.uncertainty)
+
 
 # In the example above, you need the dataset id. It can be obtained this way
 # It could be combined in a complex query, but typically you will get this once
 # and avoid doing the joined query for every event.
 def dataset_id(dataset=None, pat=None):
-  dbstore = SAMADhi.DbStore()
-  if dataset is None and pat is not None:
-    dset = dbstore.find(SAMADhi.Dataset,SAMADhi.Dataset.dataset_id==SAMADhi.Sample.source_dataset_id,SAMADhi.Sample.name==pat)
-  elif dataset is not None and pat is None:
-    dset = dbstore.find(SAMADhi.Dataset,SAMADhi.Dataset.name==dataset)
-  else:
-    return 0
-  return dset.one().dataset_id
-
+    dbstore = SAMADhi.DbStore()
+    if dataset is None and pat is not None:
+        dset = dbstore.find(
+            SAMADhi.Dataset,
+            SAMADhi.Dataset.dataset_id == SAMADhi.Sample.source_dataset_id,
+            SAMADhi.Sample.name == pat,
+        )
+    elif dataset is not None and pat is None:
+        dset = dbstore.find(SAMADhi.Dataset, SAMADhi.Dataset.name == dataset)
+    else:
+        return 0
+    return dset.one().dataset_id

installdeps_cmssw.sh

@@ -39,22 +39,24 @@ if [ $? -ne 0 ]; then
     python "${pipinstall}/get-pip.py" --prefix="${pipinstall}" --no-setuptools
   fi
   export PYTHONPATH="${pipinstall}/lib/python${pymajmin}/site-packages:${PYTHONPATH}"
+  python -m pip install --prefix="${pipinstall}" --upgrade Cython
 fi
 
 ## install dependencies
 installpath="${CMSSW_BASE}/install/samadhidep"
-echo "--> Installing MySQL-python and storm"
-python -m pip install --prefix="${installpath}" --ignore-installed --upgrade --upgrade-strategy=only-if-needed MySQL-python storm
+echo "--> Installing peewee and pymysql"
+NO_SQLITE=1 python -m pip install --prefix="${installpath}" --ignore-installed --upgrade peewee pymysql pytest pytest-console-scripts future
 
 # root_interface toolfile
 toolfile="${installpath}/samadhidep.xml"
 cat <<EOF_TOOLFILE >"${toolfile}"
-<tool name="samadhidep" version="1.2.0">
+<tool name="samadhidep" version="2.1.0">
   <info url="https://github.com/cp3-llbb/SAMADhi"/>
   <client>
     <environment name="SAMADHIDEP_BASE" default="${installpath}"/>
     <runtime name="LD_LIBRARY_PATH"     value="\$SAMADHIDEP_BASE/lib" type="path"/>
     <runtime name="PYTHONPATH"          value="\$SAMADHIDEP_BASE/lib/python${pymajmin}/site-packages" type="path"/>
+    <runtime name="PATH"                value="\$SAMADHIDEP_BASE/bin" type="path"/>
   </client>
 </tool>
 EOF_TOOLFILE

pyproject.toml

@@ -0,0 +1,9 @@
+[build-system]
+requires = ["setuptools", "wheel", "setuptools_scm[toml]>=6.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+
+[tool.black]
+line-length = 100
+target-version = ['py37']