From 9801c57f4a32630df57c0a4cdfbd739b9f348f58 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:04:59 +0000 Subject: [PATCH 01/14] Add json configuration files in installation of python module --- setup.py | 2 + .../.ipynb_checkpoints/__init__-checkpoint.py | 0 topcoffea/analysis/__init__.py | 0 .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 148 bytes topcoffea/analysis/btagMCeff/__init__.py | 0 .../analysis}/btagMCeff/btagMCeff.py | 0 .../analysis}/btagMCeff/run.py | 0 .../.ipynb_checkpoints/run-checkpoint.py | 101 +++++++++++++++ .../work_queue_run-checkpoint.py | 117 ++++++++++++++++++ .../analysis}/topEFT/Genrun.py | 0 .../analysis}/topEFT/Gentopeft.py | 0 topcoffea/analysis/topEFT/__init__.py | 0 .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 155 bytes .../topEFT/__pycache__/topeft.cpython-38.pyc | Bin 0 -> 18648 bytes .../__pycache__/topeftenv.cpython-38.pyc | Bin 0 -> 7267 bytes .../analysis}/topEFT/convert3lEFT.py | 0 .../analysis}/topEFT/drawSliders.py | 0 .../analysis}/topEFT/genPlot.py | 0 .../analysis}/topEFT/make_jsons.py | 0 .../analysis}/topEFT/plot.py | 0 .../analysis}/topEFT/run.py | 0 .../analysis}/topEFT/topeft.py | 0 .../analysis}/topEFT/topeftenv.py | 0 .../analysis}/topEFT/work_queue_run.py | 0 topcoffea/analysis/ttTest/__init__.py | 0 .../analysis}/ttTest/plot.py | 0 {analysis => topcoffea/analysis}/ttTest/tt.py | 0 27 files changed, 220 insertions(+) create mode 100644 topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py create mode 100644 topcoffea/analysis/__init__.py create mode 100644 topcoffea/analysis/__pycache__/__init__.cpython-38.pyc create mode 100644 topcoffea/analysis/btagMCeff/__init__.py rename {analysis => topcoffea/analysis}/btagMCeff/btagMCeff.py (100%) rename {analysis => topcoffea/analysis}/btagMCeff/run.py (100%) create mode 100755 topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py create mode 100644 topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py rename {analysis => topcoffea/analysis}/topEFT/Genrun.py (100%) rename {analysis => topcoffea/analysis}/topEFT/Gentopeft.py (100%) create mode 100644 topcoffea/analysis/topEFT/__init__.py create mode 100644 topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc create mode 100644 topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc create mode 100644 topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc rename {analysis => topcoffea/analysis}/topEFT/convert3lEFT.py (100%) rename {analysis => topcoffea/analysis}/topEFT/drawSliders.py (100%) rename {analysis => topcoffea/analysis}/topEFT/genPlot.py (100%) rename {analysis => topcoffea/analysis}/topEFT/make_jsons.py (100%) rename {analysis => topcoffea/analysis}/topEFT/plot.py (100%) rename {analysis => topcoffea/analysis}/topEFT/run.py (100%) rename {analysis => topcoffea/analysis}/topEFT/topeft.py (100%) rename {analysis => topcoffea/analysis}/topEFT/topeftenv.py (100%) rename {analysis => topcoffea/analysis}/topEFT/work_queue_run.py (100%) create mode 100644 topcoffea/analysis/ttTest/__init__.py rename {analysis => topcoffea/analysis}/ttTest/plot.py (100%) rename {analysis => topcoffea/analysis}/ttTest/tt.py (100%) diff --git a/setup.py b/setup.py index d987cfe98..da094cd84 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,8 @@ "data/btagSF/UL/*.pkl.gz", "data/btagSF/UL/*.csv", "data/btagSF/*.csv", + "json/*.json", + "json/signal_samples/*.json" ], } ) diff --git a/topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py b/topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py new file mode 100644 index 000000000..e69de29bb diff --git a/topcoffea/analysis/__init__.py b/topcoffea/analysis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/topcoffea/analysis/__pycache__/__init__.cpython-38.pyc b/topcoffea/analysis/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41ab022a210617d05b2dec0b3c93f578319a79f5 GIT binary patch literal 148 zcmWIL<>g`k0?EGz6F~H15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Heenx(7s(x~A zv2Ip=S!H6Leo1~oa(-G`Y9bOVF)uNvvN*F?KR!M)FS8^*Uaz3?7Kcr4eoARhsvXG4 I&p^xo02gp0X#fBK literal 0 HcmV?d00001 diff --git a/topcoffea/analysis/btagMCeff/__init__.py b/topcoffea/analysis/btagMCeff/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/analysis/btagMCeff/btagMCeff.py b/topcoffea/analysis/btagMCeff/btagMCeff.py similarity index 100% rename from analysis/btagMCeff/btagMCeff.py rename to topcoffea/analysis/btagMCeff/btagMCeff.py diff --git a/analysis/btagMCeff/run.py b/topcoffea/analysis/btagMCeff/run.py similarity index 100% rename from analysis/btagMCeff/run.py rename to topcoffea/analysis/btagMCeff/run.py diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py new file mode 100755 index 000000000..41ea9282d --- /dev/null +++ b/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +import lz4.frame as lz4f +import pickle +import json +import time +import cloudpickle +import gzip +import os +from optparse import OptionParser + +import uproot +import numpy as np +from coffea import hist, processor +from coffea.util import load, save +from coffea.nanoevents import NanoAODSchema + +import topeft +from topcoffea.modules import samples +from topcoffea.modules import fileReader + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser(description='You can customize your run') + parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') + parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') + parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') + parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk') + parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') + parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') + parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') + parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') + parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients') + + args = parser.parse_args() + cfgfile = args.cfgfile + dotest = args.test + nworkers = int(args.nworkers) + chunksize = int(args.chunksize) + nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks + outname = args.outname + outpath = args.outpath + treename = args.treename + do_errors = args.do_errors + + if dotest: + nchunks = 2 + chunksize = 10000 + nworkers = 1 + print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) + + ### Load samples + if cfgfile != '': + samplesdict = samples.main() + elif os.path.isfile('.samples.coffea'): + print('Using samples form .samples.coffea') + samplesdict = load('.samples.coffea') + else: + print('Execute as [path]/run.py [path]/samples.cfg') + exit() + + flist = {}; xsec = {}; sow = {}; isData = {} + for k in samplesdict.keys(): + samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) + flist[k] = samplesdict[k]['files'] + xsec[k] = samplesdict[k]['xsec'] + sow[k] = samplesdict[k]['nSumOfWeights'] + isData[k]= samplesdict[k]['isData'] + + # Check that all datasets have the same list of WCs + for i,k in enumerate(samplesdict.keys()): + if i == 0: + wc_lst = samplesdict[k]['WCnames'] + if wc_lst != samplesdict[k]['WCnames']: + raise Exception("Not all of the datasets have the same list of WCs.") + + processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) + + # Run the processor and get the output + tstart = time.time() + output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.futures_executor, executor_args={"schema": NanoAODSchema,'workers': nworkers, 'pre_workers': 1}, chunksize=chunksize, maxchunks=nchunks) + dt = time.time() - tstart + + nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) + nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) + print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) + print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) + + # This is taken from the DM photon analysis... + # Pickle is not very fast or memory efficient, will be replaced by something better soon + # with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: + if not outpath.endswith('/'): outpath += '/' + if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) + print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) + with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: + cloudpickle.dump(output, fout) + print('Done!') + + + + diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py new file mode 100644 index 000000000..05ed75686 --- /dev/null +++ b/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +import lz4.frame as lz4f +import pickle +import json +import time +import cloudpickle +import gzip +import os +from optparse import OptionParser + +import uproot +import numpy as np +from coffea import hist, processor +from coffea.util import load, save +from coffea.nanoevents import NanoAODSchema + +import topeft +from topcoffea.modules import samples +from topcoffea.modules import fileReader +import topeftenv + +import argparse +parser = argparse.ArgumentParser(description='You can customize your run') +parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') +parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') +parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') +parser.add_argument('--chunksize','-s' , default=100000 , help = 'Number of events per chunk') +parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') +parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') +parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') +parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') +parser.add_argument('--do-errors' , action='store_true', help = 'Save the w**2 coefficients') + +args = parser.parse_args() +cfgfile = args.cfgfile +dotest = args.test +nworkers = int(args.nworkers) +chunksize = int(args.chunksize) +nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks +outname = args.outname +outpath = args.outpath +treename = args.treename +do_errors = args.do_errors + +if dotest: + nchunks = 2 + chunksize = 10000 + nworkers = 1 + print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) + +### Load samples +if cfgfile != '': + samplesdict = samples.main() +elif os.path.isfile('.samples.coffea'): + print('Using samples form .samples.coffea') + samplesdict = load('.samples.coffea') +else: + print('Execute as [path]/run.py [path]/samples.cfg') + exit() + +flist = {}; xsec = {}; sow = {}; isData = {} +for k in samplesdict.keys(): + samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) + flist[k] = samplesdict[k]['files'] + xsec[k] = samplesdict[k]['xsec'] + sow[k] = samplesdict[k]['nSumOfWeights'] + isData[k]= samplesdict[k]['isData'] + +# Check that all datasets have the same list of WCs +for i,k in enumerate(samplesdict.keys()): + if i == 0: + wc_lst = samplesdict[k]['WCnames'] + if wc_lst != samplesdict[k]['WCnames']: + raise Exception("Not all of the datasets have the same list of WCs.") + +processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) + +executor_args = {#'flatten': True, #used for all executors + 'compression': 0, #used for all executors + 'cores': 2, + 'disk': 5000, #MB + 'memory': 10000, #MB + 'resource-monitor': True, + 'debug-log': 'debug.log', + 'transactions-log': 'tr.log', + 'stats-log': 'stats.log', + 'verbose': False, + 'port': [9123,9130], + 'environment-file': topeftenv.get_environment(), + 'master-name': '{}-workqueue-coffea'.format(os.environ['USER']), + 'print-stdout': True, + 'skipbadfiles': False, + 'schema': NanoAODSchema, + 'extra-input-files': ["topeft.py"] +} + +# Run the processor and get the output +tstart = time.time() +output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks) +#output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks, extra-input-files=["topeft.py"]) +dt = time.time() - tstart + +nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) +nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) +print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) +print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) + +# This is taken from the DM photon analysis... +# Pickle is not very fast or memory efficient, will be replaced by something better soon +# with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: +if not outpath.endswith('/'): outpath += '/' +if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) +print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) +with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: + cloudpickle.dump(output, fout) +print('Done!') + diff --git a/analysis/topEFT/Genrun.py b/topcoffea/analysis/topEFT/Genrun.py similarity index 100% rename from analysis/topEFT/Genrun.py rename to topcoffea/analysis/topEFT/Genrun.py diff --git a/analysis/topEFT/Gentopeft.py b/topcoffea/analysis/topEFT/Gentopeft.py similarity index 100% rename from analysis/topEFT/Gentopeft.py rename to topcoffea/analysis/topEFT/Gentopeft.py diff --git a/topcoffea/analysis/topEFT/__init__.py b/topcoffea/analysis/topEFT/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca8cca7a5e9d14632ec6078762204275a04cc61c GIT binary patch literal 155 zcmWIL<>g`kf|S1p6F~H15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6enx(7s(x~A zv2Ip=S!H6Leo1~oa(-G`Y9bOVF)uNvvN*FCB;@KAq8}fhnU`4-AFo$Xd5gm)H$SB` NC)EyQ@@F7s002(`CL90& literal 0 HcmV?d00001 diff --git a/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02a68d8510b6ab5cff39978cc1d501d3577f6c3e GIT binary patch literal 18648 zcmb_kd3YPwamV5&2!a$viMmA{mPJ|CLCLplOX8u!Iw+Br?6qVF#9M*@u>iEYAVvDE zd{A~S(@|n4X%aUPlemr3#!1_xN9?9a+NNplqq#O|d?ab3HvRg2ZNGkL|7m|Sy8r=7 zw&lp&Z)a!b&6_uG-ZgLWmUlNb#eDd?`0rml_5Szyd_O0m_AdltCvNk<0fbK|d7rMR ze#ONDYJhn`HOSA98scYI4f8XiM)}#G#OeW|+ClUc?x zvHWs>NoyFB-Axpv$jSwxBI3Pb*;8a zTZ`v4LK*ONwp>KreTSTIHd81XT4%rsi|kldXig+s7)#}|!U-0~G^5k+1V^%_uFnAWK{@xvu6TP*aa3=SSN;fJjk_DS`6EzLk-nr)`T-Toe<5H6 zg(Ce{=t7u!$_!x<5g;3=?3vJ15S*xp&XH(<9V8K&3C#ppqOndQEaD>0+QOnqGp zTSeDZdwiSN{w8UC(>tFB$mxWjA`5w)YILEQTEycwFk8myS~PZ8EA z1AuYi!(4dVAvHcw)(;OK*Rq)r%XGqKN-yO#rBs9-`g2we07IH8`h zu1`22L66CnLuGwPGdhaH9om>yAZ5MoGm;)j6$)D338c$dcFeL4hLO>nm^oounx3+< z>B)ryRlMWwLRvGdRJLGEIN@}$%*wjmCoN`r$}}eztkshboY3-ly6v?c>-T8KIy)}5 zk$GQX^9opa%*lECHLF7}=dElhpG{}2iH?gDOgQ1f5Y{1cvXvfl1Spu)e_MUwT_o~fBoZFaYo(F}D@(}-2JQPF0C~J>r4!V)AaTh7 zEhVyrOb4y29j>HTk&BDTgrHt0rk9I_dNC)MtCK*DJLX7)v@M#Inw(!>)(P;|UR;4f zGPv;<{XzxV>4fz9EN;1+#R=u=)7%o>H2;V-*|c{pRoJUW_E%WGJLee-QG&6Qz$yZ3 z2&^MOPBk_V*hpX#fz1TA5+HvX+X&o1;6?&B0XU7^wr&ezMUN)kerq~S$?Q)vQ~OJ)mME14w!ndBj#5>~>&pt2g*{@1wFe~-1rl*uKR z#5F6PU6!8G=PF}M7Pm$(CIm9iIKTDB*ZX6qnIfB%Mi9&WDk zGW1QUI4+Kn1H*5LBp2g+PU?R2Wo5L?{TQ093>?VLK|L zJ-(Am>;~B&{i9(+NdE;tLOn$!AniXbP=3IU0S4_xz>pmW4BJhB5j$Z;Wjxn_JBGUv zcYG=+bz;BaQ#-vCfSPc6_&xuCfVZR@NN^^2`Sjw#GYyhXB&pYZo(>bc~y?YY$IlnJqN%75N({6?%S_(g}k zOh(QJ%rjz@Oyo8!@&P8=0LAJZz8cXbR@}GY z-hun}sVKCdMd4ZWp{keB@2g%;u$L2%rg|*oc3jE6TnS%xz(aS)mE04qMY*da($L#o z&H~}~_SG^d){pwn2aScI(_T{uiVgM}$lNK{j0Wws*5(U4p=F7|<12mUU( z3fQ$(Y<(5$gm**cQ&f_@0o>idnjw1~p569Ft4CfZH^_}s0sHz2E>?-_ri1pTD&5BP zW~~5qVmb=7q(IvKJ{9^lb!0wSfxqj5WCwHV;_g?Oa;1jpqm%C2fun2uww2b=f zKG{ch*?U;mZs@vE+~n!nEBEG(Lf78hKvma1)&)Ls=jhrm_p`3MWRL8XecW;f7~jMA zLAh7%llvJz#Q0r|ANJbh0JVvSA0!-g?DyzHHED;vv?K66)i%M9+Kg&V)RCGNJPIkQ zM}i@>G1Z5t0d8@(od;f5m)qWhudB_KyZd$Zx&7nV>*{mmj$4D`X4x-qmq&&J3-J<6 zp}BcI{M<|P68G_5=IoREs+@hyK^s>&eazV-_f$D(dFVmgRXO{abCPq|E8HJAL+jZ$KV% zh<$=0~tJSp!%|GOXULzX2NQXeA=iE{hcJ;3jA`_WX) zPF}e*&-;z`sW*^=R)$79RWldD|8wSyp_+NrO|6|r)9m@7x=}8q;88BTQC{&pxJa2Z z#^Fir4ak${huP1X)T(~gm@_PgtDIrd>6T_^WJadp`4O4PU1w+INx+=EA8=HT0OsK_ zvN*wz#xq$?lslf&z|-#dM~moJF735Q;SJ<)TcpT7N3?d?Ilfdg$3tdLd!6PMDd80+ z=5TRKCcv|}!Q9i}Vm21HdR)_!t)1^JPqKF2E`xWsEY*##G3K4FYx!~1)Zo5Nkova4 zeVd?59fL28+_9=R&mf1%z9&e&H`(_DUFtY;n7QLj1>ygRH&%w%29s~B952@cZ?G)4 z_2e5Y%gtrWGS5$C_~D)z#lr^(M?WC$en707qXAyy&cgrqvea4h^s{y2>bB7!YQS+R zkBqbK9E-8@Aabd%nhO!To6CMkS}bLu9V^yYXhqi=;_jD3SFCxsYR$u#@gANtzt&sx zaMhYes@7aC*Q_;DT#G3=RUO~wnDdByq{?}eISR!gx24okw@bV9E>n<-YAMk_To^i)$$bK3# z4(^&r@^>*qJzW=3e+Yg%0d5?z9`79N56cgOcT#?s_f5BpJNTWz1$hBKZ~7j^?oRA@ z)IUt`NoM4XwOh{Q?wL}=oukD0NDXHvb3V$x`3R!%k1_U9V9&^p%8$`5Rz5=zGvP;T zZ2Ud>kvXX!sY(5P*mhX{e%+k*apwJA-JJF;M)SvcP9sRO*vENJBS_Sy6K_H$FLME&VJk`YiVag4Ft-<-S1BrD!GlthbVV2KZ;#OVmgH^ta9HdE?K1 z>-?@a{+zu*yT)h4j!VYTU$8g8N2zaAq)(=Mf+=6pZ#@l0B^th?QdA=oVb9~A`&}!{5<(}`3<}o1YGR9 zFf0y|R^tizwL(mWWO%7heyz!O=;9+pStpgcx= z@UZ)Zw!`ucd8gdz@p?SofV^8C_juR=v)mauDW%859+-I(@&RdkJnWsBr^|vYdc3_J zFDXySl*jA$cwdlTlwb0AM?BtN%FoNc@_0u*-k-`plYj2<20Y%U~D)>;<&g++$+>f$o`J}HtKdv_Q_tXy??#a^*8a3%RQIy?!h~OeUST`8o7JrU6=4w zd7J!OS4P|?PKcA@{!GyRE|-Pe(bjE=#x^De_xY+zZ|a8-xW!5N~FY4CU}$2 z)0dRzy^#NQTlphqOr=v(Yjg%#<-(aj6JQLVb=2?sp zuMBK8^Trq>-Z*|=sF}gQIkN-5lG{I#5pg?y6?DcmS_QZ20{rg|IlAQIPJ- z)lYhS=YuCd zWdBS$7fLferXD}xw_mhhlHZZvZTHFVwYyxyepx)i>B8=o|G?NW`xW^!`{(j!Vrn!D z+rErBfT*)<=}V}|>B>B7|DteAzAV4Xa=(!;BYI!)D8=Z%EGz^6JK!%jTA07w;|Jx- z_AfCXFE`d#>Lh=Tc_fax_?J{eqU>MDmotidC4;xJvYVj&Px9Njd*wep;j{l){zCpU zB*_oI!Y@R-Btb)*!>=zFy*X;R@euq0DW1Xp1)iaqYq{}W`K>DVU*(IUh&nAd{tFg9 zN_8-wgVo<-t3Q8!+3DvBt(XlD%9rHJ@G!K&vR}jVMfUu^$bZGbmzU)$vRnRI{t6y? zY`OvK2l7GlX=q+ykax%nta(luD~t}3y=)1mU!{b%R7-e??142eicv1nZ>{EfQRGZKSRinj%mJZ`qXn8lZbQp;%*7782DSQ=eYqE~Bo+TZ|(JR*R6zg~lI?Al$ zOI17Gd&N55!#Z|D#~5@hGoJJO+F>|XtmC~Ek08g#aDM>zIoxCNr#LS5zC%t#q^y*w z;h+qiA>MT+YaYU38z(rSr3@!{#?;bIQ=$GyS=Q`99yKLgLIUQ zlJ^vg;^P0|Z;;MBr(mlS9o4jw$m-_BW@O8J;QjRX?46Wf7|?*T(q?yFOBIZ@v*|y+ z)1cE>7XtulxMU>@Mcwh2EMrlHVWh^@D4j}65=J5!t>6TVks=PS#PX@hiDWvJ9>LLS z<2dqUUyh*gpZw3++zZzC?#%pR>OUn+MQ7*PA#L?j6hPxKGnIc4&He zBM!ak<&zz^cieYPc4LQopM<_HWbDd7KmOM4yodwPzCIL7mOBk8Gp!Zq(AvdD&=;GC z>DRU)3vNxr6TZ=OC!m1{=%9cB=;$ROJ{agUz&Y-LLmb@-r1E*A2rWj5z-a;o0h545 zpbUT$bEJ@O^$iV=+RkpavF+cqt7oMoCdvyQfAHRPU_qf>CRC+IqxjE9Kv zFo8!1Oc6Lo;86mP5qJxMw-R_8fwvQQ2Z6^4tO4kZUa>eI*W;6Vd_Iql>+vByKC{Os z^;SWn6Tx7~;%u85;YPW;q&6_pQyeciQMbXsjJX(b)o7vU9y>M8kYRLourp*VsNgtT z$%j&%20E9Qrbcv{2*t5HmqcfsCg~^KoagaSL;HJ=qMQgGTnOj^mmVM*LntJyiIUbC zbREM!VLy?JIvb29N#j!l*plUh(fLVZB?0oD@ooYg0Iw^~-n+awx9sJLxV$)@oxF1K zb!6g|i?4rOaf43$b;fGk!$rJ)#?FX)7WB%E;ZF71;-Q+}&n?LPo92MJCgFbjdYT@h zy7P$Qk-~j}rj$BO-0xpoQ*GDbv2x|&H;@TeF3v;mwZ*Xn;#hNvR&nDNXm>&;jz!}L zEY3MQ(dr@_q=ncC45rFXV83RmF?X@$#XBI*OsIY_sfNsKX`8^nE*4E~pHLgUwU?xf z6Qn$VqSQob_^6iOXBM|?OJcHD!}?fiaB#1h&^3`Q>-Y#qFIkk`qJ_KLJsR#$jXGf* z64z5_jJwGTg8&sX$V{~b(@M6#&sEKeqxqs~?agA=tXRT=36$4|g16putP57C(R8se zc1*L1Q~@oksezHArZ#E&3S$Seg|gWX#89cY2DjGmf;taDffyX^w+`bIDlqyjHJTN> z3Rd=D8FGxFG&PI>fC`zH6+K!yHF2;gv_sl>&q31(6w1028A+Kb%QDmk2=?(uQCPwd zIXF%9Ar+1-r%alLoQC0SK95MFps9^GA&=lea8Thdl$^*gzSgp~bvZ3V20q(LBTF)6 z7^w+UT{xUQqlrC6RvaBE_8|ew$F6)H$LJ~kFpU;!me5c+n-}Cwj+M~BP9%e-BD+G! zfW`Uf6IBTS}8m~1L^7YnA9waUD925_WbZJ-&v2jSIj#9jq>2}k{5 zJjY(!^Cr%)DdCjurum$QrlWz;ESV+n0HXk;@P%~0c( zi}V$Z?Rn!?q|@gMbi7+lRL*xNtzt26IuSPjLa#^(;Y6)s61}4|VeBH!J4tgQoyw<^ z<6Z8OW*QW38T$bobUe5nUo!B5;{>UvIH4@Qq%iS`A6)JPP~Fad#AI7s;q z5jaespFlf-qX16Si|b8C5pINWir9V~6M%(TK*^~w8UhIXl6qN<=;dSy(KjvTE)!i} zb6KGR54%u-XNzEghEK#AFCNfJkhew*O=FaJEJ+B@8QgZ%Y-$5~VacL%{A%2_-wmMQ z6&&Bkt^uO~BBmP6Ajn4tHr-%|7Ei~Y+oEzPIF7g?YwkwBBnh<%o0TGeeaIPjiFyzj zjCWCyUM$j60`(*>7lx9h z(2oaYt{P0NxN9NpLqJu`M@o(PZek6Cl;JB31d5oHsqUTxv!Xf{pCqDkR0^vEV_{9> zXQTII`x7p4xKc$_m#WBuF^prVX55Wd8TG7%aRO8;4muA-ilpCv1+YZd)+YdP5JeJtJsWu~QL&rDS zcJxcNrH+A_a#n=uR42j=5{XwcLBxrUr40TYPmR*Yo(J&KPz{b0%a+ZqR+n7iHeRMxbcS?4kF5{+N=eUM)&giTYXs_zKUxf=iSel{WuEB?h@zV1v@9 zv?|Lyx+T!1bSQB*4KNNI`IUvr!ay5zwBcHzEK*Ryf=z)Ar3F}vvd|y%g_R(1@B;*6 zRF*FeZXD??u!gim{p%IJ{H-n>SpjV=&>2+{{vhxUr6T~Izr#-@EklmVlPsZY8Ejpr ztaJ4!>;0{PR@h&?qNK6SzeU+XJlG%gZ&JFHF4o!x?kwN)M@>3v$u&foe&m6l+-nP- z>!A~{Qd(fGver*^BKs;38Ur2=HEbmxMB{8nD{Cem`J8P;zMV>s(&Hy;p|S(i&P(t` z@HI=hFfiM$gtde=27+ArYGo07imaJ)1y?Am(Q=6@zXe{QHo@ihp1>01S@$9?=r)#t z1+%51jbbQ|TGFq)%JrYImN_ZEiN)ML;lftfj9%jpqyM#_HDlx<@{2zRoYMj^UqYdq zz8j@9vY7f{Old=(YE{;Px55`B46P9Jfv()>Lz_T#kVZ(v{3JUUTV3T{4%@CwUgn!j zZ=BVAlj+BBQ*d|VHtFTF#J@M;FF|>j;^p3MFnKF9cic9X@CLxsnSmbnYf$BKU!vAX z9_nnMk8Ek0N+$7zZ&ELca-Q&bGI_e3%DXAX9u(_Dp&tLH1e-6~ix~$9e4oG%2=vy2 zJ|f!ymf$w&MJ5L2azDB&?qKvmpL=a!8L~mgNAJnxWMb21ci-5DIMm!cnee_a-sEyS z*EtRO$?cnl4f^dRaSvZ-0&FQM?m!x0%ln9!ZUl zj;9P^e2AzpwmW!(%AZBM-%VrhOMBN%WhD1yvvp6{f8O% zlkfRtf4sl)Z(nxp?q<&$KctF8uqu^O*q>6z!(P4!zj^;4=W^}pM0dD#x`R5bxdp(T e*Lf^sJi9Xi=2wg+%(_ZTq_xuEgHeq7xbpv{0XPK! literal 0 HcmV?d00001 diff --git a/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92ffd4d8773d148881fbe15f61db0e2edb605323 GIT binary patch literal 7267 zcmbVRON<-Id9LaQ*=){(Lye@7G#a(2hnE|1^(J8Rf$Z-696g98Ks6c{<=;Dc`oa>+Fa5};23a?&}61e1sJ{ng|Q zXJk7_D0NkJbv^(3|L_0)nh$4ZZH8a-AOHH}f4acfztGF{&%n#uc*6fiBADQ9R!wtW znfosytK8s61QEs@$qtD$iAODz~e)XN~fEtUBXatyw&CLKFID z8fU_&&I_|z@Yra7CKPenBE9Rc_>M1cV3aB|P7DN&G zk~kq2k)IJK#VO=x#c8pG{G7Nb&WUC8Sbm3z^J3*W6D!{NUA?;ET@V*Gwa=N{_Z)9| zm+f+_gnz)wrI(NaTN_-t@|qjgBfsO7!<$!LYkQBqcGn%hb-XZioA^@J5>tIjGL1lX z+^A}J-N)hJ=k`3p-)+le-U?(H{L z-&%d6bj>NHogQ6NgHn6?rL`4I-1O3Js#&Gy);Fn>@Y`+5w`CASA|0mD@Z8d)gJFiM zG)%`W^+Qh{S=o4AtmvS})EY`XzlY{i-_p_Sp~jzktNI#6PZyi z*7q>V9_K8wW22R)D8(w|07WAWDKT0wmJiPb2A3jWqcinc|6Yr;u)aXgJ9WW|&x*qNYmR zd%@&6EOP%#*EfQWcfH;TE3M%1j@!K+1-*JeI&t0Yy6v6N52@<@gNKxPjR+I0_I8qD zP4>GrB`c(An%6L3IERGs6;0!dyvR>+o6Ga4sgp=1KPqSO+{F`KLlUtO7ks1%ZKTKA z9yDMNDbCA?nb3t1^M(;?Kh}OoXJoDoF8oe6jIbNX1cG-UjezpUzU!zJ?el9%)=%5) zYn3_#lObVxGBJg>-fyZCk_(txt*)#m7WP%^xzPrsw&8ZWUOO>*o6S&(DNUBqJULw( zk2~HwSr+U*yg_>@B4K=qYm{c1<7@i6px=C`fOs?@NykK3t8qqrv%KE+klKqT~D$E!>a} z<{x|sRgj~SsfKGz=G!Fkavm z)NL$xrZMfm(p_g+{$6p>}RCObik+?**MnEReNiyeLgmJ;c8TU4)qGV z2(MyY*t@iLYMoLj zdU~`ZinRAJidxUC35rH%MrUdDSn+sI9i-R$_Mvm# zw3W+l3yrEnUyKQhQ=6_n11ASLWMU<{bimS%}_E+34yaDkdh}TA=8m3k-)EQ zz>l?)jO;|x^UAqob~0#f!wol*+{8}9#KghWz3S}2>5;Lg174;9uTWB<_TkgrUeANW&V+%CJW)2P77bdH?i09u$Qhm8nGc8%8ywPBEXiwK|rLjc8hf_0F^3z#I`KJP8M|T zF9134Y=dRhEB9)zG3=PnOoE*)fX|i{(~;O~iQe?1K~8#)!Fgo}LA`tbEBDGg(d$o0 ztdSe_Q&vYx+;;u$V6jr^cOwZK@d{3fSV^Q`NGrT7Y0}E z2K}~hxFD3l+lZt1`&Il+2nD%7~zlZ7EAR zqG@CrWU7Qg%6&IskdJ1D{8JY5GwhGFE#reJ+<^1YR==3=-U}|ji7r#rz`jNF8lvgl z`HTij0YGDeOC$;KYi}%()mzFa!aFETddS~HmxM_&f|m-V%_K`;M_dMg=-znqHxxw# zl$Jg+A`{#YBBn42Q7bHPs2%%ms_&VD?>Ho|MFnwywyQ(fV$3(Q z^1UcC$_fNa2ZTjKzyv@9VR#P~`HT8|@+w_5{^yuOHbM$(Xc>bXkQ6iVs0k zT3Q~JFZkLEJ<$T78(C;#fgp4!(1t(fjpHuFy47|&>%zVBCm8Kt@vy;4t={%r#3SV9 zrw==ANcO+!?F{TjA0ZI=-~ZF{WE*qH4kcTZ9EDvOqV7+rK0-3M^icZ101ynd81V{i z01GMG@_n>TjAlDnmtE>hl1T%STU1i;a6p-X8eS)O>>-Y^)u^@5AHW5VQAg3{u$r3) zAH~T=Ln&#vqbp8krp5KgXfS5c4tL;WEx-f7LFJMFTcEuRK)Ile(=r9xj#b zG>k&TqoPKNp`8kN>Ez^UAi?VhY==&z?>rq7MZ8qTGmlrI?j}&Hn3z+#JXuA_#%d=a zo7yJ6I12h4TAGdU4fFl~S3$!~#!ok^?spJB7mjZs$&Qz=RC|y+5a<*wE*#f$3;eZ5 zJ~%0^L8{^*UIXdMod;BBK9L}}AfW9&;#N{Z#cdOVPT*HF+_EWSYCoie$Z1s_o%HH{ z&-*3LbiC0+XS#=eV`o|bJ+*R(Gnw{pmQ9ZJEaFdGcI=`}k{!n-FA4LGSPyY=TDWqp ze*;}hDeAEh6j48+t#YLnNGPsA6iwBm+%9Z^F0^*}_f&}WDceOLNP!}-o$cxf3CBS# zuDZ<89O{)fXVogpDtM9xcSyF8?-N z&gH44;6oM_M<>M0Xim(I77<3`J~p<96cDo`i;S84chzPdv*WUj>J;3g+&YPXxNr~< zqh@q!k0H!-&;xh3Q$3cjMhb}$Q0}oG{a^_H1Rqu$+R-_zKNs7ygU|Was_I!h?1=z( zxpkiIf8(4u8L4p);4ZgT#HpzOce!;z1-O3+bjd4papHXIqC%I_=#n@+g)aGL`AL9# zh%V{Kpf^iv59M{7sbUh+LuoKK!C!>7_%CnJXW|=2yy6ieM4Ukpp~VRi5)2}T!%hiN znX3U(D6gUM3tr8W;it@SOh0H8Sx54>ctYaz8R+N?+!9zSdA`Zz;IxyRFrvh0xPDtc zN3X$~Uq1}3r@A@oUfso|Fs_DPCB_J^wQJ5RTO^O69RCsuYKw&-R~iA`XlYY?547qa zPL|(BYe^eD=DpPR{toqrbnoP``+)Q^K)ykp-=yR%N`8})o0NQ+5?WRoonM}sRe9gr z^r=Bf6UkI8+zNxPq_9f*lu!_*=(D17Dj+=!|fnM{uZ@- zn-b;nRcQYVuSxL$0Po>$uO0<*MtjumYNp*;An z`e%S|q``hc1^YH}j;S#JDwp5KIHb=mXLF)yn)qS%08R zM1BVi$`^4dPfDS*rHmSRmlEZi7OAAp(I7G_=jr`1B}Dm>97Oh>x|LS?kkEaYU%wl4 z8-6n}@7@3H58shW9Y3YkW;<|2D8EA8=|+9(mgil1J*JDq+fr2_3Cb!Zv^DtyO4J?D zAt^lAePZ28MRkXEAClRmCv_~v8RDA8FbcK~@0YW3_GN3qDxx%NowQ#s=r;c!#u58% literal 0 HcmV?d00001 diff --git a/analysis/topEFT/convert3lEFT.py b/topcoffea/analysis/topEFT/convert3lEFT.py similarity index 100% rename from analysis/topEFT/convert3lEFT.py rename to topcoffea/analysis/topEFT/convert3lEFT.py diff --git a/analysis/topEFT/drawSliders.py b/topcoffea/analysis/topEFT/drawSliders.py similarity index 100% rename from analysis/topEFT/drawSliders.py rename to topcoffea/analysis/topEFT/drawSliders.py diff --git a/analysis/topEFT/genPlot.py b/topcoffea/analysis/topEFT/genPlot.py similarity index 100% rename from analysis/topEFT/genPlot.py rename to topcoffea/analysis/topEFT/genPlot.py diff --git a/analysis/topEFT/make_jsons.py b/topcoffea/analysis/topEFT/make_jsons.py similarity index 100% rename from analysis/topEFT/make_jsons.py rename to topcoffea/analysis/topEFT/make_jsons.py diff --git a/analysis/topEFT/plot.py b/topcoffea/analysis/topEFT/plot.py similarity index 100% rename from analysis/topEFT/plot.py rename to topcoffea/analysis/topEFT/plot.py diff --git a/analysis/topEFT/run.py b/topcoffea/analysis/topEFT/run.py similarity index 100% rename from analysis/topEFT/run.py rename to topcoffea/analysis/topEFT/run.py diff --git a/analysis/topEFT/topeft.py b/topcoffea/analysis/topEFT/topeft.py similarity index 100% rename from analysis/topEFT/topeft.py rename to topcoffea/analysis/topEFT/topeft.py diff --git a/analysis/topEFT/topeftenv.py b/topcoffea/analysis/topEFT/topeftenv.py similarity index 100% rename from analysis/topEFT/topeftenv.py rename to topcoffea/analysis/topEFT/topeftenv.py diff --git a/analysis/topEFT/work_queue_run.py b/topcoffea/analysis/topEFT/work_queue_run.py similarity index 100% rename from analysis/topEFT/work_queue_run.py rename to topcoffea/analysis/topEFT/work_queue_run.py diff --git a/topcoffea/analysis/ttTest/__init__.py b/topcoffea/analysis/ttTest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/analysis/ttTest/plot.py b/topcoffea/analysis/ttTest/plot.py similarity index 100% rename from analysis/ttTest/plot.py rename to topcoffea/analysis/ttTest/plot.py diff --git a/analysis/ttTest/tt.py b/topcoffea/analysis/ttTest/tt.py similarity index 100% rename from analysis/ttTest/tt.py rename to topcoffea/analysis/ttTest/tt.py From 70caa2422478defd9afe8d097d74a0429314ebb6 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:08:09 +0000 Subject: [PATCH 02/14] Add UNL specific file provided by Xuan (works on coffea-casa) --- topcoffea/json/TTZToLLNuNu_M10.json | 32 +++++++++++++++++++++++++++++ topcoffea/json/__init__.py | 0 2 files changed, 32 insertions(+) create mode 100644 topcoffea/json/TTZToLLNuNu_M10.json create mode 100644 topcoffea/json/__init__.py diff --git a/topcoffea/json/TTZToLLNuNu_M10.json b/topcoffea/json/TTZToLLNuNu_M10.json new file mode 100644 index 000000000..1498dc810 --- /dev/null +++ b/topcoffea/json/TTZToLLNuNu_M10.json @@ -0,0 +1,32 @@ +{ + "xsec": 0.2529, + "year": "2018", + "treeName": "Events", + "histAxisName": "", + "options": "", + "WCnames": [], + "files": [ + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root", + "/store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root" + ], + "nEvents": 14542666, + "nGenEvents": 4876491.715182641, + "nSumOfWeights": 19992000, + "isData": false +} \ No newline at end of file diff --git a/topcoffea/json/__init__.py b/topcoffea/json/__init__.py new file mode 100644 index 000000000..e69de29bb From cf20f3e2b0a17458b073904e7dea11feded29eb6 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:08:48 +0000 Subject: [PATCH 03/14] Add a coffea-casa notebook itself --- topeft-coffea-casa.ipynb | 368 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 topeft-coffea-casa.ipynb diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb new file mode 100644 index 000000000..3608c1de9 --- /dev/null +++ b/topeft-coffea-casa.ipynb @@ -0,0 +1,368 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "f467b886", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", + "Obtaining file:///home/cms-jovyan/topcoffea\n", + "Installing collected packages: topcoffea\n", + " Attempting uninstall: topcoffea\n", + " Found existing installation: topcoffea 0.0.0\n", + " Uninstalling topcoffea-0.0.0:\n", + " Successfully uninstalled topcoffea-0.0.0\n", + " Running setup.py develop for topcoffea\n", + "Successfully installed topcoffea\n" + ] + } + ], + "source": [ + "#!pip install git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\n", + "! pip install -e ." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "781aeeb6", + "metadata": {}, + "outputs": [], + "source": [ + "import lz4.frame as lz4f\n", + "import pickle\n", + "import json\n", + "import time\n", + "import cloudpickle\n", + "import gzip\n", + "import os\n", + "from optparse import OptionParser\n", + "\n", + "import uproot\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.util import load, save\n", + "from coffea.nanoevents import NanoAODSchema\n", + "\n", + "from topcoffea.modules import samples\n", + "from topcoffea.modules import fileReader\n", + "\n", + "#FIXME: analysis is not installed anywhere (should be installed as well)\n", + "import topcoffea.analysis.topEFT.topeft\n", + "\n", + "import importlib.resources\n", + "\n", + "if hasattr(__builtins__,'__IPYTHON__'):\n", + " import sys\n", + " sys.argv = ['']\n", + " \n", + "from dask.distributed import Client, Worker, WorkerPlugin\n", + "import os\n", + "from typing import List\n", + "class DependencyInstaller(WorkerPlugin):\n", + " def __init__(self, dependencies: List[str]):\n", + " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", + " def setup(self, worker: Worker):\n", + " os.system(f\"pip install {self._depencendies}\")\n", + "dependency_installer = DependencyInstaller([\n", + " \"git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\"\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0286ca63", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" + ] + } + ], + "source": [ + "import argparse\n", + "parser = argparse.ArgumentParser(description='You can customize your run')\n", + "parser.add_argument('jsonFiles' , nargs='?', help = 'Json file(s) containing files and metadata')\n", + "parser.add_argument('--prefix', '-r' , nargs='?', help = 'Prefix or redirector to look for the files')\n", + "parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks')\n", + "parser.add_argument('--pretend' , action='store_true' , help = 'Read json files but, not execute the analysis')\n", + "#parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers')\n", + "parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk')\n", + "parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks')\n", + "parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms')\n", + "parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory')\n", + "parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files')\n", + "parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients')\n", + "\n", + "args = parser.parse_args()\n", + "\n", + "if args.jsonFiles is not None:\n", + " jsonFiles = args.jsonFiles\n", + " print('jsonFiles {}'.format(args.jsonFiles))\n", + "else:\n", + " with importlib.resources.path(\"topcoffea.json\", \"TTZToLLNuNu_M10.json\") as path:\n", + " jsonFiles = str(path)\n", + " print('jsonFile was selected for UNL {}'.format(jsonFiles))\n", + " \n", + "if args.prefix is not None:\n", + " prefix = args.prefix\n", + "else:\n", + " prefix = \"root://xcache//\"\n", + "\n", + "dotest = args.test\n", + "#nworkers = int(args.nworkers)\n", + "chunksize = int(args.chunksize)\n", + "nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks\n", + "outname = args.outname\n", + "outpath = args.outpath\n", + "pretend = args.pretend\n", + "treename = args.treename\n", + "do_errors = args.do_errors\n", + "\n", + "if dotest:\n", + " nchunks = 2\n", + " chunksize = 10000\n", + " nworkers = 1\n", + " print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "79e4d98b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", + ">> TTZToLLNuNu_M10\n", + " - isData? : NO\n", + " - year : 2018\n", + " - xsec : 0.252900\n", + " - histAxisName : \n", + " - options : \n", + " - tree : Events\n", + " - nEvents : 14542666\n", + " - nGenEvents : 4876491\n", + " - SumWeights : 19992000.000000\n", + " - Prefix : root://xcache//\n", + " - nFiles : 18\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" + ] + } + ], + "source": [ + "\n", + "samplesdict = {}\n", + "allInputFiles = []\n", + "\n", + "def LoadJsonToSampleName(jsonFile, prefix):\n", + " sampleName = jsonFile if not '/' in jsonFile else jsonFile[jsonFile.rfind('/')+1:]\n", + " if sampleName.endswith('.json'): sampleName = sampleName[:-5]\n", + " with open(jsonFile) as jf:\n", + " samplesdict[sampleName] = json.load(jf)\n", + " samplesdict[sampleName]['redirector'] = prefix\n", + "\n", + "if isinstance(jsonFiles, str) and ',' in jsonFiles: jsonFiles = jsonFiles.replace(' ', '').split(',')\n", + "elif isinstance(jsonFiles, str) : jsonFiles = [jsonFiles]\n", + "\n", + "for jsonFile in jsonFiles:\n", + " if os.path.isdir(jsonFile):\n", + " if not jsonFile.endswith('/'): jsonFile+='/'\n", + " for f in os.path.listdir(jsonFile):\n", + " if f.endswith('.json'): allInputFiles.append(jsonFile+f)\n", + " else:\n", + " allInputFiles.append(jsonFile)\n", + "\n", + "print(allInputFiles)\n", + "# Read from cfg files\n", + "for f in allInputFiles:\n", + " if not os.path.isfile(f):\n", + " print('[WARNING] Input file \"%s% not found!'%f)\n", + " continue\n", + " # This input file is a json file, not a cfg\n", + " if f.endswith('.json'): \n", + " LoadJsonToSampleName(f, prefix)\n", + " # Open cfg files\n", + " else:\n", + " with open(f) as fin:\n", + " print(' >> Reading json from cfg file...')\n", + " lines = fin.readlines()\n", + " for l in lines:\n", + " if '#' in l: l=l[:l.find('#')]\n", + " l = l.replace(' ', '').replace('\\n', '')\n", + " if l == '': continue\n", + " if ',' in l:\n", + " l = l.split(',')\n", + " for nl in l:\n", + " if not os.path.isfile(l): prefix = nl\n", + " else: LoadJsonToSampleName(nl, prefix)\n", + " else:\n", + " if not os.path.isfile(l): prefix = l\n", + " else: LoadJsonToSampleName(l, prefix)\n", + "\n", + "flist = {};\n", + "for sname in samplesdict.keys():\n", + " redirector = samplesdict[sname]['redirector']\n", + " flist[sname] = [(redirector+f) for f in samplesdict[sname]['files']]\n", + " samplesdict[sname]['year'] = int(samplesdict[sname]['year'])\n", + " samplesdict[sname]['xsec'] = float(samplesdict[sname]['xsec'])\n", + " samplesdict[sname]['nEvents'] = int(samplesdict[sname]['nEvents'])\n", + " samplesdict[sname]['nGenEvents'] = int(samplesdict[sname]['nGenEvents'])\n", + " samplesdict[sname]['nSumOfWeights'] = float(samplesdict[sname]['nSumOfWeights'])\n", + "\n", + " # Print file info\n", + " print('>> '+sname)\n", + " print(' - isData? : %s' %('YES' if samplesdict[sname]['isData'] else 'NO'))\n", + " print(' - year : %i' %samplesdict[sname]['year'])\n", + " print(' - xsec : %f' %samplesdict[sname]['xsec'])\n", + " print(' - histAxisName : %s' %samplesdict[sname]['histAxisName'])\n", + " print(' - options : %s' %samplesdict[sname]['options'])\n", + " print(' - tree : %s' %samplesdict[sname]['treeName'])\n", + " print(' - nEvents : %i' %samplesdict[sname]['nEvents'])\n", + " print(' - nGenEvents : %i' %samplesdict[sname]['nGenEvents'])\n", + " print(' - SumWeights : %f' %samplesdict[sname]['nSumOfWeights'])\n", + " print(' - Prefix : %s' %samplesdict[sname]['redirector'])\n", + " print(' - nFiles : %i' %len(samplesdict[sname]['files']))\n", + " for fname in samplesdict[sname]['files']: print(' %s'%fname)\n", + "\n", + "if pretend: \n", + " print('pretending...')\n", + " exit() \n", + "\n", + "# Check that all datasets have the same list of WCs\n", + "for i,k in enumerate(samplesdict.keys()):\n", + " if i == 0:\n", + " wc_lst = samplesdict[k]['WCnames']\n", + " if wc_lst != samplesdict[k]['WCnames']:\n", + " raise Exception(\"Not all of the datasets have the same list of WCs.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "dd0c41e4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[# ] | 2% Completed | 0.1s4s\r" + ] + }, + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'topcoffea.analysis'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m# Run the processor and get the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mtstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m output = processor.run_uproot_job(flist,\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mtreename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtreename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mrun_uproot_job\u001b[0;34m(fileset, treename, processor_instance, executor, executor_args, pre_executor, pre_args, chunksize, maxchunks, metadata_cache)\u001b[0m\n\u001b[1;32m 1233\u001b[0m }\n\u001b[1;32m 1234\u001b[0m \u001b[0mexe_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m \u001b[0mwrapped_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mexe_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1237\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"out\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mdask_executor\u001b[0;34m(items, function, accumulator, **kwargs)\u001b[0m\n\u001b[1;32m 745\u001b[0m \u001b[0;31m# FIXME: fancy widget doesn't appear, have to live with boring pbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 746\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmulti\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 747\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0maccumulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0m_decompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccumulator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 748\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 749\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"error\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cancelled\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m()\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;31m# no @wraps due to pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 142\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_compress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m_work_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mProcessorABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcloudpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlz4f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0mretry_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'topcoffea.analysis'" + ] + } + ], + "source": [ + "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", + "\n", + "client = Client(\"tls://localhost:8786\")\n", + "client.register_worker_plugin(dependency_installer)\n", + "\n", + "executor_args = {\n", + " 'schema': NanoAODSchema,\n", + " 'client': client\n", + "}\n", + "\n", + "# Run the processor and get the output \n", + "tstart = time.time()\n", + "output = processor.run_uproot_job(flist,\n", + " treename=treename,\n", + " processor_instance=processor_instance,\n", + " executor=processor.dask_executor,\n", + " executor_args=executor_args,\n", + " chunksize=chunksize,\n", + " maxchunks=nchunks\n", + " )\n", + "dt = time.time() - tstart\n", + "\n", + "nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist))\n", + "nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist))\n", + "print(\"Filled %.0f bins, nonzero bins: %1.1f %%\" % (nbins, 100*nfilled/nbins,))\n", + "print(\"Processing time: %1.2f s with %i workers (%.2f s cpu overall)\" % (dt, nworkers, dt*nworkers, ))\n", + "\n", + "# This is taken from the DM photon analysis... \n", + "# Pickle is not very fast or memory efficient, will be replaced by something better soon \n", + "# with lz4f.open(\"pods/\"+options.year+\"/\"+dataset+\".pkl.gz\", mode=\"xb\", compression_level=5) as fout: \n", + "if not outpath.endswith('/'): outpath += '/'\n", + "if not os.path.isdir(outpath): os.system(\"mkdir -p %s\"%outpath)\n", + "print('Saving output in %s...'%(outpath + outname + \".pkl.gz\"))\n", + "with gzip.open(outpath + outname + \".pkl.gz\", \"wb\") as fout:\n", + " cloudpickle.dump(output, fout)\n", + "print('Done!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d257d72", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 0de47d3621a1df6a247de2f0b2bb1a94116c8aad Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:17:56 +0000 Subject: [PATCH 04/14] Add topcoffea installation from oshadura fork repo --- topeft-coffea-casa.ipynb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 3608c1de9..9dcdd721b 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -3,7 +3,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "f467b886", + "id": "9ce0043a", "metadata": {}, "outputs": [ { @@ -30,7 +30,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "781aeeb6", + "id": "99c76a58", "metadata": {}, "outputs": [], "source": [ @@ -70,14 +70,14 @@ " def setup(self, worker: Worker):\n", " os.system(f\"pip install {self._depencendies}\")\n", "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\"\n", + " \"git+https://github.com/oshadura/topcoffea.gitt@coffea-casa-analysis#egg=topcoffea\"\n", "])" ] }, { "cell_type": "code", "execution_count": 3, - "id": "0286ca63", + "id": "f381cc91", "metadata": {}, "outputs": [ { @@ -138,7 +138,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "79e4d98b", + "id": "45f143e5", "metadata": {}, "outputs": [ { @@ -269,7 +269,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "dd0c41e4", + "id": "04cb32dd", "metadata": {}, "outputs": [ { @@ -338,7 +338,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4d257d72", + "id": "bdfbcb28", "metadata": {}, "outputs": [], "source": [] From 628a34b4caee8de9dc604bfc34e0d64d42566817 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 12:20:58 +0000 Subject: [PATCH 05/14] Fix a tiny typo in git address --- topeft-coffea-casa.ipynb | 109 ++++++--------------------------------- 1 file changed, 16 insertions(+), 93 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 9dcdd721b..7e8ce10d8 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "9ce0043a", + "execution_count": null, + "id": "d2c73461", "metadata": {}, "outputs": [ { @@ -11,14 +11,7 @@ "output_type": "stream", "text": [ "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", - "Obtaining file:///home/cms-jovyan/topcoffea\n", - "Installing collected packages: topcoffea\n", - " Attempting uninstall: topcoffea\n", - " Found existing installation: topcoffea 0.0.0\n", - " Uninstalling topcoffea-0.0.0:\n", - " Successfully uninstalled topcoffea-0.0.0\n", - " Running setup.py develop for topcoffea\n", - "Successfully installed topcoffea\n" + "Obtaining file:///home/cms-jovyan/topcoffea\n" ] } ], @@ -29,8 +22,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "99c76a58", + "execution_count": null, + "id": "0b973d1b", "metadata": {}, "outputs": [], "source": [ @@ -70,24 +63,16 @@ " def setup(self, worker: Worker):\n", " os.system(f\"pip install {self._depencendies}\")\n", "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/oshadura/topcoffea.gitt@coffea-casa-analysis#egg=topcoffea\"\n", + " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis#egg=topcoffea\"\n", "])" ] }, { "cell_type": "code", - "execution_count": 3, - "id": "f381cc91", + "execution_count": null, + "id": "8540b700", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" - ] - } - ], + "outputs": [], "source": [ "import argparse\n", "parser = argparse.ArgumentParser(description='You can customize your run')\n", @@ -137,48 +122,10 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "45f143e5", + "execution_count": null, + "id": "8bb233e8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", - ">> TTZToLLNuNu_M10\n", - " - isData? : NO\n", - " - year : 2018\n", - " - xsec : 0.252900\n", - " - histAxisName : \n", - " - options : \n", - " - tree : Events\n", - " - nEvents : 14542666\n", - " - nGenEvents : 4876491\n", - " - SumWeights : 19992000.000000\n", - " - Prefix : root://xcache//\n", - " - nFiles : 18\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" - ] - } - ], + "outputs": [], "source": [ "\n", "samplesdict = {}\n", @@ -268,34 +215,10 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "04cb32dd", + "execution_count": null, + "id": "021c1708", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[# ] | 2% Completed | 0.1s4s\r" - ] - }, - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'topcoffea.analysis'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m# Run the processor and get the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mtstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m output = processor.run_uproot_job(flist,\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mtreename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtreename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mrun_uproot_job\u001b[0;34m(fileset, treename, processor_instance, executor, executor_args, pre_executor, pre_args, chunksize, maxchunks, metadata_cache)\u001b[0m\n\u001b[1;32m 1233\u001b[0m }\n\u001b[1;32m 1234\u001b[0m \u001b[0mexe_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m \u001b[0mwrapped_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mexe_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1237\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"out\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mdask_executor\u001b[0;34m(items, function, accumulator, **kwargs)\u001b[0m\n\u001b[1;32m 745\u001b[0m \u001b[0;31m# FIXME: fancy widget doesn't appear, have to live with boring pbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 746\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmulti\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 747\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0maccumulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0m_decompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccumulator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 748\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 749\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"error\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cancelled\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m()\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;31m# no @wraps due to pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 142\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_compress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m_work_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mProcessorABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcloudpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlz4f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0mretry_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'topcoffea.analysis'" - ] - } - ], + "outputs": [], "source": [ "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", "\n", @@ -338,7 +261,7 @@ { "cell_type": "code", "execution_count": null, - "id": "bdfbcb28", + "id": "330c0d3b", "metadata": {}, "outputs": [], "source": [] From 3f229b1b5df5ef7f237344268c23455b7db8f7ec Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 13:13:13 +0000 Subject: [PATCH 06/14] Update plotting part according Zora's example --- topeft-coffea-casa.ipynb | 206 ++++++++++++++++++++++++++++++++------- 1 file changed, 170 insertions(+), 36 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 7e8ce10d8..8e660d9d1 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": null, - "id": "d2c73461", + "execution_count": 1, + "id": "a800ee7d", "metadata": {}, "outputs": [ { @@ -11,7 +11,14 @@ "output_type": "stream", "text": [ "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", - "Obtaining file:///home/cms-jovyan/topcoffea\n" + "Obtaining file:///home/cms-jovyan/topcoffea\n", + "Installing collected packages: topcoffea\n", + " Attempting uninstall: topcoffea\n", + " Found existing installation: topcoffea 0.0.0\n", + " Uninstalling topcoffea-0.0.0:\n", + " Successfully uninstalled topcoffea-0.0.0\n", + " Running setup.py develop for topcoffea\n", + "Successfully installed topcoffea\n" ] } ], @@ -22,8 +29,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "0b973d1b", + "execution_count": 2, + "id": "317b25dc", "metadata": {}, "outputs": [], "source": [ @@ -52,27 +59,23 @@ "\n", "if hasattr(__builtins__,'__IPYTHON__'):\n", " import sys\n", - " sys.argv = ['']\n", - " \n", - "from dask.distributed import Client, Worker, WorkerPlugin\n", - "import os\n", - "from typing import List\n", - "class DependencyInstaller(WorkerPlugin):\n", - " def __init__(self, dependencies: List[str]):\n", - " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", - " def setup(self, worker: Worker):\n", - " os.system(f\"pip install {self._depencendies}\")\n", - "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis#egg=topcoffea\"\n", - "])" + " sys.argv = ['']" ] }, { "cell_type": "code", - "execution_count": null, - "id": "8540b700", + "execution_count": 3, + "id": "38f87aff", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" + ] + } + ], "source": [ "import argparse\n", "parser = argparse.ArgumentParser(description='You can customize your run')\n", @@ -122,10 +125,48 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "8bb233e8", + "execution_count": 4, + "id": "c8bf7134", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", + ">> TTZToLLNuNu_M10\n", + " - isData? : NO\n", + " - year : 2018\n", + " - xsec : 0.252900\n", + " - histAxisName : \n", + " - options : \n", + " - tree : Events\n", + " - nEvents : 14542666\n", + " - nGenEvents : 4876491\n", + " - SumWeights : 19992000.000000\n", + " - Prefix : root://xcache//\n", + " - nFiles : 18\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" + ] + } + ], "source": [ "\n", "samplesdict = {}\n", @@ -215,13 +256,51 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "021c1708", + "execution_count": 5, + "id": "67d41c95", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "CommClosedError", + "evalue": "in : Stream is closed", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mStreamClosedError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m \u001b[0mframes_nbytes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 197\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mframes_nbytes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframes_nbytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mStreamClosedError\u001b[0m: Stream is closed", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mCommClosedError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"tls://localhost:8786\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdependency_installer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m executor_args = {\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mregister_worker_plugin\u001b[0;34m(self, plugin, name, **kwargs)\u001b[0m\n\u001b[1;32m 4054\u001b[0m \u001b[0mplugin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4055\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4056\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4057\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4058\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 841\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 842\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 843\u001b[0;31m return sync(\n\u001b[0m\u001b[1;32m 844\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 845\u001b[0m )\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 351\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 352\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 353\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 354\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 335\u001b[0m \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 760\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 761\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 763\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_register_worker_plugin\u001b[0;34m(self, plugin, name)\u001b[0m\n\u001b[1;32m 3974\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3975\u001b[0m \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3976\u001b[0;31m responses = await self.scheduler.register_worker_plugin(\n\u001b[0m\u001b[1;32m 3977\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3978\u001b[0m )\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv_from_rpc\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m 860\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"ConnectionPool.\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 862\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0msend_recv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 863\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 864\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv\u001b[0;34m(comm, reply, serializers, deserializers, **kwargs)\u001b[0m\n\u001b[1;32m 643\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mserializers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon_error\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"raise\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 644\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 645\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 646\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_finalizing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m \u001b[0mconvert_stream_closed_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 207\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;31m# Some OSError or a another \"low-level\" exception. We do not really know what\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mconvert_stream_closed_error\u001b[0;34m(obj, exc)\u001b[0m\n\u001b[1;32m 126\u001b[0m ) from exc\n\u001b[1;32m 127\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mCommClosedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"in %s: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mCommClosedError\u001b[0m: in : Stream is closed" + ] + } + ], "source": [ "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", "\n", + "from dask.distributed import Client, Worker, WorkerPlugin\n", + "import os\n", + "from typing import List\n", + "class DependencyInstaller(WorkerPlugin):\n", + " def __init__(self, dependencies: List[str]):\n", + " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", + " def setup(self, worker: Worker):\n", + " os.system(f\"pip install {self._depencendies}\")\n", + "dependency_installer = DependencyInstaller([\n", + " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\"\n", + "])\n", + "\n", "client = Client(\"tls://localhost:8786\")\n", "client.register_worker_plugin(dependency_installer)\n", "\n", @@ -247,21 +326,76 @@ "print(\"Filled %.0f bins, nonzero bins: %1.1f %%\" % (nbins, 100*nfilled/nbins,))\n", "print(\"Processing time: %1.2f s with %i workers (%.2f s cpu overall)\" % (dt, nworkers, dt*nworkers, ))\n", "\n", - "# This is taken from the DM photon analysis... \n", - "# Pickle is not very fast or memory efficient, will be replaced by something better soon \n", - "# with lz4f.open(\"pods/\"+options.year+\"/\"+dataset+\".pkl.gz\", mode=\"xb\", compression_level=5) as fout: \n", - "if not outpath.endswith('/'): outpath += '/'\n", - "if not os.path.isdir(outpath): os.system(\"mkdir -p %s\"%outpath)\n", - "print('Saving output in %s...'%(outpath + outname + \".pkl.gz\"))\n", - "with gzip.open(outpath + outname + \".pkl.gz\", \"wb\") as fout:\n", - " cloudpickle.dump(output, fout)\n", + "os.system(\"mkdir -p histos/\")\n", + "print('Saving output in %s...'%(\"histos/\" + outname + \".pkl.gz\"))\n", + "with gzip.open(\"histos/\" + outname + \".pkl.gz\", \"wb\") as fout:\n", + " cloudpickle.dump(output, fout)\n", "print('Done!')" ] }, { "cell_type": "code", "execution_count": null, - "id": "330c0d3b", + "id": "f2250b7d", + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import print_function, division\n", + "from collections import defaultdict, OrderedDict\n", + "import gzip\n", + "import pickle\n", + "import json\n", + "import os\n", + "import uproot\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.hist import plot\n", + "from cycler import cycler\n", + "\n", + "from topcoffea.plotter.OutText import OutText\n", + "\n", + "\n", + "path = 'histos/plotsTopEFT.pkl.gz'\n", + "outname = 'temp.png'\n", + "\n", + "# Select variable, channel and cuts\n", + "var = 'met'\n", + "channel = ['eemSSonZ', 'eemSSoffZ', 'mmeSSonZ', 'mmeSSoffZ','eeeSSonZ', 'eeeSSoffZ', 'mmmSSonZ', 'mmmSSoffZ']\n", + "cut = 'base'\n", + "\n", + "print('Opening path: ', path)\n", + "hists = {}\n", + "with gzip.open(path) as fin:\n", + " hin = pickle.load(fin)\n", + " print(' >> looking for histograms...')\n", + " for k in hin.keys():\n", + " if k in hists: hists[k]+=hin[k]\n", + " else: hists[k]=hin[k]\n", + "\n", + "\n", + "# Create figure\n", + "fig, (ax, rax) = plt.subplots(2, 1, figsize=(14,7), gridspec_kw={\"height_ratios\": (3, 1)}, sharex=True)\n", + "plt.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9)\n", + "\n", + "# Select the histogram var, channel and cut\n", + "h = hists[var]\n", + "h = h.integrate('channel', channel)\n", + "h = h.integrate('cut', cut)\n", + "\n", + "# Integrate over samples\n", + "h = h.sum('sample')\n", + " \n", + "# Plot and save figure to outname\n", + "hist.plot1d(h, ax=ax, line_opts={'color':'orange'})\n", + "fig.savefig(outname)\n", + "print('Output histogram saved in %s'%outname)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59e67150", "metadata": {}, "outputs": [], "source": [] From 972122226712dd157341e774ef400657344e4974 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 13:20:02 +0000 Subject: [PATCH 07/14] Remove accidentaly added cache (.gitignore?) --- .../.ipynb_checkpoints/run-checkpoint.py | 101 --------------- .../work_queue_run-checkpoint.py | 117 ------------------ .../__pycache__/__init__.cpython-38.pyc | Bin 155 -> 0 bytes .../topEFT/__pycache__/topeft.cpython-38.pyc | Bin 18648 -> 0 bytes .../__pycache__/topeftenv.cpython-38.pyc | Bin 7267 -> 0 bytes 5 files changed, 218 deletions(-) delete mode 100755 topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py delete mode 100644 topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py delete mode 100644 topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc delete mode 100644 topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc delete mode 100644 topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py deleted file mode 100755 index 41ea9282d..000000000 --- a/topcoffea/analysis/topEFT/.ipynb_checkpoints/run-checkpoint.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -import lz4.frame as lz4f -import pickle -import json -import time -import cloudpickle -import gzip -import os -from optparse import OptionParser - -import uproot -import numpy as np -from coffea import hist, processor -from coffea.util import load, save -from coffea.nanoevents import NanoAODSchema - -import topeft -from topcoffea.modules import samples -from topcoffea.modules import fileReader - -if __name__ == '__main__': - - import argparse - parser = argparse.ArgumentParser(description='You can customize your run') - parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') - parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') - parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') - parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk') - parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') - parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') - parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') - parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') - parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients') - - args = parser.parse_args() - cfgfile = args.cfgfile - dotest = args.test - nworkers = int(args.nworkers) - chunksize = int(args.chunksize) - nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks - outname = args.outname - outpath = args.outpath - treename = args.treename - do_errors = args.do_errors - - if dotest: - nchunks = 2 - chunksize = 10000 - nworkers = 1 - print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) - - ### Load samples - if cfgfile != '': - samplesdict = samples.main() - elif os.path.isfile('.samples.coffea'): - print('Using samples form .samples.coffea') - samplesdict = load('.samples.coffea') - else: - print('Execute as [path]/run.py [path]/samples.cfg') - exit() - - flist = {}; xsec = {}; sow = {}; isData = {} - for k in samplesdict.keys(): - samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) - flist[k] = samplesdict[k]['files'] - xsec[k] = samplesdict[k]['xsec'] - sow[k] = samplesdict[k]['nSumOfWeights'] - isData[k]= samplesdict[k]['isData'] - - # Check that all datasets have the same list of WCs - for i,k in enumerate(samplesdict.keys()): - if i == 0: - wc_lst = samplesdict[k]['WCnames'] - if wc_lst != samplesdict[k]['WCnames']: - raise Exception("Not all of the datasets have the same list of WCs.") - - processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) - - # Run the processor and get the output - tstart = time.time() - output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.futures_executor, executor_args={"schema": NanoAODSchema,'workers': nworkers, 'pre_workers': 1}, chunksize=chunksize, maxchunks=nchunks) - dt = time.time() - tstart - - nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) - nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) - print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) - print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) - - # This is taken from the DM photon analysis... - # Pickle is not very fast or memory efficient, will be replaced by something better soon - # with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: - if not outpath.endswith('/'): outpath += '/' - if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) - print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) - with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: - cloudpickle.dump(output, fout) - print('Done!') - - - - diff --git a/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py b/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py deleted file mode 100644 index 05ed75686..000000000 --- a/topcoffea/analysis/topEFT/.ipynb_checkpoints/work_queue_run-checkpoint.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -import lz4.frame as lz4f -import pickle -import json -import time -import cloudpickle -import gzip -import os -from optparse import OptionParser - -import uproot -import numpy as np -from coffea import hist, processor -from coffea.util import load, save -from coffea.nanoevents import NanoAODSchema - -import topeft -from topcoffea.modules import samples -from topcoffea.modules import fileReader -import topeftenv - -import argparse -parser = argparse.ArgumentParser(description='You can customize your run') -parser.add_argument('cfgfile' , nargs='?', default='' , help = 'Config file with dataset names') -parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') -parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') -parser.add_argument('--chunksize','-s' , default=100000 , help = 'Number of events per chunk') -parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') -parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') -parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') -parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') -parser.add_argument('--do-errors' , action='store_true', help = 'Save the w**2 coefficients') - -args = parser.parse_args() -cfgfile = args.cfgfile -dotest = args.test -nworkers = int(args.nworkers) -chunksize = int(args.chunksize) -nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks -outname = args.outname -outpath = args.outpath -treename = args.treename -do_errors = args.do_errors - -if dotest: - nchunks = 2 - chunksize = 10000 - nworkers = 1 - print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) - -### Load samples -if cfgfile != '': - samplesdict = samples.main() -elif os.path.isfile('.samples.coffea'): - print('Using samples form .samples.coffea') - samplesdict = load('.samples.coffea') -else: - print('Execute as [path]/run.py [path]/samples.cfg') - exit() - -flist = {}; xsec = {}; sow = {}; isData = {} -for k in samplesdict.keys(): - samplesdict[k]['WCnames'] = fileReader.GetListOfWCs(samplesdict[k]['files'][0]) - flist[k] = samplesdict[k]['files'] - xsec[k] = samplesdict[k]['xsec'] - sow[k] = samplesdict[k]['nSumOfWeights'] - isData[k]= samplesdict[k]['isData'] - -# Check that all datasets have the same list of WCs -for i,k in enumerate(samplesdict.keys()): - if i == 0: - wc_lst = samplesdict[k]['WCnames'] - if wc_lst != samplesdict[k]['WCnames']: - raise Exception("Not all of the datasets have the same list of WCs.") - -processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) - -executor_args = {#'flatten': True, #used for all executors - 'compression': 0, #used for all executors - 'cores': 2, - 'disk': 5000, #MB - 'memory': 10000, #MB - 'resource-monitor': True, - 'debug-log': 'debug.log', - 'transactions-log': 'tr.log', - 'stats-log': 'stats.log', - 'verbose': False, - 'port': [9123,9130], - 'environment-file': topeftenv.get_environment(), - 'master-name': '{}-workqueue-coffea'.format(os.environ['USER']), - 'print-stdout': True, - 'skipbadfiles': False, - 'schema': NanoAODSchema, - 'extra-input-files': ["topeft.py"] -} - -# Run the processor and get the output -tstart = time.time() -output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks) -#output = processor.run_uproot_job(flist, treename=treename, processor_instance=processor_instance, executor=processor.work_queue_executor, executor_args=executor_args, chunksize=chunksize, maxchunks=nchunks, extra-input-files=["topeft.py"]) -dt = time.time() - tstart - -nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) -nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) -print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) -print("Processing time: %1.2f s with %i workers (%.2f s cpu overall)" % (dt, nworkers, dt*nworkers, )) - -# This is taken from the DM photon analysis... -# Pickle is not very fast or memory efficient, will be replaced by something better soon -# with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: -if not outpath.endswith('/'): outpath += '/' -if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) -print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) -with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: - cloudpickle.dump(output, fout) -print('Done!') - diff --git a/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index ca8cca7a5e9d14632ec6078762204275a04cc61c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 155 zcmWIL<>g`kf|S1p6F~H15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!H6enx(7s(x~A zv2Ip=S!H6Leo1~oa(-G`Y9bOVF)uNvvN*FCB;@KAq8}fhnU`4-AFo$Xd5gm)H$SB` NC)EyQ@@F7s002(`CL90& diff --git a/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeft.cpython-38.pyc deleted file mode 100644 index 02a68d8510b6ab5cff39978cc1d501d3577f6c3e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18648 zcmb_kd3YPwamV5&2!a$viMmA{mPJ|CLCLplOX8u!Iw+Br?6qVF#9M*@u>iEYAVvDE zd{A~S(@|n4X%aUPlemr3#!1_xN9?9a+NNplqq#O|d?ab3HvRg2ZNGkL|7m|Sy8r=7 zw&lp&Z)a!b&6_uG-ZgLWmUlNb#eDd?`0rml_5Szyd_O0m_AdltCvNk<0fbK|d7rMR ze#ONDYJhn`HOSA98scYI4f8XiM)}#G#OeW|+ClUc?x zvHWs>NoyFB-Axpv$jSwxBI3Pb*;8a zTZ`v4LK*ONwp>KreTSTIHd81XT4%rsi|kldXig+s7)#}|!U-0~G^5k+1V^%_uFnAWK{@xvu6TP*aa3=SSN;fJjk_DS`6EzLk-nr)`T-Toe<5H6 zg(Ce{=t7u!$_!x<5g;3=?3vJ15S*xp&XH(<9V8K&3C#ppqOndQEaD>0+QOnqGp zTSeDZdwiSN{w8UC(>tFB$mxWjA`5w)YILEQTEycwFk8myS~PZ8EA z1AuYi!(4dVAvHcw)(;OK*Rq)r%XGqKN-yO#rBs9-`g2we07IH8`h zu1`22L66CnLuGwPGdhaH9om>yAZ5MoGm;)j6$)D338c$dcFeL4hLO>nm^oounx3+< z>B)ryRlMWwLRvGdRJLGEIN@}$%*wjmCoN`r$}}eztkshboY3-ly6v?c>-T8KIy)}5 zk$GQX^9opa%*lECHLF7}=dElhpG{}2iH?gDOgQ1f5Y{1cvXvfl1Spu)e_MUwT_o~fBoZFaYo(F}D@(}-2JQPF0C~J>r4!V)AaTh7 zEhVyrOb4y29j>HTk&BDTgrHt0rk9I_dNC)MtCK*DJLX7)v@M#Inw(!>)(P;|UR;4f zGPv;<{XzxV>4fz9EN;1+#R=u=)7%o>H2;V-*|c{pRoJUW_E%WGJLee-QG&6Qz$yZ3 z2&^MOPBk_V*hpX#fz1TA5+HvX+X&o1;6?&B0XU7^wr&ezMUN)kerq~S$?Q)vQ~OJ)mME14w!ndBj#5>~>&pt2g*{@1wFe~-1rl*uKR z#5F6PU6!8G=PF}M7Pm$(CIm9iIKTDB*ZX6qnIfB%Mi9&WDk zGW1QUI4+Kn1H*5LBp2g+PU?R2Wo5L?{TQ093>?VLK|L zJ-(Am>;~B&{i9(+NdE;tLOn$!AniXbP=3IU0S4_xz>pmW4BJhB5j$Z;Wjxn_JBGUv zcYG=+bz;BaQ#-vCfSPc6_&xuCfVZR@NN^^2`Sjw#GYyhXB&pYZo(>bc~y?YY$IlnJqN%75N({6?%S_(g}k zOh(QJ%rjz@Oyo8!@&P8=0LAJZz8cXbR@}GY z-hun}sVKCdMd4ZWp{keB@2g%;u$L2%rg|*oc3jE6TnS%xz(aS)mE04qMY*da($L#o z&H~}~_SG^d){pwn2aScI(_T{uiVgM}$lNK{j0Wws*5(U4p=F7|<12mUU( z3fQ$(Y<(5$gm**cQ&f_@0o>idnjw1~p569Ft4CfZH^_}s0sHz2E>?-_ri1pTD&5BP zW~~5qVmb=7q(IvKJ{9^lb!0wSfxqj5WCwHV;_g?Oa;1jpqm%C2fun2uww2b=f zKG{ch*?U;mZs@vE+~n!nEBEG(Lf78hKvma1)&)Ls=jhrm_p`3MWRL8XecW;f7~jMA zLAh7%llvJz#Q0r|ANJbh0JVvSA0!-g?DyzHHED;vv?K66)i%M9+Kg&V)RCGNJPIkQ zM}i@>G1Z5t0d8@(od;f5m)qWhudB_KyZd$Zx&7nV>*{mmj$4D`X4x-qmq&&J3-J<6 zp}BcI{M<|P68G_5=IoREs+@hyK^s>&eazV-_f$D(dFVmgRXO{abCPq|E8HJAL+jZ$KV% zh<$=0~tJSp!%|GOXULzX2NQXeA=iE{hcJ;3jA`_WX) zPF}e*&-;z`sW*^=R)$79RWldD|8wSyp_+NrO|6|r)9m@7x=}8q;88BTQC{&pxJa2Z z#^Fir4ak${huP1X)T(~gm@_PgtDIrd>6T_^WJadp`4O4PU1w+INx+=EA8=HT0OsK_ zvN*wz#xq$?lslf&z|-#dM~moJF735Q;SJ<)TcpT7N3?d?Ilfdg$3tdLd!6PMDd80+ z=5TRKCcv|}!Q9i}Vm21HdR)_!t)1^JPqKF2E`xWsEY*##G3K4FYx!~1)Zo5Nkova4 zeVd?59fL28+_9=R&mf1%z9&e&H`(_DUFtY;n7QLj1>ygRH&%w%29s~B952@cZ?G)4 z_2e5Y%gtrWGS5$C_~D)z#lr^(M?WC$en707qXAyy&cgrqvea4h^s{y2>bB7!YQS+R zkBqbK9E-8@Aabd%nhO!To6CMkS}bLu9V^yYXhqi=;_jD3SFCxsYR$u#@gANtzt&sx zaMhYes@7aC*Q_;DT#G3=RUO~wnDdByq{?}eISR!gx24okw@bV9E>n<-YAMk_To^i)$$bK3# z4(^&r@^>*qJzW=3e+Yg%0d5?z9`79N56cgOcT#?s_f5BpJNTWz1$hBKZ~7j^?oRA@ z)IUt`NoM4XwOh{Q?wL}=oukD0NDXHvb3V$x`3R!%k1_U9V9&^p%8$`5Rz5=zGvP;T zZ2Ud>kvXX!sY(5P*mhX{e%+k*apwJA-JJF;M)SvcP9sRO*vENJBS_Sy6K_H$FLME&VJk`YiVag4Ft-<-S1BrD!GlthbVV2KZ;#OVmgH^ta9HdE?K1 z>-?@a{+zu*yT)h4j!VYTU$8g8N2zaAq)(=Mf+=6pZ#@l0B^th?QdA=oVb9~A`&}!{5<(}`3<}o1YGR9 zFf0y|R^tizwL(mWWO%7heyz!O=;9+pStpgcx= z@UZ)Zw!`ucd8gdz@p?SofV^8C_juR=v)mauDW%859+-I(@&RdkJnWsBr^|vYdc3_J zFDXySl*jA$cwdlTlwb0AM?BtN%FoNc@_0u*-k-`plYj2<20Y%U~D)>;<&g++$+>f$o`J}HtKdv_Q_tXy??#a^*8a3%RQIy?!h~OeUST`8o7JrU6=4w zd7J!OS4P|?PKcA@{!GyRE|-Pe(bjE=#x^De_xY+zZ|a8-xW!5N~FY4CU}$2 z)0dRzy^#NQTlphqOr=v(Yjg%#<-(aj6JQLVb=2?sp zuMBK8^Trq>-Z*|=sF}gQIkN-5lG{I#5pg?y6?DcmS_QZ20{rg|IlAQIPJ- z)lYhS=YuCd zWdBS$7fLferXD}xw_mhhlHZZvZTHFVwYyxyepx)i>B8=o|G?NW`xW^!`{(j!Vrn!D z+rErBfT*)<=}V}|>B>B7|DteAzAV4Xa=(!;BYI!)D8=Z%EGz^6JK!%jTA07w;|Jx- z_AfCXFE`d#>Lh=Tc_fax_?J{eqU>MDmotidC4;xJvYVj&Px9Njd*wep;j{l){zCpU zB*_oI!Y@R-Btb)*!>=zFy*X;R@euq0DW1Xp1)iaqYq{}W`K>DVU*(IUh&nAd{tFg9 zN_8-wgVo<-t3Q8!+3DvBt(XlD%9rHJ@G!K&vR}jVMfUu^$bZGbmzU)$vRnRI{t6y? zY`OvK2l7GlX=q+ykax%nta(luD~t}3y=)1mU!{b%R7-e??142eicv1nZ>{EfQRGZKSRinj%mJZ`qXn8lZbQp;%*7782DSQ=eYqE~Bo+TZ|(JR*R6zg~lI?Al$ zOI17Gd&N55!#Z|D#~5@hGoJJO+F>|XtmC~Ek08g#aDM>zIoxCNr#LS5zC%t#q^y*w z;h+qiA>MT+YaYU38z(rSr3@!{#?;bIQ=$GyS=Q`99yKLgLIUQ zlJ^vg;^P0|Z;;MBr(mlS9o4jw$m-_BW@O8J;QjRX?46Wf7|?*T(q?yFOBIZ@v*|y+ z)1cE>7XtulxMU>@Mcwh2EMrlHVWh^@D4j}65=J5!t>6TVks=PS#PX@hiDWvJ9>LLS z<2dqUUyh*gpZw3++zZzC?#%pR>OUn+MQ7*PA#L?j6hPxKGnIc4&He zBM!ak<&zz^cieYPc4LQopM<_HWbDd7KmOM4yodwPzCIL7mOBk8Gp!Zq(AvdD&=;GC z>DRU)3vNxr6TZ=OC!m1{=%9cB=;$ROJ{agUz&Y-LLmb@-r1E*A2rWj5z-a;o0h545 zpbUT$bEJ@O^$iV=+RkpavF+cqt7oMoCdvyQfAHRPU_qf>CRC+IqxjE9Kv zFo8!1Oc6Lo;86mP5qJxMw-R_8fwvQQ2Z6^4tO4kZUa>eI*W;6Vd_Iql>+vByKC{Os z^;SWn6Tx7~;%u85;YPW;q&6_pQyeciQMbXsjJX(b)o7vU9y>M8kYRLourp*VsNgtT z$%j&%20E9Qrbcv{2*t5HmqcfsCg~^KoagaSL;HJ=qMQgGTnOj^mmVM*LntJyiIUbC zbREM!VLy?JIvb29N#j!l*plUh(fLVZB?0oD@ooYg0Iw^~-n+awx9sJLxV$)@oxF1K zb!6g|i?4rOaf43$b;fGk!$rJ)#?FX)7WB%E;ZF71;-Q+}&n?LPo92MJCgFbjdYT@h zy7P$Qk-~j}rj$BO-0xpoQ*GDbv2x|&H;@TeF3v;mwZ*Xn;#hNvR&nDNXm>&;jz!}L zEY3MQ(dr@_q=ncC45rFXV83RmF?X@$#XBI*OsIY_sfNsKX`8^nE*4E~pHLgUwU?xf z6Qn$VqSQob_^6iOXBM|?OJcHD!}?fiaB#1h&^3`Q>-Y#qFIkk`qJ_KLJsR#$jXGf* z64z5_jJwGTg8&sX$V{~b(@M6#&sEKeqxqs~?agA=tXRT=36$4|g16putP57C(R8se zc1*L1Q~@oksezHArZ#E&3S$Seg|gWX#89cY2DjGmf;taDffyX^w+`bIDlqyjHJTN> z3Rd=D8FGxFG&PI>fC`zH6+K!yHF2;gv_sl>&q31(6w1028A+Kb%QDmk2=?(uQCPwd zIXF%9Ar+1-r%alLoQC0SK95MFps9^GA&=lea8Thdl$^*gzSgp~bvZ3V20q(LBTF)6 z7^w+UT{xUQqlrC6RvaBE_8|ew$F6)H$LJ~kFpU;!me5c+n-}Cwj+M~BP9%e-BD+G! zfW`Uf6IBTS}8m~1L^7YnA9waUD925_WbZJ-&v2jSIj#9jq>2}k{5 zJjY(!^Cr%)DdCjurum$QrlWz;ESV+n0HXk;@P%~0c( zi}V$Z?Rn!?q|@gMbi7+lRL*xNtzt26IuSPjLa#^(;Y6)s61}4|VeBH!J4tgQoyw<^ z<6Z8OW*QW38T$bobUe5nUo!B5;{>UvIH4@Qq%iS`A6)JPP~Fad#AI7s;q z5jaespFlf-qX16Si|b8C5pINWir9V~6M%(TK*^~w8UhIXl6qN<=;dSy(KjvTE)!i} zb6KGR54%u-XNzEghEK#AFCNfJkhew*O=FaJEJ+B@8QgZ%Y-$5~VacL%{A%2_-wmMQ z6&&Bkt^uO~BBmP6Ajn4tHr-%|7Ei~Y+oEzPIF7g?YwkwBBnh<%o0TGeeaIPjiFyzj zjCWCyUM$j60`(*>7lx9h z(2oaYt{P0NxN9NpLqJu`M@o(PZek6Cl;JB31d5oHsqUTxv!Xf{pCqDkR0^vEV_{9> zXQTII`x7p4xKc$_m#WBuF^prVX55Wd8TG7%aRO8;4muA-ilpCv1+YZd)+YdP5JeJtJsWu~QL&rDS zcJxcNrH+A_a#n=uR42j=5{XwcLBxrUr40TYPmR*Yo(J&KPz{b0%a+ZqR+n7iHeRMxbcS?4kF5{+N=eUM)&giTYXs_zKUxf=iSel{WuEB?h@zV1v@9 zv?|Lyx+T!1bSQB*4KNNI`IUvr!ay5zwBcHzEK*Ryf=z)Ar3F}vvd|y%g_R(1@B;*6 zRF*FeZXD??u!gim{p%IJ{H-n>SpjV=&>2+{{vhxUr6T~Izr#-@EklmVlPsZY8Ejpr ztaJ4!>;0{PR@h&?qNK6SzeU+XJlG%gZ&JFHF4o!x?kwN)M@>3v$u&foe&m6l+-nP- z>!A~{Qd(fGver*^BKs;38Ur2=HEbmxMB{8nD{Cem`J8P;zMV>s(&Hy;p|S(i&P(t` z@HI=hFfiM$gtde=27+ArYGo07imaJ)1y?Am(Q=6@zXe{QHo@ihp1>01S@$9?=r)#t z1+%51jbbQ|TGFq)%JrYImN_ZEiN)ML;lftfj9%jpqyM#_HDlx<@{2zRoYMj^UqYdq zz8j@9vY7f{Old=(YE{;Px55`B46P9Jfv()>Lz_T#kVZ(v{3JUUTV3T{4%@CwUgn!j zZ=BVAlj+BBQ*d|VHtFTF#J@M;FF|>j;^p3MFnKF9cic9X@CLxsnSmbnYf$BKU!vAX z9_nnMk8Ek0N+$7zZ&ELca-Q&bGI_e3%DXAX9u(_Dp&tLH1e-6~ix~$9e4oG%2=vy2 zJ|f!ymf$w&MJ5L2azDB&?qKvmpL=a!8L~mgNAJnxWMb21ci-5DIMm!cnee_a-sEyS z*EtRO$?cnl4f^dRaSvZ-0&FQM?m!x0%ln9!ZUl zj;9P^e2AzpwmW!(%AZBM-%VrhOMBN%WhD1yvvp6{f8O% zlkfRtf4sl)Z(nxp?q<&$KctF8uqu^O*q>6z!(P4!zj^;4=W^}pM0dD#x`R5bxdp(T e*Lf^sJi9Xi=2wg+%(_ZTq_xuEgHeq7xbpv{0XPK! diff --git a/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc b/topcoffea/analysis/topEFT/__pycache__/topeftenv.cpython-38.pyc deleted file mode 100644 index 92ffd4d8773d148881fbe15f61db0e2edb605323..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7267 zcmbVRON<-Id9LaQ*=){(Lye@7G#a(2hnE|1^(J8Rf$Z-696g98Ks6c{<=;Dc`oa>+Fa5};23a?&}61e1sJ{ng|Q zXJk7_D0NkJbv^(3|L_0)nh$4ZZH8a-AOHH}f4acfztGF{&%n#uc*6fiBADQ9R!wtW znfosytK8s61QEs@$qtD$iAODz~e)XN~fEtUBXatyw&CLKFID z8fU_&&I_|z@Yra7CKPenBE9Rc_>M1cV3aB|P7DN&G zk~kq2k)IJK#VO=x#c8pG{G7Nb&WUC8Sbm3z^J3*W6D!{NUA?;ET@V*Gwa=N{_Z)9| zm+f+_gnz)wrI(NaTN_-t@|qjgBfsO7!<$!LYkQBqcGn%hb-XZioA^@J5>tIjGL1lX z+^A}J-N)hJ=k`3p-)+le-U?(H{L z-&%d6bj>NHogQ6NgHn6?rL`4I-1O3Js#&Gy);Fn>@Y`+5w`CASA|0mD@Z8d)gJFiM zG)%`W^+Qh{S=o4AtmvS})EY`XzlY{i-_p_Sp~jzktNI#6PZyi z*7q>V9_K8wW22R)D8(w|07WAWDKT0wmJiPb2A3jWqcinc|6Yr;u)aXgJ9WW|&x*qNYmR zd%@&6EOP%#*EfQWcfH;TE3M%1j@!K+1-*JeI&t0Yy6v6N52@<@gNKxPjR+I0_I8qD zP4>GrB`c(An%6L3IERGs6;0!dyvR>+o6Ga4sgp=1KPqSO+{F`KLlUtO7ks1%ZKTKA z9yDMNDbCA?nb3t1^M(;?Kh}OoXJoDoF8oe6jIbNX1cG-UjezpUzU!zJ?el9%)=%5) zYn3_#lObVxGBJg>-fyZCk_(txt*)#m7WP%^xzPrsw&8ZWUOO>*o6S&(DNUBqJULw( zk2~HwSr+U*yg_>@B4K=qYm{c1<7@i6px=C`fOs?@NykK3t8qqrv%KE+klKqT~D$E!>a} z<{x|sRgj~SsfKGz=G!Fkavm z)NL$xrZMfm(p_g+{$6p>}RCObik+?**MnEReNiyeLgmJ;c8TU4)qGV z2(MyY*t@iLYMoLj zdU~`ZinRAJidxUC35rH%MrUdDSn+sI9i-R$_Mvm# zw3W+l3yrEnUyKQhQ=6_n11ASLWMU<{bimS%}_E+34yaDkdh}TA=8m3k-)EQ zz>l?)jO;|x^UAqob~0#f!wol*+{8}9#KghWz3S}2>5;Lg174;9uTWB<_TkgrUeANW&V+%CJW)2P77bdH?i09u$Qhm8nGc8%8ywPBEXiwK|rLjc8hf_0F^3z#I`KJP8M|T zF9134Y=dRhEB9)zG3=PnOoE*)fX|i{(~;O~iQe?1K~8#)!Fgo}LA`tbEBDGg(d$o0 ztdSe_Q&vYx+;;u$V6jr^cOwZK@d{3fSV^Q`NGrT7Y0}E z2K}~hxFD3l+lZt1`&Il+2nD%7~zlZ7EAR zqG@CrWU7Qg%6&IskdJ1D{8JY5GwhGFE#reJ+<^1YR==3=-U}|ji7r#rz`jNF8lvgl z`HTij0YGDeOC$;KYi}%()mzFa!aFETddS~HmxM_&f|m-V%_K`;M_dMg=-znqHxxw# zl$Jg+A`{#YBBn42Q7bHPs2%%ms_&VD?>Ho|MFnwywyQ(fV$3(Q z^1UcC$_fNa2ZTjKzyv@9VR#P~`HT8|@+w_5{^yuOHbM$(Xc>bXkQ6iVs0k zT3Q~JFZkLEJ<$T78(C;#fgp4!(1t(fjpHuFy47|&>%zVBCm8Kt@vy;4t={%r#3SV9 zrw==ANcO+!?F{TjA0ZI=-~ZF{WE*qH4kcTZ9EDvOqV7+rK0-3M^icZ101ynd81V{i z01GMG@_n>TjAlDnmtE>hl1T%STU1i;a6p-X8eS)O>>-Y^)u^@5AHW5VQAg3{u$r3) zAH~T=Ln&#vqbp8krp5KgXfS5c4tL;WEx-f7LFJMFTcEuRK)Ile(=r9xj#b zG>k&TqoPKNp`8kN>Ez^UAi?VhY==&z?>rq7MZ8qTGmlrI?j}&Hn3z+#JXuA_#%d=a zo7yJ6I12h4TAGdU4fFl~S3$!~#!ok^?spJB7mjZs$&Qz=RC|y+5a<*wE*#f$3;eZ5 zJ~%0^L8{^*UIXdMod;BBK9L}}AfW9&;#N{Z#cdOVPT*HF+_EWSYCoie$Z1s_o%HH{ z&-*3LbiC0+XS#=eV`o|bJ+*R(Gnw{pmQ9ZJEaFdGcI=`}k{!n-FA4LGSPyY=TDWqp ze*;}hDeAEh6j48+t#YLnNGPsA6iwBm+%9Z^F0^*}_f&}WDceOLNP!}-o$cxf3CBS# zuDZ<89O{)fXVogpDtM9xcSyF8?-N z&gH44;6oM_M<>M0Xim(I77<3`J~p<96cDo`i;S84chzPdv*WUj>J;3g+&YPXxNr~< zqh@q!k0H!-&;xh3Q$3cjMhb}$Q0}oG{a^_H1Rqu$+R-_zKNs7ygU|Was_I!h?1=z( zxpkiIf8(4u8L4p);4ZgT#HpzOce!;z1-O3+bjd4papHXIqC%I_=#n@+g)aGL`AL9# zh%V{Kpf^iv59M{7sbUh+LuoKK!C!>7_%CnJXW|=2yy6ieM4Ukpp~VRi5)2}T!%hiN znX3U(D6gUM3tr8W;it@SOh0H8Sx54>ctYaz8R+N?+!9zSdA`Zz;IxyRFrvh0xPDtc zN3X$~Uq1}3r@A@oUfso|Fs_DPCB_J^wQJ5RTO^O69RCsuYKw&-R~iA`XlYY?547qa zPL|(BYe^eD=DpPR{toqrbnoP``+)Q^K)ykp-=yR%N`8})o0NQ+5?WRoonM}sRe9gr z^r=Bf6UkI8+zNxPq_9f*lu!_*=(D17Dj+=!|fnM{uZ@- zn-b;nRcQYVuSxL$0Po>$uO0<*MtjumYNp*;An z`e%S|q``hc1^YH}j;S#JDwp5KIHb=mXLF)yn)qS%08R zM1BVi$`^4dPfDS*rHmSRmlEZi7OAAp(I7G_=jr`1B}Dm>97Oh>x|LS?kkEaYU%wl4 z8-6n}@7@3H58shW9Y3YkW;<|2D8EA8=|+9(mgil1J*JDq+fr2_3Cb!Zv^DtyO4J?D zAt^lAePZ28MRkXEAClRmCv_~v8RDA8FbcK~@0YW3_GN3qDxx%NowQ#s=r;c!#u58% From 12c1979f142f8542eb9788807cb981c64a4c8900 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Thu, 10 Jun 2021 15:24:08 +0000 Subject: [PATCH 08/14] Add oshadura repository from which everything works --- topeft-coffea-casa.ipynb | 62 ++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 8e660d9d1..d3ce83902 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -3,7 +3,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "a800ee7d", + "id": "5d03d87f", "metadata": {}, "outputs": [ { @@ -11,26 +11,26 @@ "output_type": "stream", "text": [ "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", - "Obtaining file:///home/cms-jovyan/topcoffea\n", - "Installing collected packages: topcoffea\n", - " Attempting uninstall: topcoffea\n", - " Found existing installation: topcoffea 0.0.0\n", - " Uninstalling topcoffea-0.0.0:\n", - " Successfully uninstalled topcoffea-0.0.0\n", - " Running setup.py develop for topcoffea\n", - "Successfully installed topcoffea\n" + "Collecting git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + " Cloning https://github.com/oshadura/topcoffea.git (to revision coffea-casa-analysis) to /tmp/pip-req-build-9zf5jcvu\n", + "Requirement already satisfied (use --upgrade to upgrade): topcoffea==0.0.0 from git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis in /home/cms-jovyan/topcoffea\n", + "Building wheels for collected packages: topcoffea\n", + " Building wheel for topcoffea (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for topcoffea: filename=topcoffea-0.0.0-py3-none-any.whl size=4512948 sha256=e020a6dde8da5bedb5658fcf58b182244cc65fc5593209bae50ba8da1f429d85\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-b7hy8t1i/wheels/3a/95/24/bfbb2d1dc4571114a0106e7d0d6a8124bcd2ad9e72ec9ea14e\n", + "Successfully built topcoffea\n" ] } ], "source": [ - "#!pip install git+https://github.com/TopEFT/topcoffea.git#egg=topcoffea\n", - "! pip install -e ." + "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + "#! pip install -e ." ] }, { "cell_type": "code", "execution_count": 2, - "id": "317b25dc", + "id": "502972cd", "metadata": {}, "outputs": [], "source": [ @@ -65,7 +65,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "38f87aff", + "id": "dd76877f", "metadata": {}, "outputs": [ { @@ -126,7 +126,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "c8bf7134", + "id": "402ee04d", "metadata": {}, "outputs": [ { @@ -256,33 +256,15 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "67d41c95", + "execution_count": null, + "id": "55f6755e", "metadata": {}, "outputs": [ { - "ename": "CommClosedError", - "evalue": "in : Stream is closed", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mStreamClosedError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m \u001b[0mframes_nbytes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 197\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mframes_nbytes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstruct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframes_nbytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mStreamClosedError\u001b[0m: Stream is closed", - "\nThe above exception was the direct cause of the following exception:\n", - "\u001b[0;31mCommClosedError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"tls://localhost:8786\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdependency_installer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m executor_args = {\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mregister_worker_plugin\u001b[0;34m(self, plugin, name, **kwargs)\u001b[0m\n\u001b[1;32m 4054\u001b[0m \u001b[0mplugin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4055\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4056\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4057\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4058\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 841\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 842\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 843\u001b[0;31m return sync(\n\u001b[0m\u001b[1;32m 844\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 845\u001b[0m )\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 351\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 352\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 353\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 354\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 335\u001b[0m \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 760\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 761\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 763\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_register_worker_plugin\u001b[0;34m(self, plugin, name)\u001b[0m\n\u001b[1;32m 3974\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3975\u001b[0m \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_register_worker_plugin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3976\u001b[0;31m responses = await self.scheduler.register_worker_plugin(\n\u001b[0m\u001b[1;32m 3977\u001b[0m \u001b[0mplugin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplugin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3978\u001b[0m )\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv_from_rpc\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m 860\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"ConnectionPool.\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 862\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0msend_recv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcomm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 863\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 864\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/core.py\u001b[0m in \u001b[0;36msend_recv\u001b[0;34m(comm, reply, serializers, deserializers, **kwargs)\u001b[0m\n\u001b[1;32m 643\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mserializers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon_error\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"raise\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 644\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 645\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcomm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeserializers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 646\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, deserializers)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_finalizing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m \u001b[0mconvert_stream_closed_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 207\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;31m# Some OSError or a another \"low-level\" exception. We do not really know what\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py\u001b[0m in \u001b[0;36mconvert_stream_closed_error\u001b[0;34m(obj, exc)\u001b[0m\n\u001b[1;32m 126\u001b[0m ) from exc\n\u001b[1;32m 127\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mCommClosedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"in %s: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mCommClosedError\u001b[0m: in : Stream is closed" + "name": "stdout", + "output_type": "stream", + "text": [ + "[################################# ] | 84% Completed | 1min 35.2s" ] } ], @@ -336,7 +318,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f2250b7d", + "id": "53c45688", "metadata": {}, "outputs": [], "source": [ @@ -395,7 +377,7 @@ { "cell_type": "code", "execution_count": null, - "id": "59e67150", + "id": "01fb4952", "metadata": {}, "outputs": [], "source": [] From e41b815793427505f108938897c686a5b12671db Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Sun, 20 Jun 2021 13:57:54 +0200 Subject: [PATCH 09/14] Add working notebook (except of plotting part) --- notebooks/topeft-coffea-casa.ipynb | 325 +++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 notebooks/topeft-coffea-casa.ipynb diff --git a/notebooks/topeft-coffea-casa.ipynb b/notebooks/topeft-coffea-casa.ipynb new file mode 100644 index 000000000..cc4380d0c --- /dev/null +++ b/notebooks/topeft-coffea-casa.ipynb @@ -0,0 +1,325 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + "! pip install awkward==1.3.0\n", + "#! pip install -e ." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import lz4.frame as lz4f\n", + "import pickle\n", + "import json\n", + "import time\n", + "import cloudpickle\n", + "import gzip\n", + "import os\n", + "from optparse import OptionParser\n", + "\n", + "import uproot\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.util import load, save\n", + "from coffea.nanoevents import NanoAODSchema\n", + "\n", + "from topcoffea.modules import samples\n", + "from topcoffea.modules import fileReader\n", + "\n", + "#FIXME: analysis is not installed anywhere (should be installed as well)\n", + "import topcoffea.analysis.topEFT.topeft\n", + "\n", + "import importlib.resources\n", + "\n", + "if hasattr(__builtins__,'__IPYTHON__'):\n", + " import sys\n", + " sys.argv = ['']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import argparse\n", + "parser = argparse.ArgumentParser(description='You can customize your run')\n", + "parser.add_argument('jsonFiles' , nargs='?', help = 'Json file(s) containing files and metadata')\n", + "parser.add_argument('--prefix', '-r' , nargs='?', help = 'Prefix or redirector to look for the files')\n", + "parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks')\n", + "parser.add_argument('--pretend' , action='store_true' , help = 'Read json files but, not execute the analysis')\n", + "#parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers')\n", + "parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk')\n", + "parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks')\n", + "parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms')\n", + "parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory')\n", + "parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files')\n", + "parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients')\n", + "\n", + "args = parser.parse_args()\n", + "\n", + "if args.jsonFiles is not None:\n", + " jsonFiles = args.jsonFiles\n", + " print('jsonFiles {}'.format(args.jsonFiles))\n", + "else:\n", + " with importlib.resources.path(\"topcoffea.json\", \"TTZToLLNuNu_M10.json\") as path:\n", + " jsonFiles = str(path)\n", + " print('jsonFile was selected for UNL {}'.format(jsonFiles))\n", + " \n", + "if args.prefix is not None:\n", + " prefix = args.prefix\n", + "else:\n", + " prefix = \"root://xcache//\"\n", + "\n", + "dotest = args.test\n", + "#nworkers = int(args.nworkers)\n", + "chunksize = int(args.chunksize)\n", + "nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks\n", + "outname = args.outname\n", + "outpath = args.outpath\n", + "pretend = args.pretend\n", + "treename = args.treename\n", + "do_errors = args.do_errors\n", + "\n", + "if dotest:\n", + " nchunks = 2\n", + " chunksize = 10000\n", + " nworkers = 1\n", + " print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "samplesdict = {}\n", + "allInputFiles = []\n", + "\n", + "def LoadJsonToSampleName(jsonFile, prefix):\n", + " sampleName = jsonFile if not '/' in jsonFile else jsonFile[jsonFile.rfind('/')+1:]\n", + " if sampleName.endswith('.json'): sampleName = sampleName[:-5]\n", + " with open(jsonFile) as jf:\n", + " samplesdict[sampleName] = json.load(jf)\n", + " samplesdict[sampleName]['redirector'] = prefix\n", + "\n", + "if isinstance(jsonFiles, str) and ',' in jsonFiles: jsonFiles = jsonFiles.replace(' ', '').split(',')\n", + "elif isinstance(jsonFiles, str) : jsonFiles = [jsonFiles]\n", + "\n", + "for jsonFile in jsonFiles:\n", + " if os.path.isdir(jsonFile):\n", + " if not jsonFile.endswith('/'): jsonFile+='/'\n", + " for f in os.path.listdir(jsonFile):\n", + " if f.endswith('.json'): allInputFiles.append(jsonFile+f)\n", + " else:\n", + " allInputFiles.append(jsonFile)\n", + "\n", + "print(allInputFiles)\n", + "# Read from cfg files\n", + "for f in allInputFiles:\n", + " if not os.path.isfile(f):\n", + " print('[WARNING] Input file \"%s% not found!'%f)\n", + " continue\n", + " # This input file is a json file, not a cfg\n", + " if f.endswith('.json'): \n", + " LoadJsonToSampleName(f, prefix)\n", + " # Open cfg files\n", + " else:\n", + " with open(f) as fin:\n", + " print(' >> Reading json from cfg file...')\n", + " lines = fin.readlines()\n", + " for l in lines:\n", + " if '#' in l: l=l[:l.find('#')]\n", + " l = l.replace(' ', '').replace('\\n', '')\n", + " if l == '': continue\n", + " if ',' in l:\n", + " l = l.split(',')\n", + " for nl in l:\n", + " if not os.path.isfile(l): prefix = nl\n", + " else: LoadJsonToSampleName(nl, prefix)\n", + " else:\n", + " if not os.path.isfile(l): prefix = l\n", + " else: LoadJsonToSampleName(l, prefix)\n", + "\n", + "flist = {};\n", + "for sname in samplesdict.keys():\n", + " redirector = samplesdict[sname]['redirector']\n", + " flist[sname] = [(redirector+f) for f in samplesdict[sname]['files']]\n", + " samplesdict[sname]['year'] = int(samplesdict[sname]['year'])\n", + " samplesdict[sname]['xsec'] = float(samplesdict[sname]['xsec'])\n", + " samplesdict[sname]['nEvents'] = int(samplesdict[sname]['nEvents'])\n", + " samplesdict[sname]['nGenEvents'] = int(samplesdict[sname]['nGenEvents'])\n", + " samplesdict[sname]['nSumOfWeights'] = float(samplesdict[sname]['nSumOfWeights'])\n", + "\n", + " # Print file info\n", + " print('>> '+sname)\n", + " print(' - isData? : %s' %('YES' if samplesdict[sname]['isData'] else 'NO'))\n", + " print(' - year : %i' %samplesdict[sname]['year'])\n", + " print(' - xsec : %f' %samplesdict[sname]['xsec'])\n", + " print(' - histAxisName : %s' %samplesdict[sname]['histAxisName'])\n", + " print(' - options : %s' %samplesdict[sname]['options'])\n", + " print(' - tree : %s' %samplesdict[sname]['treeName'])\n", + " print(' - nEvents : %i' %samplesdict[sname]['nEvents'])\n", + " print(' - nGenEvents : %i' %samplesdict[sname]['nGenEvents'])\n", + " print(' - SumWeights : %f' %samplesdict[sname]['nSumOfWeights'])\n", + " print(' - Prefix : %s' %samplesdict[sname]['redirector'])\n", + " print(' - nFiles : %i' %len(samplesdict[sname]['files']))\n", + " for fname in samplesdict[sname]['files']: print(' %s'%fname)\n", + "\n", + "if pretend: \n", + " print('pretending...')\n", + " exit() \n", + "\n", + "# Check that all datasets have the same list of WCs\n", + "for i,k in enumerate(samplesdict.keys()):\n", + " if i == 0:\n", + " wc_lst = samplesdict[k]['WCnames']\n", + " if wc_lst != samplesdict[k]['WCnames']:\n", + " raise Exception(\"Not all of the datasets have the same list of WCs.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", + "\n", + "from dask.distributed import Client, Worker, WorkerPlugin\n", + "import os\n", + "from typing import List\n", + "class DependencyInstaller(WorkerPlugin):\n", + " def __init__(self, dependencies: List[str]):\n", + " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", + " def setup(self, worker: Worker):\n", + " os.system(f\"pip install {self._depencendies}\")\n", + "dependency_installer = DependencyInstaller([\n", + " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\",\"awkward==1.3.0\"\n", + "])\n", + "\n", + "client = Client(\"tls://localhost:8786\")\n", + "client.register_worker_plugin(dependency_installer)\n", + "\n", + "executor_args = {\n", + " 'schema': NanoAODSchema,\n", + " 'client': client,\n", + " 'savemetrics': True\n", + "}\n", + "\n", + "\n", + "# Run the processor and get the output \n", + "tic = time.time()\n", + "output = processor.run_uproot_job(flist,\n", + " treename=treename,\n", + " processor_instance=processor_instance,\n", + " executor=processor.dask_executor,\n", + " executor_args=executor_args,\n", + " chunksize=chunksize,\n", + " maxchunks=nchunks\n", + " )\n", + "toc = time.time()\n", + "\n", + "print(\"Dask client:\", client)\n", + "print(\"Total time: %.0f\" % (toc - tic))\n", + "print(\"Events / s / thread: {:,.0f}\".format(output[1]['entries'] / output[1]['processtime']))\n", + "print(\"Events / s: {:,.0f}\".format(output[1]['entries'] / (toc - tic)))\n", + "\n", + "os.system(\"mkdir -p histos/\")\n", + "print('Saving output in %s...'%(\"histos/\" + outname + \".pkl.gz\"))\n", + "with gzip.open(\"histos/\" + outname + \".pkl.gz\", \"wb\") as fout:\n", + " cloudpickle.dump(output, fout)\n", + "print('Done!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import print_function, division\n", + "from collections import defaultdict, OrderedDict\n", + "import gzip\n", + "import pickle\n", + "import json\n", + "import os\n", + "import uproot\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from coffea import hist, processor\n", + "from coffea.hist import plot\n", + "from cycler import cycler\n", + "\n", + "from topcoffea.plotter.OutText import OutText\n", + "\n", + "\n", + "path = 'histos/plotsTopEFT.pkl.gz'\n", + "outname = 'temp.png'\n", + "\n", + "# Select variable, channel and cuts\n", + "var = 'met'\n", + "channel = ['eemSSonZ', 'eemSSoffZ', 'mmeSSonZ', 'mmeSSoffZ','eeeSSonZ', 'eeeSSoffZ', 'mmmSSonZ', 'mmmSSoffZ']\n", + "cut = 'base'\n", + "\n", + "print('Opening path: ', path)\n", + "hists = {}\n", + "with gzip.open(path) as fin:\n", + " hin = pickle.load(fin)\n", + " print(' >> looking for histograms...')\n", + " for k in hin.keys():\n", + " if k in hists: hists[k]+=hin[k]\n", + " else: hists[k]=hin[k]\n", + "\n", + "\n", + "# Create figure\n", + "fig, (ax, rax) = plt.subplots(2, 1, figsize=(14,7), gridspec_kw={\"height_ratios\": (3, 1)}, sharex=True)\n", + "plt.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9)\n", + "\n", + "# Select the histogram var, channel and cut\n", + "h = hists[var]\n", + "h = h.integrate('channel', channel)\n", + "h = h.integrate('cut', cut)\n", + "\n", + "# Integrate over samples\n", + "h = h.sum('sample')\n", + " \n", + "# Plot and save figure to outname\n", + "hist.plot1d(h, ax=ax, line_opts={'color':'orange'})\n", + "fig.savefig(outname)\n", + "print('Output histogram saved in %s'%outname)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file From fae9c64ec401503270a42e9fc9621a081fa5ffe5 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Fri, 25 Jun 2021 13:38:33 +0200 Subject: [PATCH 10/14] Revert back location of analysis directory --- .../.ipynb_checkpoints/__init__-checkpoint.py | 0 {topcoffea/analysis => analysis}/__init__.py | 0 .../__pycache__/__init__.cpython-38.pyc | Bin .../analysis => analysis}/btagMCeff/__init__.py | 0 .../analysis => analysis}/btagMCeff/btagMCeff.py | 0 {topcoffea/analysis => analysis}/btagMCeff/run.py | 0 {topcoffea/analysis => analysis}/topEFT/Genrun.py | 0 .../analysis => analysis}/topEFT/Gentopeft.py | 0 {topcoffea/analysis => analysis}/topEFT/__init__.py | 0 .../analysis => analysis}/topEFT/convert3lEFT.py | 0 .../analysis => analysis}/topEFT/drawSliders.py | 0 {topcoffea/analysis => analysis}/topEFT/genPlot.py | 0 .../analysis => analysis}/topEFT/make_jsons.py | 0 {topcoffea/analysis => analysis}/topEFT/plot.py | 0 {topcoffea/analysis => analysis}/topEFT/run.py | 0 {topcoffea/analysis => analysis}/topEFT/topeft.py | 0 .../analysis => analysis}/topEFT/topeftenv.py | 0 .../analysis => analysis}/topEFT/work_queue_run.py | 0 {topcoffea/analysis => analysis}/ttTest/__init__.py | 0 {topcoffea/analysis => analysis}/ttTest/plot.py | 0 {topcoffea/analysis => analysis}/ttTest/tt.py | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename {topcoffea/analysis => analysis}/.ipynb_checkpoints/__init__-checkpoint.py (100%) rename {topcoffea/analysis => analysis}/__init__.py (100%) rename {topcoffea/analysis => analysis}/__pycache__/__init__.cpython-38.pyc (100%) rename {topcoffea/analysis => analysis}/btagMCeff/__init__.py (100%) rename {topcoffea/analysis => analysis}/btagMCeff/btagMCeff.py (100%) rename {topcoffea/analysis => analysis}/btagMCeff/run.py (100%) rename {topcoffea/analysis => analysis}/topEFT/Genrun.py (100%) rename {topcoffea/analysis => analysis}/topEFT/Gentopeft.py (100%) rename {topcoffea/analysis => analysis}/topEFT/__init__.py (100%) rename {topcoffea/analysis => analysis}/topEFT/convert3lEFT.py (100%) rename {topcoffea/analysis => analysis}/topEFT/drawSliders.py (100%) rename {topcoffea/analysis => analysis}/topEFT/genPlot.py (100%) rename {topcoffea/analysis => analysis}/topEFT/make_jsons.py (100%) rename {topcoffea/analysis => analysis}/topEFT/plot.py (100%) rename {topcoffea/analysis => analysis}/topEFT/run.py (100%) rename {topcoffea/analysis => analysis}/topEFT/topeft.py (100%) rename {topcoffea/analysis => analysis}/topEFT/topeftenv.py (100%) rename {topcoffea/analysis => analysis}/topEFT/work_queue_run.py (100%) rename {topcoffea/analysis => analysis}/ttTest/__init__.py (100%) rename {topcoffea/analysis => analysis}/ttTest/plot.py (100%) rename {topcoffea/analysis => analysis}/ttTest/tt.py (100%) diff --git a/topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py b/analysis/.ipynb_checkpoints/__init__-checkpoint.py similarity index 100% rename from topcoffea/analysis/.ipynb_checkpoints/__init__-checkpoint.py rename to analysis/.ipynb_checkpoints/__init__-checkpoint.py diff --git a/topcoffea/analysis/__init__.py b/analysis/__init__.py similarity index 100% rename from topcoffea/analysis/__init__.py rename to analysis/__init__.py diff --git a/topcoffea/analysis/__pycache__/__init__.cpython-38.pyc b/analysis/__pycache__/__init__.cpython-38.pyc similarity index 100% rename from topcoffea/analysis/__pycache__/__init__.cpython-38.pyc rename to analysis/__pycache__/__init__.cpython-38.pyc diff --git a/topcoffea/analysis/btagMCeff/__init__.py b/analysis/btagMCeff/__init__.py similarity index 100% rename from topcoffea/analysis/btagMCeff/__init__.py rename to analysis/btagMCeff/__init__.py diff --git a/topcoffea/analysis/btagMCeff/btagMCeff.py b/analysis/btagMCeff/btagMCeff.py similarity index 100% rename from topcoffea/analysis/btagMCeff/btagMCeff.py rename to analysis/btagMCeff/btagMCeff.py diff --git a/topcoffea/analysis/btagMCeff/run.py b/analysis/btagMCeff/run.py similarity index 100% rename from topcoffea/analysis/btagMCeff/run.py rename to analysis/btagMCeff/run.py diff --git a/topcoffea/analysis/topEFT/Genrun.py b/analysis/topEFT/Genrun.py similarity index 100% rename from topcoffea/analysis/topEFT/Genrun.py rename to analysis/topEFT/Genrun.py diff --git a/topcoffea/analysis/topEFT/Gentopeft.py b/analysis/topEFT/Gentopeft.py similarity index 100% rename from topcoffea/analysis/topEFT/Gentopeft.py rename to analysis/topEFT/Gentopeft.py diff --git a/topcoffea/analysis/topEFT/__init__.py b/analysis/topEFT/__init__.py similarity index 100% rename from topcoffea/analysis/topEFT/__init__.py rename to analysis/topEFT/__init__.py diff --git a/topcoffea/analysis/topEFT/convert3lEFT.py b/analysis/topEFT/convert3lEFT.py similarity index 100% rename from topcoffea/analysis/topEFT/convert3lEFT.py rename to analysis/topEFT/convert3lEFT.py diff --git a/topcoffea/analysis/topEFT/drawSliders.py b/analysis/topEFT/drawSliders.py similarity index 100% rename from topcoffea/analysis/topEFT/drawSliders.py rename to analysis/topEFT/drawSliders.py diff --git a/topcoffea/analysis/topEFT/genPlot.py b/analysis/topEFT/genPlot.py similarity index 100% rename from topcoffea/analysis/topEFT/genPlot.py rename to analysis/topEFT/genPlot.py diff --git a/topcoffea/analysis/topEFT/make_jsons.py b/analysis/topEFT/make_jsons.py similarity index 100% rename from topcoffea/analysis/topEFT/make_jsons.py rename to analysis/topEFT/make_jsons.py diff --git a/topcoffea/analysis/topEFT/plot.py b/analysis/topEFT/plot.py similarity index 100% rename from topcoffea/analysis/topEFT/plot.py rename to analysis/topEFT/plot.py diff --git a/topcoffea/analysis/topEFT/run.py b/analysis/topEFT/run.py similarity index 100% rename from topcoffea/analysis/topEFT/run.py rename to analysis/topEFT/run.py diff --git a/topcoffea/analysis/topEFT/topeft.py b/analysis/topEFT/topeft.py similarity index 100% rename from topcoffea/analysis/topEFT/topeft.py rename to analysis/topEFT/topeft.py diff --git a/topcoffea/analysis/topEFT/topeftenv.py b/analysis/topEFT/topeftenv.py similarity index 100% rename from topcoffea/analysis/topEFT/topeftenv.py rename to analysis/topEFT/topeftenv.py diff --git a/topcoffea/analysis/topEFT/work_queue_run.py b/analysis/topEFT/work_queue_run.py similarity index 100% rename from topcoffea/analysis/topEFT/work_queue_run.py rename to analysis/topEFT/work_queue_run.py diff --git a/topcoffea/analysis/ttTest/__init__.py b/analysis/ttTest/__init__.py similarity index 100% rename from topcoffea/analysis/ttTest/__init__.py rename to analysis/ttTest/__init__.py diff --git a/topcoffea/analysis/ttTest/plot.py b/analysis/ttTest/plot.py similarity index 100% rename from topcoffea/analysis/ttTest/plot.py rename to analysis/ttTest/plot.py diff --git a/topcoffea/analysis/ttTest/tt.py b/analysis/ttTest/tt.py similarity index 100% rename from topcoffea/analysis/ttTest/tt.py rename to analysis/ttTest/tt.py From 9be8b7a00f5c3ab56ef62e025ab7824e46ec668e Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Fri, 25 Jun 2021 13:47:32 +0200 Subject: [PATCH 11/14] Try to test topeft on coffea-casa (from terminal) --- analysis/topEFT/coffea_casa_run.py | 201 +++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 analysis/topEFT/coffea_casa_run.py diff --git a/analysis/topEFT/coffea_casa_run.py b/analysis/topEFT/coffea_casa_run.py new file mode 100644 index 000000000..bd7d6f1ef --- /dev/null +++ b/analysis/topEFT/coffea_casa_run.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +import lz4.frame as lz4f +import pickle +import json +import time +import cloudpickle +import gzip +import os +from optparse import OptionParser + +import uproot +import numpy as np +from coffea import hist, processor +from coffea.util import load, save +from coffea.nanoevents import NanoAODSchema + +import topeft +from topcoffea.modules import samples +from topcoffea.modules import fileReader + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser(description='You can customize your run') + parser.add_argument('jsonFiles' , nargs='?', default='' , help = 'Json file(s) containing files and metadata') + parser.add_argument('--prefix', '-r' , nargs='?', default='' , help = 'Prefix or redirector to look for the files') + parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks') + parser.add_argument('--pretend' , action='store_true' , help = 'Read json files but, not execute the analysis') + parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers') + parser.add_argument('--chunksize','-s' , default=100000 , help = 'Number of events per chunk') + parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks') + parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms') + parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory') + parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files') + parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients') + parser.add_argument('--do-systs', action='store_true', help = 'Run over systematic samples (takes longer)') + + args = parser.parse_args() + jsonFiles = args.jsonFiles + prefix = args.prefix + dotest = args.test + nworkers = int(args.nworkers) + chunksize = int(args.chunksize) + nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks + outname = args.outname + outpath = args.outpath + pretend = args.pretend + treename = args.treename + do_errors = args.do_errors + do_systs = args.do_systs + + if dotest: + nchunks = 2 + chunksize = 10000 + nworkers = 1 + print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize)) + + ### Load samples from json + samplesdict = {} + allInputFiles = [] + + def LoadJsonToSampleName(jsonFile, prefix): + sampleName = jsonFile if not '/' in jsonFile else jsonFile[jsonFile.rfind('/')+1:] + if sampleName.endswith('.json'): sampleName = sampleName[:-5] + with open(jsonFile) as jf: + samplesdict[sampleName] = json.load(jf) + samplesdict[sampleName]['redirector'] = prefix + + if isinstance(jsonFiles, str) and ',' in jsonFiles: jsonFiles = jsonFiles.replace(' ', '').split(',') + elif isinstance(jsonFiles, str) : jsonFiles = [jsonFiles] + for jsonFile in jsonFiles: + if os.path.isdir(jsonFile): + if not jsonFile.endswith('/'): jsonFile+='/' + for f in os.path.listdir(jsonFile): + if f.endswith('.json'): allInputFiles.append(jsonFile+f) + else: + allInputFiles.append(jsonFile) + + # Read from cfg files + for f in allInputFiles: + if not os.path.isfile(f): + print('[WARNING] Input file "%s% not found!'%f) + continue + # This input file is a json file, not a cfg + if f.endswith('.json'): + LoadJsonToSampleName(f, prefix) + # Open cfg files + else: + with open(f) as fin: + print(' >> Reading json from cfg file...') + lines = fin.readlines() + for l in lines: + if '#' in l: l=l[:l.find('#')] + l = l.replace(' ', '').replace('\n', '') + if l == '': continue + if ',' in l: + l = l.split(',') + for nl in l: + if not os.path.isfile(l): prefix = nl + else: LoadJsonToSampleName(nl, prefix) + else: + if not os.path.isfile(l): prefix = l + else: LoadJsonToSampleName(l, prefix) + + flist = {}; + for sname in samplesdict.keys(): + redirector = samplesdict[sname]['redirector'] + flist[sname] = [(redirector+f) for f in samplesdict[sname]['files']] + samplesdict[sname]['year'] = int(samplesdict[sname]['year']) + samplesdict[sname]['xsec'] = float(samplesdict[sname]['xsec']) + samplesdict[sname]['nEvents'] = int(samplesdict[sname]['nEvents']) + samplesdict[sname]['nGenEvents'] = int(samplesdict[sname]['nGenEvents']) + samplesdict[sname]['nSumOfWeights'] = float(samplesdict[sname]['nSumOfWeights']) + + # Print file info + print('>> '+sname) + print(' - isData? : %s' %('YES' if samplesdict[sname]['isData'] else 'NO')) + print(' - year : %i' %samplesdict[sname]['year']) + print(' - xsec : %f' %samplesdict[sname]['xsec']) + print(' - histAxisName : %s' %samplesdict[sname]['histAxisName']) + print(' - options : %s' %samplesdict[sname]['options']) + print(' - tree : %s' %samplesdict[sname]['treeName']) + print(' - nEvents : %i' %samplesdict[sname]['nEvents']) + print(' - nGenEvents : %i' %samplesdict[sname]['nGenEvents']) + print(' - SumWeights : %f' %samplesdict[sname]['nSumOfWeights']) + print(' - Prefix : %s' %samplesdict[sname]['redirector']) + print(' - nFiles : %i' %len(samplesdict[sname]['files'])) + for fname in samplesdict[sname]['files']: print(' %s'%fname) + + if pretend: + print('pretending...') + exit() + + # Check that all datasets have the same list of WCs + for i,k in enumerate(samplesdict.keys()): + if i == 0: + wc_lst = samplesdict[k]['WCnames'] + if wc_lst != samplesdict[k]['WCnames']: + raise Exception("Not all of the datasets have the same list of WCs.") + + processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors,do_systs) + + # Run the processor and get the output + tstart = time.time() + processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) + + from dask.distributed import Client, Worker, WorkerPlugin + import os, shutil + from typing import List + class DependencyInstaller(WorkerPlugin): + def __init__(self, dependencies: List[str]): + self._depencendies = " ".join(f"'{dep}'" for dep in dependencies) + def setup(self, worker: Worker): + os.system(f"pip install {self._depencendies}") + dependency_installer = DependencyInstaller([ + "git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis","awkward==1.3.0" + ]) + + client = Client("tls://localhost:8786") + client.register_worker_plugin(dependency_installer) + + shutil.make_archive("analysis", "zip", base_dir="analysis") + client.upload_file("analysis.zip") + + executor_args = { + 'schema': NanoAODSchema, + 'client': client, + 'savemetrics': True + } + + # Run the processor and get the output + tic = time.time() + output = processor.run_uproot_job(flist, + treename=treename, + processor_instance=processor_instance, + executor=processor.dask_executor, + executor_args=executor_args, + chunksize=chunksize, + maxchunks=nchunks + ) + toc = time.time() + + print("Dask client:", client) + print("Total time: %.0f" % (toc - tic)) + print("Events / s / thread: {:,.0f}".format(output[1]['entries'] / output[1]['processtime'])) + print("Events / s: {:,.0f}".format(output[1]['entries'] / (toc - tic))) + + nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) + nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist)) + print("Filled %.0f bins, nonzero bins: %1.1f %%" % (nbins, 100*nfilled/nbins,)) + + + # This is taken from the DM photon analysis... + # Pickle is not very fast or memory efficient, will be replaced by something better soon + # with lz4f.open("pods/"+options.year+"/"+dataset+".pkl.gz", mode="xb", compression_level=5) as fout: + if not outpath.endswith('/'): outpath += '/' + if not os.path.isdir(outpath): os.system("mkdir -p %s"%outpath) + print('Saving output in %s...'%(outpath + outname + ".pkl.gz")) + with gzip.open(outpath + outname + ".pkl.gz", "wb") as fout: + cloudpickle.dump(output, fout) + print('Done!') From 5d6a972e917007c7949a6bdd1275790ecc41d87e Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Fri, 25 Jun 2021 13:49:23 +0200 Subject: [PATCH 12/14] Move coffea-casa notebook in to parent directory --- notebooks/topeft-coffea-casa.ipynb | 325 ----------------------------- topeft-coffea-casa.ipynb | 134 +++--------- 2 files changed, 27 insertions(+), 432 deletions(-) delete mode 100644 notebooks/topeft-coffea-casa.ipynb diff --git a/notebooks/topeft-coffea-casa.ipynb b/notebooks/topeft-coffea-casa.ipynb deleted file mode 100644 index cc4380d0c..000000000 --- a/notebooks/topeft-coffea-casa.ipynb +++ /dev/null @@ -1,325 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", - "! pip install awkward==1.3.0\n", - "#! pip install -e ." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import lz4.frame as lz4f\n", - "import pickle\n", - "import json\n", - "import time\n", - "import cloudpickle\n", - "import gzip\n", - "import os\n", - "from optparse import OptionParser\n", - "\n", - "import uproot\n", - "import numpy as np\n", - "from coffea import hist, processor\n", - "from coffea.util import load, save\n", - "from coffea.nanoevents import NanoAODSchema\n", - "\n", - "from topcoffea.modules import samples\n", - "from topcoffea.modules import fileReader\n", - "\n", - "#FIXME: analysis is not installed anywhere (should be installed as well)\n", - "import topcoffea.analysis.topEFT.topeft\n", - "\n", - "import importlib.resources\n", - "\n", - "if hasattr(__builtins__,'__IPYTHON__'):\n", - " import sys\n", - " sys.argv = ['']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import argparse\n", - "parser = argparse.ArgumentParser(description='You can customize your run')\n", - "parser.add_argument('jsonFiles' , nargs='?', help = 'Json file(s) containing files and metadata')\n", - "parser.add_argument('--prefix', '-r' , nargs='?', help = 'Prefix or redirector to look for the files')\n", - "parser.add_argument('--test','-t' , action='store_true' , help = 'To perform a test, run over a few events in a couple of chunks')\n", - "parser.add_argument('--pretend' , action='store_true' , help = 'Read json files but, not execute the analysis')\n", - "#parser.add_argument('--nworkers','-n' , default=8 , help = 'Number of workers')\n", - "parser.add_argument('--chunksize','-s' , default=500000 , help = 'Number of events per chunk')\n", - "parser.add_argument('--nchunks','-c' , default=None , help = 'You can choose to run only a number of chunks')\n", - "parser.add_argument('--outname','-o' , default='plotsTopEFT', help = 'Name of the output file with histograms')\n", - "parser.add_argument('--outpath','-p' , default='histos', help = 'Name of the output directory')\n", - "parser.add_argument('--treename' , default='Events', help = 'Name of the tree inside the files')\n", - "parser.add_argument('--do-errors', action='store_true', help = 'Save the w**2 coefficients')\n", - "\n", - "args = parser.parse_args()\n", - "\n", - "if args.jsonFiles is not None:\n", - " jsonFiles = args.jsonFiles\n", - " print('jsonFiles {}'.format(args.jsonFiles))\n", - "else:\n", - " with importlib.resources.path(\"topcoffea.json\", \"TTZToLLNuNu_M10.json\") as path:\n", - " jsonFiles = str(path)\n", - " print('jsonFile was selected for UNL {}'.format(jsonFiles))\n", - " \n", - "if args.prefix is not None:\n", - " prefix = args.prefix\n", - "else:\n", - " prefix = \"root://xcache//\"\n", - "\n", - "dotest = args.test\n", - "#nworkers = int(args.nworkers)\n", - "chunksize = int(args.chunksize)\n", - "nchunks = int(args.nchunks) if not args.nchunks is None else args.nchunks\n", - "outname = args.outname\n", - "outpath = args.outpath\n", - "pretend = args.pretend\n", - "treename = args.treename\n", - "do_errors = args.do_errors\n", - "\n", - "if dotest:\n", - " nchunks = 2\n", - " chunksize = 10000\n", - " nworkers = 1\n", - " print('Running a fast test with %i workers, %i chunks of %i events'%(nworkers, nchunks, chunksize))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "samplesdict = {}\n", - "allInputFiles = []\n", - "\n", - "def LoadJsonToSampleName(jsonFile, prefix):\n", - " sampleName = jsonFile if not '/' in jsonFile else jsonFile[jsonFile.rfind('/')+1:]\n", - " if sampleName.endswith('.json'): sampleName = sampleName[:-5]\n", - " with open(jsonFile) as jf:\n", - " samplesdict[sampleName] = json.load(jf)\n", - " samplesdict[sampleName]['redirector'] = prefix\n", - "\n", - "if isinstance(jsonFiles, str) and ',' in jsonFiles: jsonFiles = jsonFiles.replace(' ', '').split(',')\n", - "elif isinstance(jsonFiles, str) : jsonFiles = [jsonFiles]\n", - "\n", - "for jsonFile in jsonFiles:\n", - " if os.path.isdir(jsonFile):\n", - " if not jsonFile.endswith('/'): jsonFile+='/'\n", - " for f in os.path.listdir(jsonFile):\n", - " if f.endswith('.json'): allInputFiles.append(jsonFile+f)\n", - " else:\n", - " allInputFiles.append(jsonFile)\n", - "\n", - "print(allInputFiles)\n", - "# Read from cfg files\n", - "for f in allInputFiles:\n", - " if not os.path.isfile(f):\n", - " print('[WARNING] Input file \"%s% not found!'%f)\n", - " continue\n", - " # This input file is a json file, not a cfg\n", - " if f.endswith('.json'): \n", - " LoadJsonToSampleName(f, prefix)\n", - " # Open cfg files\n", - " else:\n", - " with open(f) as fin:\n", - " print(' >> Reading json from cfg file...')\n", - " lines = fin.readlines()\n", - " for l in lines:\n", - " if '#' in l: l=l[:l.find('#')]\n", - " l = l.replace(' ', '').replace('\\n', '')\n", - " if l == '': continue\n", - " if ',' in l:\n", - " l = l.split(',')\n", - " for nl in l:\n", - " if not os.path.isfile(l): prefix = nl\n", - " else: LoadJsonToSampleName(nl, prefix)\n", - " else:\n", - " if not os.path.isfile(l): prefix = l\n", - " else: LoadJsonToSampleName(l, prefix)\n", - "\n", - "flist = {};\n", - "for sname in samplesdict.keys():\n", - " redirector = samplesdict[sname]['redirector']\n", - " flist[sname] = [(redirector+f) for f in samplesdict[sname]['files']]\n", - " samplesdict[sname]['year'] = int(samplesdict[sname]['year'])\n", - " samplesdict[sname]['xsec'] = float(samplesdict[sname]['xsec'])\n", - " samplesdict[sname]['nEvents'] = int(samplesdict[sname]['nEvents'])\n", - " samplesdict[sname]['nGenEvents'] = int(samplesdict[sname]['nGenEvents'])\n", - " samplesdict[sname]['nSumOfWeights'] = float(samplesdict[sname]['nSumOfWeights'])\n", - "\n", - " # Print file info\n", - " print('>> '+sname)\n", - " print(' - isData? : %s' %('YES' if samplesdict[sname]['isData'] else 'NO'))\n", - " print(' - year : %i' %samplesdict[sname]['year'])\n", - " print(' - xsec : %f' %samplesdict[sname]['xsec'])\n", - " print(' - histAxisName : %s' %samplesdict[sname]['histAxisName'])\n", - " print(' - options : %s' %samplesdict[sname]['options'])\n", - " print(' - tree : %s' %samplesdict[sname]['treeName'])\n", - " print(' - nEvents : %i' %samplesdict[sname]['nEvents'])\n", - " print(' - nGenEvents : %i' %samplesdict[sname]['nGenEvents'])\n", - " print(' - SumWeights : %f' %samplesdict[sname]['nSumOfWeights'])\n", - " print(' - Prefix : %s' %samplesdict[sname]['redirector'])\n", - " print(' - nFiles : %i' %len(samplesdict[sname]['files']))\n", - " for fname in samplesdict[sname]['files']: print(' %s'%fname)\n", - "\n", - "if pretend: \n", - " print('pretending...')\n", - " exit() \n", - "\n", - "# Check that all datasets have the same list of WCs\n", - "for i,k in enumerate(samplesdict.keys()):\n", - " if i == 0:\n", - " wc_lst = samplesdict[k]['WCnames']\n", - " if wc_lst != samplesdict[k]['WCnames']:\n", - " raise Exception(\"Not all of the datasets have the same list of WCs.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", - "\n", - "from dask.distributed import Client, Worker, WorkerPlugin\n", - "import os\n", - "from typing import List\n", - "class DependencyInstaller(WorkerPlugin):\n", - " def __init__(self, dependencies: List[str]):\n", - " self._depencendies = \" \".join(f\"'{dep}'\" for dep in dependencies)\n", - " def setup(self, worker: Worker):\n", - " os.system(f\"pip install {self._depencendies}\")\n", - "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\",\"awkward==1.3.0\"\n", - "])\n", - "\n", - "client = Client(\"tls://localhost:8786\")\n", - "client.register_worker_plugin(dependency_installer)\n", - "\n", - "executor_args = {\n", - " 'schema': NanoAODSchema,\n", - " 'client': client,\n", - " 'savemetrics': True\n", - "}\n", - "\n", - "\n", - "# Run the processor and get the output \n", - "tic = time.time()\n", - "output = processor.run_uproot_job(flist,\n", - " treename=treename,\n", - " processor_instance=processor_instance,\n", - " executor=processor.dask_executor,\n", - " executor_args=executor_args,\n", - " chunksize=chunksize,\n", - " maxchunks=nchunks\n", - " )\n", - "toc = time.time()\n", - "\n", - "print(\"Dask client:\", client)\n", - "print(\"Total time: %.0f\" % (toc - tic))\n", - "print(\"Events / s / thread: {:,.0f}\".format(output[1]['entries'] / output[1]['processtime']))\n", - "print(\"Events / s: {:,.0f}\".format(output[1]['entries'] / (toc - tic)))\n", - "\n", - "os.system(\"mkdir -p histos/\")\n", - "print('Saving output in %s...'%(\"histos/\" + outname + \".pkl.gz\"))\n", - "with gzip.open(\"histos/\" + outname + \".pkl.gz\", \"wb\") as fout:\n", - " cloudpickle.dump(output, fout)\n", - "print('Done!')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import print_function, division\n", - "from collections import defaultdict, OrderedDict\n", - "import gzip\n", - "import pickle\n", - "import json\n", - "import os\n", - "import uproot\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from coffea import hist, processor\n", - "from coffea.hist import plot\n", - "from cycler import cycler\n", - "\n", - "from topcoffea.plotter.OutText import OutText\n", - "\n", - "\n", - "path = 'histos/plotsTopEFT.pkl.gz'\n", - "outname = 'temp.png'\n", - "\n", - "# Select variable, channel and cuts\n", - "var = 'met'\n", - "channel = ['eemSSonZ', 'eemSSoffZ', 'mmeSSonZ', 'mmeSSoffZ','eeeSSonZ', 'eeeSSoffZ', 'mmmSSonZ', 'mmmSSoffZ']\n", - "cut = 'base'\n", - "\n", - "print('Opening path: ', path)\n", - "hists = {}\n", - "with gzip.open(path) as fin:\n", - " hin = pickle.load(fin)\n", - " print(' >> looking for histograms...')\n", - " for k in hin.keys():\n", - " if k in hists: hists[k]+=hin[k]\n", - " else: hists[k]=hin[k]\n", - "\n", - "\n", - "# Create figure\n", - "fig, (ax, rax) = plt.subplots(2, 1, figsize=(14,7), gridspec_kw={\"height_ratios\": (3, 1)}, sharex=True)\n", - "plt.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9)\n", - "\n", - "# Select the histogram var, channel and cut\n", - "h = hists[var]\n", - "h = h.integrate('channel', channel)\n", - "h = h.integrate('cut', cut)\n", - "\n", - "# Integrate over samples\n", - "h = h.sum('sample')\n", - " \n", - "# Plot and save figure to outname\n", - "hist.plot1d(h, ax=ax, line_opts={'color':'orange'})\n", - "fig.savefig(outname)\n", - "print('Output histogram saved in %s'%outname)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index d3ce83902..03798827a 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -2,35 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "5d03d87f", + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", - "Collecting git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", - " Cloning https://github.com/oshadura/topcoffea.git (to revision coffea-casa-analysis) to /tmp/pip-req-build-9zf5jcvu\n", - "Requirement already satisfied (use --upgrade to upgrade): topcoffea==0.0.0 from git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis in /home/cms-jovyan/topcoffea\n", - "Building wheels for collected packages: topcoffea\n", - " Building wheel for topcoffea (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for topcoffea: filename=topcoffea-0.0.0-py3-none-any.whl size=4512948 sha256=e020a6dde8da5bedb5658fcf58b182244cc65fc5593209bae50ba8da1f429d85\n", - " Stored in directory: /tmp/pip-ephem-wheel-cache-b7hy8t1i/wheels/3a/95/24/bfbb2d1dc4571114a0106e7d0d6a8124bcd2ad9e72ec9ea14e\n", - "Successfully built topcoffea\n" - ] - } - ], + "outputs": [], "source": [ "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + "! pip install awkward==1.3.0\n", "#! pip install -e ." ] }, { "cell_type": "code", - "execution_count": 2, - "id": "502972cd", + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -52,8 +35,7 @@ "from topcoffea.modules import samples\n", "from topcoffea.modules import fileReader\n", "\n", - "#FIXME: analysis is not installed anywhere (should be installed as well)\n", - "import topcoffea.analysis.topEFT.topeft\n", + "import analysis.topEFT.topeft\n", "\n", "import importlib.resources\n", "\n", @@ -64,18 +46,9 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "dd76877f", + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" - ] - } - ], + "outputs": [], "source": [ "import argparse\n", "parser = argparse.ArgumentParser(description='You can customize your run')\n", @@ -125,50 +98,10 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "402ee04d", + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", - ">> TTZToLLNuNu_M10\n", - " - isData? : NO\n", - " - year : 2018\n", - " - xsec : 0.252900\n", - " - histAxisName : \n", - " - options : \n", - " - tree : Events\n", - " - nEvents : 14542666\n", - " - nGenEvents : 4876491\n", - " - SumWeights : 19992000.000000\n", - " - Prefix : root://xcache//\n", - " - nFiles : 18\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", - " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" - ] - } - ], + "outputs": [], "source": [ - "\n", "samplesdict = {}\n", "allInputFiles = []\n", "\n", @@ -257,22 +190,13 @@ { "cell_type": "code", "execution_count": null, - "id": "55f6755e", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[################################# ] | 84% Completed | 1min 35.2s" - ] - } - ], + "outputs": [], "source": [ "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", "\n", "from dask.distributed import Client, Worker, WorkerPlugin\n", - "import os\n", + "import os, shumutil\n", "from typing import List\n", "class DependencyInstaller(WorkerPlugin):\n", " def __init__(self, dependencies: List[str]):\n", @@ -280,19 +204,24 @@ " def setup(self, worker: Worker):\n", " os.system(f\"pip install {self._depencendies}\")\n", "dependency_installer = DependencyInstaller([\n", - " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\"\n", + " \"git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\",\"awkward==1.3.0\"\n", "])\n", "\n", "client = Client(\"tls://localhost:8786\")\n", "client.register_worker_plugin(dependency_installer)\n", "\n", + "shutil.make_archive(\"analysis\", \"zip\", base_dir=\"analysis\")\n", + "dask_client.upload_file(\"myanalysis.zip\")\n", + "\n", "executor_args = {\n", " 'schema': NanoAODSchema,\n", - " 'client': client\n", + " 'client': client,\n", + " 'savemetrics': True\n", "}\n", "\n", + "\n", "# Run the processor and get the output \n", - "tstart = time.time()\n", + "tic = time.time()\n", "output = processor.run_uproot_job(flist,\n", " treename=treename,\n", " processor_instance=processor_instance,\n", @@ -301,12 +230,12 @@ " chunksize=chunksize,\n", " maxchunks=nchunks\n", " )\n", - "dt = time.time() - tstart\n", + "toc = time.time()\n", "\n", - "nbins = sum(sum(arr.size for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist))\n", - "nfilled = sum(sum(np.sum(arr > 0) for arr in h._sumw.values()) for h in output.values() if isinstance(h, hist.Hist))\n", - "print(\"Filled %.0f bins, nonzero bins: %1.1f %%\" % (nbins, 100*nfilled/nbins,))\n", - "print(\"Processing time: %1.2f s with %i workers (%.2f s cpu overall)\" % (dt, nworkers, dt*nworkers, ))\n", + "print(\"Dask client:\", client)\n", + "print(\"Total time: %.0f\" % (toc - tic))\n", + "print(\"Events / s / thread: {:,.0f}\".format(output[1]['entries'] / output[1]['processtime']))\n", + "print(\"Events / s: {:,.0f}\".format(output[1]['entries'] / (toc - tic)))\n", "\n", "os.system(\"mkdir -p histos/\")\n", "print('Saving output in %s...'%(\"histos/\" + outname + \".pkl.gz\"))\n", @@ -318,7 +247,6 @@ { "cell_type": "code", "execution_count": null, - "id": "53c45688", "metadata": {}, "outputs": [], "source": [ @@ -373,14 +301,6 @@ "fig.savefig(outname)\n", "print('Output histogram saved in %s'%outname)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "01fb4952", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -399,9 +319,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.8.5" } }, "nbformat": 4, - "nbformat_minor": 5 -} + "nbformat_minor": 4 +} \ No newline at end of file From bd0b9f8297a36e6c83548815f164ec2938469959 Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Fri, 25 Jun 2021 13:22:02 +0000 Subject: [PATCH 13/14] Misc fixes and improvements --- analysis/__pycache__/__init__.cpython-38.pyc | Bin 148 -> 0 bytes analysis/topEFT/README | 3 + analysis/topEFT/coffea_casa_run.py | 10 +- topeft-coffea-casa.ipynb | 124 ++++++++++++++++--- 4 files changed, 119 insertions(+), 18 deletions(-) delete mode 100644 analysis/__pycache__/__init__.cpython-38.pyc create mode 100644 analysis/topEFT/README diff --git a/analysis/__pycache__/__init__.cpython-38.pyc b/analysis/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 41ab022a210617d05b2dec0b3c93f578319a79f5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 148 zcmWIL<>g`k0?EGz6F~H15P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Heenx(7s(x~A zv2Ip=S!H6Leo1~oa(-G`Y9bOVF)uNvvN*F?KR!M)FS8^*Uaz3?7Kcr4eoARhsvXG4 I&p^xo02gp0X#fBK diff --git a/analysis/topEFT/README b/analysis/topEFT/README new file mode 100644 index 000000000..7b7fad2d8 --- /dev/null +++ b/analysis/topEFT/README @@ -0,0 +1,3 @@ +To run analysis on coffea-casa please use: + +`python analysis/topEFT/coffea_casa_run.py --prefix root://xcache// topcoffea/json/TTZToLLNuNu_M10.json` \ No newline at end of file diff --git a/analysis/topEFT/coffea_casa_run.py b/analysis/topEFT/coffea_casa_run.py index bd7d6f1ef..1a05ed568 100644 --- a/analysis/topEFT/coffea_casa_run.py +++ b/analysis/topEFT/coffea_casa_run.py @@ -14,7 +14,11 @@ from coffea.util import load, save from coffea.nanoevents import NanoAODSchema -import topeft +#import sys +#sys.path.append('./analysis/topEFT/') +from analysis.topEFT import topeft + +from analysis.topEFT import topeft from topcoffea.modules import samples from topcoffea.modules import fileReader @@ -75,7 +79,7 @@ def LoadJsonToSampleName(jsonFile, prefix): if f.endswith('.json'): allInputFiles.append(jsonFile+f) else: allInputFiles.append(jsonFile) - + # Read from cfg files for f in allInputFiles: if not os.path.isfile(f): @@ -142,7 +146,7 @@ def LoadJsonToSampleName(jsonFile, prefix): # Run the processor and get the output tstart = time.time() - processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) + processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors) from dask.distributed import Client, Worker, WorkerPlugin import os, shutil diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 03798827a..61947212e 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -2,9 +2,29 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", + "Collecting git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", + " Cloning https://github.com/oshadura/topcoffea.git (to revision coffea-casa-analysis) to /tmp/pip-req-build-7h8w9ss_\n", + "Requirement already satisfied (use --upgrade to upgrade): topcoffea==0.0.0 from git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis in /opt/conda/lib/python3.8/site-packages\n", + "Building wheels for collected packages: topcoffea\n", + " Building wheel for topcoffea (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for topcoffea: filename=topcoffea-0.0.0-py3-none-any.whl size=4513687 sha256=ef030747c7007667a3b9b8a7bc13d11c944b76e4cdad7aab00da5cbce93236ed\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-0j5azoud/wheels/3a/95/24/bfbb2d1dc4571114a0106e7d0d6a8124bcd2ad9e72ec9ea14e\n", + "Successfully built topcoffea\n", + "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", + "Requirement already satisfied: awkward==1.3.0 in /opt/conda/lib/python3.8/site-packages (1.3.0)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (57.0.0)\n", + "Requirement already satisfied: numpy>=1.13.1 in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (1.20.3)\n" + ] + } + ], "source": [ "!pip install git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", "! pip install awkward==1.3.0\n", @@ -13,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -35,7 +55,8 @@ "from topcoffea.modules import samples\n", "from topcoffea.modules import fileReader\n", "\n", - "import analysis.topEFT.topeft\n", + "import sys\n", + "from analysis.topEFT import topeft\n", "\n", "import importlib.resources\n", "\n", @@ -46,9 +67,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "jsonFile was selected for UNL /home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json\n" + ] + } + ], "source": [ "import argparse\n", "parser = argparse.ArgumentParser(description='You can customize your run')\n", @@ -98,9 +127,47 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['/home/cms-jovyan/topcoffea/topcoffea/json/TTZToLLNuNu_M10.json']\n", + ">> TTZToLLNuNu_M10\n", + " - isData? : NO\n", + " - year : 2018\n", + " - xsec : 0.252900\n", + " - histAxisName : \n", + " - options : \n", + " - tree : Events\n", + " - nEvents : 14542666\n", + " - nGenEvents : 4876491\n", + " - SumWeights : 19992000.000000\n", + " - Prefix : root://xcache//\n", + " - nFiles : 18\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_1.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_10.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_11.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_12.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_13.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_14.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_15.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_16.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_17.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_18.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_2.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_3.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_4.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_5.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_6.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_7.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_8.root\n", + " /store/user/jrgonzal/nanoAODcrab/TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/mc2018_28apr2021_TTZToLLNuNu_M-10_TuneCP5_13TeV-amcatnlo-pythia8/210427_230816/0000/tree_9.root\n" + ] + } + ], "source": [ "samplesdict = {}\n", "allInputFiles = []\n", @@ -189,14 +256,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[# ] | 2% Completed | 57.6s6s\r" + ] + }, + { + "ename": "ZipImportError", + "evalue": "bad local file header: '/home/cms-jovyan/dask-worker-space/worker-4w0itrwe/analysis.zip'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mZipImportError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# Run the processor and get the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mtic\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m output = processor.run_uproot_job(flist,\n\u001b[0m\u001b[1;32m 31\u001b[0m \u001b[0mtreename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtreename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mrun_uproot_job\u001b[0;34m(fileset, treename, processor_instance, executor, executor_args, pre_executor, pre_args, chunksize, maxchunks, metadata_cache)\u001b[0m\n\u001b[1;32m 1233\u001b[0m }\n\u001b[1;32m 1234\u001b[0m \u001b[0mexe_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m \u001b[0mwrapped_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mexe_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1237\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"out\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mdask_executor\u001b[0;34m(items, function, accumulator, **kwargs)\u001b[0m\n\u001b[1;32m 745\u001b[0m \u001b[0;31m# FIXME: fancy widget doesn't appear, have to live with boring pbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 746\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmulti\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 747\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0maccumulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0m_decompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccumulator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 748\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 749\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"error\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cancelled\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m()\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;31m# no @wraps due to pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 142\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_compress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m_work_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mProcessorABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcloudpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlz4f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0mretry_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mload_module\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36m_get_module_code\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36m_get_data\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mZipImportError\u001b[0m: bad local file header: '/home/cms-jovyan/dask-worker-space/worker-4w0itrwe/analysis.zip'" + ] + } + ], "source": [ - "processor_instance = topcoffea.analysis.topEFT.topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", + "processor_instance = topeft.AnalysisProcessor(samplesdict,wc_lst,do_errors)\n", "\n", "from dask.distributed import Client, Worker, WorkerPlugin\n", - "import os, shumutil\n", + "import os, shutil\n", "from typing import List\n", "class DependencyInstaller(WorkerPlugin):\n", " def __init__(self, dependencies: List[str]):\n", @@ -211,7 +305,7 @@ "client.register_worker_plugin(dependency_installer)\n", "\n", "shutil.make_archive(\"analysis\", \"zip\", base_dir=\"analysis\")\n", - "dask_client.upload_file(\"myanalysis.zip\")\n", + "client.upload_file(\"analysis.zip\")\n", "\n", "executor_args = {\n", " 'schema': NanoAODSchema,\n", @@ -319,9 +413,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} From 6072b311f53fc31798c56375d0087123947ee459 Mon Sep 17 00:00:00 2001 From: Oksana Date: Tue, 10 Aug 2021 19:09:23 +0000 Subject: [PATCH 14/14] Add missing version of notebook --- topeft-coffea-casa.ipynb | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/topeft-coffea-casa.ipynb b/topeft-coffea-casa.ipynb index 61947212e..36d04a292 100644 --- a/topeft-coffea-casa.ipynb +++ b/topeft-coffea-casa.ipynb @@ -11,17 +11,18 @@ "text": [ "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", "Collecting git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis\n", - " Cloning https://github.com/oshadura/topcoffea.git (to revision coffea-casa-analysis) to /tmp/pip-req-build-7h8w9ss_\n", - "Requirement already satisfied (use --upgrade to upgrade): topcoffea==0.0.0 from git+https://github.com/oshadura/topcoffea.git@coffea-casa-analysis in /opt/conda/lib/python3.8/site-packages\n", + " Cloning https://github.com/oshadura/topcoffea.git (to revision coffea-casa-analysis) to /tmp/pip-req-build-e8me1oel\n", "Building wheels for collected packages: topcoffea\n", " Building wheel for topcoffea (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for topcoffea: filename=topcoffea-0.0.0-py3-none-any.whl size=4513687 sha256=ef030747c7007667a3b9b8a7bc13d11c944b76e4cdad7aab00da5cbce93236ed\n", - " Stored in directory: /tmp/pip-ephem-wheel-cache-0j5azoud/wheels/3a/95/24/bfbb2d1dc4571114a0106e7d0d6a8124bcd2ad9e72ec9ea14e\n", + "\u001b[?25h Created wheel for topcoffea: filename=topcoffea-0.0.0-py3-none-any.whl size=4513687 sha256=50e1a965398095975bcbcd8e54ad498b108779a32b8dac52ff117f8a360d4318\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-j40zuryw/wheels/3a/95/24/bfbb2d1dc4571114a0106e7d0d6a8124bcd2ad9e72ec9ea14e\n", "Successfully built topcoffea\n", + "Installing collected packages: topcoffea\n", + "Successfully installed topcoffea-0.0.0\n", "/bin/bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by /bin/bash)\n", "Requirement already satisfied: awkward==1.3.0 in /opt/conda/lib/python3.8/site-packages (1.3.0)\n", - "Requirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (57.0.0)\n", - "Requirement already satisfied: numpy>=1.13.1 in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (1.20.3)\n" + "Requirement already satisfied: numpy>=1.13.1 in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (1.21.0)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from awkward==1.3.0) (57.1.0)\n" ] } ], @@ -263,26 +264,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "[# ] | 2% Completed | 57.6s6s\r" + "[###### ] | 16% Completed | 3min 16.2s\r" ] }, { - "ename": "ZipImportError", - "evalue": "bad local file header: '/home/cms-jovyan/dask-worker-space/worker-4w0itrwe/analysis.zip'", + "ename": "AttributeError", + "evalue": "Can't get attribute 'AnalysisProcessor' on ", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mZipImportError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# Run the processor and get the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mtic\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m output = processor.run_uproot_job(flist,\n\u001b[0m\u001b[1;32m 31\u001b[0m \u001b[0mtreename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtreename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mrun_uproot_job\u001b[0;34m(fileset, treename, processor_instance, executor, executor_args, pre_executor, pre_args, chunksize, maxchunks, metadata_cache)\u001b[0m\n\u001b[1;32m 1233\u001b[0m }\n\u001b[1;32m 1234\u001b[0m \u001b[0mexe_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m \u001b[0mwrapped_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mexe_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1237\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"out\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mdask_executor\u001b[0;34m(items, function, accumulator, **kwargs)\u001b[0m\n\u001b[1;32m 745\u001b[0m \u001b[0;31m# FIXME: fancy widget doesn't appear, have to live with boring pbar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 746\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmulti\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 747\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0maccumulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0m_decompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccumulator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 748\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 749\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"error\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cancelled\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m()\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;31m# no @wraps due to pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 142\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_compress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m_work_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mProcessorABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcloudpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlz4f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0mretry_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mload_module\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36m_get_module_code\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36m_get_data\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mZipImportError\u001b[0m: bad local file header: '/home/cms-jovyan/dask-worker-space/worker-4w0itrwe/analysis.zip'" + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_581/2526281362.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# Run the processor and get the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mtic\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m output = processor.run_uproot_job(flist,\n\u001b[0m\u001b[1;32m 31\u001b[0m \u001b[0mtreename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtreename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mrun_uproot_job\u001b[0;34m(fileset, treename, processor_instance, executor, executor_args, pre_executor, pre_args, chunksize, maxchunks, metadata_cache, dynamic_chunksize, dynamic_chunksize_targets)\u001b[0m\n\u001b[1;32m 1656\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1657\u001b[0m \u001b[0mexe_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexecutor_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1658\u001b[0;31m \u001b[0mwrapped_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mexe_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1659\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1660\u001b[0m \u001b[0mprocessor_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpostprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"out\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36mdask_executor\u001b[0;34m(items, function, accumulator, **kwargs)\u001b[0m\n\u001b[1;32m 1005\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmulti\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnotebook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m return accumulate(\n\u001b[0;32m-> 1007\u001b[0;31m \u001b[0;34m[\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0m_decompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1008\u001b[0m \u001b[0maccumulator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1009\u001b[0m )\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"error\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 228\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cancelled\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 229\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m()\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[0;31m# no @wraps due to pickle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 199\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_compress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/coffea/processor/executor.py\u001b[0m in \u001b[0;36m_work_function\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1144\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1145\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mProcessorABC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1146\u001b[0;31m \u001b[0mprocessor_instance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcloudpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlz4f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocessor_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1148\u001b[0m \u001b[0mretry_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mAttributeError\u001b[0m: Can't get attribute 'AnalysisProcessor' on " ] } ], @@ -399,7 +397,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" },