Move from an epoch-based optimization to a step-based optimization

.github/workflows/test.yml

@@ -9,45 +9,29 @@ on:
   workflow_dispatch:
 
 jobs:
-  remove-unneeded-software:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-      - run: |
-          sudo rm -rf \
-            "$AGENT_TOOLSDIRECTORY" \
-            /opt/google/chrome \
-            /opt/microsoft/msedge \
-            /opt/microsoft/powershell \
-            /opt/pipx \
-            /usr/lib/mono \
-            /usr/local/julia* \
-            /usr/local/lib/android \
-            /usr/local/lib/node_modules \
-            /usr/local/share/chromium \
-            /usr/local/share/powershell \
-            /usr/share/dotnet \
-            /usr/share/swift
-
-  deps-torch:
-    runs-on: ubuntu-22.04
-    needs: [remove-unneeded-software]
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10.12"
-          cache: "pip"
-      - run: pip install -r requirements.txt
+  test-in-container:
+    runs-on: ubuntu-latest
+    container:
+      image: pytorch/pytorch:2.9.1-cuda13.0-cudnn9-runtime
 
-  torch-pipeline:
-    runs-on: ubuntu-22.04
-    needs: [deps-torch]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Cache pip packages
+        uses: actions/cache@v4
         with:
-          python-version: "3.10.12"
-          cache: "pip"
-      - run: pip install -r requirements.txt
-      - run: ./scripts/local_test_torch.sh
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Install wget
+        run: |
+          apt-get update && apt-get install -y wget gcc g++ build-essential
+
+      - name: Run tests
+        run: ./scripts/local_test_torch.sh

mlpf/data/cms/genjob_pu55to75_val_v2.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -e
+set -x
+
+OUTDIR=/local/joosep/mlpf/cms/20251001_cmssw_15_0_5_e42b72/pu55to75_val/
+CMSSWDIR=/scratch/persistent/joosep/CMSSW_15_0_5/
+MLPF_PATH=/home/joosep/particleflow/
+
+#seed must be greater than 0
+SAMPLE=$1
+SEED=$2
+
+WORKDIR=/scratch/local/joosep/$SLURM_JOBID/$SAMPLE/$SEED
+#WORKDIR=`pwd`/$SAMPLE/$SEED
+mkdir -p $WORKDIR
+mkdir -p $OUTDIR/$SAMPLE/root
+
+PILEUP=Run3_Flat55To75_PoissonOOTPU
+PILEUP_INPUT=filelist:${MLPF_PATH}/mlpf/data/cms/pu_files_local.txt
+
+N=50
+
+env
+source /cvmfs/cms.cern.ch/cmsset_default.sh
+
+cd $CMSSWDIR
+eval `scramv1 runtime -sh`
+which python
+which python3
+
+env
+
+cd $WORKDIR
+
+#Generate the MC
+cmsDriver.py $SAMPLE \
+  --conditions auto:phase1_2023_realistic \
+  --beamspot Realistic25ns13p6TeVEarly2023Collision \
+  -n $N \
+  --era Run3_2023 \
+  --eventcontent FEVTDEBUGHLT \
+  -s GEN,SIM,DIGI:pdigi_valid,L1,DIGI2RAW,HLT:@relval2023 \
+  --datatier GEN-SIM \
+  --geometry DB:Extended \
+  --pileup $PILEUP \
+  --pileup_input $PILEUP_INPUT \
+  --no_exec \
+  --fileout step2_phase1_new.root \
+  --customise Validation/RecoParticleFlow/customize_pfanalysis.customize_step2 \
+  --python_filename=step2_phase1_new.py
+
+#Run the reco sequences
+cmsDriver.py step3 \
+  --conditions auto:phase1_2023_realistic \
+  --beamspot Realistic25ns13p6TeVEarly2023Collision \
+  --era Run3_2023 \
+  -n -1 \
+  --eventcontent FEVTDEBUGHLT \
+  --runUnscheduled \
+  -s RAW2DIGI,L1Reco,RECO,RECOSIM \
+  --datatier GEN-SIM-RECO \
+  --geometry DB:Extended \
+  --no_exec \
+  --filein file:step2_phase1_new.root \
+  --fileout step3_phase1_new.root \
+  --customise Validation/RecoParticleFlow/customize_pfanalysis.customize_step3 \
+  --python_filename=step3_phase1_new.py
+
+pwd
+ls -lrt
+
+echo "process.RandomNumberGeneratorService.generator.initialSeed = $SEED" >> step2_phase1_new.py
+cmsRun step2_phase1_new.py > /dev/null
+cp step2_phase1_new.root $OUTDIR/$SAMPLE/root/step2_${SEED}.root
+
+cmsRun step3_phase1_new.py > /dev/null
+cp pfntuple.root $OUTDIR/$SAMPLE/root/pfntuple_${SEED}.root
+
+rm -Rf /scratch/local/joosep/$SLURM_JOBID

mlpf/data/cms/genjob_pu55to75_val_v3.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -e
+set -x
+
+OUTDIR=/local/joosep/mlpf/cms/20251125_cmssw_15_0_5_117d32/pu55to75_val/
+CMSSWDIR=/scratch/persistent/joosep/CMSSW_15_0_5/
+MLPF_PATH=/home/joosep/particleflow/
+
+#seed must be greater than 0
+SAMPLE=$1
+SEED=$2
+
+WORKDIR=/scratch/local/joosep/$SLURM_JOBID/$SAMPLE/$SEED
+#WORKDIR=`pwd`/$SAMPLE/$SEED
+mkdir -p $WORKDIR
+mkdir -p $OUTDIR/$SAMPLE/root
+
+PILEUP=Run3_Flat55To75_PoissonOOTPU
+PILEUP_INPUT=filelist:${MLPF_PATH}/mlpf/data/cms/pu_files_local_val2.txt
+
+N=50
+
+env
+source /cvmfs/cms.cern.ch/cmsset_default.sh
+
+cd $CMSSWDIR
+eval `scramv1 runtime -sh`
+which python
+which python3
+
+env
+
+cd $WORKDIR
+
+#Generate the MC
+cmsDriver.py $SAMPLE \
+  --conditions 140X_mcRun3_2024_realistic_v26 \
+  --beamspot DBrealistic \
+  -n $N \
+  --era Run3_2024 \
+  --eventcontent FEVTDEBUGHLT \
+  -s GEN,SIM,DIGI:pdigi_valid,L1,DIGI2RAW,HLT:@relval2023 \
+  --datatier GEN-SIM \
+  --geometry DB:Extended \
+  --pileup $PILEUP \
+  --pileup_input $PILEUP_INPUT \
+  --no_exec \
+  --fileout step2_phase1_new.root \
+  --customise Validation/RecoParticleFlow/customize_pfanalysis.customize_step2 \
+  --python_filename=step2_phase1_new.py
+
+#  --customise_commands "process.mix.input.nbPileupEvents.probFunctionVariable = cms.vint32(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120) \n process.mix.input.nbPileupEvents.probValue = cms.vdouble(0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446,0.00826446)"
+
+#Run the reco sequences
+cmsDriver.py step3 \
+  --conditions 140X_mcRun3_2024_realistic_v26 \
+  --beamspot DBrealistic \
+  --era Run3_2024 \
+  -n -1 \
+  --eventcontent FEVTDEBUGHLT \
+  --runUnscheduled \
+  -s RAW2DIGI,L1Reco,RECO,RECOSIM \
+  --datatier GEN-SIM-RECO \
+  --geometry DB:Extended \
+  --no_exec \
+  --filein file:step2_phase1_new.root \
+  --fileout step3_phase1_new.root \
+  --customise Validation/RecoParticleFlow/customize_pfanalysis.customize_step3 \
+  --python_filename=step3_phase1_new.py
+
+pwd
+ls -lrt
+
+echo "process.RandomNumberGeneratorService.generator.initialSeed = $SEED" >> step2_phase1_new.py
+cmsRun step2_phase1_new.py > /dev/null
+cp step2_phase1_new.root $OUTDIR/$SAMPLE/root/step2_${SEED}.root
+
+cmsRun step3_phase1_new.py > /dev/null
+cp pfntuple.root $OUTDIR/$SAMPLE/root/pfntuple_${SEED}.root
+
+rm -Rf /scratch/local/joosep/$SLURM_JOBID

mlpf/data/cms/prepare_args_val.py

@@ -3,15 +3,16 @@
 
 import os
 
-outdir = "/local/joosep/mlpf/cms/20250618_cmssw_15_0_5_f8ae2f/"
+outdir = "/local/joosep/mlpf/cms/20251125_cmssw_15_0_5_117d32/"
 
 samples = [
-    ("QCDForPF_13p6TeV_TuneCUETP8M1_cfi", 700000, 710050, "genjob_pu55to75_val.sh", outdir + "/pu55to75_val"),
-    ("QCDForPF_13p6TeV_TuneCUETP8M1_cfi", 700000, 702050, "genjob_nopu_val.sh", outdir + "/nopu_val"),
-    ("TTbar_13p6TeV_TuneCUETP8M1_cfi", 800000, 802050, "genjob_pu55to75_val.sh", outdir + "/pu55to75_val"),
-    ("TTbar_13p6TeV_TuneCUETP8M1_cfi", 800000, 802050, "genjob_nopu_val.sh", outdir + "/nopu_val"),
-    ("PhotonJet_Pt_10_13p6TeV_TuneCUETP8M1_cfi", 900000, 902050, "genjob_pu55to75_val.sh", outdir + "/pu55to75_val"),
-    ("PhotonJet_Pt_10_13p6TeV_TuneCUETP8M1_cfi", 900000, 902050, "genjob_nopu_val.sh", outdir + "/nopu_val"),
+    ("QCDForPF_13p6TeV_TuneCUETP8M1_cfi", 700000, 701050, "genjob_pu55to75_val_v3.sh", outdir + "/pu55to75_val"),
+    # ("QCDForPF_13p6TeV_TuneCUETP8M1_cfi", 700000, 710050, "genjob_pu55to75_val.sh", outdir + "/pu55to75_val"),
+    # ("QCDForPF_13p6TeV_TuneCUETP8M1_cfi", 700000, 702050, "genjob_nopu_val.sh", outdir + "/nopu_val"),
+    # ("TTbar_13p6TeV_TuneCUETP8M1_cfi", 800000, 802050, "genjob_pu55to75_val.sh", outdir + "/pu55to75_val"),
+    # ("TTbar_13p6TeV_TuneCUETP8M1_cfi", 800000, 802050, "genjob_nopu_val.sh", outdir + "/nopu_val"),
+    # ("PhotonJet_Pt_10_13p6TeV_TuneCUETP8M1_cfi", 900000, 902050, "genjob_pu55to75_val.sh", outdir + "/pu55to75_val"),
+    # ("PhotonJet_Pt_10_13p6TeV_TuneCUETP8M1_cfi", 900000, 902050, "genjob_nopu_val.sh", outdir + "/nopu_val"),
 ]
 
 if __name__ == "__main__":