Fix Egamma PF linking in 2018 PbPb mAOD

Alignment/MillePedeAlignmentAlgorithm/test/BuildFile.xml

@@ -5,3 +5,8 @@
   <use name="millepede"/>
   <flags USE_UNITTEST_DIR="1"/>
 </test>
+<bin name="testPedeCampaign" file="TestDriver.cpp">
+  <flags TEST_RUNNER_ARGS="/bin/bash Alignment/MillePedeAlignmentAlgorithm/test test_pede.sh"/>
+  <use name="FWCore/Utilities"/>
+</bin>
+

Alignment/MillePedeAlignmentAlgorithm/test/TestDriver.cpp

@@ -0,0 +1,2 @@
+#include "FWCore/Utilities/interface/TestHelper.h"
+RUNTEST()

Alignment/MillePedeAlignmentAlgorithm/test/test_pede.py

@@ -0,0 +1,100 @@
+import FWCore.ParameterSet.Config as cms
+process = cms.Process("Alignment")
+
+process.Tracer = cms.Service("Tracer")
+
+setupGlobaltag = "121X_mcRun3_2021_realistic_forpp900GeV_v6"
+setupCollection = "ALCARECOTkAlCosmicsCosmicTF0T"
+setupCosmicsDecoMode = True
+setupCosmicsZeroTesla = False
+setupPrimaryWidth     = -1.0
+setupJson             = "placeholder_json"
+setupRunStartGeometry = 1
+setupAlgoMode = "pede"
+setupMonitorFile      = "millePedeMonitorISN.root"
+setupBinaryFile       = "milleBinaryISN.dat"
+readFiles = cms.untracked.vstring()
+
+import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.GeneralSetup as generalSetup
+generalSetup.setup(process, setupGlobaltag, setupCosmicsZeroTesla)
+
+
+################################################################################
+# setup alignment producer
+################################################################################
+import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.ConfigureAlignmentProducer as confAliProducer
+
+confAliProducer.setConfiguration(process,
+    collection   = setupCollection,
+    mode         = setupAlgoMode,
+    monitorFile  = setupMonitorFile,
+    binaryFile   = setupBinaryFile,
+    primaryWidth = setupPrimaryWidth,
+    cosmicsZeroTesla = setupCosmicsZeroTesla)
+
+
+################################################################################
+# Overwrite some conditions in global tag
+################################################################################
+import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.SetCondition as tagwriter
+
+################################################################################
+# insert Startgeometry 
+################################################################################
+# You can use tagwriter.setCondition() to overwrite conditions in globaltag
+
+################################################################################
+# insert Alignables 
+################################################################################
+process.AlignmentProducer.ParameterBuilder.parameterTypes = ["SelectorRigid,RigidBody"]
+
+################################################################################
+# Define the high-level structure alignables
+################################################################################
+process.AlignmentProducer.ParameterBuilder.SelectorRigid = cms.PSet(
+    alignParams = cms.vstring(
+        "TrackerP1PXBHalfBarrel,111111",
+        "TrackerP1PXECHalfCylinder,111111",
+        "TrackerTIBHalfBarrel,111111",
+        "TrackerTOBHalfBarrel,rrrrrr",
+        "TrackerTIDEndcap,111111",
+        "TrackerTECEndcap,111111",
+    )
+)
+
+################################################################################
+# insert Pedesettings 
+################################################################################
+import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.helper as helper
+helper.set_pede_option(process, "skipemptycons")
+
+################################################################################
+# Mille-procedure
+################################################################################
+if setupAlgoMode == "mille":
+    import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.MilleSetup as mille
+    mille.setup(process,
+                input_files        = readFiles,
+                collection         = setupCollection,
+                json_file          = setupJson,
+                cosmics_zero_tesla = setupCosmicsZeroTesla,
+                cosmics_deco_mode  = setupCosmicsDecoMode)
+
+################################################################################
+# Pede-procedure
+################################################################################
+else:
+    merge_binary_files = [
+                'milleBinary001.dat',
+                'milleBinary002.dat',
+                'milleBinary003.dat']
+    merge_tree_files   = [
+                'treeFile001.root',
+                'treeFile002.root',
+                'treeFile003.root']
+
+    import Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.PedeSetup as pede
+    pede.setup(process,
+               binary_files = merge_binary_files,
+               tree_files = merge_tree_files,
+               run_start_geometry = setupRunStartGeometry)

Alignment/MillePedeAlignmentAlgorithm/test/test_pede.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+function die { echo $1: status $2; exit $2; }
+
+if [ "${SCRAM_TEST_NAME}" != "" ] ; then
+  mkdir ${SCRAM_TEST_NAME}
+  cd ${SCRAM_TEST_NAME}
+fi
+
+if test -f "milleBinary00*"; then
+    echo "cleaning the local test area"
+    rm -fr milleBinary00* 
+    rm -fr pedeSteer* 
+fi
+
+pwd
+echo " testing Aligment/MillePedeAlignmentAlgorithm"
+
+REMOTE="/store/group/alca_global/tkal_millepede_tests/"
+TESTPACKAGE="test_pede_package.tar"
+COMMMAND=`xrdfs cms-xrd-global.cern.ch locate ${REMOTE}${TESTPACKAGE}`
+STATUS=$?
+echo "xrdfs command status = "$STATUS
+if [ $STATUS -eq 0 ]; then
+    echo "Using file ${TESTPACKAGE}. Running in ${LOCAL_TEST_DIR}."
+    xrdcp root://cms-xrd-global.cern.ch/${REMOTE}${TESTPACKAGE} ${LOCAL_TEST_DIR}
+    tar -xvf ${LOCAL_TEST_DIR}/${TESTPACKAGE} 
+    gunzip milleBinary00*    
+    (cmsRun ${LOCAL_TEST_DIR}/test_pede.py) || die 'failed running test_pede.py' $?
+    echo -e "\n MillePede Exit Status: "`cat millepede.end`
+else 
+  die "SKIPPING test, file ${TESTPACKAGE} not found" 0
+fi

CondTools/SiStrip/scripts/o2oRun_SiStripDAQ.py

@@ -50,7 +50,7 @@ def run(args):
     for jobname in processes:
         status[jobname] = {'job':None, 'upload':None, 'fast':None, 'changed':None}
         p = processes[jobname]
-        log = p.communicate()[0]
+        log = p.communicate()[0].decode()
         logging.debug('=== log from o2o run ===\n%s' % log)
         if p.returncode == 0:
             logging.info('Job for %s finished successfully!' % jobname)
@@ -86,7 +86,7 @@ def summary(args, is_ok, status, logfile):
              send_to=args.mail_to,
              send_from=args.mail_from)
     # send the detailed log
-    with open(logfile, 'rb') as log:
+    with open(logfile, 'r') as log:
         helper.send_mail(subject='%sNew O2O Log, IOV: %s' % (debugLabel, args.since),
                  message=log.read(),
                  send_to=args.mail_log_to,

Configuration/PyReleaseValidation/python/relval_steps.py

@@ -1905,7 +1905,7 @@ def lhegensim2018ml(fragment,howMuch):
 steps['HLTDR2_2018']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2018,},{'--conditions':'auto:run2_hlt_relval'},{'--era' : 'Run2_2018'},steps['HLTD'] ] )
 
 hltKey2021='relval2021'
-steps['HLTDR3_2021']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2021,},{'--conditions':'auto:run3_hlt'},{'--era':'Run3'},steps['HLTD'] ] )
+steps['HLTDR3_2021']=merge( [ {'-s':'L1REPACK:Full,HLT:@%s'%hltKey2021,},{'--conditions':'auto:run3_hlt_relval'},{'--era':'Run3'},steps['HLTD'] ] )
 
 # special setting for lumi section boundary crossing in RunEGamma2018Dml
 steps['HLTDR2_2018ml']=merge( [ {'--customise_commands':'"process.source.skipEvents=cms.untracked.uint32(7000)"'},steps['HLTDR2_2018'] ] )
@@ -2241,7 +2241,7 @@ def lhegensim2018ml(fragment,howMuch):
 #Run 3
 steps['RECODR3']=merge([{'--scenario':'pp',
                          '-s':'RAW2DIGI,L1Reco,RECO,DQM',
-                         '--conditions':'auto:run3_data',
+                         '--conditions':'auto:run3_data_relval',
                          '--era':'Run3',
                          '--customise':'Configuration/DataProcessing/RecoTLR.customisePostEra_Run3'},dataReco])
 
@@ -2535,7 +2535,7 @@ def gen2021HiMix(fragment,howMuch):
 steps['RECOUP15_PU25HS']=merge([PU25HS,step3Up2015Defaults])
 
 #Run3 reco
-steps['RECODR3_2021']=merge([{'--scenario':'pp','--conditions':'auto:run3_data','--era':'Run3','--customise':'Configuration/DataProcessing/RecoTLR.customisePostEra_Run3'},dataReco])
+steps['RECODR3_2021']=merge([{'--scenario':'pp','--conditions':'auto:run3_data_relval','--era':'Run3','--customise':'Configuration/DataProcessing/RecoTLR.customisePostEra_Run3'},dataReco])
 steps['RECODR3_MinBiasOffline']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,ALCA:SiStripCalMinBias+TkAlMinBias+EcalESAlign,DQM:@commonSiStripZeroBias+@ExtraHLT+@miniAODDQM','--procModifiers':'siPixelQualityRawToDigi'},steps['RECODR3_2021']])
 steps['RECODR3_ZBOffline']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,ALCA:SiStripCalZeroBias+SiStripCalMinBias+TkAlMinBias+EcalESAlign+HcalCalIsoTrkProducerFilter,DQM:@rerecoZeroBias+@ExtraHLT+@miniAODDQM','--procModifiers':'siPixelQualityRawToDigi'},steps['RECODR3_2021']])
 steps['RECODR3_HLTPhysics_Offline']=merge([{'-s':'RAW2DIGI,L1Reco,RECO,PAT,ALCA:TkAlMinBias+HcalCalIterativePhiSym+HcalCalIsoTrkProducerFilter+HcalCalHO+HcalCalHBHEMuonProducerFilter,DQM:@commonReduced+@miniAODDQM','--procModifiers':'siPixelQualityRawToDigi'},steps['RECODR3_2021']])
@@ -2976,7 +2976,7 @@ def gen2021HiMix(fragment,howMuch):
 
 steps['HARVESTDCRUN2']=merge([{'--conditions':'auto:run2_data','--era':'Run2_2016'},steps['HARVESTDC']])
 
-steps['HARVESTDR3']  = merge([{'--conditions':'auto:run3_data','--era':'Run3'}, steps['HARVESTD']])
+steps['HARVESTDR3']  = merge([{'--conditions':'auto:run3_data_relval','--era':'Run3'}, steps['HARVESTD']])
 steps['HARVESTD2021MB'] = merge([{'-s':'HARVESTING:@commonSiStripZeroBias+@ExtraHLT+@miniAODDQM'}, steps['HARVESTDR3'] ])
 steps['HARVESTD2021ZB'] = merge([{'-s':'HARVESTING:@rerecoZeroBias+@ExtraHLT+@miniAODDQM'}, steps['HARVESTDR3'] ])
 steps['HARVESTD2021HLTPhy'] = merge([{'-s':'HARVESTING:@commonReduced+@miniAODDQM'}, steps['HARVESTDR3'] ])

DQM/BeamMonitor/plugins/BuildFile.xml

@@ -1,5 +1,6 @@
 <use name="DQMServices/Core"/>
 <use name="DataFormats/Scalers"/>
+<use name="DataFormats/OnlineMetaData"/>
 <use name="DataFormats/TrackReco"/>
 <use name="FWCore/Framework"/>
 <use name="DataFormats/BeamSpot"/>
@@ -56,7 +57,6 @@
 
 <ifarchitecture name="_amd64_">
 <library file="BeamSpotDipServer.cc" name="BeamSpotDipServer">
-  <use name="DataFormats/OnlineMetaData"/>
   <use name="dip"/>
   <use name="log4cplus"/>
   <flags EDM_PLUGIN="1"/>

DQM/BeamMonitor/plugins/TKStatus.cc

@@ -1,50 +1,129 @@
-#include "FWCore/ServiceRegistry/interface/Service.h"
-#include "FWCore/Framework/interface/Run.h"
+/** \class TKStatus
+ * *
+ *  \author  Geng-yuan Jeng/UC Riverside
+ *           Francisco Yumiceva/FNAL
+ *
+ */
+
+// C++
+#include <array>
+#include <fstream>
+#include <string>
+
+// CMS
+#include "DataFormats/OnlineMetaData/interface/DCSRecord.h"
+#include "DataFormats/OnlineMetaData/interface/OnlineLuminosityRecord.h"
+#include "DataFormats/Scalers/interface/DcsStatus.h"
+#include "FWCore/Framework/interface/Event.h"
+#include "FWCore/Framework/interface/Frameworkfwd.h"
 #include "FWCore/Framework/interface/LuminosityBlock.h"
+#include "FWCore/Framework/interface/one/EDAnalyzer.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
-#include "DQM/BeamMonitor/plugins/TKStatus.h"
-#include <iostream>
+#include "FWCore/Framework/interface/Run.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
+#include "FWCore/Utilities/interface/EDGetToken.h"
+
+//
+// class declaration
+//
+class TKStatus : public edm::one::EDAnalyzer<> {
+public:
+  TKStatus(const edm::ParameterSet&);
+
+protected:
+  void analyze(const edm::Event& e, const edm::EventSetup& c) override;
 
-using namespace edm;
+private:
+  void dumpTkDcsStatus(std::string const&, edm::RunNumber_t, std::array<bool, 6> const&);
 
-TKStatus::TKStatus(const ParameterSet& ps) {
-  dcsTkFileName_ = ps.getParameter<ParameterSet>("BeamFitter").getUntrackedParameter<std::string>("DIPFileName");
+  std::string dcsTkFileName_;
+  const edm::EDGetTokenT<DcsStatusCollection> dcsStatusToken_;
+  const edm::EDGetTokenT<DCSRecord> dcsRecordToken_;
+
+  int lastlumi_ = -1;
+  // ----------member data ---------------------------
+};
+
+// ----------------------------------------------------------
+TKStatus::TKStatus(const edm::ParameterSet& ps)
+    : dcsStatusToken_(consumes<DcsStatusCollection>(edm::InputTag("scalersRawToDigi"))),
+      dcsRecordToken_(consumes<DCSRecord>(edm::InputTag("onlineMetaDataDigis"))) {
+  dcsTkFileName_ = ps.getParameter<edm::ParameterSet>("BeamFitter").getUntrackedParameter<std::string>("DIPFileName");
   {
     std::string tmpname = dcsTkFileName_;
     tmpname.insert(dcsTkFileName_.length() - 4, "_TkStatus");
     dcsTkFileName_ = std::move(tmpname);
   }
-  dcsStatus_ = consumes<DcsStatusCollection>(ps.getUntrackedParameter<std::string>("DCSStatus", "scalersRawToDigi"));
 }
 
 // ----------------------------------------------------------
-void TKStatus::analyze(const Event& iEvent, const EventSetup& iSetup) {
+void TKStatus::analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) {
   int nthlumi = iEvent.luminosityBlock();
   if (nthlumi > lastlumi_) {  // check every LS
     lastlumi_ = nthlumi;
+
     // Checking TK status
-    Handle<DcsStatusCollection> dcsStatus;
-    iEvent.getByToken(dcsStatus_, dcsStatus);
+    const auto& dcsStatus = iEvent.getHandle(dcsStatusToken_);
+    const auto& dcsRecord = iEvent.getHandle(dcsRecordToken_);
 
     std::array<bool, 6> dcsTk;
     for (auto& e : dcsTk) {
       e = true;
     }
 
-    for (auto const& status : *dcsStatus) {
-      if (!status.ready(DcsStatus::BPIX))
-        dcsTk[0] = false;
-      if (!status.ready(DcsStatus::FPIX))
-        dcsTk[1] = false;
-      if (!status.ready(DcsStatus::TIBTID))
-        dcsTk[2] = false;
-      if (!status.ready(DcsStatus::TOB))
-        dcsTk[3] = false;
-      if (!status.ready(DcsStatus::TECp))
-        dcsTk[4] = false;
-      if (!status.ready(DcsStatus::TECm))
-        dcsTk[5] = false;
+    // Check that the DCS information is available in some form
+    if (!dcsStatus.isValid() && !dcsRecord.isValid()) {
+      edm::LogWarning("TkStatus") << "DcsStatusCollection product with InputTag \"scalersRawToDigi\" not in event \n"
+                                  << "DCSRecord product with InputTag \"onlineMetaDataDigis\" not in event \n";
+      dumpTkDcsStatus(dcsTkFileName_, iEvent.run(), dcsTk);
+      return;
     }
+
+    if (dcsStatus.isValid() && (*dcsStatus).empty()) {
+      if (iEvent.eventAuxiliary().isRealData()) {
+        // This is the Data case for >= Run3, DCSStatus is available (unpacked), but empty
+        // because SCAL is not in data-taking. In this case we fall back to s/w FED 1022
+        if (dcsRecord.isValid()) {
+          edm::LogPrint("TkStatus") << "Using dcsRecord because dcsStatus is empty";
+          dcsTk[0] = (*dcsRecord).highVoltageReady(DCSRecord::Partition::BPIX);
+          dcsTk[1] = (*dcsRecord).highVoltageReady(DCSRecord::Partition::FPIX);
+          dcsTk[2] = (*dcsRecord).highVoltageReady(DCSRecord::Partition::TIBTID);
+          dcsTk[3] = (*dcsRecord).highVoltageReady(DCSRecord::Partition::TOB);
+          dcsTk[4] = (*dcsRecord).highVoltageReady(DCSRecord::Partition::TECp);
+          dcsTk[5] = (*dcsRecord).highVoltageReady(DCSRecord::Partition::TECm);
+        } else {
+          // DCS status is empty, and the DCS record is not available
+          edm::LogWarning("TkStatus") << "DCSRecord product with InputTag \"onlineMetaDataDigis\" empty \n";
+          dumpTkDcsStatus(dcsTkFileName_, iEvent.run(), dcsTk);
+          return;
+        }
+      } else {
+        // DCS status is empty, but it's not real data
+        edm::LogPrint("TkStatus") << "Running on MC";
+        dumpTkDcsStatus(dcsTkFileName_, iEvent.run(), dcsTk);
+        return;
+      }
+    } else {
+      // This is the case for <= Run2
+      edm::LogPrint("TkStatus") << "Using dcsStatus (Run 1 and Run 2 SCAL)";
+      for (auto const& status : *dcsStatus) {
+        if (!status.ready(DcsStatus::BPIX))
+          dcsTk[0] = false;
+        if (!status.ready(DcsStatus::FPIX))
+          dcsTk[1] = false;
+        if (!status.ready(DcsStatus::TIBTID))
+          dcsTk[2] = false;
+        if (!status.ready(DcsStatus::TOB))
+          dcsTk[3] = false;
+        if (!status.ready(DcsStatus::TECp))
+          dcsTk[4] = false;
+        if (!status.ready(DcsStatus::TECm))
+          dcsTk[5] = false;
+      }
+    }
+
     dumpTkDcsStatus(dcsTkFileName_, iEvent.run(), dcsTk);
   }
 }